update

Много всего...
2025-12-10 01:47:18 +07:00 · 2024-08-10 02:03:31 +11:00
parent 966e11db76
commit c148415781
9 changed files with 310 additions and 537 deletions
--- a/main.py
+++ b/main.py
@@ -1,114 +1,16 @@
-import configparser
+import asyncio
+import configparser
 import ipaddress
 import os
 import re
-from concurrent.futures import ThreadPoolExecutor
+from asyncio import Semaphore
+from collections import defaultdict

-import dns.resolver
-import requests
-
-# URLs
-urls = {
-    'Antifilter community edition': "https://community.antifilter.download/list/domains.lst",
-    'Youtube': "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/platforms/dns-youtube.txt",
-    'Facebook': "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/platforms/dns-facebook.txt",
-    'Openai': "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/platforms/dns-openai.txt",
-    'Tik-Tok': "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/platforms/dns-tiktok.txt",
-    'Instagram': "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/platforms/dns-instagram.txt",
-    'Twitter': "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/platforms/dns-twitter.txt",
-    'Netflix': "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/platforms/dns-netflix.txt",
-    'Bing': "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/platforms/dns-bing.txt",
-    'Adobe': "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/platforms/dns-adobe.txt",
-    'Apple': "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/platforms/dns-apple.txt",
-    'Google': "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/platforms/dns-google.txt",
-    'Tor-Truckers': "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/platforms/dns-ttruckers.txt",
-    'Search-engines': "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/platforms/dns-search-engines"
-                      ".txt",
-}
+import dns.asyncresolver
+import httpx


-# Function to resolve DNS
-def resolve_dns_and_write(service, url, unique_ips_all_services, include_cloudflare, threads):
-    try:
-        print(f"Загрузка данных - {service}")
-        response = requests.get(url)
-        response.raise_for_status()
-        dns_names = response.text.split('\n')
-
-        resolver = dns.resolver.Resolver(configure=False)
-        resolver.nameservers = ['8.8.8.8', '8.8.4.4', '208.67.222.222', '208.67.220.220', '4.2.2.1', '4.2.2.2',
-                                '149.112.112.112']  # Public DNS servers
-        resolver.rotate = True
-        resolver.timeout = 1
-        resolver.lifetime = 1
-
-        if include_cloudflare:
-            cloudflare_ips = get_cloudflare_ips()
-        else:
-            cloudflare_ips = set()
-
-        unique_ips_current_service = set()  # Set to store unique IP addresses for the current service
-
-        print(f"Анализ DNS имен платформы: {service}")
-
-        with ThreadPoolExecutor(max_workers=threads) as executor:
-            futures = []
-            for domain in dns_names:
-                if domain.strip():
-                    future = executor.submit(resolve_domain, resolver, domain, unique_ips_current_service,
-                                             unique_ips_all_services, cloudflare_ips)
-                    futures.append(future)
-
-            # Дождаться завершения всех задач
-            for future in futures:
-                future.result()
-
-        print(f"Список IP-адресов для платформы {service} создан.")
-        return '\n'.join(unique_ips_current_service) + '\n'
-    except Exception as e:
-        print(f"Не удалось сопоставить IP адреса {service} его доменным именам.", e)
-        return ""
-
-
-# Function to get Cloudflare IP addresses
-def get_cloudflare_ips():
-    try:
-        response = requests.get("https://www.cloudflare.com/ips-v4/")
-        response.raise_for_status()
-        cloudflare_ips = set()
-
-        # Extract CIDR blocks from the response text using regular expressions
-        cidr_blocks = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2})', response.text)
-
-        for cidr in cidr_blocks:
-            ip_network = ipaddress.ip_network(cidr)
-            for ip in ip_network:
-                cloudflare_ips.add(str(ip))
-
-        return cloudflare_ips
-    except Exception as e:
-        print("Ошибка при получении IP адресов Cloudflare:", e)
-        return set()
-
-
-# Function resolve domain
-def resolve_domain(resolver, domain, unique_ips_current_service, unique_ips_all_services, cloudflare_ips):
-    try:
-        ips = resolver.resolve(domain)
-        for ip in ips:
-            ip_address = ip.address
-            if (ip_address not in ('127.0.0.1', '0.0.0.1') and
-                    ip_address not in resolver.nameservers and
-                    ip_address not in cloudflare_ips and
-                    ip_address not in unique_ips_all_services):  # Check for uniqueness
-                unique_ips_current_service.add(ip_address)
-                unique_ips_all_services.add(ip_address)
-                print(f"\033[36m{domain} IP адрес: {ip_address}\033[0m")
-    except Exception as e:
-        print(f"\033[31mНе удалось обработать: {domain}\033[0m - {e}")
-
-
-# Function to read configuration file
+# Read configuration file
 def read_config(filename):
    try:
        config = configparser.ConfigParser()
@@ -117,27 +19,28 @@ def read_config(filename):
        if 'DomainMapper' in config:
            config = config['DomainMapper']
        service = config.get('service') or ''
-        threads = int(config.get('threads') or 20)
+        request_limit = int(config.get('threads') or 20)
        filename = config.get('filename') or 'domain-ip-resolve.txt'
        cloudflare = config.get('cloudflare') or ''
        filetype = config.get('filetype') or ''
        gateway = config.get('gateway') or ''
        run_command = config.get('run') or ''
+        dns_server_indices = list(map(int, config.get('dnsserver', '').split())) if config.get('dnsserver') else []

-        print("Загружена конфигурация из config.ini.")
-        return service, threads, filename, cloudflare, filetype, gateway, run_command
+        print("\033[33mЗагружена конфигурация из config.ini:\033[0m")
+        print(f"Сервисы для проверки: {service if service else 'не указаны'}")
+        print(f"Использовать DNS сервер: {dns_server_indices if dns_server_indices else 'не указано'}")
+        print(f"Количество потоков: {request_limit}")
+        print(f"Фильтр Cloudflare: {'включен' if cloudflare == 'yes' else 'вЫключен' if cloudflare == 'no' else 'не указано'}")
+        print(f"Файл результатов: {filename}")
+        print(f"Формат сохранения: {'только IP' if filetype == 'ip' else 'Linux route' if filetype == 'unix' else 'CIDR-нотация' if filetype == 'cidr' else 'Windows route' if filetype == 'win' else 'не указан'}")
+        print(f"Шлюз для маршрутов: {gateway if gateway else 'не указан'}")
+        print(f"Выполнить при заврешении: {run_command if run_command else 'не указано'}")
+        return service, request_limit, filename, cloudflare, filetype, gateway, run_command, dns_server_indices

    except Exception as e:
-        print(f"Ошибка загрузки конфигурации: {e}")
-        service = ''
-        threads = int(20)
-        filename = 'domain-ip-resolve.txt'
-        cloudflare = ''
-        filetype = ''
-        gateway = ''
-        run_command = ''
-
-        return service, threads, filename, cloudflare, filetype, gateway, run_command
+        print(f"\033[33mОшибка загрузки config.ini:\033[0m {e}\nИспользуются настройки 'по умолчанию'.")
+        return '', 20, 'domain-ip-resolve.txt', '', '', '', '', []


 def gateway_input(gateway):
@@ -149,64 +52,207 @@ def gateway_input(gateway):
        return gateway


-# Function to check if 'service' is specified in the configuration file
-def check_service_config(service):
+# Function to limit requests
+def get_semaphore(request_limit):
+    return defaultdict(lambda: Semaphore(request_limit))
+
+
+# Initialize semaphore for limiting requests
+def init_semaphores(request_limit):
+    return get_semaphore(request_limit)
+
+
+async def load_urls(url):
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(url)
+            response.raise_for_status()
+            text = response.text
+            lines = text.split('\n')
+            urls = {}
+            for line in lines:
+                if line.strip():
+                    service, url = line.split(': ', 1)
+                    urls[service.strip()] = url.strip()
+            return urls
+    except Exception as e:
+        print(f"Ошибка при загрузке списка платформ: {e}")
+        return {}
+
+
+async def load_dns_servers(url):
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(url)
+            response.raise_for_status()
+            text = response.text
+            lines = text.split('\n')
+            dns_servers = {}
+            for line in lines:
+                if line.strip():
+                    service, servers = line.split(': ', 1)
+                    dns_servers[service.strip()] = servers.strip().split()
+            return dns_servers
+    except Exception as e:
+        print(f"Ошибка при загрузке списка DNS серверов: {e}")
+        return {}
+
+
+async def get_cloudflare_ips():
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get("https://www.cloudflare.com/ips-v4/")
+            response.raise_for_status()
+            text = response.text
+            cloudflare_ips = set()
+            cidr_blocks = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2})', text)
+            for cidr in cidr_blocks:
+                ip_network = ipaddress.ip_network(cidr)
+                for ip in ip_network:
+                    cloudflare_ips.add(str(ip))
+            return cloudflare_ips
+    except Exception as e:
+        print("Ошибка при получении IP адресов Cloudflare:", e)
+        return set()
+
+
+async def resolve_domain(domain, resolver, semaphore, dns_server_name, null_ips_count, cloudflare_ips,
+                         cloudflare_ips_count):
+    async with semaphore:
+        try:
+            response = await resolver.resolve(domain)
+            ips = [ip.address for ip in response]
+            for ip_address in ips:
+                if ip_address in ('127.0.0.1', '0.0.0.0') or ip_address in resolver.nameservers:
+                    null_ips_count[0] += 1
+                elif ip_address in cloudflare_ips:
+                    cloudflare_ips_count[0] += 1
+                else:
+                    print(f"\033[36m{domain} IP адрес: {ip_address} получен от {dns_server_name}\033[0m")
+            return ips
+        except Exception as e:
+            print(f"\033[31mНе удалось разрешить {domain} через {dns_server_name}\033[0m")
+            return []
+
+
+async def resolve_dns(service, url, dns_servers, cloudflare_ips, unique_ips_all_services, semaphore, null_ips_count,
+                      cloudflare_ips_count):
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(url)
+            response.raise_for_status()
+            dns_names = response.text.split('\n')
+
+        print(f"\033[33mАнализ DNS имен платформы {service}...\033[0m")
+
+        tasks = []
+        for server_name, servers in dns_servers:
+            resolver = dns.asyncresolver.Resolver()
+            resolver.nameservers = servers
+            for domain in dns_names:
+                domain = domain.strip()
+                if domain:
+                    tasks.append(resolve_domain(domain, resolver, semaphore[server_name], server_name, null_ips_count,
+                                                cloudflare_ips, cloudflare_ips_count))
+
+        results = await asyncio.gather(*tasks)
+
+        unique_ips_current_service = set()
+        for result in results:
+            for ip_address in result:
+                if ip_address not in unique_ips_all_services:
+                    unique_ips_current_service.add(ip_address)
+                    unique_ips_all_services.add(ip_address)
+
+        return '\n'.join(unique_ips_current_service) + '\n'
+    except Exception as e:
+        print(f"Не удалось сопоставить IP адреса {service} его доменным именам.", e)
+        return ""
+
+
+def check_service_config(service, urls):
    if service:
        if service.strip().lower() == "all":
-            return list(urls.keys())  # Select all available services
+            return list(urls.keys())
        else:
            return [s.strip() for s in service.split(',')]
    else:
        selected_services = []
        while True:
-            if os.name == 'nt':  # Для пользователей Windows
-                os.system('cls')  # Очистить экран
-            else:
-                os.system('clear')
-            print("\nВыберите сервисы:\n")
-            print("0 - Отметить все")
+            print("\n\033[33mВыберите сервисы:\033[0m")
+            print("0. Выбрать все")
            for idx, (service, url) in enumerate(urls.items(), 1):
-                checkbox = "[*]" if service in selected_services else "[ ]"
-                print(f"{idx}. {service.capitalize()}  {checkbox}")
+                print(f"{idx}. {service.capitalize()}")

-            selection = input("\n\033[32mВведите номер сервиса\033[0m и нажмите Enter (Пустая строка "
-                              "и \033[32mEnter\033[0m для старта): ")
-            if selection == "0":
-                selected_services = list(urls.keys())
-            elif selection.isdigit():
-                idx = int(selection) - 1
-                if 0 <= idx < len(urls):
-                    service = list(urls.keys())[idx]
-                    if service in selected_services:
-                        selected_services.remove(service)
-                    else:
-                        selected_services.append(service)
-            elif selection == "":
-                break
+            selection = input("\nУкажите номера сервисов через пробел и нажмите \033[32mEnter\033[0m: ")
+            if selection.strip():
+                selections = selection.split()
+                if '0' in selections:  # User selected all services
+                    selected_services = list(urls.keys())
+                    break
+                else:
+                    selected_services = [list(urls.keys())[int(sel) - 1] for sel in selections if sel.isdigit()
+                                         and 1 <= int(sel) <= len(urls)]
+                    break
        return selected_services


-# Function to check if to include Cloudflare IPs based on configuration or user input
 def check_include_cloudflare(cloudflare):
    if cloudflare.lower() == 'yes':
        return True
    elif cloudflare.lower() == 'no':
        return False
    else:
-        return input("Исключить IP адреса Cloudflare из итогового списка? (\033[32myes\033[0m "
+        return input("\nИсключить IP адреса Cloudflare из итогового списка? (\033[32myes\033[0m "
                     "- исключить, \033[32mEnter\033[0m - оставить): ").strip().lower() == "yes"


-# Function to process file format
+def check_dns_servers(dns_servers, dns_server_indices):
+    system_dns_servers = dns.asyncresolver.Resolver().nameservers
+    selected_dns_servers = []
+
+    dns_server_options = [('Системный DNS', system_dns_servers)] + list(dns_servers.items())
+
+    if dns_server_indices:
+        for idx in dns_server_indices:
+            if 0 <= idx <= len(dns_server_options):
+                selected_dns_servers.append((dns_server_options[idx][0], dns_server_options[idx][1]))
+        return selected_dns_servers
+
+    while True:
+        print("\n\033[33mКакие DNS сервера использовать?\033[0m")
+        print("0. Выбрать все")
+        for idx, (name, servers) in enumerate(dns_server_options, 1):
+            print(f"{idx}. {name}: {', '.join(servers)}")
+
+        selection = input("\nУкажите номера DNS серверов через пробел и нажмите \033[32mEnter\033[0m: ")
+        if selection.strip():
+            selections = selection.split()
+            if '0' in selections:  # User selected all DNS servers
+                selected_dns_servers = dns_server_options
+                break
+            else:
+                for sel in selections:
+                    if sel.isdigit():
+                        sel = int(sel)
+                        if 1 <= sel <= len(dns_server_options):
+                            selected_dns_servers.append(
+                                (dns_server_options[sel - 1][0], dns_server_options[sel - 1][1]))
+                break
+
+    return selected_dns_servers
+
+
 def process_file_format(filename, filetype, gateway):
    if not filetype:
-        filetype = input("\nВыберите в каком формате сохранить файл: \n\033[32mwin\033[0m"
-                         " - 'route add %IP% mask %mask% %gateway%', \033[32munix\033[0m"
-                         " - 'ip route %IP%/%mask% %gateway%', \033[32mcidr\033[0m"
-                         " - 'IP/mask', \033[32mEnter\033[0m - только IP: ")
+        filetype = input("\n\033[33mВ каком формате сохранить файл?\033[0m"
+                         "\n\033[32mwin\033[0m - route add IP mask MASK GATEWAY"
+                         "\n\033[32munix\033[0m - ip route IP/MASK GATEWAY"
+                         "\n\033[32mcidr\033[0m - IP/MASK"
+                         "\n\033[32mПустое значение\033[0m - только IP"
+                         "\nВаш выбор: ")

    if filetype.lower() in ['win', 'unix']:
-        # Обработка файлов разных форматов
        gateway = gateway_input(gateway)

        try:
@@ -224,7 +270,6 @@ def process_file_format(filename, filetype, gateway):
                    elif filetype.lower() == 'unix':
                        file.write(f"ip route {ip.strip()}/32 {gateway}\n")
    elif filetype.lower() == 'cidr':
-        # Обработка CIDR формата
        try:
            with open(filename, 'r', encoding='utf-8-sig') as file:
                ips = file.readlines()
@@ -235,47 +280,69 @@ def process_file_format(filename, filetype, gateway):
        if ips:
            with open(filename, 'w', encoding='utf-8-sig') as file:
                for ip in ips:
-                    file.write(f"{ip.strip()}/32\n")  # Assuming /32 subnet mask for all IPs
+                    file.write(f"{ip.strip()}/32\n")
    else:
-        # Сохранить только IP адреса
        pass


-def main():
-    # Read parameters from the configuration file
-    service, threads, filename, cloudflare, filetype, gateway, run_command = read_config('config.ini')
+async def main():
+    # Load configuration
+    service, request_limit, filename, cloudflare, filetype, gateway, run_command, dns_server_indices = read_config('config.ini')

-    total_resolved_domains = 0
-    selected_services = check_service_config(service)
+    # Load URLs
+    platform_db_url = "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/platformdb"
+    urls = await load_urls(platform_db_url)

-    # Check if to include Cloudflare IPs based on configuration or user input
+    # Get selected services from user
+    selected_services = check_service_config(service, urls)
+
+    # Load DNS servers
+    dns_db_url = "https://raw.githubusercontent.com/Ground-Zerro/DomainMapper/main/dnsdb"
+    dns_servers = await load_dns_servers(dns_db_url)
+
+    # Get selected DNS servers from config or user
+    selected_dns_servers = check_dns_servers(dns_servers, dns_server_indices)
+
+    # Get Cloudflare IP addresses
+    cloudflare_ips = await get_cloudflare_ips()
+
+    # Check if Cloudflare IPs should be included or excluded
    include_cloudflare = check_include_cloudflare(cloudflare)

-    # Set to store unique IP addresses across all services
    unique_ips_all_services = set()
+    semaphore = init_semaphores(request_limit)
+    null_ips_count = [0]
+    cloudflare_ips_count = [0]
+    tasks = []

-    # DNS resolution for selected services
-    with open(filename, 'w', encoding='utf-8-sig') as file:  # Open file for writing
-        for service in selected_services:
-            result = resolve_dns_and_write(service, urls[service], unique_ips_all_services, include_cloudflare, threads)
-            file.write(result)  # Write unique IPs directly to the file
-            total_resolved_domains += len(result.split('\n')) - 1
+    for service in selected_services:
+        tasks.append(resolve_dns(service, urls[service], selected_dns_servers, cloudflare_ips, unique_ips_all_services,
+                                 semaphore, null_ips_count, cloudflare_ips_count))

-    print("\nПроверка завершена.")
-    print(f"Сопоставлено IP адресов доменам: {total_resolved_domains}")
+    results = await asyncio.gather(*tasks)
+
+    with open(filename, 'w', encoding='utf-8-sig') as file:
+        for result in results:
+            file.write(result)
+
+    print("\n\033[33mПроверка завершена.\033[0m")
+    print(
+        f"Использовались DNS сервера: {', '.join([f'{pair[0]} ({', '.join(pair[1])})' for pair in selected_dns_servers])}")
+    if include_cloudflare:
+        print(f"Исключено IP-адресов Cloudflare: {cloudflare_ips_count[0]}")
+    print(f"Исключено IP-адресов 'заглушек': {null_ips_count[0]}")
+    print(f"Разрешено IP-адресов из DNS имен: {len(unique_ips_all_services)}")

-    # Asking for file format if filetype is not specified in the configuration file
    process_file_format(filename, filetype, gateway)

-    # Executing the command after the program is completed, if it is specified in the configuration file
-    if run_command is not None and run_command.strip():
+    if run_command:
        print("\nВыполнение команды после завершения скрипта...")
        os.system(run_command)
    else:
-        print("Результаты сохранены в файл:", filename)
-        if os.name == 'nt':  # Для пользователей Windows при запуске из проводника
+        print("\nРезультаты сохранены в файл:", filename)
+        if os.name == 'nt':
            input("Нажмите \033[32mEnter\033[0m для выхода...")


 if __name__ == "__main__":
-    main()
+    asyncio.run(main())