diff --git a/.github/workflows/weekly_ip_list.yaml b/.github/workflows/weekly_ip_list.yaml index 06b16cb..d5416fc 100644 --- a/.github/workflows/weekly_ip_list.yaml +++ b/.github/workflows/weekly_ip_list.yaml @@ -1,4 +1,4 @@ -name: 🔁 Update Voice IP lists +name: 🔁 Update IP lists on: schedule: @@ -6,7 +6,7 @@ on: workflow_dispatch: jobs: - update_voice_lists: + update_lists: runs-on: ubuntu-latest steps: @@ -24,6 +24,10 @@ jobs: pip install dnspython pip install aiofiles + - name: ↗️ Resolve base domains + run: | + python ./src/generate_base_ip_list.py + - name: 🌐 Resolve atlanta run: | python ./src/generate_voice_ip_list.py atlanta @@ -164,11 +168,9 @@ jobs: run: | python ./src/generate_voice_ip_list.py oregon - - name: ↗️ Update ip_list.txt + - name: 📁 Generating configurations run: | - cp ./amnezia-voice-list.new.txt ./amnezia-voice-list.txt - cp ./voice-ip-list.new.txt ./voice-ip-list.txt - cp ./voice-domain-list.new.txt ./voice-domain-list.txt + python ./src/generate_singlefiles.py - name: 💫 Commit the changes run: | diff --git a/src/generate_base_ip_list.py b/src/generate_base_ip_list.py new file mode 100644 index 0000000..431bb6b --- /dev/null +++ b/src/generate_base_ip_list.py @@ -0,0 +1,71 @@ +import json +import os +import sys +import asyncio +import aiofiles +import dns.asyncresolver +from datetime import datetime + +resolver = dns.asyncresolver.Resolver() +resolver.nameservers = ['8.8.8.8', '8.8.4.4'] + +async def get_a_records(domain, retries=3, delay=1): + for attempt in range(retries): + try: + answer = await resolver.resolve(domain, 'A') + return domain, " ".join([rdata.to_text() for rdata in answer]) + except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN) as e: + return domain, f"Error: {str(e)}" + except Exception as e: + if attempt < retries - 1: + await asyncio.sleep(delay) + else: + return domain, f"Error: {str(e)}" + +async def process_domain(domain, results): + domain, a_records = await get_a_records(domain) + if a_records and "Error" not in a_records: + results.append({"hostname": domain, "ip": a_records.split()[0]}) + print(f"\r{domain} -> {a_records}", end="") + +async def main(): + tasks = [] + results = [] + max_concurrent_tasks = 1000 + semaphore = asyncio.Semaphore(max_concurrent_tasks) + + async def limited_task(domain): + async with semaphore: + await process_domain(domain, results) + + # Read the list of domains from the file + try: + async with aiofiles.open('./base-domain-list.txt', 'r') as f: + domains = await f.readlines() + except FileNotFoundError: + print("Domain list file not found.") + sys.exit(1) + + domains = [domain.strip() for domain in domains] # Clean up line breaks + + for domain in domains: + tasks.append(limited_task(domain)) + + await asyncio.gather(*tasks) + + if len(results) == 0: + print("\nNo results found.") + return + + # Display results instead of saving them + async with aiofiles.open('./base-ip-list.txt', 'w') as f: + await f.write("\n".join(map(lambda a: a["ip"], results)) + "\n") + + async with aiofiles.open('./amnezia-base-list.json', 'w') as f: + await f.write(json.dumps(results, indent=4)) + +if __name__ == "__main__": + start_time = datetime.now() + asyncio.run(main()) + end_time = datetime.now() + print(f"\nFinished! Time taken: {end_time - start_time}") diff --git a/src/generate_singlefiles.py b/src/generate_singlefiles.py new file mode 100644 index 0000000..f7a1d30 --- /dev/null +++ b/src/generate_singlefiles.py @@ -0,0 +1,67 @@ +import json +import asyncio +import aiofiles +from datetime import datetime +import os + +async def append_to_json(file_path, new_data): + # Check if the file exists and create if not + if not os.path.exists(file_path): + async with aiofiles.open(file_path, 'w') as f: + await f.write(json.dumps([], indent=4)) + + # Read and update the existing data + async with aiofiles.open(file_path, 'r') as f: + try: + existing_data = json.loads(await f.read()) + except json.JSONDecodeError: + existing_data = [] # In case of empty or invalid JSON + + existing_data.extend(new_data) # Efficiently append new data + + # Write the updated data back to the file + async with aiofiles.open(file_path, 'w') as f: + await f.write(json.dumps(existing_data, indent=4)) + + +async def process_region(region: str): + try: + region_file_path = os.path.join("regions", region, f'amnezia-{region}-voice-list.json') + async with aiofiles.open(region_file_path, 'r') as f: + return json.loads(await f.read()) + except Exception as e: + print(f"Error processing region {region}: {e}") + return [] # Return empty list on failure + + +async def main(): + regions = os.listdir("regions") + data = [] + + # Process regions concurrently + tasks = [process_region(region) for region in regions] + region_data = await asyncio.gather(*tasks) + + for row in region_data: + if row: + data.extend(row) + + # Write to text files + ip_list = "\n".join([voice["ip"] for voice in data]) + domain_list = "\n".join([voice["hostname"] for voice in data]) + + async with aiofiles.open('voice-ip-list.txt', 'w') as f: + await f.write(ip_list + "\n") + + async with aiofiles.open('voice-domain-list.txt', 'w') as f: + await f.write(domain_list + "\n") + + # Append new data to the JSON file + await append_to_json('amnezia-voice-list.json', data) + + +if __name__ == "__main__": + start_time = datetime.now() + asyncio.run(main()) + end_time = datetime.now() + print(f"\nFinished! Time taken: {end_time - start_time}") diff --git a/src/generate_voice_ip_list.py b/src/generate_voice_ip_list.py index d1e572a..27db22a 100644 --- a/src/generate_voice_ip_list.py +++ b/src/generate_voice_ip_list.py @@ -31,41 +31,17 @@ async def get_a_records(domain, retries=3, delay=1): else: return domain, f"Error: {str(e)}" - async def process_domain(region, id, results): domain = f"{region}{id}.discord.gg" domain, a_records = await get_a_records(domain) if a_records and "Error" not in a_records: results.append({"hostname": domain, "ip": a_records}) print(f"\r{domain} -> {a_records}", end="") - - -async def append_to_json(file_path, new_data): - # Check if file exists - if not os.path.exists(file_path): - # If file doesn't exist, create it and initialize with an empty list (or other default structure) - async with aiofiles.open(file_path, 'w') as f: - await f.write(json.dumps([], indent=4)) - - # Read existing data - async with aiofiles.open(file_path, 'r') as f: - try: - existing_data = json.loads(await f.read()) - except json.JSONDecodeError: - existing_data = [] # In case the file is empty or doesn't contain valid JSON - - # Append new data - for row in new_data: - existing_data.append(row) - - # Write updated data back to the file - async with aiofiles.open(file_path, 'w') as f: - await f.write(json.dumps(existing_data, indent=4)) async def main(): tasks = [] results = [] - max_concurrent_tasks = 500 + max_concurrent_tasks = 1000 semaphore = asyncio.Semaphore(max_concurrent_tasks) async def limited_task(region, id): @@ -98,15 +74,6 @@ async def limited_task(region, id): async with aiofiles.open(os.path.join(region_dir, f'amnezia-{region}-voice-list.json'), 'w') as f: await f.write(json.dumps(results, indent=4)) - async with aiofiles.open('voice-ip-list.new.txt', 'a') as f: - await f.write("\n".join(map(lambda a: a["ip"], results)) + "\n") - - async with aiofiles.open('voice-domain-list.new.txt', 'a') as f: - await f.write("\n".join(map(lambda a: a["hostname"], results)) + "\n") - - await append_to_json('amnezia-voice-list.json', results) - - if __name__ == "__main__": start_time = datetime.now() asyncio.run(main())