working on making the script importable by about_collector

fedicrawler.py

@@ -1,20 +1,8 @@
 #!/bin/env python3
-# fedicrawler v4
+# fedicrawler v5
 
-import json, requests
-from multiprocessing.dummy import Pool as ThreadPool 
 from urllib.parse import urlparse
 
-start_url = 'https://post.lurk.org'
-activity = ''
-peers_info ='/api/v1/instance/peers'
-
-proxies = {
-'http':'socks5://localhost:12345',
-'http':'socks5://localhost:12345'}
-
-pool = ThreadPool(512)
-
 def not_gab(instance):
     #gab does some weird stuff wrt enumerating subdomains
     #example: epa1pu1qcxxyzcxher0u.gab.best
@@ -117,11 +105,23 @@ def get_instance_info(instance):
         info = {'error': str(e)}
     return info
 
-filters = [not_gab, only_netloc] #what to filter out
+def fedicrawler():
+
+    start_url = 'https://post.lurk.org'
+    activity = ''
+    peers_info ='/api/v1/instance/peers'
 
-instances = set([])
-r = requests.get(start_url+peers_info) # normal brain, initial peer list
-if r. status_code == 200:
+    proxies = {
+    'http':'socks5://localhost:12345',
+    'http':'socks5://localhost:12345'}
+
+    pool = ThreadPool(512)
+
+    filters = [not_gab, only_netloc] #what to filter out
+
+    instances = set([])
+    r = requests.get(start_url+peers_info) # normal brain, initial peer list
+    if r. status_code == 200:
         start_peers = pool.map(get_peers, r.json())  #expanding brain, get all peers of those initial peers
         for i in start_peers:
             if not i:
@@ -141,19 +141,22 @@ if r. status_code == 200:
 
         instances = set(multi_filter(filters,instances))
 
-instance_info = pool.map(get_nodeinfo, instances)
+    instance_info = pool.map(get_nodeinfo, instances)
 
-scrape = {}
-instances_list = list(instances)
+    scrape = {}
+    instances_list = list(instances)
 
-for count, value in enumerate(instances_list):
+    for count, value in enumerate(instances_list):
         scrape[value] = instance_info[count]
 
-print('found {} instances'.format(len(scrape)))
+    print('found {} instances'.format(len(scrape)))
 
-pool.close()
-pool.join()
+    pool.close()
+    pool.join()
 
-with open('instance_scrape.json','w') as f:
+    with open('instance_scrape.json','w') as f:
         f.write(json.dumps(scrape,indent=4))
         #f.write('\n'.join(text)
+
+if __name__ == '__main__':
+    fedicrawler()