Browse Source

small fixes

master
rra 7 years ago
parent
commit
59fd15dd7d
  1. 5
      fedicrawler.py

5
fedicrawler.py

@ -6,8 +6,9 @@ from multiprocessing.dummy import Pool as ThreadPool
start_url = 'https://post.lurk.org' start_url = 'https://post.lurk.org'
activity = '' activity = ''
peers_info ='/api/v1/instance/peers'
pool = ThreadPool(128) pool = ThreadPool(256)
def get_peers(instance): def get_peers(instance):
@ -74,5 +75,5 @@ pool.join()
with open('instance_scrape.json','w') as f: with open('instance_scrape.json','w') as f:
f.write(json.dumps(instances,indent=4)) f.write(json.dumps(scrape,indent=4))
#f.write('\n'.join(text) #f.write('\n'.join(text)
Loading…
Cancel
Save