|
@ -6,8 +6,9 @@ from multiprocessing.dummy import Pool as ThreadPool |
|
|
|
|
|
|
|
|
start_url = 'https://post.lurk.org' |
|
|
start_url = 'https://post.lurk.org' |
|
|
activity = '' |
|
|
activity = '' |
|
|
|
|
|
peers_info ='/api/v1/instance/peers' |
|
|
|
|
|
|
|
|
pool = ThreadPool(128) |
|
|
pool = ThreadPool(256) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_peers(instance): |
|
|
def get_peers(instance): |
|
@ -74,5 +75,5 @@ pool.join() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
with open('instance_scrape.json','w') as f: |
|
|
with open('instance_scrape.json','w') as f: |
|
|
f.write(json.dumps(instances,indent=4)) |
|
|
f.write(json.dumps(scrape,indent=4)) |
|
|
#f.write('\n'.join(text) |
|
|
#f.write('\n'.join(text) |