You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
38 lines
982 B
38 lines
982 B
#!/bin/env python3
|
|
# fediscraper v1
|
|
|
|
import json, requests
|
|
|
|
|
|
start_url = 'https://post.lurk.org'
|
|
activity = ''
|
|
peers_info = '/api/v1/instance/peers'
|
|
instance_info = '/api/v1/instance'
|
|
|
|
instances = {set([])}
|
|
r = requests.get(start_url+peers_info)
|
|
if r. status_code == 200:
|
|
print('200 for '+start_url)
|
|
peers = r.json()
|
|
print('{} has {} peers'.format(start_url, len(peers)))
|
|
for count, peer in enumerate(peers):
|
|
instances.add(peer)
|
|
try:
|
|
r = requests.get('https://'+peer+peers_info, timeout=10)
|
|
if r.status_code == 200:
|
|
print(peer, 'peers with', len(r.json()))
|
|
for i in r.json():
|
|
if i not in instances:
|
|
instances.add(i)
|
|
print('added {}, n={}'.format(i,len(instances)))
|
|
else:
|
|
print(i.status_code, 'on', peer)
|
|
except Exception as e:
|
|
print('failure for', peer)
|
|
# instances[peer] = {'error':e}
|
|
print(e)
|
|
|
|
text = list(filter(None.__ne__, instances))
|
|
|
|
with open('instance_scrape.txt','w') as f:
|
|
f.write('\n'.join(text))
|