rra/dafr
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

minor tweaks to fedicrawler & scrape 05-05-2020

Browse Source
This commit is contained in:
rra 2020-05-05 15:25:05 +02:00
parent c003a6ae96
commit 3f5d2bbad0
2 changed files with 247539 additions and 286866 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
fedicrawler.py
View File

@ -1,5 +1,5 @@
#!/bin/env python3 #!/bin/env python3
# fediscraper v3 # fedicrawler v4
import json, requests import json, requests
from multiprocessing.dummy import Pool as ThreadPool from multiprocessing.dummy import Pool as ThreadPool
@ -22,12 +22,15 @@ def not_gab(instance):
if 'gab.best'in instance: if 'gab.best'in instance:
print('GAB', instance) print('GAB', instance)
return False return False
# the ones below are mostly used for testing apps
elif 'ngrok.io' in instance: elif 'ngrok.io' in instance:
print('NGROK', instance) print('NGROK', instance)
return False return False
elif 'glitch.me' in instance: elif 'glitch.me' in instance:
print('GLITCH', instance) print('GLITCH', instance)
return False return False
elif 'netlify.app' in instance:
print('NETLIFY', instance)
else: else:
return True return True
else: else:
@ -141,7 +144,6 @@ if r. status_code == 200:
instance_info = pool.map(get_nodeinfo, instances) instance_info = pool.map(get_nodeinfo, instances)
scrape = {} scrape = {}
instances_list = list(instances) instances_list = list(instances)
for count, value in enumerate(instances_list): for count, value in enumerate(instances_list):

534399
instance_scrape.json
View File

File diff suppressed because it is too large Load Diff