Browse Source

minor tweaks to fedicrawler & scrape 05-05-2020

master
rra 10 months ago
parent
commit
3f5d2bbad0
2 changed files with 158708 additions and 198035 deletions
  1. +4
    -2
      fedicrawler.py
  2. +158704
    -198033
      instance_scrape.json

+ 4
- 2
fedicrawler.py View File

@ -1,5 +1,5 @@
#!/bin/env python3
# fediscraper v3
# fedicrawler v4
import json, requests
from multiprocessing.dummy import Pool as ThreadPool
@ -22,12 +22,15 @@ def not_gab(instance):
if 'gab.best'in instance:
print('GAB', instance)
return False
# the ones below are mostly used for testing apps
elif 'ngrok.io' in instance:
print('NGROK', instance)
return False
elif 'glitch.me' in instance:
print('GLITCH', instance)
return False
elif 'netlify.app' in instance:
print('NETLIFY', instance)
else:
return True
else:
@ -141,7 +144,6 @@ if r. status_code == 200:
instance_info = pool.map(get_nodeinfo, instances)
scrape = {}
instances_list = list(instances)
for count, value in enumerate(instances_list):


+ 158704
- 198033
instance_scrape.json
File diff suppressed because it is too large
View File


Loading…
Cancel
Save