You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
83 lines
2.5 KiB
83 lines
2.5 KiB
5 years ago
|
#!/bin/env python3
|
||
|
|
# a tool for collecting mastodon /about/ & ToS and CoC pages.
|
||
|
|
# (c) roel roscam abbing 2020
|
||
|
|
# gplv3
|
||
|
|
import os, time, json
|
||
|
|
|
||
|
|
scrape_data = json.loads(open('instance_scrape.json').read())
|
||
|
|
|
||
|
|
output_dir = 'about_pages'
|
||
|
|
tos_dir = os.path.join(output_dir,'with_tos')
|
||
|
|
|
||
|
|
if not os.path.exists(output_dir):
|
||
|
|
os.mkdir(output_dir)
|
||
|
|
if not os.path.exists(tos_dir):
|
||
|
|
os.mkdir(tos_dir)
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def find_mastodon_instances(scrape_data):
|
||
|
|
mastodon_instances = []
|
||
|
|
|
||
|
|
for i in scrape_data:
|
||
|
|
if 'nodeinfo' in scrape_data[i].keys():
|
||
|
|
if 'software' in scrape_data[i]['nodeinfo'].keys():
|
||
|
|
if scrape_data[i]['nodeinfo']['software']['name'].lower() == 'mastodon':
|
||
|
|
mastodon_instances.append(i)
|
||
|
|
return mastodon_instances
|
||
|
|
|
||
|
|
|
||
|
|
def clear_cache(driver, timeout=60):
|
||
|
|
"""Clear the cookies and cache for the ChromeDriver instance."""
|
||
|
|
# navigate to the settings page
|
||
|
|
driver.get('chrome://settings/clearBrowserData')
|
||
|
|
|
||
|
|
# wait for the button to appear
|
||
|
|
wait = WebDriverWait(driver, timeout)
|
||
|
|
wait.until(get_clear_browsing_button)
|
||
|
|
|
||
|
|
# click the button to clear the cache
|
||
|
|
get_clear_browsing_button(driver).click()
|
||
|
|
|
||
|
|
# wait for the button to be gone before returning
|
||
|
|
wait.until_not(get_clear_browsing_button)
|
||
|
|
|
||
|
|
|
||
|
|
from selenium import webdriver # you need to set up a driver
|
||
|
|
from selenium.webdriver.firefox.options import Options
|
||
|
|
|
||
|
|
options = Options()
|
||
|
|
options.add_argument('-headless')
|
||
|
|
|
||
|
|
browser = webdriver.Firefox(options=options)
|
||
|
|
#browser.set_window_size(1024, 768) # set the window size that you need
|
||
|
|
|
||
|
|
for mi in find_mastodon_instances(scrape_data):
|
||
|
|
#clear_cache(browser)
|
||
|
|
|
||
|
|
browser.get('https://{}/about/more/'.format(mi))
|
||
|
|
page = browser.find_element_by_tag_name('html')
|
||
|
|
height=page.size['height']
|
||
|
|
browser.set_window_size(1400,height+100)
|
||
|
|
|
||
|
|
|
||
|
|
try:
|
||
|
|
bw = browser.find_element_by_css_selector('div .box-widget')
|
||
|
|
about_text = bw.find_element_by_css_selector('div .rich-formatting')
|
||
|
|
html = browser.execute_script('return arguments[0].innerHTML;',about_text)
|
||
|
|
if html:
|
||
|
|
output_dir = tos_dir
|
||
|
|
with open(os.path.join(output_dir,mi+'_tos.txt'),'w') as f:
|
||
|
|
f.write(html)
|
||
|
|
except Exception as e:
|
||
|
|
output_dir = 'about_pages'
|
||
|
|
print(e)
|
||
|
|
|
||
|
|
time.sleep(0.1)
|
||
|
|
print('taking screenshot of', mi)
|
||
|
|
browser.save_screenshot(os.path.join(output_dir,mi+'.png'))
|
||
|
|
with open(os.path.join(output_dir,mi+'_about.html'),'w') as f:
|
||
|
|
f.write(browser.page_source)
|
||
|
|
#browser.save_screenshot()
|
||
|
|
browser.quit()
|