This script makes Hugo page bundles out of Hashtag feeds on a Mastodon Hometown or Glitchsoc instance.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

137 lines
4.3 KiB

# hashtag publishing bot
# © 2021 roel roscam abbing agplv3
# Makes Hugo posts out of hashtag feeds on Mastodon.
# Requires an account on the Mastodon instance configured.
# Currently does not do any thread recreation and only handles images
import os
import requests
import shutil
import jinja2
from mastodon import Mastodon
import config_hashtag_bot
def login_mastodon_bot():
mastodon = Mastodon(
client_id = 'publishbot_clientcred.secret',
api_base_url = config_hashtag_bot.instance
to_file = 'publishbot_usercred.secret', scopes=['read']
return mastodon
def create_frontmatter(post_metadata):
Parse post metadata and return it as HUGO frontmatter
frontmatter = ""
return frontmatter
def download_media(post_directory, media_attachments):
Download media attached to posts. N.b. currently only images
for item in media_attachments:
if item['type'] == 'image':
image = localize_media_url(item['url'])
#TODO check whether this needs to handle delete & redraft with different images
if not os.path.exists(os.path.join(post_directory, image)):
#download image
response = requests.get(item['url'], stream=True)
with open(os.path.join(post_directory, image), 'wb') as img_file:
shutil.copyfileobj(response.raw, img_file)
print('Downloaded cover image', image)
def create_post(post_directory, post_metadata):
Create Hugo posts based on Toots/posts retuned in timeline.
if not os.path.exists(post_directory):
with open(os.path.join(post_directory,'index.html'),'w') as f:
post = template.render(post_metadata=post_metadata)
download_media(post_directory, post_metadata['media_attachments'])
def localize_media_url(url):
Returns the filename, used also as custom jinja filter
return url.split('/')[-1]
def filter_mastodon_urls(content):
Filters out Mastodon generated URLS for tags
e.g. <a href="" class="mention hashtag" rel="tag">
Used also as custom jinja filter
return content
mastodon = login_mastodon_bot()
output_dir = config_hashtag_bot.output_dir
env = jinja2.Environment(
env.filters['localize_media_url'] = localize_media_url
env.filters['filter_mastodon_urls'] = filter_mastodon_urls
template = env.get_template('')
if not os.path.exists(output_dir):
for hashtag in config_hashtag_bot.hashtags:
hashtag_dir = os.path.join(output_dir, hashtag)
if not os.path.exists(hashtag_dir):
existing_posts = os.listdir(hashtag_dir) #list all existing posts
timeline = mastodon.timeline_hashtag(hashtag, local=True, only_media=True) #returns max 20 queries and only with media
timeline = mastodon.fetch_remaining(timeline) #returns all the rest n.b. can take a while because of rate limit
for post_metadata in timeline:
post_dir = os.path.join(hashtag_dir, str(post_metadata['id']))
#if there is a post in the feed we dont already have locally, make it
if str(post_metadata['id']) not in existing_posts:
if not post_metadata['local_only']: #if you get an error here then you are using vanilla Mastodon, this is a Hometown or Glitch only feature
create_post(post_dir, post_metadata)
# if we already have the post do nothing, possibly update
elif str(post_metadata['id']) in existing_posts:
#update_post(post_dir, post_metadata)
existing_posts.remove(str(post_metadata['id'])) # create list of posts which have not been returned in the feed
for post in existing_posts:
print('deleted', post) #rm posts that exist but are no longer returned in feed