first step towards a RSS multiverse

.gitignore

@@ -0,0 +1,3 @@
+__pycache__*
+.venv*
+feeds.json

Makefile

@@ -0,0 +1,9 @@
+default: run
+
+setup:
+	@python3 -m venv .venv && \
+	.venv/bin/pip install -r requirements.txt
+
+run:
+	@python3 start.py
+

README.md

@@ -1,3 +1,19 @@
 # Multifeeder
 
 Multifeeding RSS streams into a point of access to publish from! (work-in-slow-progress)
+
+## Using the multifeeder
+
+To install: make a virtual environment + install the dependencies.
+
+`make setup`
+
+To use: activate the virtual environment ...
+
+`make activate`
+
+... and run the Flask application.
+
+`make`
+
+Open `localhost:5678` in a browser and there we go!

feeds.txt

@@ -0,0 +1,3 @@
+https://vvvvvvaria.org/feeds/all-nl.rss.xml
+https://vvvvvvaria.org/en/feeds/all-en.rss.xml
+https://post.lurk.org/tags/varia.rss

feedtools.py

@@ -0,0 +1,82 @@
+import feedparser
+from simpledatabase import SimpleDatabase
+import json
+from datetime import date, timedelta
+
+def load():
+	""" Load all feeds """
+	feeds = open('feeds.txt').readlines()
+	db = SimpleDatabase('feeds.json', 'feeds.log')
+
+	tmp = {}
+	for x, feed in enumerate(feeds):
+		parsed = feedparser.parse(feed)
+		x = str(x)
+		tmp[x] = {}
+		tmp[x]['title'] = parsed.feed.title
+		tmp[x]['link'] = parsed.feed.link
+		tmp[x]['description'] = parsed.feed.description
+		tmp[x]['entries'] = parsed.entries
+	db.update(tmp)
+	return db
+
+def latest(num):
+	""" Placeholder request """
+	request = [
+		{
+			"feedtitle" : "Varia EN",
+			"post": "hello world",
+			"date" : "Monday 15th of February 2021",
+			"url" : "https://vvvvvvaria.org/en/rr-wireless-imagination-1.html"
+		}
+	]
+	return request
+
+def today():
+	""" Collect posts from today """
+	db = load()
+
+	today = str(date.today()).split('-')
+	today_year = "{:02d}".format(int(today[0]))
+	today_month = "{:02d}".format(int(today[1]))
+	today_day = "{:02d}".format(int(today[2]))
+	print('TODAY     =', today_year, today_month, today_day)
+
+	request = []
+	for x, feed in db.items():
+		for post in feed['entries']:
+			if post['published_parsed']:				
+				year = "{:02d}".format(post['published_parsed'][0])
+				month = "{:02d}".format(post['published_parsed'][1])
+				day = "{:02d}".format(post['published_parsed'][2] + 1)
+				print('POST DATE =', year, month, day)
+
+				# Check if this post is published today
+				if year == today_year:
+					if month == today_month:
+						if day == today_day:
+							request.append(post)
+	return request
+
+def past(days):
+	""" Collect posts from this week """
+	db = load()
+
+	point_in_the_past = date.today() - timedelta(int(days))
+	print(f"{ days } days in the past =", point_in_the_past)
+	
+	request = []
+	for x, feed in db.items():
+		for post in feed['entries']:
+			if post['published_parsed']:				
+				year = post['published_parsed'][0]
+				month = post['published_parsed'][1]
+				day = post['published_parsed'][2]
+				
+				post_date = date(year, month, day)
+				print("post date =",post_date)
+				
+				if post_date > point_in_the_past:
+					request.append(post)
+
+	return request

requirements.txt

@@ -0,0 +1,3 @@
+flask
+feedparser
+pathlib

simpledatabase.py

@@ -0,0 +1,61 @@
+from os import environ, mkdir
+from os.path import exists
+from pathlib import Path 
+from logging import DEBUG, INFO, basicConfig, getLogger 
+from json import dumps, loads 
+
+class SimpleDatabase(dict):
+	"""A simple database.
+	It is a dictionary which saves to disk on all writes. It is optimised for
+	ease of hacking and accessibility and not for performance or efficiency.
+
+	Written by decentral1se, as part of:
+	https://git.vvvvvvaria.org/decentral1se/xbotlib/src/branch/main/xbotlib.py
+	"""
+
+	def __init__(self, filename, log, *args, **kwargs):
+		"""Initialise the object."""
+		self.filename = Path(filename).absolute()
+		self.log = getLogger(__name__)
+
+		self._loads()
+		self.update(*args, **kwargs)
+
+	def _loads(self):
+		"""Load the database."""
+		if not exists(self.filename):
+			return
+
+		try:
+			with open(self.filename, "r") as handle:
+				self.update(loads(handle.read()))
+		except Exception as exception:
+			message = f"Loading file storage failed: {exception}"
+			self.log.error(message, exc_info=exception)
+			exit(1)
+
+	def _dumps(self):
+		"""Save the databse to disk."""
+		try:
+			with open(self.filename, "w") as handle:
+				handle.write(dumps(self, indent=4, sort_keys=True))
+		except Exception as exception:
+			message = f"Saving file storage failed: {exception}"
+			self.log.error(message, exc_info=exception)
+			exit(1)
+
+	def __setitem__(self, key, val):
+		"""Write data to the database."""
+		super().__setitem__(key, val)
+		self._dumps()
+
+	def __delitem__(self, key):
+		"""Remove data from the database."""
+		super().__delitem__(key)
+		self._dumps()
+
+	def update(self, *args, **kwargs):
+		"""Update the database."""
+		for k, v in dict(*args, **kwargs).items():
+			self[k] = v
+		self._dumps()

start.py

@@ -0,0 +1,36 @@
+import flask
+import feedtools
+
+APP = flask.Flask(__name__, 
+	static_url_path="", 
+	static_folder="static",
+	template_folder="templates")
+
+@APP.route("/")
+def index():
+	db = feedtools.load()
+	template = flask.render_template(
+		"index.html",
+		feeds=db,
+	)
+	return template
+
+# @APP.route("/API/latest/<num>")
+# def latest(num):
+# 	request = feedtools.latest(num)
+# 	return request
+
+@APP.route("/API/today/")
+def today():
+	request = feedtools.today()
+	return str(request)
+
+@APP.route("/API/past/<days>")
+def past(days):
+	request = feedtools.past(days)
+	return str(request)
+
+if __name__ == "__main__":
+	APP.debug = True
+	APP.run(port=5678)
+

static/css/stylesheet.css

@@ -0,0 +1,37 @@
+body{
+	background-color: pink;
+	color: red;
+	margin: 1em;
+}
+h1,
+h2{
+	margin: 1em;
+}
+h2{
+	color: fuchsia;
+}
+
+table{
+	table-layout: fixed;
+	width: 100%;
+	border-collapse: collapse;
+}
+table td{
+	padding: 1em 2em;
+}
+table td:first-of-type{
+	width: 100px;
+}
+table tr{
+	border-bottom: 20px solid white;
+}
+
+section#api{
+	margin: 6em 0;
+	color: fuchsia;
+}
+section#api div.accesspoint{
+	margin: 0;
+	padding: 1em;
+	border-bottom: 20px solid yellow;
+}

templates/index.html

@@ -0,0 +1,39 @@
+<!DOCTYPE html>
+<html>
+<head>
+	<meta charset="utf-8">
+	<title>multifeeder</title>
+	<link rel="stylesheet" type="text/css" href="/css/stylesheet.css">
+</head>
+<body>
+	<h1>multifeeder</h1>
+	<table>
+		<tbody>
+		{% for x, feed in feeds.items() %}
+			<tr>
+				<td>{{ feed.title }}</td>
+				<td>{{ feed.link }}</td>
+			</tr>
+		{% endfor %}
+		</tbody>
+	</table>
+	<section id="api">
+		<h2>API</h2>
+		<!-- <div class="accesspoint">
+			/API/latest/[num] 
+			<br><br>
+			(not there yet)
+		</div> -->
+		<div class="accesspoint">
+			/API/today/
+			<br><br>
+			For example: <a href="/API/today/" target="_blank">localhost:5678/API/today/</a>
+		</div>
+		<div class="accesspoint">
+			/API/past/[days]
+			<br><br>
+			For example: <a href="/API/past/30" target="_blank">localhost:5678/API/past/30</a>
+		</div>
+	</section>
+</body>
+</html>