From 64baad0c98e7e1173a27cb8d85e9fce5eecc6d63 Mon Sep 17 00:00:00 2001 From: manetta Date: Tue, 16 Feb 2021 23:26:03 +0100 Subject: [PATCH] first step towards a RSS multiverse --- .gitignore | 3 ++ Makefile | 9 +++++ README.md | 16 ++++++++ feeds.txt | 3 ++ feedtools.py | 82 +++++++++++++++++++++++++++++++++++++++ requirements.txt | 3 ++ simpledatabase.py | 61 +++++++++++++++++++++++++++++ start.py | 36 +++++++++++++++++ static/css/stylesheet.css | 37 ++++++++++++++++++ templates/index.html | 39 +++++++++++++++++++ 10 files changed, 289 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 feeds.txt create mode 100644 feedtools.py create mode 100644 requirements.txt create mode 100644 simpledatabase.py create mode 100644 start.py create mode 100644 static/css/stylesheet.css create mode 100644 templates/index.html diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..50dc0d8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__* +.venv* +feeds.json diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..58c155b --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +default: run + +setup: + @python3 -m venv .venv && \ + .venv/bin/pip install -r requirements.txt + +run: + @python3 start.py + diff --git a/README.md b/README.md index a5be765..45b1a86 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,19 @@ # Multifeeder Multifeeding RSS streams into a point of access to publish from! (work-in-slow-progress) + +## Using the multifeeder + +To install: make a virtual environment + install the dependencies. + +`make setup` + +To use: activate the virtual environment ... + +`make activate` + +... and run the Flask application. + +`make` + +Open `localhost:5678` in a browser and there we go! diff --git a/feeds.txt b/feeds.txt new file mode 100644 index 0000000..8287da4 --- /dev/null +++ b/feeds.txt @@ -0,0 +1,3 @@ +https://vvvvvvaria.org/feeds/all-nl.rss.xml +https://vvvvvvaria.org/en/feeds/all-en.rss.xml +https://post.lurk.org/tags/varia.rss \ No newline at end of file diff --git a/feedtools.py b/feedtools.py new file mode 100644 index 0000000..0e3dfc7 --- /dev/null +++ b/feedtools.py @@ -0,0 +1,82 @@ +import feedparser +from simpledatabase import SimpleDatabase +import json +from datetime import date, timedelta + +def load(): + """ Load all feeds """ + feeds = open('feeds.txt').readlines() + db = SimpleDatabase('feeds.json', 'feeds.log') + + tmp = {} + for x, feed in enumerate(feeds): + parsed = feedparser.parse(feed) + x = str(x) + tmp[x] = {} + tmp[x]['title'] = parsed.feed.title + tmp[x]['link'] = parsed.feed.link + tmp[x]['description'] = parsed.feed.description + tmp[x]['entries'] = parsed.entries + db.update(tmp) + return db + +def latest(num): + """ Placeholder request """ + request = [ + { + "feedtitle" : "Varia EN", + "post": "hello world", + "date" : "Monday 15th of February 2021", + "url" : "https://vvvvvvaria.org/en/rr-wireless-imagination-1.html" + } + ] + return request + +def today(): + """ Collect posts from today """ + db = load() + + today = str(date.today()).split('-') + today_year = "{:02d}".format(int(today[0])) + today_month = "{:02d}".format(int(today[1])) + today_day = "{:02d}".format(int(today[2])) + print('TODAY =', today_year, today_month, today_day) + + request = [] + for x, feed in db.items(): + for post in feed['entries']: + if post['published_parsed']: + year = "{:02d}".format(post['published_parsed'][0]) + month = "{:02d}".format(post['published_parsed'][1]) + day = "{:02d}".format(post['published_parsed'][2] + 1) + print('POST DATE =', year, month, day) + + # Check if this post is published today + if year == today_year: + if month == today_month: + if day == today_day: + request.append(post) + return request + +def past(days): + """ Collect posts from this week """ + db = load() + + point_in_the_past = date.today() - timedelta(int(days)) + print(f"{ days } days in the past =", point_in_the_past) + + request = [] + for x, feed in db.items(): + for post in feed['entries']: + if post['published_parsed']: + year = post['published_parsed'][0] + month = post['published_parsed'][1] + day = post['published_parsed'][2] + + post_date = date(year, month, day) + print("post date =",post_date) + + if post_date > point_in_the_past: + request.append(post) + + return request \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e5a1bbe --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +flask +feedparser +pathlib diff --git a/simpledatabase.py b/simpledatabase.py new file mode 100644 index 0000000..bca829c --- /dev/null +++ b/simpledatabase.py @@ -0,0 +1,61 @@ +from os import environ, mkdir +from os.path import exists +from pathlib import Path +from logging import DEBUG, INFO, basicConfig, getLogger +from json import dumps, loads + +class SimpleDatabase(dict): + """A simple database. + It is a dictionary which saves to disk on all writes. It is optimised for + ease of hacking and accessibility and not for performance or efficiency. + + Written by decentral1se, as part of: + https://git.vvvvvvaria.org/decentral1se/xbotlib/src/branch/main/xbotlib.py + """ + + def __init__(self, filename, log, *args, **kwargs): + """Initialise the object.""" + self.filename = Path(filename).absolute() + self.log = getLogger(__name__) + + self._loads() + self.update(*args, **kwargs) + + def _loads(self): + """Load the database.""" + if not exists(self.filename): + return + + try: + with open(self.filename, "r") as handle: + self.update(loads(handle.read())) + except Exception as exception: + message = f"Loading file storage failed: {exception}" + self.log.error(message, exc_info=exception) + exit(1) + + def _dumps(self): + """Save the databse to disk.""" + try: + with open(self.filename, "w") as handle: + handle.write(dumps(self, indent=4, sort_keys=True)) + except Exception as exception: + message = f"Saving file storage failed: {exception}" + self.log.error(message, exc_info=exception) + exit(1) + + def __setitem__(self, key, val): + """Write data to the database.""" + super().__setitem__(key, val) + self._dumps() + + def __delitem__(self, key): + """Remove data from the database.""" + super().__delitem__(key) + self._dumps() + + def update(self, *args, **kwargs): + """Update the database.""" + for k, v in dict(*args, **kwargs).items(): + self[k] = v + self._dumps() diff --git a/start.py b/start.py new file mode 100644 index 0000000..b40a973 --- /dev/null +++ b/start.py @@ -0,0 +1,36 @@ +import flask +import feedtools + +APP = flask.Flask(__name__, + static_url_path="", + static_folder="static", + template_folder="templates") + +@APP.route("/") +def index(): + db = feedtools.load() + template = flask.render_template( + "index.html", + feeds=db, + ) + return template + +# @APP.route("/API/latest/") +# def latest(num): +# request = feedtools.latest(num) +# return request + +@APP.route("/API/today/") +def today(): + request = feedtools.today() + return str(request) + +@APP.route("/API/past/") +def past(days): + request = feedtools.past(days) + return str(request) + +if __name__ == "__main__": + APP.debug = True + APP.run(port=5678) + diff --git a/static/css/stylesheet.css b/static/css/stylesheet.css new file mode 100644 index 0000000..c04ad84 --- /dev/null +++ b/static/css/stylesheet.css @@ -0,0 +1,37 @@ +body{ + background-color: pink; + color: red; + margin: 1em; +} +h1, +h2{ + margin: 1em; +} +h2{ + color: fuchsia; +} + +table{ + table-layout: fixed; + width: 100%; + border-collapse: collapse; +} +table td{ + padding: 1em 2em; +} +table td:first-of-type{ + width: 100px; +} +table tr{ + border-bottom: 20px solid white; +} + +section#api{ + margin: 6em 0; + color: fuchsia; +} +section#api div.accesspoint{ + margin: 0; + padding: 1em; + border-bottom: 20px solid yellow; +} \ No newline at end of file diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..a5a08da --- /dev/null +++ b/templates/index.html @@ -0,0 +1,39 @@ + + + + + multifeeder + + + +

multifeeder

+ + + {% for x, feed in feeds.items() %} + + + + + {% endfor %} + +
{{ feed.title }}{{ feed.link }}
+
+

API

+ +
+ /API/today/ +

+ For example: localhost:5678/API/today/ +
+
+ /API/past/[days] +

+ For example: localhost:5678/API/past/30 +
+
+ + \ No newline at end of file