diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..50dc0d8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__* +.venv* +feeds.json diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..58c155b --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +default: run + +setup: + @python3 -m venv .venv && \ + .venv/bin/pip install -r requirements.txt + +run: + @python3 start.py + diff --git a/README.md b/README.md index a5be765..45b1a86 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,19 @@ # Multifeeder Multifeeding RSS streams into a point of access to publish from! (work-in-slow-progress) + +## Using the multifeeder + +To install: make a virtual environment + install the dependencies. + +`make setup` + +To use: activate the virtual environment ... + +`make activate` + +... and run the Flask application. + +`make` + +Open `localhost:5678` in a browser and there we go! diff --git a/feeds.txt b/feeds.txt new file mode 100644 index 0000000..8287da4 --- /dev/null +++ b/feeds.txt @@ -0,0 +1,3 @@ +https://vvvvvvaria.org/feeds/all-nl.rss.xml +https://vvvvvvaria.org/en/feeds/all-en.rss.xml +https://post.lurk.org/tags/varia.rss \ No newline at end of file diff --git a/feedtools.py b/feedtools.py new file mode 100644 index 0000000..0e3dfc7 --- /dev/null +++ b/feedtools.py @@ -0,0 +1,82 @@ +import feedparser +from simpledatabase import SimpleDatabase +import json +from datetime import date, timedelta + +def load(): + """ Load all feeds """ + feeds = open('feeds.txt').readlines() + db = SimpleDatabase('feeds.json', 'feeds.log') + + tmp = {} + for x, feed in enumerate(feeds): + parsed = feedparser.parse(feed) + x = str(x) + tmp[x] = {} + tmp[x]['title'] = parsed.feed.title + tmp[x]['link'] = parsed.feed.link + tmp[x]['description'] = parsed.feed.description + tmp[x]['entries'] = parsed.entries + db.update(tmp) + return db + +def latest(num): + """ Placeholder request """ + request = [ + { + "feedtitle" : "Varia EN", + "post": "hello world", + "date" : "Monday 15th of February 2021", + "url" : "https://vvvvvvaria.org/en/rr-wireless-imagination-1.html" + } + ] + return request + +def today(): + """ Collect posts from today """ + db = load() + + today = str(date.today()).split('-') + today_year = "{:02d}".format(int(today[0])) + today_month = "{:02d}".format(int(today[1])) + today_day = "{:02d}".format(int(today[2])) + print('TODAY =', today_year, today_month, today_day) + + request = [] + for x, feed in db.items(): + for post in feed['entries']: + if post['published_parsed']: + year = "{:02d}".format(post['published_parsed'][0]) + month = "{:02d}".format(post['published_parsed'][1]) + day = "{:02d}".format(post['published_parsed'][2] + 1) + print('POST DATE =', year, month, day) + + # Check if this post is published today + if year == today_year: + if month == today_month: + if day == today_day: + request.append(post) + return request + +def past(days): + """ Collect posts from this week """ + db = load() + + point_in_the_past = date.today() - timedelta(int(days)) + print(f"{ days } days in the past =", point_in_the_past) + + request = [] + for x, feed in db.items(): + for post in feed['entries']: + if post['published_parsed']: + year = post['published_parsed'][0] + month = post['published_parsed'][1] + day = post['published_parsed'][2] + + post_date = date(year, month, day) + print("post date =",post_date) + + if post_date > point_in_the_past: + request.append(post) + + return request \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e5a1bbe --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +flask +feedparser +pathlib diff --git a/simpledatabase.py b/simpledatabase.py new file mode 100644 index 0000000..bca829c --- /dev/null +++ b/simpledatabase.py @@ -0,0 +1,61 @@ +from os import environ, mkdir +from os.path import exists +from pathlib import Path +from logging import DEBUG, INFO, basicConfig, getLogger +from json import dumps, loads + +class SimpleDatabase(dict): + """A simple database. + It is a dictionary which saves to disk on all writes. It is optimised for + ease of hacking and accessibility and not for performance or efficiency. + + Written by decentral1se, as part of: + https://git.vvvvvvaria.org/decentral1se/xbotlib/src/branch/main/xbotlib.py + """ + + def __init__(self, filename, log, *args, **kwargs): + """Initialise the object.""" + self.filename = Path(filename).absolute() + self.log = getLogger(__name__) + + self._loads() + self.update(*args, **kwargs) + + def _loads(self): + """Load the database.""" + if not exists(self.filename): + return + + try: + with open(self.filename, "r") as handle: + self.update(loads(handle.read())) + except Exception as exception: + message = f"Loading file storage failed: {exception}" + self.log.error(message, exc_info=exception) + exit(1) + + def _dumps(self): + """Save the databse to disk.""" + try: + with open(self.filename, "w") as handle: + handle.write(dumps(self, indent=4, sort_keys=True)) + except Exception as exception: + message = f"Saving file storage failed: {exception}" + self.log.error(message, exc_info=exception) + exit(1) + + def __setitem__(self, key, val): + """Write data to the database.""" + super().__setitem__(key, val) + self._dumps() + + def __delitem__(self, key): + """Remove data from the database.""" + super().__delitem__(key) + self._dumps() + + def update(self, *args, **kwargs): + """Update the database.""" + for k, v in dict(*args, **kwargs).items(): + self[k] = v + self._dumps() diff --git a/start.py b/start.py new file mode 100644 index 0000000..b40a973 --- /dev/null +++ b/start.py @@ -0,0 +1,36 @@ +import flask +import feedtools + +APP = flask.Flask(__name__, + static_url_path="", + static_folder="static", + template_folder="templates") + +@APP.route("/") +def index(): + db = feedtools.load() + template = flask.render_template( + "index.html", + feeds=db, + ) + return template + +# @APP.route("/API/latest/") +# def latest(num): +# request = feedtools.latest(num) +# return request + +@APP.route("/API/today/") +def today(): + request = feedtools.today() + return str(request) + +@APP.route("/API/past/") +def past(days): + request = feedtools.past(days) + return str(request) + +if __name__ == "__main__": + APP.debug = True + APP.run(port=5678) + diff --git a/static/css/stylesheet.css b/static/css/stylesheet.css new file mode 100644 index 0000000..c04ad84 --- /dev/null +++ b/static/css/stylesheet.css @@ -0,0 +1,37 @@ +body{ + background-color: pink; + color: red; + margin: 1em; +} +h1, +h2{ + margin: 1em; +} +h2{ + color: fuchsia; +} + +table{ + table-layout: fixed; + width: 100%; + border-collapse: collapse; +} +table td{ + padding: 1em 2em; +} +table td:first-of-type{ + width: 100px; +} +table tr{ + border-bottom: 20px solid white; +} + +section#api{ + margin: 6em 0; + color: fuchsia; +} +section#api div.accesspoint{ + margin: 0; + padding: 1em; + border-bottom: 20px solid yellow; +} \ No newline at end of file diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..a5a08da --- /dev/null +++ b/templates/index.html @@ -0,0 +1,39 @@ + + + + + multifeeder + + + +

multifeeder

+ + + {% for x, feed in feeds.items() %} + + + + + {% endfor %} + +
{{ feed.title }}{{ feed.link }}
+
+

API

+ +
+ /API/today/ +

+ For example: localhost:5678/API/today/ +
+
+ /API/past/[days] +

+ For example: localhost:5678/API/past/30 +
+
+ + \ No newline at end of file