Browse Source

first step towards a RSS multiverse

master
manetta 3 years ago
parent
commit
64baad0c98
  1. 3
      .gitignore
  2. 9
      Makefile
  3. 16
      README.md
  4. 3
      feeds.txt
  5. 82
      feedtools.py
  6. 3
      requirements.txt
  7. 61
      simpledatabase.py
  8. 36
      start.py
  9. 37
      static/css/stylesheet.css
  10. 39
      templates/index.html

3
.gitignore

@ -0,0 +1,3 @@
__pycache__*
.venv*
feeds.json

9
Makefile

@ -0,0 +1,9 @@
default: run
setup:
@python3 -m venv .venv && \
.venv/bin/pip install -r requirements.txt
run:
@python3 start.py

16
README.md

@ -1,3 +1,19 @@
# Multifeeder
Multifeeding RSS streams into a point of access to publish from! (work-in-slow-progress)
## Using the multifeeder
To install: make a virtual environment + install the dependencies.
`make setup`
To use: activate the virtual environment ...
`make activate`
... and run the Flask application.
`make`
Open `localhost:5678` in a browser and there we go!

3
feeds.txt

@ -0,0 +1,3 @@
https://vvvvvvaria.org/feeds/all-nl.rss.xml
https://vvvvvvaria.org/en/feeds/all-en.rss.xml
https://post.lurk.org/tags/varia.rss

82
feedtools.py

@ -0,0 +1,82 @@
import feedparser
from simpledatabase import SimpleDatabase
import json
from datetime import date, timedelta
def load():
""" Load all feeds """
feeds = open('feeds.txt').readlines()
db = SimpleDatabase('feeds.json', 'feeds.log')
tmp = {}
for x, feed in enumerate(feeds):
parsed = feedparser.parse(feed)
x = str(x)
tmp[x] = {}
tmp[x]['title'] = parsed.feed.title
tmp[x]['link'] = parsed.feed.link
tmp[x]['description'] = parsed.feed.description
tmp[x]['entries'] = parsed.entries
db.update(tmp)
return db
def latest(num):
""" Placeholder request """
request = [
{
"feedtitle" : "Varia EN",
"post": "hello world",
"date" : "Monday 15th of February 2021",
"url" : "https://vvvvvvaria.org/en/rr-wireless-imagination-1.html"
}
]
return request
def today():
""" Collect posts from today """
db = load()
today = str(date.today()).split('-')
today_year = "{:02d}".format(int(today[0]))
today_month = "{:02d}".format(int(today[1]))
today_day = "{:02d}".format(int(today[2]))
print('TODAY =', today_year, today_month, today_day)
request = []
for x, feed in db.items():
for post in feed['entries']:
if post['published_parsed']:
year = "{:02d}".format(post['published_parsed'][0])
month = "{:02d}".format(post['published_parsed'][1])
day = "{:02d}".format(post['published_parsed'][2] + 1)
print('POST DATE =', year, month, day)
# Check if this post is published today
if year == today_year:
if month == today_month:
if day == today_day:
request.append(post)
return request
def past(days):
""" Collect posts from this week """
db = load()
point_in_the_past = date.today() - timedelta(int(days))
print(f"{ days } days in the past =", point_in_the_past)
request = []
for x, feed in db.items():
for post in feed['entries']:
if post['published_parsed']:
year = post['published_parsed'][0]
month = post['published_parsed'][1]
day = post['published_parsed'][2]
post_date = date(year, month, day)
print("post date =",post_date)
if post_date > point_in_the_past:
request.append(post)
return request

3
requirements.txt

@ -0,0 +1,3 @@
flask
feedparser
pathlib

61
simpledatabase.py

@ -0,0 +1,61 @@
from os import environ, mkdir
from os.path import exists
from pathlib import Path
from logging import DEBUG, INFO, basicConfig, getLogger
from json import dumps, loads
class SimpleDatabase(dict):
"""A simple database.
It is a dictionary which saves to disk on all writes. It is optimised for
ease of hacking and accessibility and not for performance or efficiency.
Written by decentral1se, as part of:
https://git.vvvvvvaria.org/decentral1se/xbotlib/src/branch/main/xbotlib.py
"""
def __init__(self, filename, log, *args, **kwargs):
"""Initialise the object."""
self.filename = Path(filename).absolute()
self.log = getLogger(__name__)
self._loads()
self.update(*args, **kwargs)
def _loads(self):
"""Load the database."""
if not exists(self.filename):
return
try:
with open(self.filename, "r") as handle:
self.update(loads(handle.read()))
except Exception as exception:
message = f"Loading file storage failed: {exception}"
self.log.error(message, exc_info=exception)
exit(1)
def _dumps(self):
"""Save the databse to disk."""
try:
with open(self.filename, "w") as handle:
handle.write(dumps(self, indent=4, sort_keys=True))
except Exception as exception:
message = f"Saving file storage failed: {exception}"
self.log.error(message, exc_info=exception)
exit(1)
def __setitem__(self, key, val):
"""Write data to the database."""
super().__setitem__(key, val)
self._dumps()
def __delitem__(self, key):
"""Remove data from the database."""
super().__delitem__(key)
self._dumps()
def update(self, *args, **kwargs):
"""Update the database."""
for k, v in dict(*args, **kwargs).items():
self[k] = v
self._dumps()

36
start.py

@ -0,0 +1,36 @@
import flask
import feedtools
APP = flask.Flask(__name__,
static_url_path="",
static_folder="static",
template_folder="templates")
@APP.route("/")
def index():
db = feedtools.load()
template = flask.render_template(
"index.html",
feeds=db,
)
return template
# @APP.route("/API/latest/<num>")
# def latest(num):
# request = feedtools.latest(num)
# return request
@APP.route("/API/today/")
def today():
request = feedtools.today()
return str(request)
@APP.route("/API/past/<days>")
def past(days):
request = feedtools.past(days)
return str(request)
if __name__ == "__main__":
APP.debug = True
APP.run(port=5678)

37
static/css/stylesheet.css

@ -0,0 +1,37 @@
body{
background-color: pink;
color: red;
margin: 1em;
}
h1,
h2{
margin: 1em;
}
h2{
color: fuchsia;
}
table{
table-layout: fixed;
width: 100%;
border-collapse: collapse;
}
table td{
padding: 1em 2em;
}
table td:first-of-type{
width: 100px;
}
table tr{
border-bottom: 20px solid white;
}
section#api{
margin: 6em 0;
color: fuchsia;
}
section#api div.accesspoint{
margin: 0;
padding: 1em;
border-bottom: 20px solid yellow;
}

39
templates/index.html

@ -0,0 +1,39 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>multifeeder</title>
<link rel="stylesheet" type="text/css" href="/css/stylesheet.css">
</head>
<body>
<h1>multifeeder</h1>
<table>
<tbody>
{% for x, feed in feeds.items() %}
<tr>
<td>{{ feed.title }}</td>
<td>{{ feed.link }}</td>
</tr>
{% endfor %}
</tbody>
</table>
<section id="api">
<h2>API</h2>
<!-- <div class="accesspoint">
/API/latest/[num]
<br><br>
(not there yet)
</div> -->
<div class="accesspoint">
/API/today/
<br><br>
For example: <a href="/API/today/" target="_blank">localhost:5678/API/today/</a>
</div>
<div class="accesspoint">
/API/past/[days]
<br><br>
For example: <a href="/API/past/30" target="_blank">localhost:5678/API/past/30</a>
</div>
</section>
</body>
</html>
Loading…
Cancel
Save