Michael Murtaugh
8 years ago
2 changed files with 135 additions and 0 deletions
@ -0,0 +1,101 @@ |
|||||
|
#!/usr/bin/env python |
||||
|
from __future__ import print_function |
||||
|
from argparse import ArgumentParser |
||||
|
import sys, json, re, os |
||||
|
from datetime import datetime |
||||
|
from urllib import urlencode |
||||
|
from urllib2 import HTTPError |
||||
|
from jinja2 import FileSystemLoader, Environment |
||||
|
from common import * |
||||
|
from time import sleep |
||||
|
import dateutil.parser |
||||
|
|
||||
|
""" |
||||
|
rss: |
||||
|
Generate an RSS feed from an etherdump. |
||||
|
""" |
||||
|
|
||||
|
def group (items, key=lambda x: x): |
||||
|
ret = [] |
||||
|
keys = {} |
||||
|
for item in items: |
||||
|
k = key(item) |
||||
|
if k not in keys: |
||||
|
keys[k] = [] |
||||
|
keys[k].append(item) |
||||
|
for k in sorted(keys): |
||||
|
keys[k].sort() |
||||
|
ret.append(keys[k]) |
||||
|
return ret |
||||
|
|
||||
|
def base (x): |
||||
|
return re.sub(r"(\.raw\.html)|(\.diff\.html)|(\.meta\.json)|(\.raw\.txt)$", "", x) |
||||
|
|
||||
|
def main (args): |
||||
|
p = ArgumentParser("Check for pads that have changed since last sync (according to .meta.json)") |
||||
|
|
||||
|
p.add_argument("input", nargs="+", help="filenames") |
||||
|
p.add_argument("--templates", default=None, help="templates path") |
||||
|
|
||||
|
p.add_argument("--padinfo", default=".etherdump/settings.json", help="settings, default: .etherdump/settings.json") |
||||
|
p.add_argument("--zerorevs", default=False, action="store_true", help="include pads with zero revisions, default: False (i.e. pads with no revisions are skipped)") |
||||
|
|
||||
|
p.add_argument("--skip", default=None, type=int, help="skip this many items, default: None") |
||||
|
p.add_argument("--type", default="lastchanges", help="type of feed, default: lastchanges") |
||||
|
|
||||
|
p.add_argument("--title", default="etherpad", help="rss feed channel title, default: etherpad") |
||||
|
p.add_argument("--description", default="", help="channel description, default: empty") |
||||
|
p.add_argument("--language", default="en-US", help="feed language, default: en-US") |
||||
|
p.add_argument("--updatePeriod", default="daily", help="updatePeriod, possible values: hourly, daily, weekly, monthly, yearly; default: daily") |
||||
|
p.add_argument("--updateFrequency", default=1, type=int, help="update frequency within the update period (where 2 would mean twice per period); default: 1") |
||||
|
p.add_argument("--padurl", default=None, help="to use as channel link, default: padurl") |
||||
|
p.add_argument("--feedurl", default="feed.xml", help="to use as feeds own (self) link, default: feed.xml") |
||||
|
p.add_argument("--generator", default="https://gitlab.com/activearchives/etherdump", help="generator, default: https://gitlab.com/activearchives/etherdump") |
||||
|
p.add_argument("--itemlink", default="pad", help="item to link to in feed, possible values: pad, text, html, dhtml; default: pad") |
||||
|
|
||||
|
p.add_argument("--chronological", default=False, action="store_true", help="order chronologically, default: False") |
||||
|
args = p.parse_args(args) |
||||
|
|
||||
|
tmpath = args.templates |
||||
|
if tmpath == None: |
||||
|
tmpath = os.path.split(os.path.abspath(__file__))[0] |
||||
|
tmpath = os.path.split(tmpath)[0] |
||||
|
tmpath = os.path.join(tmpath, "data", "templates") |
||||
|
|
||||
|
env = Environment(loader=FileSystemLoader(tmpath)) |
||||
|
template = env.get_template("rss.xml") |
||||
|
|
||||
|
info = loadpadinfo(args.padinfo) |
||||
|
|
||||
|
inputs = args.input |
||||
|
inputs.sort() |
||||
|
inputs = group(inputs, base) |
||||
|
|
||||
|
def loadmeta(paths): |
||||
|
for p in paths: |
||||
|
if p.endswith(".meta.json"): |
||||
|
with open(p) as f: |
||||
|
return json.load(f) |
||||
|
|
||||
|
def fixdates (padmeta): |
||||
|
d = dateutil.parser.parse(padmeta["lastedited_iso"]) |
||||
|
padmeta["lastedited"] = d |
||||
|
padmeta["lastedited_822"] = d.strftime("%a, %d %b %Y %H:%M:%S +0000") |
||||
|
return padmeta |
||||
|
|
||||
|
pads = map(loadmeta, inputs) |
||||
|
pads = map(fixdates, pads) |
||||
|
args.pads = pads |
||||
|
|
||||
|
# args.timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") |
||||
|
|
||||
|
padurlbase = re.sub(r"api/1.2.9/$", "p/", info["apiurl"]) |
||||
|
if type(padurlbase) == unicode: |
||||
|
padurlbase = padurlbase.encode("utf-8") |
||||
|
args.padurl = args.padurl or padurlbase |
||||
|
args.utcnow = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000") |
||||
|
|
||||
|
# from pprint import pprint |
||||
|
# pprint(inputs) |
||||
|
|
||||
|
print (template.render(vars(args)).encode("utf-8")) |
@ -0,0 +1,34 @@ |
|||||
|
<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" |
||||
|
xmlns:content="http://purl.org/rss/1.0/modules/content/" |
||||
|
xmlns:wfw="http://wellformedweb.org/CommentAPI/" |
||||
|
xmlns:dc="http://purl.org/dc/elements/1.1/" |
||||
|
xmlns:atom="http://www.w3.org/2005/Atom" |
||||
|
xmlns:sy="http://purl.org/rss/1.0/modules/syndication/" |
||||
|
xmlns:slash="http://purl.org/rss/1.0/modules/slash/" |
||||
|
> |
||||
|
|
||||
|
<channel> |
||||
|
<title>{{title}}</title> |
||||
|
<atom:link href="{{feedurl}}" rel="self" type="application/rss+xml" /> |
||||
|
<link>{{padurl}}</link> |
||||
|
<description>{{description}}</description> |
||||
|
<lastBuildDate>{{utcnow}}</lastBuildDate> |
||||
|
<language>{{language}}</language> |
||||
|
<sy:updatePeriod>{{updatePeriod}}</sy:updatePeriod> |
||||
|
<sy:updateFrequency>{{updateFrequency}}</sy:updateFrequency> |
||||
|
<generator>{{generator}}</generator> |
||||
|
|
||||
|
{% for p in pads %} |
||||
|
<item> |
||||
|
<title>{{p.pad}}</title> |
||||
|
<link>{{p.versions[0].url}}</link> |
||||
|
<pubDate>{{p.lastedited_822}}</pubDate> |
||||
|
<dc:creator><![CDATA[-]]></dc:creator> |
||||
|
<category><![CDATA[-]]></category> |
||||
|
<guid isPermaLink="false">{{p.versions[0].url}}</guid> |
||||
|
<description><![CDATA[{{p.description}}]]></description> |
||||
|
<content:encoded><![CDATA[{{p.content}}]]></content:encoded> |
||||
|
</item> |
||||
|
{% endfor %} |
||||
|
|
||||
|
</channel> |
Loading…
Reference in new issue