varia
/
etherpump
12
0
Fork 0
Code Issues 15 Pull Requests Releases Activity
Browse Source

continued tweaks

add-quote-import
Michael Murtaugh 9 years ago
parent
3957c44faa
commit
0f340433ae
3 changed files with 272 additions and 191 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 108
      etherdump/commands/dump.py
  2. 1
      etherdump/data/templates/pad_colors.html
  3. 6
      padinfo.sample.json

108
etherdump/commands/dump.py
View File

@ -1,6 +1,11 @@
#!/usr/bin/env python
# License: AGPL
#
#
# todo:
# Capture exceptions... add HTTP status errors (502) to meta!!
# so that an eventual index can show the problematic pages!
# Also: provide links to text only / html versions when diff HTML fails
from __future__ import print_function
from etherdump import DATAPATH
@ -12,6 +17,7 @@ from datetime import datetime
from xml.etree import cElementTree as ET
from urllib import urlencode
from urllib2 import urlopen, HTTPError, URLError
from time import sleep
# external dependencies (use pip to install these)
import html5lib, jinja2
@ -102,6 +108,11 @@ def get_template_env (tpath=None):
env = jinja2.Environment(loader=loader)
return env
def get_group_info(gid, info):
if 'groups' in info:
if gid in info['groups']:
return info['groups'][gid]
def main(args):
p = ArgumentParser("""
_ _ _
@ -117,7 +128,6 @@ def main(args):
p.add_argument("--verbose", default=False, action="store_true", help="flag for verbose output")
p.add_argument("--limit", type=int, default=None)
p.add_argument("--allpads", default=False, action="store_true", help="flag to process all pads")
p.add_argument("--spider", default=False, action="store_true", help="flag to spider pads")
p.add_argument("--templatepath", default=os.path.join(DATAPATH, "templates"), help="directory with templates (override default files)")
p.add_argument("--colors-template", default="pad_colors.html", help="pad with authorship colors template name: pad_colors.html")
p.add_argument("--padlink", default=[], action="append", help="give a pad link pattern, example: 'http\:\/\/10\.1\.10\.1/p/(.*)'")
@ -126,8 +136,11 @@ def main(args):
p.add_argument("--showurls", default=False, action="store_true", help="flag to display API URLs that are used (to stderr)")
p.add_argument("--hidepaths", default=False, action="store_true", help="flag to not display paths")
p.add_argument("--pretend", default=False, action="store_true", help="flag to not actually save")
p.add_argument("--linkify", default=False, action="store_true", help="flag to process [[link]] forms (and follow when --spider is used)")
p.add_argument("--spider", default=False, action="store_true", help="flag to spider pads (requires --linkify)")
p.add_argument("--add-images", default=False, action="store_true", help="flag to add image tags")
p.add_argument("--authors-css", default="authors.css", help="filename to save collected authorship css (nb: etherdump will overwrite this file!)")
p.add_argument("--force", default=False, action="store_true", help="force dump (even if not updated since last dump)")
p.add_argument("--authors-css", default=None, help="filename to save collected authorship css (nb: any existing file will be mercilessly overwritten), default: don't accumulate css")
# TODO css from pad --- ie specify a padid for a stylesheet!!!!!!
# p.add_argument("--css", default="styles.css", help="padid of stylesheet")
@ -175,14 +188,37 @@ def main(args):
done.add(padid)
data['padID'] = padid.encode("utf-8")
if args.verbose:
print ("PADID \"{0}\"".format(padid).encode("utf-8"), file=sys.stderr)
print (u"PADID \"{0}\"".format(padid).encode("utf-8"), file=sys.stderr)
# g.yIRLMysh0PMsCMHc$
grouppat = re.compile(ur"^g\.(\w+)\$(.+)$")
m = grouppat.search(padid)
if m:
group = m.group(1)
ginfo = get_group_info(group, info)
if not ginfo:
print ("No info for group '{0}', skipping".format(group), file=sys.stderr)
continue
padid = m.group(2)
else:
group = None
ginfo = None
if not args.pretend:
try:
if ginfo:
os.makedirs(os.path.join(args.path, ginfo['name']))
else:
os.makedirs(args.path)
except OSError:
pass
retry = True
tries = 1
while retry:
retry = False
try:
# _
@ -192,14 +228,18 @@ def main(args):
# |_| |_| |_|\___|\__\__,_|
meta_url = urlify(padid, ext=".json")
meta_out = "{0}/{1}".format(args.path, meta_url.encode("utf-8"))
raw_url = urlify(padid, ext=".txt")
raw_out = "{0}/{1}".format(args.path, raw_url.encode("utf-8"))
colors_url = urlify(padid, ext=".html")
if ginfo:
meta_out = "{0}/{1}/{2}".format(args.path, ginfo['name'], meta_url.encode("utf-8"))
raw_out = "{0}/{1}/{2}".format(args.path, ginfo['name'], raw_url.encode("utf-8"))
colors_out = "{0}/{1}/{2}".format(args.path, ginfo['name'], colors_url.encode("utf-8"))
else:
meta_out = "{0}/{1}".format(args.path, meta_url.encode("utf-8"))
raw_out = "{0}/{1}".format(args.path, raw_url.encode("utf-8"))
colors_out = "{0}/{1}".format(args.path, colors_url.encode("utf-8"))
if not args.hidepaths:
print (meta_out, file=sys.stderr)
if not args.pretend:
meta = {}
meta['padid'] = padid
@ -208,6 +248,19 @@ def main(args):
print (revisions_url, file=sys.stderr)
meta['total_revisions'] = json.load(urlopen(revisions_url))['data']['revisions']
# CHECK REVISIONS (against existing meta)
if meta['total_revisions'] == 0:
if args.verbose:
print (" pad has no revisions, skipping", file=sys.stderr)
continue
if os.path.exists(meta_out):
with open(meta_out) as f:
old_meta = json.load(f)
if not args.force and old_meta['total_revisions'] == meta['total_revisions']:
if args.verbose:
print (" skipping (up to date)", file=sys.stderr)
continue
lastedited_url = apiurl+'getLastEdited?'+urlencode(data)
if args.showurls:
print (lastedited_url, file=sys.stderr)
@ -223,21 +276,24 @@ def main(args):
meta['colors'] = colors_url
meta['raw'] = raw_url
meta['meta'] = meta_url
with open(meta_out, "w") as f:
json.dump(meta, f)
# defer output to LAST STEP (as confirmation)
# _ __ __ ___ __
# | '__/ _` \ \ /\ / /
# | | | (_| |\ V V /
# |_| \__,_| \_/\_/
if not args.hidepaths:
print (raw_out, file=sys.stderr)
text_url = apiurl+"getText?"+urlencode(data)
if args.showurls:
print (text_url, file=sys.stderr)
if not args.pretend:
rawText = json.load(urlopen(text_url))['data']['text']
if rawText.strip() == "":
if args.verbose:
print (" empty text, skipping", file=sys.stderr)
continue
if not args.hidepaths:
print (raw_out, file=sys.stderr)
with open(raw_out, "w") as f:
f.write(rawText.encode("utf-8"))
@ -269,6 +325,7 @@ def main(args):
# Stage 1: Process as text
# Process [[wikilink]] style links
# and (optionally) add linked page names to spider todo list
if args.linkify:
html, links = linkify(html)
if args.spider:
for l in links:
@ -338,8 +395,8 @@ def main(args):
if len(i):
selector, rule = i.split(' ',1)
authors_css_rules[selector] = rule
style = '' # strip the individual style tag from each page (only exports to authors-css file)
# nb: it's up to the template to refer to the authors-css file
# replace individual style with a ref to the authors-css
style = '<link rel="stylesheet" type="text/css" href="{0}">'.format(args.authors_css)
else:
style = ET.tostring(style, method="html")
else:
@ -362,6 +419,11 @@ def main(args):
lastedited = meta['lastedited']
).encode("utf-8"))
# OUTPUT METADATA (finally)
if not args.hidepaths:
print (meta_out, file=sys.stderr)
with open(meta_out, "w") as f:
json.dump(meta, f)
# _
# | | ___ ___ _ __
# | |/ _ \ / _ \| '_ \
@ -372,8 +434,24 @@ def main(args):
count += 1
if args.limit and count >= args.limit:
break
except TypeError:
print ("ERROR, skipping!", file=sys.stderr)
# except HTTPError as e:
# retry = True
# except TypeError as e:
# print ("TypeError, skipping!", file=sys.stderr)
except Exception as e:
print ("[{0}] Exception: {1}".format(tries, e), file=sys.stderr)
sleep(3)
retry = True
if retry:
tries += 1
if tries > 5:
print (" GIVING UP", file=sys.stderr)
retry = False
# Write the unified CSS with authors
if args.authors_css:

1
etherdump/data/templates/pad_colors.html
View File

@ -5,7 +5,6 @@
<meta charset="utf-8">
<meta revision="{{revision}}">
<link rel="stylesheet" type="text/css" href="pad.css">
<link rel="stylesheet" type="text/css" href="authors.css">
{{ style }}
</head>
<body>

6
padinfo.sample.json
View File

@ -4,5 +4,9 @@
"hostname": "localhost",
"apiversion": "1.2.9",
"apiurl": "/api/",
"apikey": "8f55f9ede1b3f5d88b3c54eb638225a7bb71c64867786b608abacfdb7d418be1"
"apikey": "8f55f9ede1b3f5d88b3c54eb638225a7bb71c64867786b608abacfdb7d418be1",
"groups": {
"71FpVh4MZBvl8VZ6": {"name": "Transmediale", "id": 43},
"HyYfoX3Q6S5utxs5": {"name": "test", "id": 42 }
}
}

Write Preview
Loading…
Cancel
Save