diff --git a/etherdump b/etherdump
index 7432968..a239b64 100755
--- a/etherdump
+++ b/etherdump
@@ -1,15 +1,17 @@
#!/usr/bin/env python
from __future__ import print_function
-from argparse import ArgumentParser
+# stdlib
import json, sys, os, re
+from argparse import ArgumentParser
from datetime import datetime
-import html5lib
+from xml.etree import cElementTree as ET
from urllib import urlencode
from urllib2 import urlopen, HTTPError, URLError
-from xml.etree import cElementTree as ET
+# dependencies
+import html5lib, jinja2
+# local mods
from trim import trim_removed_spans, contents, set_text_contents, text_contents
from linkify import linkify, urlify, filename_to_padid
-import jinja2
def get_template_env (tpath=None):
@@ -21,19 +23,34 @@ def get_template_env (tpath=None):
env = jinja2.Environment(loader=loader)
return env
-
-p = ArgumentParser("")
-p.add_argument("padid", help="the padid")
-p.add_argument("--padinfo", default="padinfo.json", help="padinfo, default: padinfo.json")
-p.add_argument("--output", default="output", help="path to save files, default: output")
-p.add_argument("--verbose", default=False, action="store_true")
+p = ArgumentParser("""
+ _ _ _
+ ___| |_| |__ ___ _ __ __| |_ _ _ __ ___ _ __
+ / _ \ __| '_ \ / _ \ '__/ _` | | | | '_ ` _ \| '_ \
+ | __/ |_| | | | __/ | | (_| | |_| | | | | | | |_) |
+ \___|\__|_| |_|\___|_| \__,_|\__,_|_| |_| |_| .__/
+ |_|
+""")
+p.add_argument("padid", default=[], nargs="*", help="the padid(s) to process")
+p.add_argument("--padinfo", default="padinfo.json", help="JSON file with login data for the pad (url, apikey etc), default: padinfo.json")
+p.add_argument("--path", default="output", help="path to save files, default: output")
+p.add_argument("--verbose", default=False, action="store_true", help="flag for verbose output")
p.add_argument("--limit", type=int, default=None)
-p.add_argument("--templates", default="templates")
-p.add_argument("--template", default="pad_html.html")
-
+p.add_argument("--allpads", default=False, action="store_true", help="flag to process all pads")
+p.add_argument("--spider", default=False, action="store_true", help="flag to spider pads")
+p.add_argument("--templatepath", default="templates", help="directory with templates, default: templates")
+p.add_argument("--colors-template", default="pad_colors.html", help="pad with authorship colors template name: pad_colors.html")
p.add_argument("--padlink", default=[], action="append", help="give a pad link pattern, example: 'http\:\/\/10\.1\.10\.1/p/(.*)'")
p.add_argument("--linksearch", default=[], action="append", help="specify a link pattern to search for")
p.add_argument("--linkreplace", default=[], action="append", help="specify a replacement pattern to replace preceding linksearch")
+p.add_argument("--showurls", default=False, action="store_true", help="flag to display API URLs that are used (to stderr)")
+p.add_argument("--hidepaths", default=False, action="store_true", help="flag to not display paths")
+p.add_argument("--pretend", default=False, action="store_true", help="flag to not actually save")
+p.add_argument("--add-images", default=False, action="store_true", help="flag to add image tags")
+
+# TODO css from pad --- ie specify a padid for a stylesheet!!!!!!
+p.add_argument("--css", default="styles.css", help="padid of stylesheet")
+
args = p.parse_args()
with open(args.padinfo) as f:
@@ -41,62 +58,135 @@ with open(args.padinfo) as f:
apiurl = "{0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info)
+# padlinkpats are for mapping internal pad links
+# linkpats are any other link replacements, both are regexps
+
padlinkpats = []
+linkpats = [] # [(pat, "\\1.html") for pat in padlinkpats]
+linkpats.extend(zip(args.linksearch, args.linkreplace))
if "padlink" in info:
if type(info['padlink']) == list:
padlinkpats.extend(info['padlink'])
else:
padlinkpats.append(info['padlink'])
padlinkpats.extend(args.padlink)
+env = get_template_env(args.templatepath)
+colors_template = env.get_template(args.colors_template)
-linkpats = [] # [(pat, "\\1.html") for pat in padlinkpats]
-linkpats.extend(zip(args.linksearch, args.linkreplace))
-
-if args.verbose:
- print ("using padlinkpats", padlinkpats)
-
-todo = [args.padid]
+todo = args.padid
done = set()
count = 0
+data = {}
+data['apikey'] = info['apikey']
-env = get_template_env(args.templates)
-template = env.get_template(args.template)
+if args.allpads:
+ # push the list of all pad names on to todo
+ list_url = apiurl+'listAllPads?'+urlencode(data)
+ if args.showurls:
+ print (list_url, file=sys.stderr)
+ results = json.load(urlopen(list_url))['data']['padIDs']
+ todo.extend(results)
while len(todo) > 0:
padid = todo[0]
todo = todo[1:]
done.add(padid)
- data = {}
- data['apikey'] = info['apikey']
data['padID'] = padid.encode("utf-8")
-
if args.verbose:
print ("PADID \"{0}\"".format(padid).encode("utf-8"), file=sys.stderr)
- out = "{0}/{1}".format(args.output, urlify(padid))
- print ("{0}".format(out), file=sys.stderr)
+ if not args.pretend:
+ try:
+ os.makedirs(args.path)
+ except OSError:
+ pass
+ # print ("{0}".format(padid).encode("utf-8"), file=sys.stderr)
- total_revisions = apiurl+'getRevisionsCount?'+urlencode(data)
- total_revisions = json.load(urlopen(total_revisions))['data']['revisions']
- if args.verbose:
- print (u"{0} has {1} total revisions...".format(padid, total_revisions).encode("utf-8"), file=sys.stderr)
+ # _
+ # _ __ ___ ___| |_ __ _
+ # | '_ ` _ \ / _ \ __/ _` |
+ # | | | | | | __/ || (_| |
+ # |_| |_| |_|\___|\__\__,_|
+
+ meta_out = "{0}/{1}".format(args.path, urlify(padid, ext=".json"))
+ if not args.hidepaths:
+ print (meta_out, file=sys.stderr)
+ if not args.pretend:
+ meta = {}
+ meta['padid'] = padid
+ revisions_url = apiurl+'getRevisionsCount?'+urlencode(data)
+ if args.showurls:
+ print (revisions_url, file=sys.stderr)
+ meta['total_revisions'] = json.load(urlopen(revisions_url))['data']['revisions']
+
+ lastedited_url = apiurl+'getLastEdited?'+urlencode(data)
+ if args.showurls:
+ print (lastedited_url, file=sys.stderr)
+ lastedited_raw = json.load(urlopen(lastedited_url))['data']['lastEdited']
+ meta['lastedited_raw'] = lastedited_raw
+ meta['lastedited'] = datetime.fromtimestamp(int(lastedited_raw)/1000).isoformat()
+
+ # author_ids (unfortunately, this is a list of internal etherpad author ids -- not the names ppl type)
+ authors_url = apiurl+'listAuthorsOfPad?'+urlencode(data)
+ if args.showurls:
+ print (authors_url, file=sys.stderr)
+ meta['author_ids'] = json.load(urlopen(authors_url))['data']['authorIDs']
+
+ with open(meta_out, "w") as f:
+ json.dump(meta, f)
+
+ # _ __ __ ___ __
+ # | '__/ _` \ \ /\ / /
+ # | | | (_| |\ V V /
+ # |_| \__,_| \_/\_/
+ raw_out = "{0}/{1}".format(args.path, urlify(padid, ext=".txt"))
+ if not args.hidepaths:
+ print (raw_out, file=sys.stderr)
+ text_url = apiurl+"getText?"+urlencode(data)
+ if args.showurls:
+ print (text_url, file=sys.stderr)
+ if not args.pretend:
+ rawText = json.load(urlopen(text_url))['data']['text']
+ with open(raw_out, "w") as f:
+ f.write(rawText.encode("utf-8"))
+
+ # _ _ _
+ # | |__ | |_ _ __ ___ | |
+ # | '_ \| __| '_ ` _ \| |
+ # | | | | |_| | | | | | |
+ # |_| |_|\__|_| |_| |_|_|
+
+ # todo ? -- regular HTML output
+
+ # _
+ # ___ ___ | | ___ _ __ ___
+ # / __/ _ \| |/ _ \| '__/ __|
+ # | (_| (_) | | (_) | | \__ \
+ # \___\___/|_|\___/|_| |___/
+
+ colors_out = "{0}/{1}".format(args.path, urlify(padid, ext=".html"))
+ if not args.hidepaths:
+ print (colors_out, file=sys.stderr)
data['startRev'] = "0"
- requesturl = apiurl+'createDiffHTML?'+urlencode(data)
- html = json.load(urlopen(requesturl))['data']['html']
+ colors_url = apiurl+'createDiffHTML?'+urlencode(data)
+ if args.showurls:
+ print (colors_url, file=sys.stderr)
+ html = json.load(urlopen(colors_url))['data']['html']
t = html5lib.parse(html, namespaceHTMLElements=False)
trim_removed_spans(t)
html = ET.tostring(t, method="html")
# Stage 1: Process as text
# Process [[wikilink]] style links
- # and add linked page names to spider todo list
+ # and (optionally) add linked page names to spider todo list
html, links = linkify(html)
- for l in links:
- if l not in todo and l not in done:
- if args.verbose:
- print (" link: {0}".format(l), file=sys.stderr)
- todo.append(l)
+ if args.spider:
+ for l in links:
+ if l not in todo and l not in done:
+ # if args.verbose:
+ # print (" link: {0}".format(l), file=sys.stderr)
+ todo.append(l)
# Stage 2: Process as ElementTree
#
@@ -135,6 +225,20 @@ while len(todo) > 0:
print (" Changed href from {0} to {1}".format(original_href, href), file=sys.stderr)
a.attrib['href'] = href
+ # SHOWIMAGES : inject img tag for (local) images
+ if args.add_images:
+ ext = os.path.splitext(href)[1].lower().lstrip(".")
+ if ext in ("png", "gif", "jpeg", "jpg"):
+ # ap = _parent(a)
+ print ("Adding img '{0}'".format(href), file=sys.stderr)
+ img = ET.SubElement(a, "img")
+ br = ET.SubElement(a, "br")
+ a.remove(img); a.insert(0, img)
+ a.remove(br); a.insert(1, br)
+ img.attrib['src'] = href
+
+
+
# extract the style tag (with authorship colors)
style = t.find(".//style")
if style != None:
@@ -144,21 +248,24 @@ while len(todo) > 0:
# and extract the contents of the body
html = contents(t.find(".//body"))
+ if not args.pretend:
+ with open(colors_out, "w") as f:
+ # f.write(html.encode("utf-8"))
+ f.write(colors_template.render(
+ html = html,
+ style = style,
+ revision = meta['total_revisions'],
+ padid = padid,
+ timestamp = datetime.now()
+ ).encode("utf-8"))
- try:
- os.makedirs(args.output)
- except OSError:
- pass
- with open(out, "w") as f:
- # f.write(html.encode("utf-8"))
- f.write(template.render(
- html = html,
- style = style,
- revision = total_revisions,
- padid = padid,
- timestamp = datetime.now()
- ).encode("utf-8"))
+ # _
+ # | | ___ ___ _ __
+ # | |/ _ \ / _ \| '_ \
+ # | | (_) | (_) | |_) |
+ # |_|\___/ \___/| .__/
+ # |_|
count += 1
if args.limit and count >= args.limit:
- break
+ break
\ No newline at end of file
diff --git a/linkify.py b/linkify.py
index b0ca66d..06f0b3f 100644
--- a/linkify.py
+++ b/linkify.py
@@ -5,8 +5,8 @@ import re, sys
def strip_tags (text):
return re.sub(r"<.*?>", "", text)
-def urlify (t):
- return t.replace(" ", "_") + ".html"
+def urlify (t, ext=".html"):
+ return t.replace(" ", "_") + ext
def filename_to_padid (t):
t = t.replace("_", " ")
diff --git a/templates/pad_html.html b/templates/pad_html.html
deleted file mode 100644
index 046b273..0000000
--- a/templates/pad_html.html
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
-{{padid}}
-
-
-
-{{ style }}
-
-
-{{ html }}
-
-
\ No newline at end of file