etherpump/etherdump

#!/usr/bin/env python
from __future__ import print_function
# stdlib
import json, sys, os, re
from argparse import ArgumentParser
from datetime import datetime
from xml.etree import cElementTree as ET 
from urllib import urlencode
from urllib2 import urlopen, HTTPError, URLError
# dependencies
import html5lib, jinja2
# local mods
from trim import trim_removed_spans, contents, set_text_contents, text_contents
from linkify import linkify, urlify, filename_to_padid


def get_template_env (tpath=None):
    paths = []
    if tpath and os.path.isdir(tpath):
        paths.append(tpath)
    # paths.append(TEMPLATES_PATH)
    loader = jinja2.FileSystemLoader(paths)
    env = jinja2.Environment(loader=loader)
    return env

p = ArgumentParser("""
          _   _                  _                       
      ___| |_| |__   ___ _ __ __| |_   _ _ __ ___  _ __  
     / _ \ __| '_ \ / _ \ '__/ _` | | | | '_ ` _ \| '_ \ 
    |  __/ |_| | | |  __/ | | (_| | |_| | | | | | | |_) |
     \___|\__|_| |_|\___|_|  \__,_|\__,_|_| |_| |_| .__/ 
                                                  |_|    
""")
p.add_argument("padid", default=[], nargs="*", help="the padid(s) to process")
p.add_argument("--padinfo", default="padinfo.json", help="JSON file with login data for the pad (url, apikey etc), default: padinfo.json")
p.add_argument("--path", default="output", help="path to save files, default: output")
p.add_argument("--verbose", default=False, action="store_true", help="flag for verbose output")
p.add_argument("--limit", type=int, default=None)
p.add_argument("--allpads", default=False, action="store_true", help="flag to process all pads")
p.add_argument("--spider", default=False, action="store_true", help="flag to spider pads")
p.add_argument("--templatepath", default="templates", help="directory with templates, default: templates")
p.add_argument("--colors-template", default="pad_colors.html", help="pad with authorship colors template name: pad_colors.html")
p.add_argument("--padlink", default=[], action="append", help="give a pad link pattern, example: 'http\:\/\/10\.1\.10\.1/p/(.*)'")
p.add_argument("--linksearch", default=[], action="append", help="specify a link pattern to search for")
p.add_argument("--linkreplace", default=[], action="append", help="specify a replacement pattern to replace preceding linksearch")
p.add_argument("--showurls", default=False, action="store_true", help="flag to display API URLs that are used (to stderr)")
p.add_argument("--hidepaths", default=False, action="store_true", help="flag to not display paths")
p.add_argument("--pretend", default=False, action="store_true", help="flag to not actually save")
p.add_argument("--add-images", default=False, action="store_true", help="flag to add image tags")

# TODO css from pad --- ie specify a padid for a stylesheet!!!!!!
p.add_argument("--css", default="styles.css", help="padid of stylesheet")


args = p.parse_args()
with open(args.padinfo) as f:
    info = json.load(f)

apiurl =  "{0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info)

# padlinkpats are for mapping internal pad links
# linkpats are any other link replacements, both are regexps

padlinkpats = []
linkpats = [] # [(pat, "\\1.html") for pat in padlinkpats]
linkpats.extend(zip(args.linksearch, args.linkreplace))
if "padlink" in info:
    if type(info['padlink']) == list:
        padlinkpats.extend(info['padlink'])
    else:
        padlinkpats.append(info['padlink'])
padlinkpats.extend(args.padlink)
env = get_template_env(args.templatepath)
colors_template = env.get_template(args.colors_template)

todo = args.padid
done = set()
count = 0
data = {}
data['apikey'] = info['apikey']

if args.allpads:
    # push the list of all pad names on to todo
    list_url = apiurl+'listAllPads?'+urlencode(data)
    if args.showurls:
        print (list_url, file=sys.stderr)
    results = json.load(urlopen(list_url))['data']['padIDs']
    todo.extend(results)

while len(todo) > 0:
    padid = todo[0]
    todo = todo[1:]
    done.add(padid)

    data['padID'] = padid.encode("utf-8")
    if args.verbose:
        print ("PADID \"{0}\"".format(padid).encode("utf-8"), file=sys.stderr)
    if not args.pretend:
        try:
            os.makedirs(args.path)
        except OSError:
            pass
    # print ("{0}".format(padid).encode("utf-8"), file=sys.stderr)

    #                 _        
    #  _ __ ___   ___| |_ __ _ 
    # | '_ ` _ \ / _ \ __/ _` |
    # | | | | | |  __/ || (_| |
    # |_| |_| |_|\___|\__\__,_|

    meta_out = "{0}/{1}".format(args.path, urlify(padid, ext=".json"))
    if not args.hidepaths:
        print (meta_out, file=sys.stderr)
    if not args.pretend:
        meta = {}
        meta['padid'] = padid
        revisions_url = apiurl+'getRevisionsCount?'+urlencode(data)
        if args.showurls:
            print (revisions_url, file=sys.stderr)
        meta['total_revisions'] = json.load(urlopen(revisions_url))['data']['revisions']

        lastedited_url = apiurl+'getLastEdited?'+urlencode(data)
        if args.showurls:
            print (lastedited_url, file=sys.stderr)
        lastedited_raw = json.load(urlopen(lastedited_url))['data']['lastEdited']
        meta['lastedited_raw'] = lastedited_raw
        meta['lastedited'] = datetime.fromtimestamp(int(lastedited_raw)/1000).isoformat()

        # author_ids (unfortunately, this is a list of internal etherpad author ids -- not the names ppl type)
        authors_url = apiurl+'listAuthorsOfPad?'+urlencode(data)
        if args.showurls:
            print (authors_url, file=sys.stderr)
        meta['author_ids'] = json.load(urlopen(authors_url))['data']['authorIDs']

        with open(meta_out, "w") as f:
            json.dump(meta, f)

    #  _ __ __ ___      __
    # | '__/ _` \ \ /\ / /
    # | | | (_| |\ V  V / 
    # |_|  \__,_| \_/\_/  

    raw_out = "{0}/{1}".format(args.path, urlify(padid, ext=".txt"))
    if not args.hidepaths:
        print (raw_out, file=sys.stderr)
    text_url = apiurl+"getText?"+urlencode(data)
    if args.showurls:
        print (text_url, file=sys.stderr)
    if not args.pretend:
        rawText = json.load(urlopen(text_url))['data']['text']
        with open(raw_out, "w") as f:
            f.write(rawText.encode("utf-8"))

    #  _     _             _ 
    # | |__ | |_ _ __ ___ | |
    # | '_ \| __| '_ ` _ \| |
    # | | | | |_| | | | | | |
    # |_| |_|\__|_| |_| |_|_|

    # todo ? -- regular HTML output

    #            _                
    #   ___ ___ | | ___  _ __ ___ 
    #  / __/ _ \| |/ _ \| '__/ __|
    # | (_| (_) | | (_) | |  \__ \
    #  \___\___/|_|\___/|_|  |___/

    colors_out = "{0}/{1}".format(args.path, urlify(padid, ext=".html"))
    if not args.hidepaths:
        print (colors_out, file=sys.stderr)
    data['startRev'] = "0"
    colors_url = apiurl+'createDiffHTML?'+urlencode(data)
    if args.showurls:
        print (colors_url, file=sys.stderr)
    html = json.load(urlopen(colors_url))['data']['html']
    t = html5lib.parse(html, namespaceHTMLElements=False)
    trim_removed_spans(t)
    html = ET.tostring(t, method="html")

    # Stage 1: Process as text
    # Process [[wikilink]] style links
    # and (optionally) add linked page names to spider todo list
    html, links = linkify(html)
    if args.spider:
        for l in links:
            if l not in todo and l not in done:
                # if args.verbose:
                #     print ("  link: {0}".format(l), file=sys.stderr)
                todo.append(l)

    # Stage 2: Process as ElementTree
    #
    t = html5lib.parse(html, namespaceHTMLElements=False)
    # apply linkpats
    for a in t.findall(".//a"):
        href = a.attrib.get("href")
        original_href = href
        if href:
            # if args.verbose:
            #     print ("searching for PADLINK: {0}".format(href))
            for pat in padlinkpats:
                if re.search(pat, href) != None:
                    # if args.verbose:
                    #     print ("  found PADLINK: {0}".format(href))
                    href = re.sub(pat, "\\1.html", href)
                    padid = filename_to_padid(href)
                    set_text_contents(a, "[[{0}]]".format(padid))
                    if padid not in todo and padid not in done:
                        if args.verbose:
                            print ("  link: {0}".format(padid), file=sys.stderr)
                        todo.append(padid)
            # apply linkpats
            for s, r in linkpats:
                href = re.sub(s, r, href)
                if href != original_href:
                    old_contents = text_contents(a)
                    # print ("OLD_CONTENTS {0}".format(old_contents))
                    if old_contents == original_href:
                        if args.verbose:
                            print ("   Updating href IN TEXT", file=sys.stderr)
                        set_text_contents(a, href)

            if original_href != href:
                if args.verbose:
                    print ("  Changed href from {0} to {1}".format(original_href, href), file=sys.stderr)
                a.attrib['href'] = href

            # SHOWIMAGES : inject img tag for (local) images
            if args.add_images:
                ext = os.path.splitext(href)[1].lower().lstrip(".")
                if ext in ("png", "gif", "jpeg", "jpg"):
                    # ap = _parent(a)
                    print ("Adding img '{0}'".format(href), file=sys.stderr)
                    img = ET.SubElement(a, "img")
                    br = ET.SubElement(a, "br")
                    a.remove(img); a.insert(0, img)
                    a.remove(br); a.insert(1, br)
                    img.attrib['src'] = href



    # extract the style tag (with authorship colors)
    style = t.find(".//style")
    if style != None:
        style = ET.tostring(style, method="html")
    else:
        style = ""
    # and extract the contents of the body
    html = contents(t.find(".//body"))

    if not args.pretend:
        with open(colors_out, "w") as f:
            # f.write(html.encode("utf-8"))
            f.write(colors_template.render(
                html = html,
                style = style,
                revision = meta['total_revisions'],
                padid = padid,
                timestamp = datetime.now()
            ).encode("utf-8"))

    #  _                   
    # | | ___   ___  _ __  
    # | |/ _ \ / _ \| '_ \ 
    # | | (_) | (_) | |_) |
    # |_|\___/ \___/| .__/ 
    #               |_|    

    count += 1
    if args.limit and count >= args.limit:
        break