varia/etherpump
14
0
Fork 0
Code Issues 15 Pull Requests Releases Activity

updated pull to deal with errors in diffhtml output that was halting the dump process

Browse Source
This commit is contained in:
Michael Murtaugh 2018-05-04 17:33:17 +02:00
parent 4f7092010c
commit 63910192b9
2 changed files with 20 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
etherdump/commands/index.py
View File

@ -274,9 +274,11 @@ def main (args):
versions_by_type[t] = v
if "text" in versions_by_type:
with open (versions_by_type["text"]["path"]) as f:
p["text"] = f.read()
try:
with open (versions_by_type["text"]["path"]) as f:
p["text"] = f.read()
except FileNotFoundError:
p['text'] = ''
# ADD IN LINK TO PAD AS "link"
for v in linkversions:
if v in versions_by_type:

24
etherdump/commands/pull.py
View File

@ -218,15 +218,21 @@ def main (args):
versions.append(ver)
ver["code"] = html["_code"]
if html["_code"] == 200:
html = html['data']['html']
ver["path"] = p+".diff.html"
ver["url"] = quote(ver["path"])
# doc = html5lib.parse(html, treebuilder="etree", override_encoding="utf-8", namespaceHTMLElements=False)
doc = html5lib.parse(html, treebuilder="etree", namespaceHTMLElements=False)
html5tidy(doc, indent=True, title=padid, scripts=args.script, links=links)
with open(ver["path"], "w") as f:
# f.write(html.encode("utf-8"))
print(ET.tostring(doc, method="html", encoding="unicode"), file=f)
try:
html = html['data']['html']
ver["path"] = p+".diff.html"
ver["url"] = quote(ver["path"])
# doc = html5lib.parse(html, treebuilder="etree", override_encoding="utf-8", namespaceHTMLElements=False)
doc = html5lib.parse(html, treebuilder="etree", namespaceHTMLElements=False)
html5tidy(doc, indent=True, title=padid, scripts=args.script, links=links)
with open(ver["path"], "w") as f:
# f.write(html.encode("utf-8"))
print(ET.tostring(doc, method="html", encoding="unicode"), file=f)
except TypeError:
# Malformed / incomplete response, record the message (such as "internal error") in the metadata and write NO file!
ver["message"] = html["message"]
# with open(ver["path"], "w") as f:
# print ("""<pre>{0}</pre>""".format(json.dumps(html, indent=2)), file=f)
# Process text, html, dhtml, all options
if args.all or args.html: