varia/etherpump
14
0
Fork 0
Code Issues 15 Pull Requests Releases Activity

new init + settings location

Browse Source
This commit is contained in:
Michael Murtaugh 2015-11-22 21:59:52 +01:00
parent cd207dc8f8
commit a16da91223
9 changed files with 163 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
README.md
View File

@ -42,3 +42,15 @@ To get help on a subcommand:
etherdump revisionscount --help etherdump revisionscount --help
file sync
----------
epfs?
pad to file
etherdump init http://localhost:9001/ --path foo
etherdump status
compare state of files to etherpad & report
etherdump pull <padid/path>
etherdump sync
push / pull file contents to pad

57
etherdump/commands/creatediffhtml.py
View File

@ -7,32 +7,33 @@ from urllib2 import urlopen, HTTPError, URLError
def main(args): def main(args):
p = ArgumentParser("calls the createDiffHTML API function for the given padid") p = ArgumentParser("calls the createDiffHTML API function for the given padid")
p.add_argument("padid", help="the padid") p.add_argument("padid", help="the padid")
p.add_argument("--padinfo", default="padinfo.json", help="padinfo, default: padinfo.json") p.add_argument("--padinfo", default=".etherdump/settings.json", help="settings, default: .etherdump/settings.json")
p.add_argument("--showurl", default=False, action="store_true") p.add_argument("--showurl", default=False, action="store_true")
p.add_argument("--format", default="text", help="output format, can be: text, json; default: text") p.add_argument("--format", default="text", help="output format, can be: text, json; default: text")
p.add_argument("--rev", type=int, default=None, help="revision, default: latest") p.add_argument("--rev", type=int, default=None, help="revision, default: latest")
args = p.parse_args(args) args = p.parse_args(args)
with open(args.padinfo) as f: with open(args.padinfo) as f:
info = json.load(f) info = json.load(f)
apiurl = "{0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info) apiurl = info.get("apiurl")
data = {} # apiurl = "{0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info)
data['apikey'] = info['apikey'] data = {}
data['padID'] = args.padid data['apikey'] = info['apikey']
data['startRev'] = "0" data['padID'] = args.padid
if args.rev != None: data['startRev'] = "0"
data['rev'] = args.rev if args.rev != None:
requesturl = apiurl+'createDiffHTML?'+urlencode(data) data['rev'] = args.rev
if args.showurl: requesturl = apiurl+'createDiffHTML?'+urlencode(data)
print requesturl if args.showurl:
else: print requesturl
try: else:
results = json.load(urlopen(requesturl))['data'] try:
if args.format == "json": results = json.load(urlopen(requesturl))['data']
print json.dumps(results) if args.format == "json":
else: print json.dumps(results)
print results['html'].encode("utf-8") else:
except HTTPError as e: print results['html'].encode("utf-8")
pass except HTTPError as e:
pass

4
etherdump/commands/dumpcsv.py
View File

@ -31,13 +31,13 @@ def jsonload (url):
def main (args): def main (args):
p = ArgumentParser("outputs a CSV of information all all pads") p = ArgumentParser("outputs a CSV of information all all pads")
p.add_argument("--padinfo", default="padinfo.json", help="padinfo, default: padinfo.json") p.add_argument("--padinfo", default=".etherdump/settings.json", help="settings, default: .etherdump/settings.json")
p.add_argument("--zerorevs", default=False, action="store_true", help="include pads with zero revisions, default: False") p.add_argument("--zerorevs", default=False, action="store_true", help="include pads with zero revisions, default: False")
args = p.parse_args(args) args = p.parse_args(args)
with open(args.padinfo) as f: with open(args.padinfo) as f:
info = json.load(f) info = json.load(f)
apiurl = "{0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info) apiurl = info.get("apiurl")
data = {} data = {}
data['apikey'] = info['apikey'] data['apikey'] = info['apikey']
requesturl = apiurl+'listAllPads?'+urlencode(data) requesturl = apiurl+'listAllPads?'+urlencode(data)

49
etherdump/commands/gethtml.py
View File

@ -7,28 +7,29 @@ from urllib2 import urlopen, HTTPError, URLError
def main(args): def main(args):
p = ArgumentParser("calls the getHTML API function for the given padid") p = ArgumentParser("calls the getHTML API function for the given padid")
p.add_argument("padid", help="the padid") p.add_argument("padid", help="the padid")
p.add_argument("--padinfo", default="padinfo.json", help="padinfo, default: padinfo.json") p.add_argument("--padinfo", default=".etherdump/settings.json", help="settings, default: .etherdump/settings.json")
p.add_argument("--showurl", default=False, action="store_true") p.add_argument("--showurl", default=False, action="store_true")
p.add_argument("--format", default="text", help="output format, can be: text, json; default: text") p.add_argument("--format", default="text", help="output format, can be: text, json; default: text")
p.add_argument("--rev", type=int, default=None, help="revision, default: latest") p.add_argument("--rev", type=int, default=None, help="revision, default: latest")
args = p.parse_args(args) args = p.parse_args(args)
with open(args.padinfo) as f: with open(args.padinfo) as f:
info = json.load(f) info = json.load(f)
apiurl = "{0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info) apiurl = info.get("apiurl")
data = {} # apiurl = "{0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info)
data['apikey'] = info['apikey'] data = {}
data['padID'] = args.padid data['apikey'] = info['apikey']
if args.rev != None: data['padID'] = args.padid
data['rev'] = args.rev if args.rev != None:
requesturl = apiurl+'getHTML?'+urlencode(data) data['rev'] = args.rev
if args.showurl: requesturl = apiurl+'getHTML?'+urlencode(data)
print requesturl if args.showurl:
else: print requesturl
results = json.load(urlopen(requesturl))['data'] else:
if args.format == "json": results = json.load(urlopen(requesturl))['data']
print json.dumps(results) if args.format == "json":
else: print json.dumps(results)
print results['html'].encode("utf-8") else:
print results['html'].encode("utf-8")

49
etherdump/commands/gettext.py
View File

@ -7,28 +7,29 @@ from urllib2 import urlopen, HTTPError, URLError
def main(args): def main(args):
p = ArgumentParser("calls the getText API function for the given padid") p = ArgumentParser("calls the getText API function for the given padid")
p.add_argument("padid", help="the padid") p.add_argument("padid", help="the padid")
p.add_argument("--padinfo", default="padinfo.json", help="padinfo, default: padinfo.json") p.add_argument("--padinfo", default=".etherdump/settings.json", help="settings, default: .etherdump/settings.json")
p.add_argument("--showurl", default=False, action="store_true") p.add_argument("--showurl", default=False, action="store_true")
p.add_argument("--format", default="text", help="output format, can be: text, json; default: text") p.add_argument("--format", default="text", help="output format, can be: text, json; default: text")
p.add_argument("--rev", type=int, default=None, help="revision, default: latest") p.add_argument("--rev", type=int, default=None, help="revision, default: latest")
args = p.parse_args(args) args = p.parse_args(args)
with open(args.padinfo) as f: with open(args.padinfo) as f:
info = json.load(f) info = json.load(f)
apiurl = "{0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info) apiurl = info.get("apiurl")
data = {} # apiurl = "{0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info)
data['apikey'] = info['apikey'] data = {}
data['padID'] = args.padid # is utf-8 encoded data['apikey'] = info['apikey']
if args.rev != None: data['padID'] = args.padid # is utf-8 encoded
data['rev'] = args.rev if args.rev != None:
requesturl = apiurl+'getText?'+urlencode(data) data['rev'] = args.rev
if args.showurl: requesturl = apiurl+'getText?'+urlencode(data)
print requesturl if args.showurl:
else: print requesturl
results = json.load(urlopen(requesturl))['data'] else:
if args.format == "json": results = json.load(urlopen(requesturl))['data']
print json.dumps(results) if args.format == "json":
else: print json.dumps(results)
print results['text'].encode("utf-8") else:
print results['text'].encode("utf-8")

41
etherdump/commands/list.py
View File

@ -6,25 +6,26 @@ from urllib import urlencode
from urllib2 import urlopen, HTTPError, URLError from urllib2 import urlopen, HTTPError, URLError
def main (args): def main (args):
p = ArgumentParser("call listAllPads and print the results") p = ArgumentParser("call listAllPads and print the results")
p.add_argument("--padinfo", default="padinfo.json", help="padinfo, default: padinfo.json") p.add_argument("--padinfo", default=".etherdump/settings.json", help="settings, default: .etherdump/settings.json")
p.add_argument("--showurl", default=False, action="store_true") p.add_argument("--showurl", default=False, action="store_true")
p.add_argument("--format", default="lines", help="output format: lines, json; default lines") p.add_argument("--format", default="lines", help="output format: lines, json; default lines")
args = p.parse_args(args) args = p.parse_args(args)
with open(args.padinfo) as f: with open(args.padinfo) as f:
info = json.load(f) info = json.load(f)
apiurl = "{0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info) apiurl = info.get("apiurl")
data = {} # apiurl = {0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info)
data['apikey'] = info['apikey'] data = {}
requesturl = apiurl+'listAllPads?'+urlencode(data) data['apikey'] = info['apikey']
if args.showurl: requesturl = apiurl+'listAllPads?'+urlencode(data)
print requesturl if args.showurl:
else: print requesturl
results = json.load(urlopen(requesturl))['data']['padIDs'] else:
if args.format == "json": results = json.load(urlopen(requesturl))['data']['padIDs']
print json.dumps(results) if args.format == "json":
else: print json.dumps(results)
for r in results: else:
print r.encode("utf-8") for r in results:
print r.encode("utf-8")

44
etherdump/commands/listauthors.py
View File

@ -7,26 +7,26 @@ from urllib2 import urlopen, HTTPError, URLError
def main(args): def main(args):
p = ArgumentParser("call listAuthorsOfPad for the padid") p = ArgumentParser("call listAuthorsOfPad for the padid")
p.add_argument("padid", help="the padid") p.add_argument("padid", help="the padid")
p.add_argument("--padinfo", default="padinfo.json", help="padinfo, default: padinfo.json") p.add_argument("--padinfo", default=".etherdump/settings.json", help="settings, default: .etherdump/settings.json")
p.add_argument("--showurl", default=False, action="store_true") p.add_argument("--showurl", default=False, action="store_true")
p.add_argument("--format", default="lines", help="output format, can be: lines, json; default: lines") p.add_argument("--format", default="lines", help="output format, can be: lines, json; default: lines")
args = p.parse_args(args) args = p.parse_args(args)
with open(args.padinfo) as f: with open(args.padinfo) as f:
info = json.load(f) info = json.load(f)
apiurl = "{0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info) apiurl = info.get("apiurl")
data = {} data = {}
data['apikey'] = info['apikey'] data['apikey'] = info['apikey']
data['padID'] = args.padid.encode("utf-8") data['padID'] = args.padid.encode("utf-8")
requesturl = apiurl+'listAuthorsOfPad?'+urlencode(data) requesturl = apiurl+'listAuthorsOfPad?'+urlencode(data)
if args.showurl: if args.showurl:
print requesturl print requesturl
else: else:
results = json.load(urlopen(requesturl))['data']['authorIDs'] results = json.load(urlopen(requesturl))['data']['authorIDs']
if args.format == "json": if args.format == "json":
print json.dumps(results) print json.dumps(results)
else: else:
for r in results: for r in results:
print r.encode("utf-8") print r.encode("utf-8")

34
etherdump/commands/revisionscount.py
View File

@ -6,21 +6,21 @@ from urllib import urlencode
from urllib2 import urlopen, HTTPError, URLError from urllib2 import urlopen, HTTPError, URLError
def main(args): def main(args):
p = ArgumentParser("call getRevisionsCount for the given padid") p = ArgumentParser("call getRevisionsCount for the given padid")
p.add_argument("padid", help="the padid") p.add_argument("padid", help="the padid")
p.add_argument("--padinfo", default="padinfo.json", help="padinfo, default: padinfo.json") p.add_argument("--padinfo", default=".etherdump/settings.json", help="settings, default: .etherdump/settings.json")
p.add_argument("--showurl", default=False, action="store_true") p.add_argument("--showurl", default=False, action="store_true")
args = p.parse_args(args) args = p.parse_args(args)
with open(args.padinfo) as f: with open(args.padinfo) as f:
info = json.load(f) info = json.load(f)
apiurl = "{0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info) apiurl = info.get("apiurl")
data = {} data = {}
data['apikey'] = info['apikey'] data['apikey'] = info['apikey']
data['padID'] = args.padid.encode("utf-8") data['padID'] = args.padid.encode("utf-8")
requesturl = apiurl+'getRevisionsCount?'+urlencode(data) requesturl = apiurl+'getRevisionsCount?'+urlencode(data)
if args.showurl: if args.showurl:
print requesturl print requesturl
else: else:
results = json.load(urlopen(requesturl))['data']['revisions'] results = json.load(urlopen(requesturl))['data']['revisions']
print results print results

21
etherdump/commands/sync.py
View File

@ -24,16 +24,15 @@ def jsonload (url):
def load_padinfo(p): def load_padinfo(p):
with open(p) as f: with open(p) as f:
info = json.load(f) info = json.load(f)
info['api'] = "{0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info)
return info return info
def main (args): def main (args):
p = ArgumentParser("Check for pads that have changed since last sync (according to .meta.json)") p = ArgumentParser("Check for pads that have changed since last sync (according to .meta.json)")
p.add_argument("padid", nargs="*", default=[]) p.add_argument("padid", nargs="*", default=[])
p.add_argument("--padinfo", default="padinfo.json", help="padinfo, default: padinfo.json") p.add_argument("--padinfo", default=".etherdump/settings.json", help="settings, default: .etherdump/settings.json")
p.add_argument("--zerorevs", default=False, action="store_true", help="include pads with zero revisions, default: False (i.e. pads with no revisions are skipped)") p.add_argument("--zerorevs", default=False, action="store_true", help="include pads with zero revisions, default: False (i.e. pads with no revisions are skipped)")
p.add_argument("--pub", default="pub", help="folder to store files for public pads, default: pub") p.add_argument("--pub", default=".", help="folder to store files for public pads, default: pub")
p.add_argument("--group", default="g", help="folder to store files for group pads, default: g") p.add_argument("--group", default="g", help="folder to store files for group pads, default: g")
p.add_argument("--skip", default=None, type=int, help="skip this many items, default: None") p.add_argument("--skip", default=None, type=int, help="skip this many items, default: None")
p.add_argument("--meta", default=False, action="store_true", help="download meta to PADID.meta.json, default: False") p.add_argument("--meta", default=False, action="store_true", help="download meta to PADID.meta.json, default: False")
@ -50,7 +49,7 @@ def main (args):
if args.padid: if args.padid:
padids = args.padid padids = args.padid
else: else:
padids = jsonload(info['api']+'listAllPads?'+urlencode(data))['data']['padIDs'] padids = jsonload(info['apiurl']+'listAllPads?'+urlencode(data))['data']['padIDs']
padids.sort() padids.sort()
numpads = len(padids) numpads = len(padids)
# maxmsglen = 0 # maxmsglen = 0
@ -80,14 +79,14 @@ def main (args):
if os.path.exists(metapath): if os.path.exists(metapath):
with open(metapath) as f: with open(metapath) as f:
meta = json.load(f) meta = json.load(f)
revisions = jsonload(info['api']+'getRevisionsCount?'+urlencode(data))['data']['revisions'] revisions = jsonload(info['apiurl']+'getRevisionsCount?'+urlencode(data))['data']['revisions']
if meta['revisions'] == revisions: if meta['revisions'] == revisions:
skip=True skip=True
break break
meta = {'padid': padid.encode("utf-8")} meta = {'padid': padid.encode("utf-8")}
if revisions == None: if revisions == None:
meta['revisions'] = jsonload(info['api']+'getRevisionsCount?'+urlencode(data))['data']['revisions'] meta['revisions'] = jsonload(info['apiurl']+'getRevisionsCount?'+urlencode(data))['data']['revisions']
else: else:
meta['revisions' ] = revisions meta['revisions' ] = revisions
@ -99,9 +98,9 @@ def main (args):
# todo: load more metadata! # todo: load more metadata!
meta['pad'], meta['group'] = splitpadname(padid) meta['pad'], meta['group'] = splitpadname(padid)
meta['pathbase'] = p meta['pathbase'] = p
meta['lastedited_raw'] = int(jsonload(info['api']+'getLastEdited?'+urlencode(data))['data']['lastEdited']) meta['lastedited_raw'] = int(jsonload(info['apiurl']+'getLastEdited?'+urlencode(data))['data']['lastEdited'])
meta['lastedited_iso'] = datetime.fromtimestamp(int(meta['lastedited_raw'])/1000).isoformat() meta['lastedited_iso'] = datetime.fromtimestamp(int(meta['lastedited_raw'])/1000).isoformat()
meta['author_ids'] = jsonload(info['api']+'listAuthorsOfPad?'+urlencode(data))['data']['authorIDs'] meta['author_ids'] = jsonload(info['apiurl']+'listAuthorsOfPad?'+urlencode(data))['data']['authorIDs']
break break
except HTTPError as e: except HTTPError as e:
tries += 1 tries += 1
@ -129,20 +128,20 @@ def main (args):
# Process text, html, dhtml, all options # Process text, html, dhtml, all options
if args.all or args.text: if args.all or args.text:
text = jsonload(info['api']+'getText?'+urlencode(data)) text = jsonload(info['apiurl']+'getText?'+urlencode(data))
text = text['data']['text'] text = text['data']['text']
with open(p+".txt", "w") as f: with open(p+".txt", "w") as f:
f.write(text.encode("utf-8")) f.write(text.encode("utf-8"))
if args.all or args.html: if args.all or args.html:
html = jsonload(info['api']+'getHTML?'+urlencode(data)) html = jsonload(info['apiurl']+'getHTML?'+urlencode(data))
html = html['data']['html'] html = html['data']['html']
with open(p+".html", "w") as f: with open(p+".html", "w") as f:
f.write(html.encode("utf-8")) f.write(html.encode("utf-8"))
if args.all or args.dhtml: if args.all or args.dhtml:
data['startRev'] = "0" data['startRev'] = "0"
html = jsonload(info['api']+'createDiffHTML?'+urlencode(data)) html = jsonload(info['apiurl']+'createDiffHTML?'+urlencode(data))
html = html['data']['html'] html = html['data']['html']
with open(p+".diff.html", "w") as f: with open(p+".diff.html", "w") as f:
f.write(html.encode("utf-8")) f.write(html.encode("utf-8"))