You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
103 lines
3.7 KiB
103 lines
3.7 KiB
#!/usr/bin/env python
|
|
from __future__ import print_function
|
|
from argparse import ArgumentParser
|
|
import sys, json, re, os
|
|
from datetime import datetime
|
|
from urllib import urlencode
|
|
from urllib2 import urlopen, HTTPError, URLError
|
|
from math import ceil, floor
|
|
from common import *
|
|
|
|
"""
|
|
sync(meta):
|
|
Update meta data files for those that have changed.
|
|
Check for changed pads by looking at revisions & comparing to existing
|
|
|
|
"""
|
|
|
|
def jsonload (url):
|
|
f = urlopen(url)
|
|
data = f.read()
|
|
f.close()
|
|
return json.loads(data)
|
|
|
|
def load_padinfo(p):
|
|
with open(p) as f:
|
|
info = json.load(f)
|
|
info['api'] = "{0[protocol]}://{0[hostname]}:{0[port]}{0[apiurl]}{0[apiversion]}/".format(info)
|
|
return info
|
|
|
|
|
|
def main (args):
|
|
p = ArgumentParser("")
|
|
p.add_argument("padid", nargs="*", default=[])
|
|
p.add_argument("--padinfo", default="padinfo.json", help="padinfo, default: padinfo.json")
|
|
p.add_argument("--zerorevs", default=False, action="store_true", help="include pads with zero revisions, default: False")
|
|
p.add_argument("--pub", default="pub", help="pub path for output, default: pub")
|
|
p.add_argument("--group", default="g", help="group path for output, default: g")
|
|
p.add_argument("--skip", default=None, type=int, help="skip this many items, default: None")
|
|
args = p.parse_args(args)
|
|
|
|
info = load_padinfo(args.padinfo)
|
|
data = {}
|
|
data['apikey'] = info['apikey']
|
|
|
|
if args.padid:
|
|
padids = args.padid
|
|
else:
|
|
padids = jsonload(info['api']+'listAllPads?'+urlencode(data))['data']['padIDs']
|
|
padids.sort()
|
|
numpads = len(padids)
|
|
maxmsglen = 0
|
|
count = 0
|
|
for i, padid in enumerate(padids):
|
|
if args.skip != None and i<args.skip:
|
|
continue
|
|
p = (float(i) / numpads)
|
|
percentage = int(floor(p*100))
|
|
bars = int(ceil(p*20))
|
|
bar = ("*"*bars) + ("-"*(20-bars))
|
|
msg = u"\r{0} {1}/{2} {3}... ".format(bar, (i+1), numpads, padid)
|
|
if len(msg) > maxmsglen:
|
|
maxmsglen = len(msg)
|
|
sys.stderr.write("\r{0}".format(" "*maxmsglen))
|
|
sys.stderr.write(msg.encode("utf-8"))
|
|
sys.stderr.flush()
|
|
data['padID'] = padid.encode("utf-8")
|
|
p = padpath(padid, args.pub, args.group)
|
|
metapath = p + ".meta.json"
|
|
revisions = None
|
|
if os.path.exists(metapath):
|
|
with open(metapath) as f:
|
|
meta = json.load(f)
|
|
revisions = jsonload(info['api']+'getRevisionsCount?'+urlencode(data))['data']['revisions']
|
|
if meta['revisions'] == revisions:
|
|
continue
|
|
|
|
meta = {'padid': padid.encode("utf-8")}
|
|
if revisions == None:
|
|
meta['revisions'] = jsonload(info['api']+'getRevisionsCount?'+urlencode(data))['data']['revisions']
|
|
else:
|
|
meta['revisions' ] = revisions
|
|
|
|
if (meta['revisions'] == 0) and (not args.zerorevs):
|
|
# print("Skipping zero revs", file=sys.stderr)
|
|
continue
|
|
|
|
count += 1
|
|
# todo: load more metadata!
|
|
meta['pad'], meta['group'] = splitpadname(padid)
|
|
meta['pathbase'] = p
|
|
meta['lastedited_raw'] = int(jsonload(info['api']+'getLastEdited?'+urlencode(data))['data']['lastEdited'])
|
|
meta['lastedited_iso'] = datetime.fromtimestamp(int(meta['lastedited_raw'])/1000).isoformat()
|
|
meta['author_ids'] = jsonload(info['api'] +'listAuthorsOfPad?'+urlencode(data))['data']['authorIDs']
|
|
|
|
# save it
|
|
try:
|
|
os.makedirs(os.path.split(metapath)[0])
|
|
except OSError:
|
|
pass
|
|
with open(metapath, "w") as f:
|
|
json.dump(meta, f)
|
|
|
|
print("\nWrote {0} files...".format(count), file=sys.stderr)
|
|
|