Move pad pulling into own function

etherpump/commands/pull.py

@@ -163,19 +163,7 @@ def build_argument_parser(args):
     return parser
 
 
-
+def get_padids(args, info, data):
-def main(args):
-    p = build_argument_parser(args)
-    args = p.parse_args(args)
-
-    raw_ext = ".raw.txt"
-    if args.no_raw_ext:
-        raw_ext = ""
-
-    info = loadpadinfo(args.padinfo)
-    data = {}
-    data['apikey'] = info['apikey']
-
     if args.padid:
         padids = args.padid
     elif args.glob:
@@ -188,18 +176,12 @@ def main(args):
             info['localapiurl'] + 'listAllPads?' + urlencode(data)
         )['data']['padIDs']
     padids.sort()
-    numpads = len(padids)
+    return padids
-    # maxmsglen = 0
-    count = 0
 
-    progress_kwargs = {}
-    if not istty():
-        progress_kwargs.update(dict(disable=True))
-    progress_pads = tqdm(iterable=padids, total=len(padids), **progress_kwargs)
 
-    for i, padid in enumerate(progress_pads):
+def handle_pad(args, index, padid, data, info, raw_ext):
-        if args.skip != None and i < args.skip:
+    if args.skip != None and index < args.skip:
-            continue
+        return
 
     data['padID'] = padid
     p = padpath(padid, args.pub, args.group, args.fix_names)
@@ -219,9 +201,7 @@ def main(args):
                 with open(metapath) as f:
                     meta.update(json.load(f))
                 revisions = getjson(
-                        info['localapiurl']
+                    info['localapiurl'] + 'getRevisionsCount?' + urlencode(data)
-                        + 'getRevisionsCount?'
-                        + urlencode(data)
                 )['data']['revisions']
                 if meta['revisions'] == revisions and not args.force:
                     skip = True
@@ -230,18 +210,12 @@ def main(args):
             meta['padid'] = padid
             versions = meta["versions"] = []
             versions.append(
-                    {
+                {"url": padurlbase + quote(padid), "type": "pad", "code": 200,}
-                        "url": padurlbase + quote(padid),
-                        "type": "pad",
-                        "code": 200,
-                    }
             )
 
-                if revisions == None:
+            if revisions is None:
                 meta['revisions'] = getjson(
-                        info['localapiurl']
+                    info['localapiurl'] + 'getRevisionsCount?' + urlencode(data)
-                        + 'getRevisionsCount?'
-                        + urlencode(data)
                 )['data']['revisions']
             else:
                 meta['revisions'] = revisions
@@ -287,9 +261,7 @@ def main(args):
             break
 
     if skip:
-            continue
+        return
-
-        count += 1
 
     if args.output:
         print(padid)
@@ -321,7 +293,7 @@ def main(args):
                         p + ".meta.json",
                     )
                 )
-                    continue
+                return
 
             ##########################################
             ## ENFORCE __PUBLISH__ MAGIC WORD
@@ -335,7 +307,7 @@ def main(args):
                         p + ".meta.json",
                     )
                 )
-                    continue
+                return
 
             ver["path"] = p + raw_ext
             ver["url"] = quote(ver["path"])
@@ -394,8 +366,6 @@ def main(args):
             }
         )
 
-        # links.append({"href":"/", "rel":"search", "type":"text/html", "title":"Index"})
-
     if args.all or args.dhtml:
         data['startRev'] = "0"
         html = getjson(
@@ -409,7 +379,6 @@ def main(args):
                 html = html['data']['html']
                 ver["path"] = p + ".diff.html"
                 ver["url"] = quote(ver["path"])
-                    # doc = html5lib.parse(html, treebuilder="etree", override_encoding="utf-8", namespaceHTMLElements=False)
                 doc = html5lib.parse(
                     html, treebuilder="etree", namespaceHTMLElements=False
                 )
@@ -426,10 +395,7 @@ def main(args):
                         file=f,
                     )
             except TypeError:
-                    # Malformed / incomplete response, record the message (such as "internal error") in the metadata and write NO file!
                 ver["message"] = html["message"]
-                    # with open(ver["path"], "w") as f:
-                    #     print ("""<pre>{0}</pre>""".format(json.dumps(html, indent=2)), file=f)
 
     # Process text, html, dhtml, all options
     if args.all or args.html:
@@ -445,16 +411,11 @@ def main(args):
                 html, treebuilder="etree", namespaceHTMLElements=False
             )
             html5tidy(
-                    doc,
+                doc, indent=True, title=padid, scripts=args.script, links=links,
-                    indent=True,
-                    title=padid,
-                    scripts=args.script,
-                    links=links,
             )
             with open(ver["path"], "w") as f:
                 print(
-                        ET.tostring(doc, method="html", encoding="unicode"),
+                    ET.tostring(doc, method="html", encoding="unicode"), file=f,
-                        file=f,
                 )
 
     # output meta
@@ -465,3 +426,26 @@ def main(args):
         ver["url"] = quote(metapath)
         with open(metapath, "w") as f:
             json.dump(meta, f, indent=2)
+
+
+def main(args):
+    p = build_argument_parser(args)
+    args = p.parse_args(args)
+
+    raw_ext = ".raw.txt"
+    if args.no_raw_ext:
+        raw_ext = ""
+
+    info = loadpadinfo(args.padinfo)
+    data = {}
+    data['apikey'] = info['apikey']
+
+    padids = get_padids(args, info, data)
+
+    progress_kwargs = {}
+    if not istty():
+        progress_kwargs.update(dict(disable=True))
+    progress_pads = tqdm(iterable=padids, total=len(padids), **progress_kwargs)
+
+    for index, padid in enumerate(progress_pads):
+        handle_pad(args, index, padid, data, info, raw_ext)