Add fancy progress bar

README.md

@@ -37,6 +37,8 @@ Improve `etherpump --help` handling to make it easier for new users.
 
 Added the `python-dateutil` and `pypandoc` dependencies
 
+Added a fancy progress bar with `tqdm` for long running `etherpump pull --all` calls
+
 **September 2019**
 
 Forking *etherpump* into *etherpump*.

etherpump/commands/common.py

@@ -3,16 +3,9 @@ import os
 import re
 import sys
 from html.entities import name2codepoint
-from math import ceil, floor
 from time import sleep
-from urllib.parse import (
+from urllib.parse import quote_plus, unquote_plus
-    quote_plus,
+from urllib.request import HTTPError, urlopen
-    unquote_plus,
-    urlencode,
-    urlparse,
-    urlunparse,
-)
-from urllib.request import HTTPError, URLError, urlopen
 
 groupnamepat = re.compile(r"^g\.(\w+)\$")
 
@@ -88,16 +81,6 @@ def loadpadinfo(p):
     return info
 
 
-def progressbar(i, num, label="", file=sys.stderr):
-    p = float(i) / num
-    percentage = int(floor(p * 100))
-    bars = int(ceil(p * 20))
-    bar = ("*" * bars) + ("-" * (20 - bars))
-    msg = "\r{0} {1}/{2} {3}... ".format(bar, (i + 1), num, label)
-    sys.stderr.write(msg)
-    sys.stderr.flush()
-
-
 # Python developer Fredrik Lundh (author of elementtree, among other things) has such a function on his website, which works with decimal, hex and named entities:
 ##
 # Removes HTML or XML character references and entities from a text string.
@@ -125,3 +108,7 @@ def unescape(text):
         return text  # leave as is
 
     return re.sub("&#?\w+;", fixup, text)
+
+
+def istty():
+    return sys.stdout.isatty() and os.environ.get('TERM') != 'dumb'

etherpump/commands/pull.py

@@ -13,6 +13,7 @@ from urllib.request import HTTPError
 from xml.etree import ElementTree as ET
 
 import html5lib
+from tqdm import tqdm
 
 from etherpump.commands.common import *  # noqa
 from etherpump.commands.html5tidy import html5tidy
@@ -190,10 +191,15 @@ def main(args):
     numpads = len(padids)
     # maxmsglen = 0
     count = 0
-    for i, padid in enumerate(padids):
+
+    progress_kwargs = {}
+    if not istty():
+        progress_kwargs.update(dict(disable=True))
+    progress_pads = tqdm(iterable=padids, total=len(padids), **progress_kwargs)
+
+    for i, padid in enumerate(progress_pads):
         if args.skip != None and i < args.skip:
             continue
-        progressbar(i, numpads, padid)
 
         data['padID'] = padid
         p = padpath(padid, args.pub, args.group, args.fix_names)
@@ -460,5 +466,3 @@ def main(args):
             ver["url"] = quote(metapath)
             with open(metapath, "w") as f:
                 json.dump(meta, f, indent=2)
-
-    print("\n{0} pad(s) loaded".format(count), file=sys.stderr)

setup.py

@@ -42,7 +42,13 @@ setup(
     description='Etherpump: pumping text from etherpads into publications',
     long_description=long_description,
     long_description_content_type='text/markdown',
-    install_requires=["html5lib", "jinja2", "python-dateutil", "pypandoc"],
+    install_requires=[
+        "html5lib",
+        "jinja2",
+        "python-dateutil",
+        "pypandoc",
+        "tqdm",
+    ],
     classifiers=[
         'Programming Language :: Python :: 3',
         'Environment :: Console',