Remove tqdm and use new logging method

Also bump to new version
This commit is contained in:
Luke Murphy 2020-01-20 18:20:05 +01:00
parent dc63cac8aa
commit a81735a1b8
No known key found for this signature in database
GPG Key ID: 5E2EF5A63E3718CC
4 changed files with 18 additions and 3 deletions

View File

@ -39,6 +39,10 @@ command which enables pads to be processed concurrently. The default
doubt, set this to a lower number (like 5). This functionality is experimental, doubt, set this to a lower number (like 5). This functionality is experimental,
be cautious and please report bugs! be cautious and please report bugs!
Removed fancy progress bars for pulling because concurrent processing makes
that hard to track. For now, we simply output whichever padid we're finished
with.
**October 2019** **October 2019**
Improve `etherpump --help` handling to make it easier for new users. Improve `etherpump --help` handling to make it easier for new users.

View File

@ -2,4 +2,4 @@ import os
DATAPATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") DATAPATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data")
__VERSION__ = '0.0.10' __VERSION__ = '0.0.11'

View File

@ -277,6 +277,7 @@ async def handle_pad(args, padid, data, info, session):
break break
if skip: if skip:
print("[x] {}".format(padid))
return return
if args.output: if args.output:
@ -310,6 +311,7 @@ async def handle_pad(args, padid, data, info, session):
p + ".meta.json", p + ".meta.json",
) )
) )
print("[x] {}".format(padid))
return return
########################################## ##########################################
@ -324,6 +326,7 @@ async def handle_pad(args, padid, data, info, session):
p + ".meta.json", p + ".meta.json",
) )
) )
print("[x] {}".format(padid))
return return
ver["path"] = p + raw_ext ver["path"] = p + raw_ext
@ -444,6 +447,8 @@ async def handle_pad(args, padid, data, info, session):
async with await trio.open_file(metapath, "w") as f: async with await trio.open_file(metapath, "w") as f:
await f.write(json.dumps(meta)) await f.write(json.dumps(meta))
print("[x] {}".format(padid))
async def handle_pads(args): async def handle_pads(args):
session = asks.Session(connections=args.connection) session = asks.Session(connections=args.connection)
@ -454,11 +459,18 @@ async def handle_pads(args):
if args.skip: if args.skip:
padids = padids[args.skip : len(padids)] padids = padids[args.skip : len(padids)]
print("=" * 79)
print("Etherpump warming up the engines ...")
print("=" * 79)
async with trio.open_nursery() as nursery: async with trio.open_nursery() as nursery:
print("Processing {} pads now...".format(len(padids)))
for padid in padids: for padid in padids:
nursery.start_soon(handle_pad, args, padid, data, info, session) nursery.start_soon(handle_pad, args, padid, data, info, session)
print("=" * 79)
print("Processed {} pads!".format(len(padids)))
print("=" * 79)
def main(args): def main(args):
p = build_argument_parser(args) p = build_argument_parser(args)

View File

@ -49,7 +49,6 @@ setup(
"pypandoc", "pypandoc",
"python-dateutil", "python-dateutil",
"requests", "requests",
"tqdm",
"trio", "trio",
], ],
python_requires=">=3.5", python_requires=">=3.5",