From 1c9af6e6e67cde1bc9222fe72ff6a2cdcf7bff95 Mon Sep 17 00:00:00 2001 From: crunk Date: Sun, 23 Jun 2024 11:02:17 +0200 Subject: [PATCH] don't add files again when updating distribusi --- verse/distribusikan/add_files_to_describer.py | 6 +- verse/file_crawler.py | 73 ------------------- verse/migrations/env.py | 20 +++-- 3 files changed, 14 insertions(+), 85 deletions(-) delete mode 100644 verse/file_crawler.py diff --git a/verse/distribusikan/add_files_to_describer.py b/verse/distribusikan/add_files_to_describer.py index 3ce1eb5..5310e90 100644 --- a/verse/distribusikan/add_files_to_describer.py +++ b/verse/distribusikan/add_files_to_describer.py @@ -65,4 +65,8 @@ def add_distribusi_files_to_db(path): files = list(filter(lambda f: not f.endswith(".html"), files)) for file in files: full_path = os.path.join(root, file) - _distribusi_file_with_type(distribusi, full_path) + distribusi_file = DistribusiFiles.query.filter_by( + path=full_path + ).first() + if distribusi_file is None: + _distribusi_file_with_type(distribusi, full_path) diff --git a/verse/file_crawler.py b/verse/file_crawler.py deleted file mode 100644 index ff57d14..0000000 --- a/verse/file_crawler.py +++ /dev/null @@ -1,73 +0,0 @@ -import os - -import magic -from distribusi.mappings import FILE_TYPES -from models.distribusi_model import Distribusis -from models.distribusi_file_model import DistribusiFiles -from app import create_app, get_app, db -from sqlalchemy.exc import ( - DatabaseError, - DataError, - IntegrityError, - InterfaceError, - InvalidRequestError, -) - -MIME_TYPE = magic.Magic(mime=True) - - -def _distribusi_file_with_type(distribusi, full_path): - mime = MIME_TYPE.from_file(full_path) - type_, subtype = mime.split("/") - if type_ in FILE_TYPES: - _add_distribusi_file_to_db(distribusi, full_path, type_) - - -def _get_distribusi_from_path(path): - distribusi = Distribusis.query.filter_by(distribusiname=path).first() - return distribusi - - -def _add_distribusi_file_to_db(distribusi, full_path, type): - app = get_app() - print(f"adding file to database: {full_path} type: {type}") - try: - new_distribusi_file = DistribusiFiles( - path=full_path, - type=type, - distribusi=distribusi.id, - ) - db.session.add(new_distribusi_file) - db.session.commit() - return - except InvalidRequestError: - db.session.rollback() - app.logger.error("Something went wrong!") - except IntegrityError: - db.session.rollback() - app.logger.error("File %s already exists!", full_path) - except DataError: - db.session.rollback() - app.logger.error("%s Invalid Entry", full_path) - except InterfaceError: - db.session.rollback() - app.logger.error("Error connecting to the database") - except DatabaseError: - db.session.rollback() - app.logger.error("Error connecting to the database") - - -def add_distribusi_files(path): - app = create_app() - with app.app_context(): - distribusi = _get_distribusi_from_path(path) - path = os.path.join("stash", path) - for root, dirs, files in os.walk(path, topdown=True): - files = list(filter(lambda f: not f.startswith("."), files)) - files = list(filter(lambda f: not f.endswith(".html"), files)) - for file in files: - full_path = os.path.join(root, file) - _distribusi_file_with_type(distribusi, full_path) - - -add_distribusi_files("2018-12-WttF-Mastodon-and-the-Fediverse") diff --git a/verse/migrations/env.py b/verse/migrations/env.py index 68a0091..68feded 100644 --- a/verse/migrations/env.py +++ b/verse/migrations/env.py @@ -14,19 +14,17 @@ config = context.config # Interpret the config file for Python logging. # This line sets up loggers basically. fileConfig(config.config_file_name) -logger = logging.getLogger("alembic.env") +logger = logging.getLogger('alembic.env') # add your model's MetaData object here # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata config.set_main_option( - "sqlalchemy.url", - str(current_app.extensions["migrate"].db.get_engine().url).replace( - "%", "%%" - ), -) -target_metadata = current_app.extensions["migrate"].db.metadata + 'sqlalchemy.url', + str(current_app.extensions['migrate'].db.get_engine().url).replace( + '%', '%%')) +target_metadata = current_app.extensions['migrate'].db.metadata # other values from the config, defined by the needs of env.py, # can be acquired: @@ -67,20 +65,20 @@ def run_migrations_online(): # when there are no changes to the schema # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html def process_revision_directives(context, revision, directives): - if getattr(config.cmd_opts, "autogenerate", False): + if getattr(config.cmd_opts, 'autogenerate', False): script = directives[0] if script.upgrade_ops.is_empty(): directives[:] = [] - logger.info("No changes in schema detected.") + logger.info('No changes in schema detected.') - connectable = current_app.extensions["migrate"].db.get_engine() + connectable = current_app.extensions['migrate'].db.get_engine() with connectable.connect() as connection: context.configure( connection=connection, target_metadata=target_metadata, process_revision_directives=process_revision_directives, - **current_app.extensions["migrate"].configure_args, + **current_app.extensions['migrate'].configure_args ) with context.begin_transaction():