don't add files again when updating distribusi

verse/distribusikan/add_files_to_describer.py

@@ -65,4 +65,8 @@ def add_distribusi_files_to_db(path):
         files = list(filter(lambda f: not f.endswith(".html"), files))
         for file in files:
             full_path = os.path.join(root, file)
+            distribusi_file = DistribusiFiles.query.filter_by(
+                path=full_path
+            ).first()
+            if distribusi_file is None:
                 _distribusi_file_with_type(distribusi, full_path)

verse/file_crawler.py

@@ -1,73 +0,0 @@
-import os
-
-import magic
-from distribusi.mappings import FILE_TYPES
-from models.distribusi_model import Distribusis
-from models.distribusi_file_model import DistribusiFiles
-from app import create_app, get_app, db
-from sqlalchemy.exc import (
-    DatabaseError,
-    DataError,
-    IntegrityError,
-    InterfaceError,
-    InvalidRequestError,
-)
-
-MIME_TYPE = magic.Magic(mime=True)
-
-
-def _distribusi_file_with_type(distribusi, full_path):
-    mime = MIME_TYPE.from_file(full_path)
-    type_, subtype = mime.split("/")
-    if type_ in FILE_TYPES:
-        _add_distribusi_file_to_db(distribusi, full_path, type_)
-
-
-def _get_distribusi_from_path(path):
-    distribusi = Distribusis.query.filter_by(distribusiname=path).first()
-    return distribusi
-
-
-def _add_distribusi_file_to_db(distribusi, full_path, type):
-    app = get_app()
-    print(f"adding file to database: {full_path} type: {type}")
-    try:
-        new_distribusi_file = DistribusiFiles(
-            path=full_path,
-            type=type,
-            distribusi=distribusi.id,
-        )
-        db.session.add(new_distribusi_file)
-        db.session.commit()
-        return
-    except InvalidRequestError:
-        db.session.rollback()
-        app.logger.error("Something went wrong!")
-    except IntegrityError:
-        db.session.rollback()
-        app.logger.error("File %s already exists!", full_path)
-    except DataError:
-        db.session.rollback()
-        app.logger.error("%s Invalid Entry", full_path)
-    except InterfaceError:
-        db.session.rollback()
-        app.logger.error("Error connecting to the database")
-    except DatabaseError:
-        db.session.rollback()
-        app.logger.error("Error connecting to the database")
-
-
-def add_distribusi_files(path):
-    app = create_app()
-    with app.app_context():
-        distribusi = _get_distribusi_from_path(path)
-        path = os.path.join("stash", path)
-        for root, dirs, files in os.walk(path, topdown=True):
-            files = list(filter(lambda f: not f.startswith("."), files))
-            files = list(filter(lambda f: not f.endswith(".html"), files))
-            for file in files:
-                full_path = os.path.join(root, file)
-                _distribusi_file_with_type(distribusi, full_path)
-
-
-add_distribusi_files("2018-12-WttF-Mastodon-and-the-Fediverse")

verse/migrations/env.py

@@ -14,19 +14,17 @@ config = context.config
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 fileConfig(config.config_file_name)
-logger = logging.getLogger("alembic.env")
+logger = logging.getLogger('alembic.env')
 
 # add your model's MetaData object here
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
 config.set_main_option(
-    "sqlalchemy.url",
+    'sqlalchemy.url',
-    str(current_app.extensions["migrate"].db.get_engine().url).replace(
+    str(current_app.extensions['migrate'].db.get_engine().url).replace(
-        "%", "%%"
+        '%', '%%'))
-    ),
+target_metadata = current_app.extensions['migrate'].db.metadata
-)
-target_metadata = current_app.extensions["migrate"].db.metadata
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
@@ -67,20 +65,20 @@ def run_migrations_online():
     # when there are no changes to the schema
     # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
     def process_revision_directives(context, revision, directives):
-        if getattr(config.cmd_opts, "autogenerate", False):
+        if getattr(config.cmd_opts, 'autogenerate', False):
             script = directives[0]
             if script.upgrade_ops.is_empty():
                 directives[:] = []
-                logger.info("No changes in schema detected.")
+                logger.info('No changes in schema detected.')
 
-    connectable = current_app.extensions["migrate"].db.get_engine()
+    connectable = current_app.extensions['migrate'].db.get_engine()
 
     with connectable.connect() as connection:
         context.configure(
             connection=connection,
             target_metadata=target_metadata,
             process_revision_directives=process_revision_directives,
-            **current_app.extensions["migrate"].configure_args,
+            **current_app.extensions['migrate'].configure_args
         )
 
         with context.begin_transaction():