file crawler for annotations,descriptions,tags

verse/deploydb.py

@@ -4,6 +4,7 @@ def deploy():
 
     from app import create_app, db
     from models.distribusi_model import Distribusis  # noqa: F401
+    from models.distribusi_file_model import DistribusiFiles  # noqa: F401
 
     # This model is required for flask_migrate to make the table
     from models.user_model import User  # noqa: F401

verse/file_crawler.py

@@ -0,0 +1,22 @@
+import os
+
+import magic
+from distribusi.mappings import CODE_TYPES, FILE_TYPES, SUB_TYPES
+
+from models.distribusi_file_model import DistribusiFiles
+
+MIME_TYPE = magic.Magic(mime=True)
+
+
+def distribusi_file_with_type(full_path):
+    mime = MIME_TYPE.from_file(full_path)
+    type_, subtype = mime.split("/")
+    if type_ in FILE_TYPES:
+        print(f"distribusi file:{full_path}  type:{type_}")
+
+
+for root, dirs, files in os.walk("stash", topdown=True):
+    files = list(filter(lambda f: not f.startswith("."), files))
+    for file in files:
+        full_path = os.path.join(root, file)
+        distribusi_file_with_type(full_path)

verse/migrations/env.py

@@ -3,9 +3,10 @@ from __future__ import with_statement
 import logging
 from logging.config import fileConfig
 
-from alembic import context
 from flask import current_app
 
+from alembic import context
+
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
@@ -13,19 +14,17 @@ config = context.config
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 fileConfig(config.config_file_name)
-logger = logging.getLogger("alembic.env")
+logger = logging.getLogger('alembic.env')
 
 # add your model's MetaData object here
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
 config.set_main_option(
-    "sqlalchemy.url",
-    str(current_app.extensions["migrate"].db.get_engine().url).replace(
-        "%", "%%"
-    ),
-)
-target_metadata = current_app.extensions["migrate"].db.metadata
+    'sqlalchemy.url',
+    str(current_app.extensions['migrate'].db.get_engine().url).replace(
+        '%', '%%'))
+target_metadata = current_app.extensions['migrate'].db.metadata
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
@@ -66,20 +65,20 @@ def run_migrations_online():
     # when there are no changes to the schema
     # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
     def process_revision_directives(context, revision, directives):
-        if getattr(config.cmd_opts, "autogenerate", False):
+        if getattr(config.cmd_opts, 'autogenerate', False):
             script = directives[0]
             if script.upgrade_ops.is_empty():
                 directives[:] = []
-                logger.info("No changes in schema detected.")
+                logger.info('No changes in schema detected.')
 
-    connectable = current_app.extensions["migrate"].db.get_engine()
+    connectable = current_app.extensions['migrate'].db.get_engine()
 
     with connectable.connect() as connection:
         context.configure(
             connection=connection,
             target_metadata=target_metadata,
             process_revision_directives=process_revision_directives,
-            **current_app.extensions["migrate"].configure_args,
+            **current_app.extensions['migrate'].configure_args
         )
 
         with context.begin_transaction():

verse/migrations/versions/8ede45e978d9_.py

@@ -0,0 +1,36 @@
+"""empty message
+
+Revision ID: 8ede45e978d9
+Revises: 
+Create Date: 2024-04-30 22:15:48.023060
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '8ede45e978d9'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('distribusi_files', schema=None) as batch_op:
+        batch_op.alter_column('type',
+               existing_type=sa.VARCHAR(length=300),
+               nullable=True)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('distribusi_files', schema=None) as batch_op:
+        batch_op.alter_column('type',
+               existing_type=sa.VARCHAR(length=300),
+               nullable=False)
+
+    # ### end Alembic commands ###

verse/models/distribusi_file_model.py

@@ -7,7 +7,7 @@ class DistribusiFiles(db.Model):
     __tablename__ = "distribusi_files"
 
     id = db.Column(db.Integer, primary_key=True)
-    type = db.Column(db.String(300), nullable=False, unique=True)
+    type = db.Column(db.String(300), nullable=True, unique=False)
     distribusi = db.Column(db.Integer, db.ForeignKey("distribusis.id"))
     path = db.Column(db.String(4096), nullable=True, unique=False)
     description = db.Column(db.String(9), nullable=True, unique=False)