ff

requirements/architecture.md

@@ -41,6 +41,7 @@
   - File inspection caches combined `ffprobe` data and crop-detection results per source and sampling window within one process to avoid repeated subprocess work.
 - Storage:
   - SQLite via SQLAlchemy ORM, with schema rooted in shows, patterns, tracks, media tags, track tags, shifted seasons, and generic properties.
+  - Ordered schema migrations are loaded dynamically from per-version-step modules under [`src/ffx/model/migration/`](/home/osgw/.local/src/codex/ffx/src/ffx/model/migration/).
   - A configuration JSON file supplies optional path, metadata-filtering, and filename-template settings.
 - Integration adapters:
   - Process execution wrapper for `ffmpeg`, `ffprobe`, `nice`, and `cpulimit`, with explicit disabled states for niceness and CPU limiting, support for both absolute `cpulimit` values and machine-wide percent input, and a combined `cpulimit -- nice -n ... <command>` execution shape when both limits are configured.
@@ -62,7 +63,7 @@
   - Config keys `databasePath`, `logDirectory`, and `outputFilenameTemplate`, plus optional metadata-filter rules.
 - Validation rules:
   - Only supported media-file extensions are accepted for conversion.
-  - Stored database version must match the runtime-required version.
+  - Stored database version must either match the runtime-required version already or have a supported sequential migration path to it.
   - A normalized descriptor may have at most one default and one forced stream per relevant track type.
   - Shifted-season ranges are intended not to overlap within the same owner scope and season, and runtime resolution prefers pattern-owned matches over show-owned matches.
   - TMDB lookups require a show ID and season and episode numbers.

requirements/project.md

@@ -49,6 +49,7 @@
   - per-pattern stream definitions,
   - show-level and pattern-level shifted-season mappings,
   - internal database version properties.
+- The system shall apply supported ordered database migrations automatically when opening an older local database file and shall fail fast when no supported path exists.
 - Detailed show, pattern, and duplicate-match management rules live in `requirements/pattern_management.md`.
 - The system shall inspect source media using `ffprobe` and derive a structured description of container metadata and streams.
 - The system shall optionally open a Textual UI to browse shows, inspect files, and create, edit, or delete shows, patterns, stream definitions, tags, and shifted-season rules.

src/ffx/database.py

@@ -1,6 +1,6 @@
 import os, click
 
-from sqlalchemy import create_engine, inspect, text
+from sqlalchemy import create_engine, inspect
 from sqlalchemy.orm import sessionmaker
 
 # Import the full model package so SQLAlchemy registers every mapped class
@@ -9,6 +9,7 @@ import ffx.model
 from ffx.model.show import Base
 
 from ffx.model.property import Property
+from ffx.model.migration import DatabaseVersionException, migrateDatabase
 
 from ffx.constants import DATABASE_VERSION
 
@@ -16,10 +17,6 @@ from ffx.constants import DATABASE_VERSION
 DATABASE_VERSION_KEY = 'database_version'
 EXPECTED_TABLE_NAMES = set(Base.metadata.tables.keys())
 
-class DatabaseVersionException(Exception):
-    def __init__(self, errorMessage):
-        super().__init__(errorMessage)
-
 def databaseContext(databasePath: str = ''):
 
     databaseContext = {}
@@ -68,6 +65,7 @@ def bootstrapDatabaseIfNeeded(databaseContext):
 
     Base.metadata.create_all(databaseContext['engine'])
 
+
 def ensureDatabaseVersion(databaseContext):
 
     currentDatabaseVersion = getDatabaseVersion(databaseContext)
@@ -81,7 +79,7 @@ def ensureDatabaseVersion(databaseContext):
         )
 
     if currentDatabaseVersion < DATABASE_VERSION:
-        migrateDatabase(databaseContext, currentDatabaseVersion, DATABASE_VERSION)
+        migrateDatabase(databaseContext, currentDatabaseVersion, DATABASE_VERSION, setDatabaseVersion)
         currentDatabaseVersion = getDatabaseVersion(databaseContext)
 
     if currentDatabaseVersion != DATABASE_VERSION:
@@ -90,93 +88,6 @@ def ensureDatabaseVersion(databaseContext):
         )
 
 
-def migrateDatabase(databaseContext, currentVersion: int, targetVersion: int):
-    version = int(currentVersion)
-
-    while version < int(targetVersion):
-        if version == 2:
-            migrateDatabaseV2ToV3(databaseContext)
-            version = 3
-            setDatabaseVersion(databaseContext, version)
-            continue
-
-        raise DatabaseVersionException(
-            f"No migration path from database version {version} to {targetVersion}"
-        )
-
-
-def migrateDatabaseV2ToV3(databaseContext):
-    engine = databaseContext['engine']
-    inspector = inspect(engine)
-    shiftedSeasonColumns = {
-        column['name']
-        for column in inspector.get_columns('shifted_seasons')
-    }
-
-    if 'pattern_id' in shiftedSeasonColumns:
-        return
-
-    with engine.begin() as connection:
-        connection.execute(text("PRAGMA foreign_keys=OFF"))
-        connection.execute(
-            text(
-                """
-                CREATE TABLE shifted_seasons_v3 (
-                    id INTEGER PRIMARY KEY,
-                    show_id INTEGER,
-                    pattern_id INTEGER,
-                    original_season INTEGER,
-                    first_episode INTEGER DEFAULT -1,
-                    last_episode INTEGER DEFAULT -1,
-                    season_offset INTEGER DEFAULT 0,
-                    episode_offset INTEGER DEFAULT 0,
-                    FOREIGN KEY(show_id) REFERENCES shows(id) ON DELETE CASCADE,
-                    FOREIGN KEY(pattern_id) REFERENCES patterns(id) ON DELETE CASCADE,
-                    CHECK (
-                        (show_id IS NOT NULL AND pattern_id IS NULL)
-                        OR (show_id IS NULL AND pattern_id IS NOT NULL)
-                    )
-                )
-                """
-            )
-        )
-        connection.execute(
-            text(
-                """
-                INSERT INTO shifted_seasons_v3 (
-                    id,
-                    show_id,
-                    pattern_id,
-                    original_season,
-                    first_episode,
-                    last_episode,
-                    season_offset,
-                    episode_offset
-                )
-                SELECT
-                    id,
-                    show_id,
-                    NULL,
-                    original_season,
-                    first_episode,
-                    last_episode,
-                    season_offset,
-                    episode_offset
-                FROM shifted_seasons
-                """
-            )
-        )
-        connection.execute(text("DROP TABLE shifted_seasons"))
-        connection.execute(text("ALTER TABLE shifted_seasons_v3 RENAME TO shifted_seasons"))
-        connection.execute(
-            text("CREATE INDEX ix_shifted_seasons_show_id ON shifted_seasons(show_id)")
-        )
-        connection.execute(
-            text("CREATE INDEX ix_shifted_seasons_pattern_id ON shifted_seasons(pattern_id)")
-        )
-        connection.execute(text("PRAGMA foreign_keys=ON"))
-
-
 def getDatabaseVersion(databaseContext):
 
     try:

src/ffx/model/conversions/conversion.py

@@ -1,47 +0,0 @@
-import os, sys, importlib, inspect, glob, re
-
-from ffx.configuration_controller import ConfigurationController
-from ffx.database import databaseContext
-
-from sqlalchemy import Engine
-from sqlalchemy.orm import sessionmaker
-
-
-class Conversion():
-
-    def __init__(self):
-
-        self._context = {}
-        self._context['config'] = ConfigurationController()
-
-        self._context['database'] = databaseContext(databasePath=self._context['config'].getDatabaseFilePath())
-
-        self.__databaseSession: sessionmaker = self._context['database']['session']
-        self.__databaseEngine: Engine = self._context['database']['engine']
-
-
-    @staticmethod    
-    def list():
-
-        basePath = os.path.dirname(__file__)
-
-        filenamePattern = re.compile("conversion_([0-9]+)_([0-9]+)\\.py")
-
-        filenameList = [os.path.basename(fp) for fp in glob.glob(f"{ basePath }/*.py") if fp != __file__]
-
-        versionTupleList = [(fm.group(1), fm.group(2)) for fn in filenameList if (fm := filenamePattern.search(fn))]
-
-        return versionTupleList
-
-
-    @staticmethod
-    def getClassReference(versionFrom, versionTo):
-        importlib.import_module(f"ffx.model.conversions.conversion_{ versionFrom }_{ versionTo }")
-        for name, obj in inspect.getmembers(sys.modules[f"ffx.model.conversions.conversion_{ versionFrom }_{ versionTo }"]):
-            #HINT: Excluding DispositionCombination as it seems to be included by import (?)
-            if inspect.isclass(obj) and name != 'Conversion' and name.startswith('Conversion'):
-                return obj
-
-    @staticmethod
-    def getAllClassReferences():
-        return [Conversion.getClassReference(verFrom, verTo) for verFrom, verTo in Conversion.list()]

src/ffx/model/conversions/conversion_2_3.py

@@ -1,17 +0,0 @@
-import os, sys, importlib, inspect, glob, re
-
-from .conversion import Conversion
-
-
-class Conversion_2_3(Conversion):
-    
-    def __init__(self):
-        super().__init__()
-
-    def applyConversion(self):
-
-        s = self.__databaseSession()
-        e = self.__databaseEngine
-
-        with e.connect() as c:
-            c.execute("ALTER TABLE user ADD COLUMN email VARCHAR(255)")

src/ffx/model/conversions/conversion_3_4.py

@@ -1,7 +0,0 @@
-import os, sys, importlib, inspect, glob, re
-
-from .conversion import Conversion
-
-
-class Conversion_3_4(Conversion):
-    pass

src/ffx/model/migration/__init__.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import importlib
+
+
+class DatabaseVersionException(Exception):
+    def __init__(self, errorMessage):
+        super().__init__(errorMessage)
+
+
+def getMigrationStepModuleName(versionFrom: int, versionTo: int) -> str:
+    return f"ffx.model.migration.step_{int(versionFrom)}_{int(versionTo)}"
+
+
+def loadMigrationStep(versionFrom: int, versionTo: int):
+    moduleName = getMigrationStepModuleName(versionFrom, versionTo)
+
+    try:
+        module = importlib.import_module(moduleName)
+    except ModuleNotFoundError as ex:
+        if ex.name == moduleName:
+            raise DatabaseVersionException(
+                f"No migration path from database version {versionFrom} to {versionTo}"
+            ) from ex
+        raise
+
+    migrationStep = getattr(module, "applyMigration", None)
+    if migrationStep is None:
+        raise DatabaseVersionException(
+            f"Migration module {moduleName} does not define applyMigration()"
+        )
+
+    return migrationStep
+
+
+def migrateDatabase(databaseContext, currentVersion: int, targetVersion: int, setDatabaseVersion):
+    version = int(currentVersion)
+    target = int(targetVersion)
+
+    while version < target:
+        nextVersion = version + 1
+        migrationStep = loadMigrationStep(version, nextVersion)
+        migrationStep(databaseContext)
+        version = nextVersion
+        setDatabaseVersion(databaseContext, version)
+

src/ffx/model/migration/step_2_3.py

@@ -0,0 +1,74 @@
+from sqlalchemy import inspect, text
+
+
+def applyMigration(databaseContext):
+    engine = databaseContext['engine']
+    inspector = inspect(engine)
+    shiftedSeasonColumns = {
+        column['name']
+        for column in inspector.get_columns('shifted_seasons')
+    }
+
+    if 'pattern_id' in shiftedSeasonColumns:
+        return
+
+    with engine.begin() as connection:
+        connection.execute(text("PRAGMA foreign_keys=OFF"))
+        connection.execute(
+            text(
+                """
+                CREATE TABLE shifted_seasons_v3 (
+                    id INTEGER PRIMARY KEY,
+                    show_id INTEGER,
+                    pattern_id INTEGER,
+                    original_season INTEGER,
+                    first_episode INTEGER DEFAULT -1,
+                    last_episode INTEGER DEFAULT -1,
+                    season_offset INTEGER DEFAULT 0,
+                    episode_offset INTEGER DEFAULT 0,
+                    FOREIGN KEY(show_id) REFERENCES shows(id) ON DELETE CASCADE,
+                    FOREIGN KEY(pattern_id) REFERENCES patterns(id) ON DELETE CASCADE,
+                    CHECK (
+                        (show_id IS NOT NULL AND pattern_id IS NULL)
+                        OR (show_id IS NULL AND pattern_id IS NOT NULL)
+                    )
+                )
+                """
+            )
+        )
+        connection.execute(
+            text(
+                """
+                INSERT INTO shifted_seasons_v3 (
+                    id,
+                    show_id,
+                    pattern_id,
+                    original_season,
+                    first_episode,
+                    last_episode,
+                    season_offset,
+                    episode_offset
+                )
+                SELECT
+                    id,
+                    show_id,
+                    NULL,
+                    original_season,
+                    first_episode,
+                    last_episode,
+                    season_offset,
+                    episode_offset
+                FROM shifted_seasons
+                """
+            )
+        )
+        connection.execute(text("DROP TABLE shifted_seasons"))
+        connection.execute(text("ALTER TABLE shifted_seasons_v3 RENAME TO shifted_seasons"))
+        connection.execute(
+            text("CREATE INDEX ix_shifted_seasons_show_id ON shifted_seasons(show_id)")
+        )
+        connection.execute(
+            text("CREATE INDEX ix_shifted_seasons_pattern_id ON shifted_seasons(pattern_id)")
+        )
+        connection.execute(text("PRAGMA foreign_keys=ON"))
+

tests/unit/test_migration.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from pathlib import Path
+import sys
+import unittest
+
+
+SRC_ROOT = Path(__file__).resolve().parents[2] / "src"
+
+if str(SRC_ROOT) not in sys.path:
+    sys.path.insert(0, str(SRC_ROOT))
+
+
+from ffx.model.migration import DatabaseVersionException, loadMigrationStep, migrateDatabase  # noqa: E402
+
+
+class MigrationTests(unittest.TestCase):
+    def test_load_migration_step_returns_known_step(self):
+        migrationStep = loadMigrationStep(2, 3)
+
+        self.assertTrue(callable(migrationStep))
+
+    def test_migrate_database_raises_when_step_module_is_missing(self):
+        updatedVersions = []
+
+        with self.assertRaises(DatabaseVersionException):
+            migrateDatabase({}, 1, 2, lambda context, version: updatedVersions.append(version))
+
+        self.assertEqual([], updatedVersions)
+
+
+if __name__ == "__main__":
+    unittest.main()
+