diff --git a/requirements/architecture.md b/requirements/architecture.md index 51b8314..6a6847d 100644 --- a/requirements/architecture.md +++ b/requirements/architecture.md @@ -41,6 +41,7 @@ - File inspection caches combined `ffprobe` data and crop-detection results per source and sampling window within one process to avoid repeated subprocess work. - Storage: - SQLite via SQLAlchemy ORM, with schema rooted in shows, patterns, tracks, media tags, track tags, shifted seasons, and generic properties. + - Ordered schema migrations are loaded dynamically from per-version-step modules under [`src/ffx/model/migration/`](/home/osgw/.local/src/codex/ffx/src/ffx/model/migration/). - A configuration JSON file supplies optional path, metadata-filtering, and filename-template settings. - Integration adapters: - Process execution wrapper for `ffmpeg`, `ffprobe`, `nice`, and `cpulimit`, with explicit disabled states for niceness and CPU limiting, support for both absolute `cpulimit` values and machine-wide percent input, and a combined `cpulimit -- nice -n ... ` execution shape when both limits are configured. @@ -62,7 +63,7 @@ - Config keys `databasePath`, `logDirectory`, and `outputFilenameTemplate`, plus optional metadata-filter rules. - Validation rules: - Only supported media-file extensions are accepted for conversion. - - Stored database version must match the runtime-required version. + - Stored database version must either match the runtime-required version already or have a supported sequential migration path to it. - A normalized descriptor may have at most one default and one forced stream per relevant track type. - Shifted-season ranges are intended not to overlap within the same owner scope and season, and runtime resolution prefers pattern-owned matches over show-owned matches. - TMDB lookups require a show ID and season and episode numbers. diff --git a/requirements/project.md b/requirements/project.md index db8d1ba..024e37d 100644 --- a/requirements/project.md +++ b/requirements/project.md @@ -49,6 +49,7 @@ - per-pattern stream definitions, - show-level and pattern-level shifted-season mappings, - internal database version properties. +- The system shall apply supported ordered database migrations automatically when opening an older local database file and shall fail fast when no supported path exists. - Detailed show, pattern, and duplicate-match management rules live in `requirements/pattern_management.md`. - The system shall inspect source media using `ffprobe` and derive a structured description of container metadata and streams. - The system shall optionally open a Textual UI to browse shows, inspect files, and create, edit, or delete shows, patterns, stream definitions, tags, and shifted-season rules. diff --git a/src/ffx/database.py b/src/ffx/database.py index aa94d87..70fc1a6 100644 --- a/src/ffx/database.py +++ b/src/ffx/database.py @@ -1,6 +1,6 @@ import os, click -from sqlalchemy import create_engine, inspect, text +from sqlalchemy import create_engine, inspect from sqlalchemy.orm import sessionmaker # Import the full model package so SQLAlchemy registers every mapped class @@ -9,6 +9,7 @@ import ffx.model from ffx.model.show import Base from ffx.model.property import Property +from ffx.model.migration import DatabaseVersionException, migrateDatabase from ffx.constants import DATABASE_VERSION @@ -16,10 +17,6 @@ from ffx.constants import DATABASE_VERSION DATABASE_VERSION_KEY = 'database_version' EXPECTED_TABLE_NAMES = set(Base.metadata.tables.keys()) -class DatabaseVersionException(Exception): - def __init__(self, errorMessage): - super().__init__(errorMessage) - def databaseContext(databasePath: str = ''): databaseContext = {} @@ -68,6 +65,7 @@ def bootstrapDatabaseIfNeeded(databaseContext): Base.metadata.create_all(databaseContext['engine']) + def ensureDatabaseVersion(databaseContext): currentDatabaseVersion = getDatabaseVersion(databaseContext) @@ -81,7 +79,7 @@ def ensureDatabaseVersion(databaseContext): ) if currentDatabaseVersion < DATABASE_VERSION: - migrateDatabase(databaseContext, currentDatabaseVersion, DATABASE_VERSION) + migrateDatabase(databaseContext, currentDatabaseVersion, DATABASE_VERSION, setDatabaseVersion) currentDatabaseVersion = getDatabaseVersion(databaseContext) if currentDatabaseVersion != DATABASE_VERSION: @@ -90,93 +88,6 @@ def ensureDatabaseVersion(databaseContext): ) -def migrateDatabase(databaseContext, currentVersion: int, targetVersion: int): - version = int(currentVersion) - - while version < int(targetVersion): - if version == 2: - migrateDatabaseV2ToV3(databaseContext) - version = 3 - setDatabaseVersion(databaseContext, version) - continue - - raise DatabaseVersionException( - f"No migration path from database version {version} to {targetVersion}" - ) - - -def migrateDatabaseV2ToV3(databaseContext): - engine = databaseContext['engine'] - inspector = inspect(engine) - shiftedSeasonColumns = { - column['name'] - for column in inspector.get_columns('shifted_seasons') - } - - if 'pattern_id' in shiftedSeasonColumns: - return - - with engine.begin() as connection: - connection.execute(text("PRAGMA foreign_keys=OFF")) - connection.execute( - text( - """ - CREATE TABLE shifted_seasons_v3 ( - id INTEGER PRIMARY KEY, - show_id INTEGER, - pattern_id INTEGER, - original_season INTEGER, - first_episode INTEGER DEFAULT -1, - last_episode INTEGER DEFAULT -1, - season_offset INTEGER DEFAULT 0, - episode_offset INTEGER DEFAULT 0, - FOREIGN KEY(show_id) REFERENCES shows(id) ON DELETE CASCADE, - FOREIGN KEY(pattern_id) REFERENCES patterns(id) ON DELETE CASCADE, - CHECK ( - (show_id IS NOT NULL AND pattern_id IS NULL) - OR (show_id IS NULL AND pattern_id IS NOT NULL) - ) - ) - """ - ) - ) - connection.execute( - text( - """ - INSERT INTO shifted_seasons_v3 ( - id, - show_id, - pattern_id, - original_season, - first_episode, - last_episode, - season_offset, - episode_offset - ) - SELECT - id, - show_id, - NULL, - original_season, - first_episode, - last_episode, - season_offset, - episode_offset - FROM shifted_seasons - """ - ) - ) - connection.execute(text("DROP TABLE shifted_seasons")) - connection.execute(text("ALTER TABLE shifted_seasons_v3 RENAME TO shifted_seasons")) - connection.execute( - text("CREATE INDEX ix_shifted_seasons_show_id ON shifted_seasons(show_id)") - ) - connection.execute( - text("CREATE INDEX ix_shifted_seasons_pattern_id ON shifted_seasons(pattern_id)") - ) - connection.execute(text("PRAGMA foreign_keys=ON")) - - def getDatabaseVersion(databaseContext): try: diff --git a/src/ffx/model/conversions/__init__.py b/src/ffx/model/conversions/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/ffx/model/conversions/conversion.py b/src/ffx/model/conversions/conversion.py deleted file mode 100644 index 927b2e8..0000000 --- a/src/ffx/model/conversions/conversion.py +++ /dev/null @@ -1,47 +0,0 @@ -import os, sys, importlib, inspect, glob, re - -from ffx.configuration_controller import ConfigurationController -from ffx.database import databaseContext - -from sqlalchemy import Engine -from sqlalchemy.orm import sessionmaker - - -class Conversion(): - - def __init__(self): - - self._context = {} - self._context['config'] = ConfigurationController() - - self._context['database'] = databaseContext(databasePath=self._context['config'].getDatabaseFilePath()) - - self.__databaseSession: sessionmaker = self._context['database']['session'] - self.__databaseEngine: Engine = self._context['database']['engine'] - - - @staticmethod - def list(): - - basePath = os.path.dirname(__file__) - - filenamePattern = re.compile("conversion_([0-9]+)_([0-9]+)\\.py") - - filenameList = [os.path.basename(fp) for fp in glob.glob(f"{ basePath }/*.py") if fp != __file__] - - versionTupleList = [(fm.group(1), fm.group(2)) for fn in filenameList if (fm := filenamePattern.search(fn))] - - return versionTupleList - - - @staticmethod - def getClassReference(versionFrom, versionTo): - importlib.import_module(f"ffx.model.conversions.conversion_{ versionFrom }_{ versionTo }") - for name, obj in inspect.getmembers(sys.modules[f"ffx.model.conversions.conversion_{ versionFrom }_{ versionTo }"]): - #HINT: Excluding DispositionCombination as it seems to be included by import (?) - if inspect.isclass(obj) and name != 'Conversion' and name.startswith('Conversion'): - return obj - - @staticmethod - def getAllClassReferences(): - return [Conversion.getClassReference(verFrom, verTo) for verFrom, verTo in Conversion.list()] diff --git a/src/ffx/model/conversions/conversion_2_3.py b/src/ffx/model/conversions/conversion_2_3.py deleted file mode 100644 index 3661a23..0000000 --- a/src/ffx/model/conversions/conversion_2_3.py +++ /dev/null @@ -1,17 +0,0 @@ -import os, sys, importlib, inspect, glob, re - -from .conversion import Conversion - - -class Conversion_2_3(Conversion): - - def __init__(self): - super().__init__() - - def applyConversion(self): - - s = self.__databaseSession() - e = self.__databaseEngine - - with e.connect() as c: - c.execute("ALTER TABLE user ADD COLUMN email VARCHAR(255)") diff --git a/src/ffx/model/conversions/conversion_3_4.py b/src/ffx/model/conversions/conversion_3_4.py deleted file mode 100644 index f1c1541..0000000 --- a/src/ffx/model/conversions/conversion_3_4.py +++ /dev/null @@ -1,7 +0,0 @@ -import os, sys, importlib, inspect, glob, re - -from .conversion import Conversion - - -class Conversion_3_4(Conversion): - pass diff --git a/src/ffx/model/migration/__init__.py b/src/ffx/model/migration/__init__.py new file mode 100644 index 0000000..4fd8681 --- /dev/null +++ b/src/ffx/model/migration/__init__.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import importlib + + +class DatabaseVersionException(Exception): + def __init__(self, errorMessage): + super().__init__(errorMessage) + + +def getMigrationStepModuleName(versionFrom: int, versionTo: int) -> str: + return f"ffx.model.migration.step_{int(versionFrom)}_{int(versionTo)}" + + +def loadMigrationStep(versionFrom: int, versionTo: int): + moduleName = getMigrationStepModuleName(versionFrom, versionTo) + + try: + module = importlib.import_module(moduleName) + except ModuleNotFoundError as ex: + if ex.name == moduleName: + raise DatabaseVersionException( + f"No migration path from database version {versionFrom} to {versionTo}" + ) from ex + raise + + migrationStep = getattr(module, "applyMigration", None) + if migrationStep is None: + raise DatabaseVersionException( + f"Migration module {moduleName} does not define applyMigration()" + ) + + return migrationStep + + +def migrateDatabase(databaseContext, currentVersion: int, targetVersion: int, setDatabaseVersion): + version = int(currentVersion) + target = int(targetVersion) + + while version < target: + nextVersion = version + 1 + migrationStep = loadMigrationStep(version, nextVersion) + migrationStep(databaseContext) + version = nextVersion + setDatabaseVersion(databaseContext, version) + diff --git a/src/ffx/model/migration/step_2_3.py b/src/ffx/model/migration/step_2_3.py new file mode 100644 index 0000000..5eacdb0 --- /dev/null +++ b/src/ffx/model/migration/step_2_3.py @@ -0,0 +1,74 @@ +from sqlalchemy import inspect, text + + +def applyMigration(databaseContext): + engine = databaseContext['engine'] + inspector = inspect(engine) + shiftedSeasonColumns = { + column['name'] + for column in inspector.get_columns('shifted_seasons') + } + + if 'pattern_id' in shiftedSeasonColumns: + return + + with engine.begin() as connection: + connection.execute(text("PRAGMA foreign_keys=OFF")) + connection.execute( + text( + """ + CREATE TABLE shifted_seasons_v3 ( + id INTEGER PRIMARY KEY, + show_id INTEGER, + pattern_id INTEGER, + original_season INTEGER, + first_episode INTEGER DEFAULT -1, + last_episode INTEGER DEFAULT -1, + season_offset INTEGER DEFAULT 0, + episode_offset INTEGER DEFAULT 0, + FOREIGN KEY(show_id) REFERENCES shows(id) ON DELETE CASCADE, + FOREIGN KEY(pattern_id) REFERENCES patterns(id) ON DELETE CASCADE, + CHECK ( + (show_id IS NOT NULL AND pattern_id IS NULL) + OR (show_id IS NULL AND pattern_id IS NOT NULL) + ) + ) + """ + ) + ) + connection.execute( + text( + """ + INSERT INTO shifted_seasons_v3 ( + id, + show_id, + pattern_id, + original_season, + first_episode, + last_episode, + season_offset, + episode_offset + ) + SELECT + id, + show_id, + NULL, + original_season, + first_episode, + last_episode, + season_offset, + episode_offset + FROM shifted_seasons + """ + ) + ) + connection.execute(text("DROP TABLE shifted_seasons")) + connection.execute(text("ALTER TABLE shifted_seasons_v3 RENAME TO shifted_seasons")) + connection.execute( + text("CREATE INDEX ix_shifted_seasons_show_id ON shifted_seasons(show_id)") + ) + connection.execute( + text("CREATE INDEX ix_shifted_seasons_pattern_id ON shifted_seasons(pattern_id)") + ) + connection.execute(text("PRAGMA foreign_keys=ON")) + diff --git a/tests/unit/test_migration.py b/tests/unit/test_migration.py new file mode 100644 index 0000000..7f3ae07 --- /dev/null +++ b/tests/unit/test_migration.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from pathlib import Path +import sys +import unittest + + +SRC_ROOT = Path(__file__).resolve().parents[2] / "src" + +if str(SRC_ROOT) not in sys.path: + sys.path.insert(0, str(SRC_ROOT)) + + +from ffx.model.migration import DatabaseVersionException, loadMigrationStep, migrateDatabase # noqa: E402 + + +class MigrationTests(unittest.TestCase): + def test_load_migration_step_returns_known_step(self): + migrationStep = loadMigrationStep(2, 3) + + self.assertTrue(callable(migrationStep)) + + def test_migrate_database_raises_when_step_module_is_missing(self): + updatedVersions = [] + + with self.assertRaises(DatabaseVersionException): + migrateDatabase({}, 1, 2, lambda context, version: updatedVersions.append(version)) + + self.assertEqual([], updatedVersions) + + +if __name__ == "__main__": + unittest.main() +