diff --git a/.gitignore b/.gitignore index c624ec6..25d914e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ __pycache__/ *.py[cod] junk/ -.vscode .ipynb_checkpoints/ tools/ansible/inventory/hawaii.yml tools/ansible/inventory/peppermint.yml @@ -17,6 +16,7 @@ dist/ *.egg-info/ .venv/ venv/ +docs/_build/ .codex @@ -24,4 +24,4 @@ venv/ *.webm *.mp4 ffmpeg2pass-0.log -*.sup \ No newline at end of file +*.sup diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..6c7e62c --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,11 @@ +{ + "recommendations": [ + "swyddfa.esbonio", + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.debugpy", + "tamasfe.even-better-toml", + "redhat.vscode-yaml", + "DavidAnson.vscode-markdownlint" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..daa653b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,18 @@ +{ + "esbonio.sphinx.pythonCommand": "${venv:.venv}/bin/python", + "esbonio.sphinx.buildCommand": [ + "sphinx-build", + "-b", + "html", + "docs", + "docs/_build/html" + ], + "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python", + "python.testing.pytestEnabled": true, + "python.testing.pytestArgs": [ + "--ignore=tests/legacy", + "--ignore=tests/support", + "tests" + ], + "restructuredtext.confPath": "${workspaceFolder}/docs" +} diff --git a/README.md b/README.md index d5c2158..0d69110 100644 --- a/README.md +++ b/README.md @@ -99,101 +99,4 @@ TMDB-backed metadata enrichment requires `TMDB_API_KEY` to be set in the environ ## Version History -### 0.4.3 - -- styled ASS subtitle sources with embedded font attachments are now detected explicitly, keep MKV output, preserve current source font attachments, and reject incompatible sidecar subtitle import -- attachment descriptors are now treated as source-runtime data instead of pattern schema data, so pattern persistence skips them and source-vs-pattern validation ignores them -- inspect differences no longer report planned changes for attachment filename/count drift while still showing attachment streams in the stream table - -### 0.4.2 - -- pattern details now show an inline `Show: ` hint next to the quality field when the pattern itself has no stored quality but the selected show does -- inspect stream tables now show attachment format labels like `TTF` in the codec column and keep attachment language cells blank instead of showing an undefined language -- ffmpeg damaged-MP3 diagnostics now recognize additional corruption lines such as `invalid new backstep`, keeping them grouped under the `warn-corrupt-mpeg-audio` review summary - -### 0.4.1 - -- `convert` now supports `--copy-video` and `--copy-audio` to keep the selected stream type in copy mode without applying the corresponding reencode flags, filters, or formatting options -- ffmpeg conversions now monitor diagnostics while the process is running, retry unset AVI packet timestamps once with `-fflags +genpts`, and stop early when a file should be skipped instead of waiting for the full job to finish -- end-of-run convert summaries now list only ffmpeg findings that still require review, including named remedy identifiers such as `warn-corrupt-mpeg-audio` -- `upgrade` now finishes by reporting the installed FFX version together with the active bundle branch - -### 0.3.1 - -- debug mode screen titles now append the active Textual screen class name, making screen-specific troubleshooting easier during inspect and edit flows -- `--cut` again works as a combined flag/option: omitted disables cutting, bare `--cut` applies the default `60,180`, and explicit duration or `START,DURATION` values stay supported -- H.265 unmux commands no longer force an invalid `-f h265` output format, keeping ffmpeg copy extraction aligned with the required Annex B bitstream filter -- H.264 encoding now falls back from `libx264` to `libopenh264` with a warning when needed, and the test fixtures use the same encoder fallback so the suite remains portable across ffmpeg builds - -### 0.3.0 - -- inspect and edit screens now refresh nested track and pattern changes more reliably, with inspect-mode tables aligned to the target pattern view shown in the differences pane -- metadata editing got a follow-up polish pass with clearer ffmpeg notifications, a shared in-screen log pane, safer apply/reload handling, and expanded cleanup and normalization coverage -- track and asset probing recognize additional codecs, and the modern test suite now covers more metadata-editor, change-set, screen-state, and asset-probe behavior -- Textual now requires version `8.0` or newer to match the UI APIs used by the current screens - -### 0.2.6 - -- DB-free `ffx edit` workflow for in-place metadata editing via temporary-file rewrite -- inspect and edit workflows split into dedicated Textual screens with shared media-workflow support -- Textual tables and row actions now separate raw data from rendered labels to avoid markup leaking into stored metadata -- responsive screen layout pass, `Esc` back handling, sortable show/inspect tables, and improved edit-screen notifications/toggles -- application-wide UTF-8 i18n catalogs with language precedence from CLI over config over system over German default -- metadata normalization extended for localized subtitle titles, ISO language cleanup, and smarter track editor language/title helpers - -### 0.2.5 - -- show-level quality and notes fields -- pattern-over-show-over-default season-shift resolution with dynamic DB migration loading -- migration prompt now reports the upgrade path and creates an in-place DB backup before applying schema changes -- `upgrade --branch ` now fetches remote-only branches before switching -- `unmux` now applies season shifting to subtitle output filenames -- convert now keeps DB-defined target subtitle dispositions authoritative over sidecar filename disposition flags when a pattern definition exists -- focused modern tests added around migrations, unmux, upgrade, and subtitle-disposition import precedence - -### 0.2.4 - -- lightweight CLI commands now stay import-light via lazy runtime loading -- setup/config templating moved to `assets/ffx.json.j2` -- aligned two-step local setup wrappers: `ffx setup` and `ffx configure_workstation` -- combined `ffprobe` payload reuse in `FileProperties` -- configurable crop-detect sampling plus per-process crop result caching -- single-query controller accessors and conditional DB schema bootstrap -- shared screen bootstrap/controller wiring for large detail screens -- configurable default season/episode digit lengths -- digit-aware `rename` and padded `unmux` filename markers - -### 0.2.3 - -- PyPI packaging -- output filename templating -- season shifting -- DB versioning - -### 0.2.2 - -- CLI overrides - -### 0.2.1 - -- signature handling -- tag cleanup -- bugfixes and refactoring - -### 0.2.0 - -- tests -- config file - -### 0.1.3 - -- subtitle file imports - -### 0.1.2 - -- bugfixes - -### 0.1.1 - -- bugfixes -- TMDB show identification +See the [version history](docs/history.rst) for release notes. diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..38f13a3 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,21 @@ +SPHINXOPTS ?= +VENV_SPHINXBUILD = ../.venv/bin/sphinx-build +SPHINXBUILD ?= $(if $(wildcard $(VENV_SPHINXBUILD)),$(VENV_SPHINXBUILD),sphinx-build) +SOURCEDIR = . +BUILDDIR = _build + +.PHONY: help clean html linkcheck + +help: + @echo "Please use 'make ' where is one of" + @echo " html to make standalone HTML files" + @echo " linkcheck to check all external links for integrity" + +clean: + rm -rf "$(BUILDDIR)" + +html: + @$(SPHINXBUILD) -b html "$(SOURCEDIR)" "$(BUILDDIR)/html" $(SPHINXOPTS) + +linkcheck: + @$(SPHINXBUILD) -b linkcheck "$(SOURCEDIR)" "$(BUILDDIR)/linkcheck" $(SPHINXOPTS) diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000..7abda7a --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,31 @@ +API Reference +============= + +This section exposes selected modules that are useful when working on tests, +diagnostics, process execution, metadata editing, and file probing. + +CLI Helpers +----------- + +.. automodule:: ffx.cli + :members: + :undoc-members: + +Process Helpers +--------------- + +.. automodule:: ffx.process + :members: + :undoc-members: + +File Probing +------------ + +.. automodule:: ffx.file_properties + +Metadata Editing +---------------- + +.. automodule:: ffx.metadata_editor + :members: + :undoc-members: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..54a69b5 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from importlib.metadata import PackageNotFoundError, version as package_version +from pathlib import Path +import sys + + +ROOT_DIR = Path(__file__).resolve().parents[1] +SRC_DIR = ROOT_DIR / "src" +sys.path.insert(0, str(SRC_DIR)) + +project = "FFX" +author = "javanaut@maveno.de" +copyright = "2026, Maveno" + +try: + release = package_version("ffx") +except PackageNotFoundError: + release = "0.0.0" +version = release + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx_copybutton", +] + +source_suffix = { + ".rst": "restructuredtext", +} + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +html_theme = "sphinx_rtd_theme" +html_title = "FFX" +html_static_path = [] + +autodoc_typehints = "description" +autodoc_member_order = "bysource" +napoleon_google_docstring = True +napoleon_numpy_docstring = True + diff --git a/docs/development.rst b/docs/development.rst new file mode 100644 index 0000000..0d9281e --- /dev/null +++ b/docs/development.rst @@ -0,0 +1,50 @@ +Development +=========== + +The repo-local ``.venv`` is the preferred environment for contributors working +on tests or documentation: + +.. code-block:: sh + + tests/prepare.sh + +The preparation script installs the package in editable mode with both test and +documentation extras: + +.. code-block:: text + + .[test,docs] + +Run Tests +--------- + +Run the modern pytest suite: + +.. code-block:: sh + + .venv/bin/python -m pytest --ignore=tests/legacy --ignore=tests/support tests + +The legacy harness remains available separately and is intentionally not part of +the default pytest run. + +Build Docs +---------- + +Build HTML documentation: + +.. code-block:: sh + + .venv/bin/sphinx-build -b html docs docs/_build/html + +The same command is wrapped by the Sphinx ``Makefile``: + +.. code-block:: sh + + make -C docs html + +VS Code +------- + +The repository includes ``.vscode/extensions.json`` with recommended +extensions, including Esbonio for Sphinx language-server support. The workspace +settings point Python tooling and Esbonio at the repo-local ``.venv``. diff --git a/docs/esbonio.db b/docs/esbonio.db new file mode 100644 index 0000000..0de02ec Binary files /dev/null and b/docs/esbonio.db differ diff --git a/docs/file_formats.md b/docs/file_formats.md deleted file mode 100644 index 25f827c..0000000 --- a/docs/file_formats.md +++ /dev/null @@ -1,170 +0,0 @@ -# File Formats - -This document captures source-file-format notes that complement the normative -requirements in `requirements/source_file_formats.md`. - -The first documented format is a Matroska source that carries styled ASS/SSA -subtitle streams together with embedded font attachments. - -## Styled ASS In Matroska With Embedded Fonts - -These files are typically `.mkv` releases where subtitle rendering quality -depends on keeping both parts of the subtitle package together: - -- one or more subtitle streams with codec `ass` -- one or more attachment streams that embed font files used by those subtitles - -This matters because ASS subtitles are not plain text subtitles in the narrow -WebVTT sense. They can carry layout, styling, positioning, karaoke, signs, and -other typesetting effects. If the matching embedded fonts are lost, consumers -can still see subtitle text but the intended styling and sometimes glyph -coverage can be degraded. - -For FFX this format is special because the ASS subtitle streams should remain -normally editable and mappable, while the related font attachments should be -transported unchanged. - -## Observed Sample - -Assessment date: `2026-04-17` - -Observed sample file: - -- `tests/assets/boruto_s01e283_ssa.mkv` - -Commands used for assessment: - -```bash -ffprobe tests/assets/boruto_s01e283_ssa.mkv -ffprobe -hide_banner -show_format -show_streams -of json tests/assets/boruto_s01e283_ssa.mkv -``` - -Observed stream layout: - -| Stream index | Kind | Key details | -| --- | --- | --- | -| `0` | video | `codec_name=h264` | -| `1` | audio | `codec_name=aac`, `language=jpn` | -| `2` | subtitle | `codec_name=ass`, `language=ger`, default | -| `3` | subtitle | `codec_name=ass`, `language=eng` | -| `4`-`13` | attachment | `tags.mimetype=font/ttf`, `.ttf` filenames | - -Observed attachment filenames: - -- `AmazonEmberTanuki-Italic.ttf` -- `AmazonEmberTanuki-Regular.ttf` -- `Arial.ttf` -- `Arial Bold.ttf` -- `Georgia.ttf` -- `Times New Roman.ttf` -- `Times New Roman Bold.ttf` -- `Trebuchet MS.ttf` -- `Verdana.ttf` -- `Verdana Bold.ttf` - -Important probe behavior from the real sample: - -- Plain `ffprobe` lists the font streams as `Attachment: none`. -- Plain `ffprobe` also prints warnings such as `Could not find codec - parameters for stream 4 (Attachment: none): unknown codec` and later - `Unsupported codec with id 0 for input stream ...`. -- The JSON produced by `FileProperties.FFPROBE_COMMAND_TOKENS` - (`ffprobe -hide_banner -show_format -show_streams -of json`) still exposes - the attachment streams clearly through `codec_type="attachment"` and the - attachment tags. -- In that JSON, the attachment streams do not expose `codec_name`. - -This last point is important for FFX: robust detection must not depend on -attachment `codec_name` being present. - -## Detection Guidance - -Current known indicators for this format are: - -- one or more subtitle streams with `codec_type="subtitle"` and - `codec_name="ass"` -- one or more attachment streams with `codec_type="attachment"` -- attachment tags that identify embedded fonts, especially - `tags.mimetype="font/ttf"` -- attachment filenames that end in `.ttf` - -The pattern can vary. FFX should therefore treat the above as a cluster of -signals rather than an exact signature tied to one file. - -Inference from the observed sample plus FFmpeg documentation: - -- MIME matching should not be limited to `font/ttf` alone. -- The Boruto sample uses `font/ttf`. -- FFmpeg's Matroska attachment example uses - `mimetype=application/x-truetype-font` for a `.ttf` attachment. -- Detection should therefore normalize multiple TTF-like MIME values rather - than depend on a single exact string. - -## Processing Expectations In FFX - -The format-specific requirements live in -`requirements/source_file_formats.md`. In practical terms, FFX should: - -- recognize the ASS-plus-font-attachment pattern even when attachment probe - data is incomplete -- tell the operator that the pattern was detected and that special handling is - being used -- reject sidecar subtitle import for such sources, because converting or - replacing these subtitle tracks with ordinary external text subtitles would - break the intended subtitle package -- continue to allow normal manipulation of the ASS subtitle tracks themselves -- preserve the font attachment streams unchanged - -## FFmpeg Notes - -Relevant FFmpeg documentation confirms several behaviors that line up with -FFX's needs: - -- FFmpeg documents `-attach` as adding an attachment stream to the output, and - explicitly names Matroska fonts used in subtitle rendering as an example. -- FFmpeg documents attachment streams as regular streams that are created after - the mapped media streams. -- FFmpeg documents `-dump_attachment` for extracting attachment streams, which - is useful for debugging or validating a source file's embedded fonts. -- FFmpeg's Matroska example requires a `mimetype` metadata tag for attached - fonts, which is consistent with using attachment tags as detection signals. -- FFmpeg also notes that attachments are implemented as codec extradata. That - helps explain why probe output for attachment streams can look different from - ordinary audio, video, and subtitle streams. - -Implication for FFX: - -- Attachment preservation is not an optional cosmetic feature for this format. - It is part of preserving the subtitle package correctly. - -## Jellyfin Notes - -Jellyfin's documentation also supports keeping this format intact: - -- Jellyfin's subtitle compatibility table lists `ASS/SSA` as supported in - `MKV` and not supported in `MP4`. -- Jellyfin notes that when subtitles must be transcoded, they are either - converted to a supported format or burned into the video, and burning them in - is the most CPU-intensive path. -- Jellyfin's subtitle-extraction example for `SSA/ASS` first dumps attachment - streams and then extracts the ASS subtitle stream, which reflects the real - relationship between ASS subtitles and embedded fonts in MKV releases. -- Jellyfin's font documentation says text-based subtitles require fonts to - render properly. -- Jellyfin's configuration documentation says the web client uses configured - fallback fonts for ASS subtitles when other fonts such as MKV attachments or - client-side fonts are not available. - -Inference from the Jellyfin compatibility tables: - -- Keeping this subtitle format in Matroska is the safest interoperability - choice for Jellyfin consumers. -- Converting the subtitle payload to WebVTT would lose styled ASS behavior. -- Dropping the attachment streams would force client or fallback font - substitution and can change appearance or glyph coverage. - -## References - -- FFmpeg documentation: https://ffmpeg.org/ffmpeg.html -- Jellyfin codec support: https://jellyfin.org/docs/general/clients/codec-support/ -- Jellyfin configuration and fonts: https://jellyfin.org/docs/general/administration/configuration/ diff --git a/docs/file_formats.rst b/docs/file_formats.rst new file mode 100644 index 0000000..df9af19 --- /dev/null +++ b/docs/file_formats.rst @@ -0,0 +1,192 @@ +File Formats +============ + +This document captures source-file-format notes that complement the normative +requirements in ``requirements/source_file_formats.md``. + +The first documented format is a Matroska source that carries styled ASS/SSA +subtitle streams together with embedded font attachments. + +Styled ASS In Matroska With Embedded Fonts +------------------------------------------ + +These files are typically ``.mkv`` releases where subtitle rendering quality +depends on keeping both parts of the subtitle package together: + +* one or more subtitle streams with codec ``ass`` +* one or more attachment streams that embed font files used by those subtitles + +This matters because ASS subtitles are not plain text subtitles in the narrow +WebVTT sense. They can carry layout, styling, positioning, karaoke, signs, and +other typesetting effects. If the matching embedded fonts are lost, consumers +can still see subtitle text but the intended styling and sometimes glyph +coverage can be degraded. + +For FFX this format is special because the ASS subtitle streams should remain +normally editable and mappable, while the related font attachments should be +transported unchanged. + +Observed Sample +--------------- + +Assessment date: ``2026-04-17`` + +Observed sample file: + +* ``tests/assets/boruto_s01e283_ssa.mkv`` + +Commands used for assessment: + +.. code-block:: bash + + ffprobe tests/assets/boruto_s01e283_ssa.mkv + ffprobe -hide_banner -show_format -show_streams -of json tests/assets/boruto_s01e283_ssa.mkv + +Observed stream layout: + +.. list-table:: + :header-rows: 1 + + * - Stream index + - Kind + - Key details + * - ``0`` + - video + - ``codec_name=h264`` + * - ``1`` + - audio + - ``codec_name=aac``, ``language=jpn`` + * - ``2`` + - subtitle + - ``codec_name=ass``, ``language=ger``, default + * - ``3`` + - subtitle + - ``codec_name=ass``, ``language=eng`` + * - ``4``-``13`` + - attachment + - ``tags.mimetype=font/ttf``, ``.ttf`` filenames + +Observed attachment filenames: + +* ``AmazonEmberTanuki-Italic.ttf`` +* ``AmazonEmberTanuki-Regular.ttf`` +* ``Arial.ttf`` +* ``Arial Bold.ttf`` +* ``Georgia.ttf`` +* ``Times New Roman.ttf`` +* ``Times New Roman Bold.ttf`` +* ``Trebuchet MS.ttf`` +* ``Verdana.ttf`` +* ``Verdana Bold.ttf`` + +Important probe behavior from the real sample: + +* Plain ``ffprobe`` lists the font streams as ``Attachment: none``. +* Plain ``ffprobe`` also prints warnings such as ``Could not find codec + parameters for stream 4 (Attachment: none): unknown codec`` and later + ``Unsupported codec with id 0 for input stream ...``. +* The JSON produced by ``FileProperties.FFPROBE_COMMAND_TOKENS`` + (``ffprobe -hide_banner -show_format -show_streams -of json``) still exposes + the attachment streams clearly through ``codec_type="attachment"`` and the + attachment tags. +* In that JSON, the attachment streams do not expose ``codec_name``. + +This last point is important for FFX: robust detection must not depend on +attachment ``codec_name`` being present. + +Detection Guidance +------------------ + +Current known indicators for this format are: + +* one or more subtitle streams with ``codec_type="subtitle"`` and + ``codec_name="ass"`` +* one or more attachment streams with ``codec_type="attachment"`` +* attachment tags that identify embedded fonts, especially + ``tags.mimetype="font/ttf"`` +* attachment filenames that end in ``.ttf`` + +The pattern can vary. FFX should therefore treat the above as a cluster of +signals rather than an exact signature tied to one file. + +Inference from the observed sample plus FFmpeg documentation: + +* MIME matching should not be limited to ``font/ttf`` alone. +* The Boruto sample uses ``font/ttf``. +* FFmpeg's Matroska attachment example uses + ``mimetype=application/x-truetype-font`` for a ``.ttf`` attachment. +* Detection should therefore normalize multiple TTF-like MIME values rather + than depend on a single exact string. + +Processing Expectations In FFX +------------------------------ + +The format-specific requirements live in +``requirements/source_file_formats.md``. In practical terms, FFX should: + +* recognize the ASS-plus-font-attachment pattern even when attachment probe data + is incomplete +* tell the operator that the pattern was detected and that special handling is + being used +* reject sidecar subtitle import for such sources, because converting or + replacing these subtitle tracks with ordinary external text subtitles would + break the intended subtitle package +* continue to allow normal manipulation of the ASS subtitle tracks themselves +* preserve the font attachment streams unchanged + +FFmpeg Notes +------------ + +Relevant FFmpeg documentation confirms several behaviors that line up with +FFX's needs: + +* FFmpeg documents ``-attach`` as adding an attachment stream to the output, and + explicitly names Matroska fonts used in subtitle rendering as an example. +* FFmpeg documents attachment streams as regular streams that are created after + the mapped media streams. +* FFmpeg documents ``-dump_attachment`` for extracting attachment streams, which + is useful for debugging or validating a source file's embedded fonts. +* FFmpeg's Matroska example requires a ``mimetype`` metadata tag for attached + fonts, which is consistent with using attachment tags as detection signals. +* FFmpeg also notes that attachments are implemented as codec extradata. That + helps explain why probe output for attachment streams can look different from + ordinary audio, video, and subtitle streams. + +Implication for FFX: + +* Attachment preservation is not an optional cosmetic feature for this format. + It is part of preserving the subtitle package correctly. + +Jellyfin Notes +-------------- + +Jellyfin's documentation also supports keeping this format intact: + +* Jellyfin's subtitle compatibility table lists ``ASS/SSA`` as supported in + ``MKV`` and not supported in ``MP4``. +* Jellyfin notes that when subtitles must be transcoded, they are either + converted to a supported format or burned into the video, and burning them in + is the most CPU-intensive path. +* Jellyfin's subtitle-extraction example for ``SSA/ASS`` first dumps attachment + streams and then extracts the ASS subtitle stream, which reflects the real + relationship between ASS subtitles and embedded fonts in MKV releases. +* Jellyfin's font documentation says text-based subtitles require fonts to + render properly. +* Jellyfin's configuration documentation says the web client uses configured + fallback fonts for ASS subtitles when other fonts such as MKV attachments or + client-side fonts are not available. + +Inference from the Jellyfin compatibility tables: + +* Keeping this subtitle format in Matroska is the safest interoperability choice + for Jellyfin consumers. +* Converting the subtitle payload to WebVTT would lose styled ASS behavior. +* Dropping the attachment streams would force client or fallback font + substitution and can change appearance or glyph coverage. + +References +---------- + +* FFmpeg documentation: https://ffmpeg.org/ffmpeg.html +* Jellyfin codec support: https://jellyfin.org/docs/general/clients/codec-support/ +* Jellyfin configuration and fonts: https://jellyfin.org/docs/general/administration/configuration/ diff --git a/docs/history.rst b/docs/history.rst new file mode 100644 index 0000000..d3c956f --- /dev/null +++ b/docs/history.rst @@ -0,0 +1,123 @@ +Version History +=============== + +0.4.4 +----- + +- External subtitle imports can now match prefixless sidecar files against the source basename, choose their extension, and confirm partial substitutions. +- Unmux now creates missing output directories with confirmation or the new ``--yes`` option. +- Project documentation is now built with Sphinx and includes installation, usage, development, file-format, and API references. + +0.4.3 +----- + +- Styled ASS subtitle sources with embedded font attachments are now detected explicitly, keep MKV output, preserve current source font attachments, and reject incompatible sidecar subtitle import. +- Attachment descriptors are now treated as source-runtime data instead of pattern schema data, so pattern persistence skips them and source-vs-pattern validation ignores them. +- Inspect differences no longer report planned changes for attachment filename/count drift while still showing attachment streams in the stream table. + +0.4.2 +----- + +- Pattern details now show an inline ``Show: `` hint next to the quality field when the pattern itself has no stored quality but the selected show does. +- Inspect stream tables now show attachment format labels like ``TTF`` in the codec column and keep attachment language cells blank instead of showing an undefined language. +- FFmpeg damaged-MP3 diagnostics now recognize additional corruption lines such as ``invalid new backstep``, keeping them grouped under the ``warn-corrupt-mpeg-audio`` review summary. + +0.4.1 +----- + +- ``convert`` now supports ``--copy-video`` and ``--copy-audio`` to keep the selected stream type in copy mode without applying the corresponding reencode flags, filters, or formatting options. +- FFmpeg conversions now monitor diagnostics while the process is running, retry unset AVI packet timestamps once with ``-fflags +genpts``, and stop early when a file should be skipped instead of waiting for the full job to finish. +- End-of-run convert summaries now list only FFmpeg findings that still require review, including named remedy identifiers such as ``warn-corrupt-mpeg-audio``. +- ``upgrade`` now finishes by reporting the installed FFX version together with the active bundle branch. + +0.3.1 +----- + +- Debug mode screen titles now append the active Textual screen class name, making screen-specific troubleshooting easier during inspect and edit flows. +- ``--cut`` again works as a combined flag/option: omitted disables cutting, bare ``--cut`` applies the default ``60,180``, and explicit duration or ``START,DURATION`` values stay supported. +- H.265 unmux commands no longer force an invalid ``-f h265`` output format, keeping FFmpeg copy extraction aligned with the required Annex B bitstream filter. +- H.264 encoding now falls back from ``libx264`` to ``libopenh264`` with a warning when needed, and the test fixtures use the same encoder fallback so the suite remains portable across FFmpeg builds. + +0.3.0 +----- + +- Inspect and edit screens now refresh nested track and pattern changes more reliably, with inspect-mode tables aligned to the target pattern view shown in the differences pane. +- Metadata editing got a follow-up polish pass with clearer FFmpeg notifications, a shared in-screen log pane, safer apply/reload handling, and expanded cleanup and normalization coverage. +- Track and asset probing recognize additional codecs, and the modern test suite now covers more metadata-editor, change-set, screen-state, and asset-probe behavior. +- Textual now requires version ``8.0`` or newer to match the UI APIs used by the current screens. + +0.2.6 +----- + +- DB-free ``ffx edit`` workflow for in-place metadata editing via temporary-file rewrite. +- Inspect and edit workflows split into dedicated Textual screens with shared media-workflow support. +- Textual tables and row actions now separate raw data from rendered labels to avoid markup leaking into stored metadata. +- Responsive screen layout pass, ``Esc`` back handling, sortable show/inspect tables, and improved edit-screen notifications/toggles. +- Application-wide UTF-8 i18n catalogs with language precedence from CLI over config over system over German default. +- Metadata normalization extended for localized subtitle titles, ISO language cleanup, and smarter track editor language/title helpers. + +0.2.5 +----- + +- Show-level quality and notes fields. +- Pattern-over-show-over-default season-shift resolution with dynamic DB migration loading. +- Migration prompt now reports the upgrade path and creates an in-place DB backup before applying schema changes. +- ``upgrade --branch `` now fetches remote-only branches before switching. +- ``unmux`` now applies season shifting to subtitle output filenames. +- Convert now keeps DB-defined target subtitle dispositions authoritative over sidecar filename disposition flags when a pattern definition exists. +- Focused modern tests added around migrations, unmux, upgrade, and subtitle-disposition import precedence. + +0.2.4 +----- + +- Lightweight CLI commands now stay import-light via lazy runtime loading. +- Setup/config templating moved to ``assets/ffx.json.j2``. +- Aligned two-step local setup wrappers: ``ffx setup`` and ``ffx configure_workstation``. +- Combined ``ffprobe`` payload reuse in ``FileProperties``. +- Configurable crop-detect sampling plus per-process crop result caching. +- Single-query controller accessors and conditional DB schema bootstrap. +- Shared screen bootstrap/controller wiring for large detail screens. +- Configurable default season/episode digit lengths. +- Digit-aware ``rename`` and padded ``unmux`` filename markers. + +0.2.3 +----- + +- PyPI packaging. +- Output filename templating. +- Season shifting. +- DB versioning. + +0.2.2 +----- + +- CLI overrides. + +0.2.1 +----- + +- Signature handling. +- Tag cleanup. +- Bugfixes and refactoring. + +0.2.0 +----- + +- Tests. +- Config file. + +0.1.3 +----- + +- Subtitle file imports. + +0.1.2 +----- + +- Bugfixes. + +0.1.1 +----- + +- Bugfixes. +- TMDB show identification. diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..9cf0229 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,31 @@ +FFX Documentation +================= + +FFX is a local command-line and Textual terminal UI for inspecting TV episode +files, storing normalization rules, and converting media into predictable +archive-ready outputs. + +This documentation covers operator setup, day-to-day command usage, contributor +workflow, format-specific notes, and generated API references for the smaller +utility modules. + +.. toctree:: + :maxdepth: 2 + :caption: User Guide + + installation + usage + file_formats + +.. toctree:: + :maxdepth: 1 + :caption: Release Notes + + history + +.. toctree:: + :maxdepth: 2 + :caption: Contributor Guide + + development + api diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..a2bface --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,52 @@ +Installation +============ + +FFX is designed for a Linux-like workstation with local command execution. The +runtime media tools must be available on ``PATH``: + +* ``ffmpeg`` +* ``ffprobe`` +* ``cpulimit`` + +User Bundle +----------- + +The persistent user installation is prepared with the two-step flow described in +the project README: + +.. code-block:: sh + + bash tools/setup.sh + bash tools/configure_workstation.sh + +``tools/setup.sh`` creates the long-lived bundle virtualenv at +``~/.local/share/ffx.venv`` and exposes the ``ffx`` command. The workstation +script checks system tools and seeds local config directories. + +Local Test And Docs Environment +------------------------------- + +Contributor test and documentation work uses the repo-local virtualenv: + +.. code-block:: sh + + tests/prepare.sh + +The script creates ``.venv``, installs FFX in editable mode with test and docs +extras, and verifies the Sphinx toolchain. Use check-only mode when you only +want to inspect readiness: + +.. code-block:: sh + + tests/prepare.sh --check + +Documentation Build +------------------- + +After preparation, build the documentation with: + +.. code-block:: sh + + .venv/bin/sphinx-build -b html docs docs/_build/html + +The generated site starts at ``docs/_build/html/index.html``. diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..2ec8f45 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,42 @@ +@ECHO OFF + +pushd %~dp0 + +if "%SPHINXBUILD%" == "" if exist ..\.venv\Scripts\sphinx-build.exe ( + set SPHINXBUILD=..\.venv\Scripts\sphinx-build.exe +) +if "%SPHINXBUILD%" == "" set SPHINXBUILD=sphinx-build +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo The 'sphinx-build' command was not found. Make sure Sphinx is installed, + echo then set SPHINXBUILD to the full path if needed. + exit /b 1 +) + +if "%1" == "" goto help +if "%1" == "html" goto html +if "%1" == "linkcheck" goto linkcheck +echo. +echo Unknown target "%1". +goto help + +:html +%SPHINXBUILD% -b html %SOURCEDIR% %BUILDDIR%\html %SPHINXOPTS% +goto end + +:linkcheck +%SPHINXBUILD% -b linkcheck %SOURCEDIR% %BUILDDIR%\linkcheck %SPHINXOPTS% +goto end + +:help +echo. +echo Please use 'make.bat ^' where ^ is one of +echo html to make standalone HTML files +echo linkcheck to check all external links for integrity + +:end +popd diff --git a/docs/usage.rst b/docs/usage.rst new file mode 100644 index 0000000..9e84b83 --- /dev/null +++ b/docs/usage.rst @@ -0,0 +1,97 @@ +Usage +===== + +FFX exposes a single ``ffx`` command with subcommands for inspection, +conversion, metadata editing, setup, and maintenance. + +Inspect Files +------------- + +Open the inspection workflow for one or more files: + +.. code-block:: sh + + ffx inspect /path/to/episode.mkv + +Print resolved season-shift mappings without opening the TUI: + +.. code-block:: sh + + ffx inspect --shift /path/to/episode.mkv + +Convert Files +------------- + +Convert one or more source files using stored rules where available: + +.. code-block:: sh + + ffx convert /path/to/episode.mkv + +Useful overrides include: + +* ``--no-pattern`` to skip database pattern matching +* ``--show``, ``--season``, and ``--episode`` for explicit episode identity +* ``--output-directory`` for generated output placement +* ``--subtitle-directory`` for source-basename sidecar subtitle imports +* ``--subtitle-prefix`` for explicit or configured-prefix subtitle imports +* ``--subtitle-extension`` to select the imported sidecar format (default: + ``vtt``) +* ``--yes`` to accept a valid partial sidecar set without prompting +* ``--copy-video`` or ``--copy-audio`` to preserve selected stream types +* ``--rename-only`` for filename normalization without media rewriting + +Directory-only subtitle import matches the source basename. For example, +``A2_t01.mkv`` discovers files such as ``A2_t01_2_deu_DEF.vtt`` in the +provided directory: + +.. code-block:: sh + + ffx convert --subtitle-directory /path/to/subtitles A2_t01.mkv + +Select a different sidecar set by extension, with or without the leading dot: + +.. code-block:: sh + + ffx convert --subtitle-directory /path/to/subtitles \ + --subtitle-extension .mkv A2_t01.mkv + +When only some source subtitle tracks have matching sidecar files, conversion +asks for confirmation. Use ``--yes`` to substitute that valid subset without +prompting. ``--yes`` also permits this case when ``--no-prompt`` is set. + +Manage Shows And Patterns +------------------------- + +Open the Textual interface for show and pattern management: + +.. code-block:: sh + + ffx shows + +Extract Streams +--------------- + +Extract streams from a file: + +.. code-block:: sh + + ffx unmux /path/to/episode.mkv + +For subtitle-only extraction: + +.. code-block:: sh + + ffx unmux --subtitles-only --label show-name /path/to/episode.mkv + +Detect Crop +----------- + +Ask FFmpeg to suggest crop parameters: + +.. code-block:: sh + + ffx cropdetect /path/to/episode.mkv + +The default sampling window is controlled by the application defaults and can be +overridden with command options. diff --git a/pyproject.toml b/pyproject.toml index 648fc05..3fdee87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "ffx" description = "FFX recoding and metadata managing tool" -version = "0.4.3" +version = "0.4.4" license = {file = "LICENSE.md"} dependencies = [ "requests", @@ -31,6 +31,12 @@ Issues = "https://gitea.maveno.de/Javanaut/ffx/issues" test = [ "pytest", ] +docs = [ + "esbonio", + "sphinx", + "sphinx-copybutton", + "sphinx-rtd-theme", +] [build-system] requires = [ diff --git a/src/ffx/cli.py b/src/ffx/cli.py index 5e3b784..74e28de 100755 --- a/src/ffx/cli.py +++ b/src/ffx/cli.py @@ -41,13 +41,17 @@ CPU_OPTION_HELP = ( + "Omit to disable; 0 also disables." ) SUBTITLE_DIRECTORY_OPTION_HELP = ( - "Load subtitles from here. When omitted and --subtitle-prefix is set, " + "Load subtitles from here. Without --subtitle-prefix, match the source filename " + + "basename. When omitted and --subtitle-prefix is set, " + "FFX uses the configured subtitlesDirectory base path plus the prefix as a subdirectory." ) SUBTITLE_PREFIX_OPTION_HELP = ( "Subtitle filename prefix. Requires --subtitle-directory, or a configured " + "subtitlesDirectory base path that contains a matching / subdirectory." ) +SUBTITLE_EXTENSION_OPTION_HELP = ( + "External subtitle filename extension. A leading dot is optional." +) UNMUX_OUTPUT_DIRECTORY_OPTION_HELP = ( "Write extracted streams here. When omitted together with --subtitles-only and " + "--label, FFX uses the configured subtitlesDirectory base path plus the label." @@ -96,6 +100,18 @@ def normalizeCpuOption(ctx, param, value): raise click.BadParameter(str(ex)) from ex +def normalizeSubtitleExtension(ctx, param, value): + normalizedExtension = str(value).strip().lower() + if normalizedExtension.startswith('.'): + normalizedExtension = normalizedExtension[1:] + if not normalizedExtension or not normalizedExtension.isalnum(): + raise click.BadParameter( + "Subtitle extension must contain only letters and numbers, " + + "with an optional leading dot." + ) + return normalizedExtension + + def parseCutOptionValue(value) -> tuple[int, int] | None: if value is None: return None @@ -146,11 +162,21 @@ def resolveSubtitleImportOptions(context, subtitleDirectory, subtitlePrefix): else '' ) - if not resolvedSubtitlePrefix: - return False, resolvedSubtitleDirectory, resolvedSubtitlePrefix - if resolvedSubtitleDirectory: - return True, resolvedSubtitleDirectory, resolvedSubtitlePrefix + if not os.path.isdir(resolvedSubtitleDirectory): + raise click.ClickException( + "The provided subtitle directory does not exist: " + + resolvedSubtitleDirectory + ) + return ( + True, + resolvedSubtitleDirectory, + resolvedSubtitlePrefix, + not resolvedSubtitlePrefix, + ) + + if not resolvedSubtitlePrefix: + return False, resolvedSubtitleDirectory, resolvedSubtitlePrefix, False configuredSubtitlesBaseDirectory = context['config'].getSubtitlesDirectoryPath() if not configuredSubtitlesBaseDirectory: @@ -170,7 +196,85 @@ def resolveSubtitleImportOptions(context, subtitleDirectory, subtitlePrefix): + resolvedSubtitleDirectory ) - return True, resolvedSubtitleDirectory, resolvedSubtitlePrefix + return True, resolvedSubtitleDirectory, resolvedSubtitlePrefix, False + + +def importExternalSubtitles( + context, + mediaDescriptor, + sourceFileBasename, + season, + episode, + preserveDispositions=False, +): + matchSourceBasename = context['subtitle_match_source_basename'] + subtitlePrefix = ( + sourceFileBasename + if matchSourceBasename + else context['subtitle_prefix'] + ) + + try: + importResult = mediaDescriptor.importSubtitles( + context['subtitle_directory'], + subtitlePrefix, + season, + episode, + preserve_dispositions=preserveDispositions, + extension=context['subtitle_extension'], + strict=matchSourceBasename, + ) + except (OSError, ValueError) as ex: + raise click.ClickException( + f"External subtitle discovery failed for '{sourceFileBasename}': {ex}" + ) from ex + + if not matchSourceBasename: + return importResult + + importedTrackIndices = importResult['imported_track_indices'] + missingTrackIndices = importResult['missing_track_indices'] + extension = context['subtitle_extension'] + importedDescription = ( + ', '.join(f"#{index}" for index in importedTrackIndices) + if importedTrackIndices + else 'none' + ) + click.echo( + f"External subtitle scan for '{sourceFileBasename}': found " + + f"{importResult['candidate_count']} .{extension} file(s); " + + f"matched subtitle tracks {importedDescription}." + ) + + if not missingTrackIndices: + return importResult + + missingDescription = ', '.join(f"#{index}" for index in missingTrackIndices) + incompleteMessage = ( + f"External subtitle files are missing for subtitle tracks " + + f"{missingDescription} in '{sourceFileBasename}'." + ) + if context.get('yes', False): + click.echo( + incompleteMessage + + " Continuing with the matching subtitle files because --yes is set." + ) + return importResult + + if context['no_prompt']: + raise click.ClickException( + incompleteMessage + + " Partial subtitle substitution requires confirmation, but --no-prompt is set." + ) + + click.echo(incompleteMessage) + if not click.confirm( + "Continue and substitute only the subtitle tracks with matching files?", + default=False, + ): + raise click.ClickException("External subtitle substitution aborted by user.") + + return importResult def resolveUnmuxOutputDirectory(context, outputDirectory, subtitlesOnly, label): @@ -181,7 +285,10 @@ def resolveUnmuxOutputDirectory(context, outputDirectory, subtitlesOnly, label): ) resolvedLabel = str(label).strip() - if resolvedOutputDirectory or not subtitlesOnly or not resolvedLabel: + if resolvedOutputDirectory: + return resolvedOutputDirectory, True + + if not subtitlesOnly or not resolvedLabel: return resolvedOutputDirectory, False configuredSubtitlesBaseDirectory = context['config'].getSubtitlesDirectoryPath() @@ -194,6 +301,63 @@ def resolveUnmuxOutputDirectory(context, outputDirectory, subtitlesOnly, label): return os.path.join(configuredSubtitlesBaseDirectory, resolvedLabel), True +def confirmUnmuxOutputDirectoryCreation(outputDirectory): + message = ( + "Create unmux output directory and missing parents: " + + str(outputDirectory) + ) + + if not sys.stdin.isatty(): + return click.confirm(message, default=True) + + click.echo(f"{message} [Y/n]: ", nl=False) + while True: + char = click.getchar() + if char in ('\r', '\n'): + click.echo() + return True + + normalizedChar = char.lower() + if normalizedChar == 'y': + click.echo(char) + return True + if normalizedChar == 'n': + click.echo(char) + return False + if char in ('\x03', '\x04'): + raise click.Abort() + + click.echo("\nPlease respond with 'y' or 'n': ", nl=False) + + +def ensureUnmuxOutputDirectory(context, outputDirectory): + resolvedOutputDirectory = os.path.expanduser(str(outputDirectory).strip()) + if not resolvedOutputDirectory: + return False + + if os.path.isdir(resolvedOutputDirectory): + return False + + if os.path.exists(resolvedOutputDirectory): + raise click.ClickException( + "Unmux output path exists but is not a directory: " + + resolvedOutputDirectory + ) + + if context.get('dry_run', False): + return False + + if context.get('yes', False): + os.makedirs(resolvedOutputDirectory, exist_ok=True) + return True + + if not confirmUnmuxOutputDirectoryCreation(resolvedOutputDirectory): + raise click.ClickException("Unmux output directory creation aborted by user.") + + os.makedirs(resolvedOutputDirectory, exist_ok=True) + return True + + def resolveIndicatorDigitLengths(context=None, showDescriptor=None): from ffx.show_descriptor import ShowDescriptor @@ -716,6 +880,12 @@ def getUnmuxSequence(trackDescriptor: TrackDescriptor, sourcePath, targetPrefix, @click.option('-l', '--label', type=str, default='', help='Label to be used as filename prefix') @click.option("-o", "--output-directory", type=str, default='', help=UNMUX_OUTPUT_DIRECTORY_OPTION_HELP) @click.option("-s", "--subtitles-only", is_flag=True, default=False) +@click.option( + "--yes", + is_flag=True, + default=False, + help="Create a missing unmux output directory without prompting.", +) @click.option( '--nice', type=int, @@ -737,6 +907,7 @@ def unmux(ctx, label, output_directory, subtitles_only, + yes, nice, cpu): from ffx.file_properties import FileProperties @@ -752,15 +923,16 @@ def unmux(ctx, ctx.obj['resource_limits']['niceness'] = nice ctx.obj['resource_limits']['cpu_limit'] = cpu ctx.obj['resource_limits']['cpu_percent'] = cpu + ctx.obj['yes'] = bool(yes) - output_directory, create_output_directory = resolveUnmuxOutputDirectory( + output_directory, requires_output_directory = resolveUnmuxOutputDirectory( ctx.obj, output_directory, subtitles_only, label, ) - if create_output_directory and existingSourcePaths and not ctx.obj.get('dry_run', False): - os.makedirs(output_directory, exist_ok=True) + if requires_output_directory and existingSourcePaths: + ensureUnmuxOutputDirectory(ctx.obj, output_directory) shiftedSeasonController = ShiftedSeasonController(ctx.obj) @@ -974,6 +1146,14 @@ def checkUniqueDispositions(context, mediaDescriptor: MediaDescriptor): @click.option('--subtitle-directory', type=str, default='', help=SUBTITLE_DIRECTORY_OPTION_HELP) @click.option('--subtitle-prefix', type=str, default='', help=SUBTITLE_PREFIX_OPTION_HELP) +@click.option( + '--subtitle-extension', + type=str, + default='vtt', + callback=normalizeSubtitleExtension, + show_default=True, + help=SUBTITLE_EXTENSION_OPTION_HELP, +) @click.option('--language', type=str, multiple=True, help='Set stream language. Use format :<3 letter iso code>') @click.option('--title', type=str, multiple=True, help='Set stream title. Use format :') @@ -1034,6 +1214,12 @@ def checkUniqueDispositions(context, mediaDescriptor: MediaDescriptor): @click.option("--dont-pass-dispositions", is_flag=True, default=False) @click.option("--no-prompt", is_flag=True, default=False) +@click.option( + "--yes", + is_flag=True, + default=False, + help="Confirm partial external subtitle substitution without prompting.", +) @click.option("--no-signature", is_flag=True, default=False) @click.option("--keep-mkvmerge-metadata", is_flag=True, default=False) @@ -1070,6 +1256,7 @@ def convert(ctx, subtitle_directory, subtitle_prefix, + subtitle_extension, language, title, @@ -1108,6 +1295,7 @@ def convert(ctx, no_pattern, dont_pass_dispositions, no_prompt, + yes, no_signature, keep_mkvmerge_metadata, @@ -1162,6 +1350,7 @@ def convert(ctx, context['use_tmdb'] = not no_tmdb context['use_pattern'] = not no_pattern context['no_prompt'] = no_prompt + context['yes'] = yes context['no_signature'] = no_signature context['keep_mkvmerge_metadata'] = keep_mkvmerge_metadata @@ -1180,6 +1369,7 @@ def convert(ctx, context['import_subtitles'], resolvedSubtitleDirectory, resolvedSubtitlePrefix, + context['subtitle_match_source_basename'], ) = resolveSubtitleImportOptions( context, subtitle_directory, @@ -1188,6 +1378,7 @@ def convert(ctx, if context['import_subtitles']: context['subtitle_directory'] = resolvedSubtitleDirectory context['subtitle_prefix'] = resolvedSubtitlePrefix + context['subtitle_extension'] = subtitle_extension existingSourcePaths = [p for p in paths if os.path.isfile(p) and p.split('.')[-1] in SUPPORTED_INPUT_FILE_EXTENSIONS] @@ -1431,10 +1622,13 @@ def convert(ctx, currentShowDescriptor = None if context['import_subtitles']: - sourceMediaDescriptor.importSubtitles(context['subtitle_directory'], - context['subtitle_prefix'], - showSeason, - showEpisode) + importExternalSubtitles( + context, + sourceMediaDescriptor, + sourceFileBasename, + showSeason, + showEpisode, + ) if cliOverrides: sourceMediaDescriptor.applyOverrides(cliOverrides) @@ -1478,11 +1672,14 @@ def convert(ctx, if context['import_subtitles']: - targetMediaDescriptor.importSubtitles(context['subtitle_directory'], - context['subtitle_prefix'], - showSeason, - showEpisode, - preserve_dispositions=True) + importExternalSubtitles( + context, + targetMediaDescriptor, + sourceFileBasename, + showSeason, + showEpisode, + preserveDispositions=True, + ) # ctx.obj['logger'].debug(f"tmd subindices: {[t.getIndex() for t in targetMediaDescriptor.getAllTrackDescriptors()]} {[t.getSubIndex() for t in targetMediaDescriptor.getAllTrackDescriptors()]} {[t.getDispositionFlag(TrackDisposition.DEFAULT) for t in targetMediaDescriptor.getAllTrackDescriptors()]}") ctx.obj['logger'].debug(f"tmd subindices: {[t.getIndex() for t in targetMediaDescriptor.getTrackDescriptors()]} {[t.getSubIndex() for t in targetMediaDescriptor.getTrackDescriptors()]} {[t.getDispositionFlag(TrackDisposition.DEFAULT) for t in targetMediaDescriptor.getTrackDescriptors()]}") diff --git a/src/ffx/constants.py b/src/ffx/constants.py index cf735c1..ba77693 100644 --- a/src/ffx/constants.py +++ b/src/ffx/constants.py @@ -1,4 +1,4 @@ -VERSION='0.4.3' +VERSION='0.4.4' DATABASE_VERSION = 3 DEFAULT_QUALITY = 32 diff --git a/src/ffx/media_descriptor.py b/src/ffx/media_descriptor.py index 1dde796..92eb25c 100644 --- a/src/ffx/media_descriptor.py +++ b/src/ffx/media_descriptor.py @@ -431,10 +431,13 @@ class MediaDescriptor: importedFilePath = td.getExternalSourceFilePath() if importedFilePath: - - self.__logger.info(f"Substituting subtitle stream #{td.getIndex()} " - + f"({td.getType().label()}:{td.getSubIndex()}) " - + f"with import from file {td.getExternalSourceFilePath()}") + substitutionMessage = ( + f"Substituting subtitle stream #{td.getIndex()} " + + f"({td.getType().label()}:{td.getSubIndex()}) " + + f"with import from file {td.getExternalSourceFilePath()}" + ) + click.echo(substitutionMessage) + self.__logger.debug(substitutionMessage) importFileTokens += [ "-i", @@ -524,66 +527,153 @@ class MediaDescriptor: return inputMappingTokens - def searchSubtitleFiles(self, searchDirectory, prefix): + def searchSubtitleFiles( + self, + searchDirectory, + prefix, + extension=SUBTITLE_FILE_EXTENSION, + strict=False, + ): - sesld_match = re.compile(f"{prefix}_{MediaDescriptor.SEASON_EPISODE_STREAM_LANGUAGE_DISPOSITIONS_MATCH}") - sld_match = re.compile(f"{prefix}_{MediaDescriptor.STREAM_LANGUAGE_DISPOSITIONS_MATCH}") + normalizedExtension = str(extension).strip().lower() + if normalizedExtension.startswith('.'): + normalizedExtension = normalizedExtension[1:] + escapedPrefix = re.escape(prefix) + sesld_match = re.compile( + f"{escapedPrefix}_{MediaDescriptor.SEASON_EPISODE_STREAM_LANGUAGE_DISPOSITIONS_MATCH}" + ) + sld_match = re.compile( + f"{escapedPrefix}_{MediaDescriptor.STREAM_LANGUAGE_DISPOSITIONS_MATCH}" + ) subtitleFileDescriptors = [] + subtitleFilenames = [] - for subtitleFilename in os.listdir(searchDirectory): - if subtitleFilename.startswith(prefix) and subtitleFilename.endswith( - "." + MediaDescriptor.SUBTITLE_FILE_EXTENSION + for subtitleFilename in sorted(os.listdir(searchDirectory)): + subtitleFilePath = os.path.join(searchDirectory, subtitleFilename) + subtitleFilenameStem, subtitleFilenameExtension = os.path.splitext( + subtitleFilename + ) + if ( + os.path.isfile(subtitleFilePath) + and subtitleFilenameStem.startswith(prefix + '_') + and subtitleFilenameExtension.lower() == '.' + normalizedExtension ): + subtitleFilenames.append(subtitleFilename) - sesld_result = sesld_match.search(subtitleFilename) - sld_result = None if not sesld_result is None else sld_match.search(subtitleFilename) - - if not sesld_result is None: + expectedSubtitleTrackIndices = { + subtitleTrack.getIndex() + for subtitleTrack in self.getSubtitleTracks() + } + if strict and len(subtitleFilenames) > len(expectedSubtitleTrackIndices): + raise ValueError( + f"Found {len(subtitleFilenames)} matching .{normalizedExtension} files " + + f"for {len(expectedSubtitleTrackIndices)} subtitle tracks." + ) - subtitleFilePath = os.path.join(searchDirectory, subtitleFilename) - if os.path.isfile(subtitleFilePath): + for subtitleFilename in subtitleFilenames: + subtitleFilenameStem = os.path.splitext(subtitleFilename)[0] + sesld_result = ( + None + if strict + else sesld_match.fullmatch(subtitleFilenameStem) + ) + sld_result = ( + None + if sesld_result is not None + else sld_match.fullmatch(subtitleFilenameStem) + ) - subtitleFileDescriptor = {} - subtitleFileDescriptor["path"] = subtitleFilePath - subtitleFileDescriptor["season"] = int(sesld_result.group(1)) - subtitleFileDescriptor["episode"] = int(sesld_result.group(2)) - subtitleFileDescriptor["index"] = int(sesld_result.group(3)) - subtitleFileDescriptor["language"] = sesld_result.group(4) + if strict and sesld_result is None and sld_result is None: + raise ValueError( + f"Subtitle filename does not match the expected pattern: " + + subtitleFilename + ) - dispSet = set() - dispCaptGroups = sesld_result.groups() - numCaptGroups = len(dispCaptGroups) - if numCaptGroups > 4: - for groupIndex in range(numCaptGroups - 4): - disp = TrackDisposition.fromIndicator(dispCaptGroups[groupIndex + 4]) - if disp is not None: - dispSet.add(disp) - subtitleFileDescriptor["disposition_set"] = dispSet + if sesld_result is not None: - subtitleFileDescriptors.append(subtitleFileDescriptor) + subtitleFilePath = os.path.join(searchDirectory, subtitleFilename) - if not sld_result is None: + subtitleFileDescriptor = {} + subtitleFileDescriptor["path"] = subtitleFilePath + subtitleFileDescriptor["season"] = int(sesld_result.group(1)) + subtitleFileDescriptor["episode"] = int(sesld_result.group(2)) + subtitleFileDescriptor["index"] = int(sesld_result.group(3)) + subtitleFileDescriptor["language"] = sesld_result.group(4) - subtitleFilePath = os.path.join(searchDirectory, subtitleFilename) - if os.path.isfile(subtitleFilePath): + dispSet = set() + dispCaptGroups = sesld_result.groups() + numCaptGroups = len(dispCaptGroups) + if numCaptGroups > 4: + for groupIndex in range(numCaptGroups - 4): + disp = TrackDisposition.fromIndicator( + dispCaptGroups[groupIndex + 4] + ) + if disp is not None: + dispSet.add(disp) + subtitleFileDescriptor["disposition_set"] = dispSet - subtitleFileDescriptor = {} - subtitleFileDescriptor["path"] = subtitleFilePath - subtitleFileDescriptor["index"] = int(sld_result.group(1)) - subtitleFileDescriptor["language"] = sld_result.group(2) + subtitleFileDescriptors.append(subtitleFileDescriptor) - dispSet = set() - dispCaptGroups = sld_result.groups() - numCaptGroups = len(dispCaptGroups) - if numCaptGroups > 2: - for groupIndex in range(numCaptGroups - 2): - disp = TrackDisposition.fromIndicator(dispCaptGroups[groupIndex + 2]) - if disp is not None: - dispSet.add(disp) - subtitleFileDescriptor["disposition_set"] = dispSet + if sld_result is not None: - subtitleFileDescriptors.append(subtitleFileDescriptor) + subtitleFilePath = os.path.join(searchDirectory, subtitleFilename) + + subtitleFileDescriptor = {} + subtitleFileDescriptor["path"] = subtitleFilePath + subtitleFileDescriptor["index"] = int(sld_result.group(1)) + subtitleFileDescriptor["language"] = sld_result.group(2) + + dispSet = set() + dispCaptGroups = sld_result.groups() + numCaptGroups = len(dispCaptGroups) + if numCaptGroups > 2: + for groupIndex in range(numCaptGroups - 2): + disp = TrackDisposition.fromIndicator( + dispCaptGroups[groupIndex + 2] + ) + if disp is not None: + dispSet.add(disp) + subtitleFileDescriptor["disposition_set"] = dispSet + + subtitleFileDescriptors.append(subtitleFileDescriptor) + + if strict: + discoveredTrackIndices = [ + descriptor['index'] for descriptor in subtitleFileDescriptors + ] + duplicateTrackIndices = sorted( + { + trackIndex + for trackIndex in discoveredTrackIndices + if discoveredTrackIndices.count(trackIndex) > 1 + } + ) + if duplicateTrackIndices: + duplicateDescription = ', '.join( + f"#{index}" for index in duplicateTrackIndices + ) + raise ValueError( + "Multiple external subtitle files refer to subtitle track(s) " + + duplicateDescription + + "." + ) + + unexpectedTrackIndices = sorted( + set(discoveredTrackIndices) - expectedSubtitleTrackIndices + ) + if unexpectedTrackIndices: + unexpectedDescription = ', '.join( + f"#{index}" for index in unexpectedTrackIndices + ) + expectedDescription = ', '.join( + f"#{index}" for index in sorted(expectedSubtitleTrackIndices) + ) or 'none' + raise ValueError( + "External subtitle track index pattern does not match the media " + + f"subtitle tracks: found {unexpectedDescription}; " + + f"expected a subset of {expectedDescription}." + ) self.__logger.debug(f"searchSubtitleFiles(): Available subtitle files {subtitleFileDescriptors}") @@ -598,12 +688,19 @@ class MediaDescriptor: season: int = -1, episode: int = -1, preserve_dispositions: bool = False, + extension: str = SUBTITLE_FILE_EXTENSION, + strict: bool = False, ): # click.echo(f"Season: {season} Episode: {episode}") self.__logger.debug(f"importSubtitles(): Season: {season} Episode: {episode}") - availableFileSubtitleDescriptors = self.searchSubtitleFiles(searchDirectory, prefix) + availableFileSubtitleDescriptors = self.searchSubtitleFiles( + searchDirectory, + prefix, + extension=extension, + strict=strict, + ) self.__logger.debug(f"importSubtitles(): availableFileSubtitleDescriptors: {availableFileSubtitleDescriptors}") @@ -616,7 +713,8 @@ class MediaDescriptor: [ d for d in availableFileSubtitleDescriptors - if ((season == -1 and episode == -1) + if (strict + or (season == -1 and episode == -1) or ( d.get("season") == int(season) and d.get("episode") == int(episode) @@ -630,6 +728,7 @@ class MediaDescriptor: self.__logger.debug(f"importSubtitles(): matchingSubtitleFileDescriptors: {matchingSubtitleFileDescriptors}") + importedTrackIndices = [] for msfd in matchingSubtitleFileDescriptors: matchingSubtitleTrackDescriptor = [s for s in subtitleTracks if s.getIndex() == msfd["index"]] if matchingSubtitleTrackDescriptor: @@ -643,6 +742,19 @@ class MediaDescriptor: matchingTrack.getTags()["language"] = msfd["language"] if msfd["disposition_set"] and not preserve_dispositions: matchingTrack.setDispositionSet(msfd["disposition_set"]) + importedTrackIndices.append(matchingTrack.getIndex()) + + expectedTrackIndices = sorted( + subtitleTrack.getIndex() for subtitleTrack in subtitleTracks + ) + importedTrackIndices = sorted(set(importedTrackIndices)) + return { + "candidate_count": len(availableFileSubtitleDescriptors), + "imported_track_indices": importedTrackIndices, + "missing_track_indices": sorted( + set(expectedTrackIndices) - set(importedTrackIndices) + ), + } def getConfiguration(self, label: str = ''): diff --git a/tests/integration/subtrack_mapping/test_cli_bundle.py b/tests/integration/subtrack_mapping/test_cli_bundle.py index b472c2c..b73ba56 100644 --- a/tests/integration/subtrack_mapping/test_cli_bundle.py +++ b/tests/integration/subtrack_mapping/test_cli_bundle.py @@ -421,6 +421,59 @@ class SubtrackMappingBundleTests(unittest.TestCase): self.assertIn("external subtitle payload", extracted_subtitle) self.assertNotIn("embedded subtitle payload", extracted_subtitle) + def test_subtitle_directory_without_prefix_uses_source_basename(self): + source_filename = "basename_substitute.mkv" + subtitle_directory = self.workdir / "sidecars" + subtitle_directory.mkdir() + source_path = create_source_fixture( + self.workdir, + source_filename, + [ + SourceTrackSpec(TrackType.VIDEO, identity="video-0"), + SourceTrackSpec(TrackType.AUDIO, identity="audio-1", language="eng"), + SourceTrackSpec( + TrackType.SUBTITLE, + identity="embedded-subtitle", + language="eng", + subtitle_lines=("embedded subtitle payload",), + ), + ], + ) + write_vtt( + subtitle_directory / "basename_substitute_2_deu_DEF.vtt", + ("external subtitle payload",), + ) + + completed = run_ffx_convert( + self.workdir, + self.home_dir, + self.database_path, + "--video-encoder", + "copy", + "--no-pattern", + "--no-tmdb", + "--no-prompt", + "--no-signature", + "--subtitle-directory", + str(subtitle_directory), + str(source_path), + ) + self.assertCompleted(completed) + self.assertIn("matched subtitle tracks #2", completed.stdout) + self.assertIn("Substituting subtitle stream #2", completed.stdout) + + output_path = expected_output_path(self.workdir, source_filename) + subtitle_stream = [ + stream + for stream in ffprobe_json(output_path)["streams"] + if stream["codec_type"] == "subtitle" + ][0] + self.assertEqual("deu", get_tag(subtitle_stream, "language")) + + extracted_subtitle = extract_first_subtitle_text(self.workdir, output_path) + self.assertIn("external subtitle payload", extracted_subtitle) + self.assertNotIn("embedded subtitle payload", extracted_subtitle) + def test_subtitle_prefix_uses_configured_base_directory_when_directory_is_omitted(self): source_filename = "substitute_default_s01e01.mkv" subtitle_prefix = "substitute_default" diff --git a/tests/integration/test_cli_unmux.py b/tests/integration/test_cli_unmux.py index a09728f..9729ef9 100644 --- a/tests/integration/test_cli_unmux.py +++ b/tests/integration/test_cli_unmux.py @@ -35,7 +35,13 @@ if pytest is not None: SRC_ROOT = Path(__file__).resolve().parents[2] / "src" -def run_ffx_unmux(workdir: Path, home_dir: Path, database_path: Path, *args: str) -> subprocess.CompletedProcess[str]: +def run_ffx_unmux( + workdir: Path, + home_dir: Path, + database_path: Path, + *args: str, + input_text: str | None = None, +) -> subprocess.CompletedProcess[str]: env = os.environ.copy() env["HOME"] = str(home_dir) existing_pythonpath = env.get("PYTHONPATH", "") @@ -50,7 +56,14 @@ def run_ffx_unmux(workdir: Path, home_dir: Path, database_path: Path, *args: str "unmux", *args, ] - return subprocess.run(command, cwd=workdir, env=env, capture_output=True, text=True) + return subprocess.run( + command, + cwd=workdir, + env=env, + capture_output=True, + input=input_text, + text=True, + ) class UnmuxCliTests(unittest.TestCase): @@ -161,6 +174,7 @@ class UnmuxCliTests(unittest.TestCase): self.home_dir, self.database_path, "--subtitles-only", + "--yes", "--label", "dball", str(source_path), diff --git a/tests/prepare.sh b/tests/prepare.sh new file mode 100755 index 0000000..f5a3ec5 --- /dev/null +++ b/tests/prepare.sh @@ -0,0 +1,471 @@ +#!/usr/bin/env bash + +set -u + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd -- "${SCRIPT_DIR}/.." && pwd)" +VENV_DIR="${FFX_TEST_VENV_DIR:-${ROOT_DIR}/.venv}" +VENV_BIN_DIR="${VENV_DIR}/bin" +VENV_PYTHON="${VENV_BIN_DIR}/python" +VENV_PIP="${VENV_BIN_DIR}/pip" + +CHECK_ONLY=0 +READINESS_FAILURES=0 +INSTALL_FAILURES=0 + +MISSING_REQUIRED_SYSTEM=() + +COLOR_RESET="" +COLOR_GREEN="" +COLOR_YELLOW="" +COLOR_RED="" + +if [ -t 1 ]; then + COLOR_RESET="$(printf '\033[0m')" + COLOR_GREEN="$(printf '\033[32m')" + COLOR_YELLOW="$(printf '\033[33m')" + COLOR_RED="$(printf '\033[31m')" +fi + +usage() { + cat <<EOF +Usage: $(basename "$0") [--check] [--help] + +Prepare the repo-local FFX test environment at: + ${VENV_DIR} + +Actions: + - verify or install required system commands for tests + - create or reuse the repo-local test virtualenv + - install this repository into the venv with Python test and docs extras + +Options: + --check Report readiness only. Do not create, install, or modify. + --help Show this help text. + +Environment overrides: + FFX_TEST_VENV_DIR Override the test virtualenv path. Defaults to ${ROOT_DIR}/.venv. + +Notes: + - This script prepares a project-local test environment, not the persistent user bundle. + - The persistent bundle setup remains owned by tools/setup.sh. +EOF +} + +status_ok() { + printf '%sok%s' "${COLOR_GREEN}" "${COLOR_RESET}" +} + +status_warn() { + printf '%swarn%s' "${COLOR_YELLOW}" "${COLOR_RESET}" +} + +status_fail() { + printf '%sfailed%s' "${COLOR_RED}" "${COLOR_RESET}" +} + +report_component() { + local level="$1" + local label="$2" + local detail="$3" + local rendered_status="" + + case "${level}" in + ok) + rendered_status="$(status_ok)" + ;; + warn) + rendered_status="$(status_warn)" + ;; + *) + rendered_status="$(status_fail)" + ;; + esac + + printf '[%s] %s%s\n' "${rendered_status}" "${label}" "${detail:+: $detail}" +} + +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +check_python_venv_support() { + python3 -m venv --help >/dev/null 2>&1 +} + +check_system_command() { + command_exists "$1" +} + +check_venv_python() { + [ -x "${VENV_PYTHON}" ] +} + +check_venv_pip() { + check_venv_python && "${VENV_PIP}" --version >/dev/null 2>&1 +} + +check_venv_ffx() { + check_venv_python && "${VENV_PYTHON}" -m ffx version >/dev/null 2>&1 +} + +check_venv_pytest() { + check_venv_python && "${VENV_PYTHON}" -m pytest --version >/dev/null 2>&1 +} + +check_venv_sphinx() { + check_venv_python && "${VENV_BIN_DIR}/sphinx-build" --version >/dev/null 2>&1 +} + +check_venv_docs_packages() { + check_venv_python && "${VENV_PYTHON}" - <<'PY' >/dev/null 2>&1 +import esbonio +import sphinx +import sphinx_rtd_theme +PY +} + +check_editable_install() { + check_venv_python && FFX_REPO_ROOT="${ROOT_DIR}" "${VENV_PYTHON}" - <<'PY' >/dev/null 2>&1 +from __future__ import annotations + +import os +from pathlib import Path +import ffx + +repo_root = Path(os.environ["FFX_REPO_ROOT"]).resolve() +package_path = Path(ffx.__file__).resolve() + +raise SystemExit(0 if repo_root in package_path.parents else 1) +PY +} + +check_python_environment_ready() { + check_venv_python && + check_venv_pip && + check_venv_pytest && + check_venv_sphinx && + check_venv_docs_packages && + check_venv_ffx && + check_editable_install +} + +command_detail() { + command -v "$1" || printf "command '%s' not found" "$1" +} + +python_venv_detail() { + if check_python_venv_support; then + printf 'python3 -m venv is available' + else + printf 'python3 venv support is unavailable' + fi +} + +venv_python_detail() { + if check_venv_python; then + printf '%s' "${VENV_PYTHON}" + else + printf 'missing %s' "${VENV_PYTHON}" + fi +} + +venv_pip_detail() { + if check_venv_pip; then + "${VENV_PIP}" --version + else + printf 'missing pip in %s' "${VENV_DIR}" + fi +} + +venv_ffx_detail() { + if check_venv_ffx; then + printf 'ffx import and CLI entry are available' + else + printf 'ffx is not installed in %s' "${VENV_DIR}" + fi +} + +venv_pytest_detail() { + if check_venv_pytest; then + "${VENV_PYTHON}" -m pytest --version 2>/dev/null | head -n 1 + else + printf 'pytest is not installed in %s' "${VENV_DIR}" + fi +} + +venv_sphinx_detail() { + if check_venv_sphinx; then + "${VENV_BIN_DIR}/sphinx-build" --version 2>&1 + else + printf 'sphinx-build is not installed in %s' "${VENV_DIR}" + fi +} + +venv_docs_packages_detail() { + if check_venv_docs_packages; then + printf 'Sphinx, Read the Docs theme, and Esbonio packages are importable' + else + printf 'one or more docs packages are missing in %s' "${VENV_DIR}" + fi +} + +editable_install_detail() { + if check_editable_install; then + printf 'ffx resolves from %s' "${ROOT_DIR}" + else + printf 'ffx does not resolve from the project source tree' + fi +} + +report_required_command() { + local label="$1" + local command_name="$2" + + if check_system_command "${command_name}"; then + report_component ok "${label}" "$(command_detail "${command_name}")" + else + report_component failed "${label}" "$(command_detail "${command_name}")" + MISSING_REQUIRED_SYSTEM+=("${command_name}") + READINESS_FAILURES=$((READINESS_FAILURES + 1)) + fi +} + +print_system_status() { + MISSING_REQUIRED_SYSTEM=() + + echo "System toolchain status:" + report_required_command "git" "git" + report_required_command "python3" "python3" + + if check_system_command "python3" && check_python_venv_support; then + report_component ok "python3 venv" "$(python_venv_detail)" + else + report_component failed "python3 venv" "$(python_venv_detail)" + MISSING_REQUIRED_SYSTEM+=("python3-venv") + READINESS_FAILURES=$((READINESS_FAILURES + 1)) + fi + + report_required_command "ffmpeg" "ffmpeg" + report_required_command "ffprobe" "ffprobe" + report_required_command "cpulimit" "cpulimit" +} + +print_python_status() { + echo "Repo test and docs virtualenv status:" + + if check_venv_python; then + report_component ok "test virtualenv" "$(venv_python_detail)" + else + report_component failed "test virtualenv" "$(venv_python_detail)" + READINESS_FAILURES=$((READINESS_FAILURES + 1)) + fi + + if check_venv_pip; then + report_component ok "test pip" "$(venv_pip_detail)" + else + report_component failed "test pip" "$(venv_pip_detail)" + READINESS_FAILURES=$((READINESS_FAILURES + 1)) + fi + + if check_venv_pytest; then + report_component ok "test pytest" "$(venv_pytest_detail)" + else + report_component failed "test pytest" "$(venv_pytest_detail)" + READINESS_FAILURES=$((READINESS_FAILURES + 1)) + fi + + if check_venv_sphinx; then + report_component ok "docs sphinx" "$(venv_sphinx_detail)" + else + report_component failed "docs sphinx" "$(venv_sphinx_detail)" + READINESS_FAILURES=$((READINESS_FAILURES + 1)) + fi + + if check_venv_docs_packages; then + report_component ok "docs packages" "$(venv_docs_packages_detail)" + else + report_component failed "docs packages" "$(venv_docs_packages_detail)" + READINESS_FAILURES=$((READINESS_FAILURES + 1)) + fi + + if check_venv_ffx; then + report_component ok "test ffx" "$(venv_ffx_detail)" + else + report_component failed "test ffx" "$(venv_ffx_detail)" + READINESS_FAILURES=$((READINESS_FAILURES + 1)) + fi + + if check_editable_install; then + report_component ok "editable source" "$(editable_install_detail)" + else + report_component failed "editable source" "$(editable_install_detail)" + READINESS_FAILURES=$((READINESS_FAILURES + 1)) + fi +} + +print_status_report() { + READINESS_FAILURES=0 + + print_system_status + echo + print_python_status +} + +detect_package_manager() { + if command_exists apt-get; then + printf 'apt-get\n' + return 0 + fi + if command_exists pacman; then + printf 'pacman\n' + return 0 + fi + return 1 +} + +run_root_command() { + if [ "${EUID}" -eq 0 ]; then + "$@" + elif command_exists sudo; then + sudo -n "$@" + else + return 1 + fi +} + +install_system_requirements() { + local package_manager + + if [ "${#MISSING_REQUIRED_SYSTEM[@]}" -eq 0 ]; then + return 0 + fi + + if ! package_manager="$(detect_package_manager)"; then + printf 'No supported package manager found for automatic system preparation.\n' >&2 + INSTALL_FAILURES=$((INSTALL_FAILURES + 1)) + return 1 + fi + + case "${package_manager}" in + apt-get) + printf 'Installing required system dependencies via apt-get...\n' + if ! run_root_command apt-get update; then + printf 'apt-get update failed or requires interactive sudo.\n' >&2 + INSTALL_FAILURES=$((INSTALL_FAILURES + 1)) + return 1 + fi + if ! run_root_command apt-get install -y git python3 python3-venv ffmpeg cpulimit; then + printf 'apt-get install failed or requires interactive sudo.\n' >&2 + INSTALL_FAILURES=$((INSTALL_FAILURES + 1)) + return 1 + fi + ;; + pacman) + printf 'Installing required system dependencies via pacman...\n' + if ! run_root_command pacman -Sy --noconfirm git python ffmpeg cpulimit; then + printf 'pacman install failed or requires interactive sudo.\n' >&2 + INSTALL_FAILURES=$((INSTALL_FAILURES + 1)) + return 1 + fi + ;; + esac + + return 0 +} + +ensure_test_venv() { + if ! check_venv_python; then + printf 'Creating repo test virtualenv at %s...\n' "${VENV_DIR}" + if ! python3 -m venv "${VENV_DIR}"; then + printf 'Failed to create test virtualenv at %s.\n' "${VENV_DIR}" >&2 + INSTALL_FAILURES=$((INSTALL_FAILURES + 1)) + return 1 + fi + fi + + if ! check_venv_pip; then + printf 'Missing pip in %s.\n' "${VENV_DIR}" >&2 + INSTALL_FAILURES=$((INSTALL_FAILURES + 1)) + return 1 + fi + + printf 'Installing FFX package with test and docs extras into %s...\n' "${VENV_DIR}" + if ! ( + cd "${ROOT_DIR}" && + "${VENV_PIP}" install --editable '.[test,docs]' + ); then + printf 'Failed to install FFX package with test and docs extras into %s.\n' "${VENV_DIR}" >&2 + INSTALL_FAILURES=$((INSTALL_FAILURES + 1)) + return 1 + fi + + return 0 +} + +parse_args() { + while [ "$#" -gt 0 ]; do + case "$1" in + --check) + CHECK_ONLY=1 + ;; + --help|-h) + usage + exit 0 + ;; + *) + printf 'Unknown option: %s\n\n' "$1" >&2 + usage >&2 + exit 2 + ;; + esac + shift + done +} + +main() { + parse_args "$@" + + print_status_report + + if [ "${CHECK_ONLY}" -eq 0 ]; then + if [ "${#MISSING_REQUIRED_SYSTEM[@]}" -gt 0 ]; then + echo + install_system_requirements + fi + + if check_python_environment_ready; then + echo + report_component ok "Python package install" "repo test and docs virtualenv is already ready" + elif check_system_command "python3" && check_python_venv_support; then + echo + ensure_test_venv + fi + + echo + print_status_report + fi + + echo + if [ "${INSTALL_FAILURES}" -gt 0 ]; then + echo "One or more test preparation steps failed; see the status checks above." >&2 + exit 1 + fi + + if [ "${READINESS_FAILURES}" -gt 0 ]; then + if [ "${CHECK_ONLY}" -eq 1 ]; then + echo "The FFX test and docs environment is incomplete." >&2 + else + echo "Required test or docs components are still missing after preparation." >&2 + fi + exit 1 + fi + + if [ "${CHECK_ONLY}" -eq 1 ]; then + echo "The FFX test and docs environment is ready." + else + echo "The FFX test and docs environment is prepared." + fi +} + +main "$@" diff --git a/tests/unit/test_cli_subtitle_directory.py b/tests/unit/test_cli_subtitle_directory.py index d0ef29c..3b802c3 100644 --- a/tests/unit/test_cli_subtitle_directory.py +++ b/tests/unit/test_cli_subtitle_directory.py @@ -6,7 +6,9 @@ from pathlib import Path import sys import tempfile import unittest +from unittest.mock import patch +import click from click.testing import CliRunner @@ -17,6 +19,10 @@ if str(SRC_ROOT) not in sys.path: from ffx import cli # noqa: E402 +from ffx.logging_utils import get_ffx_logger # noqa: E402 +from ffx.media_descriptor import MediaDescriptor # noqa: E402 +from ffx.track_descriptor import TrackDescriptor # noqa: E402 +from ffx.track_type import TrackType # noqa: E402 class SubtitleDirectoryCliTests(unittest.TestCase): @@ -48,6 +54,35 @@ class SubtitleDirectoryCliTests(unittest.TestCase): env={**os.environ, "HOME": str(self.home_dir)}, ) + def make_subtitle_descriptor(self, indices=(2, 3, 4)) -> MediaDescriptor: + return MediaDescriptor( + context={"logger": get_ffx_logger()}, + track_descriptors=[ + TrackDescriptor( + index=index, + source_index=index, + sub_index=subIndex, + track_type=TrackType.SUBTITLE, + ) + for subIndex, index in enumerate(indices) + ], + ) + + def make_import_context( + self, + subtitleDirectory: Path, + noPrompt: bool, + yes: bool = False, + ) -> dict: + return { + "subtitle_match_source_basename": True, + "subtitle_directory": str(subtitleDirectory), + "subtitle_prefix": "", + "subtitle_extension": "vtt", + "no_prompt": noPrompt, + "yes": yes, + } + def test_subtitle_prefix_without_directory_or_default_fails(self): result = self.invoke_convert("--subtitle-prefix", "dball") @@ -79,6 +114,153 @@ class SubtitleDirectoryCliTests(unittest.TestCase): self.assertEqual(0, result.exit_code, result.output) + def test_explicit_directory_without_prefix_enables_basename_matching(self): + explicitSubtitleDirectory = self.home_dir / "manual-subtitles" + explicitSubtitleDirectory.mkdir(parents=True, exist_ok=True) + + enabled, directory, prefix, matchBasename = cli.resolveSubtitleImportOptions( + {}, + str(explicitSubtitleDirectory), + "", + ) + + self.assertTrue(enabled) + self.assertEqual(str(explicitSubtitleDirectory), directory) + self.assertEqual("", prefix) + self.assertTrue(matchBasename) + + def test_subtitle_extension_accepts_optional_leading_dot(self): + self.assertEqual("mkv", cli.normalizeSubtitleExtension(None, None, "mkv")) + self.assertEqual("mkv", cli.normalizeSubtitleExtension(None, None, ".mkv")) + + def test_subtitle_extension_rejects_multiple_leading_dots(self): + with self.assertRaises(click.BadParameter): + cli.normalizeSubtitleExtension(None, None, "..mkv") + + def test_complete_basename_set_does_not_prompt(self): + subtitleDirectory = self.home_dir / "complete-subtitles" + subtitleDirectory.mkdir() + for basename in ( + "A2_t01_2_deu_DEF", + "A2_t01_3_eng", + "A2_t01_4_eng", + ): + (subtitleDirectory / f"{basename}.vtt").write_text( + "WEBVTT\n\n", + encoding="utf-8", + ) + descriptor = self.make_subtitle_descriptor() + context = self.make_import_context(subtitleDirectory, noPrompt=True) + + with patch("ffx.cli.click.confirm") as mockedConfirm: + result = cli.importExternalSubtitles( + context, + descriptor, + "A2_t01", + -1, + -1, + ) + + self.assertEqual([], result["missing_track_indices"]) + mockedConfirm.assert_not_called() + + def test_incomplete_basename_set_fails_with_no_prompt(self): + descriptor = self.make_subtitle_descriptor() + subtitleDirectory = self.home_dir / "partial-subtitles" + subtitleDirectory.mkdir() + (subtitleDirectory / "episode_2_deu.vtt").write_text( + "WEBVTT\n\n", + encoding="utf-8", + ) + context = self.make_import_context(subtitleDirectory, noPrompt=True) + + with patch("ffx.cli.click.confirm") as mockedConfirm: + with self.assertRaisesRegex(click.ClickException, "--no-prompt is set"): + cli.importExternalSubtitles( + context, + descriptor, + "episode", + -1, + -1, + ) + + mockedConfirm.assert_not_called() + + def test_incomplete_basename_set_can_be_confirmed(self): + descriptor = self.make_subtitle_descriptor() + subtitleDirectory = self.home_dir / "partial-subtitles" + subtitleDirectory.mkdir() + (subtitleDirectory / "episode_2_deu.vtt").write_text( + "WEBVTT\n\n", + encoding="utf-8", + ) + context = self.make_import_context(subtitleDirectory, noPrompt=False) + + with patch("ffx.cli.click.confirm", return_value=True) as mockedConfirm: + result = cli.importExternalSubtitles( + context, + descriptor, + "episode", + -1, + -1, + ) + + self.assertEqual([3, 4], result["missing_track_indices"]) + mockedConfirm.assert_called_once() + + def test_incomplete_basename_set_with_yes_does_not_prompt(self): + descriptor = self.make_subtitle_descriptor() + subtitleDirectory = self.home_dir / "partial-subtitles" + subtitleDirectory.mkdir() + (subtitleDirectory / "episode_2_deu.vtt").write_text( + "WEBVTT\n\n", + encoding="utf-8", + ) + context = self.make_import_context( + subtitleDirectory, + noPrompt=False, + yes=True, + ) + + with patch("ffx.cli.click.confirm") as mockedConfirm: + result = cli.importExternalSubtitles( + context, + descriptor, + "episode", + -1, + -1, + ) + + self.assertEqual([2], result["imported_track_indices"]) + self.assertEqual([3, 4], result["missing_track_indices"]) + mockedConfirm.assert_not_called() + + def test_yes_takes_precedence_over_no_prompt_for_incomplete_set(self): + descriptor = self.make_subtitle_descriptor() + subtitleDirectory = self.home_dir / "partial-subtitles" + subtitleDirectory.mkdir() + (subtitleDirectory / "episode_2_deu.vtt").write_text( + "WEBVTT\n\n", + encoding="utf-8", + ) + context = self.make_import_context( + subtitleDirectory, + noPrompt=True, + yes=True, + ) + + with patch("ffx.cli.click.confirm") as mockedConfirm: + result = cli.importExternalSubtitles( + context, + descriptor, + "episode", + -1, + -1, + ) + + self.assertEqual([3, 4], result["missing_track_indices"]) + mockedConfirm.assert_not_called() + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_cli_unmux_output_directory.py b/tests/unit/test_cli_unmux_output_directory.py index f417fc6..760f889 100644 --- a/tests/unit/test_cli_unmux_output_directory.py +++ b/tests/unit/test_cli_unmux_output_directory.py @@ -4,6 +4,7 @@ from pathlib import Path import sys import tempfile import unittest +from unittest.mock import patch import click @@ -42,7 +43,7 @@ class UnmuxOutputDirectoryTests(unittest.TestCase): self.assertEqual(str(Path(tempdir) / "subtitles" / "dball"), resolved_output_directory) self.assertTrue(should_create) - def test_explicit_output_directory_keeps_existing_behavior(self): + def test_explicit_output_directory_requires_directory(self): with tempfile.TemporaryDirectory() as tempdir: context = { "config": StaticConfig(str(Path(tempdir) / "subtitles")), @@ -57,7 +58,7 @@ class UnmuxOutputDirectoryTests(unittest.TestCase): ) self.assertEqual(explicit_output_directory, resolved_output_directory) - self.assertFalse(should_create) + self.assertTrue(should_create) def test_subtitles_only_without_label_keeps_existing_behavior(self): context = { @@ -89,6 +90,110 @@ class UnmuxOutputDirectoryTests(unittest.TestCase): self.assertIn("subtitlesDirectory default", str(caught.exception)) + def test_missing_output_directory_can_be_confirmed_and_created_with_parents(self): + with tempfile.TemporaryDirectory() as tempdir: + output_directory = Path(tempdir) / "missing" / "parents" / "manual" + + with patch("ffx.cli.click.confirm", return_value=True) as mocked_confirm: + created = cli.ensureUnmuxOutputDirectory( + {"dry_run": False}, + str(output_directory), + ) + + self.assertTrue(created) + self.assertTrue(output_directory.is_dir()) + mocked_confirm.assert_called_once() + + def test_tty_carriage_return_accepts_default_directory_creation(self): + with tempfile.TemporaryDirectory() as tempdir: + output_directory = Path(tempdir) / "missing" / "manual" + + with patch("ffx.cli.sys.stdin.isatty", return_value=True), patch( + "ffx.cli.click.getchar", + return_value="\r", + ) as mocked_getchar, patch("ffx.cli.click.confirm") as mocked_confirm: + created = cli.ensureUnmuxOutputDirectory( + {"dry_run": False}, + str(output_directory), + ) + + self.assertTrue(created) + self.assertTrue(output_directory.is_dir()) + mocked_getchar.assert_called_once() + mocked_confirm.assert_not_called() + + def test_yes_creates_missing_output_directory_without_prompt(self): + with tempfile.TemporaryDirectory() as tempdir: + output_directory = Path(tempdir) / "missing" / "parents" / "manual" + + with patch("ffx.cli.click.confirm") as mocked_confirm: + created = cli.ensureUnmuxOutputDirectory( + {"dry_run": False, "yes": True}, + str(output_directory), + ) + + self.assertTrue(created) + self.assertTrue(output_directory.is_dir()) + mocked_confirm.assert_not_called() + + def test_missing_output_directory_can_be_rejected(self): + with tempfile.TemporaryDirectory() as tempdir: + output_directory = Path(tempdir) / "missing" / "manual" + + with patch("ffx.cli.click.confirm", return_value=False) as mocked_confirm: + with self.assertRaises(click.ClickException) as caught: + cli.ensureUnmuxOutputDirectory( + {"dry_run": False}, + str(output_directory), + ) + + self.assertFalse(output_directory.exists()) + self.assertIn("aborted by user", str(caught.exception)) + mocked_confirm.assert_called_once() + + def test_existing_output_directory_does_not_prompt(self): + with tempfile.TemporaryDirectory() as tempdir: + output_directory = Path(tempdir) / "manual" + output_directory.mkdir() + + with patch("ffx.cli.click.confirm") as mocked_confirm: + created = cli.ensureUnmuxOutputDirectory( + {"dry_run": False}, + str(output_directory), + ) + + self.assertFalse(created) + mocked_confirm.assert_not_called() + + def test_existing_non_directory_output_path_fails_without_prompt(self): + with tempfile.TemporaryDirectory() as tempdir: + output_path = Path(tempdir) / "manual" + output_path.write_text("not a directory", encoding="utf-8") + + with patch("ffx.cli.click.confirm") as mocked_confirm: + with self.assertRaises(click.ClickException) as caught: + cli.ensureUnmuxOutputDirectory( + {"dry_run": False}, + str(output_path), + ) + + self.assertIn("not a directory", str(caught.exception)) + mocked_confirm.assert_not_called() + + def test_dry_run_does_not_prompt_or_create_missing_output_directory(self): + with tempfile.TemporaryDirectory() as tempdir: + output_directory = Path(tempdir) / "missing" / "manual" + + with patch("ffx.cli.click.confirm") as mocked_confirm: + created = cli.ensureUnmuxOutputDirectory( + {"dry_run": True}, + str(output_directory), + ) + + self.assertFalse(created) + self.assertFalse(output_directory.exists()) + mocked_confirm.assert_not_called() + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_media_descriptor_import_subtitles.py b/tests/unit/test_media_descriptor_import_subtitles.py index 6754b0e..a103b55 100644 --- a/tests/unit/test_media_descriptor_import_subtitles.py +++ b/tests/unit/test_media_descriptor_import_subtitles.py @@ -20,18 +20,32 @@ from ffx.track_type import TrackType # noqa: E402 class MediaDescriptorImportSubtitlesTests(unittest.TestCase): - def make_descriptor(self) -> MediaDescriptor: + COMPLETE_SIDECAR_NAMES = ( + "A2_t01_2_deu_DEF", + "A2_t01_3_eng", + "A2_t01_4_eng", + ) + + def write_complete_sidecar_set(self, directory: str, extension: str) -> None: + for basename in self.COMPLETE_SIDECAR_NAMES: + (Path(directory) / f"{basename}.{extension}").write_text( + "WEBVTT\n\n", + encoding="utf-8", + ) + + def make_descriptor(self, indices=(3,)) -> MediaDescriptor: return MediaDescriptor( context={"logger": get_ffx_logger()}, track_descriptors=[ TrackDescriptor( - index=3, - source_index=3, - sub_index=0, + index=index, + source_index=index, + sub_index=subIndex, track_type=TrackType.SUBTITLE, tags={"language": "eng", "title": "DB Subtitle"}, disposition_set={TrackDisposition.DEFAULT}, ) + for subIndex, index in enumerate(indices) ], ) @@ -74,6 +88,114 @@ class MediaDescriptorImportSubtitlesTests(unittest.TestCase): self.assertEqual("deu", track.getTags()["language"]) self.assertEqual({TrackDisposition.FORCED}, track.getDispositionSet()) + def test_strict_basename_import_recognizes_vtt_asset_set(self): + descriptor = self.make_descriptor(indices=(2, 3, 4)) + + with tempfile.TemporaryDirectory() as tmpdir: + self.write_complete_sidecar_set(tmpdir, "vtt") + result = descriptor.importSubtitles( + tmpdir, + "A2_t01", + strict=True, + ) + + self.assertEqual(3, result["candidate_count"]) + self.assertEqual([2, 3, 4], result["imported_track_indices"]) + self.assertEqual([], result["missing_track_indices"]) + self.assertEqual( + [ + "A2_t01_2_deu_DEF.vtt", + "A2_t01_3_eng.vtt", + "A2_t01_4_eng.vtt", + ], + [ + Path(track.getExternalSourceFilePath()).name + for track in descriptor.getSubtitleTracks() + ], + ) + + def test_strict_basename_import_accepts_dotted_mkv_extension(self): + descriptor = self.make_descriptor(indices=(2, 3, 4)) + + with tempfile.TemporaryDirectory() as tmpdir: + self.write_complete_sidecar_set(tmpdir, "mkv") + result = descriptor.importSubtitles( + tmpdir, + "A2_t01", + extension=".mkv", + strict=True, + ) + + self.assertEqual(3, result["candidate_count"]) + self.assertEqual([2, 3, 4], result["imported_track_indices"]) + self.assertEqual([], result["missing_track_indices"]) + self.assertTrue( + all( + track.getExternalSourceFilePath().endswith(".mkv") + for track in descriptor.getSubtitleTracks() + ) + ) + + def test_strict_basename_import_reports_missing_tracks(self): + descriptor = self.make_descriptor(indices=(2, 3, 4)) + + with tempfile.TemporaryDirectory() as tmpdir: + sidecarPath = Path(tmpdir) / "episode_2_deu.vtt" + sidecarPath.write_text("WEBVTT\n\n", encoding="utf-8") + + result = descriptor.importSubtitles( + tmpdir, + "episode", + strict=True, + ) + + self.assertEqual([2], result["imported_track_indices"]) + self.assertEqual([3, 4], result["missing_track_indices"]) + + def test_strict_basename_import_rejects_too_many_files(self): + descriptor = self.make_descriptor(indices=(2,)) + + with tempfile.TemporaryDirectory() as tmpdir: + for filename in ("episode_2_deu.vtt", "episode_3_eng.vtt"): + (Path(tmpdir) / filename).write_text("WEBVTT\n\n", encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "2 matching .* for 1 subtitle tracks"): + descriptor.importSubtitles(tmpdir, "episode", strict=True) + + def test_strict_basename_import_rejects_unknown_track_index(self): + descriptor = self.make_descriptor(indices=(2, 3, 4)) + + with tempfile.TemporaryDirectory() as tmpdir: + (Path(tmpdir) / "episode_9_eng.vtt").write_text( + "WEBVTT\n\n", + encoding="utf-8", + ) + + with self.assertRaisesRegex(ValueError, "track index pattern does not match"): + descriptor.importSubtitles(tmpdir, "episode", strict=True) + + def test_strict_basename_import_rejects_malformed_filtered_filename(self): + descriptor = self.make_descriptor(indices=(2, 3, 4)) + + with tempfile.TemporaryDirectory() as tmpdir: + (Path(tmpdir) / "episode_s01e01_2_deu.vtt").write_text( + "WEBVTT\n\n", + encoding="utf-8", + ) + + with self.assertRaisesRegex(ValueError, "expected pattern"): + descriptor.importSubtitles(tmpdir, "episode", strict=True) + + def test_strict_basename_import_rejects_duplicate_track_indices(self): + descriptor = self.make_descriptor(indices=(2, 3, 4)) + + with tempfile.TemporaryDirectory() as tmpdir: + for filename in ("episode_2_deu.vtt", "episode_2_eng.vtt"): + (Path(tmpdir) / filename).write_text("WEBVTT\n\n", encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "Multiple external subtitle files"): + descriptor.importSubtitles(tmpdir, "episode", strict=True) + if __name__ == "__main__": unittest.main()