From ce2f3993e11218335fcab32129ed3052c4a8af47 Mon Sep 17 00:00:00 2001 From: Javanaut Date: Thu, 17 Oct 2024 22:54:29 +0200 Subject: [PATCH] nightl --- bin/ffx.py | 265 ++++------------------------ bin/ffx/ffx_controller.py | 173 +++++++++++++------ bin/ffx/file_properties.py | 101 +++++------ bin/ffx/media_descriptor.py | 39 ++++- bin/ffx/model/show.py | 24 ++- bin/ffx/show_descriptor.py | 334 ++++++++++++++++++++++++++++++++++++ bin/ffx/video_encoder.py | 32 ++++ 7 files changed, 624 insertions(+), 344 deletions(-) create mode 100644 bin/ffx/show_descriptor.py create mode 100644 bin/ffx/video_encoder.py diff --git a/bin/ffx.py b/bin/ffx.py index 4b2480f..8753d9b 100755 --- a/bin/ffx.py +++ b/bin/ffx.py @@ -194,11 +194,17 @@ def convert(ctx, context = ctx.obj + context['dry_run'] = dry_run + context['jellyfin'] = jellyfin context['tmdb'] = tmdb + + context['import_subtitles'] = (subtitle_directory and subtitle_prefix) + if context['import_subtitles']: + context['subtitle_directory'] = subtitle_directory + context['subtitle_prefix'] = subtitle_prefix # click.echo(f"\nVideo encoder: {video_encoder}") - qualityTokens = quality.split(',') q_list = [q for q in qualityTokens if q.isnumeric()] @@ -220,26 +226,22 @@ def convert(ctx, if context['perform_crop']: cTokens = crop.split(',') if cTokens and len(cTokens) == 2: - cropStart = int(cTokens[0]) - cropLength = int(cTokens[1]) - cropTokens = FfxController.generateCropTokens(cropStart, cropLength) - else: - cropTokens = FfxController.generateCropTokens() - else: - cropTokens = [] + context['crop_start'] = int(cTokens[0]) + context['crop_lenght'] = int(cTokens[1]) - click.echo(f"Crop tokens={cropTokens}") + click.echo(f"Crop start={context['crop_start']} length={context['crop_length']}") # ## Conversion parameters # # # Parse subtitle files -# context['import_subtitles'] = (subtitle_directory and subtitle_prefix) +# # availableFileSubtitleDescriptors = searchSubtitleFiles(subtitle_directory, subtitle_prefix) if context['import_subtitles'] else [] existingSourcePaths = [p for p in paths if os.path.isfile(p) and p.split('.')[-1] in FfxController.INPUT_FILE_EXTENSIONS] click.echo(f"\nRunning {len(existingSourcePaths) * len(q_list)} jobs") + jobIndex = 0 for sourcePath in existingSourcePaths: @@ -303,10 +305,18 @@ def convert(ctx, sourceMediaDescriptor.setForcedSubTrack(TrackType.SUBTITLE, forcedSubtitleTrackSubIndex) fc = FfxController(context, sourceMediaDescriptor) - + + # mappingTokens = fc.generateMetadataTokens() + # click.echo(f"Metadata Tokens: {mappingTokens}") + dispositionTokens = fc.generateDispositionTokens() click.echo(f"Disposition Tokens: {dispositionTokens}") + audioTokens = fc.generateAudioEncodingTokens() + click.echo(f"Audio Tokens: {audioTokens}") + + audioTokens = fc + click.echo(f"Audio Tokens: {audioTokens}") else: @@ -314,6 +324,9 @@ def convert(ctx, targetMediaDescriptor = currentPattern.getMediaDescriptor() if currentPattern is not None else None + if context['import_subtitles']: + targetMediaDescriptor.importSubtitles(context['subtitle_directory'], context['subtitle_prefix']) + targetMediaDescriptor.setJellyfinOrder(context['jellyfin']) click.echo(f"Input mapping tokens: {targetMediaDescriptor.getInputMappingTokens()}") @@ -332,236 +345,16 @@ def convert(ctx, click.echo(f"Season={mediaFileProperties.getSeason()} Episode={mediaFileProperties.getEpisode()}") -# matchingFileSubtitleDescriptors = sorted([d for d in availableFileSubtitleDescriptors if d['season'] == season and d['episode'] == episode], key=lambda d: d['stream']) if availableFileSubtitleDescriptors else [] -# -# print(f"season={season} episode={episode} file={file_index}") -# -# -# # Assemble target filename tokens -# targetFilenameTokens = [] -# targetFilenameExtension = FfxController.DEFAULT_FILE_EXTENSION -# -# if label: -# targetFilenameTokens = [label] -# -# if season > -1 and episode > -1: -# targetFilenameTokens += [f"S{season:0{season_digits}d}E{episode:0{episode_digits}d}"] -# elif episode > -1: -# targetFilenameTokens += [f"E{episode:0{episode_digits}d}"] -# else: -# targetFilenameTokens += [f"{file_index:0{index_digits}d}"] -# -# else: -# targetFilenameTokens = [sourceFileBasename] -# -# ### -# ### -# -# # Load source stream descriptor -# try: -# ### -# sourceStreamDescriptor = getStreamDescriptor(sourcePath) -# ### -# -# except Exception: -# click.echo(f"File with path {sourcePath} does not contain any audiovisual data, skipping ...") -# continue -# -# -# ## ## ## -# targetStreamDescriptor = sourceStreamDescriptor.copy() -# ## ## ## - + for q in q_list: + click.echo(f"\nRunning job {jobIndex} file={sourcePath} q={q}") + jobIndex += 1 -# commandTokens = COMMAND_TOKENS + ['-i', sourcePath] -# -# -# # matchingSubtitles = [] -# # if context['import_subtitles']: -# # -# -# -# # -# # for streamIndex in range(len(mSubtitles)): -# # mSubtitles[streamIndex]['forced'] = 1 if forcedSubtitle != -1 and streamIndex == forcedSubtitle else 0 -# # mSubtitles[streamIndex]['default'] = 1 if defaultSubtitle != -1 and streamIndex == defaultSubtitle else 0 -# # -# # if streamIndex <= len(subtitleTitles) -1: -# # mSubtitles[streamIndex]['title'] = subtitleTitles[streamIndex] -# # -# # if defaultSubtitle != -1 and jellyfin: -# # matchingSubtitles = getReorderedSubstreams(mSubtitles, defaultSubtitle) -# # else: -# # matchingSubtitles = mSubtitles -# -# -# -# for q in q_list: -# -# click.echo(f"\nRunning job {job_index} file={sourcePath} q={q}") -# job_index += 1 -# -# -# # # Reorder audio stream descriptors and create disposition options if default is given per command line option -# # if defaultAudio == -1: -# # sourceAudioStreams = audioStreams -# # else: -# # for streamIndex in range(len(audioStreams)): -# # audioStreams[streamIndex]['disposition']['default'] = 1 if streamIndex == defaultAudio else 0 -# # -# # sourceAudioStreams = getReorderedSubstreams(audioStreams, defaultAudio) if jellyfin else audioStreams -# # -# # dispositionTokens += generateDispositionTokens(sourceAudioStreams) -# # -# # # Set forced tag in subtitle descriptor if given per command line option -# # if forcedSubtitle != -1: -# # for streamIndex in range(len(subtitleStreams)): -# # subtitleStreams[streamIndex]['disposition']['forced'] = 1 if streamIndex == forcedSubtitle else 0 -# # -# # # Reorder subtitle stream descriptors and create disposition options if default is given per command line option -# # if defaultSubtitle == -1: -# # sourceSubtitleStreams = subtitleStreams -# # else: -# # for streamIndex in range(len(subtitleStreams)): -# # subtitleStreams[streamIndex]['disposition']['default'] = 1 if streamIndex == defaultSubtitle else 0 -# # -# # sourceSubtitleStreams = getReorderedSubstreams(subtitleStreams, defaultSubtitle) if jellyfin else subtitleStreams -# # -# # dispositionTokens += generateDispositionTokens(sourceSubtitleStreams) -# # -# -# -# # # Create mapping and ffmpeg options for subtitle streams -# -# # if context['import_subtitles']: -# # -# # numMatchingSubtitles = len(matchingSubtitles) -# # -# # if jellyfin and defaultSubtitle != -1: -# # subtitleSequence = getModifiedStreamOrder(numMatchingSubtitles, default_subtitle) #! -# # else: -# # subtitleSequence = range(numMatchingSubtitles) -# # -# # for fileIndex in range(numMatchingSubtitles): -# # -# # # Create mapping for subtitle streams when imported from files -# # mappingTokens += ['-map', f"{subtitleSequence[fileIndex]+1}:s:0"] -# # -# # msg = matchingSubtitles[fileIndex] -# # subtitleMetadataTokens += [f"-metadata:s:s:{fileIndex}", f"language={msg['language']}"] -# # if 'title' in matchingSubtitles[fileIndex].keys(): -# # subtitleMetadataTokens += [f"-metadata:s:s:{fileIndex}", f"title={matchingSubtitles[fileIndex]['title']}"] -# # -# # else: -# # -# # for subtitleStreamIndex in range(len(sourceSubtitleStreams)): -# # -# # subtitleStream = sourceSubtitleStreams[subtitleStreamIndex] -# # -# # # Create mapping for subtitle streams -# # mappingTokens += ['-map', f"s:{subtitleStream['src_sub_index']}"] -# # -# # if 'tags' in subtitleStream.keys(): -# # if 'language' in subtitleStream['tags'].keys(): -# # subtitleMetadataTokens += [f"-metadata:s:s:{subtitleStreamIndex}", f"language={subtitleStream['tags']['language']}"] -# # if 'title' in subtitleStream['tags'].keys(): -# # subtitleMetadataTokens += [f"-metadata:s:s:{subtitleStreamIndex}", f"title={subtitleStream['tags']['title']}"] -# -# -# # Job specific tokens -# targetFilenameJobTokens = targetFilenameTokens.copy() -# -# if len(q_list) > 1: -# targetFilenameJobTokens += [f"q{q}"] -# -# # In case source and target filenames are the same add an extension to distinct output from input -# if not label and sourceFilenameExtension == targetFilenameExtension: -# targetFilenameJobTokens += ['ffx'] -# -# targetFilename = '_'.join(targetFilenameJobTokens) # + '.' + targetFilenameExtension -# -# click.echo(f"target filename: {targetFilename}") -# -# -# if video_encoder == 'av1': -# -# commandSequence = (commandTokens -# + subtitleImportFileTokens -# + mappingTokens -# + audioMetadataTokens -# + subtitleMetadataTokens -# + audioDispositionTokens -# + subtitleDispositionTokens -# + audioEncodingTokens -# + generateAV1Tokens(q, preset) + audioEncodingTokens) -# -# if clear_metadata: -# commandSequence += generateClearTokens(sourceStreamDescriptor) -# -# commandSequence += cropTokens -# -# commandSequence += generateOutputTokens(targetFilename, DEFAULT_FILE_FORMAT, DEFAULT_FILE_EXTENSION) -# -# click.echo(f"Command: {' '.join(commandSequence)}") -# -# if not dry_run: -# executeProcess(commandSequence) -# -# -# if video_encoder == 'vp9': -# -# commandSequence1 = commandTokens + mappingVideoTokens + generateVP9Pass1Tokens(q) -# -# commandSequence1 += cropTokens -# -# commandSequence1 += NULL_TOKENS -# -# click.echo(f"Command 1: {' '.join(commandSequence1)}") -# -# if os.path.exists(TEMP_FILE_NAME): -# os.remove(TEMP_FILE_NAME) -# -# if not dry_run: -# executeProcess(commandSequence1) -# -# -# commandSequence2 = (commandTokens -# + subtitleImportFileTokens -# + mappingTokens -# + audioMetadataTokens -# + subtitleMetadataTokens -# + audioDispositionTokens -# + subtitleDispositionTokens -# + dispositionTokens) -# -# if denoise: -# commandSequence2 += generateDenoiseTokens() -# -# commandSequence2 += generateVP9Pass2Tokens(q) + audioEncodingTokens -# -# if clear_metadata: -# commandSequence2 += generateClearTokens(sourceStreamDescriptor) -# -# commandSequence2 += cropTokens -# -# commandSequence2 += generateOutputTokens(targetFilename, DEFAULT_FILE_FORMAT, DEFAULT_FILE_EXTENSION) -# -# click.echo(f"Command 2: {' '.join(commandSequence2)}") -# -# if not dry_run: -# executeProcess(commandSequence2) -# -# -# #app = ModesApp(context) -# #app.run() -# + # #click.confirm('Warning! This file is not compliant to the defined source schema! Do you want to continue?', abort=True) -# -# click.echo('\nDONE\n') endTime = time.perf_counter() - click.echo(f"Time elapsed {endTime - startTime}") + click.echo(f"\nDONE\nTime elapsed {endTime - startTime}") if __name__ == '__main__': diff --git a/bin/ffx/ffx_controller.py b/bin/ffx/ffx_controller.py index 96dab3a..136f996 100644 --- a/bin/ffx/ffx_controller.py +++ b/bin/ffx/ffx_controller.py @@ -1,4 +1,4 @@ -import click +import os, click, re from ffx.media_descriptor import MediaDescriptor from ffx.helper import DIFF_ADDED_KEY, DIFF_REMOVED_KEY, DIFF_CHANGED_KEY @@ -6,6 +6,9 @@ from ffx.track_descriptor import TrackDescriptor from ffx.model.track import Track from ffx.audio_layout import AudioLayout from ffx.track_type import TrackType +from ffx.video_encoder import VideoEncoder +from ffx.process import executeProcess +from ffx.file_properties import FileProperties class FfxController(): @@ -37,6 +40,9 @@ class FfxController(): '_STATISTICS_TAGS'] INPUT_FILE_EXTENSIONS = ['mkv', 'mp4', 'avi', 'flv', 'webm'] + + CHANNEL_MAP_5_1 = 'FL-FL|FR-FR|FC-FC|LFE-LFE|SL-BL|SR-BR:5.1' + def __init__(self, context : dict, @@ -48,26 +54,18 @@ class FfxController(): self.__targetMediaDescriptor = targetMediaDescriptor - # def getReorderedSubstreams(subDescriptor, last): - # numSubStreams = len(subDescriptor) - # modifiedOrder = getModifiedStreamOrder(numSubStreams, last) - # reorderedDescriptor = [] - # for streamIndex in range(numSubStreams): - # reorderedDescriptor.append(subDescriptor[modifiedOrder[streamIndex]]) - # return reorderedDescriptor - - - def generateAV1Tokens(self, quality, preset): + def generateAV1Tokens(self, quality, preset, subIndex : int = 0): - return ['-c:v:0', 'libsvtav1', + return [f"-c:v:{int(subIndex)}", 'libsvtav1', '-svtav1-params', f"crf={quality}:preset={preset}:tune=0:enable-overlays=1:scd=1:scm=0", '-pix_fmt', 'yuv420p10le'] # -c:v:0 libvpx-vp9 -row-mt 1 -crf 32 -pass 1 -speed 4 -frame-parallel 0 -g 9999 -aq-mode 0 - def generateVP9Pass1Tokens(self, quality): + def generateVP9Pass1Tokens(self, quality, subIndex : int = 0): - return ['-c:v:0', 'libvpx-vp9', + return [f"-c:v:{int(subIndex)}", + 'libvpx-vp9', '-row-mt', '1', '-crf', str(quality), '-pass', '1', @@ -77,9 +75,10 @@ class FfxController(): '-aq-mode', '0'] # -c:v:0 libvpx-vp9 -row-mt 1 -crf 32 -pass 2 -frame-parallel 0 -g 9999 -aq-mode 0 -auto-alt-ref 1 -lag-in-frames 25 - def generateVP9Pass2Tokens(self, quality): + def generateVP9Pass2Tokens(self, quality, subIndex : int = 0): - return ['-c:v:0', 'libvpx-vp9', + return [f"-c:v:{int(subIndex)}", + 'libvpx-vp9', '-row-mt', '1', '-crf', str(quality), '-pass', '2', @@ -90,14 +89,16 @@ class FfxController(): '-lag-in-frames', '25'] + def generateCropTokens(self): - @staticmethod - def generateCropTokens(cropStart : int = -1, cropLength : int = -1): + if 'crop_start' in self.__context.keys() and 'crop_length' in self.__context.keys(): + cropStart = int(self.__context['crop_start']) + cropLength = int(self.__context['crop_length']) + else: + cropStart = FfxController.DEFAULT_CROP_START + cropLength = FfxController.DEFAULT_CROP_LENGTH - start = int(cropStart if cropStart > -1 else FfxController.DEFAULT_CROP_START) - length = int(cropLength if cropLength > -1 else FfxController.DEFAULT_CROP_LENGTH) - - return ['-ss', str(start), '-t', str(length)] + return ['-ss', str(cropStart), '-t', str(cropLength)] def generateDenoiseTokens(self, spatial=5, patch=7, research=7, hw=False): @@ -143,7 +144,7 @@ class FfxController(): audioTokens += [f"-c:a:{trackSubIndex}", 'libopus', f"-filter:a:{trackSubIndex}", - "channelmap=FL-FL|FR-FR|FC-FC|LFE-LFE|SL-BL|SR-BR:5.1", + f"channelmap={FfxController.CHANNEL_MAP_5_1}", f"-b:a:{trackSubIndex}", self.__context['bitrates']['ac3']] @@ -157,25 +158,16 @@ class FfxController(): audioTokens += [f"-c:a:{trackSubIndex}", 'libopus', f"-filter:a:{trackSubIndex}", - "channelmap=FL-FL|FR-FR|FC-FC|LFE-LFE|SL-BL|SR-BR:5.1", + f"channelmap={FfxController.CHANNEL_MAP_5_1}", f"-b:a:{trackSubIndex}", self.__context['bitrates']['ac3']] trackSubIndex += 1 return audioTokens - # def generateClearTokens(self, streams): - # clearTokens = [] - # for s in streams: - # for k in FfxController.MKVMERGE_METADATA_KEYS: - # clearTokens += [f"-metadata:s:{s['type'][0]}:{s['sub_index']}", f"{k}="] - # return clearTokens - - - def generateDispositionTokens(self): - """-disposition:s:X default+forced""" - + """Source media descriptor is optional""" + sourceTrackDescriptors = [] if self.__sourceMediaDescriptor is None else self.__sourceMediaDescriptor.getAllTrackDescriptors() targetTrackDescriptors = self.__targetMediaDescriptor.getReorderedTrackDescriptors() @@ -203,8 +195,8 @@ class FfxController(): return dispositionTokens - def generateMetadataTokens(self): + """Source media descriptor is mandatory""" mappingTokens = [] @@ -337,18 +329,101 @@ class FfxController(): mappingTokens += [f"-metadata:s:{changedTargetTrackDescriptor.getType().indicator()}:{changedTargetSourceSubIndex}", f"{changedTagKey}={changedTagValue}"] -# if TrackDescriptor.DISPOSITION_SET_KEY in changedTrackDiff.keys(): -# -# if DIFF_ADDED_KEY in changedTrackDiff[TrackDescriptor.DISPOSITION_SET_KEY]: -# for addedDisposition in changedTrackDiff[TrackDescriptor.DISPOSITION_SET_KEY][DIFF_ADDED_KEY]: -# # row = (f"changed {changedTargetTrackDescriptor.getType().label()} track index={changedTrackIndex} added disposition={addedDisposition.label()}",) -# # self.differencesTable.add_row(*map(str, row)) -# pass -# -# if DIFF_REMOVED_KEY in changedTrackDiff[TrackDescriptor.DISPOSITION_SET_KEY]: -# for removedDisposition in changedTrackDiff[TrackDescriptor.DISPOSITION_SET_KEY][DIFF_REMOVED_KEY]: -# # row = (f"changed {changedTargetTrackDescriptor.getType().label()} track index={changedTrackIndex} removed disposition={removedDisposition.label()}",) -# # self.differencesTable.add_row(*map(str, row)) -# pass + # if TrackDescriptor.DISPOSITION_SET_KEY in changedTrackDiff.keys(): + + # if DIFF_ADDED_KEY in changedTrackDiff[TrackDescriptor.DISPOSITION_SET_KEY]: + # for addedDisposition in changedTrackDiff[TrackDescriptor.DISPOSITION_SET_KEY][DIFF_ADDED_KEY]: + # # row = (f"changed {changedTargetTrackDescriptor.getType().label()} track index={changedTrackIndex} added disposition={addedDisposition.label()}",) + # # self.differencesTable.add_row(*map(str, row)) + # pass + + # if DIFF_REMOVED_KEY in changedTrackDiff[TrackDescriptor.DISPOSITION_SET_KEY]: + # for removedDisposition in changedTrackDiff[TrackDescriptor.DISPOSITION_SET_KEY][DIFF_REMOVED_KEY]: + # # row = (f"changed {changedTargetTrackDescriptor.getType().label()} track index={changedTrackIndex} removed disposition={removedDisposition.label()}",) + # # self.differencesTable.add_row(*map(str, row)) + # pass return mappingTokens + + + def runJob(self, + sourcePath, + targetPath, + videoEncoder : VideoEncoder = VideoEncoder.VP9, + quality : int = DEFAULT_QUALITY, + preset : int = DEFAULT_AV1_PRESET, + denoise : bool = False): + + + commandTokens = FfxController.COMMAND_TOKENS + ['-i', sourcePath] + + if videoEncoder == VideoEncoder.AV1: + + commandSequence = (commandTokens + #+ subtitleImportFileTokens + + self.__targetMediaDescriptor.getInputMappingTokens() + + self.generateDispositionTokens()) + + if not self.__sourceMediaDescriptor is None: + commandSequence += self.generateMetadataTokens() + + commandSequence += (self.generateAudioEncodingTokens() + + self.generateAV1Tokens(quality, preset) + + self.generateAudioEncodingTokens()) + + if self.__context['perform_crop']: + commandSequence += FfxController.generateCropTokens() + + commandSequence += self.generateOutputTokens(targetPath, + FfxController.DEFAULT_FILE_FORMAT, + FfxController.DEFAULT_FILE_EXTENSION) + + click.echo(f"Command: {' '.join(commandSequence)}") + + if not self.__context['dry_run']: + executeProcess(commandSequence) + + + if videoEncoder == VideoEncoder.VP9: + + commandSequence1 = (commandTokens + + self.__targetMediaDescriptor.getInputMappingTokens() + + self.generateVP9Pass1Tokens(quality)) + + if self.__context['perform_crop']: + commandSequence1 += FfxController.generateCropTokens() + + commandSequence1 += FfxController.NULL_TOKENS + + click.echo(f"Command 1: {' '.join(commandSequence1)}") + + if os.path.exists(FfxController.TEMP_FILE_NAME): + os.remove(FfxController.TEMP_FILE_NAME) + + if not self.__context['dry_run']: + executeProcess(commandSequence1) + + commandSequence2 = (commandTokens + #+ subtitleImportFileTokens + + self.__targetMediaDescriptor.getInputMappingTokens() + + self.generateDispositionTokens()) + + if not self.__sourceMediaDescriptor is None: + commandSequence += self.generateMetadataTokens() + + if denoise: + commandSequence2 += self.generateDenoiseTokens() + + commandSequence2 += self.generateVP9Pass2Tokens(quality) + self.generateAudioEncodingTokens() + + if self.__context['perform_crop']: + commandSequence2 += FfxController.generateCropTokens() + + commandSequence2 += self.generateOutputTokens(targetPath, + FfxController.DEFAULT_FILE_FORMAT, + FfxController.DEFAULT_FILE_EXTENSION) + + click.echo(f"Command 2: {' '.join(commandSequence2)}") + + if not self.__context['dry_run']: + executeProcess(commandSequence2) diff --git a/bin/ffx/file_properties.py b/bin/ffx/file_properties.py index 9845472..725e807 100644 --- a/bin/ffx/file_properties.py +++ b/bin/ffx/file_properties.py @@ -6,6 +6,8 @@ from .pattern_controller import PatternController from .process import executeProcess from ffx.model.pattern import Pattern +from ffx.ffx_controller import FfxController +from ffx.show_descriptor import ShowDescriptor class FileProperties(): @@ -15,10 +17,10 @@ class FileProperties(): SEASON_EPISODE_INDICATOR_MATCH = '[sS]([0-9]+)[eE]([0-9]+)' EPISODE_INDICATOR_MATCH = '[eE]([0-9]+)' - SEASON_EPISODE_STREAM_LANGUAGE_MATCH = '[sS]([0-9]+)[eE]([0-9]+)_([0-9]+)_([a-z]{3})' SUBTITLE_FILE_EXTENSION = 'vtt' + DEFAULT_INDEX_DIGITS = 3 def __init__(self, context, sourcePath): @@ -62,59 +64,6 @@ class FileProperties(): self.__episode = int(e_match.group(1)) - # click.echo(pattern) -# matchingFileSubtitleDescriptors = sorted([d for d in availableFileSubtitleDescriptors if d['season'] == season and d['episode'] == episode], key=lambda d: d['stream']) if availableFileSubtitleDescriptors else [] -# -# print(f"season={season} episode={episode} file={file_index}") -# - - def assembleTargetFilename(self): - - targetFilenameTokens = [] -# targetFilenameExtension = DEFAULT_FILE_EXTENSION -# -# if label: -# targetFilenameTokens = [label] -# -# if self.__season > -1 and self.__episode > -1: -# targetFilenameTokens += [f"S{self.__season:0{season_digits}d}E{self.__episode:0{episode_digits}d}"] -# elif self.__episode > -1: -# targetFilenameTokens += [f"E{self.__episode:0{episode_digits}d}"] -# else: -# targetFilenameTokens += [f"{file_index:0{index_digits}d}"] -# -# else: -# targetFilenameTokens = [self.__sourceFileBasename] - - - - def searchSubtitleFiles(dir, prefix): - - sesl_match = re.compile(FileProperties.SEASON_EPISODE_STREAM_LANGUAGE_MATCH) - - availableFileSubtitleDescriptors = [] - for subtitleFilename in os.listdir(dir): - if subtitleFilename.startswith(prefix) and subtitleFilename.endswith('.' + FileProperties.SUBTITLE_FILE_EXTENSION): - sesl_result = sesl_match.search(subtitleFilename) - if sesl_result is not None: - subtitleFilePath = os.path.join(dir, subtitleFilename) - if os.path.isfile(subtitleFilePath): - - subtitleFileDescriptor = {} - subtitleFileDescriptor['path'] = subtitleFilePath - subtitleFileDescriptor['season'] = int(sesl_result.group(1)) - subtitleFileDescriptor['episode'] = int(sesl_result.group(2)) - subtitleFileDescriptor['stream'] = int(sesl_result.group(3)) - subtitleFileDescriptor['language'] = sesl_result.group(4) - - availableFileSubtitleDescriptors.append(subtitleFileDescriptor) - - click.echo(f"Found {len(availableFileSubtitleDescriptors)} subtitles in files\n") - - return availableFileSubtitleDescriptors - - - def getFormatData(self): """ "format": { @@ -240,3 +189,47 @@ class FileProperties(): return int(self.__episode) + def assembleTargetFilename(self, + label = None, + quality : int = -1, + fileIndex : int = -1, + indexDigits : int = DEFAULT_INDEX_DIGITS, + extension : str = None): + + if 'show_descriptor' in self.context['show_descriptor'].keys(): + season_digits = self.context['show_descriptor'][ShowDescriptor.INDICATOR_SEASON_DIGITS_KEY] + episode_digits = self.context['show_descriptor'][ShowDescriptor.INDICATOR_EPISODE_DIGITS_KEY] + else: + season_digits = ShowDescriptor.DEFAULT_INDICATOR_SEASON_DIGITS + episode_digits = ShowDescriptor.DEFAULT_INDICATOR_EPISODE_DIGITS + + targetFilenameTokens = [] + + targetFilenameExtension = FfxController.DEFAULT_FILE_EXTENSION if extension is None else str(extension) + + if label is None: + targetFilenameTokens = [self.__sourceFileBasename] + + else: + targetFilenameTokens = [label] + + if fileIndex > -1: + targetFilenameTokens += [f"{fileIndex:0{indexDigits}d}"] + + elif self.__season > -1 and self.__episode > -1: + targetFilenameTokens += [f"S{self.__season:0{season_digits}d}E{self.__episode:0{episode_digits}d}"] + elif self.__episode > -1: + targetFilenameTokens += [f"E{self.__episode:0{episode_digits}d}"] + + if len(quality) != 1: + targetFilenameTokens += [f"q{quality}"] + + # In case source and target filenames are the same add an extension to distinct output from input + if label is None and self.__sourceFilenameExtension == targetFilenameExtension: + targetFilenameTokens += ['ffx'] + + targetFilename = '_'.join(targetFilenameTokens) + + click.echo(f"Target filename: {targetFilename}") + + return targetFilename diff --git a/bin/ffx/media_descriptor.py b/bin/ffx/media_descriptor.py index 9bd9e99..377e0c5 100644 --- a/bin/ffx/media_descriptor.py +++ b/bin/ffx/media_descriptor.py @@ -1,9 +1,10 @@ -import click +import os, re, click from typing import List, Self from ffx.track_type import TrackType from ffx.track_disposition import TrackDisposition +from ffx.file_properties import FileProperties from ffx.track_descriptor import TrackDescriptor @@ -300,3 +301,39 @@ class MediaDescriptor(): return inputMappingTokens + + # matchingFileSubtitleDescriptors = sorted([d for d in availableFileSubtitleDescriptors if d['season'] == season and d['episode'] == episode], key=lambda d: d['stream']) if availableFileSubtitleDescriptors else [] + + + def searchSubtitleFiles(searchDirectory, prefix): + + sesl_match = re.compile(FileProperties.SEASON_EPISODE_STREAM_LANGUAGE_MATCH) + + availableFileSubtitleDescriptors = [] + for subtitleFilename in os.listdir(searchDirectory): + if subtitleFilename.startswith(prefix) and subtitleFilename.endswith('.' + FileProperties.SUBTITLE_FILE_EXTENSION): + sesl_result = sesl_match.search(subtitleFilename) + if sesl_result is not None: + subtitleFilePath = os.path.join(searchDirectory, subtitleFilename) + if os.path.isfile(subtitleFilePath): + + subtitleFileDescriptor = {} + subtitleFileDescriptor['path'] = subtitleFilePath + subtitleFileDescriptor['season'] = int(sesl_result.group(1)) + subtitleFileDescriptor['episode'] = int(sesl_result.group(2)) + subtitleFileDescriptor['stream'] = int(sesl_result.group(3)) + subtitleFileDescriptor['language'] = sesl_result.group(4) + + availableFileSubtitleDescriptors.append(subtitleFileDescriptor) + + click.echo(f"Found {len(availableFileSubtitleDescriptors)} subtitles in files\n") + + return availableFileSubtitleDescriptors + + + def importSubtitles(self, searchDirectory, prefix): + + availableFileSubtitleDescriptors = self.searchSubtitleFiles(searchDirectory, prefix) + + if len(availableFileSubtitleDescriptors) != len(self.getSubtitleTracks()): + raise click.ClickException(f"MediaDescriptor.importSubtitles(): Number if subtitle files not matching number of subtitle tracks") diff --git a/bin/ffx/model/show.py b/bin/ffx/model/show.py index 864a0e6..372b75e 100644 --- a/bin/ffx/model/show.py +++ b/bin/ffx/model/show.py @@ -2,6 +2,9 @@ from sqlalchemy import create_engine, Column, Integer, String, ForeignKey from sqlalchemy.orm import relationship, declarative_base, sessionmaker +from ffx.show_descriptor import ShowDescriptor + + Base = declarative_base() class Show(Base): @@ -35,10 +38,10 @@ class Show(Base): # v2.0 # patterns: Mapped[List["Pattern"]] = relationship(back_populates="show", cascade="all, delete") - index_season_digits = Column(Integer, default=2) - index_episode_digits = Column(Integer, default=2) - indicator_season_digits = Column(Integer, default=2) - indicator_episode_digits = Column(Integer, default=2) + index_season_digits = Column(Integer, default=ShowDescriptor.DEFAULT_INDEX_SEASON_DIGITS) + index_episode_digits = Column(Integer, default=ShowDescriptor.DEFAULT_INDEX_EPISODE_DIGITS) + indicator_season_digits = Column(Integer, default=ShowDescriptor.DEFAULT_INDICATOR_SEASON_DIGITS) + indicator_episode_digits = Column(Integer, default=ShowDescriptor.DEFAULT_INDICATOR_EPISODE_DIGITS) def getDesciptor(self): @@ -55,3 +58,16 @@ class Show(Base): descriptor['indicator_episode_digits'] = int(self.indicator_episode_digits) return descriptor + +# def getDescriptor(self): +# +# kwargs = {} +# +# kwargs[] +# kwargs[] +# kwargs[] +# +# kwargs[] +# kwargs[] +# kwargs[] +# kwargs[] \ No newline at end of file diff --git a/bin/ffx/show_descriptor.py b/bin/ffx/show_descriptor.py new file mode 100644 index 0000000..1ab79ed --- /dev/null +++ b/bin/ffx/show_descriptor.py @@ -0,0 +1,334 @@ +import click + +from typing import List, Self + +from ffx.track_type import TrackType +from ffx.track_disposition import TrackDisposition + +from ffx.track_descriptor import TrackDescriptor + +from ffx.helper import dictDiff, DIFF_ADDED_KEY, DIFF_CHANGED_KEY, DIFF_REMOVED_KEY + + +class ShowDescriptor(): + """This class represents the structural content of a media file including streams and metadata""" + + CONTEXT_KEY = 'context' + + ID_KEY = 'id' + NAME_KEY = 'name' + YEAR_KEY = 'year' + + INDEX_SEASON_DIGITS_KEY = 'index_season_digits' + INDEX_EPISODE_DIGITS_KEY = 'index_episode_digits' + INDICATOR_SEASON_DIGITS_KEY = 'indicator_season_digits' + INDICATOR_EPISODE_DIGITS_KEY = 'indicator_episode_digits' + + DEFAULT_INDEX_SEASON_DIGITS = 2 + DEFAULT_INDEX_EPISODE_DIGITS = 2 + DEFAULT_INDICATOR_SEASON_DIGITS = 2 + DEFAULT_INDICATOR_EPISODE_DIGITS = 2 + + def getDesciptor(self): + + descriptor = {} + + descriptor['id'] = int(self.id) + descriptor['name'] = str(self.name) + descriptor['year'] = int(self.year) + + descriptor['index_season_digits'] = int(self.index_season_digits) + descriptor['index_episode_digits'] = int(self.index_episode_digits) + descriptor['indicator_season_digits'] = int(self.indicator_season_digits) + descriptor['indicator_episode_digits'] = int(self.indicator_episode_digits) + + return descriptor + + def __init__(self, **kwargs): + + if ShowDescriptor.ID_KEY in kwargs.keys(): + if type(kwargs[ShowDescriptor.ID_KEY]) is not dict: + raise TypeError(f"ShowDescriptor.__init__(): Argument {ShowDescriptor.ID_KEY} is required to be of type dict") + self.__showId = kwargs[ShowDescriptor.ID_KEY] + else: + self.__showId = {} + + if ShowDescriptor.NAME_KEY in kwargs.keys(): + if type(kwargs[ShowDescriptor.NAME_KEY]) is not dict: + raise TypeError(f"ShowDescriptor.__init__(): Argument {ShowDescriptor.NAME_KEY} is required to be of type dict") + self.__showName = kwargs[ShowDescriptor.NAME_KEY] + else: + self.__showName = {} + + if ShowDescriptor.YEAR_KEY in kwargs.keys(): + if type(kwargs[ShowDescriptor.YEAR_KEY]) is not dict: + raise TypeError(f"ShowDescriptor.__init__(): Argument {ShowDescriptor.YEAR_KEY} is required to be of type dict") + self.__showYear = kwargs[ShowDescriptor.YEAR_KEY] + else: + self.__showYear = {} + + if ShowDescriptor.INDEX_SEASON_DIGITS_KEY in kwargs.keys(): + if type(kwargs[ShowDescriptor.INDEX_SEASON_DIGITS_KEY]) is not dict: + raise TypeError(f"ShowDescriptor.__init__(): Argument {ShowDescriptor.INDEX_SEASON_DIGITS_KEY} is required to be of type dict") + self.__indexSeasonDigits = kwargs[ShowDescriptor.INDEX_SEASON_DIGITS_KEY] + else: + self.__indexSeasonDigits = {} + + if ShowDescriptor.INDEX_EPISODE_DIGITS_KEY in kwargs.keys(): + if type(kwargs[ShowDescriptor.INDEX_EPISODE_DIGITS_KEY]) is not dict: + raise TypeError(f"ShowDescriptor.__init__(): Argument {ShowDescriptor.INDEX_EPISODE_DIGITS_KEY} is required to be of type dict") + self.__indexEpisodeDigits = kwargs[ShowDescriptor.INDEX_EPISODE_DIGITS_KEY] + else: + self.__indexEpisodeDigits = {} + + if ShowDescriptor.INDICATOR_SEASON_DIGITS_KEY in kwargs.keys(): + if type(kwargs[ShowDescriptor.INDICATOR_SEASON_DIGITS_KEY]) is not dict: + raise TypeError(f"ShowDescriptor.__init__(): Argument {ShowDescriptor.INDICATOR_SEASON_DIGITS_KEY} is required to be of type dict") + self.__indicatorSeasonDigits = kwargs[ShowDescriptor.INDICATOR_SEASON_DIGITS_KEY] + else: + self.__indicatorSeasonDigits = {} + + if ShowDescriptor.INDICATOR_EPISODE_DIGITS_KEY in kwargs.keys(): + if type(kwargs[ShowDescriptor.INDICATOR_EPISODE_DIGITS_KEY]) is not dict: + raise TypeError(f"ShowDescriptor.__init__(): Argument {ShowDescriptor.INDICATOR_EPISODE_DIGITS_KEY} is required to be of type dict") + self.__indicatorEpisodeDigits = kwargs[ShowDescriptor.INDICATOR_EPISODE_DIGITS_KEY] + else: + self.__indicatorEpisodeDigits = {} + + + + def getDefaultVideoTrack(self): + videoDefaultTracks = [v for v in self.getVideoTracks() if TrackDisposition.DEFAULT in v.getDispositionSet()] + if len(videoDefaultTracks) > 1: + raise ValueError('ShowDescriptor.getDefaultVideoTrack(): More than one default video track is not supported') + return videoDefaultTracks[0] if videoDefaultTracks else None + + def getForcedVideoTrack(self): + videoForcedTracks = [v for v in self.getVideoTracks() if TrackDisposition.FORCED in v.getDispositionSet()] + if len(videoForcedTracks) > 1: + raise ValueError('ShowDescriptor.getForcedVideoTrack(): More than one forced video track is not supported') + return videoForcedTracks[0] if videoForcedTracks else None + + def getDefaultAudioTrack(self): + audioDefaultTracks = [a for a in self.getAudioTracks() if TrackDisposition.DEFAULT in a.getDispositionSet()] + if len(audioDefaultTracks) > 1: + raise ValueError('ShowDescriptor.getDefaultAudioTrack(): More than one default audio track is not supported') + return audioDefaultTracks[0] if audioDefaultTracks else None + + def getForcedAudioTrack(self): + audioForcedTracks = [a for a in self.getAudioTracks() if TrackDisposition.FORCED in a.getDispositionSet()] + if len(audioForcedTracks) > 1: + raise ValueError('ShowDescriptor.getForcedAudioTrack(): More than one forced audio track is not supported') + return audioForcedTracks[0] if audioForcedTracks else None + + def getDefaultSubtitleTrack(self): + subtitleDefaultTracks = [s for s in self.getSubtitleTracks() if TrackDisposition.DEFAULT in s.getDispositionSet()] + if len(subtitleDefaultTracks) > 1: + raise ValueError('ShowDescriptor.getDefaultSubtitleTrack(): More than one default subtitle track is not supported') + return subtitleDefaultTracks[0] if subtitleDefaultTracks else None + + def getForcedSubtitleTrack(self): + subtitleForcedTracks = [s for s in self.getSubtitleTracks() if TrackDisposition.FORCED in s.getDispositionSet()] + if len(subtitleForcedTracks) > 1: + raise ValueError('ShowDescriptor.getForcedSubtitleTrack(): More than one forced subtitle track is not supported') + return subtitleForcedTracks[0] if subtitleForcedTracks else None + + + def setDefaultSubTrack(self, trackType : TrackType, subIndex : int): + for t in self.getAllTrackDescriptors(): + if t.getType() == trackType: + t.setDispositionFlag(TrackDisposition.DEFAULT, t.getSubIndex() == int(subIndex)) + + def setForcedSubTrack(self, trackType : TrackType, subIndex : int): + for t in self.getAllTrackDescriptors(): + if t.getType() == trackType: + t.setDispositionFlag(TrackDisposition.FORCED, t.getSubIndex() == int(subIndex)) + + + def getReorderedTrackDescriptors(self): + + videoTracks = self.sortSubIndices(self.getVideoTracks()) + audioTracks = self.sortSubIndices(self.getAudioTracks()) + subtitleTracks = self.sortSubIndices(self.getSubtitleTracks()) + + videoDefaultTrack = self.getDefaultVideoTrack() + self.getForcedVideoTrack() + audioDefaultTrack = self.getDefaultAudioTrack() + self.getForcedAudioTrack() + subtitleDefaultTrack = self.getDefaultSubtitleTrack() + self.getForcedSubtitleTrack() + + if self.__jellyfinOrder: + if not videoDefaultTrack is None: + videoTracks.append(videoTracks.pop(videoTracks.index(videoDefaultTrack))) + if not audioDefaultTrack is None: + audioTracks.append(audioTracks.pop(audioTracks.index(audioDefaultTrack))) + if not subtitleDefaultTrack is None: + subtitleTracks.append(subtitleTracks.pop(subtitleTracks.index(subtitleDefaultTrack))) + + reorderedTrackDescriptors = videoTracks + audioTracks + subtitleTracks + orderedSourceTrackSequence = [t.getSourceIndex() for t in reorderedTrackDescriptors] + + if len(set(orderedSourceTrackSequence)) < len(orderedSourceTrackSequence): + raise ValueError(f"Multiple streams originating from the same source stream not supported") + + return reorderedTrackDescriptors + + + @classmethod + def fromFfprobe(cls, formatData, streamData): + + kwargs = {} + + if ShowDescriptor.FFPROBE_TAGS_KEY in formatData.keys(): + kwargs[ShowDescriptor.TAGS_KEY] = formatData[ShowDescriptor.FFPROBE_TAGS_KEY] + + kwargs[ShowDescriptor.TRACK_DESCRIPTOR_LIST_KEY] = [] + + #TODO: Evtl obsolet + subIndexCounters = {} + + for streamObj in streamData: + + ffprobeCodecType = streamObj[ShowDescriptor.FFPROBE_CODEC_TYPE_KEY] + trackType = TrackType.fromLabel(ffprobeCodecType) + + if trackType != TrackType.UNKNOWN: + + if trackType not in subIndexCounters.keys(): + subIndexCounters[trackType] = 0 + + kwargs[ShowDescriptor.TRACK_DESCRIPTOR_LIST_KEY].append(TrackDescriptor.fromFfprobe(streamObj, + subIndex=subIndexCounters[trackType])) + subIndexCounters[trackType] += 1 + + + return cls(**kwargs) + + + def getTags(self): + return self.__mediaTags + + + def sortSubIndices(self, descriptors : List[TrackDescriptor]) -> List[TrackDescriptor]: + subIndex = 0 + for t in descriptors: + t.setSubIndex(subIndex) + subIndex += 1 + return descriptors + + + def getAllTrackDescriptors(self) -> List[TrackDescriptor]: + return self.getVideoTracks() + self.getAudioTracks() + self.getSubtitleTracks() + + def getVideoTracks(self) -> List[TrackDescriptor]: + return [v for v in self.__trackDescriptors.copy() if v.getType() == TrackType.VIDEO] + + + def getAudioTracks(self) -> List[TrackDescriptor]: + return [a for a in self.__trackDescriptors.copy() if a.getType() == TrackType.AUDIO] + + def getSubtitleTracks(self) -> List[TrackDescriptor]: + return [s for s in self.__trackDescriptors.copy() if s.getType() == TrackType.SUBTITLE] + + def getJellyfin(self): + return self.__jellyfinOrder + + def setJellyfinOrder(self, state): + self.__jellyfinOrder = state + + def getClearTags(self): + return self.__clearTags + + + + def compare(self, vsShowDescriptor : Self): + + if not isinstance(vsShowDescriptor, self.__class__): + raise click.ClickException(f"ShowDescriptor.compare(): Argument is required to be of type {self.__class__}") + + vsTags = vsShowDescriptor.getTags() + tags = self.getTags() + + #HINT: Some tags differ per file, for example creation_time, so these are removed before diff + for emt in ShowDescriptor.EXCLUDED_MEDIA_TAGS: + if emt in tags.keys(): + del tags[emt] + if emt in vsTags.keys(): + del vsTags[emt] + + tagsDiff = dictDiff(vsTags, tags) + + compareResult = {} + + if tagsDiff: + compareResult[ShowDescriptor.TAGS_KEY] = tagsDiff + + + # Target track configuration (from DB) + #tracks = self.getAllTrackDescriptors() + tracks = self.getReorderedTrackDescriptors() + numTracks = len(tracks) + + # Current track configuration (of file) + vsTracks = vsShowDescriptor.getAllTrackDescriptors() + numVsTracks = len(vsTracks) + + maxNumOfTracks = max(numVsTracks, numTracks) + + + trackCompareResult = {} + + for tp in range(maxNumOfTracks): + + # inspect/update funktionier nur so + if self.__jellyfinOrder: + vsTrackIndex = tracks[tp].getSourceIndex() + else: + vsTrackIndex = tp + # vsTrackIndex = tracks[tp].getSourceIndex() + + # Will trigger if tracks are missing in file + if tp > (numVsTracks - 1): + if DIFF_ADDED_KEY not in trackCompareResult.keys(): + trackCompareResult[DIFF_ADDED_KEY] = set() + trackCompareResult[DIFF_ADDED_KEY].add(tracks[tp].getIndex()) + continue + + # Will trigger if tracks are missing in DB definition + # New tracks will be added per update via this way + if tp > (numTracks - 1): + if DIFF_REMOVED_KEY not in trackCompareResult.keys(): + trackCompareResult[DIFF_REMOVED_KEY] = {} + trackCompareResult[DIFF_REMOVED_KEY][vsTracks[vsTrackIndex].getIndex()] = vsTracks[vsTrackIndex] + continue + + # assumption is made here that the track order will not change for all files of a sequence + trackDiff = tracks[tp].compare(vsTracks[vsTrackIndex]) + + if trackDiff: + if DIFF_CHANGED_KEY not in trackCompareResult.keys(): + trackCompareResult[DIFF_CHANGED_KEY] = {} + trackCompareResult[DIFF_CHANGED_KEY][vsTracks[vsTrackIndex].getIndex()] = trackDiff + + if trackCompareResult: + compareResult[ShowDescriptor.TRACKS_KEY] = trackCompareResult + + return compareResult + + + + def getInputMappingTokens(self, use_sub_index : bool = True): + + reorderedTrackDescriptors = self.getReorderedTrackDescriptors() + inputMappingTokens = [] + + for rtd in reorderedTrackDescriptors: + trackType = rtd.getType() + if use_sub_index: + inputMappingTokens += ['-map', f"0:{trackType.indicator()}:{rtd.getSubIndex()}"] + else: + inputMappingTokens += ['-map', f"0:{rtd.getIndex()}"] + + return inputMappingTokens + diff --git a/bin/ffx/video_encoder.py b/bin/ffx/video_encoder.py new file mode 100644 index 0000000..6726848 --- /dev/null +++ b/bin/ffx/video_encoder.py @@ -0,0 +1,32 @@ +from enum import Enum + +class VideoEncoder(Enum): + + AV1 = {'label': 'av1', 'index': 1} + VP9 = {'label': 'vp9', 'index': 2} + + UNDEFINED = {'label': 'undefined', 'index': 0} + + def label(self): + """Returns the stream type as string""" + return str(self.value['label']) + + def index(self): + """Returns the stream type index""" + return int(self.value['index']) + + @staticmethod + def fromLabel(label : str): + tlist = [t for t in VideoEncoder if t.value['label'] == str(label)] + if tlist: + return tlist[0] + else: + return VideoEncoder.UNDEFINED + + @staticmethod + def fromIndex(index : int): + tlist = [t for t in VideoEncoder if t.value['index'] == int(index)] + if tlist: + return tlist[0] + else: + return VideoEncoder.UNDEFINED