#! /usr/bin/python3 import os, click, time, logging from ffx.configuration_controller import ConfigurationController from ffx.file_properties import FileProperties from ffx.ffx_app import FfxApp from ffx.ffx_controller import FfxController from ffx.tmdb_controller import TmdbController from ffx.database import databaseContext from ffx.media_descriptor import MediaDescriptor from ffx.track_descriptor import TrackDescriptor from ffx.track_type import TrackType from ffx.video_encoder import VideoEncoder from ffx.track_disposition import TrackDisposition from ffx.process import executeProcess from ffx.helper import filterFilename from ffx.constants import DEFAULT_QUALITY, DEFAULT_AV1_PRESET from ffx.constants import DEFAULT_STEREO_BANDWIDTH, DEFAULT_AC3_BANDWIDTH, DEFAULT_DTS_BANDWIDTH, DEFAULT_7_1_BANDWIDTH VERSION='0.2.1' # 0.1.1 # Bugfixes, TMBD identify shows # 0.1.2 # Bugfixes # 0.1.3 # Subtitle file imports # 0.2.0 # Tests, Config-File # 0.2.1 # Signature, Tags cleaning, Bugfixes, Refactoring @click.group() @click.pass_context @click.option('--database-file', type=str, default='', help='Path to database file') @click.option('-v', '--verbose', type=int, default=0, help='Set verbosity of output') @click.option("--dry-run", is_flag=True, default=False) def ffx(ctx, database_file, verbose, dry_run): """FFX""" ctx.obj = {} ctx.obj['config'] = ConfigurationController() ctx.obj['database'] = databaseContext(databasePath=database_file if database_file else ctx.obj['config'].getDatabaseFilePath()) ctx.obj['dry_run'] = dry_run ctx.obj['verbosity'] = verbose # Critical 50 # Error 40 # Warning 30 # Info 20 # Debug 10 fileLogVerbosity = max(40 - verbose * 10, 10) consoleLogVerbosity = max(20 - verbose * 10, 10) ctx.obj['logger'] = logging.getLogger('FFX') ctx.obj['logger'].setLevel(logging.DEBUG) ffxFileHandler = logging.FileHandler(ctx.obj['config'].getLogFilePath()) ffxFileHandler.setLevel(fileLogVerbosity) ffxConsoleHandler = logging.StreamHandler() ffxConsoleHandler.setLevel(consoleLogVerbosity) fileFormatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ffxFileHandler.setFormatter(fileFormatter) consoleFormatter = logging.Formatter( '%(message)s') ffxConsoleHandler.setFormatter(consoleFormatter) ctx.obj['logger'].addHandler(ffxConsoleHandler) ctx.obj['logger'].addHandler(ffxFileHandler) # Define a subcommand @ffx.command() def version(): click.echo(VERSION) # Another subcommand @ffx.command() def help(): click.echo(f"ffx {VERSION}\n") click.echo(f"Usage: ffx [input file] [output file] [vp9|av1] [q=[nn[,nn,...]]] [p=nn] [a=nnn[k]] [ac3=nnn[k]] [dts=nnn[k]] [crop]") @ffx.command() @click.pass_context @click.argument('filename', nargs=1) def inspect(ctx, filename): ctx.obj['command'] = 'inspect' ctx.obj['arguments'] = {} ctx.obj['arguments']['filename'] = filename app = FfxApp(ctx.obj) app.run() #TODO: TrackCodec Klasse CODEC_LOOKUP_TABLE = { 'h264': {'format': 'h264', 'extension': 'h264'}, 'aac': { 'extension': 'aac'}, 'ac3': {'format': 'ac3', 'extension': 'ac3'}, 'ass': {'format': 'ass', 'extension': 'ass'}, 'hdmv_pgs_subtitle': {'format': 'sup', 'extension': 'sup'} } def getUnmuxSequence(trackDescriptor: TrackDescriptor, sourcePath, targetPrefix, targetDirectory = ''): trackCodec = trackDescriptor.getCodec() if not trackCodec in CODEC_LOOKUP_TABLE.keys(): return [] commandTokens = FfxController.COMMAND_TOKENS + ['-i', sourcePath] trackType = trackDescriptor.getType() targetPathBase = os.path.join(targetDirectory, targetPrefix) if targetDirectory else targetPrefix commandTokens += ['-map', f"0:{trackType.indicator()}:{trackDescriptor.getSubIndex()}", '-c', 'copy'] if 'format' in CODEC_LOOKUP_TABLE[trackCodec].keys(): commandTokens += ['-f', CODEC_LOOKUP_TABLE[trackCodec]['format']] commandTokens += [f"{targetPathBase}.{CODEC_LOOKUP_TABLE[trackCodec]['extension']}"] return commandTokens @ffx.command() @click.pass_context @click.argument('paths', nargs=-1) @click.option('-l', '--label', type=str, default='', help='Label to be used as filename prefix') @click.option("-o", "--output-directory", type=str, default='') @click.option("-s", "--subtitles-only", is_flag=True, default=False) def unmux(ctx, paths, label, output_directory, subtitles_only): existingSourcePaths = [p for p in paths if os.path.isfile(p)] ctx.obj['logger'].debug(f"\nUnmuxing {len(existingSourcePaths)} files") for sourcePath in existingSourcePaths: fp = FileProperties(ctx.obj, sourcePath) try: sourceMediaDescriptor = fp.getMediaDescriptor() season = fp.getSeason() episode = fp.getEpisode() #TODO: Recognition für alle Formate anpassen targetLabel = label if label else fp.getFileBasename() targetIndicator = f"_S{season}E{episode}" if label and season != -1 and episode != -1 else '' if label and not targetIndicator: ctx.obj['logger'].warning(f"Skipping file {fp.getFilename()}: Label set but no indicator recognized") continue else: ctx.obj['logger'].debug(f"\nUnmuxing file {fp.getFilename()}\n") for trackDescriptor in sourceMediaDescriptor.getAllTrackDescriptors(): if trackDescriptor.getType() == TrackType.SUBTITLE or not subtitles_only: # SEASON_EPISODE_STREAM_LANGUAGE_MATCH = '[sS]([0-9]+)[eE]([0-9]+)_([0-9]+)_([a-z]{3})' targetPrefix = f"{targetLabel}{targetIndicator}_{trackDescriptor.getIndex()}_{trackDescriptor.getLanguage().threeLetter()}" unmuxSequence = getUnmuxSequence(trackDescriptor, sourcePath, targetPrefix, targetDirectory = output_directory) if unmuxSequence: if not ctx.obj['dry_run']: ctx.obj['logger'].debug(f"Executing unmuxing sequence: {' '.join(unmuxSequence)}") out, err, rc = executeProcess(unmuxSequence) if rc: ctx.obj['logger'].error(f"Unmuxing of stream {trackDescriptor.getIndex()} failed with error ({rc}) {err}") else: ctx.obj['logger'].warning(f"Skipping stream with unknown codec {trackDescriptor.getCodec()}") except Exception as ex: ctx.obj['logger'].warning(f"Skipping File {sourcePath} ({ex})") @ffx.command() @click.pass_context def shows(ctx): ctx.obj['command'] = 'shows' app = FfxApp(ctx.obj) app.run() def checkUniqueDispositions(context, mediaDescriptor: MediaDescriptor): # Check for multiple default or forced dispositions if not set by user input or database requirements # # Query user for the correct sub indices, then configure flags in track descriptors associated with media descriptor accordingly. # The correct tokens should then be created by if len([v for v in mediaDescriptor.getVideoTracks() if v.getDispositionFlag(TrackDisposition.DEFAULT)]) > 1: if context['no_prompt']: raise click.ClickException('More than one default video stream detected and no prompt set') defaultVideoTrackSubIndex = click.prompt("More than one default video stream detected! Please select stream", type=int) mediaDescriptor.setDefaultSubTrack(TrackType.VIDEO, defaultVideoTrackSubIndex) if len([v for v in mediaDescriptor.getVideoTracks() if v.getDispositionFlag(TrackDisposition.FORCED)]) > 1: if context['no_prompt']: raise click.ClickException('More than one forced video stream detected and no prompt set') forcedVideoTrackSubIndex = click.prompt("More than one forced video stream detected! Please select stream", type=int) mediaDescriptor.setForcedSubTrack(TrackType.VIDEO, forcedVideoTrackSubIndex) if len([a for a in mediaDescriptor.getAudioTracks() if a.getDispositionFlag(TrackDisposition.DEFAULT)]) > 1: if context['no_prompt']: raise click.ClickException('More than one default audio stream detected and no prompt set') defaultAudioTrackSubIndex = click.prompt("More than one default audio stream detected! Please select stream", type=int) mediaDescriptor.setDefaultSubTrack(TrackType.AUDIO, defaultAudioTrackSubIndex) if len([a for a in mediaDescriptor.getAudioTracks() if a.getDispositionFlag(TrackDisposition.FORCED)]) > 1: if context['no_prompt']: raise click.ClickException('More than one forced audio stream detected and no prompt set') forcedAudioTrackSubIndex = click.prompt("More than one forced audio stream detected! Please select stream", type=int) mediaDescriptor.setForcedSubTrack(TrackType.AUDIO, forcedAudioTrackSubIndex) if len([s for s in mediaDescriptor.getSubtitleTracks() if s.getDispositionFlag(TrackDisposition.DEFAULT)]) > 1: if context['no_prompt']: raise click.ClickException('More than one default subtitle stream detected and no prompt set') defaultSubtitleTrackSubIndex = click.prompt("More than one default subtitle stream detected! Please select stream", type=int) mediaDescriptor.setDefaultSubTrack(TrackType.SUBTITLE, defaultSubtitleTrackSubIndex) if len([s for s in mediaDescriptor.getSubtitleTracks() if s.getDispositionFlag(TrackDisposition.FORCED)]) > 1: if context['no_prompt']: raise click.ClickException('More than one forced subtitle stream detected and no prompt set') forcedSubtitleTrackSubIndex = click.prompt("More than one forced subtitle stream detected! Please select stream", type=int) mediaDescriptor.setForcedSubTrack(TrackType.SUBTITLE, forcedSubtitleTrackSubIndex) @ffx.command() @click.pass_context @click.argument('paths', nargs=-1) @click.option('-l', '--label', type=str, default='', help='Label to be used as filename prefix') @click.option('-v', '--video-encoder', type=str, default=FfxController.DEFAULT_VIDEO_ENCODER, help=f"Target video encoder (vp9 or av1) default: {FfxController.DEFAULT_VIDEO_ENCODER}") @click.option('-q', '--quality', type=str, default=DEFAULT_QUALITY, help=f"Quality settings to be used with VP9 encoder (default: {DEFAULT_QUALITY})") @click.option('-p', '--preset', type=str, default=DEFAULT_AV1_PRESET, help=f"Quality preset to be used with AV1 encoder (default: {DEFAULT_AV1_PRESET})") @click.option('-a', '--stereo-bitrate', type=int, default=DEFAULT_STEREO_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode stereo audio streams (default: {DEFAULT_STEREO_BANDWIDTH})") @click.option('--ac3', type=int, default=DEFAULT_AC3_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode 5.1 audio streams (default: {DEFAULT_AC3_BANDWIDTH})") @click.option('--dts', type=int, default=DEFAULT_DTS_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode 6.1 audio streams (default: {DEFAULT_DTS_BANDWIDTH})") @click.option('--subtitle-directory', type=str, default='', help='Load subtitles from here') @click.option('--subtitle-prefix', type=str, default='', help='Subtitle filename prefix') @click.option('--audio-language', type=str, multiple=True, help='Audio stream language(s)') @click.option('--audio-title', type=str, multiple=True, help='Audio stream title(s)') @click.option('--default-audio', type=int, default=-1, help='Index of default audio stream') @click.option('--forced-audio', type=int, default=-1, help='Index of forced audio stream') @click.option('--subtitle-language', type=str, multiple=True, help='Subtitle stream language(s)') @click.option('--subtitle-title', type=str, multiple=True, help='Subtitle stream title(s)') @click.option('--default-subtitle', type=int, default=-1, help='Index of default subtitle stream') @click.option('--forced-subtitle', type=int, default=-1, help='Index of forced subtitle stream') # (including default audio stream tag) @click.option("--crop", is_flag=False, flag_value="default", default="none") @click.option("--output-directory", type=str, default='') @click.option("--denoise", is_flag=True, default=False) @click.option("--no-tmdb", is_flag=True, default=False) @click.option("--no-jellyfin", is_flag=True, default=False) @click.option("--no-pattern", is_flag=True, default=False) @click.option("--dont-pass-dispositions", is_flag=True, default=False) @click.option("--no-prompt", is_flag=True, default=False) @click.option("--no-signature", is_flag=True, default=False) @click.option("--keep-mkvmerge-metadata", is_flag=True, default=False) def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, ac3, dts, subtitle_directory, subtitle_prefix, audio_language, audio_title, default_audio, forced_audio, subtitle_language, subtitle_title, default_subtitle, forced_subtitle, crop, output_directory, denoise, no_tmdb, no_jellyfin, no_pattern, dont_pass_dispositions, no_prompt, no_signature, keep_mkvmerge_metadata): """Batch conversion of audiovideo files in format suitable for web playback, e.g. jellyfin Files found under PATHS will be converted according to parameters. Filename extensions will be changed appropriately. Suffices will we appended to filename in case of multiple created files or if the filename has not changed.""" startTime = time.perf_counter() context = ctx.obj context['video_encoder'] = VideoEncoder.fromLabel(video_encoder) context['use_jellyfin'] = not no_jellyfin context['use_tmdb'] = not no_tmdb context['use_pattern'] = not no_pattern context['no_prompt'] = no_prompt context['no_signature'] = no_signature context['keep_mkvmerge_metadata'] = keep_mkvmerge_metadata context['import_subtitles'] = (subtitle_directory and subtitle_prefix) if context['import_subtitles']: context['subtitle_directory'] = subtitle_directory context['subtitle_prefix'] = subtitle_prefix ctx.obj['logger'].debug(f"\nVideo encoder: {video_encoder}") qualityTokens = quality.split(',') q_list = [q for q in qualityTokens if q.isnumeric()] ctx.obj['logger'].debug(f"Qualities: {q_list}") context['bitrates'] = {} context['bitrates']['stereo'] = str(stereo_bitrate) if str(stereo_bitrate).endswith('k') else f"{stereo_bitrate}k" context['bitrates']['ac3'] = str(ac3) if str(ac3).endswith('k') else f"{ac3}k" context['bitrates']['dts'] = str(dts) if str(dts).endswith('k') else f"{dts}k" ctx.obj['logger'].debug(f"Stereo bitrate: {context['bitrates']['stereo']}") ctx.obj['logger'].debug(f"AC3 bitrate: {context['bitrates']['ac3']}") ctx.obj['logger'].debug(f"DTS bitrate: {context['bitrates']['dts']}") # Process crop parameters context['perform_crop'] = (crop != 'none') if context['perform_crop']: cTokens = crop.split(',') if cTokens and len(cTokens) == 2: context['crop_start'] = int(cTokens[0]) context['crop_length'] = int(cTokens[1]) ctx.obj['logger'].debug(f"Crop start={context['crop_start']} length={context['crop_length']}") tc = TmdbController() if context['use_tmdb'] else None existingSourcePaths = [p for p in paths if os.path.isfile(p) and p.split('.')[-1] in FfxController.INPUT_FILE_EXTENSIONS] ctx.obj['logger'].info(f"\nRunning {len(existingSourcePaths) * len(q_list)} jobs") jobIndex = 0 for sourcePath in existingSourcePaths: # Separate basedir, basename and extension for current source file sourceDirectory = os.path.dirname(sourcePath) sourceFilename = os.path.basename(sourcePath) sourcePathTokens = sourceFilename.split('.') sourceFileBasename = '.'.join(sourcePathTokens[:-1]) sourceFilenameExtension = sourcePathTokens[-1] ctx.obj['logger'].info(f"\nProcessing file {sourcePath}") mediaFileProperties = FileProperties(context, sourceFilename) sourceMediaDescriptor = mediaFileProperties.getMediaDescriptor() #HINT: This is None if the filename did not match anything in database currentPattern = mediaFileProperties.getPattern() if context['use_pattern'] else None ctx.obj['logger'].debug(f"Pattern matching: {'No' if currentPattern is None else 'Yes'}") # fileBasename = '' if currentPattern is None: # Case no pattern matching # fileBasename = currentShowDescriptor.getFilenamePrefix() checkUniqueDispositions(context, sourceMediaDescriptor) if context['import_subtitles']: sourceMediaDescriptor.importSubtitles(context['subtitle_directory'], context['subtitle_prefix'], mediaFileProperties.getSeason(), mediaFileProperties.getEpisode()) if context['use_jellyfin']: # Reorder subtracks in types with default the last, then make subindices flat again sourceMediaDescriptor.applyJellyfinOrder() fc = FfxController(context, sourceMediaDescriptor) else: # Case pattern matching targetMediaDescriptor = currentPattern.getMediaDescriptor(ctx.obj) checkUniqueDispositions(context, targetMediaDescriptor) currentShowDescriptor = currentPattern.getShowDescriptor(ctx.obj) if context['use_tmdb']: ctx.obj['logger'].debug(f"Querying TMDB for show_id={currentShowDescriptor.getId()} season={mediaFileProperties.getSeason()} episode{mediaFileProperties.getEpisode()}") tmdbEpisodeResult = tc.queryEpisode(currentShowDescriptor.getId(), mediaFileProperties.getSeason(), mediaFileProperties.getEpisode()) ctx.obj['logger'].debug(f"tmdbEpisodeResult={tmdbEpisodeResult}") if tmdbEpisodeResult: filteredEpisodeName = filterFilename(tmdbEpisodeResult['name']) sourceFileBasename = TmdbController.getEpisodeFileBasename(currentShowDescriptor.getFilenamePrefix(), filteredEpisodeName, mediaFileProperties.getSeason(), mediaFileProperties.getEpisode(), currentShowDescriptor.getIndexSeasonDigits(), currentShowDescriptor.getIndexEpisodeDigits(), currentShowDescriptor.getIndicatorSeasonDigits(), currentShowDescriptor.getIndicatorEpisodeDigits()) else: sourceFileBasename = currentShowDescriptor.getFilenamePrefix() if context['import_subtitles']: targetMediaDescriptor.importSubtitles(context['subtitle_directory'], context['subtitle_prefix'], mediaFileProperties.getSeason(), mediaFileProperties.getEpisode()) ctx.obj['logger'].debug(f"tmd subindices: {[t.getIndex() for t in targetMediaDescriptor.getAllTrackDescriptors()]} {[t.getSubIndex() for t in targetMediaDescriptor.getAllTrackDescriptors()]} {[t.getDispositionFlag(TrackDisposition.DEFAULT) for t in targetMediaDescriptor.getAllTrackDescriptors()]}") if context['use_jellyfin']: # Reorder subtracks in types with default the last, then make subindices flat again targetMediaDescriptor.applyJellyfinOrder() ctx.obj['logger'].debug(f"tmd subindices: {[t.getIndex() for t in targetMediaDescriptor.getAllTrackDescriptors()]} {[t.getSubIndex() for t in targetMediaDescriptor.getAllTrackDescriptors()]} {[t.getDispositionFlag(TrackDisposition.DEFAULT) for t in targetMediaDescriptor.getAllTrackDescriptors()]}") ctx.obj['logger'].debug(f"Input mapping tokens (2nd pass): {targetMediaDescriptor.getInputMappingTokens()}") fc = FfxController(context, targetMediaDescriptor, sourceMediaDescriptor) ctx.obj['logger'].debug(f"Season={mediaFileProperties.getSeason()} Episode={mediaFileProperties.getEpisode()}") ctx.obj['logger'].debug(f"fileBasename={sourceFileBasename}") for q in q_list: ctx.obj['logger'].debug(f"\nRunning job {jobIndex} file={sourcePath} q={q}") jobIndex += 1 extra = ['ffx'] if sourceFilenameExtension == FfxController.DEFAULT_FILE_EXTENSION else [] ctx.obj['logger'].debug(f"label={label if label else 'Falsy'}") ctx.obj['logger'].debug(f"sourceFileBasename={sourceFileBasename}") targetFileBasename = mediaFileProperties.assembleTargetFileBasename(label, q if len(q_list) > 1 else -1, extraTokens = extra) #TODO #387 targetFilename = ((f"{sourceFileBasename}_q{q}" if len(q_list) > 1 else sourceFileBasename) if context['use_tmdb'] else targetFileBasename) targetPath = os.path.join(output_directory if output_directory else sourceDirectory, targetFilename) #TODO: target extension anpassen ctx.obj['logger'].info(f"Creating file {targetFilename}.webm") fc.runJob(sourcePath, targetPath, context['video_encoder'], q, preset, denoise) #TODO: click.confirm('Warning! This file is not compliant to the defined source schema! Do you want to continue?', abort=True) endTime = time.perf_counter() ctx.obj['logger'].info(f"\nDONE\nTime elapsed {endTime - startTime}") if __name__ == '__main__': ffx()