#! /usr/bin/python3 import os, sys, subprocess, json, click, time, re from ffx.ffx_app import FfxApp VERSION='0.1.0' DEFAULT_VIDEO_ENCODER = 'vp9' DEFAULT_QUALITY = 23 DEFAULT_AV1_PRESET = 5 DEFAULT_FILE_FORMAT = 'webm' DEFAULT_FILE_EXTENSION = 'webm' DEFAULT_STEREO_BANDWIDTH = "128" DEFAULT_AC3_BANDWIDTH = "256" DEFAULT_DTS_BANDWIDTH = "320" DEFAULT_CROP_START = 60 DEFAULT_CROP_LENGTH = 180 TEMP_FILE_NAME = "ffmpeg2pass-0.log" MKVMERGE_METADATA_KEYS = ['BPS', 'NUMBER_OF_FRAMES', 'NUMBER_OF_BYTES', '_STATISTICS_WRITING_APP', '_STATISTICS_WRITING_DATE_UTC', '_STATISTICS_TAGS'] FILE_EXTENSIONS = ['mkv', 'mp4', 'avi', 'flv', 'webm'] COMMAND_TOKENS = ['ffmpeg', '-y'] NULL_TOKENS = ['-f', 'null', '/dev/null'] STREAM_TYPE_VIDEO = 'video' STREAM_TYPE_AUDIO = 'audio' STREAM_TYPE_SUBTITLE = 'subtitle' STREAM_LAYOUT_6_1 = '6.1' STREAM_LAYOUT_5_1 = '5.1(side)' STREAM_LAYOUT_STEREO = 'stereo' STREAM_LAYOUT_6CH = '6ch' SEASON_EPISODE_INDICATOR_MATCH = '[sS]([0-9]+)[eE]([0-9]+)' EPISODE_INDICATOR_MATCH = '[eE]([0-9]+)' SEASON_EPISODE_STREAM_LANGUAGE_MATCH = '[sS]([0-9]+)[eE]([0-9]+)_([0-9]+)_([a-z]{3})' SUBTITLE_FILE_EXTENSION = 'vtt' def executeProcess(commandSequence): process = subprocess.Popen(commandSequence, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = process.communicate() return output.decode('utf-8'), error.decode('utf-8'), process.returncode #[{'index': 0, 'codec_name': 'vp9', 'codec_long_name': 'Google VP9', 'profile': 'Profile 0', 'codec_type': 'video', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'width': 1920, 'height': 1080, 'coded_width': 1920, 'coded_height': 1080, 'closed_captions': 0, 'film_grain': 0, 'has_b_frames': 0, 'sample_aspect_ratio': '1:1', 'display_aspect_ratio': '16:9', 'pix_fmt': 'yuv420p', 'level': -99, 'color_range': 'tv', 'chroma_location': 'left', 'field_order': 'progressive', 'refs': 1, 'r_frame_rate': '24000/1001', 'avg_frame_rate': '24000/1001', 'time_base': '1/1000', 'start_pts': 0, 'start_time': '0.000000', 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'BPS': '7974017', 'NUMBER_OF_FRAMES': '34382', 'NUMBER_OF_BYTES': '1429358655', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 libvpx-vp9', 'DURATION': '00:23:54.016000000'}}] #[{'index': 1, 'codec_name': 'opus', 'codec_long_name': 'Opus (Opus Interactive Audio Codec)', 'codec_type': 'audio', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'sample_fmt': 'fltp', 'sample_rate': '48000', 'channels': 2, 'channel_layout': 'stereo', 'bits_per_sample': 0, 'initial_padding': 312, 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'extradata_size': 19, 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'jpn', 'title': 'Japanisch', 'BPS': '128000', 'NUMBER_OF_FRAMES': '61763', 'NUMBER_OF_BYTES': '22946145', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 libopus', 'DURATION': '00:23:54.141000000'}}] #[{'index': 2, 'codec_name': 'webvtt', 'codec_long_name': 'WebVTT subtitle', 'codec_type': 'subtitle', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'duration_ts': 1434141, 'duration': '1434.141000', 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'ger', 'title': 'Deutsch [Full]', 'BPS': '118', 'NUMBER_OF_FRAMES': '300', 'NUMBER_OF_BYTES': '21128', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 webvtt', 'DURATION': '00:23:54.010000000'}}, {'index': 3, 'codec_name': 'webvtt', 'codec_long_name': 'WebVTT subtitle', 'codec_type': 'subtitle', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'duration_ts': 1434141, 'duration': '1434.141000', 'disposition': {'default': 0, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'eng', 'title': 'Englisch [Full]', 'BPS': '101', 'NUMBER_OF_FRAMES': '276', 'NUMBER_OF_BYTES': '16980', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 webvtt', 'DURATION': '00:23:53.230000000'}}] def getStreamData(filepath): """Returns ffprobe stream data as array with elements according to the following example { "index": 4, "codec_name": "hdmv_pgs_subtitle", "codec_long_name": "HDMV Presentation Graphic Stream subtitles", "codec_type": "subtitle", "codec_tag_string": "[0][0][0][0]", "codec_tag": "0x0000", "r_frame_rate": "0/0", "avg_frame_rate": "0/0", "time_base": "1/1000", "start_pts": 0, "start_time": "0.000000", "duration_ts": 1421035, "duration": "1421.035000", "disposition": { "default": 1, "dub": 0, "original": 0, "comment": 0, "lyrics": 0, "karaoke": 0, "forced": 0, "hearing_impaired": 0, "visual_impaired": 0, "clean_effects": 0, "attached_pic": 0, "timed_thumbnails": 0, "non_diegetic": 0, "captions": 0, "descriptions": 0, "metadata": 0, "dependent": 0, "still_image": 0 }, "tags": { "language": "ger", "title": "German Full" } } """ ffprobeOutput, ffprobeError, returnCode = executeProcess(["ffprobe", "-show_streams", "-of", "json", filepath]) if 'Invalid data found when processing input' in ffprobeError: raise Exception(f"File {filepath} does not contain valid stream data") if returnCode != 0: raise Exception(f"ffprobe returned with error {returnCode}") return json.loads(ffprobeOutput)['streams'] def getStreamDescriptor(filename): streamData = getStreamData(filename) descriptor = {} descriptor['video'] = [] descriptor['audio'] = [] descriptor['subtitle'] = [] for subStream in streamData: if subStream['codec_type'] in ['video', 'audio', 'subtitle']: if not 'disposition' in subStream.keys(): subStream['disposition'] = {} if not 'default' in subStream['disposition'].keys(): subStream['disposition']['default'] = 0 if not 'forced' in subStream['disposition'].keys(): subStream['disposition']['forced'] = 0 if not 'tags' in subStream.keys(): subStream['tags'] = {} if not 'language' in subStream['tags'].keys(): subStream['tags']['language'] = 'undefined' if not 'title' in subStream['tags'].keys(): subStream['tags']['title'] = 'undefined' if subStream['codec_type'] == STREAM_TYPE_AUDIO: if 'channel_layout' in subStream.keys(): subStream['audio_layout'] = subStream['channel_layout'] elif subStream['channels'] == 6: subStream['audio_layout'] = STREAM_LAYOUT_6CH else: subStream['audio_layout'] = 'undefined' descriptor[subStream['codec_type']].append(subStream) descriptor[subStream['codec_type']][-1]['sub_index'] = len(descriptor[subStream['codec_type']]) - 1 return descriptor def getModifiedStreamOrder(length, last): """This is jellyfin specific as the last stream in the order is set as default""" seq = list(range(length)) if last < 0 or last > length -1: return seq seq.pop(last) seq.append(last) return seq # def getReorderedSubstreams(subDescriptor, last): # numSubStreams = len(subDescriptor) # modifiedOrder = getModifiedStreamOrder(numSubStreams, last) # reorderedDescriptor = [] # for streamIndex in range(numSubStreams): # reorderedDescriptor.append(subDescriptor[modifiedOrder[streamIndex]]) # return reorderedDescriptor def generateAV1Tokens(q, p): return ['-c:v:0', 'libsvtav1', '-svtav1-params', f"crf={q}:preset={p}:tune=0:enable-overlays=1:scd=1:scm=0", '-pix_fmt', 'yuv420p10le'] def generateVP9Pass1Tokens(q): return ['-c:v:0', 'libvpx-vp9', '-row-mt', '1', '-crf', str(q), '-pass', '1', '-speed', '4', '-frame-parallel', '0', '-g', '9999', '-aq-mode', '0'] def generateVP9Pass2Tokens(q): return ['-c:v:0', 'libvpx-vp9', '-row-mt', '1', '-crf', str(q), '-pass', '2', '-frame-parallel', '0', '-g', '9999', '-aq-mode', '0', '-auto-alt-ref', '1', '-lag-in-frames', '25'] def generateCropTokens(start, length): return ['-ss', str(start), '-t', str(length)] def generateDenoiseTokens(spatial=5, patch=7, research=7, hw=False): filterName = 'nlmeans_opencl' if hw else 'nlmeans' return ['-vf', f"{filterName}=s={spatial}:p={patch}:r={research}"] def generateOutputTokens(filepath, format, ext): return ['-f', format, f"{filepath}.{ext}"] def generateAudioEncodingTokens(context, index, layout): """Generates ffmpeg options for one output audio stream including channel remapping, codec and bitrate""" if layout == STREAM_LAYOUT_6_1: return [f"-c:a:{index}", 'libopus', f"-filter:a:{index}", 'channelmap=channel_layout=6.1', f"-b:a:{index}", context['bitrates']['dts']] elif layout == STREAM_LAYOUT_5_1: return [f"-c:a:{index}", 'libopus', f"-filter:a:{index}", "channelmap=FL-FL|FR-FR|FC-FC|LFE-LFE|SL-BL|SR-BR:5.1", f"-b:a:{index}", context['bitrates']['ac3']] elif layout == STREAM_LAYOUT_STEREO: return [f"-c:a:{index}", 'libopus', f"-b:a:{index}", context['bitrates']['stereo']] elif layout == STREAM_LAYOUT_6CH: return [f"-c:a:{index}", 'libopus', f"-filter:a:{index}", "channelmap=FL-FL|FR-FR|FC-FC|LFE-LFE|SL-BL|SR-BR:5.1", f"-b:a:{index}", context['bitrates']['ac3']] else: return [] def generateClearTokens(streams): clearTokens = [] for s in streams: for k in MKVMERGE_METADATA_KEYS: clearTokens += [f"-metadata:s:{s['type'][0]}:{s['sub_index']}", f"{k}="] return clearTokens def getDispositionFlags(subStreamDescriptor): return {k for (k,v) in subStreamDescriptor['disposition'].items() if v == 1} if 'disposition' in subStreamDescriptor.keys() else set() # def generateDispositionTokens(subDescriptor): def generateDispositionTokens(subDescriptor, modifyOrder = []): """-disposition:s:X default+forced""" dispositionTokens = [] for subStreamIndex in range(len(subDescriptor)): sourceSubStreamIndex = modifyOrder[subStreamIndex] if modifyOrder else subStreamIndex subStream = subDescriptor[sourceSubStreamIndex] streamType = subStream['codec_type'][0] # v|a|s dispositionFlags = getDispositionFlags(subStream) if dispositionFlags: dispositionTokens += [f"-disposition:{streamType}:{subStreamIndex}", '+'.join(dispositionFlags)] else: dispositionTokens += [f"-disposition:{streamType}:{subStreamIndex}", '0'] return dispositionTokens # def countStreamDispositions(subStreamDescriptor): # return len([l for (k,v) in subStreamDescriptor['disposition'].items()]) def searchSubtitleFiles(dir, prefix): sesl_match = re.compile(SEASON_EPISODE_STREAM_LANGUAGE_MATCH) availableFileSubtitleDescriptors = [] for subtitleFilename in os.listdir(dir): if subtitleFilename.startswith(prefix) and subtitleFilename.endswith('.' + SUBTITLE_FILE_EXTENSION): sesl_result = sesl_match.search(subtitleFilename) if sesl_result is not None: subtitleFilePath = os.path.join(dir, subtitleFilename) if os.path.isfile(subtitleFilePath): subtitleFileDescriptor = {} subtitleFileDescriptor['path'] = subtitleFilePath subtitleFileDescriptor['season'] = int(sesl_result.group(1)) subtitleFileDescriptor['episode'] = int(sesl_result.group(2)) subtitleFileDescriptor['stream'] = int(sesl_result.group(3)) subtitleFileDescriptor['language'] = sesl_result.group(4) availableFileSubtitleDescriptors.append(subtitleFileDescriptor) click.echo(f"Found {len(availableFileSubtitleDescriptors)} subtitles in files\n") return availableFileSubtitleDescriptors @click.group() @click.pass_context def ffx(ctx): """FFX""" ctx.obj = {} pass # Define a subcommand @ffx.command() def version(): click.echo(VERSION) # Another subcommand @ffx.command() def help(): click.echo(f"ffx {VERSION}\n") click.echo(f"Usage: ffx [input file] [output file] [vp9|av1] [q=[nn[,nn,...]]] [p=nn] [a=nnn[k]] [ac3=nnn[k]] [dts=nnn[k]] [crop]") @click.argument('filename', nargs=1) @ffx.command() def streams(filename): try: sd = getStreamDescriptor(filename) except Exception as ex: raise click.ClickException(f"This file does not contain any audiovisual data: {ex}") for d in sd: click.echo(f"{d['codec_name']}{' (' + str(d['channels']) + ')' if d['codec_type'] == 'audio' else ''}") @ffx.command() @click.pass_context @click.argument('paths', nargs=-1) @click.option('-l', '--label', type=str, default='', help='Label to be used as filename prefix') @click.option('-sd', '--subtitle-directory', type=str, default='', help='Load subtitles from here') @click.option('-sp', '--subtitle-prefix', type=str, default='', help='Subtitle filename prefix') @click.option("-o", "--output-directory", type=str, default='') @click.option("--dry-run", is_flag=True, default=False) def unmux(ctx, label, paths, subtitle_directory, subtitle_prefix, output_directory, dry_run): existingSourcePaths = [p for p in paths if os.path.isfile(p)] click.echo(f"\nUnmuxing {len(existingSourcePaths)} files") for sourcePath in existingSourcePaths: sd = getStreamDescriptor(sourcePath) print(f"\nFile {sourcePath}\n") for v in sd['video']: if v['codec_name'] == 'h264': commandSequence = ['ffmpeg', '-i', sourcePath, '-map', '0:v:0', '-c', 'copy', '-f', 'h264'] executeProcess() for a in sd['audio']: print(f"A: {a}\n") for s in sd['subtitle']: print(f"S: {s}\n") @ffx.command() @click.pass_context def shows(ctx): app = FfxApp(ctx.obj) app.run() @ffx.command() @click.pass_context @click.argument('paths', nargs=-1) @click.option('-l', '--label', type=str, default='', help='Label to be used as filename prefix') @click.option('-v', '--video-encoder', type=str, default=DEFAULT_VIDEO_ENCODER, help=f"Target video encoder (vp9 or av1) default: {DEFAULT_VIDEO_ENCODER}") @click.option('-q', '--quality', type=str, default=DEFAULT_QUALITY, help=f"Quality settings to be used with VP9 encoder (default: {DEFAULT_QUALITY})") @click.option('-p', '--preset', type=str, default=DEFAULT_AV1_PRESET, help=f"Quality preset to be used with AV1 encoder (default: {DEFAULT_AV1_PRESET})") @click.option('-a', '--stereo-bitrate', type=int, default=DEFAULT_STEREO_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode stereo audio streams (default: {DEFAULT_STEREO_BANDWIDTH})") @click.option('-ac3', '--ac3-bitrate', type=int, default=DEFAULT_AC3_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode 5.1 audio streams (default: {DEFAULT_AC3_BANDWIDTH})") @click.option('-dts', '--dts-bitrate', type=int, default=DEFAULT_DTS_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode 6.1 audio streams (default: {DEFAULT_DTS_BANDWIDTH})") @click.option('-sd', '--subtitle-directory', type=str, default='', help='Load subtitles from here') @click.option('-sp', '--subtitle-prefix', type=str, default='', help='Subtitle filename prefix') @click.option('-ss', '--subtitle-language', type=str, multiple=True, help='Subtitle stream language(s)') @click.option('-st', '--subtitle-title', type=str, multiple=True, help='Subtitle stream title(s)') @click.option('-ds', '--default-subtitle', type=int, default=-1, help='Index of default subtitle stream') @click.option('-fs', '--forced-subtitle', type=int, default=-1, help='Index of forced subtitle stream') # (including default audio stream tag) @click.option('-as', '--audio-language', type=str, multiple=True, help='Audio stream language(s)') @click.option('-at', '--audio-title', type=str, multiple=True, help='Audio stream title(s)') @click.option('-da', '--default-audio', type=int, default=-1, help='Index of default audio stream') @click.option('-da', '--forced-audio', type=int, default=-1, help='Index of forced audio stream') @click.option("--crop", is_flag=False, flag_value="default", default="none") @click.option("-o", "--output-directory", type=str, default='') @click.option("-c", "--clear-metadata", is_flag=True, default=False) @click.option("-d", "--denoise", is_flag=True, default=False) @click.option("-j", "--jellyfin", is_flag=True, default=False) @click.option("--dry-run", is_flag=True, default=False) def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, ac3_bitrate, dts_bitrate, subtitle_directory, subtitle_prefix, subtitle_language, subtitle_title, default_subtitle, forced_subtitle, audio_language, audio_title, default_audio, forced_audio, crop, output_directory, clear_metadata, denoise, jellyfin, dry_run): """Batch conversion of audiovideo files in format suitable for web playback, e.g. jellyfin Files found under PATHS will be converted according to parameters. Filename extensions will be changed appropriately. Suffices will we appended to filename in case of multiple created files or if the filename has not changed.""" startTime = time.perf_counter() context = ctx.obj click.echo(f"\nVideo encoder: {video_encoder}") qualityTokens = quality.split(',') q_list = [q for q in qualityTokens if q.isnumeric()] click.echo(f"Qualities: {q_list}") context['bitrates'] = {} context['bitrates']['stereo'] = str(stereo_bitrate) if str(stereo_bitrate).endswith('k') else f"{stereo_bitrate}k" context['bitrates']['ac3'] = str(ac3_bitrate) if str(ac3_bitrate).endswith('k') else f"{ac3_bitrate}k" context['bitrates']['dts'] = str(dts_bitrate) if str(dts_bitrate).endswith('k') else f"{dts_bitrate}k" click.echo(f"Stereo bitrate: {context['bitrates']['stereo']}") click.echo(f"AC3 bitrate: {context['bitrates']['ac3']}") click.echo(f"DTS bitrate: {context['bitrates']['dts']}") se_match = re.compile(SEASON_EPISODE_INDICATOR_MATCH) e_match = re.compile(EPISODE_INDICATOR_MATCH) ## Conversion parameters # Parse subtitle files context['import_subtitles'] = (subtitle_directory and subtitle_prefix) availableFileSubtitleDescriptors = searchSubtitleFiles(subtitle_directory, subtitle_prefix) if context['import_subtitles'] else [] # Overwrite audio tags if set audioLanguages = audio_language audioTitles = audio_title # Overwrite subtitle tags if set subtitleLanguages = subtitle_language subtitleTitles = subtitle_title defaultAudio = default_audio defaultSubtitle = default_subtitle forcedAudio = forced_audio forcedSubtitle = forced_subtitle # Process crop parameters context['perform_crop'] = (crop != 'none') if context['perform_crop']: cTokens = crop.split(',') if cTokens and len(cTokens) == 2: cropStart, cropLength = crop.split(',') else: cropStart = DEFAULT_CROP_START cropLength = DEFAULT_CROP_LENGTH click.echo(f"crop start={cropStart} length={cropLength}") cropTokens = generateCropTokens(int(cropStart), int(cropLength)) else: cropTokens = [] job_index = 0 existingSourcePaths = [p for p in paths if os.path.isfile(p)] click.echo(f"\nRunning {len(existingSourcePaths) * len(q_list)} jobs") for sourcePath in existingSourcePaths: ### ### # Separate basedir, basename and extension for current source file sourceDirectory = os.path.dirname(sourcePath) sourceFilename = os.path.basename(sourcePath) sourcePathTokens = sourceFilename.split('.') if sourcePathTokens[-1] in FILE_EXTENSIONS: sourceFileBasename = '.'.join(sourcePathTokens[:-1]) sourceFilenameExtension = sourcePathTokens[-1] else: sourceFileBasename = sourceFilename sourceFilenameExtension = '' click.echo(f"\nProcessing file {sourcePath}") # Determine season and episode if present in current filename season_digits = 2 episode_digits = 2 index_digits = 3 se_result = se_match.search(sourceFilename) e_result = e_match.search(sourceFilename) season = -1 episode = -1 file_index = 0 if se_result is not None: season = int(se_result.group(1)) episode = int(se_result.group(2)) elif e_result is not None: episode = int(e_result.group(1)) else: file_index += 1 matchingFileSubtitleDescriptors = sorted([d for d in availableFileSubtitleDescriptors if d['season'] == season and d['episode'] == episode], key=lambda d: d['stream']) if availableFileSubtitleDescriptors else [] print(f"season={season} episode={episode} file={file_index}") # Assemble target filename tokens targetFilenameTokens = [] targetFilenameExtension = DEFAULT_FILE_EXTENSION if label: targetFilenameTokens = [label] if season > -1 and episode > -1: targetFilenameTokens += [f"S{season:0{season_digits}d}E{episode:0{episode_digits}d}"] elif episode > -1: targetFilenameTokens += [f"E{episode:0{episode_digits}d}"] else: targetFilenameTokens += [f"{file_index:0{index_digits}d}"] else: targetFilenameTokens = [sourceFileBasename] ### ### # Load source stream descriptor try: ### sourceStreamDescriptor = getStreamDescriptor(sourcePath) ### except Exception: click.echo(f"File with path {sourcePath} does not contain any audiovisual data, skipping ...") continue ## ## ## targetStreamDescriptor = sourceStreamDescriptor.copy() ## ## ## click.echo('\nSource streams:') for aStream in sourceStreamDescriptor[STREAM_TYPE_AUDIO]: click.echo(f"audio stream {aStream['sub_index']} lang={aStream['tags']['language']} title={aStream['tags']['title']} default={aStream['disposition']['default']} forced={aStream['disposition']['forced']}") for sStream in sourceStreamDescriptor[STREAM_TYPE_SUBTITLE]: click.echo(f"subtitle stream {sStream['sub_index']} lang={sStream['tags']['language']} title={sStream['tags']['title']} default={sStream['disposition']['default']} forced={sStream['disposition']['forced']}") # Check for multiple default or forced dispositions if not set by user input or database requirements #NOTE: It is currently expected that all source file have the same substream pattern, e.g. coming from the same encoder numDefaultAudioStreams = len([a for a in sourceStreamDescriptor[STREAM_TYPE_AUDIO] if a['disposition']['default'] == 1]) if defaultAudio == -1 and numDefaultAudioStreams > 1: defaultAudio = click.prompt("More than one default audio stream detected! Please select stream", type=int) numForcedAudioStreams = len([a for a in sourceStreamDescriptor[STREAM_TYPE_AUDIO] if a['disposition']['forced'] == 1]) if forcedAudio == -1 and numForcedAudioStreams > 1: forcedAudio = click.prompt("More than one forced audio stream detected! Please select stream", type=int) numDefaultSubtitleStreams = len([s for s in sourceStreamDescriptor[STREAM_TYPE_SUBTITLE] if s['disposition']['default'] == 1]) if defaultSubtitle == -1 and numDefaultSubtitleStreams > 1: defaultSubtitle = click.prompt("More than one default subtitle stream detected! Please select stream", type=int) numForcedSubtitleStreams = len([s for s in sourceStreamDescriptor[STREAM_TYPE_SUBTITLE] if s['disposition']['forced'] == 1]) if forcedSubtitle == -1 and numForcedSubtitleStreams > 1: forcedSubtitle = click.prompt("More than one forced subtitle stream detected! Please select stream", type=int) #Define default/forced tags if defaultAudio != -1: for substreamIndex in range(len(targetStreamDescriptor[STREAM_TYPE_AUDIO])): targetStreamDescriptor[STREAM_TYPE_AUDIO][substreamIndex]['disposition']['default'] = 1 if substreamIndex == defaultAudio else 0 if forcedAudio != -1: for substreamIndex in range(len(targetStreamDescriptor[STREAM_TYPE_AUDIO])): targetStreamDescriptor[STREAM_TYPE_AUDIO][substreamIndex]['disposition']['forced'] = 1 if substreamIndex == forcedAudio else 0 if defaultSubtitle != -1: for substreamIndex in range(len(targetStreamDescriptor[STREAM_TYPE_SUBTITLE])): targetStreamDescriptor[STREAM_TYPE_SUBTITLE][substreamIndex]['disposition']['default'] = 1 if substreamIndex == defaultSubtitle else 0 if forcedSubtitle != -1: for substreamIndex in range(len(targetStreamDescriptor[STREAM_TYPE_SUBTITLE])): targetStreamDescriptor[STREAM_TYPE_SUBTITLE][substreamIndex]['disposition']['forced'] = 1 if substreamIndex == forcedSubtitle else 0 # Set language and title in source stream descriptors if given per command line option for streamIndex in range(len(targetStreamDescriptor[STREAM_TYPE_AUDIO])): if streamIndex <= len(audioLanguages) - 1: targetStreamDescriptor[STREAM_TYPE_AUDIO][streamIndex]['tags']['language'] = audioLanguages[streamIndex] if streamIndex <= len(audioTitles) - 1: targetStreamDescriptor[STREAM_TYPE_AUDIO][streamIndex]['tags']['title'] = audioTitles[streamIndex] for streamIndex in range(len(targetStreamDescriptor[STREAM_TYPE_SUBTITLE])): if streamIndex <= len(subtitleLanguages) - 1: targetStreamDescriptor[STREAM_TYPE_SUBTITLE][streamIndex]['tags']['language'] = subtitleLanguages[streamIndex] if streamIndex <= len(subtitleTitles) - 1: targetStreamDescriptor[STREAM_TYPE_SUBTITLE][streamIndex]['tags']['title'] = subtitleTitles[streamIndex] click.echo('\nTarget streams:') for aStream in targetStreamDescriptor[STREAM_TYPE_AUDIO]: click.echo(f"audio stream {aStream['sub_index']} lang={aStream['tags']['language']} title={aStream['tags']['title']} default={aStream['disposition']['default']} forced={aStream['disposition']['forced']}") for sStream in targetStreamDescriptor[STREAM_TYPE_SUBTITLE]: click.echo(f"subtitle stream {sStream['sub_index']} lang={sStream['tags']['language']} title={sStream['tags']['title']} default={sStream['disposition']['default']} forced={sStream['disposition']['forced']}") numSourceAudioSubStreams = len(sourceStreamDescriptor[STREAM_TYPE_AUDIO]) numSourceSubtitleSubStreams = len(sourceStreamDescriptor[STREAM_TYPE_SUBTITLE]) # Stream order is just a list of integer audioStreamSourceOrder = list(range(numSourceAudioSubStreams)) subtitleStreamSourceOrder = list(range(numSourceSubtitleSubStreams)) # In order for the jellyfin media web UI to work properly the default/forced stream has to be the last in the sequence if jellyfin: defaultTargetAudioStreams = [a for a in targetStreamDescriptor[STREAM_TYPE_AUDIO] if a['disposition']['default'] == 1] if defaultTargetAudioStreams: audioStreamSourceOrder = getModifiedStreamOrder(len(sourceStreamDescriptor[STREAM_TYPE_AUDIO]), defaultTargetAudioStreams[0]['sub_index']) defaultTargetSubtitleStreams = [a for a in targetStreamDescriptor[STREAM_TYPE_SUBTITLE] if a['disposition']['default'] == 1] if defaultTargetSubtitleStreams: subtitleStreamSourceOrder = getModifiedStreamOrder(len(sourceStreamDescriptor[STREAM_TYPE_SUBTITLE]), defaultTargetSubtitleStreams[0]['sub_index']) # audioDispositionTokens = generateDispositionTokens(targetStreamDescriptor[STREAM_TYPE_AUDIO]) # subtitleDispositionTokens = generateDispositionTokens(targetStreamDescriptor[STREAM_TYPE_SUBTITLE]) audioDispositionTokens = generateDispositionTokens(targetStreamDescriptor[STREAM_TYPE_AUDIO], modifyOrder = audioStreamSourceOrder) subtitleDispositionTokens = generateDispositionTokens(targetStreamDescriptor[STREAM_TYPE_SUBTITLE], modifyOrder = subtitleStreamSourceOrder) mappingVideoTokens = ['-map', '0:v:0'] mappingTokens = mappingVideoTokens.copy() dispositionTokens = [] audioEncodingTokens = [] audioMetadataTokens = [] for audioStreamIndex in range(len(targetStreamDescriptor[STREAM_TYPE_AUDIO])): # Modify selected source audio stream for jellyfin if required sourceAudioStreamIndex = audioStreamSourceOrder[audioStreamIndex] # Add audio mapping tokens to list of general mapping tokens mappingTokens += ['-map', f"0:a:{sourceAudioStreamIndex}"] targetAudioStream = targetStreamDescriptor[STREAM_TYPE_AUDIO][audioStreamIndex] # audioEncodingTokens += generateAudioEncodingTokens(context, sourceAudioStream['src_sub_index'], sourceAudioStream['layout']) audioEncodingTokens += generateAudioEncodingTokens(context, audioStreamIndex, targetAudioStream['audio_layout']) if sourceStreamDescriptor[STREAM_TYPE_AUDIO][sourceAudioStreamIndex]['tags']['language'] != targetStreamDescriptor[STREAM_TYPE_AUDIO][audioStreamIndex]['tags']['language']: audioMetadataTokens += [f"-metadata:s:a:{audioStreamIndex}", f"language={targetStreamDescriptor[STREAM_TYPE_AUDIO][sourceAudioStreamIndex]['tags']['language']}"] if sourceStreamDescriptor[STREAM_TYPE_AUDIO][sourceAudioStreamIndex]['tags']['title'] != targetStreamDescriptor[STREAM_TYPE_AUDIO][audioStreamIndex]['tags']['title']: audioMetadataTokens += [f"-metadata:s:a:{audioStreamIndex}", f"title={targetStreamDescriptor[STREAM_TYPE_AUDIO][sourceAudioStreamIndex]['tags']['title']}"] # targetStreamDescriptor[STREAM_TYPE_AUDIO][audioStreamIndex]['disposition']['default'] = 1 if streamIndex == defaultAudio else 0 subtitleImportFileTokens = [] subtitleMetadataTokens = [] if context['import_subtitles'] and numSourceSubtitleSubStreams != len(matchingFileSubtitleDescriptors): click.echo(f"The number of subtitle streams found in file with path {sourcePath} is different from the number of subtitle streams provided by matching imported files, skipping ...") continue # 0: Quelle f1 = forced # 1: QUelle f2 = full for subtitleStreamIndex in range(len(targetStreamDescriptor[STREAM_TYPE_SUBTITLE])): # Modify selected source subtitle stream for jellyfin if required sourceSubtitleStreamIndex = subtitleStreamSourceOrder[subtitleStreamIndex] if context['import_subtitles']: fileSubtitleDescriptor = matchingFileSubtitleDescriptors[subtitleStreamIndex] # original order subtitleImportFileTokens += ['-i', fileSubtitleDescriptor['path']] # original order # Create mapping for subtitle streams when imported from files mappingTokens += ['-map', f"{sourceSubtitleStreamIndex+1}:s:0"] # modified order if fileSubtitleDescriptor['language'] != targetStreamDescriptor[STREAM_TYPE_SUBTITLE][subtitleStreamIndex]['tags']['language']: subtitleMetadataTokens += [f"-metadata:s:s:{sourceSubtitleStreamIndex}", f"language={targetStreamDescriptor[STREAM_TYPE_SUBTITLE][subtitleStreamIndex]['tags']['language']}"] subtitleMetadataTokens += [f"-metadata:s:s:{sourceSubtitleStreamIndex}", f"title={targetStreamDescriptor[STREAM_TYPE_SUBTITLE][subtitleStreamIndex]['tags']['title']}"] else: # Add subtitle mapping tokens to list of general mapping tokens mappingTokens += ['-map', f"0:s:{sourceSubtitleStreamIndex}"] if sourceStreamDescriptor[STREAM_TYPE_SUBTITLE][sourceSubtitleStreamIndex]['tags']['language'] != targetStreamDescriptor[STREAM_TYPE_SUBTITLE][subtitleStreamIndex]['tags']['language']: subtitleMetadataTokens += [f"-metadata:s:s:{subtitleStreamIndex}", f"language={targetStreamDescriptor[STREAM_TYPE_SUBTITLE][subtitleStreamIndex]['tags']['language']}"] if sourceStreamDescriptor[STREAM_TYPE_SUBTITLE][sourceSubtitleStreamIndex]['tags']['title'] != targetStreamDescriptor[STREAM_TYPE_SUBTITLE][subtitleStreamIndex]['tags']['title']: subtitleMetadataTokens += [f"-metadata:s:s:{subtitleStreamIndex}", f"title={targetStreamDescriptor[STREAM_TYPE_SUBTITLE][subtitleStreamIndex]['tags']['title']}"] # # Reorder audio stream descriptors and create disposition options if default is given per command line option # if defaultAudio == -1: # sourceAudioStreams = audioStreams # else: # for streamIndex in range(len(audioStreams)): # audioStreams[streamIndex]['disposition']['default'] = 1 if streamIndex == defaultAudio else 0 # # sourceAudioStreams = getReorderedSubstreams(audioStreams, defaultAudio) if jellyfin else audioStreams # # dispositionTokens += generateDispositionTokens(sourceAudioStreams) # # # Set forced tag in subtitle descriptor if given per command line option # if forcedSubtitle != -1: # for streamIndex in range(len(subtitleStreams)): # subtitleStreams[streamIndex]['disposition']['forced'] = 1 if streamIndex == forcedSubtitle else 0 # # # Reorder subtitle stream descriptors and create disposition options if default is given per command line option # if defaultSubtitle == -1: # sourceSubtitleStreams = subtitleStreams # else: # for streamIndex in range(len(subtitleStreams)): # subtitleStreams[streamIndex]['disposition']['default'] = 1 if streamIndex == defaultSubtitle else 0 # # sourceSubtitleStreams = getReorderedSubstreams(subtitleStreams, defaultSubtitle) if jellyfin else subtitleStreams # # dispositionTokens += generateDispositionTokens(sourceSubtitleStreams) # click.echo(f"Audio stream source order {audioStreamSourceOrder}") click.echo(f"Subtitle stream source order {subtitleStreamSourceOrder}") commandTokens = COMMAND_TOKENS + ['-i', sourcePath] # matchingSubtitles = [] # if context['import_subtitles']: # # # for streamIndex in range(len(mSubtitles)): # mSubtitles[streamIndex]['forced'] = 1 if forcedSubtitle != -1 and streamIndex == forcedSubtitle else 0 # mSubtitles[streamIndex]['default'] = 1 if defaultSubtitle != -1 and streamIndex == defaultSubtitle else 0 # # if streamIndex <= len(subtitleTitles) -1: # mSubtitles[streamIndex]['title'] = subtitleTitles[streamIndex] # # if defaultSubtitle != -1 and jellyfin: # matchingSubtitles = getReorderedSubstreams(mSubtitles, defaultSubtitle) # else: # matchingSubtitles = mSubtitles for q in q_list: click.echo(f"\nRunning job {job_index} file={sourcePath} q={q}") job_index += 1 # # Reorder audio stream descriptors and create disposition options if default is given per command line option # if defaultAudio == -1: # sourceAudioStreams = audioStreams # else: # for streamIndex in range(len(audioStreams)): # audioStreams[streamIndex]['disposition']['default'] = 1 if streamIndex == defaultAudio else 0 # # sourceAudioStreams = getReorderedSubstreams(audioStreams, defaultAudio) if jellyfin else audioStreams # # dispositionTokens += generateDispositionTokens(sourceAudioStreams) # # # Set forced tag in subtitle descriptor if given per command line option # if forcedSubtitle != -1: # for streamIndex in range(len(subtitleStreams)): # subtitleStreams[streamIndex]['disposition']['forced'] = 1 if streamIndex == forcedSubtitle else 0 # # # Reorder subtitle stream descriptors and create disposition options if default is given per command line option # if defaultSubtitle == -1: # sourceSubtitleStreams = subtitleStreams # else: # for streamIndex in range(len(subtitleStreams)): # subtitleStreams[streamIndex]['disposition']['default'] = 1 if streamIndex == defaultSubtitle else 0 # # sourceSubtitleStreams = getReorderedSubstreams(subtitleStreams, defaultSubtitle) if jellyfin else subtitleStreams # # dispositionTokens += generateDispositionTokens(sourceSubtitleStreams) # # # Create mapping and ffmpeg options for subtitle streams # if context['import_subtitles']: # # numMatchingSubtitles = len(matchingSubtitles) # # if jellyfin and defaultSubtitle != -1: # subtitleSequence = getModifiedStreamOrder(numMatchingSubtitles, default_subtitle) #! # else: # subtitleSequence = range(numMatchingSubtitles) # # for fileIndex in range(numMatchingSubtitles): # # # Create mapping for subtitle streams when imported from files # mappingTokens += ['-map', f"{subtitleSequence[fileIndex]+1}:s:0"] # # msg = matchingSubtitles[fileIndex] # subtitleMetadataTokens += [f"-metadata:s:s:{fileIndex}", f"language={msg['language']}"] # if 'title' in matchingSubtitles[fileIndex].keys(): # subtitleMetadataTokens += [f"-metadata:s:s:{fileIndex}", f"title={matchingSubtitles[fileIndex]['title']}"] # # else: # # for subtitleStreamIndex in range(len(sourceSubtitleStreams)): # # subtitleStream = sourceSubtitleStreams[subtitleStreamIndex] # # # Create mapping for subtitle streams # mappingTokens += ['-map', f"s:{subtitleStream['src_sub_index']}"] # # if 'tags' in subtitleStream.keys(): # if 'language' in subtitleStream['tags'].keys(): # subtitleMetadataTokens += [f"-metadata:s:s:{subtitleStreamIndex}", f"language={subtitleStream['tags']['language']}"] # if 'title' in subtitleStream['tags'].keys(): # subtitleMetadataTokens += [f"-metadata:s:s:{subtitleStreamIndex}", f"title={subtitleStream['tags']['title']}"] # Job specific tokens targetFilenameJobTokens = targetFilenameTokens.copy() if len(q_list) > 1: targetFilenameJobTokens += [f"q{q}"] # In case source and target filenames are the same add an extension to distinct output from input if not label and sourceFilenameExtension == targetFilenameExtension: targetFilenameJobTokens += ['ffx'] targetFilename = '_'.join(targetFilenameJobTokens) # + '.' + targetFilenameExtension click.echo(f"target filename: {targetFilename}") if video_encoder == 'av1': commandSequence = (commandTokens + subtitleImportFileTokens + mappingTokens + audioMetadataTokens + subtitleMetadataTokens + audioDispositionTokens + subtitleDispositionTokens + audioEncodingTokens + generateAV1Tokens(q, preset) + audioEncodingTokens) if clear_metadata: commandSequence += generateClearTokens(sourceStreamDescriptor) commandSequence += cropTokens commandSequence += generateOutputTokens(targetFilename, DEFAULT_FILE_FORMAT, DEFAULT_FILE_EXTENSION) click.echo(f"Command: {' '.join(commandSequence)}") if not dry_run: executeProcess(commandSequence) if video_encoder == 'vp9': commandSequence1 = commandTokens + mappingVideoTokens + generateVP9Pass1Tokens(q) commandSequence1 += cropTokens commandSequence1 += NULL_TOKENS click.echo(f"Command 1: {' '.join(commandSequence1)}") if os.path.exists(TEMP_FILE_NAME): os.remove(TEMP_FILE_NAME) if not dry_run: executeProcess(commandSequence1) commandSequence2 = (commandTokens + subtitleImportFileTokens + mappingTokens + audioMetadataTokens + subtitleMetadataTokens + audioDispositionTokens + subtitleDispositionTokens + dispositionTokens) if denoise: commandSequence2 += generateDenoiseTokens() commandSequence2 += generateVP9Pass2Tokens(q) + audioEncodingTokens if clear_metadata: commandSequence2 += generateClearTokens(sourceStreamDescriptor) commandSequence2 += cropTokens commandSequence2 += generateOutputTokens(targetFilename, DEFAULT_FILE_FORMAT, DEFAULT_FILE_EXTENSION) click.echo(f"Command 2: {' '.join(commandSequence2)}") if not dry_run: executeProcess(commandSequence2) #app = ModesApp(context) #app.run() #click.confirm('Warning! This file is not compliant to the defined source schema! Do you want to continue?', abort=True) click.echo('\nDONE\n') endTime = time.perf_counter() click.echo(f"Time elapsed {endTime - startTime}") # click.echo(f"app result: {app.getContext()}") if __name__ == '__main__': ffx()