#! /usr/bin/python3 import os, sys, subprocess, json, click, time, re from textual.app import App, ComposeResult from textual.screen import Screen from textual.widgets import Header, Footer, Placeholder, Label VERSION='0.1.0' DEFAULT_VIDEO_ENCODER = 'vp9' DEFAULT_QUALITY = 23 DEFAULT_AV1_PRESET = 5 DEFAULT_FILE_EXTENSION = 'webm' DEFAULT_STEREO_BANDWIDTH = "128" DEFAULT_AC3_BANDWIDTH = "256" DEFAULT_DTS_BANDWIDTH = "320" DEFAULT_CROP_START = 60 DEFAULT_CROP_LENGTH = 180 TEMP_FILE_NAME = "ffmpeg2pass-0.log" MKVMERGE_METADATA_KEYS = ['BPS', 'NUMBER_OF_FRAMES', 'NUMBER_OF_BYTES', '_STATISTICS_WRITING_APP', '_STATISTICS_WRITING_DATE_UTC', '_STATISTICS_TAGS'] FILE_EXTENSIONS = ['mkv', 'mp4', 'avi', 'flv', 'webm'] COMMAND_TOKENS = ['ffmpeg', '-y', '-i'] NULL_TOKENS = ['-f', 'null', '/dev/null'] STREAM_TYPE_VIDEO = 'video' STREAM_TYPE_AUDIO = 'audio' STREAM_TYPE_SUBTITLE = 'subtitle' STREAM_LAYOUT_6_1 = '6.1' STREAM_LAYOUT_5_1 = '5.1(side)' STREAM_LAYOUT_STEREO = 'stereo' STREAM_LAYOUT_6CH = '6ch' SEASON_EPISODE_INDICATOR_MATCH = '[sS]([0-9]+)[eE]([0-9]+)' EPISODE_INDICATOR_MATCH = '[eE]([0-9]+)' class DashboardScreen(Screen): def __init__(self): super().__init__() context = self.app.getContext() context['dashboard'] = 'dashboard' def compose(self) -> ComposeResult: yield Header(show_clock=True) yield Placeholder("Dashboard Screen") yield Footer() class WarningScreen(Screen): def __init__(self): super().__init__() context = self.app.getContext() def compose(self) -> ComposeResult: yield Label("Warning! This file is not compliant to the defined source schema!") yield Footer() class SettingsScreen(Screen): def __init__(self): super().__init__() context = self.app.getContext() def compose(self) -> ComposeResult: yield Placeholder("Settings Screen") yield Footer() class HelpScreen(Screen): def __init__(self): super().__init__() context = self.app.getContext() def compose(self) -> ComposeResult: yield Placeholder("Help Screen") yield Footer() class ModesApp(App): BINDINGS = [ ("q", "quit()", "Quit"), # ("d", "switch_mode('dashboard')", "Dashboard"), # ("s", "switch_mode('settings')", "Settings"), # ("h", "switch_mode('help')", "Help"), ] MODES = { "warning": WarningScreen, "dashboard": DashboardScreen, "settings": SettingsScreen, "help": HelpScreen, } def __init__(self, context = {}): super().__init__() self.context = context def on_mount(self) -> None: self.switch_mode("warning") def getContext(self): return self.context def executeProcess(commandSequence): process = subprocess.Popen(commandSequence, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = process.communicate() return output.decode('utf-8'), error.decode('utf-8') #[{'index': 0, 'codec_name': 'vp9', 'codec_long_name': 'Google VP9', 'profile': 'Profile 0', 'codec_type': 'video', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'width': 1920, 'height': 1080, 'coded_width': 1920, 'coded_height': 1080, 'closed_captions': 0, 'film_grain': 0, 'has_b_frames': 0, 'sample_aspect_ratio': '1:1', 'display_aspect_ratio': '16:9', 'pix_fmt': 'yuv420p', 'level': -99, 'color_range': 'tv', 'chroma_location': 'left', 'field_order': 'progressive', 'refs': 1, 'r_frame_rate': '24000/1001', 'avg_frame_rate': '24000/1001', 'time_base': '1/1000', 'start_pts': 0, 'start_time': '0.000000', 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'BPS': '7974017', 'NUMBER_OF_FRAMES': '34382', 'NUMBER_OF_BYTES': '1429358655', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 libvpx-vp9', 'DURATION': '00:23:54.016000000'}}] #[{'index': 1, 'codec_name': 'opus', 'codec_long_name': 'Opus (Opus Interactive Audio Codec)', 'codec_type': 'audio', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'sample_fmt': 'fltp', 'sample_rate': '48000', 'channels': 2, 'channel_layout': 'stereo', 'bits_per_sample': 0, 'initial_padding': 312, 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'extradata_size': 19, 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'jpn', 'title': 'Japanisch', 'BPS': '128000', 'NUMBER_OF_FRAMES': '61763', 'NUMBER_OF_BYTES': '22946145', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 libopus', 'DURATION': '00:23:54.141000000'}}] #[{'index': 2, 'codec_name': 'webvtt', 'codec_long_name': 'WebVTT subtitle', 'codec_type': 'subtitle', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'duration_ts': 1434141, 'duration': '1434.141000', 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'ger', 'title': 'Deutsch [Full]', 'BPS': '118', 'NUMBER_OF_FRAMES': '300', 'NUMBER_OF_BYTES': '21128', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 webvtt', 'DURATION': '00:23:54.010000000'}}, {'index': 3, 'codec_name': 'webvtt', 'codec_long_name': 'WebVTT subtitle', 'codec_type': 'subtitle', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'duration_ts': 1434141, 'duration': '1434.141000', 'disposition': {'default': 0, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'eng', 'title': 'Englisch [Full]', 'BPS': '101', 'NUMBER_OF_FRAMES': '276', 'NUMBER_OF_BYTES': '16980', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 webvtt', 'DURATION': '00:23:53.230000000'}}] def getStreamDescriptor(filename): ffprobeOutput, ffprobeError = executeProcess(["ffprobe", "-show_streams", "-of", "json", filename]) if 'Invalid data found when processing input' in ffprobeError: return None streamData = json.loads(ffprobeOutput)['streams'] descriptor = [] i = 0 for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_VIDEO]: descriptor.append({ 'index': d['index'], 'sub_index': i, 'type': STREAM_TYPE_VIDEO, 'codec': d['codec_name'] }) i += 1 i = 0 for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_AUDIO]: streamDescriptor = { 'index': d['index'], 'sub_index': i, 'type': STREAM_TYPE_AUDIO, 'codec': d['codec_name'], 'channels': d['channels'] } if 'channel_layout' in d.keys(): streamDescriptor['layout'] = d['channel_layout'] elif d['channels'] == 6: streamDescriptor['layout'] = STREAM_LAYOUT_6CH else: streamDescriptor['layout'] = 'undefined' descriptor.append(streamDescriptor) i += 1 i = 0 for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_SUBTITLE]: descriptor.append({ 'index': d['index'], 'sub_index': i, 'type': STREAM_TYPE_SUBTITLE, 'codec': d['codec_name'] }) i += 1 return descriptor def generateAV1Tokens(q, p): return ['-c:v:0', 'libsvtav1', '-svtav1-params', f"crf={q}:preset={p}:tune=0:enable-overlays=1:scd=1:scm=0", '-pix_fmt', 'yuv420p10le'] def generateVP9Pass1Tokens(q): return ['-c:v:0', 'libvpx-vp9', '-row-mt', '1', '-crf', str(q), '-pass', '1', '-speed', '4', '-frame-parallel', '0', '-g', '9999', '-aq-mode', '0'] def generateVP9Pass2Tokens(q): return ['-c:v:0', 'libvpx-vp9', '-row-mt', '1', '-crf', str(q), '-pass', '2', '-frame-parallel', '0', '-g', '9999', '-aq-mode', '0', '-auto-alt-ref', '1', '-lag-in-frames', '25'] def generateCropTokens(start, length): return ['-ss', str(start), '-t', str(length)] def generateDenoiseTokens(spatial=5, patch=7, research=7, hw=False): filterName = 'nlmeans_opencl' if hw else 'nlmeans' return ['-vf', f"{filterName}=s={spatial}:p={patch}:r={research}"] def generateOutputTokens(f, suffix, q=None): if q is None: return ['-f', 'webm', f"{f}.{suffix}"] else: return ['-f', 'webm', f"{f}_q{q}.{suffix}"] # preset = DEFAULT_AV1_PRESET # presetTokens = [p for p in sys.argv if p.startswith('p=')] # if presetTokens: # preset = int(presetTokens[0].split('=')[1]) # ctx.obj['crop_start'] = '' # ctx.obj['crop_length'] = '' # cropTokens = [c for c in sys.argv if c.startswith('crop')] # if cropTokens: # if '=' in cropTokens[0]: # cropString = cropTokens[0].split('=')[1] # ctx.obj['crop_start'], ctx.obj['crop_length'] = cropString.split(',') # else: # ctx.obj['crop_start'] = 60 # ctx.obj['crop_length'] = 180 # # denoiseTokens = [d for d in sys.argv if d.startswith('denoise')] # # for aStream in audioStreams: # if 'channel_layout' in aStream: # print(f"audio stream: {aStream['channel_layout']}") #channel_layout # else: # print(f"unknown audio stream with {aStream['channels']} channels") #channel_layout def generateAudioTokens(context, index, layout): if layout == STREAM_LAYOUT_6_1: return [f"-c:a:{index}", 'libopus', f"-filter:a:{index}", 'channelmap=channel_layout=6.1', f"-b:a:{index}", context['bitrates']['dts']] elif layout == STREAM_LAYOUT_5_1: return [f"-c:a:{index}", 'libopus', f"-filter:a:{index}", "channelmap=FL-FL|FR-FR|FC-FC|LFE-LFE|SL-BL|SR-BR:5.1", f"-b:a:{index}", context['bitrates']['ac3']] elif layout == STREAM_LAYOUT_STEREO: return [f"-c:a:{index}", 'libopus', f"-b:a:{index}", context['bitrates']['stereo']] elif layout == STREAM_LAYOUT_6CH: return [f"-c:a:{index}", 'libopus', f"-filter:a:{index}", "channelmap=FL-FL|FR-FR|FC-FC|LFE-LFE|SL-BL|SR-BR:5.1", f"-b:a:{index}", context['bitrates']['ac3']] else: return [] def generateClearTokens(streams): clearTokens = [] for s in streams: for k in MKVMERGE_METADATA_KEYS: clearTokens += [f"-metadata:s:{s['type'][0]}:{s['sub_index']}", f"{k}="] return clearTokens @click.group() @click.pass_context def ffx(ctx): """FFX""" ctx.obj = {} pass # Define a subcommand @ffx.command() def version(): click.echo(VERSION) # Another subcommand @ffx.command() def help(): click.echo(f"ffx {VERSION}\n") click.echo(f"Usage: ffx [input file] [output file] [vp9|av1] [q=[nn[,nn,...]]] [p=nn] [a=nnn[k]] [ac3=nnn[k]] [dts=nnn[k]] [crop]") @click.argument('filename', nargs=1) @ffx.command() def streams(filename): sd = getStreamDescriptor(filename) if sd is None: raise click.ClickException('This file does not contain any audiovisual data') for d in sd: click.echo(f"{d['codec']}{' (' + str(d['channels']) + ')' if d['type'] == 'audio' else ''}") @ffx.command() @click.pass_context @click.argument('paths', nargs=-1) @click.option('-l', '--label', type=str, default='', help='Label to be used as filename prefix') @click.option('-v', '--video-encoder', type=str, default=DEFAULT_VIDEO_ENCODER, help='Target video encoder (vp9 or av1) default: vp9') @click.option('-q', '--quality', type=str, default=DEFAULT_QUALITY, help='Quality settings to be used with VP9 encoder (default: 23)') @click.option('-p', '--preset', type=str, default=DEFAULT_QUALITY, help='Quality preset to be used with AV1 encoder (default: 5)') @click.option('-a', '--stereo-bitrate', type=int, default=DEFAULT_STEREO_BANDWIDTH, help='Bitrate in kbit/s to be used to encode stereo audio streams') @click.option('-ac3', '--ac3-bitrate', type=int, default=DEFAULT_AC3_BANDWIDTH, help='Bitrate in kbit/s to be used to encode 5.1 audio streams') @click.option('-dts', '--dts-bitrate', type=int, default=DEFAULT_DTS_BANDWIDTH, help='Bitrate in kbit/s to be used to encode 6.1 audio streams') @click.option('-ds', '--default-subtitle', type=int, help='Index of default subtitle stream') @click.option('-fa', '--forced-audio', type=int, help='Index of forced audio stream (including default audio stream tag)') @click.option('-da', '--default-audio', type=int, help='Index of default audio stream') @click.option("--crop", is_flag=False, flag_value="default", default="none") @click.option("-c", "--clear-metadata", is_flag=True, default=False) @click.option("-d", "--denoise", is_flag=True, default=False) @click.option("-o", "--output-directory", type=str, default='') def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, ac3_bitrate, dts_bitrate, crop, clear_metadata, default_subtitle, forced_audio, default_audio, denoise, output_directory): """Batch conversion of audiovideo files in format suitable for web playback, e.g. jellyfin Files found under PATHS will be converted according to parameters. Filename extensions will be changed appropriately. Suffices will we appended to filename in case of multiple created files or if the filename has not changed.""" startTime = time.perf_counter() context = ctx.obj click.echo(f"\nVideo encoder: {video_encoder}") qualityTokens = quality.split(',') q_list = [q for q in qualityTokens if q.isnumeric()] click.echo(f"Qualities: {q_list}") ctx.obj['bitrates'] = {} ctx.obj['bitrates']['stereo'] = str(stereo_bitrate) if str(stereo_bitrate).endswith('k') else f"{stereo_bitrate}k" ctx.obj['bitrates']['ac3'] = str(ac3_bitrate) if str(ac3_bitrate).endswith('k') else f"{ac3_bitrate}k" ctx.obj['bitrates']['dts'] = str(dts_bitrate) if str(dts_bitrate).endswith('k') else f"{dts_bitrate}k" click.echo(f"Stereo bitrate: {ctx.obj['bitrates']['stereo']}") click.echo(f"AC3 bitrate: {ctx.obj['bitrates']['ac3']}") click.echo(f"DTS bitrate: {ctx.obj['bitrates']['dts']}") ctx.obj['perform_crop'] = (crop != 'none') if ctx.obj['perform_crop']: cropTokens = crop.split(',') if cropTokens and len(cropTokens) == 2: ctx.obj['crop_start'], ctx.obj['crop_length'] = crop.split(',') else: ctx.obj['crop_start'] = DEFAULT_CROP_START ctx.obj['crop_length'] = DEFAULT_CROP_LENGTH click.echo(f"crop start={ctx.obj['crop_start']} length={ctx.obj['crop_length']}") click.echo(f"\nRunning {len(paths) * len(q_list)} jobs") job_index = 0 se_match = re.compile(SEASON_EPISODE_INDICATOR_MATCH) e_match = re.compile(EPISODE_INDICATOR_MATCH) for sourcePath in paths: if not os.path.isfile(sourcePath): click.echo(f"There is no file with path {sourcePath}, skipping ...") continue sourceDirectory = os.path.dirname(sourcePath) sourceFilename = os.path.basename(sourcePath) sourcePathTokens = sourceFilename.split('.') if sourcePathTokens[-1] in FILE_EXTENSIONS: sourceFileBasename = '.'.join(sourcePathTokens[:-1]) sourceFilenameExtension = sourcePathTokens[-1] else: sourceFileBasename = sourceFilename sourceFilenameExtension = '' #click.echo(f"dir={sourceDirectory} base={sourceFileBasename} ext={sourceFilenameExtension}") click.echo(f"\nProcessing file {sourcePath}") season_digits = 2 episode_digits = 2 index_digits = 3 se_result = se_match.search(sourceFilename) e_result = e_match.search(sourceFilename) season = -1 episode = -1 file_index = 0 if se_result is not None: season = int(se_result.group(1)) episode = int(se_result.group(2)) elif e_result is not None: episode = int(e_result.group(1)) else: file_index += 1 print(f"season={season} episode={episode} file={file_index}") targetFilenameTokens = [] targetFilenameExtension = DEFAULT_FILE_EXTENSION if label: targetFilenameTokens = [label] if season > -1 and episode > -1: targetFilenameTokens += [f"S{season:0{season_digits}d}E{episode:0{episode_digits}d}"] elif episode > -1: targetFilenameTokens += [f"E{episode:0{episode_digits}d}"] else: targetFilenameTokens += [f"{file_index:0{index_digits}d}"] else: targetFilenameTokens = [sourceFileBasename] # In case source and target filenames are the same add an extension to distinct output from input if sourceFilenameExtension == targetFilenameExtension: targetFilenameTokens += ['ffx'] targetFilename = '_'.join(targetFilenameTokens) + '.' + targetFilenameExtension click.echo(f"target filename: {targetFilename}") streamDescriptor = getStreamDescriptor(sourcePath) if streamDescriptor is None: click.echo(f"File with path {sourcePath} does not contain any audiovisual data, skipping ...") continue commandTokens = COMMAND_TOKENS + [sourcePath] for q in q_list: click.echo(f"\nRunning job {job_index} file={sourcePath} q={q}") job_index += 1 mappingVideoTokens = ['-map', 'v:0'] mappingTokens = mappingVideoTokens.copy() audioTokens = [] audioIndex = 0 for audioStreamDescriptor in streamDescriptor: if audioStreamDescriptor['type'] == STREAM_TYPE_AUDIO: mappingTokens += ['-map', f"a:{audioIndex}"] audioTokens += generateAudioTokens(ctx.obj, audioIndex, audioStreamDescriptor['layout']) audioIndex += 1 for s in range(len([d for d in streamDescriptor if d['type'] == STREAM_TYPE_SUBTITLE])): mappingTokens += ['-map', f"s:{s}"] if video_encoder == 'av1': commandSequence = commandTokens + mappingTokens + audioTokens + generateAV1Tokens(q, preset) + audioTokens if clear_metadata: commandSequence += generateClearTokens(streamDescriptor) if ctx.obj['perform_crop']: commandSequence += generateCropTokens(ctx.obj['crop_start'], ctx.obj['crop_length']) commandSequence += generateOutputTokens(targetFilename, DEFAULT_FILE_EXTENSION, q) click.echo(f"Command: {' '.join(commandSequence)}") # executeProcess(commandSequence) if video_encoder == 'vp9': commandSequence1 = commandTokens + mappingVideoTokens + generateVP9Pass1Tokens(q) if ctx.obj['perform_crop']: commandSequence1 += generateCropTokens(ctx.obj['crop_start'], ctx.obj['crop_length']) commandSequence1 += NULL_TOKENS click.echo(f"Command 1: {' '.join(commandSequence1)}") if os.path.exists(TEMP_FILE_NAME): os.remove(TEMP_FILE_NAME) # executeProcess(commandSequence1) commandSequence2 = commandTokens + mappingTokens if denoise: commandSequence2 += generateDenoiseTokens() commandSequence2 += generateVP9Pass2Tokens(q) + audioTokens if clear_metadata: commandSequence2 += generateClearTokens(streamDescriptor) if ctx.obj['perform_crop']: commandSequence2 += generateCropTokens(ctx.obj['crop_start'], ctx.obj['crop_length']) commandSequence2 += generateOutputTokens(targetFilename, DEFAULT_FILE_EXTENSION, q) click.echo(f"Command 2: {' '.join(commandSequence2)}") # executeProcess(commandSequence2) #app = ModesApp(ctx.obj) #app.run() #click.confirm('Warning! This file is not compliant to the defined source schema! Do you want to continue?', abort=True) click.echo('\nDONE\n') endTime = time.perf_counter() click.echo(f"Time elapsed {endTime - startTime}") # click.echo(f"app result: {app.getContext()}") if __name__ == '__main__': ffx()