diff --git a/bin/.ipynb_checkpoints/ffx-checkpoint.py b/bin/.ipynb_checkpoints/ffx-checkpoint.py new file mode 100755 index 0000000..3401df4 --- /dev/null +++ b/bin/.ipynb_checkpoints/ffx-checkpoint.py @@ -0,0 +1,444 @@ +#! /usr/bin/python3 + +import os, sys, subprocess, json, click, time + +VERSION='0.1.0' + +DEFAULT_VIDEO_ENCODER = 'vp9' + +DEFAULT_QUALITY = 23 + +DEFAULT_AV1_PRESET = 5 + +DEFAULT_LABEL='output' +DEFAULT_FILE_SUFFIX = 'webm' + +DEFAULT_STEREO_BANDWIDTH = "128" +DEFAULT_AC3_BANDWIDTH = "256" +DEFAULT_DTS_BANDWIDTH = "320" + +DEFAULT_CROP_START = 60 +DEFAULT_CROP_LENGTH = 180 + +TEMP_FILE_NAME = "ffmpeg2pass-0.log" + + +MKVMERGE_METADATA_KEYS = ['BPS', + 'NUMBER_OF_FRAMES', + 'NUMBER_OF_BYTES', + '_STATISTICS_WRITING_APP', + '_STATISTICS_WRITING_DATE_UTC', + '_STATISTICS_TAGS'] + +FILE_EXTENSION = ['mkv', 'mp4', 'avi', 'flv', 'webm'] + + +COMMAND_TOKENS = ['ffmpeg', '-y', '-i'] +NULL_TOKENS = ['-f', 'null', '/dev/null'] + +STREAM_TYPE_VIDEO = 'video' +STREAM_TYPE_AUDIO = 'audio' +STREAM_TYPE_SUBTITLE = 'subtitle' + +STREAM_LAYOUT_6_1 = '6.1' +STREAM_LAYOUT_5_1 = '5.1(side)' +STREAM_LAYOUT_STEREO = 'stereo' +STREAM_LAYOUT_6CH = '6ch' + + +def executeProcess(commandSequence): + + process = subprocess.Popen(commandSequence, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + output, error = process.communicate() + + return output + + + +#[{'index': 0, 'codec_name': 'vp9', 'codec_long_name': 'Google VP9', 'profile': 'Profile 0', 'codec_type': 'video', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'width': 1920, 'height': 1080, 'coded_width': 1920, 'coded_height': 1080, 'closed_captions': 0, 'film_grain': 0, 'has_b_frames': 0, 'sample_aspect_ratio': '1:1', 'display_aspect_ratio': '16:9', 'pix_fmt': 'yuv420p', 'level': -99, 'color_range': 'tv', 'chroma_location': 'left', 'field_order': 'progressive', 'refs': 1, 'r_frame_rate': '24000/1001', 'avg_frame_rate': '24000/1001', 'time_base': '1/1000', 'start_pts': 0, 'start_time': '0.000000', 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'BPS': '7974017', 'NUMBER_OF_FRAMES': '34382', 'NUMBER_OF_BYTES': '1429358655', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 libvpx-vp9', 'DURATION': '00:23:54.016000000'}}] +#[{'index': 1, 'codec_name': 'opus', 'codec_long_name': 'Opus (Opus Interactive Audio Codec)', 'codec_type': 'audio', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'sample_fmt': 'fltp', 'sample_rate': '48000', 'channels': 2, 'channel_layout': 'stereo', 'bits_per_sample': 0, 'initial_padding': 312, 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'extradata_size': 19, 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'jpn', 'title': 'Japanisch', 'BPS': '128000', 'NUMBER_OF_FRAMES': '61763', 'NUMBER_OF_BYTES': '22946145', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 libopus', 'DURATION': '00:23:54.141000000'}}] + +#[{'index': 2, 'codec_name': 'webvtt', 'codec_long_name': 'WebVTT subtitle', 'codec_type': 'subtitle', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'duration_ts': 1434141, 'duration': '1434.141000', 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'ger', 'title': 'Deutsch [Full]', 'BPS': '118', 'NUMBER_OF_FRAMES': '300', 'NUMBER_OF_BYTES': '21128', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 webvtt', 'DURATION': '00:23:54.010000000'}}, {'index': 3, 'codec_name': 'webvtt', 'codec_long_name': 'WebVTT subtitle', 'codec_type': 'subtitle', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'duration_ts': 1434141, 'duration': '1434.141000', 'disposition': {'default': 0, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'eng', 'title': 'Englisch [Full]', 'BPS': '101', 'NUMBER_OF_FRAMES': '276', 'NUMBER_OF_BYTES': '16980', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 webvtt', 'DURATION': '00:23:53.230000000'}}] + + + +def getStreamDescriptor(filename): + + ffprobeOutput = executeProcess(["ffprobe", + "-show_streams", + "-of", "json", + filename]) + + streamData = json.loads(ffprobeOutput)['streams'] + + descriptor = [] + + i = 0 + for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_VIDEO]: + descriptor.append({ + 'index': d['index'], + 'sub_index': i, + 'type': STREAM_TYPE_VIDEO, + 'codec': d['codec_name'] + }) + i += 1 + + i = 0 + for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_AUDIO]: + + streamDescriptor = { + 'index': d['index'], + 'sub_index': i, + 'type': STREAM_TYPE_AUDIO, + 'codec': d['codec_name'], + 'channels': d['channels'] + } + + if 'channel_layout' in d.keys(): + streamDescriptor['layout'] = d['channel_layout'] + elif d['channels'] == 6: + streamDescriptor['layout'] = STREAM_LAYOUT_6CH + else: + streamDescriptor['layout'] = 'undefined' + + descriptor.append(streamDescriptor) + i += 1 + + i = 0 + for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_SUBTITLE]: + descriptor.append({ + 'index': d['index'], + 'sub_index': i, + 'type': STREAM_TYPE_SUBTITLE, + 'codec': d['codec_name'] + }) + i += 1 + + return descriptor + + + +def generateAV1Tokens(q, p): + + return ['-c:v:0', 'libsvtav1', + '-svtav1-params', f"crf={q}:preset={p}:tune=0:enable-overlays=1:scd=1:scm=0", + '-pix_fmt', 'yuv420p10le'] + +def generateVP9Pass1Tokens(q): + + return ['-c:v:0', 'libvpx-vp9', + '-row-mt', '1', + '-crf', str(q), + '-pass', '1', + '-speed', '4', + '-frame-parallel', '0', + '-g', '9999', + '-aq-mode', '0'] + +def generateVP9Pass2Tokens(q): + + return ['-c:v:0', 'libvpx-vp9', + '-row-mt', '1', + '-crf', str(q), + '-pass', '2', + '-frame-parallel', '0', + '-g', '9999', + '-aq-mode', '0', + '-auto-alt-ref', '1', + '-lag-in-frames', '25'] + + +def generateCropTokens(start, length): + + return ['-ss', str(start), '-t', str(length)] + + +def generateDenoiseTokens(spatial=5, patch=7, research=7, hw=False): + filterName = 'nlmeans_opencl' if hw else 'nlmeans' + return ['-vf', f"{filterName}=s={spatial}:p={patch}:r={research}"] + + +def generateOutputTokens(f, suffix, q=None): + + if q is None: + return ['-f', 'webm', f"{f}.{suffix}"] + else: + return ['-f', 'webm', f"{f}_q{q}.{suffix}"] + + +# preset = DEFAULT_AV1_PRESET +# presetTokens = [p for p in sys.argv if p.startswith('p=')] +# if presetTokens: +# preset = int(presetTokens[0].split('=')[1]) + +# cropStart = '' +# cropLength = '' +# cropTokens = [c for c in sys.argv if c.startswith('crop')] +# if cropTokens: +# if '=' in cropTokens[0]: +# cropString = cropTokens[0].split('=')[1] +# cropStart, cropLength = cropString.split(',') +# else: +# cropStart = 60 +# cropLength = 180 +# +# denoiseTokens = [d for d in sys.argv if d.startswith('denoise')] +# + +# for aStream in audioStreams: +# if 'channel_layout' in aStream: +# print(f"audio stream: {aStream['channel_layout']}") #channel_layout +# else: +# print(f"unknown audio stream with {aStream['channels']} channels") #channel_layout + +def generateAudioTokens(context, index, layout): + + if layout == STREAM_LAYOUT_6_1: + return [f"-c:a:{index}", + 'libopus', + f"-filter:a:{index}", + 'channelmap=channel_layout=6.1', + f"-b:a:{index}", + context['bitrates']['dts']] + + elif layout == STREAM_LAYOUT_5_1: + return [f"-c:a:{index}", + 'libopus', + f"-filter:a:{index}", + "channelmap=FL-FL|FR-FR|FC-FC|LFE-LFE|SL-BL|SR-BR:5.1", + f"-b:a:{index}", + context['bitrates']['ac3']] + + elif layout == STREAM_LAYOUT_STEREO: + return [f"-c:a:{index}", + 'libopus', + f"-b:a:{index}", + context['bitrates']['stereo']] + + elif layout == STREAM_LAYOUT_6CH: + return [f"-c:a:{index}", + 'libopus', + f"-filter:a:{index}", + "channelmap=FL-FL|FR-FR|FC-FC|LFE-LFE|SL-BL|SR-BR:5.1", + f"-b:a:{index}", + context['bitrates']['ac3']] + else: + return [] + + +def generateClearTokens(streams): + clearTokens = [] + for s in streams: + for k in MKVMERGE_METADATA_KEYS: + clearTokens += [f"-metadata:s:{s['type'][0]}:{s['sub_index']}", f"{k}="] + return clearTokens + + +@click.group() +@click.pass_context +def ffx(ctx): + """FFX""" + ctx.obj = {} + pass + + +# Define a subcommand +@ffx.command() +def version(): + click.echo(VERSION) + + +# Another subcommand +@ffx.command() +def help(): + click.echo(f"ffx {VERSION}\n") + click.echo(f"Usage: ffx [input file] [output file] [vp9|av1] [q=[nn[,nn,...]]] [p=nn] [a=nnn[k]] [ac3=nnn[k]] [dts=nnn[k]] [crop]") + + +@click.argument('filename', nargs=1) +@ffx.command() +def streams(filename): + for d in getStreamDescriptor(filename): + click.echo(f"{d['codec']}{' (' + str(d['channels']) + ')' if d['type'] == 'audio' else ''}") + + + +@ffx.command() +@click.pass_context + +@click.argument('paths', nargs=-1) +@click.option('-l', '--label', type=str, default=DEFAULT_LABEL, help='Label to be used as filename prefix') + +@click.option('-v', '--video-encoder', type=str, default=DEFAULT_VIDEO_ENCODER, help='Target video encoder (vp9 or av1) default: vp9') + +@click.option('-q', '--quality', type=str, default=DEFAULT_QUALITY, help='Quality settings to be used with VP9 encoder (default: 23)') +@click.option('-p', '--preset', type=str, default=DEFAULT_QUALITY, help='Quality preset to be used with AV1 encoder (default: 5)') + +@click.option('-a', '--stereo-bitrate', type=int, default=DEFAULT_STEREO_BANDWIDTH, help='Bitrate in kbit/s to be used to encode stereo audio streams') +@click.option('-ac3', '--ac3-bitrate', type=int, default=DEFAULT_AC3_BANDWIDTH, help='Bitrate in kbit/s to be used to encode 5.1 audio streams') +@click.option('-dts', '--dts-bitrate', type=int, default=DEFAULT_DTS_BANDWIDTH, help='Bitrate in kbit/s to be used to encode 6.1 audio streams') + +@click.option('-ds', '--default-subtitle', type=int, help='Index of default subtitle stream') + +@click.option('-fa', '--forced-audio', type=int, help='Index of forced audio stream (including default audio stream tag)') +@click.option('-da', '--default-audio', type=int, help='Index of default audio stream') + + +@click.option("--crop", is_flag=False, flag_value="default", default="none") + +@click.option("-c", "--clear-metadata", is_flag=True, default=False) +@click.option("-d", "--denoise", is_flag=True, default=False) + + +def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, ac3_bitrate, dts_bitrate, crop, clear_metadata, default_subtitle, default_audio, denoise): + """Batch conversion of audiovideo files in format suitable for web playback, e.g. jellyfin + + Files found under PATHS will be converted according to parameters. + Filename extensions will be changed appropriately. + Suffices will we appended to filename in case of multiple created files + or if the filename has not changed.""" + + startTime = time.perf_counter() + + sourcePath = paths[0] + targetFilename = paths[1] + + if not os.path.isfile(sourcePath): + raise click.ClickException(f"There is no file with path {sourcePath}") + + click.echo(f"src: {sourcePath} tgt: {targetFilename}") + + + click.echo(f"ve={video_encoder}") + + + qualityTokens = quality.split(',') + + q_list = [q for q in qualityTokens if q.isnumeric()] + + click.echo(q_list) + + ctx.obj['bitrates'] = {} + ctx.obj['bitrates']['stereo'] = str(stereo_bitrate) if str(stereo_bitrate).endswith('k') else f"{stereo_bitrate}k" + ctx.obj['bitrates']['ac3'] = str(ac3_bitrate) if str(ac3_bitrate).endswith('k') else f"{ac3_bitrate}k" + ctx.obj['bitrates']['dts'] = str(dts_bitrate) if str(dts_bitrate).endswith('k') else f"{dts_bitrate}k" + + + click.echo(f"a={ctx.obj['bitrates']['stereo']}") + click.echo(f"ac3={ctx.obj['bitrates']['ac3']}") + click.echo(f"dts={ctx.obj['bitrates']['dts']}") + + + performCrop = (crop != 'none') + + + if performCrop: + + cropTokens = crop.split(',') + + if cropTokens and len(cropTokens) == 2: + + cropStart, cropLength = crop.split(',') + else: + cropStart = DEFAULT_CROP_START + cropLength = DEFAULT_CROP_LENGTH + + click.echo(f"crop start={cropStart} length={cropLength}") + + + + click.echo(f"\nRunning {len(q_list)} jobs") + + + streamDescriptor = getStreamDescriptor(sourcePath) + + commandTokens = COMMAND_TOKENS + [sourcePath] + + + for q in q_list: + + click.echo(f"\nRunning job q={q}") + + mappingVideoTokens = ['-map', 'v:0'] + mappingTokens = mappingVideoTokens.copy() + audioTokens = [] + + audioIndex = 0 + for audioStreamDescriptor in streamDescriptor: + + if audioStreamDescriptor['type'] == STREAM_TYPE_AUDIO: + + mappingTokens += ['-map', f"a:{audioIndex}"] + audioTokens += generateAudioTokens(ctx.obj, audioIndex, audioStreamDescriptor['layout']) + audioIndex += 1 + + + for s in range(len([d for d in streamDescriptor if d['type'] == STREAM_TYPE_SUBTITLE])): + mappingTokens += ['-map', f"s:{s}"] + + + + if video_encoder == 'av1': + + commandSequence = commandTokens + mappingTokens + audioTokens + generateAV1Tokens(q, preset) + audioTokens + + if clear_metadata: + commandSequence += generateClearTokens(streamDescriptor) + + if performCrop: + commandSequence += generateCropTokens(cropStart, cropLength) + + commandSequence += generateOutputTokens(targetFilename, DEFAULT_FILE_SUFFIX, q) + + click.echo(f"Command: {' '.join(commandSequence)}") + + executeProcess(commandSequence) + + + if video_encoder == 'vp9': + + commandSequence1 = commandTokens + mappingVideoTokens + generateVP9Pass1Tokens(q) + + if performCrop: + commandSequence1 += generateCropTokens(cropStart, cropLength) + + commandSequence1 += NULL_TOKENS + + click.echo(f"Command 1: {' '.join(commandSequence1)}") + + if os.path.exists(TEMP_FILE_NAME): + os.remove(TEMP_FILE_NAME) + + executeProcess(commandSequence1) + + + commandSequence2 = commandTokens + mappingTokens + + if denoise: + commandSequence2 += generateDenoiseTokens() + + commandSequence2 += generateVP9Pass2Tokens(q) + audioTokens + + if clear_metadata: + commandSequence2 += generateClearTokens(streamDescriptor) + + if performCrop: + commandSequence2 += generateCropTokens(cropStart, cropLength) + + commandSequence2 += generateOutputTokens(targetFilename, DEFAULT_FILE_SUFFIX, q) + + click.echo(f"Command 2: {' '.join(commandSequence2)}") + + executeProcess(commandSequence2) + + + click.echo('\nDONE\n') + + endTime = time.perf_counter() + click.echo(f"Time elapsed {endTime - startTime}") + + + +if __name__ == '__main__': + ffx() diff --git a/bin/ffx.py b/bin/ffx.py index 8607bda..3401df4 100755 --- a/bin/ffx.py +++ b/bin/ffx.py @@ -30,6 +30,8 @@ MKVMERGE_METADATA_KEYS = ['BPS', '_STATISTICS_WRITING_DATE_UTC', '_STATISTICS_TAGS'] +FILE_EXTENSION = ['mkv', 'mp4', 'avi', 'flv', 'webm'] + COMMAND_TOKENS = ['ffmpeg', '-y', '-i'] NULL_TOKENS = ['-f', 'null', '/dev/null']