diff --git a/bin/ffx.py b/bin/ffx.py index f20f89b..6ecd3b1 100755 --- a/bin/ffx.py +++ b/bin/ffx.py @@ -15,6 +15,7 @@ DEFAULT_QUALITY = 23 DEFAULT_AV1_PRESET = 5 +DEFAULT_FILE_FORMAT = 'webm' DEFAULT_FILE_EXTENSION = 'webm' DEFAULT_STEREO_BANDWIDTH = "128" @@ -126,8 +127,8 @@ def executeProcess(commandSequence): process = subprocess.Popen(commandSequence, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = process.communicate() - - return output.decode('utf-8'), error.decode('utf-8') + + return output.decode('utf-8'), error.decode('utf-8'), process.returncode @@ -137,64 +138,100 @@ def executeProcess(commandSequence): #[{'index': 2, 'codec_name': 'webvtt', 'codec_long_name': 'WebVTT subtitle', 'codec_type': 'subtitle', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'duration_ts': 1434141, 'duration': '1434.141000', 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'ger', 'title': 'Deutsch [Full]', 'BPS': '118', 'NUMBER_OF_FRAMES': '300', 'NUMBER_OF_BYTES': '21128', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 webvtt', 'DURATION': '00:23:54.010000000'}}, {'index': 3, 'codec_name': 'webvtt', 'codec_long_name': 'WebVTT subtitle', 'codec_type': 'subtitle', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'duration_ts': 1434141, 'duration': '1434.141000', 'disposition': {'default': 0, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'eng', 'title': 'Englisch [Full]', 'BPS': '101', 'NUMBER_OF_FRAMES': '276', 'NUMBER_OF_BYTES': '16980', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 webvtt', 'DURATION': '00:23:53.230000000'}}] +def getStreamData(filepath): + """Returns ffprobe stream data as array with elements according to the following example + { + "index": 4, + "codec_name": "hdmv_pgs_subtitle", + "codec_long_name": "HDMV Presentation Graphic Stream subtitles", + "codec_type": "subtitle", + "codec_tag_string": "[0][0][0][0]", + "codec_tag": "0x0000", + "r_frame_rate": "0/0", + "avg_frame_rate": "0/0", + "time_base": "1/1000", + "start_pts": 0, + "start_time": "0.000000", + "duration_ts": 1421035, + "duration": "1421.035000", + "disposition": { + "default": 1, + "dub": 0, + "original": 0, + "comment": 0, + "lyrics": 0, + "karaoke": 0, + "forced": 0, + "hearing_impaired": 0, + "visual_impaired": 0, + "clean_effects": 0, + "attached_pic": 0, + "timed_thumbnails": 0, + "non_diegetic": 0, + "captions": 0, + "descriptions": 0, + "metadata": 0, + "dependent": 0, + "still_image": 0 + }, + "tags": { + "language": "ger", + "title": "German Full" + } + } + """ -def getStreamDescriptor(filename): - - ffprobeOutput, ffprobeError = executeProcess(["ffprobe", + ffprobeOutput, ffprobeError, returnCode = executeProcess(["ffprobe", "-show_streams", "-of", "json", - filename]) + filepath]) if 'Invalid data found when processing input' in ffprobeError: - return None - - streamData = json.loads(ffprobeOutput)['streams'] - - descriptor = [] - - i = 0 - for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_VIDEO]: - descriptor.append({ - 'index': d['index'], - 'sub_index': i, - 'type': STREAM_TYPE_VIDEO, - 'codec': d['codec_name'] - }) - i += 1 - - i = 0 - for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_AUDIO]: - - streamDescriptor = { - 'index': d['index'], - 'sub_index': i, - 'type': STREAM_TYPE_AUDIO, - 'codec': d['codec_name'], - 'channels': d['channels'] - } + raise Exception(f"File {filepath} does not contain valid stream data") + + if returnCode != 0: + raise Exception(f"ffprobe returned with error {returnCode}") + + return json.loads(ffprobeOutput)['streams'] + + + +def getStreamDescriptor(filename): + + streamData = getStreamData(filename) - if 'channel_layout' in d.keys(): - streamDescriptor['layout'] = d['channel_layout'] - elif d['channels'] == 6: - streamDescriptor['layout'] = STREAM_LAYOUT_6CH + descriptor = {} + descriptor['video'] = [] + descriptor['audio'] = [] + descriptor['subtitle'] = [] + + for subStream in streamData: + + s = subStream.copy() + + #Defaulting to undefined if tag not defined for stream + if 'tags' in subStream.keys() and 'language' in subStream['tags'].keys(): + s['language'] = subStream['tags']['language'] else: - streamDescriptor['layout'] = 'undefined' + s['language'] = 'undefined' - descriptor.append(streamDescriptor) - i += 1 + #Defaulting to undefined if tag not defined for stream + if 'tags' in subStream.keys() and 'title' in subStream['tags'].keys(): + s['title'] = subStream['tags']['title'] + else: + s['title'] = 'undefined' - i = 0 - for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_SUBTITLE]: - descriptor.append({ - 'index': d['index'], - 'sub_index': i, - 'type': STREAM_TYPE_SUBTITLE, - 'codec': d['codec_name'] - }) - i += 1 + if subStream['codec_type'] == STREAM_TYPE_AUDIO: + if 'channel_layout' in subStream.keys(): + s['layout'] = subStream['channel_layout'] + elif subStream['channels'] == 6: + s['layout'] = STREAM_LAYOUT_6CH + else: + s['layout'] = 'undefined' - return descriptor + descriptor[s['codec_type']].append(s) + return descriptor def generateAV1Tokens(q, p): @@ -237,38 +274,9 @@ def generateDenoiseTokens(spatial=5, patch=7, research=7, hw=False): return ['-vf', f"{filterName}=s={spatial}:p={patch}:r={research}"] -def generateOutputTokens(f, suffix, q=None): +def generateOutputTokens(filepath, format, ext): + return ['-f', format, f"{filepath}.{ext}"] - if q is None: - return ['-f', 'webm', f"{f}.{suffix}"] - else: - return ['-f', 'webm', f"{f}_q{q}.{suffix}"] - - -# preset = DEFAULT_AV1_PRESET -# presetTokens = [p for p in sys.argv if p.startswith('p=')] -# if presetTokens: -# preset = int(presetTokens[0].split('=')[1]) - -# ctx.obj['crop_start'] = '' -# ctx.obj['crop_length'] = '' -# cropTokens = [c for c in sys.argv if c.startswith('crop')] -# if cropTokens: -# if '=' in cropTokens[0]: -# cropString = cropTokens[0].split('=')[1] -# ctx.obj['crop_start'], ctx.obj['crop_length'] = cropString.split(',') -# else: -# ctx.obj['crop_start'] = 60 -# ctx.obj['crop_length'] = 180 -# -# denoiseTokens = [d for d in sys.argv if d.startswith('denoise')] -# - -# for aStream in audioStreams: -# if 'channel_layout' in aStream: -# print(f"audio stream: {aStream['channel_layout']}") #channel_layout -# else: -# print(f"unknown audio stream with {aStream['channels']} channels") #channel_layout def generateAudioTokens(context, index, layout): @@ -337,11 +345,13 @@ def help(): @click.argument('filename', nargs=1) @ffx.command() def streams(filename): - sd = getStreamDescriptor(filename) - if sd is None: - raise click.ClickException('This file does not contain any audiovisual data') + + try: + sd = getStreamDescriptor(filename) + except Exception as ex: + raise click.ClickException(f"This file does not contain any audiovisual data: {ex}") for d in sd: - click.echo(f"{d['codec']}{' (' + str(d['channels']) + ')' if d['type'] == 'audio' else ''}") + click.echo(f"{d['codec_name']}{' (' + str(d['channels']) + ')' if d['codec_type'] == 'audio' else ''}") @@ -351,14 +361,14 @@ def streams(filename): @click.argument('paths', nargs=-1) @click.option('-l', '--label', type=str, default='', help='Label to be used as filename prefix') -@click.option('-v', '--video-encoder', type=str, default=DEFAULT_VIDEO_ENCODER, help='Target video encoder (vp9 or av1) default: vp9') +@click.option('-v', '--video-encoder', type=str, default=DEFAULT_VIDEO_ENCODER, help=f"Target video encoder (vp9 or av1) default: {DEFAULT_VIDEO_ENCODER}") -@click.option('-q', '--quality', type=str, default=DEFAULT_QUALITY, help='Quality settings to be used with VP9 encoder (default: 23)') -@click.option('-p', '--preset', type=str, default=DEFAULT_QUALITY, help='Quality preset to be used with AV1 encoder (default: 5)') +@click.option('-q', '--quality', type=str, default=DEFAULT_QUALITY, help=f"Quality settings to be used with VP9 encoder (default: {DEFAULT_QUALITY})") +@click.option('-p', '--preset', type=str, default=DEFAULT_AV1_PRESET, help=f"Quality preset to be used with AV1 encoder (default: {DEFAULT_AV1_PRESET})") -@click.option('-a', '--stereo-bitrate', type=int, default=DEFAULT_STEREO_BANDWIDTH, help='Bitrate in kbit/s to be used to encode stereo audio streams') -@click.option('-ac3', '--ac3-bitrate', type=int, default=DEFAULT_AC3_BANDWIDTH, help='Bitrate in kbit/s to be used to encode 5.1 audio streams') -@click.option('-dts', '--dts-bitrate', type=int, default=DEFAULT_DTS_BANDWIDTH, help='Bitrate in kbit/s to be used to encode 6.1 audio streams') +@click.option('-a', '--stereo-bitrate', type=int, default=DEFAULT_STEREO_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode stereo audio streams (default: {DEFAULT_STEREO_BANDWIDTH})") +@click.option('-ac3', '--ac3-bitrate', type=int, default=DEFAULT_AC3_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode 5.1 audio streams (default: {DEFAULT_AC3_BANDWIDTH})") +@click.option('-dts', '--dts-bitrate', type=int, default=DEFAULT_DTS_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode 6.1 audio streams (default: {DEFAULT_DTS_BANDWIDTH})") @click.option('-ds', '--default-subtitle', type=int, help='Index of default subtitle stream') @@ -368,13 +378,16 @@ def streams(filename): @click.option("--crop", is_flag=False, flag_value="default", default="none") +@click.option("-o", "--output-directory", type=str, default='') + + @click.option("-c", "--clear-metadata", is_flag=True, default=False) @click.option("-d", "--denoise", is_flag=True, default=False) -@click.option("-o", "--output-directory", type=str, default='') +@click.option("--dry-run", is_flag=True, default=False) -def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, ac3_bitrate, dts_bitrate, crop, clear_metadata, default_subtitle, forced_audio, default_audio, denoise, output_directory): +def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, ac3_bitrate, dts_bitrate, default_subtitle, forced_audio, default_audio, crop, output_directory, clear_metadata, denoise, dry_run): """Batch conversion of audiovideo files in format suitable for web playback, e.g. jellyfin Files found under PATHS will be converted according to parameters. @@ -395,32 +408,36 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a click.echo(f"Qualities: {q_list}") - ctx.obj['bitrates'] = {} - ctx.obj['bitrates']['stereo'] = str(stereo_bitrate) if str(stereo_bitrate).endswith('k') else f"{stereo_bitrate}k" - ctx.obj['bitrates']['ac3'] = str(ac3_bitrate) if str(ac3_bitrate).endswith('k') else f"{ac3_bitrate}k" - ctx.obj['bitrates']['dts'] = str(dts_bitrate) if str(dts_bitrate).endswith('k') else f"{dts_bitrate}k" + context['bitrates'] = {} + context['bitrates']['stereo'] = str(stereo_bitrate) if str(stereo_bitrate).endswith('k') else f"{stereo_bitrate}k" + context['bitrates']['ac3'] = str(ac3_bitrate) if str(ac3_bitrate).endswith('k') else f"{ac3_bitrate}k" + context['bitrates']['dts'] = str(dts_bitrate) if str(dts_bitrate).endswith('k') else f"{dts_bitrate}k" - click.echo(f"Stereo bitrate: {ctx.obj['bitrates']['stereo']}") - click.echo(f"AC3 bitrate: {ctx.obj['bitrates']['ac3']}") - click.echo(f"DTS bitrate: {ctx.obj['bitrates']['dts']}") + click.echo(f"Stereo bitrate: {context['bitrates']['stereo']}") + click.echo(f"AC3 bitrate: {context['bitrates']['ac3']}") + click.echo(f"DTS bitrate: {context['bitrates']['dts']}") - ctx.obj['perform_crop'] = (crop != 'none') + context['perform_crop'] = (crop != 'none') - if ctx.obj['perform_crop']: + if context['perform_crop']: cropTokens = crop.split(',') if cropTokens and len(cropTokens) == 2: - ctx.obj['crop_start'], ctx.obj['crop_length'] = crop.split(',') + context['crop_start'], context['crop_length'] = crop.split(',') else: - ctx.obj['crop_start'] = DEFAULT_CROP_START - ctx.obj['crop_length'] = DEFAULT_CROP_LENGTH + context['crop_start'] = DEFAULT_CROP_START + context['crop_length'] = DEFAULT_CROP_LENGTH + + click.echo(f"crop start={context['crop_start']} length={context['crop_length']}") - click.echo(f"crop start={ctx.obj['crop_start']} length={ctx.obj['crop_length']}") + existingSourcePaths = [p for p in paths if os.path.isfile(p)] + + + click.echo(f"\nRunning {len(existingSourcePaths) * len(q_list)} jobs") - click.echo(f"\nRunning {len(paths) * len(q_list)} jobs") job_index = 0 @@ -428,12 +445,7 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a e_match = re.compile(EPISODE_INDICATOR_MATCH) - for sourcePath in paths: - - - if not os.path.isfile(sourcePath): - click.echo(f"There is no file with path {sourcePath}, skipping ...") - continue + for sourcePath in existingSourcePaths: sourceDirectory = os.path.dirname(sourcePath) sourceFilename = os.path.basename(sourcePath) @@ -447,8 +459,6 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a sourceFilenameExtension = '' - #click.echo(f"dir={sourceDirectory} base={sourceFileBasename} ext={sourceFilenameExtension}") - click.echo(f"\nProcessing file {sourcePath}") season_digits = 2 @@ -474,11 +484,8 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a targetFilenameTokens = [] - - targetFilenameExtension = DEFAULT_FILE_EXTENSION - if label: targetFilenameTokens = [label] @@ -492,24 +499,23 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a else: targetFilenameTokens = [sourceFileBasename] - # In case source and target filenames are the same add an extension to distinct output from input - if sourceFilenameExtension == targetFilenameExtension: - targetFilenameTokens += ['ffx'] + try: + streamDescriptor = getStreamDescriptor(sourcePath) + except Exception: + click.echo(f"File with path {sourcePath} does not contain any audiovisual data, skipping ...") + continue - targetFilename = '_'.join(targetFilenameTokens) + '.' + targetFilenameExtension + for aStream in streamDescriptor[STREAM_TYPE_AUDIO]: + click.echo(f"audio stream lang={aStream['language']}") - click.echo(f"target filename: {targetFilename}") + for sStream in streamDescriptor[STREAM_TYPE_SUBTITLE]: + click.echo(f"subtitle stream lang={sStream['language']}") - streamDescriptor = getStreamDescriptor(sourcePath) - if streamDescriptor is None: - click.echo(f"File with path {sourcePath} does not contain any audiovisual data, skipping ...") - continue commandTokens = COMMAND_TOKENS + [sourcePath] - for q in q_list: click.echo(f"\nRunning job {job_index} file={sourcePath} q={q}") @@ -520,17 +526,30 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a audioTokens = [] audioIndex = 0 - for audioStreamDescriptor in streamDescriptor: - - if audioStreamDescriptor['type'] == STREAM_TYPE_AUDIO: + for audioStreamDescriptor in streamDescriptor[STREAM_TYPE_AUDIO]: - mappingTokens += ['-map', f"a:{audioIndex}"] - audioTokens += generateAudioTokens(ctx.obj, audioIndex, audioStreamDescriptor['layout']) - audioIndex += 1 + mappingTokens += ['-map', f"a:{audioIndex}"] + audioTokens += generateAudioTokens(context, audioIndex, audioStreamDescriptor['layout']) + audioIndex += 1 + subtitleIndex = 0 + for subtitleStreamDescriptor in streamDescriptor[STREAM_TYPE_SUBTITLE]: + mappingTokens += ['-map', f"s:{subtitleIndex}"] + subtitleIndex += 1 - for s in range(len([d for d in streamDescriptor if d['type'] == STREAM_TYPE_SUBTITLE])): - mappingTokens += ['-map', f"s:{s}"] + + targetFilenameJobTokens = targetFilenameTokens.copy() + + if len(q_list) > 1: + targetFilenameJobTokens += [f"q{q}"] + + # In case source and target filenames are the same add an extension to distinct output from input + if not label and sourceFilenameExtension == targetFilenameExtension: + targetFilenameJobTokens += ['ffx'] + + targetFilename = '_'.join(targetFilenameJobTokens) # + '.' + targetFilenameExtension + + click.echo(f"target filename: {targetFilename}") @@ -541,22 +560,23 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a if clear_metadata: commandSequence += generateClearTokens(streamDescriptor) - if ctx.obj['perform_crop']: - commandSequence += generateCropTokens(ctx.obj['crop_start'], ctx.obj['crop_length']) + if context['perform_crop']: + commandSequence += generateCropTokens(context['crop_start'], context['crop_length']) - commandSequence += generateOutputTokens(targetFilename, DEFAULT_FILE_EXTENSION, q) + commandSequence += generateOutputTokens(targetFilename, DEFAULT_FILE_FORMAT, DEFAULT_FILE_EXTENSION) click.echo(f"Command: {' '.join(commandSequence)}") - # executeProcess(commandSequence) + if not dry_run: + executeProcess(commandSequence) if video_encoder == 'vp9': commandSequence1 = commandTokens + mappingVideoTokens + generateVP9Pass1Tokens(q) - if ctx.obj['perform_crop']: - commandSequence1 += generateCropTokens(ctx.obj['crop_start'], ctx.obj['crop_length']) + if context['perform_crop']: + commandSequence1 += generateCropTokens(context['crop_start'], context['crop_length']) commandSequence1 += NULL_TOKENS @@ -565,7 +585,8 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a if os.path.exists(TEMP_FILE_NAME): os.remove(TEMP_FILE_NAME) - # executeProcess(commandSequence1) + if not dry_run: + executeProcess(commandSequence1) commandSequence2 = commandTokens + mappingTokens @@ -578,17 +599,18 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a if clear_metadata: commandSequence2 += generateClearTokens(streamDescriptor) - if ctx.obj['perform_crop']: - commandSequence2 += generateCropTokens(ctx.obj['crop_start'], ctx.obj['crop_length']) + if context['perform_crop']: + commandSequence2 += generateCropTokens(context['crop_start'], context['crop_length']) - commandSequence2 += generateOutputTokens(targetFilename, DEFAULT_FILE_EXTENSION, q) + commandSequence2 += generateOutputTokens(targetFilename, DEFAULT_FILE_FORMAT, DEFAULT_FILE_EXTENSION) click.echo(f"Command 2: {' '.join(commandSequence2)}") - - # executeProcess(commandSequence2) + + if not dry_run: + executeProcess(commandSequence2) - #app = ModesApp(ctx.obj) + #app = ModesApp(context) #app.run() #click.confirm('Warning! This file is not compliant to the defined source schema! Do you want to continue?', abort=True)