Javanaut/ffx
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

impl multifile handling

Browse Source
This commit is contained in:
Maveno
2024-09-14 09:25:03 +02:00
parent ad7562f387
commit 4d7f728f25
1 changed files with 173 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

312
bin/ffx.py
View File

@@ -15,6 +15,7 @@ DEFAULT_QUALITY = 23
DEFAULT_AV1_PRESET = 5
DEFAULT_FILE_FORMAT = 'webm'
DEFAULT_FILE_EXTENSION = 'webm'
DEFAULT_STEREO_BANDWIDTH = "128"
@@ -127,7 +128,7 @@ def executeProcess(commandSequence):
output, error = process.communicate()
return output.decode('utf-8'), error.decode('utf-8')
return output.decode('utf-8'), error.decode('utf-8'), process.returncode
@@ -137,66 +138,102 @@ def executeProcess(commandSequence):
#[{'index': 2, 'codec_name': 'webvtt', 'codec_long_name': 'WebVTT subtitle', 'codec_type': 'subtitle', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'duration_ts': 1434141, 'duration': '1434.141000', 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'ger', 'title': 'Deutsch [Full]', 'BPS': '118', 'NUMBER_OF_FRAMES': '300', 'NUMBER_OF_BYTES': '21128', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 webvtt', 'DURATION': '00:23:54.010000000'}}, {'index': 3, 'codec_name': 'webvtt', 'codec_long_name': 'WebVTT subtitle', 'codec_type': 'subtitle', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'duration_ts': 1434141, 'duration': '1434.141000', 'disposition': {'default': 0, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'eng', 'title': 'Englisch [Full]', 'BPS': '101', 'NUMBER_OF_FRAMES': '276', 'NUMBER_OF_BYTES': '16980', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 webvtt', 'DURATION': '00:23:53.230000000'}}]
def getStreamData(filepath):
"""Returns ffprobe stream data as array with elements according to the following example
{
"index": 4,
"codec_name": "hdmv_pgs_subtitle",
"codec_long_name": "HDMV Presentation Graphic Stream subtitles",
"codec_type": "subtitle",
"codec_tag_string": "[0][0][0][0]",
"codec_tag": "0x0000",
"r_frame_rate": "0/0",
"avg_frame_rate": "0/0",
"time_base": "1/1000",
"start_pts": 0,
"start_time": "0.000000",
"duration_ts": 1421035,
"duration": "1421.035000",
"disposition": {
"default": 1,
"dub": 0,
"original": 0,
"comment": 0,
"lyrics": 0,
"karaoke": 0,
"forced": 0,
"hearing_impaired": 0,
"visual_impaired": 0,
"clean_effects": 0,
"attached_pic": 0,
"timed_thumbnails": 0,
"non_diegetic": 0,
"captions": 0,
"descriptions": 0,
"metadata": 0,
"dependent": 0,
"still_image": 0
},
"tags": {
"language": "ger",
"title": "German Full"
}
}
"""
ffprobeOutput, ffprobeError, returnCode = executeProcess(["ffprobe",
"-show_streams",
"-of", "json",
filepath])
if 'Invalid data found when processing input' in ffprobeError:
raise Exception(f"File {filepath} does not contain valid stream data")
if returnCode != 0:
raise Exception(f"ffprobe returned with error {returnCode}")
return json.loads(ffprobeOutput)['streams']
def getStreamDescriptor(filename):
ffprobeOutput, ffprobeError = executeProcess(["ffprobe",
"-show_streams",
"-of", "json",
filename])
streamData = getStreamData(filename)
if 'Invalid data found when processing input' in ffprobeError:
return None
descriptor = {}
descriptor['video'] = []
descriptor['audio'] = []
descriptor['subtitle'] = []
streamData = json.loads(ffprobeOutput)['streams']
for subStream in streamData:
descriptor = []
s = subStream.copy()
i = 0
for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_VIDEO]:
descriptor.append({
'index': d['index'],
'sub_index': i,
'type': STREAM_TYPE_VIDEO,
'codec': d['codec_name']
})
i += 1
i = 0
for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_AUDIO]:
streamDescriptor = {
'index': d['index'],
'sub_index': i,
'type': STREAM_TYPE_AUDIO,
'codec': d['codec_name'],
'channels': d['channels']
}
if 'channel_layout' in d.keys():
streamDescriptor['layout'] = d['channel_layout']
elif d['channels'] == 6:
streamDescriptor['layout'] = STREAM_LAYOUT_6CH
#Defaulting to undefined if tag not defined for stream
if 'tags' in subStream.keys() and 'language' in subStream['tags'].keys():
s['language'] = subStream['tags']['language']
else:
streamDescriptor['layout'] = 'undefined'
s['language'] = 'undefined'
descriptor.append(streamDescriptor)
i += 1
#Defaulting to undefined if tag not defined for stream
if 'tags' in subStream.keys() and 'title' in subStream['tags'].keys():
s['title'] = subStream['tags']['title']
else:
s['title'] = 'undefined'
i = 0
for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_SUBTITLE]:
descriptor.append({
'index': d['index'],
'sub_index': i,
'type': STREAM_TYPE_SUBTITLE,
'codec': d['codec_name']
})
i += 1
if subStream['codec_type'] == STREAM_TYPE_AUDIO:
if 'channel_layout' in subStream.keys():
s['layout'] = subStream['channel_layout']
elif subStream['channels'] == 6:
s['layout'] = STREAM_LAYOUT_6CH
else:
s['layout'] = 'undefined'
descriptor[s['codec_type']].append(s)
return descriptor
def generateAV1Tokens(q, p):
return ['-c:v:0', 'libsvtav1',
@@ -237,38 +274,9 @@ def generateDenoiseTokens(spatial=5, patch=7, research=7, hw=False):
return ['-vf', f"{filterName}=s={spatial}:p={patch}:r={research}"]
def generateOutputTokens(f, suffix, q=None):
def generateOutputTokens(filepath, format, ext):
return ['-f', format, f"{filepath}.{ext}"]
if q is None:
return ['-f', 'webm', f"{f}.{suffix}"]
else:
return ['-f', 'webm', f"{f}_q{q}.{suffix}"]
# preset = DEFAULT_AV1_PRESET
# presetTokens = [p for p in sys.argv if p.startswith('p=')]
# if presetTokens:
# preset = int(presetTokens[0].split('=')[1])
# ctx.obj['crop_start'] = ''
# ctx.obj['crop_length'] = ''
# cropTokens = [c for c in sys.argv if c.startswith('crop')]
# if cropTokens:
# if '=' in cropTokens[0]:
# cropString = cropTokens[0].split('=')[1]
# ctx.obj['crop_start'], ctx.obj['crop_length'] = cropString.split(',')
# else:
# ctx.obj['crop_start'] = 60
# ctx.obj['crop_length'] = 180
#
# denoiseTokens = [d for d in sys.argv if d.startswith('denoise')]
#
# for aStream in audioStreams:
# if 'channel_layout' in aStream:
# print(f"audio stream: {aStream['channel_layout']}") #channel_layout
# else:
# print(f"unknown audio stream with {aStream['channels']} channels") #channel_layout
def generateAudioTokens(context, index, layout):
@@ -337,11 +345,13 @@ def help():
@click.argument('filename', nargs=1)
@ffx.command()
def streams(filename):
try:
sd = getStreamDescriptor(filename)
if sd is None:
raise click.ClickException('This file does not contain any audiovisual data')
except Exception as ex:
raise click.ClickException(f"This file does not contain any audiovisual data: {ex}")
for d in sd:
click.echo(f"{d['codec']}{' (' + str(d['channels']) + ')' if d['type'] == 'audio' else ''}")
click.echo(f"{d['codec_name']}{' (' + str(d['channels']) + ')' if d['codec_type'] == 'audio' else ''}")
@@ -351,14 +361,14 @@ def streams(filename):
@click.argument('paths', nargs=-1)
@click.option('-l', '--label', type=str, default='', help='Label to be used as filename prefix')
@click.option('-v', '--video-encoder', type=str, default=DEFAULT_VIDEO_ENCODER, help='Target video encoder (vp9 or av1) default: vp9')
@click.option('-v', '--video-encoder', type=str, default=DEFAULT_VIDEO_ENCODER, help=f"Target video encoder (vp9 or av1) default: {DEFAULT_VIDEO_ENCODER}")
@click.option('-q', '--quality', type=str, default=DEFAULT_QUALITY, help='Quality settings to be used with VP9 encoder (default: 23)')
@click.option('-p', '--preset', type=str, default=DEFAULT_QUALITY, help='Quality preset to be used with AV1 encoder (default: 5)')
@click.option('-q', '--quality', type=str, default=DEFAULT_QUALITY, help=f"Quality settings to be used with VP9 encoder (default: {DEFAULT_QUALITY})")
@click.option('-p', '--preset', type=str, default=DEFAULT_AV1_PRESET, help=f"Quality preset to be used with AV1 encoder (default: {DEFAULT_AV1_PRESET})")
@click.option('-a', '--stereo-bitrate', type=int, default=DEFAULT_STEREO_BANDWIDTH, help='Bitrate in kbit/s to be used to encode stereo audio streams')
@click.option('-ac3', '--ac3-bitrate', type=int, default=DEFAULT_AC3_BANDWIDTH, help='Bitrate in kbit/s to be used to encode 5.1 audio streams')
@click.option('-dts', '--dts-bitrate', type=int, default=DEFAULT_DTS_BANDWIDTH, help='Bitrate in kbit/s to be used to encode 6.1 audio streams')
@click.option('-a', '--stereo-bitrate', type=int, default=DEFAULT_STEREO_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode stereo audio streams (default: {DEFAULT_STEREO_BANDWIDTH})")
@click.option('-ac3', '--ac3-bitrate', type=int, default=DEFAULT_AC3_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode 5.1 audio streams (default: {DEFAULT_AC3_BANDWIDTH})")
@click.option('-dts', '--dts-bitrate', type=int, default=DEFAULT_DTS_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode 6.1 audio streams (default: {DEFAULT_DTS_BANDWIDTH})")
@click.option('-ds', '--default-subtitle', type=int, help='Index of default subtitle stream')
@@ -368,13 +378,16 @@ def streams(filename):
@click.option("--crop", is_flag=False, flag_value="default", default="none")
@click.option("-c", "--clear-metadata", is_flag=True, default=False)
@click.option("-d", "--denoise", is_flag=True, default=False)
@click.option("-o", "--output-directory", type=str, default='')
def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, ac3_bitrate, dts_bitrate, crop, clear_metadata, default_subtitle, forced_audio, default_audio, denoise, output_directory):
@click.option("-c", "--clear-metadata", is_flag=True, default=False)
@click.option("-d", "--denoise", is_flag=True, default=False)
@click.option("--dry-run", is_flag=True, default=False)
def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, ac3_bitrate, dts_bitrate, default_subtitle, forced_audio, default_audio, crop, output_directory, clear_metadata, denoise, dry_run):
"""Batch conversion of audiovideo files in format suitable for web playback, e.g. jellyfin
Files found under PATHS will be converted according to parameters.
@@ -395,32 +408,36 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
click.echo(f"Qualities: {q_list}")
ctx.obj['bitrates'] = {}
ctx.obj['bitrates']['stereo'] = str(stereo_bitrate) if str(stereo_bitrate).endswith('k') else f"{stereo_bitrate}k"
ctx.obj['bitrates']['ac3'] = str(ac3_bitrate) if str(ac3_bitrate).endswith('k') else f"{ac3_bitrate}k"
ctx.obj['bitrates']['dts'] = str(dts_bitrate) if str(dts_bitrate).endswith('k') else f"{dts_bitrate}k"
context['bitrates'] = {}
context['bitrates']['stereo'] = str(stereo_bitrate) if str(stereo_bitrate).endswith('k') else f"{stereo_bitrate}k"
context['bitrates']['ac3'] = str(ac3_bitrate) if str(ac3_bitrate).endswith('k') else f"{ac3_bitrate}k"
context['bitrates']['dts'] = str(dts_bitrate) if str(dts_bitrate).endswith('k') else f"{dts_bitrate}k"
click.echo(f"Stereo bitrate: {ctx.obj['bitrates']['stereo']}")
click.echo(f"AC3 bitrate: {ctx.obj['bitrates']['ac3']}")
click.echo(f"DTS bitrate: {ctx.obj['bitrates']['dts']}")
click.echo(f"Stereo bitrate: {context['bitrates']['stereo']}")
click.echo(f"AC3 bitrate: {context['bitrates']['ac3']}")
click.echo(f"DTS bitrate: {context['bitrates']['dts']}")
ctx.obj['perform_crop'] = (crop != 'none')
context['perform_crop'] = (crop != 'none')
if ctx.obj['perform_crop']:
if context['perform_crop']:
cropTokens = crop.split(',')
if cropTokens and len(cropTokens) == 2:
ctx.obj['crop_start'], ctx.obj['crop_length'] = crop.split(',')
context['crop_start'], context['crop_length'] = crop.split(',')
else:
ctx.obj['crop_start'] = DEFAULT_CROP_START
ctx.obj['crop_length'] = DEFAULT_CROP_LENGTH
context['crop_start'] = DEFAULT_CROP_START
context['crop_length'] = DEFAULT_CROP_LENGTH
click.echo(f"crop start={ctx.obj['crop_start']} length={ctx.obj['crop_length']}")
click.echo(f"crop start={context['crop_start']} length={context['crop_length']}")
click.echo(f"\nRunning {len(paths) * len(q_list)} jobs")
existingSourcePaths = [p for p in paths if os.path.isfile(p)]
click.echo(f"\nRunning {len(existingSourcePaths) * len(q_list)} jobs")
job_index = 0
@@ -428,12 +445,7 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
e_match = re.compile(EPISODE_INDICATOR_MATCH)
for sourcePath in paths:
if not os.path.isfile(sourcePath):
click.echo(f"There is no file with path {sourcePath}, skipping ...")
continue
for sourcePath in existingSourcePaths:
sourceDirectory = os.path.dirname(sourcePath)
sourceFilename = os.path.basename(sourcePath)
@@ -447,8 +459,6 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
sourceFilenameExtension = ''
#click.echo(f"dir={sourceDirectory} base={sourceFileBasename} ext={sourceFilenameExtension}")
click.echo(f"\nProcessing file {sourcePath}")
season_digits = 2
@@ -474,11 +484,8 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
targetFilenameTokens = []
targetFilenameExtension = DEFAULT_FILE_EXTENSION
if label:
targetFilenameTokens = [label]
@@ -492,23 +499,22 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
else:
targetFilenameTokens = [sourceFileBasename]
# In case source and target filenames are the same add an extension to distinct output from input
if sourceFilenameExtension == targetFilenameExtension:
targetFilenameTokens += ['ffx']
targetFilename = '_'.join(targetFilenameTokens) + '.' + targetFilenameExtension
click.echo(f"target filename: {targetFilename}")
try:
streamDescriptor = getStreamDescriptor(sourcePath)
if streamDescriptor is None:
except Exception:
click.echo(f"File with path {sourcePath} does not contain any audiovisual data, skipping ...")
continue
commandTokens = COMMAND_TOKENS + [sourcePath]
for aStream in streamDescriptor[STREAM_TYPE_AUDIO]:
click.echo(f"audio stream lang={aStream['language']}")
for sStream in streamDescriptor[STREAM_TYPE_SUBTITLE]:
click.echo(f"subtitle stream lang={sStream['language']}")
commandTokens = COMMAND_TOKENS + [sourcePath]
for q in q_list:
@@ -520,17 +526,30 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
audioTokens = []
audioIndex = 0
for audioStreamDescriptor in streamDescriptor:
if audioStreamDescriptor['type'] == STREAM_TYPE_AUDIO:
for audioStreamDescriptor in streamDescriptor[STREAM_TYPE_AUDIO]:
mappingTokens += ['-map', f"a:{audioIndex}"]
audioTokens += generateAudioTokens(ctx.obj, audioIndex, audioStreamDescriptor['layout'])
audioTokens += generateAudioTokens(context, audioIndex, audioStreamDescriptor['layout'])
audioIndex += 1
subtitleIndex = 0
for subtitleStreamDescriptor in streamDescriptor[STREAM_TYPE_SUBTITLE]:
mappingTokens += ['-map', f"s:{subtitleIndex}"]
subtitleIndex += 1
for s in range(len([d for d in streamDescriptor if d['type'] == STREAM_TYPE_SUBTITLE])):
mappingTokens += ['-map', f"s:{s}"]
targetFilenameJobTokens = targetFilenameTokens.copy()
if len(q_list) > 1:
targetFilenameJobTokens += [f"q{q}"]
# In case source and target filenames are the same add an extension to distinct output from input
if not label and sourceFilenameExtension == targetFilenameExtension:
targetFilenameJobTokens += ['ffx']
targetFilename = '_'.join(targetFilenameJobTokens) # + '.' + targetFilenameExtension
click.echo(f"target filename: {targetFilename}")
@@ -541,22 +560,23 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
if clear_metadata:
commandSequence += generateClearTokens(streamDescriptor)
if ctx.obj['perform_crop']:
commandSequence += generateCropTokens(ctx.obj['crop_start'], ctx.obj['crop_length'])
if context['perform_crop']:
commandSequence += generateCropTokens(context['crop_start'], context['crop_length'])
commandSequence += generateOutputTokens(targetFilename, DEFAULT_FILE_EXTENSION, q)
commandSequence += generateOutputTokens(targetFilename, DEFAULT_FILE_FORMAT, DEFAULT_FILE_EXTENSION)
click.echo(f"Command: {' '.join(commandSequence)}")
# executeProcess(commandSequence)
if not dry_run:
executeProcess(commandSequence)
if video_encoder == 'vp9':
commandSequence1 = commandTokens + mappingVideoTokens + generateVP9Pass1Tokens(q)
if ctx.obj['perform_crop']:
commandSequence1 += generateCropTokens(ctx.obj['crop_start'], ctx.obj['crop_length'])
if context['perform_crop']:
commandSequence1 += generateCropTokens(context['crop_start'], context['crop_length'])
commandSequence1 += NULL_TOKENS
@@ -565,7 +585,8 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
if os.path.exists(TEMP_FILE_NAME):
os.remove(TEMP_FILE_NAME)
# executeProcess(commandSequence1)
if not dry_run:
executeProcess(commandSequence1)
commandSequence2 = commandTokens + mappingTokens
@@ -578,17 +599,18 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
if clear_metadata:
commandSequence2 += generateClearTokens(streamDescriptor)
if ctx.obj['perform_crop']:
commandSequence2 += generateCropTokens(ctx.obj['crop_start'], ctx.obj['crop_length'])
if context['perform_crop']:
commandSequence2 += generateCropTokens(context['crop_start'], context['crop_length'])
commandSequence2 += generateOutputTokens(targetFilename, DEFAULT_FILE_EXTENSION, q)
commandSequence2 += generateOutputTokens(targetFilename, DEFAULT_FILE_FORMAT, DEFAULT_FILE_EXTENSION)
click.echo(f"Command 2: {' '.join(commandSequence2)}")
# executeProcess(commandSequence2)
if not dry_run:
executeProcess(commandSequence2)
#app = ModesApp(ctx.obj)
#app = ModesApp(context)
#app.run()
#click.confirm('Warning! This file is not compliant to the defined source schema! Do you want to continue?', abort=True)