impl multifile handling

bin/ffx.py

@@ -15,6 +15,7 @@ DEFAULT_QUALITY = 23
 
 DEFAULT_AV1_PRESET = 5
 
+DEFAULT_FILE_FORMAT = 'webm'
 DEFAULT_FILE_EXTENSION = 'webm'
 
 DEFAULT_STEREO_BANDWIDTH = "128"
@@ -127,7 +128,7 @@ def executeProcess(commandSequence):
 
     output, error = process.communicate()
     
-    return output.decode('utf-8'), error.decode('utf-8')
+    return output.decode('utf-8'), error.decode('utf-8'), process.returncode
 
 
 
@@ -137,66 +138,102 @@ def executeProcess(commandSequence):
 #[{'index': 2, 'codec_name': 'webvtt', 'codec_long_name': 'WebVTT subtitle', 'codec_type': 'subtitle', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'duration_ts': 1434141, 'duration': '1434.141000', 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'ger', 'title': 'Deutsch [Full]', 'BPS': '118', 'NUMBER_OF_FRAMES': '300', 'NUMBER_OF_BYTES': '21128', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 webvtt', 'DURATION': '00:23:54.010000000'}}, {'index': 3, 'codec_name': 'webvtt', 'codec_long_name': 'WebVTT subtitle', 'codec_type': 'subtitle', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'duration_ts': 1434141, 'duration': '1434.141000', 'disposition': {'default': 0, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'eng', 'title': 'Englisch [Full]', 'BPS': '101', 'NUMBER_OF_FRAMES': '276', 'NUMBER_OF_BYTES': '16980', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 webvtt', 'DURATION': '00:23:53.230000000'}}]
 
 
+def getStreamData(filepath):
+    """Returns ffprobe stream data as array with elements according to the following example      
+    {
+        "index": 4,
+        "codec_name": "hdmv_pgs_subtitle",
+        "codec_long_name": "HDMV Presentation Graphic Stream subtitles",
+        "codec_type": "subtitle",
+        "codec_tag_string": "[0][0][0][0]",
+        "codec_tag": "0x0000",
+        "r_frame_rate": "0/0",
+        "avg_frame_rate": "0/0",
+        "time_base": "1/1000",
+        "start_pts": 0,
+        "start_time": "0.000000",
+        "duration_ts": 1421035,
+        "duration": "1421.035000",
+        "disposition": {
+            "default": 1,
+            "dub": 0,
+            "original": 0,
+            "comment": 0,
+            "lyrics": 0,
+            "karaoke": 0,
+            "forced": 0,
+            "hearing_impaired": 0,
+            "visual_impaired": 0,
+            "clean_effects": 0,
+            "attached_pic": 0,
+            "timed_thumbnails": 0,
+            "non_diegetic": 0,
+            "captions": 0,
+            "descriptions": 0,
+            "metadata": 0,
+            "dependent": 0,
+            "still_image": 0
+        },
+        "tags": {
+            "language": "ger",
+            "title": "German Full"
+        }
+    }
+    """
+
+    ffprobeOutput, ffprobeError, returnCode = executeProcess(["ffprobe",
+                                                    "-show_streams",
+                                                    "-of", "json",
+                                                    filepath])
+
+    if 'Invalid data found when processing input' in ffprobeError:
+        raise Exception(f"File {filepath} does not contain valid stream data")
+    
+    if returnCode != 0:
+        raise Exception(f"ffprobe returned with error {returnCode}")
+
+    return json.loads(ffprobeOutput)['streams']
+
+
 
 def getStreamDescriptor(filename):
 
-    ffprobeOutput, ffprobeError = executeProcess(["ffprobe",
+    streamData = getStreamData(filename)
-                                                    "-show_streams",
-                                                    "-of", "json",
-                                                    filename])
 
-    if 'Invalid data found when processing input' in ffprobeError:
+    descriptor = {}
-        return None
+    descriptor['video'] = []
+    descriptor['audio'] = []
+    descriptor['subtitle'] = []
 
-    streamData = json.loads(ffprobeOutput)['streams']
+    for subStream in streamData:
 
-    descriptor = []
+        s = subStream.copy()
 
-    i = 0
+        #Defaulting to undefined if tag not defined for stream
-    for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_VIDEO]:
+        if 'tags' in subStream.keys() and 'language' in subStream['tags'].keys():
-        descriptor.append({
+            s['language'] = subStream['tags']['language']
-            'index': d['index'],
-            'sub_index': i,
-            'type': STREAM_TYPE_VIDEO,
-            'codec': d['codec_name']
-        })
-        i += 1
-
-    i = 0
-    for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_AUDIO]:
-
-        streamDescriptor = {
-            'index': d['index'],
-            'sub_index': i,
-            'type': STREAM_TYPE_AUDIO,
-            'codec': d['codec_name'],
-            'channels': d['channels']
-        }
-
-        if 'channel_layout' in d.keys():
-            streamDescriptor['layout'] = d['channel_layout']
-        elif d['channels'] == 6:
-            streamDescriptor['layout'] = STREAM_LAYOUT_6CH
         else:
-            streamDescriptor['layout'] = 'undefined'
+            s['language'] = 'undefined'
 
-        descriptor.append(streamDescriptor)
+        #Defaulting to undefined if tag not defined for stream
-        i += 1
+        if 'tags' in subStream.keys() and 'title' in subStream['tags'].keys():
+            s['title'] = subStream['tags']['title']
+        else:
+            s['title'] = 'undefined'
 
-    i = 0
+        if subStream['codec_type'] == STREAM_TYPE_AUDIO:
-    for d in [s for s in streamData if s['codec_type'] == STREAM_TYPE_SUBTITLE]:
+            if 'channel_layout' in subStream.keys():
-        descriptor.append({
+                s['layout'] = subStream['channel_layout']
-            'index': d['index'],
+            elif subStream['channels'] == 6:
-            'sub_index': i,
+                s['layout'] = STREAM_LAYOUT_6CH
-            'type': STREAM_TYPE_SUBTITLE,
+            else:
-            'codec': d['codec_name']
+                s['layout'] = 'undefined'
-        })
+
-        i += 1
+        descriptor[s['codec_type']].append(s)
 
     return descriptor
 
 
-
 def generateAV1Tokens(q, p):
 
     return ['-c:v:0', 'libsvtav1',
@@ -237,38 +274,9 @@ def generateDenoiseTokens(spatial=5, patch=7, research=7, hw=False):
     return ['-vf', f"{filterName}=s={spatial}:p={patch}:r={research}"]
 
 
-def generateOutputTokens(f, suffix, q=None):
+def generateOutputTokens(filepath, format, ext):
+    return ['-f', format, f"{filepath}.{ext}"]
 
-    if q is None:
-        return ['-f', 'webm', f"{f}.{suffix}"]
-    else:        
-        return ['-f', 'webm', f"{f}_q{q}.{suffix}"]
-
-
-# preset = DEFAULT_AV1_PRESET
-# presetTokens = [p for p in sys.argv if p.startswith('p=')]
-# if presetTokens:
-#   preset = int(presetTokens[0].split('=')[1])
-
-# ctx.obj['crop_start'] = ''
-# ctx.obj['crop_length'] = ''
-# cropTokens = [c for c in sys.argv if c.startswith('crop')]
-# if cropTokens:
-#     if '=' in cropTokens[0]:
-#         cropString = cropTokens[0].split('=')[1]
-#         ctx.obj['crop_start'], ctx.obj['crop_length'] = cropString.split(',')
-#     else:
-#         ctx.obj['crop_start'] = 60
-#         ctx.obj['crop_length'] = 180
-# 
-# denoiseTokens = [d for d in sys.argv if d.startswith('denoise')]
-# 
-
-# for aStream in audioStreams:
-#     if 'channel_layout' in aStream:
-#         print(f"audio stream: {aStream['channel_layout']}") #channel_layout
-#     else:
-#         print(f"unknown audio stream with {aStream['channels']} channels") #channel_layout
 
 def generateAudioTokens(context, index, layout):
 
@@ -337,11 +345,13 @@ def help():
 @click.argument('filename', nargs=1)
 @ffx.command()
 def streams(filename):
-    sd = getStreamDescriptor(filename)
+    
-    if sd is None:
+    try:
-        raise click.ClickException('This file does not contain any audiovisual data')
+        sd = getStreamDescriptor(filename)
+    except Exception as ex:
+        raise click.ClickException(f"This file does not contain any audiovisual data: {ex}")
     for d in sd:
-        click.echo(f"{d['codec']}{' (' + str(d['channels']) + ')' if d['type'] == 'audio' else ''}")
+        click.echo(f"{d['codec_name']}{' (' + str(d['channels']) + ')' if d['codec_type'] == 'audio' else ''}")
 
 
 
@@ -351,14 +361,14 @@ def streams(filename):
 @click.argument('paths', nargs=-1)
 @click.option('-l', '--label', type=str, default='', help='Label to be used as filename prefix')
 
-@click.option('-v', '--video-encoder', type=str, default=DEFAULT_VIDEO_ENCODER, help='Target video encoder (vp9 or av1) default: vp9')
+@click.option('-v', '--video-encoder', type=str, default=DEFAULT_VIDEO_ENCODER, help=f"Target video encoder (vp9 or av1) default: {DEFAULT_VIDEO_ENCODER}")
 
-@click.option('-q', '--quality', type=str, default=DEFAULT_QUALITY, help='Quality settings to be used with VP9 encoder (default: 23)')
+@click.option('-q', '--quality', type=str, default=DEFAULT_QUALITY, help=f"Quality settings to be used with VP9 encoder (default: {DEFAULT_QUALITY})")
-@click.option('-p', '--preset', type=str, default=DEFAULT_QUALITY, help='Quality preset to be used with AV1 encoder (default: 5)')
+@click.option('-p', '--preset', type=str, default=DEFAULT_AV1_PRESET, help=f"Quality preset to be used with AV1 encoder (default: {DEFAULT_AV1_PRESET})")
 
-@click.option('-a', '--stereo-bitrate', type=int, default=DEFAULT_STEREO_BANDWIDTH, help='Bitrate in kbit/s to be used to encode stereo audio streams')
+@click.option('-a', '--stereo-bitrate', type=int, default=DEFAULT_STEREO_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode stereo audio streams (default: {DEFAULT_STEREO_BANDWIDTH})")
-@click.option('-ac3', '--ac3-bitrate', type=int, default=DEFAULT_AC3_BANDWIDTH, help='Bitrate in kbit/s to be used to encode 5.1 audio streams')
+@click.option('-ac3', '--ac3-bitrate', type=int, default=DEFAULT_AC3_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode 5.1 audio streams (default: {DEFAULT_AC3_BANDWIDTH})")
-@click.option('-dts', '--dts-bitrate', type=int, default=DEFAULT_DTS_BANDWIDTH, help='Bitrate in kbit/s to be used to encode 6.1 audio streams')
+@click.option('-dts', '--dts-bitrate', type=int, default=DEFAULT_DTS_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode 6.1 audio streams (default: {DEFAULT_DTS_BANDWIDTH})")
 
 @click.option('-ds', '--default-subtitle', type=int, help='Index of default subtitle stream')
 
@@ -368,13 +378,16 @@ def streams(filename):
 
 @click.option("--crop", is_flag=False, flag_value="default", default="none")
 
-@click.option("-c", "--clear-metadata", is_flag=True, default=False)
-@click.option("-d", "--denoise", is_flag=True, default=False)
-
 @click.option("-o", "--output-directory", type=str, default='')
 
 
-def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, ac3_bitrate, dts_bitrate, crop, clear_metadata, default_subtitle, forced_audio, default_audio, denoise, output_directory):
+@click.option("-c", "--clear-metadata", is_flag=True, default=False)
+@click.option("-d", "--denoise", is_flag=True, default=False)
+
+@click.option("--dry-run", is_flag=True, default=False)
+
+
+def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, ac3_bitrate, dts_bitrate, default_subtitle, forced_audio, default_audio, crop, output_directory, clear_metadata, denoise, dry_run):
     """Batch conversion of audiovideo files in format suitable for web playback, e.g. jellyfin
     
     Files found under PATHS will be converted according to parameters.
@@ -395,32 +408,36 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
     click.echo(f"Qualities: {q_list}")
 
     
-    ctx.obj['bitrates'] = {}
+    context['bitrates'] = {}
-    ctx.obj['bitrates']['stereo'] = str(stereo_bitrate) if str(stereo_bitrate).endswith('k') else f"{stereo_bitrate}k"
+    context['bitrates']['stereo'] = str(stereo_bitrate) if str(stereo_bitrate).endswith('k') else f"{stereo_bitrate}k"
-    ctx.obj['bitrates']['ac3'] = str(ac3_bitrate) if str(ac3_bitrate).endswith('k') else f"{ac3_bitrate}k"
+    context['bitrates']['ac3'] = str(ac3_bitrate) if str(ac3_bitrate).endswith('k') else f"{ac3_bitrate}k"
-    ctx.obj['bitrates']['dts'] = str(dts_bitrate) if str(dts_bitrate).endswith('k') else f"{dts_bitrate}k"
+    context['bitrates']['dts'] = str(dts_bitrate) if str(dts_bitrate).endswith('k') else f"{dts_bitrate}k"
 
-    click.echo(f"Stereo bitrate: {ctx.obj['bitrates']['stereo']}")
+    click.echo(f"Stereo bitrate: {context['bitrates']['stereo']}")
-    click.echo(f"AC3 bitrate: {ctx.obj['bitrates']['ac3']}")
+    click.echo(f"AC3 bitrate: {context['bitrates']['ac3']}")
-    click.echo(f"DTS bitrate: {ctx.obj['bitrates']['dts']}")
+    click.echo(f"DTS bitrate: {context['bitrates']['dts']}")
 
-    ctx.obj['perform_crop'] = (crop != 'none')
+    context['perform_crop'] = (crop != 'none')
 
-    if ctx.obj['perform_crop']:
+    if context['perform_crop']:
 
         cropTokens = crop.split(',')
 
         if cropTokens and len(cropTokens) == 2:
 
-            ctx.obj['crop_start'], ctx.obj['crop_length'] = crop.split(',')
+            context['crop_start'], context['crop_length'] = crop.split(',')
         else:
-            ctx.obj['crop_start'] = DEFAULT_CROP_START
+            context['crop_start'] = DEFAULT_CROP_START
-            ctx.obj['crop_length'] = DEFAULT_CROP_LENGTH
+            context['crop_length'] = DEFAULT_CROP_LENGTH
 
-        click.echo(f"crop start={ctx.obj['crop_start']} length={ctx.obj['crop_length']}")
+        click.echo(f"crop start={context['crop_start']} length={context['crop_length']}")
 
 
-    click.echo(f"\nRunning {len(paths) * len(q_list)} jobs")
+    existingSourcePaths = [p for p in paths if os.path.isfile(p)]
+
+
+    click.echo(f"\nRunning {len(existingSourcePaths) * len(q_list)} jobs")
+
 
     job_index = 0
 
@@ -428,12 +445,7 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
     e_match = re.compile(EPISODE_INDICATOR_MATCH)
 
 
-    for sourcePath in paths:
+    for sourcePath in existingSourcePaths:
-
-
-        if not os.path.isfile(sourcePath):
-            click.echo(f"There is no file with path {sourcePath}, skipping ...")
-            continue
 
         sourceDirectory = os.path.dirname(sourcePath)
         sourceFilename = os.path.basename(sourcePath)
@@ -447,8 +459,6 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
             sourceFilenameExtension = ''
 
 
-        #click.echo(f"dir={sourceDirectory} base={sourceFileBasename} ext={sourceFilenameExtension}")
-
         click.echo(f"\nProcessing file {sourcePath}")
 
         season_digits = 2
@@ -474,11 +484,8 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
 
 
         targetFilenameTokens = []
-
-
         targetFilenameExtension = DEFAULT_FILE_EXTENSION
 
-
         if label:
             targetFilenameTokens = [label]
 
@@ -492,23 +499,22 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
         else:
             targetFilenameTokens = [sourceFileBasename]
 
-            # In case source and target filenames are the same add an extension to distinct output from input
-            if sourceFilenameExtension == targetFilenameExtension:
-                targetFilenameTokens += ['ffx']
 
-
+        try:
-        targetFilename = '_'.join(targetFilenameTokens) + '.' + targetFilenameExtension
+            streamDescriptor = getStreamDescriptor(sourcePath)
-
+        except Exception:
-        click.echo(f"target filename: {targetFilename}")
-
-        streamDescriptor = getStreamDescriptor(sourcePath)
-
-        if streamDescriptor is None:
             click.echo(f"File with path {sourcePath} does not contain any audiovisual data, skipping ...")
             continue
 
-        commandTokens = COMMAND_TOKENS + [sourcePath]
+        for aStream in streamDescriptor[STREAM_TYPE_AUDIO]:
+            click.echo(f"audio stream lang={aStream['language']}")
 
+        for sStream in streamDescriptor[STREAM_TYPE_SUBTITLE]:
+            click.echo(f"subtitle stream lang={sStream['language']}")
+
+
+
+        commandTokens = COMMAND_TOKENS + [sourcePath]
 
         for q in q_list:
 
@@ -520,17 +526,30 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
             audioTokens = []
 
             audioIndex = 0
-            for audioStreamDescriptor in streamDescriptor:
+            for audioStreamDescriptor in streamDescriptor[STREAM_TYPE_AUDIO]:
 
-                if audioStreamDescriptor['type'] == STREAM_TYPE_AUDIO:
+                mappingTokens += ['-map', f"a:{audioIndex}"]
+                audioTokens += generateAudioTokens(context, audioIndex, audioStreamDescriptor['layout'])
+                audioIndex += 1
 
-                    mappingTokens += ['-map', f"a:{audioIndex}"]
+            subtitleIndex = 0
-                    audioTokens += generateAudioTokens(ctx.obj, audioIndex, audioStreamDescriptor['layout'])
+            for subtitleStreamDescriptor in streamDescriptor[STREAM_TYPE_SUBTITLE]:
-                    audioIndex += 1
+                mappingTokens += ['-map', f"s:{subtitleIndex}"]
+                subtitleIndex += 1
 
 
-            for s in range(len([d for d in streamDescriptor if d['type'] == STREAM_TYPE_SUBTITLE])):
+            targetFilenameJobTokens = targetFilenameTokens.copy()
-                mappingTokens += ['-map', f"s:{s}"]
+
+            if len(q_list) > 1:
+                targetFilenameJobTokens += [f"q{q}"]
+
+            # In case source and target filenames are the same add an extension to distinct output from input
+            if not label and sourceFilenameExtension == targetFilenameExtension:
+                targetFilenameJobTokens += ['ffx']
+
+            targetFilename = '_'.join(targetFilenameJobTokens) # + '.' + targetFilenameExtension
+
+            click.echo(f"target filename: {targetFilename}")
 
 
 
@@ -541,22 +560,23 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
                 if clear_metadata:
                     commandSequence += generateClearTokens(streamDescriptor)
 
-                if ctx.obj['perform_crop']:
+                if context['perform_crop']:
-                    commandSequence += generateCropTokens(ctx.obj['crop_start'], ctx.obj['crop_length'])
+                    commandSequence += generateCropTokens(context['crop_start'], context['crop_length'])
                 
-                commandSequence += generateOutputTokens(targetFilename, DEFAULT_FILE_EXTENSION, q)
+                commandSequence += generateOutputTokens(targetFilename, DEFAULT_FILE_FORMAT, DEFAULT_FILE_EXTENSION)
 
                 click.echo(f"Command: {' '.join(commandSequence)}")
 
-                # executeProcess(commandSequence)
+                if not dry_run:
+                    executeProcess(commandSequence)
 
 
             if video_encoder == 'vp9':
 
                 commandSequence1 = commandTokens + mappingVideoTokens + generateVP9Pass1Tokens(q)
 
-                if ctx.obj['perform_crop']:
+                if context['perform_crop']:
-                    commandSequence1 += generateCropTokens(ctx.obj['crop_start'], ctx.obj['crop_length'])
+                    commandSequence1 += generateCropTokens(context['crop_start'], context['crop_length'])
 
                 commandSequence1 += NULL_TOKENS
 
@@ -565,7 +585,8 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
                 if os.path.exists(TEMP_FILE_NAME):
                     os.remove(TEMP_FILE_NAME)
 
-                # executeProcess(commandSequence1)
+                if not dry_run:
+                    executeProcess(commandSequence1)
 
 
                 commandSequence2 = commandTokens + mappingTokens
@@ -578,17 +599,18 @@ def convert(ctx, paths, label, video_encoder, quality, preset, stereo_bitrate, a
                 if clear_metadata:
                     commandSequence2 += generateClearTokens(streamDescriptor)
 
-                if ctx.obj['perform_crop']:
+                if context['perform_crop']:
-                    commandSequence2 += generateCropTokens(ctx.obj['crop_start'], ctx.obj['crop_length'])
+                    commandSequence2 += generateCropTokens(context['crop_start'], context['crop_length'])
 
-                commandSequence2 += generateOutputTokens(targetFilename, DEFAULT_FILE_EXTENSION, q)
+                commandSequence2 += generateOutputTokens(targetFilename, DEFAULT_FILE_FORMAT, DEFAULT_FILE_EXTENSION)
 
                 click.echo(f"Command 2: {' '.join(commandSequence2)}")
 
-                # executeProcess(commandSequence2)
+                if not dry_run:            
+                    executeProcess(commandSequence2)
 
 
-    #app = ModesApp(ctx.obj)
+    #app = ModesApp(context)
     #app.run()
 
     #click.confirm('Warning! This file is not compliant to the defined source schema! Do you want to continue?', abort=True)