From 8c7eee580d1e9dfa13c3b7fd2c3e7c0df3ace348 Mon Sep 17 00:00:00 2001 From: Maveno Date: Tue, 12 Nov 2024 20:17:50 +0100 Subject: [PATCH] #401 tag filter in config file --- ansible/setup_node.yml | 22 +++++++++++ bin/ffx/configuration_controller.py | 4 ++ bin/ffx/ffx_controller.py | 60 ++++++++++++----------------- bin/ffx/file_properties.py | 11 ------ 4 files changed, 50 insertions(+), 47 deletions(-) diff --git a/ansible/setup_node.yml b/ansible/setup_node.yml index c6da0b9..14b7502 100644 --- a/ansible/setup_node.yml +++ b/ansible/setup_node.yml @@ -105,6 +105,28 @@ vars: ffxConfiguration: databasePath: "{{ ffxHomeDirectory }}/.local/var/sync/ffx/ffx.db" + metadata: + signature: + RECODED_WITH: FFX + remove: + - VERSION-eng + - creation_time + - NAME + streams: + remove: + - BPS + - NUMBER_OF_FRAMES + - NUMBER_OF_BYTES + - _STATISTICS_WRITING_APP + - _STATISTICS_WRITING_DATE_UTC + - _STATISTICS_TAGS + - BPS-eng + - DURATION-eng + - NUMBER_OF_FRAMES-eng + - NUMBER_OF_BYTES-eng + - _STATISTICS_WRITING_APP-eng + - _STATISTICS_WRITING_DATE_UTC-eng + - _STATISTICS_TAGS-eng ansible.builtin.copy: content: "{{ ffxConfiguration | to_json }}" dest: "{{ ffxHomeDirectory }}/.local/etc/ffx.json" diff --git a/bin/ffx/configuration_controller.py b/bin/ffx/configuration_controller.py index b1882c2..dc6d125 100644 --- a/bin/ffx/configuration_controller.py +++ b/bin/ffx/configuration_controller.py @@ -46,3 +46,7 @@ class ConfigurationController(): def getDatabaseFilePath(self): return self.__databaseFilePath + + + def getData(self): + return self.__configurationData \ No newline at end of file diff --git a/bin/ffx/ffx_controller.py b/bin/ffx/ffx_controller.py index a05074f..fed41e0 100644 --- a/bin/ffx/ffx_controller.py +++ b/bin/ffx/ffx_controller.py @@ -26,27 +26,6 @@ class FfxController(): DEFAULT_FILE_FORMAT = 'webm' DEFAULT_FILE_EXTENSION = 'webm' - MKVMERGE_METADATA_KEYS = ['BPS', - 'NUMBER_OF_FRAMES', - 'NUMBER_OF_BYTES', - '_STATISTICS_WRITING_APP', - '_STATISTICS_WRITING_DATE_UTC', - '_STATISTICS_TAGS', - - 'BPS-eng', - 'DURATION-eng', - 'NUMBER_OF_FRAMES-eng', - 'NUMBER_OF_BYTES-eng', - '_STATISTICS_WRITING_APP-eng', - '_STATISTICS_WRITING_DATE_UTC-eng', - '_STATISTICS_TAGS-eng'] - - IGNORED_METADATA_KEYS = ['VERSION-eng', - 'creation_time', - 'NAME'] - - - INPUT_FILE_EXTENSIONS = ['mkv', 'mp4', 'avi', 'flv', 'webm'] CHANNEL_MAP_5_1 = 'FL-FL|FR-FR|FC-FC|LFE-LFE|SL-BL|SR-BR:5.1' @@ -62,6 +41,8 @@ class FfxController(): self.__sourceMediaDescriptor = sourceMediaDescriptor self.__targetMediaDescriptor = targetMediaDescriptor + self.__configurationData = self.__context['config'].getData() + self.__logger = context['logger'] @@ -206,11 +187,17 @@ class FfxController(): metadataTokens = [] - mediaTags = self.__targetMediaDescriptor.getTags() + metadataConfiguration = self.__configurationData['metadata'] if 'metadata' in self.__configurationData.keys() else {} + + signatureTags = metadataConfiguration['signature'] if 'signature' in metadataConfiguration.keys() else {} + removeGlobalKeys = metadataConfiguration['remove'] if 'remove' in metadataConfiguration.keys() else [] + removeTrackKeys = metadataConfiguration['streams']['remove'] if 'streams' in metadataConfiguration.keys() and 'remove' in metadataConfiguration['streams'].keys() else [] + + mediaTags = {k:v for k,v in self.__targetMediaDescriptor.getTags().items() if not k in removeGlobalKeys} if (not 'no_signature' in self.__context.keys() or not self.__context['no_signature']): - outputMediaTags = mediaTags | FfxController.SIGNATURE_TAGS + outputMediaTags = mediaTags | signatureTags else: outputMediaTags = mediaTags @@ -218,28 +205,29 @@ class FfxController(): metadataTokens += [f"-metadata:g", f"{tagKey}={tagValue}"] + for removeKey in removeGlobalKeys: + metadataTokens += [f"-metadata:g", + f"{removeKey}="] + + removeMkvmergeMetadata = (not 'keep_mkvmerge_metadata' in self.__context.keys() or not self.__context['keep_mkvmerge_metadata']) #HINT: With current ffmpeg version track metadata tags are not passed to the outfile for td in self.__targetMediaDescriptor.getAllTrackDescriptors(): - - for tagKey, tagValue in td.getTags().items(): - typeIndicator = td.getType().indicator() - subIndex = td.getSubIndex() - metadataTokens += [f"-metadata:s:{typeIndicator}:{subIndex}", - f"{tagKey}={tagValue}"] + typeIndicator = td.getType().indicator() + subIndex = td.getSubIndex() - if removeMkvmergeMetadata: + for tagKey, tagValue in td.getTags().items(): - #TODO: #401 - for ignKey in FfxController.IGNORED_METADATA_KEYS: - metadataTokens += [f"-metadata:g", - f"{ignKey}="] - for mmKey in FfxController.MKVMERGE_METADATA_KEYS: + if not tagKey in removeTrackKeys: metadataTokens += [f"-metadata:s:{typeIndicator}:{subIndex}", - f"{mmKey}="] + f"{tagKey}={tagValue}"] + + for removeKey in removeTrackKeys: + metadataTokens += [f"-metadata:s:{typeIndicator}:{subIndex}", + f"{removeKey}="] return metadataTokens diff --git a/bin/ffx/file_properties.py b/bin/ffx/file_properties.py index d9f25be..e3dae06 100644 --- a/bin/ffx/file_properties.py +++ b/bin/ffx/file_properties.py @@ -41,11 +41,8 @@ class FileProperties(): self.__sourceFileBasename = self.__sourceFilename self.__sourceFilenameExtension = '' - self.__pc = PatternController(context) - # db pattern boruto_[sS]([0-9]+)[eE]([0-9]+).mkv - # Checking if database contains matching pattern matchResult = self.__pc.matchFilename(self.__sourceFilename) @@ -106,22 +103,14 @@ class FileProperties(): "-of", "json", self.__sourcePath]) - if 'Invalid data found when processing input' in ffprobeError: raise Exception(f"File {self.__sourcePath} does not contain valid stream data") - if returnCode != 0: raise Exception(f"ffprobe returned with error {returnCode}") - return json.loads(ffprobeOutput)['format'] - #[{'index': 0, 'codec_name': 'vp9', 'codec_long_name': 'Google VP9', 'profile': 'Profile 0', 'codec_type': 'video', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'width': 1920, 'height': 1080, 'coded_width': 1920, 'coded_height': 1080, 'closed_captions': 0, 'film_grain': 0, 'has_b_frames': 0, 'sample_aspect_ratio': '1:1', 'display_aspect_ratio': '16:9', 'pix_fmt': 'yuv420p', 'level': -99, 'color_range': 'tv', 'chroma_location': 'left', 'field_order': 'progressive', 'refs': 1, 'r_frame_rate': '24000/1001', 'avg_frame_rate': '24000/1001', 'time_base': '1/1000', 'start_pts': 0, 'start_time': '0.000000', 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'BPS': '7974017', 'NUMBER_OF_FRAMES': '34382', 'NUMBER_OF_BYTES': '1429358655', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 libvpx-vp9', 'DURATION': '00:23:54.016000000'}}] - #[{'index': 1, 'codec_name': 'opus', 'codec_long_name': 'Opus (Opus Interactive Audio Codec)', 'codec_type': 'audio', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'sample_fmt': 'fltp', 'sample_rate': '48000', 'channels': 2, 'channel_layout': 'stereo', 'bits_per_sample': 0, 'initial_padding': 312, 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'extradata_size': 19, 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'jpn', 'title': 'Japanisch', 'BPS': '128000', 'NUMBER_OF_FRAMES': '61763', 'NUMBER_OF_BYTES': '22946145', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 libopus', 'DURATION': '00:23:54.141000000'}}] - - #[{'index': 2, 'codec_name': 'webvtt', 'codec_long_name': 'WebVTT subtitle', 'codec_type': 'subtitle', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'duration_ts': 1434141, 'duration': '1434.141000', 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'ger', 'title': 'Deutsch [Full]', 'BPS': '118', 'NUMBER_OF_FRAMES': '300', 'NUMBER_OF_BYTES': '21128', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 webvtt', 'DURATION': '00:23:54.010000000'}}, {'index': 3, 'codec_name': 'webvtt', 'codec_long_name': 'WebVTT subtitle', 'codec_type': 'subtitle', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'r_frame_rate': '0/0', 'avg_frame_rate': '0/0', 'time_base': '1/1000', 'start_pts': -7, 'start_time': '-0.007000', 'duration_ts': 1434141, 'duration': '1434.141000', 'disposition': {'default': 0, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'eng', 'title': 'Englisch [Full]', 'BPS': '101', 'NUMBER_OF_FRAMES': '276', 'NUMBER_OF_BYTES': '16980', '_STATISTICS_WRITING_APP': "mkvmerge v63.0.0 ('Everything') 64-bit", '_STATISTICS_WRITING_DATE_UTC': '2023-10-07 13:59:46', '_STATISTICS_TAGS': 'BPS DURATION NUMBER_OF_FRAMES NUMBER_OF_BYTES', 'ENCODER': 'Lavc61.3.100 webvtt', 'DURATION': '00:23:53.230000000'}}] - def getStreamData(self): """Returns ffprobe stream data as array with elements according to the following example