You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
ffx/bin/ffx.py

538 lines
23 KiB
Python

#! /usr/bin/python3
import os, click, time, logging
from ffx.configuration_controller import ConfigurationController
from ffx.file_properties import FileProperties
from ffx.ffx_app import FfxApp
from ffx.ffx_controller import FfxController
from ffx.tmdb_controller import TmdbController
from ffx.database import databaseContext
from ffx.media_descriptor import MediaDescriptor
from ffx.track_descriptor import TrackDescriptor
from ffx.track_type import TrackType
from ffx.video_encoder import VideoEncoder
from ffx.track_disposition import TrackDisposition
from ffx.process import executeProcess
VERSION='0.2.0'
# 0.1.1
# Bugfixes, TMBD identify shows
# 0.1.2
# Bugfixes
# 0.1.3
# Subtitle file imports
# 0.2.0
# Tests, Config-File
@click.group()
@click.pass_context
@click.option('--database-file', type=str, default='', help='Path to database file')
@click.option('-v', '--verbose', type=int, default=0, help='Set verbosity of output')
@click.option("--dry-run", is_flag=True, default=False)
def ffx(ctx, database_file, verbose, dry_run):
"""FFX"""
ctx.obj = {}
ctx.obj['config'] = ConfigurationController()
ctx.obj['database'] = databaseContext(databasePath=database_file
if database_file else ctx.obj['config'].getDatabaseFilePath())
ctx.obj['dry_run'] = dry_run
ctx.obj['verbosity'] = verbose
# Critical 50
# Error 40
# Warning 30
# Info 20
# Debug 10
fileLogVerbosity = max(40 - verbose * 10, 10)
consoleLogVerbosity = max(20 - verbose * 10, 10)
ctx.obj['logger'] = logging.getLogger('FFX')
ctx.obj['logger'].setLevel(logging.DEBUG)
ffxFileHandler = logging.FileHandler(ctx.obj['config'].getLogFilePath())
ffxFileHandler.setLevel(fileLogVerbosity)
ffxConsoleHandler = logging.StreamHandler()
ffxConsoleHandler.setLevel(consoleLogVerbosity)
fileFormatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ffxFileHandler.setFormatter(fileFormatter)
consoleFormatter = logging.Formatter(
'%(message)s')
ffxConsoleHandler.setFormatter(consoleFormatter)
ctx.obj['logger'].addHandler(ffxConsoleHandler)
ctx.obj['logger'].addHandler(ffxFileHandler)
# Define a subcommand
@ffx.command()
def version():
click.echo(VERSION)
# Another subcommand
@ffx.command()
def help():
click.echo(f"ffx {VERSION}\n")
click.echo(f"Usage: ffx [input file] [output file] [vp9|av1] [q=[nn[,nn,...]]] [p=nn] [a=nnn[k]] [ac3=nnn[k]] [dts=nnn[k]] [crop]")
@ffx.command()
@click.pass_context
@click.argument('filename', nargs=1)
def inspect(ctx, filename):
ctx.obj['command'] = 'inspect'
ctx.obj['arguments'] = {}
ctx.obj['arguments']['filename'] = filename
app = FfxApp(ctx.obj)
app.run()
#TODO: TrackCodec Klasse
CODEC_LOOKUP_TABLE = {
'h264': {'format': 'h264', 'extension': 'h264'},
'aac': { 'extension': 'aac'},
'ac3': {'format': 'ac3', 'extension': 'ac3'},
'ass': {'format': 'ass', 'extension': 'ass'},
'hdmv_pgs_subtitle': {'format': 'sup', 'extension': 'sup'}
}
def getUnmuxSequence(trackDescriptor: TrackDescriptor, sourcePath, targetPrefix, targetDirectory = ''):
trackCodec = trackDescriptor.getCodec()
if not trackCodec in CODEC_LOOKUP_TABLE.keys():
return []
commandTokens = FfxController.COMMAND_TOKENS + ['-i', sourcePath]
trackType = trackDescriptor.getType()
targetPathBase = os.path.join(targetDirectory, targetPrefix) if targetDirectory else targetPrefix
commandTokens += ['-map',
f"0:{trackType.indicator()}:{trackDescriptor.getSubIndex()}",
'-c',
'copy']
if 'format' in CODEC_LOOKUP_TABLE[trackCodec].keys():
commandTokens += ['-f', CODEC_LOOKUP_TABLE[trackCodec]['format']]
commandTokens += [f"{targetPathBase}.{CODEC_LOOKUP_TABLE[trackCodec]['extension']}"]
return commandTokens
@ffx.command()
@click.pass_context
@click.argument('paths', nargs=-1)
@click.option('-l', '--label', type=str, default='', help='Label to be used as filename prefix')
@click.option("-o", "--output-directory", type=str, default='')
@click.option("-s", "--subtitles-only", is_flag=True, default=False)
def unmux(ctx,
paths,
label,
output_directory,
subtitles_only):
existingSourcePaths = [p for p in paths if os.path.isfile(p)]
if ctx.obj['verbosity'] > 0:
click.echo(f"\nUnmuxing {len(existingSourcePaths)} files")
for sourcePath in existingSourcePaths:
fp = FileProperties(ctx.obj, sourcePath)
try:
sourceMediaDescriptor = fp.getMediaDescriptor()
season = fp.getSeason()
episode = fp.getEpisode()
#TODO: Recognition für alle Formate anpassen
targetLabel = label if label else fp.getFileBasename()
targetIndicator = f"_S{season}E{episode}" if label and season != -1 and episode != -1 else ''
if label and not targetIndicator:
if ctx.obj['verbosity'] > 0:
click.echo(f"Skipping file {fp.getFilename()}: Label set but no indicator recognized")
continue
else:
if ctx.obj['verbosity'] > 0:
click.echo(f"\nUnmuxing file {fp.getFilename()}\n")
for trackDescriptor in sourceMediaDescriptor.getAllTrackDescriptors():
if trackDescriptor.getType() == TrackType.SUBTITLE or not subtitles_only:
# SEASON_EPISODE_STREAM_LANGUAGE_MATCH = '[sS]([0-9]+)[eE]([0-9]+)_([0-9]+)_([a-z]{3})'
targetPrefix = f"{targetLabel}{targetIndicator}_{trackDescriptor.getIndex()}_{trackDescriptor.getLanguage().threeLetter()}"
unmuxSequence = getUnmuxSequence(trackDescriptor, sourcePath, targetPrefix, targetDirectory = output_directory)
if unmuxSequence:
if not ctx.obj['dry_run']:
if ctx.obj['verbosity'] > 0:
click.echo(f"Executing unmuxing sequence: {' '.join(unmuxSequence)}")
out, err, rc = executeProcess(unmuxSequence)
if rc:
if ctx.obj['verbosity'] > 0:
click.echo(f"Unmuxing of stream {trackDescriptor.getIndex()} failed with error ({rc}) {err}")
else:
if ctx.obj['verbosity'] > 0:
click.echo(f"Skipping stream with unknown codec {trackDescriptor.getCodec()}")
except Exception as ex:
if ctx.obj['verbosity'] > 0:
click.echo(f"Skipping File {sourcePath} ({ex})")
@ffx.command()
@click.pass_context
def shows(ctx):
ctx.obj['command'] = 'shows'
app = FfxApp(ctx.obj)
app.run()
def checkUniqueDispositions(context, mediaDescriptor: MediaDescriptor):
# Check for multiple default or forced dispositions if not set by user input or database requirements
#
# Query user for the correct sub indices, then configure flags in track descriptors associated with media descriptor accordingly.
# The correct tokens should then be created by
if len([v for v in mediaDescriptor.getVideoTracks() if v.getDispositionFlag(TrackDisposition.DEFAULT)]) > 1:
if context['no_prompt']:
raise click.ClickException('More than one default video stream detected and no prompt set')
defaultVideoTrackSubIndex = click.prompt("More than one default video stream detected! Please select stream", type=int)
mediaDescriptor.setDefaultSubTrack(TrackType.VIDEO, defaultVideoTrackSubIndex)
if len([v for v in mediaDescriptor.getVideoTracks() if v.getDispositionFlag(TrackDisposition.FORCED)]) > 1:
if context['no_prompt']:
raise click.ClickException('More than one forced video stream detected and no prompt set')
forcedVideoTrackSubIndex = click.prompt("More than one forced video stream detected! Please select stream", type=int)
mediaDescriptor.setForcedSubTrack(TrackType.VIDEO, forcedVideoTrackSubIndex)
if len([a for a in mediaDescriptor.getAudioTracks() if a.getDispositionFlag(TrackDisposition.DEFAULT)]) > 1:
if context['no_prompt']:
raise click.ClickException('More than one default audio stream detected and no prompt set')
defaultAudioTrackSubIndex = click.prompt("More than one default audio stream detected! Please select stream", type=int)
mediaDescriptor.setDefaultSubTrack(TrackType.AUDIO, defaultAudioTrackSubIndex)
if len([a for a in mediaDescriptor.getAudioTracks() if a.getDispositionFlag(TrackDisposition.FORCED)]) > 1:
if context['no_prompt']:
raise click.ClickException('More than one forced audio stream detected and no prompt set')
forcedAudioTrackSubIndex = click.prompt("More than one forced audio stream detected! Please select stream", type=int)
mediaDescriptor.setForcedSubTrack(TrackType.AUDIO, forcedAudioTrackSubIndex)
if len([s for s in mediaDescriptor.getSubtitleTracks() if s.getDispositionFlag(TrackDisposition.DEFAULT)]) > 1:
if context['no_prompt']:
raise click.ClickException('More than one default subtitle stream detected and no prompt set')
defaultSubtitleTrackSubIndex = click.prompt("More than one default subtitle stream detected! Please select stream", type=int)
mediaDescriptor.setDefaultSubTrack(TrackType.SUBTITLE, defaultSubtitleTrackSubIndex)
if len([s for s in mediaDescriptor.getSubtitleTracks() if s.getDispositionFlag(TrackDisposition.FORCED)]) > 1:
if context['no_prompt']:
raise click.ClickException('More than one forced subtitle stream detected and no prompt set')
forcedSubtitleTrackSubIndex = click.prompt("More than one forced subtitle stream detected! Please select stream", type=int)
mediaDescriptor.setForcedSubTrack(TrackType.SUBTITLE, forcedSubtitleTrackSubIndex)
@ffx.command()
@click.pass_context
@click.argument('paths', nargs=-1)
@click.option('-l', '--label', type=str, default='', help='Label to be used as filename prefix')
@click.option('-v', '--video-encoder', type=str, default=FfxController.DEFAULT_VIDEO_ENCODER, help=f"Target video encoder (vp9 or av1) default: {FfxController.DEFAULT_VIDEO_ENCODER}")
@click.option('-q', '--quality', type=str, default=FfxController.DEFAULT_QUALITY, help=f"Quality settings to be used with VP9 encoder (default: {FfxController.DEFAULT_QUALITY})")
@click.option('-p', '--preset', type=str, default=FfxController.DEFAULT_AV1_PRESET, help=f"Quality preset to be used with AV1 encoder (default: {FfxController.DEFAULT_AV1_PRESET})")
@click.option('-s', '--stereo-bitrate', type=int, default=FfxController.DEFAULT_STEREO_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode stereo audio streams (default: {FfxController.DEFAULT_STEREO_BANDWIDTH})")
@click.option('--ac3', type=int, default=FfxController.DEFAULT_AC3_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode 5.1 audio streams (default: {FfxController.DEFAULT_AC3_BANDWIDTH})")
@click.option('--dts', type=int, default=FfxController.DEFAULT_DTS_BANDWIDTH, help=f"Bitrate in kbit/s to be used to encode 6.1 audio streams (default: {FfxController.DEFAULT_DTS_BANDWIDTH})")
@click.option('--subtitle-directory', type=str, default='', help='Load subtitles from here')
@click.option('--subtitle-prefix', type=str, default='', help='Subtitle filename prefix')
@click.option('--audio-language', type=str, multiple=True, help='Audio stream language(s)')
@click.option('--audio-title', type=str, multiple=True, help='Audio stream title(s)')
@click.option('--default-audio', type=int, default=-1, help='Index of default audio stream')
@click.option('--forced-audio', type=int, default=-1, help='Index of forced audio stream')
@click.option('--subtitle-language', type=str, multiple=True, help='Subtitle stream language(s)')
@click.option('--subtitle-title', type=str, multiple=True, help='Subtitle stream title(s)')
@click.option('--default-subtitle', type=int, default=-1, help='Index of default subtitle stream')
@click.option('--forced-subtitle', type=int, default=-1, help='Index of forced subtitle stream') # (including default audio stream tag)
@click.option("--crop", is_flag=False, flag_value="default", default="none")
@click.option("--output-directory", type=str, default='')
@click.option("--denoise", is_flag=True, default=False)
@click.option("--no-tmdb", is_flag=True, default=False)
@click.option("--no-jellyfin", is_flag=True, default=False)
@click.option("--no-pattern", is_flag=True, default=False)
@click.option("--dont-pass-dispositions", is_flag=True, default=False)
@click.option("--no-prompt", is_flag=True, default=False)
def convert(ctx,
paths,
label,
video_encoder,
quality,
preset,
stereo_bitrate,
ac3,
dts,
subtitle_directory,
subtitle_prefix,
audio_language,
audio_title,
default_audio,
forced_audio,
subtitle_language,
subtitle_title,
default_subtitle,
forced_subtitle,
crop,
output_directory,
denoise,
no_tmdb,
no_jellyfin,
no_pattern,
dont_pass_dispositions,
no_prompt):
"""Batch conversion of audiovideo files in format suitable for web playback, e.g. jellyfin
Files found under PATHS will be converted according to parameters.
Filename extensions will be changed appropriately.
Suffices will we appended to filename in case of multiple created files
or if the filename has not changed."""
startTime = time.perf_counter()
context = ctx.obj
context['video_encoder'] = VideoEncoder.fromLabel(video_encoder)
context['use_jellyfin'] = not no_jellyfin
context['use_tmdb'] = not no_tmdb
context['use_pattern'] = not no_pattern
context['no_prompt'] = no_prompt
context['import_subtitles'] = (subtitle_directory and subtitle_prefix)
if context['import_subtitles']:
context['subtitle_directory'] = subtitle_directory
context['subtitle_prefix'] = subtitle_prefix
# click.echo(f"\nVideo encoder: {video_encoder}")
qualityTokens = quality.split(',')
q_list = [q for q in qualityTokens if q.isnumeric()]
if ctx.obj['verbosity'] > 0:
click.echo(f"Qualities: {q_list}")
context['bitrates'] = {}
context['bitrates']['stereo'] = str(stereo_bitrate) if str(stereo_bitrate).endswith('k') else f"{stereo_bitrate}k"
context['bitrates']['ac3'] = str(ac3) if str(ac3).endswith('k') else f"{ac3}k"
context['bitrates']['dts'] = str(dts) if str(dts).endswith('k') else f"{dts}k"
if ctx.obj['verbosity'] > 0:
click.echo(f"Stereo bitrate: {context['bitrates']['stereo']}")
click.echo(f"AC3 bitrate: {context['bitrates']['ac3']}")
click.echo(f"DTS bitrate: {context['bitrates']['dts']}")
# Process crop parameters
context['perform_crop'] = (crop != 'none')
if context['perform_crop']:
cTokens = crop.split(',')
if cTokens and len(cTokens) == 2:
context['crop_start'] = int(cTokens[0])
context['crop_length'] = int(cTokens[1])
if ctx.obj['verbosity'] > 0:
click.echo(f"Crop start={context['crop_start']} length={context['crop_length']}")
tc = TmdbController() if context['use_tmdb'] else None
existingSourcePaths = [p for p in paths if os.path.isfile(p) and p.split('.')[-1] in FfxController.INPUT_FILE_EXTENSIONS]
if ctx.obj['verbosity'] > 0:
click.echo(f"\nRunning {len(existingSourcePaths) * len(q_list)} jobs")
jobIndex = 0
for sourcePath in existingSourcePaths:
# Separate basedir, basename and extension for current source file
sourceDirectory = os.path.dirname(sourcePath)
sourceFilename = os.path.basename(sourcePath)
sourcePathTokens = sourceFilename.split('.')
sourceFileBasename = '.'.join(sourcePathTokens[:-1])
sourceFilenameExtension = sourcePathTokens[-1]
if ctx.obj['verbosity'] > 0:
click.echo(f"\nProcessing file {sourcePath}")
mediaFileProperties = FileProperties(context, sourceFilename)
sourceMediaDescriptor = mediaFileProperties.getMediaDescriptor()
#HINT: This is None if the filename did not match anything in database
currentPattern = mediaFileProperties.getPattern() if context['use_pattern'] else None
if ctx.obj['verbosity'] > 0:
click.echo(f"Pattern matching: {'No' if currentPattern is None else 'Yes'}")
# fileBasename = ''
if currentPattern is None:
# Case no pattern matching
# fileBasename = currentShowDescriptor.getFilenamePrefix()
checkUniqueDispositions(context, sourceMediaDescriptor)
if context['import_subtitles']:
sourceMediaDescriptor.importSubtitles(context['subtitle_directory'],
context['subtitle_prefix'],
mediaFileProperties.getSeason(),
mediaFileProperties.getEpisode())
if context['use_jellyfin']:
# Reorder subtracks in types with default the last, then make subindices flat again
sourceMediaDescriptor.applyJellyfinOrder()
fc = FfxController(context, sourceMediaDescriptor)
else:
# Case pattern matching
targetMediaDescriptor = currentPattern.getMediaDescriptor(ctx.obj)
checkUniqueDispositions(context, targetMediaDescriptor)
currentShowDescriptor = currentPattern.getShowDescriptor(ctx.obj)
if context['use_tmdb']:
click.echo(f"Querying TMDB for show_id={currentShowDescriptor.getId()} season={mediaFileProperties.getSeason()} episode{mediaFileProperties.getEpisode()}")
tmdbEpisodeResult = tc.queryEpisode(currentShowDescriptor.getId(), mediaFileProperties.getSeason(), mediaFileProperties.getEpisode())
click.echo(f"tmdbEpisodeResult={tmdbEpisodeResult}")
if tmdbEpisodeResult:
sourceFileBasename = TmdbController.getEpisodeFileBasename(currentShowDescriptor.getFilenamePrefix(),
tmdbEpisodeResult['name'],
mediaFileProperties.getSeason(),
mediaFileProperties.getEpisode(),
currentShowDescriptor.getIndexSeasonDigits(),
currentShowDescriptor.getIndexEpisodeDigits(),
currentShowDescriptor.getIndicatorSeasonDigits(),
currentShowDescriptor.getIndicatorEpisodeDigits())
else:
sourceFileBasename = currentShowDescriptor.getFilenamePrefix()
if context['import_subtitles']:
targetMediaDescriptor.importSubtitles(context['subtitle_directory'],
context['subtitle_prefix'],
mediaFileProperties.getSeason(),
mediaFileProperties.getEpisode())
# raise click.ClickException(f"tmd subindices: {[t.getSubIndex() for t in targetMediaDescriptor.getAllTrackDescriptors()]}")
# click.echo(f"tmd subindices: {[t.getIndex() for t in targetMediaDescriptor.getAllTrackDescriptors()]} {[t.getSubIndex() for t in targetMediaDescriptor.getAllTrackDescriptors()]} {[t.getDispositionFlag(TrackDisposition.DEFAULT) for t in targetMediaDescriptor.getAllTrackDescriptors()]}")
if context['use_jellyfin']:
# Reorder subtracks in types with default the last, then make subindices flat again
targetMediaDescriptor.applyJellyfinOrder()
# sourceMediaDescriptor.applyJellyfinOrder()
# click.echo(f"tmd subindices: {[t.getIndex() for t in targetMediaDescriptor.getAllTrackDescriptors()]} {[t.getSubIndex() for t in targetMediaDescriptor.getAllTrackDescriptors()]} {[t.getDispositionFlag(TrackDisposition.DEFAULT) for t in targetMediaDescriptor.getAllTrackDescriptors()]}")
# raise click.Abort
if ctx.obj['verbosity'] > 0:
click.echo(f"Input mapping tokens (2nd pass): {targetMediaDescriptor.getInputMappingTokens()}")
fc = FfxController(context, targetMediaDescriptor, sourceMediaDescriptor)
if ctx.obj['verbosity'] > 0:
click.echo(f"Season={mediaFileProperties.getSeason()} Episode={mediaFileProperties.getEpisode()}")
if ctx.obj['verbosity'] > 0:
click.echo(f"fileBasename={sourceFileBasename}")
for q in q_list:
if ctx.obj['verbosity'] > 0:
click.echo(f"\nRunning job {jobIndex} file={sourcePath} q={q}")
jobIndex += 1
extra = ['ffx'] if sourceFilenameExtension == FfxController.DEFAULT_FILE_EXTENSION else []
click.echo(f"label={label if label else 'Falsy'}")
click.echo(f"sourceFileBasename={sourceFileBasename}")
targetFilename = (sourceFileBasename if context['use_tmdb']
else mediaFileProperties.assembleTargetFileBasename(label,
q if len(q_list) > 1 else -1,
extraTokens = extra))
targetPath = os.path.join(output_directory if output_directory else sourceDirectory, targetFilename)
# media_S01E02_S01E02
click.echo(f"targetPath={targetPath}")
fc.runJob(sourcePath,
targetPath,
context['video_encoder'],
q,
preset,
denoise)
#TODO: click.confirm('Warning! This file is not compliant to the defined source schema! Do you want to continue?', abort=True)
endTime = time.perf_counter()
if ctx.obj['verbosity'] > 0:
click.echo(f"\nDONE\nTime elapsed {endTime - startTime}")
if __name__ == '__main__':
ffx()