From e92caff5d50e60bfd33563d631f0c49ce176dc70 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 9 Mar 2021 07:47:21 +0530 Subject: [PATCH] Refactor (See desc) * Create `FFmpegPostProcessor.real_run_ffmpeg` that can accept multiple input/output files along with switches for each * Rewrite `cli_configuration_args` and related functions * Create `YoutubeDL._ensure_dir_exists` - this was previously defined in multiple places --- yt_dlp/YoutubeDL.py | 25 +++++++++---------- yt_dlp/downloader/external.py | 3 ++- yt_dlp/postprocessor/common.py | 12 +++++++-- yt_dlp/postprocessor/embedthumbnail.py | 2 +- yt_dlp/postprocessor/ffmpeg.py | 27 ++++++++++++++------ yt_dlp/postprocessor/sponskrub.py | 2 +- yt_dlp/utils.py | 34 +++++++++----------------- 7 files changed, 56 insertions(+), 49 deletions(-) diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 7e0a69528..397d40503 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -1171,6 +1171,9 @@ def _fixup(r): else: raise Exception('Invalid result type: %s' % result_type) + def _ensure_dir_exists(self, path): + return make_dir(path, self.report_error) + def __process_playlist(self, ie_result, download): # We process each entry in the playlist playlist = ie_result.get('title') or ie_result.get('id') @@ -1187,12 +1190,9 @@ def __process_playlist(self, ie_result, download): } ie_copy.update(dict(ie_result)) - def ensure_dir_exists(path): - return make_dir(path, self.report_error) - if self.params.get('writeinfojson', False): infofn = self.prepare_filename(ie_copy, 'pl_infojson') - if not ensure_dir_exists(encodeFilename(infofn)): + if not self._ensure_dir_exists(encodeFilename(infofn)): return if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(infofn)): self.to_screen('[info] Playlist metadata is already present') @@ -1208,7 +1208,7 @@ def ensure_dir_exists(path): if self.params.get('writedescription', False): descfn = self.prepare_filename(ie_copy, 'pl_description') - if not ensure_dir_exists(encodeFilename(descfn)): + if not self._ensure_dir_exists(encodeFilename(descfn)): return if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(descfn)): self.to_screen('[info] Playlist description is already present') @@ -2089,17 +2089,14 @@ def process_info(self, info_dict): if full_filename is None: return - def ensure_dir_exists(path): - return make_dir(path, self.report_error) - - if not ensure_dir_exists(encodeFilename(full_filename)): + if not self._ensure_dir_exists(encodeFilename(full_filename)): return - if not ensure_dir_exists(encodeFilename(temp_filename)): + if not self._ensure_dir_exists(encodeFilename(temp_filename)): return if self.params.get('writedescription', False): descfn = self.prepare_filename(info_dict, 'description') - if not ensure_dir_exists(encodeFilename(descfn)): + if not self._ensure_dir_exists(encodeFilename(descfn)): return if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(descfn)): self.to_screen('[info] Video description is already present') @@ -2116,7 +2113,7 @@ def ensure_dir_exists(path): if self.params.get('writeannotations', False): annofn = self.prepare_filename(info_dict, 'annotation') - if not ensure_dir_exists(encodeFilename(annofn)): + if not self._ensure_dir_exists(encodeFilename(annofn)): return if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(annofn)): self.to_screen('[info] Video annotations are already present') @@ -2204,7 +2201,7 @@ def dl(name, info, subtitle=False): if self.params.get('writeinfojson', False): infofn = self.prepare_filename(info_dict, 'infojson') - if not ensure_dir_exists(encodeFilename(infofn)): + if not self._ensure_dir_exists(encodeFilename(infofn)): return if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(infofn)): self.to_screen('[info] Video metadata is already present') @@ -2360,7 +2357,7 @@ def correct_ext(filename): fname = prepend_extension( self.prepare_filename(new_info, 'temp'), 'f%s' % f['format_id'], new_info['ext']) - if not ensure_dir_exists(fname): + if not self._ensure_dir_exists(fname): return downloaded.append(fname) partial_success, real_download = dl(fname, new_info) diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index 7c2d56d9c..2c37860af 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -108,7 +108,8 @@ def _valueless_option(self, command_option, param, expected_value=True): def _configuration_args(self, *args, **kwargs): return cli_configuration_args( self.params.get('external_downloader_args'), - self.get_basename(), *args, **kwargs) + [self.get_basename(), 'default'], + *args, **kwargs) def _call_downloader(self, tmpfilename, info_dict): """ Either overwrite this or implement _make_cmd """ diff --git a/yt_dlp/postprocessor/common.py b/yt_dlp/postprocessor/common.py index f8f4e7693..3c316b349 100644 --- a/yt_dlp/postprocessor/common.py +++ b/yt_dlp/postprocessor/common.py @@ -91,10 +91,18 @@ def try_utime(self, path, atime, mtime, errnote='Cannot update utime of file'): except Exception: self.report_warning(errnote) - def _configuration_args(self, *args, **kwargs): + def _configuration_args(self, exe, keys=None, default=[], use_compat=True): + pp_key = self.pp_key().lower() + exe = exe.lower() + root_key = exe if pp_key == exe else '%s+%s' % (pp_key, exe) + keys = ['%s%s' % (root_key, k) for k in (keys or [''])] + if root_key in keys: + keys += [root_key] + ([] if pp_key == exe else [(self.pp_key(), exe)]) + ['default'] + else: + use_compat = False return cli_configuration_args( self._downloader.params.get('postprocessor_args'), - self.pp_key().lower(), *args, **kwargs) + keys, default, use_compat) class AudioConversionError(PostProcessingError): diff --git a/yt_dlp/postprocessor/embedthumbnail.py b/yt_dlp/postprocessor/embedthumbnail.py index cc6cd1c8e..3c68e0298 100644 --- a/yt_dlp/postprocessor/embedthumbnail.py +++ b/yt_dlp/postprocessor/embedthumbnail.py @@ -139,7 +139,7 @@ def is_webp(path): encodeFilename(thumbnail_filename, True), encodeArgument('-o'), encodeFilename(temp_filename, True)] - cmd += [encodeArgument(o) for o in self._configuration_args(exe='AtomicParsley')] + cmd += [encodeArgument(o) for o in self._configuration_args('AtomicParsley')] self.to_screen('Adding thumbnail to "%s"' % filename) self.write_debug('AtomicParsley command line: %s' % shell_quote(cmd)) diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 944a4bc17..a8635c1d1 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -234,25 +234,35 @@ def get_stream_number(self, path, keys, value): return num, len(streams) def run_ffmpeg_multiple_files(self, input_paths, out_path, opts): + return self.real_run_ffmpeg( + [(path, []) for path in input_paths], + [(out_path, opts)]) + + def real_run_ffmpeg(self, input_path_opts, output_path_opts): self.check_version() oldest_mtime = min( - os.stat(encodeFilename(path)).st_mtime for path in input_paths) + os.stat(encodeFilename(path)).st_mtime for path, _ in input_path_opts) cmd = [encodeFilename(self.executable, True), encodeArgument('-y')] # avconv does not have repeat option if self.basename == 'ffmpeg': cmd += [encodeArgument('-loglevel'), encodeArgument('repeat+info')] - def make_args(file, pre=[], post=[], *args, **kwargs): - args = pre + self._configuration_args(*args, **kwargs) + post + def make_args(file, args, name, number): + keys = ['_%s%d' % (name, number), '_%s' % name] + if name == 'o' and number == 1: + keys.append('') + args += self._configuration_args(self.basename, keys) + if name == 'i': + args.append('-i') return ( - [encodeArgument(o) for o in args] + [encodeArgument(arg) for arg in args] + [encodeFilename(self._ffmpeg_filename_argument(file), True)]) - for i, path in enumerate(input_paths): - cmd += make_args(path, post=['-i'], exe='%s_i%d' % (self.basename, i + 1), use_default_arg=False) - cmd += make_args(out_path, pre=opts, exe=self.basename) + for arg_type, path_opts in (('i', input_path_opts), ('o', output_path_opts)): + cmd += [arg for i, o in enumerate(path_opts) + for arg in make_args(o[0], o[1], arg_type, i + 1)] self.write_debug('ffmpeg command line: %s' % shell_quote(cmd)) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) @@ -262,7 +272,8 @@ def make_args(file, pre=[], post=[], *args, **kwargs): if self.get_param('verbose', False): self.report_error(stderr) raise FFmpegPostProcessorError(stderr.split('\n')[-1]) - self.try_utime(out_path, oldest_mtime, oldest_mtime) + for out_path, _ in output_path_opts: + self.try_utime(out_path, oldest_mtime, oldest_mtime) return stderr.decode('utf-8', 'replace') def run_ffmpeg(self, path, out_path, opts): diff --git a/yt_dlp/postprocessor/sponskrub.py b/yt_dlp/postprocessor/sponskrub.py index 2f30acc8e..4ba33398e 100644 --- a/yt_dlp/postprocessor/sponskrub.py +++ b/yt_dlp/postprocessor/sponskrub.py @@ -71,7 +71,7 @@ def run(self, information): if not self.cutout: cmd += ['-chapter'] cmd += compat_shlex_split(self.args) # For backward compatibility - cmd += self._configuration_args(exe=self._exe_name, use_default_arg='no_compat') + cmd += self._configuration_args(self._exe_name, use_compat=False) cmd += ['--', information['id'], filename, temp_filename] cmd = [encodeArgument(i) for i in cmd] diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 12bc637f8..77f8c0f4d 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -4692,36 +4692,26 @@ def cli_valueless_option(params, command_option, param, expected_value=True): return [command_option] if param == expected_value else [] -def cli_configuration_args(argdict, key, default=[], exe=None, use_default_arg=True): - # use_default_arg can be True, False, or 'no_compat' +def cli_configuration_args(argdict, keys, default=[], use_compat=True): if isinstance(argdict, (list, tuple)): # for backward compatibility - if use_default_arg is True: + if use_compat: return argdict else: argdict = None - if argdict is None: return default assert isinstance(argdict, dict) - key = key.lower() - args = exe_args = None - if exe is not None: - assert isinstance(exe, compat_str) - exe = exe.lower() - args = argdict.get('%s+%s' % (key, exe)) - if args is None: - exe_args = argdict.get(exe) - - if args is None: - args = argdict.get(key) if key != exe else None - if args is None and exe_args is None: - args = argdict.get('default', default) if use_default_arg else default - - args, exe_args = args or [], exe_args or [] - assert isinstance(args, (list, tuple)) - assert isinstance(exe_args, (list, tuple)) - return args + exe_args + assert isinstance(keys, (list, tuple)) + for key_list in keys: + if isinstance(key_list, compat_str): + key_list = (key_list,) + arg_list = list(filter( + lambda x: x is not None, + [argdict.get(key.lower()) for key in key_list])) + if arg_list: + return [arg for args in arg_list for arg in args] + return default class ISO639Utils(object):