Add format types j, l, q for outtmpl

Closes #345
This commit is contained in:
pukkandan 2021-07-29 08:26:17 +05:30
parent 901130bbcf
commit 7d1eb38af1
No known key found for this signature in database
GPG Key ID: 0F00D95A001F4698
5 changed files with 49 additions and 17 deletions

View File

@ -789,10 +789,11 @@ ## Post-Processing Options:
command. An additional field "filepath" command. An additional field "filepath"
that contains the final path of the that contains the final path of the
downloaded file is also available. If no downloaded file is also available. If no
fields are passed, "%(filepath)s" is fields are passed, %(filepath)q is appended
appended to the end of the command to the end of the command
--exec-before-download CMD Execute a command before the actual --exec-before-download CMD Execute a command before the actual
download. The syntax is the same as --exec download. The syntax is the same as --exec
but "filepath" is not available
--convert-subs FORMAT Convert the subtitles to another format --convert-subs FORMAT Convert the subtitles to another format
(currently supported: srt|vtt|ass|lrc) (currently supported: srt|vtt|ass|lrc)
(Alias: --convert-subtitles) (Alias: --convert-subtitles)
@ -917,10 +918,11 @@ # OUTPUT TEMPLATE
It may however also contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [python string formatting operations](https://docs.python.org/2/library/stdtypes.html#string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations. It may however also contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [python string formatting operations](https://docs.python.org/2/library/stdtypes.html#string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations.
The field names themselves (the part inside the parenthesis) can also have some special formatting: The field names themselves (the part inside the parenthesis) can also have some special formatting:
1. **Object traversal**: The dictionaries and lists available in metadata can be traversed by using a `.` (dot) separator. You can also do python slicing using `:`. Eg: `%(tags.0)s`, `%(subtitles.en.-1.ext)`, `%(id.3:7:-1)s`. Note that the fields that become available using this method are not listed below. Use `-j` to see such fields 1. **Object traversal**: The dictionaries and lists available in metadata can be traversed by using a `.` (dot) separator. You can also do python slicing using `:`. Eg: `%(tags.0)s`, `%(subtitles.en.-1.ext)`, `%(id.3:7:-1)s`, `%(formats.:.format_id)s`. Note that all the fields that become available using this method are not listed below. Use `-j` to see such fields
1. **Addition**: Addition and subtraction of numeric fields can be done using `+` and `-` respectively. Eg: `%(playlist_index+10)03d`, `%(n_entries+1-playlist_index)d` 1. **Addition**: Addition and subtraction of numeric fields can be done using `+` and `-` respectively. Eg: `%(playlist_index+10)03d`, `%(n_entries+1-playlist_index)d`
1. **Date/time Formatting**: Date/time fields can be formatted according to [strftime formatting](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) by specifying it separated from the field name using a `>`. Eg: `%(duration>%H-%M-%S)s`, `%(upload_date>%Y-%m-%d)s`, `%(epoch-3600>%H-%M-%S)s` 1. **Date/time Formatting**: Date/time fields can be formatted according to [strftime formatting](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) by specifying it separated from the field name using a `>`. Eg: `%(duration>%H-%M-%S)s`, `%(upload_date>%Y-%m-%d)s`, `%(epoch-3600>%H-%M-%S)s`
1. **Default**: A default value can be specified for when the field is empty using a `|` seperator. This overrides `--output-na-template`. Eg: `%(uploader|Unknown)s` 1. **Default**: A default value can be specified for when the field is empty using a `|` seperator. This overrides `--output-na-template`. Eg: `%(uploader|Unknown)s`
1. **More Conversions**: In addition to the normal format types `diouxXeEfFgGcrs`, `j`, `l`, `q` can be used for converting to **j**son, a comma seperated **l**ist and a string **q**uoted for the terminal respectively
To summarize, the general syntax for a field is: To summarize, the general syntax for a field is:
``` ```

View File

@ -10,6 +10,7 @@
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import copy import copy
import json
from test.helper import FakeYDL, assertRegexpMatches from test.helper import FakeYDL, assertRegexpMatches
from yt_dlp import YoutubeDL from yt_dlp import YoutubeDL
@ -647,6 +648,7 @@ def test_add_extra_info(self):
'title1': '$PATH', 'title1': '$PATH',
'title2': '%PATH%', 'title2': '%PATH%',
'title3': 'foo/bar\\test', 'title3': 'foo/bar\\test',
'title4': 'foo "bar" test',
'timestamp': 1618488000, 'timestamp': 1618488000,
'duration': 100000, 'duration': 100000,
'playlist_index': 1, 'playlist_index': 1,
@ -669,10 +671,12 @@ def test(tmpl, expected, *, info=None, **params):
if callable(expected): if callable(expected):
self.assertTrue(expected(out)) self.assertTrue(expected(out))
self.assertTrue(expected(fname)) self.assertTrue(expected(fname))
elif isinstance(expected, compat_str): elif isinstance(expected, str):
self.assertEqual((out, fname), (expected, expected)) self.assertEqual(out, expected)
self.assertEqual(fname, expected)
else: else:
self.assertEqual((out, fname), expected) self.assertEqual(out, expected[0])
self.assertEqual(fname, expected[1])
# Auto-generated fields # Auto-generated fields
test('%(id)s.%(ext)s', '1234.mp4') test('%(id)s.%(ext)s', '1234.mp4')
@ -741,14 +745,26 @@ def test(tmpl, expected, *, info=None, **params):
test('%(width|0)04d', '0000') test('%(width|0)04d', '0000')
test('a%(width|)d', 'a', outtmpl_na_placeholder='none') test('a%(width|)d', 'a', outtmpl_na_placeholder='none')
# Internal formatting
FORMATS = self.outtmpl_info['formats'] FORMATS = self.outtmpl_info['formats']
sanitize = lambda x: x.replace(':', ' -').replace('"', "'")
# Custom type casting
test('%(formats.:.id)l', 'id1, id2, id3')
test('%(ext)l', 'mp4')
test('%(formats.:.id) 15l', ' id1, id2, id3')
test('%(formats)j', (json.dumps(FORMATS), sanitize(json.dumps(FORMATS))))
if compat_os_name == 'nt':
test('%(title4)q', ('"foo \\"bar\\" test"', "'foo _'bar_' test'"))
else:
test('%(title4)q', ('\'foo "bar" test\'', "'foo 'bar' test'"))
# Internal formatting
test('%(timestamp-1000>%H-%M-%S)s', '11-43-20') test('%(timestamp-1000>%H-%M-%S)s', '11-43-20')
test('%(title|%)s %(title|%%)s', '% %%') test('%(title|%)s %(title|%%)s', '% %%')
test('%(id+1-height+3)05d', '00158') test('%(id+1-height+3)05d', '00158')
test('%(width+100)05d', 'NA') test('%(width+100)05d', 'NA')
test('%(formats.0) 15s', ('% 15s' % FORMATS[0], '% 15s' % str(FORMATS[0]).replace(':', ' -'))) test('%(formats.0) 15s', ('% 15s' % FORMATS[0], '% 15s' % sanitize(str(FORMATS[0]))))
test('%(formats.0)r', (repr(FORMATS[0]), repr(FORMATS[0]).replace(':', ' -'))) test('%(formats.0)r', (repr(FORMATS[0]), sanitize(repr(FORMATS[0]))))
test('%(height.0)03d', '001') test('%(height.0)03d', '001')
test('%(-height.0)04d', '-001') test('%(-height.0)04d', '-001')
test('%(formats.-1.id)s', FORMATS[-1]['id']) test('%(formats.-1.id)s', FORMATS[-1]['id'])

View File

@ -35,6 +35,7 @@
compat_kwargs, compat_kwargs,
compat_numeric_types, compat_numeric_types,
compat_os_name, compat_os_name,
compat_shlex_quote,
compat_str, compat_str,
compat_tokenize_tokenize, compat_tokenize_tokenize,
compat_urllib_error, compat_urllib_error,
@ -108,6 +109,7 @@
try_get, try_get,
UnavailableVideoError, UnavailableVideoError,
url_basename, url_basename,
variadic,
version_tuple, version_tuple,
write_json_file, write_json_file,
write_string, write_string,
@ -871,9 +873,12 @@ def escape_outtmpl(outtmpl):
@classmethod @classmethod
def validate_outtmpl(cls, outtmpl): def validate_outtmpl(cls, outtmpl):
''' @return None or Exception object ''' ''' @return None or Exception object '''
outtmpl = cls.escape_outtmpl(cls._outtmpl_expandpath(outtmpl)) outtmpl = re.sub(
STR_FORMAT_RE_TMPL.format('[^)]*', '[ljq]'),
lambda mobj: f'{mobj.group(0)[:-1]}s',
cls._outtmpl_expandpath(outtmpl))
try: try:
outtmpl % collections.defaultdict(int) cls.escape_outtmpl(outtmpl) % collections.defaultdict(int)
return None return None
except ValueError as err: except ValueError as err:
return err return err
@ -900,7 +905,7 @@ def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
} }
TMPL_DICT = {} TMPL_DICT = {}
EXTERNAL_FORMAT_RE = re.compile(STR_FORMAT_RE_TMPL.format('[^)]*', f'[{STR_FORMAT_TYPES}]')) EXTERNAL_FORMAT_RE = re.compile(STR_FORMAT_RE_TMPL.format('[^)]*', f'[{STR_FORMAT_TYPES}ljq]'))
MATH_FUNCTIONS = { MATH_FUNCTIONS = {
'+': float.__add__, '+': float.__add__,
'-': float.__sub__, '-': float.__sub__,
@ -977,8 +982,15 @@ def create_key(outer_mobj):
value = default if value is None else value value = default if value is None else value
if fmt == 'c': str_fmt = f'{fmt[:-1]}s'
value = compat_str(value) if fmt[-1] == 'l':
value, fmt = ', '.join(variadic(value)), str_fmt
elif fmt[-1] == 'j':
value, fmt = json.dumps(value), str_fmt
elif fmt[-1] == 'q':
value, fmt = compat_shlex_quote(str(value)), str_fmt
elif fmt[-1] == 'c':
value = str(value)
if value is None: if value is None:
value, fmt = default, 's' value, fmt = default, 's'
else: else:
@ -992,7 +1004,7 @@ def create_key(outer_mobj):
if fmt[-1] == 'r': if fmt[-1] == 'r':
# If value is an object, sanitize might convert it to a string # If value is an object, sanitize might convert it to a string
# So we convert it to repr first # So we convert it to repr first
value, fmt = repr(value), '%ss' % fmt[:-1] value, fmt = repr(value), str_fmt
if fmt[-1] in 'csr': if fmt[-1] in 'csr':
value = sanitize(mobj['fields'].split('.')[-1], value) value = sanitize(mobj['fields'].split('.')[-1], value)

View File

@ -1286,11 +1286,11 @@ def _dict_from_options_callback(
'Execute a command on the file after downloading and post-processing. ' 'Execute a command on the file after downloading and post-processing. '
'Similar syntax to the output template can be used to pass any field as arguments to the command. ' 'Similar syntax to the output template can be used to pass any field as arguments to the command. '
'An additional field "filepath" that contains the final path of the downloaded file is also available. ' 'An additional field "filepath" that contains the final path of the downloaded file is also available. '
'If no fields are passed, "%(filepath)s" is appended to the end of the command')) 'If no fields are passed, %(filepath)q is appended to the end of the command'))
postproc.add_option( postproc.add_option(
'--exec-before-download', '--exec-before-download',
metavar='CMD', dest='exec_before_dl_cmd', metavar='CMD', dest='exec_before_dl_cmd',
help='Execute a command before the actual download. The syntax is the same as --exec') help='Execute a command before the actual download. The syntax is the same as --exec but "filepath" is not available')
postproc.add_option( postproc.add_option(
'--convert-subs', '--convert-sub', '--convert-subtitles', '--convert-subs', '--convert-sub', '--convert-subtitles',
metavar='FORMAT', dest='convertsubtitles', default=None, metavar='FORMAT', dest='convertsubtitles', default=None,

View File

@ -4451,8 +4451,10 @@ def q(qid):
) )
''' '''
STR_FORMAT_TYPES = 'diouxXeEfFgGcrs' STR_FORMAT_TYPES = 'diouxXeEfFgGcrs'
def limit_length(s, length): def limit_length(s, length):
""" Add ellipses to overly long strings """ """ Add ellipses to overly long strings """
if s is None: if s is None: