[utils, etc] Kill child processes when yt-dl is killed

* derived from PR #26592, closes #26592

Authored by: Unrud
This commit is contained in:
pukkandan 2021-01-09 17:56:12 +05:30 committed by dirkf
parent 811c480f7b
commit 0700fde640
8 changed files with 42 additions and 21 deletions

View File

@ -73,6 +73,7 @@ from .utils import (
PostProcessingError, PostProcessingError,
preferredencoding, preferredencoding,
prepend_extension, prepend_extension,
process_communicate_or_kill,
register_socks_protocols, register_socks_protocols,
render_table, render_table,
replace_extension, replace_extension,
@ -2323,7 +2324,7 @@ class YoutubeDL(object):
['git', 'rev-parse', '--short', 'HEAD'], ['git', 'rev-parse', '--short', 'HEAD'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
cwd=os.path.dirname(os.path.abspath(__file__))) cwd=os.path.dirname(os.path.abspath(__file__)))
out, err = sp.communicate() out, err = process_communicate_or_kill(sp)
out = out.decode().strip() out = out.decode().strip()
if re.match('[0-9a-f]+', out): if re.match('[0-9a-f]+', out):
self._write_string('[debug] Git HEAD: ' + out + '\n') self._write_string('[debug] Git HEAD: ' + out + '\n')

View File

@ -2890,6 +2890,7 @@ else:
_terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines']) _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
def compat_get_terminal_size(fallback=(80, 24)): def compat_get_terminal_size(fallback=(80, 24)):
from .utils import process_communicate_or_kill
columns = compat_getenv('COLUMNS') columns = compat_getenv('COLUMNS')
if columns: if columns:
columns = int(columns) columns = int(columns)
@ -2906,7 +2907,7 @@ else:
sp = subprocess.Popen( sp = subprocess.Popen(
['stty', 'size'], ['stty', 'size'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = sp.communicate() out, err = process_communicate_or_kill(sp)
_lines, _columns = map(int, out.split()) _lines, _columns = map(int, out.split())
except Exception: except Exception:
_columns, _lines = _terminal_size(*fallback) _columns, _lines = _terminal_size(*fallback)

View File

@ -22,6 +22,7 @@ from ..utils import (
handle_youtubedl_headers, handle_youtubedl_headers,
check_executable, check_executable,
is_outdated_version, is_outdated_version,
process_communicate_or_kill,
) )
@ -104,7 +105,7 @@ class ExternalFD(FileDownloader):
p = subprocess.Popen( p = subprocess.Popen(
cmd, stderr=subprocess.PIPE) cmd, stderr=subprocess.PIPE)
_, stderr = p.communicate() _, stderr = process_communicate_or_kill(p)
if p.returncode != 0: if p.returncode != 0:
self.to_stderr(stderr.decode('utf-8', 'replace')) self.to_stderr(stderr.decode('utf-8', 'replace'))
return p.returncode return p.returncode
@ -141,7 +142,7 @@ class CurlFD(ExternalFD):
# curl writes the progress to stderr so don't capture it. # curl writes the progress to stderr so don't capture it.
p = subprocess.Popen(cmd) p = subprocess.Popen(cmd)
p.communicate() process_communicate_or_kill(p)
return p.returncode return p.returncode
@ -336,14 +337,17 @@ class FFmpegFD(ExternalFD):
proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env) proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
try: try:
retval = proc.wait() retval = proc.wait()
except KeyboardInterrupt: except BaseException as e:
# subprocces.run would send the SIGKILL signal to ffmpeg and the # subprocess.run would send the SIGKILL signal to ffmpeg and the
# mp4 file couldn't be played, but if we ask ffmpeg to quit it # mp4 file couldn't be played, but if we ask ffmpeg to quit it
# produces a file that is playable (this is mostly useful for live # produces a file that is playable (this is mostly useful for live
# streams). Note that Windows is not affected and produces playable # streams). Note that Windows is not affected and produces playable
# files (see https://github.com/ytdl-org/youtube-dl/issues/8300). # files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
if sys.platform != 'win32': if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
proc.communicate(b'q') process_communicate_or_kill(proc, b'q')
else:
proc.kill()
proc.wait()
raise raise
return retval return retval

View File

@ -89,11 +89,13 @@ class RtmpFD(FileDownloader):
self.to_screen('') self.to_screen('')
cursor_in_new_line = True cursor_in_new_line = True
self.to_screen('[rtmpdump] ' + line) self.to_screen('[rtmpdump] ' + line)
finally:
proc.wait()
if not cursor_in_new_line: if not cursor_in_new_line:
self.to_screen('') self.to_screen('')
return proc.returncode return proc.wait()
except BaseException: # Including KeyboardInterrupt
proc.kill()
proc.wait()
raise
url = info_dict['url'] url = info_dict['url']
player_url = info_dict.get('player_url') player_url = info_dict.get('player_url')

View File

@ -16,6 +16,7 @@ from ..utils import (
ExtractorError, ExtractorError,
get_exe_version, get_exe_version,
is_outdated_version, is_outdated_version,
process_communicate_or_kill,
std_headers, std_headers,
) )
@ -226,7 +227,7 @@ class PhantomJSwrapper(object):
self.exe, '--ssl-protocol=any', self.exe, '--ssl-protocol=any',
self._TMP_FILES['script'].name self._TMP_FILES['script'].name
], stdout=subprocess.PIPE, stderr=subprocess.PIPE) ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate() out, err = process_communicate_or_kill(p)
if p.returncode != 0: if p.returncode != 0:
raise ExtractorError( raise ExtractorError(
'Executing JS failed\n:' + encodeArgument(err)) 'Executing JS failed\n:' + encodeArgument(err))

View File

@ -13,8 +13,9 @@ from ..utils import (
encodeFilename, encodeFilename,
PostProcessingError, PostProcessingError,
prepend_extension, prepend_extension,
process_communicate_or_kill,
replace_extension, replace_extension,
shell_quote shell_quote,
) )
@ -109,7 +110,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
self._downloader.to_screen('[debug] AtomicParsley command line: %s' % shell_quote(cmd)) self._downloader.to_screen('[debug] AtomicParsley command line: %s' % shell_quote(cmd))
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate() stdout, stderr = process_communicate_or_kill(p)
if p.returncode != 0: if p.returncode != 0:
msg = stderr.decode('utf-8', 'replace').strip() msg = stderr.decode('utf-8', 'replace').strip()

View File

@ -16,6 +16,7 @@ from ..utils import (
is_outdated_version, is_outdated_version,
PostProcessingError, PostProcessingError,
prepend_extension, prepend_extension,
process_communicate_or_kill,
shell_quote, shell_quote,
subtitles_filename, subtitles_filename,
dfxp2srt, dfxp2srt,
@ -180,7 +181,7 @@ class FFmpegPostProcessor(PostProcessor):
handle = subprocess.Popen( handle = subprocess.Popen(
cmd, stderr=subprocess.PIPE, cmd, stderr=subprocess.PIPE,
stdout=subprocess.PIPE, stdin=subprocess.PIPE) stdout=subprocess.PIPE, stdin=subprocess.PIPE)
stdout_data, stderr_data = handle.communicate() stdout_data, stderr_data = process_communicate_or_kill(handle)
expected_ret = 0 if self.probe_available else 1 expected_ret = 0 if self.probe_available else 1
if handle.wait() != expected_ret: if handle.wait() != expected_ret:
return None return None
@ -228,7 +229,7 @@ class FFmpegPostProcessor(PostProcessor):
if self._downloader.params.get('verbose', False): if self._downloader.params.get('verbose', False):
self._downloader.to_screen('[debug] ffmpeg command line: %s' % shell_quote(cmd)) self._downloader.to_screen('[debug] ffmpeg command line: %s' % shell_quote(cmd))
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
stdout, stderr = p.communicate() stdout, stderr = process_communicate_or_kill(p)
if p.returncode != 0: if p.returncode != 0:
stderr = stderr.decode('utf-8', 'replace') stderr = stderr.decode('utf-8', 'replace')
msgs = stderr.strip().split('\n') msgs = stderr.strip().split('\n')

View File

@ -2212,6 +2212,15 @@ def unescapeHTML(s):
r'&([^&;]+;)', lambda m: _htmlentity_transform(m.group(1)), s) r'&([^&;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
def process_communicate_or_kill(p, *args, **kwargs):
try:
return p.communicate(*args, **kwargs)
except BaseException: # Including KeyboardInterrupt
p.kill()
p.wait()
raise
def get_subprocess_encoding(): def get_subprocess_encoding():
if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5: if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
# For subprocess calls, encode with locale encoding # For subprocess calls, encode with locale encoding
@ -3788,7 +3797,8 @@ def check_executable(exe, args=[]):
""" Checks if the given binary is installed somewhere in PATH, and returns its name. """ Checks if the given binary is installed somewhere in PATH, and returns its name.
args can be a list of arguments for a short output (like -version) """ args can be a list of arguments for a short output (like -version) """
try: try:
subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() process_communicate_or_kill(subprocess.Popen(
[exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE))
except OSError: except OSError:
return False return False
return exe return exe
@ -3802,10 +3812,10 @@ def get_exe_version(exe, args=['--version'],
# STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers # STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers
# SIGTTOU if youtube-dl is run in the background. # SIGTTOU if youtube-dl is run in the background.
# See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656 # See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656
out, _ = subprocess.Popen( out, _ = process_communicate_or_kill(subprocess.Popen(
[encodeArgument(exe)] + args, [encodeArgument(exe)] + args,
stdin=subprocess.PIPE, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate() stdout=subprocess.PIPE, stderr=subprocess.STDOUT))
except OSError: except OSError:
return False return False
if isinstance(out, bytes): # Python 2.x if isinstance(out, bytes): # Python 2.x
@ -5744,7 +5754,7 @@ def write_xattr(path, key, value):
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
except EnvironmentError as e: except EnvironmentError as e:
raise XAttrMetadataError(e.errno, e.strerror) raise XAttrMetadataError(e.errno, e.strerror)
stdout, stderr = p.communicate() stdout, stderr = process_communicate_or_kill(p)
stderr = stderr.decode('utf-8', 'replace') stderr = stderr.decode('utf-8', 'replace')
if p.returncode != 0: if p.returncode != 0:
raise XAttrMetadataError(p.returncode, stderr) raise XAttrMetadataError(p.returncode, stderr)