Coverage report: + 55% +
+
+
+
+
+
+
+ 
+ Hot-keys on this page
++ n + s + m + x + + c change column sorting +
+| Module | +statements | +missing | +excluded | + +coverage | +
|---|---|---|---|---|
| Total | +3474 | +1568 | +0 | + +55% | +
| youtube_dl | +330 | +303 | +0 | + +8% | +
| youtube_dl.FileDownloader | +477 | +209 | +0 | + +56% | +
| youtube_dl.InfoExtractors | +2213 | +923 | +0 | + +58% | +
| youtube_dl.PostProcessor | +130 | +111 | +0 | + +15% | +
| youtube_dl.utils | +323 | +22 | +0 | + +93% | +
| youtube_dl.version | +1 | +0 | +0 | + +100% | +
| t |
| + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | +
+ #!/usr/bin/env python +# -*- coding: utf-8 -*- ++ + + + + 'Ricardo Garcia Gonzalez', +'Danny Colligan', +'Benjamin Johnson', +'Vasyl\' Vavrychuk', +'Witold Baryluk', +'Paweł Paprota', +'Gergely Imreh', +'Rogério Brito', +'Philipp Hagemeister', +'Sören Schulze', +'Kevin Ngo', +'Ori Avtalion', +'shizeeg', +'Filippo Valsorda', +'Christian Albrecht', +'Dave Vasilevsky', +) ++ + + + + + + + + + + + + + + + + + + + """Update the program file with the latest version from the repository""" ++ # TODO: at least, check https certificates ++ from zipimport import zipimporter ++ API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads" +BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl" +EXE_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl.exe" ++ if hasattr(sys, "frozen"): # PY2EXE +if not os.access(filename, os.W_OK): +sys.exit('ERROR: no write permissions on %s' % filename) ++ downloader.to_screen(u'Updating to latest version...') ++ urla = compat_urllib_request.urlopen(API_URL) +download = filter(lambda x: x["name"] == "youtube-dl.exe", json.loads(urla.read())) +if not download: +downloader.to_screen(u'ERROR: can\'t find the current version. Please try again later.') +return +newversion = download[0]["description"].strip() +if newversion == __version__: +downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') +return +urla.close() ++ exe = os.path.abspath(filename) +directory = os.path.dirname(exe) +if not os.access(directory, os.W_OK): +sys.exit('ERROR: no write permissions on %s' % directory) ++ try: +urlh = compat_urllib_request.urlopen(EXE_URL) +newcontent = urlh.read() +urlh.close() +with open(exe + '.new', 'wb') as outf: +outf.write(newcontent) +except (IOError, OSError) as err: +sys.exit('ERROR: unable to download latest version') ++ try: +bat = os.path.join(directory, 'youtube-dl-updater.bat') +b = open(bat, 'w') +b.write(""" +echo Updating youtube-dl... +ping 127.0.0.1 -n 5 -w 1000 > NUL +move /Y "%s.new" "%s" +del "%s" +\n""" %(exe, exe, bat)) +b.close() ++ os.startfile(bat) +except (IOError, OSError) as err: +sys.exit('ERROR: unable to overwrite current version') ++ elif isinstance(globals().get('__loader__'), zipimporter): # UNIX ZIP +if not os.access(filename, os.W_OK): +sys.exit('ERROR: no write permissions on %s' % filename) ++ downloader.to_screen(u'Updating to latest version...') ++ urla = compat_urllib_request.urlopen(API_URL) +download = [x for x in json.loads(urla.read().decode('utf8')) if x["name"] == "youtube-dl"] +if not download: +downloader.to_screen(u'ERROR: can\'t find the current version. Please try again later.') +return +newversion = download[0]["description"].strip() +if newversion == __version__: +downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') +return +urla.close() ++ try: +urlh = compat_urllib_request.urlopen(BIN_URL) +newcontent = urlh.read() +urlh.close() +except (IOError, OSError) as err: +sys.exit('ERROR: unable to download latest version') ++ try: +with open(filename, 'wb') as outf: +outf.write(newcontent) +except (IOError, OSError) as err: +sys.exit('ERROR: unable to overwrite current version') ++ else: +downloader.to_screen(u'It looks like you installed youtube-dl with pip or setup.py. Please use that to update.') +return ++ downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.') ++ + def _readOptions(filename_bytes): +try: +optionf = open(filename_bytes) +except IOError: +return [] # silently skip if file is not present +try: +res = [] +for l in optionf: +res += shlex.split(l, comments=True) +finally: +optionf.close() +return res ++ def _format_option_string(option): +''' ('-o', '--option') -> -o, --format METAVAR''' ++ opts = [] ++ if option._short_opts: +opts.append(option._short_opts[0]) +if option._long_opts: +opts.append(option._long_opts[0]) +if len(opts) > 1: +opts.insert(1, ', ') ++ if option.takes_value(): opts.append(' %s' % option.metavar) ++ return "".join(opts) ++ def _find_term_columns(): +columns = os.environ.get('COLUMNS', None) +if columns: +return int(columns) ++ try: +sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) +out,err = sp.communicate() +return int(out.split()[1]) +except: +pass +return None ++ max_width = 80 +max_help_position = 80 ++ # No need to wrap help messages if we're on a wide console +columns = _find_term_columns() +if columns: max_width = columns ++ fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position) +fmt.format_option_strings = _format_option_string ++ kw = { +'version' : __version__, +'formatter' : fmt, +'usage' : '%prog [options] url [url...]', +'conflict_handler' : 'resolve', +} ++ parser = optparse.OptionParser(**kw) ++ # option groups +general = optparse.OptionGroup(parser, 'General Options') +selection = optparse.OptionGroup(parser, 'Video Selection') +authentication = optparse.OptionGroup(parser, 'Authentication Options') +video_format = optparse.OptionGroup(parser, 'Video Format Options') +postproc = optparse.OptionGroup(parser, 'Post-processing Options') +filesystem = optparse.OptionGroup(parser, 'Filesystem Options') +verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options') ++ general.add_option('-h', '--help', +action='help', help='print this help text and exit') +general.add_option('-v', '--version', +action='version', help='print program version and exit') +general.add_option('-U', '--update', +action='store_true', dest='update_self', help='update this program to latest version') +general.add_option('-i', '--ignore-errors', +action='store_true', dest='ignoreerrors', help='continue on download errors', default=False) +general.add_option('-r', '--rate-limit', +dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)') +general.add_option('-R', '--retries', +dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10) +general.add_option('--buffer-size', +dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16k) (default is %default)', default="1024") +general.add_option('--no-resize-buffer', +action='store_true', dest='noresizebuffer', +help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False) +general.add_option('--dump-user-agent', +action='store_true', dest='dump_user_agent', +help='display the current browser identification', default=False) +general.add_option('--user-agent', +dest='user_agent', help='specify a custom user agent', metavar='UA') +general.add_option('--list-extractors', +action='store_true', dest='list_extractors', +help='List all supported extractors and the URLs they would handle', default=False) +general.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP) ++ selection.add_option('--playlist-start', +dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1) +selection.add_option('--playlist-end', +dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1) +selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)') +selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)') +selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None) ++ authentication.add_option('-u', '--username', +dest='username', metavar='USERNAME', help='account username') +authentication.add_option('-p', '--password', +dest='password', metavar='PASSWORD', help='account password') +authentication.add_option('-n', '--netrc', +action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False) ++ + video_format.add_option('-f', '--format', +action='store', dest='format', metavar='FORMAT', help='video format code') +video_format.add_option('--all-formats', +action='store_const', dest='format', help='download all available video formats', const='all') +video_format.add_option('--prefer-free-formats', +action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested') +video_format.add_option('--max-quality', +action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') +video_format.add_option('-F', '--list-formats', +action='store_true', dest='listformats', help='list all available formats (currently youtube only)') +video_format.add_option('--write-srt', +action='store_true', dest='writesubtitles', +help='write video closed captions to a .srt file (currently youtube only)', default=False) +video_format.add_option('--srt-lang', +action='store', dest='subtitleslang', metavar='LANG', +help='language of the closed captions to download (optional) use IETF language tags like \'en\'') ++ + verbosity.add_option('-q', '--quiet', +action='store_true', dest='quiet', help='activates quiet mode', default=False) +verbosity.add_option('-s', '--simulate', +action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False) +verbosity.add_option('--skip-download', +action='store_true', dest='skip_download', help='do not download the video', default=False) +verbosity.add_option('-g', '--get-url', +action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False) +verbosity.add_option('-e', '--get-title', +action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False) +verbosity.add_option('--get-thumbnail', +action='store_true', dest='getthumbnail', +help='simulate, quiet but print thumbnail URL', default=False) +verbosity.add_option('--get-description', +action='store_true', dest='getdescription', +help='simulate, quiet but print video description', default=False) +verbosity.add_option('--get-filename', +action='store_true', dest='getfilename', +help='simulate, quiet but print output filename', default=False) +verbosity.add_option('--get-format', +action='store_true', dest='getformat', +help='simulate, quiet but print output format', default=False) +verbosity.add_option('--no-progress', +action='store_true', dest='noprogress', help='do not print progress bar', default=False) +verbosity.add_option('--console-title', +action='store_true', dest='consoletitle', +help='display progress in console titlebar', default=False) +verbosity.add_option('-v', '--verbose', +action='store_true', dest='verbose', help='print various debugging information', default=False) ++ + filesystem.add_option('-t', '--title', +action='store_true', dest='usetitle', help='use title in file name', default=False) +filesystem.add_option('--id', +action='store_true', dest='useid', help='use video ID in file name', default=False) +filesystem.add_option('-l', '--literal', +action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False) +filesystem.add_option('-A', '--auto-number', +action='store_true', dest='autonumber', +help='number downloaded files starting from 00000', default=False) +filesystem.add_option('-o', '--output', +dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .') +filesystem.add_option('--restrict-filenames', +action='store_true', dest='restrictfilenames', +help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False) +filesystem.add_option('-a', '--batch-file', +dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)') +filesystem.add_option('-w', '--no-overwrites', +action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) +filesystem.add_option('-c', '--continue', +action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True) +filesystem.add_option('--no-continue', +action='store_false', dest='continue_dl', +help='do not resume partially downloaded files (restart from beginning)') +filesystem.add_option('--cookies', +dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in') +filesystem.add_option('--no-part', +action='store_true', dest='nopart', help='do not use .part files', default=False) +filesystem.add_option('--no-mtime', +action='store_false', dest='updatetime', +help='do not use the Last-modified header to set the file modification time', default=True) +filesystem.add_option('--write-description', +action='store_true', dest='writedescription', +help='write video description to a .description file', default=False) +filesystem.add_option('--write-info-json', +action='store_true', dest='writeinfojson', +help='write video metadata to a .info.json file', default=False) ++ + postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False, +help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)') +postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best', +help='"best", "aac", "vorbis", "mp3", "m4a", or "wav"; best by default') +postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5', +help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)') +postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False, +help='keeps the video file on disk after the post-processing; the video is erased by default') ++ + parser.add_option_group(general) +parser.add_option_group(selection) +parser.add_option_group(filesystem) +parser.add_option_group(verbosity) +parser.add_option_group(video_format) +parser.add_option_group(authentication) +parser.add_option_group(postproc) ++ xdg_config_home = os.environ.get('XDG_CONFIG_HOME') +if xdg_config_home: +userConf = os.path.join(xdg_config_home, 'youtube-dl.conf') +else: +userConf = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf') +argv = _readOptions('/etc/youtube-dl.conf') + _readOptions(userConf) + sys.argv[1:] +opts, args = parser.parse_args(argv) ++ return parser, opts, args ++ + """ Return a list of an instance of every supported extractor. +The order does matter; the first extractor matched is the one handling the URL. +""" + +YoutubePlaylistIE(), +YoutubeChannelIE(), +YoutubeUserIE(), +YoutubeSearchIE(), +YoutubeIE(), +MetacafeIE(), +DailymotionIE(), +GoogleSearchIE(), +PhotobucketIE(), +YahooIE(), +YahooSearchIE(), +DepositFilesIE(), +FacebookIE(), +BlipTVUserIE(), +BlipTVIE(), +VimeoIE(), +MyVideoIE(), +ComedyCentralIE(), +EscapistIE(), +CollegeHumorIE(), +XVideosIE(), +SoundcloudIE(), +InfoQIE(), +MixcloudIE(), +StanfordOpenClassroomIE(), +MTVIE(), +YoukuIE(), +XNXXIE(), +GooglePlusIE(), +ArteTvIE(), +NBAIE(), +JustinTVIE(), +GenericIE() +] ++ + + + # Open appropriate CookieJar +if opts.cookiefile is None: +jar = compat_cookiejar.CookieJar() +else: +try: +jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile) +if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK): +jar.load() +except (IOError, OSError) as err: +sys.exit(u'ERROR: unable to open cookie file') +# Set user agent +if opts.user_agent is not None: +std_headers['User-Agent'] = opts.user_agent ++ # Dump user agent +if opts.dump_user_agent: +print(std_headers['User-Agent']) +sys.exit(0) ++ # Batch file verification +batchurls = [] +if opts.batchfile is not None: +try: +if opts.batchfile == '-': +batchfd = sys.stdin +else: +batchfd = open(opts.batchfile, 'r') +batchurls = batchfd.readlines() +batchurls = [x.strip() for x in batchurls] +batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)] +except IOError: +sys.exit(u'ERROR: batch file could not be read') +all_urls = batchurls + args +all_urls = [url.strip() for url in all_urls] ++ # General configuration +cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar) +proxy_handler = compat_urllib_request.ProxyHandler() +opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler()) +compat_urllib_request.install_opener(opener) +socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words) ++ extractors = gen_extractors() ++ if opts.list_extractors: +for ie in extractors: +print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '')) +matchedUrls = filter(lambda url: ie.suitable(url), all_urls) +all_urls = filter(lambda url: url not in matchedUrls, all_urls) +for mu in matchedUrls: +print(u' ' + mu) +sys.exit(0) ++ # Conflicting, missing and erroneous options +if opts.usenetrc and (opts.username is not None or opts.password is not None): +parser.error(u'using .netrc conflicts with giving username/password') +if opts.password is not None and opts.username is None: +parser.error(u'account username missing') +if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid): +parser.error(u'using output template conflicts with using title, video ID or auto number') +if opts.usetitle and opts.useid: +parser.error(u'using title conflicts with using video ID') +if opts.username is not None and opts.password is None: +opts.password = getpass.getpass(u'Type account password and press return:') +if opts.ratelimit is not None: +numeric_limit = FileDownloader.parse_bytes(opts.ratelimit) +if numeric_limit is None: +parser.error(u'invalid rate limit specified') +opts.ratelimit = numeric_limit +if opts.retries is not None: +try: +opts.retries = int(opts.retries) +except (TypeError, ValueError) as err: +parser.error(u'invalid retry count specified') +if opts.buffersize is not None: +numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize) +if numeric_buffersize is None: +parser.error(u'invalid buffer size specified') +opts.buffersize = numeric_buffersize +try: +opts.playliststart = int(opts.playliststart) +if opts.playliststart <= 0: +raise ValueError(u'Playlist start must be positive') +except (TypeError, ValueError) as err: +parser.error(u'invalid playlist start number specified') +try: +opts.playlistend = int(opts.playlistend) +if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart): +raise ValueError(u'Playlist end must be greater than playlist start') +except (TypeError, ValueError) as err: +parser.error(u'invalid playlist end number specified') +if opts.extractaudio: +if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a', 'wav']: +parser.error(u'invalid audio format specified') +if opts.audioquality: +opts.audioquality = opts.audioquality.strip('k').strip('K') +if not opts.audioquality.isdigit(): +parser.error(u'invalid audio quality specified') ++ if sys.version_info < (3,): +# In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems) +if opts.outtmpl is not None: +opts.outtmpl = opts.outtmpl.decode(preferredencoding()) +outtmpl =((opts.outtmpl is not None and opts.outtmpl) +or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s') +or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s') +or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s') +or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s') +or (opts.useid and u'%(id)s.%(ext)s') +or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s') +or u'%(id)s.%(ext)s') +# File downloader +fd = FileDownloader({ +'usenetrc': opts.usenetrc, +'username': opts.username, +'password': opts.password, +'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat), +'forceurl': opts.geturl, +'forcetitle': opts.gettitle, +'forcethumbnail': opts.getthumbnail, +'forcedescription': opts.getdescription, +'forcefilename': opts.getfilename, +'forceformat': opts.getformat, +'simulate': opts.simulate, +'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat), +'format': opts.format, +'format_limit': opts.format_limit, +'listformats': opts.listformats, +'outtmpl': outtmpl, +'restrictfilenames': opts.restrictfilenames, +'ignoreerrors': opts.ignoreerrors, +'ratelimit': opts.ratelimit, +'nooverwrites': opts.nooverwrites, +'retries': opts.retries, +'buffersize': opts.buffersize, +'noresizebuffer': opts.noresizebuffer, +'continuedl': opts.continue_dl, +'noprogress': opts.noprogress, +'playliststart': opts.playliststart, +'playlistend': opts.playlistend, +'logtostderr': opts.outtmpl == '-', +'consoletitle': opts.consoletitle, +'nopart': opts.nopart, +'updatetime': opts.updatetime, +'writedescription': opts.writedescription, +'writeinfojson': opts.writeinfojson, +'writesubtitles': opts.writesubtitles, +'subtitleslang': opts.subtitleslang, +'matchtitle': opts.matchtitle, +'rejecttitle': opts.rejecttitle, +'max_downloads': opts.max_downloads, +'prefer_free_formats': opts.prefer_free_formats, +'verbose': opts.verbose, +'test': opts.test, +}) ++ if opts.verbose: +fd.to_screen(u'[debug] Proxy map: ' + str(proxy_handler.proxies)) ++ for extractor in extractors: +fd.add_info_extractor(extractor) ++ # PostProcessors +if opts.extractaudio: +fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, keepvideo=opts.keepvideo)) ++ # Update version +if opts.update_self: +updateSelf(fd, sys.argv[0]) ++ # Maybe do nothing +if len(all_urls) < 1: +if not opts.update_self: +parser.error(u'you must provide at least one URL') +else: +sys.exit() ++ try: +retcode = fd.download(all_urls) +except MaxDownloadsReached: +fd.to_screen(u'--max-download limit reached, aborting.') +retcode = 101 ++ # Dump cookie jar if requested +if opts.cookiefile is not None: + +jar.save() +except (IOError, OSError) as err: +sys.exit(u'ERROR: unable to save cookie jar') ++ sys.exit(retcode) ++ + + _real_main() +except DownloadError: +sys.exit(1) +except SameFileError: +sys.exit(u'ERROR: fixed output name but more than one file to download') +except KeyboardInterrupt: +sys.exit(u'\nERROR: Interrupted by user') + + |
+
+
+Hot-keys on this page
++ r + m + x + p toggle line displays +
++ j + k next/prev highlighted chunk +
++ 0 (zero) top of page +
++ 1 (one) first highlighted chunk +
+| + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | +
+ #!/usr/bin/env python +# -*- coding: utf-8 -*- ++ + + + + + + + + + + + + import ctypes ++ + + + + """File Downloader class. ++ File downloader objects are the ones responsible of downloading the +actual video file and writing it to disk if the user has requested +it, among some other tasks. In most cases there should be one per +program. As, given a video URL, the downloader doesn't know how to +extract all the needed information, task that InfoExtractors do, it +has to pass the URL to one of them. ++ For this, file downloader objects have a method that allows +InfoExtractors to be registered in a given order. When it is passed +a URL, the file downloader handles it to the first InfoExtractor it +finds that reports being able to handle it. The InfoExtractor extracts +all the information about the video or videos the URL refers to, and +asks the FileDownloader to process the video information, possibly +downloading the video. ++ File downloaders accept a lot of parameters. In order not to saturate +the object constructor with arguments, it receives a dictionary of +options instead. These options are available through the params +attribute for the InfoExtractors to use. The FileDownloader also +registers itself as the downloader in charge for the InfoExtractors +that are added to it, so this is a "mutual registration". ++ Available options: ++ username: Username for authentication purposes. +password: Password for authentication purposes. +usenetrc: Use netrc for authentication instead. +quiet: Do not print messages to stdout. +forceurl: Force printing final URL. +forcetitle: Force printing title. +forcethumbnail: Force printing thumbnail URL. +forcedescription: Force printing description. +forcefilename: Force printing final filename. +simulate: Do not download the video files. +format: Video format code. +format_limit: Highest quality format to try. +outtmpl: Template for output names. +restrictfilenames: Do not allow "&" and spaces in file names +ignoreerrors: Do not stop on download errors. +ratelimit: Download speed limit, in bytes/sec. +nooverwrites: Prevent overwriting files. +retries: Number of times to retry for HTTP error 5xx +buffersize: Size of download buffer in bytes. +noresizebuffer: Do not automatically resize the download buffer. +continuedl: Try to continue downloads if possible. +noprogress: Do not print the progress bar. +playliststart: Playlist item to start at. +playlistend: Playlist item to end at. +matchtitle: Download only matching titles. +rejecttitle: Reject downloads for matching titles. +logtostderr: Log messages to stderr instead of stdout. +consoletitle: Display progress in console window's titlebar. +nopart: Do not use temporary .part files. +updatetime: Use the Last-modified header to set output file timestamps. +writedescription: Write the video description to a .description file +writeinfojson: Write the video description to a .info.json file +writesubtitles: Write the video subtitles to a .srt file +subtitleslang: Language of the subtitles to download +test: Download only first bytes to test the downloader. +""" ++ + + + + + + + + """Create a FileDownloader object with the given options.""" + + + + + + ++ + self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.') ++ + def format_bytes(bytes): + +return 'N/A' + +bytes = float(bytes) + +exponent = 0 +else: + + + + ++ + def calc_percent(byte_counter, data_len): + +return '---.-%' + ++ + def calc_eta(start, now, total, current): + +return '--:--' + + + + + + + +return '--:--' + ++ + def calc_speed(start, now, bytes): + + + + ++ + def best_block_size(elapsed_time, bytes): + + + + + + + + + + ++ + def parse_bytes(bytestr): +"""Parse a string indicating a byte quantity into an integer.""" +matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr) +if matchobj is None: +return None +number = float(matchobj.group(1)) +multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) +return int(round(number * multiplier)) ++ + """Add an InfoExtractor object to the end of the list.""" + + ++ + """Add a PostProcessor object to the end of the chain.""" +self._pps.append(pp) +pp.set_downloader(self) ++ + """Print message to stdout if not in quiet mode.""" + + + + + + + + ++ + """Print message to stderr.""" +assert type(message) == type(u'') +output = message + u'\n' +if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr +output = output.encode(preferredencoding()) +sys.stderr.write(output) ++ + """Set console/terminal window title to message.""" + + +if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow(): +# c_wchar_p() might not be necessary if `message` is +# already of type unicode() +ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message)) +elif 'TERM' in os.environ: +sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding())) ++ + """Checks if the output template is fixed.""" +return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None) ++ + """Determine action to take when a download problem appears. ++ Depending on if the downloader has been configured to ignore +download errors or not, this method may throw an exception or +not when errors are found, after printing the message. +""" +if message is not None: +self.to_stderr(message) +if self.params.get('verbose'): +self.to_stderr(u''.join(traceback.format_list(traceback.extract_stack()))) +if not self.params.get('ignoreerrors', False): +raise DownloadError(message) +self._download_retcode = 1 ++ + """Sleep if the download speed is over the rate limit.""" + + + +now = time.time() +elapsed = now - start_time +if elapsed <= 0.0: +return +speed = float(byte_counter) / elapsed +if speed > rate_limit: +time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) ++ + """Returns a temporary filename for the given filename.""" + +(os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))): +return filename + ++ + + + return filename ++ + + + return + +except (IOError, OSError) as err: +self.trouble(u'ERROR: unable to rename file') ++ + """Try to set the last-modified time of the given file.""" + +return + +return + + +return + + +return filetime + + +except: +pass + ++ + """ Report that the description file is being written """ +self.to_screen(u'[info] Writing video description to: ' + descfn) ++ + """ Report that the subtitles file is being written """ + ++ + """ Report that the metadata file has been written """ +self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn) ++ + """Report destination filename.""" + ++ + """Report download progress.""" + +return + +(percent_str, data_len_str, speed_str, eta_str), skip_eol=True) + +(percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip())) ++ + """Report attempt to resume at given byte.""" +self.to_screen(u'[download] Resuming download at byte %s' % resume_len) ++ + """Report retry in case of HTTP error 5xx""" +self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) ++ + """Report file has already been fully downloaded.""" + + +except (UnicodeEncodeError) as err: +self.to_screen(u'[download] The file has already been downloaded') ++ + """Report it was impossible to resume download.""" +self.to_screen(u'[download] Unable to resume') ++ + """Report download finished.""" + +self.to_screen(u'[download] Download completed') +else: + ++ + """Increment the ordinal that assigns a number to each file.""" + ++ + """Generate the output filename.""" + + ++ + + + + u'NA' if v is None else compat_str(v), +restricted=self.params.get('restrictfilenames'), +is_id=(k==u'id')) + ++ + + except (ValueError, KeyError) as err: +self.trouble(u'ERROR: invalid system charset or erroneous output template') +return None ++ + """ Returns None iff the file should be downloaded """ ++ + + + matchtitle = matchtitle.decode('utf8') +if not re.search(matchtitle, title, re.IGNORECASE): +return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"' + + +rejecttitle = rejecttitle.decode('utf8') +if re.search(rejecttitle, title, re.IGNORECASE): +return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' + ++ + """Process a single dictionary returned by an InfoExtractor.""" ++ # Keep for backwards compatibility + ++ + + + + + self.to_screen(u'[download] ' + reason) +return ++ + + if self._num_downloads > int(max_downloads): +raise MaxDownloadsReached() ++ + + # Forced printings + +compat_print(info_dict['title']) + +compat_print(info_dict['url']) + +compat_print(info_dict['thumbnail']) + +compat_print(info_dict['description']) + +compat_print(filename) + +compat_print(info_dict['format']) ++ # Do nothing else if in simulate mode + +return ++ + return ++ + + + os.makedirs(dn) +except (OSError, IOError) as err: +self.trouble(u'ERROR: unable to create directory ' + compat_str(err)) +return ++ + try: +descfn = filename + u'.description' +self.report_writedescription(descfn) +descfile = open(encodeFilename(descfn), 'wb') +try: +descfile.write(info_dict['description'].encode('utf-8')) +finally: +descfile.close() +except (OSError, IOError): +self.trouble(u'ERROR: Cannot write description file ' + descfn) +return ++ + # subtitles download errors are already managed as troubles in relevant IE +# that way it will silently go on when used with unsupporting IE + + + + + + +finally: + +except (OSError, IOError): +self.trouble(u'ERROR: Cannot write subtitles file ' + descfn) +return ++ + infofn = filename + u'.info.json' +self.report_writeinfojson(infofn) +try: +json.dump +except (NameError,AttributeError): +self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.') +return +try: +infof = open(encodeFilename(infofn), 'w') +try: +json_info_dict = dict((k, info_dict[k]) for k in info_dict if not k in ['urlhandle']) +json.dump(json_info_dict, infof) +finally: +infof.close() +except (OSError, IOError): +self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn) +return ++ + + success = True +else: + + +except (OSError, IOError) as err: +raise UnavailableVideoError() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self.trouble(u'ERROR: unable to download video data: %s' % str(err)) +return +except (ContentTooShortError, ) as err: +self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) +return ++ + + + except (PostProcessingError) as err: +self.trouble(u'ERROR: postprocessing: %s' % str(err)) +return ++ + """Download a given list of URLs.""" + +raise SameFileError(self.params['outtmpl']) ++ + + + # Go to next InfoExtractor if not suitable + + ++ # Warn if the _WORKING attribute is False + +self.trouble(u'WARNING: the program functionality for this site has been marked as broken, ' +u'and will probably not work. If you want to go on, use the -i option.') ++ # Suitable InfoExtractor found + ++ # Extract information from URL and process it + + + + + + +except UnavailableVideoError: +self.trouble(u'\nERROR: unable to download video') ++ # Suitable InfoExtractor had been found; go to next URL + ++ + self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url) ++ + + + """Run the postprocessing chain on the given file.""" + + + +info = pp.run(info) +if info is None: +break ++ + self.report_destination(filename) +tmpfilename = self.temp_name(filename) ++ # Check for rtmpdump first +try: +subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT) +except (OSError, IOError): +self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run') +return False ++ # Download using rtmpdump. rtmpdump returns exit code 2 when +# the connection was interrumpted and resuming appears to be +# possible. This is part of rtmpdump's normal usage, AFAIK. +basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename] +args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)] +if self.params.get('verbose', False): +try: +import pipes +shell_quote = lambda args: ' '.join(map(pipes.quote, args)) +except ImportError: +shell_quote = repr +self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args)) +retval = subprocess.call(args) +while retval == 2 or retval == 1: +prevsize = os.path.getsize(encodeFilename(tmpfilename)) +self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True) +time.sleep(5.0) # This seems to be needed +retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) +cursize = os.path.getsize(encodeFilename(tmpfilename)) +if prevsize == cursize and retval == 1: +break +# Some rtmp streams seem abort after ~ 99.8%. Don't complain for those +if prevsize == cursize and retval == 2 and cursize > 1024: +self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.') +retval = 0 +break +if retval == 0: +self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename))) +self.try_rename(tmpfilename, filename) +return True +else: +self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval) +return False ++ + + + + # Check file already present + + + ++ # Attempt to download using rtmpdump + +return self._download_with_rtmpdump(filename, url, player_url) ++ + + + # Do not include the Accept-Encoding header + + + ++ + + + # Establish possible resume length + +resume_len = os.path.getsize(encodeFilename(tmpfilename)) +else: + ++ + + if self.params.get('continuedl', False): +self.report_resuming_byte(resume_len) +request.add_header('Range','bytes=%d-' % resume_len) +open_mode = 'ab' +else: +resume_len = 0 ++ + + + # Establish connection + + +data = info_dict['urlhandle'] + + +except (compat_urllib_error.HTTPError, ) as err: +if (err.code < 500 or err.code >= 600) and err.code != 416: +# Unexpected HTTP error +raise +elif err.code == 416: +# Unable to resume (requested range not satisfiable) +try: +# Open the connection again without the range header +data = compat_urllib_request.urlopen(basic_request) +content_length = data.info()['Content-Length'] +except (compat_urllib_error.HTTPError, ) as err: +if err.code < 500 or err.code >= 600: +raise +else: +# Examine the reported length +if (content_length is not None and +(resume_len - 100 < int(content_length) < resume_len + 100)): +# The file had already been fully downloaded. +# Explanation to the above condition: in issue #175 it was revealed that +# YouTube sometimes adds or removes a few bytes from the end of the file, +# changing the file size slightly and causing problems for some users. So +# I decided to implement a suggested change and consider the file +# completely downloaded if the file size differs less than 100 bytes from +# the one in the hard drive. +self.report_file_already_downloaded(filename) +self.try_rename(tmpfilename, filename) +return True +else: +# The length does not match, we start the download over +self.report_unable_to_resume() +open_mode = 'wb' +break +# Retry +count += 1 +if count <= retries: +self.report_retry(count, retries) ++ + self.trouble(u'ERROR: giving up after %s retries' % retries) +return False ++ + + + + + + + + # Download and write + + + + + + ++ # Open file just in time + + + + + + +except (OSError, IOError) as err: +self.trouble(u'ERROR: unable to open for writing: %s' % str(err)) +return False + + +except (IOError, OSError) as err: +self.trouble(u'\nERROR: unable to write data: %s' % str(err)) +return False + + ++ # Progress message + + +self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA') +else: + + + ++ # Apply rate limit + ++ + self.trouble(u'\nERROR: Did not get any data blocks') +return False + + + +raise ContentTooShortError(byte_counter, int(data_len)) + ++ # Update file modification time + + ++ + + |
+
+
+Hot-keys on this page
++ r + m + x + p toggle line displays +
++ j + k next/prev highlighted chunk +
++ 0 (zero) top of page +
++ 1 (one) first highlighted chunk +
+| + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | +
+ #!/usr/bin/env python +# -*- coding: utf-8 -*- ++ + + + + + + + + + + + + + + + + + """Information Extractor class. ++ Information extractors are the classes that, given a URL, extract +information about the video (or videos) the URL refers to. This +information includes the real video URL, the video title, author and +others. The information is stored in a dictionary which is then +passed to the FileDownloader. The FileDownloader processes this +information possibly downloading the video to the file system, among +other possible outcomes. ++ The dictionaries must include the following fields: ++ id: Video identifier. +url: Final video URL. +uploader: Nickname of the video uploader, unescaped. +upload_date: Video upload date (YYYYMMDD). +title: Video title, unescaped. +ext: Video filename extension. ++ The following fields are optional: ++ format: The video format, defaults to ext (used for --get-format) +thumbnail: Full URL to a video thumbnail image. +description: One-line video description. +player_url: SWF Player URL (used for rtmpdump). +subtitles: The .srt file contents. +urlhandle: [internal] The urlHandle to be used to download the file, +like returned by urllib.request.urlopen ++ The fields should all be Unicode strings. ++ Subclasses of this one should re-define the _real_initialize() and +_real_extract() methods and define a _VALID_URL regexp. +Probably, they should also be added to the list of extractors. ++ _real_extract() must return a *list* of information dictionaries as +described above. ++ Finally, the _WORKING attribute should be set to False for broken IEs +in order to warn the users and skip the tests. +""" ++ + + + + + """Constructor. Receives an optional downloader.""" + + ++ + """Receives a URL and returns True if suitable for this IE.""" + ++ + """Getter method for _WORKING.""" + ++ + """Initializes an instance (authentication, etc).""" + + + ++ + """Extracts URL information and returns it in list of dicts.""" + + ++ + """Sets the downloader for this IE.""" + ++ + """Real initialization process. Redefine in subclasses.""" + ++ + """Real extraction process. Redefine in subclasses.""" +pass ++ + + """Information extractor for youtube.com.""" ++ + ( +(?:https?://)? # http(s):// (optional) +(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/| +tube\.majestyc\.net/) # the various hostnames, with wildcard subdomains +(?:.*?\#/)? # handle anchor (#/) redirect urls +(?!view_play_list|my_playlists|artist|playlist) # ignore playlist URLs +(?: # the various things that can precede the ID: +(?:(?:v|embed|e)/) # v/ or embed/ or e/ +|(?: # or the v= param in all its forms +(?:watch(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx) +(?:\?|\#!?) # the params delimiter ? or # or #! +(?:.+&)? # any other preceding param (like /?s=tuff&v=xxxx) +v= +) +)? # optional -> youtube.com/xxxx is OK +)? # all until now is optional -> you can pass the naked ID +([0-9A-Za-z_-]+) # here is it! the YouTube video ID +(?(1).+)? # if we found the ID, everything can follow +$""" + + + + + +# Listed in order of quality + + + +'13': '3gp', +'17': 'mp4', +'18': 'mp4', +'22': 'mp4', +'37': 'mp4', +'38': 'video', # You actually don't know if this will be MOV, AVI or whatever +'43': 'webm', +'44': 'webm', +'45': 'webm', +'46': 'webm', +} + +'5': '240x400', +'6': '???', +'13': '???', +'17': '144x176', +'18': '360x640', +'22': '720x1280', +'34': '360x640', +'35': '480x854', +'37': '1080x1920', +'38': '3072x4096', +'43': '360x640', +'44': '480x854', +'45': '720x1280', +'46': '1080x1920', +} + ++ + """Receives a URL and returns True if suitable for this IE.""" + ++ + """Report attempt to set language.""" + ++ + """Report attempt to log in.""" +self._downloader.to_screen(u'[youtube] Logging in') ++ + """Report attempt to confirm age.""" +self._downloader.to_screen(u'[youtube] Confirming age') ++ + """Report attempt to download video webpage.""" + ++ + """Report attempt to download video info webpage.""" + ++ + """Report attempt to download video info webpage.""" + ++ + """Report attempt to extract video information.""" + ++ + """Report extracted video URL.""" +self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format)) ++ + """Indicate the download will use the RTMP protocol.""" +self._downloader.to_screen(u'[youtube] RTMP download detected') ++ + + + # TODO parse xml instead of regex + + + + + + + + + + + + ++ + print('Available formats:') +for x in formats: +print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???'))) ++ + + return ++ + + + + # Attempt to use provided username and password or .netrc data + +username = downloader_params['username'] +password = downloader_params['password'] + +try: +info = netrc.netrc().authenticators(self._NETRC_MACHINE) +if info is not None: +username = info[0] +password = info[2] +else: +raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) +except (IOError, netrc.NetrcParseError) as err: +self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err)) +return ++ # Set language + + + + +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err)) +return ++ # No authentication to be performed + + ++ # Log in +login_form = { +'current_form': 'loginForm', +'next': '/', +'action_login': 'Log In', +'username': username, +'password': password, +} +request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form)) +try: +self.report_login() +login_results = compat_urllib_request.urlopen(request).read().decode('utf-8') +if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None: +self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password') +return +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err)) +return ++ # Confirm age +age_form = { +'next_url': '/', +'action_confirm': 'Confirm', +} +request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form)) +try: +self.report_age_confirmation() +age_results = compat_urllib_request.urlopen(request).read().decode('utf-8') +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err)) +return ++ + # Extract original video URL from URL with redirection, like age verification, using next_url parameter + + +url = 'http://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/') ++ # Extract video id from URL + + +self._downloader.trouble(u'ERROR: invalid URL: %s' % url) +return + ++ # Get video webpage + + + + +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) +return ++ + + # Attempt to extract SWF player URL + + +player_url = re.sub(r'\\(.)', r'\1', mobj.group(1)) +else: + ++ # Get video info + + + +% (video_id, el_type)) + + + + + + + +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err)) +return + +if 'reason' in video_info: +self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0]) +else: +self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason') +return ++ # Check for "rental" videos + +self._downloader.trouble(u'ERROR: "rental" videos not supported') +return ++ # Start extracting information + ++ # uploader + +self._downloader.trouble(u'ERROR: unable to extract uploader nickname') +return + ++ # title + +self._downloader.trouble(u'ERROR: unable to extract video title') +return + ++ # thumbnail image + +self._downloader.trouble(u'WARNING: unable to extract video thumbnail') +video_thumbnail = '' +else: # don't panic if we can't find it + ++ # upload date + + + + + + + + + + ++ # description + + + +else: +video_description = '' ++ # closed captions + + + + + + + +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err)) + + + +raise Trouble(u'WARNING: video has no closed captions') + + + + +else: +srt_lang = srt_lang_list.keys()[0] + +raise Trouble(u'WARNING: no closed captions found in the specified language') + + + +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err)) + +raise Trouble(u'WARNING: unable to download video subtitles') + +except Trouble as trouble: +self._downloader.trouble(str(trouble)) ++ + self._downloader.trouble(u'WARNING: unable to extract video duration') +video_duration = '' +else: + ++ # token + ++ # Decide which formats to download + ++ + self.report_rtmp_download() +video_url_list = [(None, video_info['conn'][0])] + + + + + ++ + + + format_list = available_formats[available_formats.index(format_limit):] +else: + + + +self._downloader.trouble(u'ERROR: no known formats available for video') +return + +self._print_formats(existing_formats) +return + + +elif req_format == 'worst': +video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality +elif req_format in ('-1', 'all'): +video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats +else: +# Specific formats. We pick the first in a slash-delimeted sequence. +# For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'. +req_formats = req_format.split('/') +video_url_list = None +for rf in req_formats: +if rf in url_map: +video_url_list = [(rf, url_map[rf])] +break +if video_url_list is None: +self._downloader.trouble(u'ERROR: requested format not available') +return +else: +self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info') +return ++ + + # Extension + ++ + self._video_dimensions.get(format_param, '???')) ++ + 'id': video_id, +'url': video_real_url, +'uploader': video_uploader, +'upload_date': upload_date, +'title': video_title, +'ext': video_extension, +'format': video_format, +'thumbnail': video_thumbnail, +'description': video_description, +'player_url': player_url, +'subtitles': video_subtitles, +'duration': video_duration +}) + ++ + + """Information Extractor for metacafe.com.""" ++ + + + + + + + + + """Report disclaimer retrieval.""" + ++ + """Report attempt to confirm age.""" + ++ + """Report webpage download.""" +self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id) ++ + """Report information extraction.""" +self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id) ++ + # Retrieve disclaimer + + + + +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % compat_str(err)) +return ++ # Confirm age + +'filters': '0', +'submit': "Continue - I'm over 18", +} + + + + +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err)) +return ++ + # Extract id and simplified title from URL + + +self._downloader.trouble(u'ERROR: invalid URL: %s' % url) +return ++ + + # Check if video comes from YouTube + + + + ++ # Retrieve video webpage to extract further information +request = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id) +try: +self.report_download_webpage(video_id) +webpage = compat_urllib_request.urlopen(request).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err)) +return ++ # Extract URL, uploader and title from webpage +self.report_extraction(video_id) +mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage) +if mobj is not None: +mediaURL = compat_urllib_parse.unquote(mobj.group(1)) +video_extension = mediaURL[-3:] ++ # Extract gdaKey if available +mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage) +if mobj is None: +video_url = mediaURL +else: +gdaKey = mobj.group(1) +video_url = '%s?__gda__=%s' % (mediaURL, gdaKey) +else: +mobj = re.search(r' name="flashvars" value="(.*?)"', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract media URL') +return +vardict = compat_parse_qs(mobj.group(1)) +if 'mediaData' not in vardict: +self._downloader.trouble(u'ERROR: unable to extract media URL') +return +mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0]) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract media URL') +return +mediaURL = mobj.group(1).replace('\\/', '/') +video_extension = mediaURL[-3:] +video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2)) ++ mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract title') +return +video_title = mobj.group(1).decode('utf-8') ++ mobj = re.search(r'submitter=(.*?);', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract uploader nickname') +return +video_uploader = mobj.group(1) ++ return [{ +'id': video_id.decode('utf-8'), +'url': video_url.decode('utf-8'), +'uploader': video_uploader.decode('utf-8'), +'upload_date': None, +'title': video_title, +'ext': video_extension.decode('utf-8'), +}] ++ + + """Information Extractor for Dailymotion""" ++ + + + + + + + """Report webpage download.""" + ++ + """Report information extraction.""" + ++ + # Extract id and simplified title from URL + + +self._downloader.trouble(u'ERROR: invalid URL: %s' % url) +return ++ + + + + # Retrieve video webpage to extract further information + + + + + + +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err)) +return ++ # Extract URL, uploader and title from webpage + + + +self._downloader.trouble(u'ERROR: unable to extract media URL') + + ++ + + + + + else: +self._downloader.trouble(u'ERROR: unable to extract video URL') +return ++ + + self._downloader.trouble(u'ERROR: unable to extract video URL') +return ++ + + # TODO: support choosing qualities ++ + + self._downloader.trouble(u'ERROR: unable to extract title') + + ++ + + + # lookin for official user +mobj_official = re.search(r'<span rel="author"[^>]+?>([^<]+?)</span>', webpage) +if mobj_official is None: +self._downloader.trouble(u'WARNING: unable to extract uploader nickname') +else: +video_uploader = mobj_official.group(1) +else: + ++ + + + video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1) ++ + 'id': video_id, +'url': video_url, +'uploader': video_uploader, +'upload_date': video_upload_date, +'title': video_title, +'ext': video_extension, +}] ++ + + """Information extractor for photobucket.com.""" ++ + + + + InfoExtractor.__init__(self, downloader) ++ + """Report webpage download.""" +self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id) ++ + """Report information extraction.""" +self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id) ++ + # Extract id from URL +mobj = re.match(self._VALID_URL, url) +if mobj is None: +self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) +return ++ video_id = mobj.group(1) ++ video_extension = 'flv' ++ # Retrieve video webpage to extract further information +request = compat_urllib_request.Request(url) +try: +self.report_download_webpage(video_id) +webpage = compat_urllib_request.urlopen(request).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) +return ++ # Extract URL, uploader, and title from webpage +self.report_extraction(video_id) +mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract media URL') +return +mediaURL = compat_urllib_parse.unquote(mobj.group(1)) ++ video_url = mediaURL ++ mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract title') +return +video_title = mobj.group(1).decode('utf-8') ++ video_uploader = mobj.group(2).decode('utf-8') ++ return [{ +'id': video_id.decode('utf-8'), +'url': video_url.decode('utf-8'), +'uploader': video_uploader, +'upload_date': None, +'title': video_title, +'ext': video_extension.decode('utf-8'), +}] ++ + + """Information extractor for video.yahoo.com.""" ++ + # _VALID_URL matches all Yahoo! Video URLs +# _VPAGE_URL matches only the extractable '/watch/' URLs + + + ++ + InfoExtractor.__init__(self, downloader) ++ + """Report webpage download.""" +self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id) ++ + """Report information extraction.""" +self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id) ++ + # Extract ID from URL +mobj = re.match(self._VALID_URL, url) +if mobj is None: +self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) +return ++ + + + # Rewrite valid but non-extractable URLs as +# extractable English language /watch/ URLs + + +try: + + +self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) +return ++ + if mobj is None: +self._downloader.trouble(u'ERROR: Unable to extract id field') + + ++ mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: Unable to extract vid field') +return +yahoo_vid = mobj.group(1) ++ url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id) +return self._real_extract(url, new_video=False) ++ # Retrieve video webpage to extract further information +request = compat_urllib_request.Request(url) +try: +self.report_download_webpage(video_id) +webpage = compat_urllib_request.urlopen(request).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) +return ++ # Extract uploader and title from webpage +self.report_extraction(video_id) +mobj = re.search(r'<meta name="title" content="(.*)" />', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract video title') +return +video_title = mobj.group(1).decode('utf-8') ++ mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract video uploader') +return +video_uploader = mobj.group(1).decode('utf-8') ++ # Extract video thumbnail +mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract video thumbnail') +return +video_thumbnail = mobj.group(1).decode('utf-8') ++ # Extract video description +mobj = re.search(r'<meta name="description" content="(.*)" />', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract video description') +return +video_description = mobj.group(1).decode('utf-8') + + ++ # Extract video height and width +mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage) + + + + ++ + if mobj is None: + + +yv_video_width = mobj.group(1) ++ # Retrieve video playlist to extract media URL +# I'm not completely sure what all these options are, but we +# seem to need most of them, otherwise the server sends a 401. +yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents + + +'&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height + +'&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797') +try: +self.report_download_webpage(video_id) +webpage = compat_urllib_request.urlopen(request).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) +return ++ # Extract media URL from playlist XML +mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: Unable to extract media URL') +return +video_url = compat_urllib_parse.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8') +video_url = unescapeHTML(video_url) ++ return [{ +'id': video_id.decode('utf-8'), +'url': video_url, +'uploader': video_uploader, +'upload_date': None, +'title': video_title, +'ext': video_extension.decode('utf-8'), +'thumbnail': video_thumbnail.decode('utf-8'), +'description': video_description, +}] ++ + + """Information extractor for vimeo.com.""" ++ # _VALID_URL matches Vimeo URLs + + ++ + + + + """Report webpage download.""" + ++ + """Report information extraction.""" + ++ + # Extract ID from URL + + +self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) +return ++ + + # Retrieve video webpage to extract further information + + + + + +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) +return ++ # Now we begin extracting as much information as we can from what we +# retrieved. First we extract the information common to all extractors, +# and latter we extract those that are Vimeo specific. + ++ # Extract the config JSON + + + +except: +self._downloader.trouble(u'ERROR: unable to extract info section') +return ++ # Extract title + ++ # Extract uploader + ++ # Extract video thumbnail + ++ # Extract video description + + + ++ # Extract upload date + + + +video_upload_date = mobj.group(1) ++ # Vimeo specific: extract request signature and timestamp + + ++ # Vimeo specific: extract video codec and quality information +# First consider quality, then codecs, then take everything +# TODO bind to format param + + + + + + +elif 'sd' in config["video"]["files"][codec_name]: +files['sd'].append((codec_name, codec_extension, 'sd')) +else: +files['other'].append((codec_name, codec_extension, config["video"]["files"][codec_name][0])) ++ + + + + + + + else: + + ++ + %(video_id, sig, timestamp, video_quality, video_codec.upper()) ++ + 'id': video_id, +'url': video_url, +'uploader': video_uploader, +'upload_date': video_upload_date, +'title': video_title, +'ext': video_extension, +'thumbnail': video_thumbnail, +'description': video_description, +}] ++ + + """arte.tv information extractor.""" ++ + + + + + + + + + """Report webpage download.""" +self._downloader.to_screen(u'[arte.tv] %s: Downloading webpage' % video_id) ++ + """Report information extraction.""" +self._downloader.to_screen(u'[arte.tv] %s: Extracting information' % video_id) ++ + + + + + webpage = compat_urllib_request.urlopen(request).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) + +except ValueError as err: + + +return webpage ++ + page = self.fetch_webpage(url) + + ++ + + + + + + + + else: + ++ + + + video_lang = url.split('/')[-4] + +url, +r'src="(.*?/videothek_js.*?\.js)', +0, +[ +(1, 'url', u'ERROR: Invalid URL: %s' % url) +] +) +http_host = url.split('/')[2] +next_url = 'http://%s%s' % (http_host, compat_urllib_parse.unquote(info.get('url'))) + +next_url, +r'(s_artestras_scst_geoFRDE_' + video_lang + '.*?)\'.*?' + +'(http://.*?\.swf).*?' + +'(rtmp://.*?)\'', +re.DOTALL, +[ +(1, 'path', u'ERROR: could not extract video path: %s' % url), +(2, 'player', u'ERROR: could not extract video player: %s' % url), +(3, 'url', u'ERROR: could not extract video url: %s' % url) +] +) + ++ + + + url, +r'param name="movie".*?videorefFileUrl=(http[^\'"&]*)', +0, +[ +(1, 'url', u'ERROR: Invalid URL: %s' % url) +] +) + + +next_url, +r'<video lang="%s" ref="(http[^\'"&]*)' % video_lang, +0, +[ +(1, 'url', u'ERROR: Could not find <video> tag: %s' % url) +] +) + ++ + next_url, +r'<video id="(.*?)".*?>.*?' + +'<name>(.*?)</name>.*?' + +'<dateVideo>(.*?)</dateVideo>.*?' + +'<url quality="hd">(.*?)</url>', +re.DOTALL, +[ +(1, 'id', u'ERROR: could not extract video id: %s' % url), +(2, 'title', u'ERROR: could not extract video title: %s' % url), +(3, 'date', u'ERROR: could not extract video date: %s' % url), +(4, 'url', u'ERROR: could not extract video url: %s' % url) +] +) ++ return { +'id': info.get('id'), +'url': compat_urllib_parse.unquote(info.get('url')), +'uploader': u'arte.tv', +'upload_date': info.get('date'), +'title': info.get('title').decode('utf-8'), +'ext': u'mp4', +'format': u'NA', +'player_url': None, +} ++ + + self.report_extraction(video_id) ++ if re.search(self._LIVE_URL, video_id) is not None: +self.extractLiveStream(url) +return +else: +info = self.extractPlus7Stream(url) ++ return [info] ++ + + """Generic last-resort information extractor.""" ++ + + + + InfoExtractor.__init__(self, downloader) ++ + """Report webpage download.""" +self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.') +self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id) ++ + """Report information extraction.""" +self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id) ++ + """Report information extraction.""" +self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url) ++ + """Check if it is a redirect, like url shorteners, in case restart chain.""" +class HeadRequest(compat_urllib_request.Request): +def get_method(self): +return "HEAD" ++ class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler): +""" +Subclass the HTTPRedirectHandler to make it use our +HeadRequest also on the redirected URL +""" +def redirect_request(self, req, fp, code, msg, headers, newurl): +if code in (301, 302, 303, 307): +newurl = newurl.replace(' ', '%20') +newheaders = dict((k,v) for k,v in req.headers.items() +if k.lower() not in ("content-length", "content-type")) +return HeadRequest(newurl, +headers=newheaders, +origin_req_host=req.get_origin_req_host(), +unverifiable=True) +else: +raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp) ++ class HTTPMethodFallback(compat_urllib_request.BaseHandler): +""" +Fallback to GET if HEAD is not allowed (405 HTTP error) +""" +def http_error_405(self, req, fp, code, msg, headers): +fp.read() +fp.close() ++ newheaders = dict((k,v) for k,v in req.headers.items() +if k.lower() not in ("content-length", "content-type")) +return self.parent.open(compat_urllib_request.Request(req.get_full_url(), +headers=newheaders, +origin_req_host=req.get_origin_req_host(), +unverifiable=True)) ++ # Build our opener +opener = compat_urllib_request.OpenerDirector() +for handler in [compat_urllib_request.HTTPHandler, compat_urllib_request.HTTPDefaultErrorHandler, +HTTPMethodFallback, HEADRedirectHandler, +compat_urllib_error.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]: +opener.add_handler(handler()) ++ response = opener.open(HeadRequest(url)) +new_url = response.geturl() ++ if url == new_url: +return False ++ self.report_following_redirect(new_url) +self._downloader.download([new_url]) +return True ++ + if self._test_redirect(url): return ++ video_id = url.split('/')[-1] +request = compat_urllib_request.Request(url) +try: +self.report_download_webpage(video_id) +webpage = compat_urllib_request.urlopen(request).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) +return +except ValueError as err: +# since this is the last-resort InfoExtractor, if +# this error is thrown, it'll be thrown here + + ++ + # Start with something easy: JW Player in SWFObject + +if mobj is None: +# Broaden the search a little bit + +if mobj is None: + +return ++ # It's possible that one of the regexes +# matched, but returned an empty group: + +self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + ++ + video_id = os.path.basename(video_url) ++ # here's a fun little line of code for you: + +video_id = os.path.splitext(video_id)[0] ++ # it's tempting to parse this further, but you would +# have to take into account all the variations like +# Video Title - Site Name +# Site Name | Video Title +# Video Title - Tagline | Site Name +# and so on and so forth; it's just not practical +mobj = re.search(r'<title>(.*)</title>', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract title') +return +video_title = mobj.group(1) ++ # video uploader is domain name +mobj = re.match(r'(?:https?://)?([^/]*)/.*', url) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract title') +return +video_uploader = mobj.group(1) ++ return [{ +'id': video_id, +'url': video_url, +'uploader': video_uploader, +'upload_date': None, +'title': video_title, +'ext': video_extension, +}] ++ + + """Information Extractor for YouTube search queries.""" + + + + ++ + InfoExtractor.__init__(self, downloader) ++ + """Report attempt to download search page with given number.""" +query = query.decode(preferredencoding()) +self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum)) ++ + mobj = re.match(self._VALID_URL, query) +if mobj is None: +self._downloader.trouble(u'ERROR: invalid search query "%s"' % query) +return ++ + prefix = prefix[8:] + +if prefix == '': +self._download_n_results(query, 1) +return +elif prefix == 'all': +self._download_n_results(query, self._max_youtube_results) +return +else: +try: +n = int(prefix) +if n <= 0: +self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) +return +elif n > self._max_youtube_results: +self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n)) +n = self._max_youtube_results +self._download_n_results(query, n) +return +except ValueError: # parsing prefix as integer fails +self._download_n_results(query, 1) +return ++ + """Downloads a specified number of results for a query""" ++ video_ids = [] +pagenum = 0 +limit = n ++ while (50 * pagenum) < limit: +self.report_download_page(query, pagenum+1) +result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1) +request = compat_urllib_request.Request(result_url) +try: +data = compat_urllib_request.urlopen(request).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download API page: %s' % compat_str(err)) +return +api_response = json.loads(data)['data'] ++ new_ids = list(video['id'] for video in api_response['items']) +video_ids += new_ids ++ limit = min(n, api_response['totalItems']) +pagenum += 1 ++ if len(video_ids) > n: +video_ids = video_ids[:n] +for id in video_ids: +self._downloader.download(['http://www.youtube.com/watch?v=%s' % id]) +return ++ + + """Information Extractor for Google Video search queries.""" + + + + + + ++ + InfoExtractor.__init__(self, downloader) ++ + """Report attempt to download playlist page with given number.""" + + ++ + + + + + + + + query = query.encode('utf-8') + +self._download_n_results(query, 1) +return + +self._download_n_results(query, self._max_google_results) + +else: +try: +n = int(prefix) +if n <= 0: +self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) +return +elif n > self._max_google_results: +self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n)) +n = self._max_google_results +self._download_n_results(query, n) +return +except ValueError: # parsing prefix as integer fails +self._download_n_results(query, 1) +return ++ + """Downloads a specified number of results for a query""" ++ video_ids = [] +pagenum = 0 ++ while True: +self.report_download_page(query, pagenum) +result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum*10) +request = compat_urllib_request.Request(result_url) +try: +page = compat_urllib_request.urlopen(request).read() + +self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) + ++ # Extract video identifiers +for mobj in re.finditer(self._VIDEO_INDICATOR, page): +video_id = mobj.group(1) +if video_id not in video_ids: +video_ids.append(video_id) +if len(video_ids) == n: +# Specified n videos reached +for id in video_ids: +self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id]) +return ++ if re.search(self._MORE_PAGES_INDICATOR, page) is None: +for id in video_ids: +self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id]) +return ++ pagenum = pagenum + 1 ++ + + """Information Extractor for Yahoo! Video search queries.""" ++ + + + + + + + + + + + + """Report attempt to download playlist page with given number.""" + + ++ + + if mobj is None: + +return ++ + prefix = prefix[8:] + +if prefix == '': +self._download_n_results(query, 1) +return +elif prefix == 'all': +self._download_n_results(query, self._max_yahoo_results) +return +else: +try: +n = int(prefix) +if n <= 0: +self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) +return +elif n > self._max_yahoo_results: +self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n)) +n = self._max_yahoo_results +self._download_n_results(query, n) +return +except ValueError: # parsing prefix as integer fails +self._download_n_results(query, 1) +return ++ + """Downloads a specified number of results for a query""" ++ video_ids = [] +already_seen = set() +pagenum = 1 ++ while True: + +result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum) +request = compat_urllib_request.Request(result_url) +try: +page = compat_urllib_request.urlopen(request).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) +return ++ # Extract video identifiers +for mobj in re.finditer(self._VIDEO_INDICATOR, page): +video_id = mobj.group(1) +if video_id not in already_seen: +video_ids.append(video_id) +already_seen.add(video_id) +if len(video_ids) == n: +# Specified n videos reached +for id in video_ids: +self._downloader.download(['http://video.yahoo.com/watch/%s' % id]) +return ++ if re.search(self._MORE_PAGES_INDICATOR, page) is None: +for id in video_ids: +self._downloader.download(['http://video.yahoo.com/watch/%s' % id]) +return ++ pagenum = pagenum + 1 ++ + + """Information Extractor for YouTube playlists.""" ++ + + + + + + + + + + """Report attempt to download playlist page with given number.""" + ++ + # Extract playlist id + + +self._downloader.trouble(u'ERROR: invalid url: %s' % url) + ++ # Single video case + +self._downloader.download([mobj.group(3)]) +return ++ # Download playlist pages +# prefix is 'p' as default for playlists but there are other types that need extra care + + +playlist_access = 'artist' +else: + + + + + ++ + + + + + + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) +return ++ # Extract video identifiers + + + + + ++ + + + + + + + + + + else: +video_ids = video_ids[playliststart:playlistend] ++ + + else: +self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(video_ids))) ++ + + + + + + """Information Extractor for YouTube channels.""" ++ + + + + + + """Report attempt to download channel page with given number.""" + ++ + # Extract channel id + + + + ++ # Download channel pages + +video_ids = [] + ++ + self.report_download_page(channel_id, pagenum) + + + + +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) + ++ # Extract video identifiers +ids_in_page = [] +for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&', page): +if mobj.group(1) not in ids_in_page: + + ++ + + + + + + + + + + + + """Information Extractor for YouTube users.""" ++ + + + + + + + + + + + """Report attempt to download user page.""" + +(username, start_index, start_index + self._GDATA_PAGE_SIZE)) ++ + # Extract username + + + + ++ + + # Download video ids using YouTube Data API. Result size per +# query is limited (currently to 50 videos) so we need to query +# page by page until there are no video ids - it means we got +# all of them. ++ + + + + + + + + + + + + + return ++ # Extract video identifiers + ++ + + + + + + # A little optimization - if current page is not +# "full", ie. does not contain PAGE_SIZE video ids then +# we can assume that this page is the last one - there +# are no more ids on further pages - no need to query +# again. ++ + + + + + + + + + + + else: +video_ids = video_ids[playliststart:playlistend] ++ + (username, all_ids_count, len(video_ids))) ++ + + + + + """Information Extractor for blip.tv users.""" ++ + + + + + + + + """Report attempt to download user page.""" + +(self.IE_NAME, username, pagenum)) ++ + # Extract username + + +self._downloader.trouble(u'ERROR: invalid url: %s' % url) + ++ + + page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1' ++ + + + page = compat_urllib_request.urlopen(request).read().decode('utf-8') + +page_base = page_base % mobj.group(1) + +self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) +return ++ + # Download video ids using BlipTV Ajax calls. Result size per +# query is limited (currently to 12 videos) so we need to query +# page by page until there are no video ids - it means we got +# all of them. ++ + + + while True: + ++ + + try: +page = compat_urllib_request.urlopen(request).read().decode('utf-8') +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + +return ++ # Extract video identifiers + ++ + if mobj.group(1) not in ids_in_page: + ++ video_ids.extend(ids_in_page) ++ # A little optimization - if current page is not +# "full", ie. does not contain PAGE_SIZE video ids then +# we can assume that this page is the last one - there +# are no more ids on further pages - no need to query +# again. ++ if len(ids_in_page) < self._PAGE_SIZE: + ++ + + + + + + if playlistend == -1: + +else: + ++ + (self.IE_NAME, username, all_ids_count, len(video_ids))) ++ + + + + + """Information extractor for depositfiles.com""" ++ + + + + InfoExtractor.__init__(self, downloader) ++ + """Report webpage download.""" + ++ + """Report information extraction.""" +self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id) ++ + file_id = url.split('/')[-1] +# Rebuild url in english locale +url = 'http://depositfiles.com/en/files/' + file_id ++ # Retrieve file webpage with 'Free download' button pressed +free_download_indication = { 'gateway_result' : '1' } +request = compat_urllib_request.Request(url, compat_urllib_parse.urlencode(free_download_indication)) +try: +self.report_download_webpage(file_id) +webpage = compat_urllib_request.urlopen(request).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % compat_str(err)) +return ++ # Search for the real file URL +mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage) +if (mobj is None) or (mobj.group(1) is None): +# Try to figure out reason of the error. +mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL) +if (mobj is not None) and (mobj.group(1) is not None): +restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip() +self._downloader.trouble(u'ERROR: %s' % restriction_message) +else: +self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url) +return ++ file_url = mobj.group(1) +file_extension = os.path.splitext(file_url)[1][1:] ++ # Search for file title +mobj = re.search(r'<b title="(.*?)">', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract title') +return +file_title = mobj.group(1).decode('utf-8') ++ return [{ +'id': file_id.decode('utf-8'), +'url': file_url.decode('utf-8'), +'uploader': None, +'upload_date': None, +'title': file_title, +'ext': file_extension.decode('utf-8'), +}] ++ + + """Information Extractor for Facebook""" ++ + + + + + + 'video': 'mp4', +'highqual': 'mp4', +'lowqual': 'mp4', +} + ++ + InfoExtractor.__init__(self, downloader) ++ + """Add header and report message.""" +self._downloader.to_screen(u'[facebook] %s' % message) ++ + """Report attempt to log in.""" + ++ + """Report attempt to download video webpage.""" + ++ + """Report attempt to extract video information.""" + ++ + """Extract video information from page""" +# General data + +'description': r'<div class="datawrap">(.*?)</div>', +'owner': r'\("video_owner_name", "(.*?)"\)', +'thumbnail': r'\("thumb_url", "(?P<THUMB>.*?)"\)', +} +video_info = {} + +mobj = re.search(data[piece], video_webpage) +if mobj is not None: +video_info[piece] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape")) ++ # Video urls +video_urls = {} +for fmt in self._available_formats: +mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage) +if mobj is not None: +# URL is in a Javascript segment inside an escaped Unicode format within +# the generally utf-8 page +video_urls[fmt] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape")) +video_info['video_urls'] = video_urls ++ return video_info ++ + if self._downloader is None: +return ++ useremail = None +password = None +downloader_params = self._downloader.params ++ # Attempt to use provided username and password or .netrc data +if downloader_params.get('username', None) is not None: +useremail = downloader_params['username'] +password = downloader_params['password'] +elif downloader_params.get('usenetrc', False): +try: +info = netrc.netrc().authenticators(self._NETRC_MACHINE) +if info is not None: +useremail = info[0] +password = info[2] +else: +raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) +except (IOError, netrc.NetrcParseError) as err: +self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err)) +return ++ if useremail is None: +return ++ # Log in + +'email': useremail, +'pass': password, +'login': 'Log+In' +} + + + + + + + + + +return ++ + + + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + +video_id = mobj.group('ID') ++ # Get video webpage + +request = compat_urllib_request.Request('https://www.facebook.com/video/video.php?v=%s' % video_id) + +page = compat_urllib_request.urlopen(request) + +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + +return ++ # Start extracting information + ++ # Extract information +video_info = self._parse_page(video_webpage) ++ # uploader +if 'owner' not in video_info: +self._downloader.trouble(u'ERROR: unable to extract uploader nickname') +return +video_uploader = video_info['owner'] ++ # title +if 'title' not in video_info: +self._downloader.trouble(u'ERROR: unable to extract video title') +return +video_title = video_info['title'] +video_title = video_title.decode('utf-8') ++ # thumbnail image +if 'thumbnail' not in video_info: +self._downloader.trouble(u'WARNING: unable to extract video thumbnail') +video_thumbnail = '' +else: +video_thumbnail = video_info['thumbnail'] ++ # upload date + +if 'upload_date' in video_info: + +timetuple = email.utils.parsedate_tz(upload_time) +if timetuple is not None: +try: +upload_date = time.strftime('%Y%m%d', timetuple[0:9]) +except: +pass ++ # description +video_description = video_info.get('description', 'No description available.') ++ url_map = video_info['video_urls'] +if len(url_map.keys()) > 0: +# Decide which formats to download +req_format = self._downloader.params.get('format', None) +format_limit = self._downloader.params.get('format_limit', None) ++ if format_limit is not None and format_limit in self._available_formats: +format_list = self._available_formats[self._available_formats.index(format_limit):] +else: +format_list = self._available_formats +existing_formats = [x for x in format_list if x in url_map] +if len(existing_formats) == 0: +self._downloader.trouble(u'ERROR: no known formats available for video') +return +if req_format is None: +video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality +elif req_format == 'worst': +video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality +elif req_format == '-1': +video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats +else: +# Specific format +if req_format not in url_map: +self._downloader.trouble(u'ERROR: requested format not available') +return +video_url_list = [(req_format, url_map[req_format])] # Specific format ++ results = [] +for format_param, video_real_url in video_url_list: +# Extension +video_extension = self._video_extensions.get(format_param, 'mp4') ++ + 'id': video_id.decode('utf-8'), +'url': video_real_url.decode('utf-8'), +'uploader': video_uploader.decode('utf-8'), +'upload_date': upload_date, +'title': video_title, +'ext': video_extension.decode('utf-8'), +'format': (format_param is None and u'NA' or format_param.decode('utf-8')), +'thumbnail': video_thumbnail.decode('utf-8'), +'description': video_description.decode('utf-8'), +}) +return results ++ + """Information extractor for blip.tv""" ++ + + + + + """Report information extraction.""" + ++ + """Report information extraction.""" +self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title)) ++ + + + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) +return ++ + cchar = '&' +else: + + + + + + + + +basename = url.split('/')[-1] +title,ext = os.path.splitext(basename) +title = title.decode('UTF-8') +ext = ext.replace('.', '') +self.report_direct_download(title) +info = { +'id': title, +'url': url, +'uploader': None, +'upload_date': None, +'title': title, +'ext': ext, +'urlhandle': urlh +} +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err)) +return + + + + +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err)) +return ++ + + + + else: +data = json_data ++ + + + + raise ValueError('Can not determine filename extension') + ++ + 'id': data['item_id'], +'url': video_url, +'uploader': data['display_name'], +'upload_date': upload_date, +'title': data['title'], +'ext': ext, +'format': data['media']['mimeType'], +'thumbnail': data['thumbnailUrl'], +'description': data['description'], +'player_url': data['embedUrl'] +} +except (ValueError,KeyError) as err: +self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err)) +return ++ + + + + + """Information Extractor for myvideo.de.""" ++ + + + + + + + """Report webpage download.""" + ++ + """Report information extraction.""" + ++ + + + + + + + + # Get video webpage + + + + + +self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) +return ++ + + webpage) + +self._downloader.trouble(u'ERROR: unable to extract media URL') +return + ++ + + self._downloader.trouble(u'ERROR: unable to extract title') +return ++ + + + 'id': video_id, +'url': video_url, +'uploader': None, +'upload_date': None, +'title': video_title, +'ext': u'flv', +}] ++ + """Information extractor for The Daily Show and Colbert Report """ ++ # urls can be abbreviations like :thedailyshow or :colbert +# urls for episodes like: +# or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day +# or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news +# or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524 + +|(https?://)?(www\.)? +(?P<showname>thedailyshow|colbertnation)\.com/ +(full-episodes/(?P<episode>.*)| +(?P<clip> +(the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?)) +|(watch/(?P<date>[^/]*)/(?P<tdstitle>.*))))) +$""" + ++ + + + '3500': 'mp4', +'2200': 'mp4', +'1700': 'mp4', +'1200': 'mp4', +'750': 'mp4', +'400': 'mp4', +} + +'3500': '1280x720', +'2200': '960x540', +'1700': '768x432', +'1200': '640x360', +'750': '512x288', +'400': '384x216', +} ++ + """Receives a URL and returns True if suitable for this IE.""" + ++ + self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id) ++ + + + + self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id) ++ + self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id) ++ + + print('Available formats:') +for x in formats: +print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???'))) ++ + + mobj = re.match(self._VALID_URL, url, re.VERBOSE) +if mobj is None: +self._downloader.trouble(u'ERROR: invalid URL: %s' % url) +return ++ if mobj.group('shortname'): +if mobj.group('shortname') in ('tds', 'thedailyshow'): +url = u'http://www.thedailyshow.com/full-episodes/' +else: +url = u'http://www.colbertnation.com/full-episodes/' +mobj = re.match(self._VALID_URL, url, re.VERBOSE) +assert mobj is not None ++ if mobj.group('clip'): +if mobj.group('showname') == 'thedailyshow': +epTitle = mobj.group('tdstitle') +else: +epTitle = mobj.group('cntitle') +dlNewest = False +else: +dlNewest = not mobj.group('episode') +if dlNewest: +epTitle = mobj.group('showname') +else: +epTitle = mobj.group('episode') ++ req = compat_urllib_request.Request(url) + + + + +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) +return +if dlNewest: +url = htmlHandle.geturl() +mobj = re.match(self._VALID_URL, url, re.VERBOSE) +if mobj is None: +self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url) +return +if mobj.group('episode') == '': +self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url) + + ++ + + + # The Colbert Report embeds the information in a without +# a URL prefix; so extract the alternate reference +# and then add the URL prefix manually. ++ + + + + else: + ++ + + + + + + self._downloader.trouble(u'ERROR: unable to find out player URL: ' + compat_str(err)) + ++ + indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + compat_urllib_parse.urlencode({'uri': uri}) + + +indexXml = compat_urllib_request.urlopen(indexUrl).read() + + +return ++ + + + itemEls = idoc.findall('.//item') + +mediaId = itemEl.findall('./guid')[0].text + +showId = mediaId.split(':')[-2].replace('.com', '') +officialTitle = itemEl.findall('./title')[0].text +officialDate = itemEl.findall('./pubDate')[0].text ++ + compat_urllib_parse.urlencode({'uri': mediaId})) +configReq = compat_urllib_request.Request(configUrl) +self.report_config_download(epTitle) +try: +configXml = compat_urllib_request.urlopen(configReq).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) +return ++ cdoc = xml.etree.ElementTree.fromstring(configXml) +turls = [] +for rendition in cdoc.findall('.//rendition'): +finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text) +turls.append(finfo) ++ if len(turls) == 0: +self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found') +continue ++ if self._downloader.params.get('listformats', None): +self._print_formats([i[0] for i in turls]) +return ++ # For now, just pick the highest bitrate +format,video_url = turls[-1] ++ # Get the format arg from the arg stream +req_format = self._downloader.params.get('format', None) ++ # Select format if we can find one +for f,v in turls: +if f == req_format: +format, video_url = f, v +break ++ # Patch to download from alternative CDN, which does not +# break on current RTMPDump builds +broken_cdn = "rtmpe://viacomccstrmfs.fplive.net/viacomccstrm/gsp.comedystor/" +better_cdn = "rtmpe://cp10740.edgefcs.net/ondemand/mtvnorigin/gsp.comedystor/" ++ if video_url.startswith(broken_cdn): +video_url = video_url.replace(broken_cdn, better_cdn) ++ effTitle = showId + u'-' + epTitle +info = { +'id': shortMediaId, +'url': video_url, +'uploader': showId, +'upload_date': officialDate, +'title': effTitle, +'ext': 'mp4', +'format': format, +'thumbnail': None, +'description': officialTitle, +'player_url': None #playerUrl +} ++ results.append(info) ++ return results ++ + + """Information extractor for The Escapist """ ++ + + + + + + + + + + + + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) +return + + ++ + + + + + + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download webpage: ' + compat_str(err)) +return ++ + + + + + + + + + + + + + + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download configuration: ' + compat_str(err)) +return ++ # Technically, it's JavaScript, not JSON + ++ + + except (ValueError,) as err: +self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + compat_str(err)) + ++ + + + + 'id': videoId, +'url': videoUrl, +'uploader': showName, +'upload_date': None, +'title': showName, +'ext': 'flv', +'thumbnail': imgUrl, +'description': description, +'player_url': playerUrl, +} ++ + + + + """Information extractor for collegehumor.com""" ++ + + + + + """Report information extraction.""" + ++ + """Report information extraction.""" + ++ + + + + + video_id = mobj.group('videoid') ++ + 'id': video_id, +'uploader': None, +'upload_date': None, +} ++ self.report_extraction(video_id) +xmlUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id +try: +metaXml = compat_urllib_request.urlopen(xmlUrl).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err)) +return ++ mdoc = xml.etree.ElementTree.fromstring(metaXml) +try: +videoNode = mdoc.findall('./video')[0] +info['description'] = videoNode.findall('./description')[0].text +info['title'] = videoNode.findall('./caption')[0].text +info['thumbnail'] = videoNode.findall('./thumbnail')[0].text +manifest_url = videoNode.findall('./file')[0].text +except IndexError: +self._downloader.trouble(u'\nERROR: Invalid metadata XML file') +return ++ manifest_url += '?hdcore=2.10.3' +self.report_manifest(video_id) +try: +manifestXml = compat_urllib_request.urlopen(manifest_url).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err)) +return ++ adoc = xml.etree.ElementTree.fromstring(manifestXml) +try: +media_node = adoc.findall('./{http://ns.adobe.com/f4m/1.0}media')[0] +node_id = media_node.attrib['url'] +video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text +except IndexError as err: +self._downloader.trouble(u'\nERROR: Invalid manifest file') +return ++ url_pr = compat_urllib_parse_urlparse(manifest_url) +url = url_pr.scheme + '://' + url_pr.netloc + '/z' + video_id[:-2] + '/' + node_id + 'Seg1-Frag1' ++ info['url'] = url +info['ext'] = 'f4f' +return [info] ++ + + """Information extractor for xvideos.com""" ++ + + + + """Report information extraction.""" + ++ + """Report information extraction.""" + ++ + + + + + + + + + + + + + + self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) + ++ + + + # Extract video URL + + + +return + ++ + # Extract title + + + +return + ++ + # Extract video thumbnail + + + +return + ++ + 'id': video_id, +'url': video_url, +'uploader': None, +'upload_date': None, +'title': video_title, +'ext': 'flv', +'thumbnail': video_thumbnail, +'description': None, +} ++ + + + + """Information extractor for soundcloud.com +To access the media, the uid of the song and a stream token +must be extracted from the page source and the script must make +a request to media.soundcloud.com/crossdomain.xml. Then +the media can be grabbed by requesting from an url composed +of the stream token and uid +""" ++ + + + + + + + """Report information extraction.""" + ++ + """Report information extraction.""" + ++ + + + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + ++ # extract uploader (which is in the url) + +# extract simple title (uploader + slug of song title) + + ++ + + + + + + + + + self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) + ++ + + + + + + + + + + + + + + + + + 'id': info['id'], +'url': mediaURL, +'uploader': info['user']['username'], +'upload_date': info['created_at'], +'title': info['title'], +'ext': u'mp3', +'description': info['description'], +}] ++ + + """Information extractor for infoq.com""" ++ + + + + """Report information extraction.""" + ++ + """Report information extraction.""" + ++ + mobj = re.match(self._VALID_URL, url) +if mobj is None: +self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + ++ + + request = compat_urllib_request.Request(url) + + +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + + ++ + + + # Extract video URL + + + + + ++ + # Extract title + +if mobj is None: + + +video_title = mobj.group(1).decode('utf-8') ++ # Extract description + + + + ++ + + + + 'id': video_id, +'url': video_url, +'uploader': None, +'upload_date': None, +'title': video_title, +'ext': extension, # Extension is always(?) mp4, but seems to be flv +'thumbnail': None, +'description': video_description, +} ++ + + + """Information extractor for www.mixcloud.com""" ++ + + + + + + + + """Report JSON download.""" + ++ + """Report information extraction.""" + ++ + """Get urls from 'audio_formats' section in json""" + +try: + + +bitrate = max(bitrate_list) # select highest ++ + + url_list = jsonData[fmt] + ++ + """Returns 1st active url from list""" + +try: +compat_urllib_request.urlopen(url) +return url +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + ++ + + + print('Available formats:') +for fmt in formats.keys(): +for b in formats[fmt]: +try: +ext = formats[fmt][b][0] +print('%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1])) +except TypeError: # we have no bitrate info + +print('%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1])) + ++ + mobj = re.match(self._VALID_URL, url) +if mobj is None: +self._downloader.trouble(u'ERROR: invalid URL: %s' % url) +return +# extract uploader & filename from url +uploader = mobj.group(1).decode('utf-8') +file_id = uploader + "-" + mobj.group(2).decode('utf-8') ++ # construct API request +file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json' +# retrieve .json file with links to files +request = compat_urllib_request.Request(file_url) +try: +self.report_download_json(file_url) +jsonData = compat_urllib_request.urlopen(request).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % compat_str(err)) +return ++ # parse JSON +json_data = json.loads(jsonData) +player_url = json_data['player_swf_url'] +formats = dict(json_data['audio_formats']) ++ req_format = self._downloader.params.get('format', None) +bitrate = None ++ if self._downloader.params.get('listformats', None): +self._print_formats(formats) +return ++ + + + # check urls + + +break # got it! +else: +if req_format not in formats.keys(): +self._downloader.trouble(u'ERROR: format is not available') + ++ url_list = self.get_urls(formats, req_format) +file_url = self.check_urls(url_list) + ++ + 'id': file_id.decode('utf-8'), +'url': file_url.decode('utf-8'), +'uploader': uploader.decode('utf-8'), +'upload_date': None, +'title': json_data['name'], +'ext': file_url.split('.')[-1].decode('utf-8'), +'format': (format_param is None and u'NA' or format_param.decode('utf-8')), +'thumbnail': json_data['thumbnail_url'], +'description': json_data['description'], +'player_url': player_url.decode('utf-8'), +}] ++ + """Information extractor for Stanford's Open ClassRoom""" ++ + + + + """Report information extraction.""" + ++ + """Report information extraction.""" + ++ + + + + + + + + + + 'id': course + '_' + video, +'uploader': None, +'upload_date': None, +} ++ + + + + + + + + + + + + + + + + + elif mobj.group('course'): # A course page +course = mobj.group('course') +info = { +'id': course, +'type': 'playlist', +'uploader': None, +'upload_date': None, +} ++ self.report_download_webpage(info['id']) +try: +coursepage = compat_urllib_request.urlopen(url).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err)) +return ++ m = re.search('<h1>([^<]+)</h1>', coursepage) +if m: +info['title'] = unescapeHTML(m.group(1)) +else: +info['title'] = info['id'] ++ m = re.search('<description>([^<]+)</description>', coursepage) +if m: +info['description'] = unescapeHTML(m.group(1)) ++ links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage)) +info['list'] = [ +{ +'type': 'reference', +'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage), +} +for vpage in links] +results = [] +for entry in info['list']: +assert entry['type'] == 'reference' +results += self.extract(entry['url']) + ++ else: # Root page + +'id': 'Stanford OpenClassroom', +'type': 'playlist', +'uploader': None, +'upload_date': None, +} ++ + rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php' + + + + + ++ + + + + { +'type': 'reference', +'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage), +} +for cpage in links] ++ + + + + + + + """Information extractor for MTV.com""" ++ + + + + """Report information extraction.""" + ++ + """Report information extraction.""" + ++ + mobj = re.match(self._VALID_URL, url) +if mobj is None: +self._downloader.trouble(u'ERROR: invalid URL: %s' % url) +return +if not mobj.group('proto'): +url = 'http://' + url +video_id = mobj.group('videoid') +self.report_webpage(video_id) ++ request = compat_urllib_request.Request(url) +try: +webpage = compat_urllib_request.urlopen(request).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) +return ++ mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract song name') +return +song_name = unescapeHTML(mobj.group(1).decode('iso-8859-1')) +mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract performer') +return +performer = unescapeHTML(mobj.group(1).decode('iso-8859-1')) +video_title = performer + ' - ' + song_name ++ mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to mtvn_uri') +return +mtvn_uri = mobj.group(1) ++ mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage) +if mobj is None: +self._downloader.trouble(u'ERROR: unable to extract content id') +return +content_id = mobj.group(1) ++ videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri +self.report_extraction(video_id) +request = compat_urllib_request.Request(videogen_url) +try: +metadataXml = compat_urllib_request.urlopen(request).read() +except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: +self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % compat_str(err)) +return ++ mdoc = xml.etree.ElementTree.fromstring(metadataXml) +renditions = mdoc.findall('.//rendition') ++ # For now, always pick the highest quality. +rendition = renditions[-1] ++ try: +_,_,ext = rendition.attrib['type'].partition('/') +format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate'] +video_url = rendition.find('./src').text +except KeyError: +self._downloader.trouble('Invalid rendition field.') +return ++ + 'id': video_id, +'url': video_url, +'uploader': performer, +'upload_date': None, +'title': video_title, +'ext': ext, +'format': format, +} ++ return [info] ++ + + + + + + + + + + """Report webpage download.""" + ++ + """Report information extraction.""" + ++ + + + + + + + + + + + + + + + + #return ''.join(mixed) + ++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + else: + + +elif format == 'worst': +format = 'mp4' +ext = u'mp4' +else: + + ++ + + + + + + + + + + + #column 8,9 of fileid represent the segment number +#fileid[7:9] should be changed + ++ + + + + 'id': '%s_part%02d' % (video_id, index), +'url': download_url, +'uploader': None, +'upload_date': None, +'title': video_title, +'ext': ext, +} + ++ + + + + """Information extractor for xnxx.com""" ++ + + + + + + + """Report information extraction""" + ++ + """Report information extraction""" + ++ + + + + + + + + + # Get webpage content + + + + + + ++ + + + + + + + + + + + + + + + + + + + 'id': video_id, +'url': video_url, +'uploader': None, +'upload_date': None, +'title': video_title, +'ext': 'flv', +'thumbnail': video_thumbnail, +'description': None, +}] ++ + + """Information extractor for plus.google.com.""" ++ + + + + + + + """Report downloading extry""" + ++ + """Report downloading extry""" + ++ + """Report downloading extry""" + ++ + """Report downloading extry""" + ++ + """Report information extraction.""" + ++ + # Extract id from URL + + + + ++ + + + + + # Step 1, Retrieve post webpage to extract further information + + + + + + +return ++ # Extract update date + + + + + +# Convert timestring to a format suitable for filename + + + ++ # Extract uploader + + + + + + ++ # Extract title +# Get the first line for title + + + + + + ++ # Step 2, Stimulate clicking the image box to launch video + + + + ++ + + + + + + + + + + # Extract video links on video page +"""Extract video links of all sizes""" + + + + ++ # Sort in resolution + ++ # Choose the lowest of the sort, i.e. highest resolution + +# Only get the url. The resolution part in the tuple has no use anymore + +# Treat escaped \u0026 style hex + + + + ++ + + 'id': video_id, +'url': video_url, +'uploader': uploader, +'upload_date': upload_date, +'title': video_title, +'ext': video_extension, +}] ++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + else: + ++ + + + 'id': shortened_video_id, +'url': video_url, +'ext': 'mp4', +'title': title, +'uploader_date': _findProp(r'<b>Date:</b> (.*?)</div>'), +'description': _findProp(r'<div class="description">(.*?)</h1>'), +} + ++ + """Information extractor for justin.tv and twitch.tv""" +# TODO: One broadcast may be split into multiple videos. The key +# 'broadcast_id' is the same for all parts, and 'broadcast_part' +# starts at 1 and increases. Can we treat all parts as one video? ++ + ([^/]+)(?:/b/([^/]+))?/?(?:\#.*)?$""" + + ++ + """Report information extraction.""" + ++ + """Report attempt to download a single page of videos.""" + +(self.IE_NAME, channel, offset, offset + self._JUSTIN_PAGE_LIMIT)) ++ # Return count of items, list of *valid* items + + + + + + + + ++ + + + + + + + + 'id': clip['id'], +'url': video_url, +'title': clip['title'], +'uploader': clip.get('user_id', clip.get('channel_id')), +'upload_date': video_date, +'ext': video_extension, +}) + ++ + + + + + + + + + + + + else: + + ++ + + + + + + + + + + + + + + + + |
+
+
+Hot-keys on this page
++ r + m + x + p toggle line displays +
++ j + k next/prev highlighted chunk +
++ 0 (zero) top of page +
++ 1 (one) first highlighted chunk +
+| + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | +
+ #!/usr/bin/env python +# -*- coding: utf-8 -*- ++ + + + + + + + + + + + """Post Processor class. ++ PostProcessor objects can be added to downloaders with their +add_post_processor() method. When the downloader has finished a +successful download, it will take its internal chain of PostProcessors +and start calling the run() method on each one of them, first with +an initial argument and then with the returned value of the previous +PostProcessor. ++ The chain will be stopped if one of them ever returns None or the end +of the chain is reached. ++ PostProcessor objects follow a "mutual registration" process similar +to InfoExtractor objects. +""" ++ + + + self._downloader = downloader ++ + """Sets the downloader for this PP.""" +self._downloader = downloader ++ + """Run the PostProcessor. ++ The "information" argument is a dictionary like the ones +composed by InfoExtractors. The only difference is that this +one has an extra field called "filepath" that points to the +downloaded file. ++ When this method returns None, the postprocessing chain is +stopped. However, this method may return an information +dictionary that will be passed to the next postprocessing +object in the chain. It can be the one it received after +changing some fields. ++ In addition, this method may raise a PostProcessingError +exception that will be taken into account by the downloader +it was called from. +""" +return information # by default, do nothing ++ + + self.message = message ++ + + PostProcessor.__init__(self, downloader) +if preferredcodec is None: +preferredcodec = 'best' +self._preferredcodec = preferredcodec +self._preferredquality = preferredquality +self._keepvideo = keepvideo +self._exes = self.detect_executables() ++ + def detect_executables(): +def executable(exe): +try: +subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() +except OSError: +return False +return exe +programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe'] +return dict((program, executable(program)) for program in programs) ++ + if not self._exes['ffprobe'] and not self._exes['avprobe']: return None +try: +cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', '--', encodeFilename(path)] +handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE) +output = handle.communicate()[0] +if handle.wait() != 0: +return None +except (IOError, OSError): +return None +audio_codec = None +for line in output.decode('ascii', 'ignore').split('\n'): +if line.startswith('codec_name='): +audio_codec = line.split('=')[1].strip() +elif line.strip() == 'codec_type=audio' and audio_codec is not None: +return audio_codec +return None ++ + if not self._exes['ffmpeg'] and not self._exes['avconv']: +raise AudioConversionError('ffmpeg or avconv not found. Please install one.') +if codec is None: +acodec_opts = [] +else: +acodec_opts = ['-acodec', codec] +cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path), '-vn'] ++ acodec_opts + more_opts + +['--', encodeFilename(out_path)]) +p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) +stdout,stderr = p.communicate() +if p.returncode != 0: +msg = stderr.strip().split('\n')[-1] +raise AudioConversionError(msg) ++ + path = information['filepath'] ++ filecodec = self.get_audio_codec(path) +if filecodec is None: +self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe') +return None ++ more_opts = [] +if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'): +if self._preferredcodec == 'm4a' and filecodec == 'aac': +# Lossless, but in another container +acodec = 'copy' +extension = self._preferredcodec +more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc'] +elif filecodec in ['aac', 'mp3', 'vorbis']: +# Lossless if possible +acodec = 'copy' +extension = filecodec +if filecodec == 'aac': +more_opts = ['-f', 'adts'] +if filecodec == 'vorbis': +extension = 'ogg' +else: +# MP3 otherwise. +acodec = 'libmp3lame' +extension = 'mp3' +more_opts = [] +if self._preferredquality is not None: +if int(self._preferredquality) < 10: +more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality] +else: +more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k'] +else: +# We convert the audio (lossy) +acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec] +extension = self._preferredcodec +more_opts = [] +if self._preferredquality is not None: +if int(self._preferredquality) < 10: +more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality] +else: +more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k'] +if self._preferredcodec == 'aac': +more_opts += ['-f', 'adts'] +if self._preferredcodec == 'm4a': +more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc'] +if self._preferredcodec == 'vorbis': +extension = 'ogg' +if self._preferredcodec == 'wav': +extension = 'wav' +more_opts += ['-f', 'wav'] ++ prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups +new_path = prefix + sep + extension +self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path) +try: +self.run_ffmpeg(path, new_path, acodec, more_opts) +except: +etype,e,tb = sys.exc_info() +if isinstance(e, AudioConversionError): +self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message) +else: +self._downloader.to_stderr(u'ERROR: error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg')) +return None ++ # Try to update the date time for extracted audio file. +if information.get('filetime') is not None: +try: +os.utime(encodeFilename(new_path), (time.time(), information['filetime'])) +except: +self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file') ++ if not self._keepvideo: +try: +os.remove(encodeFilename(path)) +except (IOError, OSError): +self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file') +return None ++ information['filepath'] = new_path +return information + + |
+
+
+Hot-keys on this page
++ r + m + x + p toggle line displays +
++ j + k next/prev highlighted chunk +
++ 0 (zero) top of page +
++ 1 (one) first highlighted chunk +
+| + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | +
+ #!/usr/bin/env python +# -*- coding: utf-8 -*- ++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib. +# Python 2's version is apparently totally broken + + + + + + + + + + +# pct_sequence: contiguous sequence of percent-encoded bytes, decoded + + + + + +raise ValueError + + + +# This segment was just a single percent-encoded character. +# May be part of a sequence of code units, so delay decoding. +# (Stored in pct_sequence). + + +rest = '%' + item +# Encountered non-percent-encoded characters. Flush the current +# pct_sequence. + + + +# Flush the final pct_sequence + + ++ + encoding='utf-8', errors='replace'): + + + + + +continue + + +if strict_parsing: + +# Handle case of a control-name with no equal sign + + +else: + + + + + + + + + + ++ + encoding='utf-8', errors='replace'): + + +encoding=encoding, errors=errors) + + + +else: + + ++ + + + + + + + + + + + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0', +'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', +'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +'Accept-Encoding': 'gzip, deflate', +'Accept-Language': 'en-us,en;q=0.5', +} + +"""Get preferred encoding. ++ Returns the best encoding scheme for the system, based on +locale.getpreferredencoding() and some further tweaks. +""" + + + + +pref = 'UTF-8' ++ + + + + + else: + +assert type(s) == type(u'') +print(s) ++ + """Transforms an HTML entity to a character. ++ This function receives a match object and is intended to be used with +the re.sub() function. +""" + ++ # Known non-numeric HTML entity + + ++ + + + + base = 16 +numstr = u'0%s' % numstr +else: + + ++ # Unknown entity in name, return its literal representation + ++ + + """Modified HTMLParser that isolates a tag with the specified id""" + + + + + + + + + ++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + """Needed to put the start position of the result (self.result[1]) +after the opening tag with the requested id""" + + + + +handle_decl = handle_pi = unknown_decl = find_startpos ++ + + + + + + + + + + + + + + """Return the content of the tag with the specified id in the passed HTML document""" + + + + + + ++ + + """Clean an HTML snippet into a readable string""" +# Newline vs <br /> + + +# Strip html tags + +# Replace html entities + + ++ + + """Try to open the given filename, and slightly tweak it if this fails. ++ Attempts to open the given filename. If this fails, it tries to change +the filename slightly, step by step, until it's either able to open it +or it fails and raises a final exception, like the standard open() +function. ++ It returns the tuple (stream, definitive_file_name). +""" + + +if sys.platform == 'win32': +import msvcrt +msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) +return (sys.stdout, filename) + + +except (IOError, OSError) as err: +# In case of error, try to remove win32 forbidden chars +filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename) ++ # An exception here should be caught in the caller + + ++ + + """Convert RFC 2822 defined time string into system timestamp""" + + + + + ++ + """Sanitizes a string so it could be used as part of a filename. +If restricted is set, use a stricter subset of allowed characters. +Set is_id if this is not an arbitrary string, but an ID that should be kept if possible +""" + + + + + + + + + + + + + + ++ + + + + + # Common case of "Foreign band name - English song title" + + + + + ++ + """ Remove all duplicates from the input iterable """ + + + + + ++ + """ +@param s a string +""" + ++ + + + + """ +@param s The name of the file +""" ++ + + # Python 3 has a Unicode API + + ++ + # Pass u'' directly to use Unicode APIs on Windows 2000 and up +# (Detecting Windows NT 4 is tricky because 'major >= 4' would +# match Windows 9x series as well. Besides, NT 4 is obsolete.) +return s +else: + ++ + """Download Error exception. ++ This exception may be thrown by FileDownloader objects if they are not +configured to continue on errors. They will contain the appropriate +error message. +""" + ++ + + """Same File exception. ++ This exception will be thrown by FileDownloader objects if they detect +multiple files would have to be downloaded to the same file on disk. +""" + ++ + + """Post Processing exception. ++ This exception may be raised by PostProcessor's .run() method to +indicate an error in the postprocessing task. +""" + ++ + """ --max-downloads limit has been reached. """ + ++ + + """Unavailable Format exception. ++ This exception will be thrown when a video is requested +in a format that is not available for that video. +""" + ++ + + """Content Too Short exception. ++ This exception may be raised by FileDownloader objects when a file they +download is too small for what the server announced first, indicating +the connection was probably interrupted. +""" +# Both in bytes + + ++ + self.downloaded = downloaded +self.expected = expected ++ + + """Trouble helper exception ++ This is an exception to be handled with +FileDownloader.trouble +""" ++ + """Handler for HTTP requests and responses. ++ This class, when installed with an OpenerDirector, automatically adds +the standard headers to every HTTP request and handles gzipped and +deflated responses from web servers. If compression is to be avoided in +a particular request, the original request in the program code only has +to include the HTTP header "Youtubedl-No-Compression", which will be +removed before making the real request. ++ Part of this code was copied from: ++ http://techknack.net/python-urllib2-handlers/ ++ Andrew Rowls, the author of that code, agreed to release it to the +public domain. +""" ++ + def deflate(data): +try: + + +return zlib.decompress(data) ++ + def addinfourl_wrapper(stream, headers, url, code): + + + + + ++ + + + + + + + + + + + + + # gzip + + + + +# deflate + + +resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code) +resp.msg = old_resp.msg + ++ + + + |
+
+
+Hot-keys on this page
++ r + m + x + p toggle line displays +
++ j + k next/prev highlighted chunk +
++ 0 (zero) top of page +
++ 1 (one) first highlighted chunk +
+| + + + + | +
+ + + + |
+