[fc2:live] Add extractor (#2418)

Authored by: Lesmiscore
This commit is contained in:
Lesmiscore (Naoya Ozaki) 2022-02-25 11:16:23 +09:00 committed by GitHub
parent 3e9b66d761
commit 15dfb3929c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 197 additions and 1 deletions

View File

@ -30,6 +30,7 @@ def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=N
from .common import FileDownloader from .common import FileDownloader
from .dash import DashSegmentsFD from .dash import DashSegmentsFD
from .f4m import F4mFD from .f4m import F4mFD
from .fc2 import FC2LiveFD
from .hls import HlsFD from .hls import HlsFD
from .http import HttpFD from .http import HttpFD
from .rtmp import RtmpFD from .rtmp import RtmpFD
@ -58,6 +59,7 @@ def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=N
'ism': IsmFD, 'ism': IsmFD,
'mhtml': MhtmlFD, 'mhtml': MhtmlFD,
'niconico_dmc': NiconicoDmcFD, 'niconico_dmc': NiconicoDmcFD,
'fc2_live': FC2LiveFD,
'websocket_frag': WebSocketFragmentFD, 'websocket_frag': WebSocketFragmentFD,
'youtube_live_chat': YoutubeLiveChatFD, 'youtube_live_chat': YoutubeLiveChatFD,
'youtube_live_chat_replay': YoutubeLiveChatFD, 'youtube_live_chat_replay': YoutubeLiveChatFD,

41
yt_dlp/downloader/fc2.py Normal file
View File

@ -0,0 +1,41 @@
from __future__ import division, unicode_literals
import threading
from .common import FileDownloader
from .external import FFmpegFD
class FC2LiveFD(FileDownloader):
"""
Downloads FC2 live without being stopped. <br>
Note, this is not a part of public API, and will be removed without notice.
DO NOT USE
"""
def real_download(self, filename, info_dict):
ws = info_dict['ws']
heartbeat_lock = threading.Lock()
heartbeat_state = [None, 1]
def heartbeat():
try:
heartbeat_state[1] += 1
ws.send('{"name":"heartbeat","arguments":{},"id":%d}' % heartbeat_state[1])
except Exception:
self.to_screen('[fc2:live] Heartbeat failed')
with heartbeat_lock:
heartbeat_state[0] = threading.Timer(30, heartbeat)
heartbeat_state[0]._daemonic = True
heartbeat_state[0].start()
heartbeat()
new_info_dict = info_dict.copy()
new_info_dict.update({
'ws': None,
'protocol': 'live_ffmpeg',
})
return FFmpegFD(self.ydl, self.params or {}).download(filename, new_info_dict)

View File

@ -478,6 +478,7 @@
from .fc2 import ( from .fc2 import (
FC2IE, FC2IE,
FC2EmbedIE, FC2EmbedIE,
FC2LiveIE,
) )
from .fczenit import FczenitIE from .fczenit import FczenitIE
from .filmmodu import FilmmoduIE from .filmmodu import FilmmoduIE

View File

@ -1,14 +1,21 @@
# coding: utf-8 # coding: utf-8
from __future__ import unicode_literals from __future__ import unicode_literals
import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import ( from ..compat import (
compat_parse_qs, compat_parse_qs,
) )
from ..utils import ( from ..utils import (
ExtractorError, ExtractorError,
WebSocketsWrapper,
has_websockets,
js_to_json,
sanitized_Request, sanitized_Request,
std_headers,
traverse_obj, traverse_obj,
update_url_query,
urlencode_postdata, urlencode_postdata,
urljoin, urljoin,
) )
@ -147,3 +154,146 @@ def _real_extract(self, url):
'title': title, 'title': title,
'thumbnail': thumbnail, 'thumbnail': thumbnail,
} }
class FC2LiveIE(InfoExtractor):
_VALID_URL = r'https?://live\.fc2\.com/(?P<id>\d+)'
IE_NAME = 'fc2:live'
_TESTS = [{
'url': 'https://live.fc2.com/57892267/',
'info_dict': {
'id': '57892267',
'title': 'どこまで・・・',
'uploader': 'あつあげ',
'uploader_id': '57892267',
'thumbnail': r're:https?://.+fc2.+',
},
'skip': 'livestream',
}]
def _real_extract(self, url):
if not has_websockets:
raise ExtractorError('websockets library is not available. Please install it.', expected=True)
video_id = self._match_id(url)
webpage = self._download_webpage('https://live.fc2.com/%s/' % video_id, video_id)
self._set_cookie('live.fc2.com', 'js-player_size', '1')
member_api = self._download_json(
'https://live.fc2.com/api/memberApi.php', video_id, data=urlencode_postdata({
'channel': '1',
'profile': '1',
'user': '1',
'streamid': video_id
}), note='Requesting member info')
control_server = self._download_json(
'https://live.fc2.com/api/getControlServer.php', video_id, note='Downloading ControlServer data',
data=urlencode_postdata({
'channel_id': video_id,
'mode': 'play',
'orz': '',
'channel_version': member_api['data']['channel_data']['version'],
'client_version': '2.1.0\n [1]',
'client_type': 'pc',
'client_app': 'browser_hls',
'ipv6': '',
}), headers={'X-Requested-With': 'XMLHttpRequest'})
self._set_cookie('live.fc2.com', 'l_ortkn', control_server['orz_raw'])
ws_url = update_url_query(control_server['url'], {'control_token': control_server['control_token']})
playlist_data = None
self.to_screen('%s: Fetching HLS playlist info via WebSocket' % video_id)
ws = WebSocketsWrapper(ws_url, {
'Cookie': str(self._get_cookies('https://live.fc2.com/'))[12:],
'Origin': 'https://live.fc2.com',
'Accept': '*/*',
'User-Agent': std_headers['User-Agent'],
})
ws.__enter__()
self.write_debug('[debug] Sending HLS server request')
while True:
recv = ws.recv()
if not recv:
continue
data = self._parse_json(recv, video_id, fatal=False)
if not data or not isinstance(data, dict):
continue
if data.get('name') == 'connect_complete':
break
ws.send(r'{"name":"get_hls_information","arguments":{},"id":1}')
while True:
recv = ws.recv()
if not recv:
continue
data = self._parse_json(recv, video_id, fatal=False)
if not data or not isinstance(data, dict):
continue
if data.get('name') == '_response_' and data.get('id') == 1:
self.write_debug('[debug] Goodbye.')
playlist_data = data
break
elif self._downloader.params.get('verbose', False):
if len(recv) > 100:
recv = recv[:100] + '...'
self.to_screen('[debug] Server said: %s' % recv)
if not playlist_data:
raise ExtractorError('Unable to fetch HLS playlist info via WebSocket')
formats = []
for name, playlists in playlist_data['arguments'].items():
if not isinstance(playlists, list):
continue
for pl in playlists:
if pl.get('status') == 0 and 'master_playlist' in pl.get('url'):
formats.extend(self._extract_m3u8_formats(
pl['url'], video_id, ext='mp4', m3u8_id=name, live=True,
headers={
'Origin': 'https://live.fc2.com',
'Referer': url,
}))
self._sort_formats(formats)
for fmt in formats:
fmt.update({
'protocol': 'fc2_live',
'ws': ws,
})
title = self._html_search_meta(('og:title', 'twitter:title'), webpage, 'live title', fatal=False)
if not title:
title = self._html_extract_title(webpage, 'html title', fatal=False)
if title:
# remove service name in <title>
title = re.sub(r'\s+-\s+.+$', '', title)
uploader = None
if title:
match = self._search_regex(r'^(.+?)\s*\[(.+?)\]$', title, 'title and uploader', default=None, group=(1, 2))
if match and all(match):
title, uploader = match
live_info_view = self._search_regex(r'(?s)liveInfoView\s*:\s*({.+?}),\s*premiumStateView', webpage, 'user info', fatal=False) or None
if live_info_view:
# remove jQuery code from object literal
live_info_view = re.sub(r'\$\(.+?\)[^,]+,', '"",', live_info_view)
live_info_view = self._parse_json(js_to_json(live_info_view), video_id)
return {
'id': video_id,
'title': title or traverse_obj(live_info_view, 'title'),
'description': self._html_search_meta(
('og:description', 'twitter:description'),
webpage, 'live description', fatal=False) or traverse_obj(live_info_view, 'info'),
'formats': formats,
'uploader': uploader or traverse_obj(live_info_view, 'name'),
'uploader_id': video_id,
'thumbnail': traverse_obj(live_info_view, 'thumb'),
'is_live': True,
}

View File

@ -4,6 +4,7 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import asyncio import asyncio
import atexit
import base64 import base64
import binascii import binascii
import calendar import calendar
@ -5348,6 +5349,7 @@ def __init__(self, url, headers=None):
self.conn = compat_websockets.connect( self.conn = compat_websockets.connect(
url, extra_headers=headers, ping_interval=None, url, extra_headers=headers, ping_interval=None,
close_timeout=float('inf'), loop=self.loop, ping_timeout=float('inf')) close_timeout=float('inf'), loop=self.loop, ping_timeout=float('inf'))
atexit.register(self.__exit__, None, None, None)
def __enter__(self): def __enter__(self):
self.pool = self.run_with_loop(self.conn.__aenter__(), self.loop) self.pool = self.run_with_loop(self.conn.__aenter__(), self.loop)
@ -5364,7 +5366,7 @@ def __exit__(self, type, value, traceback):
return self.run_with_loop(self.conn.__aexit__(type, value, traceback), self.loop) return self.run_with_loop(self.conn.__aexit__(type, value, traceback), self.loop)
finally: finally:
self.loop.close() self.loop.close()
self.r_cancel_all_tasks(self.loop) self._cancel_all_tasks(self.loop)
# taken from https://github.com/python/cpython/blob/3.9/Lib/asyncio/runners.py with modifications # taken from https://github.com/python/cpython/blob/3.9/Lib/asyncio/runners.py with modifications
# for contributors: If there's any new library using asyncio needs to be run in non-async, move these function out of this class # for contributors: If there's any new library using asyncio needs to be run in non-async, move these function out of this class