test_YoutubeDL.py 48 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192
  1. #!/usr/bin/env python
  2. # coding: utf-8
  3. from __future__ import unicode_literals
  4. # Allow direct execution
  5. import os
  6. import sys
  7. import unittest
  8. sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  9. import copy
  10. import json
  11. from test.helper import (
  12. FakeYDL,
  13. assertRegexpMatches,
  14. try_rm,
  15. )
  16. from youtube_dl import YoutubeDL
  17. from youtube_dl.compat import (
  18. compat_http_cookiejar_Cookie,
  19. compat_http_cookies_SimpleCookie,
  20. compat_kwargs,
  21. compat_open as open,
  22. compat_str,
  23. compat_urllib_error,
  24. )
  25. from youtube_dl.extractor import YoutubeIE
  26. from youtube_dl.extractor.common import InfoExtractor
  27. from youtube_dl.postprocessor.common import PostProcessor
  28. from youtube_dl.utils import (
  29. ExtractorError,
  30. match_filter_func,
  31. traverse_obj,
  32. )
  33. TEST_URL = 'http://localhost/sample.mp4'
  34. class YDL(FakeYDL):
  35. def __init__(self, *args, **kwargs):
  36. super(YDL, self).__init__(*args, **kwargs)
  37. self.downloaded_info_dicts = []
  38. self.msgs = []
  39. def process_info(self, info_dict):
  40. self.downloaded_info_dicts.append(info_dict.copy())
  41. def to_screen(self, msg):
  42. self.msgs.append(msg)
  43. def dl(self, *args, **kwargs):
  44. assert False, 'Downloader must not be invoked for test_YoutubeDL'
  45. def _make_result(formats, **kwargs):
  46. res = {
  47. 'formats': formats,
  48. 'id': 'testid',
  49. 'title': 'testttitle',
  50. 'extractor': 'testex',
  51. 'extractor_key': 'TestEx',
  52. 'webpage_url': 'http://example.com/watch?v=shenanigans',
  53. }
  54. res.update(**compat_kwargs(kwargs))
  55. return res
  56. class TestFormatSelection(unittest.TestCase):
  57. def test_prefer_free_formats(self):
  58. # Same resolution => download webm
  59. ydl = YDL()
  60. ydl.params['prefer_free_formats'] = True
  61. formats = [
  62. {'ext': 'webm', 'height': 460, 'url': TEST_URL},
  63. {'ext': 'mp4', 'height': 460, 'url': TEST_URL},
  64. ]
  65. info_dict = _make_result(formats)
  66. yie = YoutubeIE(ydl)
  67. yie._sort_formats(info_dict['formats'])
  68. ydl.process_ie_result(info_dict)
  69. downloaded = ydl.downloaded_info_dicts[0]
  70. self.assertEqual(downloaded['ext'], 'webm')
  71. # Different resolution => download best quality (mp4)
  72. ydl = YDL()
  73. ydl.params['prefer_free_formats'] = True
  74. formats = [
  75. {'ext': 'webm', 'height': 720, 'url': TEST_URL},
  76. {'ext': 'mp4', 'height': 1080, 'url': TEST_URL},
  77. ]
  78. info_dict['formats'] = formats
  79. yie = YoutubeIE(ydl)
  80. yie._sort_formats(info_dict['formats'])
  81. ydl.process_ie_result(info_dict)
  82. downloaded = ydl.downloaded_info_dicts[0]
  83. self.assertEqual(downloaded['ext'], 'mp4')
  84. # No prefer_free_formats => prefer mp4 and flv for greater compatibility
  85. ydl = YDL()
  86. ydl.params['prefer_free_formats'] = False
  87. formats = [
  88. {'ext': 'webm', 'height': 720, 'url': TEST_URL},
  89. {'ext': 'mp4', 'height': 720, 'url': TEST_URL},
  90. {'ext': 'flv', 'height': 720, 'url': TEST_URL},
  91. ]
  92. info_dict['formats'] = formats
  93. yie = YoutubeIE(ydl)
  94. yie._sort_formats(info_dict['formats'])
  95. ydl.process_ie_result(info_dict)
  96. downloaded = ydl.downloaded_info_dicts[0]
  97. self.assertEqual(downloaded['ext'], 'mp4')
  98. ydl = YDL()
  99. ydl.params['prefer_free_formats'] = False
  100. formats = [
  101. {'ext': 'flv', 'height': 720, 'url': TEST_URL},
  102. {'ext': 'webm', 'height': 720, 'url': TEST_URL},
  103. ]
  104. info_dict['formats'] = formats
  105. yie = YoutubeIE(ydl)
  106. yie._sort_formats(info_dict['formats'])
  107. ydl.process_ie_result(info_dict)
  108. downloaded = ydl.downloaded_info_dicts[0]
  109. self.assertEqual(downloaded['ext'], 'flv')
  110. def test_format_selection(self):
  111. formats = [
  112. {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
  113. {'format_id': 'example-with-dashes', 'ext': 'webm', 'preference': 1, 'url': TEST_URL},
  114. {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL},
  115. {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL},
  116. {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL},
  117. ]
  118. info_dict = _make_result(formats)
  119. ydl = YDL({'format': '20/47'})
  120. ydl.process_ie_result(info_dict.copy())
  121. downloaded = ydl.downloaded_info_dicts[0]
  122. self.assertEqual(downloaded['format_id'], '47')
  123. ydl = YDL({'format': '20/71/worst'})
  124. ydl.process_ie_result(info_dict.copy())
  125. downloaded = ydl.downloaded_info_dicts[0]
  126. self.assertEqual(downloaded['format_id'], '35')
  127. ydl = YDL()
  128. ydl.process_ie_result(info_dict.copy())
  129. downloaded = ydl.downloaded_info_dicts[0]
  130. self.assertEqual(downloaded['format_id'], '2')
  131. ydl = YDL({'format': 'webm/mp4'})
  132. ydl.process_ie_result(info_dict.copy())
  133. downloaded = ydl.downloaded_info_dicts[0]
  134. self.assertEqual(downloaded['format_id'], '47')
  135. ydl = YDL({'format': '3gp/40/mp4'})
  136. ydl.process_ie_result(info_dict.copy())
  137. downloaded = ydl.downloaded_info_dicts[0]
  138. self.assertEqual(downloaded['format_id'], '35')
  139. ydl = YDL({'format': 'example-with-dashes'})
  140. ydl.process_ie_result(info_dict.copy())
  141. downloaded = ydl.downloaded_info_dicts[0]
  142. self.assertEqual(downloaded['format_id'], 'example-with-dashes')
  143. def test_format_selection_audio(self):
  144. formats = [
  145. {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
  146. {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
  147. {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': TEST_URL},
  148. {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': TEST_URL},
  149. ]
  150. info_dict = _make_result(formats)
  151. ydl = YDL({'format': 'bestaudio'})
  152. ydl.process_ie_result(info_dict.copy())
  153. downloaded = ydl.downloaded_info_dicts[0]
  154. self.assertEqual(downloaded['format_id'], 'audio-high')
  155. ydl = YDL({'format': 'worstaudio'})
  156. ydl.process_ie_result(info_dict.copy())
  157. downloaded = ydl.downloaded_info_dicts[0]
  158. self.assertEqual(downloaded['format_id'], 'audio-low')
  159. formats = [
  160. {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
  161. {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': TEST_URL},
  162. ]
  163. info_dict = _make_result(formats)
  164. ydl = YDL({'format': 'bestaudio/worstaudio/best'})
  165. ydl.process_ie_result(info_dict.copy())
  166. downloaded = ydl.downloaded_info_dicts[0]
  167. self.assertEqual(downloaded['format_id'], 'vid-high')
  168. def test_format_selection_audio_exts(self):
  169. formats = [
  170. {'format_id': 'mp3-64', 'ext': 'mp3', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
  171. {'format_id': 'ogg-64', 'ext': 'ogg', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
  172. {'format_id': 'aac-64', 'ext': 'aac', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
  173. {'format_id': 'mp3-32', 'ext': 'mp3', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
  174. {'format_id': 'aac-32', 'ext': 'aac', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
  175. ]
  176. info_dict = _make_result(formats)
  177. ydl = YDL({'format': 'best'})
  178. ie = YoutubeIE(ydl)
  179. ie._sort_formats(info_dict['formats'])
  180. ydl.process_ie_result(copy.deepcopy(info_dict))
  181. downloaded = ydl.downloaded_info_dicts[0]
  182. self.assertEqual(downloaded['format_id'], 'aac-64')
  183. ydl = YDL({'format': 'mp3'})
  184. ie = YoutubeIE(ydl)
  185. ie._sort_formats(info_dict['formats'])
  186. ydl.process_ie_result(copy.deepcopy(info_dict))
  187. downloaded = ydl.downloaded_info_dicts[0]
  188. self.assertEqual(downloaded['format_id'], 'mp3-64')
  189. ydl = YDL({'prefer_free_formats': True})
  190. ie = YoutubeIE(ydl)
  191. ie._sort_formats(info_dict['formats'])
  192. ydl.process_ie_result(copy.deepcopy(info_dict))
  193. downloaded = ydl.downloaded_info_dicts[0]
  194. self.assertEqual(downloaded['format_id'], 'ogg-64')
  195. def test_format_selection_video(self):
  196. formats = [
  197. {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': TEST_URL},
  198. {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': TEST_URL},
  199. {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': TEST_URL},
  200. ]
  201. info_dict = _make_result(formats)
  202. ydl = YDL({'format': 'bestvideo'})
  203. ydl.process_ie_result(info_dict.copy())
  204. downloaded = ydl.downloaded_info_dicts[0]
  205. self.assertEqual(downloaded['format_id'], 'dash-video-high')
  206. ydl = YDL({'format': 'worstvideo'})
  207. ydl.process_ie_result(info_dict.copy())
  208. downloaded = ydl.downloaded_info_dicts[0]
  209. self.assertEqual(downloaded['format_id'], 'dash-video-low')
  210. ydl = YDL({'format': 'bestvideo[format_id^=dash][format_id$=low]'})
  211. ydl.process_ie_result(info_dict.copy())
  212. downloaded = ydl.downloaded_info_dicts[0]
  213. self.assertEqual(downloaded['format_id'], 'dash-video-low')
  214. formats = [
  215. {'format_id': 'vid-vcodec-dot', 'ext': 'mp4', 'preference': 1, 'vcodec': 'avc1.123456', 'acodec': 'none', 'url': TEST_URL},
  216. ]
  217. info_dict = _make_result(formats)
  218. ydl = YDL({'format': 'bestvideo[vcodec=avc1.123456]'})
  219. ydl.process_ie_result(info_dict.copy())
  220. downloaded = ydl.downloaded_info_dicts[0]
  221. self.assertEqual(downloaded['format_id'], 'vid-vcodec-dot')
  222. def test_format_selection_string_ops(self):
  223. formats = [
  224. {'format_id': 'abc-cba', 'ext': 'mp4', 'url': TEST_URL},
  225. {'format_id': 'zxc-cxz', 'ext': 'webm', 'url': TEST_URL},
  226. ]
  227. info_dict = _make_result(formats)
  228. # equals (=)
  229. ydl = YDL({'format': '[format_id=abc-cba]'})
  230. ydl.process_ie_result(info_dict.copy())
  231. downloaded = ydl.downloaded_info_dicts[0]
  232. self.assertEqual(downloaded['format_id'], 'abc-cba')
  233. # does not equal (!=)
  234. ydl = YDL({'format': '[format_id!=abc-cba]'})
  235. ydl.process_ie_result(info_dict.copy())
  236. downloaded = ydl.downloaded_info_dicts[0]
  237. self.assertEqual(downloaded['format_id'], 'zxc-cxz')
  238. ydl = YDL({'format': '[format_id!=abc-cba][format_id!=zxc-cxz]'})
  239. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  240. # starts with (^=)
  241. ydl = YDL({'format': '[format_id^=abc]'})
  242. ydl.process_ie_result(info_dict.copy())
  243. downloaded = ydl.downloaded_info_dicts[0]
  244. self.assertEqual(downloaded['format_id'], 'abc-cba')
  245. # does not start with (!^=)
  246. ydl = YDL({'format': '[format_id!^=abc]'})
  247. ydl.process_ie_result(info_dict.copy())
  248. downloaded = ydl.downloaded_info_dicts[0]
  249. self.assertEqual(downloaded['format_id'], 'zxc-cxz')
  250. ydl = YDL({'format': '[format_id!^=abc][format_id!^=zxc]'})
  251. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  252. # ends with ($=)
  253. ydl = YDL({'format': '[format_id$=cba]'})
  254. ydl.process_ie_result(info_dict.copy())
  255. downloaded = ydl.downloaded_info_dicts[0]
  256. self.assertEqual(downloaded['format_id'], 'abc-cba')
  257. # does not end with (!$=)
  258. ydl = YDL({'format': '[format_id!$=cba]'})
  259. ydl.process_ie_result(info_dict.copy())
  260. downloaded = ydl.downloaded_info_dicts[0]
  261. self.assertEqual(downloaded['format_id'], 'zxc-cxz')
  262. ydl = YDL({'format': '[format_id!$=cba][format_id!$=cxz]'})
  263. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  264. # contains (*=)
  265. ydl = YDL({'format': '[format_id*=bc-cb]'})
  266. ydl.process_ie_result(info_dict.copy())
  267. downloaded = ydl.downloaded_info_dicts[0]
  268. self.assertEqual(downloaded['format_id'], 'abc-cba')
  269. # does not contain (!*=)
  270. ydl = YDL({'format': '[format_id!*=bc-cb]'})
  271. ydl.process_ie_result(info_dict.copy())
  272. downloaded = ydl.downloaded_info_dicts[0]
  273. self.assertEqual(downloaded['format_id'], 'zxc-cxz')
  274. ydl = YDL({'format': '[format_id!*=abc][format_id!*=zxc]'})
  275. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  276. ydl = YDL({'format': '[format_id!*=-]'})
  277. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  278. def test_youtube_format_selection(self):
  279. order = [
  280. '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '17', '36', '13',
  281. # Apple HTTP Live Streaming
  282. '96', '95', '94', '93', '92', '132', '151',
  283. # 3D
  284. '85', '84', '102', '83', '101', '82', '100',
  285. # Dash video
  286. '137', '248', '136', '247', '135', '246',
  287. '245', '244', '134', '243', '133', '242', '160',
  288. # Dash audio
  289. '141', '172', '140', '171', '139',
  290. ]
  291. def format_info(f_id):
  292. info = YoutubeIE._formats[f_id].copy()
  293. # XXX: In real cases InfoExtractor._parse_mpd_formats() fills up 'acodec'
  294. # and 'vcodec', while in tests such information is incomplete since
  295. # commit a6c2c24479e5f4827ceb06f64d855329c0a6f593
  296. # test_YoutubeDL.test_youtube_format_selection is broken without
  297. # this fix
  298. if 'acodec' in info and 'vcodec' not in info:
  299. info['vcodec'] = 'none'
  300. elif 'vcodec' in info and 'acodec' not in info:
  301. info['acodec'] = 'none'
  302. info['format_id'] = f_id
  303. info['url'] = 'url:' + f_id
  304. return info
  305. formats_order = [format_info(f_id) for f_id in order]
  306. info_dict = _make_result(list(formats_order), extractor='youtube')
  307. ydl = YDL({'format': 'bestvideo+bestaudio'})
  308. yie = YoutubeIE(ydl)
  309. yie._sort_formats(info_dict['formats'])
  310. ydl.process_ie_result(info_dict)
  311. downloaded = ydl.downloaded_info_dicts[0]
  312. self.assertEqual(downloaded['format_id'], '137+141')
  313. self.assertEqual(downloaded['ext'], 'mp4')
  314. info_dict = _make_result(list(formats_order), extractor='youtube')
  315. ydl = YDL({'format': 'bestvideo[height>=999999]+bestaudio/best'})
  316. yie = YoutubeIE(ydl)
  317. yie._sort_formats(info_dict['formats'])
  318. ydl.process_ie_result(info_dict)
  319. downloaded = ydl.downloaded_info_dicts[0]
  320. self.assertEqual(downloaded['format_id'], '38')
  321. info_dict = _make_result(list(formats_order), extractor='youtube')
  322. ydl = YDL({'format': 'bestvideo/best,bestaudio'})
  323. yie = YoutubeIE(ydl)
  324. yie._sort_formats(info_dict['formats'])
  325. ydl.process_ie_result(info_dict)
  326. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  327. self.assertEqual(downloaded_ids, ['137', '141'])
  328. info_dict = _make_result(list(formats_order), extractor='youtube')
  329. ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
  330. yie = YoutubeIE(ydl)
  331. yie._sort_formats(info_dict['formats'])
  332. ydl.process_ie_result(info_dict)
  333. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  334. self.assertEqual(downloaded_ids, ['137+141', '248+141'])
  335. info_dict = _make_result(list(formats_order), extractor='youtube')
  336. ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])[height<=720]+bestaudio'})
  337. yie = YoutubeIE(ydl)
  338. yie._sort_formats(info_dict['formats'])
  339. ydl.process_ie_result(info_dict)
  340. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  341. self.assertEqual(downloaded_ids, ['136+141', '247+141'])
  342. info_dict = _make_result(list(formats_order), extractor='youtube')
  343. ydl = YDL({'format': '(bestvideo[ext=none]/bestvideo[ext=webm])+bestaudio'})
  344. yie = YoutubeIE(ydl)
  345. yie._sort_formats(info_dict['formats'])
  346. ydl.process_ie_result(info_dict)
  347. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  348. self.assertEqual(downloaded_ids, ['248+141'])
  349. for f1, f2 in zip(formats_order, formats_order[1:]):
  350. info_dict = _make_result([f1, f2], extractor='youtube')
  351. ydl = YDL({'format': 'best/bestvideo'})
  352. yie = YoutubeIE(ydl)
  353. yie._sort_formats(info_dict['formats'])
  354. ydl.process_ie_result(info_dict)
  355. downloaded = ydl.downloaded_info_dicts[0]
  356. self.assertEqual(downloaded['format_id'], f1['format_id'])
  357. info_dict = _make_result([f2, f1], extractor='youtube')
  358. ydl = YDL({'format': 'best/bestvideo'})
  359. yie = YoutubeIE(ydl)
  360. yie._sort_formats(info_dict['formats'])
  361. ydl.process_ie_result(info_dict)
  362. downloaded = ydl.downloaded_info_dicts[0]
  363. self.assertEqual(downloaded['format_id'], f1['format_id'])
  364. def test_audio_only_extractor_format_selection(self):
  365. # For extractors with incomplete formats (all formats are audio-only or
  366. # video-only) best and worst should fallback to corresponding best/worst
  367. # video-only or audio-only formats (as per
  368. # https://github.com/ytdl-org/youtube-dl/pull/5556)
  369. formats = [
  370. {'format_id': 'low', 'ext': 'mp3', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
  371. {'format_id': 'high', 'ext': 'mp3', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
  372. ]
  373. info_dict = _make_result(formats)
  374. ydl = YDL({'format': 'best'})
  375. ydl.process_ie_result(info_dict.copy())
  376. downloaded = ydl.downloaded_info_dicts[0]
  377. self.assertEqual(downloaded['format_id'], 'high')
  378. ydl = YDL({'format': 'worst'})
  379. ydl.process_ie_result(info_dict.copy())
  380. downloaded = ydl.downloaded_info_dicts[0]
  381. self.assertEqual(downloaded['format_id'], 'low')
  382. def test_format_not_available(self):
  383. formats = [
  384. {'format_id': 'regular', 'ext': 'mp4', 'height': 360, 'url': TEST_URL},
  385. {'format_id': 'video', 'ext': 'mp4', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
  386. ]
  387. info_dict = _make_result(formats)
  388. # This must fail since complete video-audio format does not match filter
  389. # and extractor does not provide incomplete only formats (i.e. only
  390. # video-only or audio-only).
  391. ydl = YDL({'format': 'best[height>360]'})
  392. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  393. def test_format_selection_issue_10083(self):
  394. # See https://github.com/ytdl-org/youtube-dl/issues/10083
  395. formats = [
  396. {'format_id': 'regular', 'height': 360, 'url': TEST_URL},
  397. {'format_id': 'video', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
  398. {'format_id': 'audio', 'vcodec': 'none', 'url': TEST_URL},
  399. ]
  400. info_dict = _make_result(formats)
  401. ydl = YDL({'format': 'best[height>360]/bestvideo[height>360]+bestaudio'})
  402. ydl.process_ie_result(info_dict.copy())
  403. self.assertEqual(ydl.downloaded_info_dicts[0]['format_id'], 'video+audio')
  404. def test_invalid_format_specs(self):
  405. def assert_syntax_error(format_spec):
  406. ydl = YDL({'format': format_spec})
  407. info_dict = _make_result([{'format_id': 'foo', 'url': TEST_URL}])
  408. self.assertRaises(SyntaxError, ydl.process_ie_result, info_dict)
  409. assert_syntax_error('bestvideo,,best')
  410. assert_syntax_error('+bestaudio')
  411. assert_syntax_error('bestvideo+')
  412. assert_syntax_error('/')
  413. assert_syntax_error('bestvideo+bestvideo+bestaudio')
  414. def test_format_filtering(self):
  415. formats = [
  416. {'format_id': 'A', 'filesize': 500, 'width': 1000},
  417. {'format_id': 'B', 'filesize': 1000, 'width': 500},
  418. {'format_id': 'C', 'filesize': 1000, 'width': 400},
  419. {'format_id': 'D', 'filesize': 2000, 'width': 600},
  420. {'format_id': 'E', 'filesize': 3000},
  421. {'format_id': 'F'},
  422. {'format_id': 'G', 'filesize': 1000000},
  423. ]
  424. for f in formats:
  425. f['url'] = 'http://_/'
  426. f['ext'] = 'unknown'
  427. info_dict = _make_result(formats)
  428. ydl = YDL({'format': 'best[filesize<3000]'})
  429. ydl.process_ie_result(info_dict)
  430. downloaded = ydl.downloaded_info_dicts[0]
  431. self.assertEqual(downloaded['format_id'], 'D')
  432. ydl = YDL({'format': 'best[filesize<=3000]'})
  433. ydl.process_ie_result(info_dict)
  434. downloaded = ydl.downloaded_info_dicts[0]
  435. self.assertEqual(downloaded['format_id'], 'E')
  436. ydl = YDL({'format': 'best[filesize <= ? 3000]'})
  437. ydl.process_ie_result(info_dict)
  438. downloaded = ydl.downloaded_info_dicts[0]
  439. self.assertEqual(downloaded['format_id'], 'F')
  440. ydl = YDL({'format': 'best [filesize = 1000] [width>450]'})
  441. ydl.process_ie_result(info_dict)
  442. downloaded = ydl.downloaded_info_dicts[0]
  443. self.assertEqual(downloaded['format_id'], 'B')
  444. ydl = YDL({'format': 'best [filesize = 1000] [width!=450]'})
  445. ydl.process_ie_result(info_dict)
  446. downloaded = ydl.downloaded_info_dicts[0]
  447. self.assertEqual(downloaded['format_id'], 'C')
  448. ydl = YDL({'format': '[filesize>?1]'})
  449. ydl.process_ie_result(info_dict)
  450. downloaded = ydl.downloaded_info_dicts[0]
  451. self.assertEqual(downloaded['format_id'], 'G')
  452. ydl = YDL({'format': '[filesize<1M]'})
  453. ydl.process_ie_result(info_dict)
  454. downloaded = ydl.downloaded_info_dicts[0]
  455. self.assertEqual(downloaded['format_id'], 'E')
  456. ydl = YDL({'format': '[filesize<1MiB]'})
  457. ydl.process_ie_result(info_dict)
  458. downloaded = ydl.downloaded_info_dicts[0]
  459. self.assertEqual(downloaded['format_id'], 'G')
  460. ydl = YDL({'format': 'all[width>=400][width<=600]'})
  461. ydl.process_ie_result(info_dict)
  462. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  463. self.assertEqual(downloaded_ids, ['B', 'C', 'D'])
  464. ydl = YDL({'format': 'best[height<40]'})
  465. try:
  466. ydl.process_ie_result(info_dict)
  467. except ExtractorError:
  468. pass
  469. self.assertEqual(ydl.downloaded_info_dicts, [])
  470. def test_default_format_spec(self):
  471. ydl = YDL({'simulate': True})
  472. self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
  473. ydl = YDL({})
  474. self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')
  475. ydl = YDL({'simulate': True})
  476. self.assertEqual(ydl._default_format_spec({'is_live': True}), 'bestvideo+bestaudio/best')
  477. ydl = YDL({'outtmpl': '-'})
  478. self.assertEqual(ydl._default_format_spec({}), 'best/bestvideo+bestaudio')
  479. ydl = YDL({})
  480. self.assertEqual(ydl._default_format_spec({}, download=False), 'bestvideo+bestaudio/best')
  481. self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')
  482. class TestYoutubeDL(unittest.TestCase):
  483. def test_subtitles(self):
  484. def s_formats(lang, autocaption=False):
  485. return [{
  486. 'ext': ext,
  487. 'url': 'http://localhost/video.%s.%s' % (lang, ext),
  488. '_auto': autocaption,
  489. } for ext in ['vtt', 'srt', 'ass']]
  490. subtitles = dict((l, s_formats(l)) for l in ['en', 'fr', 'es'])
  491. auto_captions = dict((l, s_formats(l, True)) for l in ['it', 'pt', 'es'])
  492. info_dict = {
  493. 'id': 'test',
  494. 'title': 'Test',
  495. 'url': 'http://localhost/video.mp4',
  496. 'subtitles': subtitles,
  497. 'automatic_captions': auto_captions,
  498. 'extractor': 'TEST',
  499. }
  500. def get_info(params={}):
  501. params.setdefault('simulate', True)
  502. ydl = YDL(params)
  503. ydl.report_warning = lambda *args, **kargs: None
  504. return ydl.process_video_result(info_dict, download=False)
  505. result = get_info()
  506. self.assertFalse(result.get('requested_subtitles'))
  507. self.assertEqual(result['subtitles'], subtitles)
  508. self.assertEqual(result['automatic_captions'], auto_captions)
  509. result = get_info({'writesubtitles': True})
  510. subs = result['requested_subtitles']
  511. self.assertTrue(subs)
  512. self.assertEqual(set(subs.keys()), set(['en']))
  513. self.assertTrue(subs['en'].get('data') is None)
  514. self.assertEqual(subs['en']['ext'], 'ass')
  515. result = get_info({'writesubtitles': True, 'subtitlesformat': 'foo/srt'})
  516. subs = result['requested_subtitles']
  517. self.assertEqual(subs['en']['ext'], 'srt')
  518. result = get_info({'writesubtitles': True, 'subtitleslangs': ['es', 'fr', 'it']})
  519. subs = result['requested_subtitles']
  520. self.assertTrue(subs)
  521. self.assertEqual(set(subs.keys()), set(['es', 'fr']))
  522. result = get_info({'writesubtitles': True, 'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
  523. subs = result['requested_subtitles']
  524. self.assertTrue(subs)
  525. self.assertEqual(set(subs.keys()), set(['es', 'pt']))
  526. self.assertFalse(subs['es']['_auto'])
  527. self.assertTrue(subs['pt']['_auto'])
  528. result = get_info({'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
  529. subs = result['requested_subtitles']
  530. self.assertTrue(subs)
  531. self.assertEqual(set(subs.keys()), set(['es', 'pt']))
  532. self.assertTrue(subs['es']['_auto'])
  533. self.assertTrue(subs['pt']['_auto'])
  534. def test_add_extra_info(self):
  535. test_dict = {
  536. 'extractor': 'Foo',
  537. }
  538. extra_info = {
  539. 'extractor': 'Bar',
  540. 'playlist': 'funny videos',
  541. }
  542. YDL.add_extra_info(test_dict, extra_info)
  543. self.assertEqual(test_dict['extractor'], 'Foo')
  544. self.assertEqual(test_dict['playlist'], 'funny videos')
  545. def test_prepare_filename(self):
  546. info = {
  547. 'id': '1234',
  548. 'ext': 'mp4',
  549. 'width': None,
  550. 'height': 1080,
  551. 'title1': '$PATH',
  552. 'title2': '%PATH%',
  553. }
  554. def fname(templ, na_placeholder='NA'):
  555. params = {'outtmpl': templ}
  556. if na_placeholder != 'NA':
  557. params['outtmpl_na_placeholder'] = na_placeholder
  558. ydl = YoutubeDL(params)
  559. return ydl.prepare_filename(info)
  560. self.assertEqual(fname('%(id)s.%(ext)s'), '1234.mp4')
  561. self.assertEqual(fname('%(id)s-%(width)s.%(ext)s'), '1234-NA.mp4')
  562. NA_TEST_OUTTMPL = '%(uploader_date)s-%(width)d-%(id)s.%(ext)s'
  563. # Replace missing fields with 'NA' by default
  564. self.assertEqual(fname(NA_TEST_OUTTMPL), 'NA-NA-1234.mp4')
  565. # Or by provided placeholder
  566. self.assertEqual(fname(NA_TEST_OUTTMPL, na_placeholder='none'), 'none-none-1234.mp4')
  567. self.assertEqual(fname(NA_TEST_OUTTMPL, na_placeholder=''), '--1234.mp4')
  568. self.assertEqual(fname('%(height)d.%(ext)s'), '1080.mp4')
  569. self.assertEqual(fname('%(height)6d.%(ext)s'), ' 1080.mp4')
  570. self.assertEqual(fname('%(height)-6d.%(ext)s'), '1080 .mp4')
  571. self.assertEqual(fname('%(height)06d.%(ext)s'), '001080.mp4')
  572. self.assertEqual(fname('%(height) 06d.%(ext)s'), ' 01080.mp4')
  573. self.assertEqual(fname('%(height) 06d.%(ext)s'), ' 01080.mp4')
  574. self.assertEqual(fname('%(height)0 6d.%(ext)s'), ' 01080.mp4')
  575. self.assertEqual(fname('%(height)0 6d.%(ext)s'), ' 01080.mp4')
  576. self.assertEqual(fname('%(height) 0 6d.%(ext)s'), ' 01080.mp4')
  577. self.assertEqual(fname('%%'), '%')
  578. self.assertEqual(fname('%%%%'), '%%')
  579. self.assertEqual(fname('%%(height)06d.%(ext)s'), '%(height)06d.mp4')
  580. self.assertEqual(fname('%(width)06d.%(ext)s'), 'NA.mp4')
  581. self.assertEqual(fname('%(width)06d.%%(ext)s'), 'NA.%(ext)s')
  582. self.assertEqual(fname('%%(width)06d.%(ext)s'), '%(width)06d.mp4')
  583. self.assertEqual(fname('Hello %(title1)s'), 'Hello $PATH')
  584. self.assertEqual(fname('Hello %(title2)s'), 'Hello %PATH%')
  585. def test_format_note(self):
  586. ydl = YoutubeDL()
  587. self.assertEqual(ydl._format_note({}), '')
  588. assertRegexpMatches(self, ydl._format_note({
  589. 'vbr': 10,
  590. }), r'^\s*10k$')
  591. assertRegexpMatches(self, ydl._format_note({
  592. 'fps': 30,
  593. }), r'^30fps$')
  594. def test_postprocessors(self):
  595. filename = 'post-processor-testfile.mp4'
  596. audiofile = filename + '.mp3'
  597. class SimplePP(PostProcessor):
  598. def run(self, info):
  599. with open(audiofile, 'w') as f:
  600. f.write('EXAMPLE')
  601. return [info['filepath']], info
  602. def run_pp(params, PP):
  603. with open(filename, 'w') as f:
  604. f.write('EXAMPLE')
  605. ydl = YoutubeDL(params)
  606. ydl.add_post_processor(PP())
  607. ydl.post_process(filename, {'filepath': filename})
  608. run_pp({'keepvideo': True}, SimplePP)
  609. self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
  610. self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
  611. os.unlink(filename)
  612. os.unlink(audiofile)
  613. run_pp({'keepvideo': False}, SimplePP)
  614. self.assertFalse(os.path.exists(filename), '%s exists' % filename)
  615. self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
  616. os.unlink(audiofile)
  617. class ModifierPP(PostProcessor):
  618. def run(self, info):
  619. with open(info['filepath'], 'w') as f:
  620. f.write('MODIFIED')
  621. return [], info
  622. run_pp({'keepvideo': False}, ModifierPP)
  623. self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
  624. os.unlink(filename)
  625. def test_match_filter(self):
  626. class FilterYDL(YDL):
  627. def __init__(self, *args, **kwargs):
  628. super(FilterYDL, self).__init__(*args, **kwargs)
  629. self.params['simulate'] = True
  630. def process_info(self, info_dict):
  631. super(YDL, self).process_info(info_dict)
  632. def _match_entry(self, info_dict, incomplete):
  633. res = super(FilterYDL, self)._match_entry(info_dict, incomplete)
  634. if res is None:
  635. self.downloaded_info_dicts.append(info_dict)
  636. return res
  637. first = {
  638. 'id': '1',
  639. 'url': TEST_URL,
  640. 'title': 'one',
  641. 'extractor': 'TEST',
  642. 'duration': 30,
  643. 'filesize': 10 * 1024,
  644. 'playlist_id': '42',
  645. 'uploader': "變態妍字幕版 太妍 тест",
  646. 'creator': "тест ' 123 ' тест--",
  647. }
  648. second = {
  649. 'id': '2',
  650. 'url': TEST_URL,
  651. 'title': 'two',
  652. 'extractor': 'TEST',
  653. 'duration': 10,
  654. 'description': 'foo',
  655. 'filesize': 5 * 1024,
  656. 'playlist_id': '43',
  657. 'uploader': "тест 123",
  658. }
  659. videos = [first, second]
  660. def get_videos(filter_=None):
  661. ydl = FilterYDL({'match_filter': filter_})
  662. for v in videos:
  663. ydl.process_ie_result(v, download=True)
  664. return [v['id'] for v in ydl.downloaded_info_dicts]
  665. res = get_videos()
  666. self.assertEqual(res, ['1', '2'])
  667. def f(v):
  668. if v['id'] == '1':
  669. return None
  670. else:
  671. return 'Video id is not 1'
  672. res = get_videos(f)
  673. self.assertEqual(res, ['1'])
  674. f = match_filter_func('duration < 30')
  675. res = get_videos(f)
  676. self.assertEqual(res, ['2'])
  677. f = match_filter_func('description = foo')
  678. res = get_videos(f)
  679. self.assertEqual(res, ['2'])
  680. f = match_filter_func('description =? foo')
  681. res = get_videos(f)
  682. self.assertEqual(res, ['1', '2'])
  683. f = match_filter_func('filesize > 5KiB')
  684. res = get_videos(f)
  685. self.assertEqual(res, ['1'])
  686. f = match_filter_func('playlist_id = 42')
  687. res = get_videos(f)
  688. self.assertEqual(res, ['1'])
  689. f = match_filter_func('uploader = "變態妍字幕版 太妍 тест"')
  690. res = get_videos(f)
  691. self.assertEqual(res, ['1'])
  692. f = match_filter_func('uploader != "變態妍字幕版 太妍 тест"')
  693. res = get_videos(f)
  694. self.assertEqual(res, ['2'])
  695. f = match_filter_func('creator = "тест \' 123 \' тест--"')
  696. res = get_videos(f)
  697. self.assertEqual(res, ['1'])
  698. f = match_filter_func("creator = 'тест \\' 123 \\' тест--'")
  699. res = get_videos(f)
  700. self.assertEqual(res, ['1'])
  701. f = match_filter_func(r"creator = 'тест \' 123 \' тест--' & duration > 30")
  702. res = get_videos(f)
  703. self.assertEqual(res, [])
  704. def test_playlist_items_selection(self):
  705. entries = [{
  706. 'id': compat_str(i),
  707. 'title': compat_str(i),
  708. 'url': TEST_URL,
  709. } for i in range(1, 5)]
  710. playlist = {
  711. '_type': 'playlist',
  712. 'id': 'test',
  713. 'entries': entries,
  714. 'extractor': 'test:playlist',
  715. 'extractor_key': 'test:playlist',
  716. 'webpage_url': 'http://example.com',
  717. }
  718. def get_downloaded_info_dicts(params):
  719. ydl = YDL(params)
  720. # make a deep copy because the dictionary and nested entries
  721. # can be modified
  722. ydl.process_ie_result(copy.deepcopy(playlist))
  723. return ydl.downloaded_info_dicts
  724. def get_ids(params):
  725. return [int(v['id']) for v in get_downloaded_info_dicts(params)]
  726. result = get_ids({})
  727. self.assertEqual(result, [1, 2, 3, 4])
  728. result = get_ids({'playlistend': 10})
  729. self.assertEqual(result, [1, 2, 3, 4])
  730. result = get_ids({'playlistend': 2})
  731. self.assertEqual(result, [1, 2])
  732. result = get_ids({'playliststart': 10})
  733. self.assertEqual(result, [])
  734. result = get_ids({'playliststart': 2})
  735. self.assertEqual(result, [2, 3, 4])
  736. result = get_ids({'playlist_items': '2-4'})
  737. self.assertEqual(result, [2, 3, 4])
  738. result = get_ids({'playlist_items': '2,4'})
  739. self.assertEqual(result, [2, 4])
  740. result = get_ids({'playlist_items': '10'})
  741. self.assertEqual(result, [])
  742. result = get_ids({'playlist_items': '3-10'})
  743. self.assertEqual(result, [3, 4])
  744. result = get_ids({'playlist_items': '2-4,3-4,3'})
  745. self.assertEqual(result, [2, 3, 4])
  746. # Tests for https://github.com/ytdl-org/youtube-dl/issues/10591
  747. # @{
  748. result = get_downloaded_info_dicts({'playlist_items': '2-4,3-4,3'})
  749. self.assertEqual(result[0]['playlist_index'], 2)
  750. self.assertEqual(result[1]['playlist_index'], 3)
  751. result = get_downloaded_info_dicts({'playlist_items': '2-4,3-4,3'})
  752. self.assertEqual(result[0]['playlist_index'], 2)
  753. self.assertEqual(result[1]['playlist_index'], 3)
  754. self.assertEqual(result[2]['playlist_index'], 4)
  755. result = get_downloaded_info_dicts({'playlist_items': '4,2'})
  756. self.assertEqual(result[0]['playlist_index'], 4)
  757. self.assertEqual(result[1]['playlist_index'], 2)
  758. # @}
  759. def test_urlopen_no_file_protocol(self):
  760. # see https://github.com/ytdl-org/youtube-dl/issues/8227
  761. ydl = YDL()
  762. self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')
  763. def test_do_not_override_ie_key_in_url_transparent(self):
  764. ydl = YDL()
  765. class Foo1IE(InfoExtractor):
  766. _VALID_URL = r'foo1:'
  767. def _real_extract(self, url):
  768. return {
  769. '_type': 'url_transparent',
  770. 'url': 'foo2:',
  771. 'ie_key': 'Foo2',
  772. 'title': 'foo1 title',
  773. 'id': 'foo1_id',
  774. }
  775. class Foo2IE(InfoExtractor):
  776. _VALID_URL = r'foo2:'
  777. def _real_extract(self, url):
  778. return {
  779. '_type': 'url',
  780. 'url': 'foo3:',
  781. 'ie_key': 'Foo3',
  782. }
  783. class Foo3IE(InfoExtractor):
  784. _VALID_URL = r'foo3:'
  785. def _real_extract(self, url):
  786. return _make_result([{'url': TEST_URL}], title='foo3 title')
  787. ydl.add_info_extractor(Foo1IE(ydl))
  788. ydl.add_info_extractor(Foo2IE(ydl))
  789. ydl.add_info_extractor(Foo3IE(ydl))
  790. ydl.extract_info('foo1:')
  791. downloaded = ydl.downloaded_info_dicts[0]
  792. self.assertEqual(downloaded['url'], TEST_URL)
  793. self.assertEqual(downloaded['title'], 'foo1 title')
  794. self.assertEqual(downloaded['id'], 'testid')
  795. self.assertEqual(downloaded['extractor'], 'testex')
  796. self.assertEqual(downloaded['extractor_key'], 'TestEx')
  797. # Test case for https://github.com/ytdl-org/youtube-dl/issues/27064
  798. def test_ignoreerrors_for_playlist_with_url_transparent_iterable_entries(self):
  799. ydl = YDL({
  800. 'format': 'extra',
  801. 'ignoreerrors': True,
  802. })
  803. ydl.trouble = lambda *_, **__: None
  804. class VideoIE(InfoExtractor):
  805. _VALID_URL = r'video:(?P<id>\d+)'
  806. def _real_extract(self, url):
  807. video_id = self._match_id(url)
  808. formats = [{
  809. 'format_id': 'default',
  810. 'url': 'url:',
  811. }]
  812. if video_id == '0':
  813. raise ExtractorError('foo')
  814. if video_id == '2':
  815. formats.append({
  816. 'format_id': 'extra',
  817. 'url': TEST_URL,
  818. })
  819. return {
  820. 'id': video_id,
  821. 'title': 'Video %s' % video_id,
  822. 'formats': formats,
  823. }
  824. class PlaylistIE(InfoExtractor):
  825. _VALID_URL = r'playlist:'
  826. def _entries(self):
  827. for n in range(3):
  828. video_id = compat_str(n)
  829. yield {
  830. '_type': 'url_transparent',
  831. 'ie_key': VideoIE.ie_key(),
  832. 'id': video_id,
  833. 'url': 'video:%s' % video_id,
  834. 'title': 'Video Transparent %s' % video_id,
  835. }
  836. def _real_extract(self, url):
  837. return self.playlist_result(self._entries())
  838. ydl.add_info_extractor(VideoIE(ydl))
  839. ydl.add_info_extractor(PlaylistIE(ydl))
  840. info = ydl.extract_info('playlist:')
  841. entries = info['entries']
  842. self.assertEqual(len(entries), 3)
  843. self.assertTrue(entries[0] is None)
  844. self.assertTrue(entries[1] is None)
  845. self.assertEqual(len(ydl.downloaded_info_dicts), 1)
  846. downloaded = ydl.downloaded_info_dicts[0]
  847. self.assertEqual(entries[2], downloaded)
  848. self.assertEqual(downloaded['url'], TEST_URL)
  849. self.assertEqual(downloaded['title'], 'Video Transparent 2')
  850. self.assertEqual(downloaded['id'], '2')
  851. self.assertEqual(downloaded['extractor'], 'Video')
  852. self.assertEqual(downloaded['extractor_key'], 'Video')
  853. def test_default_times(self):
  854. """Test addition of missing upload/release/_date from /release_/timestamp"""
  855. info = {
  856. 'id': '1234',
  857. 'url': TEST_URL,
  858. 'title': 'Title',
  859. 'ext': 'mp4',
  860. 'timestamp': 1631352900,
  861. 'release_timestamp': 1632995931,
  862. }
  863. params = {'simulate': True, }
  864. ydl = FakeYDL(params)
  865. out_info = ydl.process_ie_result(info)
  866. self.assertTrue(isinstance(out_info['upload_date'], compat_str))
  867. self.assertEqual(out_info['upload_date'], '20210911')
  868. self.assertTrue(isinstance(out_info['release_date'], compat_str))
  869. self.assertEqual(out_info['release_date'], '20210930')
  870. class TestYoutubeDLCookies(unittest.TestCase):
  871. @staticmethod
  872. def encode_cookie(cookie):
  873. if not isinstance(cookie, dict):
  874. cookie = vars(cookie)
  875. for name, value in cookie.items():
  876. yield name, compat_str(value)
  877. @classmethod
  878. def comparable_cookies(cls, cookies):
  879. # Work around cookiejar cookies not being unicode strings
  880. return sorted(map(tuple, map(sorted, map(cls.encode_cookie, cookies))))
  881. def assertSameCookies(self, c1, c2, msg=None):
  882. return self.assertEqual(
  883. *map(self.comparable_cookies, (c1, c2)),
  884. msg=msg)
  885. def assertSameCookieStrings(self, c1, c2, msg=None):
  886. return self.assertSameCookies(
  887. *map(lambda c: compat_http_cookies_SimpleCookie(c).values(), (c1, c2)),
  888. msg=msg)
  889. def test_header_cookies(self):
  890. ydl = FakeYDL()
  891. ydl.report_warning = lambda *_, **__: None
  892. def cookie(name, value, version=None, domain='', path='', secure=False, expires=None):
  893. return compat_http_cookiejar_Cookie(
  894. version or 0, name, value, None, False,
  895. domain, bool(domain), bool(domain), path, bool(path),
  896. secure, expires, False, None, None, rest={})
  897. test_url, test_domain = (t % ('yt.dl',) for t in ('https://%s/test', '.%s'))
  898. def test(encoded_cookies, cookies, headers=False, round_trip=None, error_re=None):
  899. def _test():
  900. ydl.cookiejar.clear()
  901. ydl._load_cookies(encoded_cookies, autoscope=headers)
  902. if headers:
  903. ydl._apply_header_cookies(test_url)
  904. data = {'url': test_url}
  905. ydl._calc_headers(data)
  906. self.assertSameCookies(
  907. cookies, ydl.cookiejar,
  908. 'Extracted cookiejar.Cookie is not the same')
  909. if not headers:
  910. self.assertSameCookieStrings(
  911. data.get('cookies'), round_trip or encoded_cookies,
  912. msg='Cookie is not the same as round trip')
  913. ydl.__dict__['_YoutubeDL__header_cookies'] = []
  914. try:
  915. _test()
  916. except AssertionError:
  917. raise
  918. except Exception as e:
  919. if not error_re:
  920. raise
  921. assertRegexpMatches(self, e.args[0], error_re.join(('.*',) * 2))
  922. test('test=value; Domain=' + test_domain, [cookie('test', 'value', domain=test_domain)])
  923. test('test=value', [cookie('test', 'value')], error_re='Unscoped cookies are not allowed')
  924. test('cookie1=value1; Domain={0}; Path=/test; cookie2=value2; Domain={0}; Path=/'.format(test_domain), [
  925. cookie('cookie1', 'value1', domain=test_domain, path='/test'),
  926. cookie('cookie2', 'value2', domain=test_domain, path='/')])
  927. cookie_kw = compat_kwargs(
  928. {'domain': test_domain, 'path': '/test', 'secure': True, 'expires': '9999999999', })
  929. test('test=value; Domain={domain}; Path={path}; Secure; Expires={expires}'.format(**cookie_kw), [
  930. cookie('test', 'value', **cookie_kw)])
  931. test('test="value; "; path=/test; domain=' + test_domain, [
  932. cookie('test', 'value; ', domain=test_domain, path='/test')],
  933. round_trip='test="value\\073 "; Domain={0}; Path=/test'.format(test_domain))
  934. test('name=; Domain=' + test_domain, [cookie('name', '', domain=test_domain)],
  935. round_trip='name=""; Domain=' + test_domain)
  936. test('test=value', [cookie('test', 'value', domain=test_domain)], headers=True)
  937. test('cookie1=value; Domain={0}; cookie2=value'.format(test_domain), [],
  938. headers=True, error_re='Invalid syntax')
  939. ydl.report_warning = ydl.report_error
  940. test('test=value', [], headers=True, error_re='Passing cookies as a header is a potential security risk')
  941. def test_infojson_cookies(self):
  942. TEST_FILE = 'test_infojson_cookies.info.json'
  943. TEST_URL = 'https://example.com/example.mp4'
  944. COOKIES = 'a=b; Domain=.example.com; c=d; Domain=.example.com'
  945. COOKIE_HEADER = {'Cookie': 'a=b; c=d'}
  946. ydl = FakeYDL()
  947. ydl.process_info = lambda x: ydl._write_info_json('test', x, TEST_FILE)
  948. def make_info(info_header_cookies=False, fmts_header_cookies=False, cookies_field=False):
  949. fmt = {'url': TEST_URL}
  950. if fmts_header_cookies:
  951. fmt['http_headers'] = COOKIE_HEADER
  952. if cookies_field:
  953. fmt['cookies'] = COOKIES
  954. return _make_result([fmt], http_headers=COOKIE_HEADER if info_header_cookies else None)
  955. def test(initial_info, note):
  956. def failure_msg(why):
  957. return ' when '.join((why, note))
  958. result = {}
  959. result['processed'] = ydl.process_ie_result(initial_info)
  960. self.assertTrue(ydl.cookiejar.get_cookies_for_url(TEST_URL),
  961. msg=failure_msg('No cookies set in cookiejar after initial process'))
  962. ydl.cookiejar.clear()
  963. with open(TEST_FILE) as infojson:
  964. result['loaded'] = ydl.sanitize_info(json.load(infojson), True)
  965. result['final'] = ydl.process_ie_result(result['loaded'].copy(), download=False)
  966. self.assertTrue(ydl.cookiejar.get_cookies_for_url(TEST_URL),
  967. msg=failure_msg('No cookies set in cookiejar after final process'))
  968. ydl.cookiejar.clear()
  969. for key in ('processed', 'loaded', 'final'):
  970. info = result[key]
  971. self.assertIsNone(
  972. traverse_obj(info, ((None, ('formats', 0)), 'http_headers', 'Cookie'), casesense=False, get_all=False),
  973. msg=failure_msg('Cookie header not removed in {0} result'.format(key)))
  974. self.assertSameCookieStrings(
  975. traverse_obj(info, ((None, ('formats', 0)), 'cookies'), get_all=False), COOKIES,
  976. msg=failure_msg('No cookies field found in {0} result'.format(key)))
  977. test({'url': TEST_URL, 'http_headers': COOKIE_HEADER, 'id': '1', 'title': 'x'}, 'no formats field')
  978. test(make_info(info_header_cookies=True), 'info_dict header cokies')
  979. test(make_info(fmts_header_cookies=True), 'format header cookies')
  980. test(make_info(info_header_cookies=True, fmts_header_cookies=True), 'info_dict and format header cookies')
  981. test(make_info(info_header_cookies=True, fmts_header_cookies=True, cookies_field=True), 'all cookies fields')
  982. test(make_info(cookies_field=True), 'cookies format field')
  983. test({'url': TEST_URL, 'cookies': COOKIES, 'id': '1', 'title': 'x'}, 'info_dict cookies field only')
  984. try_rm(TEST_FILE)
  985. def test_add_headers_cookie(self):
  986. def check_for_cookie_header(result):
  987. return traverse_obj(result, ((None, ('formats', 0)), 'http_headers', 'Cookie'), casesense=False, get_all=False)
  988. ydl = FakeYDL({'http_headers': {'Cookie': 'a=b'}})
  989. ydl._apply_header_cookies(_make_result([])['webpage_url']) # Scope to input webpage URL: .example.com
  990. fmt = {'url': 'https://example.com/video.mp4'}
  991. result = ydl.process_ie_result(_make_result([fmt]), download=False)
  992. self.assertIsNone(check_for_cookie_header(result), msg='http_headers cookies in result info_dict')
  993. self.assertEqual(result.get('cookies'), 'a=b; Domain=.example.com', msg='No cookies were set in cookies field')
  994. self.assertIn('a=b', ydl.cookiejar.get_cookie_header(fmt['url']), msg='No cookies were set in cookiejar')
  995. fmt = {'url': 'https://wrong.com/video.mp4'}
  996. result = ydl.process_ie_result(_make_result([fmt]), download=False)
  997. self.assertIsNone(check_for_cookie_header(result), msg='http_headers cookies for wrong domain')
  998. self.assertFalse(result.get('cookies'), msg='Cookies set in cookies field for wrong domain')
  999. self.assertFalse(ydl.cookiejar.get_cookie_header(fmt['url']), msg='Cookies set in cookiejar for wrong domain')
  1000. if __name__ == '__main__':
  1001. unittest.main()