build 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681
  1. #!/usr/bin/env python3
  2. import collections
  3. import copy
  4. import itertools
  5. import math
  6. import os
  7. import re
  8. import subprocess
  9. import cli_function
  10. import common
  11. import lkmc
  12. import shell_helpers
  13. from shell_helpers import LF
  14. class _Component:
  15. '''
  16. Yes, we are re-inventing a crappy dependency resolution system,
  17. reminiscent of scons or apt or Buildroot. I can't believe it.
  18. The hard part is that we have optional dependencies as well...
  19. e.g. Buildroot optionally depends on m5 to put m5 in the root filesystem,
  20. and Buildroot optionally depends on QEMU to build the qcow2 version
  21. of the image.
  22. '''
  23. def __init__(
  24. self,
  25. build_callback=None,
  26. supported_archs=None,
  27. dependencies=None,
  28. apt_get_pkgs=None,
  29. apt_build_deps=None,
  30. submodules=None,
  31. submodules_shallow=None,
  32. python3_pkgs=None,
  33. ruby_pkgs=None,
  34. ):
  35. self.build_callback = build_callback
  36. self.supported_archs = supported_archs
  37. self.dependencies = dependencies or set()
  38. self.apt_get_pkgs = apt_get_pkgs or set()
  39. self.apt_build_deps = apt_build_deps or set()
  40. self.submodules = submodules or set()
  41. self.submodules_shallow = submodules_shallow or set()
  42. self.python3_pkgs = python3_pkgs or set()
  43. self.ruby_pkgs = ruby_pkgs or set()
  44. def build(self, arch):
  45. if (
  46. (self.build_callback is not None) and
  47. (self.supported_archs is None or arch in self.supported_archs)
  48. ):
  49. return self.build_callback()
  50. else:
  51. # Component that does not build anything itself, only has dependencies.
  52. return 0
  53. submodule_extra_remotes = {
  54. 'binutils-gdb': {
  55. 'up': 'git://sourceware.org/git/binutils-gdb.git',
  56. },
  57. 'buildroot': {
  58. 'up': 'https://github.com/buildroot/buildroot',
  59. },
  60. 'crosstool-ng': {
  61. 'up': 'https://github.com/crosstool-ng/crosstool-ng',
  62. },
  63. 'freebsd': {
  64. 'up': 'https://github.com/freebsd/freebsd',
  65. },
  66. 'gcc': {
  67. 'up': 'git://gcc.gnu.org/git/gcc.git',
  68. },
  69. 'gem5': {
  70. 'up': 'https://gem5.googlesource.com/public/gem5',
  71. },
  72. 'glibc': {
  73. 'up': 'git://sourceware.org/git/glibc.git',
  74. },
  75. 'linux': {
  76. # https://cirosantilli.com/linux-kernel-module-cheat#gem5-arm-linux-kernel-patches
  77. 'gem5-arm': 'https://gem5.googlesource.com/arm/linux',
  78. 'up': 'git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git',
  79. },
  80. 'qemu': {
  81. 'up': 'https://git.qemu.org/git/qemu.git',
  82. },
  83. 'xen': {
  84. 'up': 'git://xenbits.xen.org/xen.git',
  85. },
  86. }
  87. class Main(common.LkmcCliFunction):
  88. def __init__(self):
  89. super().__init__(
  90. description='''\
  91. Build a component and all its dependencies.
  92. Our build-* scripts don't build any dependencies to make iterative
  93. development fast and more predictable.
  94. It is currently not possible to configure individual components from the command line
  95. when you build with this script. TODO.
  96. Without any args, build only what is necessary for:
  97. https://cirosantilli.com/linux-kernel-module-cheat#qemu-buildroot-setup
  98. ....
  99. ./%(prog)s
  100. ....
  101. This is equivalent to:
  102. ....
  103. ./%(prog)s --arch x86_64 qemu-buildroot
  104. ....
  105. Another important target is `all`:
  106. ....
  107. ./%(prog)s all
  108. ....
  109. This does not truly build ALL configurations: that would be impractical.
  110. But more precisely: build the reference configuration of each major component.
  111. So e.g.: one config of Linux kernel, Buildroot, gem5 and QEMU.
  112. Don't do for example all possible gem5 configs: debug, opt and fast,
  113. as that would be huge. This ensures that every piece of software
  114. builds in at least one config.
  115. TODO looping over emulators is not currently supported by this script, e.g.:
  116. ....
  117. ./%(prog)s --arch x86_64 --arch aarch64 all
  118. ....
  119. Instead, for the targets that are emulator dependent, you must select the
  120. taret version for the desired emulator, e.g.:
  121. ....
  122. ./build --arch aarch64 baremetal-qemu baremetal-gem5
  123. ....
  124. The reason is that some targets depend on emulator, while others don't,
  125. so looping over all of them would waste time.
  126. ''',
  127. )
  128. buildroot_component = _Component(
  129. self._build_file('build-buildroot'),
  130. submodules_shallow = {
  131. 'buildroot',
  132. 'binutils-gdb',
  133. 'gcc',
  134. 'glibc',
  135. },
  136. # https://buildroot.org/downloads/manual/manual.html#requirement
  137. apt_get_pkgs={
  138. 'bash',
  139. 'bc',
  140. 'binutils',
  141. 'build-essential',
  142. 'bzip2',
  143. 'cpio',
  144. 'g++',
  145. 'gcc',
  146. 'graphviz',
  147. 'gzip',
  148. 'make',
  149. 'patch',
  150. 'perl',
  151. 'python3',
  152. 'rsync',
  153. 'sed',
  154. 'tar',
  155. 'unzip',
  156. },
  157. python3_pkgs={
  158. # Generate graphs of config.ini under m5out.
  159. 'matplotlib',
  160. },
  161. )
  162. buildroot_overlay_qemu_component = copy.copy(buildroot_component)
  163. # We need to build QEMU before the final Buildroot to get qemu-img.
  164. buildroot_overlay_qemu_component.dependencies = ['overlay', 'qemu']
  165. buildroot_overlay_gem5_component = copy.copy(buildroot_component)
  166. buildroot_overlay_gem5_component.dependencies = ['overlay-gem5']
  167. gem5_deps = {
  168. 'apt_get_pkgs': {
  169. # https://askubuntu.com/questions/350475/how-can-i-install-gem5/1275773#1275773
  170. 'build-essential',
  171. 'doxygen',
  172. 'git',
  173. 'libboost-all-dev',
  174. 'libelf-dev',
  175. 'libgoogle-perftools-dev',
  176. 'libhdf5-serial-dev',
  177. 'libpng-dev',
  178. 'libprotobuf-dev',
  179. 'libprotoc-dev',
  180. 'lld',
  181. 'm4',
  182. 'protobuf-compiler',
  183. 'python-is-python3',
  184. 'python3-dev',
  185. 'python3-pydot',
  186. 'python3-six',
  187. 'scons',
  188. 'zlib1g',
  189. 'zlib1g-dev',
  190. # Some extra ones.
  191. 'device-tree-compiler',
  192. 'diod',
  193. # For prebuilt qcow2 unpack.
  194. 'qemu-utils',
  195. # Otherwise HDF5 not found.
  196. 'pkg-config',
  197. },
  198. 'python3_pkgs': {
  199. # https://cirosantilli.com/linux-kernel-module-cheat#gem5-config-dot
  200. 'pydot',
  201. },
  202. 'submodules_shallow': {'gem5'},
  203. }
  204. self.name_to_component_map = {
  205. 'all': _Component(dependencies=[
  206. 'qemu-gem5-buildroot',
  207. 'all-baremetal',
  208. 'user-mode-qemu',
  209. 'doc',
  210. ]),
  211. 'all-baremetal': _Component(dependencies=[
  212. 'qemu-baremetal',
  213. 'gem5-baremetal',
  214. 'baremetal-gem5-pbx',
  215. ],
  216. supported_archs=common.consts['crosstool_ng_supported_archs'],
  217. ),
  218. 'baremetal': _Component(dependencies=[
  219. 'baremetal-gem5',
  220. 'baremetal-qemu',
  221. ]),
  222. 'baremetal-qemu': _Component(
  223. self._build_file('build-baremetal', emulators=['qemu']),
  224. supported_archs=common.consts['crosstool_ng_supported_archs'],
  225. dependencies=['crosstool-ng'],
  226. ),
  227. 'baremetal-gem5': _Component(
  228. self._build_file('build-baremetal', emulators=['gem5']),
  229. supported_archs=common.consts['crosstool_ng_supported_archs'],
  230. dependencies=['crosstool-ng'],
  231. ),
  232. 'baremetal-gem5-pbx': _Component(
  233. self._build_file('build-baremetal', emulators=['gem5'], machine='RealViewPBX'),
  234. supported_archs=common.consts['crosstool_ng_supported_archs'],
  235. dependencies=['crosstool-ng'],
  236. ),
  237. 'buildroot': buildroot_component,
  238. # We need those to avoid circular dependencies, since we need to run Buildroot
  239. # twice: once to get the toolchain, and a second time to put the overlay into
  240. # the root filesystem.
  241. 'buildroot-overlay-qemu': buildroot_overlay_qemu_component,
  242. 'buildroot-overlay-gem5': buildroot_overlay_gem5_component,
  243. 'copy-overlay': _Component(
  244. self._build_file('copy-overlay'),
  245. ),
  246. 'crosstool-ng': _Component(
  247. self._build_file('build-crosstool-ng'),
  248. supported_archs=common.consts['crosstool_ng_supported_archs'],
  249. # http://crosstool-ng.github.io/docs/os-setup/
  250. apt_get_pkgs={
  251. 'bison',
  252. 'docbook2x',
  253. 'flex',
  254. 'gawk',
  255. 'gcc',
  256. 'gperf',
  257. 'help2man',
  258. 'libncurses5-dev',
  259. 'libtool-bin',
  260. 'make',
  261. 'python3-dev',
  262. 'texinfo',
  263. },
  264. submodules_shallow={'crosstool-ng'},
  265. ),
  266. 'doc': _Component(
  267. self._build_file('build-doc'),
  268. ruby_pkgs={
  269. 'asciidoctor',
  270. },
  271. ),
  272. 'gem5': _Component(
  273. self._build_file('build-gem5'),
  274. **gem5_deps
  275. ),
  276. 'gem5-baremetal': _Component(dependencies=[
  277. 'gem5',
  278. 'baremetal-gem5',
  279. ]),
  280. 'gem5-buildroot': _Component(dependencies=[
  281. 'buildroot-overlay-gem5',
  282. 'linux',
  283. 'gem5',
  284. ]),
  285. 'gem5-debug': _Component(
  286. self._build_file('build-gem5', gem5_build_type='debug'),
  287. **gem5_deps
  288. ),
  289. 'gem5-fast': _Component(
  290. self._build_file('build-gem5', gem5_build_type='fast'),
  291. **gem5_deps
  292. ),
  293. 'linux': _Component(
  294. self._build_file('build-linux'),
  295. dependencies={'buildroot'},
  296. submodules_shallow={'linux'},
  297. apt_get_pkgs={
  298. 'bison',
  299. 'flex',
  300. # Without this started failing in kernel 4.15 with:
  301. # Makefile:932: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel". Stop.
  302. 'libelf-dev',
  303. },
  304. ),
  305. 'modules': _Component(
  306. self._build_file('build-modules'),
  307. dependencies=['buildroot', 'linux'],
  308. ),
  309. 'm5': _Component(
  310. self._build_file('build-m5'),
  311. dependencies=['buildroot'],
  312. submodules_shallow={'gem5'},
  313. ),
  314. 'overlay': _Component(dependencies=[
  315. 'copy-overlay',
  316. 'modules',
  317. 'userland',
  318. ]),
  319. 'overlay-gem5': _Component(dependencies=[
  320. 'm5',
  321. 'overlay',
  322. ]),
  323. 'parsec-benchmark': _Component(
  324. submodules_shallow={'parsec-benchmark'},
  325. dependencies=['buildroot'],
  326. ),
  327. 'qemu': _Component(
  328. self._build_file('build-qemu'),
  329. apt_build_deps={'qemu'},
  330. apt_get_pkgs={
  331. 'libsdl2-dev',
  332. 'ninja-build',
  333. },
  334. submodules={'qemu'},
  335. ),
  336. 'qemu-baremetal': _Component(dependencies=[
  337. 'qemu',
  338. 'baremetal-qemu',
  339. ]),
  340. 'qemu-buildroot': _Component(dependencies=[
  341. 'buildroot-overlay-qemu',
  342. 'linux',
  343. ]),
  344. 'qemu-gem5-buildroot': _Component(dependencies=[
  345. 'qemu',
  346. 'gem5-buildroot',
  347. ]),
  348. 'qemu-user': _Component(
  349. self._build_file('build-qemu', mode='userland'),
  350. apt_build_deps = {'qemu'},
  351. apt_get_pkgs={'libsdl2-dev'},
  352. submodules={'qemu'},
  353. ),
  354. 'release': _Component(dependencies=[
  355. 'baremetal-qemu',
  356. 'qemu-buildroot',
  357. 'doc',
  358. ]),
  359. 'test-gdb': _Component(dependencies=[
  360. 'all-baremetal',
  361. 'userland',
  362. ],
  363. ),
  364. 'test-executables-userland': _Component(dependencies=[
  365. 'test-executables-userland-qemu',
  366. 'test-executables-userland-gem5',
  367. ]),
  368. 'test-executables-userland-qemu': _Component(dependencies=[
  369. 'user-mode-qemu',
  370. 'userland',
  371. ]),
  372. 'test-executables-userland-gem5': _Component(dependencies=[
  373. 'gem5',
  374. 'userland-gem5',
  375. ]),
  376. 'user-mode-qemu': _Component(
  377. dependencies=['qemu-user', 'userland'],
  378. ),
  379. 'userland': _Component(
  380. self._build_file('build-userland'),
  381. dependencies=['buildroot'],
  382. ),
  383. 'userland-host': _Component(
  384. self._build_file('build-userland-in-tree'),
  385. apt_get_pkgs={
  386. 'libboost-all-dev',
  387. 'libdrm-dev',
  388. 'libeigen3-dev',
  389. 'libhdf5-dev',
  390. 'libopenblas-dev',
  391. },
  392. ),
  393. 'userland-gem5': _Component(
  394. self._build_file('build-userland', static=True, userland_build_id='static'),
  395. dependencies=['buildroot'],
  396. ),
  397. }
  398. self.component_to_name_map = {self.name_to_component_map[key]:key for key in self.name_to_component_map}
  399. self.add_argument(
  400. '--apt',
  401. default=True,
  402. help='''\
  403. Don't run any apt-get commands. To make it easier to use with other archs:
  404. https://cirosantilli.com/linux-kernel-module-cheat#supported-hosts
  405. '''
  406. )
  407. self.add_argument(
  408. '-D', '--download-dependencies',
  409. default=False,
  410. help='''\
  411. Also download all dependencies required for a given build: Ubuntu packages,
  412. Python packages and git submodules.
  413. '''
  414. )
  415. self.add_argument(
  416. '--print-components',
  417. default=False,
  418. help='''\
  419. Print the components that would be built, including dependencies, but don't
  420. build them, nor show the build commands.
  421. '''
  422. )
  423. self.add_argument(
  424. '--travis',
  425. default=False,
  426. help='''\
  427. Extra args to pass to all scripts.
  428. '''
  429. )
  430. self.add_argument(
  431. 'components',
  432. choices=list(self.name_to_component_map.keys()) + [[]],
  433. default=[],
  434. nargs='*',
  435. help='''\
  436. Which components to build. Default: qemu-buildroot
  437. '''
  438. )
  439. def _build_file(self, component_file, **extra_args):
  440. '''
  441. Build something based on a component file that defines a Main class.
  442. '''
  443. def f():
  444. args = self.get_common_args()
  445. args.update(extra_args)
  446. args['show_time'] = False
  447. return lkmc.import_path.import_path_main(component_file)(**args)
  448. return f
  449. def timed_main(self):
  450. self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run'])
  451. # Decide components.
  452. components = self.env['components']
  453. if components == []:
  454. components = ['qemu-buildroot']
  455. selected_components = []
  456. for component_name in components:
  457. todo = [component_name]
  458. while todo:
  459. current_name = todo.pop(0)
  460. component = self.name_to_component_map[current_name]
  461. selected_components.insert(0, component)
  462. todo.extend(component.dependencies)
  463. # Remove duplicates, keep only the first one of each.
  464. # https://stackoverflow.com/questions/7961363/removing-duplicates-in-lists/7961390#7961390
  465. selected_components = collections.OrderedDict.fromkeys(selected_components)
  466. if self.env['download_dependencies']:
  467. apt_get_pkgs = {
  468. # Core requirements for this repo.
  469. 'git',
  470. 'moreutils', # ts
  471. 'rr',
  472. 'squashfs-tools',
  473. 'tmux',
  474. 'vinagre',
  475. 'wget',
  476. }
  477. # E.g. on an ARM host, the package gcc-arm-linux-gnueabihf
  478. # is called just gcc.
  479. processor = self.env['host_arch']
  480. if processor != 'arm':
  481. apt_get_pkgs.update({
  482. 'gcc-arm-linux-gnueabihf',
  483. 'g++-arm-linux-gnueabihf',
  484. })
  485. if processor != 'aarch64':
  486. apt_get_pkgs.update({
  487. 'gcc-aarch64-linux-gnu',
  488. 'g++-aarch64-linux-gnu',
  489. })
  490. apt_build_deps = set()
  491. submodules = set()
  492. submodules_shallow = set()
  493. python3_pkgs = {
  494. 'pexpect==4.6.0',
  495. }
  496. ruby_pkgs = set()
  497. for component in selected_components:
  498. apt_get_pkgs.update(component.apt_get_pkgs)
  499. apt_build_deps.update(component.apt_build_deps)
  500. submodules.update(component.submodules)
  501. submodules_shallow.update(component.submodules_shallow)
  502. python3_pkgs.update(component.python3_pkgs)
  503. python3_pkgs.update(component.python3_pkgs)
  504. ruby_pkgs.update(component.ruby_pkgs)
  505. if ruby_pkgs:
  506. apt_get_pkgs.add('ruby')
  507. if python3_pkgs:
  508. apt_get_pkgs.add('python3-pip')
  509. if apt_get_pkgs or apt_build_deps:
  510. if self.env['travis']:
  511. interacive_pkgs = {
  512. 'libsdl2-dev',
  513. }
  514. apt_get_pkgs.difference_update(interacive_pkgs)
  515. if common.consts['in_docker']:
  516. sudo = []
  517. # https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai
  518. os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
  519. # https://askubuntu.com/questions/496549/error-you-must-put-some-source-uris-in-your-sources-list
  520. sources_path = os.path.join('/etc', 'apt', 'sources.list')
  521. with open(sources_path, 'r') as f:
  522. sources_txt = f.read()
  523. sources_txt = re.sub('^# deb-src ', 'deb-src ', sources_txt, flags=re.MULTILINE)
  524. with open(sources_path, 'w') as f:
  525. f.write(sources_txt)
  526. else:
  527. sudo = ['sudo']
  528. if common.consts['in_docker'] or self.env['travis']:
  529. y = ['-y']
  530. else:
  531. y = []
  532. if self.env['apt']:
  533. self.sh.run_cmd(
  534. sudo + ['apt-get', 'update', LF]
  535. )
  536. if apt_get_pkgs:
  537. self.sh.run_cmd(
  538. sudo + ['apt-get', 'install'] + y + [LF] +
  539. self.sh.add_newlines(sorted(apt_get_pkgs))
  540. )
  541. if apt_build_deps:
  542. self.sh.run_cmd(
  543. sudo +
  544. ['apt-get', 'build-dep'] + y + [LF] +
  545. self.sh.add_newlines(sorted(apt_build_deps))
  546. )
  547. if python3_pkgs:
  548. # Not with pip executable directly:
  549. # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
  550. self.sh.run_cmd(
  551. ['python3', '-m', 'pip', 'install', '--user', LF] +
  552. self.sh.add_newlines(sorted(python3_pkgs))
  553. )
  554. self.sh.run_cmd(['python3', '-m', 'pip', 'install', '-r', 'requirements.txt', '--user', LF])
  555. if ruby_pkgs:
  556. # The right thing to do would be to use RVM and avoid sudo,
  557. # but we felt it was too much boilerplate for now.
  558. #
  559. # --user-install does work here, but we still need to add
  560. # the binary to PATH which is a pain:
  561. # https://stackoverflow.com/questions/31596273/install-gem-in-local-folder
  562. self.sh.run_cmd(['sudo', 'gem', 'install', 'bundler', LF])
  563. # No ones knows how to do this without sudo:
  564. # https://stackoverflow.com/questions/16376995/bundler-cannot-install-any-gems-without-sudo/27677094
  565. self.sh.run_cmd([
  566. 'sudo', LF,
  567. 'bundle', LF,
  568. 'install', LF,
  569. '--gemfile', os.path.join(self.env['root_dir'], 'Gemfile'), LF,
  570. ])
  571. git_cmd_common = [
  572. 'git', LF,
  573. 'submodule', LF,
  574. 'update', LF,
  575. '--init', LF,
  576. '--recursive', LF,
  577. ]
  578. if self.env['dry_run']:
  579. git_version_tuple = (math.inf, math.inf, math.inf)
  580. else:
  581. git_version_tuple = tuple(
  582. int(x) for x in self.sh.check_output(['git', '--version']) \
  583. .decode().split(' ')[-1].split('.')[:3]
  584. )
  585. if git_version_tuple >= (2, 9, 0):
  586. # https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
  587. git_cmd_common.extend(['--jobs', str(len(os.sched_getaffinity(0))), LF])
  588. if git_version_tuple >= (2, 10, 0):
  589. # * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching
  590. # * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone
  591. git_cmd_common.extend(['--progress', LF])
  592. def submodule_ids_to_cmd(submodules):
  593. return self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules)])
  594. if submodules:
  595. self.sh.run_cmd(git_cmd_common + ['--', LF] + submodule_ids_to_cmd(submodules))
  596. if submodules_shallow:
  597. # TODO Ideally we should shallow clone --depth 1 all of them.
  598. #
  599. # However, most git servers out there are crap or craply configured
  600. # and don't allow shallow cloning except for branches.
  601. #
  602. # So for now I'm keeping all mirrors in my GitHub.
  603. # and always have a lkmc-* branch pointint to it.
  604. #
  605. # However, QEMU has a bunch of submodules itself, and I'm not in the mood
  606. # to mirror all of them...
  607. #
  608. # See also:
  609. #
  610. # * https://stackoverflow.com/questions/3489173/how-to-clone-git-repository-with-specific-revision-changeset
  611. # * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702
  612. # * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac
  613. cmd = git_cmd_common.copy()
  614. if git_version_tuple > (2, 7, 4):
  615. # Then there is a bug in Ubuntu 16.04 git 2.7.4 where --depth 1 fails...
  616. # OMG git submodules implementation sucks:
  617. # * https://stackoverflow.com/questions/2155887/git-submodule-head-reference-is-not-a-tree-error/25875273#25875273
  618. # * https://github.com/boostorg/boost/issues/245
  619. cmd.extend(['--depth', '1', LF])
  620. else:
  621. self.log_warn('your git is too old for git submodule update --depth 1')
  622. self.log_warn('update to git 2.17 or newer and you will save clone time')
  623. self.log_warn('see: https://github.com/cirosantilli/linux-kernel-module-cheat/issues/44')
  624. self.sh.run_cmd(cmd + ['--', LF] + submodule_ids_to_cmd(submodules_shallow))
  625. for submodule_name in submodule_extra_remotes:
  626. submodule = submodule_extra_remotes[submodule_name]
  627. for remote_name in submodule:
  628. self.sh.run_cmd(
  629. [
  630. 'git', LF,
  631. '-C', os.path.join(common.consts['submodules_dir'], submodule_name), LF,
  632. 'remote', LF,
  633. 'add', LF,
  634. remote_name, LF,
  635. submodule[remote_name], LF,
  636. ],
  637. # In case the remote already exists.
  638. raise_on_failure=False
  639. )
  640. # Do the build.
  641. for component in selected_components:
  642. if self.env['print_components']:
  643. print(self.component_to_name_map[component])
  644. else:
  645. ret = component.build(self.env['arch'])
  646. if (ret != 0):
  647. return ret
  648. if __name__ == '__main__':
  649. Main().cli()