123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681 |
- #!/usr/bin/env python3
- import collections
- import copy
- import itertools
- import math
- import os
- import re
- import subprocess
- import cli_function
- import common
- import lkmc
- import shell_helpers
- from shell_helpers import LF
- class _Component:
- '''
- Yes, we are re-inventing a crappy dependency resolution system,
- reminiscent of scons or apt or Buildroot. I can't believe it.
- The hard part is that we have optional dependencies as well...
- e.g. Buildroot optionally depends on m5 to put m5 in the root filesystem,
- and Buildroot optionally depends on QEMU to build the qcow2 version
- of the image.
- '''
- def __init__(
- self,
- build_callback=None,
- supported_archs=None,
- dependencies=None,
- apt_get_pkgs=None,
- apt_build_deps=None,
- submodules=None,
- submodules_shallow=None,
- python3_pkgs=None,
- ruby_pkgs=None,
- ):
- self.build_callback = build_callback
- self.supported_archs = supported_archs
- self.dependencies = dependencies or set()
- self.apt_get_pkgs = apt_get_pkgs or set()
- self.apt_build_deps = apt_build_deps or set()
- self.submodules = submodules or set()
- self.submodules_shallow = submodules_shallow or set()
- self.python3_pkgs = python3_pkgs or set()
- self.ruby_pkgs = ruby_pkgs or set()
- def build(self, arch):
- if (
- (self.build_callback is not None) and
- (self.supported_archs is None or arch in self.supported_archs)
- ):
- return self.build_callback()
- else:
- # Component that does not build anything itself, only has dependencies.
- return 0
- submodule_extra_remotes = {
- 'binutils-gdb': {
- 'up': 'git://sourceware.org/git/binutils-gdb.git',
- },
- 'buildroot': {
- 'up': 'https://github.com/buildroot/buildroot',
- },
- 'crosstool-ng': {
- 'up': 'https://github.com/crosstool-ng/crosstool-ng',
- },
- 'freebsd': {
- 'up': 'https://github.com/freebsd/freebsd',
- },
- 'gcc': {
- 'up': 'git://gcc.gnu.org/git/gcc.git',
- },
- 'gem5': {
- 'up': 'https://gem5.googlesource.com/public/gem5',
- },
- 'glibc': {
- 'up': 'git://sourceware.org/git/glibc.git',
- },
- 'linux': {
- # https://cirosantilli.com/linux-kernel-module-cheat#gem5-arm-linux-kernel-patches
- 'gem5-arm': 'https://gem5.googlesource.com/arm/linux',
- 'up': 'git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git',
- },
- 'qemu': {
- 'up': 'https://git.qemu.org/git/qemu.git',
- },
- 'xen': {
- 'up': 'git://xenbits.xen.org/xen.git',
- },
- }
- class Main(common.LkmcCliFunction):
- def __init__(self):
- super().__init__(
- description='''\
- Build a component and all its dependencies.
- Our build-* scripts don't build any dependencies to make iterative
- development fast and more predictable.
- It is currently not possible to configure individual components from the command line
- when you build with this script. TODO.
- Without any args, build only what is necessary for:
- https://cirosantilli.com/linux-kernel-module-cheat#qemu-buildroot-setup
- ....
- ./%(prog)s
- ....
- This is equivalent to:
- ....
- ./%(prog)s --arch x86_64 qemu-buildroot
- ....
- Another important target is `all`:
- ....
- ./%(prog)s all
- ....
- This does not truly build ALL configurations: that would be impractical.
- But more precisely: build the reference configuration of each major component.
- So e.g.: one config of Linux kernel, Buildroot, gem5 and QEMU.
- Don't do for example all possible gem5 configs: debug, opt and fast,
- as that would be huge. This ensures that every piece of software
- builds in at least one config.
- TODO looping over emulators is not currently supported by this script, e.g.:
- ....
- ./%(prog)s --arch x86_64 --arch aarch64 all
- ....
- Instead, for the targets that are emulator dependent, you must select the
- taret version for the desired emulator, e.g.:
- ....
- ./build --arch aarch64 baremetal-qemu baremetal-gem5
- ....
- The reason is that some targets depend on emulator, while others don't,
- so looping over all of them would waste time.
- ''',
- )
- buildroot_component = _Component(
- self._build_file('build-buildroot'),
- submodules_shallow = {
- 'buildroot',
- 'binutils-gdb',
- 'gcc',
- 'glibc',
- },
- # https://buildroot.org/downloads/manual/manual.html#requirement
- apt_get_pkgs={
- 'bash',
- 'bc',
- 'binutils',
- 'build-essential',
- 'bzip2',
- 'cpio',
- 'g++',
- 'gcc',
- 'graphviz',
- 'gzip',
- 'make',
- 'patch',
- 'perl',
- 'python3',
- 'rsync',
- 'sed',
- 'tar',
- 'unzip',
- },
- python3_pkgs={
- # Generate graphs of config.ini under m5out.
- 'matplotlib',
- },
- )
- buildroot_overlay_qemu_component = copy.copy(buildroot_component)
- # We need to build QEMU before the final Buildroot to get qemu-img.
- buildroot_overlay_qemu_component.dependencies = ['overlay', 'qemu']
- buildroot_overlay_gem5_component = copy.copy(buildroot_component)
- buildroot_overlay_gem5_component.dependencies = ['overlay-gem5']
- gem5_deps = {
- 'apt_get_pkgs': {
- # https://askubuntu.com/questions/350475/how-can-i-install-gem5/1275773#1275773
- 'build-essential',
- 'doxygen',
- 'git',
- 'libboost-all-dev',
- 'libelf-dev',
- 'libgoogle-perftools-dev',
- 'libhdf5-serial-dev',
- 'libpng-dev',
- 'libprotobuf-dev',
- 'libprotoc-dev',
- 'lld',
- 'm4',
- 'protobuf-compiler',
- 'python-is-python3',
- 'python3-dev',
- 'python3-pydot',
- 'python3-six',
- 'scons',
- 'zlib1g',
- 'zlib1g-dev',
- # Some extra ones.
- 'device-tree-compiler',
- 'diod',
- # For prebuilt qcow2 unpack.
- 'qemu-utils',
- # Otherwise HDF5 not found.
- 'pkg-config',
- },
- 'python3_pkgs': {
- # https://cirosantilli.com/linux-kernel-module-cheat#gem5-config-dot
- 'pydot',
- },
- 'submodules_shallow': {'gem5'},
- }
- self.name_to_component_map = {
- 'all': _Component(dependencies=[
- 'qemu-gem5-buildroot',
- 'all-baremetal',
- 'user-mode-qemu',
- 'doc',
- ]),
- 'all-baremetal': _Component(dependencies=[
- 'qemu-baremetal',
- 'gem5-baremetal',
- 'baremetal-gem5-pbx',
- ],
- supported_archs=common.consts['crosstool_ng_supported_archs'],
- ),
- 'baremetal': _Component(dependencies=[
- 'baremetal-gem5',
- 'baremetal-qemu',
- ]),
- 'baremetal-qemu': _Component(
- self._build_file('build-baremetal', emulators=['qemu']),
- supported_archs=common.consts['crosstool_ng_supported_archs'],
- dependencies=['crosstool-ng'],
- ),
- 'baremetal-gem5': _Component(
- self._build_file('build-baremetal', emulators=['gem5']),
- supported_archs=common.consts['crosstool_ng_supported_archs'],
- dependencies=['crosstool-ng'],
- ),
- 'baremetal-gem5-pbx': _Component(
- self._build_file('build-baremetal', emulators=['gem5'], machine='RealViewPBX'),
- supported_archs=common.consts['crosstool_ng_supported_archs'],
- dependencies=['crosstool-ng'],
- ),
- 'buildroot': buildroot_component,
- # We need those to avoid circular dependencies, since we need to run Buildroot
- # twice: once to get the toolchain, and a second time to put the overlay into
- # the root filesystem.
- 'buildroot-overlay-qemu': buildroot_overlay_qemu_component,
- 'buildroot-overlay-gem5': buildroot_overlay_gem5_component,
- 'copy-overlay': _Component(
- self._build_file('copy-overlay'),
- ),
- 'crosstool-ng': _Component(
- self._build_file('build-crosstool-ng'),
- supported_archs=common.consts['crosstool_ng_supported_archs'],
- # http://crosstool-ng.github.io/docs/os-setup/
- apt_get_pkgs={
- 'bison',
- 'docbook2x',
- 'flex',
- 'gawk',
- 'gcc',
- 'gperf',
- 'help2man',
- 'libncurses5-dev',
- 'libtool-bin',
- 'make',
- 'python3-dev',
- 'texinfo',
- },
- submodules_shallow={'crosstool-ng'},
- ),
- 'doc': _Component(
- self._build_file('build-doc'),
- ruby_pkgs={
- 'asciidoctor',
- },
- ),
- 'gem5': _Component(
- self._build_file('build-gem5'),
- **gem5_deps
- ),
- 'gem5-baremetal': _Component(dependencies=[
- 'gem5',
- 'baremetal-gem5',
- ]),
- 'gem5-buildroot': _Component(dependencies=[
- 'buildroot-overlay-gem5',
- 'linux',
- 'gem5',
- ]),
- 'gem5-debug': _Component(
- self._build_file('build-gem5', gem5_build_type='debug'),
- **gem5_deps
- ),
- 'gem5-fast': _Component(
- self._build_file('build-gem5', gem5_build_type='fast'),
- **gem5_deps
- ),
- 'linux': _Component(
- self._build_file('build-linux'),
- dependencies={'buildroot'},
- submodules_shallow={'linux'},
- apt_get_pkgs={
- 'bison',
- 'flex',
- # Without this started failing in kernel 4.15 with:
- # Makefile:932: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel". Stop.
- 'libelf-dev',
- },
- ),
- 'modules': _Component(
- self._build_file('build-modules'),
- dependencies=['buildroot', 'linux'],
- ),
- 'm5': _Component(
- self._build_file('build-m5'),
- dependencies=['buildroot'],
- submodules_shallow={'gem5'},
- ),
- 'overlay': _Component(dependencies=[
- 'copy-overlay',
- 'modules',
- 'userland',
- ]),
- 'overlay-gem5': _Component(dependencies=[
- 'm5',
- 'overlay',
- ]),
- 'parsec-benchmark': _Component(
- submodules_shallow={'parsec-benchmark'},
- dependencies=['buildroot'],
- ),
- 'qemu': _Component(
- self._build_file('build-qemu'),
- apt_build_deps={'qemu'},
- apt_get_pkgs={
- 'libsdl2-dev',
- 'ninja-build',
- },
- submodules={'qemu'},
- ),
- 'qemu-baremetal': _Component(dependencies=[
- 'qemu',
- 'baremetal-qemu',
- ]),
- 'qemu-buildroot': _Component(dependencies=[
- 'buildroot-overlay-qemu',
- 'linux',
- ]),
- 'qemu-gem5-buildroot': _Component(dependencies=[
- 'qemu',
- 'gem5-buildroot',
- ]),
- 'qemu-user': _Component(
- self._build_file('build-qemu', mode='userland'),
- apt_build_deps = {'qemu'},
- apt_get_pkgs={'libsdl2-dev'},
- submodules={'qemu'},
- ),
- 'release': _Component(dependencies=[
- 'baremetal-qemu',
- 'qemu-buildroot',
- 'doc',
- ]),
- 'test-gdb': _Component(dependencies=[
- 'all-baremetal',
- 'userland',
- ],
- ),
- 'test-executables-userland': _Component(dependencies=[
- 'test-executables-userland-qemu',
- 'test-executables-userland-gem5',
- ]),
- 'test-executables-userland-qemu': _Component(dependencies=[
- 'user-mode-qemu',
- 'userland',
- ]),
- 'test-executables-userland-gem5': _Component(dependencies=[
- 'gem5',
- 'userland-gem5',
- ]),
- 'user-mode-qemu': _Component(
- dependencies=['qemu-user', 'userland'],
- ),
- 'userland': _Component(
- self._build_file('build-userland'),
- dependencies=['buildroot'],
- ),
- 'userland-host': _Component(
- self._build_file('build-userland-in-tree'),
- apt_get_pkgs={
- 'libboost-all-dev',
- 'libdrm-dev',
- 'libeigen3-dev',
- 'libhdf5-dev',
- 'libopenblas-dev',
- },
- ),
- 'userland-gem5': _Component(
- self._build_file('build-userland', static=True, userland_build_id='static'),
- dependencies=['buildroot'],
- ),
- }
- self.component_to_name_map = {self.name_to_component_map[key]:key for key in self.name_to_component_map}
- self.add_argument(
- '--apt',
- default=True,
- help='''\
- Don't run any apt-get commands. To make it easier to use with other archs:
- https://cirosantilli.com/linux-kernel-module-cheat#supported-hosts
- '''
- )
- self.add_argument(
- '-D', '--download-dependencies',
- default=False,
- help='''\
- Also download all dependencies required for a given build: Ubuntu packages,
- Python packages and git submodules.
- '''
- )
- self.add_argument(
- '--print-components',
- default=False,
- help='''\
- Print the components that would be built, including dependencies, but don't
- build them, nor show the build commands.
- '''
- )
- self.add_argument(
- '--travis',
- default=False,
- help='''\
- Extra args to pass to all scripts.
- '''
- )
- self.add_argument(
- 'components',
- choices=list(self.name_to_component_map.keys()) + [[]],
- default=[],
- nargs='*',
- help='''\
- Which components to build. Default: qemu-buildroot
- '''
- )
- def _build_file(self, component_file, **extra_args):
- '''
- Build something based on a component file that defines a Main class.
- '''
- def f():
- args = self.get_common_args()
- args.update(extra_args)
- args['show_time'] = False
- return lkmc.import_path.import_path_main(component_file)(**args)
- return f
- def timed_main(self):
- self.sh = shell_helpers.ShellHelpers(dry_run=self.env['dry_run'])
- # Decide components.
- components = self.env['components']
- if components == []:
- components = ['qemu-buildroot']
- selected_components = []
- for component_name in components:
- todo = [component_name]
- while todo:
- current_name = todo.pop(0)
- component = self.name_to_component_map[current_name]
- selected_components.insert(0, component)
- todo.extend(component.dependencies)
- # Remove duplicates, keep only the first one of each.
- # https://stackoverflow.com/questions/7961363/removing-duplicates-in-lists/7961390#7961390
- selected_components = collections.OrderedDict.fromkeys(selected_components)
- if self.env['download_dependencies']:
- apt_get_pkgs = {
- # Core requirements for this repo.
- 'git',
- 'moreutils', # ts
- 'rr',
- 'squashfs-tools',
- 'tmux',
- 'vinagre',
- 'wget',
- }
- # E.g. on an ARM host, the package gcc-arm-linux-gnueabihf
- # is called just gcc.
- processor = self.env['host_arch']
- if processor != 'arm':
- apt_get_pkgs.update({
- 'gcc-arm-linux-gnueabihf',
- 'g++-arm-linux-gnueabihf',
- })
- if processor != 'aarch64':
- apt_get_pkgs.update({
- 'gcc-aarch64-linux-gnu',
- 'g++-aarch64-linux-gnu',
- })
- apt_build_deps = set()
- submodules = set()
- submodules_shallow = set()
- python3_pkgs = {
- 'pexpect==4.6.0',
- }
- ruby_pkgs = set()
- for component in selected_components:
- apt_get_pkgs.update(component.apt_get_pkgs)
- apt_build_deps.update(component.apt_build_deps)
- submodules.update(component.submodules)
- submodules_shallow.update(component.submodules_shallow)
- python3_pkgs.update(component.python3_pkgs)
- python3_pkgs.update(component.python3_pkgs)
- ruby_pkgs.update(component.ruby_pkgs)
- if ruby_pkgs:
- apt_get_pkgs.add('ruby')
- if python3_pkgs:
- apt_get_pkgs.add('python3-pip')
- if apt_get_pkgs or apt_build_deps:
- if self.env['travis']:
- interacive_pkgs = {
- 'libsdl2-dev',
- }
- apt_get_pkgs.difference_update(interacive_pkgs)
- if common.consts['in_docker']:
- sudo = []
- # https://askubuntu.com/questions/909277/avoiding-user-interaction-with-tzdata-when-installing-certbot-in-a-docker-contai
- os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
- # https://askubuntu.com/questions/496549/error-you-must-put-some-source-uris-in-your-sources-list
- sources_path = os.path.join('/etc', 'apt', 'sources.list')
- with open(sources_path, 'r') as f:
- sources_txt = f.read()
- sources_txt = re.sub('^# deb-src ', 'deb-src ', sources_txt, flags=re.MULTILINE)
- with open(sources_path, 'w') as f:
- f.write(sources_txt)
- else:
- sudo = ['sudo']
- if common.consts['in_docker'] or self.env['travis']:
- y = ['-y']
- else:
- y = []
- if self.env['apt']:
- self.sh.run_cmd(
- sudo + ['apt-get', 'update', LF]
- )
- if apt_get_pkgs:
- self.sh.run_cmd(
- sudo + ['apt-get', 'install'] + y + [LF] +
- self.sh.add_newlines(sorted(apt_get_pkgs))
- )
- if apt_build_deps:
- self.sh.run_cmd(
- sudo +
- ['apt-get', 'build-dep'] + y + [LF] +
- self.sh.add_newlines(sorted(apt_build_deps))
- )
- if python3_pkgs:
- # Not with pip executable directly:
- # https://stackoverflow.com/questions/49836676/error-after-upgrading-pip-cannot-import-name-main/51846054#51846054
- self.sh.run_cmd(
- ['python3', '-m', 'pip', 'install', '--user', LF] +
- self.sh.add_newlines(sorted(python3_pkgs))
- )
- self.sh.run_cmd(['python3', '-m', 'pip', 'install', '-r', 'requirements.txt', '--user', LF])
- if ruby_pkgs:
- # The right thing to do would be to use RVM and avoid sudo,
- # but we felt it was too much boilerplate for now.
- #
- # --user-install does work here, but we still need to add
- # the binary to PATH which is a pain:
- # https://stackoverflow.com/questions/31596273/install-gem-in-local-folder
- self.sh.run_cmd(['sudo', 'gem', 'install', 'bundler', LF])
- # No ones knows how to do this without sudo:
- # https://stackoverflow.com/questions/16376995/bundler-cannot-install-any-gems-without-sudo/27677094
- self.sh.run_cmd([
- 'sudo', LF,
- 'bundle', LF,
- 'install', LF,
- '--gemfile', os.path.join(self.env['root_dir'], 'Gemfile'), LF,
- ])
- git_cmd_common = [
- 'git', LF,
- 'submodule', LF,
- 'update', LF,
- '--init', LF,
- '--recursive', LF,
- ]
- if self.env['dry_run']:
- git_version_tuple = (math.inf, math.inf, math.inf)
- else:
- git_version_tuple = tuple(
- int(x) for x in self.sh.check_output(['git', '--version']) \
- .decode().split(' ')[-1].split('.')[:3]
- )
- if git_version_tuple >= (2, 9, 0):
- # https://stackoverflow.com/questions/26957237/how-to-make-git-clone-faster-with-multiple-threads/52327638#52327638
- git_cmd_common.extend(['--jobs', str(len(os.sched_getaffinity(0))), LF])
- if git_version_tuple >= (2, 10, 0):
- # * https://stackoverflow.com/questions/32944468/how-to-show-progress-for-submodule-fetching
- # * https://stackoverflow.com/questions/4640020/progress-indicator-for-git-clone
- git_cmd_common.extend(['--progress', LF])
- def submodule_ids_to_cmd(submodules):
- return self.sh.add_newlines([os.path.join(common.consts['submodules_dir'], x) for x in sorted(submodules)])
- if submodules:
- self.sh.run_cmd(git_cmd_common + ['--', LF] + submodule_ids_to_cmd(submodules))
- if submodules_shallow:
- # TODO Ideally we should shallow clone --depth 1 all of them.
- #
- # However, most git servers out there are crap or craply configured
- # and don't allow shallow cloning except for branches.
- #
- # So for now I'm keeping all mirrors in my GitHub.
- # and always have a lkmc-* branch pointint to it.
- #
- # However, QEMU has a bunch of submodules itself, and I'm not in the mood
- # to mirror all of them...
- #
- # See also:
- #
- # * https://stackoverflow.com/questions/3489173/how-to-clone-git-repository-with-specific-revision-changeset
- # * https://stackoverflow.com/questions/2144406/git-shallow-submodules/47374702#47374702
- # * https://unix.stackexchange.com/questions/338578/why-is-the-git-clone-of-the-linux-kernel-source-code-much-larger-than-the-extrac
- cmd = git_cmd_common.copy()
- if git_version_tuple > (2, 7, 4):
- # Then there is a bug in Ubuntu 16.04 git 2.7.4 where --depth 1 fails...
- # OMG git submodules implementation sucks:
- # * https://stackoverflow.com/questions/2155887/git-submodule-head-reference-is-not-a-tree-error/25875273#25875273
- # * https://github.com/boostorg/boost/issues/245
- cmd.extend(['--depth', '1', LF])
- else:
- self.log_warn('your git is too old for git submodule update --depth 1')
- self.log_warn('update to git 2.17 or newer and you will save clone time')
- self.log_warn('see: https://github.com/cirosantilli/linux-kernel-module-cheat/issues/44')
- self.sh.run_cmd(cmd + ['--', LF] + submodule_ids_to_cmd(submodules_shallow))
- for submodule_name in submodule_extra_remotes:
- submodule = submodule_extra_remotes[submodule_name]
- for remote_name in submodule:
- self.sh.run_cmd(
- [
- 'git', LF,
- '-C', os.path.join(common.consts['submodules_dir'], submodule_name), LF,
- 'remote', LF,
- 'add', LF,
- remote_name, LF,
- submodule[remote_name], LF,
- ],
- # In case the remote already exists.
- raise_on_failure=False
- )
- # Do the build.
- for component in selected_components:
- if self.env['print_components']:
- print(self.component_to_name_map[component])
- else:
- ret = component.build(self.env['arch'])
- if (ret != 0):
- return ret
- if __name__ == '__main__':
- Main().cli()
|