diff --git a/build_with_bazel.py b/build_with_bazel.py index ed212f0e54a4..6f384f12f7c7 100755 --- a/build_with_bazel.py +++ b/build_with_bazel.py @@ -5,100 +5,219 @@ import argparse import logging import os +import re import sys -from subprocess import Popen +import subprocess -BAZEL_BIN = "./tools/bazel" CPU = "aarch64" HOST_CROSSTOOL = "@bazel_tools//tools/cpp:toolchain" +HOST_TARGETS = ["dtc"] +DEFAULT_SKIP_LIST = ["abi", "test_mapping"] -def get_cross_cli_opts(toolchain): - return [ - "--cpu={}".format(CPU), - "--crosstool_top={}".format(toolchain), - "--host_crosstool_top={}".format(HOST_CROSSTOOL), - ] +class BazelBuilder: + """Helper class for building with Bazel""" + def __init__(self, target_list, skip_list, user_opts): + self.workspace = os.path.realpath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") + ) + self.bazel_bin = os.path.join(self.workspace, "tools", "bazel") + if not os.path.exists(self.bazel_bin): + logging.error("failed to find Bazel binary at %s", self.bazel_bin) + sys.exit(1) + self.kernel_dir = os.path.basename( + (os.path.dirname(os.path.realpath(__file__))) + ) + self.target_list = target_list + self.skip_list = skip_list + self.user_opts = user_opts + self.process_list = [] -def bazel(bazel_subcommand, targets, extra_options=[], us_cross_toolchain=None): - cmdline = [BAZEL_BIN, bazel_subcommand] - cmdline.extend(extra_options) - if us_cross_toolchain: - cmdline.extend(get_cross_cli_opts(us_cross_toolchain)) - cmdline.extend(targets) - cmdline_str = " ".join(cmdline) - try: - logging.debug('Running "%s"', cmdline_str) - build_proc = Popen(cmdline_str, shell=True) - build_proc.wait() - if build_proc.returncode != 0: - sys.exit(build_proc.returncode) - except Exception as e: - logging.error(e), - sys.exit(1) + def __del__(self): + for proc in self.process_list: + proc.kill() + proc.wait() + @staticmethod + def get_cross_cli_opts(toolchain): + """Form cross toolchain Bazel options""" + return [ + "--cpu={}".format(CPU), + "--crosstool_top={}".format(toolchain), + "--host_crosstool_top={}".format(HOST_CROSSTOOL), + ] -def build_targets(targets, user_opts=[], us_cross_toolchain=None): - if not targets: - logging.warning("no targets to build") - bazel("build", targets, user_opts, us_cross_toolchain) + def get_build_targets(self): + """Query for build targets""" + logging.info("Querying build targets...") + cross_target_list = [] + host_target_list = [] + for t, v in self.target_list: + if v == "ALL": + skip_list_re = [ + re.compile(r"//{}:{}_.*_{}_dist".format(self.kernel_dir, t, s)) + for s in self.skip_list + ] + host_target_list_re = [ + re.compile(r"//{}:{}_.*_{}_dist".format(self.kernel_dir, t, h)) + for h in HOST_TARGETS + ] + query = 'filter("{}_.*_dist$", attr(generator_function, define_msm_platforms, {}/...))'.format( + t, self.kernel_dir + ) + else: + skip_list_re = [ + re.compile(r"//{}:{}_{}_{}_dist".format(self.kernel_dir, t, v, s)) + for s in self.skip_list + ] + host_target_list_re = [ + re.compile(r"//{}:{}_{}_{}_dist".format(self.kernel_dir, t, v, h)) + for h in HOST_TARGETS + ] + query = 'filter("{}_{}.*_dist$", attr(generator_function, define_msm_platforms, {}/...))'.format( + t, v, self.kernel_dir + ) -def run_targets(targets, user_opts=[], us_cross_toolchain=None): - # bazel cannot _run_ multiple targets at once, so run them in serial - for target in targets: - bazel("run", [target], user_opts, us_cross_toolchain) + cmdline = [ + self.bazel_bin, + "query", + "--ui_event_filters=-info", + "--noshow_progress", + query, + ] + logging.debug('Running "%s"', " ".join(cmdline)) -def build( - target_list, - user_opts=[], - us_cross_toolchain=None, - extra_targets=[], - skip_kernel=False, - skip_abl=False, - skip_host_tools=False, - **ignored -): - msm_targets = ["//msm-kernel:{}_{}".format(t, v) for t, v in target_list] - extra_msm_targets = [] - for extra in extra_targets: - extra_msm_targets.extend( - ["//msm-kernel:{}_{}_{}".format(t, v, extra) for t, v in target_list] + try: + query_cmd = subprocess.Popen( + cmdline, cwd=self.workspace, stdout=subprocess.PIPE + ) + self.process_list.append(query_cmd) + target_list = query_cmd.stdout.read().splitlines() + except Exception as e: + logging.error(e) + sys.exit(1) + + self.process_list.remove(query_cmd) + + if not target_list: + logging.error( + "failed to find any Bazel targets for target/variant combo %s_%s", + t, + v, + ) + sys.exit(1) + + for target in target_list: + if any((skip_re.match(target) for skip_re in skip_list_re)): + continue + if any((host_re.match(target) for host_re in host_target_list_re)): + host_target_list.append(target) + else: + cross_target_list.append(target) + + return (cross_target_list, host_target_list) + + def get_cross_toolchain(self): + """Query for a custom toolchain if one is defined""" + logging.info("Querying toolchains...") + query = 'filter("crosstool_suite", {}/...)'.format(self.kernel_dir) + cmdline = [ + self.bazel_bin, + "query", + "--ui_event_filters=-info", + "--noshow_progress", + query, + ] + + logging.debug('Running "%s"', " ".join(cmdline)) + + try: + query_cmd = subprocess.Popen( + cmdline, cwd=self.workspace, stdout=subprocess.PIPE + ) + self.process_list.append(query_cmd) + toolchain = query_cmd.stdout.read().strip() + except Exception as e: + logging.error(e) + sys.exit(1) + + self.process_list.remove(query_cmd) + + if not toolchain: + logging.debug("no userspace cross toolchain found") + return None + + logging.debug("using userspace cross toolchain %s", toolchain) + return toolchain + + def bazel( + self, bazel_subcommand, targets, extra_options=None, us_cross_toolchain=None + ): + """Execute a bazel command""" + cmdline = [self.bazel_bin, bazel_subcommand] + if extra_options: + cmdline.extend(extra_options) + if us_cross_toolchain: + cmdline.extend(self.get_cross_cli_opts(us_cross_toolchain)) + cmdline.extend(targets) + cmdline_str = " ".join(cmdline) + try: + logging.info('Running "%s"', cmdline_str) + build_proc = subprocess.Popen(cmdline_str, cwd=self.workspace, shell=True) + self.process_list.append(build_proc) + build_proc.wait() + if build_proc.returncode != 0: + sys.exit(build_proc.returncode) + except Exception as e: + logging.error(e) + sys.exit(1) + + self.process_list.remove(build_proc) + + def build_targets(self, targets, user_opts=None, us_cross_toolchain=None): + """Run "bazel build" on all targets in parallel""" + if not targets: + logging.warning("no targets to build") + self.bazel("build", targets, user_opts, us_cross_toolchain) + + def run_targets(self, targets, user_opts=None, us_cross_toolchain=None): + """Run "bazel run" on all targets in serial (since bazel run cannot have multiple targets)""" + for target in targets: + self.bazel("run", [target], user_opts, us_cross_toolchain) + + def build(self): + """Determine which targets to build, then build them""" + cross_targets_to_build, host_targets_to_build = self.get_build_targets() + + logging.debug( + "Building the following %s targets:\n%s", + CPU, + "\n".join(cross_targets_to_build), + ) + logging.debug( + "Building the following host targets:\n%s", "\n".join(host_targets_to_build) ) - targets_to_build = [] - host_targets_to_build = [] - for t in msm_targets: - logging.info( - "skipping kernel build" - ) if skip_kernel else targets_to_build.append(t) - logging.info("skipping ABL build") if skip_abl else targets_to_build.append( - t + "_abl" - ) - logging.info( - "skipping host tools build" - ) if skip_host_tools else host_targets_to_build.append(t + "_dtc") + us_cross_toolchain = self.get_cross_toolchain() - targets_to_build.append(t + "_test_mapping") + if not cross_targets_to_build and not host_targets_to_build: + logging.error("no targets to build") + sys.exit(1) - targets_to_build += extra_msm_targets + logging.info("Building %s targets...", CPU) + self.build_targets(cross_targets_to_build, self.user_opts, us_cross_toolchain) + self.run_targets(cross_targets_to_build, self.user_opts, us_cross_toolchain) - if not targets_to_build: - logging.error("no targets to build") - sys.exit(1) - - logging.info("Building {} targets...".format(CPU)) - build_targets(targets_to_build, user_opts, us_cross_toolchain) - run_targets([t + "_dist" for t in targets_to_build], user_opts, us_cross_toolchain) - - logging.info("Building host targets...") - build_targets(host_targets_to_build, user_opts) - run_targets([t + "_dist" for t in host_targets_to_build], user_opts) + logging.info("Building host targets...") + self.build_targets(host_targets_to_build, self.user_opts) + self.run_targets(host_targets_to_build, self.user_opts) def main(): + """Main script entrypoint""" parser = argparse.ArgumentParser(description="Build kernel platform with Bazel") parser.add_argument( @@ -108,47 +227,22 @@ def main(): action="append", nargs=2, required=True, - help="Target and variant to build (e.g. -t kalama gki). May be passed multiple times.", + help='Target and variant to build (e.g. -t kalama gki). May be passed multiple times. A special VARIANT may be passed, "ALL", which will build all variants for a particular target', ) parser.add_argument( - "-u", - "--userspace_cross_toolchain", - default=None, - metavar="TOOLCHAIN", - help="Set a Bazel toolchain for cross-compiled userspace programs", - ) - parser.add_argument( - "-e", - "--extra_build_target", - metavar="BUILD_TARGET", + "-s", + "--skip", + metavar="BUILD_RULE", action="append", default=[], - help=""" - Additional Bazel target to build and run dist. For example, "-t kalama gki --extra_target foo" - would run "bazel build //msm-kernel:kalama_gki_foo && bazel run //msms-kernel:kalama_gki_foo_dist". - May be passed multiple times. - """, - ) - parser.add_argument( - "--skip_kernel", - action="store_true", - help="Skip building the kernel (note: may still need to build kernel if building ABL or tests)", - ) - parser.add_argument( - "--skip_abl", - action="store_true", - help="Skip building the Android Boot Loader (ABL)", - ) - parser.add_argument( - "--skip_host_tools", - action="store_true", - help="Skip building host tools (DTC, etc.)", + help="Skip specific build rules (e.g. --skip abl will skip the //msm-kernel:__abl build)", ) parser.add_argument( "--log", - help="Log level (debug, info, warning, error)", - default="debug", + metavar="LEVEL", + default="info", choices=["debug", "info", "warning", "error"], + help="Log level (debug, info, warning, error)", ) args, user_opts = parser.parse_known_args(sys.argv[1:]) @@ -158,15 +252,15 @@ def main(): format="[{}] %(levelname)s: %(message)s".format(os.path.basename(sys.argv[0])), ) - extra_targets = list(args.extra_build_target) + args.skip.extend(DEFAULT_SKIP_LIST) - build( - args.target, - user_opts, - args.userspace_cross_toolchain, - extra_targets, - **vars(args) - ) + builder = BazelBuilder(args.target, args.skip, user_opts) + try: + builder.build() + except KeyboardInterrupt: + logging.info("Received keyboard interrupt... exiting") + del builder + sys.exit(1) logging.info("Build completed successfully!")