#!/usr/bin/env python3 # Copyright (c) Meta Platforms, Inc. and affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. import argparse import json import os import shutil import subprocess import sys import tarfile import tempfile # We don't import cache.create_cache directly as the facebook # specific import below may monkey patch it, and we want to # observe the patched version of this function! import getdeps.cache as cache_module from getdeps.buildopts import setup_build_options from getdeps.dyndeps import create_dyn_dep_munger from getdeps.errors import TransientFailure from getdeps.fetcher import ( file_name_is_cmake_file, list_files_under_dir_newer_than_timestamp, SystemPackageFetcher, ) from getdeps.load import ManifestLoader from getdeps.manifest import ManifestParser from getdeps.platform import HostType from getdeps.runcmd import run_cmd from getdeps.subcmd import add_subcommands, cmd, SubCmd try: import getdeps.facebook # noqa: F401 except ImportError: # we don't ship the facebook specific subdir, # so allow that to fail silently pass sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "getdeps")) class UsageError(Exception): pass @cmd("validate-manifest", "parse a manifest and validate that it is correct") class ValidateManifest(SubCmd): def run(self, args): try: ManifestParser(file_name=args.file_name) print("OK", file=sys.stderr) return 0 except Exception as exc: print("ERROR: %s" % str(exc), file=sys.stderr) return 1 def setup_parser(self, parser): parser.add_argument("file_name", help="path to the manifest file") @cmd("show-host-type", "outputs the host type tuple for the host machine") class ShowHostType(SubCmd): def run(self, args): host = HostType() print("%s" % host.as_tuple_string()) return 0 class ProjectCmdBase(SubCmd): def run(self, args): opts = setup_build_options(args) if args.current_project is not None: opts.repo_project = args.current_project if args.project is None: if opts.repo_project is None: raise UsageError( "no project name specified, and no .projectid file found" ) if opts.repo_project == "fbsource": # The fbsource repository is a little special. There is no project # manifest file for it. A specific project must always be explicitly # specified when building from fbsource. raise UsageError( "no project name specified (required when building in fbsource)" ) args.project = opts.repo_project ctx_gen = opts.get_context_generator() if args.test_dependencies: ctx_gen.set_value_for_all_projects("test", "on") if args.enable_tests: ctx_gen.set_value_for_project(args.project, "test", "on") else: ctx_gen.set_value_for_project(args.project, "test", "off") if opts.shared_libs: ctx_gen.set_value_for_all_projects("shared_libs", "on") loader = ManifestLoader(opts, ctx_gen) self.process_project_dir_arguments(args, loader) manifest = loader.load_manifest(args.project) self.run_project_cmd(args, loader, manifest) def process_project_dir_arguments(self, args, loader): def parse_project_arg(arg, arg_type): parts = arg.split(":") if len(parts) == 2: project, path = parts elif len(parts) == 1: project = args.project path = parts[0] # On Windows path contains colon, e.g. C:\open elif os.name == "nt" and len(parts) == 3: project = parts[0] path = parts[1] + ":" + parts[2] else: raise UsageError( "invalid %s argument; too many ':' characters: %s" % (arg_type, arg) ) return project, os.path.abspath(path) # If we are currently running from a project repository, # use the current repository for the project sources. build_opts = loader.build_opts if build_opts.repo_project is not None and build_opts.repo_root is not None: loader.set_project_src_dir(build_opts.repo_project, build_opts.repo_root) for arg in args.src_dir: project, path = parse_project_arg(arg, "--src-dir") loader.set_project_src_dir(project, path) for arg in args.build_dir: project, path = parse_project_arg(arg, "--build-dir") loader.set_project_build_dir(project, path) for arg in args.install_dir: project, path = parse_project_arg(arg, "--install-dir") loader.set_project_install_dir(project, path) for arg in args.project_install_prefix: project, path = parse_project_arg(arg, "--install-prefix") loader.set_project_install_prefix(project, path) def setup_parser(self, parser): parser.add_argument( "project", nargs="?", help=( "name of the project or path to a manifest " "file describing the project" ), ) parser.add_argument( "--no-tests", action="store_false", dest="enable_tests", default=True, help="Disable building tests for this project.", ) parser.add_argument( "--test-dependencies", action="store_true", help="Enable building tests for dependencies as well.", ) parser.add_argument( "--current-project", help="Specify the name of the fbcode_builder manifest file for the " "current repository. If not specified, the code will attempt to find " "this in a .projectid file in the repository root.", ) parser.add_argument( "--src-dir", default=[], action="append", help="Specify a local directory to use for the project source, " "rather than fetching it.", ) parser.add_argument( "--build-dir", default=[], action="append", help="Explicitly specify the build directory to use for the " "project, instead of the default location in the scratch path. " "This only affects the project specified, and not its dependencies.", ) parser.add_argument( "--install-dir", default=[], action="append", help="Explicitly specify the install directory to use for the " "project, instead of the default location in the scratch path. " "This only affects the project specified, and not its dependencies.", ) parser.add_argument( "--project-install-prefix", default=[], action="append", help="Specify the final deployment installation path for a project", ) self.setup_project_cmd_parser(parser) def setup_project_cmd_parser(self, parser): pass class CachedProject(object): """A helper that allows calling the cache logic for a project from both the build and the fetch code""" def __init__(self, cache, loader, m): self.m = m self.inst_dir = loader.get_project_install_dir(m) self.project_hash = loader.get_project_hash(m) self.ctx = loader.ctx_gen.get_context(m.name) self.loader = loader self.cache = cache self.cache_file_name = "-".join( ( m.name, self.ctx.get("os"), self.ctx.get("distro") or "none", self.ctx.get("distro_vers") or "none", self.project_hash, "buildcache.tgz", ) ) def is_cacheable(self): """We only cache third party projects""" return self.cache and self.m.shipit_project is None def was_cached(self): cached_marker = os.path.join(self.inst_dir, ".getdeps-cached-build") return os.path.exists(cached_marker) def download(self): if self.is_cacheable() and not os.path.exists(self.inst_dir): print("check cache for %s" % self.cache_file_name) dl_dir = os.path.join(self.loader.build_opts.scratch_dir, "downloads") if not os.path.exists(dl_dir): os.makedirs(dl_dir) try: target_file_name = os.path.join(dl_dir, self.cache_file_name) if self.cache.download_to_file(self.cache_file_name, target_file_name): tf = tarfile.open(target_file_name, "r") print( "Extracting %s -> %s..." % (self.cache_file_name, self.inst_dir) ) tf.extractall(self.inst_dir) cached_marker = os.path.join(self.inst_dir, ".getdeps-cached-build") with open(cached_marker, "w") as f: f.write("\n") return True except Exception as exc: print("%s" % str(exc)) return False def upload(self): if self.is_cacheable(): # We can prepare an archive and stick it in LFS tempdir = tempfile.mkdtemp() tarfilename = os.path.join(tempdir, self.cache_file_name) print("Archiving for cache: %s..." % tarfilename) tf = tarfile.open(tarfilename, "w:gz") tf.add(self.inst_dir, arcname=".") tf.close() try: self.cache.upload_from_file(self.cache_file_name, tarfilename) except Exception as exc: print( "Failed to upload to cache (%s), continue anyway" % str(exc), file=sys.stderr, ) shutil.rmtree(tempdir) @cmd("fetch", "fetch the code for a given project") class FetchCmd(ProjectCmdBase): def setup_project_cmd_parser(self, parser): parser.add_argument( "--recursive", help="fetch the transitive deps also", action="store_true", default=False, ) parser.add_argument( "--host-type", help=( "When recursively fetching, fetch deps for " "this host type rather than the current system" ), ) def run_project_cmd(self, args, loader, manifest): if args.recursive: projects = loader.manifests_in_dependency_order() else: projects = [manifest] cache = cache_module.create_cache() for m in projects: cached_project = CachedProject(cache, loader, m) if cached_project.download(): continue inst_dir = loader.get_project_install_dir(m) built_marker = os.path.join(inst_dir, ".built-by-getdeps") if os.path.exists(built_marker): with open(built_marker, "r") as f: built_hash = f.read().strip() project_hash = loader.get_project_hash(m) if built_hash == project_hash: continue # We need to fetch the sources fetcher = loader.create_fetcher(m) fetcher.update() @cmd("install-system-deps", "Install system packages to satisfy the deps for a project") class InstallSysDepsCmd(ProjectCmdBase): def setup_project_cmd_parser(self, parser): parser.add_argument( "--recursive", help="install the transitive deps also", action="store_true", default=False, ) parser.add_argument( "--dry-run", action="store_true", default=False, help="Don't install, just print the commands specs we would run", ) parser.add_argument( "--os-type", help="Filter to just this OS type to run", choices=["linux", "darwin", "windows"], action="store", dest="ostype", default=None, ) parser.add_argument( "--distro", help="Filter to just this distro to run", choices=["ubuntu", "centos_stream"], action="store", dest="distro", default=None, ) parser.add_argument( "--distro-version", help="Filter to just this distro version", action="store", dest="distrovers", default=None, ) def run_project_cmd(self, args, loader, manifest): if args.recursive: projects = loader.manifests_in_dependency_order() else: projects = [manifest] rebuild_ctx_gen = False if args.ostype: loader.build_opts.host_type.ostype = args.ostype loader.build_opts.host_type.distro = None loader.build_opts.host_type.distrovers = None rebuild_ctx_gen = True if args.distro: loader.build_opts.host_type.distro = args.distro loader.build_opts.host_type.distrovers = None rebuild_ctx_gen = True if args.distrovers: loader.build_opts.host_type.distrovers = args.distrovers rebuild_ctx_gen = True if rebuild_ctx_gen: loader.ctx_gen = loader.build_opts.get_context_generator() manager = loader.build_opts.host_type.get_package_manager() all_packages = {} for m in projects: ctx = loader.ctx_gen.get_context(m.name) packages = m.get_required_system_packages(ctx) for k, v in packages.items(): merged = all_packages.get(k, []) merged += v all_packages[k] = merged cmd_args = None if manager == "rpm": packages = sorted(set(all_packages["rpm"])) if packages: cmd_args = ["sudo", "dnf", "install", "-y"] + packages elif manager == "deb": packages = sorted(set(all_packages["deb"])) if packages: cmd_args = ["sudo", "apt", "install", "-y"] + packages elif manager == "homebrew": packages = sorted(set(all_packages["homebrew"])) if packages: cmd_args = ["brew", "install"] + packages else: host_tuple = loader.build_opts.host_type.as_tuple_string() print( f"I don't know how to install any packages on this system {host_tuple}" ) return if cmd_args: if args.dry_run: print(" ".join(cmd_args)) else: run_cmd(cmd_args) else: print("no packages to install") @cmd("list-deps", "lists the transitive deps for a given project") class ListDepsCmd(ProjectCmdBase): def run_project_cmd(self, args, loader, manifest): for m in loader.manifests_in_dependency_order(): print(m.name) return 0 def setup_project_cmd_parser(self, parser): parser.add_argument( "--host-type", help=( "Produce the list for the specified host type, " "rather than that of the current system" ), ) def clean_dirs(opts): for d in ["build", "installed", "extracted", "shipit"]: d = os.path.join(opts.scratch_dir, d) print("Cleaning %s..." % d) if os.path.exists(d): shutil.rmtree(d) @cmd("clean", "clean up the scratch dir") class CleanCmd(SubCmd): def run(self, args): opts = setup_build_options(args) clean_dirs(opts) @cmd("show-scratch-dir", "show the scratch dir") class ShowScratchDirCmd(SubCmd): def run(self, args): opts = setup_build_options(args) print(opts.scratch_dir) @cmd("show-build-dir", "print the build dir for a given project") class ShowBuildDirCmd(ProjectCmdBase): def run_project_cmd(self, args, loader, manifest): if args.recursive: manifests = loader.manifests_in_dependency_order() else: manifests = [manifest] for m in manifests: inst_dir = loader.get_project_build_dir(m) print(inst_dir) def setup_project_cmd_parser(self, parser): parser.add_argument( "--recursive", help="print the transitive deps also", action="store_true", default=False, ) @cmd("show-inst-dir", "print the installation dir for a given project") class ShowInstDirCmd(ProjectCmdBase): def run_project_cmd(self, args, loader, manifest): if args.recursive: manifests = loader.manifests_in_dependency_order() else: manifests = [manifest] for m in manifests: fetcher = loader.create_fetcher(m) if isinstance(fetcher, SystemPackageFetcher): # We are guaranteed that if the fetcher is set to # SystemPackageFetcher then this item is completely # satisfied by the appropriate system packages continue inst_dir = loader.get_project_install_dir_respecting_install_prefix(m) print(inst_dir) def setup_project_cmd_parser(self, parser): parser.add_argument( "--recursive", help="print the transitive deps also", action="store_true", default=False, ) @cmd("show-source-dir", "print the source dir for a given project") class ShowSourceDirCmd(ProjectCmdBase): def run_project_cmd(self, args, loader, manifest): if args.recursive: manifests = loader.manifests_in_dependency_order() else: manifests = [manifest] for m in manifests: fetcher = loader.create_fetcher(m) print(fetcher.get_src_dir()) def setup_project_cmd_parser(self, parser): parser.add_argument( "--recursive", help="print the transitive deps also", action="store_true", default=False, ) @cmd("build", "build a given project") class BuildCmd(ProjectCmdBase): def run_project_cmd(self, args, loader, manifest): if args.clean: clean_dirs(loader.build_opts) print("Building on %s" % loader.ctx_gen.get_context(args.project)) projects = loader.manifests_in_dependency_order() cache = cache_module.create_cache() if args.use_build_cache else None # Accumulate the install directories so that the build steps # can find their dep installation install_dirs = [] for m in projects: fetcher = loader.create_fetcher(m) if args.build_skip_lfs_download and hasattr(fetcher, "skip_lfs_download"): print("skipping lfs download for %s" % m.name) fetcher.skip_lfs_download() if isinstance(fetcher, SystemPackageFetcher): # We are guaranteed that if the fetcher is set to # SystemPackageFetcher then this item is completely # satisfied by the appropriate system packages continue if args.clean: fetcher.clean() build_dir = loader.get_project_build_dir(m) inst_dir = loader.get_project_install_dir(m) if ( m == manifest and not args.only_deps or m != manifest and not args.no_deps ): print("Assessing %s..." % m.name) project_hash = loader.get_project_hash(m) ctx = loader.ctx_gen.get_context(m.name) built_marker = os.path.join(inst_dir, ".built-by-getdeps") cached_project = CachedProject(cache, loader, m) reconfigure, sources_changed = self.compute_source_change_status( cached_project, fetcher, m, built_marker, project_hash ) if os.path.exists(built_marker) and not cached_project.was_cached(): # We've previously built this. We may need to reconfigure if # our deps have changed, so let's check them. dep_reconfigure, dep_build = self.compute_dep_change_status( m, built_marker, loader ) if dep_reconfigure: reconfigure = True if dep_build: sources_changed = True extra_cmake_defines = ( json.loads(args.extra_cmake_defines) if args.extra_cmake_defines else {} ) extra_b2_args = args.extra_b2_args or [] if sources_changed or reconfigure or not os.path.exists(built_marker): if os.path.exists(built_marker): os.unlink(built_marker) src_dir = fetcher.get_src_dir() # Prepare builders write out config before the main builder runs prepare_builders = m.create_prepare_builders( loader.build_opts, ctx, src_dir, build_dir, inst_dir, loader, ) for preparer in prepare_builders: preparer.prepare(install_dirs, reconfigure=reconfigure) builder = m.create_builder( loader.build_opts, src_dir, build_dir, inst_dir, ctx, loader, final_install_prefix=loader.get_project_install_prefix(m), extra_cmake_defines=extra_cmake_defines, cmake_target=args.cmake_target if m == manifest else "install", extra_b2_args=extra_b2_args, ) builder.build(install_dirs, reconfigure=reconfigure) # If we are building the project (not dependency) and a specific # cmake_target (not 'install') has been requested, then we don't # set the built_marker. This allows subsequent runs of getdeps.py # for the project to run with different cmake_targets to trigger # cmake has_built_marker = False if not (m == manifest and args.cmake_target != "install"): with open(built_marker, "w") as f: f.write(project_hash) has_built_marker = True # Only populate the cache from continuous build runs, and # only if we have a built_marker. if ( not args.skip_upload and args.schedule_type == "continuous" and has_built_marker ): cached_project.upload() elif args.verbose: print("found good %s" % built_marker) # Paths are resolved from front. We prepend rather than append as # the last project in topo order is the project itself, which # should be first in the path, then its deps and so on. install_dirs.insert(0, inst_dir) def compute_dep_change_status(self, m, built_marker, loader): reconfigure = False sources_changed = False st = os.lstat(built_marker) ctx = loader.ctx_gen.get_context(m.name) dep_list = m.get_dependencies(ctx) for dep in dep_list: if reconfigure and sources_changed: break dep_manifest = loader.load_manifest(dep) dep_root = loader.get_project_install_dir(dep_manifest) for dep_file in list_files_under_dir_newer_than_timestamp( dep_root, st.st_mtime ): if os.path.basename(dep_file) == ".built-by-getdeps": continue if file_name_is_cmake_file(dep_file): if not reconfigure: reconfigure = True print( f"Will reconfigure cmake because {dep_file} is newer than {built_marker}" ) else: if not sources_changed: sources_changed = True print( f"Will run build because {dep_file} is newer than {built_marker}" ) if reconfigure and sources_changed: break return reconfigure, sources_changed def compute_source_change_status( self, cached_project, fetcher, m, built_marker, project_hash ): reconfigure = False sources_changed = False if cached_project.download(): if not os.path.exists(built_marker): fetcher.update() else: check_fetcher = True if os.path.exists(built_marker): check_fetcher = False with open(built_marker, "r") as f: built_hash = f.read().strip() if built_hash == project_hash: if cached_project.is_cacheable(): # We can blindly trust the build status reconfigure = False sources_changed = False else: # Otherwise, we may have changed the source, so let's # check in with the fetcher layer check_fetcher = True else: # Some kind of inconsistency with a prior build, # let's run it again to be sure os.unlink(built_marker) reconfigure = True sources_changed = True # While we don't need to consult the fetcher for the # status in this case, we may still need to have eg: shipit # run in order to have a correct source tree. fetcher.update() if check_fetcher: change_status = fetcher.update() reconfigure = change_status.build_changed() sources_changed = change_status.sources_changed() return reconfigure, sources_changed def setup_project_cmd_parser(self, parser): parser.add_argument( "--clean", action="store_true", default=False, help=( "Clean up the build and installation area prior to building, " "causing the projects to be built from scratch" ), ) parser.add_argument( "--no-deps", action="store_true", default=False, help=( "Only build the named project, not its deps. " "This is most useful after you've built all of the deps, " "and helps to avoid waiting for relatively " "slow up-to-date-ness checks" ), ) parser.add_argument( "--only-deps", action="store_true", default=False, help=( "Only build the named project's deps. " "This is most useful when you want to separate out building " "of all of the deps and your project" ), ) parser.add_argument( "--no-build-cache", action="store_false", default=True, dest="use_build_cache", help="Do not attempt to use the build cache.", ) parser.add_argument( "--schedule-type", help="Indicates how the build was activated" ) parser.add_argument( "--extra-cmake-defines", help=( "Input json map that contains extra cmake defines to be used " "when compiling the current project and all its deps. " 'e.g: \'{"CMAKE_CXX_FLAGS": "--bla"}\'' ), ) parser.add_argument( "--cmake-target", help=("Target for cmake build."), default="install", ) parser.add_argument( "--extra-b2-args", help=( "Repeatable argument that contains extra arguments to pass " "to b2, which compiles boost. " "e.g.: 'cxxflags=-fPIC' 'cflags=-fPIC'" ), action="append", ) parser.add_argument( "--shared-libs", help="Build shared libraries if possible", action="store_true", default=False, ) parser.add_argument( "--free-up-disk", help="Remove unused tools and clean up intermediate files if possible to maximise space for the build", action="store_true", default=False, ) parser.add_argument( "--build-type", help="Set the build type explicitly. Cmake and cargo builders act on them. Only Debug and RelWithDebInfo widely supported.", choices=["Debug", "Release", "RelWithDebInfo", "MinSizeRel"], action="store", default=None, ) @cmd("fixup-dyn-deps", "Adjusts dynamic dependencies for packaging purposes") class FixupDeps(ProjectCmdBase): def run_project_cmd(self, args, loader, manifest): projects = loader.manifests_in_dependency_order() # Accumulate the install directories so that the build steps # can find their dep installation install_dirs = [] for m in projects: inst_dir = loader.get_project_install_dir_respecting_install_prefix(m) install_dirs.append(inst_dir) if m == manifest: dep_munger = create_dyn_dep_munger( loader.build_opts, install_dirs, args.strip ) if dep_munger is None: print(f"dynamic dependency fixups not supported on {sys.platform}") else: dep_munger.process_deps(args.destdir, args.final_install_prefix) def setup_project_cmd_parser(self, parser): parser.add_argument("destdir", help="Where to copy the fixed up executables") parser.add_argument( "--final-install-prefix", help="specify the final installation prefix" ) parser.add_argument( "--strip", action="store_true", default=False, help="Strip debug info while processing executables", ) @cmd("test", "test a given project") class TestCmd(ProjectCmdBase): def run_project_cmd(self, args, loader, manifest): projects = loader.manifests_in_dependency_order() # Accumulate the install directories so that the test steps # can find their dep installation install_dirs = [] for m in projects: inst_dir = loader.get_project_install_dir(m) if m == manifest or args.test_dependencies: built_marker = os.path.join(inst_dir, ".built-by-getdeps") if not os.path.exists(built_marker): print("project %s has not been built" % m.name) # TODO: we could just go ahead and build it here, but I # want to tackle that as part of adding build-for-test # support. return 1 fetcher = loader.create_fetcher(m) src_dir = fetcher.get_src_dir() ctx = loader.ctx_gen.get_context(m.name) build_dir = loader.get_project_build_dir(m) builder = m.create_builder( loader.build_opts, src_dir, build_dir, inst_dir, ctx, loader ) builder.run_tests( install_dirs, schedule_type=args.schedule_type, owner=args.test_owner, test_filter=args.filter, retry=args.retry, no_testpilot=args.no_testpilot, ) install_dirs.append(inst_dir) def setup_project_cmd_parser(self, parser): parser.add_argument( "--schedule-type", help="Indicates how the build was activated" ) parser.add_argument("--test-owner", help="Owner for testpilot") parser.add_argument("--filter", help="Only run the tests matching the regex") parser.add_argument( "--retry", type=int, default=3, help="Number of immediate retries for failed tests " "(noop in continuous and testwarden runs)", ) parser.add_argument( "--no-testpilot", help="Do not use Test Pilot even when available", action="store_true", ) @cmd("generate-github-actions", "generate a GitHub actions configuration") class GenerateGitHubActionsCmd(ProjectCmdBase): RUN_ON_ALL = """ [push, pull_request]""" def run_project_cmd(self, args, loader, manifest): platforms = [ HostType("linux", "ubuntu", "18"), HostType("darwin", None, None), HostType("windows", None, None), ] for p in platforms: if args.os_types and p.ostype not in args.os_types: continue self.write_job_for_platform(p, args) def get_run_on(self, args): if args.run_on_all_branches: return self.RUN_ON_ALL if args.cron: return f""" schedule: - cron: '{args.cron}'""" return f""" push: branches: - {args.main_branch} pull_request: branches: - {args.main_branch}""" # TODO: Break up complex function def write_job_for_platform(self, platform, args): # noqa: C901 build_opts = setup_build_options(args, platform) ctx_gen = build_opts.get_context_generator() loader = ManifestLoader(build_opts, ctx_gen) manifest = loader.load_manifest(args.project) manifest_ctx = loader.ctx_gen.get_context(manifest.name) run_on = self.get_run_on(args) # Some projects don't do anything "useful" as a leaf project, only # as a dep for a leaf project. Check for those here; we don't want # to waste the effort scheduling them on CI. # We do this by looking at the builder type in the manifest file # rather than creating a builder and checking its type because we # don't know enough to create the full builder instance here. builder_name = manifest.get("build", "builder", ctx=manifest_ctx) if builder_name == "nop": return None # We want to be sure that we're running things with python 3 # but python versioning is honestly a bit of a frustrating mess. # `python` may be version 2 or version 3 depending on the system. # python3 may not be a thing at all! # Assume an optimistic default py3 = "python3" if build_opts.is_linux(): artifacts = "linux" runs_on = f"ubuntu-{args.ubuntu_version}" elif build_opts.is_windows(): artifacts = "windows" runs_on = "windows-2019" # The windows runners are python 3 by default; python2.exe # is available if needed. py3 = "python" else: artifacts = "mac" runs_on = "macOS-latest" os.makedirs(args.output_dir, exist_ok=True) job_file_prefix = "getdeps_" if args.job_file_prefix: job_file_prefix = args.job_file_prefix output_file = os.path.join(args.output_dir, f"{job_file_prefix}{artifacts}.yml") if args.job_name_prefix: job_name = args.job_name_prefix + artifacts.capitalize() else: job_name = artifacts with open(output_file, "w") as out: # Deliberate line break here because the @ and the generated # symbols are meaningful to our internal tooling when they # appear in a single token out.write("# This file was @") out.write("generated by getdeps.py\n") out.write( f""" name: {job_name} on:{run_on} permissions: contents: read # to fetch code (actions/checkout) jobs: """ ) getdepscmd = f"{py3} build/fbcode_builder/getdeps.py" out.write(" build:\n") out.write(" runs-on: %s\n" % runs_on) out.write(" steps:\n") if build_opts.is_windows(): # cmake relies on BOOST_ROOT but GH deliberately don't set it in order # to avoid versioning issues: # https://github.com/actions/virtual-environments/issues/319 # Instead, set the version we think we need; this is effectively # coupled with the boost manifest # This is the unusual syntax for setting an env var for the rest of # the steps in a workflow: # https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/ out.write(" - name: Export boost environment\n") out.write( ' run: "echo BOOST_ROOT=%BOOST_ROOT_1_83_0% >> %GITHUB_ENV%"\n' ) out.write(" shell: cmd\n") # The git installation may not like long filenames, so tell it # that we want it to use them! out.write(" - name: Fix Git config\n") out.write(" run: git config --system core.longpaths true\n") out.write(" - name: Disable autocrlf\n") out.write(" run: git config --system core.autocrlf false\n") out.write(" - uses: actions/checkout@v4\n") build_type_arg = "" if args.build_type: build_type_arg = f"--build-type {args.build_type} " if build_opts.free_up_disk: free_up_disk = "--free-up-disk " if not build_opts.is_windows(): out.write(" - name: Show disk space at start\n") out.write(" run: df -h\n") # remove the unused github supplied android dev tools out.write(" - name: Free up disk space\n") out.write(" run: sudo rm -rf /usr/local/lib/android\n") out.write(" - name: Show disk space after freeing up\n") out.write(" run: df -h\n") else: free_up_disk = "" allow_sys_arg = "" if ( build_opts.allow_system_packages and build_opts.host_type.get_package_manager() ): sudo_arg = "sudo " allow_sys_arg = " --allow-system-packages" if build_opts.host_type.get_package_manager() == "deb": out.write(" - name: Update system package info\n") out.write(f" run: {sudo_arg}apt-get update\n") out.write(" - name: Install system deps\n") if build_opts.is_darwin(): # brew is installed as regular user sudo_arg = "" out.write( f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n" ) if build_opts.is_linux() or build_opts.is_freebsd(): out.write(" - name: Install packaging system deps\n") out.write( f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf\n" ) projects = loader.manifests_in_dependency_order() main_repo_url = manifest.get_repo_url(manifest_ctx) has_same_repo_dep = False # Add the rust dep which doesn't have a manifest for m in projects: if m == manifest: continue mbuilder_name = m.get("build", "builder", ctx=manifest_ctx) if ( m.name == "rust" or builder_name == "cargo" or mbuilder_name == "cargo" ): out.write(" - name: Install Rust Stable\n") out.write(" uses: dtolnay/rust-toolchain@stable\n") break # Normal deps that have manifests for m in projects: if m == manifest or m.name == "rust": continue ctx = loader.ctx_gen.get_context(m.name) if m.get_repo_url(ctx) != main_repo_url: out.write(" - name: Fetch %s\n" % m.name) out.write( f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n" ) for m in projects: if m != manifest: if m.name == "rust": continue else: src_dir_arg = "" ctx = loader.ctx_gen.get_context(m.name) if main_repo_url and m.get_repo_url(ctx) == main_repo_url: # Its in the same repo, so src-dir is also . src_dir_arg = "--src-dir=. " has_same_repo_dep = True out.write(" - name: Build %s\n" % m.name) out.write( f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{src_dir_arg}{free_up_disk}--no-tests {m.name}\n" ) out.write(" - name: Build %s\n" % manifest.name) project_prefix = "" if not build_opts.is_windows(): project_prefix = ( " --project-install-prefix %s:/usr/local" % manifest.name ) # If we have dep from same repo, we already built it and don't want to rebuild it again no_deps_arg = "" if has_same_repo_dep: no_deps_arg = "--no-deps " no_tests_arg = "" if not args.enable_tests: no_tests_arg = "--no-tests " out.write( f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{no_tests_arg}{no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n" ) out.write(" - name: Copy artifacts\n") if build_opts.is_linux(): # Strip debug info from the binaries, but only on linux. # While the `strip` utility is also available on macOS, # attempting to strip there results in an error. # The `strip` utility is not available on Windows. strip = " --strip" else: strip = "" out.write( f" run: {getdepscmd}{allow_sys_arg} fixup-dyn-deps{strip} " f"--src-dir=. {manifest.name} _artifacts/{artifacts} {project_prefix} " f"--final-install-prefix /usr/local\n" ) out.write(" - uses: actions/upload-artifact@v2\n") out.write(" with:\n") out.write(" name: %s\n" % manifest.name) out.write(" path: _artifacts\n") if ( args.enable_tests and manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off" ): out.write(" - name: Test %s\n" % manifest.name) out.write( f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n" ) if build_opts.free_up_disk and not build_opts.is_windows(): out.write(" - name: Show disk space at end\n") out.write(" run: df -h\n") def setup_project_cmd_parser(self, parser): parser.add_argument( "--disallow-system-packages", help="Disallow satisfying third party deps from installed system packages", action="store_true", default=False, ) parser.add_argument( "--output-dir", help="The directory that will contain the yml files" ) parser.add_argument( "--run-on-all-branches", action="store_true", help="Allow CI to fire on all branches - Handy for testing", ) parser.add_argument( "--ubuntu-version", default="20.04", help="Version of Ubuntu to use" ) parser.add_argument( "--cron", help="Specify that the job runs on a cron schedule instead of on pushes", ) parser.add_argument( "--main-branch", default="main", help="Main branch to trigger GitHub Action on", ) parser.add_argument( "--os-type", help="Filter to just this OS type to run", choices=["linux", "darwin", "windows"], action="append", dest="os_types", default=[], ) parser.add_argument( "--job-file-prefix", type=str, help="add a prefix to all job file names", default=None, ) parser.add_argument( "--job-name-prefix", type=str, help="add a prefix to all job names", default=None, ) parser.add_argument( "--free-up-disk", help="Remove unused tools and clean up intermediate files if possible to maximise space for the build", action="store_true", default=False, ) parser.add_argument( "--build-type", help="Set the build type explicitly. Cmake and cargo builders act on them. Only Debug and RelWithDebInfo widely supported.", choices=["Debug", "Release", "RelWithDebInfo", "MinSizeRel"], action="store", default=None, ) def get_arg_var_name(args): for arg in args: if arg.startswith("--"): return arg[2:].replace("-", "_") raise Exception("unable to determine argument variable name from %r" % (args,)) def parse_args(): # We want to allow common arguments to be specified either before or after # the subcommand name. In order to do this we add them to the main parser # and to subcommand parsers. In order for this to work, we need to tell # argparse that the default value is SUPPRESS, so that the default values # from the subparser arguments won't override values set by the user from # the main parser. We maintain our own list of desired defaults in the # common_defaults dictionary, and manually set those if the argument wasn't # present at all. common_args = argparse.ArgumentParser(add_help=False) common_defaults = {} def add_common_arg(*args, **kwargs): var_name = get_arg_var_name(args) default_value = kwargs.pop("default", None) common_defaults[var_name] = default_value kwargs["default"] = argparse.SUPPRESS common_args.add_argument(*args, **kwargs) add_common_arg("--scratch-path", help="Where to maintain checkouts and build dirs") add_common_arg( "--vcvars-path", default=None, help="Path to the vcvarsall.bat on Windows." ) add_common_arg( "--install-prefix", help=( "Where the final build products will be installed " "(default is [scratch-path]/installed)" ), ) add_common_arg( "--num-jobs", type=int, help=( "Number of concurrent jobs to use while building. " "(default=number of cpu cores)" ), ) add_common_arg( "--use-shipit", help="use the real ShipIt instead of the simple shipit transformer", action="store_true", default=False, ) add_common_arg( "--facebook-internal", help="Setup the build context as an FB internal build", action="store_true", default=None, ) add_common_arg( "--no-facebook-internal", help="Perform a non-FB internal build, even when in an fbsource repository", action="store_false", dest="facebook_internal", ) add_common_arg( "--allow-system-packages", help="Allow satisfying third party deps from installed system packages", action="store_true", default=False, ) add_common_arg( "-v", "--verbose", help="Print more output", action="store_true", default=False, ) add_common_arg( "-su", "--skip-upload", help="skip upload steps", action="store_true", default=False, ) add_common_arg( "--lfs-path", help="Provide a parent directory for lfs when fbsource is unavailable", default=None, ) add_common_arg( "--build-skip-lfs-download", action="store_true", default=False, help=( "Download from the URL, rather than LFS. This is useful " "in cases where the upstream project has uploaded a new " "version of the archive with a different hash" ), ) ap = argparse.ArgumentParser( description="Get and build dependencies and projects", parents=[common_args] ) sub = ap.add_subparsers( # metavar suppresses the long and ugly default list of subcommands on a # single line. We still render the nicer list below where we would # have shown the nasty one. metavar="", title="Available commands", help="", ) add_subcommands(sub, common_args) args = ap.parse_args() for var_name, default_value in common_defaults.items(): if not hasattr(args, var_name): setattr(args, var_name, default_value) return ap, args def main(): ap, args = parse_args() if getattr(args, "func", None) is None: ap.print_help() return 0 try: return args.func(args) except UsageError as exc: ap.error(str(exc)) return 1 except TransientFailure as exc: print("TransientFailure: %s" % str(exc)) # This return code is treated as a retryable transient infrastructure # error by Facebook's internal CI, rather than eg: a build or code # related error that needs to be fixed before progress can be made. return 128 except subprocess.CalledProcessError as exc: print("%s" % str(exc), file=sys.stderr) print("!! Failed", file=sys.stderr) return 1 if __name__ == "__main__": sys.exit(main())