530 lines
18 KiB
Python
530 lines
18 KiB
Python
# Copyright (C) 2020 The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Splits a manifest to the minimum set of projects needed to build the targets.
|
|
|
|
Usage: manifest_split [options] targets
|
|
|
|
targets: Space-separated list of targets that should be buildable
|
|
using the split manifest.
|
|
|
|
options:
|
|
--manifest <path>
|
|
Path to the repo manifest to split. [Required]
|
|
--split-manifest <path>
|
|
Path to write the resulting split manifest. [Required]
|
|
--config <path>
|
|
Optional path(s) to a config XML file containing projects to add or
|
|
remove. See default_config.xml for an example. This flag can be passed
|
|
more than once to use multiple config files.
|
|
Sample file my_config.xml:
|
|
<config>
|
|
<add_project name="vendor/my/needed/project" />
|
|
<remove_project name="vendor/my/unused/project" />
|
|
</config>
|
|
--repo-list <path>
|
|
Optional path to the output of the 'repo list' command. Used if the
|
|
output of 'repo list' needs pre-processing before being used by
|
|
this tool.
|
|
--ninja-build <path>
|
|
Optional path to the combined-<target>.ninja file found in an out dir.
|
|
If not provided, the default file is used based on the lunch environment.
|
|
--ninja-binary <path>
|
|
Optional path to the ninja binary. Uses the standard binary by default.
|
|
--module-info <path>
|
|
Optional path to the module-info.json file found in an out dir.
|
|
If not provided, the default file is used based on the lunch environment.
|
|
--kati-stamp <path>
|
|
Optional path to the .kati_stamp file found in an out dir.
|
|
If not provided, the default file is used based on the lunch environment.
|
|
--overlay <path>
|
|
Optional path(s) to treat as overlays when parsing the kati stamp file
|
|
and scanning for makefiles. See the tools/treble/build/sandbox directory
|
|
for more info about overlays. This flag can be passed more than once.
|
|
--debug
|
|
Print debug messages.
|
|
-h (--help)
|
|
Display this usage message and exit.
|
|
"""
|
|
|
|
from __future__ import print_function
|
|
|
|
import getopt
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import pkg_resources
|
|
import subprocess
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
|
|
logging.basicConfig(
|
|
stream=sys.stdout,
|
|
level=logging.INFO,
|
|
format="%(asctime)s - %(filename)s - %(levelname)-8s: %(message)s",
|
|
datefmt="%Y-%m-%d %H:%M:%S")
|
|
logger = logging.getLogger(os.path.basename(__file__))
|
|
|
|
# Projects determined to be needed despite the dependency not being visible
|
|
# to ninja.
|
|
DEFAULT_CONFIG_PATH = pkg_resources.resource_filename(__name__,
|
|
"default_config.xml")
|
|
|
|
|
|
def read_config(config_file):
|
|
"""Reads a config XML file to find extra projects to add or remove.
|
|
|
|
Args:
|
|
config_file: The filename of the config XML.
|
|
|
|
Returns:
|
|
A tuple of (set of remove_projects, set of add_projects) from the config.
|
|
"""
|
|
root = ET.parse(config_file).getroot()
|
|
remove_projects = set(
|
|
[child.attrib["name"] for child in root.findall("remove_project")])
|
|
add_projects = set(
|
|
[child.attrib["name"] for child in root.findall("add_project")])
|
|
return remove_projects, add_projects
|
|
|
|
|
|
def get_repo_projects(repo_list_file):
|
|
"""Returns a dict of { project path : project name } using 'repo list'.
|
|
|
|
Args:
|
|
repo_list_file: An optional filename to read instead of calling the repo
|
|
list command.
|
|
"""
|
|
repo_list = []
|
|
|
|
if repo_list_file:
|
|
with open(repo_list_file) as repo_list_lines:
|
|
repo_list = [line.strip() for line in repo_list_lines if line.strip()]
|
|
else:
|
|
repo_list = subprocess.check_output([
|
|
"repo",
|
|
"list",
|
|
]).decode().strip("\n").split("\n")
|
|
return dict([entry.split(" : ") for entry in repo_list])
|
|
|
|
|
|
def get_module_info(module_info_file, repo_projects):
|
|
"""Returns a dict of { project name : set of modules } in each project.
|
|
|
|
Args:
|
|
module_info_file: The path to a module-info.json file from a build.
|
|
repo_projects: The output of the get_repo_projects function.
|
|
|
|
Raises:
|
|
ValueError: A module from module-info.json belongs to a path not
|
|
known by the repo projects output.
|
|
"""
|
|
project_modules = {}
|
|
|
|
with open(module_info_file) as module_info_file:
|
|
module_info = json.load(module_info_file)
|
|
|
|
def module_has_valid_path(module):
|
|
return ("path" in module_info[module] and module_info[module]["path"] and
|
|
not module_info[module]["path"][0].startswith("out/"))
|
|
|
|
module_paths = {
|
|
module: module_info[module]["path"][0]
|
|
for module in module_info
|
|
if module_has_valid_path(module)
|
|
}
|
|
module_project_paths = {
|
|
module: scan_repo_projects(repo_projects, module_paths[module])
|
|
for module in module_paths
|
|
}
|
|
|
|
for module, project_path in module_project_paths.items():
|
|
if not project_path:
|
|
raise ValueError("Unknown module path for module %s: %s" %
|
|
(module, module_info[module]))
|
|
project_modules.setdefault(repo_projects[project_path], set()).add(module)
|
|
return project_modules
|
|
|
|
|
|
def get_ninja_inputs(ninja_binary, ninja_build_file, modules):
|
|
"""Returns the set of input file path strings for the given modules.
|
|
|
|
Uses the `ninja -t inputs` tool.
|
|
|
|
Args:
|
|
ninja_binary: The path to a ninja binary.
|
|
ninja_build_file: The path to a .ninja file from a build.
|
|
modules: The set of modules to scan for inputs.
|
|
"""
|
|
inputs = set(
|
|
subprocess.check_output([
|
|
ninja_binary,
|
|
"-f",
|
|
ninja_build_file,
|
|
"-t",
|
|
"inputs",
|
|
"-d",
|
|
] + list(modules)).decode().strip("\n").split("\n"))
|
|
return {path.strip() for path in inputs}
|
|
|
|
|
|
def get_kati_makefiles(kati_stamp_file, overlays):
|
|
"""Returns the set of makefile paths from the kati stamp file.
|
|
|
|
Uses the ckati_stamp_dump prebuilt binary.
|
|
Also includes symlink sources in the resulting set for any
|
|
makefiles that are symlinks.
|
|
|
|
Args:
|
|
kati_stamp_file: The path to a .kati_stamp file from a build.
|
|
overlays: A list of paths to treat as overlays when parsing the kati stamp
|
|
file.
|
|
"""
|
|
# Get a set of all makefiles that were parsed by Kati during the build.
|
|
makefiles = set(
|
|
subprocess.check_output([
|
|
"prebuilts/build-tools/linux-x86/bin/ckati_stamp_dump",
|
|
"--files",
|
|
kati_stamp_file,
|
|
]).decode().strip("\n").split("\n"))
|
|
|
|
def is_product_makefile(makefile):
|
|
"""Returns True if the makefile path meets certain criteria."""
|
|
banned_prefixes = [
|
|
"out/",
|
|
# Ignore product makefiles for sample AOSP boards.
|
|
"device/amlogic",
|
|
"device/generic",
|
|
"device/google",
|
|
"device/linaro",
|
|
"device/sample",
|
|
]
|
|
banned_suffixes = [
|
|
# All Android.mk files in the source are always parsed by Kati,
|
|
# so including them here would bring in lots of unnecessary projects.
|
|
"Android.mk",
|
|
# The ckati stamp file always includes a line for the ckati bin at
|
|
# the beginnning.
|
|
"bin/ckati",
|
|
]
|
|
return (all([not makefile.startswith(p) for p in banned_prefixes]) and
|
|
all([not makefile.endswith(s) for s in banned_suffixes]))
|
|
|
|
# Limit the makefiles to only product makefiles.
|
|
product_makefiles = {
|
|
os.path.normpath(path) for path in makefiles if is_product_makefile(path)
|
|
}
|
|
|
|
def strip_overlay(makefile):
|
|
"""Remove any overlays from a makefile path."""
|
|
for overlay in overlays:
|
|
if makefile.startswith(overlay):
|
|
return makefile[len(overlay):]
|
|
return makefile
|
|
|
|
makefiles_and_symlinks = set()
|
|
for makefile in product_makefiles:
|
|
# Search for the makefile, possibly scanning overlays as well.
|
|
for overlay in [""] + overlays:
|
|
makefile_with_overlay = os.path.join(overlay, makefile)
|
|
if os.path.exists(makefile_with_overlay):
|
|
makefile = makefile_with_overlay
|
|
break
|
|
|
|
if not os.path.exists(makefile):
|
|
logger.warning("Unknown kati makefile: %s" % makefile)
|
|
continue
|
|
|
|
# Ensure the project that contains the makefile is included, as well as
|
|
# the project that any makefile symlinks point to.
|
|
makefiles_and_symlinks.add(strip_overlay(makefile))
|
|
if os.path.islink(makefile):
|
|
makefiles_and_symlinks.add(
|
|
strip_overlay(os.path.relpath(os.path.realpath(makefile))))
|
|
|
|
return makefiles_and_symlinks
|
|
|
|
|
|
def scan_repo_projects(repo_projects, input_path):
|
|
"""Returns the project path of the given input path if it exists.
|
|
|
|
Args:
|
|
repo_projects: The output of the get_repo_projects function.
|
|
input_path: The path of an input file used in the build, as given by the
|
|
ninja inputs tool.
|
|
|
|
Returns:
|
|
The path string, or None if not found.
|
|
"""
|
|
parts = input_path.split("/")
|
|
|
|
for index in reversed(range(0, len(parts))):
|
|
project_path = os.path.join(*parts[:index + 1])
|
|
if project_path in repo_projects:
|
|
return project_path
|
|
|
|
return None
|
|
|
|
|
|
def get_input_projects(repo_projects, inputs):
|
|
"""Returns the set of project names that contain the given input paths.
|
|
|
|
Args:
|
|
repo_projects: The output of the get_repo_projects function.
|
|
inputs: The paths of input files used in the build, as given by the ninja
|
|
inputs tool.
|
|
"""
|
|
input_project_paths = [
|
|
scan_repo_projects(repo_projects, input_path)
|
|
for input_path in inputs
|
|
if (not input_path.startswith("out/") and not input_path.startswith("/"))
|
|
]
|
|
return {
|
|
repo_projects[project_path]
|
|
for project_path in input_project_paths
|
|
if project_path is not None
|
|
}
|
|
|
|
|
|
def update_manifest(manifest, input_projects, remove_projects):
|
|
"""Modifies and returns a manifest ElementTree by modifying its projects.
|
|
|
|
Args:
|
|
manifest: The manifest object to modify.
|
|
input_projects: A set of projects that should stay in the manifest.
|
|
remove_projects: A set of projects that should be removed from the manifest.
|
|
Projects in this set override input_projects.
|
|
|
|
Returns:
|
|
The modified manifest object.
|
|
"""
|
|
projects_to_keep = input_projects.difference(remove_projects)
|
|
root = manifest.getroot()
|
|
for child in root.findall("project"):
|
|
if child.attrib["name"] not in projects_to_keep:
|
|
root.remove(child)
|
|
return manifest
|
|
|
|
|
|
def create_manifest_sha1_element(manifest, name):
|
|
"""Creates and returns an ElementTree 'hash' Element using a sha1 hash.
|
|
|
|
Args:
|
|
manifest: The manifest ElementTree to hash.
|
|
name: The name string to give this element.
|
|
|
|
Returns:
|
|
The ElementTree 'hash' Element.
|
|
"""
|
|
sha1_element = ET.Element("hash")
|
|
sha1_element.set("type", "sha1")
|
|
sha1_element.set("name", name)
|
|
sha1_element.set("value",
|
|
hashlib.sha1(ET.tostring(manifest.getroot())).hexdigest())
|
|
return sha1_element
|
|
|
|
|
|
def create_split_manifest(targets, manifest_file, split_manifest_file,
|
|
config_files, repo_list_file, ninja_build_file,
|
|
ninja_binary, module_info_file, kati_stamp_file,
|
|
overlays):
|
|
"""Creates and writes a split manifest by inspecting build inputs.
|
|
|
|
Args:
|
|
targets: List of targets that should be buildable using the split manifest.
|
|
manifest_file: Path to the repo manifest to split.
|
|
split_manifest_file: Path to write the resulting split manifest.
|
|
config_files: Paths to a config XML file containing projects to add or
|
|
remove. See default_config.xml for an example. This flag can be passed
|
|
more than once to use multiple config files.
|
|
repo_list_file: Path to the output of the 'repo list' command.
|
|
ninja_build_file: Path to the combined-<target>.ninja file found in an out
|
|
dir.
|
|
ninja_binary: Path to the ninja binary.
|
|
module_info_file: Path to the module-info.json file found in an out dir.
|
|
kati_stamp_file: The path to a .kati_stamp file from a build.
|
|
overlays: A list of paths to treat as overlays when parsing the kati stamp
|
|
file.
|
|
"""
|
|
remove_projects = set()
|
|
add_projects = set()
|
|
for config_file in config_files:
|
|
config_remove_projects, config_add_projects = read_config(config_file)
|
|
remove_projects = remove_projects.union(config_remove_projects)
|
|
add_projects = add_projects.union(config_add_projects)
|
|
|
|
repo_projects = get_repo_projects(repo_list_file)
|
|
module_info = get_module_info(module_info_file, repo_projects)
|
|
|
|
inputs = get_ninja_inputs(ninja_binary, ninja_build_file, targets)
|
|
input_projects = get_input_projects(repo_projects, inputs)
|
|
if logger.isEnabledFor(logging.DEBUG):
|
|
for project in sorted(input_projects):
|
|
logger.debug("Direct dependency: %s", project)
|
|
logger.info("%s projects needed for targets \"%s\"", len(input_projects),
|
|
" ".join(targets))
|
|
|
|
kati_makefiles = get_kati_makefiles(kati_stamp_file, overlays)
|
|
kati_makefiles_projects = get_input_projects(repo_projects, kati_makefiles)
|
|
if logger.isEnabledFor(logging.DEBUG):
|
|
for project in sorted(kati_makefiles_projects.difference(input_projects)):
|
|
logger.debug("Kati makefile dependency: %s", project)
|
|
input_projects = input_projects.union(kati_makefiles_projects)
|
|
logger.info("%s projects after including Kati makefiles projects.",
|
|
len(input_projects))
|
|
|
|
if logger.isEnabledFor(logging.DEBUG):
|
|
manual_projects = add_projects.difference(input_projects)
|
|
for project in sorted(manual_projects):
|
|
logger.debug("Manual inclusion: %s", project)
|
|
input_projects = input_projects.union(add_projects)
|
|
logger.info("%s projects after including manual additions.",
|
|
len(input_projects))
|
|
|
|
# Remove projects from our set of input projects before adding adjacent
|
|
# modules, so that no project is added only because of an adjacent
|
|
# dependency in a to-be-removed project.
|
|
input_projects = input_projects.difference(remove_projects)
|
|
|
|
# While we still have projects whose modules we haven't checked yet,
|
|
checked_projects = set()
|
|
projects_to_check = input_projects.difference(checked_projects)
|
|
while projects_to_check:
|
|
# check all modules in each project,
|
|
modules = []
|
|
for project in projects_to_check:
|
|
checked_projects.add(project)
|
|
if project not in module_info:
|
|
continue
|
|
modules += module_info[project]
|
|
|
|
# adding those modules' input projects to our list of projects.
|
|
inputs = get_ninja_inputs(ninja_binary, ninja_build_file, modules)
|
|
adjacent_module_additions = get_input_projects(repo_projects, inputs)
|
|
if logger.isEnabledFor(logging.DEBUG):
|
|
for project in sorted(
|
|
adjacent_module_additions.difference(input_projects)):
|
|
logger.debug("Adjacent module dependency: %s", project)
|
|
input_projects = input_projects.union(adjacent_module_additions)
|
|
logger.info("%s total projects so far.", len(input_projects))
|
|
|
|
projects_to_check = input_projects.difference(checked_projects)
|
|
|
|
original_manifest = ET.parse(manifest_file)
|
|
original_sha1 = create_manifest_sha1_element(original_manifest, "original")
|
|
split_manifest = update_manifest(original_manifest, input_projects,
|
|
remove_projects)
|
|
split_manifest.getroot().append(original_sha1)
|
|
split_manifest.getroot().append(
|
|
create_manifest_sha1_element(split_manifest, "self"))
|
|
split_manifest.write(split_manifest_file)
|
|
|
|
|
|
def main(argv):
|
|
try:
|
|
opts, args = getopt.getopt(argv, "h", [
|
|
"help",
|
|
"debug",
|
|
"manifest=",
|
|
"split-manifest=",
|
|
"config=",
|
|
"repo-list=",
|
|
"ninja-build=",
|
|
"ninja-binary=",
|
|
"module-info=",
|
|
"kati-stamp=",
|
|
"overlay=",
|
|
])
|
|
except getopt.GetoptError as err:
|
|
print(__doc__, file=sys.stderr)
|
|
print("**%s**" % str(err), file=sys.stderr)
|
|
sys.exit(2)
|
|
|
|
manifest_file = None
|
|
split_manifest_file = None
|
|
config_files = [DEFAULT_CONFIG_PATH]
|
|
repo_list_file = None
|
|
ninja_build_file = None
|
|
module_info_file = None
|
|
ninja_binary = "ninja"
|
|
kati_stamp_file = None
|
|
overlays = []
|
|
|
|
for o, a in opts:
|
|
if o in ("-h", "--help"):
|
|
print(__doc__, file=sys.stderr)
|
|
sys.exit()
|
|
elif o in ("--debug"):
|
|
logger.setLevel(logging.DEBUG)
|
|
elif o in ("--manifest"):
|
|
manifest_file = a
|
|
elif o in ("--split-manifest"):
|
|
split_manifest_file = a
|
|
elif o in ("--config"):
|
|
config_files.append(a)
|
|
elif o in ("--repo-list"):
|
|
repo_list_file = a
|
|
elif o in ("--ninja-build"):
|
|
ninja_build_file = a
|
|
elif o in ("--ninja-binary"):
|
|
ninja_binary = a
|
|
elif o in ("--module-info"):
|
|
module_info_file = a
|
|
elif o in ("--kati-stamp"):
|
|
kati_stamp_file = a
|
|
elif o in ("--overlay"):
|
|
overlays.append(a)
|
|
else:
|
|
assert False, "unknown option \"%s\"" % o
|
|
|
|
if not args:
|
|
print(__doc__, file=sys.stderr)
|
|
print("**Missing targets**", file=sys.stderr)
|
|
sys.exit(2)
|
|
if not manifest_file:
|
|
print(__doc__, file=sys.stderr)
|
|
print("**Missing required flag --manifest**", file=sys.stderr)
|
|
sys.exit(2)
|
|
if not split_manifest_file:
|
|
print(__doc__, file=sys.stderr)
|
|
print("**Missing required flag --split-manifest**", file=sys.stderr)
|
|
sys.exit(2)
|
|
if not module_info_file:
|
|
module_info_file = os.path.join(os.environ["ANDROID_PRODUCT_OUT"],
|
|
"module-info.json")
|
|
if not kati_stamp_file:
|
|
kati_stamp_file = os.path.join(
|
|
os.environ["ANDROID_BUILD_TOP"], "out",
|
|
".kati_stamp-%s" % os.environ["TARGET_PRODUCT"])
|
|
if not ninja_build_file:
|
|
ninja_build_file = os.path.join(
|
|
os.environ["ANDROID_BUILD_TOP"], "out",
|
|
"combined-%s.ninja" % os.environ["TARGET_PRODUCT"])
|
|
|
|
create_split_manifest(
|
|
targets=args,
|
|
manifest_file=manifest_file,
|
|
split_manifest_file=split_manifest_file,
|
|
config_files=config_files,
|
|
repo_list_file=repo_list_file,
|
|
ninja_build_file=ninja_build_file,
|
|
ninja_binary=ninja_binary,
|
|
module_info_file=module_info_file,
|
|
kati_stamp_file=kati_stamp_file,
|
|
overlays=overlays)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.argv[1:])
|