1// Copyright 2017 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package build 16 17import ( 18 "encoding/json" 19 "errors" 20 "fmt" 21 "io/ioutil" 22 "math/rand" 23 "os" 24 "os/exec" 25 "os/user" 26 "path/filepath" 27 "runtime" 28 "strconv" 29 "strings" 30 "syscall" 31 "time" 32 33 "android/soong/shared" 34 "android/soong/ui/metrics" 35 36 "google.golang.org/protobuf/proto" 37 38 smpb "android/soong/ui/metrics/metrics_proto" 39) 40 41const ( 42 envConfigDir = "vendor/google/tools/soong_config" 43 jsonSuffix = "json" 44) 45 46var ( 47 rbeRandPrefix int 48 googleProdCredsExistCache bool 49) 50 51func init() { 52 rand.Seed(time.Now().UnixNano()) 53 rbeRandPrefix = rand.Intn(1000) 54} 55 56type Config struct{ *configImpl } 57 58type configImpl struct { 59 // Some targets that are implemented in soong_build 60 // (bp2build, json-module-graph) are not here and have their own bits below. 61 arguments []string 62 goma bool 63 environ *Environment 64 distDir string 65 buildDateTime string 66 logsPrefix string 67 68 // From the arguments 69 parallel int 70 keepGoing int 71 verbose bool 72 checkbuild bool 73 dist bool 74 jsonModuleGraph bool 75 queryview bool 76 reportMkMetrics bool // Collect and report mk2bp migration progress metrics. 77 soongDocs bool 78 skipConfig bool 79 skipKati bool 80 skipKatiNinja bool 81 skipSoong bool 82 skipNinja bool 83 skipSoongTests bool 84 searchApiDir bool // Scan the Android.bp files generated in out/api_surfaces 85 skipMetricsUpload bool 86 buildStartedTime int64 // For metrics-upload-only - manually specify a build-started time 87 buildFromSourceStub bool 88 ensureAllowlistIntegrity bool // For CI builds - make sure modules are mixed-built 89 90 // From the product config 91 katiArgs []string 92 ninjaArgs []string 93 katiSuffix string 94 targetDevice string 95 targetDeviceDir string 96 sandboxConfig *SandboxConfig 97 98 // Autodetected 99 totalRAM uint64 100 101 brokenDupRules bool 102 brokenUsesNetwork bool 103 brokenNinjaEnvVars []string 104 105 pathReplaced bool 106 107 // Set by multiproduct_kati 108 emptyNinjaFile bool 109 110 metricsUploader string 111 112 includeTags []string 113 sourceRootDirs []string 114 115 // Data source to write ninja weight list 116 ninjaWeightListSource NinjaWeightListSource 117 118 // This file is a detailed dump of all soong-defined modules for debugging purposes. 119 // There's quite a bit of overlap with module-info.json and soong module graph. We 120 // could consider merging them. 121 moduleDebugFile string 122} 123 124type NinjaWeightListSource uint 125 126const ( 127 // ninja doesn't use weight list. 128 NOT_USED NinjaWeightListSource = iota 129 // ninja uses weight list based on previous builds by ninja log 130 NINJA_LOG 131 // ninja thinks every task has the same weight. 132 EVENLY_DISTRIBUTED 133 // ninja uses an external custom weight list 134 EXTERNAL_FILE 135 // ninja uses a prioritized module list from Soong 136 HINT_FROM_SOONG 137 // If ninja log exists, use NINJA_LOG, if not, use HINT_FROM_SOONG instead. 138 // We can assume it is an incremental build if ninja log exists. 139 DEFAULT 140) 141const srcDirFileCheck = "build/soong/root.bp" 142 143var buildFiles = []string{"Android.mk", "Android.bp"} 144 145type BuildAction uint 146 147const ( 148 // Builds all of the modules and their dependencies of a specified directory, relative to the root 149 // directory of the source tree. 150 BUILD_MODULES_IN_A_DIRECTORY BuildAction = iota 151 152 // Builds all of the modules and their dependencies of a list of specified directories. All specified 153 // directories are relative to the root directory of the source tree. 154 BUILD_MODULES_IN_DIRECTORIES 155 156 // Build a list of specified modules. If none was specified, simply build the whole source tree. 157 BUILD_MODULES 158) 159 160// checkTopDir validates that the current directory is at the root directory of the source tree. 161func checkTopDir(ctx Context) { 162 if _, err := os.Stat(srcDirFileCheck); err != nil { 163 if os.IsNotExist(err) { 164 ctx.Fatalf("Current working directory must be the source tree. %q not found.", srcDirFileCheck) 165 } 166 ctx.Fatalln("Error verifying tree state:", err) 167 } 168} 169 170func loadEnvConfig(ctx Context, config *configImpl, bc string) error { 171 if bc == "" { 172 return nil 173 } 174 175 configDirs := []string{ 176 config.OutDir(), 177 os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG_DIR"), 178 envConfigDir, 179 } 180 for _, dir := range configDirs { 181 cfgFile := filepath.Join(os.Getenv("TOP"), dir, fmt.Sprintf("%s.%s", bc, jsonSuffix)) 182 envVarsJSON, err := ioutil.ReadFile(cfgFile) 183 if err != nil { 184 continue 185 } 186 ctx.Verbosef("Loading config file %v\n", cfgFile) 187 var envVars map[string]map[string]string 188 if err := json.Unmarshal(envVarsJSON, &envVars); err != nil { 189 fmt.Fprintf(os.Stderr, "Env vars config file %s did not parse correctly: %s", cfgFile, err.Error()) 190 continue 191 } 192 for k, v := range envVars["env"] { 193 if os.Getenv(k) != "" { 194 continue 195 } 196 config.environ.Set(k, v) 197 } 198 ctx.Verbosef("Finished loading config file %v\n", cfgFile) 199 break 200 } 201 202 return nil 203} 204 205func NewConfig(ctx Context, args ...string) Config { 206 ret := &configImpl{ 207 environ: OsEnvironment(), 208 sandboxConfig: &SandboxConfig{}, 209 ninjaWeightListSource: DEFAULT, 210 } 211 212 // Skip soong tests by default on Linux 213 if runtime.GOOS == "linux" { 214 ret.skipSoongTests = true 215 } 216 217 // Default matching ninja 218 ret.parallel = runtime.NumCPU() + 2 219 ret.keepGoing = 1 220 221 ret.totalRAM = detectTotalRAM(ctx) 222 ret.parseArgs(ctx, args) 223 224 if ret.ninjaWeightListSource == HINT_FROM_SOONG { 225 ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "always") 226 } else if ret.ninjaWeightListSource == DEFAULT { 227 defaultNinjaWeightListSource := NINJA_LOG 228 if _, err := os.Stat(filepath.Join(ret.OutDir(), ninjaLogFileName)); errors.Is(err, os.ErrNotExist) { 229 ctx.Verboseln("$OUT/.ninja_log doesn't exist, use HINT_FROM_SOONG instead") 230 defaultNinjaWeightListSource = HINT_FROM_SOONG 231 } else { 232 ctx.Verboseln("$OUT/.ninja_log exist, use NINJA_LOG") 233 } 234 ret.ninjaWeightListSource = defaultNinjaWeightListSource 235 // soong_build generates ninja hint depending on ninja log existence. 236 // Set it "depend" to avoid soong re-run due to env variable change. 237 ret.environ.Set("SOONG_GENERATES_NINJA_HINT", "depend") 238 } 239 240 // Make sure OUT_DIR is set appropriately 241 if outDir, ok := ret.environ.Get("OUT_DIR"); ok { 242 ret.environ.Set("OUT_DIR", filepath.Clean(outDir)) 243 } else { 244 outDir := "out" 245 if baseDir, ok := ret.environ.Get("OUT_DIR_COMMON_BASE"); ok { 246 if wd, err := os.Getwd(); err != nil { 247 ctx.Fatalln("Failed to get working directory:", err) 248 } else { 249 outDir = filepath.Join(baseDir, filepath.Base(wd)) 250 } 251 } 252 ret.environ.Set("OUT_DIR", outDir) 253 } 254 255 // loadEnvConfig needs to know what the OUT_DIR is, so it should 256 // be called after we determine the appropriate out directory. 257 bc := os.Getenv("ANDROID_BUILD_ENVIRONMENT_CONFIG") 258 259 if bc != "" { 260 if err := loadEnvConfig(ctx, ret, bc); err != nil { 261 ctx.Fatalln("Failed to parse env config files: %v", err) 262 } 263 if !ret.canSupportRBE() { 264 // Explicitly set USE_RBE env variable to false when we cannot run 265 // an RBE build to avoid ninja local execution pool issues. 266 ret.environ.Set("USE_RBE", "false") 267 } 268 } 269 270 if distDir, ok := ret.environ.Get("DIST_DIR"); ok { 271 ret.distDir = filepath.Clean(distDir) 272 } else { 273 ret.distDir = filepath.Join(ret.OutDir(), "dist") 274 } 275 276 if srcDirIsWritable, ok := ret.environ.Get("BUILD_BROKEN_SRC_DIR_IS_WRITABLE"); ok { 277 ret.sandboxConfig.SetSrcDirIsRO(srcDirIsWritable == "false") 278 } 279 280 if os.Getenv("GENERATE_SOONG_DEBUG") == "true" { 281 ret.moduleDebugFile, _ = filepath.Abs(shared.JoinPath(ret.SoongOutDir(), "soong-debug-info.json")) 282 } 283 284 ret.environ.Unset( 285 // We're already using it 286 "USE_SOONG_UI", 287 288 // We should never use GOROOT/GOPATH from the shell environment 289 "GOROOT", 290 "GOPATH", 291 292 // These should only come from Soong, not the environment. 293 "CLANG", 294 "CLANG_CXX", 295 "CCC_CC", 296 "CCC_CXX", 297 298 // Used by the goma compiler wrapper, but should only be set by 299 // gomacc 300 "GOMACC_PATH", 301 302 // We handle this above 303 "OUT_DIR_COMMON_BASE", 304 305 // This is handled above too, and set for individual commands later 306 "DIST_DIR", 307 308 // Variables that have caused problems in the past 309 "BASH_ENV", 310 "CDPATH", 311 "DISPLAY", 312 "GREP_OPTIONS", 313 "JAVAC", 314 "NDK_ROOT", 315 "POSIXLY_CORRECT", 316 317 // Drop make flags 318 "MAKEFLAGS", 319 "MAKELEVEL", 320 "MFLAGS", 321 322 // Set in envsetup.sh, reset in makefiles 323 "ANDROID_JAVA_TOOLCHAIN", 324 325 // Set by envsetup.sh, but shouldn't be used inside the build because envsetup.sh is optional 326 "ANDROID_BUILD_TOP", 327 "ANDROID_HOST_OUT", 328 "ANDROID_PRODUCT_OUT", 329 "ANDROID_HOST_OUT_TESTCASES", 330 "ANDROID_TARGET_OUT_TESTCASES", 331 "ANDROID_TOOLCHAIN", 332 "ANDROID_TOOLCHAIN_2ND_ARCH", 333 "ANDROID_DEV_SCRIPTS", 334 "ANDROID_EMULATOR_PREBUILTS", 335 "ANDROID_PRE_BUILD_PATHS", 336 337 // We read it here already, don't let others share in the fun 338 "GENERATE_SOONG_DEBUG", 339 ) 340 341 if ret.UseGoma() || ret.ForceUseGoma() { 342 ctx.Println("Goma for Android has been deprecated and replaced with RBE. See go/rbe_for_android for instructions on how to use RBE.") 343 ctx.Fatalln("USE_GOMA / FORCE_USE_GOMA flag is no longer supported.") 344 } 345 346 // Tell python not to spam the source tree with .pyc files. 347 ret.environ.Set("PYTHONDONTWRITEBYTECODE", "1") 348 349 tmpDir := absPath(ctx, ret.TempDir()) 350 ret.environ.Set("TMPDIR", tmpDir) 351 352 // Always set ASAN_SYMBOLIZER_PATH so that ASAN-based tools can symbolize any crashes 353 symbolizerPath := filepath.Join("prebuilts/clang/host", ret.HostPrebuiltTag(), 354 "llvm-binutils-stable/llvm-symbolizer") 355 ret.environ.Set("ASAN_SYMBOLIZER_PATH", absPath(ctx, symbolizerPath)) 356 357 // Precondition: the current directory is the top of the source tree 358 checkTopDir(ctx) 359 360 srcDir := absPath(ctx, ".") 361 if strings.ContainsRune(srcDir, ' ') { 362 ctx.Println("You are building in a directory whose absolute path contains a space character:") 363 ctx.Println() 364 ctx.Printf("%q\n", srcDir) 365 ctx.Println() 366 ctx.Fatalln("Directory names containing spaces are not supported") 367 } 368 369 ret.metricsUploader = GetMetricsUploader(srcDir, ret.environ) 370 371 if outDir := ret.OutDir(); strings.ContainsRune(outDir, ' ') { 372 ctx.Println("The absolute path of your output directory ($OUT_DIR) contains a space character:") 373 ctx.Println() 374 ctx.Printf("%q\n", outDir) 375 ctx.Println() 376 ctx.Fatalln("Directory names containing spaces are not supported") 377 } 378 379 if distDir := ret.RealDistDir(); strings.ContainsRune(distDir, ' ') { 380 ctx.Println("The absolute path of your dist directory ($DIST_DIR) contains a space character:") 381 ctx.Println() 382 ctx.Printf("%q\n", distDir) 383 ctx.Println() 384 ctx.Fatalln("Directory names containing spaces are not supported") 385 } 386 387 // Configure Java-related variables, including adding it to $PATH 388 java8Home := filepath.Join("prebuilts/jdk/jdk8", ret.HostPrebuiltTag()) 389 java21Home := filepath.Join("prebuilts/jdk/jdk21", ret.HostPrebuiltTag()) 390 javaHome := func() string { 391 if override, ok := ret.environ.Get("OVERRIDE_ANDROID_JAVA_HOME"); ok { 392 return override 393 } 394 if toolchain11, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN"); ok && toolchain11 != "true" { 395 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK11_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.") 396 } 397 if toolchain17, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN"); ok && toolchain17 != "true" { 398 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK17_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.") 399 } 400 if toolchain21, ok := ret.environ.Get("EXPERIMENTAL_USE_OPENJDK21_TOOLCHAIN"); ok && toolchain21 != "true" { 401 ctx.Fatalln("The environment variable EXPERIMENTAL_USE_OPENJDK21_TOOLCHAIN is no longer supported. An OpenJDK 21 toolchain is now the global default.") 402 } 403 return java21Home 404 }() 405 absJavaHome := absPath(ctx, javaHome) 406 407 ret.configureLocale(ctx) 408 409 newPath := []string{filepath.Join(absJavaHome, "bin")} 410 if path, ok := ret.environ.Get("PATH"); ok && path != "" { 411 newPath = append(newPath, path) 412 } 413 414 ret.environ.Unset("OVERRIDE_ANDROID_JAVA_HOME") 415 ret.environ.Set("JAVA_HOME", absJavaHome) 416 ret.environ.Set("ANDROID_JAVA_HOME", javaHome) 417 ret.environ.Set("ANDROID_JAVA8_HOME", java8Home) 418 ret.environ.Set("PATH", strings.Join(newPath, string(filepath.ListSeparator))) 419 420 // b/286885495, https://bugzilla.redhat.com/show_bug.cgi?id=2227130: some versions of Fedora include patches 421 // to unzip to enable zipbomb detection that incorrectly handle zip64 and data descriptors and fail on large 422 // zip files produced by soong_zip. Disable zipbomb detection. 423 ret.environ.Set("UNZIP_DISABLE_ZIPBOMB_DETECTION", "TRUE") 424 425 outDir := ret.OutDir() 426 buildDateTimeFile := filepath.Join(outDir, "build_date.txt") 427 if buildDateTime, ok := ret.environ.Get("BUILD_DATETIME"); ok && buildDateTime != "" { 428 ret.buildDateTime = buildDateTime 429 } else { 430 ret.buildDateTime = strconv.FormatInt(time.Now().Unix(), 10) 431 } 432 433 ret.environ.Set("BUILD_DATETIME_FILE", buildDateTimeFile) 434 435 if _, ok := ret.environ.Get("BUILD_USERNAME"); !ok { 436 username := "unknown" 437 if u, err := user.Current(); err == nil { 438 username = u.Username 439 } else { 440 ctx.Println("Failed to get current user:", err) 441 } 442 ret.environ.Set("BUILD_USERNAME", username) 443 } 444 445 if ret.UseRBE() { 446 for k, v := range getRBEVars(ctx, Config{ret}) { 447 ret.environ.Set(k, v) 448 } 449 } 450 451 c := Config{ret} 452 storeConfigMetrics(ctx, c) 453 return c 454} 455 456// NewBuildActionConfig returns a build configuration based on the build action. The arguments are 457// processed based on the build action and extracts any arguments that belongs to the build action. 458func NewBuildActionConfig(action BuildAction, dir string, ctx Context, args ...string) Config { 459 return NewConfig(ctx, getConfigArgs(action, dir, ctx, args)...) 460} 461 462// Prepare for getting make variables. For them to be accurate, we need to have 463// obtained PRODUCT_RELEASE_CONFIG_MAPS. 464// 465// Returns: 466// 467// Whether config should be called again. 468// 469// TODO: when converting product config to a declarative language, make sure 470// that PRODUCT_RELEASE_CONFIG_MAPS is properly handled as a separate step in 471// that process. 472func SetProductReleaseConfigMaps(ctx Context, config Config) bool { 473 ctx.BeginTrace(metrics.RunKati, "SetProductReleaseConfigMaps") 474 defer ctx.EndTrace() 475 476 if config.SkipConfig() { 477 // This duplicates the logic from Build to skip product config 478 // if the user has explicitly said to. 479 return false 480 } 481 482 releaseConfigVars := []string{ 483 "PRODUCT_RELEASE_CONFIG_MAPS", 484 } 485 486 origValue, _ := config.environ.Get("PRODUCT_RELEASE_CONFIG_MAPS") 487 // Get the PRODUCT_RELEASE_CONFIG_MAPS for this product, to avoid polluting the environment 488 // when we run product config to get the rest of the make vars. 489 releaseMapVars, err := dumpMakeVars(ctx, config, nil, releaseConfigVars, false, "") 490 if err != nil { 491 ctx.Fatalln("Error getting PRODUCT_RELEASE_CONFIG_MAPS:", err) 492 } 493 productReleaseConfigMaps := releaseMapVars["PRODUCT_RELEASE_CONFIG_MAPS"] 494 os.Setenv("PRODUCT_RELEASE_CONFIG_MAPS", productReleaseConfigMaps) 495 return origValue != productReleaseConfigMaps 496} 497 498// storeConfigMetrics selects a set of configuration information and store in 499// the metrics system for further analysis. 500func storeConfigMetrics(ctx Context, config Config) { 501 if ctx.Metrics == nil { 502 return 503 } 504 505 ctx.Metrics.BuildConfig(buildConfig(config)) 506 507 s := &smpb.SystemResourceInfo{ 508 TotalPhysicalMemory: proto.Uint64(config.TotalRAM()), 509 AvailableCpus: proto.Int32(int32(runtime.NumCPU())), 510 } 511 ctx.Metrics.SystemResourceInfo(s) 512} 513 514func getNinjaWeightListSourceInMetric(s NinjaWeightListSource) *smpb.BuildConfig_NinjaWeightListSource { 515 switch s { 516 case NINJA_LOG: 517 return smpb.BuildConfig_NINJA_LOG.Enum() 518 case EVENLY_DISTRIBUTED: 519 return smpb.BuildConfig_EVENLY_DISTRIBUTED.Enum() 520 case EXTERNAL_FILE: 521 return smpb.BuildConfig_EXTERNAL_FILE.Enum() 522 case HINT_FROM_SOONG: 523 return smpb.BuildConfig_HINT_FROM_SOONG.Enum() 524 default: 525 return smpb.BuildConfig_NOT_USED.Enum() 526 } 527} 528 529func buildConfig(config Config) *smpb.BuildConfig { 530 c := &smpb.BuildConfig{ 531 ForceUseGoma: proto.Bool(config.ForceUseGoma()), 532 UseGoma: proto.Bool(config.UseGoma()), 533 UseRbe: proto.Bool(config.UseRBE()), 534 NinjaWeightListSource: getNinjaWeightListSourceInMetric(config.NinjaWeightListSource()), 535 } 536 c.Targets = append(c.Targets, config.arguments...) 537 538 return c 539} 540 541// getConfigArgs processes the command arguments based on the build action and creates a set of new 542// arguments to be accepted by Config. 543func getConfigArgs(action BuildAction, dir string, ctx Context, args []string) []string { 544 // The next block of code verifies that the current directory is the root directory of the source 545 // tree. It then finds the relative path of dir based on the root directory of the source tree 546 // and verify that dir is inside of the source tree. 547 checkTopDir(ctx) 548 topDir, err := os.Getwd() 549 if err != nil { 550 ctx.Fatalf("Error retrieving top directory: %v", err) 551 } 552 dir, err = filepath.EvalSymlinks(dir) 553 if err != nil { 554 ctx.Fatalf("Unable to evaluate symlink of %s: %v", dir, err) 555 } 556 dir, err = filepath.Abs(dir) 557 if err != nil { 558 ctx.Fatalf("Unable to find absolute path %s: %v", dir, err) 559 } 560 relDir, err := filepath.Rel(topDir, dir) 561 if err != nil { 562 ctx.Fatalf("Unable to find relative path %s of %s: %v", relDir, topDir, err) 563 } 564 // If there are ".." in the path, it's not in the source tree. 565 if strings.Contains(relDir, "..") { 566 ctx.Fatalf("Directory %s is not under the source tree %s", dir, topDir) 567 } 568 569 configArgs := args[:] 570 571 // If the arguments contains GET-INSTALL-PATH, change the target name prefix from MODULES-IN- to 572 // GET-INSTALL-PATH-IN- to extract the installation path instead of building the modules. 573 targetNamePrefix := "MODULES-IN-" 574 if inList("GET-INSTALL-PATH", configArgs) { 575 targetNamePrefix = "GET-INSTALL-PATH-IN-" 576 configArgs = removeFromList("GET-INSTALL-PATH", configArgs) 577 } 578 579 var targets []string 580 581 switch action { 582 case BUILD_MODULES: 583 // No additional processing is required when building a list of specific modules or all modules. 584 case BUILD_MODULES_IN_A_DIRECTORY: 585 // If dir is the root source tree, all the modules are built of the source tree are built so 586 // no need to find the build file. 587 if topDir == dir { 588 break 589 } 590 591 buildFile := findBuildFile(ctx, relDir) 592 if buildFile == "" { 593 ctx.Fatalf("Build file not found for %s directory", relDir) 594 } 595 targets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} 596 case BUILD_MODULES_IN_DIRECTORIES: 597 newConfigArgs, dirs := splitArgs(configArgs) 598 configArgs = newConfigArgs 599 targets = getTargetsFromDirs(ctx, relDir, dirs, targetNamePrefix) 600 } 601 602 // Tidy only override all other specified targets. 603 tidyOnly := os.Getenv("WITH_TIDY_ONLY") 604 if tidyOnly == "true" || tidyOnly == "1" { 605 configArgs = append(configArgs, "tidy_only") 606 } else { 607 configArgs = append(configArgs, targets...) 608 } 609 610 return configArgs 611} 612 613// convertToTarget replaces "/" to "-" in dir and pre-append the targetNamePrefix to the target name. 614func convertToTarget(dir string, targetNamePrefix string) string { 615 return targetNamePrefix + strings.ReplaceAll(dir, "/", "-") 616} 617 618// hasBuildFile returns true if dir contains an Android build file. 619func hasBuildFile(ctx Context, dir string) bool { 620 for _, buildFile := range buildFiles { 621 _, err := os.Stat(filepath.Join(dir, buildFile)) 622 if err == nil { 623 return true 624 } 625 if !os.IsNotExist(err) { 626 ctx.Fatalf("Error retrieving the build file stats: %v", err) 627 } 628 } 629 return false 630} 631 632// findBuildFile finds a build file (makefile or blueprint file) by looking if there is a build file 633// in the current and any sub directory of dir. If a build file is not found, traverse the path 634// up by one directory and repeat again until either a build file is found or reached to the root 635// source tree. The returned filename of build file is "Android.mk". If one was not found, a blank 636// string is returned. 637func findBuildFile(ctx Context, dir string) string { 638 // If the string is empty or ".", assume it is top directory of the source tree. 639 if dir == "" || dir == "." { 640 return "" 641 } 642 643 found := false 644 for buildDir := dir; buildDir != "."; buildDir = filepath.Dir(buildDir) { 645 err := filepath.Walk(buildDir, func(path string, info os.FileInfo, err error) error { 646 if err != nil { 647 return err 648 } 649 if found { 650 return filepath.SkipDir 651 } 652 if info.IsDir() { 653 return nil 654 } 655 for _, buildFile := range buildFiles { 656 if info.Name() == buildFile { 657 found = true 658 return filepath.SkipDir 659 } 660 } 661 return nil 662 }) 663 if err != nil { 664 ctx.Fatalf("Error finding Android build file: %v", err) 665 } 666 667 if found { 668 return filepath.Join(buildDir, "Android.mk") 669 } 670 } 671 672 return "" 673} 674 675// splitArgs iterates over the arguments list and splits into two lists: arguments and directories. 676func splitArgs(args []string) (newArgs []string, dirs []string) { 677 specialArgs := map[string]bool{ 678 "showcommands": true, 679 "snod": true, 680 "dist": true, 681 "checkbuild": true, 682 } 683 684 newArgs = []string{} 685 dirs = []string{} 686 687 for _, arg := range args { 688 // It's a dash argument if it starts with "-" or it's a key=value pair, it's not a directory. 689 if strings.IndexRune(arg, '-') == 0 || strings.IndexRune(arg, '=') != -1 { 690 newArgs = append(newArgs, arg) 691 continue 692 } 693 694 if _, ok := specialArgs[arg]; ok { 695 newArgs = append(newArgs, arg) 696 continue 697 } 698 699 dirs = append(dirs, arg) 700 } 701 702 return newArgs, dirs 703} 704 705// getTargetsFromDirs iterates over the dirs list and creates a list of targets to build. If a 706// directory from the dirs list does not exist, a fatal error is raised. relDir is related to the 707// source root tree where the build action command was invoked. Each directory is validated if the 708// build file can be found and follows the format "dir1:target1,target2,...". Target is optional. 709func getTargetsFromDirs(ctx Context, relDir string, dirs []string, targetNamePrefix string) (targets []string) { 710 for _, dir := range dirs { 711 // The directory may have specified specific modules to build. ":" is the separator to separate 712 // the directory and the list of modules. 713 s := strings.Split(dir, ":") 714 l := len(s) 715 if l > 2 { // more than one ":" was specified. 716 ctx.Fatalf("%s not in proper directory:target1,target2,... format (\":\" was specified more than once)", dir) 717 } 718 719 dir = filepath.Join(relDir, s[0]) 720 if _, err := os.Stat(dir); err != nil { 721 ctx.Fatalf("couldn't find directory %s", dir) 722 } 723 724 // Verify that if there are any targets specified after ":". Each target is separated by ",". 725 var newTargets []string 726 if l == 2 && s[1] != "" { 727 newTargets = strings.Split(s[1], ",") 728 if inList("", newTargets) { 729 ctx.Fatalf("%s not in proper directory:target1,target2,... format", dir) 730 } 731 } 732 733 // If there are specified targets to build in dir, an android build file must exist for the one 734 // shot build. For the non-targets case, find the appropriate build file and build all the 735 // modules in dir (or the closest one in the dir path). 736 if len(newTargets) > 0 { 737 if !hasBuildFile(ctx, dir) { 738 ctx.Fatalf("Couldn't locate a build file from %s directory", dir) 739 } 740 } else { 741 buildFile := findBuildFile(ctx, dir) 742 if buildFile == "" { 743 ctx.Fatalf("Build file not found for %s directory", dir) 744 } 745 newTargets = []string{convertToTarget(filepath.Dir(buildFile), targetNamePrefix)} 746 } 747 748 targets = append(targets, newTargets...) 749 } 750 751 return targets 752} 753 754func (c *configImpl) parseArgs(ctx Context, args []string) { 755 for i := 0; i < len(args); i++ { 756 arg := strings.TrimSpace(args[i]) 757 if arg == "showcommands" { 758 c.verbose = true 759 } else if arg == "--empty-ninja-file" { 760 c.emptyNinjaFile = true 761 } else if arg == "--skip-ninja" { 762 c.skipNinja = true 763 } else if arg == "--skip-make" { 764 // TODO(ccross): deprecate this, it has confusing behaviors. It doesn't run kati, 765 // but it does run a Kati ninja file if the .kati_enabled marker file was created 766 // by a previous build. 767 c.skipConfig = true 768 c.skipKati = true 769 } else if arg == "--soong-only" { 770 c.skipKati = true 771 c.skipKatiNinja = true 772 } else if arg == "--config-only" { 773 c.skipKati = true 774 c.skipKatiNinja = true 775 c.skipSoong = true 776 } else if arg == "--skip-config" { 777 c.skipConfig = true 778 } else if arg == "--skip-soong-tests" { 779 c.skipSoongTests = true 780 } else if arg == "--no-skip-soong-tests" { 781 c.skipSoongTests = false 782 } else if arg == "--skip-metrics-upload" { 783 c.skipMetricsUpload = true 784 } else if arg == "--mk-metrics" { 785 c.reportMkMetrics = true 786 } else if arg == "--search-api-dir" { 787 c.searchApiDir = true 788 } else if strings.HasPrefix(arg, "--ninja_weight_source=") { 789 source := strings.TrimPrefix(arg, "--ninja_weight_source=") 790 if source == "ninja_log" { 791 c.ninjaWeightListSource = NINJA_LOG 792 } else if source == "evenly_distributed" { 793 c.ninjaWeightListSource = EVENLY_DISTRIBUTED 794 } else if source == "not_used" { 795 c.ninjaWeightListSource = NOT_USED 796 } else if source == "soong" { 797 c.ninjaWeightListSource = HINT_FROM_SOONG 798 } else if strings.HasPrefix(source, "file,") { 799 c.ninjaWeightListSource = EXTERNAL_FILE 800 filePath := strings.TrimPrefix(source, "file,") 801 err := validateNinjaWeightList(filePath) 802 if err != nil { 803 ctx.Fatalf("Malformed weight list from %s: %s", filePath, err) 804 } 805 _, err = copyFile(filePath, filepath.Join(c.OutDir(), ".ninja_weight_list")) 806 if err != nil { 807 ctx.Fatalf("Error to copy ninja weight list from %s: %s", filePath, err) 808 } 809 } else { 810 ctx.Fatalf("unknown option for ninja_weight_source: %s", source) 811 } 812 } else if arg == "--build-from-source-stub" { 813 c.buildFromSourceStub = true 814 } else if strings.HasPrefix(arg, "--build-command=") { 815 buildCmd := strings.TrimPrefix(arg, "--build-command=") 816 // remove quotations 817 buildCmd = strings.TrimPrefix(buildCmd, "\"") 818 buildCmd = strings.TrimSuffix(buildCmd, "\"") 819 ctx.Metrics.SetBuildCommand([]string{buildCmd}) 820 } else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") { 821 buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=") 822 val, err := strconv.ParseInt(buildTimeStr, 10, 64) 823 if err == nil { 824 c.buildStartedTime = val 825 } else { 826 ctx.Fatalf("Error parsing build-time-started-unix-millis", err) 827 } 828 } else if arg == "--ensure-allowlist-integrity" { 829 c.ensureAllowlistIntegrity = true 830 } else if len(arg) > 0 && arg[0] == '-' { 831 parseArgNum := func(def int) int { 832 if len(arg) > 2 { 833 p, err := strconv.ParseUint(arg[2:], 10, 31) 834 if err != nil { 835 ctx.Fatalf("Failed to parse %q: %v", arg, err) 836 } 837 return int(p) 838 } else if i+1 < len(args) { 839 p, err := strconv.ParseUint(args[i+1], 10, 31) 840 if err == nil { 841 i++ 842 return int(p) 843 } 844 } 845 return def 846 } 847 848 if len(arg) > 1 && arg[1] == 'j' { 849 c.parallel = parseArgNum(c.parallel) 850 } else if len(arg) > 1 && arg[1] == 'k' { 851 c.keepGoing = parseArgNum(0) 852 } else { 853 ctx.Fatalln("Unknown option:", arg) 854 } 855 } else if k, v, ok := decodeKeyValue(arg); ok && len(k) > 0 { 856 if k == "OUT_DIR" { 857 ctx.Fatalln("OUT_DIR may only be set in the environment, not as a command line option.") 858 } 859 c.environ.Set(k, v) 860 } else if arg == "dist" { 861 c.dist = true 862 } else if arg == "json-module-graph" { 863 c.jsonModuleGraph = true 864 } else if arg == "queryview" { 865 c.queryview = true 866 } else if arg == "soong_docs" { 867 c.soongDocs = true 868 } else { 869 if arg == "checkbuild" { 870 c.checkbuild = true 871 } 872 c.arguments = append(c.arguments, arg) 873 } 874 } 875} 876 877func validateNinjaWeightList(weightListFilePath string) (err error) { 878 data, err := os.ReadFile(weightListFilePath) 879 if err != nil { 880 return 881 } 882 lines := strings.Split(strings.TrimSpace(string(data)), "\n") 883 for _, line := range lines { 884 fields := strings.Split(line, ",") 885 if len(fields) != 2 { 886 return fmt.Errorf("wrong format, each line should have two fields, but '%s'", line) 887 } 888 _, err = strconv.Atoi(fields[1]) 889 if err != nil { 890 return 891 } 892 } 893 return 894} 895 896func (c *configImpl) configureLocale(ctx Context) { 897 cmd := Command(ctx, Config{c}, "locale", "locale", "-a") 898 output, err := cmd.Output() 899 900 var locales []string 901 if err == nil { 902 locales = strings.Split(string(output), "\n") 903 } else { 904 // If we're unable to list the locales, let's assume en_US.UTF-8 905 locales = []string{"en_US.UTF-8"} 906 ctx.Verbosef("Failed to list locales (%q), falling back to %q", err, locales) 907 } 908 909 // gettext uses LANGUAGE, which is passed directly through 910 911 // For LANG and LC_*, only preserve the evaluated version of 912 // LC_MESSAGES 913 userLang := "" 914 if lc_all, ok := c.environ.Get("LC_ALL"); ok { 915 userLang = lc_all 916 } else if lc_messages, ok := c.environ.Get("LC_MESSAGES"); ok { 917 userLang = lc_messages 918 } else if lang, ok := c.environ.Get("LANG"); ok { 919 userLang = lang 920 } 921 922 c.environ.UnsetWithPrefix("LC_") 923 924 if userLang != "" { 925 c.environ.Set("LC_MESSAGES", userLang) 926 } 927 928 // The for LANG, use C.UTF-8 if it exists (Debian currently, proposed 929 // for others) 930 if inList("C.UTF-8", locales) { 931 c.environ.Set("LANG", "C.UTF-8") 932 } else if inList("C.utf8", locales) { 933 // These normalize to the same thing 934 c.environ.Set("LANG", "C.UTF-8") 935 } else if inList("en_US.UTF-8", locales) { 936 c.environ.Set("LANG", "en_US.UTF-8") 937 } else if inList("en_US.utf8", locales) { 938 // These normalize to the same thing 939 c.environ.Set("LANG", "en_US.UTF-8") 940 } else { 941 ctx.Fatalln("System doesn't support either C.UTF-8 or en_US.UTF-8") 942 } 943} 944 945func (c *configImpl) Environment() *Environment { 946 return c.environ 947} 948 949func (c *configImpl) Arguments() []string { 950 return c.arguments 951} 952 953func (c *configImpl) SoongBuildInvocationNeeded() bool { 954 if len(c.Arguments()) > 0 { 955 // Explicit targets requested that are not special targets like b2pbuild 956 // or the JSON module graph 957 return true 958 } 959 960 if !c.JsonModuleGraph() && !c.Queryview() && !c.SoongDocs() { 961 // Command line was empty, the default Ninja target is built 962 return true 963 } 964 965 if c.Dist() { 966 return true 967 } 968 969 // build.ninja doesn't need to be generated 970 return false 971} 972 973func (c *configImpl) OutDir() string { 974 if outDir, ok := c.environ.Get("OUT_DIR"); ok { 975 return outDir 976 } 977 return "out" 978} 979 980func (c *configImpl) DistDir() string { 981 return c.distDir 982} 983 984func (c *configImpl) RealDistDir() string { 985 return c.distDir 986} 987 988func (c *configImpl) NinjaArgs() []string { 989 if c.skipKati { 990 return c.arguments 991 } 992 return c.ninjaArgs 993} 994 995func (c *configImpl) SoongOutDir() string { 996 return filepath.Join(c.OutDir(), "soong") 997} 998 999func (c *configImpl) ApiSurfacesOutDir() string { 1000 return filepath.Join(c.OutDir(), "api_surfaces") 1001} 1002 1003func (c *configImpl) PrebuiltOS() string { 1004 switch runtime.GOOS { 1005 case "linux": 1006 return "linux-x86" 1007 case "darwin": 1008 return "darwin-x86" 1009 default: 1010 panic("Unknown GOOS") 1011 } 1012} 1013 1014func (c *configImpl) HostToolDir() string { 1015 if c.SkipKatiNinja() { 1016 return filepath.Join(c.SoongOutDir(), "host", c.PrebuiltOS(), "bin") 1017 } else { 1018 return filepath.Join(c.OutDir(), "host", c.PrebuiltOS(), "bin") 1019 } 1020} 1021 1022func (c *configImpl) NamedGlobFile(name string) string { 1023 return shared.JoinPath(c.SoongOutDir(), "globs-"+name+".ninja") 1024} 1025 1026func (c *configImpl) UsedEnvFile(tag string) string { 1027 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { 1028 return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+v+"."+tag) 1029 } 1030 return shared.JoinPath(c.SoongOutDir(), usedEnvFile+"."+tag) 1031} 1032 1033func (c *configImpl) SoongDocsHtml() string { 1034 return shared.JoinPath(c.SoongOutDir(), "docs/soong_build.html") 1035} 1036 1037func (c *configImpl) QueryviewMarkerFile() string { 1038 return shared.JoinPath(c.SoongOutDir(), "queryview.marker") 1039} 1040 1041func (c *configImpl) ModuleGraphFile() string { 1042 return shared.JoinPath(c.SoongOutDir(), "module-graph.json") 1043} 1044 1045func (c *configImpl) ModuleActionsFile() string { 1046 return shared.JoinPath(c.SoongOutDir(), "module-actions.json") 1047} 1048 1049func (c *configImpl) TempDir() string { 1050 return shared.TempDirForOutDir(c.SoongOutDir()) 1051} 1052 1053func (c *configImpl) FileListDir() string { 1054 return filepath.Join(c.OutDir(), ".module_paths") 1055} 1056 1057func (c *configImpl) KatiSuffix() string { 1058 if c.katiSuffix != "" { 1059 return c.katiSuffix 1060 } 1061 panic("SetKatiSuffix has not been called") 1062} 1063 1064// Checkbuild returns true if "checkbuild" was one of the build goals, which means that the 1065// user is interested in additional checks at the expense of build time. 1066func (c *configImpl) Checkbuild() bool { 1067 return c.checkbuild 1068} 1069 1070func (c *configImpl) Dist() bool { 1071 return c.dist 1072} 1073 1074func (c *configImpl) JsonModuleGraph() bool { 1075 return c.jsonModuleGraph 1076} 1077 1078func (c *configImpl) Queryview() bool { 1079 return c.queryview 1080} 1081 1082func (c *configImpl) SoongDocs() bool { 1083 return c.soongDocs 1084} 1085 1086func (c *configImpl) IsVerbose() bool { 1087 return c.verbose 1088} 1089 1090func (c *configImpl) NinjaWeightListSource() NinjaWeightListSource { 1091 return c.ninjaWeightListSource 1092} 1093 1094func (c *configImpl) SkipKati() bool { 1095 return c.skipKati 1096} 1097 1098func (c *configImpl) SkipKatiNinja() bool { 1099 return c.skipKatiNinja 1100} 1101 1102func (c *configImpl) SkipSoong() bool { 1103 return c.skipSoong 1104} 1105 1106func (c *configImpl) SkipNinja() bool { 1107 return c.skipNinja 1108} 1109 1110func (c *configImpl) SetSkipNinja(v bool) { 1111 c.skipNinja = v 1112} 1113 1114func (c *configImpl) SkipConfig() bool { 1115 return c.skipConfig 1116} 1117 1118func (c *configImpl) BuildFromTextStub() bool { 1119 return !c.buildFromSourceStub 1120} 1121 1122func (c *configImpl) TargetProduct() string { 1123 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { 1124 return v 1125 } 1126 panic("TARGET_PRODUCT is not defined") 1127} 1128 1129func (c *configImpl) TargetProductOrErr() (string, error) { 1130 if v, ok := c.environ.Get("TARGET_PRODUCT"); ok { 1131 return v, nil 1132 } 1133 return "", fmt.Errorf("TARGET_PRODUCT is not defined") 1134} 1135 1136func (c *configImpl) TargetDevice() string { 1137 return c.targetDevice 1138} 1139 1140func (c *configImpl) SetTargetDevice(device string) { 1141 c.targetDevice = device 1142} 1143 1144func (c *configImpl) TargetBuildVariant() string { 1145 if v, ok := c.environ.Get("TARGET_BUILD_VARIANT"); ok { 1146 return v 1147 } 1148 panic("TARGET_BUILD_VARIANT is not defined") 1149} 1150 1151func (c *configImpl) KatiArgs() []string { 1152 return c.katiArgs 1153} 1154 1155func (c *configImpl) Parallel() int { 1156 return c.parallel 1157} 1158 1159func (c *configImpl) GetSourceRootDirs() []string { 1160 return c.sourceRootDirs 1161} 1162 1163func (c *configImpl) SetSourceRootDirs(i []string) { 1164 c.sourceRootDirs = i 1165} 1166 1167func (c *configImpl) GetLogsPrefix() string { 1168 return c.logsPrefix 1169} 1170 1171func (c *configImpl) SetLogsPrefix(prefix string) { 1172 c.logsPrefix = prefix 1173} 1174 1175func (c *configImpl) HighmemParallel() int { 1176 if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok { 1177 return i 1178 } 1179 1180 const minMemPerHighmemProcess = 8 * 1024 * 1024 * 1024 1181 parallel := c.Parallel() 1182 if c.UseRemoteBuild() { 1183 // Ninja doesn't support nested pools, and when remote builds are enabled the total ninja parallelism 1184 // is set very high (i.e. 500). Using a large value here would cause the total number of running jobs 1185 // to be the sum of the sizes of the local and highmem pools, which will cause extra CPU contention. 1186 // Return 1/16th of the size of the local pool, rounding up. 1187 return (parallel + 15) / 16 1188 } else if c.totalRAM == 0 { 1189 // Couldn't detect the total RAM, don't restrict highmem processes. 1190 return parallel 1191 } else if c.totalRAM <= 16*1024*1024*1024 { 1192 // Less than 16GB of ram, restrict to 1 highmem processes 1193 return 1 1194 } else if c.totalRAM <= 32*1024*1024*1024 { 1195 // Less than 32GB of ram, restrict to 2 highmem processes 1196 return 2 1197 } else if p := int(c.totalRAM / minMemPerHighmemProcess); p < parallel { 1198 // If less than 8GB total RAM per process, reduce the number of highmem processes 1199 return p 1200 } 1201 // No restriction on highmem processes 1202 return parallel 1203} 1204 1205func (c *configImpl) TotalRAM() uint64 { 1206 return c.totalRAM 1207} 1208 1209// ForceUseGoma determines whether we should override Goma deprecation 1210// and use Goma for the current build or not. 1211func (c *configImpl) ForceUseGoma() bool { 1212 if v, ok := c.environ.Get("FORCE_USE_GOMA"); ok { 1213 v = strings.TrimSpace(v) 1214 if v != "" && v != "false" { 1215 return true 1216 } 1217 } 1218 return false 1219} 1220 1221func (c *configImpl) UseGoma() bool { 1222 if v, ok := c.environ.Get("USE_GOMA"); ok { 1223 v = strings.TrimSpace(v) 1224 if v != "" && v != "false" { 1225 return true 1226 } 1227 } 1228 return false 1229} 1230 1231func (c *configImpl) StartGoma() bool { 1232 if !c.UseGoma() { 1233 return false 1234 } 1235 1236 if v, ok := c.environ.Get("NOSTART_GOMA"); ok { 1237 v = strings.TrimSpace(v) 1238 if v != "" && v != "false" { 1239 return false 1240 } 1241 } 1242 return true 1243} 1244 1245func (c *configImpl) canSupportRBE() bool { 1246 // Do not use RBE with prod credentials in scenarios when stubby doesn't exist, since 1247 // its unlikely that we will be able to obtain necessary creds without stubby. 1248 authType, _ := c.rbeAuth() 1249 if !c.StubbyExists() && strings.Contains(authType, "use_google_prod_creds") { 1250 return false 1251 } 1252 return true 1253} 1254 1255func (c *configImpl) UseRBE() bool { 1256 // These alternate modes of running Soong do not use RBE / reclient. 1257 if c.Queryview() || c.JsonModuleGraph() { 1258 return false 1259 } 1260 1261 if !c.canSupportRBE() { 1262 return false 1263 } 1264 1265 if v, ok := c.Environment().Get("USE_RBE"); ok { 1266 v = strings.TrimSpace(v) 1267 if v != "" && v != "false" { 1268 return true 1269 } 1270 } 1271 return false 1272} 1273 1274func (c *configImpl) StartRBE() bool { 1275 if !c.UseRBE() { 1276 return false 1277 } 1278 1279 if v, ok := c.environ.Get("NOSTART_RBE"); ok { 1280 v = strings.TrimSpace(v) 1281 if v != "" && v != "false" { 1282 return false 1283 } 1284 } 1285 return true 1286} 1287 1288func (c *configImpl) rbeProxyLogsDir() string { 1289 for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} { 1290 if v, ok := c.environ.Get(f); ok { 1291 return v 1292 } 1293 } 1294 return c.rbeTmpDir() 1295} 1296 1297func (c *configImpl) rbeDownloadTmpDir() string { 1298 for _, f := range []string{"RBE_download_tmp_dir", "FLAG_download_tmp_dir"} { 1299 if v, ok := c.environ.Get(f); ok { 1300 return v 1301 } 1302 } 1303 return c.rbeTmpDir() 1304} 1305 1306func (c *configImpl) rbeTmpDir() string { 1307 buildTmpDir := shared.TempDirForOutDir(c.SoongOutDir()) 1308 return filepath.Join(buildTmpDir, "rbe") 1309} 1310 1311func (c *configImpl) rbeCacheDir() string { 1312 for _, f := range []string{"RBE_cache_dir", "FLAG_cache_dir"} { 1313 if v, ok := c.environ.Get(f); ok { 1314 return v 1315 } 1316 } 1317 return shared.JoinPath(c.SoongOutDir(), "rbe") 1318} 1319 1320func (c *configImpl) shouldCleanupRBELogsDir() bool { 1321 // Perform a log directory cleanup only when the log directory 1322 // is auto created by the build rather than user-specified. 1323 for _, f := range []string{"RBE_proxy_log_dir", "FLAG_output_dir"} { 1324 if _, ok := c.environ.Get(f); ok { 1325 return false 1326 } 1327 } 1328 return true 1329} 1330 1331func (c *configImpl) rbeExecRoot() string { 1332 for _, f := range []string{"RBE_exec_root", "FLAG_exec_root"} { 1333 if v, ok := c.environ.Get(f); ok { 1334 return v 1335 } 1336 } 1337 wd, err := os.Getwd() 1338 if err != nil { 1339 return "" 1340 } 1341 return wd 1342} 1343 1344func (c *configImpl) rbeDir() string { 1345 if v, ok := c.environ.Get("RBE_DIR"); ok { 1346 return v 1347 } 1348 return "prebuilts/remoteexecution-client/live/" 1349} 1350 1351func (c *configImpl) rbeReproxy() string { 1352 for _, f := range []string{"RBE_re_proxy", "FLAG_re_proxy"} { 1353 if v, ok := c.environ.Get(f); ok { 1354 return v 1355 } 1356 } 1357 return filepath.Join(c.rbeDir(), "reproxy") 1358} 1359 1360func (c *configImpl) rbeAuth() (string, string) { 1361 credFlags := []string{ 1362 "use_application_default_credentials", 1363 "use_gce_credentials", 1364 "credential_file", 1365 "use_google_prod_creds", 1366 } 1367 for _, cf := range credFlags { 1368 for _, f := range []string{"RBE_" + cf, "FLAG_" + cf} { 1369 if v, ok := c.environ.Get(f); ok { 1370 v = strings.TrimSpace(v) 1371 if v != "" && v != "false" && v != "0" { 1372 return "RBE_" + cf, v 1373 } 1374 } 1375 } 1376 } 1377 return "RBE_use_application_default_credentials", "true" 1378} 1379 1380func (c *configImpl) rbeSockAddr(dir string) (string, error) { 1381 // Absolute path socket addresses have a prefix of //. This should 1382 // be included in the length limit. 1383 maxNameLen := len(syscall.RawSockaddrUnix{}.Path) - 2 1384 base := fmt.Sprintf("reproxy_%v.sock", rbeRandPrefix) 1385 1386 name := filepath.Join(dir, base) 1387 if len(name) < maxNameLen { 1388 return name, nil 1389 } 1390 1391 name = filepath.Join("/tmp", base) 1392 if len(name) < maxNameLen { 1393 return name, nil 1394 } 1395 1396 return "", fmt.Errorf("cannot generate a proxy socket address shorter than the limit of %v", maxNameLen) 1397} 1398 1399// IsGooglerEnvironment returns true if the current build is running 1400// on a Google developer machine and false otherwise. 1401func (c *configImpl) IsGooglerEnvironment() bool { 1402 cf := "ANDROID_BUILD_ENVIRONMENT_CONFIG" 1403 if v, ok := c.environ.Get(cf); ok { 1404 return v == "googler" 1405 } 1406 return false 1407} 1408 1409// GoogleProdCredsExist determine whether credentials exist on the 1410// Googler machine to use remote execution. 1411func (c *configImpl) GoogleProdCredsExist() bool { 1412 if googleProdCredsExistCache { 1413 return googleProdCredsExistCache 1414 } 1415 if _, err := exec.Command("/usr/bin/gcertstatus", "-nocheck_ssh").Output(); err != nil { 1416 return false 1417 } 1418 googleProdCredsExistCache = true 1419 return true 1420} 1421 1422// UseRemoteBuild indicates whether to use a remote build acceleration system 1423// to speed up the build. 1424func (c *configImpl) UseRemoteBuild() bool { 1425 return c.UseGoma() || c.UseRBE() 1426} 1427 1428// StubbyExists checks whether the stubby binary exists on the machine running 1429// the build. 1430func (c *configImpl) StubbyExists() bool { 1431 if _, err := exec.LookPath("stubby"); err != nil { 1432 return false 1433 } 1434 return true 1435} 1436 1437// RemoteParallel controls how many remote jobs (i.e., commands which contain 1438// gomacc) are run in parallel. Note the parallelism of all other jobs is 1439// still limited by Parallel() 1440func (c *configImpl) RemoteParallel() int { 1441 if !c.UseRemoteBuild() { 1442 return 0 1443 } 1444 if i, ok := c.environ.GetInt("NINJA_REMOTE_NUM_JOBS"); ok { 1445 return i 1446 } 1447 return 500 1448} 1449 1450func (c *configImpl) SetKatiArgs(args []string) { 1451 c.katiArgs = args 1452} 1453 1454func (c *configImpl) SetNinjaArgs(args []string) { 1455 c.ninjaArgs = args 1456} 1457 1458func (c *configImpl) SetKatiSuffix(suffix string) { 1459 c.katiSuffix = suffix 1460} 1461 1462func (c *configImpl) LastKatiSuffixFile() string { 1463 return filepath.Join(c.OutDir(), "last_kati_suffix") 1464} 1465 1466func (c *configImpl) HasKatiSuffix() bool { 1467 return c.katiSuffix != "" 1468} 1469 1470func (c *configImpl) KatiEnvFile() string { 1471 return filepath.Join(c.OutDir(), "env"+c.KatiSuffix()+".sh") 1472} 1473 1474func (c *configImpl) KatiBuildNinjaFile() string { 1475 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiBuildSuffix+".ninja") 1476} 1477 1478func (c *configImpl) KatiPackageNinjaFile() string { 1479 return filepath.Join(c.OutDir(), "build"+c.KatiSuffix()+katiPackageSuffix+".ninja") 1480} 1481 1482func (c *configImpl) SoongVarsFile() string { 1483 targetProduct, err := c.TargetProductOrErr() 1484 if err != nil { 1485 return filepath.Join(c.SoongOutDir(), "soong.variables") 1486 } else { 1487 return filepath.Join(c.SoongOutDir(), "soong."+targetProduct+".variables") 1488 } 1489} 1490 1491func (c *configImpl) SoongNinjaFile() string { 1492 targetProduct, err := c.TargetProductOrErr() 1493 if err != nil { 1494 return filepath.Join(c.SoongOutDir(), "build.ninja") 1495 } else { 1496 return filepath.Join(c.SoongOutDir(), "build."+targetProduct+".ninja") 1497 } 1498} 1499 1500func (c *configImpl) CombinedNinjaFile() string { 1501 if c.katiSuffix == "" { 1502 return filepath.Join(c.OutDir(), "combined.ninja") 1503 } 1504 return filepath.Join(c.OutDir(), "combined"+c.KatiSuffix()+".ninja") 1505} 1506 1507func (c *configImpl) SoongAndroidMk() string { 1508 return filepath.Join(c.SoongOutDir(), "Android-"+c.TargetProduct()+".mk") 1509} 1510 1511func (c *configImpl) SoongMakeVarsMk() string { 1512 return filepath.Join(c.SoongOutDir(), "make_vars-"+c.TargetProduct()+".mk") 1513} 1514 1515func (c *configImpl) SoongBuildMetrics() string { 1516 return filepath.Join(c.LogsDir(), "soong_build_metrics.pb") 1517} 1518 1519func (c *configImpl) ProductOut() string { 1520 return filepath.Join(c.OutDir(), "target", "product", c.TargetDevice()) 1521} 1522 1523func (c *configImpl) DevicePreviousProductConfig() string { 1524 return filepath.Join(c.ProductOut(), "previous_build_config.mk") 1525} 1526 1527func (c *configImpl) KatiPackageMkDir() string { 1528 return filepath.Join(c.ProductOut(), "obj", "CONFIG", "kati_packaging") 1529} 1530 1531func (c *configImpl) hostOutRoot() string { 1532 return filepath.Join(c.OutDir(), "host") 1533} 1534 1535func (c *configImpl) HostOut() string { 1536 return filepath.Join(c.hostOutRoot(), c.HostPrebuiltTag()) 1537} 1538 1539// This probably needs to be multi-valued, so not exporting it for now 1540func (c *configImpl) hostCrossOut() string { 1541 if runtime.GOOS == "linux" { 1542 return filepath.Join(c.hostOutRoot(), "windows-x86") 1543 } else { 1544 return "" 1545 } 1546} 1547 1548func (c *configImpl) HostPrebuiltTag() string { 1549 if runtime.GOOS == "linux" { 1550 return "linux-x86" 1551 } else if runtime.GOOS == "darwin" { 1552 return "darwin-x86" 1553 } else { 1554 panic("Unsupported OS") 1555 } 1556} 1557 1558func (c *configImpl) PrebuiltBuildTool(name string) string { 1559 if v, ok := c.environ.Get("SANITIZE_HOST"); ok { 1560 if sanitize := strings.Fields(v); inList("address", sanitize) { 1561 asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name) 1562 if _, err := os.Stat(asan); err == nil { 1563 return asan 1564 } 1565 } 1566 } 1567 return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name) 1568} 1569 1570func (c *configImpl) SetBuildBrokenDupRules(val bool) { 1571 c.brokenDupRules = val 1572} 1573 1574func (c *configImpl) BuildBrokenDupRules() bool { 1575 return c.brokenDupRules 1576} 1577 1578func (c *configImpl) SetBuildBrokenUsesNetwork(val bool) { 1579 c.brokenUsesNetwork = val 1580} 1581 1582func (c *configImpl) BuildBrokenUsesNetwork() bool { 1583 return c.brokenUsesNetwork 1584} 1585 1586func (c *configImpl) SetBuildBrokenNinjaUsesEnvVars(val []string) { 1587 c.brokenNinjaEnvVars = val 1588} 1589 1590func (c *configImpl) BuildBrokenNinjaUsesEnvVars() []string { 1591 return c.brokenNinjaEnvVars 1592} 1593 1594func (c *configImpl) SetTargetDeviceDir(dir string) { 1595 c.targetDeviceDir = dir 1596} 1597 1598func (c *configImpl) TargetDeviceDir() string { 1599 return c.targetDeviceDir 1600} 1601 1602func (c *configImpl) BuildDateTime() string { 1603 return c.buildDateTime 1604} 1605 1606func (c *configImpl) MetricsUploaderApp() string { 1607 return c.metricsUploader 1608} 1609 1610// LogsDir returns the absolute path to the logs directory where build log and 1611// metrics files are located. By default, the logs directory is the out 1612// directory. If the argument dist is specified, the logs directory 1613// is <dist_dir>/logs. 1614func (c *configImpl) LogsDir() string { 1615 dir := c.OutDir() 1616 if c.Dist() { 1617 // Always write logs to the real dist dir, even if Bazel is using a rigged dist dir for other files 1618 dir = filepath.Join(c.RealDistDir(), "logs") 1619 } 1620 absDir, err := filepath.Abs(dir) 1621 if err != nil { 1622 fmt.Fprintf(os.Stderr, "\nError making log dir '%s' absolute: %s\n", dir, err.Error()) 1623 os.Exit(1) 1624 } 1625 return absDir 1626} 1627 1628// MkFileMetrics returns the file path for make-related metrics. 1629func (c *configImpl) MkMetrics() string { 1630 return filepath.Join(c.LogsDir(), "mk_metrics.pb") 1631} 1632 1633func (c *configImpl) SetEmptyNinjaFile(v bool) { 1634 c.emptyNinjaFile = v 1635} 1636 1637func (c *configImpl) EmptyNinjaFile() bool { 1638 return c.emptyNinjaFile 1639} 1640 1641func (c *configImpl) SkipMetricsUpload() bool { 1642 return c.skipMetricsUpload 1643} 1644 1645func (c *configImpl) EnsureAllowlistIntegrity() bool { 1646 return c.ensureAllowlistIntegrity 1647} 1648 1649// Returns a Time object if one was passed via a command-line flag. 1650// Otherwise returns the passed default. 1651func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time { 1652 if c.buildStartedTime == 0 { 1653 return defaultTime 1654 } 1655 return time.UnixMilli(c.buildStartedTime) 1656} 1657 1658func GetMetricsUploader(topDir string, env *Environment) string { 1659 if p, ok := env.Get("METRICS_UPLOADER"); ok { 1660 metricsUploader := filepath.Join(topDir, p) 1661 if _, err := os.Stat(metricsUploader); err == nil { 1662 return metricsUploader 1663 } 1664 } 1665 1666 return "" 1667} 1668