1// Copyright 2015 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
18	"bytes"
19	"errors"
20	"flag"
21	"fmt"
22	"os"
23	"path/filepath"
24	"strings"
25	"time"
26
27	"android/soong/android"
28	"android/soong/android/allowlists"
29	"android/soong/bp2build"
30	"android/soong/shared"
31
32	"github.com/google/blueprint"
33	"github.com/google/blueprint/bootstrap"
34	"github.com/google/blueprint/deptools"
35	"github.com/google/blueprint/metrics"
36	androidProtobuf "google.golang.org/protobuf/android"
37)
38
39var (
40	topDir           string
41	availableEnvFile string
42	usedEnvFile      string
43
44	globFile    string
45	globListDir string
46	delveListen string
47	delvePath   string
48
49	cmdlineArgs android.CmdArgs
50)
51
52func init() {
53	// Flags that make sense in every mode
54	flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree")
55	flag.StringVar(&cmdlineArgs.SoongOutDir, "soong_out", "", "Soong output directory (usually $TOP/out/soong)")
56	flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables")
57	flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables")
58	flag.StringVar(&globFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output")
59	flag.StringVar(&globListDir, "globListDir", "", "the directory containing the glob list files")
60	flag.StringVar(&cmdlineArgs.OutDir, "out", "", "the ninja builddir directory")
61	flag.StringVar(&cmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse")
62
63	// Debug flags
64	flag.StringVar(&delveListen, "delve_listen", "", "Delve port to listen on for debugging")
65	flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set")
66	flag.StringVar(&cmdlineArgs.Cpuprofile, "cpuprofile", "", "write cpu profile to file")
67	flag.StringVar(&cmdlineArgs.TraceFile, "trace", "", "write trace to file")
68	flag.StringVar(&cmdlineArgs.Memprofile, "memprofile", "", "write memory profile to file")
69	flag.BoolVar(&cmdlineArgs.NoGC, "nogc", false, "turn off GC for debugging")
70
71	// Flags representing various modes soong_build can run in
72	flag.StringVar(&cmdlineArgs.ModuleGraphFile, "module_graph_file", "", "JSON module graph file to output")
73	flag.StringVar(&cmdlineArgs.ModuleActionsFile, "module_actions_file", "", "JSON file to output inputs/outputs of actions of modules")
74	flag.StringVar(&cmdlineArgs.DocFile, "soong_docs", "", "build documentation file to output")
75	flag.StringVar(&cmdlineArgs.BazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top")
76	flag.StringVar(&cmdlineArgs.OutFile, "o", "build.ninja", "the Ninja file to output")
77	flag.StringVar(&cmdlineArgs.SoongVariables, "soong_variables", "soong.variables", "the file contains all build variables")
78	flag.BoolVar(&cmdlineArgs.EmptyNinjaFile, "empty-ninja-file", false, "write out a 0-byte ninja file")
79	flag.BoolVar(&cmdlineArgs.BuildFromSourceStub, "build-from-source-stub", false, "build Java stubs from source files instead of API text files")
80	flag.BoolVar(&cmdlineArgs.EnsureAllowlistIntegrity, "ensure-allowlist-integrity", false, "verify that allowlisted modules are mixed-built")
81	flag.StringVar(&cmdlineArgs.ModuleDebugFile, "soong_module_debug", "", "soong module debug info file to write")
82	// Flags that probably shouldn't be flags of soong_build, but we haven't found
83	// the time to remove them yet
84	flag.BoolVar(&cmdlineArgs.RunGoTests, "t", false, "build and run go tests during bootstrap")
85
86	// Disable deterministic randomization in the protobuf package, so incremental
87	// builds with unrelated Soong changes don't trigger large rebuilds (since we
88	// write out text protos in command lines, and command line changes trigger
89	// rebuilds).
90	androidProtobuf.DisableRand()
91}
92
93func newNameResolver(config android.Config) *android.NameResolver {
94	return android.NewNameResolver(config)
95}
96
97func newContext(configuration android.Config) *android.Context {
98	ctx := android.NewContext(configuration)
99	ctx.SetNameInterface(newNameResolver(configuration))
100	ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies())
101	ctx.AddSourceRootDirs(configuration.SourceRootDirs()...)
102	return ctx
103}
104
105func needToWriteNinjaHint(ctx *android.Context) bool {
106	switch ctx.Config().GetenvWithDefault("SOONG_GENERATES_NINJA_HINT", "") {
107	case "always":
108		return true
109	case "depend":
110		if _, err := os.Stat(filepath.Join(topDir, ctx.Config().OutDir(), ".ninja_log")); errors.Is(err, os.ErrNotExist) {
111			return true
112		}
113	}
114	return false
115}
116
117// Run the code-generation phase to convert BazelTargetModules to BUILD files.
118func runQueryView(queryviewDir, queryviewMarker string, ctx *android.Context) {
119	ctx.EventHandler.Begin("queryview")
120	defer ctx.EventHandler.End("queryview")
121	codegenContext := bp2build.NewCodegenContext(ctx.Config(), ctx, bp2build.QueryView, topDir)
122	err := createBazelWorkspace(codegenContext, shared.JoinPath(topDir, queryviewDir), false)
123	maybeQuit(err, "")
124	touch(shared.JoinPath(topDir, queryviewMarker))
125}
126
127func writeNinjaHint(ctx *android.Context) error {
128	ctx.BeginEvent("ninja_hint")
129	defer ctx.EndEvent("ninja_hint")
130	// The current predictor focuses on reducing false negatives.
131	// If there are too many false positives (e.g., most modules are marked as positive),
132	// real long-running jobs cannot run early.
133	// Therefore, the model should be adjusted in this case.
134	// The model should also be adjusted if there are critical false negatives.
135	predicate := func(j *blueprint.JsonModule) (prioritized bool, weight int) {
136		prioritized = false
137		weight = 0
138		for prefix, w := range allowlists.HugeModuleTypePrefixMap {
139			if strings.HasPrefix(j.Type, prefix) {
140				prioritized = true
141				weight = w
142				return
143			}
144		}
145		dep_count := len(j.Deps)
146		src_count := 0
147		for _, a := range j.Module["Actions"].([]blueprint.JSONAction) {
148			src_count += len(a.Inputs)
149		}
150		input_size := dep_count + src_count
151
152		// Current threshold is an arbitrary value which only consider recall rather than accuracy.
153		if input_size > allowlists.INPUT_SIZE_THRESHOLD {
154			prioritized = true
155			weight += ((input_size) / allowlists.INPUT_SIZE_THRESHOLD) * allowlists.DEFAULT_PRIORITIZED_WEIGHT
156
157			// To prevent some modules from having too large a priority value.
158			if weight > allowlists.HIGH_PRIORITIZED_WEIGHT {
159				weight = allowlists.HIGH_PRIORITIZED_WEIGHT
160			}
161		}
162		return
163	}
164
165	outputsMap := ctx.Context.GetWeightedOutputsFromPredicate(predicate)
166	var outputBuilder strings.Builder
167	for output, weight := range outputsMap {
168		outputBuilder.WriteString(fmt.Sprintf("%s,%d\n", output, weight))
169	}
170	weightListFile := filepath.Join(topDir, ctx.Config().OutDir(), ".ninja_weight_list")
171
172	err := os.WriteFile(weightListFile, []byte(outputBuilder.String()), 0644)
173	if err != nil {
174		return fmt.Errorf("could not write ninja weight list file %s", err)
175	}
176	return nil
177}
178
179func writeMetrics(configuration android.Config, eventHandler *metrics.EventHandler, metricsDir string) {
180	if len(metricsDir) < 1 {
181		fmt.Fprintf(os.Stderr, "\nMissing required env var for generating soong metrics: LOG_DIR\n")
182		os.Exit(1)
183	}
184	metricsFile := filepath.Join(metricsDir, "soong_build_metrics.pb")
185	err := android.WriteMetrics(configuration, eventHandler, metricsFile)
186	maybeQuit(err, "error writing soong_build metrics %s", metricsFile)
187}
188
189func writeJsonModuleGraphAndActions(ctx *android.Context, cmdArgs android.CmdArgs) {
190	graphFile, graphErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleGraphFile))
191	maybeQuit(graphErr, "graph err")
192	defer graphFile.Close()
193	actionsFile, actionsErr := os.Create(shared.JoinPath(topDir, cmdArgs.ModuleActionsFile))
194	maybeQuit(actionsErr, "actions err")
195	defer actionsFile.Close()
196	ctx.Context.PrintJSONGraphAndActions(graphFile, actionsFile)
197}
198
199func writeBuildGlobsNinjaFile(ctx *android.Context) {
200	ctx.EventHandler.Begin("globs_ninja_file")
201	defer ctx.EventHandler.End("globs_ninja_file")
202
203	globDir := bootstrap.GlobDirectory(ctx.Config().SoongOutDir(), globListDir)
204	err := bootstrap.WriteBuildGlobsNinjaFile(&bootstrap.GlobSingleton{
205		GlobLister: ctx.Globs,
206		GlobFile:   globFile,
207		GlobDir:    globDir,
208		SrcDir:     ctx.SrcDir(),
209	}, ctx.Config())
210	maybeQuit(err, "")
211}
212
213func writeDepFile(outputFile string, eventHandler *metrics.EventHandler, ninjaDeps []string) {
214	eventHandler.Begin("ninja_deps")
215	defer eventHandler.End("ninja_deps")
216	depFile := shared.JoinPath(topDir, outputFile+".d")
217	err := deptools.WriteDepFile(depFile, outputFile, ninjaDeps)
218	maybeQuit(err, "error writing depfile '%s'", depFile)
219}
220
221// runSoongOnlyBuild runs the standard Soong build in a number of different modes.
222func runSoongOnlyBuild(ctx *android.Context, extraNinjaDeps []string) string {
223	ctx.EventHandler.Begin("soong_build")
224	defer ctx.EventHandler.End("soong_build")
225
226	var stopBefore bootstrap.StopBefore
227	switch ctx.Config().BuildMode {
228	case android.GenerateModuleGraph:
229		stopBefore = bootstrap.StopBeforeWriteNinja
230	case android.GenerateQueryView, android.GenerateDocFile:
231		stopBefore = bootstrap.StopBeforePrepareBuildActions
232	default:
233		stopBefore = bootstrap.DoEverything
234	}
235
236	ninjaDeps, err := bootstrap.RunBlueprint(cmdlineArgs.Args, stopBefore, ctx.Context, ctx.Config())
237	maybeQuit(err, "")
238	ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
239
240	writeBuildGlobsNinjaFile(ctx)
241
242	// Convert the Soong module graph into Bazel BUILD files.
243	switch ctx.Config().BuildMode {
244	case android.GenerateQueryView:
245		queryviewMarkerFile := cmdlineArgs.BazelQueryViewDir + ".marker"
246		runQueryView(cmdlineArgs.BazelQueryViewDir, queryviewMarkerFile, ctx)
247		writeDepFile(queryviewMarkerFile, ctx.EventHandler, ninjaDeps)
248		return queryviewMarkerFile
249	case android.GenerateModuleGraph:
250		writeJsonModuleGraphAndActions(ctx, cmdlineArgs)
251		writeDepFile(cmdlineArgs.ModuleGraphFile, ctx.EventHandler, ninjaDeps)
252		return cmdlineArgs.ModuleGraphFile
253	case android.GenerateDocFile:
254		// TODO: we could make writeDocs() return the list of documentation files
255		// written and add them to the .d file. Then soong_docs would be re-run
256		// whenever one is deleted.
257		err := writeDocs(ctx, shared.JoinPath(topDir, cmdlineArgs.DocFile))
258		maybeQuit(err, "error building Soong documentation")
259		writeDepFile(cmdlineArgs.DocFile, ctx.EventHandler, ninjaDeps)
260		return cmdlineArgs.DocFile
261	default:
262		// The actual output (build.ninja) was written in the RunBlueprint() call
263		// above
264		writeDepFile(cmdlineArgs.OutFile, ctx.EventHandler, ninjaDeps)
265		if needToWriteNinjaHint(ctx) {
266			writeNinjaHint(ctx)
267		}
268		return cmdlineArgs.OutFile
269	}
270}
271
272// soong_ui dumps the available environment variables to
273// soong.environment.available . Then soong_build itself is run with an empty
274// environment so that the only way environment variables can be accessed is
275// using Config, which tracks access to them.
276
277// At the end of the build, a file called soong.environment.used is written
278// containing the current value of all used environment variables. The next
279// time soong_ui is run, it checks whether any environment variables that was
280// used had changed and if so, it deletes soong.environment.used to cause a
281// rebuild.
282//
283// The dependency of build.ninja on soong.environment.used is declared in
284// build.ninja.d
285func parseAvailableEnv() map[string]string {
286	if availableEnvFile == "" {
287		fmt.Fprintf(os.Stderr, "--available_env not set\n")
288		os.Exit(1)
289	}
290	result, err := shared.EnvFromFile(shared.JoinPath(topDir, availableEnvFile))
291	maybeQuit(err, "error reading available environment file '%s'", availableEnvFile)
292	return result
293}
294
295func main() {
296	flag.Parse()
297
298	shared.ReexecWithDelveMaybe(delveListen, delvePath)
299	android.InitSandbox(topDir)
300
301	availableEnv := parseAvailableEnv()
302	configuration, err := android.NewConfig(cmdlineArgs, availableEnv)
303	maybeQuit(err, "")
304	if configuration.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
305		configuration.SetAllowMissingDependencies()
306	}
307
308	extraNinjaDeps := []string{configuration.ProductVariablesFileName, usedEnvFile}
309	if shared.IsDebugging() {
310		// Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is
311		// enabled even if it completed successfully.
312		extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.SoongOutDir(), "always_rerun_for_delve"))
313	}
314
315	// Bypass configuration.Getenv, as LOG_DIR does not need to be dependency tracked. By definition, it will
316	// change between every CI build, so tracking it would require re-running Soong for every build.
317	metricsDir := availableEnv["LOG_DIR"]
318
319	ctx := newContext(configuration)
320	android.StartBackgroundMetrics(configuration)
321
322	ctx.Register()
323	finalOutputFile := runSoongOnlyBuild(ctx, extraNinjaDeps)
324	writeMetrics(configuration, ctx.EventHandler, metricsDir)
325
326	writeUsedEnvironmentFile(configuration)
327
328	// Touch the output file so that it's the newest file created by soong_build.
329	// This is necessary because, if soong_build generated any files which
330	// are ninja inputs to the main output file, then ninja would superfluously
331	// rebuild this output file on the next build invocation.
332	touch(shared.JoinPath(topDir, finalOutputFile))
333}
334
335func writeUsedEnvironmentFile(configuration android.Config) {
336	if usedEnvFile == "" {
337		return
338	}
339
340	path := shared.JoinPath(topDir, usedEnvFile)
341	data, err := shared.EnvFileContents(configuration.EnvDeps())
342	maybeQuit(err, "error writing used environment file '%s'\n", usedEnvFile)
343
344	if preexistingData, err := os.ReadFile(path); err != nil {
345		if !os.IsNotExist(err) {
346			maybeQuit(err, "error reading used environment file '%s'", usedEnvFile)
347		}
348	} else if bytes.Equal(preexistingData, data) {
349		// used environment file is unchanged
350		return
351	}
352	err = os.WriteFile(path, data, 0666)
353	maybeQuit(err, "error writing used environment file '%s'", usedEnvFile)
354}
355
356func touch(path string) {
357	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
358	maybeQuit(err, "Error touching '%s'", path)
359	err = f.Close()
360	maybeQuit(err, "Error touching '%s'", path)
361
362	currentTime := time.Now().Local()
363	err = os.Chtimes(path, currentTime, currentTime)
364	maybeQuit(err, "error touching '%s'", path)
365}
366
367func maybeQuit(err error, format string, args ...interface{}) {
368	if err == nil {
369		return
370	}
371	if format != "" {
372		fmt.Fprintln(os.Stderr, fmt.Sprintf(format, args...)+": "+err.Error())
373	} else {
374		fmt.Fprintln(os.Stderr, err)
375	}
376	os.Exit(1)
377}
378