1// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
18	"bufio"
19	"context"
20	"flag"
21	"fmt"
22	"io"
23	"io/ioutil"
24	"log"
25	"os"
26	"os/exec"
27	"path/filepath"
28	"regexp"
29	"runtime"
30	"strings"
31	"sync"
32	"syscall"
33	"time"
34
35	"android/soong/ui/logger"
36	"android/soong/ui/signal"
37	"android/soong/ui/status"
38	"android/soong/ui/terminal"
39	"android/soong/ui/tracer"
40	"android/soong/zip"
41)
42
43var numJobs = flag.Int("j", 0, "number of parallel jobs [0=autodetect]")
44
45var keepArtifacts = flag.Bool("keep", false, "keep archives of artifacts")
46var incremental = flag.Bool("incremental", false, "run in incremental mode (saving intermediates)")
47
48var outDir = flag.String("out", "", "path to store output directories (defaults to tmpdir under $OUT when empty)")
49var alternateResultDir = flag.Bool("dist", false, "write select results to $DIST_DIR (or <out>/dist when empty)")
50
51var bazelMode = flag.Bool("bazel-mode", false, "use bazel for analysis of certain modules")
52var bazelModeStaging = flag.Bool("bazel-mode-staging", false, "use bazel for analysis of certain near-ready modules")
53
54var onlyConfig = flag.Bool("only-config", false, "Only run product config (not Soong or Kati)")
55var onlySoong = flag.Bool("only-soong", false, "Only run product config and Soong (not Kati)")
56
57var buildVariant = flag.String("variant", "eng", "build variant to use")
58
59var shardCount = flag.Int("shard-count", 1, "split the products into multiple shards (to spread the build onto multiple machines, etc)")
60var shard = flag.Int("shard", 1, "1-indexed shard to execute")
61
62var skipProducts multipleStringArg
63var includeProducts multipleStringArg
64
65func init() {
66	flag.Var(&skipProducts, "skip-products", "comma-separated list of products to skip (known failures, etc)")
67	flag.Var(&includeProducts, "products", "comma-separated list of products to build")
68}
69
70// multipleStringArg is a flag.Value that takes comma separated lists and converts them to a
71// []string.  The argument can be passed multiple times to append more values.
72type multipleStringArg []string
73
74func (m *multipleStringArg) String() string {
75	return strings.Join(*m, `, `)
76}
77
78func (m *multipleStringArg) Set(s string) error {
79	*m = append(*m, strings.Split(s, ",")...)
80	return nil
81}
82
83const errorLeadingLines = 20
84const errorTrailingLines = 20
85
86func errMsgFromLog(filename string) string {
87	if filename == "" {
88		return ""
89	}
90
91	data, err := ioutil.ReadFile(filename)
92	if err != nil {
93		return ""
94	}
95
96	lines := strings.Split(strings.TrimSpace(string(data)), "\n")
97	if len(lines) > errorLeadingLines+errorTrailingLines+1 {
98		lines[errorLeadingLines] = fmt.Sprintf("... skipping %d lines ...",
99			len(lines)-errorLeadingLines-errorTrailingLines)
100
101		lines = append(lines[:errorLeadingLines+1],
102			lines[len(lines)-errorTrailingLines:]...)
103	}
104	var buf strings.Builder
105	for _, line := range lines {
106		buf.WriteString("> ")
107		buf.WriteString(line)
108		buf.WriteString("\n")
109	}
110	return buf.String()
111}
112
113// TODO(b/70370883): This tool uses a lot of open files -- over the default
114// soft limit of 1024 on some systems. So bump up to the hard limit until I fix
115// the algorithm.
116func setMaxFiles(log logger.Logger) {
117	var limits syscall.Rlimit
118
119	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limits)
120	if err != nil {
121		log.Println("Failed to get file limit:", err)
122		return
123	}
124
125	log.Verbosef("Current file limits: %d soft, %d hard", limits.Cur, limits.Max)
126	if limits.Cur == limits.Max {
127		return
128	}
129
130	limits.Cur = limits.Max
131	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limits)
132	if err != nil {
133		log.Println("Failed to increase file limit:", err)
134	}
135}
136
137func inList(str string, list []string) bool {
138	for _, other := range list {
139		if str == other {
140			return true
141		}
142	}
143	return false
144}
145
146func copyFile(from, to string) error {
147	fromFile, err := os.Open(from)
148	if err != nil {
149		return err
150	}
151	defer fromFile.Close()
152
153	toFile, err := os.Create(to)
154	if err != nil {
155		return err
156	}
157	defer toFile.Close()
158
159	_, err = io.Copy(toFile, fromFile)
160	return err
161}
162
163type mpContext struct {
164	Logger logger.Logger
165	Status status.ToolStatus
166
167	SoongUi     string
168	MainOutDir  string
169	MainLogsDir string
170}
171
172func findNamedProducts(soongUi string, log logger.Logger) []string {
173	cmd := exec.Command(soongUi, "--dumpvars-mode", "--vars=all_named_products")
174	output, err := cmd.Output()
175	if err != nil {
176		log.Fatalf("Cannot determine named products: %v", err)
177	}
178
179	rx := regexp.MustCompile(`^all_named_products='(.*)'$`)
180	match := rx.FindStringSubmatch(strings.TrimSpace(string(output)))
181	return strings.Fields(match[1])
182}
183
184// ensureEmptyFileExists ensures that the containing directory exists, and the
185// specified file exists. If it doesn't exist, it will write an empty file.
186func ensureEmptyFileExists(file string, log logger.Logger) {
187	if _, err := os.Stat(file); os.IsNotExist(err) {
188		f, err := os.Create(file)
189		if err != nil {
190			log.Fatalf("Error creating %s: %q\n", file, err)
191		}
192		f.Close()
193	} else if err != nil {
194		log.Fatalf("Error checking %s: %q\n", file, err)
195	}
196}
197
198func outDirBase() string {
199	outDirBase := os.Getenv("OUT_DIR")
200	if outDirBase == "" {
201		return "out"
202	} else {
203		return outDirBase
204	}
205}
206
207func distDir(outDir string) string {
208	if distDir := os.Getenv("DIST_DIR"); distDir != "" {
209		return filepath.Clean(distDir)
210	} else {
211		return filepath.Join(outDir, "dist")
212	}
213}
214
215func forceAnsiOutput() bool {
216	value := os.Getenv("SOONG_UI_ANSI_OUTPUT")
217	return value == "1" || value == "y" || value == "yes" || value == "on" || value == "true"
218}
219
220func getBazelArg() string {
221	count := 0
222	str := ""
223	if *bazelMode {
224		count++
225		str = "--bazel-mode"
226	}
227	if *bazelModeStaging {
228		count++
229		str = "--bazel-mode-staging"
230	}
231
232	if count > 1 {
233		// Can't set more than one
234		fmt.Errorf("Only one bazel mode is permitted to be set.")
235		os.Exit(1)
236	}
237
238	return str
239}
240
241func main() {
242	stdio := terminal.StdioImpl{}
243
244	output := terminal.NewStatusOutput(stdio.Stdout(), "", false, false,
245		forceAnsiOutput())
246	log := logger.New(output)
247	defer log.Cleanup()
248
249	for _, v := range os.Environ() {
250		log.Println("Environment: " + v)
251	}
252
253	log.Printf("Argv: %v\n", os.Args)
254
255	flag.Parse()
256
257	_, cancel := context.WithCancel(context.Background())
258	defer cancel()
259
260	trace := tracer.New(log)
261	defer trace.Close()
262
263	stat := &status.Status{}
264	defer stat.Finish()
265	stat.AddOutput(output)
266
267	var failures failureCount
268	stat.AddOutput(&failures)
269
270	signal.SetupSignals(log, cancel, func() {
271		trace.Close()
272		log.Cleanup()
273		stat.Finish()
274	})
275
276	soongUi := "build/soong/soong_ui.bash"
277
278	var outputDir string
279	if *outDir != "" {
280		outputDir = *outDir
281	} else {
282		name := "multiproduct"
283		if !*incremental {
284			name += "-" + time.Now().Format("20060102150405")
285		}
286		outputDir = filepath.Join(outDirBase(), name)
287	}
288
289	log.Println("Output directory:", outputDir)
290
291	// The ninja_build file is used by our buildbots to understand that the output
292	// can be parsed as ninja output.
293	if err := os.MkdirAll(outputDir, 0777); err != nil {
294		log.Fatalf("Failed to create output directory: %v", err)
295	}
296	ensureEmptyFileExists(filepath.Join(outputDir, "ninja_build"), log)
297
298	logsDir := filepath.Join(outputDir, "logs")
299	os.MkdirAll(logsDir, 0777)
300
301	var configLogsDir string
302	if *alternateResultDir {
303		configLogsDir = filepath.Join(distDir(outDirBase()), "logs")
304	} else {
305		configLogsDir = outputDir
306	}
307
308	log.Println("Logs dir: " + configLogsDir)
309
310	os.MkdirAll(configLogsDir, 0777)
311	log.SetOutput(filepath.Join(configLogsDir, "soong.log"))
312	trace.SetOutput(filepath.Join(configLogsDir, "build.trace"))
313
314	var jobs = *numJobs
315	if jobs < 1 {
316		jobs = runtime.NumCPU() / 4
317
318		ramGb := int(detectTotalRAM() / (1024 * 1024 * 1024))
319		if ramJobs := ramGb / 40; ramGb > 0 && jobs > ramJobs {
320			jobs = ramJobs
321		}
322
323		if jobs < 1 {
324			jobs = 1
325		}
326	}
327	log.Verbosef("Using %d parallel jobs", jobs)
328
329	setMaxFiles(log)
330
331	allProducts := findNamedProducts(soongUi, log)
332	var productsList []string
333
334	if len(includeProducts) > 0 {
335		var missingProducts []string
336		for _, product := range includeProducts {
337			if inList(product, allProducts) {
338				productsList = append(productsList, product)
339			} else {
340				missingProducts = append(missingProducts, product)
341			}
342		}
343		if len(missingProducts) > 0 {
344			log.Fatalf("Products don't exist: %s\n", missingProducts)
345		}
346	} else {
347		productsList = allProducts
348	}
349
350	finalProductsList := make([]string, 0, len(productsList))
351	skipProduct := func(p string) bool {
352		for _, s := range skipProducts {
353			if p == s {
354				return true
355			}
356		}
357		return false
358	}
359	for _, product := range productsList {
360		if !skipProduct(product) {
361			finalProductsList = append(finalProductsList, product)
362		} else {
363			log.Verbose("Skipping: ", product)
364		}
365	}
366
367	if *shard < 1 {
368		log.Fatalf("--shard value must be >= 1, not %d\n", *shard)
369	} else if *shardCount < 1 {
370		log.Fatalf("--shard-count value must be >= 1, not %d\n", *shardCount)
371	} else if *shard > *shardCount {
372		log.Fatalf("--shard (%d) must not be greater than --shard-count (%d)\n", *shard,
373			*shardCount)
374	} else if *shardCount > 1 {
375		finalProductsList = splitList(finalProductsList, *shardCount)[*shard-1]
376	}
377
378	log.Verbose("Got product list: ", finalProductsList)
379
380	s := stat.StartTool()
381	s.SetTotalActions(len(finalProductsList))
382
383	mpCtx := &mpContext{
384		Logger:      log,
385		Status:      s,
386		SoongUi:     soongUi,
387		MainOutDir:  outputDir,
388		MainLogsDir: logsDir,
389	}
390
391	products := make(chan string, len(productsList))
392	go func() {
393		defer close(products)
394		for _, product := range finalProductsList {
395			products <- product
396		}
397	}()
398
399	var wg sync.WaitGroup
400	for i := 0; i < jobs; i++ {
401		wg.Add(1)
402		// To smooth out the spikes in memory usage, skew the
403		// initial starting time of the jobs by a small amount.
404		time.Sleep(15 * time.Second)
405		go func() {
406			defer wg.Done()
407			for {
408				select {
409				case product := <-products:
410					if product == "" {
411						return
412					}
413					runSoongUiForProduct(mpCtx, product)
414				}
415			}
416		}()
417	}
418	wg.Wait()
419
420	if *alternateResultDir {
421		args := zip.ZipArgs{
422			FileArgs: []zip.FileArg{
423				{GlobDir: logsDir, SourcePrefixToStrip: logsDir},
424			},
425			OutputFilePath:   filepath.Join(distDir(outDirBase()), "logs.zip"),
426			NumParallelJobs:  runtime.NumCPU(),
427			CompressionLevel: 5,
428		}
429		log.Printf("Logs zip: %v\n", args.OutputFilePath)
430		if err := zip.Zip(args); err != nil {
431			log.Fatalf("Error zipping logs: %v", err)
432		}
433	}
434
435	s.Finish()
436
437	if failures.count == 1 {
438		log.Fatal("1 failure")
439	} else if failures.count > 1 {
440		log.Fatalf("%d failures %q", failures.count, failures.fails)
441	} else {
442		fmt.Fprintln(output, "Success")
443	}
444}
445
446func cleanupAfterProduct(outDir, productZip string) {
447	if *keepArtifacts {
448		args := zip.ZipArgs{
449			FileArgs: []zip.FileArg{
450				{
451					GlobDir:             outDir,
452					SourcePrefixToStrip: outDir,
453				},
454			},
455			OutputFilePath:   productZip,
456			NumParallelJobs:  runtime.NumCPU(),
457			CompressionLevel: 5,
458		}
459		if err := zip.Zip(args); err != nil {
460			log.Fatalf("Error zipping artifacts: %v", err)
461		}
462	}
463	if !*incremental {
464		os.RemoveAll(outDir)
465	}
466}
467
468func runSoongUiForProduct(mpctx *mpContext, product string) {
469	outDir := filepath.Join(mpctx.MainOutDir, product)
470	logsDir := filepath.Join(mpctx.MainLogsDir, product)
471	productZip := filepath.Join(mpctx.MainOutDir, product+".zip")
472	consoleLogPath := filepath.Join(logsDir, "std.log")
473
474	if err := os.MkdirAll(outDir, 0777); err != nil {
475		mpctx.Logger.Fatalf("Error creating out directory: %v", err)
476	}
477	if err := os.MkdirAll(logsDir, 0777); err != nil {
478		mpctx.Logger.Fatalf("Error creating log directory: %v", err)
479	}
480
481	consoleLogFile, err := os.Create(consoleLogPath)
482	if err != nil {
483		mpctx.Logger.Fatalf("Error creating console log file: %v", err)
484	}
485	defer consoleLogFile.Close()
486
487	consoleLogWriter := bufio.NewWriter(consoleLogFile)
488	defer consoleLogWriter.Flush()
489
490	args := []string{"--make-mode", "--skip-soong-tests", "--skip-ninja"}
491
492	if !*keepArtifacts {
493		args = append(args, "--empty-ninja-file")
494	}
495
496	if *onlyConfig {
497		args = append(args, "--config-only")
498	} else if *onlySoong {
499		args = append(args, "--soong-only")
500	}
501
502	bazelStr := getBazelArg()
503	if bazelStr != "" {
504		args = append(args, bazelStr)
505	}
506
507	cmd := exec.Command(mpctx.SoongUi, args...)
508	cmd.Stdout = consoleLogWriter
509	cmd.Stderr = consoleLogWriter
510	cmd.Env = append(os.Environ(),
511		"OUT_DIR="+outDir,
512		"TARGET_PRODUCT="+product,
513		"TARGET_BUILD_VARIANT="+*buildVariant,
514		"TARGET_BUILD_TYPE=release",
515		"TARGET_BUILD_APPS=",
516		"TARGET_BUILD_UNBUNDLED=",
517		"USE_RBE=false") // Disabling RBE saves ~10 secs per product
518
519	if *alternateResultDir {
520		cmd.Env = append(cmd.Env,
521			"DIST_DIR="+filepath.Join(distDir(outDirBase()), "products/"+product))
522	}
523
524	action := &status.Action{
525		Description: product,
526		Outputs:     []string{product},
527	}
528
529	mpctx.Status.StartAction(action)
530	defer cleanupAfterProduct(outDir, productZip)
531
532	before := time.Now()
533	err = cmd.Run()
534
535	if !*onlyConfig && !*onlySoong {
536		katiBuildNinjaFile := filepath.Join(outDir, "build-"+product+".ninja")
537		if after, err := os.Stat(katiBuildNinjaFile); err == nil && after.ModTime().After(before) {
538			err := copyFile(consoleLogPath, filepath.Join(filepath.Dir(consoleLogPath), "std_full.log"))
539			if err != nil {
540				log.Fatalf("Error copying log file: %s", err)
541			}
542		}
543	}
544	var errOutput string
545	if err == nil {
546		errOutput = ""
547	} else {
548		errOutput = errMsgFromLog(consoleLogPath)
549	}
550
551	mpctx.Status.FinishAction(status.ActionResult{
552		Action: action,
553		Error:  err,
554		Output: errOutput,
555	})
556}
557
558type failureCount struct {
559	count int
560	fails []string
561}
562
563func (f *failureCount) StartAction(action *status.Action, counts status.Counts) {}
564
565func (f *failureCount) FinishAction(result status.ActionResult, counts status.Counts) {
566	if result.Error != nil {
567		f.count += 1
568		f.fails = append(f.fails, result.Action.Description)
569	}
570}
571
572func (f *failureCount) Message(level status.MsgLevel, message string) {
573	if level >= status.ErrorLvl {
574		f.count += 1
575	}
576}
577
578func (f *failureCount) Flush() {}
579
580func (f *failureCount) Write(p []byte) (int, error) {
581	// discard writes
582	return len(p), nil
583}
584
585func splitList(list []string, shardCount int) (ret [][]string) {
586	each := len(list) / shardCount
587	extra := len(list) % shardCount
588	for i := 0; i < shardCount; i++ {
589		count := each
590		if extra > 0 {
591			count += 1
592			extra -= 1
593		}
594		ret = append(ret, list[:count])
595		list = list[count:]
596	}
597	return
598}
599