1// Copyright 2019 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// This executable runs a series of build commands to test and benchmark some critical user journeys.
16package main
17
18import (
19	"context"
20	"fmt"
21	"os"
22	"path/filepath"
23	"strconv"
24	"strings"
25	"time"
26
27	"android/soong/ui/build"
28	"android/soong/ui/logger"
29	"android/soong/ui/metrics"
30	"android/soong/ui/signal"
31	"android/soong/ui/status"
32	"android/soong/ui/terminal"
33	"android/soong/ui/tracer"
34)
35
36type Test struct {
37	name   string
38	args   []string
39	before func() error
40
41	results TestResults
42}
43
44type TestResults struct {
45	metrics *metrics.Metrics
46	err     error
47}
48
49// Run runs a single build command.  It emulates the "m" command line by calling into Soong UI directly.
50func (t *Test) Run(logsDir string) {
51	output := terminal.NewStatusOutput(os.Stdout, "", false, false, false)
52
53	log := logger.New(output)
54	defer log.Cleanup()
55
56	ctx, cancel := context.WithCancel(context.Background())
57	defer cancel()
58
59	trace := tracer.New(log)
60	defer trace.Close()
61
62	met := metrics.New()
63
64	stat := &status.Status{}
65	defer stat.Finish()
66	stat.AddOutput(output)
67	stat.AddOutput(trace.StatusTracer())
68
69	signal.SetupSignals(log, cancel, func() {
70		trace.Close()
71		log.Cleanup()
72		stat.Finish()
73	})
74
75	buildCtx := build.Context{ContextImpl: &build.ContextImpl{
76		Context: ctx,
77		Logger:  log,
78		Metrics: met,
79		Tracer:  trace,
80		Writer:  output,
81		Status:  stat,
82	}}
83
84	defer logger.Recover(func(err error) {
85		t.results.err = err
86	})
87
88	config := build.NewConfig(buildCtx, t.args...)
89	build.SetupOutDir(buildCtx, config)
90
91	os.MkdirAll(logsDir, 0777)
92	log.SetOutput(filepath.Join(logsDir, "soong.log"))
93	trace.SetOutput(filepath.Join(logsDir, "build.trace"))
94	stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, "verbose.log")))
95	stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, "error.log")))
96	stat.AddOutput(status.NewProtoErrorLog(log, filepath.Join(logsDir, "build_error")))
97	stat.AddOutput(status.NewCriticalPathLogger(log, nil))
98
99	defer met.Dump(filepath.Join(logsDir, "soong_metrics"))
100
101	if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
102		if !strings.HasSuffix(start, "N") {
103			if start_time, err := strconv.ParseUint(start, 10, 64); err == nil {
104				log.Verbosef("Took %dms to start up.",
105					time.Since(time.Unix(0, int64(start_time))).Nanoseconds()/time.Millisecond.Nanoseconds())
106				buildCtx.CompleteTrace(metrics.RunSetupTool, "startup", start_time, uint64(time.Now().UnixNano()))
107			}
108		}
109
110		if executable, err := os.Executable(); err == nil {
111			trace.ImportMicrofactoryLog(filepath.Join(filepath.Dir(executable), "."+filepath.Base(executable)+".trace"))
112		}
113	}
114
115	f := build.NewSourceFinder(buildCtx, config)
116	defer f.Shutdown()
117	build.FindSources(buildCtx, config, f)
118
119	build.Build(buildCtx, config)
120
121	t.results.metrics = met
122}
123
124// Touch the Intent.java file to cause a rebuild of the frameworks to monitor the
125// incremental build speed as mentioned b/152046247. Intent.java file was chosen
126// as it is a key component of the framework and is often modified.
127func touchIntentFile() error {
128	const intentFileName = "frameworks/base/core/java/android/content/Intent.java"
129	currentTime := time.Now().Local()
130	return os.Chtimes(intentFileName, currentTime, currentTime)
131}
132
133func main() {
134	outDir := os.Getenv("OUT_DIR")
135	if outDir == "" {
136		outDir = "out"
137	}
138
139	cujDir := filepath.Join(outDir, "cuj_tests")
140
141	wd, _ := os.Getwd()
142	os.Setenv("TOP", wd)
143	// Use a subdirectory for the out directory for the tests to keep them isolated.
144	os.Setenv("OUT_DIR", filepath.Join(cujDir, "out"))
145
146	// Each of these tests is run in sequence without resetting the output tree.  The state of the output tree will
147	// affect each successive test.  To maintain the validity of the benchmarks across changes, care must be taken
148	// to avoid changing the state of the tree when a test is run.  This is most easily accomplished by adding tests
149	// at the end.
150	tests := []Test{
151		{
152			// Reset the out directory to get reproducible results.
153			name: "clean",
154			args: []string{"clean"},
155		},
156		{
157			// Parse the build files.
158			name: "nothing",
159			args: []string{"nothing"},
160		},
161		{
162			// Parse the build files again to monitor issues like globs rerunning.
163			name: "nothing_rebuild",
164			args: []string{"nothing"},
165		},
166		{
167			// Parse the build files again, this should always be very short.
168			name: "nothing_rebuild_twice",
169			args: []string{"nothing"},
170		},
171		{
172			// Build the framework as a common developer task and one that keeps getting longer.
173			name: "framework",
174			args: []string{"framework"},
175		},
176		{
177			// Build the framework again to make sure it doesn't rebuild anything.
178			name: "framework_rebuild",
179			args: []string{"framework"},
180		},
181		{
182			// Build the framework again to make sure it doesn't rebuild anything even if it did the second time.
183			name: "framework_rebuild_twice",
184			args: []string{"framework"},
185		},
186		{
187			// Scenario major_inc_build (b/152046247): tracking build speed of major incremental build.
188			name: "major_inc_build_droid",
189			args: []string{"droid"},
190		},
191		{
192			name:   "major_inc_build_framework_minus_apex_after_droid_build",
193			args:   []string{"framework-minus-apex"},
194			before: touchIntentFile,
195		},
196		{
197			name:   "major_inc_build_framework_after_droid_build",
198			args:   []string{"framework"},
199			before: touchIntentFile,
200		},
201		{
202			name:   "major_inc_build_sync_after_droid_build",
203			args:   []string{"sync"},
204			before: touchIntentFile,
205		},
206		{
207			name:   "major_inc_build_droid_rebuild",
208			args:   []string{"droid"},
209			before: touchIntentFile,
210		},
211		{
212			name:   "major_inc_build_update_api_after_droid_rebuild",
213			args:   []string{"update-api"},
214			before: touchIntentFile,
215		},
216	}
217
218	cujMetrics := metrics.NewCriticalUserJourneysMetrics()
219	defer cujMetrics.Dump(filepath.Join(cujDir, "logs", "cuj_metrics.pb"))
220
221	for i, t := range tests {
222		logsSubDir := fmt.Sprintf("%02d_%s", i, t.name)
223		logsDir := filepath.Join(cujDir, "logs", logsSubDir)
224		if t.before != nil {
225			if err := t.before(); err != nil {
226				fmt.Printf("error running before function on test %q: %v\n", t.name, err)
227				break
228			}
229		}
230		t.Run(logsDir)
231		if t.results.err != nil {
232			fmt.Printf("error running test %q: %s\n", t.name, t.results.err)
233			break
234		}
235		if t.results.metrics != nil {
236			cujMetrics.Add(t.name, t.results.metrics)
237		}
238	}
239}
240