1// Copyright 2019 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15// This executable runs a series of build commands to test and benchmark some critical user journeys. 16package main 17 18import ( 19 "context" 20 "fmt" 21 "os" 22 "path/filepath" 23 "strconv" 24 "strings" 25 "time" 26 27 "android/soong/ui/build" 28 "android/soong/ui/logger" 29 "android/soong/ui/metrics" 30 "android/soong/ui/signal" 31 "android/soong/ui/status" 32 "android/soong/ui/terminal" 33 "android/soong/ui/tracer" 34) 35 36type Test struct { 37 name string 38 args []string 39 before func() error 40 41 results TestResults 42} 43 44type TestResults struct { 45 metrics *metrics.Metrics 46 err error 47} 48 49// Run runs a single build command. It emulates the "m" command line by calling into Soong UI directly. 50func (t *Test) Run(logsDir string) { 51 output := terminal.NewStatusOutput(os.Stdout, "", false, false, false) 52 53 log := logger.New(output) 54 defer log.Cleanup() 55 56 ctx, cancel := context.WithCancel(context.Background()) 57 defer cancel() 58 59 trace := tracer.New(log) 60 defer trace.Close() 61 62 met := metrics.New() 63 64 stat := &status.Status{} 65 defer stat.Finish() 66 stat.AddOutput(output) 67 stat.AddOutput(trace.StatusTracer()) 68 69 signal.SetupSignals(log, cancel, func() { 70 trace.Close() 71 log.Cleanup() 72 stat.Finish() 73 }) 74 75 buildCtx := build.Context{ContextImpl: &build.ContextImpl{ 76 Context: ctx, 77 Logger: log, 78 Metrics: met, 79 Tracer: trace, 80 Writer: output, 81 Status: stat, 82 }} 83 84 defer logger.Recover(func(err error) { 85 t.results.err = err 86 }) 87 88 config := build.NewConfig(buildCtx, t.args...) 89 build.SetupOutDir(buildCtx, config) 90 91 os.MkdirAll(logsDir, 0777) 92 log.SetOutput(filepath.Join(logsDir, "soong.log")) 93 trace.SetOutput(filepath.Join(logsDir, "build.trace")) 94 stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, "verbose.log"))) 95 stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, "error.log"))) 96 stat.AddOutput(status.NewProtoErrorLog(log, filepath.Join(logsDir, "build_error"))) 97 stat.AddOutput(status.NewCriticalPathLogger(log, nil)) 98 99 defer met.Dump(filepath.Join(logsDir, "soong_metrics")) 100 101 if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok { 102 if !strings.HasSuffix(start, "N") { 103 if start_time, err := strconv.ParseUint(start, 10, 64); err == nil { 104 log.Verbosef("Took %dms to start up.", 105 time.Since(time.Unix(0, int64(start_time))).Nanoseconds()/time.Millisecond.Nanoseconds()) 106 buildCtx.CompleteTrace(metrics.RunSetupTool, "startup", start_time, uint64(time.Now().UnixNano())) 107 } 108 } 109 110 if executable, err := os.Executable(); err == nil { 111 trace.ImportMicrofactoryLog(filepath.Join(filepath.Dir(executable), "."+filepath.Base(executable)+".trace")) 112 } 113 } 114 115 f := build.NewSourceFinder(buildCtx, config) 116 defer f.Shutdown() 117 build.FindSources(buildCtx, config, f) 118 119 build.Build(buildCtx, config) 120 121 t.results.metrics = met 122} 123 124// Touch the Intent.java file to cause a rebuild of the frameworks to monitor the 125// incremental build speed as mentioned b/152046247. Intent.java file was chosen 126// as it is a key component of the framework and is often modified. 127func touchIntentFile() error { 128 const intentFileName = "frameworks/base/core/java/android/content/Intent.java" 129 currentTime := time.Now().Local() 130 return os.Chtimes(intentFileName, currentTime, currentTime) 131} 132 133func main() { 134 outDir := os.Getenv("OUT_DIR") 135 if outDir == "" { 136 outDir = "out" 137 } 138 139 cujDir := filepath.Join(outDir, "cuj_tests") 140 141 wd, _ := os.Getwd() 142 os.Setenv("TOP", wd) 143 // Use a subdirectory for the out directory for the tests to keep them isolated. 144 os.Setenv("OUT_DIR", filepath.Join(cujDir, "out")) 145 146 // Each of these tests is run in sequence without resetting the output tree. The state of the output tree will 147 // affect each successive test. To maintain the validity of the benchmarks across changes, care must be taken 148 // to avoid changing the state of the tree when a test is run. This is most easily accomplished by adding tests 149 // at the end. 150 tests := []Test{ 151 { 152 // Reset the out directory to get reproducible results. 153 name: "clean", 154 args: []string{"clean"}, 155 }, 156 { 157 // Parse the build files. 158 name: "nothing", 159 args: []string{"nothing"}, 160 }, 161 { 162 // Parse the build files again to monitor issues like globs rerunning. 163 name: "nothing_rebuild", 164 args: []string{"nothing"}, 165 }, 166 { 167 // Parse the build files again, this should always be very short. 168 name: "nothing_rebuild_twice", 169 args: []string{"nothing"}, 170 }, 171 { 172 // Build the framework as a common developer task and one that keeps getting longer. 173 name: "framework", 174 args: []string{"framework"}, 175 }, 176 { 177 // Build the framework again to make sure it doesn't rebuild anything. 178 name: "framework_rebuild", 179 args: []string{"framework"}, 180 }, 181 { 182 // Build the framework again to make sure it doesn't rebuild anything even if it did the second time. 183 name: "framework_rebuild_twice", 184 args: []string{"framework"}, 185 }, 186 { 187 // Scenario major_inc_build (b/152046247): tracking build speed of major incremental build. 188 name: "major_inc_build_droid", 189 args: []string{"droid"}, 190 }, 191 { 192 name: "major_inc_build_framework_minus_apex_after_droid_build", 193 args: []string{"framework-minus-apex"}, 194 before: touchIntentFile, 195 }, 196 { 197 name: "major_inc_build_framework_after_droid_build", 198 args: []string{"framework"}, 199 before: touchIntentFile, 200 }, 201 { 202 name: "major_inc_build_sync_after_droid_build", 203 args: []string{"sync"}, 204 before: touchIntentFile, 205 }, 206 { 207 name: "major_inc_build_droid_rebuild", 208 args: []string{"droid"}, 209 before: touchIntentFile, 210 }, 211 { 212 name: "major_inc_build_update_api_after_droid_rebuild", 213 args: []string{"update-api"}, 214 before: touchIntentFile, 215 }, 216 } 217 218 cujMetrics := metrics.NewCriticalUserJourneysMetrics() 219 defer cujMetrics.Dump(filepath.Join(cujDir, "logs", "cuj_metrics.pb")) 220 221 for i, t := range tests { 222 logsSubDir := fmt.Sprintf("%02d_%s", i, t.name) 223 logsDir := filepath.Join(cujDir, "logs", logsSubDir) 224 if t.before != nil { 225 if err := t.before(); err != nil { 226 fmt.Printf("error running before function on test %q: %v\n", t.name, err) 227 break 228 } 229 } 230 t.Run(logsDir) 231 if t.results.err != nil { 232 fmt.Printf("error running test %q: %s\n", t.name, t.results.err) 233 break 234 } 235 if t.results.metrics != nil { 236 cujMetrics.Add(t.name, t.results.metrics) 237 } 238 } 239} 240