1// Copyright 2017 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17import (
18	"bytes"
19	"io/ioutil"
20	"os"
21	"path/filepath"
22	"strings"
23
24	"android/soong/finder"
25	"android/soong/finder/fs"
26	"android/soong/ui/logger"
27
28	"android/soong/ui/metrics"
29)
30
31// This file provides an interface to the Finder type for soong_ui. Finder is
32// used to recursively traverse the source tree to gather paths of files, such
33// as Android.bp or Android.mk, and store the lists/database of paths in files
34// under `$OUT_DIR/.module_paths`. This directory can also be dist'd.
35
36// NewSourceFinder returns a new Finder configured to search for source files.
37// Callers of NewSourceFinder should call <f.Shutdown()> when done
38func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) {
39	ctx.BeginTrace(metrics.RunSetupTool, "find modules")
40	defer ctx.EndTrace()
41
42	// Set up the working directory for the Finder.
43	dir, err := os.Getwd()
44	if err != nil {
45		ctx.Fatalf("No working directory for module-finder: %v", err.Error())
46	}
47	filesystem := fs.OsFs
48
49	// .out-dir and .find-ignore are markers for Finder to ignore siblings and
50	// subdirectories of the directory Finder finds them in, hence stopping the
51	// search recursively down those branches. It's possible that these files
52	// are in the root directory, and if they are, then the subsequent error
53	// messages are very confusing, so check for that here.
54	pruneFiles := []string{".out-dir", ".find-ignore"}
55	for _, name := range pruneFiles {
56		prunePath := filepath.Join(dir, name)
57		_, statErr := filesystem.Lstat(prunePath)
58		if statErr == nil {
59			ctx.Fatalf("%v must not exist", prunePath)
60		}
61	}
62
63	// Set up configuration parameters for the Finder cache.
64	cacheParams := finder.CacheParams{
65		WorkingDirectory: dir,
66		RootDirs:         androidBpSearchDirs(config),
67		FollowSymlinks:   config.environ.IsEnvTrue("ALLOW_BP_UNDER_SYMLINKS"),
68		ExcludeDirs:      []string{".git", ".repo"},
69		PruneFiles:       pruneFiles,
70		IncludeFiles: []string{
71			// Kati build definitions.
72			"Android.mk",
73			// Product configuration files.
74			"AndroidProducts.mk",
75			// General Soong build definitions, using the Blueprint syntax.
76			"Android.bp",
77			// Kati clean definitions.
78			"CleanSpec.mk",
79			// Ownership definition.
80			"OWNERS",
81			// Test configuration for modules in directories that contain this
82			// file.
83			"TEST_MAPPING",
84			// METADATA file of packages
85			"METADATA",
86		},
87		// .mk files for product/board configuration.
88		IncludeSuffixes: []string{".mk"},
89	}
90	dumpDir := config.FileListDir()
91	f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard),
92		filepath.Join(dumpDir, "files.db"))
93	if err != nil {
94		ctx.Fatalf("Could not create module-finder: %v", err)
95	}
96	return f
97}
98
99func androidBpSearchDirs(config Config) []string {
100	dirs := []string{"."} // always search from root of source tree.
101	if config.searchApiDir {
102		// Search in out/api_surfaces
103		dirs = append(dirs, config.ApiSurfacesOutDir())
104	}
105	return dirs
106}
107
108func findProductAndBoardConfigFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) {
109	matches := []string{}
110	for _, foundName := range entries.FileNames {
111		if foundName != "Android.mk" &&
112			foundName != "AndroidProducts.mk" &&
113			foundName != "CleanSpec.mk" &&
114			strings.HasSuffix(foundName, ".mk") {
115			matches = append(matches, foundName)
116		}
117	}
118	return entries.DirNames, matches
119}
120
121// FindSources searches for source files known to <f> and writes them to the filesystem for
122// use later.
123func FindSources(ctx Context, config Config, f *finder.Finder) {
124	// note that dumpDir in FindSources may be different than dumpDir in NewSourceFinder
125	// if a caller such as multiproduct_kati wants to share one Finder among several builds
126	dumpDir := config.FileListDir()
127	os.MkdirAll(dumpDir, 0777)
128
129	// Stop searching a subdirectory recursively after finding an Android.mk.
130	androidMks := f.FindFirstNamedAt(".", "Android.mk")
131	blockAndroidMks(ctx, androidMks)
132	err := dumpListToFile(ctx, config, androidMks, filepath.Join(dumpDir, "Android.mk.list"))
133	if err != nil {
134		ctx.Fatalf("Could not export module list: %v", err)
135	}
136
137	// Gate collecting/reporting mk metrics on builds that specifically request
138	// it, as identifying the total number of mk files adds 4-5ms onto null
139	// builds.
140	if config.reportMkMetrics {
141		androidMksTotal := f.FindNamedAt(".", "Android.mk")
142
143		ctx.Metrics.SetToplevelMakefiles(len(androidMks))
144		ctx.Metrics.SetTotalMakefiles(len(androidMksTotal))
145		ctx.Metrics.DumpMkMetrics(config.MkMetrics())
146	}
147
148	// Stop searching a subdirectory recursively after finding a CleanSpec.mk.
149	cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk")
150	err = dumpListToFile(ctx, config, cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list"))
151	if err != nil {
152		ctx.Fatalf("Could not export module list: %v", err)
153	}
154
155	// Only consider AndroidProducts.mk in device/, vendor/ and product/, recursively in these directories.
156	androidProductsMks := f.FindNamedAt("device", "AndroidProducts.mk")
157	androidProductsMks = append(androidProductsMks, f.FindNamedAt("vendor", "AndroidProducts.mk")...)
158	androidProductsMks = append(androidProductsMks, f.FindNamedAt("product", "AndroidProducts.mk")...)
159	err = dumpListToFile(ctx, config, androidProductsMks, filepath.Join(dumpDir, "AndroidProducts.mk.list"))
160	if err != nil {
161		ctx.Fatalf("Could not export product list: %v", err)
162	}
163
164	// Recursively look for all OWNERS files.
165	owners := f.FindNamedAt(".", "OWNERS")
166	err = dumpListToFile(ctx, config, owners, filepath.Join(dumpDir, "OWNERS.list"))
167	if err != nil {
168		ctx.Fatalf("Could not find OWNERS: %v", err)
169	}
170
171	// Recursively look for all METADATA files.
172	metadataFiles := f.FindNamedAt(".", "METADATA")
173	err = dumpListToFile(ctx, config, metadataFiles, filepath.Join(dumpDir, "METADATA.list"))
174	if err != nil {
175		ctx.Fatalf("Could not find METADATA: %v", err)
176	}
177
178	// Recursively look for all TEST_MAPPING files.
179	testMappings := f.FindNamedAt(".", "TEST_MAPPING")
180	err = dumpListToFile(ctx, config, testMappings, filepath.Join(dumpDir, "TEST_MAPPING.list"))
181	if err != nil {
182		ctx.Fatalf("Could not find TEST_MAPPING: %v", err)
183	}
184
185	// Recursively look for all Android.bp files
186	androidBps := f.FindNamedAt(".", "Android.bp")
187	if len(androidBps) == 0 {
188		ctx.Fatalf("No Android.bp found")
189	}
190	err = dumpListToFile(ctx, config, androidBps, filepath.Join(dumpDir, "Android.bp.list"))
191	if err != nil {
192		ctx.Fatalf("Could not find modules: %v", err)
193	}
194
195	// Recursively look for all product/board config files.
196	configurationFiles := f.FindMatching(".", findProductAndBoardConfigFiles)
197	err = dumpListToFile(ctx, config, configurationFiles, filepath.Join(dumpDir, "configuration.list"))
198	if err != nil {
199		ctx.Fatalf("Could not export product/board configuration list: %v", err)
200	}
201
202	if config.Dist() {
203		f.WaitForDbDump()
204		// Dist the files.db plain text database.
205		distFile(ctx, config, f.DbPath, "module_paths")
206	}
207}
208
209// Write the .list files to disk.
210func dumpListToFile(ctx Context, config Config, list []string, filePath string) (err error) {
211	desiredText := strings.Join(list, "\n")
212	desiredBytes := []byte(desiredText)
213	actualBytes, readErr := ioutil.ReadFile(filePath)
214	if readErr != nil || !bytes.Equal(desiredBytes, actualBytes) {
215		err = ioutil.WriteFile(filePath, desiredBytes, 0777)
216		if err != nil {
217			return err
218		}
219	}
220
221	distFile(ctx, config, filePath, "module_paths")
222
223	return nil
224}
225