1// Copyright 2023 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package android
16
17import (
18	"crypto/sha1"
19	"encoding/hex"
20	"fmt"
21	"github.com/google/blueprint"
22	"io"
23	"io/fs"
24	"os"
25	"path/filepath"
26	"strings"
27	"testing"
28
29	"github.com/google/blueprint/proptools"
30)
31
32// WriteFileRule creates a ninja rule to write contents to a file by immediately writing the
33// contents, plus a trailing newline, to a file in out/soong/raw-${TARGET_PRODUCT}, and then creating
34// a ninja rule to copy the file into place.
35func WriteFileRule(ctx BuilderContext, outputFile WritablePath, content string) {
36	writeFileRule(ctx, outputFile, content, true, false)
37}
38
39// WriteFileRuleVerbatim creates a ninja rule to write contents to a file by immediately writing the
40// contents to a file in out/soong/raw-${TARGET_PRODUCT}, and then creating a ninja rule to copy the file into place.
41func WriteFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) {
42	writeFileRule(ctx, outputFile, content, false, false)
43}
44
45// WriteExecutableFileRuleVerbatim is the same as WriteFileRuleVerbatim, but runs chmod +x on the result
46func WriteExecutableFileRuleVerbatim(ctx BuilderContext, outputFile WritablePath, content string) {
47	writeFileRule(ctx, outputFile, content, false, true)
48}
49
50// tempFile provides a testable wrapper around a file in out/soong/.temp.  It writes to a temporary file when
51// not in tests, but writes to a buffer in memory when used in tests.
52type tempFile struct {
53	// tempFile contains wraps an io.Writer, which will be file if testMode is false, or testBuf if it is true.
54	io.Writer
55
56	file    *os.File
57	testBuf *strings.Builder
58}
59
60func newTempFile(ctx BuilderContext, pattern string, testMode bool) *tempFile {
61	if testMode {
62		testBuf := &strings.Builder{}
63		return &tempFile{
64			Writer:  testBuf,
65			testBuf: testBuf,
66		}
67	} else {
68		f, err := os.CreateTemp(absolutePath(ctx.Config().tempDir()), pattern)
69		if err != nil {
70			panic(fmt.Errorf("failed to open temporary raw file: %w", err))
71		}
72		return &tempFile{
73			Writer: f,
74			file:   f,
75		}
76	}
77}
78
79func (t *tempFile) close() error {
80	if t.file != nil {
81		return t.file.Close()
82	}
83	return nil
84}
85
86func (t *tempFile) name() string {
87	if t.file != nil {
88		return t.file.Name()
89	}
90	return "temp_file_in_test"
91}
92
93func (t *tempFile) rename(to string) {
94	if t.file != nil {
95		os.MkdirAll(filepath.Dir(to), 0777)
96		err := os.Rename(t.file.Name(), to)
97		if err != nil {
98			panic(fmt.Errorf("failed to rename %s to %s: %w", t.file.Name(), to, err))
99		}
100	}
101}
102
103func (t *tempFile) remove() error {
104	if t.file != nil {
105		return os.Remove(t.file.Name())
106	}
107	return nil
108}
109
110func writeContentToTempFileAndHash(ctx BuilderContext, content string, newline bool) (*tempFile, string) {
111	tempFile := newTempFile(ctx, "raw", ctx.Config().captureBuild)
112	defer tempFile.close()
113
114	hash := sha1.New()
115	w := io.MultiWriter(tempFile, hash)
116
117	_, err := io.WriteString(w, content)
118	if err == nil && newline {
119		_, err = io.WriteString(w, "\n")
120	}
121	if err != nil {
122		panic(fmt.Errorf("failed to write to temporary raw file %s: %w", tempFile.name(), err))
123	}
124	return tempFile, hex.EncodeToString(hash.Sum(nil))
125}
126
127func writeFileRule(ctx BuilderContext, outputFile WritablePath, content string, newline bool, executable bool) {
128	// Write the contents to a temporary file while computing its hash.
129	tempFile, hash := writeContentToTempFileAndHash(ctx, content, newline)
130
131	// Shard the final location of the raw file into a subdirectory based on the first two characters of the
132	// hash to avoid making the raw directory too large and slowing down accesses.
133	relPath := filepath.Join(hash[0:2], hash)
134
135	// These files are written during soong_build.  If something outside the build deleted them there would be no
136	// trigger to rerun soong_build, and the build would break with dependencies on missing files.  Writing them
137	// to their final locations would risk having them deleted when cleaning a module, and would also pollute the
138	// output directory with files for modules that have never been built.
139	// Instead, the files are written to a separate "raw" directory next to the build.ninja file, and a ninja
140	// rule is created to copy the files into their final location as needed.
141	// Obsolete files written by previous runs of soong_build must be cleaned up to avoid continually growing
142	// disk usage as the hashes of the files change over time.  The cleanup must not remove files that were
143	// created by previous runs of soong_build for other products, as the build.ninja files for those products
144	// may still exist and still reference those files.  The raw files from different products are kept
145	// separate by appending the Make_suffix to the directory name.
146	rawPath := PathForOutput(ctx, "raw"+proptools.String(ctx.Config().productVariables.Make_suffix), relPath)
147
148	rawFileInfo := rawFileInfo{
149		relPath: relPath,
150	}
151
152	if ctx.Config().captureBuild {
153		// When running tests tempFile won't write to disk, instead store the contents for later retrieval by
154		// ContentFromFileRuleForTests.
155		rawFileInfo.contentForTests = tempFile.testBuf.String()
156	}
157
158	rawFileSet := getRawFileSet(ctx.Config())
159	if _, exists := rawFileSet.LoadOrStore(hash, rawFileInfo); exists {
160		// If a raw file with this hash has already been created delete the temporary file.
161		tempFile.remove()
162	} else {
163		// If this is the first time this hash has been seen then move it from the temporary directory
164		// to the raw directory.  If the file already exists in the raw directory assume it has the correct
165		// contents.
166		absRawPath := absolutePath(rawPath.String())
167		_, err := os.Stat(absRawPath)
168		if os.IsNotExist(err) {
169			tempFile.rename(absRawPath)
170		} else if err != nil {
171			panic(fmt.Errorf("failed to stat %q: %w", absRawPath, err))
172		} else {
173			tempFile.remove()
174		}
175	}
176
177	// Emit a rule to copy the file from raw directory to the final requested location in the output tree.
178	// Restat is used to ensure that two different products that produce identical files copied from their
179	// own raw directories they don't cause everything downstream to rebuild.
180	rule := rawFileCopy
181	if executable {
182		rule = rawFileCopyExecutable
183	}
184	ctx.Build(pctx, BuildParams{
185		Rule:        rule,
186		Input:       rawPath,
187		Output:      outputFile,
188		Description: "raw " + outputFile.Base(),
189	})
190}
191
192var (
193	rawFileCopy = pctx.AndroidStaticRule("rawFileCopy",
194		blueprint.RuleParams{
195			Command:     "if ! cmp -s $in $out; then cp $in $out; fi",
196			Description: "copy raw file $out",
197			Restat:      true,
198		})
199	rawFileCopyExecutable = pctx.AndroidStaticRule("rawFileCopyExecutable",
200		blueprint.RuleParams{
201			Command:     "if ! cmp -s $in $out; then cp $in $out; fi && chmod +x $out",
202			Description: "copy raw exectuable file $out",
203			Restat:      true,
204		})
205)
206
207type rawFileInfo struct {
208	relPath         string
209	contentForTests string
210}
211
212var rawFileSetKey OnceKey = NewOnceKey("raw file set")
213
214func getRawFileSet(config Config) *SyncMap[string, rawFileInfo] {
215	return config.Once(rawFileSetKey, func() any {
216		return &SyncMap[string, rawFileInfo]{}
217	}).(*SyncMap[string, rawFileInfo])
218}
219
220// ContentFromFileRuleForTests returns the content that was passed to a WriteFileRule for use
221// in tests.
222func ContentFromFileRuleForTests(t *testing.T, ctx *TestContext, params TestingBuildParams) string {
223	t.Helper()
224	if params.Rule != rawFileCopy && params.Rule != rawFileCopyExecutable {
225		t.Errorf("expected params.Rule to be rawFileCopy or rawFileCopyExecutable, was %q", params.Rule)
226		return ""
227	}
228
229	key := filepath.Base(params.Input.String())
230	rawFileSet := getRawFileSet(ctx.Config())
231	rawFileInfo, _ := rawFileSet.Load(key)
232
233	return rawFileInfo.contentForTests
234}
235
236func rawFilesSingletonFactory() Singleton {
237	return &rawFilesSingleton{}
238}
239
240type rawFilesSingleton struct{}
241
242func (rawFilesSingleton) GenerateBuildActions(ctx SingletonContext) {
243	if ctx.Config().captureBuild {
244		// Nothing to do when running in tests, no temporary files were created.
245		return
246	}
247	rawFileSet := getRawFileSet(ctx.Config())
248	rawFilesDir := PathForOutput(ctx, "raw"+proptools.String(ctx.Config().productVariables.Make_suffix)).String()
249	absRawFilesDir := absolutePath(rawFilesDir)
250	err := filepath.WalkDir(absRawFilesDir, func(path string, d fs.DirEntry, err error) error {
251		if err != nil {
252			return err
253		}
254		if d.IsDir() {
255			// Ignore obsolete directories for now.
256			return nil
257		}
258
259		// Assume the basename of the file is a hash
260		key := filepath.Base(path)
261		relPath, err := filepath.Rel(absRawFilesDir, path)
262		if err != nil {
263			return err
264		}
265
266		// Check if a file with the same hash was written by this run of soong_build.  If the file was not written,
267		// or if a file with the same hash was written but to a different path in the raw directory, then delete it.
268		// Checking that the path matches allows changing the structure of the raw directory, for example to increase
269		// the sharding.
270		rawFileInfo, written := rawFileSet.Load(key)
271		if !written || rawFileInfo.relPath != relPath {
272			os.Remove(path)
273		}
274		return nil
275	})
276	if err != nil {
277		panic(fmt.Errorf("failed to clean %q: %w", rawFilesDir, err))
278	}
279}
280