1// Copyright 2024 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//	http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package tradefed_modules
15
16import (
17	"android/soong/android"
18	"android/soong/java"
19	"fmt"
20	"strconv"
21	"strings"
22	"testing"
23
24	"github.com/google/blueprint"
25)
26
27const bp = `
28		android_app {
29			name: "foo",
30			srcs: ["a.java"],
31			sdk_version: "current",
32		}
33
34                android_test_helper_app {
35                        name: "HelperApp",
36                        srcs: ["helper.java"],
37                }
38
39		android_test {
40			name: "base",
41			sdk_version: "current",
42                        data: [":HelperApp", "data/testfile"],
43		}
44
45                test_module_config {
46                        name: "derived_test",
47                        base: "base",
48                        exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"],
49                        include_annotations: ["android.platform.test.annotations.LargeTest"],
50                        test_suites: ["general-tests"],
51                }
52
53`
54
55// Ensure we create files needed and set the AndroidMkEntries needed
56func TestModuleConfigAndroidTest(t *testing.T) {
57
58	ctx := android.GroupFixturePreparers(
59		java.PrepareForTestWithJavaDefaultModules,
60		android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
61	).RunTestWithBp(t, bp)
62
63	derived := ctx.ModuleForTests("derived_test", "android_common")
64	// Assert there are rules to create these files.
65	derived.Output("test_module_config.manifest")
66	derived.Output("test_config_fixer/derived_test.config")
67
68	// Ensure some basic rules exist.
69	ctx.ModuleForTests("base", "android_common").Output("package-res.apk")
70	entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0]
71
72	// Ensure some entries from base are there, specifically support files for data and helper apps.
73	// Do not use LOCAL_COMPATIBILITY_SUPPORT_FILES, but instead use LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES
74	android.AssertStringPathsRelativeToTopEquals(t, "support-files", ctx.Config,
75		[]string{"out/soong/target/product/test_device/testcases/derived_test/arm64/base.apk",
76			"out/soong/target/product/test_device/testcases/derived_test/HelperApp.apk",
77			"out/soong/target/product/test_device/testcases/derived_test/data/testfile"},
78		entries.EntryMap["LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES"])
79	android.AssertArrayString(t, "", entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"], []string{})
80
81	android.AssertArrayString(t, "", entries.EntryMap["LOCAL_REQUIRED_MODULES"], []string{"base"})
82	android.AssertArrayString(t, "", entries.EntryMap["LOCAL_CERTIFICATE"], []string{"build/make/target/product/security/testkey.x509.pem"})
83	android.AssertStringEquals(t, "", entries.Class, "APPS")
84
85	// And some new derived entries are there.
86	android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE_TAGS"], []string{"tests"})
87
88	android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], "derived_test/android_common/test_config_fixer/derived_test.config")
89
90	// Check the footer lines.  Our support files should depend on base's support files.
91	convertedActual := make([]string, 5)
92	for i, e := range entries.FooterLinesForTests() {
93		// AssertStringPathsRelativeToTop doesn't replace both instances
94		convertedActual[i] = strings.Replace(e, ctx.Config.SoongOutDir(), "", 2)
95	}
96	android.AssertArrayString(t, fmt.Sprintf("%s", ctx.Config.SoongOutDir()), convertedActual, []string{
97		"include $(BUILD_SYSTEM)/soong_app_prebuilt.mk",
98		"/target/product/test_device/testcases/derived_test/arm64/base.apk: /target/product/test_device/testcases/base/arm64/base.apk",
99		"/target/product/test_device/testcases/derived_test/HelperApp.apk: /target/product/test_device/testcases/base/HelperApp.apk",
100		"/target/product/test_device/testcases/derived_test/data/testfile: /target/product/test_device/testcases/base/data/testfile",
101		"",
102	})
103}
104
105// Make sure we call test-config-fixer with the right args.
106func TestModuleConfigOptions(t *testing.T) {
107
108	ctx := android.GroupFixturePreparers(
109		java.PrepareForTestWithJavaDefaultModules,
110		android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
111	).RunTestWithBp(t, bp)
112
113	// Check that we generate a rule to make a new AndroidTest.xml/Module.config file.
114	derived := ctx.ModuleForTests("derived_test", "android_common")
115	rule_cmd := derived.Rule("fix_test_config").RuleParams.Command
116	android.AssertStringDoesContain(t, "Bad FixConfig rule inputs", rule_cmd,
117		`--test-runner-options='[{"Name":"exclude-filter","Key":"","Value":"android.test.example.devcodelab.DevCodelabTest#testHelloFail"},{"Name":"include-annotation","Key":"","Value":"android.platform.test.annotations.LargeTest"}]'`)
118}
119
120// Ensure we error for a base we don't support.
121func TestModuleConfigWithHostBaseShouldFailWithExplicitMessage(t *testing.T) {
122	badBp := `
123		java_test_host {
124			name: "base",
125                        srcs: ["a.java"],
126		}
127
128                test_module_config {
129                        name: "derived_test",
130                        base: "base",
131                        exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"],
132                        include_annotations: ["android.platform.test.annotations.LargeTest"],
133                        test_suites: ["general-tests"],
134                }`
135
136	android.GroupFixturePreparers(
137		java.PrepareForTestWithJavaDefaultModules,
138		android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
139	).ExtendWithErrorHandler(
140		android.FixtureExpectsAtLeastOneErrorMatchingPattern("'java_test_host' module used as base, but 'android_test' expected")).
141		RunTestWithBp(t, badBp)
142}
143
144func TestModuleConfigBadBaseShouldFailWithGeneralMessage(t *testing.T) {
145	badBp := `
146		java_library {
147			name: "base",
148                        srcs: ["a.java"],
149		}
150
151                test_module_config {
152                        name: "derived_test",
153                        base: "base",
154                        exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"],
155                        include_annotations: ["android.platform.test.annotations.LargeTest"],
156                        test_suites: ["general-tests"],
157                }`
158
159	android.GroupFixturePreparers(
160		java.PrepareForTestWithJavaDefaultModules,
161		android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
162	).ExtendWithErrorHandler(
163		android.FixtureExpectsOneErrorPattern("'base' module used as base but it is not a 'android_test' module.")).
164		RunTestWithBp(t, badBp)
165}
166
167func TestModuleConfigNoBaseShouldFail(t *testing.T) {
168	badBp := `
169		java_library {
170			name: "base",
171                        srcs: ["a.java"],
172		}
173
174                test_module_config {
175                        name: "derived_test",
176                        exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"],
177                        include_annotations: ["android.platform.test.annotations.LargeTest"],
178                        test_suites: ["general-tests"],
179                }`
180
181	android.GroupFixturePreparers(
182		java.PrepareForTestWithJavaDefaultModules,
183		android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
184	).ExtendWithErrorHandler(
185		android.FixtureExpectsOneErrorPattern("'base' field must be set to a 'android_test' module.")).
186		RunTestWithBp(t, badBp)
187}
188
189// Ensure we error for a base we don't support.
190func TestModuleConfigNoFiltersOrAnnotationsShouldFail(t *testing.T) {
191	badBp := `
192		android_test {
193			name: "base",
194			sdk_version: "current",
195                        srcs: ["a.java"],
196		}
197
198                test_module_config {
199                        name: "derived_test",
200                        base: "base",
201                        test_suites: ["general-tests"],
202                }`
203
204	ctx := android.GroupFixturePreparers(
205		java.PrepareForTestWithJavaDefaultModules,
206		android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
207	).ExtendWithErrorHandler(
208		android.FixtureExpectsAtLeastOneErrorMatchingPattern("Test options must be given")).
209		RunTestWithBp(t, badBp)
210
211	ctx.ModuleForTests("derived_test", "android_common")
212}
213
214func TestModuleConfigMultipleDerivedTestsWriteDistinctMakeEntries(t *testing.T) {
215	multiBp := `
216		android_test {
217			name: "base",
218			sdk_version: "current",
219                        srcs: ["a.java"],
220                        data: [":HelperApp", "data/testfile"],
221		}
222
223                android_test_helper_app {
224                        name: "HelperApp",
225                        srcs: ["helper.java"],
226                }
227
228                test_module_config {
229                        name: "derived_test",
230                        base: "base",
231                        include_annotations: ["android.platform.test.annotations.LargeTest"],
232                        test_suites: ["general-tests"],
233                }
234
235                test_module_config {
236                        name: "another_derived_test",
237                        base: "base",
238                        include_annotations: ["android.platform.test.annotations.LargeTest"],
239                        test_suites: ["general-tests"],
240                }`
241
242	ctx := android.GroupFixturePreparers(
243		java.PrepareForTestWithJavaDefaultModules,
244		android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
245	).RunTestWithBp(t, multiBp)
246
247	{
248		derived := ctx.ModuleForTests("derived_test", "android_common")
249		entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0]
250		// All these should be the same in both derived tests
251		android.AssertStringPathsRelativeToTopEquals(t, "support-files", ctx.Config,
252			[]string{"out/soong/target/product/test_device/testcases/derived_test/arm64/base.apk",
253				"out/soong/target/product/test_device/testcases/derived_test/HelperApp.apk",
254				"out/soong/target/product/test_device/testcases/derived_test/data/testfile"},
255			entries.EntryMap["LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES"])
256
257		// Except this one, which points to the updated tradefed xml file.
258		android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], "derived_test/android_common/test_config_fixer/derived_test.config")
259		// And this one, the module name.
260		android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE"], []string{"derived_test"})
261	}
262
263	{
264		derived := ctx.ModuleForTests("another_derived_test", "android_common")
265		entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0]
266		// All these should be the same in both derived tests
267		android.AssertStringPathsRelativeToTopEquals(t, "support-files", ctx.Config,
268			[]string{"out/soong/target/product/test_device/testcases/another_derived_test/arm64/base.apk",
269				"out/soong/target/product/test_device/testcases/another_derived_test/HelperApp.apk",
270				"out/soong/target/product/test_device/testcases/another_derived_test/data/testfile"},
271			entries.EntryMap["LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES"])
272		// Except this one, which points to the updated tradefed xml file.
273		android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], "another_derived_test/android_common/test_config_fixer/another_derived_test.config")
274		// And this one, the module name.
275		android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE"], []string{"another_derived_test"})
276	}
277}
278
279// Test_module_config_host rule is allowed to depend on java_test_host
280func TestModuleConfigHostBasics(t *testing.T) {
281	bp := `
282               java_test_host {
283                        name: "base",
284                        srcs: ["a.java"],
285                        test_suites: ["suiteA", "general-tests",  "suiteB"],
286               }
287
288                test_module_config_host {
289                        name: "derived_test",
290                        base: "base",
291                        exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"],
292                        include_annotations: ["android.platform.test.annotations.LargeTest"],
293                        test_suites: ["general-tests"],
294                }`
295
296	ctx := android.GroupFixturePreparers(
297		java.PrepareForTestWithJavaDefaultModules,
298		android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
299	).RunTestWithBp(t, bp)
300
301	variant := ctx.Config.BuildOS.String() + "_common"
302	derived := ctx.ModuleForTests("derived_test", variant)
303	mod := derived.Module().(*testModuleConfigHostModule)
304	allEntries := android.AndroidMkEntriesForTest(t, ctx.TestContext, mod)
305	entries := allEntries[0]
306	android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE"], []string{"derived_test"})
307	android.AssertArrayString(t, "", entries.EntryMap["LOCAL_SDK_VERSION"], []string{"private_current"})
308	android.AssertStringEquals(t, "", entries.Class, "JAVA_LIBRARIES")
309
310	if !mod.Host() {
311		t.Errorf("host bit is not set for a java_test_host module.")
312	}
313	actualData, _ := strconv.ParseBool(entries.EntryMap["LOCAL_IS_UNIT_TEST"][0])
314	android.AssertBoolEquals(t, "LOCAL_IS_UNIT_TEST", true, actualData)
315
316}
317
318// When you pass an 'android_test' as base, the warning message is a bit obscure,
319// talking about variants, but it is something.  Ideally we could do better.
320func TestModuleConfigHostBadBaseShouldFailWithVariantWarning(t *testing.T) {
321	badBp := `
322		android_test {
323			name: "base",
324			sdk_version: "current",
325                        srcs: ["a.java"],
326		}
327
328                test_module_config_host {
329                        name: "derived_test",
330                        base: "base",
331                        exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"],
332                        include_annotations: ["android.platform.test.annotations.LargeTest"],
333                }`
334
335	android.GroupFixturePreparers(
336		java.PrepareForTestWithJavaDefaultModules,
337		android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
338	).ExtendWithErrorHandler(
339		android.FixtureExpectsAtLeastOneErrorMatchingPattern("missing variant")).
340		RunTestWithBp(t, badBp)
341}
342
343func TestModuleConfigHostNeedsATestSuite(t *testing.T) {
344	badBp := `
345		java_test_host {
346			name: "base",
347                        srcs: ["a.java"],
348		}
349
350                test_module_config_host {
351                        name: "derived_test",
352                        base: "base",
353                        exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"],
354                        include_annotations: ["android.platform.test.annotations.LargeTest"],
355                }`
356
357	android.GroupFixturePreparers(
358		java.PrepareForTestWithJavaDefaultModules,
359		android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
360	).ExtendWithErrorHandler(
361		android.FixtureExpectsAtLeastOneErrorMatchingPattern("At least one test-suite must be set")).
362		RunTestWithBp(t, badBp)
363}
364
365func TestTestOnlyProvider(t *testing.T) {
366	t.Parallel()
367	ctx := android.GroupFixturePreparers(
368		java.PrepareForTestWithJavaDefaultModules,
369		android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents),
370	).RunTestWithBp(t, `
371                // These should be test-only
372                test_module_config_host {
373                        name: "host-derived-test",
374                        base: "host-base",
375                        exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"],
376                        include_annotations: ["android.platform.test.annotations.LargeTest"],
377                        test_suites: ["general-tests"],
378                }
379
380                test_module_config {
381                        name: "derived-test",
382                        base: "base",
383                        exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"],
384                        include_annotations: ["android.platform.test.annotations.LargeTest"],
385                        test_suites: ["general-tests"],
386                }
387
388		android_test {
389			name: "base",
390			sdk_version: "current",
391                        data: ["data/testfile"],
392		}
393
394		java_test_host {
395			name: "host-base",
396                        srcs: ["a.java"],
397                        test_suites: ["general-tests"],
398		}`,
399	)
400
401	// Visit all modules and ensure only the ones that should
402	// marked as test-only are marked as test-only.
403
404	actualTestOnly := []string{}
405	ctx.VisitAllModules(func(m blueprint.Module) {
406		if provider, ok := android.OtherModuleProvider(ctx.TestContext.OtherModuleProviderAdaptor(), m, android.TestOnlyProviderKey); ok {
407			if provider.TestOnly {
408				actualTestOnly = append(actualTestOnly, m.Name())
409			}
410		}
411	})
412	expectedTestOnlyModules := []string{
413		"host-derived-test",
414		"derived-test",
415		// android_test and java_test_host are tests too.
416		"host-base",
417		"base",
418	}
419
420	notEqual, left, right := android.ListSetDifference(expectedTestOnlyModules, actualTestOnly)
421	if notEqual {
422		t.Errorf("test-only: Expected but not found: %v, Found but not expected: %v", left, right)
423	}
424}
425