1// Copyright 2024 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14package tradefed_modules 15 16import ( 17 "android/soong/android" 18 "android/soong/java" 19 "fmt" 20 "strconv" 21 "strings" 22 "testing" 23 24 "github.com/google/blueprint" 25) 26 27const bp = ` 28 android_app { 29 name: "foo", 30 srcs: ["a.java"], 31 sdk_version: "current", 32 } 33 34 android_test_helper_app { 35 name: "HelperApp", 36 srcs: ["helper.java"], 37 } 38 39 android_test { 40 name: "base", 41 sdk_version: "current", 42 data: [":HelperApp", "data/testfile"], 43 } 44 45 test_module_config { 46 name: "derived_test", 47 base: "base", 48 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 49 include_annotations: ["android.platform.test.annotations.LargeTest"], 50 test_suites: ["general-tests"], 51 } 52 53` 54 55// Ensure we create files needed and set the AndroidMkEntries needed 56func TestModuleConfigAndroidTest(t *testing.T) { 57 58 ctx := android.GroupFixturePreparers( 59 java.PrepareForTestWithJavaDefaultModules, 60 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 61 ).RunTestWithBp(t, bp) 62 63 derived := ctx.ModuleForTests("derived_test", "android_common") 64 // Assert there are rules to create these files. 65 derived.Output("test_module_config.manifest") 66 derived.Output("test_config_fixer/derived_test.config") 67 68 // Ensure some basic rules exist. 69 ctx.ModuleForTests("base", "android_common").Output("package-res.apk") 70 entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0] 71 72 // Ensure some entries from base are there, specifically support files for data and helper apps. 73 // Do not use LOCAL_COMPATIBILITY_SUPPORT_FILES, but instead use LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES 74 android.AssertStringPathsRelativeToTopEquals(t, "support-files", ctx.Config, 75 []string{"out/soong/target/product/test_device/testcases/derived_test/arm64/base.apk", 76 "out/soong/target/product/test_device/testcases/derived_test/HelperApp.apk", 77 "out/soong/target/product/test_device/testcases/derived_test/data/testfile"}, 78 entries.EntryMap["LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES"]) 79 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_COMPATIBILITY_SUPPORT_FILES"], []string{}) 80 81 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_REQUIRED_MODULES"], []string{"base"}) 82 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_CERTIFICATE"], []string{"build/make/target/product/security/testkey.x509.pem"}) 83 android.AssertStringEquals(t, "", entries.Class, "APPS") 84 85 // And some new derived entries are there. 86 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE_TAGS"], []string{"tests"}) 87 88 android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], "derived_test/android_common/test_config_fixer/derived_test.config") 89 90 // Check the footer lines. Our support files should depend on base's support files. 91 convertedActual := make([]string, 5) 92 for i, e := range entries.FooterLinesForTests() { 93 // AssertStringPathsRelativeToTop doesn't replace both instances 94 convertedActual[i] = strings.Replace(e, ctx.Config.SoongOutDir(), "", 2) 95 } 96 android.AssertArrayString(t, fmt.Sprintf("%s", ctx.Config.SoongOutDir()), convertedActual, []string{ 97 "include $(BUILD_SYSTEM)/soong_app_prebuilt.mk", 98 "/target/product/test_device/testcases/derived_test/arm64/base.apk: /target/product/test_device/testcases/base/arm64/base.apk", 99 "/target/product/test_device/testcases/derived_test/HelperApp.apk: /target/product/test_device/testcases/base/HelperApp.apk", 100 "/target/product/test_device/testcases/derived_test/data/testfile: /target/product/test_device/testcases/base/data/testfile", 101 "", 102 }) 103} 104 105// Make sure we call test-config-fixer with the right args. 106func TestModuleConfigOptions(t *testing.T) { 107 108 ctx := android.GroupFixturePreparers( 109 java.PrepareForTestWithJavaDefaultModules, 110 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 111 ).RunTestWithBp(t, bp) 112 113 // Check that we generate a rule to make a new AndroidTest.xml/Module.config file. 114 derived := ctx.ModuleForTests("derived_test", "android_common") 115 rule_cmd := derived.Rule("fix_test_config").RuleParams.Command 116 android.AssertStringDoesContain(t, "Bad FixConfig rule inputs", rule_cmd, 117 `--test-runner-options='[{"Name":"exclude-filter","Key":"","Value":"android.test.example.devcodelab.DevCodelabTest#testHelloFail"},{"Name":"include-annotation","Key":"","Value":"android.platform.test.annotations.LargeTest"}]'`) 118} 119 120// Ensure we error for a base we don't support. 121func TestModuleConfigWithHostBaseShouldFailWithExplicitMessage(t *testing.T) { 122 badBp := ` 123 java_test_host { 124 name: "base", 125 srcs: ["a.java"], 126 } 127 128 test_module_config { 129 name: "derived_test", 130 base: "base", 131 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 132 include_annotations: ["android.platform.test.annotations.LargeTest"], 133 test_suites: ["general-tests"], 134 }` 135 136 android.GroupFixturePreparers( 137 java.PrepareForTestWithJavaDefaultModules, 138 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 139 ).ExtendWithErrorHandler( 140 android.FixtureExpectsAtLeastOneErrorMatchingPattern("'java_test_host' module used as base, but 'android_test' expected")). 141 RunTestWithBp(t, badBp) 142} 143 144func TestModuleConfigBadBaseShouldFailWithGeneralMessage(t *testing.T) { 145 badBp := ` 146 java_library { 147 name: "base", 148 srcs: ["a.java"], 149 } 150 151 test_module_config { 152 name: "derived_test", 153 base: "base", 154 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 155 include_annotations: ["android.platform.test.annotations.LargeTest"], 156 test_suites: ["general-tests"], 157 }` 158 159 android.GroupFixturePreparers( 160 java.PrepareForTestWithJavaDefaultModules, 161 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 162 ).ExtendWithErrorHandler( 163 android.FixtureExpectsOneErrorPattern("'base' module used as base but it is not a 'android_test' module.")). 164 RunTestWithBp(t, badBp) 165} 166 167func TestModuleConfigNoBaseShouldFail(t *testing.T) { 168 badBp := ` 169 java_library { 170 name: "base", 171 srcs: ["a.java"], 172 } 173 174 test_module_config { 175 name: "derived_test", 176 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 177 include_annotations: ["android.platform.test.annotations.LargeTest"], 178 test_suites: ["general-tests"], 179 }` 180 181 android.GroupFixturePreparers( 182 java.PrepareForTestWithJavaDefaultModules, 183 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 184 ).ExtendWithErrorHandler( 185 android.FixtureExpectsOneErrorPattern("'base' field must be set to a 'android_test' module.")). 186 RunTestWithBp(t, badBp) 187} 188 189// Ensure we error for a base we don't support. 190func TestModuleConfigNoFiltersOrAnnotationsShouldFail(t *testing.T) { 191 badBp := ` 192 android_test { 193 name: "base", 194 sdk_version: "current", 195 srcs: ["a.java"], 196 } 197 198 test_module_config { 199 name: "derived_test", 200 base: "base", 201 test_suites: ["general-tests"], 202 }` 203 204 ctx := android.GroupFixturePreparers( 205 java.PrepareForTestWithJavaDefaultModules, 206 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 207 ).ExtendWithErrorHandler( 208 android.FixtureExpectsAtLeastOneErrorMatchingPattern("Test options must be given")). 209 RunTestWithBp(t, badBp) 210 211 ctx.ModuleForTests("derived_test", "android_common") 212} 213 214func TestModuleConfigMultipleDerivedTestsWriteDistinctMakeEntries(t *testing.T) { 215 multiBp := ` 216 android_test { 217 name: "base", 218 sdk_version: "current", 219 srcs: ["a.java"], 220 data: [":HelperApp", "data/testfile"], 221 } 222 223 android_test_helper_app { 224 name: "HelperApp", 225 srcs: ["helper.java"], 226 } 227 228 test_module_config { 229 name: "derived_test", 230 base: "base", 231 include_annotations: ["android.platform.test.annotations.LargeTest"], 232 test_suites: ["general-tests"], 233 } 234 235 test_module_config { 236 name: "another_derived_test", 237 base: "base", 238 include_annotations: ["android.platform.test.annotations.LargeTest"], 239 test_suites: ["general-tests"], 240 }` 241 242 ctx := android.GroupFixturePreparers( 243 java.PrepareForTestWithJavaDefaultModules, 244 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 245 ).RunTestWithBp(t, multiBp) 246 247 { 248 derived := ctx.ModuleForTests("derived_test", "android_common") 249 entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0] 250 // All these should be the same in both derived tests 251 android.AssertStringPathsRelativeToTopEquals(t, "support-files", ctx.Config, 252 []string{"out/soong/target/product/test_device/testcases/derived_test/arm64/base.apk", 253 "out/soong/target/product/test_device/testcases/derived_test/HelperApp.apk", 254 "out/soong/target/product/test_device/testcases/derived_test/data/testfile"}, 255 entries.EntryMap["LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES"]) 256 257 // Except this one, which points to the updated tradefed xml file. 258 android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], "derived_test/android_common/test_config_fixer/derived_test.config") 259 // And this one, the module name. 260 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE"], []string{"derived_test"}) 261 } 262 263 { 264 derived := ctx.ModuleForTests("another_derived_test", "android_common") 265 entries := android.AndroidMkEntriesForTest(t, ctx.TestContext, derived.Module())[0] 266 // All these should be the same in both derived tests 267 android.AssertStringPathsRelativeToTopEquals(t, "support-files", ctx.Config, 268 []string{"out/soong/target/product/test_device/testcases/another_derived_test/arm64/base.apk", 269 "out/soong/target/product/test_device/testcases/another_derived_test/HelperApp.apk", 270 "out/soong/target/product/test_device/testcases/another_derived_test/data/testfile"}, 271 entries.EntryMap["LOCAL_SOONG_INSTALLED_COMPATIBILITY_SUPPORT_FILES"]) 272 // Except this one, which points to the updated tradefed xml file. 273 android.AssertStringMatches(t, "", entries.EntryMap["LOCAL_FULL_TEST_CONFIG"][0], "another_derived_test/android_common/test_config_fixer/another_derived_test.config") 274 // And this one, the module name. 275 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE"], []string{"another_derived_test"}) 276 } 277} 278 279// Test_module_config_host rule is allowed to depend on java_test_host 280func TestModuleConfigHostBasics(t *testing.T) { 281 bp := ` 282 java_test_host { 283 name: "base", 284 srcs: ["a.java"], 285 test_suites: ["suiteA", "general-tests", "suiteB"], 286 } 287 288 test_module_config_host { 289 name: "derived_test", 290 base: "base", 291 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 292 include_annotations: ["android.platform.test.annotations.LargeTest"], 293 test_suites: ["general-tests"], 294 }` 295 296 ctx := android.GroupFixturePreparers( 297 java.PrepareForTestWithJavaDefaultModules, 298 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 299 ).RunTestWithBp(t, bp) 300 301 variant := ctx.Config.BuildOS.String() + "_common" 302 derived := ctx.ModuleForTests("derived_test", variant) 303 mod := derived.Module().(*testModuleConfigHostModule) 304 allEntries := android.AndroidMkEntriesForTest(t, ctx.TestContext, mod) 305 entries := allEntries[0] 306 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_MODULE"], []string{"derived_test"}) 307 android.AssertArrayString(t, "", entries.EntryMap["LOCAL_SDK_VERSION"], []string{"private_current"}) 308 android.AssertStringEquals(t, "", entries.Class, "JAVA_LIBRARIES") 309 310 if !mod.Host() { 311 t.Errorf("host bit is not set for a java_test_host module.") 312 } 313 actualData, _ := strconv.ParseBool(entries.EntryMap["LOCAL_IS_UNIT_TEST"][0]) 314 android.AssertBoolEquals(t, "LOCAL_IS_UNIT_TEST", true, actualData) 315 316} 317 318// When you pass an 'android_test' as base, the warning message is a bit obscure, 319// talking about variants, but it is something. Ideally we could do better. 320func TestModuleConfigHostBadBaseShouldFailWithVariantWarning(t *testing.T) { 321 badBp := ` 322 android_test { 323 name: "base", 324 sdk_version: "current", 325 srcs: ["a.java"], 326 } 327 328 test_module_config_host { 329 name: "derived_test", 330 base: "base", 331 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 332 include_annotations: ["android.platform.test.annotations.LargeTest"], 333 }` 334 335 android.GroupFixturePreparers( 336 java.PrepareForTestWithJavaDefaultModules, 337 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 338 ).ExtendWithErrorHandler( 339 android.FixtureExpectsAtLeastOneErrorMatchingPattern("missing variant")). 340 RunTestWithBp(t, badBp) 341} 342 343func TestModuleConfigHostNeedsATestSuite(t *testing.T) { 344 badBp := ` 345 java_test_host { 346 name: "base", 347 srcs: ["a.java"], 348 } 349 350 test_module_config_host { 351 name: "derived_test", 352 base: "base", 353 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 354 include_annotations: ["android.platform.test.annotations.LargeTest"], 355 }` 356 357 android.GroupFixturePreparers( 358 java.PrepareForTestWithJavaDefaultModules, 359 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 360 ).ExtendWithErrorHandler( 361 android.FixtureExpectsAtLeastOneErrorMatchingPattern("At least one test-suite must be set")). 362 RunTestWithBp(t, badBp) 363} 364 365func TestTestOnlyProvider(t *testing.T) { 366 t.Parallel() 367 ctx := android.GroupFixturePreparers( 368 java.PrepareForTestWithJavaDefaultModules, 369 android.FixtureRegisterWithContext(RegisterTestModuleConfigBuildComponents), 370 ).RunTestWithBp(t, ` 371 // These should be test-only 372 test_module_config_host { 373 name: "host-derived-test", 374 base: "host-base", 375 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 376 include_annotations: ["android.platform.test.annotations.LargeTest"], 377 test_suites: ["general-tests"], 378 } 379 380 test_module_config { 381 name: "derived-test", 382 base: "base", 383 exclude_filters: ["android.test.example.devcodelab.DevCodelabTest#testHelloFail"], 384 include_annotations: ["android.platform.test.annotations.LargeTest"], 385 test_suites: ["general-tests"], 386 } 387 388 android_test { 389 name: "base", 390 sdk_version: "current", 391 data: ["data/testfile"], 392 } 393 394 java_test_host { 395 name: "host-base", 396 srcs: ["a.java"], 397 test_suites: ["general-tests"], 398 }`, 399 ) 400 401 // Visit all modules and ensure only the ones that should 402 // marked as test-only are marked as test-only. 403 404 actualTestOnly := []string{} 405 ctx.VisitAllModules(func(m blueprint.Module) { 406 if provider, ok := android.OtherModuleProvider(ctx.TestContext.OtherModuleProviderAdaptor(), m, android.TestOnlyProviderKey); ok { 407 if provider.TestOnly { 408 actualTestOnly = append(actualTestOnly, m.Name()) 409 } 410 } 411 }) 412 expectedTestOnlyModules := []string{ 413 "host-derived-test", 414 "derived-test", 415 // android_test and java_test_host are tests too. 416 "host-base", 417 "base", 418 } 419 420 notEqual, left, right := android.ListSetDifference(expectedTestOnlyModules, actualTestOnly) 421 if notEqual { 422 t.Errorf("test-only: Expected but not found: %v, Found but not expected: %v", left, right) 423 } 424} 425