1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.result.suite; 17 18 import static org.junit.Assert.assertEquals; 19 import static org.junit.Assert.assertFalse; 20 import static org.junit.Assert.assertNotNull; 21 import static org.junit.Assert.assertTrue; 22 23 import com.android.tradefed.build.BuildInfo; 24 import com.android.tradefed.invoker.IInvocationContext; 25 import com.android.tradefed.invoker.InvocationContext; 26 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 27 import com.android.tradefed.result.FailureDescription; 28 import com.android.tradefed.result.LogDataType; 29 import com.android.tradefed.result.LogFile; 30 import com.android.tradefed.result.TestDescription; 31 import com.android.tradefed.result.TestResult; 32 import com.android.tradefed.result.TestRunResult; 33 import com.android.tradefed.result.TestStatus; 34 import com.android.tradefed.result.error.TestErrorIdentifier; 35 import com.android.tradefed.testtype.Abi; 36 import com.android.tradefed.testtype.IAbi; 37 import com.android.tradefed.util.FileUtil; 38 import com.android.tradefed.util.proto.TfMetricProtoUtil; 39 40 import org.junit.After; 41 import org.junit.Before; 42 import org.junit.Test; 43 import org.junit.runner.RunWith; 44 import org.junit.runners.JUnit4; 45 import org.w3c.dom.Element; 46 import org.w3c.dom.NodeList; 47 import org.xml.sax.InputSource; 48 49 import java.io.File; 50 import java.io.StringReader; 51 import java.util.ArrayList; 52 import java.util.Arrays; 53 import java.util.Collection; 54 import java.util.HashMap; 55 import java.util.List; 56 import java.util.Map; 57 import java.util.Map.Entry; 58 59 import javax.xml.xpath.XPath; 60 import javax.xml.xpath.XPathConstants; 61 import javax.xml.xpath.XPathExpressionException; 62 import javax.xml.xpath.XPathFactory; 63 64 /** Unit tests for {@link XmlSuiteResultFormatter}. */ 65 @RunWith(JUnit4.class) 66 public class XmlSuiteResultFormatterTest { 67 private XmlSuiteResultFormatter mFormatter; 68 private SuiteResultHolder mResultHolder; 69 private IInvocationContext mContext; 70 private File mResultDir; 71 72 @Before setUp()73 public void setUp() throws Exception { 74 mFormatter = 75 new XmlSuiteResultFormatter() { 76 @Override 77 protected String sanitizeXmlContent(String s) { 78 // Stub sanitize to avoid some versioning issues 79 return s.replace("\0", "0"); 80 } 81 }; 82 mResultHolder = new SuiteResultHolder(); 83 mContext = new InvocationContext(); 84 mResultDir = FileUtil.createTempDir("result-dir"); 85 } 86 87 @After tearDown()88 public void tearDown() throws Exception { 89 FileUtil.recursiveDelete(mResultDir); 90 } 91 92 /** Check that the basic overall structure is good an contains all the information. */ 93 @Test testBasicFormat()94 public void testBasicFormat() throws Exception { 95 mResultHolder.context = mContext; 96 // Ensure attributes are escaped if needed 97 mContext.addInvocationAttribute("test-type-count:JarHostTest", "5"); 98 99 Collection<TestRunResult> runResults = new ArrayList<>(); 100 runResults.add(createFakeResult("module1", 2, 0, 0, 0)); 101 runResults.add(createFakeResult("module2", 1, 0, 0, 0)); 102 mResultHolder.runResults = runResults; 103 104 Map<String, IAbi> modulesAbi = new HashMap<>(); 105 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 106 modulesAbi.put("module2", new Abi("armeabi-v7a", "32")); 107 mResultHolder.modulesAbi = modulesAbi; 108 109 mResultHolder.completeModules = 2; 110 mResultHolder.totalModules = 2; 111 mResultHolder.passedTests = 2; 112 mResultHolder.failedTests = 0; 113 mResultHolder.startTime = 0L; 114 mResultHolder.endTime = 10L; 115 File res = mFormatter.writeResults(mResultHolder, mResultDir); 116 String content = FileUtil.readStringFromFile(res); 117 assertXmlContainsNode(content, "Result"); 118 // Verify that RunHistory tag should not exist if run history is empty. 119 assertXmlNotContainNode(content, "Result/RunHistory"); 120 // Verify that the summary has been populated 121 assertXmlContainsNode(content, "Result/Summary"); 122 assertXmlContainsAttribute(content, "Result/Summary", "pass", "2"); 123 assertXmlContainsAttribute(content, "Result/Summary", "failed", "0"); 124 assertXmlContainsAttribute(content, "Result/Summary", "modules_done", "2"); 125 assertXmlContainsAttribute(content, "Result/Summary", "modules_total", "2"); 126 // Verify that each module results are available 127 assertXmlContainsNode(content, "Result/Module"); 128 assertXmlContainsAttribute(content, "Result/Module", "name", "module1"); 129 assertXmlContainsAttribute(content, "Result/Module", "abi", "armeabi-v7a"); 130 assertXmlContainsAttribute(content, "Result/Module", "runtime", "10"); 131 assertXmlContainsAttribute(content, "Result/Module", "done", "true"); 132 assertXmlContainsAttribute(content, "Result/Module", "pass", "2"); 133 // Verify the test cases that passed are present 134 assertXmlContainsNode(content, "Result/Module/TestCase"); 135 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 136 assertXmlContainsAttribute( 137 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 138 assertXmlContainsAttribute( 139 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 140 141 assertXmlContainsAttribute(content, "Result/Module", "name", "module2"); 142 assertXmlContainsAttribute(content, "Result/Module", "pass", "1"); 143 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module2"); 144 assertXmlContainsAttribute( 145 content, "Result/Module/TestCase/Test", "name", "module2.method0"); 146 } 147 148 /** Check that the test failures are properly reported. */ 149 @Test testFailuresReporting()150 public void testFailuresReporting() throws Exception { 151 mResultHolder.context = mContext; 152 153 Collection<TestRunResult> runResults = new ArrayList<>(); 154 runResults.add(createFakeResult("module1", 2, 1, 0, 0)); 155 mResultHolder.runResults = runResults; 156 157 Map<String, IAbi> modulesAbi = new HashMap<>(); 158 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 159 mResultHolder.modulesAbi = modulesAbi; 160 161 mResultHolder.completeModules = 2; 162 mResultHolder.totalModules = 1; 163 mResultHolder.passedTests = 2; 164 mResultHolder.failedTests = 1; 165 mResultHolder.startTime = 0L; 166 mResultHolder.endTime = 10L; 167 File res = mFormatter.writeResults(mResultHolder, mResultDir); 168 String content = FileUtil.readStringFromFile(res); 169 170 assertXmlContainsNode(content, "Result/Module"); 171 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 172 assertXmlContainsAttribute( 173 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 174 assertXmlContainsAttribute( 175 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 176 // Check that failures are showing in the xml for the test cases with error identifiers 177 assertXmlContainsAttribute( 178 content, "Result/Module/TestCase/Test", "name", "module1.failed0"); 179 assertXmlContainsAttribute(content, "Result/Module/TestCase/Test", "result", "fail"); 180 assertXmlContainsAttribute( 181 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 182 assertXmlContainsAttribute( 183 content, 184 "Result/Module/TestCase/Test/Failure", 185 "error_name", 186 TestErrorIdentifier.TEST_ABORTED.name()); 187 assertXmlContainsAttribute( 188 content, 189 "Result/Module/TestCase/Test/Failure", 190 "error_code", 191 Long.toString(TestErrorIdentifier.TEST_ABORTED.code())); 192 assertXmlContainsValue( 193 content, 194 "Result/Module/TestCase/Test/Failure/StackTrace", 195 mFormatter.sanitizeXmlContent("module1 failed.\nstack\nstack\0")); 196 // Test that we can read back the informations 197 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 198 assertEquals(holder.completeModules, mResultHolder.completeModules); 199 assertEquals(holder.totalModules, mResultHolder.totalModules); 200 assertEquals(holder.passedTests, mResultHolder.passedTests); 201 assertEquals(holder.failedTests, mResultHolder.failedTests); 202 assertEquals(holder.startTime, mResultHolder.startTime); 203 assertEquals(holder.endTime, mResultHolder.endTime); 204 assertEquals( 205 holder.modulesAbi.get("armeabi-v7a module1"), 206 mResultHolder.modulesAbi.get("module1")); 207 assertEquals(holder.runResults.size(), mResultHolder.runResults.size()); 208 } 209 210 @Test testFailuresReporting_notDone()211 public void testFailuresReporting_notDone() throws Exception { 212 mResultHolder.context = mContext; 213 214 List<TestRunResult> runResults = new ArrayList<>(); 215 runResults.add(createFakeResult("module1", 2, 1, 0, 0)); 216 FailureDescription failureDescription = 217 FailureDescription.create("Failed module") 218 .setErrorIdentifier(TestErrorIdentifier.TEST_ABORTED); 219 runResults.get(0).testRunFailed(failureDescription); 220 mResultHolder.runResults = runResults; 221 222 Map<String, IAbi> modulesAbi = new HashMap<>(); 223 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 224 mResultHolder.modulesAbi = modulesAbi; 225 226 mResultHolder.completeModules = 2; 227 mResultHolder.totalModules = 1; 228 mResultHolder.passedTests = 2; 229 mResultHolder.failedTests = 1; 230 mResultHolder.startTime = 0L; 231 mResultHolder.endTime = 10L; 232 File res = mFormatter.writeResults(mResultHolder, mResultDir); 233 String content = FileUtil.readStringFromFile(res); 234 235 assertXmlContainsNode(content, "Result/Module"); 236 assertXmlContainsAttribute(content, "Result/Module", "done", "false"); 237 assertXmlContainsNode(content, "Result/Module/Reason"); 238 // Check that Reason contains error identifiers 239 assertXmlContainsAttribute( 240 content, 241 "Result/Module/Reason", 242 "error_name", 243 TestErrorIdentifier.TEST_ABORTED.name()); 244 assertXmlContainsAttribute( 245 content, 246 "Result/Module/Reason", 247 "error_code", 248 Long.toString(TestErrorIdentifier.TEST_ABORTED.code())); 249 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 250 assertXmlContainsAttribute( 251 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 252 assertXmlContainsAttribute( 253 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 254 // Check that failures are showing in the xml for the test cases 255 assertXmlContainsAttribute( 256 content, "Result/Module/TestCase/Test", "name", "module1.failed0"); 257 assertXmlContainsAttribute(content, "Result/Module/TestCase/Test", "result", "fail"); 258 assertXmlContainsAttribute( 259 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 260 assertXmlContainsValue( 261 content, 262 "Result/Module/TestCase/Test/Failure/StackTrace", 263 mFormatter.sanitizeXmlContent("module1 failed.\nstack\nstack\0")); 264 // Test that we can read back the informations 265 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 266 assertEquals(holder.completeModules, mResultHolder.completeModules); 267 assertEquals(holder.totalModules, mResultHolder.totalModules); 268 assertEquals(holder.passedTests, mResultHolder.passedTests); 269 assertEquals(holder.failedTests, mResultHolder.failedTests); 270 assertEquals(holder.startTime, mResultHolder.startTime); 271 assertEquals(holder.endTime, mResultHolder.endTime); 272 assertEquals( 273 holder.modulesAbi.get("armeabi-v7a module1"), 274 mResultHolder.modulesAbi.get("module1")); 275 assertEquals(1, holder.runResults.size()); 276 TestRunResult result = new ArrayList<>(holder.runResults).get(0); 277 assertFalse(result.isRunComplete()); 278 } 279 280 @Test testFailuresReporting_largeStackTrace()281 public void testFailuresReporting_largeStackTrace() throws Exception { 282 mResultHolder.context = mContext; 283 284 List<TestRunResult> runResults = new ArrayList<>(); 285 runResults.add(createFakeResult("module1", 2, 1, 0, 0, 1024 * 1024, false, false)); 286 mResultHolder.runResults = runResults; 287 288 Map<String, IAbi> modulesAbi = new HashMap<>(); 289 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 290 mResultHolder.modulesAbi = modulesAbi; 291 292 mResultHolder.completeModules = 2; 293 mResultHolder.totalModules = 1; 294 mResultHolder.passedTests = 2; 295 mResultHolder.failedTests = 1; 296 mResultHolder.startTime = 0L; 297 mResultHolder.endTime = 10L; 298 File res = mFormatter.writeResults(mResultHolder, mResultDir); 299 String content = FileUtil.readStringFromFile(res); 300 301 assertXmlContainsNode(content, "Result/Module"); 302 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 303 assertXmlContainsAttribute( 304 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 305 assertXmlContainsAttribute( 306 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 307 // Check that failures are showing in the xml for the test cases with error identifiers 308 assertXmlContainsAttribute( 309 content, "Result/Module/TestCase/Test", "name", "module1.failed0"); 310 assertXmlContainsAttribute(content, "Result/Module/TestCase/Test", "result", "fail"); 311 assertXmlContainsAttribute( 312 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 313 assertXmlContainsAttribute( 314 content, 315 "Result/Module/TestCase/Test/Failure", 316 "error_name", 317 TestErrorIdentifier.TEST_ABORTED.name()); 318 assertXmlContainsAttribute( 319 content, 320 "Result/Module/TestCase/Test/Failure", 321 "error_code", 322 Long.toString(TestErrorIdentifier.TEST_ABORTED.code())); 323 assertXmlContainsValue( 324 content, 325 "Result/Module/TestCase/Test/Failure/StackTrace", 326 mFormatter.sanitizeXmlContent("module1 failed." + "\nstack".repeat(174760) + "\n")); 327 // Test that we can read back the informations 328 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 329 assertEquals(holder.completeModules, mResultHolder.completeModules); 330 assertEquals(holder.totalModules, mResultHolder.totalModules); 331 assertEquals(holder.passedTests, mResultHolder.passedTests); 332 assertEquals(holder.failedTests, mResultHolder.failedTests); 333 assertEquals(holder.startTime, mResultHolder.startTime); 334 assertEquals(holder.endTime, mResultHolder.endTime); 335 assertEquals( 336 holder.modulesAbi.get("armeabi-v7a module1"), 337 mResultHolder.modulesAbi.get("module1")); 338 assertEquals(holder.runResults.size(), mResultHolder.runResults.size()); 339 } 340 341 /** Test that assumption failures and ignored tests are correctly reported in the xml. */ 342 @Test testAssumptionFailures_Ignore_Reporting()343 public void testAssumptionFailures_Ignore_Reporting() throws Exception { 344 mResultHolder.context = mContext; 345 346 Collection<TestRunResult> runResults = new ArrayList<>(); 347 runResults.add(createFakeResult("module1", 2, 0, 1, 1)); 348 mResultHolder.runResults = runResults; 349 350 Map<String, IAbi> modulesAbi = new HashMap<>(); 351 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 352 mResultHolder.modulesAbi = modulesAbi; 353 354 mResultHolder.completeModules = 1; 355 mResultHolder.totalModules = 1; 356 mResultHolder.passedTests = 2; 357 mResultHolder.failedTests = 0L; 358 mResultHolder.startTime = 0L; 359 mResultHolder.endTime = 10L; 360 File res = mFormatter.writeResults(mResultHolder, mResultDir); 361 String content = FileUtil.readStringFromFile(res); 362 363 assertXmlContainsNode(content, "Result/Module"); 364 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 365 assertXmlContainsAttribute( 366 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 367 assertXmlContainsAttribute( 368 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 369 // Check that failures are showing in the xml for the test cases 370 assertXmlContainsAttribute( 371 content, "Result/Module/TestCase/Test", "name", "module1.assumpFail0"); 372 assertXmlContainsAttribute( 373 content, "Result/Module/TestCase/Test", "result", "ASSUMPTION_FAILURE"); 374 assertXmlContainsAttribute( 375 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 376 assertXmlContainsValue( 377 content, 378 "Result/Module/TestCase/Test/Failure/StackTrace", 379 "module1 failed.\nstack\nstack"); 380 // Test that we can read back the informations 381 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 382 assertEquals(holder.completeModules, mResultHolder.completeModules); 383 assertEquals(holder.totalModules, mResultHolder.totalModules); 384 assertEquals(holder.passedTests, mResultHolder.passedTests); 385 assertEquals(holder.failedTests, mResultHolder.failedTests); 386 assertEquals(holder.startTime, mResultHolder.startTime); 387 assertEquals(holder.endTime, mResultHolder.endTime); 388 assertEquals( 389 holder.modulesAbi.get("armeabi-v7a module1"), 390 mResultHolder.modulesAbi.get("module1")); 391 assertEquals(holder.runResults.size(), mResultHolder.runResults.size()); 392 393 // Test that the results are loadable with expected values 394 SuiteResultHolder reloaded = mFormatter.parseResults(mResultDir, false); 395 assertEquals(1, reloaded.runResults.size()); 396 TestRunResult result = reloaded.runResults.iterator().next(); 397 assertEquals(0, result.getNumTestsInState(TestStatus.FAILURE)); 398 assertEquals(1, result.getNumTestsInState(TestStatus.ASSUMPTION_FAILURE)); 399 assertEquals(1, result.getNumTestsInState(TestStatus.IGNORED)); 400 } 401 402 /** Check that the logs for each test case are reported. */ 403 @Test testLogReporting()404 public void testLogReporting() throws Exception { 405 mResultHolder.context = mContext; 406 407 Collection<TestRunResult> runResults = new ArrayList<>(); 408 runResults.add(createResultWithLog("armeabi-v7a module1", 1, LogDataType.LOGCAT)); 409 runResults.add(createResultWithLog("module2", 1, LogDataType.BUGREPORT)); 410 runResults.add(createResultWithLog("module3", 1, LogDataType.PNG)); 411 mResultHolder.runResults = runResults; 412 413 Map<String, IAbi> modulesAbi = new HashMap<>(); 414 modulesAbi.put("armeabi-v7a module1", new Abi("armeabi-v7a", "32")); 415 mResultHolder.modulesAbi = modulesAbi; 416 417 mResultHolder.completeModules = 2; 418 mResultHolder.totalModules = 2; 419 mResultHolder.passedTests = 2; 420 mResultHolder.failedTests = 0; 421 mResultHolder.startTime = 0L; 422 mResultHolder.endTime = 10L; 423 File res = mFormatter.writeResults(mResultHolder, mResultDir); 424 String content = FileUtil.readStringFromFile(res); 425 // One logcat and one bugreport are found in the report 426 assertXmlContainsValue( 427 content, "Result/Module/TestCase/Test/Logcat", "http:url/armeabi-v7a module1"); 428 assertXmlContainsValue( 429 content, "Result/Module/TestCase/Test/BugReport", "http:url/module2"); 430 assertXmlContainsValue( 431 content, "Result/Module/TestCase/Test/Screenshot", "http:url/module3"); 432 433 // Test that we can read back the informations for log files 434 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 435 assertEquals( 436 holder.modulesAbi.get("armeabi-v7a module1"), 437 mResultHolder.modulesAbi.get("armeabi-v7a module1")); 438 assertEquals(holder.runResults.size(), mResultHolder.runResults.size()); 439 for (TestRunResult result : holder.runResults) { 440 TestDescription description = 441 new TestDescription( 442 "com.class." + result.getName(), result.getName() + ".method0"); 443 // Check that we reloaded the logged files. 444 assertTrue( 445 result.getTestResults() 446 .get(description) 447 .getLoggedFiles() 448 .get(result.getName() + "log0") 449 != null); 450 } 451 } 452 453 /** Check that the metrics with test cases are properly reported. */ 454 @Test testMetricReporting()455 public void testMetricReporting() throws Exception { 456 mResultHolder.context = mContext; 457 458 Collection<TestRunResult> runResults = new ArrayList<>(); 459 runResults.add(createFakeResult("module1", 2, 1, 0, 0, true, false)); 460 mResultHolder.runResults = runResults; 461 462 Map<String, IAbi> modulesAbi = new HashMap<>(); 463 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 464 mResultHolder.modulesAbi = modulesAbi; 465 466 mResultHolder.completeModules = 1; 467 mResultHolder.totalModules = 1; 468 mResultHolder.passedTests = 2; 469 mResultHolder.failedTests = 1; 470 mResultHolder.startTime = 0L; 471 mResultHolder.endTime = 10L; 472 File res = mFormatter.writeResults(mResultHolder, mResultDir); 473 String content = FileUtil.readStringFromFile(res); 474 475 assertXmlContainsNode(content, "Result/Module"); 476 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 477 assertXmlContainsAttribute( 478 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 479 assertXmlContainsAttribute( 480 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 481 // Check that failures are showing in the xml for the test cases 482 assertXmlContainsAttribute( 483 content, "Result/Module/TestCase/Test", "name", "module1.failed0"); 484 assertXmlContainsAttribute(content, "Result/Module/TestCase/Test", "result", "fail"); 485 assertXmlContainsAttribute( 486 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 487 // Test that we can read back the informations 488 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 489 assertEquals(holder.completeModules, mResultHolder.completeModules); 490 assertEquals(holder.totalModules, mResultHolder.totalModules); 491 assertEquals(holder.passedTests, mResultHolder.passedTests); 492 assertEquals(holder.failedTests, mResultHolder.failedTests); 493 assertEquals(holder.startTime, mResultHolder.startTime); 494 assertEquals(holder.endTime, mResultHolder.endTime); 495 assertEquals( 496 holder.modulesAbi.get("armeabi-v7a module1"), 497 mResultHolder.modulesAbi.get("module1")); 498 assertEquals(1, holder.runResults.size()); 499 TestRunResult result = holder.runResults.iterator().next(); 500 // 2 passed tests and 1 failed with metrics 501 assertEquals(3, result.getTestResults().size()); 502 assertEquals(1, result.getNumTestsInState(TestStatus.FAILURE)); 503 for (Entry<TestDescription, TestResult> entry : result.getTestResults().entrySet()) { 504 if (TestStatus.FAILURE.equals(entry.getValue().getResultStatus())) { 505 assertEquals("value00", entry.getValue().getMetrics().get("metric00")); 506 assertEquals("value10", entry.getValue().getMetrics().get("metric10")); 507 } 508 } 509 } 510 511 /** Test that the device format is properly done. */ 512 @Test testDeviceSerials()513 public void testDeviceSerials() throws Exception { 514 mResultHolder.context = mContext; 515 mResultHolder.context.addSerialsFromShard(0, Arrays.asList("serial1", "serial2")); 516 mResultHolder.context.addSerialsFromShard(1, Arrays.asList("serial3", "serial4")); 517 518 Collection<TestRunResult> runResults = new ArrayList<>(); 519 runResults.add(createFakeResult("module1", 2, 0, 0, 0)); 520 runResults.add(createFakeResult("module2", 1, 0, 0, 0)); 521 mResultHolder.runResults = runResults; 522 523 Map<String, IAbi> modulesAbi = new HashMap<>(); 524 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 525 modulesAbi.put("module2", new Abi("armeabi-v7a", "32")); 526 mResultHolder.modulesAbi = modulesAbi; 527 528 mResultHolder.completeModules = 2; 529 mResultHolder.totalModules = 2; 530 mResultHolder.passedTests = 2; 531 mResultHolder.failedTests = 0; 532 mResultHolder.startTime = 0L; 533 mResultHolder.endTime = 10L; 534 File res = mFormatter.writeResults(mResultHolder, mResultDir); 535 String content = FileUtil.readStringFromFile(res); 536 assertXmlContainsNode(content, "Result"); 537 assertXmlContainsAttribute(content, "Result", "devices", "serial1,serial2,serial3,serial4"); 538 } 539 540 /** Test writing then loading a shallow representation of the results. */ 541 @Test testBasicFormat_shallow()542 public void testBasicFormat_shallow() throws Exception { 543 mResultHolder.context = mContext; 544 545 Collection<TestRunResult> runResults = new ArrayList<>(); 546 runResults.add(createFakeResult("module1", 2, 1, 0, 0, true, false)); 547 mResultHolder.runResults = runResults; 548 549 Map<String, IAbi> modulesAbi = new HashMap<>(); 550 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 551 mResultHolder.modulesAbi = modulesAbi; 552 553 mResultHolder.completeModules = 1; 554 mResultHolder.totalModules = 1; 555 mResultHolder.passedTests = 2; 556 mResultHolder.failedTests = 1; 557 mResultHolder.startTime = 0L; 558 mResultHolder.endTime = 10L; 559 File res = mFormatter.writeResults(mResultHolder, mResultDir); 560 String content = FileUtil.readStringFromFile(res); 561 562 assertXmlContainsNode(content, "Result/Module"); 563 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 564 assertXmlContainsAttribute( 565 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 566 assertXmlContainsAttribute( 567 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 568 // Check that failures are showing in the xml for the test cases 569 assertXmlContainsAttribute( 570 content, "Result/Module/TestCase/Test", "name", "module1.failed0"); 571 assertXmlContainsAttribute(content, "Result/Module/TestCase/Test", "result", "fail"); 572 assertXmlContainsAttribute( 573 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 574 assertXmlContainsValue( 575 content, 576 "Result/Module/TestCase/Test/Failure/StackTrace", 577 mFormatter.sanitizeXmlContent("module1 failed.\nstack\nstack\0")); 578 // Test that we can read back the informations 579 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, true); 580 assertEquals(holder.completeModules, mResultHolder.completeModules); 581 assertEquals(holder.totalModules, mResultHolder.totalModules); 582 assertEquals(holder.passedTests, mResultHolder.passedTests); 583 assertEquals(holder.failedTests, mResultHolder.failedTests); 584 assertEquals(holder.startTime, mResultHolder.startTime); 585 assertEquals(holder.endTime, mResultHolder.endTime); 586 587 // Shallow loading doesn't load complex run informations. 588 assertTrue(holder.runResults == null); 589 assertTrue(holder.modulesAbi == null); 590 } 591 592 @Test testMetricReporting_badKey()593 public void testMetricReporting_badKey() throws Exception { 594 mResultHolder.context = mContext; 595 596 Collection<TestRunResult> runResults = new ArrayList<>(); 597 runResults.add(createFakeResult("module1", 2, 1, 0, 0, true, true)); 598 mResultHolder.runResults = runResults; 599 600 Map<String, IAbi> modulesAbi = new HashMap<>(); 601 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 602 mResultHolder.modulesAbi = modulesAbi; 603 604 mResultHolder.completeModules = 1; 605 mResultHolder.totalModules = 1; 606 mResultHolder.passedTests = 2; 607 mResultHolder.failedTests = 1; 608 mResultHolder.startTime = 0L; 609 mResultHolder.endTime = 10L; 610 File res = mFormatter.writeResults(mResultHolder, mResultDir); 611 String content = FileUtil.readStringFromFile(res); 612 613 assertXmlContainsNode(content, "Result/Module"); 614 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 615 assertXmlContainsAttribute( 616 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 617 assertXmlContainsAttribute( 618 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 619 // Check that failures are showing in the xml for the test cases 620 assertXmlContainsAttribute( 621 content, "Result/Module/TestCase/Test", "name", "module1.failed0"); 622 assertXmlContainsAttribute(content, "Result/Module/TestCase/Test", "result", "fail"); 623 assertXmlContainsAttribute( 624 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 625 // Test that we can read back the informations 626 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 627 assertEquals(holder.completeModules, mResultHolder.completeModules); 628 assertEquals(holder.totalModules, mResultHolder.totalModules); 629 assertEquals(holder.passedTests, mResultHolder.passedTests); 630 assertEquals(holder.failedTests, mResultHolder.failedTests); 631 assertEquals(holder.startTime, mResultHolder.startTime); 632 assertEquals(holder.endTime, mResultHolder.endTime); 633 assertEquals( 634 holder.modulesAbi.get("armeabi-v7a module1"), 635 mResultHolder.modulesAbi.get("module1")); 636 assertEquals(1, holder.runResults.size()); 637 TestRunResult result = holder.runResults.iterator().next(); 638 // 2 passed tests and 1 failed with metrics 639 assertEquals(3, result.getTestResults().size()); 640 assertEquals(1, result.getNumTestsInState(TestStatus.FAILURE)); 641 for (Entry<TestDescription, TestResult> entry : result.getTestResults().entrySet()) { 642 if (TestStatus.FAILURE.equals(entry.getValue().getResultStatus())) { 643 assertEquals("value00", entry.getValue().getMetrics().get("metric00")); 644 assertEquals("value10", entry.getValue().getMetrics().get("metric10")); 645 } 646 } 647 } 648 649 /** Check that run history is properly reported. */ 650 @Test testRunHistoryReporting()651 public void testRunHistoryReporting() throws Exception { 652 final String RUN_HISTORY = 653 "[{\"startTime\":10000000000000,\"endTime\":10000000100000,\"passedTests\":10," 654 + "\"failedTests\":5,\"commandLineArgs\":\"cts\"," 655 + "\"hostName\":\"user.android.com\"}," 656 + "{\"startTime\":10000000200000,\"endTime\":10000000300000," 657 + "\"passedTests\":3,\"failedTests\":2," 658 + "\"commandLineArgs\":\"cts\"," 659 + "\"hostName\":\"user.android.com\"}]"; 660 mResultHolder.context = mContext; 661 mResultHolder.context.addInvocationAttribute("run_history", RUN_HISTORY); 662 663 Collection<TestRunResult> runResults = new ArrayList<>(); 664 runResults.add(createFakeResult("module1", 2, 1, 0, 0)); 665 mResultHolder.runResults = runResults; 666 667 Map<String, IAbi> modulesAbi = new HashMap<>(); 668 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 669 mResultHolder.modulesAbi = modulesAbi; 670 671 mResultHolder.completeModules = 1; 672 mResultHolder.totalModules = 1; 673 mResultHolder.passedTests = 1; 674 mResultHolder.failedTests = 0; 675 mResultHolder.startTime = 0L; 676 mResultHolder.endTime = 10L; 677 File res = mFormatter.writeResults(mResultHolder, mResultDir); 678 String content = FileUtil.readStringFromFile(res); 679 680 assertXmlContainsAttribute(content, "Result/Build", "run_history", RUN_HISTORY); 681 assertXmlContainsNode(content, "Result/RunHistory"); 682 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "start", "10000000000000"); 683 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "end", "10000000100000"); 684 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "pass", "10"); 685 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "failed", "5"); 686 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "command_line_args", "cts"); 687 assertXmlContainsAttribute( 688 content, "Result/RunHistory/Run", "host_name", "user.android.com"); 689 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "start", "10000000200000"); 690 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "end", "10000000300000"); 691 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "pass", "3"); 692 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "failed", "2"); 693 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "command_line_args", "cts"); 694 assertXmlContainsAttribute( 695 content, "Result/RunHistory/Run", "host_name", "user.android.com"); 696 // Test that we can read back the information. 697 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 698 assertEquals(RUN_HISTORY, holder.context.getAttributes().getUniqueMap().get("run_history")); 699 } 700 701 /** Ensure the order is sorted according to module name and abi. */ 702 @Test testSortModules()703 public void testSortModules() { 704 List<TestRunResult> originalList = new ArrayList<>(); 705 originalList.add(createFakeResult("armeabi-v7a module1", 1, 0, 0, 0)); 706 originalList.add(createFakeResult("arm64-v8a module3", 1, 0, 0, 0)); 707 originalList.add(createFakeResult("armeabi-v7a module2", 1, 0, 0, 0)); 708 originalList.add(createFakeResult("arm64-v8a module1", 1, 0, 0, 0)); 709 originalList.add(createFakeResult("armeabi-v7a module4", 1, 0, 0, 0)); 710 originalList.add(createFakeResult("arm64-v8a module2", 1, 0, 0, 0)); 711 Map<String, IAbi> moduleAbis = new HashMap<>(); 712 moduleAbis.put("armeabi-v7a module1", new Abi("armeabi-v7a", "32")); 713 moduleAbis.put("arm64-v8a module1", new Abi("arm64-v8a", "64")); 714 moduleAbis.put("armeabi-v7a module2", new Abi("armeabi-v7a", "32")); 715 moduleAbis.put("arm64-v8a module2", new Abi("arm64-v8a", "64")); 716 moduleAbis.put("arm64-v8a module3", new Abi("arm64-v8a", "64")); 717 moduleAbis.put("armeabi-v7a module4", new Abi("armeabi-v7a", "32")); 718 719 List<TestRunResult> sortedResult = mFormatter.sortModules(originalList, moduleAbis); 720 assertEquals(6, sortedResult.size()); 721 assertEquals("arm64-v8a module1", sortedResult.get(0).getName()); 722 assertEquals("armeabi-v7a module1", sortedResult.get(1).getName()); 723 assertEquals("arm64-v8a module2", sortedResult.get(2).getName()); 724 assertEquals("armeabi-v7a module2", sortedResult.get(3).getName()); 725 assertEquals("arm64-v8a module3", sortedResult.get(4).getName()); 726 assertEquals("armeabi-v7a module4", sortedResult.get(5).getName()); 727 } 728 729 /** Check that the build info contains all information. */ 730 @Test testBasicFormat_buildInfo()731 public void testBasicFormat_buildInfo() throws Exception { 732 mResultHolder.context = mContext; 733 mContext.addInvocationAttribute("invocation-attr", "attr"); 734 BuildInfo buildInfo = new BuildInfo(); 735 buildInfo.addBuildAttribute("device_kernel_info", "kernel info"); 736 buildInfo.addBuildAttribute("system_img_info", "system img info"); 737 buildInfo.addBuildAttribute("vendor_img_info", "vendor img info"); 738 mContext.addDeviceBuildInfo("device", buildInfo); 739 740 Collection<TestRunResult> runResults = new ArrayList<>(); 741 runResults.add(createFakeResult("module1", 2, 0, 0, 0)); 742 runResults.add(createFakeResult("module2", 1, 0, 0, 0)); 743 mResultHolder.runResults = runResults; 744 745 Map<String, IAbi> modulesAbi = new HashMap<>(); 746 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 747 modulesAbi.put("module2", new Abi("armeabi-v7a", "32")); 748 mResultHolder.modulesAbi = modulesAbi; 749 750 mResultHolder.completeModules = 2; 751 mResultHolder.totalModules = 2; 752 mResultHolder.passedTests = 2; 753 mResultHolder.failedTests = 0; 754 mResultHolder.startTime = 0L; 755 mResultHolder.endTime = 10L; 756 File res = mFormatter.writeResults(mResultHolder, mResultDir); 757 String content = FileUtil.readStringFromFile(res); 758 759 assertXmlContainsNode(content, "Result/Build"); 760 assertXmlContainsAttribute(content, "Result/Build", "invocation-attr", "attr"); 761 assertXmlContainsAttribute(content, "Result/Build", "device_kernel_info", "kernel info"); 762 assertXmlContainsAttribute(content, "Result/Build", "system_img_info", "system img info"); 763 assertXmlContainsAttribute(content, "Result/Build", "vendor_img_info", "vendor img info"); 764 } 765 createResultWithLog(String runName, int count, LogDataType type)766 private TestRunResult createResultWithLog(String runName, int count, LogDataType type) { 767 TestRunResult fakeRes = new TestRunResult(); 768 fakeRes.testRunStarted(runName, count); 769 for (int i = 0; i < count; i++) { 770 TestDescription description = 771 new TestDescription("com.class." + runName, runName + ".method" + i); 772 fakeRes.testStarted(description); 773 fakeRes.testLogSaved( 774 runName + "log" + i, new LogFile("path", "http:url/" + runName, type)); 775 fakeRes.testEnded(description, new HashMap<String, Metric>()); 776 } 777 fakeRes.testRunEnded(10L, new HashMap<String, String>()); 778 return fakeRes; 779 } 780 createFakeResult( String runName, int passed, int failed, int assumptionFailures, int testIgnored)781 private TestRunResult createFakeResult( 782 String runName, int passed, int failed, int assumptionFailures, int testIgnored) { 783 return createFakeResult( 784 runName, passed, failed, assumptionFailures, testIgnored, false, false); 785 } 786 createFakeResult( String runName, int passed, int failed, int assumptionFailures, int testIgnored, boolean withMetrics, boolean withBadKey)787 private TestRunResult createFakeResult( 788 String runName, 789 int passed, 790 int failed, 791 int assumptionFailures, 792 int testIgnored, 793 boolean withMetrics, 794 boolean withBadKey) { 795 return createFakeResult( 796 runName, 797 passed, 798 failed, 799 assumptionFailures, 800 testIgnored, 801 2, 802 withMetrics, 803 withBadKey); 804 } 805 createFakeResult( String runName, int passed, int failed, int assumptionFailures, int testIgnored, int stackDepth, boolean withMetrics, boolean withBadKey)806 private TestRunResult createFakeResult( 807 String runName, 808 int passed, 809 int failed, 810 int assumptionFailures, 811 int testIgnored, 812 int stackDepth, 813 boolean withMetrics, 814 boolean withBadKey) { 815 TestRunResult fakeRes = new TestRunResult(); 816 fakeRes.testRunStarted(runName, passed + failed); 817 for (int i = 0; i < passed; i++) { 818 TestDescription description = 819 new TestDescription("com.class." + runName, runName + ".method" + i); 820 fakeRes.testStarted(description); 821 fakeRes.testEnded(description, new HashMap<String, Metric>()); 822 } 823 for (int i = 0; i < failed; i++) { 824 TestDescription description = 825 new TestDescription("com.class." + runName, runName + ".failed" + i); 826 fakeRes.testStarted(description); 827 // Include a null character \0 that is not XML supported 828 FailureDescription failureDescription = 829 FailureDescription.create( 830 runName + " failed." + "\nstack".repeat(stackDepth) + "\0") 831 .setErrorIdentifier(TestErrorIdentifier.TEST_ABORTED); 832 fakeRes.testFailed(description, failureDescription); 833 HashMap<String, Metric> metrics = new HashMap<String, Metric>(); 834 if (withMetrics) { 835 metrics.put("metric0" + i, TfMetricProtoUtil.stringToMetric("value0" + i)); 836 metrics.put("metric1" + i, TfMetricProtoUtil.stringToMetric("value1" + i)); 837 } 838 if (withBadKey) { 839 metrics.put("%_capacity" + i, TfMetricProtoUtil.stringToMetric("0.00")); 840 metrics.put("&_capacity" + i, TfMetricProtoUtil.stringToMetric("0.00")); 841 } 842 fakeRes.testEnded(description, metrics); 843 } 844 for (int i = 0; i < assumptionFailures; i++) { 845 TestDescription description = 846 new TestDescription("com.class." + runName, runName + ".assumpFail" + i); 847 fakeRes.testStarted(description); 848 fakeRes.testAssumptionFailure( 849 description, runName + " failed." + "\nstack".repeat(stackDepth)); 850 fakeRes.testEnded(description, new HashMap<String, Metric>()); 851 } 852 for (int i = 0; i < testIgnored; i++) { 853 TestDescription description = 854 new TestDescription("com.class." + runName, runName + ".ignored" + i); 855 fakeRes.testStarted(description); 856 fakeRes.testIgnored(description); 857 fakeRes.testEnded(description, new HashMap<String, Metric>()); 858 } 859 fakeRes.testRunEnded(10L, new HashMap<String, Metric>()); 860 return fakeRes; 861 } 862 863 /** Return all XML nodes that match the given xPathExpression. */ getXmlNodes(String xml, String xPathExpression)864 private NodeList getXmlNodes(String xml, String xPathExpression) 865 throws XPathExpressionException { 866 867 InputSource inputSource = new InputSource(new StringReader(xml)); 868 XPath xpath = XPathFactory.newInstance().newXPath(); 869 return (NodeList) xpath.evaluate(xPathExpression, inputSource, XPathConstants.NODESET); 870 } 871 872 /** Assert that the XML contains a node matching the given xPathExpression. */ assertXmlContainsNode(String xml, String xPathExpression)873 private NodeList assertXmlContainsNode(String xml, String xPathExpression) 874 throws XPathExpressionException { 875 NodeList nodes = getXmlNodes(xml, xPathExpression); 876 assertNotNull( 877 String.format("XML '%s' returned null for xpath '%s'.", xml, xPathExpression), 878 nodes); 879 assertTrue( 880 String.format( 881 "XML '%s' should have returned at least 1 node for xpath '%s', " 882 + "but returned %s nodes instead.", 883 xml, xPathExpression, nodes.getLength()), 884 nodes.getLength() >= 1); 885 return nodes; 886 } 887 888 /** Assert that the XML does not contain a node matching the given xPathExpression. */ assertXmlNotContainNode(String xml, String xPathExpression)889 private void assertXmlNotContainNode(String xml, String xPathExpression) 890 throws XPathExpressionException { 891 NodeList nodes = getXmlNodes(xml, xPathExpression); 892 assertNotNull( 893 String.format("XML '%s' returned null for xpath '%s'.", xml, xPathExpression), 894 nodes); 895 assertEquals( 896 String.format( 897 "XML '%s' should have returned at least 1 node for xpath '%s', " 898 + "but returned %s nodes instead.", 899 xml, xPathExpression, nodes.getLength()), 900 0, 901 nodes.getLength()); 902 } 903 904 /** 905 * Assert that the XML contains a node matching the given xPathExpression and that the node has 906 * a given value. 907 */ assertXmlContainsValue(String xml, String xPathExpression, String value)908 private void assertXmlContainsValue(String xml, String xPathExpression, String value) 909 throws XPathExpressionException { 910 NodeList nodes = assertXmlContainsNode(xml, xPathExpression); 911 boolean found = false; 912 913 for (int i = 0; i < nodes.getLength(); i++) { 914 Element element = (Element) nodes.item(i); 915 if (element.getTextContent().equals(value)) { 916 found = true; 917 break; 918 } 919 } 920 921 assertTrue( 922 String.format( 923 "xPath '%s' should contain value '%s' but does not. XML: '%s'", 924 xPathExpression, value, xml), 925 found); 926 } 927 928 /** 929 * Assert that the XML contains a node matching the given xPathExpression and that the node has 930 * a given value. 931 */ assertXmlContainsAttribute( String xml, String xPathExpression, String attributeName, String attributeValue)932 private void assertXmlContainsAttribute( 933 String xml, String xPathExpression, String attributeName, String attributeValue) 934 throws XPathExpressionException { 935 NodeList nodes = assertXmlContainsNode(xml, xPathExpression); 936 boolean found = false; 937 938 for (int i = 0; i < nodes.getLength(); i++) { 939 Element element = (Element) nodes.item(i); 940 String value = element.getAttribute(attributeName); 941 if (attributeValue.equals(value)) { 942 found = true; 943 break; 944 } 945 } 946 947 assertTrue( 948 String.format( 949 "xPath '%s' should contain attribute '%s' but does not. XML: '%s'", 950 xPathExpression, attributeName, xml), 951 found); 952 } 953 } 954