1 /* 2 * Copyright (C) 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.binary; 17 18 import com.android.tradefed.log.LogUtil.CLog; 19 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 20 import com.android.tradefed.result.ITestInvocationListener; 21 import com.android.tradefed.result.TestDescription; 22 23 import com.google.common.annotations.VisibleForTesting; 24 25 import java.util.ArrayList; 26 import java.util.Arrays; 27 import java.util.HashMap; 28 import java.util.Iterator; 29 import java.util.List; 30 import java.util.function.Predicate; 31 import java.util.regex.Matcher; 32 import java.util.regex.Pattern; 33 34 /** 35 * Reads KTAP output as that produced by a KUnit test module and placed in a `results` file under 36 * debugfs. 37 * 38 * <p>This implementation is based off of the official documentation, kunit_parser.py and specific 39 * caveats found during testing. Additional logic needed: 40 * 41 * <ul> 42 * <li>Indentation is ignored because it's not consistent across usage. 43 * <li>Line starting with "# Subtest:" is required to properly nest into subtest groups. This 44 * approach was taken from kunit_parser.py. 45 * <li>Sometimes a "- " proceeds the test name and diagnostic data when a '#' isn't used. When 46 * it's encountered it's stripped off. 47 * <li>The test name can technically have any character besides '#'. This will probably become an 48 * issue when getting translated to TF test results. For now only post processing is to 49 * replace spaces with underscores. 50 * </ul> 51 * 52 * @See <a href="The Kernel Test Anything Protocol (KTAP), version 53 * 1">https://docs.kernel.org/dev-tools/ktap.html</a> 54 */ 55 public class KTapResultParser { 56 57 public enum ParseResolution { 58 INDIVIDUAL_LEAVES, 59 AGGREGATED_TOP_LEVEL, 60 } 61 62 enum ResultDirective { 63 NOTSET, 64 SKIP, 65 TODO, 66 XFAIL, 67 TIMEOUT, 68 ERROR 69 } 70 71 @VisibleForTesting 72 static class TestResult { 73 public String version; 74 public boolean expectSubtests; 75 public int numberOfSubtests; 76 public List<TestResult> subtests; 77 public boolean isOk; 78 public int testNum; 79 public String name; 80 public ResultDirective directive; 81 public String diagnosticData; 82 public List<String> diagnosticLines; 83 public boolean isRoot; 84 TestResult()85 public TestResult() { 86 numberOfSubtests = 0; 87 subtests = new ArrayList<TestResult>(); 88 diagnosticLines = new ArrayList<String>(); 89 } 90 createDiagnosticTrace()91 public String createDiagnosticTrace() { 92 return String.format("%s\n%s", String.join("\n", diagnosticLines), diagnosticData) 93 .trim(); 94 } 95 toStringWithSubtests(String prefix)96 public String toStringWithSubtests(String prefix) { 97 StringBuilder builder = new StringBuilder(); 98 builder.append(prefix + name + System.lineSeparator()); 99 for (TestResult tr : subtests) { 100 builder.append(tr.toStringWithSubtests(prefix + " ")); 101 } 102 return builder.toString(); 103 } 104 } 105 106 static final String DIR_OPTIONS = 107 String.format( 108 "%s", 109 Arrays.toString( 110 new ResultDirective[] { 111 ResultDirective.SKIP, 112 ResultDirective.TODO, 113 ResultDirective.XFAIL, 114 ResultDirective.TIMEOUT, 115 ResultDirective.ERROR 116 }) 117 .replace("[", "") 118 .replace(", ", "|") 119 .replace("]", "")); 120 121 // Regex Group Names 122 static final String SUBTEST_NAME = "name"; 123 static final String SUBTEST_COUNT = "count"; 124 static final String OK_OR_NOT = "okOrNot"; 125 static final String TEST_NUM = "testNum"; 126 static final String TEST_NAME = "testName"; 127 static final String DIRECTIVE = "directive"; 128 static final String DIAGNOSTIC = "diagnosticData"; 129 130 static final Pattern VERSION_PATTERN = Pattern.compile("^\\s*K?TAP version.*"); 131 static final Pattern SUBTEST_PATTERN = 132 Pattern.compile(String.format("^\\s*# Subtest: (?<%s>\\S+)", SUBTEST_NAME)); 133 static final Pattern PLAN_PATTERN = 134 Pattern.compile(String.format("^\\s*1..(?<%s>\\d+)", SUBTEST_COUNT)); 135 static final Pattern TEST_CASE_RESULT_PATTERN = 136 Pattern.compile( 137 String.format( 138 "^\\s*(?<%s>ok|not ok)\\s+(?<%s>\\d+)\\s+(-" 139 + " )?(?<%s>[^#]+)?(\\s*#\\s+)?(?<%s>%s)?\\s*(?<%s>.*)?$", 140 OK_OR_NOT, TEST_NUM, TEST_NAME, DIRECTIVE, DIR_OPTIONS, DIAGNOSTIC)); 141 142 static final Predicate<String> IS_VERSION = VERSION_PATTERN.asPredicate(); 143 static final Predicate<String> IS_SUBTEST = SUBTEST_PATTERN.asPredicate(); 144 static final Predicate<String> IS_PLAN = PLAN_PATTERN.asPredicate(); 145 static final Predicate<String> IS_TEST_CASE_RESULT = TEST_CASE_RESULT_PATTERN.asPredicate(); 146 applyKTapResultToListener( ITestInvocationListener listener, String testRunName, String ktapFileContent, ParseResolution resolution)147 public static void applyKTapResultToListener( 148 ITestInvocationListener listener, 149 String testRunName, 150 String ktapFileContent, 151 ParseResolution resolution) { 152 KTapResultParser parser = new KTapResultParser(); 153 TestResult root = parser.processResultsFileContent(ktapFileContent); 154 parser.applyToListener(listener, testRunName, root, resolution); 155 } 156 157 @VisibleForTesting processResultsFileContent(String fileContent)158 TestResult processResultsFileContent(String fileContent) { 159 TestResult root = new TestResult(); 160 root.isRoot = true; 161 root.expectSubtests = true; 162 Iterator<String> lineIterator = fileContent.lines().iterator(); 163 processLines(root, lineIterator); 164 return root; 165 } 166 processLines(TestResult currentTest, Iterator<String> lineIterator)167 private void processLines(TestResult currentTest, Iterator<String> lineIterator) { 168 if (!lineIterator.hasNext()) { 169 // Reached the end of file, confirm that the final test is in fact the root test 170 if (!currentTest.isRoot) { 171 throw new RuntimeException( 172 String.format("Incomplete KTap results. All tests not closed properly.")); 173 } 174 return; 175 } 176 177 String line = lineIterator.next().trim(); 178 179 if (IS_VERSION.test(line)) { 180 if (currentTest.isRoot && currentTest.version == null || !currentTest.expectSubtests) { 181 currentTest.version = line; 182 } else { 183 TestResult subtest = new TestResult(); 184 subtest.version = line; 185 currentTest.subtests.add(subtest); 186 processLines(subtest, lineIterator); 187 } 188 } else if (IS_SUBTEST.test(line)) { 189 Matcher matcher = SUBTEST_PATTERN.matcher(line); 190 matcher.matches(); 191 192 String subtestName = matcher.group(SUBTEST_NAME); 193 if (currentTest.isRoot 194 || (currentTest.expectSubtests && currentTest.numberOfSubtests > 0)) { 195 // If we've already seen a PLAN line then this subtest line 196 // belongs to a different test. 197 TestResult subtest = new TestResult(); 198 subtest.expectSubtests = true; 199 subtest.name = subtestName; 200 currentTest.subtests.add(subtest); 201 processLines(subtest, lineIterator); 202 } else { 203 currentTest.expectSubtests = true; 204 currentTest.name = subtestName; 205 } 206 } else if (IS_PLAN.test(line)) { 207 Matcher matcher = PLAN_PATTERN.matcher(line); 208 matcher.matches(); 209 210 currentTest.expectSubtests = true; 211 currentTest.numberOfSubtests = Integer.parseInt(matcher.group(SUBTEST_COUNT)); 212 } else if (IS_TEST_CASE_RESULT.test(line)) { 213 Matcher matcher = TEST_CASE_RESULT_PATTERN.matcher(line); 214 matcher.matches(); 215 216 boolean isOk = matcher.group(OK_OR_NOT).equals("ok"); 217 int testNum = Integer.parseInt(matcher.group(TEST_NUM)); 218 219 String name = 220 matcher.group(TEST_NAME) != null 221 ? matcher.group(TEST_NAME).trim().replace(" ", "_") 222 : String.format("unnamed_test_%d", testNum); 223 ResultDirective directive = 224 matcher.group(DIRECTIVE) != null 225 ? ResultDirective.valueOf(matcher.group(DIRECTIVE)) 226 : ResultDirective.NOTSET; 227 String diagnosticData = 228 matcher.group(DIAGNOSTIC) != null ? matcher.group(DIAGNOSTIC) : ""; 229 230 if (name.equals(currentTest.name) 231 || (currentTest.name == null 232 && (currentTest.numberOfSubtests != 0 233 && currentTest.subtests.size() 234 == currentTest.numberOfSubtests))) { 235 // This test with subtests has completed. 236 currentTest.name = name; 237 currentTest.isOk = isOk; 238 currentTest.testNum = testNum; 239 currentTest.directive = directive; 240 currentTest.diagnosticData = diagnosticData; 241 return; // No more recurse with this test. 242 } else { 243 // This is a subtest within currentTest. 244 TestResult leafSubtest = new TestResult(); 245 leafSubtest.name = name; 246 leafSubtest.isOk = isOk; 247 248 // For situations where the number of subtests is known 249 // validate that the testNum is not out of order. 250 if (currentTest.numberOfSubtests != 0 251 && testNum != currentTest.subtests.size() + 1) { 252 throw new RuntimeException( 253 String.format( 254 "Test encountered out of order expected '%d' but received '%d'." 255 + " Line: '%s'", 256 currentTest.subtests.size() + 1, testNum, line)); 257 } 258 leafSubtest.testNum = testNum; 259 leafSubtest.directive = directive; 260 leafSubtest.diagnosticData = diagnosticData; 261 262 // Any diagnostic lines encountered up until this point actually 263 // belong to this leaf test. Copy over from the parent and then clear 264 // out the parents lines. 265 leafSubtest.diagnosticLines.addAll(currentTest.diagnosticLines); 266 currentTest.diagnosticLines.clear(); 267 currentTest.subtests.add(leafSubtest); 268 } 269 } else if (!line.isEmpty()) { 270 // Diagnostic lines or unknown lines. 271 currentTest.diagnosticLines.add(line); 272 } 273 274 processLines(currentTest, lineIterator); 275 } 276 applyToListener( ITestInvocationListener listener, String testRunName, TestResult root, ParseResolution resolution)277 private void applyToListener( 278 ITestInvocationListener listener, 279 String testRunName, 280 TestResult root, 281 ParseResolution resolution) { 282 if (resolution == ParseResolution.INDIVIDUAL_LEAVES) { 283 applySubtestLeavesToListener(listener, testRunName, root, ""); 284 } else if (resolution == ParseResolution.AGGREGATED_TOP_LEVEL) { 285 applyAggregatedTopLevelToListener(listener, testRunName, root, ""); 286 } 287 } 288 applySubtestLeavesToListener( ITestInvocationListener listener, String testRunName, TestResult test, String prefix)289 private void applySubtestLeavesToListener( 290 ITestInvocationListener listener, String testRunName, TestResult test, String prefix) { 291 String testName = prefix == null || prefix.isEmpty() ? test.name : prefix + "." + test.name; 292 if (test.subtests.size() > 0) { 293 for (TestResult subtest : test.subtests) { 294 applySubtestLeavesToListener(listener, testRunName, subtest, testName); 295 } 296 } else { 297 applyTestResultToListener(listener, testRunName, testName, test); 298 } 299 } 300 applyAggregatedTopLevelToListener( ITestInvocationListener listener, String testRunName, TestResult root, String fullKTapResult)301 private void applyAggregatedTopLevelToListener( 302 ITestInvocationListener listener, 303 String testRunName, 304 TestResult root, 305 String fullKTapResult) { 306 // Here we want to apply a single test result based on the one or more top level KTAP 307 // results. If there are more than one top level result, their names are concatenated 308 // and pass/fail results are AND'd into a final value. 309 310 if (root.subtests.isEmpty()) { 311 throw new IllegalArgumentException( 312 "No valid test results in KTAP results. " + fullKTapResult); 313 } 314 315 String testName = root.subtests.get(0).name; 316 boolean isOk = root.subtests.get(0).isOk; 317 for (int i = 1; i < root.subtests.size(); ++i) { 318 testName += "." + root.subtests.get(i).name; 319 isOk &= root.subtests.get(i).isOk; 320 } 321 322 TestDescription testDescription = new TestDescription(testRunName, testName); 323 listener.testStarted(testDescription); 324 if (!isOk) { 325 listener.testFailed(testDescription, fullKTapResult); 326 } 327 listener.testEnded(testDescription, new HashMap<String, Metric>()); 328 } 329 applyTestResultToListener( ITestInvocationListener listener, String testRunName, String testName, TestResult test)330 private void applyTestResultToListener( 331 ITestInvocationListener listener, 332 String testRunName, 333 String testName, 334 TestResult test) { 335 TestDescription testDescription = new TestDescription(testRunName, testName); 336 listener.testStarted(testDescription); 337 switch (test.directive) { 338 case NOTSET: 339 if (!test.isOk) { 340 listener.testFailed(testDescription, test.createDiagnosticTrace()); 341 } 342 break; 343 case TIMEOUT: 344 // fall through 345 case ERROR: 346 listener.testFailed(testDescription, test.createDiagnosticTrace()); 347 if (!test.isOk) { 348 CLog.w( 349 "%s has directive '%s' but also shows 'ok', forcing 'not ok'", 350 testName, test.directive); 351 } 352 break; 353 case SKIP: 354 // fall through 355 case TODO: 356 // fall through 357 case XFAIL: 358 listener.testIgnored(testDescription); 359 break; 360 } 361 listener.testEnded(testDescription, new HashMap<String, Metric>()); 362 } 363 } 364