1#!/usr/bin/env python3
2#
3# [VPYTHON:BEGIN]
4# python_version: "3.8"
5# [VPYTHON:END]
6#
7# Copyright (C) 2021 The Android Open Source Project
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13#      http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20
21import sys, os, argparse, subprocess, shlex, re, concurrent.futures, multiprocessing
22
23def parse_args():
24  parser = argparse.ArgumentParser(description="Run libcore tests using the vogar testing tool.")
25  parser.add_argument('--mode', choices=['device', 'host', 'jvm'], required=True,
26                      help='Specify where tests should be run.')
27  parser.add_argument('--variant', choices=['X32', 'X64'],
28                      help='Which dalvikvm variant to execute with.')
29  parser.add_argument('-j', '--jobs', type=int,
30                      help='Number of tests to run simultaneously.')
31  parser.add_argument('--timeout', type=int,
32                      help='How long to run the test before aborting (seconds).')
33  parser.add_argument('--debug', action='store_true',
34                      help='Use debug version of ART (device|host only).')
35  parser.add_argument('--dry-run', action='store_true',
36                      help='Print vogar command-line, but do not run.')
37  parser.add_argument('--no-getrandom', action='store_false', dest='getrandom',
38                      help='Ignore failures from getrandom() (for kernel < 3.17).')
39  parser.add_argument('--no-jit', action='store_false', dest='jit',
40                      help='Disable JIT (device|host only).')
41  parser.add_argument('--gcstress', action='store_true',
42                      help='Enable GC stress configuration (device|host only).')
43  parser.add_argument('tests', nargs="*",
44                      help='Name(s) of the test(s) to run')
45  parser.add_argument('--verbose', action='store_true', help='Print verbose output from vogar.')
46  return parser.parse_args()
47
48ART_TEST_ANDROID_ROOT = os.environ.get("ART_TEST_ANDROID_ROOT", "/system")
49ART_TEST_CHROOT = os.environ.get("ART_TEST_CHROOT")
50ANDROID_PRODUCT_OUT = os.environ.get("ANDROID_PRODUCT_OUT")
51
52LIBCORE_TEST_NAMES = [
53  ### luni tests. ###
54  # Naive critical path optimization: Run the longest tests first.
55  "org.apache.harmony.tests.java.util",  # 90min under gcstress
56  "libcore.java.lang",                   # 90min under gcstress
57  "jsr166",                              # 60min under gcstress
58  "libcore.java.util",                   # 60min under gcstress
59  "libcore.java.math",                   # 50min under gcstress
60  "org.apache.harmony.crypto",           # 30min under gcstress
61  "org.apache.harmony.tests.java.io",    # 30min under gcstress
62  "org.apache.harmony.tests.java.text",  # 30min under gcstress
63  # Split highmemorytest to individual classes since it is too big.
64  "libcore.highmemorytest.java.text.DateFormatTest",
65  "libcore.highmemorytest.java.text.DecimalFormatTest",
66  "libcore.highmemorytest.java.text.SimpleDateFormatTest",
67  "libcore.highmemorytest.java.time.format.DateTimeFormatterTest",
68  "libcore.highmemorytest.java.util.CalendarTest",
69  "libcore.highmemorytest.java.util.CurrencyTest",
70  "libcore.highmemorytest.libcore.icu.SimpleDateFormatDataTest",
71  # All other luni tests in alphabetical order.
72  "libcore.android.system",
73  "libcore.build",
74  "libcore.dalvik.system",
75  "libcore.java.awt",
76  "libcore.java.text",
77  "libcore.javax.crypto",
78  "libcore.javax.net",
79  "libcore.javax.security",
80  "libcore.javax.sql",
81  "libcore.javax.xml",
82  "libcore.libcore.icu",
83  "libcore.libcore.internal",
84  "libcore.libcore.io",
85  "libcore.libcore.net",
86  "libcore.libcore.reflect",
87  "libcore.libcore.util",
88  "libcore.sun.invoke",
89  "libcore.sun.misc",
90  "libcore.sun.net",
91  "libcore.sun.security",
92  "libcore.sun.util",
93  "libcore.xml",
94  "org.apache.harmony.annotation",
95  "org.apache.harmony.luni.tests.internal.net.www.protocol.http.HttpURLConnection",
96  "org.apache.harmony.luni.tests.internal.net.www.protocol.https.HttpsURLConnection",
97  "org.apache.harmony.luni.tests.java.io",
98  "org.apache.harmony.luni.tests.java.net",
99  "org.apache.harmony.nio",
100  "org.apache.harmony.regex",
101  "org.apache.harmony.testframework",
102  "org.apache.harmony.tests.java.lang",
103  "org.apache.harmony.tests.java.math",
104  "org.apache.harmony.tests.javax.security",
105  "tests.java.lang.String",
106  ### OpenJDK upstream tests (ojluni). ###
107  # "test.java.awt",
108  "test.java.awt",
109  # test.java.io
110  "test.java.io.ByteArrayInputStream",
111  "test.java.io.ByteArrayOutputStream",
112  "test.java.io.FileReader",
113  "test.java.io.FileWriter",
114  "test.java.io.InputStream",
115  "test.java.io.OutputStream",
116  "test.java.io.PrintStream",
117  "test.java.io.PrintWriter",
118  "test.java.io.Reader",
119  "test.java.io.Writer",
120  # test.java.lang
121  "test.java.lang.Boolean",
122  "test.java.lang.ClassLoader",
123  "test.java.lang.Double",
124  "test.java.lang.Float",
125  "test.java.lang.Integer",
126  "test.java.lang.Long",
127  # Sharded test.java.lang.StrictMath
128  "test.java.lang.StrictMath.CubeRootTests",
129  # TODO: disable the test until b/248208762 is fixed.
130  # "test.java.lang.StrictMath.ExactArithTests",
131  "test.java.lang.StrictMath.Expm1Tests",
132  "test.java.lang.StrictMath.ExpTests",
133  "test.java.lang.StrictMath.HyperbolicTests",
134  "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard1",
135  "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard2",
136  "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard3",
137  "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard4",
138  "test.java.lang.StrictMath.HypotTests#testHypot",
139  "test.java.lang.StrictMath.Log1pTests",
140  "test.java.lang.StrictMath.Log10Tests",
141  "test.java.lang.StrictMath.MultiplicationTests",
142  "test.java.lang.StrictMath.PowTests",
143  "test.java.lang.String",
144  "test.java.lang.Thread",
145  # test.java.lang.invoke
146  "test.java.lang.invoke",
147  # test.java.lang.ref
148  "test.java.lang.ref.SoftReference",
149  "test.java.lang.ref.BasicTest",
150  "test.java.lang.ref.EnqueueNullRefTest",
151  "test.java.lang.ref.EnqueuePollRaceTest",
152  "test.java.lang.ref.ReferenceCloneTest",
153  "test.java.lang.ref.ReferenceEnqueuePendingTest",
154  # test.java.math
155  "test.java.math.BigDecimal",
156  # Sharded test.java.math.BigInteger
157  "test.java.math.BigInteger#testArithmetic",
158  "test.java.math.BigInteger#testBitCount",
159  "test.java.math.BigInteger#testBitLength",
160  "test.java.math.BigInteger#testbitOps",
161  "test.java.math.BigInteger#testBitwise",
162  "test.java.math.BigInteger#testByteArrayConv",
163  "test.java.math.BigInteger#testConstructor",
164  "test.java.math.BigInteger#testDivideAndReminder",
165  "test.java.math.BigInteger#testDivideLarge",
166  "test.java.math.BigInteger#testModExp",
167  "test.java.math.BigInteger#testMultiplyLarge",
168  "test.java.math.BigInteger#testNextProbablePrime",
169  "test.java.math.BigInteger#testPow",
170  "test.java.math.BigInteger#testSerialize",
171  "test.java.math.BigInteger#testShift",
172  "test.java.math.BigInteger#testSquare",
173  "test.java.math.BigInteger#testSquareLarge",
174  "test.java.math.BigInteger#testSquareRootAndReminder",
175  "test.java.math.BigInteger#testStringConv_generic",
176  "test.java.math.RoundingMode",
177  # test.java.net
178  "test.java.net.DatagramSocket",
179  "test.java.net.Socket",
180  "test.java.net.SocketOptions",
181  "test.java.net.URLDecoder",
182  "test.java.net.URLEncoder",
183  # test.java.nio
184  "test.java.nio.channels.Channels",
185  "test.java.nio.channels.SelectionKey",
186  "test.java.nio.channels.Selector",
187  "test.java.nio.file",
188  # test.java.security
189  "test.java.security.cert",
190  # Sharded test.java.security.KeyAgreement
191  "test.java.security.KeyAgreement.KeyAgreementTest",
192  "test.java.security.KeyAgreement.KeySizeTest#testECDHKeySize",
193  "test.java.security.KeyAgreement.KeySpecTest",
194  "test.java.security.KeyAgreement.MultiThreadTest",
195  "test.java.security.KeyAgreement.NegativeTest",
196  "test.java.security.KeyStore",
197  "test.java.security.Provider",
198  # test.java.time
199  "test.java.time",
200  # test.java.util
201  "test.java.util.Arrays",
202  "test.java.util.Collection",
203  "test.java.util.Collections",
204  "test.java.util.Date",
205  "test.java.util.EnumMap",
206  "test.java.util.EnumSet",
207  "test.java.util.GregorianCalendar",
208  "test.java.util.LinkedHashMap",
209  "test.java.util.LinkedHashSet",
210  "test.java.util.List",
211  "test.java.util.Map",
212  "test.java.util.Optional",
213  "test.java.util.TestFormatter",
214  "test.java.util.TimeZone",
215  # test.java.util.concurrent
216  "test.java.util.concurrent",
217  # test.java.util.function
218  "test.java.util.function",
219  # test.java.util.stream
220  "test.java.util.stream",
221  # test.java.util.zip
222  "test.java.util.zip.ZipFile",
223  # tck.java.time
224  "tck.java.time",
225]
226# "org.apache.harmony.security",  # We don't have rights to revert changes in case of failures.
227
228# Note: This must start with the CORE_IMG_JARS in Android.common_path.mk
229# because that's what we use for compiling the boot.art image.
230# It may contain additional modules from TEST_CORE_JARS.
231BOOT_CLASSPATH = [
232  "/apex/com.android.art/javalib/core-oj.jar",
233  "/apex/com.android.art/javalib/core-libart.jar",
234  "/apex/com.android.art/javalib/okhttp.jar",
235  "/apex/com.android.art/javalib/bouncycastle.jar",
236  "/apex/com.android.art/javalib/apache-xml.jar",
237  "/apex/com.android.i18n/javalib/core-icu4j.jar",
238  "/apex/com.android.conscrypt/javalib/conscrypt.jar",
239]
240
241CLASSPATH = ["core-tests", "core-ojtests", "jsr166-tests", "mockito-target"]
242
243SLOW_OJLUNI_TESTS = {
244  "test.java.awt",
245  "test.java.lang.String",
246  "test.java.lang.invoke",
247  "test.java.nio.channels.Selector",
248  "test.java.time",
249  "test.java.util.Arrays",
250  "test.java.util.Map",
251  "test.java.util.concurrent",
252  "test.java.util.stream",
253  "test.java.util.zip.ZipFile",
254  "tck.java.time",
255}
256
257# Disabled to unblock art-buildbot
258# These tests fail with "java.io.IOException: Stream closed", tracked in
259# http://b/235566533 and http://b/208639267
260DISABLED_GCSTRESS_DEBUG_TESTS = {
261  "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard1",
262  "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard2",
263  "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard3",
264  "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard4",
265  "test.java.math.BigDecimal",
266  "test.java.math.BigInteger#testConstructor",
267  "test.java.util.TestFormatter",
268  "test.java.util.Collection",
269}
270
271DISABLED_FUGU_TESTS = {
272  "org.apache.harmony.luni.tests.internal.net.www.protocol.http.HttpURLConnection",
273  "org.apache.harmony.luni.tests.internal.net.www.protocol.https.HttpsURLConnection",
274  "test.java.awt",
275  "test.java.io.ByteArrayInputStream",
276  "test.java.io.ByteArrayOutputStream",
277  "test.java.io.InputStream",
278  "test.java.io.OutputStream",
279  "test.java.io.PrintStream",
280  "test.java.io.PrintWriter",
281  "test.java.io.Reader",
282  "test.java.io.Writer",
283  "test.java.lang.Boolean",
284  "test.java.lang.ClassLoader",
285  "test.java.lang.Double",
286  "test.java.lang.Float",
287  "test.java.lang.Integer",
288  "test.java.lang.Long",
289  "test.java.lang.StrictMath.CubeRootTests",
290  "test.java.lang.StrictMath.Expm1Tests",
291  "test.java.lang.StrictMath.ExpTests",
292  "test.java.lang.StrictMath.HyperbolicTests",
293  "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard1",
294  "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard2",
295  "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard3",
296  "test.java.lang.StrictMath.HypotTests#testAgainstTranslit_shard4",
297  "test.java.lang.StrictMath.HypotTests#testHypot",
298  "test.java.lang.StrictMath.Log1pTests",
299  "test.java.lang.StrictMath.Log10Tests",
300  "test.java.lang.StrictMath.MultiplicationTests",
301  "test.java.lang.StrictMath.PowTests",
302  "test.java.lang.String",
303  "test.java.lang.Thread",
304  "test.java.lang.invoke",
305  "test.java.lang.ref.SoftReference",
306  "test.java.lang.ref.BasicTest",
307  "test.java.lang.ref.EnqueueNullRefTest",
308  "test.java.lang.ref.EnqueuePollRaceTest",
309  "test.java.lang.ref.ReferenceCloneTest",
310  "test.java.lang.ref.ReferenceEnqueuePendingTest",
311  "test.java.math.BigDecimal",
312  "test.java.math.BigInteger#testArithmetic",
313  "test.java.math.BigInteger#testBitCount",
314  "test.java.math.BigInteger#testBitLength",
315  "test.java.math.BigInteger#testbitOps",
316  "test.java.math.BigInteger#testBitwise",
317  "test.java.math.BigInteger#testByteArrayConv",
318  "test.java.math.BigInteger#testConstructor",
319  "test.java.math.BigInteger#testDivideAndReminder",
320  "test.java.math.BigInteger#testDivideLarge",
321  "test.java.math.BigInteger#testModExp",
322  "test.java.math.BigInteger#testMultiplyLarge",
323  "test.java.math.BigInteger#testNextProbablePrime",
324  "test.java.math.BigInteger#testPow",
325  "test.java.math.BigInteger#testSerialize",
326  "test.java.math.BigInteger#testShift",
327  "test.java.math.BigInteger#testSquare",
328  "test.java.math.BigInteger#testSquareLarge",
329  "test.java.math.BigInteger#testSquareRootAndReminder",
330  "test.java.math.BigInteger#testStringConv_generic",
331  "test.java.math.RoundingMode",
332  "test.java.net.DatagramSocket",
333  "test.java.net.Socket",
334  "test.java.net.SocketOptions",
335  "test.java.net.URLDecoder",
336  "test.java.net.URLEncoder",
337  "test.java.nio.channels.Channels",
338  "test.java.nio.channels.SelectionKey",
339  "test.java.nio.channels.Selector",
340  "test.java.nio.file",
341  "test.java.security.cert",
342  "test.java.security.KeyAgreement.KeyAgreementTest",
343  "test.java.security.KeyAgreement.KeySizeTest#testECDHKeySize",
344  "test.java.security.KeyAgreement.KeySpecTest",
345  "test.java.security.KeyAgreement.MultiThreadTest",
346  "test.java.security.KeyAgreement.NegativeTest",
347  "test.java.security.KeyStore",
348  "test.java.security.Provider",
349  "test.java.time",
350  "test.java.util.Arrays",
351  "test.java.util.Collection",
352  "test.java.util.Collections",
353  "test.java.util.Date",
354  "test.java.util.EnumMap",
355  "test.java.util.EnumSet",
356  "test.java.util.GregorianCalendar",
357  "test.java.util.LinkedHashMap",
358  "test.java.util.LinkedHashSet",
359  "test.java.util.List",
360  "test.java.util.Map",
361  "test.java.util.Optional",
362  "test.java.util.TestFormatter",
363  "test.java.util.TimeZone",
364  "test.java.util.function",
365  "test.java.util.stream",
366  "tck.java.time",
367}
368
369def get_jar_filename(classpath):
370  base_path = (ANDROID_PRODUCT_OUT + "/../..") if ANDROID_PRODUCT_OUT else "out/target"
371  base_path = os.path.normpath(base_path)  # Normalize ".." components for readability.
372  return f"{base_path}/common/obj/JAVA_LIBRARIES/{classpath}_intermediates/classes.jar"
373
374def get_timeout_secs():
375  default_timeout_secs = 600
376  if args.gcstress:
377    default_timeout_secs = 1200
378    if args.debug:
379      default_timeout_secs = 1800
380  return args.timeout or default_timeout_secs
381
382def get_expected_failures():
383  failures = ["art/tools/libcore_failures.txt"]
384  if args.mode != "jvm":
385    if args.gcstress:
386      failures.append("art/tools/libcore_gcstress_failures.txt")
387    if args.gcstress and args.debug:
388      failures.append("art/tools/libcore_gcstress_debug_failures.txt")
389    if args.debug and not args.gcstress and args.getrandom:
390      failures.append("art/tools/libcore_debug_failures.txt")
391    if not args.getrandom:
392      failures.append("art/tools/libcore_fugu_failures.txt")
393  return failures
394
395def get_test_names():
396  if args.tests:
397    return args.tests
398  test_names = list(LIBCORE_TEST_NAMES)
399  # See b/78228743 and b/178351808.
400  if args.gcstress or args.debug or args.mode == "jvm":
401    test_names = list(t for t in test_names if not t.startswith("libcore.highmemorytest"))
402    test_names = list(filter(lambda x: x not in SLOW_OJLUNI_TESTS, test_names))
403  if args.gcstress and args.debug:
404    test_names = list(filter(lambda x: x not in DISABLED_GCSTRESS_DEBUG_TESTS, test_names))
405  if not args.getrandom:
406    # Disable libcore.highmemorytest due to limited ram on fugu. http://b/258173036
407    test_names = list(filter(lambda x: x not in DISABLED_FUGU_TESTS and
408                                       not x.startswith("libcore.highmemorytest"), test_names))
409  return test_names
410
411def get_vogar_command(test_name):
412  cmd = ["vogar"]
413  if args.mode == "device":
414    cmd.append("--mode=device --vm-arg -Ximage:/system/framework/art_boot_images/boot.art")
415    cmd.append("--vm-arg -Xbootclasspath:" + ":".join(BOOT_CLASSPATH))
416
417  if args.mode == "host":
418    # We explicitly give a wrong path for the image, to ensure vogar
419    # will create a boot image with the default compiler. Note that
420    # giving an existing image on host does not work because of
421    # classpath/resources differences when compiling the boot image.
422    cmd.append("--mode=host --vm-arg -Ximage:/non/existent/vogar.art")
423  if args.mode == "jvm":
424    cmd.append("--mode=jvm")
425  if args.variant:
426    cmd.append("--variant=" + args.variant)
427  if args.gcstress:
428    cmd.append("--vm-arg -Xgc:gcstress")
429    cmd.append('--vm-arg -Djsr166.delay.factor="1.50"')
430  if args.debug:
431    cmd.append("--vm-arg -XXlib:libartd.so --vm-arg -XX:SlowDebug=true")
432
433  # The only device in go/art-buildbot without getrandom is fugu. We limit the amount of memory
434  # per runtime for fugu to avoid low memory killer, fugu has 4-cores 1GB RAM (b/258171768).
435  if not args.getrandom:
436    cmd.append("--vm-arg -Xmx128M")
437
438  if args.mode == "device":
439    if ART_TEST_CHROOT:
440      cmd.append(f"--chroot {ART_TEST_CHROOT} --device-dir=/tmp/vogar/test-{test_name}")
441    else:
442      cmd.append(f"--device-dir=/data/local/tmp/vogar/test-{test_name}")
443    cmd.append(f"--vm-command={ART_TEST_ANDROID_ROOT}/bin/art")
444  else:
445    cmd.append(f"--device-dir=/tmp/vogar/test-{test_name}")
446
447  if args.mode != "jvm":
448    cmd.append("--timeout {}".format(get_timeout_secs()))
449    cmd.append("--toolchain d8 --language CUR")
450    if args.jit:
451      cmd.append("--vm-arg -Xcompiler-option --vm-arg --compiler-filter=verify")
452    cmd.append("--vm-arg -Xusejit:{}".format(str(args.jit).lower()))
453
454  if args.verbose:
455    cmd.append("--verbose")
456
457  # Suppress color codes if not attached to a terminal
458  if not sys.stdout.isatty():
459    cmd.append("--no-color")
460
461  cmd.extend("--expectations " + f for f in get_expected_failures())
462  cmd.extend("--classpath " + get_jar_filename(cp) for cp in CLASSPATH)
463  cmd.append(test_name)
464
465  # vogar target options
466  if not os.path.exists('frameworks/base'):
467    cmd.append("--")
468    # Skip @NonMts test in thin manifest which uses prebuilt Conscrypt and ICU.
469    # It's similar to running libcore tests on the older platforms.
470    # @NonMts means that the test doesn't pass on a older platform version.
471    cmd.append("--exclude-filter libcore.test.annotation.NonMts")
472  return cmd
473
474def get_target_cpu_count():
475  adb_command = 'adb shell cat /sys/devices/system/cpu/present'
476  with subprocess.Popen(adb_command.split(),
477                        stderr=subprocess.STDOUT,
478                        stdout=subprocess.PIPE,
479                        universal_newlines=True) as proc:
480    assert(proc.wait() == 0)  # Check the exit code.
481    match = re.match(r'\d*-(\d*)', proc.stdout.read())
482    assert(match)
483    return int(match.group(1)) + 1  # Add one to convert from "last-index" to "count"
484
485def main():
486  global args
487  args = parse_args()
488
489  if not os.path.exists('build/envsetup.sh'):
490    raise AssertionError("Script needs to be run at the root of the android tree")
491  for jar in map(get_jar_filename, CLASSPATH):
492    if not os.path.exists(jar):
493      raise AssertionError(f"Missing {jar}. Run buildbot-build.sh first.")
494
495  if not args.jobs:
496    if args.mode == "device":
497      args.jobs = get_target_cpu_count()
498    else:
499      args.jobs = multiprocessing.cpu_count()
500      if args.gcstress:
501        # TODO: Investigate and fix the underlying issues.
502        args.jobs = args.jobs // 2
503
504  def run_test(test_name):
505    cmd = " ".join(get_vogar_command(test_name))
506    if args.dry_run:
507      return test_name, cmd, "Dry-run: skipping execution", 0
508    with subprocess.Popen(shlex.split(cmd),
509                          stderr=subprocess.STDOUT,
510                          stdout=subprocess.PIPE,
511                          universal_newlines=True) as proc:
512      return test_name, cmd, proc.communicate()[0], proc.wait()
513
514  failed_regex = re.compile(r"^.* FAIL \((?:EXEC_FAILED|ERROR)\)$", re.MULTILINE)
515  failed_tests, max_exit_code = [], 0
516  with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as pool:
517    futures = [pool.submit(run_test, test_name) for test_name in get_test_names()]
518    print(f"Running {len(futures)} tasks on {args.jobs} core(s)...\n")
519    for i, future in enumerate(concurrent.futures.as_completed(futures)):
520      test_name, cmd, stdout, exit_code = future.result()
521      if exit_code != 0 or args.dry_run or args.verbose:
522        print(cmd)
523        print(stdout.strip())
524      else:
525        print(stdout.strip().split("\n")[-1])  # Vogar final summary line.
526      failed_match = failed_regex.findall(stdout)
527      failed_tests.extend(failed_match)
528      max_exit_code = max(max_exit_code, exit_code)
529      result = "PASSED" if exit_code == 0 else f"FAILED ({len(failed_match)} test(s) failed)"
530      print(f"[{i+1}/{len(futures)}] Test set {test_name} {result}\n")
531  print(f"Overall, {len(failed_tests)} test(s) failed:")
532  print("\n".join(failed_tests))
533  sys.exit(max_exit_code)
534
535if __name__ == '__main__':
536  main()
537