1 /*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <fcntl.h>
18 #if !defined(__APPLE__)
19 #include <malloc.h>
20 #endif
21 #include <signal.h>
22 #include <stdio.h>
23 #include <stdlib.h>
24 #include <sys/types.h>
25 #include <sys/wait.h>
26 #include <unistd.h>
27
28 #include <map>
29 #include <regex>
30 #include <sstream>
31 #include <string>
32 #include <vector>
33
34 #include <android-base/file.h>
35 #include <android-base/properties.h>
36 #include <android-base/strings.h>
37 #include <android-base/test_utils.h>
38 #include <gtest/gtest.h>
39
40 #include "NanoTime.h"
41
42 #if defined(__APPLE__)
43 extern char** environ;
44
clearenv()45 int clearenv() {
46 *environ = nullptr;
47 return 0;
48 }
49 #endif
50
51 // Change the slow threshold for these tests since a few can take around
52 // 20 seconds.
GetInitialArgs(const char *** args,size_t * num_args)53 extern "C" bool GetInitialArgs(const char*** args, size_t* num_args) {
54 static const char* initial_args[] = {"--slow_threshold_ms=25000"};
55 *args = initial_args;
56 *num_args = 1;
57 return true;
58 }
59
60 namespace android {
61 namespace gtest_extras {
62
63 class SystemTests : public ::testing::Test {
64 protected:
SetUp()65 void SetUp() override {
66 raw_output_ = "";
67 sanitized_output_ = "";
68 exitcode_ = 0;
69
70 // Clear all environment variables to make sure the test isn't affected
71 // by GTEST_XXX ones.
72 clearenv();
73 }
74
75 void SanitizeOutput();
76
77 void Exec(std::vector<const char*> args);
78 void ExecAndCapture(std::vector<const char*> args);
79 void RunTest(const std::string& test_name, std::vector<const char*> extra_args = {});
80 void RunTestCaptureFooter(const std::string& test_name, std::string* footer,
81 std::vector<const char*> extra_args = {});
82 void Verify(const std::string& test_name, const std::string& expected_output,
83 int expected_exitcode, std::vector<const char*> extra_args = {});
84 void VerifySortedOutput(const std::string& test_name, const std::string& expected_output,
85 int expected_exitcode, std::vector<const char*> extra_args = {});
86
delay(int seconds)87 static void delay(int seconds) { sleep(seconds * android::base::HwTimeoutMultiplier()); }
88
89 std::string raw_output_;
90 std::string sanitized_output_;
91 int exitcode_;
92 pid_t pid_;
93 int fd_;
94 };
95
SortTestOutput(const std::string & output)96 static std::string SortTestOutput(const std::string& output) {
97 std::map<std::string, std::string> tests;
98
99 std::vector<std::string> lines(android::base::Split(output, "\n"));
100 std::string prefix;
101 std::string suffix;
102 bool capture_prefix = true;
103 for (auto iter = lines.begin(); iter != lines.end(); ++iter) {
104 const char* line = iter->c_str();
105 if (line[0] == '\x1B') {
106 // Skip the escape sequence.
107 line = &line[7];
108 }
109 if (strncmp(line, "[ RUN ]", 12) == 0) {
110 std::string test_name(iter->substr(12));
111 std::string test_body(*iter + '\n');
112 bool ended = false;
113 for (++iter; iter != lines.end(); ++iter) {
114 test_body += *iter + '\n';
115 line = iter->c_str();
116 if (line[0] == '\x1B') {
117 // Skip the escape sequence.
118 line = &line[7];
119 }
120 if (strncmp(line, "[ ", 2) == 0) {
121 ended = true;
122 break;
123 }
124 }
125 if (!ended) {
126 return output;
127 }
128 tests[test_name] = test_body;
129 capture_prefix = false;
130 } else if (capture_prefix) {
131 prefix += *iter + '\n';
132 } else {
133 suffix += *iter + '\n';
134 }
135 }
136
137 std::string new_output("Output Sorted\n" + prefix);
138 for (auto entry : tests) {
139 new_output += entry.second;
140 }
141 new_output += suffix;
142 if (android::base::EndsWith(new_output, "\n\n")) {
143 new_output.resize(new_output.size() - 1);
144 }
145 return new_output;
146 }
147
SanitizeOutput()148 void SystemTests::SanitizeOutput() {
149 // Change (100 ms to (XX ms
150 sanitized_output_ =
151 std::regex_replace(raw_output_, std::regex("\\(\\d+ ms(\\)|\\s|,)"), "(XX ms$1");
152
153 // Change (elapsed time 100 ms to (elapsed time XX ms
154 sanitized_output_ = std::regex_replace(
155 sanitized_output_, std::regex("\\(elapsed time \\d+ ms(\\)|\\s|,)"), "(elapsed time XX ms$1");
156
157 // Change stopped|timeout at 100 ms to stopped|timeout at XX ms
158 sanitized_output_ = std::regex_replace(sanitized_output_,
159 std::regex("(stopped|timeout) at \\d+ ms"), "$1 at XX ms");
160
161 // Change any error message like .../file.cc:(200) to file:(XX)
162 sanitized_output_ = std::regex_replace(
163 sanitized_output_, std::regex("\\b([^/\\s]+/)*[^/\\s]+:\\(\\d+\\)\\s"), "file:(XX) ");
164
165 // Change any terminated by signal message to ignore the actual signal name.
166 sanitized_output_ =
167 std::regex_replace(sanitized_output_, std::regex("( terminated by signal:) .*"), "$1 XXX");
168 }
169
Exec(std::vector<const char * > args)170 void SystemTests::Exec(std::vector<const char*> args) {
171 int fds[2];
172 #if !defined(__APPLE__)
173 ASSERT_NE(-1, pipe2(fds, O_NONBLOCK));
174 #else
175 ASSERT_NE(-1, pipe(fds));
176 ASSERT_NE(-1, fcntl(fds[0], F_SETFL, O_NONBLOCK));
177 #endif
178
179 if ((pid_ = fork()) == 0) {
180 // Run the test.
181 close(fds[0]);
182 close(STDIN_FILENO);
183 close(STDOUT_FILENO);
184 close(STDERR_FILENO);
185 ASSERT_NE(0, dup2(fds[1], STDOUT_FILENO));
186 ASSERT_NE(0, dup2(fds[1], STDERR_FILENO));
187 close(fds[1]);
188
189 std::string exe_name = android::base::GetExecutablePath();
190 args.insert(args.begin(), exe_name.c_str());
191 args.push_back(nullptr);
192 execv(args[0], reinterpret_cast<char* const*>(const_cast<char**>(args.data())));
193 exit(1);
194 }
195 ASSERT_NE(-1, pid_);
196
197 close(fds[1]);
198 fd_ = fds[0];
199 }
200
ExecAndCapture(std::vector<const char * > args)201 void SystemTests::ExecAndCapture(std::vector<const char*> args) {
202 Exec(args);
203
204 int flags = fcntl(fd_, F_GETFL);
205 ASSERT_NE(-1, flags);
206 flags &= ~O_NONBLOCK;
207 ASSERT_NE(-1, fcntl(fd_, F_SETFL, flags));
208 ASSERT_TRUE(android::base::ReadFdToString(fd_, &raw_output_));
209 close(fd_);
210
211 int status;
212 ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, &status, 0))) << "Test output:\n" << raw_output_;
213 exitcode_ = WEXITSTATUS(status);
214 SanitizeOutput();
215 }
216
RunTest(const std::string & test_name,std::vector<const char * > extra_args)217 void SystemTests::RunTest(const std::string& test_name, std::vector<const char*> extra_args) {
218 std::vector<const char*> args;
219 bool job_count = false;
220 for (const auto& arg : extra_args) {
221 if (strncmp(arg, "-j", 2) == 0) {
222 job_count = true;
223 }
224 args.push_back(arg);
225 }
226 if (!job_count) {
227 // Always set to only 20 jobs if no job count option is set.
228 args.push_back("-j20");
229 }
230 args.push_back("--gtest_also_run_disabled_tests");
231 std::string filter_arg("--gtest_filter=" + test_name);
232 args.push_back(filter_arg.c_str());
233
234 ExecAndCapture(args);
235 }
236
RunTestCaptureFooter(const std::string & test_name,std::string * footer,std::vector<const char * > extra_args)237 void SystemTests::RunTestCaptureFooter(const std::string& test_name, std::string* footer,
238 std::vector<const char*> extra_args) {
239 ASSERT_NO_FATAL_FAILURE(RunTest(test_name, extra_args));
240
241 // Verify the order of the output messages.
242 std::stringstream stream(sanitized_output_);
243 std::string line;
244 footer->clear();
245 while (std::getline(stream, line, '\n')) {
246 if (!footer->empty()) {
247 *footer += line + '\n';
248 } else if (android::base::StartsWith(line, "[ PASSED ] ")) {
249 *footer = line + '\n';
250 }
251 }
252 ASSERT_FALSE(footer->empty()) << "Test output:\n" << raw_output_;
253 }
254
Verify(const std::string & test_name,const std::string & expected_output,int expected_exitcode,std::vector<const char * > extra_args)255 void SystemTests::Verify(const std::string& test_name, const std::string& expected_output,
256 int expected_exitcode, std::vector<const char*> extra_args) {
257 ASSERT_NO_FATAL_FAILURE(RunTest(test_name, extra_args));
258 ASSERT_EQ(expected_exitcode, exitcode_) << "Test output:\n" << raw_output_;
259 if (!expected_output.empty()) {
260 ASSERT_EQ(expected_output, sanitized_output_);
261 }
262 }
263
VerifySortedOutput(const std::string & test_name,const std::string & expected_output,int expected_exitcode,std::vector<const char * > extra_args)264 void SystemTests::VerifySortedOutput(const std::string& test_name,
265 const std::string& expected_output, int expected_exitcode,
266 std::vector<const char*> extra_args) {
267 ASSERT_NO_FATAL_FAILURE(RunTest(test_name, extra_args));
268 ASSERT_EQ(expected_exitcode, exitcode_) << "Test output:\n" << raw_output_;
269 if (!expected_output.empty()) {
270 std::string sorted_output = SortTestOutput(sanitized_output_);
271 ASSERT_EQ(expected_output, sorted_output);
272 }
273 }
274
TEST_F(SystemTests,verify_pass)275 TEST_F(SystemTests, verify_pass) {
276 std::string expected =
277 "Note: Google Test filter = *.DISABLED_pass\n"
278 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
279 "[ RUN ] SystemTests.DISABLED_pass\n"
280 "[ OK ] SystemTests.DISABLED_pass (XX ms)\n"
281 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
282 "[ PASSED ] 1 test.\n";
283 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_pass", expected, 0));
284 }
285
TEST_F(SystemTests,verify_pass_no_print_time)286 TEST_F(SystemTests, verify_pass_no_print_time) {
287 std::string expected =
288 "Note: Google Test filter = *.DISABLED_pass\n"
289 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
290 "[ RUN ] SystemTests.DISABLED_pass\n"
291 "[ OK ] SystemTests.DISABLED_pass\n"
292 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
293 "[ PASSED ] 1 test.\n";
294 ASSERT_NO_FATAL_FAILURE(
295 Verify("*.DISABLED_pass", expected, 0, std::vector<const char*>{"--gtest_print_time=0"}));
296 }
297
TEST_F(SystemTests,verify_pass_color)298 TEST_F(SystemTests, verify_pass_color) {
299 std::string expected =
300 "\x1B[0;33mNote: Google Test filter = *.DISABLED_pass\x1B[m\n"
301 "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
302 "\x1B[0;32m[ RUN ]\x1B[m SystemTests.DISABLED_pass\n"
303 "\x1B[0;32m[ OK ]\x1B[m SystemTests.DISABLED_pass (XX ms)\n"
304 "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
305 "\x1B[0;32m[ PASSED ]\x1B[m 1 test.\n";
306 ASSERT_NO_FATAL_FAILURE(
307 Verify("*.DISABLED_pass", expected, 0, std::vector<const char*>{"--gtest_color=yes"}));
308 }
309
TEST_F(SystemTests,verify_skip)310 TEST_F(SystemTests, verify_skip) {
311 std::string expected =
312 "Note: Google Test filter = *.DISABLED_skip_no_message\n"
313 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
314 "[ RUN ] SystemTests.DISABLED_skip_no_message\n"
315 "file:(XX) Skipped\n"
316 "[ SKIPPED ] SystemTests.DISABLED_skip_no_message (XX ms)\n"
317 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
318 "[ PASSED ] 0 tests.\n"
319 "[ SKIPPED ] 1 test, listed below:\n"
320 "[ SKIPPED ] SystemTests.DISABLED_skip_no_message\n";
321 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_no_message", expected, 0));
322 }
323
TEST_F(SystemTests,verify_skip_with_message)324 TEST_F(SystemTests, verify_skip_with_message) {
325 std::string expected =
326 "Note: Google Test filter = *.DISABLED_skip_with_message\n"
327 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
328 "[ RUN ] SystemTests.DISABLED_skip_with_message\n"
329 "file:(XX) Skipped\n"
330 "This is a skip message\n"
331 "[ SKIPPED ] SystemTests.DISABLED_skip_with_message (XX ms)\n"
332 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
333 "[ PASSED ] 0 tests.\n"
334 "[ SKIPPED ] 1 test, listed below:\n"
335 "[ SKIPPED ] SystemTests.DISABLED_skip_with_message\n";
336 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_with_message", expected, 0));
337 }
338
TEST_F(SystemTests,verify_skip_with_output_before_message)339 TEST_F(SystemTests, verify_skip_with_output_before_message) {
340 std::string expected =
341 "Note: Google Test filter = *.DISABLED_skip_with_output_before\n"
342 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
343 "[ RUN ] SystemTests.DISABLED_skip_with_output_before\n"
344 "This is the message before the skip message\n"
345 "file:(XX) Skipped\n"
346 "This is the skip message\n"
347 "[ SKIPPED ] SystemTests.DISABLED_skip_with_output_before (XX ms)\n"
348 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
349 "[ PASSED ] 0 tests.\n"
350 "[ SKIPPED ] 1 test, listed below:\n"
351 "[ SKIPPED ] SystemTests.DISABLED_skip_with_output_before\n";
352 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_with_output_before", expected, 0));
353 }
354
TEST_F(SystemTests,verify_skip_with_output_after_message)355 TEST_F(SystemTests, verify_skip_with_output_after_message) {
356 std::string expected =
357 "Note: Google Test filter = *.DISABLED_skip_with_output_after\n"
358 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
359 "[ RUN ] SystemTests.DISABLED_skip_with_output_after\n"
360 "file:(XX) Skipped\n"
361 "This is the skip message\n"
362 "This is the message after the skip message\n"
363 "[ SKIPPED ] SystemTests.DISABLED_skip_with_output_after (XX ms)\n"
364 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
365 "[ PASSED ] 0 tests.\n"
366 "[ SKIPPED ] 1 test, listed below:\n"
367 "[ SKIPPED ] SystemTests.DISABLED_skip_with_output_after\n";
368 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_with_output_after", expected, 0));
369 }
370
TEST_F(SystemTests,verify_skip_with_skipped_line)371 TEST_F(SystemTests, verify_skip_with_skipped_line) {
372 std::string expected =
373 "Note: Google Test filter = *.DISABLED_skip_with_skipped_line\n"
374 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
375 "[ RUN ] SystemTests.DISABLED_skip_with_skipped_line\n"
376 "\n"
377 "Skipped\n"
378 "file:(XX) Skipped\n"
379 "This is the skip message 1\n"
380 "Skipped\n"
381 "file:(XX) Skipped\n"
382 "This is the skip message 2\n"
383 "Skipped\n"
384 "[ SKIPPED ] SystemTests.DISABLED_skip_with_skipped_line (XX ms)\n"
385 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
386 "[ PASSED ] 0 tests.\n"
387 "[ SKIPPED ] 1 test, listed below:\n"
388 "[ SKIPPED ] SystemTests.DISABLED_skip_with_skipped_line\n";
389 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_with_skipped_line", expected, 0));
390 }
391
TEST_F(SystemTests,verify_skip_multiple)392 TEST_F(SystemTests, verify_skip_multiple) {
393 std::string expected =
394 "Note: Google Test filter = *.DISABLED_skip_multiple\n"
395 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
396 "[ RUN ] SystemTests.DISABLED_skip_multiple\n"
397 "This is not a skip message 1\n"
398 "file:(XX) Skipped\n"
399 "This is the skip message 1\n"
400 "This is not a skip message 2\n"
401 "file:(XX) Skipped\n"
402 "This is the skip message 2\n"
403 "file:(XX) Skipped\n"
404 "This is the skip message 3\n"
405 "This is not a skip message 4\n"
406 "[ SKIPPED ] SystemTests.DISABLED_skip_multiple (XX ms)\n"
407 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
408 "[ PASSED ] 0 tests.\n"
409 "[ SKIPPED ] 1 test, listed below:\n"
410 "[ SKIPPED ] SystemTests.DISABLED_skip_multiple\n";
411 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_multiple", expected, 0));
412 }
413
TEST_F(SystemTests,verify_skip_no_print_time)414 TEST_F(SystemTests, verify_skip_no_print_time) {
415 std::string expected =
416 "Note: Google Test filter = *.DISABLED_skip_no_message\n"
417 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
418 "[ RUN ] SystemTests.DISABLED_skip_no_message\n"
419 "file:(XX) Skipped\n"
420 "[ SKIPPED ] SystemTests.DISABLED_skip_no_message\n"
421 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
422 "[ PASSED ] 0 tests.\n"
423 "[ SKIPPED ] 1 test, listed below:\n"
424 "[ SKIPPED ] SystemTests.DISABLED_skip_no_message\n";
425 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_no_message", expected, 0,
426 std::vector<const char*>{"--gtest_print_time=0"}));
427 }
428
TEST_F(SystemTests,verify_skip_color)429 TEST_F(SystemTests, verify_skip_color) {
430 std::string expected =
431 "\x1B[0;33mNote: Google Test filter = *.DISABLED_skip_no_message\x1B[m\n"
432 "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
433 "\x1B[0;32m[ RUN ]\x1B[m SystemTests.DISABLED_skip_no_message\n"
434 "file:(XX) Skipped\n"
435 "\x1B[0;32m[ SKIPPED ]\x1B[m SystemTests.DISABLED_skip_no_message (XX ms)\n"
436 "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
437 "\x1B[0;32m[ PASSED ]\x1B[m 0 tests.\n"
438 "\x1B[0;32m[ SKIPPED ]\x1B[m 1 test, listed below:\n"
439 "\x1B[0;32m[ SKIPPED ]\x1B[m SystemTests.DISABLED_skip_no_message\n";
440 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_no_message", expected, 0,
441 std::vector<const char*>{"--gtest_color=yes"}));
442 }
443
TEST_F(SystemTests,verify_xfail_fail_expect_to_fail)444 TEST_F(SystemTests, verify_xfail_fail_expect_to_fail) {
445 std::string expected =
446 "Note: Google Test filter = *.xfail_fail\n"
447 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
448 "[ RUN ] DISABLED_SystemTestsXfail.xfail_fail\n"
449 "file:(XX) Failure in test DISABLED_SystemTestsXfail.xfail_fail\n"
450 "Expected equality of these values:\n"
451 " 1\n"
452 " 0\n"
453 "DISABLED_SystemTestsXfail.xfail_fail exited with exitcode 1.\n"
454 "[ OK ] DISABLED_SystemTestsXfail.xfail_fail (XX ms)\n"
455 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
456 "[ PASSED ] 1 test. (1 expected failure)\n";
457 ASSERT_NO_FATAL_FAILURE(Verify("*.xfail_fail", expected, 0));
458 }
459
TEST_F(SystemTests,verify_xfail_pass_expect_to_fail)460 TEST_F(SystemTests, verify_xfail_pass_expect_to_fail) {
461 std::string expected =
462 "Note: Google Test filter = *.xfail_pass\n"
463 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
464 "[ RUN ] DISABLED_SystemTestsXfail.xfail_pass\n"
465 "[ FAILED ] DISABLED_SystemTestsXfail.xfail_pass (XX ms)\n"
466 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
467 "[ PASSED ] 0 tests.\n"
468 "[ FAILED ] 1 test should have failed, listed below:\n"
469 "[ FAILED ] DISABLED_SystemTestsXfail.xfail_pass\n"
470 "\n"
471 " 1 SHOULD HAVE FAILED TEST\n";
472 ASSERT_NO_FATAL_FAILURE(Verify("*.xfail_pass", expected, 1));
473 }
474
TEST_F(SystemTests,verify_xfail_pass_expect_to_fail_color)475 TEST_F(SystemTests, verify_xfail_pass_expect_to_fail_color) {
476 std::string expected =
477 "\x1B[0;33mNote: Google Test filter = *.xfail_pass\x1B[m\n"
478 "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
479 "\x1B[0;32m[ RUN ]\x1B[m DISABLED_SystemTestsXfail.xfail_pass\n"
480 "\x1B[0;31m[ FAILED ]\x1B[m DISABLED_SystemTestsXfail.xfail_pass (XX ms)\n"
481 "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
482 "\x1B[0;32m[ PASSED ]\x1B[m 0 tests.\n"
483 "\x1B[0;31m[ FAILED ]\x1B[m 1 test should have failed, listed below:\n"
484 "\x1B[0;31m[ FAILED ]\x1B[m DISABLED_SystemTestsXfail.xfail_pass\n"
485 "\n"
486 " 1 SHOULD HAVE FAILED TEST\n";
487 ASSERT_NO_FATAL_FAILURE(
488 Verify("*.xfail_pass", expected, 1, std::vector<const char*>{"--gtest_color=yes"}));
489 }
490
TEST_F(SystemTests,verify_deathtest_pass)491 TEST_F(SystemTests, verify_deathtest_pass) {
492 std::string expected =
493 "Note: Google Test filter = *.DISABLED_death_pass\n"
494 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
495 "[ RUN ] SystemTestsDeathTest.DISABLED_death_pass\n"
496 "[ OK ] SystemTestsDeathTest.DISABLED_death_pass (XX ms)\n"
497 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
498 "[ PASSED ] 1 test.\n";
499 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_death_pass", expected, 0));
500 }
501
TEST_F(SystemTests,verify_fail)502 TEST_F(SystemTests, verify_fail) {
503 std::string expected =
504 "Note: Google Test filter = *.DISABLED_fail\n"
505 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
506 "[ RUN ] SystemTests.DISABLED_fail\n"
507 "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
508 "Expected equality of these values:\n"
509 " 1\n"
510 " 0\n"
511 "SystemTests.DISABLED_fail exited with exitcode 1.\n"
512 "[ FAILED ] SystemTests.DISABLED_fail (XX ms)\n"
513 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
514 "[ PASSED ] 0 tests.\n"
515 "[ FAILED ] 1 test, listed below:\n"
516 "[ FAILED ] SystemTests.DISABLED_fail\n"
517 "\n"
518 " 1 FAILED TEST\n";
519 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_fail", expected, 1));
520 }
521
TEST_F(SystemTests,verify_fail_color)522 TEST_F(SystemTests, verify_fail_color) {
523 std::string expected =
524 "\x1B[0;33mNote: Google Test filter = *.DISABLED_fail\x1B[m\n"
525 "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
526 "\x1B[0;32m[ RUN ]\x1B[m SystemTests.DISABLED_fail\n"
527 "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
528 "Expected equality of these values:\n"
529 " 1\n"
530 " 0\n"
531 "SystemTests.DISABLED_fail exited with exitcode 1.\n"
532 "\x1B[0;31m[ FAILED ]\x1B[m SystemTests.DISABLED_fail (XX ms)\n"
533 "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
534 "\x1B[0;32m[ PASSED ]\x1B[m 0 tests.\n"
535 "\x1B[0;31m[ FAILED ]\x1B[m 1 test, listed below:\n"
536 "\x1B[0;31m[ FAILED ]\x1B[m SystemTests.DISABLED_fail\n"
537 "\n"
538 " 1 FAILED TEST\n";
539 ASSERT_NO_FATAL_FAILURE(
540 Verify("*.DISABLED_fail", expected, 1, std::vector<const char*>{"--gtest_color=yes"}));
541 }
542
TEST_F(SystemTests,verify_deathtest_fail)543 TEST_F(SystemTests, verify_deathtest_fail) {
544 std::string expected =
545 "Note: Google Test filter = *.DISABLED_death_fail\n"
546 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
547 "[ RUN ] SystemTestsDeathTest.DISABLED_death_fail\n"
548 "file:(XX) Failure in test SystemTestsDeathTest.DISABLED_death_fail\n"
549 "Death test: DeathTestHelperFail()\n"
550 " Result: failed to die.\n"
551 " Error msg:\n"
552 "[ DEATH ] \n"
553 "SystemTestsDeathTest.DISABLED_death_fail exited with exitcode 1.\n"
554 "[ FAILED ] SystemTestsDeathTest.DISABLED_death_fail (XX ms)\n"
555 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
556 "[ PASSED ] 0 tests.\n"
557 "[ FAILED ] 1 test, listed below:\n"
558 "[ FAILED ] SystemTestsDeathTest.DISABLED_death_fail\n"
559 "\n"
560 " 1 FAILED TEST\n";
561 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_death_fail", expected, 1));
562 }
563
TEST_F(SystemTests,verify_crash)564 TEST_F(SystemTests, verify_crash) {
565 std::string expected =
566 "Note: Google Test filter = *.DISABLED_crash\n"
567 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
568 "[ RUN ] SystemTests.DISABLED_crash\n"
569 "SystemTests.DISABLED_crash terminated by signal: XXX\n"
570 "[ FAILED ] SystemTests.DISABLED_crash (XX ms)\n"
571 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
572 "[ PASSED ] 0 tests.\n"
573 "[ FAILED ] 1 test, listed below:\n"
574 "[ FAILED ] SystemTests.DISABLED_crash\n"
575 "\n"
576 " 1 FAILED TEST\n";
577 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_crash", expected, 1));
578 }
579
TEST_F(SystemTests,verify_warning_slow)580 TEST_F(SystemTests, verify_warning_slow) {
581 std::string expected =
582 "Note: Google Test filter = *.DISABLED_sleep5\n"
583 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
584 "[ RUN ] SystemTests.DISABLED_sleep5\n"
585 "[ OK ] SystemTests.DISABLED_sleep5 (XX ms)\n"
586 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
587 "[ PASSED ] 1 test.\n"
588 "[ SLOW ] 1 test, listed below:\n"
589 "[ SLOW ] SystemTests.DISABLED_sleep5 (XX ms, exceeded 3000 ms)\n"
590 "\n"
591 " 1 SLOW TEST\n";
592 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_sleep5", expected, 0,
593 std::vector<const char*>{"--slow_threshold_ms=3000"}));
594 }
595
TEST_F(SystemTests,verify_warning_slow_color)596 TEST_F(SystemTests, verify_warning_slow_color) {
597 std::string expected =
598 "\x1B[0;33mNote: Google Test filter = *.DISABLED_sleep5\x1B[m\n"
599 "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
600 "\x1B[0;32m[ RUN ]\x1B[m SystemTests.DISABLED_sleep5\n"
601 "\x1B[0;32m[ OK ]\x1B[m SystemTests.DISABLED_sleep5 (XX ms)\n"
602 "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
603 "\x1B[0;32m[ PASSED ]\x1B[m 1 test.\n"
604 "\x1B[0;33m[ SLOW ]\x1B[m 1 test, listed below:\n"
605 "\x1B[0;33m[ SLOW ]\x1B[m SystemTests.DISABLED_sleep5 (XX ms, exceeded 3000 ms)\n"
606 "\n"
607 " 1 SLOW TEST\n";
608 ASSERT_NO_FATAL_FAILURE(
609 Verify("*.DISABLED_sleep5", expected, 0,
610 std::vector<const char*>{"--slow_threshold_ms=3000", "--gtest_color=yes"}));
611 }
612
TEST_F(SystemTests,verify_timeout)613 TEST_F(SystemTests, verify_timeout) {
614 std::string expected =
615 "Note: Google Test filter = *.DISABLED_sleep_forever\n"
616 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
617 "[ RUN ] SystemTests.DISABLED_sleep_forever\n"
618 "SystemTests.DISABLED_sleep_forever killed because of timeout at XX ms.\n"
619 "[ FAILED ] SystemTests.DISABLED_sleep_forever (XX ms)\n"
620 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
621 "[ PASSED ] 0 tests.\n"
622 "[ TIMEOUT ] 1 test, listed below:\n"
623 "[ TIMEOUT ] SystemTests.DISABLED_sleep_forever (stopped at XX ms)\n"
624 "\n"
625 " 1 TIMEOUT TEST\n";
626 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_sleep_forever", expected, 1,
627 std::vector<const char*>{"--deadline_threshold_ms=3000"}));
628 }
629
630 // Verify that tests that timeout do not get marked as slow too when
631 // another test is marked as slow.
TEST_F(SystemTests,verify_timeout_not_slow)632 TEST_F(SystemTests, verify_timeout_not_slow) {
633 std::string expected =
634 "Note: Google Test filter = *.DISABLED_sleep*\n"
635 "[==========] Running 2 tests from 1 test suite (20 jobs).\n"
636 "[ RUN ] SystemTests.DISABLED_sleep5\n"
637 "[ OK ] SystemTests.DISABLED_sleep5 (XX ms)\n"
638 "[ RUN ] SystemTests.DISABLED_sleep_forever\n"
639 "SystemTests.DISABLED_sleep_forever killed because of timeout at XX ms.\n"
640 "[ FAILED ] SystemTests.DISABLED_sleep_forever (XX ms)\n"
641 "[==========] 2 tests from 1 test suite ran. (XX ms total)\n"
642 "[ PASSED ] 1 test.\n"
643 "[ SLOW ] 1 test, listed below:\n"
644 "[ SLOW ] SystemTests.DISABLED_sleep5 (XX ms, exceeded 1000 ms)\n"
645 "[ TIMEOUT ] 1 test, listed below:\n"
646 "[ TIMEOUT ] SystemTests.DISABLED_sleep_forever (stopped at XX ms)\n"
647 "\n"
648 " 1 SLOW TEST\n"
649 " 1 TIMEOUT TEST\n";
650 ASSERT_NO_FATAL_FAILURE(Verify(
651 "*.DISABLED_sleep*", expected, 1,
652 std::vector<const char*>{"--slow_threshold_ms=1000", "--deadline_threshold_ms=10000"}));
653 }
654
TEST_F(SystemTests,verify_timeout_color)655 TEST_F(SystemTests, verify_timeout_color) {
656 std::string expected =
657 "\x1B[0;33mNote: Google Test filter = *.DISABLED_sleep_forever\x1B[m\n"
658 "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
659 "\x1B[0;32m[ RUN ]\x1B[m SystemTests.DISABLED_sleep_forever\n"
660 "SystemTests.DISABLED_sleep_forever killed because of timeout at XX ms.\n"
661 "\x1B[0;31m[ FAILED ]\x1B[m SystemTests.DISABLED_sleep_forever (XX ms)\n"
662 "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
663 "\x1B[0;32m[ PASSED ]\x1B[m 0 tests.\n"
664 "\x1B[0;31m[ TIMEOUT ]\x1B[m 1 test, listed below:\n"
665 "\x1B[0;31m[ TIMEOUT ]\x1B[m SystemTests.DISABLED_sleep_forever (stopped at XX ms)\n"
666 "\n"
667 " 1 TIMEOUT TEST\n";
668 ASSERT_NO_FATAL_FAILURE(
669 Verify("*.DISABLED_sleep_forever", expected, 1,
670 std::vector<const char*>{"--deadline_threshold_ms=3000", "--gtest_color=yes"}));
671 }
672
TEST_F(SystemTests,verify_order_isolated)673 TEST_F(SystemTests, verify_order_isolated) {
674 std::string expected =
675 "Note: Google Test filter = *.DISABLED_order_*\n"
676 "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
677 "[ RUN ] SystemTests.DISABLED_order_3\n"
678 "[ OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
679 "[ RUN ] SystemTests.DISABLED_order_2\n"
680 "[ OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
681 "[ RUN ] SystemTests.DISABLED_order_1\n"
682 "[ OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
683 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
684 "[ PASSED ] 3 tests.\n";
685 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_order_*", expected, 0));
686 }
687
TEST_F(SystemTests,verify_order_not_isolated)688 TEST_F(SystemTests, verify_order_not_isolated) {
689 std::string expected =
690 "Note: Google Test filter = *.DISABLED_order_*\n"
691 "[==========] Running 3 tests from 1 test suite.\n"
692 "[----------] Global test environment set-up.\n"
693 "[----------] 3 tests from SystemTests\n"
694 "[ RUN ] SystemTests.DISABLED_order_1\n"
695 "[ OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
696 "[ RUN ] SystemTests.DISABLED_order_2\n"
697 "[ OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
698 "[ RUN ] SystemTests.DISABLED_order_3\n"
699 "[ OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
700 "[----------] 3 tests from SystemTests (XX ms total)\n"
701 "\n"
702 "[----------] Global test environment tear-down\n"
703 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
704 "[ PASSED ] 3 tests.\n";
705 ASSERT_NO_FATAL_FAILURE(
706 Verify("*.DISABLED_order_*", expected, 0, std::vector<const char*>{"--no_isolate"}));
707 }
708
TEST_F(SystemTests,verify_fail_ge10)709 TEST_F(SystemTests, verify_fail_ge10) {
710 ASSERT_NO_FATAL_FAILURE(RunTest("*.DISABLED_fail_*"));
711 // Verify the failed output at the end has no space in front.
712 std::regex regex("\\n.*\\d+ FAILED TESTS\\n");
713 std::cmatch match;
714 ASSERT_TRUE(std::regex_search(sanitized_output_.c_str(), match, regex)) << "Test Output:\n"
715 << raw_output_;
716 ASSERT_EQ("\n10 FAILED TESTS\n", match[0]);
717 ASSERT_NE(0, exitcode_);
718 }
719
TEST_F(SystemTests,verify_title_order)720 TEST_F(SystemTests, verify_title_order) {
721 std::string footer;
722 ASSERT_NO_FATAL_FAILURE(RunTestCaptureFooter(
723 "*.DISABLED_all_*", &footer,
724 std::vector<const char*>{"--slow_threshold_ms=2000", "--deadline_threshold_ms=4000"}));
725
726 ASSERT_EQ(
727 "[ PASSED ] 4 tests.\n"
728 "[ SKIPPED ] 2 tests, listed below:\n"
729 "[ SKIPPED ] SystemTests.DISABLED_all_skip_1\n"
730 "[ SKIPPED ] SystemTests.DISABLED_all_skip_2\n"
731 "[ SLOW ] 2 tests, listed below:\n"
732 "[ SLOW ] SystemTests.DISABLED_all_slow_1 (XX ms, exceeded 2000 ms)\n"
733 "[ SLOW ] SystemTests.DISABLED_all_slow_2 (XX ms, exceeded 2000 ms)\n"
734 "[ TIMEOUT ] 2 tests, listed below:\n"
735 "[ TIMEOUT ] SystemTests.DISABLED_all_timeout_1 (stopped at XX ms)\n"
736 "[ TIMEOUT ] SystemTests.DISABLED_all_timeout_2 (stopped at XX ms)\n"
737 "[ FAILED ] 2 tests, listed below:\n"
738 "[ FAILED ] SystemTests.DISABLED_all_fail_1\n"
739 "[ FAILED ] SystemTests.DISABLED_all_fail_2\n"
740 "\n"
741 " 2 SLOW TESTS\n"
742 " 2 TIMEOUT TESTS\n"
743 " 2 FAILED TESTS\n",
744 footer);
745 }
746
TEST_F(SystemTests,verify_job_count_single)747 TEST_F(SystemTests, verify_job_count_single) {
748 std::string expected =
749 "Note: Google Test filter = *.DISABLED_job_*\n"
750 "[==========] Running 3 tests from 1 test suite (1 job).\n"
751 "[ RUN ] SystemTests.DISABLED_job_1\n"
752 "[ OK ] SystemTests.DISABLED_job_1 (XX ms)\n"
753 "[ RUN ] SystemTests.DISABLED_job_2\n"
754 "[ OK ] SystemTests.DISABLED_job_2 (XX ms)\n"
755 "[ RUN ] SystemTests.DISABLED_job_3\n"
756 "[ OK ] SystemTests.DISABLED_job_3 (XX ms)\n"
757 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
758 "[ PASSED ] 3 tests.\n";
759 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_job_*", expected, 0, std::vector<const char*>{"-j1"}));
760 }
761
TEST_F(SystemTests,verify_job_count_multiple)762 TEST_F(SystemTests, verify_job_count_multiple) {
763 std::string expected =
764 "Note: Google Test filter = *.DISABLED_job_*\n"
765 "[==========] Running 3 tests from 1 test suite (2 jobs).\n"
766 "[ RUN ] SystemTests.DISABLED_job_2\n"
767 "[ OK ] SystemTests.DISABLED_job_2 (XX ms)\n"
768 "[ RUN ] SystemTests.DISABLED_job_1\n"
769 "[ OK ] SystemTests.DISABLED_job_1 (XX ms)\n"
770 "[ RUN ] SystemTests.DISABLED_job_3\n"
771 "[ OK ] SystemTests.DISABLED_job_3 (XX ms)\n"
772 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
773 "[ PASSED ] 3 tests.\n";
774 ASSERT_NO_FATAL_FAILURE(
775 Verify("*.DISABLED_job_*", expected, 0, std::vector<const char*>{"-j", "2"}));
776 }
777
TEST_F(SystemTests,verify_help)778 TEST_F(SystemTests, verify_help) {
779 // This tests verifies that the help options display the help for
780 // the isolated test run, and for the gtest data.
781 std::vector<const char*> help_args{"-h", "--help"};
782 for (auto arg : help_args) {
783 ASSERT_NO_FATAL_FAILURE(RunTest("*.DISABLED_pass", std::vector<const char*>{arg}));
784 ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
785 // First find something from the isolation help.
786 std::size_t isolation_help = sanitized_output_.find("In isolation mode,");
787 ASSERT_NE(std::string::npos, isolation_help) << "Cannot find isolation help:\n" << raw_output_;
788 std::size_t gtest_help = sanitized_output_.find("Assertion Behavior:");
789 ASSERT_NE(std::string::npos, gtest_help) << "Cannot find gtest help:\n" << raw_output_;
790
791 ASSERT_GT(gtest_help, isolation_help) << "Gtest help before isolation help:\n" << raw_output_;
792 }
793 }
794
TEST_F(SystemTests,verify_help_color)795 TEST_F(SystemTests, verify_help_color) {
796 // Verify that the color option does change the help display.
797 std::vector<const char*> help_args{"-h", "--help"};
798 for (auto arg : help_args) {
799 ASSERT_NO_FATAL_FAILURE(
800 RunTest("*.DISABLED_pass", std::vector<const char*>{arg, "--gtest_color=yes"}));
801 ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
802 // First find something from the isolation help that is in color.
803 std::size_t isolation_help =
804 sanitized_output_.find("Unit Test Options:\n\x1B[0;32m -j \x1B[m");
805 ASSERT_NE(std::string::npos, isolation_help) << "Cannot find isolation help:\n" << raw_output_;
806 std::size_t gtest_help = sanitized_output_.find("\x1B[0;32m--gtest_list_tests\x1B[m");
807 ASSERT_NE(std::string::npos, gtest_help) << "Cannot find gtest help:\n" << raw_output_;
808
809 ASSERT_GT(gtest_help, isolation_help) << "Gtest help before isolation help:\n" << raw_output_;
810 }
811 }
812
TEST_F(SystemTests,verify_repeat)813 TEST_F(SystemTests, verify_repeat) {
814 std::string expected =
815 "Note: Google Test filter = *.DISABLED_order_*\n"
816 "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
817 "[ RUN ] SystemTests.DISABLED_order_3\n"
818 "[ OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
819 "[ RUN ] SystemTests.DISABLED_order_2\n"
820 "[ OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
821 "[ RUN ] SystemTests.DISABLED_order_1\n"
822 "[ OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
823 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
824 "[ PASSED ] 3 tests.\n"
825 "\n"
826 "Repeating all tests (iteration 2) . . .\n"
827 "\n"
828 "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
829 "[ RUN ] SystemTests.DISABLED_order_3\n"
830 "[ OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
831 "[ RUN ] SystemTests.DISABLED_order_2\n"
832 "[ OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
833 "[ RUN ] SystemTests.DISABLED_order_1\n"
834 "[ OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
835 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
836 "[ PASSED ] 3 tests.\n"
837 "\n"
838 "Repeating all tests (iteration 3) . . .\n"
839 "\n"
840 "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
841 "[ RUN ] SystemTests.DISABLED_order_3\n"
842 "[ OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
843 "[ RUN ] SystemTests.DISABLED_order_2\n"
844 "[ OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
845 "[ RUN ] SystemTests.DISABLED_order_1\n"
846 "[ OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
847 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
848 "[ PASSED ] 3 tests.\n";
849 uint64_t time_ns = NanoTime();
850 ASSERT_NO_FATAL_FAILURE(
851 Verify("*.DISABLED_order_*", expected, 0, std::vector<const char*>{"--gtest_repeat=3"}));
852 time_ns = NanoTime() - time_ns;
853 // Make sure that the total test time is about 18 seconds.
854 double seconds = double(time_ns) / 1000000000;
855 // Adjust expected times by multipliers.
856 double min_time = 18.0 * android::base::HwTimeoutMultiplier();
857 double max_time = 20.0 * android::base::HwTimeoutMultiplier();
858 int expected_seconds = 18 * android::base::HwTimeoutMultiplier();
859 ASSERT_LE(min_time, seconds) << "Repeat test should take at least " << expected_seconds
860 << " seconds.\n"
861 << "Test output:\n"
862 << raw_output_;
863 ASSERT_GT(max_time, seconds) << "Repeat test should take about " << expected_seconds
864 << " seconds.\n"
865 << "Test output:\n"
866 << raw_output_;
867 }
868
TEST_F(SystemTests,verify_results_as_tests_finish)869 TEST_F(SystemTests, verify_results_as_tests_finish) {
870 // This test verifies that test output comes out as the test finishes.
871 Exec(std::vector<const char*>{"--gtest_filter=*.DISABLED_order_*",
872 "--gtest_also_run_disabled_tests", "-j20"});
873
874 std::string output;
875 std::vector<char> buffer(4096);
876 uint64_t time_ns = NanoTime();
877 while (true) {
878 ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
879 if (bytes == -1 && errno == EAGAIN) {
880 continue;
881 }
882 ASSERT_NE(-1, bytes);
883 ASSERT_NE(0, bytes) << "Did not find test output before test finished:\n" << output;
884 buffer[bytes] = '\0';
885 output += buffer.data();
886 // See if the output has come out now.
887 if (output.find("[ OK ] SystemTests.DISABLED_order_2") != std::string::npos) {
888 uint64_t test_ns = NanoTime() - time_ns;
889 double test_sec = double(test_ns) / 1000000000;
890 double min_time = 3.0 * android::base::HwTimeoutMultiplier();
891 double max_time = 4.5 * android::base::HwTimeoutMultiplier();
892 // This should happen after 3 seconds, but before 4.5 seconds.
893 ASSERT_LE(min_time, test_sec) << "Test output:\n" << output;
894 ASSERT_GT(max_time, test_sec) << "Test output:\n" << output;
895 break;
896 }
897 }
898
899 // Read the rest of the output.
900 while (true) {
901 ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
902 if (bytes == -1 && errno == EAGAIN) {
903 continue;
904 }
905 ASSERT_NE(-1, bytes);
906 if (bytes == 0) {
907 break;
908 }
909 buffer[bytes] = '\0';
910 output += buffer.data();
911 }
912 close(fd_);
913 time_ns = NanoTime() - time_ns;
914 ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, nullptr, 0))) << "Test output:\n" << output;
915 // Verify that the total test time is > 6 seconds.
916 double max_time = 6.0 * android::base::HwTimeoutMultiplier();
917 ASSERT_LE(max_time, double(time_ns) / 1000000000) << "Test output:\n" << output;
918 }
919
TEST_F(SystemTests,verify_xml)920 TEST_F(SystemTests, verify_xml) {
921 std::string tmp_arg("--gtest_output=xml:");
922 TemporaryFile tf;
923 ASSERT_TRUE(tf.fd != -1);
924 close(tf.fd);
925 tmp_arg += tf.path;
926
927 ASSERT_NO_FATAL_FAILURE(RunTest("*.DISABLED_xml_*", std::vector<const char*>{tmp_arg.c_str()}));
928 ASSERT_EQ(1, exitcode_) << "Test output:\n" << raw_output_;
929
930 // Check that the xml file exists.
931 FILE* xml_file = fopen(tf.path, "r");
932 ASSERT_TRUE(xml_file != nullptr) << "Failed to find xml file:\n" << raw_output_;
933 // Read the entire file in.
934 std::string xml_output;
935 std::vector<char> buffer(4096);
936 size_t bytes;
937 while ((bytes = fread(buffer.data(), 1, buffer.size(), xml_file)) > 0) {
938 xml_output += std::string(buffer.data(), bytes);
939 }
940 fclose(xml_file);
941 unlink(tf.path);
942
943 // Change time|timestamp="" to time|timestamp="XX"
944 xml_output =
945 std::regex_replace(xml_output, std::regex("(time|timestamp)=\"[^\"]+\""), "$1=\"XX\"");
946 // Change ".*.cc:(XX) to "file:(XX)
947 xml_output = std::regex_replace(xml_output, std::regex("\"([^/\\s]+/)*[^/\\s]+:\\(\\d+\\)\\s"),
948 "\"file:(XX) ");
949
950 std::string expected =
951 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
952 "<testsuites tests=\"6\" failures=\"3\" disabled=\"0\" errors=\"0\" timestamp=\"XX\" "
953 "time=\"XX\" name=\"AllTests\">\n"
954 " <testsuite name=\"SystemTestsXml1\" tests=\"2\" failures=\"1\" disabled=\"0\" "
955 "errors=\"0\" time=\"XX\">\n"
956 " <testcase name=\"DISABLED_xml_1\" status=\"run\" time=\"XX\" "
957 "classname=\"SystemTestsXml1\" />\n"
958 " <testcase name=\"DISABLED_xml_2\" status=\"run\" time=\"XX\" "
959 "classname=\"SystemTestsXml1\">\n"
960 " <failure message=\"file:(XX) Failure in test SystemTestsXml1.DISABLED_xml_2\n"
961 "Expected equality of these values:\n"
962 " 1\n"
963 " 0\n"
964 "SystemTestsXml1.DISABLED_xml_2 exited with exitcode 1.\n"
965 "\" type=\"\">\n"
966 " </failure>\n"
967 " </testcase>\n"
968 " </testsuite>\n"
969 " <testsuite name=\"SystemTestsXml2\" tests=\"2\" failures=\"1\" disabled=\"0\" "
970 "errors=\"0\" time=\"XX\">\n"
971 " <testcase name=\"DISABLED_xml_1\" status=\"run\" time=\"XX\" "
972 "classname=\"SystemTestsXml2\">\n"
973 " <failure message=\"file:(XX) Failure in test SystemTestsXml2.DISABLED_xml_1\n"
974 "Expected equality of these values:\n"
975 " 1\n"
976 " 0\n"
977 "SystemTestsXml2.DISABLED_xml_1 exited with exitcode 1.\n"
978 "\" type=\"\">\n"
979 " </failure>\n"
980 " </testcase>\n"
981 " <testcase name=\"DISABLED_xml_2\" status=\"run\" time=\"XX\" "
982 "classname=\"SystemTestsXml2\" />\n"
983 " </testsuite>\n"
984 " <testsuite name=\"SystemTestsXml3\" tests=\"2\" failures=\"1\" disabled=\"0\" "
985 "errors=\"0\" time=\"XX\">\n"
986 " <testcase name=\"DISABLED_xml_1\" status=\"run\" time=\"XX\" "
987 "classname=\"SystemTestsXml3\" />\n"
988 " <testcase name=\"DISABLED_xml_2\" status=\"run\" time=\"XX\" "
989 "classname=\"SystemTestsXml3\">\n"
990 " <failure message=\"file:(XX) Failure in test SystemTestsXml3.DISABLED_xml_2\n"
991 "Expected equality of these values:\n"
992 " 1\n"
993 " 0\n"
994 "SystemTestsXml3.DISABLED_xml_2 exited with exitcode 1.\n"
995 "\" type=\"\">\n"
996 " </failure>\n"
997 " </testcase>\n"
998 " </testsuite>\n"
999 "</testsuites>\n";
1000 ASSERT_EQ(expected, xml_output);
1001 }
1002
TEST_F(SystemTests,verify_disabled_not_displayed_with_no_tests)1003 TEST_F(SystemTests, verify_disabled_not_displayed_with_no_tests) {
1004 std::vector<const char*> args{"--gtest_filter=NO_TEST_FILTER_MATCH", "-j2"};
1005
1006 ASSERT_NO_FATAL_FAILURE(ExecAndCapture(args));
1007 ASSERT_EQ(0, exitcode_);
1008 std::string expected =
1009 "Note: Google Test filter = NO_TEST_FILTER_MATCH\n"
1010 "[==========] Running 0 tests from 0 test suites (2 jobs).\n"
1011 "[==========] 0 tests from 0 test suites ran. (XX ms total)\n"
1012 "[ PASSED ] 0 tests.\n";
1013 ASSERT_EQ(expected, sanitized_output_) << "Test output:\n" << raw_output_;
1014 }
1015
TEST_F(SystemTests,verify_disabled)1016 TEST_F(SystemTests, verify_disabled) {
1017 std::vector<const char*> args{"--gtest_filter=*always_pass", "-j2"};
1018
1019 ASSERT_NO_FATAL_FAILURE(ExecAndCapture(args));
1020 ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
1021 std::string expected =
1022 "Note: Google Test filter = *always_pass\n"
1023 "[==========] Running 1 test from 1 test suite (2 jobs).\n"
1024 "[ RUN ] SystemTests.always_pass\n"
1025 "[ OK ] SystemTests.always_pass (XX ms)\n"
1026 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
1027 "[ PASSED ] 1 test.\n"
1028 "\n"
1029 " YOU HAVE 1 DISABLED TEST\n"
1030 "\n";
1031 ASSERT_EQ(expected, sanitized_output_);
1032 }
1033
TEST_F(SystemTests,verify_disabled_color)1034 TEST_F(SystemTests, verify_disabled_color) {
1035 std::vector<const char*> args{"--gtest_filter=*always_pass", "-j2", "--gtest_color=yes"};
1036
1037 ASSERT_NO_FATAL_FAILURE(ExecAndCapture(args));
1038 ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
1039 std::string expected =
1040 "\x1B[0;33mNote: Google Test filter = *always_pass\x1B[m\n"
1041 "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (2 jobs).\n"
1042 "\x1B[0;32m[ RUN ]\x1B[m SystemTests.always_pass\n"
1043 "\x1B[0;32m[ OK ]\x1B[m SystemTests.always_pass (XX ms)\n"
1044 "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
1045 "\x1B[0;32m[ PASSED ]\x1B[m 1 test.\n"
1046 "\n"
1047 "\x1B[0;33m YOU HAVE 1 DISABLED TEST\n"
1048 "\n\x1B[m";
1049 ASSERT_EQ(expected, sanitized_output_);
1050 }
1051
TEST_F(SystemTests,verify_SIGINT)1052 TEST_F(SystemTests, verify_SIGINT) {
1053 // Verify that SIGINT kills all of the tests.
1054 Exec(std::vector<const char*>{"--gtest_filter=*.DISABLED_job*", "--gtest_also_run_disabled_tests",
1055 "-j20"});
1056 // It is expected that all of the tests will be sleeping so nothing will
1057 // complete by the time the signal is sent.
1058 delay(1);
1059 ASSERT_NE(-1, kill(pid_, SIGINT));
1060
1061 std::string output;
1062 std::vector<char> buffer(4096);
1063 while (true) {
1064 ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
1065 if (bytes == -1 && errno == EAGAIN) {
1066 continue;
1067 }
1068 ASSERT_NE(-1, bytes);
1069 if (bytes == 0) {
1070 break;
1071 }
1072 buffer[bytes] = '\0';
1073 output += buffer.data();
1074 }
1075 close(fd_);
1076 int status;
1077 ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, &status, 0))) << "Test output:\n" << output;
1078 ASSERT_EQ(
1079 "Note: Google Test filter = *.DISABLED_job*\n"
1080 "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
1081 "Terminating due to signal...\n",
1082 output);
1083 ASSERT_EQ(1, WEXITSTATUS(status));
1084 }
1085
TEST_F(SystemTests,verify_SIGQUIT)1086 TEST_F(SystemTests, verify_SIGQUIT) {
1087 // Verify that SIGQUIT prints all of the running tests.
1088 Exec(std::vector<const char*>{"--gtest_filter=*.DISABLED_job*", "--gtest_also_run_disabled_tests",
1089 "-j20"});
1090 // It is expected that all of the tests will be sleeping so nothing will
1091 // complete by the time the signal is sent.
1092 delay(1);
1093 ASSERT_NE(-1, kill(pid_, SIGQUIT));
1094
1095 std::vector<char> buffer(4096);
1096 while (true) {
1097 ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
1098 if (bytes == -1 && errno == EAGAIN) {
1099 continue;
1100 }
1101 ASSERT_NE(-1, bytes);
1102 if (bytes == 0) {
1103 break;
1104 }
1105 buffer[bytes] = '\0';
1106 raw_output_ += buffer.data();
1107 }
1108 close(fd_);
1109 int status;
1110 ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, &status, 0))) << "Test output:\n" << raw_output_;
1111 SanitizeOutput();
1112 ASSERT_EQ(
1113 "Note: Google Test filter = *.DISABLED_job*\n"
1114 "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
1115 "List of current running tests:\n"
1116 " SystemTests.DISABLED_job_1 (elapsed time XX ms)\n"
1117 " SystemTests.DISABLED_job_2 (elapsed time XX ms)\n"
1118 " SystemTests.DISABLED_job_3 (elapsed time XX ms)\n"
1119 "[ RUN ] SystemTests.DISABLED_job_2\n"
1120 "[ OK ] SystemTests.DISABLED_job_2 (XX ms)\n"
1121 "[ RUN ] SystemTests.DISABLED_job_3\n"
1122 "[ OK ] SystemTests.DISABLED_job_3 (XX ms)\n"
1123 "[ RUN ] SystemTests.DISABLED_job_1\n"
1124 "[ OK ] SystemTests.DISABLED_job_1 (XX ms)\n"
1125 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
1126 "[ PASSED ] 3 tests.\n",
1127 sanitized_output_);
1128 ASSERT_EQ(0, WEXITSTATUS(status));
1129 }
1130
TEST_F(SystemTests,verify_SIGQUIT_after_test_finish)1131 TEST_F(SystemTests, verify_SIGQUIT_after_test_finish) {
1132 // Verify that SIGQUIT prints all of the tests after a test finishes.
1133 Exec(std::vector<const char*>{"--gtest_filter=*.DISABLED_sigquit_*",
1134 "--gtest_also_run_disabled_tests", "-j20"});
1135 // It is expected that one tests will have finished, but the rest will still
1136 // be running.
1137 delay(1);
1138 ASSERT_NE(-1, kill(pid_, SIGQUIT));
1139
1140 std::vector<char> buffer(4096);
1141 while (true) {
1142 ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
1143 if (bytes == -1 && errno == EAGAIN) {
1144 continue;
1145 }
1146 ASSERT_NE(-1, bytes);
1147 if (bytes == 0) {
1148 break;
1149 }
1150 buffer[bytes] = '\0';
1151 raw_output_ += buffer.data();
1152 }
1153 close(fd_);
1154 int status;
1155 ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, &status, 0))) << "Test output:\n" << raw_output_;
1156 SanitizeOutput();
1157 ASSERT_EQ(
1158 "Note: Google Test filter = *.DISABLED_sigquit_*\n"
1159 "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
1160 "[ RUN ] SystemTests.DISABLED_sigquit_no_sleep\n"
1161 "[ OK ] SystemTests.DISABLED_sigquit_no_sleep (XX ms)\n"
1162 "List of current running tests:\n"
1163 " SystemTests.DISABLED_sigquit_sleep_5 (elapsed time XX ms)\n"
1164 " SystemTests.DISABLED_sigquit_sleep_6 (elapsed time XX ms)\n"
1165 "[ RUN ] SystemTests.DISABLED_sigquit_sleep_5\n"
1166 "[ OK ] SystemTests.DISABLED_sigquit_sleep_5 (XX ms)\n"
1167 "[ RUN ] SystemTests.DISABLED_sigquit_sleep_6\n"
1168 "[ OK ] SystemTests.DISABLED_sigquit_sleep_6 (XX ms)\n"
1169 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
1170 "[ PASSED ] 3 tests.\n",
1171 sanitized_output_);
1172 ASSERT_EQ(0, WEXITSTATUS(status));
1173 }
1174
TEST_F(SystemTests,verify_memory)1175 TEST_F(SystemTests, verify_memory) {
1176 // This test verifies that memory isn't leaking when running repeatedly.
1177 std::vector<const char*> args{"--gtest_filter=*.DISABLED_memory",
1178 "--gtest_also_run_disabled_tests", "--gtest_repeat=400"};
1179 ASSERT_NO_FATAL_FAILURE(ExecAndCapture(args));
1180 ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
1181 std::vector<std::string> lines(android::base::Split(raw_output_, "\n"));
1182
1183 constexpr static size_t kMaxLeakBytes = 32 * 1024;
1184 size_t memory_iteration = 0;
1185 size_t memory_start = 0;
1186 size_t memory_last = 0;
1187 for (auto& line : lines) {
1188 size_t memory;
1189 if (android::base::StartsWith(line, "Allocated ") &&
1190 sscanf(line.c_str(), "Allocated %zu", &memory) == 1) {
1191 if (memory_iteration == 0) {
1192 memory_start = memory;
1193 } else {
1194 // Check the increase from the last loop.
1195 if (memory > memory_last) {
1196 ASSERT_GT(kMaxLeakBytes, memory - memory_last)
1197 << "On iteration " << memory_iteration << " memory increased beyond expected value."
1198 << std::endl
1199 << "Last memory bytes " << memory_last << std::endl
1200 << "Current memory bytes " << memory;
1201 }
1202 // Check the increase from the first loop.
1203 if (memory > memory_start) {
1204 ASSERT_GT(kMaxLeakBytes, memory - memory_start)
1205 << "On iteration " << memory_iteration
1206 << " total memory increased beyond expected value." << std::endl
1207 << "Starting memory bytes " << memory_start << std::endl
1208 << "Current memory bytes " << memory;
1209 }
1210 }
1211 memory_last = memory;
1212 memory_iteration++;
1213 }
1214 }
1215 ASSERT_EQ(400U, memory_iteration)
1216 << "Did not find the expected 400 lines of memory data." << std::endl
1217 << "Raw output:" << std::endl
1218 << raw_output_;
1219 }
1220
TEST_F(SystemTests,verify_sharding)1221 TEST_F(SystemTests, verify_sharding) {
1222 std::string expected =
1223 "Output Sorted\n"
1224 "Note: Google Test filter = SystemTestsShard*.DISABLED*\n"
1225 "Note: This is test shard 1 of 4\n"
1226 "[==========] Running 3 tests from 3 test suites (20 jobs).\n"
1227 "[ RUN ] SystemTestsShard1.DISABLED_case1_test1\n"
1228 "[ OK ] SystemTestsShard1.DISABLED_case1_test1 (XX ms)\n"
1229 "[ RUN ] SystemTestsShard2.DISABLED_case2_test1\n"
1230 "[ OK ] SystemTestsShard2.DISABLED_case2_test1 (XX ms)\n"
1231 "[ RUN ] SystemTestsShard3.DISABLED_case3_test1\n"
1232 "[ OK ] SystemTestsShard3.DISABLED_case3_test1 (XX ms)\n"
1233 "[==========] 3 tests from 3 test suites ran. (XX ms total)\n"
1234 "[ PASSED ] 3 tests.\n";
1235 ASSERT_NE(-1, setenv("GTEST_TOTAL_SHARDS", "4", 1));
1236 ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "0", 1));
1237 ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0));
1238
1239 expected =
1240 "Output Sorted\n"
1241 "Note: Google Test filter = SystemTestsShard*.DISABLED*\n"
1242 "Note: This is test shard 2 of 4\n"
1243 "[==========] Running 3 tests from 3 test suites (20 jobs).\n"
1244 "[ RUN ] SystemTestsShard1.DISABLED_case1_test2\n"
1245 "[ OK ] SystemTestsShard1.DISABLED_case1_test2 (XX ms)\n"
1246 "[ RUN ] SystemTestsShard2.DISABLED_case2_test2\n"
1247 "[ OK ] SystemTestsShard2.DISABLED_case2_test2 (XX ms)\n"
1248 "[ RUN ] SystemTestsShard3.DISABLED_case3_test2\n"
1249 "[ OK ] SystemTestsShard3.DISABLED_case3_test2 (XX ms)\n"
1250 "[==========] 3 tests from 3 test suites ran. (XX ms total)\n"
1251 "[ PASSED ] 3 tests.\n";
1252 ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "1", 1));
1253 ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0));
1254
1255 expected =
1256 "Output Sorted\n"
1257 "Note: Google Test filter = SystemTestsShard*.DISABLED*\n"
1258 "Note: This is test shard 3 of 4\n"
1259 "[==========] Running 3 tests from 3 test suites (20 jobs).\n"
1260 "[ RUN ] SystemTestsShard1.DISABLED_case1_test3\n"
1261 "[ OK ] SystemTestsShard1.DISABLED_case1_test3 (XX ms)\n"
1262 "[ RUN ] SystemTestsShard2.DISABLED_case2_test3\n"
1263 "[ OK ] SystemTestsShard2.DISABLED_case2_test3 (XX ms)\n"
1264 "[ RUN ] SystemTestsShard3.DISABLED_case3_test3\n"
1265 "[ OK ] SystemTestsShard3.DISABLED_case3_test3 (XX ms)\n"
1266 "[==========] 3 tests from 3 test suites ran. (XX ms total)\n"
1267 "[ PASSED ] 3 tests.\n";
1268 ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "2", 1));
1269 ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0));
1270
1271 expected =
1272 "Output Sorted\n"
1273 "Note: Google Test filter = SystemTestsShard*.DISABLED*\n"
1274 "Note: This is test shard 4 of 4\n"
1275 "[==========] Running 3 tests from 3 test suites (20 jobs).\n"
1276 "[ RUN ] SystemTestsShard1.DISABLED_case1_test4\n"
1277 "[ OK ] SystemTestsShard1.DISABLED_case1_test4 (XX ms)\n"
1278 "[ RUN ] SystemTestsShard2.DISABLED_case2_test4\n"
1279 "[ OK ] SystemTestsShard2.DISABLED_case2_test4 (XX ms)\n"
1280 "[ RUN ] SystemTestsShard3.DISABLED_case3_test4\n"
1281 "[ OK ] SystemTestsShard3.DISABLED_case3_test4 (XX ms)\n"
1282 "[==========] 3 tests from 3 test suites ran. (XX ms total)\n"
1283 "[ PASSED ] 3 tests.\n";
1284 ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "3", 1));
1285 ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0));
1286 }
1287
TEST_F(SystemTests,verify_sharding_color)1288 TEST_F(SystemTests, verify_sharding_color) {
1289 std::string expected =
1290 "Output Sorted\n"
1291 "\x1B[0;33mNote: Google Test filter = SystemTestsShard*.DISABLED*\x1B[m\n"
1292 "\x1B[0;33mNote: This is test shard 1 of 4\x1B[m\n"
1293 "\x1B[0;32m[==========]\x1B[m Running 3 tests from 3 test suites (20 jobs).\n"
1294 "\x1B[0;32m[ RUN ]\x1B[m SystemTestsShard1.DISABLED_case1_test1\n"
1295 "\x1B[0;32m[ OK ]\x1B[m SystemTestsShard1.DISABLED_case1_test1 (XX ms)\n"
1296 "\x1B[0;32m[ RUN ]\x1B[m SystemTestsShard2.DISABLED_case2_test1\n"
1297 "\x1B[0;32m[ OK ]\x1B[m SystemTestsShard2.DISABLED_case2_test1 (XX ms)\n"
1298 "\x1B[0;32m[ RUN ]\x1B[m SystemTestsShard3.DISABLED_case3_test1\n"
1299 "\x1B[0;32m[ OK ]\x1B[m SystemTestsShard3.DISABLED_case3_test1 (XX ms)\n"
1300 "\x1B[0;32m[==========]\x1B[m 3 tests from 3 test suites ran. (XX ms total)\n"
1301 "\x1B[0;32m[ PASSED ]\x1B[m 3 tests.\n";
1302 ASSERT_NE(-1, setenv("GTEST_TOTAL_SHARDS", "4", 1));
1303 ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "0", 1));
1304 ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0,
1305 std::vector<const char*>{"--gtest_color=yes"}));
1306 }
1307
TEST_F(SystemTests,verify_sharding_error)1308 TEST_F(SystemTests, verify_sharding_error) {
1309 std::string expected =
1310 "Invalid environment variables: we require 0 <= GTEST_SHARD_INDEX < GTEST_TOTAL_SHARDS, but "
1311 "you have GTEST_SHARD_INDEX=4, GTEST_TOTAL_SHARDS=4\n";
1312 ASSERT_NE(-1, setenv("GTEST_TOTAL_SHARDS", "4", 1));
1313 ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "4", 1));
1314 ASSERT_NO_FATAL_FAILURE(Verify("SystemTestsShard*.DISABLED*", expected, 1));
1315 }
1316
TEST_F(SystemTests,verify_sharding_error_color)1317 TEST_F(SystemTests, verify_sharding_error_color) {
1318 std::string expected =
1319 "\x1B[0;31mInvalid environment variables: we require 0 <= GTEST_SHARD_INDEX < "
1320 "GTEST_TOTAL_SHARDS, but you have GTEST_SHARD_INDEX=4, GTEST_TOTAL_SHARDS=4\x1B[m\n";
1321 ASSERT_NE(-1, setenv("GTEST_TOTAL_SHARDS", "4", 1));
1322 ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "4", 1));
1323 ASSERT_NO_FATAL_FAILURE(Verify("SystemTestsShard*.DISABLED*", expected, 1,
1324 std::vector<const char*>{"--gtest_color=yes"}));
1325 }
1326
TEST_F(SystemTests,verify_gtest_flagfile)1327 TEST_F(SystemTests, verify_gtest_flagfile) {
1328 TemporaryFile tf;
1329 ASSERT_TRUE(android::base::WriteStringToFile("--gtest_print_time=0\n", tf.path));
1330 std::string flagfile("--gtest_flagfile=");
1331 flagfile += tf.path;
1332 std::string expected =
1333 "Note: Google Test filter = *.DISABLED_pass\n"
1334 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
1335 "[ RUN ] SystemTests.DISABLED_pass\n"
1336 "[ OK ] SystemTests.DISABLED_pass\n"
1337 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
1338 "[ PASSED ] 1 test.\n";
1339 ASSERT_NO_FATAL_FAILURE(
1340 Verify("*.DISABLED_pass", expected, 0, std::vector<const char*>{flagfile.c_str()}));
1341 }
1342
TEST_F(SystemTests,verify_repeat_stop_on_error)1343 TEST_F(SystemTests, verify_repeat_stop_on_error) {
1344 std::string expected =
1345 "Note: Google Test filter = *.DISABLED_fail\n"
1346 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
1347 "[ RUN ] SystemTests.DISABLED_fail\n"
1348 "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
1349 "Expected equality of these values:\n"
1350 " 1\n"
1351 " 0\n"
1352 "SystemTests.DISABLED_fail exited with exitcode 1.\n"
1353 "[ FAILED ] SystemTests.DISABLED_fail (XX ms)\n"
1354 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
1355 "[ PASSED ] 0 tests.\n"
1356 "[ FAILED ] 1 test, listed below:\n"
1357 "[ FAILED ] SystemTests.DISABLED_fail\n"
1358 "\n"
1359 " 1 FAILED TEST\n"
1360 "\n"
1361 "Terminating repeat run due to failing tests (iteration 1).\n";
1362 ASSERT_NO_FATAL_FAILURE(
1363 Verify("*.DISABLED_fail", expected, 1,
1364 std::vector<const char*>{"--gtest_repeat=2", "--gtest_break_on_failure"}));
1365 }
1366
TEST_F(SystemTests,verify_repeat_no_stop_on_error)1367 TEST_F(SystemTests, verify_repeat_no_stop_on_error) {
1368 std::string expected =
1369 "Note: Google Test filter = *.DISABLED_fail\n"
1370 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
1371 "[ RUN ] SystemTests.DISABLED_fail\n"
1372 "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
1373 "Expected equality of these values:\n"
1374 " 1\n"
1375 " 0\n"
1376 "SystemTests.DISABLED_fail exited with exitcode 1.\n"
1377 "[ FAILED ] SystemTests.DISABLED_fail (XX ms)\n"
1378 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
1379 "[ PASSED ] 0 tests.\n"
1380 "[ FAILED ] 1 test, listed below:\n"
1381 "[ FAILED ] SystemTests.DISABLED_fail\n"
1382 "\n"
1383 " 1 FAILED TEST\n"
1384 "\n"
1385 "Repeating all tests (iteration 2) . . .\n"
1386 "\n"
1387 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
1388 "[ RUN ] SystemTests.DISABLED_fail\n"
1389 "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
1390 "Expected equality of these values:\n"
1391 " 1\n"
1392 " 0\n"
1393 "SystemTests.DISABLED_fail exited with exitcode 1.\n"
1394 "[ FAILED ] SystemTests.DISABLED_fail (XX ms)\n"
1395 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
1396 "[ PASSED ] 0 tests.\n"
1397 "[ FAILED ] 1 test, listed below:\n"
1398 "[ FAILED ] SystemTests.DISABLED_fail\n"
1399 "\n"
1400 " 1 FAILED TEST\n";
1401 ASSERT_NO_FATAL_FAILURE(
1402 Verify("*.DISABLED_fail", expected, 1, std::vector<const char*>{"--gtest_repeat=2"}));
1403 }
1404
TEST_F(SystemTests,verify_single_no_terminate_message)1405 TEST_F(SystemTests, verify_single_no_terminate_message) {
1406 std::string expected =
1407 "Note: Google Test filter = *.DISABLED_fail\n"
1408 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
1409 "[ RUN ] SystemTests.DISABLED_fail\n"
1410 "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
1411 "Expected equality of these values:\n"
1412 " 1\n"
1413 " 0\n"
1414 "SystemTests.DISABLED_fail exited with exitcode 1.\n"
1415 "[ FAILED ] SystemTests.DISABLED_fail (XX ms)\n"
1416 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
1417 "[ PASSED ] 0 tests.\n"
1418 "[ FAILED ] 1 test, listed below:\n"
1419 "[ FAILED ] SystemTests.DISABLED_fail\n"
1420 "\n"
1421 " 1 FAILED TEST\n";
1422 ASSERT_NO_FATAL_FAILURE(
1423 Verify("*.DISABLED_fail", expected, 1, std::vector<const char*>{"--gtest_break_on_failure"}));
1424 }
1425
1426 // These tests are used by the verify_disabled tests.
TEST_F(SystemTests,always_pass)1427 TEST_F(SystemTests, always_pass) {}
1428
TEST_F(SystemTests,DISABLED_always_pass)1429 TEST_F(SystemTests, DISABLED_always_pass) {}
1430
1431 // The tests listed below will not run by default. They are executed by
1432 // the above tests.
TEST_F(SystemTests,DISABLED_pass)1433 TEST_F(SystemTests, DISABLED_pass) {}
1434
TEST_F(SystemTests,DISABLED_fail)1435 TEST_F(SystemTests, DISABLED_fail) {
1436 ASSERT_EQ(1, 0);
1437 }
1438
TEST_F(SystemTests,DISABLED_crash)1439 TEST_F(SystemTests, DISABLED_crash) __attribute__((optnone)) {
1440 char* p = reinterpret_cast<char*>(static_cast<intptr_t>(atoi("0")));
1441 *p = 3;
1442 }
1443
TEST_F(SystemTests,DISABLED_sigquit_no_sleep)1444 TEST_F(SystemTests, DISABLED_sigquit_no_sleep) {}
1445
TEST_F(SystemTests,DISABLED_sigquit_sleep_5)1446 TEST_F(SystemTests, DISABLED_sigquit_sleep_5) {
1447 delay(5);
1448 }
1449
TEST_F(SystemTests,DISABLED_sigquit_sleep_6)1450 TEST_F(SystemTests, DISABLED_sigquit_sleep_6) {
1451 delay(6);
1452 }
1453
TEST_F(SystemTests,DISABLED_sleep_forever)1454 TEST_F(SystemTests, DISABLED_sleep_forever) {
1455 while (true) {
1456 delay(10000);
1457 }
1458 }
1459
TEST_F(SystemTests,DISABLED_sleep5)1460 TEST_F(SystemTests, DISABLED_sleep5) {
1461 delay(5);
1462 }
1463
1464 // These tests will finish 1, 2, 3 in non-isolated mode and 3, 2, 1 in isolated
1465 // mode.
TEST_F(SystemTests,DISABLED_order_1)1466 TEST_F(SystemTests, DISABLED_order_1) {
1467 delay(6);
1468 }
1469
TEST_F(SystemTests,DISABLED_order_2)1470 TEST_F(SystemTests, DISABLED_order_2) {
1471 delay(3);
1472 }
1473
TEST_F(SystemTests,DISABLED_order_3)1474 TEST_F(SystemTests, DISABLED_order_3) {}
1475
TEST_F(SystemTests,DISABLED_fail_0)1476 TEST_F(SystemTests, DISABLED_fail_0) {
1477 ASSERT_EQ(1, 0);
1478 }
1479
TEST_F(SystemTests,DISABLED_fail_1)1480 TEST_F(SystemTests, DISABLED_fail_1) {
1481 ASSERT_EQ(1, 0);
1482 }
1483
TEST_F(SystemTests,DISABLED_fail_2)1484 TEST_F(SystemTests, DISABLED_fail_2) {
1485 ASSERT_EQ(1, 0);
1486 }
1487
TEST_F(SystemTests,DISABLED_fail_3)1488 TEST_F(SystemTests, DISABLED_fail_3) {
1489 ASSERT_EQ(1, 0);
1490 }
1491
TEST_F(SystemTests,DISABLED_fail_4)1492 TEST_F(SystemTests, DISABLED_fail_4) {
1493 ASSERT_EQ(1, 0);
1494 }
1495
TEST_F(SystemTests,DISABLED_fail_5)1496 TEST_F(SystemTests, DISABLED_fail_5) {
1497 ASSERT_EQ(1, 0);
1498 }
1499
TEST_F(SystemTests,DISABLED_fail_6)1500 TEST_F(SystemTests, DISABLED_fail_6) {
1501 ASSERT_EQ(1, 0);
1502 }
1503
TEST_F(SystemTests,DISABLED_fail_7)1504 TEST_F(SystemTests, DISABLED_fail_7) {
1505 ASSERT_EQ(1, 0);
1506 }
1507
TEST_F(SystemTests,DISABLED_fail_8)1508 TEST_F(SystemTests, DISABLED_fail_8) {
1509 ASSERT_EQ(1, 0);
1510 }
1511
TEST_F(SystemTests,DISABLED_fail_9)1512 TEST_F(SystemTests, DISABLED_fail_9) {
1513 ASSERT_EQ(1, 0);
1514 }
1515
TEST_F(SystemTests,DISABLED_all_pass_1)1516 TEST_F(SystemTests, DISABLED_all_pass_1) {}
1517
TEST_F(SystemTests,DISABLED_all_pass_2)1518 TEST_F(SystemTests, DISABLED_all_pass_2) {}
1519
TEST_F(SystemTests,DISABLED_all_skip_1)1520 TEST_F(SystemTests, DISABLED_all_skip_1) {
1521 GTEST_SKIP();
1522 }
1523
TEST_F(SystemTests,DISABLED_all_skip_2)1524 TEST_F(SystemTests, DISABLED_all_skip_2) {
1525 GTEST_SKIP() << "Skip message present";
1526 }
1527
TEST_F(SystemTests,DISABLED_all_slow_1)1528 TEST_F(SystemTests, DISABLED_all_slow_1) {
1529 delay(3);
1530 }
1531
TEST_F(SystemTests,DISABLED_all_slow_2)1532 TEST_F(SystemTests, DISABLED_all_slow_2) {
1533 delay(3);
1534 }
1535
TEST_F(SystemTests,DISABLED_all_fail_1)1536 TEST_F(SystemTests, DISABLED_all_fail_1) {
1537 ASSERT_EQ(1, 0);
1538 }
1539
TEST_F(SystemTests,DISABLED_all_fail_2)1540 TEST_F(SystemTests, DISABLED_all_fail_2) {
1541 ASSERT_EQ(1, 0);
1542 }
1543
TEST_F(SystemTests,DISABLED_all_timeout_1)1544 TEST_F(SystemTests, DISABLED_all_timeout_1) {
1545 delay(6);
1546 }
1547
TEST_F(SystemTests,DISABLED_all_timeout_2)1548 TEST_F(SystemTests, DISABLED_all_timeout_2) {
1549 delay(6);
1550 }
1551
TEST_F(SystemTests,DISABLED_job_1)1552 TEST_F(SystemTests, DISABLED_job_1) {
1553 delay(5);
1554 }
1555
TEST_F(SystemTests,DISABLED_job_2)1556 TEST_F(SystemTests, DISABLED_job_2) {
1557 delay(3);
1558 }
1559
TEST_F(SystemTests,DISABLED_job_3)1560 TEST_F(SystemTests, DISABLED_job_3) {
1561 delay(4);
1562 }
1563
TEST_F(SystemTests,DISABLED_skip_no_message)1564 TEST_F(SystemTests, DISABLED_skip_no_message) {
1565 GTEST_SKIP();
1566 }
1567
TEST_F(SystemTests,DISABLED_skip_with_message)1568 TEST_F(SystemTests, DISABLED_skip_with_message) {
1569 GTEST_SKIP() << "This is a skip message";
1570 }
1571
TEST_F(SystemTests,DISABLED_skip_with_output_before)1572 TEST_F(SystemTests, DISABLED_skip_with_output_before) {
1573 printf("This is the message before the skip message\n");
1574 GTEST_SKIP() << "This is the skip message";
1575 }
1576
1577 // Do not optimize this call away so that the print after the skip
1578 // will actually occur.
AvoidSkipStopping(int tag=0)1579 void AvoidSkipStopping(int tag = 0) __attribute__((optnone)) {
1580 if (tag == 0) {
1581 GTEST_SKIP() << "This is the skip message";
1582 } else {
1583 GTEST_SKIP() << "This is the skip message " << std::to_string(tag);
1584 }
1585 }
1586
TEST_F(SystemTests,DISABLED_skip_with_output_after)1587 TEST_F(SystemTests, DISABLED_skip_with_output_after) {
1588 AvoidSkipStopping();
1589 printf("This is the message after the skip message\n");
1590 }
1591
TEST_F(SystemTests,DISABLED_skip_with_skipped_line)1592 TEST_F(SystemTests, DISABLED_skip_with_skipped_line) {
1593 printf("\nSkipped\n");
1594 AvoidSkipStopping(1);
1595 printf("Skipped\n");
1596 AvoidSkipStopping(2);
1597 printf("Skipped\n");
1598 }
1599
TEST_F(SystemTests,DISABLED_skip_multiple)1600 TEST_F(SystemTests, DISABLED_skip_multiple) {
1601 printf("This is not a skip message 1\n");
1602 AvoidSkipStopping(1);
1603 printf("This is not a skip message 2\n");
1604 AvoidSkipStopping(2);
1605 AvoidSkipStopping(3);
1606 printf("This is not a skip message 4\n");
1607 }
1608
1609 class DISABLED_SystemTestsXfail : public ::testing::Test {};
1610
TEST_F(DISABLED_SystemTestsXfail,xfail_fail)1611 TEST_F(DISABLED_SystemTestsXfail, xfail_fail) {
1612 ASSERT_EQ(1, 0);
1613 }
1614
TEST_F(DISABLED_SystemTestsXfail,xfail_pass)1615 TEST_F(DISABLED_SystemTestsXfail, xfail_pass) {}
1616
1617 class SystemTestsDeathTest : public ::testing::Test {
1618 protected:
SetUp()1619 virtual void SetUp() { ::testing::FLAGS_gtest_death_test_style = "threadsafe"; }
1620 };
1621
DeathTestHelperPass()1622 static void DeathTestHelperPass() {
1623 ASSERT_EQ(1, 1);
1624 exit(0);
1625 }
1626
TEST_F(SystemTestsDeathTest,DISABLED_death_pass)1627 TEST_F(SystemTestsDeathTest, DISABLED_death_pass) {
1628 ASSERT_EXIT(DeathTestHelperPass(), ::testing::ExitedWithCode(0), "");
1629 }
1630
DeathTestHelperFail()1631 static void DeathTestHelperFail() {
1632 ASSERT_EQ(1, 0);
1633 }
1634
TEST_F(SystemTestsDeathTest,DISABLED_death_fail)1635 TEST_F(SystemTestsDeathTest, DISABLED_death_fail) {
1636 ASSERT_EXIT(DeathTestHelperFail(), ::testing::ExitedWithCode(0), "");
1637 }
1638
TEST(SystemTestsXml1,DISABLED_xml_1)1639 TEST(SystemTestsXml1, DISABLED_xml_1) {}
1640
TEST(SystemTestsXml1,DISABLED_xml_2)1641 TEST(SystemTestsXml1, DISABLED_xml_2) {
1642 ASSERT_EQ(1, 0);
1643 }
1644
TEST(SystemTestsXml2,DISABLED_xml_1)1645 TEST(SystemTestsXml2, DISABLED_xml_1) {
1646 ASSERT_EQ(1, 0);
1647 }
1648
TEST(SystemTestsXml2,DISABLED_xml_2)1649 TEST(SystemTestsXml2, DISABLED_xml_2) {}
1650
TEST(SystemTestsXml3,DISABLED_xml_1)1651 TEST(SystemTestsXml3, DISABLED_xml_1) {}
1652
TEST(SystemTestsXml3,DISABLED_xml_2)1653 TEST(SystemTestsXml3, DISABLED_xml_2) {
1654 ASSERT_EQ(1, 0);
1655 }
1656
TEST(SystemTestsMemory,DISABLED_memory)1657 TEST(SystemTestsMemory, DISABLED_memory) {
1658 #if !defined(__APPLE__)
1659 struct mallinfo info = mallinfo();
1660 #if defined(__ANDROID__)
1661 printf("Allocated %zu\n", info.uordblks);
1662 #else
1663 printf("Allocated %d\n", info.uordblks);
1664 #endif
1665 #else
1666 printf("Allocated 0\n");
1667 #endif
1668 }
1669
TEST(SystemTestsShard1,DISABLED_case1_test1)1670 TEST(SystemTestsShard1, DISABLED_case1_test1) {}
1671
TEST(SystemTestsShard1,DISABLED_case1_test2)1672 TEST(SystemTestsShard1, DISABLED_case1_test2) {}
1673
TEST(SystemTestsShard1,DISABLED_case1_test3)1674 TEST(SystemTestsShard1, DISABLED_case1_test3) {}
1675
TEST(SystemTestsShard1,DISABLED_case1_test4)1676 TEST(SystemTestsShard1, DISABLED_case1_test4) {}
1677
TEST(SystemTestsShard2,DISABLED_case2_test1)1678 TEST(SystemTestsShard2, DISABLED_case2_test1) {}
1679
TEST(SystemTestsShard2,DISABLED_case2_test2)1680 TEST(SystemTestsShard2, DISABLED_case2_test2) {}
1681
TEST(SystemTestsShard2,DISABLED_case2_test3)1682 TEST(SystemTestsShard2, DISABLED_case2_test3) {}
1683
TEST(SystemTestsShard2,DISABLED_case2_test4)1684 TEST(SystemTestsShard2, DISABLED_case2_test4) {}
1685
TEST(SystemTestsShard3,DISABLED_case3_test1)1686 TEST(SystemTestsShard3, DISABLED_case3_test1) {}
1687
TEST(SystemTestsShard3,DISABLED_case3_test2)1688 TEST(SystemTestsShard3, DISABLED_case3_test2) {}
1689
TEST(SystemTestsShard3,DISABLED_case3_test3)1690 TEST(SystemTestsShard3, DISABLED_case3_test3) {}
1691
TEST(SystemTestsShard3,DISABLED_case3_test4)1692 TEST(SystemTestsShard3, DISABLED_case3_test4) {}
1693
1694 } // namespace gtest_extras
1695 } // namespace android
1696