1# Copyright 2023, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Unittests for mobly_test_runner.""" 16# pylint: disable=protected-access 17# pylint: disable=invalid-name 18 19import argparse 20import os 21import pathlib 22import unittest 23from unittest import mock 24 25from atest import constants 26from atest import result_reporter 27from atest import unittest_constants 28from atest.test_finders import test_info 29from atest.test_runners import mobly_test_runner 30from atest.test_runners import test_runner_base 31 32 33TEST_NAME = 'SampleMoblyTest' 34MOBLY_PKG = 'mobly/SampleMoblyTest' 35REQUIREMENTS_TXT = 'mobly/requirements.txt' 36APK_1 = 'mobly/snippet1.apk' 37APK_2 = 'mobly/snippet2.apk' 38MISC_FILE = 'mobly/misc_file.txt' 39RESULTS_DIR = 'atest_results/sample_test' 40SERIAL_1 = 'serial1' 41SERIAL_2 = 'serial2' 42ADB_DEVICE = 'adb_device' 43MOBLY_SUMMARY_FILE = os.path.join( 44 unittest_constants.TEST_DATA_DIR, 'mobly', 'sample_test_summary.yaml' 45) 46MOCK_TEST_FILES = mobly_test_runner.MoblyTestFiles('', None, [], []) 47 48 49class MoblyResultUploaderUnittests(unittest.TestCase): 50 """Unit tests for MoblyResultUploader.""" 51 52 def setUp(self) -> None: 53 self.patchers = [ 54 mock.patch( 55 'atest.logstorage.logstorage_utils.is_upload_enabled', 56 return_value=True, 57 ), 58 mock.patch( 59 'atest.logstorage.logstorage_utils.do_upload_flow', 60 return_value=('creds', {'invocationId': 'I00001'}), 61 ), 62 mock.patch('atest.logstorage.logstorage_utils.BuildClient'), 63 ] 64 for patcher in self.patchers: 65 patcher.start() 66 self.uploader = mobly_test_runner.MoblyResultUploader({}) 67 self.uploader._root_workunit = {'id': 'WU00001', 'runCount': 0} 68 self.uploader._current_workunit = {'id': 'WU00010'} 69 70 def tearDown(self) -> None: 71 mock.patch.stopall() 72 73 def test_start_new_workunit(self): 74 """Tests that start_new_workunit sets correct workunit fields.""" 75 self.uploader._build_client.insert_work_unit.return_value = {} 76 self.uploader.start_new_workunit() 77 78 self.assertEqual( 79 self.uploader.current_workunit, 80 { 81 'type': mobly_test_runner.WORKUNIT_ATEST_MOBLY_TEST_RUN, 82 'parentId': 'WU00001', 83 }, 84 ) 85 86 def test_set_workunit_iteration_details_with_repeats(self): 87 """Tests that set_workunit_iteration_details sets the run number for 88 89 repeated tests. 90 """ 91 rerun_options = mobly_test_runner.RerunOptions(3, False, False) 92 self.uploader.set_workunit_iteration_details(1, rerun_options) 93 94 self.assertEqual(self.uploader.current_workunit['childRunNumber'], 1) 95 96 def test_set_workunit_iteration_details_with_retries(self): 97 """Tests that set_workunit_iteration_details sets the run number for 98 99 retried tests. 100 """ 101 rerun_options = mobly_test_runner.RerunOptions(3, False, True) 102 self.uploader.set_workunit_iteration_details(1, rerun_options) 103 104 self.assertEqual(self.uploader.current_workunit['childAttemptNumber'], 1) 105 106 def test_finalize_current_workunit(self): 107 """Tests that finalize_current_workunit sets correct workunit fields.""" 108 workunit = self.uploader.current_workunit 109 self.uploader.finalize_current_workunit() 110 111 self.assertEqual(workunit['schedulerState'], 'completed') 112 self.assertEqual(self.uploader._root_workunit['runCount'], 1) 113 self.assertIsNone(self.uploader.current_workunit) 114 115 def test_finalize_invocation(self): 116 """Tests that finalize_invocation sets correct fields.""" 117 invocation = self.uploader.invocation 118 root_workunit = self.uploader._root_workunit 119 self.uploader.finalize_invocation() 120 121 self.assertEqual(root_workunit['schedulerState'], 'completed') 122 self.assertEqual(root_workunit['runCount'], 0) 123 self.assertEqual(invocation['runner'], 'mobly') 124 self.assertEqual(invocation['schedulerState'], 'completed') 125 self.assertFalse(self.uploader.enabled) 126 127 @mock.patch('atest.constants.RESULT_LINK', 'link:%s') 128 def test_add_result_link(self): 129 """Tests that add_result_link correctly sets the result link.""" 130 reporter = result_reporter.ResultReporter() 131 132 reporter.test_result_link = ['link:I00000'] 133 self.uploader.add_result_link(reporter) 134 self.assertEqual(reporter.test_result_link, ['link:I00000', 'link:I00001']) 135 136 reporter.test_result_link = 'link:I00000' 137 self.uploader.add_result_link(reporter) 138 self.assertEqual(reporter.test_result_link, ['link:I00000', 'link:I00001']) 139 140 reporter.test_result_link = None 141 self.uploader.add_result_link(reporter) 142 self.assertEqual(reporter.test_result_link, ['link:I00001']) 143 144 145class MoblyTestRunnerUnittests(unittest.TestCase): 146 """Unit tests for MoblyTestRunner.""" 147 148 def setUp(self) -> None: 149 self.runner = mobly_test_runner.MoblyTestRunner(RESULTS_DIR) 150 self.tinfo = test_info.TestInfo( 151 test_name=TEST_NAME, 152 test_runner=mobly_test_runner.MoblyTestRunner.EXECUTABLE, 153 build_targets=[], 154 ) 155 self.reporter = result_reporter.ResultReporter() 156 self.mobly_args = argparse.Namespace(config='', testbed='', testparam=[]) 157 158 @mock.patch.object(pathlib.Path, 'is_file') 159 def test_get_test_files_all_files_present(self, is_file) -> None: 160 """Tests _get_test_files with all files present.""" 161 is_file.return_value = True 162 files = [MOBLY_PKG, REQUIREMENTS_TXT, APK_1, APK_2, MISC_FILE] 163 file_paths = [pathlib.Path(f) for f in files] 164 self.tinfo.data[constants.MODULE_INSTALLED] = file_paths 165 166 test_files = self.runner._get_test_files(self.tinfo) 167 168 self.assertTrue(test_files.mobly_pkg.endswith(MOBLY_PKG)) 169 self.assertTrue(test_files.requirements_txt.endswith(REQUIREMENTS_TXT)) 170 self.assertTrue(test_files.test_apks[0].endswith(APK_1)) 171 self.assertTrue(test_files.test_apks[1].endswith(APK_2)) 172 self.assertTrue(test_files.misc_data[0].endswith(MISC_FILE)) 173 174 @mock.patch.object(pathlib.Path, 'is_file') 175 def test_get_test_files_no_mobly_pkg(self, is_file) -> None: 176 """Tests _get_test_files with missing mobly_pkg.""" 177 is_file.return_value = True 178 files = [REQUIREMENTS_TXT, APK_1, APK_2] 179 self.tinfo.data[constants.MODULE_INSTALLED] = [ 180 pathlib.Path(f) for f in files 181 ] 182 183 with self.assertRaisesRegex( 184 mobly_test_runner.MoblyTestRunnerError, 'No Mobly test package' 185 ): 186 self.runner._get_test_files(self.tinfo) 187 188 @mock.patch.object(pathlib.Path, 'is_file') 189 def test_get_test_files_file_not_found(self, is_file) -> None: 190 """Tests _get_test_files with file not found in file system.""" 191 is_file.return_value = False 192 files = [MOBLY_PKG, REQUIREMENTS_TXT, APK_1, APK_2] 193 self.tinfo.data[constants.MODULE_INSTALLED] = [ 194 pathlib.Path(f) for f in files 195 ] 196 197 with self.assertRaisesRegex( 198 mobly_test_runner.MoblyTestRunnerError, 'Required test file' 199 ): 200 self.runner._get_test_files(self.tinfo) 201 202 @mock.patch('builtins.open') 203 @mock.patch('os.makedirs') 204 @mock.patch('yaml.safe_dump') 205 def test_generate_mobly_config_no_serials(self, yaml_dump, *_) -> None: 206 """Tests _generate_mobly_config with no serials provided.""" 207 self.runner._generate_mobly_config(self.mobly_args, None, MOCK_TEST_FILES) 208 209 expected_config = { 210 'TestBeds': [{ 211 'Name': 'LocalTestBed', 212 'Controllers': { 213 'AndroidDevice': '*', 214 }, 215 'TestParams': {}, 216 }], 217 'MoblyParams': { 218 'LogPath': 'atest_results/sample_test/mobly_logs', 219 }, 220 } 221 self.assertEqual(yaml_dump.call_args.args[0], expected_config) 222 223 @mock.patch('builtins.open') 224 @mock.patch('os.makedirs') 225 @mock.patch('yaml.safe_dump') 226 def test_generate_mobly_config_with_serials(self, yaml_dump, *_) -> None: 227 """Tests _generate_mobly_config with serials provided.""" 228 self.runner._generate_mobly_config( 229 self.mobly_args, [SERIAL_1, SERIAL_2], MOCK_TEST_FILES 230 ) 231 232 expected_config = { 233 'TestBeds': [{ 234 'Name': 'LocalTestBed', 235 'Controllers': { 236 'AndroidDevice': [SERIAL_1, SERIAL_2], 237 }, 238 'TestParams': {}, 239 }], 240 'MoblyParams': { 241 'LogPath': 'atest_results/sample_test/mobly_logs', 242 }, 243 } 244 self.assertEqual(yaml_dump.call_args.args[0], expected_config) 245 246 @mock.patch('builtins.open') 247 @mock.patch('os.makedirs') 248 @mock.patch('yaml.safe_dump') 249 def test_generate_mobly_config_with_testparams(self, yaml_dump, *_) -> None: 250 """Tests _generate_mobly_config with custom testparams.""" 251 self.mobly_args.testparam = ['foo=bar'] 252 self.runner._generate_mobly_config(self.mobly_args, None, MOCK_TEST_FILES) 253 254 expected_config = { 255 'TestBeds': [{ 256 'Name': 'LocalTestBed', 257 'Controllers': { 258 'AndroidDevice': '*', 259 }, 260 'TestParams': { 261 'foo': 'bar', 262 }, 263 }], 264 'MoblyParams': { 265 'LogPath': 'atest_results/sample_test/mobly_logs', 266 }, 267 } 268 self.assertEqual(yaml_dump.call_args.args[0], expected_config) 269 270 def test_generate_mobly_config_with_invalid_testparams(self) -> None: 271 """Tests _generate_mobly_config with invalid testparams.""" 272 self.mobly_args.testparam = ['foobar'] 273 with self.assertRaisesRegex( 274 mobly_test_runner.MoblyTestRunnerError, 'Invalid testparam values' 275 ): 276 self.runner._generate_mobly_config(self.mobly_args, None, []) 277 278 @mock.patch('builtins.open') 279 @mock.patch('os.makedirs') 280 @mock.patch('yaml.safe_dump') 281 def test_generate_mobly_config_with_test_files(self, yaml_dump, *_) -> None: 282 """Tests _generate_mobly_config with test files.""" 283 test_apks = ['files/my_app1.apk', 'files/my_app2.apk'] 284 misc_data = ['files/some_file.txt'] 285 test_files = mobly_test_runner.MoblyTestFiles('', '', test_apks, misc_data) 286 self.runner._generate_mobly_config(self.mobly_args, None, test_files) 287 288 expected_config = { 289 'TestBeds': [{ 290 'Name': 'LocalTestBed', 291 'Controllers': { 292 'AndroidDevice': '*', 293 }, 294 'TestParams': { 295 'files': { 296 'my_app1': ['files/my_app1.apk'], 297 'my_app2': ['files/my_app2.apk'], 298 'some_file.txt': ['files/some_file.txt'], 299 }, 300 }, 301 }], 302 'MoblyParams': { 303 'LogPath': 'atest_results/sample_test/mobly_logs', 304 }, 305 } 306 self.assertEqual(yaml_dump.call_args.args[0], expected_config) 307 308 @mock.patch('atest.atest_configs.GLOBAL_ARGS.acloud_create', True) 309 @mock.patch('atest.atest_utils.get_adb_devices') 310 def test_get_cvd_serials(self, get_adb_devices) -> None: 311 """Tests _get_cvd_serials returns correct serials.""" 312 devices = ['localhost:1234', '127.0.0.1:5678', 'AD12345'] 313 get_adb_devices.return_value = devices 314 315 self.assertEqual(self.runner._get_cvd_serials(), devices[:2]) 316 317 @mock.patch('atest.atest_utils.get_adb_devices', return_value=[ADB_DEVICE]) 318 @mock.patch('subprocess.check_call') 319 def test_install_apks_no_serials(self, check_call, _) -> None: 320 """Tests _install_apks with no serials provided.""" 321 self.runner._install_apks([APK_1], None) 322 323 expected_cmds = [['adb', '-s', ADB_DEVICE, 'install', '-r', '-g', APK_1]] 324 self.assertEqual( 325 [call.args[0] for call in check_call.call_args_list], expected_cmds 326 ) 327 328 @mock.patch('atest.atest_utils.get_adb_devices', return_value=[ADB_DEVICE]) 329 @mock.patch('subprocess.check_call') 330 def test_install_apks_with_serials(self, check_call, _) -> None: 331 """Tests _install_apks with serials provided.""" 332 self.runner._install_apks([APK_1], [SERIAL_1, SERIAL_2]) 333 334 expected_cmds = [ 335 ['adb', '-s', SERIAL_1, 'install', '-r', '-g', APK_1], 336 ['adb', '-s', SERIAL_2, 'install', '-r', '-g', APK_1], 337 ] 338 self.assertEqual( 339 [call.args[0] for call in check_call.call_args_list], expected_cmds 340 ) 341 342 def test_get_test_cases_from_spec_with_class_and_methods(self) -> None: 343 """Tests _get_test_cases_from_spec with both class and methods defined.""" 344 self.tinfo.data = { 345 'filter': frozenset({ 346 test_info.TestFilter( 347 class_name='SampleClass', methods=frozenset({'test1', 'test2'}) 348 ) 349 }) 350 } 351 352 self.assertCountEqual( 353 self.runner._get_test_cases_from_spec(self.tinfo), 354 ['SampleClass.test1', 'SampleClass.test2'], 355 ) 356 357 def test_get_test_cases_from_spec_with_class_only(self) -> None: 358 """Tests _get_test_cases_from_spec with only test class defined.""" 359 self.tinfo.data = { 360 'filter': frozenset({ 361 test_info.TestFilter(class_name='SampleClass', methods=frozenset()) 362 }) 363 } 364 365 self.assertCountEqual( 366 self.runner._get_test_cases_from_spec(self.tinfo), ['SampleClass'] 367 ) 368 369 def test_get_test_cases_from_spec_with_method_only(self) -> None: 370 """Tests _get_test_cases_from_spec with only methods defined.""" 371 self.tinfo.data = { 372 'filter': frozenset({ 373 test_info.TestFilter( 374 class_name='.', methods=frozenset({'test1', 'test2'}) 375 ) 376 }) 377 } 378 379 self.assertCountEqual( 380 self.runner._get_test_cases_from_spec(self.tinfo), ['test1', 'test2'] 381 ) 382 383 @mock.patch.object( 384 mobly_test_runner.MoblyTestRunner, 385 '_process_test_results_from_summary', 386 return_value=(), 387 ) 388 @mock.patch('atest.test_runners.mobly_test_runner.MoblyResultUploader') 389 def test_run_and_handle_results_with_iterations(self, uploader, _) -> None: 390 """Tests _run_and_handle_results with multiple iterations.""" 391 with mock.patch.object( 392 mobly_test_runner.MoblyTestRunner, 393 '_run_mobly_command', 394 side_effect=(1, 1, 0, 0, 1), 395 ) as run_mobly_command: 396 runner = mobly_test_runner.MoblyTestRunner(RESULTS_DIR) 397 runner._run_and_handle_results( 398 [], 399 self.tinfo, 400 mobly_test_runner.RerunOptions(5, False, False), 401 self.mobly_args, 402 self.reporter, 403 uploader, 404 ) 405 self.assertEqual(run_mobly_command.call_count, 5) 406 407 @mock.patch.object( 408 mobly_test_runner.MoblyTestRunner, 409 '_process_test_results_from_summary', 410 return_value=(), 411 ) 412 @mock.patch('atest.test_runners.mobly_test_runner.MoblyResultUploader') 413 def test_run_and_handle_results_with_rerun_until_failure( 414 self, uploader, _ 415 ) -> None: 416 """Tests _run_and_handle_results with rerun_until_failure.""" 417 with mock.patch.object( 418 mobly_test_runner.MoblyTestRunner, 419 '_run_mobly_command', 420 side_effect=(0, 0, 1, 0, 1), 421 ) as run_mobly_command: 422 runner = mobly_test_runner.MoblyTestRunner(RESULTS_DIR) 423 runner._run_and_handle_results( 424 [], 425 self.tinfo, 426 mobly_test_runner.RerunOptions(5, True, False), 427 self.mobly_args, 428 self.reporter, 429 uploader, 430 ) 431 self.assertEqual(run_mobly_command.call_count, 3) 432 433 @mock.patch.object( 434 mobly_test_runner.MoblyTestRunner, 435 '_process_test_results_from_summary', 436 return_value=(), 437 ) 438 @mock.patch('atest.test_runners.mobly_test_runner.MoblyResultUploader') 439 def test_run_and_handle_results_with_retry_any_failure( 440 self, uploader, _ 441 ) -> None: 442 """Tests _run_and_handle_results with retry_any_failure.""" 443 with mock.patch.object( 444 mobly_test_runner.MoblyTestRunner, 445 '_run_mobly_command', 446 side_effect=(1, 1, 1, 0, 0), 447 ) as run_mobly_command: 448 runner = mobly_test_runner.MoblyTestRunner(RESULTS_DIR) 449 runner._run_and_handle_results( 450 [], 451 self.tinfo, 452 mobly_test_runner.RerunOptions(5, False, True), 453 self.mobly_args, 454 self.reporter, 455 uploader, 456 ) 457 self.assertEqual(run_mobly_command.call_count, 4) 458 459 @mock.patch('atest.test_runners.mobly_test_runner.MoblyResultUploader') 460 def test_process_test_results_from_summary_show_correct_names( 461 self, uploader 462 ) -> None: 463 """Tests _process_results_from_summary outputs correct test names.""" 464 test_results = self.runner._process_test_results_from_summary( 465 MOBLY_SUMMARY_FILE, self.tinfo, 0, 1, uploader 466 ) 467 468 result = test_results[0] 469 self.assertEqual(result.runner_name, self.runner.NAME) 470 self.assertEqual(result.group_name, TEST_NAME) 471 self.assertEqual(result.test_run_name, 'SampleTest') 472 self.assertEqual(result.test_name, 'SampleTest.test_should_pass') 473 474 test_results = self.runner._process_test_results_from_summary( 475 MOBLY_SUMMARY_FILE, self.tinfo, 2, 3, uploader 476 ) 477 478 result = test_results[0] 479 self.assertEqual(result.test_run_name, 'SampleTest (#3)') 480 self.assertEqual(result.test_name, 'SampleTest.test_should_pass (#3)') 481 482 @mock.patch('atest.test_runners.mobly_test_runner.MoblyResultUploader') 483 def test_process_test_results_from_summary_show_correct_status_and_details( 484 self, uploader 485 ) -> None: 486 """Tests _process_results_from_summary outputs correct test status and 487 488 details. 489 """ 490 test_results = self.runner._process_test_results_from_summary( 491 MOBLY_SUMMARY_FILE, self.tinfo, 0, 1, uploader 492 ) 493 494 # passed case 495 self.assertEqual(test_results[0].status, test_runner_base.PASSED_STATUS) 496 self.assertEqual(test_results[0].details, None) 497 # failed case 498 self.assertEqual(test_results[1].status, test_runner_base.FAILED_STATUS) 499 self.assertEqual(test_results[1].details, 'mobly.signals.TestFailure') 500 # errored case 501 self.assertEqual(test_results[2].status, test_runner_base.FAILED_STATUS) 502 self.assertEqual(test_results[2].details, 'Exception: error') 503 # skipped case 504 self.assertEqual(test_results[3].status, test_runner_base.IGNORED_STATUS) 505 self.assertEqual(test_results[3].details, 'mobly.signals.TestSkip') 506 507 @mock.patch('atest.test_runners.mobly_test_runner.MoblyResultUploader') 508 def test_process_test_results_from_summary_show_correct_stats( 509 self, uploader 510 ) -> None: 511 """Tests _process_results_from_summary outputs correct stats.""" 512 test_results = self.runner._process_test_results_from_summary( 513 MOBLY_SUMMARY_FILE, self.tinfo, 0, 1, uploader 514 ) 515 516 self.assertEqual(test_results[0].test_count, 1) 517 self.assertEqual(test_results[0].group_total, 4) 518 self.assertEqual(test_results[0].test_time, '0:00:01') 519 self.assertEqual(test_results[1].test_count, 2) 520 self.assertEqual(test_results[1].group_total, 4) 521 self.assertEqual(test_results[1].test_time, '0:00:00') 522 523 @mock.patch('atest.test_runners.mobly_test_runner.MoblyResultUploader') 524 def test_process_test_results_from_summary_create_correct_uploader_result( 525 self, uploader 526 ) -> None: 527 """Tests _process_results_from_summary creates correct result for the 528 529 uploader. 530 """ 531 uploader.enabled = True 532 uploader.invocation = {'invocationId': 'I12345'} 533 uploader.current_workunit = {'id': 'WU12345'} 534 self.runner._process_test_results_from_summary( 535 MOBLY_SUMMARY_FILE, self.tinfo, 0, 1, uploader 536 ) 537 538 expected_results = { 539 'invocationId': 'I12345', 540 'workUnitId': 'WU12345', 541 'testIdentifier': { 542 'module': TEST_NAME, 543 'testClass': 'SampleTest', 544 'method': 'test_should_error', 545 }, 546 'testStatus': mobly_test_runner.TEST_STORAGE_ERROR, 547 'timing': {'creationTimestamp': 1000, 'completeTimestamp': 2000}, 548 'debugInfo': {'errorMessage': 'error', 'trace': 'Exception: error'}, 549 } 550 551 self.assertEqual( 552 uploader.record_test_result.call_args_list[2].args[0], expected_results 553 ) 554 555 556if __name__ == '__main__': 557 unittest.main() 558