1# Copyright 2023, The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""Unittests for mobly_test_runner."""
16# pylint: disable=protected-access
17# pylint: disable=invalid-name
18
19import argparse
20import os
21import pathlib
22import unittest
23from unittest import mock
24
25from atest import constants
26from atest import result_reporter
27from atest import unittest_constants
28from atest.test_finders import test_info
29from atest.test_runners import mobly_test_runner
30from atest.test_runners import test_runner_base
31
32
33TEST_NAME = 'SampleMoblyTest'
34MOBLY_PKG = 'mobly/SampleMoblyTest'
35REQUIREMENTS_TXT = 'mobly/requirements.txt'
36APK_1 = 'mobly/snippet1.apk'
37APK_2 = 'mobly/snippet2.apk'
38MISC_FILE = 'mobly/misc_file.txt'
39RESULTS_DIR = 'atest_results/sample_test'
40SERIAL_1 = 'serial1'
41SERIAL_2 = 'serial2'
42ADB_DEVICE = 'adb_device'
43MOBLY_SUMMARY_FILE = os.path.join(
44    unittest_constants.TEST_DATA_DIR, 'mobly', 'sample_test_summary.yaml'
45)
46MOCK_TEST_FILES = mobly_test_runner.MoblyTestFiles('', None, [], [])
47
48
49class MoblyResultUploaderUnittests(unittest.TestCase):
50  """Unit tests for MoblyResultUploader."""
51
52  def setUp(self) -> None:
53    self.patchers = [
54        mock.patch(
55            'atest.logstorage.logstorage_utils.is_upload_enabled',
56            return_value=True,
57        ),
58        mock.patch(
59            'atest.logstorage.logstorage_utils.do_upload_flow',
60            return_value=('creds', {'invocationId': 'I00001'}),
61        ),
62        mock.patch('atest.logstorage.logstorage_utils.BuildClient'),
63    ]
64    for patcher in self.patchers:
65      patcher.start()
66    self.uploader = mobly_test_runner.MoblyResultUploader({})
67    self.uploader._root_workunit = {'id': 'WU00001', 'runCount': 0}
68    self.uploader._current_workunit = {'id': 'WU00010'}
69
70  def tearDown(self) -> None:
71    mock.patch.stopall()
72
73  def test_start_new_workunit(self):
74    """Tests that start_new_workunit sets correct workunit fields."""
75    self.uploader._build_client.insert_work_unit.return_value = {}
76    self.uploader.start_new_workunit()
77
78    self.assertEqual(
79        self.uploader.current_workunit,
80        {
81            'type': mobly_test_runner.WORKUNIT_ATEST_MOBLY_TEST_RUN,
82            'parentId': 'WU00001',
83        },
84    )
85
86  def test_set_workunit_iteration_details_with_repeats(self):
87    """Tests that set_workunit_iteration_details sets the run number for
88
89    repeated tests.
90    """
91    rerun_options = mobly_test_runner.RerunOptions(3, False, False)
92    self.uploader.set_workunit_iteration_details(1, rerun_options)
93
94    self.assertEqual(self.uploader.current_workunit['childRunNumber'], 1)
95
96  def test_set_workunit_iteration_details_with_retries(self):
97    """Tests that set_workunit_iteration_details sets the run number for
98
99    retried tests.
100    """
101    rerun_options = mobly_test_runner.RerunOptions(3, False, True)
102    self.uploader.set_workunit_iteration_details(1, rerun_options)
103
104    self.assertEqual(self.uploader.current_workunit['childAttemptNumber'], 1)
105
106  def test_finalize_current_workunit(self):
107    """Tests that finalize_current_workunit sets correct workunit fields."""
108    workunit = self.uploader.current_workunit
109    self.uploader.finalize_current_workunit()
110
111    self.assertEqual(workunit['schedulerState'], 'completed')
112    self.assertEqual(self.uploader._root_workunit['runCount'], 1)
113    self.assertIsNone(self.uploader.current_workunit)
114
115  def test_finalize_invocation(self):
116    """Tests that finalize_invocation sets correct fields."""
117    invocation = self.uploader.invocation
118    root_workunit = self.uploader._root_workunit
119    self.uploader.finalize_invocation()
120
121    self.assertEqual(root_workunit['schedulerState'], 'completed')
122    self.assertEqual(root_workunit['runCount'], 0)
123    self.assertEqual(invocation['runner'], 'mobly')
124    self.assertEqual(invocation['schedulerState'], 'completed')
125    self.assertFalse(self.uploader.enabled)
126
127  @mock.patch('atest.constants.RESULT_LINK', 'link:%s')
128  def test_add_result_link(self):
129    """Tests that add_result_link correctly sets the result link."""
130    reporter = result_reporter.ResultReporter()
131
132    reporter.test_result_link = ['link:I00000']
133    self.uploader.add_result_link(reporter)
134    self.assertEqual(reporter.test_result_link, ['link:I00000', 'link:I00001'])
135
136    reporter.test_result_link = 'link:I00000'
137    self.uploader.add_result_link(reporter)
138    self.assertEqual(reporter.test_result_link, ['link:I00000', 'link:I00001'])
139
140    reporter.test_result_link = None
141    self.uploader.add_result_link(reporter)
142    self.assertEqual(reporter.test_result_link, ['link:I00001'])
143
144
145class MoblyTestRunnerUnittests(unittest.TestCase):
146  """Unit tests for MoblyTestRunner."""
147
148  def setUp(self) -> None:
149    self.runner = mobly_test_runner.MoblyTestRunner(RESULTS_DIR)
150    self.tinfo = test_info.TestInfo(
151        test_name=TEST_NAME,
152        test_runner=mobly_test_runner.MoblyTestRunner.EXECUTABLE,
153        build_targets=[],
154    )
155    self.reporter = result_reporter.ResultReporter()
156    self.mobly_args = argparse.Namespace(config='', testbed='', testparam=[])
157
158  @mock.patch.object(pathlib.Path, 'is_file')
159  def test_get_test_files_all_files_present(self, is_file) -> None:
160    """Tests _get_test_files with all files present."""
161    is_file.return_value = True
162    files = [MOBLY_PKG, REQUIREMENTS_TXT, APK_1, APK_2, MISC_FILE]
163    file_paths = [pathlib.Path(f) for f in files]
164    self.tinfo.data[constants.MODULE_INSTALLED] = file_paths
165
166    test_files = self.runner._get_test_files(self.tinfo)
167
168    self.assertTrue(test_files.mobly_pkg.endswith(MOBLY_PKG))
169    self.assertTrue(test_files.requirements_txt.endswith(REQUIREMENTS_TXT))
170    self.assertTrue(test_files.test_apks[0].endswith(APK_1))
171    self.assertTrue(test_files.test_apks[1].endswith(APK_2))
172    self.assertTrue(test_files.misc_data[0].endswith(MISC_FILE))
173
174  @mock.patch.object(pathlib.Path, 'is_file')
175  def test_get_test_files_no_mobly_pkg(self, is_file) -> None:
176    """Tests _get_test_files with missing mobly_pkg."""
177    is_file.return_value = True
178    files = [REQUIREMENTS_TXT, APK_1, APK_2]
179    self.tinfo.data[constants.MODULE_INSTALLED] = [
180        pathlib.Path(f) for f in files
181    ]
182
183    with self.assertRaisesRegex(
184        mobly_test_runner.MoblyTestRunnerError, 'No Mobly test package'
185    ):
186      self.runner._get_test_files(self.tinfo)
187
188  @mock.patch.object(pathlib.Path, 'is_file')
189  def test_get_test_files_file_not_found(self, is_file) -> None:
190    """Tests _get_test_files with file not found in file system."""
191    is_file.return_value = False
192    files = [MOBLY_PKG, REQUIREMENTS_TXT, APK_1, APK_2]
193    self.tinfo.data[constants.MODULE_INSTALLED] = [
194        pathlib.Path(f) for f in files
195    ]
196
197    with self.assertRaisesRegex(
198        mobly_test_runner.MoblyTestRunnerError, 'Required test file'
199    ):
200      self.runner._get_test_files(self.tinfo)
201
202  @mock.patch('builtins.open')
203  @mock.patch('os.makedirs')
204  @mock.patch('yaml.safe_dump')
205  def test_generate_mobly_config_no_serials(self, yaml_dump, *_) -> None:
206    """Tests _generate_mobly_config with no serials provided."""
207    self.runner._generate_mobly_config(self.mobly_args, None, MOCK_TEST_FILES)
208
209    expected_config = {
210        'TestBeds': [{
211            'Name': 'LocalTestBed',
212            'Controllers': {
213                'AndroidDevice': '*',
214            },
215            'TestParams': {},
216        }],
217        'MoblyParams': {
218            'LogPath': 'atest_results/sample_test/mobly_logs',
219        },
220    }
221    self.assertEqual(yaml_dump.call_args.args[0], expected_config)
222
223  @mock.patch('builtins.open')
224  @mock.patch('os.makedirs')
225  @mock.patch('yaml.safe_dump')
226  def test_generate_mobly_config_with_serials(self, yaml_dump, *_) -> None:
227    """Tests _generate_mobly_config with serials provided."""
228    self.runner._generate_mobly_config(
229        self.mobly_args, [SERIAL_1, SERIAL_2], MOCK_TEST_FILES
230    )
231
232    expected_config = {
233        'TestBeds': [{
234            'Name': 'LocalTestBed',
235            'Controllers': {
236                'AndroidDevice': [SERIAL_1, SERIAL_2],
237            },
238            'TestParams': {},
239        }],
240        'MoblyParams': {
241            'LogPath': 'atest_results/sample_test/mobly_logs',
242        },
243    }
244    self.assertEqual(yaml_dump.call_args.args[0], expected_config)
245
246  @mock.patch('builtins.open')
247  @mock.patch('os.makedirs')
248  @mock.patch('yaml.safe_dump')
249  def test_generate_mobly_config_with_testparams(self, yaml_dump, *_) -> None:
250    """Tests _generate_mobly_config with custom testparams."""
251    self.mobly_args.testparam = ['foo=bar']
252    self.runner._generate_mobly_config(self.mobly_args, None, MOCK_TEST_FILES)
253
254    expected_config = {
255        'TestBeds': [{
256            'Name': 'LocalTestBed',
257            'Controllers': {
258                'AndroidDevice': '*',
259            },
260            'TestParams': {
261                'foo': 'bar',
262            },
263        }],
264        'MoblyParams': {
265            'LogPath': 'atest_results/sample_test/mobly_logs',
266        },
267    }
268    self.assertEqual(yaml_dump.call_args.args[0], expected_config)
269
270  def test_generate_mobly_config_with_invalid_testparams(self) -> None:
271    """Tests _generate_mobly_config with invalid testparams."""
272    self.mobly_args.testparam = ['foobar']
273    with self.assertRaisesRegex(
274        mobly_test_runner.MoblyTestRunnerError, 'Invalid testparam values'
275    ):
276      self.runner._generate_mobly_config(self.mobly_args, None, [])
277
278  @mock.patch('builtins.open')
279  @mock.patch('os.makedirs')
280  @mock.patch('yaml.safe_dump')
281  def test_generate_mobly_config_with_test_files(self, yaml_dump, *_) -> None:
282    """Tests _generate_mobly_config with test files."""
283    test_apks = ['files/my_app1.apk', 'files/my_app2.apk']
284    misc_data = ['files/some_file.txt']
285    test_files = mobly_test_runner.MoblyTestFiles('', '', test_apks, misc_data)
286    self.runner._generate_mobly_config(self.mobly_args, None, test_files)
287
288    expected_config = {
289        'TestBeds': [{
290            'Name': 'LocalTestBed',
291            'Controllers': {
292                'AndroidDevice': '*',
293            },
294            'TestParams': {
295                'files': {
296                    'my_app1': ['files/my_app1.apk'],
297                    'my_app2': ['files/my_app2.apk'],
298                    'some_file.txt': ['files/some_file.txt'],
299                },
300            },
301        }],
302        'MoblyParams': {
303            'LogPath': 'atest_results/sample_test/mobly_logs',
304        },
305    }
306    self.assertEqual(yaml_dump.call_args.args[0], expected_config)
307
308  @mock.patch('atest.atest_configs.GLOBAL_ARGS.acloud_create', True)
309  @mock.patch('atest.atest_utils.get_adb_devices')
310  def test_get_cvd_serials(self, get_adb_devices) -> None:
311    """Tests _get_cvd_serials returns correct serials."""
312    devices = ['localhost:1234', '127.0.0.1:5678', 'AD12345']
313    get_adb_devices.return_value = devices
314
315    self.assertEqual(self.runner._get_cvd_serials(), devices[:2])
316
317  @mock.patch('atest.atest_utils.get_adb_devices', return_value=[ADB_DEVICE])
318  @mock.patch('subprocess.check_call')
319  def test_install_apks_no_serials(self, check_call, _) -> None:
320    """Tests _install_apks with no serials provided."""
321    self.runner._install_apks([APK_1], None)
322
323    expected_cmds = [['adb', '-s', ADB_DEVICE, 'install', '-r', '-g', APK_1]]
324    self.assertEqual(
325        [call.args[0] for call in check_call.call_args_list], expected_cmds
326    )
327
328  @mock.patch('atest.atest_utils.get_adb_devices', return_value=[ADB_DEVICE])
329  @mock.patch('subprocess.check_call')
330  def test_install_apks_with_serials(self, check_call, _) -> None:
331    """Tests _install_apks with serials provided."""
332    self.runner._install_apks([APK_1], [SERIAL_1, SERIAL_2])
333
334    expected_cmds = [
335        ['adb', '-s', SERIAL_1, 'install', '-r', '-g', APK_1],
336        ['adb', '-s', SERIAL_2, 'install', '-r', '-g', APK_1],
337    ]
338    self.assertEqual(
339        [call.args[0] for call in check_call.call_args_list], expected_cmds
340    )
341
342  def test_get_test_cases_from_spec_with_class_and_methods(self) -> None:
343    """Tests _get_test_cases_from_spec with both class and methods defined."""
344    self.tinfo.data = {
345        'filter': frozenset({
346            test_info.TestFilter(
347                class_name='SampleClass', methods=frozenset({'test1', 'test2'})
348            )
349        })
350    }
351
352    self.assertCountEqual(
353        self.runner._get_test_cases_from_spec(self.tinfo),
354        ['SampleClass.test1', 'SampleClass.test2'],
355    )
356
357  def test_get_test_cases_from_spec_with_class_only(self) -> None:
358    """Tests _get_test_cases_from_spec with only test class defined."""
359    self.tinfo.data = {
360        'filter': frozenset({
361            test_info.TestFilter(class_name='SampleClass', methods=frozenset())
362        })
363    }
364
365    self.assertCountEqual(
366        self.runner._get_test_cases_from_spec(self.tinfo), ['SampleClass']
367    )
368
369  def test_get_test_cases_from_spec_with_method_only(self) -> None:
370    """Tests _get_test_cases_from_spec with only methods defined."""
371    self.tinfo.data = {
372        'filter': frozenset({
373            test_info.TestFilter(
374                class_name='.', methods=frozenset({'test1', 'test2'})
375            )
376        })
377    }
378
379    self.assertCountEqual(
380        self.runner._get_test_cases_from_spec(self.tinfo), ['test1', 'test2']
381    )
382
383  @mock.patch.object(
384      mobly_test_runner.MoblyTestRunner,
385      '_process_test_results_from_summary',
386      return_value=(),
387  )
388  @mock.patch('atest.test_runners.mobly_test_runner.MoblyResultUploader')
389  def test_run_and_handle_results_with_iterations(self, uploader, _) -> None:
390    """Tests _run_and_handle_results with multiple iterations."""
391    with mock.patch.object(
392        mobly_test_runner.MoblyTestRunner,
393        '_run_mobly_command',
394        side_effect=(1, 1, 0, 0, 1),
395    ) as run_mobly_command:
396      runner = mobly_test_runner.MoblyTestRunner(RESULTS_DIR)
397      runner._run_and_handle_results(
398          [],
399          self.tinfo,
400          mobly_test_runner.RerunOptions(5, False, False),
401          self.mobly_args,
402          self.reporter,
403          uploader,
404      )
405      self.assertEqual(run_mobly_command.call_count, 5)
406
407  @mock.patch.object(
408      mobly_test_runner.MoblyTestRunner,
409      '_process_test_results_from_summary',
410      return_value=(),
411  )
412  @mock.patch('atest.test_runners.mobly_test_runner.MoblyResultUploader')
413  def test_run_and_handle_results_with_rerun_until_failure(
414      self, uploader, _
415  ) -> None:
416    """Tests _run_and_handle_results with rerun_until_failure."""
417    with mock.patch.object(
418        mobly_test_runner.MoblyTestRunner,
419        '_run_mobly_command',
420        side_effect=(0, 0, 1, 0, 1),
421    ) as run_mobly_command:
422      runner = mobly_test_runner.MoblyTestRunner(RESULTS_DIR)
423      runner._run_and_handle_results(
424          [],
425          self.tinfo,
426          mobly_test_runner.RerunOptions(5, True, False),
427          self.mobly_args,
428          self.reporter,
429          uploader,
430      )
431      self.assertEqual(run_mobly_command.call_count, 3)
432
433  @mock.patch.object(
434      mobly_test_runner.MoblyTestRunner,
435      '_process_test_results_from_summary',
436      return_value=(),
437  )
438  @mock.patch('atest.test_runners.mobly_test_runner.MoblyResultUploader')
439  def test_run_and_handle_results_with_retry_any_failure(
440      self, uploader, _
441  ) -> None:
442    """Tests _run_and_handle_results with retry_any_failure."""
443    with mock.patch.object(
444        mobly_test_runner.MoblyTestRunner,
445        '_run_mobly_command',
446        side_effect=(1, 1, 1, 0, 0),
447    ) as run_mobly_command:
448      runner = mobly_test_runner.MoblyTestRunner(RESULTS_DIR)
449      runner._run_and_handle_results(
450          [],
451          self.tinfo,
452          mobly_test_runner.RerunOptions(5, False, True),
453          self.mobly_args,
454          self.reporter,
455          uploader,
456      )
457      self.assertEqual(run_mobly_command.call_count, 4)
458
459  @mock.patch('atest.test_runners.mobly_test_runner.MoblyResultUploader')
460  def test_process_test_results_from_summary_show_correct_names(
461      self, uploader
462  ) -> None:
463    """Tests _process_results_from_summary outputs correct test names."""
464    test_results = self.runner._process_test_results_from_summary(
465        MOBLY_SUMMARY_FILE, self.tinfo, 0, 1, uploader
466    )
467
468    result = test_results[0]
469    self.assertEqual(result.runner_name, self.runner.NAME)
470    self.assertEqual(result.group_name, TEST_NAME)
471    self.assertEqual(result.test_run_name, 'SampleTest')
472    self.assertEqual(result.test_name, 'SampleTest.test_should_pass')
473
474    test_results = self.runner._process_test_results_from_summary(
475        MOBLY_SUMMARY_FILE, self.tinfo, 2, 3, uploader
476    )
477
478    result = test_results[0]
479    self.assertEqual(result.test_run_name, 'SampleTest (#3)')
480    self.assertEqual(result.test_name, 'SampleTest.test_should_pass (#3)')
481
482  @mock.patch('atest.test_runners.mobly_test_runner.MoblyResultUploader')
483  def test_process_test_results_from_summary_show_correct_status_and_details(
484      self, uploader
485  ) -> None:
486    """Tests _process_results_from_summary outputs correct test status and
487
488    details.
489    """
490    test_results = self.runner._process_test_results_from_summary(
491        MOBLY_SUMMARY_FILE, self.tinfo, 0, 1, uploader
492    )
493
494    # passed case
495    self.assertEqual(test_results[0].status, test_runner_base.PASSED_STATUS)
496    self.assertEqual(test_results[0].details, None)
497    # failed case
498    self.assertEqual(test_results[1].status, test_runner_base.FAILED_STATUS)
499    self.assertEqual(test_results[1].details, 'mobly.signals.TestFailure')
500    # errored case
501    self.assertEqual(test_results[2].status, test_runner_base.FAILED_STATUS)
502    self.assertEqual(test_results[2].details, 'Exception: error')
503    # skipped case
504    self.assertEqual(test_results[3].status, test_runner_base.IGNORED_STATUS)
505    self.assertEqual(test_results[3].details, 'mobly.signals.TestSkip')
506
507  @mock.patch('atest.test_runners.mobly_test_runner.MoblyResultUploader')
508  def test_process_test_results_from_summary_show_correct_stats(
509      self, uploader
510  ) -> None:
511    """Tests _process_results_from_summary outputs correct stats."""
512    test_results = self.runner._process_test_results_from_summary(
513        MOBLY_SUMMARY_FILE, self.tinfo, 0, 1, uploader
514    )
515
516    self.assertEqual(test_results[0].test_count, 1)
517    self.assertEqual(test_results[0].group_total, 4)
518    self.assertEqual(test_results[0].test_time, '0:00:01')
519    self.assertEqual(test_results[1].test_count, 2)
520    self.assertEqual(test_results[1].group_total, 4)
521    self.assertEqual(test_results[1].test_time, '0:00:00')
522
523  @mock.patch('atest.test_runners.mobly_test_runner.MoblyResultUploader')
524  def test_process_test_results_from_summary_create_correct_uploader_result(
525      self, uploader
526  ) -> None:
527    """Tests _process_results_from_summary creates correct result for the
528
529    uploader.
530    """
531    uploader.enabled = True
532    uploader.invocation = {'invocationId': 'I12345'}
533    uploader.current_workunit = {'id': 'WU12345'}
534    self.runner._process_test_results_from_summary(
535        MOBLY_SUMMARY_FILE, self.tinfo, 0, 1, uploader
536    )
537
538    expected_results = {
539        'invocationId': 'I12345',
540        'workUnitId': 'WU12345',
541        'testIdentifier': {
542            'module': TEST_NAME,
543            'testClass': 'SampleTest',
544            'method': 'test_should_error',
545        },
546        'testStatus': mobly_test_runner.TEST_STORAGE_ERROR,
547        'timing': {'creationTimestamp': 1000, 'completeTimestamp': 2000},
548        'debugInfo': {'errorMessage': 'error', 'trace': 'Exception: error'},
549    }
550
551    self.assertEqual(
552        uploader.record_test_result.call_args_list[2].args[0], expected_results
553    )
554
555
556if __name__ == '__main__':
557  unittest.main()
558