1# Copyright 2018, The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""Robolectric test runner class.
16
17This test runner will be short lived, once robolectric support v2 is in, then
18robolectric tests will be invoked through AtestTFTestRunner.
19"""
20
21from functools import partial
22import json
23import logging
24import os
25from pathlib import Path
26import re
27import tempfile
28import time
29from typing import List
30
31from atest import atest_utils
32from atest import constants
33from atest.atest_enum import ExitCode
34from atest.test_finders import test_info
35from atest.test_runners import test_runner_base
36from atest.test_runners.event_handler import EventHandler
37
38POLL_FREQ_SECS = 0.1
39# A pattern to match event like below
40# TEST_FAILED {'className':'SomeClass', 'testName':'SomeTestName',
41#            'trace':'{"trace":"AssertionError: <true> is equal to <false>\n
42#               at FailureStrategy.fail(FailureStrategy.java:24)\n
43#               at FailureStrategy.fail(FailureStrategy.java:20)\n"}\n\n
44EVENT_RE = re.compile(
45    r'^(?P<event_name>[A-Z_]+) (?P<json_data>{(.\r*|\n)*})(?:\n|$)'
46)
47
48
49class RobolectricTestRunner(test_runner_base.TestRunnerBase):
50  """Robolectric Test Runner class."""
51
52  NAME = 'RobolectricTestRunner'
53  # We don't actually use EXECUTABLE because we're going to use
54  # atest_utils.build to kick off the test but if we don't set it, the base
55  # class will raise an exception.
56  EXECUTABLE = 'make'
57
58  # pylint: disable=useless-super-delegation
59  def __init__(self, results_dir, **kwargs):
60    """Init stuff for robolectric runner class."""
61    super().__init__(results_dir, **kwargs)
62    # TODO: Rollback when found a solution to b/183335046.
63    if not os.getenv(test_runner_base.OLD_OUTPUT_ENV_VAR):
64      self.is_verbose = True
65    else:
66      self.is_verbose = logging.getLogger().isEnabledFor(logging.DEBUG)
67
68  def run_tests(self, test_infos, extra_args, reporter):
69    """Run the list of test_infos. See base class for more.
70
71    Args:
72        test_infos: A list of TestInfos.
73        extra_args: Dict of extra args to add to test run.
74        reporter: An instance of result_report.ResultReporter.
75
76    Returns:
77        0 if tests succeed, non-zero otherwise.
78    """
79    # TODO: Rollback when found a solution to b/183335046.
80    if os.getenv(test_runner_base.OLD_OUTPUT_ENV_VAR):
81      return self.run_tests_pretty(test_infos, extra_args, reporter)
82    return self.run_tests_raw(test_infos, extra_args, reporter)
83
84  def run_tests_raw(self, test_infos, extra_args, reporter):
85    """Run the list of test_infos with raw output.
86
87    Args:
88        test_infos: List of TestInfo.
89        extra_args: Dict of extra args to add to test run.
90        reporter: A ResultReporter Instance.
91
92    Returns:
93        0 if tests succeed, non-zero otherwise.
94    """
95    reporter.register_unsupported_runner(self.NAME)
96    ret_code = ExitCode.SUCCESS
97    for test_info in test_infos:
98      full_env_vars = self._get_full_build_environ(test_info, extra_args)
99      run_cmd = self.generate_run_commands([test_info], extra_args)[0]
100      subproc = self.run(
101          run_cmd, output_to_stdout=self.is_verbose, env_vars=full_env_vars
102      )
103      ret_code |= self.wait_for_subprocess(subproc)
104    if not ret_code:
105      ret_code = self._check_robo_tests_result(test_infos)
106    return ret_code
107
108  def run_tests_pretty(self, test_infos, extra_args, reporter):
109    """Run the list of test_infos with pretty output mode.
110
111    Args:
112        test_infos: List of TestInfo.
113        extra_args: Dict of extra args to add to test run.
114        reporter: A ResultReporter Instance.
115
116    Returns:
117        0 if tests succeed, non-zero otherwise.
118    """
119    ret_code = ExitCode.SUCCESS
120    for test_info in test_infos:
121      # Create a temp communication file.
122      with tempfile.NamedTemporaryFile(dir=self.results_dir) as event_file:
123        # Prepare build environment parameter.
124        full_env_vars = self._get_full_build_environ(
125            test_info, extra_args, event_file
126        )
127        run_cmd = self.generate_run_commands([test_info], extra_args)[0]
128        subproc = self.run(
129            run_cmd, output_to_stdout=self.is_verbose, env_vars=full_env_vars
130        )
131        event_handler = EventHandler(reporter, self.NAME)
132        # Start polling.
133        self.handle_subprocess(
134            subproc,
135            partial(
136                self._exec_with_robo_polling, event_file, subproc, event_handler
137            ),
138        )
139        ret_code |= self.wait_for_subprocess(subproc)
140    if not ret_code:
141      ret_code = self._check_robo_tests_result(test_infos)
142    return ret_code
143
144  def _get_full_build_environ(
145      self, test_info=None, extra_args=None, event_file=None
146  ):
147    """Helper to get full build environment.
148
149    Args:
150        test_info: TestInfo object.
151        extra_args: Dict of extra args to add to test run.
152        event_file: A file-like object that can be used as a temporary storage
153          area.
154    """
155    full_env_vars = os.environ.copy()
156    env_vars = self.generate_env_vars(test_info, extra_args, event_file)
157    full_env_vars.update(env_vars)
158    return full_env_vars
159
160  def _exec_with_robo_polling(
161      self, communication_file, robo_proc, event_handler
162  ):
163    """Polling data from communication file
164
165    Polling data from communication file. Exit when communication file
166    is empty and subprocess ended.
167
168    Args:
169        communication_file: A monitored communication file.
170        robo_proc: The build process.
171        event_handler: A file-like object storing the events of robolectric
172          tests.
173    """
174    buf = ''
175    while True:
176      # Make sure that ATest gets content from current position.
177      communication_file.seek(0, 1)
178      data = communication_file.read()
179      if isinstance(data, bytes):
180        data = data.decode()
181      buf += data
182      reg = re.compile(r'(.|\n)*}\n\n')
183      if not reg.match(buf) or data == '':
184        if robo_proc.poll() is not None:
185          logging.debug('Build process exited early')
186          return
187        time.sleep(POLL_FREQ_SECS)
188      else:
189        # Read all new data and handle it at one time.
190        for event in re.split(r'\n\n', buf):
191          match = EVENT_RE.match(event)
192          if match:
193            try:
194              event_data = json.loads(match.group('json_data'), strict=False)
195            except ValueError:
196              # Parse event fail, continue to parse next one.
197              logging.debug(
198                  '"%s" is not valid json format.', match.group('json_data')
199              )
200              continue
201            event_name = match.group('event_name')
202            event_handler.process_event(event_name, event_data)
203        buf = ''
204
205  @staticmethod
206  def generate_env_vars(test_info, extra_args, event_file=None):
207    """Turn the args into env vars.
208
209    Robolectric tests specify args through env vars, so look for class
210    filters and debug args to apply to the env.
211
212    Args:
213        test_info: TestInfo class that holds the class filter info.
214        extra_args: Dict of extra args to apply for test run.
215        event_file: A file-like object storing the events of robolectric tests.
216
217    Returns:
218        Dict of env vars to pass into invocation.
219    """
220    env_var = {}
221    for arg in extra_args:
222      if constants.WAIT_FOR_DEBUGGER == arg:
223        env_var['DEBUG_ROBOLECTRIC'] = 'true'
224        continue
225    filters = test_info.data.get(constants.TI_FILTER)
226    if filters:
227      robo_filter = next(iter(filters))
228      env_var['ROBOTEST_FILTER'] = robo_filter.class_name
229      if robo_filter.methods:
230        logging.debug(
231            'method filtering not supported for robolectric tests yet.'
232        )
233    if event_file:
234      env_var['EVENT_FILE_ROBOLECTRIC'] = event_file.name
235    return env_var
236
237  # pylint: disable=unnecessary-pass
238  # Please keep above disable flag to ensure host_env_check is overridden.
239  def host_env_check(self):
240    """Check that host env has everything we need.
241
242    We actually can assume the host env is fine because we have the same
243    requirements that atest has. Update this to check for android env vars
244    if that changes.
245    """
246    pass
247
248  def get_test_runner_build_reqs(self, test_infos: List[test_info.TestInfo]):
249    """Return the build requirements.
250
251    Args:
252        test_infos: List of TestInfo.
253
254    Returns:
255        Set of build targets.
256    """
257    build_targets = set()
258    build_targets |= test_runner_base.gather_build_targets(test_infos)
259    return build_targets
260
261  # pylint: disable=unused-argument
262  def generate_run_commands(self, test_infos, extra_args, port=None):
263    """Generate a list of run commands from TestInfos.
264
265    Args:
266        test_infos: A set of TestInfo instances.
267        extra_args: A Dict of extra args to append.
268        port: Optional. An int of the port number to send events to. Subprocess
269          reporter in TF won't try to connect if it's None.
270
271    Returns:
272        A list of run commands to run the tests.
273    """
274    run_cmds = []
275    for test_info in test_infos:
276      robo_command = atest_utils.get_build_cmd() + [str(test_info.test_name)]
277      run_cmd = ' '.join(x for x in robo_command)
278      if constants.DRY_RUN in extra_args:
279        run_cmd = run_cmd.replace(
280            os.environ.get(constants.ANDROID_BUILD_TOP) + os.sep, ''
281        )
282      run_cmds.append(run_cmd)
283    return run_cmds
284
285  @staticmethod
286  def _check_robo_tests_result(test_infos):
287    """Check the result of test_infos with raw output.
288
289    Args:
290        test_infos: List of TestInfo.
291
292    Returns:
293        0 if tests succeed, non-zero otherwise.
294    """
295    for test_info in test_infos:
296      result_output = Path(
297          os.getenv(constants.ANDROID_PRODUCT_OUT, '')
298      ).joinpath(
299          f'obj/ROBOLECTRIC/{test_info.test_name}_intermediates/output.out'
300      )
301      if result_output.exists():
302        with result_output.open() as f:
303          for line in f.readlines():
304            if str(line).find('FAILURES!!!') >= 0:
305              logging.debug(
306                  '%s is failed from %s', test_info.test_name, result_output
307              )
308              return ExitCode.TEST_FAILURE
309    return ExitCode.SUCCESS
310