1#!/usr/bin/env python3.4 2# 3# Copyright 2017 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the 'License'); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an 'AS IS' BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import collections 18import itertools 19import json 20import logging 21import numpy 22import os 23import time 24from acts import asserts 25from acts import base_test 26from acts import utils 27from acts.controllers import iperf_server as ipf 28from acts.controllers.utils_lib import ssh 29from acts.metrics.loggers.blackbox import BlackboxMappedMetricLogger 30from acts_contrib.test_utils.wifi import ota_chamber 31from acts_contrib.test_utils.wifi import ota_sniffer 32from acts_contrib.test_utils.wifi import wifi_performance_test_utils as wputils 33from acts_contrib.test_utils.wifi.wifi_performance_test_utils.bokeh_figure import BokehFigure 34from acts_contrib.test_utils.wifi import wifi_retail_ap as retail_ap 35from acts_contrib.test_utils.wifi import wifi_test_utils as wutils 36from functools import partial 37 38 39class WifiRvrTest(base_test.BaseTestClass): 40 """Class to test WiFi rate versus range. 41 42 This class implements WiFi rate versus range tests on single AP single STA 43 links. The class setups up the AP in the desired configurations, configures 44 and connects the phone to the AP, and runs iperf throughput test while 45 sweeping attenuation. For an example config file to run this test class see 46 example_connectivity_performance_ap_sta.json. 47 """ 48 49 TEST_TIMEOUT = 6 50 MAX_CONSECUTIVE_ZEROS = 3 51 52 def __init__(self, controllers): 53 base_test.BaseTestClass.__init__(self, controllers) 54 self.testcase_metric_logger = ( 55 BlackboxMappedMetricLogger.for_test_case()) 56 self.testclass_metric_logger = ( 57 BlackboxMappedMetricLogger.for_test_class()) 58 self.publish_testcase_metrics = True 59 60 def setup_class(self): 61 """Initializes common test hardware and parameters. 62 63 This function initializes hardwares and compiles parameters that are 64 common to all tests in this class. 65 """ 66 self.sta_dut = self.android_devices[0] 67 req_params = [ 68 'RetailAccessPoints', 'rvr_test_params', 'testbed_params', 69 'RemoteServer', 'main_network' 70 ] 71 opt_params = ['golden_files_list', 'OTASniffer'] 72 self.unpack_userparams(req_params, opt_params) 73 self.testclass_params = self.rvr_test_params 74 self.num_atten = self.attenuators[0].instrument.num_atten 75 self.iperf_server = self.iperf_servers[0] 76 self.remote_server = ssh.connection.SshConnection( 77 ssh.settings.from_config(self.RemoteServer[0]['ssh_config'])) 78 self.iperf_client = self.iperf_clients[0] 79 self.access_point = retail_ap.create(self.RetailAccessPoints)[0] 80 if hasattr(self, 81 'OTASniffer') and self.testbed_params['sniffer_enable']: 82 try: 83 self.sniffer = ota_sniffer.create(self.OTASniffer)[0] 84 except: 85 self.log.warning('Could not start sniffer. Disabling sniffs.') 86 self.testbed_params['sniffer_enable'] = 0 87 self.log.info('Access Point Configuration: {}'.format( 88 self.access_point.ap_settings)) 89 self.log_path = os.path.join(logging.log_path, 'results') 90 os.makedirs(self.log_path, exist_ok=True) 91 if not hasattr(self, 'golden_files_list'): 92 if 'golden_results_path' in self.testbed_params: 93 self.golden_files_list = [ 94 os.path.join(self.testbed_params['golden_results_path'], 95 file) for file in 96 os.listdir(self.testbed_params['golden_results_path']) 97 ] 98 else: 99 self.log.warning('No golden files found.') 100 self.golden_files_list = [] 101 self.testclass_results = [] 102 103 # Turn WiFi ON 104 if self.testclass_params.get('airplane_mode', 1): 105 for dev in self.android_devices: 106 self.log.info('Turning on airplane mode.') 107 asserts.assert_true(utils.force_airplane_mode(dev, True), 108 'Can not turn on airplane mode.') 109 wutils.reset_wifi(dev) 110 wutils.wifi_toggle_state(dev, True) 111 112 def teardown_test(self): 113 self.iperf_server.stop() 114 115 def teardown_class(self): 116 # Turn WiFi OFF 117 self.access_point.teardown() 118 for dev in self.android_devices: 119 wutils.wifi_toggle_state(dev, False) 120 dev.go_to_sleep() 121 self.process_testclass_results() 122 123 def process_testclass_results(self): 124 """Saves plot with all test results to enable comparison.""" 125 # Plot and save all results 126 plots = collections.OrderedDict() 127 for result in self.testclass_results: 128 plot_id = (result['testcase_params']['channel'], 129 result['testcase_params']['mode']) 130 if plot_id not in plots: 131 plots[plot_id] = BokehFigure( 132 title='Channel {} {} ({})'.format( 133 result['testcase_params']['channel'], 134 result['testcase_params']['mode'], 135 result['testcase_params']['traffic_type']), 136 x_label='Attenuation (dB)', 137 primary_y_label='Throughput (Mbps)') 138 plots[plot_id].add_line(result['total_attenuation'], 139 result['throughput_receive'], 140 result['test_name'].strip('test_rvr_'), 141 hover_text=result['hover_text'], 142 marker='circle') 143 plots[plot_id].add_line(result['total_attenuation'], 144 result['rx_phy_rate'], 145 result['test_name'].strip('test_rvr_') + 146 ' (Rx PHY)', 147 hover_text=result['hover_text'], 148 style='dashed', 149 marker='inverted_triangle') 150 plots[plot_id].add_line(result['total_attenuation'], 151 result['tx_phy_rate'], 152 result['test_name'].strip('test_rvr_') + 153 ' (Tx PHY)', 154 hover_text=result['hover_text'], 155 style='dashed', 156 marker='triangle') 157 158 figure_list = [] 159 for plot_id, plot in plots.items(): 160 plot.generate_figure() 161 figure_list.append(plot) 162 output_file_path = os.path.join(self.log_path, 'results.html') 163 BokehFigure.save_figures(figure_list, output_file_path) 164 165 def pass_fail_check(self, rvr_result): 166 """Check the test result and decide if it passed or failed. 167 168 Checks the RvR test result and compares to a throughput limites for 169 the same configuration. The pass/fail tolerances are provided in the 170 config file. 171 172 Args: 173 rvr_result: dict containing attenuation, throughput and other data 174 """ 175 try: 176 throughput_limits = self.compute_throughput_limits(rvr_result) 177 except: 178 asserts.explicit_pass( 179 'Test passed by default. Golden file not found') 180 181 failure_count = 0 182 for idx, current_throughput in enumerate( 183 rvr_result['throughput_receive']): 184 if (current_throughput < throughput_limits['lower_limit'][idx] 185 or current_throughput > 186 throughput_limits['upper_limit'][idx]): 187 failure_count = failure_count + 1 188 189 # Set test metrics 190 rvr_result['metrics']['failure_count'] = failure_count 191 if self.publish_testcase_metrics: 192 self.testcase_metric_logger.add_metric('failure_count', 193 failure_count) 194 195 # Assert pass or fail 196 if failure_count >= self.testclass_params['failure_count_tolerance']: 197 asserts.fail('Test failed. Found {} points outside limits.'.format( 198 failure_count)) 199 asserts.explicit_pass( 200 'Test passed. Found {} points outside throughput limits.'.format( 201 failure_count)) 202 203 def compute_throughput_limits(self, rvr_result): 204 """Compute throughput limits for current test. 205 206 Checks the RvR test result and compares to a throughput limites for 207 the same configuration. The pass/fail tolerances are provided in the 208 config file. 209 210 Args: 211 rvr_result: dict containing attenuation, throughput and other meta 212 data 213 Returns: 214 throughput_limits: dict containing attenuation and throughput limit data 215 """ 216 test_name = self.current_test_name 217 golden_path = next(file_name for file_name in self.golden_files_list 218 if test_name in file_name) 219 with open(golden_path, 'r') as golden_file: 220 golden_results = json.load(golden_file) 221 golden_attenuation = [ 222 att + golden_results['fixed_attenuation'] 223 for att in golden_results['attenuation'] 224 ] 225 attenuation = [] 226 lower_limit = [] 227 upper_limit = [] 228 for idx, current_throughput in enumerate( 229 rvr_result['throughput_receive']): 230 current_att = rvr_result['attenuation'][idx] + rvr_result[ 231 'fixed_attenuation'] 232 att_distances = [ 233 abs(current_att - golden_att) 234 for golden_att in golden_attenuation 235 ] 236 sorted_distances = sorted(enumerate(att_distances), 237 key=lambda x: x[1]) 238 closest_indeces = [dist[0] for dist in sorted_distances[0:3]] 239 closest_throughputs = [ 240 golden_results['throughput_receive'][index] 241 for index in closest_indeces 242 ] 243 closest_throughputs.sort() 244 245 attenuation.append(current_att) 246 lower_limit.append( 247 max( 248 closest_throughputs[0] - max( 249 self.testclass_params['abs_tolerance'], 250 closest_throughputs[0] * 251 self.testclass_params['pct_tolerance'] / 100), 0)) 252 upper_limit.append(closest_throughputs[-1] + max( 253 self.testclass_params['abs_tolerance'], closest_throughputs[-1] 254 * self.testclass_params['pct_tolerance'] / 100)) 255 throughput_limits = { 256 'attenuation': attenuation, 257 'lower_limit': lower_limit, 258 'upper_limit': upper_limit 259 } 260 return throughput_limits 261 262 def plot_rvr_result(self, rvr_result): 263 """Saves plots and JSON formatted results. 264 265 Args: 266 rvr_result: dict containing attenuation, throughput and other meta 267 data 268 """ 269 # Save output as text file 270 results_file_path = os.path.join( 271 self.log_path, '{}.json'.format(self.current_test_name)) 272 with open(results_file_path, 'w') as results_file: 273 json.dump(wputils.serialize_dict(rvr_result), 274 results_file, 275 indent=4) 276 # Plot and save 277 figure = BokehFigure(title=self.current_test_name, 278 x_label='Attenuation (dB)', 279 primary_y_label='Throughput (Mbps)') 280 try: 281 golden_path = next(file_name 282 for file_name in self.golden_files_list 283 if self.current_test_name in file_name) 284 with open(golden_path, 'r') as golden_file: 285 golden_results = json.load(golden_file) 286 golden_attenuation = [ 287 att + golden_results['fixed_attenuation'] 288 for att in golden_results['attenuation'] 289 ] 290 throughput_limits = self.compute_throughput_limits(rvr_result) 291 shaded_region = { 292 'x_vector': throughput_limits['attenuation'], 293 'lower_limit': throughput_limits['lower_limit'], 294 'upper_limit': throughput_limits['upper_limit'] 295 } 296 figure.add_line(golden_attenuation, 297 golden_results['throughput_receive'], 298 'Golden Results', 299 color='green', 300 marker='circle', 301 shaded_region=shaded_region) 302 except: 303 self.log.warning('ValueError: Golden file not found') 304 305 # Generate graph annotatios 306 rvr_result['hover_text'] = { 307 'llstats': [ 308 'TX MCS = {0} ({1:.1f}%). RX MCS = {2} ({3:.1f}%)'.format( 309 curr_llstats['summary']['common_tx_mcs'], 310 curr_llstats['summary']['common_tx_mcs_freq'] * 100, 311 curr_llstats['summary']['common_rx_mcs'], 312 curr_llstats['summary']['common_rx_mcs_freq'] * 100) 313 for curr_llstats in rvr_result['llstats'] 314 ], 315 'rssi': [ 316 '{0:.2f} [{1:.2f},{2:.2f}]'.format( 317 rssi['signal_poll_rssi'], 318 rssi['chain_0_rssi'], 319 rssi['chain_1_rssi'], 320 ) for rssi in rvr_result['rssi'] 321 ] 322 } 323 324 figure.add_line(rvr_result['total_attenuation'], 325 rvr_result['throughput_receive'], 326 'Measured Throughput', 327 hover_text=rvr_result['hover_text'], 328 color='black', 329 marker='circle') 330 figure.add_line( 331 rvr_result['total_attenuation'][0:len(rvr_result['rx_phy_rate'])], 332 rvr_result['rx_phy_rate'], 333 'Rx PHY Rate', 334 hover_text=rvr_result['hover_text'], 335 color='blue', 336 style='dashed', 337 marker='inverted_triangle') 338 figure.add_line( 339 rvr_result['total_attenuation'][0:len(rvr_result['rx_phy_rate'])], 340 rvr_result['tx_phy_rate'], 341 'Tx PHY Rate', 342 hover_text=rvr_result['hover_text'], 343 color='red', 344 style='dashed', 345 marker='triangle') 346 347 output_file_path = os.path.join( 348 self.log_path, '{}.html'.format(self.current_test_name)) 349 figure.generate_figure(output_file_path) 350 351 def compute_test_metrics(self, rvr_result): 352 # Set test metrics 353 rvr_result['metrics'] = {} 354 rvr_result['metrics']['peak_tput'] = max( 355 rvr_result['throughput_receive']) 356 if self.publish_testcase_metrics: 357 self.testcase_metric_logger.add_metric( 358 'peak_tput', rvr_result['metrics']['peak_tput']) 359 360 test_mode = rvr_result['ap_settings'][rvr_result['testcase_params'] 361 ['band']]['bandwidth'] 362 tput_below_limit = [ 363 tput < 364 self.testclass_params['tput_metric_targets'][test_mode]['high'] 365 for tput in rvr_result['throughput_receive'] 366 ] 367 rvr_result['metrics']['high_tput_range'] = -1 368 for idx in range(len(tput_below_limit)): 369 if all(tput_below_limit[idx:]): 370 if idx == 0: 371 # Throughput was never above limit 372 rvr_result['metrics']['high_tput_range'] = -1 373 else: 374 rvr_result['metrics']['high_tput_range'] = rvr_result[ 375 'total_attenuation'][max(idx, 1) - 1] 376 break 377 if self.publish_testcase_metrics: 378 self.testcase_metric_logger.add_metric( 379 'high_tput_range', rvr_result['metrics']['high_tput_range']) 380 381 tput_below_limit = [ 382 tput < 383 self.testclass_params['tput_metric_targets'][test_mode]['low'] 384 for tput in rvr_result['throughput_receive'] 385 ] 386 for idx in range(len(tput_below_limit)): 387 if all(tput_below_limit[idx:]): 388 rvr_result['metrics']['low_tput_range'] = rvr_result[ 389 'total_attenuation'][max(idx, 1) - 1] 390 break 391 else: 392 rvr_result['metrics']['low_tput_range'] = -1 393 if self.publish_testcase_metrics: 394 self.testcase_metric_logger.add_metric( 395 'low_tput_range', rvr_result['metrics']['low_tput_range']) 396 397 def process_test_results(self, rvr_result): 398 self.plot_rvr_result(rvr_result) 399 self.compute_test_metrics(rvr_result) 400 401 def run_rvr_test(self, testcase_params): 402 """Test function to run RvR. 403 404 The function runs an RvR test in the current device/AP configuration. 405 Function is called from another wrapper function that sets up the 406 testbed for the RvR test 407 408 Args: 409 testcase_params: dict containing test-specific parameters 410 Returns: 411 rvr_result: dict containing rvr_results and meta data 412 """ 413 self.log.info('Start running RvR') 414 # Refresh link layer stats before test 415 llstats_obj = wputils.LinkLayerStats( 416 self.monitored_dut, 417 self.testclass_params.get('monitor_llstats', 1)) 418 zero_counter = 0 419 throughput = [] 420 rx_phy_rate = [] 421 tx_phy_rate = [] 422 llstats = [] 423 rssi = [] 424 for atten in testcase_params['atten_range']: 425 for dev in self.android_devices: 426 if not wputils.health_check(dev, 5, 50): 427 asserts.skip('DUT health check failed. Skipping test.') 428 # Set Attenuation 429 for attenuator in self.attenuators: 430 attenuator.set_atten(atten, strict=False, retry=True) 431 # Refresh link layer stats 432 llstats_obj.update_stats() 433 # Setup sniffer 434 if self.testbed_params['sniffer_enable']: 435 self.sniffer.start_capture( 436 network=testcase_params['test_network'], 437 chan=testcase_params['channel'], 438 bw=testcase_params['bandwidth'], 439 duration=self.testclass_params['iperf_duration'] / 5) 440 # Start iperf session 441 if self.testclass_params.get('monitor_rssi', 1): 442 rssi_future = wputils.get_connected_rssi_nb( 443 self.monitored_dut, 444 self.testclass_params['iperf_duration'] - 1, 445 1, 446 1, 447 interface=self.monitored_interface) 448 self.iperf_server.start(tag=str(atten)) 449 client_output_path = self.iperf_client.start( 450 testcase_params['iperf_server_address'], 451 testcase_params['iperf_args'], str(atten), 452 self.testclass_params['iperf_duration'] + self.TEST_TIMEOUT) 453 server_output_path = self.iperf_server.stop() 454 if self.testclass_params.get('monitor_rssi', 1): 455 rssi_result = rssi_future.result() 456 current_rssi = { 457 'signal_poll_rssi': 458 rssi_result['signal_poll_rssi']['mean'], 459 'chain_0_rssi': rssi_result['chain_0_rssi']['mean'], 460 'chain_1_rssi': rssi_result['chain_1_rssi']['mean'] 461 } 462 else: 463 current_rssi = { 464 'signal_poll_rssi': float('nan'), 465 'chain_0_rssi': float('nan'), 466 'chain_1_rssi': float('nan') 467 } 468 rssi.append(current_rssi) 469 # Stop sniffer 470 if self.testbed_params['sniffer_enable']: 471 self.sniffer.stop_capture(tag=str(atten)) 472 # Parse and log result 473 if testcase_params['use_client_output']: 474 iperf_file = client_output_path 475 else: 476 iperf_file = server_output_path 477 try: 478 iperf_result = ipf.IPerfResult(iperf_file) 479 curr_throughput = numpy.mean(iperf_result.instantaneous_rates[ 480 self.testclass_params['iperf_ignored_interval']:-1] 481 ) * 8 * (1.024**2) 482 except: 483 self.log.warning( 484 'ValueError: Cannot get iperf result. Setting to 0') 485 curr_throughput = 0 486 throughput.append(curr_throughput) 487 llstats_obj.update_stats() 488 curr_llstats = llstats_obj.llstats_incremental.copy() 489 llstats.append(curr_llstats) 490 rx_phy_rate.append(curr_llstats['summary'].get( 491 'mean_rx_phy_rate', 0)) 492 tx_phy_rate.append(curr_llstats['summary'].get( 493 'mean_tx_phy_rate', 0)) 494 self.log.info( 495 ('Throughput at {0:.2f} dB is {1:.2f} Mbps. ' 496 'RSSI = {2:.2f} [{3:.2f}, {4:.2f}].').format( 497 atten, curr_throughput, current_rssi['signal_poll_rssi'], 498 current_rssi['chain_0_rssi'], 499 current_rssi['chain_1_rssi'])) 500 if curr_throughput == 0: 501 zero_counter = zero_counter + 1 502 else: 503 zero_counter = 0 504 if zero_counter == self.MAX_CONSECUTIVE_ZEROS: 505 self.log.info( 506 'Throughput stable at 0 Mbps. Stopping test now.') 507 zero_padding = len( 508 testcase_params['atten_range']) - len(throughput) 509 throughput.extend([0] * zero_padding) 510 rx_phy_rate.extend([0] * zero_padding) 511 tx_phy_rate.extend([0] * zero_padding) 512 break 513 for attenuator in self.attenuators: 514 attenuator.set_atten(0, strict=False, retry=True) 515 # Compile test result and meta data 516 rvr_result = collections.OrderedDict() 517 rvr_result['test_name'] = self.current_test_name 518 rvr_result['testcase_params'] = testcase_params.copy() 519 rvr_result['ap_settings'] = self.access_point.ap_settings.copy() 520 rvr_result['fixed_attenuation'] = self.testbed_params[ 521 'fixed_attenuation'][str(testcase_params['channel'])] 522 rvr_result['attenuation'] = list(testcase_params['atten_range']) 523 rvr_result['total_attenuation'] = [ 524 att + rvr_result['fixed_attenuation'] 525 for att in rvr_result['attenuation'] 526 ] 527 rvr_result['rssi'] = rssi 528 rvr_result['throughput_receive'] = throughput 529 rvr_result['rx_phy_rate'] = rx_phy_rate 530 rvr_result['tx_phy_rate'] = tx_phy_rate 531 rvr_result['llstats'] = llstats 532 return rvr_result 533 534 def setup_ap(self, testcase_params): 535 """Sets up the access point in the configuration required by the test. 536 537 Args: 538 testcase_params: dict containing AP and other test params 539 """ 540 band = self.access_point.band_lookup_by_channel( 541 testcase_params['channel']) 542 if '6G' in band: 543 frequency = wutils.WifiEnums.channel_6G_to_freq[int( 544 testcase_params['channel'].strip('6g'))] 545 else: 546 if testcase_params['channel'] < 13: 547 frequency = wutils.WifiEnums.channel_2G_to_freq[ 548 testcase_params['channel']] 549 else: 550 frequency = wutils.WifiEnums.channel_5G_to_freq[ 551 testcase_params['channel']] 552 if frequency in wutils.WifiEnums.DFS_5G_FREQUENCIES: 553 self.access_point.set_region(self.testbed_params['DFS_region']) 554 else: 555 self.access_point.set_region(self.testbed_params['default_region']) 556 self.access_point.set_channel_and_bandwidth(testcase_params['band'], 557 testcase_params['channel'], 558 testcase_params['mode']) 559 self.log.info('Access Point Configuration: {}'.format( 560 self.access_point.ap_settings)) 561 562 def setup_dut(self, testcase_params): 563 """Sets up the DUT in the configuration required by the test. 564 565 Args: 566 testcase_params: dict containing AP and other test params 567 """ 568 # Turn screen off to preserve battery 569 if self.testbed_params.get('screen_on', 570 False) or self.testclass_params.get( 571 'screen_on', False): 572 self.sta_dut.droid.wakeLockAcquireDim() 573 else: 574 self.sta_dut.go_to_sleep() 575 # Enable Tune Code 576 band = self.access_point.band_lookup_by_channel(testcase_params['channel']) 577 if 'tune_code' in self.testbed_params: 578 if int(self.testbed_params['tune_code']['manual_tune_code']): 579 self.log.info('Tune Code forcing enabled in config file') 580 wputils.write_antenna_tune_code(self.sta_dut, self.testbed_params['tune_code'][band]) 581 if (wputils.validate_network(self.sta_dut, 582 testcase_params['test_network']['SSID']) 583 and not self.testclass_params.get('force_reconnect', 0)): 584 self.log.info('Already connected to desired network') 585 else: 586 wutils.wifi_toggle_state(self.sta_dut, False) 587 wutils.set_wifi_country_code(self.sta_dut, 588 self.testclass_params['country_code']) 589 wutils.wifi_toggle_state(self.sta_dut, True) 590 wutils.reset_wifi(self.sta_dut) 591 if self.testbed_params.get('txbf_off', False): 592 wputils.disable_beamforming(self.sta_dut) 593 wutils.set_wifi_country_code(self.sta_dut, 594 self.testclass_params['country_code']) 595 if self.testbed_params['sniffer_enable']: 596 self.sniffer.start_capture( 597 network={'SSID': testcase_params['test_network']['SSID']}, 598 chan=testcase_params['channel'], 599 bw=testcase_params['bandwidth'], 600 duration=180) 601 try: 602 wutils.wifi_connect(self.sta_dut, 603 testcase_params['test_network'], 604 num_of_tries=5, 605 check_connectivity=True) 606 if self.testclass_params.get('num_streams', 2) == 1: 607 wputils.set_nss_capability(self.sta_dut, 1) 608 finally: 609 if self.testbed_params['sniffer_enable']: 610 self.sniffer.stop_capture(tag='connection_setup') 611 612 def setup_rvr_test(self, testcase_params): 613 """Function that gets devices ready for the test. 614 615 Args: 616 testcase_params: dict containing test-specific parameters 617 """ 618 # Configure AP 619 self.setup_ap(testcase_params) 620 # Set attenuator to 0 dB 621 for attenuator in self.attenuators: 622 attenuator.set_atten(0, strict=False, retry=True) 623 # Reset, configure, and connect DUT 624 self.setup_dut(testcase_params) 625 # Wait before running the first wifi test 626 first_test_delay = self.testclass_params.get('first_test_delay', 600) 627 if first_test_delay > 0 and len(self.testclass_results) == 0: 628 self.log.info('Waiting before the first RvR test.') 629 time.sleep(first_test_delay) 630 self.setup_dut(testcase_params) 631 # Get iperf_server address 632 sta_dut_ip = self.sta_dut.droid.connectivityGetIPv4Addresses( 633 'wlan0')[0] 634 if isinstance(self.iperf_server, ipf.IPerfServerOverAdb): 635 testcase_params['iperf_server_address'] = sta_dut_ip 636 else: 637 if self.testbed_params.get('lan_traffic_only', True): 638 testcase_params[ 639 'iperf_server_address'] = wputils.get_server_address( 640 self.remote_server, sta_dut_ip, '255.255.255.0') 641 else: 642 testcase_params[ 643 'iperf_server_address'] = wputils.get_server_address( 644 self.remote_server, sta_dut_ip, 'public') 645 # Set DUT to monitor RSSI and LLStats on 646 self.monitored_dut = self.sta_dut 647 self.monitored_interface = 'wlan0' 648 649 def compile_test_params(self, testcase_params): 650 """Function that completes all test params based on the test name. 651 652 Args: 653 testcase_params: dict containing test-specific parameters 654 """ 655 # Check if test should be skipped based on parameters. 656 wputils.check_skip_conditions(testcase_params, self.sta_dut, 657 self.access_point, 658 getattr(self, 'ota_chamber', None)) 659 660 band = wputils.CHANNEL_TO_BAND_MAP[testcase_params['channel']] 661 start_atten = self.testclass_params['atten_start'].get(band, 0) 662 num_atten_steps = int( 663 (self.testclass_params['atten_stop'] - start_atten) / 664 self.testclass_params['atten_step']) 665 testcase_params['atten_range'] = [ 666 start_atten + x * self.testclass_params['atten_step'] 667 for x in range(0, num_atten_steps) 668 ] 669 band = self.access_point.band_lookup_by_channel( 670 testcase_params['channel']) 671 testcase_params['band'] = band 672 testcase_params['test_network'] = self.main_network[band] 673 if testcase_params['traffic_type'] == 'TCP': 674 testcase_params['iperf_socket_size'] = self.testclass_params.get( 675 'tcp_socket_size', None) 676 testcase_params['iperf_processes'] = self.testclass_params.get( 677 'tcp_processes', 1) 678 elif testcase_params['traffic_type'] == 'UDP': 679 testcase_params['iperf_socket_size'] = self.testclass_params.get( 680 'udp_socket_size', None) 681 testcase_params['iperf_processes'] = self.testclass_params.get( 682 'udp_processes', 1) 683 if (testcase_params['traffic_direction'] == 'DL' 684 and not isinstance(self.iperf_server, ipf.IPerfServerOverAdb) 685 ) or (testcase_params['traffic_direction'] == 'UL' 686 and isinstance(self.iperf_server, ipf.IPerfServerOverAdb)): 687 testcase_params['iperf_args'] = wputils.get_iperf_arg_string( 688 duration=self.testclass_params['iperf_duration'], 689 reverse_direction=1, 690 traffic_type=testcase_params['traffic_type'], 691 socket_size=testcase_params['iperf_socket_size'], 692 num_processes=testcase_params['iperf_processes'], 693 udp_throughput=self.testclass_params['UDP_rates'][ 694 testcase_params['mode']]) 695 testcase_params['use_client_output'] = True 696 else: 697 testcase_params['iperf_args'] = wputils.get_iperf_arg_string( 698 duration=self.testclass_params['iperf_duration'], 699 reverse_direction=0, 700 traffic_type=testcase_params['traffic_type'], 701 socket_size=testcase_params['iperf_socket_size'], 702 num_processes=testcase_params['iperf_processes'], 703 udp_throughput=self.testclass_params['UDP_rates'][ 704 testcase_params['mode']]) 705 testcase_params['use_client_output'] = False 706 return testcase_params 707 708 def _test_rvr(self, testcase_params): 709 """ Function that gets called for each test case 710 711 Args: 712 testcase_params: dict containing test-specific parameters 713 """ 714 # Compile test parameters from config and test name 715 testcase_params = self.compile_test_params(testcase_params) 716 717 # Prepare devices and run test 718 self.setup_rvr_test(testcase_params) 719 rvr_result = self.run_rvr_test(testcase_params) 720 721 # Post-process results 722 self.testclass_results.append(rvr_result) 723 self.process_test_results(rvr_result) 724 self.pass_fail_check(rvr_result) 725 726 def generate_test_cases(self, channels, modes, traffic_types, 727 traffic_directions): 728 """Function that auto-generates test cases for a test class.""" 729 test_cases = [] 730 allowed_configs = { 731 20: [ 732 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 36, 40, 44, 48, 64, 100, 733 116, 132, 140, 149, 153, 157, 161, '6g37', '6g117', '6g213' 734 ], 735 40: [36, 44, 100, 149, 157, '6g37', '6g117', '6g213'], 736 80: [36, 100, 149, '6g37', '6g117', '6g213'], 737 160: [36, '6g37', '6g117', '6g213'] 738 } 739 740 for channel, mode, traffic_type, traffic_direction in itertools.product( 741 channels, modes, traffic_types, traffic_directions): 742 bandwidth = int(''.join([x for x in mode if x.isdigit()])) 743 if channel not in allowed_configs[bandwidth]: 744 continue 745 test_name = 'test_rvr_{}_{}_ch{}_{}'.format( 746 traffic_type, traffic_direction, channel, mode) 747 test_params = collections.OrderedDict( 748 channel=channel, 749 mode=mode, 750 bandwidth=bandwidth, 751 traffic_type=traffic_type, 752 traffic_direction=traffic_direction) 753 setattr(self, test_name, partial(self._test_rvr, test_params)) 754 test_cases.append(test_name) 755 return test_cases 756 757 758class WifiRvr_TCP_Test(WifiRvrTest): 759 760 def __init__(self, controllers): 761 super().__init__(controllers) 762 self.tests = self.generate_test_cases( 763 channels=[ 764 1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161, '6g37', '6g117', 765 '6g213' 766 ], 767 modes=['bw20', 'bw40', 'bw80', 'bw160'], 768 traffic_types=['TCP'], 769 traffic_directions=['DL', 'UL']) 770 771 772class WifiRvr_VHT_TCP_Test(WifiRvrTest): 773 774 def __init__(self, controllers): 775 super().__init__(controllers) 776 self.tests = self.generate_test_cases( 777 channels=[1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161], 778 modes=['VHT20', 'VHT40', 'VHT80'], 779 traffic_types=['TCP'], 780 traffic_directions=['DL', 'UL']) 781 782 783class WifiRvr_HE_TCP_Test(WifiRvrTest): 784 785 def __init__(self, controllers): 786 super().__init__(controllers) 787 self.tests = self.generate_test_cases( 788 channels=[ 789 1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161, '6g37', '6g117', 790 '6g213' 791 ], 792 modes=['HE20', 'HE40', 'HE80', 'HE160'], 793 traffic_types=['TCP'], 794 traffic_directions=['DL', 'UL']) 795 796 797class WifiRvr_SampleUDP_Test(WifiRvrTest): 798 799 def __init__(self, controllers): 800 super().__init__(controllers) 801 self.tests = self.generate_test_cases( 802 channels=[6, 36, 149, '6g37'], 803 modes=['bw20', 'bw40', 'bw80', 'bw160'], 804 traffic_types=['UDP'], 805 traffic_directions=['DL', 'UL']) 806 807 808class WifiRvr_VHT_SampleUDP_Test(WifiRvrTest): 809 810 def __init__(self, controllers): 811 super().__init__(controllers) 812 self.tests = self.generate_test_cases( 813 channels=[6, 36, 149], 814 modes=['VHT20', 'VHT40', 'VHT80', 'VHT160'], 815 traffic_types=['UDP'], 816 traffic_directions=['DL', 'UL']) 817 818 819class WifiRvr_HE_SampleUDP_Test(WifiRvrTest): 820 821 def __init__(self, controllers): 822 super().__init__(controllers) 823 self.tests = self.generate_test_cases( 824 channels=[6, 36, 149], 825 modes=['HE20', 'HE40', 'HE80', 'HE160', '6g37'], 826 traffic_types=['UDP'], 827 traffic_directions=['DL', 'UL']) 828 829 830class WifiRvr_SampleDFS_Test(WifiRvrTest): 831 832 def __init__(self, controllers): 833 super().__init__(controllers) 834 self.tests = self.generate_test_cases( 835 channels=[64, 100, 116, 132, 140], 836 modes=['bw20', 'bw40', 'bw80'], 837 traffic_types=['TCP'], 838 traffic_directions=['DL', 'UL']) 839 840 841class WifiRvr_SingleChain_TCP_Test(WifiRvrTest): 842 843 def __init__(self, controllers): 844 super().__init__(controllers) 845 self.tests = self.generate_test_cases( 846 channels=[ 847 1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161, '6g37', '6g117', 848 '6g213' 849 ], 850 modes=['bw20', 'bw40', 'bw80', 'bw160'], 851 traffic_types=['TCP'], 852 traffic_directions=['DL', 'UL'], 853 chains=[0, 1, '2x2']) 854 855 def setup_dut(self, testcase_params): 856 self.sta_dut = self.android_devices[0] 857 wputils.set_chain_mask(self.sta_dut, testcase_params['chain']) 858 WifiRvrTest.setup_dut(self, testcase_params) 859 860 def generate_test_cases(self, channels, modes, traffic_types, 861 traffic_directions, chains): 862 """Function that auto-generates test cases for a test class.""" 863 test_cases = [] 864 allowed_configs = { 865 20: [ 866 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 36, 40, 44, 48, 64, 100, 867 116, 132, 140, 149, 153, 157, 161, '6g37', '6g117', '6g213' 868 ], 869 40: [36, 44, 100, 149, 157, '6g37', '6g117', '6g213'], 870 80: [36, 100, 149, '6g37', '6g117', '6g213'], 871 160: [36, '6g37', '6g117', '6g213'] 872 } 873 874 for channel, mode, chain, traffic_type, traffic_direction in itertools.product( 875 channels, modes, chains, traffic_types, traffic_directions): 876 bandwidth = int(''.join([x for x in mode if x.isdigit()])) 877 if channel not in allowed_configs[bandwidth]: 878 continue 879 test_name = 'test_rvr_{}_{}_ch{}_{}_ch{}'.format( 880 traffic_type, traffic_direction, channel, mode, chain) 881 test_params = collections.OrderedDict( 882 channel=channel, 883 mode=mode, 884 bandwidth=bandwidth, 885 traffic_type=traffic_type, 886 traffic_direction=traffic_direction, 887 chain=chain) 888 setattr(self, test_name, partial(self._test_rvr, test_params)) 889 test_cases.append(test_name) 890 return test_cases 891 892 893# Over-the air version of RVR tests 894class WifiOtaRvrTest(WifiRvrTest): 895 """Class to test over-the-air RvR 896 897 This class implements measures WiFi RvR tests in an OTA chamber. It enables 898 setting turntable orientation and other chamber parameters to study 899 performance in varying channel conditions 900 """ 901 902 def __init__(self, controllers): 903 base_test.BaseTestClass.__init__(self, controllers) 904 self.testcase_metric_logger = ( 905 BlackboxMappedMetricLogger.for_test_case()) 906 self.testclass_metric_logger = ( 907 BlackboxMappedMetricLogger.for_test_class()) 908 self.publish_testcase_metrics = False 909 910 def setup_class(self): 911 WifiRvrTest.setup_class(self) 912 self.ota_chamber = ota_chamber.create( 913 self.user_params['OTAChamber'])[0] 914 915 def teardown_class(self): 916 WifiRvrTest.teardown_class(self) 917 self.ota_chamber.reset_chamber() 918 919 def extract_test_id(self, testcase_params, id_fields): 920 test_id = collections.OrderedDict( 921 (param, testcase_params.get(param, None)) for param in id_fields) 922 return test_id 923 924 def process_testclass_results(self): 925 """Saves plot with all test results to enable comparison.""" 926 # Plot individual test id results raw data and compile metrics 927 plots = collections.OrderedDict() 928 compiled_data = collections.OrderedDict() 929 for result in self.testclass_results: 930 test_id = tuple( 931 self.extract_test_id(result['testcase_params'], [ 932 'channel', 'mode', 'traffic_type', 'traffic_direction', 933 'chain' 934 ]).items()) 935 if test_id not in plots: 936 # Initialize test id data when not present 937 compiled_data[test_id] = { 938 'throughput': [], 939 'rx_phy_rate': [], 940 'tx_phy_rate': [], 941 'metrics': {} 942 } 943 compiled_data[test_id]['metrics'] = { 944 key: [] 945 for key in result['metrics'].keys() 946 } 947 plots[test_id] = BokehFigure( 948 title='Channel {} {} ({} {})'.format( 949 result['testcase_params']['channel'], 950 result['testcase_params']['mode'], 951 result['testcase_params']['traffic_type'], 952 result['testcase_params']['traffic_direction']), 953 x_label='Attenuation (dB)', 954 primary_y_label='Throughput (Mbps)') 955 test_id_phy = test_id + tuple('PHY') 956 plots[test_id_phy] = BokehFigure( 957 title='Channel {} {} ({} {}) (PHY Rate)'.format( 958 result['testcase_params']['channel'], 959 result['testcase_params']['mode'], 960 result['testcase_params']['traffic_type'], 961 result['testcase_params']['traffic_direction']), 962 x_label='Attenuation (dB)', 963 primary_y_label='PHY Rate (Mbps)') 964 # Compile test id data and metrics 965 compiled_data[test_id]['throughput'].append( 966 result['throughput_receive']) 967 compiled_data[test_id]['rx_phy_rate'].append(result['rx_phy_rate']) 968 compiled_data[test_id]['tx_phy_rate'].append(result['tx_phy_rate']) 969 compiled_data[test_id]['total_attenuation'] = result[ 970 'total_attenuation'] 971 for metric_key, metric_value in result['metrics'].items(): 972 compiled_data[test_id]['metrics'][metric_key].append( 973 metric_value) 974 # Add test id to plots 975 plots[test_id].add_line(result['total_attenuation'], 976 result['throughput_receive'], 977 result['test_name'].strip('test_rvr_'), 978 hover_text=result['hover_text'], 979 width=1, 980 style='dashed', 981 marker='circle') 982 plots[test_id_phy].add_line( 983 result['total_attenuation'], 984 result['rx_phy_rate'], 985 result['test_name'].strip('test_rvr_') + ' Rx PHY Rate', 986 hover_text=result['hover_text'], 987 width=1, 988 style='dashed', 989 marker='inverted_triangle') 990 plots[test_id_phy].add_line( 991 result['total_attenuation'], 992 result['tx_phy_rate'], 993 result['test_name'].strip('test_rvr_') + ' Tx PHY Rate', 994 hover_text=result['hover_text'], 995 width=1, 996 style='dashed', 997 marker='triangle') 998 999 # Compute average RvRs and compute metrics over orientations 1000 for test_id, test_data in compiled_data.items(): 1001 test_id_dict = dict(test_id) 1002 metric_tag = '{}_{}_ch{}_{}'.format( 1003 test_id_dict['traffic_type'], 1004 test_id_dict['traffic_direction'], test_id_dict['channel'], 1005 test_id_dict['mode']) 1006 high_tput_hit_freq = numpy.mean( 1007 numpy.not_equal(test_data['metrics']['high_tput_range'], -1)) 1008 self.testclass_metric_logger.add_metric( 1009 '{}.high_tput_hit_freq'.format(metric_tag), high_tput_hit_freq) 1010 for metric_key, metric_value in test_data['metrics'].items(): 1011 metric_key = '{}.avg_{}'.format(metric_tag, metric_key) 1012 metric_value = numpy.mean(metric_value) 1013 self.testclass_metric_logger.add_metric( 1014 metric_key, metric_value) 1015 test_data['avg_rvr'] = numpy.mean(test_data['throughput'], 0) 1016 test_data['median_rvr'] = numpy.median(test_data['throughput'], 0) 1017 test_data['avg_rx_phy_rate'] = numpy.mean(test_data['rx_phy_rate'], 1018 0) 1019 test_data['avg_tx_phy_rate'] = numpy.mean(test_data['tx_phy_rate'], 1020 0) 1021 plots[test_id].add_line(test_data['total_attenuation'], 1022 test_data['avg_rvr'], 1023 legend='Average Throughput', 1024 marker='circle') 1025 plots[test_id].add_line(test_data['total_attenuation'], 1026 test_data['median_rvr'], 1027 legend='Median Throughput', 1028 marker='square') 1029 test_id_phy = test_id + tuple('PHY') 1030 plots[test_id_phy].add_line(test_data['total_attenuation'], 1031 test_data['avg_rx_phy_rate'], 1032 legend='Average Rx Rate', 1033 marker='inverted_triangle') 1034 plots[test_id_phy].add_line(test_data['total_attenuation'], 1035 test_data['avg_tx_phy_rate'], 1036 legend='Average Tx Rate', 1037 marker='triangle') 1038 1039 figure_list = [] 1040 for plot_id, plot in plots.items(): 1041 plot.generate_figure() 1042 figure_list.append(plot) 1043 output_file_path = os.path.join(self.log_path, 'results.html') 1044 BokehFigure.save_figures(figure_list, output_file_path) 1045 1046 def setup_rvr_test(self, testcase_params): 1047 # Continue test setup 1048 WifiRvrTest.setup_rvr_test(self, testcase_params) 1049 # Set turntable orientation 1050 self.ota_chamber.set_orientation(testcase_params['orientation']) 1051 1052 def generate_test_cases(self, channels, modes, angles, traffic_types, 1053 directions): 1054 test_cases = [] 1055 allowed_configs = { 1056 20: [ 1057 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 36, 40, 44, 48, 64, 100, 1058 116, 132, 140, 149, 153, 157, 161 1059 ], 1060 40: [36, 44, 100, 149, 157], 1061 80: [36, 100, 149], 1062 160: [36, '6g37', '6g117', '6g213'] 1063 } 1064 for channel, mode, angle, traffic_type, direction in itertools.product( 1065 channels, modes, angles, traffic_types, directions): 1066 bandwidth = int(''.join([x for x in mode if x.isdigit()])) 1067 if channel not in allowed_configs[bandwidth]: 1068 continue 1069 testcase_name = 'test_rvr_{}_{}_ch{}_{}_{}deg'.format( 1070 traffic_type, direction, channel, mode, angle) 1071 test_params = collections.OrderedDict(channel=channel, 1072 mode=mode, 1073 bandwidth=bandwidth, 1074 traffic_type=traffic_type, 1075 traffic_direction=direction, 1076 orientation=angle) 1077 setattr(self, testcase_name, partial(self._test_rvr, test_params)) 1078 test_cases.append(testcase_name) 1079 return test_cases 1080 1081 1082class WifiOtaRvr_StandardOrientation_Test(WifiOtaRvrTest): 1083 1084 def __init__(self, controllers): 1085 WifiOtaRvrTest.__init__(self, controllers) 1086 self.tests = self.generate_test_cases( 1087 [1, 6, 11, 36, 40, 44, 48, 149, 153, 157, 161, '6g37'], 1088 ['bw20', 'bw40', 'bw80', 'bw160'], list(range(0, 360, 45)), 1089 ['TCP'], ['DL', 'UL']) 1090 1091 1092class WifiOtaRvr_SampleChannel_Test(WifiOtaRvrTest): 1093 1094 def __init__(self, controllers): 1095 WifiOtaRvrTest.__init__(self, controllers) 1096 self.tests = self.generate_test_cases([6], ['bw20'], 1097 list(range(0, 360, 45)), ['TCP'], 1098 ['DL']) 1099 self.tests.extend( 1100 self.generate_test_cases([36, 149], ['bw80', 'bw160'], 1101 list(range(0, 360, 45)), ['TCP'], ['DL'])) 1102 self.tests.extend( 1103 self.generate_test_cases(['6g37'], ['bw160'], 1104 list(range(0, 360, 45)), ['TCP'], ['DL'])) 1105 1106class WifiOtaRvr_SampleChannel_UDP_Test(WifiOtaRvrTest): 1107 1108 def __init__(self, controllers): 1109 WifiOtaRvrTest.__init__(self, controllers) 1110 self.tests = self.generate_test_cases([6], ['bw20'], 1111 list(range(0, 360, 45)), ['UDP'], 1112 ['DL', 'UL']) 1113 self.tests.extend( 1114 self.generate_test_cases([36, 149], ['bw80', 'bw160'], 1115 list(range(0, 360, 45)), ['UDP'], ['DL', 'UL'])) 1116 self.tests.extend( 1117 self.generate_test_cases(['6g37'], ['bw160'], 1118 list(range(0, 360, 45)), ['UDP'], ['DL', 'UL'])) 1119 1120class WifiOtaRvr_SingleOrientation_Test(WifiOtaRvrTest): 1121 1122 def __init__(self, controllers): 1123 WifiOtaRvrTest.__init__(self, controllers) 1124 self.tests = self.generate_test_cases( 1125 [6, 36, 40, 44, 48, 149, 153, 157, 161, '6g37'], 1126 ['bw20', 'bw40', 'bw80', 'bw160'], [0], ['TCP'], ['DL', 'UL']) 1127 1128 1129class WifiOtaRvr_SingleChain_Test(WifiOtaRvrTest): 1130 1131 def __init__(self, controllers): 1132 WifiOtaRvrTest.__init__(self, controllers) 1133 self.tests = self.generate_test_cases([6], ['bw20'], 1134 list(range(0, 360, 45)), ['TCP'], 1135 ['DL', 'UL'], [0, 1]) 1136 self.tests.extend( 1137 self.generate_test_cases([36, 149], ['bw20', 'bw80', 'bw160'], 1138 list(range(0, 360, 45)), ['TCP'], 1139 ['DL', 'UL'], [0, 1, '2x2'])) 1140 self.tests.extend( 1141 self.generate_test_cases(['6g37'], ['bw20', 'bw80', 'bw160'], 1142 list(range(0, 360, 45)), ['TCP'], 1143 ['DL', 'UL'], [0, 1, '2x2'])) 1144 1145 def setup_dut(self, testcase_params): 1146 self.sta_dut = self.android_devices[0] 1147 wputils.set_chain_mask(self.sta_dut, testcase_params['chain']) 1148 WifiRvrTest.setup_dut(self, testcase_params) 1149 1150 def generate_test_cases(self, channels, modes, angles, traffic_types, 1151 directions, chains): 1152 test_cases = [] 1153 allowed_configs = { 1154 20: [ 1155 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 36, 40, 44, 48, 64, 100, 1156 116, 132, 140, 149, 153, 157, 161 1157 ], 1158 40: [36, 44, 100, 149, 157], 1159 80: [36, 100, 149], 1160 160: [36, '6g37', '6g117', '6g213'] 1161 } 1162 for channel, mode, chain, angle, traffic_type, direction in itertools.product( 1163 channels, modes, chains, angles, traffic_types, directions): 1164 bandwidth = int(''.join([x for x in mode if x.isdigit()])) 1165 if channel not in allowed_configs[bandwidth]: 1166 continue 1167 testcase_name = 'test_rvr_{}_{}_ch{}_{}_ch{}_{}deg'.format( 1168 traffic_type, direction, channel, mode, chain, angle) 1169 test_params = collections.OrderedDict(channel=channel, 1170 mode=mode, 1171 bandwidth=bandwidth, 1172 chain=chain, 1173 traffic_type=traffic_type, 1174 traffic_direction=direction, 1175 orientation=angle) 1176 setattr(self, testcase_name, partial(self._test_rvr, test_params)) 1177 test_cases.append(testcase_name) 1178 return test_cases 1179