1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.nn.crashtest.app;
18 
19 import android.app.Activity;
20 import android.content.Intent;
21 import android.os.Bundle;
22 import android.test.ActivityInstrumentationTestCase2;
23 import android.test.UiThreadTest;
24 import android.util.Log;
25 
26 import androidx.test.InstrumentationRegistry;
27 import androidx.test.filters.LargeTest;
28 
29 import com.android.nn.benchmark.app.AcceleratorSpecificTestSupport;
30 import com.android.nn.benchmark.app.BenchmarkTestBase;
31 import com.android.nn.benchmark.core.NNTestBase;
32 import com.android.nn.benchmark.core.NnApiDelegationFailure;
33 import com.android.nn.benchmark.core.TestModels;
34 
35 import org.junit.After;
36 import org.junit.Before;
37 import org.junit.Rule;
38 import org.junit.Test;
39 import org.junit.rules.TestName;
40 import org.junit.runners.Parameterized.Parameters;
41 
42 import java.time.Duration;
43 import java.util.Arrays;
44 import java.util.stream.IntStream;
45 
46 abstract class NNParallelInferenceTest
47         extends ActivityInstrumentationTestCase2<NNParallelTestActivity>
48         implements AcceleratorSpecificTestSupport {
49     static final String TAG = "NNParallelInferenceTest";
50 
51     @Rule
52     public TestName mTestName = new TestName();
53 
54     private final int mThreadCount;
55     private final Duration mTestDuration;
56     private final String mAcceleratorName;
57 
runTestsInSeparateProcess()58     protected abstract boolean runTestsInSeparateProcess();
59 
NNParallelInferenceTest(int threadCount, Duration testDuration, String acceleratorName)60     protected NNParallelInferenceTest(int threadCount, Duration testDuration,
61             String acceleratorName) {
62         super(NNParallelTestActivity.class);
63         mThreadCount = threadCount;
64         mTestDuration = testDuration;
65         mAcceleratorName = acceleratorName;
66     }
67 
68     @Before
69     @Override
setUp()70     public void setUp() {
71         injectInstrumentation(InstrumentationRegistry.getInstrumentation());
72         BenchmarkTestBase.waitUntilCharged(getInstrumentation().getTargetContext(), 90);
73         try {
74             setActivityIntent(
75                     runAllModelsOnNThreadsForOnAccelerator(mThreadCount, mTestDuration,
76                             mAcceleratorName));
77         } catch (NnApiDelegationFailure nnApiDelegationFailure) {
78             throw new RuntimeException(
79                     "Cannot initialize test, failure looking for supported models, please check "
80                             + "the driver status",
81                     nnApiDelegationFailure);
82         }
83     }
84 
85     @Test
86     @LargeTest
87     @UiThreadTest
shouldNotFailWithParallelThreads()88     public void shouldNotFailWithParallelThreads() {
89         Bundle testData = new Bundle();
90         testData.putString("Test name", mTestName.getMethodName());
91         testData.putString("Test status", "Started");
92         getInstrumentation().sendStatus(Activity.RESULT_FIRST_USER, testData);
93 
94         CrashTestStatus.TestResult testResult = getActivity().testResult();
95         assertEquals("Test didn't complete successfully", CrashTestStatus.TestResult.SUCCESS,
96                 testResult);
97 
98         testData.putString("Test status", "Completed");
99         getInstrumentation().sendStatus(Activity.RESULT_OK, testData);
100     }
101 
102     @After
103     @Override
tearDown()104     public void tearDown() throws Exception {
105         Log.i(TAG, "Tearing down test");
106         super.tearDown();
107     }
108 
109     @Parameters(name = "{0} threads for {1} on accelerator {2}")
threadCountValues()110     public static Iterable<Object[]> threadCountValues() {
111         return AcceleratorSpecificTestSupport.perAcceleratorTestConfig(
112                 Arrays.asList(
113                         new Object[]{8, Duration.ofMinutes(30)},
114                         new Object[]{12, Duration.ofMinutes(20)}));
115     }
116 
runAllModelsOnNThreadsForOnAccelerator(int threadCount, Duration testDuration, String acceleratorName)117     private Intent runAllModelsOnNThreadsForOnAccelerator(int threadCount, Duration testDuration,
118             String acceleratorName) throws NnApiDelegationFailure {
119         Intent intent = new Intent();
120 
121         int modelsCount = TestModels.modelsList().size();
122         intent.putExtra(
123                 NNParallelTestActivity.EXTRA_TEST_LIST, IntStream.range(0, modelsCount).toArray());
124         intent.putExtra(NNParallelTestActivity.EXTRA_THREAD_COUNT, threadCount);
125         intent.putExtra(NNParallelTestActivity.EXTRA_TEST_DURATION_MILLIS, testDuration.toMillis());
126         intent.putExtra(NNParallelTestActivity.EXTRA_RUN_IN_SEPARATE_PROCESS,
127                 runTestsInSeparateProcess());
128         intent.putExtra(NNParallelTestActivity.EXTRA_TEST_NAME, mTestName.getMethodName());
129         if (acceleratorName != null) {
130             intent.putExtra(NNParallelTestActivity.EXTRA_ACCELERATOR_NAME, acceleratorName);
131             intent.putExtra(NNParallelTestActivity.EXTRA_IGNORE_UNSUPPORTED_MODELS, true);
132         }
133         intent.putExtra(NNParallelTestActivity.EXTRA_USE_NNAPI_SL,
134             NNTestBase.shouldUseNnApiSupportLibrary());
135         intent.putExtra(NNParallelTestActivity.EXTRA_EXTRACT_NNAPI_SL,
136             NNTestBase.shouldExtractNnApiSupportLibrary());
137         return intent;
138     }
139 }
140