1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.nn.crashtest.app;
18 
19 import static com.android.nn.crashtest.app.CrashTestStatus.TestResult.SUCCESS;
20 
21 import android.content.Intent;
22 import android.test.ActivityInstrumentationTestCase2;
23 import android.test.UiThreadTest;
24 
25 import androidx.test.InstrumentationRegistry;
26 import androidx.test.filters.LargeTest;
27 
28 import com.android.nn.benchmark.app.AcceleratorSpecificTestSupport;
29 import com.android.nn.benchmark.app.BenchmarkTestBase;
30 import com.android.nn.benchmark.core.NnApiDelegationFailure;
31 import com.android.nn.benchmark.core.TestModels;
32 import com.android.nn.crashtest.core.test.RandomGraphTest;
33 
34 import org.junit.Before;
35 import org.junit.Rule;
36 import org.junit.Test;
37 import org.junit.rules.TestName;
38 import org.junit.runner.RunWith;
39 import org.junit.runners.Parameterized;
40 import org.junit.runners.Parameterized.Parameters;
41 
42 import java.time.Duration;
43 import java.util.Arrays;
44 import java.util.Optional;
45 import java.util.concurrent.ExecutionException;
46 
47 @RunWith(Parameterized.class)
48 public abstract class NNRandomGraphTest
49         extends ActivityInstrumentationTestCase2<NNRandomGraphTestActivity>
50         implements AcceleratorSpecificTestSupport {
51     private static final String TAG = "NN_RAND_MODEL";
52 
53     protected final String mAcceleratorName;
54     private final int mModelCount;
55     private final int mGraphSize;
56     private final Duration mDuration;
57     private final int mDimensionRange;
58 
59     private final static int SMALL_MODEL_SIZE = 10;
60     private final static int LARGE_MODEL_SIZE = 600;
61     private final static int NARROW_DIMENSIONS_RANGE = 10;
62     private final static int WIDE_DIMENSIONS_RANGE = 1000;
63     private final static Duration MAX_TEST_DURATION = Duration.ofMinutes(15);
64     private final static int NUMBER_OF_MODELS = 600;
65     private final boolean mRunModelCompilationOnly;
66 
67     @Parameters(name = "{0} models of size {1} and dimensions range {2} for max duration of {3} "
68             + "on accelerator {4}")
testConfiguration()69     public static Iterable<Object[]> testConfiguration() {
70         return AcceleratorSpecificTestSupport.perAcceleratorTestConfig(
71                 Arrays.asList(
72                         new Object[]{NUMBER_OF_MODELS, SMALL_MODEL_SIZE, WIDE_DIMENSIONS_RANGE,
73                                 MAX_TEST_DURATION},
74                         new Object[]{NUMBER_OF_MODELS, LARGE_MODEL_SIZE, NARROW_DIMENSIONS_RANGE,
75                                 MAX_TEST_DURATION}));
76     }
77 
78     @Rule
79     public TestName mTestName = new TestName();
80 
NNRandomGraphTest(int modelCount, int graphSize, int dimensionRange, Duration duration, String acceleratorName, boolean runModelCompilationOnly)81     public NNRandomGraphTest(int modelCount, int graphSize, int dimensionRange,
82             Duration duration, String acceleratorName, boolean runModelCompilationOnly) {
83         super(NNRandomGraphTestActivity.class);
84         mModelCount = modelCount;
85         mGraphSize = graphSize;
86         mDuration = duration;
87         mAcceleratorName = acceleratorName;
88         mDimensionRange = dimensionRange;
89         mRunModelCompilationOnly = runModelCompilationOnly;
90     }
91 
92     @Before
93     @Override
setUp()94     public void setUp() {
95         injectInstrumentation(InstrumentationRegistry.getInstrumentation());
96         BenchmarkTestBase.waitUntilCharged(getInstrumentation().getTargetContext(), 60);
97         setActivityIntent(getTestModelsOfSizeAndRangeForMaxTimeIntent(mGraphSize, mDimensionRange,
98                 mModelCount, mAcceleratorName, mDuration, mTestName.getMethodName()));
99     }
100 
findModelForLivenessTest()101     protected Optional<TestModels.TestModelEntry> findModelForLivenessTest()
102             throws NnApiDelegationFailure {
103         return AcceleratorSpecificTestSupport.findTestModelRunningOnAccelerator(
104                 getInstrumentation().getTargetContext(), mAcceleratorName);
105     }
106 
107     @Test
108     @LargeTest
109     @UiThreadTest
testDriverDoesNotFailWithParallelWorkload()110     public void testDriverDoesNotFailWithParallelWorkload()
111             throws ExecutionException, InterruptedException, NnApiDelegationFailure {
112         final NNRandomGraphTestActivity activity = getActivity();
113 
114         assertEquals(SUCCESS, activity.testResult());
115     }
116 
117     /**
118      * @return the intent to use to initialise the RandomGraphTest test class
119      */
getTestModelsOfSizeAndRangeForMaxTimeIntent(int graphSize, int dimensionsRange, int modelsCount, String deviceName, Duration maxTestDuration, String testName)120     protected Intent getTestModelsOfSizeAndRangeForMaxTimeIntent(int graphSize, int dimensionsRange,
121             int modelsCount, String deviceName, Duration maxTestDuration, String testName) {
122         Intent result = new Intent();
123         RandomGraphTest
124                 .intentInitializer(graphSize, dimensionsRange, modelsCount,
125                         RandomGraphTest.DEFAULT_PAUSE_BETWEEN_MODELS_MILLIS,
126                         mRunModelCompilationOnly, deviceName, maxTestDuration.toMillis(), testName)
127                 .addIntentParams(result);
128         return result;
129     }
130 }