1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.nn.crashtest.app;
18 
19 import android.content.Intent;
20 import android.os.RemoteException;
21 import android.test.ActivityInstrumentationTestCase2;
22 import android.test.UiThreadTest;
23 
24 import androidx.test.InstrumentationRegistry;
25 import androidx.test.filters.LargeTest;
26 
27 import com.android.nn.benchmark.app.AcceleratorSpecificTestSupport;
28 import com.android.nn.benchmark.core.NNTestBase;
29 import com.android.nn.benchmark.core.NnApiDelegationFailure;
30 import com.android.nn.benchmark.core.TestModels;
31 
32 import org.junit.Before;
33 import org.junit.Rule;
34 import org.junit.Test;
35 import org.junit.rules.TestName;
36 import org.junit.runner.RunWith;
37 import org.junit.runners.Parameterized;
38 import org.junit.runners.Parameterized.Parameters;
39 
40 import java.time.Duration;
41 import java.util.Optional;
42 import java.util.concurrent.ExecutionException;
43 import java.util.concurrent.ExecutorService;
44 import java.util.concurrent.Executors;
45 import java.util.concurrent.Future;
46 import java.util.stream.IntStream;
47 
48 @RunWith(Parameterized.class)
49 public class NNClientEarlyTerminationTest extends
50         ActivityInstrumentationTestCase2<NNParallelTestActivity> implements
51     AcceleratorSpecificTestSupport {
52 
53     private static final String TAG = "NNClientEarlyTermination";
54     private static final Duration MAX_SEPARATE_PROCESS_EXECUTION_TIME = Duration.ofSeconds(70);
55     public static final int NNAPI_CLIENTS_COUNT = 4;
56 
57     private final ExecutorService mDriverLivenessValidationExecutor =
58             Executors.newSingleThreadExecutor();
59     private final String mAcceleratorName;
60 
61     @Rule
62     public TestName mTestName = new TestName();
63 
NNClientEarlyTerminationTest(String acceleratorName)64     public NNClientEarlyTerminationTest(String acceleratorName) {
65         super(NNParallelTestActivity.class);
66         mAcceleratorName = acceleratorName;
67     }
68 
69     @Parameters(name = "Accelerator({0})")
targetAccelerators()70     public static Iterable<String> targetAccelerators() {
71         return AcceleratorSpecificTestSupport.getTargetAcceleratorNames();
72     }
73 
74     @Before
75     @Override
setUp()76     public void setUp() {
77         injectInstrumentation(InstrumentationRegistry.getInstrumentation());
78         try {
79             final Intent runSomeInferencesInASeparateProcess = compileSupportedModelsOnNThreadsFor(
80                     NNAPI_CLIENTS_COUNT, MAX_SEPARATE_PROCESS_EXECUTION_TIME);
81             setActivityIntent(runSomeInferencesInASeparateProcess);
82         } catch (NnApiDelegationFailure nnApiDelegationFailure) {
83             throw new RuntimeException(
84                     "Cannot initialize test, failure looking for supported models, please check "
85                             + "the driver status",
86                     nnApiDelegationFailure);
87         }
88     }
89 
90     @Test
91     @LargeTest
92     @UiThreadTest
testDriverDoesNotFailWithParallelThreads()93     public void testDriverDoesNotFailWithParallelThreads()
94             throws ExecutionException, InterruptedException, RemoteException,
95             NnApiDelegationFailure {
96         final NNParallelTestActivity activity = getActivity();
97 
98         Optional<TestModels.TestModelEntry> modelForLivenessTest =
99                 AcceleratorSpecificTestSupport.findTestModelRunningOnAccelerator(activity, mAcceleratorName);
100         assertTrue("No model available to be run on accelerator " + mAcceleratorName,
101                 modelForLivenessTest.isPresent());
102 
103         final DriverLivenessChecker driverLivenessChecker = new DriverLivenessChecker(activity,
104                 mAcceleratorName, modelForLivenessTest.get());
105         Future<Boolean> driverDidNotCrash = mDriverLivenessValidationExecutor.submit(
106                 driverLivenessChecker);
107 
108         // Causing failure before tests would end on their own.
109         final long maxTerminationTime = MAX_SEPARATE_PROCESS_EXECUTION_TIME.toMillis() / 2;
110         final long minTerminationTime = MAX_SEPARATE_PROCESS_EXECUTION_TIME.toMillis() / 4;
111         Thread.sleep(ramdomInRange(minTerminationTime, maxTerminationTime));
112 
113         try {
114             activity.killTestProcess();
115         } catch (RemoteException e) {
116             driverLivenessChecker.stop();
117             throw e;
118         }
119 
120         CrashTestStatus.TestResult testResult = activity.testResult();
121         driverLivenessChecker.stop();
122 
123         assertEquals("Remote process is expected to be killed",
124                 CrashTestStatus.TestResult.CRASH,
125                 testResult);
126 
127         assertTrue("Driver shouldn't crash if a client process is terminated",
128                 driverDidNotCrash.get());
129     }
130 
compileSupportedModelsOnNThreadsFor(int threadCount, Duration testDuration)131     private Intent compileSupportedModelsOnNThreadsFor(int threadCount, Duration testDuration)
132         throws NnApiDelegationFailure {
133         Intent intent = new Intent();
134         intent.putExtra(
135             NNParallelTestActivity.EXTRA_TEST_LIST, IntStream.range(0,
136                 TestModels.modelsList().size()).toArray());
137         intent.putExtra(NNParallelTestActivity.EXTRA_THREAD_COUNT, threadCount);
138         intent.putExtra(NNParallelTestActivity.EXTRA_TEST_DURATION_MILLIS, testDuration.toMillis());
139         intent.putExtra(NNParallelTestActivity.EXTRA_RUN_IN_SEPARATE_PROCESS, true);
140         intent.putExtra(NNParallelTestActivity.EXTRA_TEST_NAME, mTestName.getMethodName());
141         intent.putExtra(NNParallelTestActivity.EXTRA_ACCELERATOR_NAME, mAcceleratorName);
142         intent.putExtra(NNParallelTestActivity.EXTRA_IGNORE_UNSUPPORTED_MODELS, true);
143         intent.putExtra(NNParallelTestActivity.EXTRA_RUN_MODEL_COMPILATION_ONLY, true);
144         intent.putExtra(NNParallelTestActivity.EXTRA_USE_NNAPI_SL,
145             NNTestBase.shouldUseNnApiSupportLibrary());
146         intent.putExtra(NNParallelTestActivity.EXTRA_EXTRACT_NNAPI_SL,
147             NNTestBase.shouldExtractNnApiSupportLibrary());
148         return intent;
149     }
150 }
151