1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License
15  */
16 
17 package android.keystore.cts.performance;
18 
19 import android.keystore.cts.util.TestUtils;
20 import android.os.Build;
21 import android.os.SystemClock;
22 import android.security.keystore.KeyGenParameterSpec;
23 
24 import androidx.test.platform.app.InstrumentationRegistry;
25 import androidx.test.runner.AndroidJUnit4;
26 
27 import com.android.compatibility.common.util.DeviceReportLog;
28 import com.android.compatibility.common.util.ResultType;
29 import com.android.compatibility.common.util.ResultUnit;
30 
31 import java.security.KeyPair;
32 import java.security.KeyPairGenerator;
33 import java.security.KeyStore;
34 import java.security.cert.Certificate;
35 import java.util.ArrayList;
36 
37 import javax.crypto.KeyGenerator;
38 import javax.crypto.SecretKey;
39 
40 import org.junit.Test;
41 import org.junit.runner.RunWith;
42 
43 public class PerformanceTestBase {
44 
45     public static final long MS_PER_NS = 1000000L;
46     protected static final String TAG = "KeystorePerformanceTest";
47     private static final String REPORT_LOG_NAME = "CtsKeystorePerformanceTests";
48     /**
49      * Number of milliseconds to spend repeating a single test.
50      *
51      * <p>For each algorithm we run the test repeatedly up to a maximum of this time limit (or up to
52      * {@link #TEST_ITERATION_LIMIT} whichever is reached first), then report back the number of
53      * repetitions. We don't abort a test at the time limit but let it run to completion, so we're
54      * guaranteed to always get at least one repetition, even if it takes longer than the limit.
55      */
56     private static int TEST_TIME_LIMIT = 20000;
57 
58     /** Maximum number of iterations to run a single test. */
59     static int TEST_ITERATION_LIMIT = 20;
60 
measure(Measurable... measurables)61     protected void measure(Measurable... measurables) throws Exception {
62         ArrayList<PerformanceTestResult> results = new ArrayList<>();
63         results.ensureCapacity(measurables.length);
64         for (Measurable measurable : measurables) {
65             DeviceReportLog reportLog =
66                     new DeviceReportLog(
67                             REPORT_LOG_NAME, "performance_test", TestUtils.getFilesDir());
68             PerformanceTestResult result = measure(measurable);
69 
70             reportLog.addValue(
71                     "test_environment",
72                     measurable.getEnvironment(),
73                     ResultType.NEUTRAL,
74                     ResultUnit.NONE);
75             reportLog.addValue(
76                     "test_name", measurable.getName(), ResultType.NEUTRAL, ResultUnit.NONE);
77             reportLog.addValue(
78                     "sample_count", result.getSampleCount(), ResultType.NEUTRAL, ResultUnit.COUNT);
79             reportLog.addValue(
80                     "setup_time", result.getSetupTime(), ResultType.LOWER_BETTER, ResultUnit.MS);
81             reportLog.addValue(
82                     "mean_time", result.getMean(), ResultType.LOWER_BETTER, ResultUnit.MS);
83             reportLog.addValue(
84                     "sample_std_dev",
85                     result.getSampleStdDev(),
86                     ResultType.LOWER_BETTER,
87                     ResultUnit.MS);
88             reportLog.addValue(
89                     "median_time", result.getMedian(), ResultType.LOWER_BETTER, ResultUnit.MS);
90             reportLog.addValue(
91                     "percentile_90_time",
92                     result.getPercentile(0.9),
93                     ResultType.LOWER_BETTER,
94                     ResultUnit.MS);
95             reportLog.addValue(
96                     "teardown_time",
97                     result.getTearDownTime(),
98                     ResultType.LOWER_BETTER,
99                     ResultUnit.MS);
100             reportLog.addValue(
101                     "teardown_time",
102                     result.getTearDownTime(),
103                     ResultType.LOWER_BETTER,
104                     ResultUnit.MS);
105             reportLog.submit(InstrumentationRegistry.getInstrumentation());
106         }
107     }
108 
measure(Measurable measurable)109     private PerformanceTestResult measure(Measurable measurable) throws Exception {
110         // One un-measured time through everything, to warm caches, etc.
111 
112         PerformanceTestResult result = new PerformanceTestResult();
113 
114         measurable.initialSetUp();
115         measurable.setUp();
116         measurable.measure();
117         measurable.tearDown();
118 
119         long runLimit = now() + TEST_TIME_LIMIT * MS_PER_NS;
120         while (now() < runLimit && result.getSampleCount() < TEST_ITERATION_LIMIT) {
121             long setupBegin = now();
122             measurable.setUp();
123             result.addSetupTime(now() - setupBegin);
124 
125             long runBegin = now();
126             measurable.measure();
127             result.addMeasurement(now() - runBegin);
128 
129             long tearDownBegin = now();
130             measurable.tearDown();
131             result.addTeardownTime(now() - tearDownBegin);
132         }
133         measurable.finalTearDown();
134 
135         return result;
136     }
137 
now()138     protected long now() {
139         return SystemClock.elapsedRealtimeNanos();
140     }
141 
142     public abstract class Measurable {
143 
Measurable()144         private Measurable() {}
145         ;
146 
getEnvironment()147         public abstract String getEnvironment();
148 
getName()149         public abstract String getName();
150 
initialSetUp()151         public void initialSetUp() throws Exception {}
152 
setUp()153         public void setUp() throws Exception {}
154 
measure()155         public abstract void measure() throws Exception;
156 
tearDown()157         public void tearDown() throws Exception {}
158 
finalTearDown()159         public void finalTearDown() throws Exception {}
160     }
161 
162     /** Base class for measuring Keystore operations. */
163     abstract class KeystoreMeasurable extends Measurable {
164         private final String mName;
165         private final byte[] mMessage;
166         private final KeystoreKeyGenerator mGenerator;
167 
KeystoreMeasurable( KeystoreKeyGenerator generator, String operation, int keySize, int messageSize)168         KeystoreMeasurable(
169                 KeystoreKeyGenerator generator, String operation, int keySize, int messageSize)
170                 throws Exception {
171             super();
172             mGenerator = generator;
173             if (messageSize < 0) {
174                 mName = (operation
175                                 + "/" + getAlgorithm()
176                                 + "/" + keySize);
177                 mMessage = null;
178             } else {
179                 mName = (operation
180                                 + "/" + getAlgorithm()
181                                 + "/" + keySize
182                                 + "/" + messageSize);
183                 mMessage = TestUtils.generateRandomMessage(messageSize);
184             }
185         }
186 
KeystoreMeasurable(KeystoreKeyGenerator generator, String operation, int keySize)187         KeystoreMeasurable(KeystoreKeyGenerator generator, String operation, int keySize)
188                 throws Exception {
189             this(generator, operation, keySize, -1);
190         }
191 
192         @Override
getEnvironment()193         public String getEnvironment() {
194             return mGenerator.getProvider() + "/" + Build.CPU_ABI;
195         }
196 
197         @Override
getName()198         public String getName() {
199             return mName;
200         }
201 
getMessage()202         byte[] getMessage() {
203             return mMessage;
204         }
205 
getAlgorithm()206         String getAlgorithm() {
207             return mGenerator.getAlgorithm();
208         }
209 
210         @Override
finalTearDown()211         public void finalTearDown() throws Exception {
212             deleteKey();
213         }
214 
deleteKey()215         public void deleteKey() throws Exception {
216             mGenerator.deleteKey();
217         }
218 
generateSecretKey()219         SecretKey generateSecretKey() throws Exception {
220             return mGenerator.getSecretKeyGenerator().generateKey();
221         }
222 
generateKeyPair()223         KeyPair generateKeyPair() throws Exception {
224             return mGenerator.getKeyPairGenerator().generateKeyPair();
225         }
226     }
227 
228     /**
229      * Measurable for generating key pairs.
230      *
231      * <p>This class is ignostic to the Keystore provider or key algorithm.
232      */
233     class KeystoreKeyPairGenMeasurable extends KeystoreMeasurable {
234 
KeystoreKeyPairGenMeasurable(KeystoreKeyGenerator keyGenerator, int keySize)235         KeystoreKeyPairGenMeasurable(KeystoreKeyGenerator keyGenerator, int keySize)
236                 throws Exception {
237             super(keyGenerator, "keygen", keySize);
238         }
239 
240         @Override
measure()241         public void measure() throws Exception {
242             generateKeyPair();
243         }
244 
245         @Override
tearDown()246         public void tearDown() throws Exception {
247             deleteKey();
248         }
249     }
250 
251     /**
252      * Measurable for generating a secret key.
253      *
254      * <p>This class is ignostic to the Keystore provider or key algorithm.
255      */
256     class KeystoreSecretKeyGenMeasurable extends KeystoreMeasurable {
257 
KeystoreSecretKeyGenMeasurable(KeystoreKeyGenerator keyGenerator, int keySize)258         KeystoreSecretKeyGenMeasurable(KeystoreKeyGenerator keyGenerator, int keySize)
259                 throws Exception {
260             super(keyGenerator, "keygen", keySize);
261         }
262 
263         @Override
measure()264         public void measure() throws Exception {
265             generateSecretKey();
266         }
267 
268         @Override
tearDown()269         public void tearDown() throws Exception {
270             deleteKey();
271         }
272     }
273 
274     /**
275      * Wrapper for generating Keystore keys.
276      *
277      * <p>Abstracts the provider and initilization of the generator.
278      */
279     abstract class KeystoreKeyGenerator {
280         private final String mAlgorithm;
281         private final String mProvider;
282         private KeyGenerator mSecretKeyGenerator = null;
283         private KeyPairGenerator mKeyPairGenerator = null;
284 
KeystoreKeyGenerator(String algorithm, String provider)285         KeystoreKeyGenerator(String algorithm, String provider) throws Exception {
286             mAlgorithm = algorithm;
287             mProvider = provider;
288         }
289 
KeystoreKeyGenerator(String algorithm)290         KeystoreKeyGenerator(String algorithm) throws Exception {
291             // This is a hack to get the default provider.
292             this(algorithm, KeyGenerator.getInstance("AES").getProvider().getName());
293         }
294 
getAlgorithm()295         String getAlgorithm() {
296             return mAlgorithm;
297         }
298 
getProvider()299         String getProvider() {
300             return mProvider;
301         }
302 
303         /** By default, deleteKey is a nop */
deleteKey()304         void deleteKey() throws Exception {}
305 
getSecretKeyGenerator()306         KeyGenerator getSecretKeyGenerator() throws Exception {
307             if (mSecretKeyGenerator == null) {
308                 mSecretKeyGenerator =
309                         KeyGenerator.getInstance(TestUtils.getKeyAlgorithm(mAlgorithm), mProvider);
310             }
311             return mSecretKeyGenerator;
312         }
313 
getKeyPairGenerator()314         KeyPairGenerator getKeyPairGenerator() throws Exception {
315             if (mKeyPairGenerator == null) {
316                 mKeyPairGenerator =
317                         KeyPairGenerator.getInstance(
318                                 TestUtils.getKeyAlgorithm(mAlgorithm), mProvider);
319             }
320             return mKeyPairGenerator;
321         }
322     }
323 
324     /**
325      * Wrapper for generating Android Keystore keys.
326      *
327      * <p>Provides Android Keystore specific functionality, like deleting the key.
328      */
329     abstract class AndroidKeystoreKeyGenerator extends KeystoreKeyGenerator {
330         private final KeyStore mKeyStore;
331         private final String KEY_ALIAS = "perf_key";
332 
AndroidKeystoreKeyGenerator(String algorithm)333         AndroidKeystoreKeyGenerator(String algorithm) throws Exception {
334             super(algorithm, TestUtils.EXPECTED_PROVIDER_NAME);
335             mKeyStore = KeyStore.getInstance(getProvider());
336             mKeyStore.load(null);
337         }
338 
339         @Override
deleteKey()340         void deleteKey() throws Exception {
341             mKeyStore.deleteEntry(KEY_ALIAS);
342         }
343 
getKeyGenParameterSpecBuilder(int purpose)344         KeyGenParameterSpec.Builder getKeyGenParameterSpecBuilder(int purpose) {
345             return new KeyGenParameterSpec.Builder(KEY_ALIAS, purpose);
346         }
347 
getCertificateChain()348         Certificate[] getCertificateChain() throws Exception {
349             return mKeyStore.getCertificateChain(KEY_ALIAS);
350         }
351     }
352 
353     /** Basic generator for KeyPairs that uses the default provider. */
354     class DefaultKeystoreKeyPairGenerator extends KeystoreKeyGenerator {
DefaultKeystoreKeyPairGenerator(String algorithm, int keySize)355         DefaultKeystoreKeyPairGenerator(String algorithm, int keySize) throws Exception {
356             super(algorithm);
357             getKeyPairGenerator().initialize(keySize);
358         }
359     }
360 
361     /** Basic generator for SecretKeys that uses the default provider. */
362     class DefaultKeystoreSecretKeyGenerator extends KeystoreKeyGenerator {
DefaultKeystoreSecretKeyGenerator(String algorithm, int keySize)363         DefaultKeystoreSecretKeyGenerator(String algorithm, int keySize) throws Exception {
364             super(algorithm);
365             getSecretKeyGenerator().init(keySize);
366         }
367     }
368 }
369