1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.tradefed.config;
18 
19 import com.android.tradefed.log.LogUtil.CLog;
20 import com.android.tradefed.util.FileUtil;
21 import com.android.tradefed.util.MultiMap;
22 
23 import org.kxml2.io.KXmlSerializer;
24 
25 import java.io.File;
26 import java.io.IOException;
27 import java.io.PrintWriter;
28 import java.lang.reflect.Field;
29 import java.lang.reflect.InvocationTargetException;
30 import java.util.ArrayList;
31 import java.util.Collection;
32 import java.util.HashSet;
33 import java.util.LinkedHashMap;
34 import java.util.LinkedHashSet;
35 import java.util.LinkedList;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Map.Entry;
39 import java.util.Set;
40 
41 /** Utility functions to handle configuration files. */
42 public class ConfigurationUtil {
43 
44     // Element names used for emitting the configuration XML.
45     public static final String CONFIGURATION_NAME = "configuration";
46     public static final String OPTION_NAME = "option";
47     public static final String CLASS_NAME = "class";
48     public static final String NAME_NAME = "name";
49     public static final String KEY_NAME = "key";
50     public static final String VALUE_NAME = "value";
51 
52     /**
53      * Create a serializer to be used to create a new configuration file.
54      *
55      * @param outputXml the XML file to write to
56      * @return a {@link KXmlSerializer}
57      */
createSerializer(File outputXml)58     static KXmlSerializer createSerializer(File outputXml) throws IOException {
59         PrintWriter output = new PrintWriter(outputXml);
60         KXmlSerializer serializer = new KXmlSerializer();
61         serializer.setOutput(output);
62         serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true);
63         serializer.startDocument("UTF-8", null);
64         return serializer;
65     }
66 
67     /**
68      * Add a class to the configuration XML dump.
69      *
70      * @param serializer a {@link KXmlSerializer} to create the XML dump
71      * @param classTypeName a {@link String} of the class type's name
72      * @param obj {@link Object} to be added to the XML dump
73      * @param excludeClassFilter list of object configuration type or fully qualified class names to
74      *     be excluded from the dump. for example: {@link Configuration#TARGET_PREPARER_TYPE_NAME}.
75      *     com.android.tradefed.testtype.StubTest
76      * @param printDeprecatedOptions whether or not to print deprecated options
77      * @param printUnchangedOptions whether or not to print options that haven't been changed
78      */
dumpClassToXml( KXmlSerializer serializer, String classTypeName, Object obj, List<String> excludeClassFilter, boolean printDeprecatedOptions, boolean printUnchangedOptions)79     static void dumpClassToXml(
80             KXmlSerializer serializer,
81             String classTypeName,
82             Object obj,
83             List<String> excludeClassFilter,
84             boolean printDeprecatedOptions,
85             boolean printUnchangedOptions)
86             throws IOException {
87         dumpClassToXml(
88                 serializer,
89                 classTypeName,
90                 obj,
91                 false,
92                 excludeClassFilter,
93                 new NoOpConfigOptionValueTransformer(),
94                 printDeprecatedOptions,
95                 printUnchangedOptions);
96     }
97 
98     /**
99      * Add a class to the configuration XML dump.
100      *
101      * @param serializer a {@link KXmlSerializer} to create the XML dump
102      * @param classTypeName a {@link String} of the class type's name
103      * @param obj {@link Object} to be added to the XML dump
104      * @param isGenericObject Whether or not the object is specified as <object> in the xml
105      * @param excludeClassFilter list of object configuration type or fully qualified class names to
106      *     be excluded from the dump. for example: {@link Configuration#TARGET_PREPARER_TYPE_NAME}.
107      *     com.android.tradefed.testtype.StubTest
108      * @param printDeprecatedOptions whether or not to print deprecated options
109      * @param printUnchangedOptions whether or not to print options that haven't been changed
110      */
dumpClassToXml( KXmlSerializer serializer, String classTypeName, Object obj, boolean isGenericObject, List<String> excludeClassFilter, IConfigOptionValueTransformer transformer, boolean printDeprecatedOptions, boolean printUnchangedOptions)111     static void dumpClassToXml(
112             KXmlSerializer serializer,
113             String classTypeName,
114             Object obj,
115             boolean isGenericObject,
116             List<String> excludeClassFilter,
117             IConfigOptionValueTransformer transformer,
118             boolean printDeprecatedOptions,
119             boolean printUnchangedOptions)
120             throws IOException {
121         if (excludeClassFilter.contains(classTypeName)) {
122             return;
123         }
124         if (excludeClassFilter.contains(obj.getClass().getName())) {
125             return;
126         }
127         if (isGenericObject) {
128             serializer.startTag(null, "object");
129             serializer.attribute(null, "type", classTypeName);
130             serializer.attribute(null, CLASS_NAME, obj.getClass().getName());
131             dumpOptionsToXml(
132                     serializer, obj, transformer, printDeprecatedOptions, printUnchangedOptions);
133             serializer.endTag(null, "object");
134         } else {
135             serializer.startTag(null, classTypeName);
136             serializer.attribute(null, CLASS_NAME, obj.getClass().getName());
137             dumpOptionsToXml(
138                     serializer, obj, transformer, printDeprecatedOptions, printUnchangedOptions);
139             serializer.endTag(null, classTypeName);
140         }
141     }
142 
143     /**
144      * Add all the options of class to the command XML dump.
145      *
146      * @param serializer a {@link KXmlSerializer} to create the XML dump
147      * @param obj {@link Object} to be added to the XML dump
148      * @param printDeprecatedOptions whether or not to skip the deprecated options
149      * @param printUnchangedOptions whether or not to print options that haven't been changed
150      */
151     @SuppressWarnings({"rawtypes", "unchecked"})
dumpOptionsToXml( KXmlSerializer serializer, Object obj, IConfigOptionValueTransformer transformer, boolean printDeprecatedOptions, boolean printUnchangedOptions)152     private static void dumpOptionsToXml(
153             KXmlSerializer serializer,
154             Object obj,
155             IConfigOptionValueTransformer transformer,
156             boolean printDeprecatedOptions,
157             boolean printUnchangedOptions)
158             throws IOException {
159         Object comparisonBaseObj = null;
160         if (!printUnchangedOptions) {
161             try {
162                 comparisonBaseObj = obj.getClass().getDeclaredConstructor().newInstance();
163             } catch (InstantiationException
164                     | IllegalAccessException
165                     | InvocationTargetException
166                     | NoSuchMethodException e) {
167                 throw new RuntimeException(e);
168             }
169         }
170 
171         for (Field field : OptionSetter.getOptionFieldsForClass(obj.getClass())) {
172             Option option = field.getAnnotation(Option.class);
173             Deprecated deprecatedAnnotation = field.getAnnotation(Deprecated.class);
174             // If enabled, skip @Deprecated options
175             if (!printDeprecatedOptions && deprecatedAnnotation != null) {
176                 continue;
177             }
178             Object fieldVal = OptionSetter.getFieldValue(field, obj);
179             if (fieldVal == null) {
180                 continue;
181             }
182             if (comparisonBaseObj != null) {
183                 Object compField = OptionSetter.getFieldValue(field, comparisonBaseObj);
184                 if (fieldVal.equals(compField)) {
185                     continue;
186                 }
187             }
188 
189             if (fieldVal instanceof Collection) {
190                 for (Object entry : (Collection) fieldVal) {
191                     entry = transformer.transform(obj, option, entry);
192                     dumpOptionToXml(serializer, option.name(), null, entry.toString());
193                 }
194             } else if (fieldVal instanceof Map) {
195                 Map map = (Map) fieldVal;
196                 for (Object entryObj : map.entrySet()) {
197                     Map.Entry entry = (Entry) entryObj;
198                     Object value = entry.getValue();
199                     value = transformer.transform(obj, option, value);
200                     dumpOptionToXml(
201                             serializer, option.name(), entry.getKey().toString(), value.toString());
202                 }
203             } else if (fieldVal instanceof MultiMap) {
204                 MultiMap multimap = (MultiMap) fieldVal;
205                 for (Object keyObj : multimap.keySet()) {
206                     for (Object valueObj : multimap.get(keyObj)) {
207                         valueObj = transformer.transform(obj, option, valueObj);
208                         dumpOptionToXml(
209                                 serializer, option.name(), keyObj.toString(), valueObj.toString());
210                     }
211                 }
212             } else {
213                 fieldVal = transformer.transform(obj, option, fieldVal);
214                 dumpOptionToXml(serializer, option.name(), null, fieldVal.toString());
215             }
216         }
217     }
218 
219     /**
220      * Add a single option to the command XML dump.
221      *
222      * @param serializer a {@link KXmlSerializer} to create the XML dump
223      * @param name a {@link String} of the option's name
224      * @param key a {@link String} of the option's key, used as name if param name is null
225      * @param value a {@link String} of the option's value
226      */
dumpOptionToXml( KXmlSerializer serializer, String name, String key, String value)227     private static void dumpOptionToXml(
228             KXmlSerializer serializer, String name, String key, String value) throws IOException {
229         serializer.startTag(null, OPTION_NAME);
230         serializer.attribute(null, NAME_NAME, name);
231         if (key != null) {
232             serializer.attribute(null, KEY_NAME, key);
233         }
234         serializer.attribute(null, VALUE_NAME, value);
235         serializer.endTag(null, OPTION_NAME);
236     }
237 
238     /**
239      * Helper to get the test config files from given directories.
240      *
241      * @param subPath where to look for configuration. Can be null.
242      * @param dirs a list of {@link File} of extra directories to search for test configs
243      */
getConfigNamesFromDirs(String subPath, List<File> dirs)244     public static Set<String> getConfigNamesFromDirs(String subPath, List<File> dirs) {
245         Set<File> res = getConfigNamesFileFromDirs(subPath, dirs);
246         if (res.isEmpty()) {
247             return new HashSet<>();
248         }
249         Set<String> files = new HashSet<>();
250         res.forEach(file -> files.add(file.getAbsolutePath()));
251         return files;
252     }
253 
254     /**
255      * Helper to get the test config files from given directories.
256      *
257      * @param subPath The location where to look for configuration. Can be null.
258      * @param dirs A list of {@link File} of extra directories to search for test configs
259      * @return the set of {@link File} that were found.
260      */
getConfigNamesFileFromDirs(String subPath, List<File> dirs)261     public static Set<File> getConfigNamesFileFromDirs(String subPath, List<File> dirs) {
262         List<String> patterns = new ArrayList<>();
263         patterns.add(".*\\.config$");
264         patterns.add(".*\\.xml$");
265         return getConfigNamesFileFromDirs(subPath, dirs, patterns);
266     }
267 
268     /**
269      * Search a particular pattern of in the given directories.
270      *
271      * @param subPath The location where to look for configuration. Can be null.
272      * @param dirs A list of {@link File} of extra directories to search for test configs
273      * @param configNamePatterns the list of patterns for files to be found.
274      * @return the set of {@link File} that were found.
275      */
getConfigNamesFileFromDirs( String subPath, List<File> dirs, List<String> configNamePatterns)276     public static Set<File> getConfigNamesFileFromDirs(
277             String subPath, List<File> dirs, List<String> configNamePatterns) {
278         return getConfigNamesFileFromDirs(subPath, dirs, configNamePatterns, false);
279     }
280 
281     /**
282      * Search a particular pattern of in the given directories.
283      *
284      * @param subPath The location where to look for configuration. Can be null.
285      * @param dirs A list of {@link File} of extra directories to search for test configs
286      * @param configNamePatterns the list of patterns for files to be found.
287      * @param includeDuplicateFileNames whether to include config files with same name but different
288      *     content.
289      * @return the set of {@link File} that were found.
290      */
getConfigNamesFileFromDirs( String subPath, List<File> dirs, List<String> configNamePatterns, boolean includeDuplicateFileNames)291     public static Set<File> getConfigNamesFileFromDirs(
292             String subPath,
293             List<File> dirs,
294             List<String> configNamePatterns,
295             boolean includeDuplicateFileNames) {
296         Set<File> configNames = new LinkedHashSet<>();
297         for (File dir : dirs) {
298             if (subPath != null) {
299                 dir = new File(dir, subPath);
300             }
301             if (!dir.isDirectory()) {
302                 CLog.d("%s doesn't exist or is not a directory.", dir.getAbsolutePath());
303                 continue;
304             }
305             try {
306                 for (String configNamePattern : configNamePatterns) {
307                     configNames.addAll(FileUtil.findFilesObject(dir, configNamePattern));
308                 }
309             } catch (IOException e) {
310                 CLog.w("Failed to get test config files from directory %s", dir.getAbsolutePath());
311             }
312         }
313         return dedupFiles(configNames, includeDuplicateFileNames);
314     }
315 
316     /**
317      * From a same tests dir we only expect a single instance of each names, so we dedup the files
318      * if that happens.
319      */
dedupFiles(Set<File> origSet, boolean includeDuplicateFileNames)320     private static Set<File> dedupFiles(Set<File> origSet, boolean includeDuplicateFileNames) {
321         Map<String, List<File>> newMap = new LinkedHashMap<>();
322         for (File f : origSet) {
323             try {
324                 if (!FileUtil.readStringFromFile(f).contains("<configuration")) {
325                     CLog.e("%s doesn't look like a test configuration.", f);
326                     continue;
327                 }
328             } catch (IOException e) {
329                 CLog.e(e);
330                 continue;
331             }
332             // Always keep the first found
333             if (!newMap.keySet().contains(f.getName())) {
334                 List<File> newList = new LinkedList<>();
335                 newList.add(f);
336                 newMap.put(f.getName(), newList);
337             } else if (includeDuplicateFileNames) {
338                 // Two files with same name may have different contents. Make sure they are
339                 // identical. if not, add them to the list.
340                 boolean isSameContent = false;
341                 for (File uniqueFiles : newMap.get(f.getName())) {
342                     try {
343                         isSameContent = FileUtil.compareFileContents(uniqueFiles, f);
344                         if (isSameContent) {
345                             break;
346                         }
347                     } catch (IOException e) {
348                         CLog.e(e);
349                     }
350                 }
351                 if (!isSameContent) {
352                     newMap.get(f.getName()).add(f);
353                     CLog.d(
354                             "Config %s already exists, but content is different. Not skipping.",
355                             f.getName());
356                 }
357             }
358         }
359         Set<File> uniqueFiles = new LinkedHashSet<>();
360         for (List<File> files : newMap.values()) {
361             uniqueFiles.addAll(files);
362         }
363         return uniqueFiles;
364     }
365 }
366