1// Copyright 2018 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package main
16
17import (
18	"bytes"
19	"fmt"
20	"hash/crc32"
21	"os"
22	"strconv"
23	"strings"
24	"testing"
25	"time"
26
27	"android/soong/jar"
28	"android/soong/third_party/zip"
29)
30
31type testZipEntry struct {
32	name      string
33	mode      os.FileMode
34	data      []byte
35	method    uint16
36	timestamp time.Time
37}
38
39var (
40	A     = testZipEntry{"A", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime}
41	a     = testZipEntry{"a", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime}
42	a2    = testZipEntry{"a", 0755, []byte("FOO2"), zip.Deflate, jar.DefaultTime}
43	a3    = testZipEntry{"a", 0755, []byte("Foo3"), zip.Deflate, jar.DefaultTime}
44	bDir  = testZipEntry{"b/", os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime}
45	bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime}
46	bbb   = testZipEntry{"b/b/b", 0755, nil, zip.Deflate, jar.DefaultTime}
47	ba    = testZipEntry{"b/a", 0755, []byte("foo"), zip.Deflate, jar.DefaultTime}
48	bc    = testZipEntry{"b/c", 0755, []byte("bar"), zip.Deflate, jar.DefaultTime}
49	bd    = testZipEntry{"b/d", 0700, []byte("baz"), zip.Deflate, jar.DefaultTime}
50	be    = testZipEntry{"b/e", 0700, []byte(""), zip.Deflate, jar.DefaultTime}
51
52	withTimestamp    = testZipEntry{"timestamped", 0755, nil, zip.Store, jar.DefaultTime.Add(time.Hour)}
53	withoutTimestamp = testZipEntry{"timestamped", 0755, nil, zip.Store, jar.DefaultTime}
54
55	service1a        = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\n"), zip.Store, jar.DefaultTime}
56	service1b        = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass3\n"), zip.Deflate, jar.DefaultTime}
57	service1combined = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\nclass3\n"), zip.Store, jar.DefaultTime}
58	service2         = testZipEntry{"META-INF/services/service2", 0755, []byte("class1\nclass2\n"), zip.Deflate, jar.DefaultTime}
59
60	metainfDir     = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil, zip.Deflate, jar.DefaultTime}
61	manifestFile   = testZipEntry{jar.ManifestFile, 0755, []byte("manifest"), zip.Deflate, jar.DefaultTime}
62	manifestFile2  = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2"), zip.Deflate, jar.DefaultTime}
63	moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info"), zip.Deflate, jar.DefaultTime}
64)
65
66type testInputZip struct {
67	name    string
68	entries []testZipEntry
69	reader  *zip.Reader
70}
71
72func (tiz *testInputZip) Name() string {
73	return tiz.name
74}
75
76func (tiz *testInputZip) Open() error {
77	if tiz.reader == nil {
78		tiz.reader = testZipEntriesToZipReader(tiz.entries)
79	}
80	return nil
81}
82
83func (tiz *testInputZip) Close() error {
84	tiz.reader = nil
85	return nil
86}
87
88func (tiz *testInputZip) Entries() []*zip.File {
89	if tiz.reader == nil {
90		panic(fmt.Errorf("%s: should be open to get entries", tiz.Name()))
91	}
92	return tiz.reader.File
93}
94
95func (tiz *testInputZip) IsOpen() bool {
96	return tiz.reader != nil
97}
98
99func TestMergeZips(t *testing.T) {
100	testCases := []struct {
101		name             string
102		in               [][]testZipEntry
103		stripFiles       []string
104		stripDirs        []string
105		jar              bool
106		par              bool
107		sort             bool
108		ignoreDuplicates bool
109		stripDirEntries  bool
110		zipsToNotStrip   map[string]bool
111
112		out []testZipEntry
113		err string
114	}{
115		{
116			name: "duplicates error",
117			in: [][]testZipEntry{
118				{a},
119				{a2},
120				{a3},
121			},
122			out: []testZipEntry{a},
123			err: "duplicate",
124		},
125		{
126			name: "duplicates take first",
127			in: [][]testZipEntry{
128				{a},
129				{a2},
130				{a3},
131			},
132			out: []testZipEntry{a},
133
134			ignoreDuplicates: true,
135		},
136		{
137			name: "duplicates identical",
138			in: [][]testZipEntry{
139				{a},
140				{a},
141			},
142			out: []testZipEntry{a},
143		},
144		{
145			name: "sort",
146			in: [][]testZipEntry{
147				{be, bc, bDir, bbDir, bbb, A, metainfDir, manifestFile},
148			},
149			out: []testZipEntry{A, metainfDir, manifestFile, bDir, bbDir, bbb, bc, be},
150
151			sort: true,
152		},
153		{
154			name: "jar sort",
155			in: [][]testZipEntry{
156				{be, bc, bDir, A, metainfDir, manifestFile},
157			},
158			out: []testZipEntry{metainfDir, manifestFile, A, bDir, bc, be},
159
160			jar: true,
161		},
162		{
163			name: "jar merge",
164			in: [][]testZipEntry{
165				{metainfDir, manifestFile, bDir, be},
166				{metainfDir, manifestFile2, bDir, bc},
167				{metainfDir, manifestFile2, A},
168			},
169			out: []testZipEntry{metainfDir, manifestFile, A, bDir, bc, be},
170
171			jar: true,
172		},
173		{
174			name: "merge",
175			in: [][]testZipEntry{
176				{bDir, be},
177				{bDir, bc},
178				{A},
179			},
180			out: []testZipEntry{bDir, be, bc, A},
181		},
182		{
183			name: "strip dir entries",
184			in: [][]testZipEntry{
185				{a, bDir, bbDir, bbb, bc, bd, be},
186			},
187			out: []testZipEntry{a, bbb, bc, bd, be},
188
189			stripDirEntries: true,
190		},
191		{
192			name: "strip files",
193			in: [][]testZipEntry{
194				{a, bDir, bbDir, bbb, bc, bd, be},
195			},
196			out: []testZipEntry{a, bDir, bbDir, bbb, bc},
197
198			stripFiles: []string{"b/d", "b/e"},
199		},
200		{
201			// merge_zips used to treat -stripFile a as stripping any file named a, it now only strips a in the
202			// root of the zip.
203			name: "strip file name",
204			in: [][]testZipEntry{
205				{a, bDir, ba},
206			},
207			out: []testZipEntry{bDir, ba},
208
209			stripFiles: []string{"a"},
210		},
211		{
212			name: "strip files glob",
213			in: [][]testZipEntry{
214				{a, bDir, ba},
215			},
216			out: []testZipEntry{bDir},
217
218			stripFiles: []string{"**/a"},
219		},
220		{
221			name: "strip dirs",
222			in: [][]testZipEntry{
223				{a, bDir, bbDir, bbb, bc, bd, be},
224			},
225			out: []testZipEntry{a},
226
227			stripDirs: []string{"b"},
228		},
229		{
230			name: "strip dirs glob",
231			in: [][]testZipEntry{
232				{a, bDir, bbDir, bbb, bc, bd, be},
233			},
234			out: []testZipEntry{a, bDir, bc, bd, be},
235
236			stripDirs: []string{"b/*"},
237		},
238		{
239			name: "zips to not strip",
240			in: [][]testZipEntry{
241				{a, bDir, bc},
242				{bDir, bd},
243				{bDir, be},
244			},
245			out: []testZipEntry{a, bDir, bd},
246
247			stripDirs: []string{"b"},
248			zipsToNotStrip: map[string]bool{
249				"in1": true,
250			},
251		},
252		{
253			name: "services",
254			in: [][]testZipEntry{
255				{service1a, service2},
256				{service1b},
257			},
258			jar: true,
259			out: []testZipEntry{service1combined, service2},
260		},
261		{
262			name: "strip timestamps",
263			in: [][]testZipEntry{
264				{withTimestamp},
265				{a},
266			},
267			out: []testZipEntry{withoutTimestamp, a},
268		},
269		{
270			name: "emulate par",
271			in: [][]testZipEntry{
272				{
273					testZipEntry{name: "3/main.py"},
274					testZipEntry{name: "c/main.py"},
275					testZipEntry{name: "a/main.py"},
276					testZipEntry{name: "2/main.py"},
277					testZipEntry{name: "b/main.py"},
278					testZipEntry{name: "1/main.py"},
279				},
280			},
281			out: []testZipEntry{
282				testZipEntry{name: "3/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
283				testZipEntry{name: "c/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
284				testZipEntry{name: "a/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
285				testZipEntry{name: "2/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
286				testZipEntry{name: "b/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
287				testZipEntry{name: "1/__init__.py", mode: 0700, timestamp: jar.DefaultTime},
288				testZipEntry{name: "3/main.py", timestamp: jar.DefaultTime},
289				testZipEntry{name: "c/main.py", timestamp: jar.DefaultTime},
290				testZipEntry{name: "a/main.py", timestamp: jar.DefaultTime},
291				testZipEntry{name: "2/main.py", timestamp: jar.DefaultTime},
292				testZipEntry{name: "b/main.py", timestamp: jar.DefaultTime},
293				testZipEntry{name: "1/main.py", timestamp: jar.DefaultTime},
294			},
295			par: true,
296		},
297	}
298
299	for _, test := range testCases {
300		t.Run(test.name, func(t *testing.T) {
301			inputZips := make([]InputZip, len(test.in))
302			for i, in := range test.in {
303				inputZips[i] = &testInputZip{name: "in" + strconv.Itoa(i), entries: in}
304			}
305
306			want := testZipEntriesToBuf(test.out)
307
308			out := &bytes.Buffer{}
309			writer := zip.NewWriter(out)
310
311			err := mergeZips(inputZips, writer, "", "",
312				test.sort, test.jar, test.par, test.stripDirEntries, test.ignoreDuplicates,
313				test.stripFiles, test.stripDirs, test.zipsToNotStrip)
314
315			closeErr := writer.Close()
316			if closeErr != nil {
317				t.Fatal(closeErr)
318			}
319
320			if test.err != "" {
321				if err == nil {
322					t.Fatal("missing err, expected: ", test.err)
323				} else if !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(test.err)) {
324					t.Fatal("incorrect err, want:", test.err, "got:", err)
325				}
326				return
327			} else if err != nil {
328				t.Fatal("unexpected err: ", err)
329			}
330
331			if !bytes.Equal(want, out.Bytes()) {
332				t.Error("incorrect zip output")
333				t.Errorf("want:\n%s", dumpZip(want))
334				t.Errorf("got:\n%s", dumpZip(out.Bytes()))
335				os.WriteFile("/tmp/got.zip", out.Bytes(), 0755)
336				os.WriteFile("/tmp/want.zip", want, 0755)
337			}
338		})
339	}
340}
341
342func testZipEntriesToBuf(entries []testZipEntry) []byte {
343	b := &bytes.Buffer{}
344	zw := zip.NewWriter(b)
345
346	for _, e := range entries {
347		fh := zip.FileHeader{
348			Name: e.name,
349		}
350		fh.SetMode(e.mode)
351		fh.Method = e.method
352		fh.SetModTime(e.timestamp)
353		fh.UncompressedSize64 = uint64(len(e.data))
354		fh.CRC32 = crc32.ChecksumIEEE(e.data)
355		if fh.Method == zip.Store {
356			fh.CompressedSize64 = fh.UncompressedSize64
357		}
358
359		w, err := zw.CreateHeaderAndroid(&fh)
360		if err != nil {
361			panic(err)
362		}
363
364		_, err = w.Write(e.data)
365		if err != nil {
366			panic(err)
367		}
368	}
369
370	err := zw.Close()
371	if err != nil {
372		panic(err)
373	}
374
375	return b.Bytes()
376}
377
378func testZipEntriesToZipReader(entries []testZipEntry) *zip.Reader {
379	b := testZipEntriesToBuf(entries)
380	r := bytes.NewReader(b)
381
382	zr, err := zip.NewReader(r, int64(len(b)))
383	if err != nil {
384		panic(err)
385	}
386
387	return zr
388}
389
390func dumpZip(buf []byte) string {
391	r := bytes.NewReader(buf)
392	zr, err := zip.NewReader(r, int64(len(buf)))
393	if err != nil {
394		panic(err)
395	}
396
397	var ret string
398
399	for _, f := range zr.File {
400		ret += fmt.Sprintf("%v: %v %v %08x %s\n", f.Name, f.Mode(), f.UncompressedSize64, f.CRC32, f.ModTime())
401	}
402
403	return ret
404}
405
406type DummyInpuZip struct {
407	isOpen bool
408}
409
410func (diz *DummyInpuZip) Name() string {
411	return "dummy"
412}
413
414func (diz *DummyInpuZip) Open() error {
415	diz.isOpen = true
416	return nil
417}
418
419func (diz *DummyInpuZip) Close() error {
420	diz.isOpen = false
421	return nil
422}
423
424func (DummyInpuZip) Entries() []*zip.File {
425	panic("implement me")
426}
427
428func (diz *DummyInpuZip) IsOpen() bool {
429	return diz.isOpen
430}
431
432func TestInputZipsManager(t *testing.T) {
433	const nInputZips = 20
434	const nMaxOpenZips = 10
435	izm := NewInputZipsManager(20, 10)
436	managedZips := make([]InputZip, nInputZips)
437	for i := 0; i < nInputZips; i++ {
438		managedZips[i] = izm.Manage(&DummyInpuZip{})
439	}
440
441	t.Run("InputZipsManager", func(t *testing.T) {
442		for i, iz := range managedZips {
443			if err := iz.Open(); err != nil {
444				t.Fatalf("Step %d: open failed: %s", i, err)
445				return
446			}
447			if izm.nOpenZips > nMaxOpenZips {
448				t.Errorf("Step %d: should be <=%d open zips", i, nMaxOpenZips)
449			}
450		}
451		if !managedZips[nInputZips-1].IsOpen() {
452			t.Error("The last input should stay open")
453		}
454		for _, iz := range managedZips {
455			iz.Close()
456		}
457		if izm.nOpenZips > 0 {
458			t.Error("Some input zips are still open")
459		}
460	})
461}
462