1// Copyright 2016 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package pathtools
16
17import (
18	"bytes"
19	"fmt"
20	"io"
21	"io/ioutil"
22	"os"
23	"path/filepath"
24	"sort"
25	"strings"
26	"syscall"
27	"time"
28)
29
30// Based on Andrew Gerrand's "10 things you (probably) dont' know about Go"
31
32type ShouldFollowSymlinks bool
33
34const (
35	FollowSymlinks     = ShouldFollowSymlinks(true)
36	DontFollowSymlinks = ShouldFollowSymlinks(false)
37)
38
39var OsFs FileSystem = &osFs{}
40
41func MockFs(files map[string][]byte) FileSystem {
42	fs := &mockFs{
43		files:    make(map[string][]byte, len(files)),
44		dirs:     make(map[string]bool),
45		symlinks: make(map[string]string),
46		all:      []string(nil),
47	}
48
49	for f, b := range files {
50		if tokens := strings.SplitN(f, "->", 2); len(tokens) == 2 {
51			fs.symlinks[strings.TrimSpace(tokens[0])] = strings.TrimSpace(tokens[1])
52			continue
53		}
54
55		fs.files[filepath.Clean(f)] = b
56		dir := filepath.Dir(f)
57		for dir != "." && dir != "/" {
58			fs.dirs[dir] = true
59			dir = filepath.Dir(dir)
60		}
61		fs.dirs[dir] = true
62	}
63
64	fs.dirs["."] = true
65	fs.dirs["/"] = true
66
67	for f := range fs.files {
68		fs.all = append(fs.all, f)
69	}
70
71	for d := range fs.dirs {
72		fs.all = append(fs.all, d)
73	}
74
75	for s := range fs.symlinks {
76		fs.all = append(fs.all, s)
77	}
78
79	sort.Strings(fs.all)
80
81	return fs
82}
83
84type ReaderAtSeekerCloser interface {
85	io.Reader
86	io.ReaderAt
87	io.Seeker
88	io.Closer
89}
90
91type FileSystem interface {
92	// Open opens a file for reading. Follows symlinks.
93	Open(name string) (ReaderAtSeekerCloser, error)
94
95	// Exists returns whether the file exists and whether it is a directory.  Follows symlinks.
96	Exists(name string) (bool, bool, error)
97
98	Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (GlobResult, error)
99	glob(pattern string) (matches []string, err error)
100
101	// IsDir returns true if the path points to a directory, false it it points to a file.  Follows symlinks.
102	// Returns os.ErrNotExist if the path does not exist or is a symlink to a path that does not exist.
103	IsDir(name string) (bool, error)
104
105	// IsSymlink returns true if the path points to a symlink, even if that symlink points to a path that does
106	// not exist.  Returns os.ErrNotExist if the path does not exist.
107	IsSymlink(name string) (bool, error)
108
109	// Lstat returns info on a file without following symlinks.
110	Lstat(name string) (os.FileInfo, error)
111
112	// Lstat returns info on a file.
113	Stat(name string) (os.FileInfo, error)
114
115	// ListDirsRecursive returns a list of all the directories in a path, following symlinks if requested.
116	ListDirsRecursive(name string, follow ShouldFollowSymlinks) (dirs []string, err error)
117
118	// ReadDirNames returns a list of everything in a directory.
119	ReadDirNames(name string) ([]string, error)
120
121	// Readlink returns the destination of the named symbolic link.
122	Readlink(name string) (string, error)
123}
124
125// osFs implements FileSystem using the local disk.
126type osFs struct {
127	srcDir        string
128	openFilesChan chan bool
129}
130
131func NewOsFs(path string) FileSystem {
132	// Darwin has a default limit of 256 open files, rate limit open files to 200
133	limit := 200
134	return &osFs{
135		srcDir:        path,
136		openFilesChan: make(chan bool, limit),
137	}
138}
139
140func (fs *osFs) acquire() {
141	if fs.openFilesChan != nil {
142		fs.openFilesChan <- true
143	}
144}
145
146func (fs *osFs) release() {
147	if fs.openFilesChan != nil {
148		<-fs.openFilesChan
149	}
150}
151
152func (fs *osFs) toAbs(path string) string {
153	if filepath.IsAbs(path) {
154		return path
155	}
156	return filepath.Join(fs.srcDir, path)
157}
158
159func (fs *osFs) removeSrcDirPrefix(path string) string {
160	if fs.srcDir == "" {
161		return path
162	}
163	rel, err := filepath.Rel(fs.srcDir, path)
164	if err != nil {
165		panic(fmt.Errorf("unexpected failure in removeSrcDirPrefix filepath.Rel(%s, %s): %s",
166			fs.srcDir, path, err))
167	}
168	if strings.HasPrefix(rel, "../") {
169		panic(fmt.Errorf("unexpected relative path outside directory in removeSrcDirPrefix filepath.Rel(%s, %s): %s",
170			fs.srcDir, path, rel))
171	}
172	return rel
173}
174
175func (fs *osFs) removeSrcDirPrefixes(paths []string) []string {
176	if fs.srcDir != "" {
177		for i, path := range paths {
178			paths[i] = fs.removeSrcDirPrefix(path)
179		}
180	}
181	return paths
182}
183
184// OsFile wraps an os.File to also release open file descriptors semaphore on close
185type OsFile struct {
186	*os.File
187	fs *osFs
188}
189
190// Close closes file and releases the open file descriptor semaphore
191func (f *OsFile) Close() error {
192	err := f.File.Close()
193	f.fs.release()
194	return err
195}
196
197func (fs *osFs) Open(name string) (ReaderAtSeekerCloser, error) {
198	fs.acquire()
199	f, err := os.Open(fs.toAbs(name))
200	if err != nil {
201		return nil, err
202	}
203	return &OsFile{f, fs}, nil
204}
205
206func (fs *osFs) Exists(name string) (bool, bool, error) {
207	stat, err := os.Stat(fs.toAbs(name))
208	if err == nil {
209		return true, stat.IsDir(), nil
210	} else if os.IsNotExist(err) {
211		return false, false, nil
212	} else {
213		return false, false, err
214	}
215}
216
217func (fs *osFs) IsDir(name string) (bool, error) {
218	info, err := os.Stat(fs.toAbs(name))
219	if err != nil {
220		return false, err
221	}
222	return info.IsDir(), nil
223}
224
225func (fs *osFs) IsSymlink(name string) (bool, error) {
226	if info, err := os.Lstat(fs.toAbs(name)); err != nil {
227		return false, err
228	} else {
229		return info.Mode()&os.ModeSymlink != 0, nil
230	}
231}
232
233func (fs *osFs) Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (GlobResult, error) {
234	return startGlob(fs, pattern, excludes, follow)
235}
236
237func (fs *osFs) glob(pattern string) ([]string, error) {
238	fs.acquire()
239	defer fs.release()
240	paths, err := filepath.Glob(fs.toAbs(pattern))
241	fs.removeSrcDirPrefixes(paths)
242	return paths, err
243}
244
245func (fs *osFs) Lstat(path string) (stats os.FileInfo, err error) {
246	return os.Lstat(fs.toAbs(path))
247}
248
249func (fs *osFs) Stat(path string) (stats os.FileInfo, err error) {
250	return os.Stat(fs.toAbs(path))
251}
252
253// Returns a list of all directories under dir
254func (fs *osFs) ListDirsRecursive(name string, follow ShouldFollowSymlinks) (dirs []string, err error) {
255	return listDirsRecursive(fs, name, follow)
256}
257
258func (fs *osFs) ReadDirNames(name string) ([]string, error) {
259	fs.acquire()
260	defer fs.release()
261	dir, err := os.Open(fs.toAbs(name))
262	if err != nil {
263		return nil, err
264	}
265	defer dir.Close()
266
267	contents, err := dir.Readdirnames(-1)
268	if err != nil {
269		return nil, err
270	}
271
272	sort.Strings(contents)
273	return contents, nil
274}
275
276func (fs *osFs) Readlink(name string) (string, error) {
277	return os.Readlink(fs.toAbs(name))
278}
279
280type mockFs struct {
281	files    map[string][]byte
282	dirs     map[string]bool
283	symlinks map[string]string
284	all      []string
285}
286
287func (m *mockFs) followSymlinks(name string) string {
288	dir, file := quickSplit(name)
289	if dir != "." && dir != "/" {
290		dir = m.followSymlinks(dir)
291	}
292	name = filepath.Join(dir, file)
293
294	for i := 0; i < 255; i++ {
295		i++
296		if i > 255 {
297			panic("symlink loop")
298		}
299		to, exists := m.symlinks[name]
300		if !exists {
301			break
302		}
303		if filepath.IsAbs(to) {
304			name = to
305		} else {
306			name = filepath.Join(dir, to)
307		}
308	}
309	return name
310}
311
312func (m *mockFs) Open(name string) (ReaderAtSeekerCloser, error) {
313	name = filepath.Clean(name)
314	name = m.followSymlinks(name)
315	if f, ok := m.files[name]; ok {
316		return struct {
317			io.Closer
318			*bytes.Reader
319		}{
320			ioutil.NopCloser(nil),
321			bytes.NewReader(f),
322		}, nil
323	}
324
325	return nil, &os.PathError{
326		Op:   "open",
327		Path: name,
328		Err:  os.ErrNotExist,
329	}
330}
331
332func (m *mockFs) Exists(name string) (bool, bool, error) {
333	name = filepath.Clean(name)
334	name = m.followSymlinks(name)
335	if _, ok := m.files[name]; ok {
336		return ok, false, nil
337	}
338	if _, ok := m.dirs[name]; ok {
339		return ok, true, nil
340	}
341	return false, false, nil
342}
343
344func (m *mockFs) IsDir(name string) (bool, error) {
345	dir := filepath.Dir(name)
346	if dir != "." && dir != "/" {
347		isDir, err := m.IsDir(dir)
348
349		if serr, ok := err.(*os.SyscallError); ok && serr.Err == syscall.ENOTDIR {
350			isDir = false
351		} else if err != nil {
352			return false, err
353		}
354
355		if !isDir {
356			return false, os.NewSyscallError("stat "+name, syscall.ENOTDIR)
357		}
358	}
359
360	name = filepath.Clean(name)
361	name = m.followSymlinks(name)
362
363	if _, ok := m.dirs[name]; ok {
364		return true, nil
365	}
366	if _, ok := m.files[name]; ok {
367		return false, nil
368	}
369	return false, os.ErrNotExist
370}
371
372func (m *mockFs) IsSymlink(name string) (bool, error) {
373	dir, file := quickSplit(name)
374	dir = m.followSymlinks(dir)
375	name = filepath.Join(dir, file)
376
377	if _, isSymlink := m.symlinks[name]; isSymlink {
378		return true, nil
379	}
380	if _, isDir := m.dirs[name]; isDir {
381		return false, nil
382	}
383	if _, isFile := m.files[name]; isFile {
384		return false, nil
385	}
386	return false, os.ErrNotExist
387}
388
389func (m *mockFs) Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (GlobResult, error) {
390	return startGlob(m, pattern, excludes, follow)
391}
392
393func unescapeGlob(s string) string {
394	i := 0
395	for i < len(s) {
396		if s[i] == '\\' {
397			s = s[:i] + s[i+1:]
398		} else {
399			i++
400		}
401	}
402	return s
403}
404
405func (m *mockFs) glob(pattern string) ([]string, error) {
406	dir, file := quickSplit(pattern)
407
408	dir = unescapeGlob(dir)
409	toDir := m.followSymlinks(dir)
410
411	var matches []string
412	for _, f := range m.all {
413		fDir, fFile := quickSplit(f)
414		if toDir == fDir {
415			match, err := filepath.Match(file, fFile)
416			if err != nil {
417				return nil, err
418			}
419			if (f == "." || f == "/") && f != pattern {
420				// filepath.Glob won't return "." or "/" unless the pattern was "." or "/"
421				match = false
422			}
423			if match {
424				matches = append(matches, filepath.Join(dir, fFile))
425			}
426		}
427	}
428	return matches, nil
429}
430
431type mockStat struct {
432	name string
433	size int64
434	mode os.FileMode
435}
436
437func (ms *mockStat) Name() string       { return ms.name }
438func (ms *mockStat) IsDir() bool        { return ms.Mode().IsDir() }
439func (ms *mockStat) Size() int64        { return ms.size }
440func (ms *mockStat) Mode() os.FileMode  { return ms.mode }
441func (ms *mockStat) ModTime() time.Time { return time.Time{} }
442func (ms *mockStat) Sys() interface{}   { return nil }
443
444func (m *mockFs) Lstat(name string) (os.FileInfo, error) {
445	dir, file := quickSplit(name)
446	dir = m.followSymlinks(dir)
447	name = filepath.Join(dir, file)
448
449	ms := mockStat{
450		name: file,
451	}
452
453	if symlink, isSymlink := m.symlinks[name]; isSymlink {
454		ms.mode = os.ModeSymlink
455		ms.size = int64(len(symlink))
456	} else if _, isDir := m.dirs[name]; isDir {
457		ms.mode = os.ModeDir
458	} else if _, isFile := m.files[name]; isFile {
459		ms.mode = 0
460		ms.size = int64(len(m.files[name]))
461	} else {
462		return nil, os.ErrNotExist
463	}
464
465	return &ms, nil
466}
467
468func (m *mockFs) Stat(name string) (os.FileInfo, error) {
469	name = filepath.Clean(name)
470	origName := name
471	name = m.followSymlinks(name)
472
473	ms := mockStat{
474		name: filepath.Base(origName),
475		size: int64(len(m.files[name])),
476	}
477
478	if _, isDir := m.dirs[name]; isDir {
479		ms.mode = os.ModeDir
480	} else if _, isFile := m.files[name]; isFile {
481		ms.mode = 0
482		ms.size = int64(len(m.files[name]))
483	} else {
484		return nil, os.ErrNotExist
485	}
486
487	return &ms, nil
488}
489
490func (m *mockFs) ReadDirNames(name string) ([]string, error) {
491	name = filepath.Clean(name)
492	name = m.followSymlinks(name)
493
494	exists, isDir, err := m.Exists(name)
495	if err != nil {
496		return nil, err
497	}
498	if !exists {
499		return nil, os.ErrNotExist
500	}
501	if !isDir {
502		return nil, os.NewSyscallError("readdir", syscall.ENOTDIR)
503	}
504
505	var ret []string
506	for _, f := range m.all {
507		dir, file := quickSplit(f)
508		if dir == name && len(file) > 0 && file[0] != '.' {
509			ret = append(ret, file)
510		}
511	}
512	return ret, nil
513}
514
515func (m *mockFs) ListDirsRecursive(name string, follow ShouldFollowSymlinks) ([]string, error) {
516	return listDirsRecursive(m, name, follow)
517}
518
519func (m *mockFs) Readlink(name string) (string, error) {
520	dir, file := quickSplit(name)
521	dir = m.followSymlinks(dir)
522
523	origName := name
524	name = filepath.Join(dir, file)
525
526	if dest, isSymlink := m.symlinks[name]; isSymlink {
527		return dest, nil
528	}
529
530	if exists, _, err := m.Exists(name); err != nil {
531		return "", err
532	} else if !exists {
533		return "", os.ErrNotExist
534	} else {
535		return "", os.NewSyscallError("readlink: "+origName, syscall.EINVAL)
536	}
537}
538
539func listDirsRecursive(fs FileSystem, name string, follow ShouldFollowSymlinks) ([]string, error) {
540	name = filepath.Clean(name)
541
542	isDir, err := fs.IsDir(name)
543	if err != nil {
544		return nil, err
545	}
546
547	if !isDir {
548		return nil, nil
549	}
550
551	dirs := []string{name}
552
553	subDirs, err := listDirsRecursiveRelative(fs, name, follow, 0)
554	if err != nil {
555		return nil, err
556	}
557
558	for _, d := range subDirs {
559		dirs = append(dirs, filepath.Join(name, d))
560	}
561
562	return dirs, nil
563}
564
565func listDirsRecursiveRelative(fs FileSystem, name string, follow ShouldFollowSymlinks, depth int) ([]string, error) {
566	depth++
567	if depth > 255 {
568		return nil, fmt.Errorf("too many symlinks")
569	}
570	contents, err := fs.ReadDirNames(name)
571	if err != nil {
572		return nil, err
573	}
574
575	var dirs []string
576	for _, f := range contents {
577		if f[0] == '.' {
578			continue
579		}
580		f = filepath.Join(name, f)
581		var info os.FileInfo
582		if follow == DontFollowSymlinks {
583			info, err = fs.Lstat(f)
584			if err != nil {
585				continue
586			}
587			if info.Mode()&os.ModeSymlink != 0 {
588				continue
589			}
590		} else {
591			info, err = fs.Stat(f)
592			if err != nil {
593				continue
594			}
595		}
596		if info.IsDir() {
597			dirs = append(dirs, f)
598			subDirs, err := listDirsRecursiveRelative(fs, f, follow, depth)
599			if err != nil {
600				return nil, err
601			}
602			for _, s := range subDirs {
603				dirs = append(dirs, filepath.Join(f, s))
604			}
605		}
606	}
607
608	for i, d := range dirs {
609		rel, err := filepath.Rel(name, d)
610		if err != nil {
611			return nil, err
612		}
613		dirs[i] = rel
614	}
615
616	return dirs, nil
617}
618