1// Copyright 2022 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//      http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package projectmetadata
16
17import (
18	"fmt"
19	"io"
20	"io/fs"
21	"path/filepath"
22	"strings"
23	"sync"
24
25	"android/soong/compliance/project_metadata_proto"
26
27	"google.golang.org/protobuf/encoding/prototext"
28)
29
30var (
31	// ConcurrentReaders is the size of the task pool for limiting resource usage e.g. open files.
32	ConcurrentReaders = 5
33)
34
35// ProjectMetadata contains the METADATA for a git project.
36type ProjectMetadata struct {
37	proto project_metadata_proto.Metadata
38
39	// project is the path to the directory containing the METADATA file.
40	project string
41}
42
43// ProjectUrlMap maps url type name to url value
44type ProjectUrlMap map[string]string
45
46// DownloadUrl returns the address of a download location
47func (m ProjectUrlMap) DownloadUrl() string {
48	for _, urlType := range []string{"GIT", "SVN", "HG", "DARCS"} {
49		if url, ok := m[urlType]; ok {
50			return url
51		}
52	}
53	return ""
54}
55
56// String returns a string representation of the metadata for error messages.
57func (pm *ProjectMetadata) String() string {
58	return fmt.Sprintf("project: %q\n%s", pm.project, pm.proto.String())
59}
60
61// Project returns the path to the directory containing the METADATA file
62func (pm *ProjectMetadata) Project() string {
63	return pm.project
64}
65
66// Name returns the name of the project.
67func (pm *ProjectMetadata) Name() string {
68	return pm.proto.GetName()
69}
70
71// Version returns the version of the project if available.
72func (pm *ProjectMetadata) Version() string {
73	tp := pm.proto.GetThirdParty()
74	if tp != nil {
75		version := tp.GetVersion()
76		return version
77	}
78	return ""
79}
80
81// VersionedName returns the name of the project including the version if any.
82func (pm *ProjectMetadata) VersionedName() string {
83	name := pm.proto.GetName()
84	if name != "" {
85		tp := pm.proto.GetThirdParty()
86		if tp != nil {
87			version := tp.GetVersion()
88			if version != "" {
89				if version[0] == 'v' || version[0] == 'V' {
90					return name + "_" + version
91				} else {
92					return name + "_v_" + version
93				}
94			}
95		}
96		return name
97	}
98	return pm.proto.GetDescription()
99}
100
101// UrlsByTypeName returns a map of URLs by Type Name
102func (pm *ProjectMetadata) UrlsByTypeName() ProjectUrlMap {
103	tp := pm.proto.GetThirdParty()
104	if tp == nil {
105		return nil
106	}
107	if len(tp.Url) == 0 {
108		return nil
109	}
110	urls := make(ProjectUrlMap)
111
112	for _, url := range tp.Url {
113		uri := url.GetValue()
114		if uri == "" {
115			continue
116		}
117		urls[project_metadata_proto.URL_Type_name[int32(url.GetType())]] = uri
118	}
119	return urls
120}
121
122// projectIndex describes a project to be read; after `wg.Wait()`, will contain either
123// a `ProjectMetadata`, pm (can be nil even without error), or a non-nil `err`.
124type projectIndex struct {
125	project string
126	path    string
127	pm      *ProjectMetadata
128	err     error
129	done    chan struct{}
130}
131
132// finish marks the task to read the `projectIndex` completed.
133func (pi *projectIndex) finish() {
134	close(pi.done)
135}
136
137// wait suspends execution until the `projectIndex` task completes.
138func (pi *projectIndex) wait() {
139	<-pi.done
140}
141
142// Index reads and caches ProjectMetadata (thread safe)
143type Index struct {
144	// projecs maps project name to a wait group if read has already started, and
145	// to a `ProjectMetadata` or to an `error` after the read completes.
146	projects sync.Map
147
148	// task provides a fixed-size task pool to limit concurrent open files etc.
149	task chan bool
150
151	// rootFS locates the root of the file system from which to read the files.
152	rootFS fs.FS
153}
154
155// NewIndex constructs a project metadata `Index` for the given file system.
156func NewIndex(rootFS fs.FS) *Index {
157	ix := &Index{task: make(chan bool, ConcurrentReaders), rootFS: rootFS}
158	for i := 0; i < ConcurrentReaders; i++ {
159		ix.task <- true
160	}
161	return ix
162}
163
164// MetadataForProjects returns 0..n ProjectMetadata for n `projects`, or an error.
165// Each project that has a METADATA.android or a METADATA file in the root of the project will have
166// a corresponding ProjectMetadata in the result. Projects with neither file get skipped. A nil
167// result with no error indicates none of the given `projects` has a METADATA file.
168// (thread safe -- can be called concurrently from multiple goroutines)
169func (ix *Index) MetadataForProjects(projects ...string) ([]*ProjectMetadata, error) {
170	if ConcurrentReaders < 1 {
171		return nil, fmt.Errorf("need at least one task in project metadata pool")
172	}
173	if len(projects) == 0 {
174		return nil, nil
175	}
176	// Identify the projects that have never been read
177	projectsToRead := make([]*projectIndex, 0, len(projects))
178	projectIndexes := make([]*projectIndex, 0, len(projects))
179	for _, p := range projects {
180		pi, loaded := ix.projects.LoadOrStore(p, &projectIndex{project: p, done: make(chan struct{})})
181		if !loaded {
182			projectsToRead = append(projectsToRead, pi.(*projectIndex))
183		}
184		projectIndexes = append(projectIndexes, pi.(*projectIndex))
185	}
186	// findMeta locates and reads the appropriate METADATA file, if any.
187	findMeta := func(pi *projectIndex) {
188		<-ix.task
189		defer func() {
190			ix.task <- true
191			pi.finish()
192		}()
193
194		// Support METADATA.android for projects that already have a different sort of METADATA file.
195		path := filepath.Join(pi.project, "METADATA.android")
196		fi, err := fs.Stat(ix.rootFS, path)
197		if err == nil {
198			if fi.Mode().IsRegular() {
199				ix.readMetadataFile(pi, path)
200				return
201			}
202		}
203		// No METADATA.android try METADATA file.
204		path = filepath.Join(pi.project, "METADATA")
205		fi, err = fs.Stat(ix.rootFS, path)
206		if err == nil {
207			if fi.Mode().IsRegular() {
208				ix.readMetadataFile(pi, path)
209				return
210			}
211		}
212		// no METADATA file exists -- leave nil and finish
213	}
214	// Look for the METADATA files to read, and record any missing.
215	for _, p := range projectsToRead {
216		go findMeta(p)
217	}
218	// Wait until all of the projects have been read.
219	var msg strings.Builder
220	result := make([]*ProjectMetadata, 0, len(projects))
221	for _, pi := range projectIndexes {
222		pi.wait()
223		// Combine any errors into a single error.
224		if pi.err != nil {
225			fmt.Fprintf(&msg, "  %v\n", pi.err)
226		} else if pi.pm != nil {
227			result = append(result, pi.pm)
228		}
229	}
230	if msg.Len() > 0 {
231		return nil, fmt.Errorf("error reading project(s):\n%s", msg.String())
232	}
233	if len(result) == 0 {
234		return nil, nil
235	}
236	return result, nil
237}
238
239// AllMetadataFiles returns the sorted list of all METADATA files read thus far.
240func (ix *Index) AllMetadataFiles() []string {
241	var files []string
242	ix.projects.Range(func(key, value any) bool {
243		pi := value.(*projectIndex)
244		if pi.path != "" {
245			files = append(files, pi.path)
246		}
247		return true
248	})
249	return files
250}
251
252// readMetadataFile tries to read and parse a METADATA file at `path` for `project`.
253func (ix *Index) readMetadataFile(pi *projectIndex, path string) {
254	f, err := ix.rootFS.Open(path)
255	if err != nil {
256		pi.err = fmt.Errorf("error opening project %q metadata %q: %w", pi.project, path, err)
257		return
258	}
259
260	// read the file
261	data, err := io.ReadAll(f)
262	if err != nil {
263		pi.err = fmt.Errorf("error reading project %q metadata %q: %w", pi.project, path, err)
264		return
265	}
266	f.Close()
267
268	uo := prototext.UnmarshalOptions{DiscardUnknown: true}
269	pm := &ProjectMetadata{project: pi.project}
270	err = uo.Unmarshal(data, &pm.proto)
271	if err != nil {
272		pi.err = fmt.Errorf(`error in project %q METADATA %q: %v
273
274METADATA and METADATA.android files must parse as text protobufs
275defined by
276   build/soong/compliance/project_metadata_proto/project_metadata.proto
277
278* unknown fields don't matter
279* check invalid ENUM names
280* check quoting
281* check unescaped nested quotes
282* check the comment marker for protobuf is '#' not '//'
283
284if importing a library that uses a different sort of METADATA file, add
285a METADATA.android file beside it to parse instead
286`, pi.project, path, err)
287		return
288	}
289
290	pi.path = path
291	pi.pm = pm
292}
293