1/*
2 * Copyright (C) 2024 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17// Binary ide_query generates and analyzes build artifacts.
18// The produced result can be consumed by IDEs to provide language features.
19package main
20
21import (
22	"bytes"
23	"container/list"
24	"context"
25	"encoding/json"
26	"flag"
27	"fmt"
28	"log"
29	"os"
30	"os/exec"
31	"path"
32	"slices"
33	"strings"
34
35	"google.golang.org/protobuf/proto"
36	pb "ide_query/ide_query_proto"
37)
38
39// Env contains information about the current environment.
40type Env struct {
41	LunchTarget    LunchTarget
42	RepoDir        string
43	OutDir         string
44	ClangToolsRoot string
45
46	CcFiles   []string
47	JavaFiles []string
48}
49
50// LunchTarget is a parsed Android lunch target.
51// Input format: <product_name>-<release_type>-<build_variant>
52type LunchTarget struct {
53	Product string
54	Release string
55	Variant string
56}
57
58var _ flag.Value = (*LunchTarget)(nil)
59
60// // Get implements flag.Value.
61// func (l *LunchTarget) Get() any {
62// 	return l
63// }
64
65// Set implements flag.Value.
66func (l *LunchTarget) Set(s string) error {
67	parts := strings.Split(s, "-")
68	if len(parts) != 3 {
69		return fmt.Errorf("invalid lunch target: %q, must have form <product_name>-<release_type>-<build_variant>", s)
70	}
71	*l = LunchTarget{
72		Product: parts[0],
73		Release: parts[1],
74		Variant: parts[2],
75	}
76	return nil
77}
78
79// String implements flag.Value.
80func (l *LunchTarget) String() string {
81	return fmt.Sprintf("%s-%s-%s", l.Product, l.Release, l.Variant)
82}
83
84func main() {
85	var env Env
86	env.OutDir = os.Getenv("OUT_DIR")
87	env.RepoDir = os.Getenv("ANDROID_BUILD_TOP")
88	env.ClangToolsRoot = os.Getenv("PREBUILTS_CLANG_TOOLS_ROOT")
89	flag.Var(&env.LunchTarget, "lunch_target", "The lunch target to query")
90	flag.Parse()
91	files := flag.Args()
92	if len(files) == 0 {
93		fmt.Println("No files provided.")
94		os.Exit(1)
95		return
96	}
97
98	for _, f := range files {
99		switch {
100		case strings.HasSuffix(f, ".java") || strings.HasSuffix(f, ".kt"):
101			env.JavaFiles = append(env.JavaFiles, f)
102		case strings.HasSuffix(f, ".cc") || strings.HasSuffix(f, ".cpp") || strings.HasSuffix(f, ".h"):
103			env.CcFiles = append(env.CcFiles, f)
104		default:
105			log.Printf("File %q is supported - will be skipped.", f)
106		}
107	}
108
109	ctx := context.Background()
110	// TODO(michaelmerg): Figure out if module_bp_java_deps.json and compile_commands.json is outdated.
111	runMake(ctx, env, "nothing")
112
113	javaModules, javaFileToModuleMap, err := loadJavaModules(&env)
114	if err != nil {
115		log.Printf("Failed to load java modules: %v", err)
116	}
117	toMake := getJavaTargets(javaFileToModuleMap)
118
119	ccTargets, status := getCCTargets(ctx, &env)
120	if status != nil && status.Code != pb.Status_OK {
121		log.Fatalf("Failed to query cc targets: %v", *status.Message)
122	}
123	toMake = append(toMake, ccTargets...)
124	fmt.Fprintf(os.Stderr, "Running make for modules: %v\n", strings.Join(toMake, ", "))
125	if err := runMake(ctx, env, toMake...); err != nil {
126		log.Printf("Building deps failed: %v", err)
127	}
128
129	res := getJavaInputs(&env, javaModules, javaFileToModuleMap)
130	ccAnalysis := getCCInputs(ctx, &env)
131	proto.Merge(res, ccAnalysis)
132
133	res.BuildArtifactRoot = env.OutDir
134	data, err := proto.Marshal(res)
135	if err != nil {
136		log.Fatalf("Failed to marshal result proto: %v", err)
137	}
138
139	_, err = os.Stdout.Write(data)
140	if err != nil {
141		log.Fatalf("Failed to write result proto: %v", err)
142	}
143
144	for _, s := range res.Sources {
145		fmt.Fprintf(os.Stderr, "%s: %v (Deps: %d, Generated: %d)\n", s.GetPath(), s.GetStatus(), len(s.GetDeps()), len(s.GetGenerated()))
146	}
147}
148
149func repoState(env *Env) *pb.RepoState {
150	const compDbPath = "soong/development/ide/compdb/compile_commands.json"
151	return &pb.RepoState{
152		RepoDir:        env.RepoDir,
153		ActiveFilePath: env.CcFiles,
154		OutDir:         env.OutDir,
155		CompDbPath:     path.Join(env.OutDir, compDbPath),
156	}
157}
158
159func runCCanalyzer(ctx context.Context, env *Env, mode string, in []byte) ([]byte, error) {
160	ccAnalyzerPath := path.Join(env.ClangToolsRoot, "bin/ide_query_cc_analyzer")
161	outBuffer := new(bytes.Buffer)
162
163	inBuffer := new(bytes.Buffer)
164	inBuffer.Write(in)
165
166	cmd := exec.CommandContext(ctx, ccAnalyzerPath, "--mode="+mode)
167	cmd.Dir = env.RepoDir
168
169	cmd.Stdin = inBuffer
170	cmd.Stdout = outBuffer
171	cmd.Stderr = os.Stderr
172
173	err := cmd.Run()
174
175	return outBuffer.Bytes(), err
176}
177
178// Execute cc_analyzer and get all the targets that needs to be build for analyzing files.
179func getCCTargets(ctx context.Context, env *Env) ([]string, *pb.Status) {
180	state := repoState(env)
181	bytes, err := proto.Marshal(state)
182	if err != nil {
183		log.Fatalln("Failed to serialize state:", err)
184	}
185
186	resp := new(pb.DepsResponse)
187	result, err := runCCanalyzer(ctx, env, "deps", bytes)
188	if marshal_err := proto.Unmarshal(result, resp); marshal_err != nil {
189		return nil, &pb.Status{
190			Code:    pb.Status_FAILURE,
191			Message: proto.String("Malformed response from cc_analyzer: " + marshal_err.Error()),
192		}
193	}
194
195	var targets []string
196	if resp.Status != nil && resp.Status.Code != pb.Status_OK {
197		return targets, resp.Status
198	}
199	for _, deps := range resp.Deps {
200		targets = append(targets, deps.BuildTarget...)
201	}
202
203	status := &pb.Status{Code: pb.Status_OK}
204	if err != nil {
205		status = &pb.Status{
206			Code:    pb.Status_FAILURE,
207			Message: proto.String(err.Error()),
208		}
209	}
210	return targets, status
211}
212
213func getCCInputs(ctx context.Context, env *Env) *pb.IdeAnalysis {
214	state := repoState(env)
215	bytes, err := proto.Marshal(state)
216	if err != nil {
217		log.Fatalln("Failed to serialize state:", err)
218	}
219
220	resp := new(pb.IdeAnalysis)
221	result, err := runCCanalyzer(ctx, env, "inputs", bytes)
222	if marshal_err := proto.Unmarshal(result, resp); marshal_err != nil {
223		resp.Status = &pb.Status{
224			Code:    pb.Status_FAILURE,
225			Message: proto.String("Malformed response from cc_analyzer: " + marshal_err.Error()),
226		}
227		return resp
228	}
229
230	if err != nil && (resp.Status == nil || resp.Status.Code == pb.Status_OK) {
231		resp.Status = &pb.Status{
232			Code:    pb.Status_FAILURE,
233			Message: proto.String(err.Error()),
234		}
235	}
236	return resp
237}
238
239func getJavaTargets(javaFileToModuleMap map[string]*javaModule) []string {
240	var targets []string
241	for _, m := range javaFileToModuleMap {
242		targets = append(targets, m.Name)
243	}
244	return targets
245}
246
247func getJavaInputs(env *Env, javaModules map[string]*javaModule, javaFileToModuleMap map[string]*javaModule) *pb.IdeAnalysis {
248	var sources []*pb.SourceFile
249	type depsAndGenerated struct {
250		Deps      []string
251		Generated []*pb.GeneratedFile
252	}
253	moduleToDeps := make(map[string]*depsAndGenerated)
254	for _, f := range env.JavaFiles {
255		file := &pb.SourceFile{
256			Path: f,
257		}
258		sources = append(sources, file)
259
260		m := javaFileToModuleMap[f]
261		if m == nil {
262			file.Status = &pb.Status{
263				Code:    pb.Status_FAILURE,
264				Message: proto.String("File not found in any module."),
265			}
266			continue
267		}
268
269		file.Status = &pb.Status{Code: pb.Status_OK}
270		if moduleToDeps[m.Name] != nil {
271			file.Generated = moduleToDeps[m.Name].Generated
272			file.Deps = moduleToDeps[m.Name].Deps
273			continue
274		}
275
276		deps := transitiveDeps(m, javaModules)
277		var generated []*pb.GeneratedFile
278		outPrefix := env.OutDir + "/"
279		for _, d := range deps {
280			if relPath, ok := strings.CutPrefix(d, outPrefix); ok {
281				contents, err := os.ReadFile(d)
282				if err != nil {
283					fmt.Printf("Generated file %q not found - will be skipped.\n", d)
284					continue
285				}
286
287				generated = append(generated, &pb.GeneratedFile{
288					Path:     relPath,
289					Contents: contents,
290				})
291			}
292		}
293		moduleToDeps[m.Name] = &depsAndGenerated{deps, generated}
294		file.Generated = generated
295		file.Deps = deps
296	}
297	return &pb.IdeAnalysis{
298		Sources: sources,
299	}
300}
301
302// runMake runs Soong build for the given modules.
303func runMake(ctx context.Context, env Env, modules ...string) error {
304	args := []string{
305		"--make-mode",
306		"ANDROID_BUILD_ENVIRONMENT_CONFIG=googler-cog",
307		"SOONG_GEN_COMPDB=1",
308		"TARGET_PRODUCT=" + env.LunchTarget.Product,
309		"TARGET_RELEASE=" + env.LunchTarget.Release,
310		"TARGET_BUILD_VARIANT=" + env.LunchTarget.Variant,
311		"-k",
312	}
313	args = append(args, modules...)
314	cmd := exec.CommandContext(ctx, "build/soong/soong_ui.bash", args...)
315	cmd.Dir = env.RepoDir
316	cmd.Stdout = os.Stderr
317	cmd.Stderr = os.Stderr
318	return cmd.Run()
319}
320
321type javaModule struct {
322	Name    string
323	Path    []string `json:"path,omitempty"`
324	Deps    []string `json:"dependencies,omitempty"`
325	Srcs    []string `json:"srcs,omitempty"`
326	Jars    []string `json:"jars,omitempty"`
327	SrcJars []string `json:"srcjars,omitempty"`
328}
329
330func loadJavaModules(env *Env) (map[string]*javaModule, map[string]*javaModule, error) {
331	javaDepsPath := path.Join(env.RepoDir, env.OutDir, "soong/module_bp_java_deps.json")
332	data, err := os.ReadFile(javaDepsPath)
333	if err != nil {
334		return nil, nil, err
335	}
336
337	var moduleMapping map[string]*javaModule // module name -> module
338	if err = json.Unmarshal(data, &moduleMapping); err != nil {
339		return nil, nil, err
340	}
341
342	javaModules := make(map[string]*javaModule)
343	javaFileToModuleMap := make(map[string]*javaModule)
344	for name, module := range moduleMapping {
345		if strings.HasSuffix(name, "-jarjar") || strings.HasSuffix(name, ".impl") {
346			continue
347		}
348		module.Name = name
349		javaModules[name] = module
350		for _, src := range module.Srcs {
351			if !slices.Contains(env.JavaFiles, src) {
352				// We are only interested in active files.
353				continue
354			}
355			if javaFileToModuleMap[src] != nil {
356				// TODO(michaelmerg): Handle the case where a file is covered by multiple modules.
357				log.Printf("File %q found in module %q but is already covered by module %q", src, module.Name, javaFileToModuleMap[src].Name)
358				continue
359			}
360			javaFileToModuleMap[src] = module
361		}
362	}
363	return javaModules, javaFileToModuleMap, nil
364}
365
366func transitiveDeps(m *javaModule, modules map[string]*javaModule) []string {
367	var ret []string
368	q := list.New()
369	q.PushBack(m.Name)
370	seen := make(map[string]bool) // module names -> true
371	for q.Len() > 0 {
372		name := q.Remove(q.Front()).(string)
373		mod := modules[name]
374		if mod == nil {
375			continue
376		}
377
378		ret = append(ret, mod.Srcs...)
379		ret = append(ret, mod.SrcJars...)
380		ret = append(ret, mod.Jars...)
381		for _, d := range mod.Deps {
382			if seen[d] {
383				continue
384			}
385			seen[d] = true
386			q.PushBack(d)
387		}
388	}
389	slices.Sort(ret)
390	ret = slices.Compact(ret)
391	return ret
392}
393