1 // Copyright 2014 The Bazel Authors. All rights reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 //    http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 //
15 // This program creates a "runfiles tree" from a "runfiles manifest".
16 //
17 // The command line arguments are an input manifest INPUT and an output
18 // directory RUNFILES. First, the files in the RUNFILES directory are scanned
19 // and any extraneous ones are removed. Second, any missing files are created.
20 // Finally, a copy of the input manifest is written to RUNFILES/MANIFEST.
21 //
22 // The input manifest consists of lines, each containing a relative path within
23 // the runfiles, a space, and an optional absolute path.  If this second path
24 // is present, a symlink is created pointing to it; otherwise an empty file is
25 // created.
26 //
27 // Given the line
28 //   <workspace root>/output/path /real/path
29 // we will create directories
30 //   RUNFILES/<workspace root>
31 //   RUNFILES/<workspace root>/output
32 // a symlink
33 //   RUNFILES/<workspace root>/output/path -> /real/path
34 // and the output manifest will contain a line
35 //   <workspace root>/output/path /real/path
36 //
37 // If --use_metadata is supplied, every other line is treated as opaque
38 // metadata, and is ignored here.
39 //
40 // All output paths must be relative and generally (but not always) begin with
41 // <workspace root>. No output path may be equal to another.  No output path may
42 // be a path prefix of another.
43 
44 #define _FILE_OFFSET_BITS 64
45 
46 #include <dirent.h>
47 #include <err.h>
48 #include <errno.h>
49 #include <fcntl.h>
50 #include <limits.h>
51 #include <stdio.h>
52 #include <string.h>
53 #include <stdlib.h>
54 #include <sys/stat.h>
55 #include <unistd.h>
56 
57 #include <map>
58 #include <string>
59 
60 // program_invocation_short_name is not portable.
61 static const char *argv0;
62 
63 const char *input_filename;
64 const char *output_base_dir;
65 
66 enum FileType {
67   FILE_TYPE_REGULAR,
68   FILE_TYPE_DIRECTORY,
69   FILE_TYPE_SYMLINK
70 };
71 
72 struct FileInfo {
73   FileType type;
74   std::string symlink_target;
75 
operator ==FileInfo76   bool operator==(const FileInfo &other) const {
77     return type == other.type && symlink_target == other.symlink_target;
78   }
79 
operator !=FileInfo80   bool operator!=(const FileInfo &other) const {
81     return !(*this == other);
82   }
83 };
84 
85 typedef std::map<std::string, FileInfo> FileInfoMap;
86 
87 class RunfilesCreator {
88  public:
RunfilesCreator(const std::string & output_base)89   explicit RunfilesCreator(const std::string &output_base)
90       : output_base_(output_base),
91         output_filename_("MANIFEST"),
92         temp_filename_(output_filename_ + ".tmp") {
93     SetupOutputBase();
94     if (chdir(output_base_.c_str()) != 0) {
95       err(2, "chdir '%s'", output_base_.c_str());
96     }
97   }
98 
ReadManifest(const std::string & manifest_file,bool allow_relative,bool use_metadata)99   void ReadManifest(const std::string &manifest_file, bool allow_relative,
100                     bool use_metadata) {
101     FILE *outfile = fopen(temp_filename_.c_str(), "w");
102     if (!outfile) {
103       err(2, "opening '%s/%s' for writing", output_base_.c_str(),
104            temp_filename_.c_str());
105     }
106     FILE *infile = fopen(manifest_file.c_str(), "r");
107     if (!infile) {
108       err(2, "opening '%s' for reading", manifest_file.c_str());
109     }
110 
111     // read input manifest
112     int lineno = 0;
113     char buf[3 * PATH_MAX];
114     while (fgets(buf, sizeof buf, infile)) {
115       // copy line to output manifest
116       if (fputs(buf, outfile) == EOF) {
117         err(2, "writing to '%s/%s'", output_base_.c_str(),
118              temp_filename_.c_str());
119       }
120 
121       // parse line
122       ++lineno;
123       // Skip metadata lines. They are used solely for
124       // dependency checking.
125       if (use_metadata && lineno % 2 == 0) continue;
126 
127       char *tok = strtok(buf, " \n");
128       if (tok == nullptr) {
129         continue;
130       } else if (*tok == '/') {
131         errx(2, "%s:%d: paths must not be absolute", input_filename, lineno);
132       }
133       std::string link(tok);
134 
135       const char *target = strtok(nullptr, " \n");
136       if (target == nullptr) {
137         target = "";
138       } else if (strtok(nullptr, " \n") != nullptr) {
139         errx(2, "%s:%d: link or target filename contains space", input_filename, lineno);
140       } else if (!allow_relative && target[0] != '/') {
141         errx(2, "%s:%d: expected absolute path", input_filename, lineno);
142       }
143 
144       FileInfo *info = &manifest_[link];
145       if (target[0] == '\0') {
146         // No target means an empty file.
147         info->type = FILE_TYPE_REGULAR;
148       } else {
149         info->type = FILE_TYPE_SYMLINK;
150         info->symlink_target = target;
151       }
152 
153       FileInfo parent_info;
154       parent_info.type = FILE_TYPE_DIRECTORY;
155 
156       while (true) {
157         int k = link.rfind('/');
158         if (k < 0) break;
159         link.erase(k, std::string::npos);
160         if (!manifest_.insert(std::make_pair(link, parent_info)).second) break;
161       }
162     }
163     if (fclose(outfile) != 0) {
164       err(2, "writing to '%s/%s'", output_base_.c_str(),
165            temp_filename_.c_str());
166     }
167     fclose(infile);
168 
169     // Don't delete the temp manifest file.
170     manifest_[temp_filename_].type = FILE_TYPE_REGULAR;
171   }
172 
CreateRunfiles()173   void CreateRunfiles() {
174     if (unlink(output_filename_.c_str()) != 0 && errno != ENOENT) {
175       err(2, "removing previous file at '%s/%s'", output_base_.c_str(),
176            output_filename_.c_str());
177     }
178 
179     ScanTreeAndPrune(".");
180     CreateFiles();
181 
182     // rename output file into place
183     if (rename(temp_filename_.c_str(), output_filename_.c_str()) != 0) {
184       err(2, "renaming '%s/%s' to '%s/%s'",
185            output_base_.c_str(), temp_filename_.c_str(),
186            output_base_.c_str(), output_filename_.c_str());
187     }
188   }
189 
190  private:
SetupOutputBase()191   void SetupOutputBase() {
192     struct stat st;
193     if (stat(output_base_.c_str(), &st) != 0) {
194       // Technically, this will cause problems if the user's umask contains
195       // 0200, but we don't care. Anyone who does that deserves what's coming.
196       if (mkdir(output_base_.c_str(), 0777) != 0) {
197         err(2, "creating directory '%s'", output_base_.c_str());
198       }
199     } else {
200       EnsureDirReadAndWritePerms(output_base_);
201     }
202   }
203 
ScanTreeAndPrune(const std::string & path)204   void ScanTreeAndPrune(const std::string &path) {
205     // A note on non-empty files:
206     // We don't distinguish between empty and non-empty files. That is, if
207     // there's a file that has contents, we don't truncate it here, even though
208     // the manifest supports creation of empty files, only. Given that
209     // .runfiles are *supposed* to be immutable, this shouldn't be a problem.
210     EnsureDirReadAndWritePerms(path);
211 
212     struct dirent *entry;
213     DIR *dh = opendir(path.c_str());
214     if (!dh) {
215       err(2, "opendir '%s'", path.c_str());
216     }
217 
218     errno = 0;
219     const std::string prefix = (path == "." ? "" : path + "/");
220     while ((entry = readdir(dh)) != nullptr) {
221       if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) continue;
222 
223       std::string entry_path = prefix + entry->d_name;
224       FileInfo actual_info;
225       actual_info.type = DentryToFileType(entry_path, entry);
226 
227       if (actual_info.type == FILE_TYPE_SYMLINK) {
228         ReadLinkOrDie(entry_path, &actual_info.symlink_target);
229       }
230 
231       FileInfoMap::iterator expected_it = manifest_.find(entry_path);
232       if (expected_it == manifest_.end() ||
233           expected_it->second != actual_info) {
234         DelTree(entry_path, actual_info.type);
235       } else {
236         manifest_.erase(expected_it);
237         if (actual_info.type == FILE_TYPE_DIRECTORY) {
238           ScanTreeAndPrune(entry_path);
239         }
240       }
241 
242       errno = 0;
243     }
244     if (errno != 0) {
245       err(2, "reading directory '%s'", path.c_str());
246     }
247     closedir(dh);
248   }
249 
CreateFiles()250   void CreateFiles() {
251     for (FileInfoMap::const_iterator it = manifest_.begin();
252          it != manifest_.end(); ++it) {
253       const std::string &path = it->first;
254       switch (it->second.type) {
255         case FILE_TYPE_DIRECTORY:
256           if (mkdir(path.c_str(), 0777) != 0) {
257             err(2, "mkdir '%s'", path.c_str());
258           }
259           break;
260         case FILE_TYPE_REGULAR:
261           {
262             int fd = open(path.c_str(), O_CREAT|O_EXCL|O_WRONLY, 0555);
263             if (fd < 0) {
264               err(2, "creating empty file '%s'", path.c_str());
265             }
266             close(fd);
267           }
268           break;
269         case FILE_TYPE_SYMLINK:
270           {
271             const std::string& target = it->second.symlink_target;
272             if (symlink(target.c_str(), path.c_str()) != 0) {
273               err(2, "symlinking '%s' -> '%s'", path.c_str(), target.c_str());
274             }
275           }
276           break;
277       }
278     }
279   }
280 
DentryToFileType(const std::string & path,struct dirent * ent)281   FileType DentryToFileType(const std::string &path, struct dirent *ent) {
282 #ifdef _DIRENT_HAVE_D_TYPE
283     if (ent->d_type != DT_UNKNOWN) {
284       if (ent->d_type == DT_DIR) {
285         return FILE_TYPE_DIRECTORY;
286       } else if (ent->d_type == DT_LNK) {
287         return FILE_TYPE_SYMLINK;
288       } else {
289         return FILE_TYPE_REGULAR;
290       }
291     } else  // NOLINT (the brace is in the next line)
292 #endif
293     {
294       struct stat st;
295       LStatOrDie(path, &st);
296       if (S_ISDIR(st.st_mode)) {
297         return FILE_TYPE_DIRECTORY;
298       } else if (S_ISLNK(st.st_mode)) {
299         return FILE_TYPE_SYMLINK;
300       } else {
301         return FILE_TYPE_REGULAR;
302       }
303     }
304   }
305 
LStatOrDie(const std::string & path,struct stat * st)306   void LStatOrDie(const std::string &path, struct stat *st) {
307     if (lstat(path.c_str(), st) != 0) {
308       err(2, "lstating file '%s'", path.c_str());
309     }
310   }
311 
StatOrDie(const std::string & path,struct stat * st)312   void StatOrDie(const std::string &path, struct stat *st) {
313     if (stat(path.c_str(), st) != 0) {
314       err(2, "stating file '%s'", path.c_str());
315     }
316   }
317 
ReadLinkOrDie(const std::string & path,std::string * output)318   void ReadLinkOrDie(const std::string &path, std::string *output) {
319     char readlink_buffer[PATH_MAX];
320     int sz = readlink(path.c_str(), readlink_buffer, sizeof(readlink_buffer));
321     if (sz < 0) {
322       err(2, "reading symlink '%s'", path.c_str());
323     }
324     // readlink returns a non-null terminated string.
325     std::string(readlink_buffer, sz).swap(*output);
326   }
327 
EnsureDirReadAndWritePerms(const std::string & path)328   void EnsureDirReadAndWritePerms(const std::string &path) {
329     const int kMode = 0700;
330     struct stat st;
331     LStatOrDie(path, &st);
332     if ((st.st_mode & kMode) != kMode) {
333       int new_mode = st.st_mode | kMode;
334       if (chmod(path.c_str(), new_mode) != 0) {
335         err(2, "chmod '%s'", path.c_str());
336       }
337     }
338   }
339 
DelTree(const std::string & path,FileType file_type)340   bool DelTree(const std::string &path, FileType file_type) {
341     if (file_type != FILE_TYPE_DIRECTORY) {
342       if (unlink(path.c_str()) != 0) {
343         err(2, "unlinking '%s'", path.c_str());
344         return false;
345       }
346       return true;
347     }
348 
349     EnsureDirReadAndWritePerms(path);
350 
351     struct dirent *entry;
352     DIR *dh = opendir(path.c_str());
353     if (!dh) {
354       err(2, "opendir '%s'", path.c_str());
355     }
356     errno = 0;
357     while ((entry = readdir(dh)) != nullptr) {
358       if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, "..")) continue;
359       const std::string entry_path = path + '/' + entry->d_name;
360       FileType entry_file_type = DentryToFileType(entry_path, entry);
361       DelTree(entry_path, entry_file_type);
362       errno = 0;
363     }
364     if (errno != 0) {
365       err(2, "readdir '%s'", path.c_str());
366     }
367     closedir(dh);
368     if (rmdir(path.c_str()) != 0) {
369       err(2, "rmdir '%s'", path.c_str());
370     }
371     return true;
372   }
373 
374  private:
375   std::string output_base_;
376   std::string output_filename_;
377   std::string temp_filename_;
378 
379   FileInfoMap manifest_;
380 };
381 
main(int argc,char ** argv)382 int main(int argc, char **argv) {
383   argv0 = argv[0];
384 
385   argc--; argv++;
386   bool allow_relative = false;
387   bool use_metadata = false;
388 
389   while (argc >= 1) {
390     if (strcmp(argv[0], "--allow_relative") == 0) {
391       allow_relative = true;
392       argc--; argv++;
393     } else if (strcmp(argv[0], "--use_metadata") == 0) {
394       use_metadata = true;
395       argc--; argv++;
396     } else {
397       break;
398     }
399   }
400 
401   if (argc != 2) {
402     fprintf(stderr, "usage: %s "
403             "[--allow_relative] [--use_metadata] "
404             "INPUT RUNFILES\n",
405             argv0);
406     return 1;
407   }
408 
409   input_filename = argv[0];
410   output_base_dir = argv[1];
411 
412   std::string manifest_file = input_filename;
413   if (input_filename[0] != '/') {
414     char cwd_buf[PATH_MAX];
415     if (getcwd(cwd_buf, sizeof(cwd_buf)) == nullptr) {
416       err(2, "getcwd failed");
417     }
418     manifest_file = std::string(cwd_buf) + '/' + manifest_file;
419   }
420 
421   RunfilesCreator runfiles_creator(output_base_dir);
422   runfiles_creator.ReadManifest(manifest_file, allow_relative, use_metadata);
423   runfiles_creator.CreateRunfiles();
424 
425   return 0;
426 }
427