Lines Matching refs:f
173 logger Logger, dbPath string) (f *Finder, err error) {
179 logger Logger, dbPath string, numThreads int) (f *Finder, err error) {
191 f = &Finder{
204 f.loadFromFilesystem()
207 err = f.getErr()
215 path = filepath.Join(f.cacheMetadata.Config.WorkingDirectory, path)
217 node := f.nodes.GetNode(filepath.Clean(path), false)
223 return f, nil
227 func (f *Finder) FindAll() []string {
228 return f.FindAt("/")
232 func (f *Finder) FindAt(rootDir string) []string {
236 return f.FindMatching(rootDir, filter)
240 func (f *Finder) FindNamed(fileName string) []string {
241 return f.FindNamedAt("/", fileName)
247 func (f *Finder) FindNamedAt(rootPath string, fileName string) []string {
258 return f.FindMatching(rootPath, filter)
263 func (f *Finder) FindFirstNamed(fileName string) []string {
264 return f.FindFirstNamedAt("/", fileName)
269 func (f *Finder) FindFirstNamedAt(rootPath string, fileName string) []string {
283 return f.FindMatching(rootPath, filter)
290 func (f *Finder) FindMatching(rootPath string, filter WalkFunc) []string {
294 workingDir := f.cacheMetadata.Config.WorkingDirectory
304 f.verbosef("FindMatching waiting for finder to be idle\n")
305 f.lock()
306 defer f.unlock()
308 node := f.nodes.GetNode(rootPath, false)
310 f.verbosef("No data for path %v ; apparently not included in cache params: %v\n",
311 rootPath, f.cacheMetadata.Config.CacheParams)
317 f.verbosef("Finder finding %v using cache\n", rootPath)
318 results := f.findInCacheMultithreaded(node, filter, f.numSearchingThreads)
327 f.verbosef("Found %v files under %v in %v using cache\n",
334 func (f *Finder) Shutdown() {
335 f.WaitForDbDump()
339 func (f *Finder) WaitForDbDump() {
340 f.shutdownWaitgroup.Wait()
345 func (f *Finder) goDumpDb() {
346 if f.wasModified() {
347 f.shutdownWaitgroup.Add(1)
349 err := f.dumpDb()
351 f.verbosef("%v\n", err)
353 f.shutdownWaitgroup.Done()
356 f.verbosef("Skipping dumping unmodified db\n")
375 func (f *Finder) verbosef(format string, args ...interface{}) {
376 f.logger.Output(2, fmt.Sprintf(format, args...))
380 func (f *Finder) loadFromFilesystem() {
381 f.threadPool = newThreadPool(f.numDbLoadingThreads)
383 err := f.startFromExternalCache()
385 f.startWithoutExternalCache()
388 f.goDumpDb()
390 f.threadPool = nil
393 func (f *Finder) startFind(path string) {
395 path = filepath.Join(f.cacheMetadata.Config.WorkingDirectory, path)
397 node := f.nodes.GetNode(path, true)
398 f.statDirAsync(node)
401 func (f *Finder) lock() {
402 f.mutex.Lock()
405 func (f *Finder) unlock() {
406 f.mutex.Unlock()
655 func (f *Finder) serializeCacheEntry(dirInfos []dirFullInfo) ([]byte, error) {
704 func (f *Finder) parseCacheEntry(bytes []byte) ([]dirFullInfo, error) {
750 func (f *Finder) readLine(reader *bufio.Reader) ([]byte, error) {
755 func (f *Finder) validateCacheHeader(cacheReader *bufio.Reader) bool {
756 cacheVersionBytes, err := f.readLine(cacheReader)
758 f.verbosef("Failed to read database header; database is invalid\n")
765 currentVersion := f.cacheMetadata.Version
767 …f.verbosef("Version changed from %q to %q, database is not applicable\n", cacheVersionString, curr…
771 cacheParamBytes, err := f.readLine(cacheReader)
773 f.verbosef("Failed to read database search params; database is invalid\n")
781 currentParamBytes, err := f.cacheMetadata.Config.Dump()
788 …f.verbosef("Params changed from %q to %q, database is not applicable\n", cacheParamString, current…
796 func (f *Finder) loadBytes(id int, data []byte) (m *pathMap, dirsToWalk []string, err error) {
800 cachedNodes, err := f.parseCacheEntry(data)
806 f.verbosef("Unmarshaled %v objects for %v in %v\n",
814 stats[i] = f.statDirSync(node.Path)
826 if !f.isInfoUpToDate(cachedNode.statResponse, updated) && updated.ModTime != 0 {
827 f.setModified()
838 f.verbosef("Statted inodes of block %v in %v\n", id, time.Now().Sub(unmarshalDate))
845 func (f *Finder) startFromExternalCache() (err error) {
847 dbPath := f.DbPath
850 reader, err := f.filesystem.Open(dbPath)
856 if !f.validateCacheHeader(bufferedReader) {
859 f.verbosef("Database header matches, will attempt to use database %v\n", f.DbPath)
871 blockChannel := make(chan dataBlock, f.numDbLoadingThreads)
879 data, err := f.readLine(bufferedReader)
892 f.verbosef("Read block %v after %v\n", index, duration)
894 f.verbosef("Read %v blocks in %v\n", index, duration)
912 threadPool := newThreadPool(f.numDbLoadingThreads)
929 f.verbosef("Starting to process block %v after %v\n",
931 tempMap, updatedDirs, err := f.loadBytes(block.id, block.data)
934 f.verbosef(
946 f.verbosef("Processed block %v in %v\n",
954 f.verbosef("Finished processing %v blocks in %v\n",
996 f.nodes = *mainTree
1002 f.listDirsAsync(nodesToWalk[i])
1004 f.verbosef("Loaded db and statted known dirs in %v\n", time.Since(startTime))
1005 f.threadPool.Wait()
1006 f.verbosef("Loaded db and statted all dirs in %v\n", time.Now().Sub(startTime))
1013 func (f *Finder) startWithoutExternalCache() {
1015 configDirs := f.cacheMetadata.Config.RootDirs
1039 f.verbosef("Starting find of %v\n", path)
1040 f.startFind(path)
1043 f.threadPool.Wait()
1045 f.verbosef("Scanned filesystem (not using cache) in %v\n", time.Now().Sub(startTime))
1049 func (f *Finder) isInfoUpToDate(old statResponse, new statResponse) (equal bool) {
1062 func (f *Finder) wasModified() bool {
1063 return atomic.LoadInt32(&f.modifiedFlag) > 0
1066 func (f *Finder) setModified() {
1069 atomic.StoreInt32(&f.modifiedFlag, newVal)
1073 func (f *Finder) sortedDirEntries() []dirFullInfo {
1076 for _, node := range f.nodes.DumpAll() {
1082 f.verbosef("Generated %v cache entries in %v\n", len(nodes), discoveryDate.Sub(startTime))
1088 f.verbosef("Sorted %v cache entries in %v\n", len(nodes), sortDate.Sub(discoveryDate))
1094 func (f *Finder) serializeDb() ([]byte, error) {
1096 var entryList = f.sortedDirEntries()
1104 header = append(header, []byte(f.cacheMetadata.Version)...)
1106 configDump, err := f.cacheMetadata.Config.Dump()
1113 numBlocks := f.numDbLoadingThreads
1131 byteBlock, subErr := f.serializeCacheEntry(block)
1132 f.verbosef("Serialized block %v into %v bytes\n", index, len(byteBlock))
1134 f.verbosef("%v\n", subErr.Error())
1159 func (f *Finder) dumpDb() error {
1161 f.verbosef("Dumping db\n")
1163 tempPath := f.DbPath + ".tmp"
1165 bytes, err := f.serializeDb()
1170 f.verbosef("Serialized db in %v\n", serializeDate.Sub(startTime))
1172 err = f.filesystem.WriteFile(tempPath, bytes, 0777)
1176 err = f.filesystem.Rename(tempPath, f.DbPath)
1181 f.verbosef("Wrote db in %v\n", time.Now().Sub(serializeDate))
1187 func (f *Finder) canIgnoreFsErr(err error) bool {
1208 func (f *Finder) onFsError(path string, err error) {
1209 if !f.canIgnoreFsErr(err) {
1215 f.errlock.Lock()
1216 f.fsErrs = append(f.fsErrs, fsErr{path: path, err: err})
1217 f.errlock.Unlock()
1222 func (f *Finder) discardErrsForPrunedPaths() {
1225 relevantErrs := make([]fsErr, 0, len(f.fsErrs))
1226 for _, fsErr := range f.fsErrs {
1228 node := f.nodes.GetNode(path, false)
1235 f.fsErrs = relevantErrs
1239 func (f *Finder) getErr() error {
1240 f.discardErrsForPrunedPaths()
1242 numErrs := len(f.fsErrs)
1250 …message = fmt.Sprintf("finder encountered %v errors: %v...", numErrs, f.fsErrs[:maxNumErrsToInclud…
1252 message = fmt.Sprintf("finder encountered %v errors: %v", numErrs, f.fsErrs)
1258 func (f *Finder) statDirAsync(dir *pathMap) {
1261 f.threadPool.Run(
1263 updatedStats := f.statDirSync(path)
1265 if !f.isInfoUpToDate(node.statResponse, updatedStats) {
1270 f.setModified()
1274 f.listDirAsync(dir)
1281 func (f *Finder) statDirSync(path string) statResponse {
1283 fileInfo, err := f.filesystem.Lstat(path)
1288 f.onFsError(path, err)
1294 inode, err := f.filesystem.InodeNumber(fileInfo)
1299 device, err := f.filesystem.DeviceNumber(fileInfo)
1304 permissionsChangeTime, err := f.filesystem.PermTime(fileInfo)
1321 func (f *Finder) shouldIncludeFile(fileName string) bool {
1322 for _, includedName := range f.cacheMetadata.Config.IncludeFiles {
1327 for _, includeSuffix := range f.cacheMetadata.Config.IncludeSuffixes {
1336 func (f *Finder) pruneCacheCandidates(items *DirEntries) {
1339 for _, abortedName := range f.cacheMetadata.Config.PruneFiles {
1351 if f.shouldIncludeFile(fileName) {
1364 for _, excludedName := range f.cacheMetadata.Config.ExcludeDirs {
1379 func (f *Finder) listDirsAsync(nodes []*pathMap) {
1380 f.threadPool.Run(
1383 f.listDirSync(nodes[i])
1389 func (f *Finder) listDirAsync(node *pathMap) {
1390 f.threadPool.Run(
1392 f.listDirSync(node)
1397 func (f *Finder) listDirSync(dir *pathMap) {
1399 children, err := f.filesystem.ReadDir(path)
1403 f.onFsError(path, err)
1417 childStat, err := f.filesystem.Stat(childPath)
1427 if f.cacheMetadata.Config.FollowSymlinks {
1446 f.pruneCacheCandidates(entry)
1456 f.statDirAsync(childNode)
1476 func (f *Finder) listMatches(node *pathMap,
1505 func (f *Finder) findInCacheMultithreaded(node *pathMap, filter WalkFunc,
1510 return f.findInCacheSinglethreaded(node, filter)
1519 subDirs, filePaths := f.listMatches(node, filter)
1525 childResults := f.findInCacheMultithreaded(child, filter, numChildThreads)
1546 func (f *Finder) findInCacheSinglethreaded(node *pathMap, filter WalkFunc) []string {
1558 subDirs, filePaths := f.listMatches(currentNode, filter)