/* * Copyright © 2020 Valve Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ /* This is a basic c implementation of a fossilize db like format intended for * use with the Mesa shader cache. * * The format is compatible enough to allow the fossilize db tools to be used * to do things like merge db collections. */ #include "fossilize_db.h" #ifdef FOZ_DB_UTIL #include #include #include #include #include #include #include #include #ifdef FOZ_DB_UTIL_DYNAMIC_LIST #include #endif #include "util/u_debug.h" #include "crc32.h" #include "hash_table.h" #include "mesa-sha1.h" #include "ralloc.h" #define FOZ_REF_MAGIC_SIZE 16 static const uint8_t stream_reference_magic_and_version[FOZ_REF_MAGIC_SIZE] = { 0x81, 'F', 'O', 'S', 'S', 'I', 'L', 'I', 'Z', 'E', 'D', 'B', 0, 0, 0, FOSSILIZE_FORMAT_VERSION, /* 4 bytes to use for versioning. */ }; /* Mesa uses 160bit hashes to identify cache entries, a hash of this size * makes collisions virtually impossible for our use case. However the foz db * format uses a 64bit hash table to lookup file offsets for reading cache * entries so we must shorten our hash. */ static uint64_t truncate_hash_to_64bits(const uint8_t *cache_key) { uint64_t hash = 0; unsigned shift = 7; for (unsigned i = 0; i < 8; i++) { hash |= ((uint64_t)cache_key[i]) << shift * 8; shift--; } return hash; } static bool check_files_opened_successfully(FILE *file, FILE *db_idx) { if (!file) { if (db_idx) fclose(db_idx); return false; } if (!db_idx) { if (file) fclose(file); return false; } return true; } static bool create_foz_db_filenames(const char *cache_path, char *name, char **filename, char **idx_filename) { if (asprintf(filename, "%s/%s.foz", cache_path, name) == -1) return false; if (asprintf(idx_filename, "%s/%s_idx.foz", cache_path, name) == -1) { free(*filename); return false; } return true; } /* This looks at stuff that was added to the index since the last time we looked at it. This is safe * to do without locking the file as we assume the file is append only */ static void update_foz_index(struct foz_db *foz_db, FILE *db_idx, unsigned file_idx) { uint64_t offset = ftell(db_idx); fseek(db_idx, 0, SEEK_END); uint64_t len = ftell(db_idx); uint64_t parsed_offset = offset; if (offset == len) return; fseek(db_idx, offset, SEEK_SET); while (offset < len) { char bytes_to_read[FOSSILIZE_BLOB_HASH_LENGTH + sizeof(struct foz_payload_header)]; struct foz_payload_header *header; /* Corrupt entry. Our process might have been killed before we * could write all data. */ if (offset + sizeof(bytes_to_read) > len) break; /* NAME + HEADER in one read */ if (fread(bytes_to_read, 1, sizeof(bytes_to_read), db_idx) != sizeof(bytes_to_read)) break; offset += sizeof(bytes_to_read); header = (struct foz_payload_header*)&bytes_to_read[FOSSILIZE_BLOB_HASH_LENGTH]; /* Corrupt entry. Our process might have been killed before we * could write all data. */ if (offset + header->payload_size > len || header->payload_size != sizeof(uint64_t)) break; char hash_str[FOSSILIZE_BLOB_HASH_LENGTH + 1] = {0}; memcpy(hash_str, bytes_to_read, FOSSILIZE_BLOB_HASH_LENGTH); /* read cache item offset from index file */ uint64_t cache_offset; if (fread(&cache_offset, 1, sizeof(cache_offset), db_idx) != sizeof(cache_offset)) break; offset += header->payload_size; parsed_offset = offset; struct foz_db_entry *entry = ralloc(foz_db->mem_ctx, struct foz_db_entry); entry->header = *header; entry->file_idx = file_idx; _mesa_sha1_hex_to_sha1(entry->key, hash_str); /* Truncate the entry's hash string to a 64bit hash for use with a * 64bit hash table for looking up file offsets. */ hash_str[16] = '\0'; uint64_t key = strtoull(hash_str, NULL, 16); entry->offset = cache_offset; _mesa_hash_table_u64_insert(foz_db->index_db, key, entry); } fseek(db_idx, parsed_offset, SEEK_SET); } /* exclusive flock with timeout. timeout is in nanoseconds */ static int lock_file_with_timeout(FILE *f, int64_t timeout) { int err; int fd = fileno(f); int64_t iterations = MAX2(DIV_ROUND_UP(timeout, 1000000), 1); /* Since there is no blocking flock with timeout and we don't want to totally spin on getting the * lock, use a nonblocking method and retry every millisecond. */ for (int64_t iter = 0; iter < iterations; ++iter) { err = flock(fd, LOCK_EX | LOCK_NB); if (err == 0 || errno != EAGAIN) break; usleep(1000); } return err; } static bool load_foz_dbs(struct foz_db *foz_db, FILE *db_idx, uint8_t file_idx, bool read_only) { /* Scan through the archive and get the list of cache entries. */ fseek(db_idx, 0, SEEK_END); size_t len = ftell(db_idx); rewind(db_idx); /* Try not to take the lock if len >= the size of the header, but if it is smaller we take the * lock to potentially initialize the files. */ if (len < sizeof(stream_reference_magic_and_version)) { /* Wait for 100 ms in case of contention, after that we prioritize getting the app started. */ int err = lock_file_with_timeout(foz_db->file[file_idx], 100000000); if (err == -1) goto fail; /* Compute length again so we know nobody else did it in the meantime */ fseek(db_idx, 0, SEEK_END); len = ftell(db_idx); rewind(db_idx); } if (len != 0) { uint8_t magic[FOZ_REF_MAGIC_SIZE]; if (fread(magic, 1, FOZ_REF_MAGIC_SIZE, db_idx) != FOZ_REF_MAGIC_SIZE) goto fail; if (memcmp(magic, stream_reference_magic_and_version, FOZ_REF_MAGIC_SIZE - 1)) goto fail; int version = magic[FOZ_REF_MAGIC_SIZE - 1]; if (version > FOSSILIZE_FORMAT_VERSION || version < FOSSILIZE_FORMAT_MIN_COMPAT_VERSION) goto fail; } else { /* Appending to a fresh file. Make sure we have the magic. */ if (fwrite(stream_reference_magic_and_version, 1, sizeof(stream_reference_magic_and_version), foz_db->file[file_idx]) != sizeof(stream_reference_magic_and_version)) goto fail; if (fwrite(stream_reference_magic_and_version, 1, sizeof(stream_reference_magic_and_version), db_idx) != sizeof(stream_reference_magic_and_version)) goto fail; fflush(foz_db->file[file_idx]); fflush(db_idx); } flock(fileno(foz_db->file[file_idx]), LOCK_UN); if (foz_db->updater.thrd) { /* If MESA_DISK_CACHE_READ_ONLY_FOZ_DBS_DYNAMIC_LIST is enabled, access to * the foz_db hash table requires locking to prevent racing between this * updated thread loading DBs at runtime and cache entry read/writes. */ simple_mtx_lock(&foz_db->mtx); update_foz_index(foz_db, db_idx, file_idx); simple_mtx_unlock(&foz_db->mtx); } else { update_foz_index(foz_db, db_idx, file_idx); } foz_db->alive = true; return true; fail: flock(fileno(foz_db->file[file_idx]), LOCK_UN); return false; } static void load_foz_dbs_ro(struct foz_db *foz_db, char *foz_dbs_ro) { uint8_t file_idx = 1; char *filename = NULL; char *idx_filename = NULL; for (unsigned n; n = strcspn(foz_dbs_ro, ","), *foz_dbs_ro; foz_dbs_ro += MAX2(1, n)) { char *foz_db_filename = strndup(foz_dbs_ro, n); filename = NULL; idx_filename = NULL; if (!create_foz_db_filenames(foz_db->cache_path, foz_db_filename, &filename, &idx_filename)) { free(foz_db_filename); continue; /* Ignore invalid user provided filename and continue */ } free(foz_db_filename); /* Open files as read only */ foz_db->file[file_idx] = fopen(filename, "rb"); FILE *db_idx = fopen(idx_filename, "rb"); free(filename); free(idx_filename); if (!check_files_opened_successfully(foz_db->file[file_idx], db_idx)) { /* Prevent foz_destroy from destroying it a second time. */ foz_db->file[file_idx] = NULL; continue; /* Ignore invalid user provided filename and continue */ } if (!load_foz_dbs(foz_db, db_idx, file_idx, true)) { fclose(db_idx); fclose(foz_db->file[file_idx]); foz_db->file[file_idx] = NULL; continue; /* Ignore invalid user provided foz db */ } fclose(db_idx); file_idx++; if (file_idx >= FOZ_MAX_DBS) break; } } #ifdef FOZ_DB_UTIL_DYNAMIC_LIST static bool check_file_already_loaded(struct foz_db *foz_db, FILE *db_file, uint8_t max_file_idx) { struct stat new_file_stat; if (fstat(fileno(db_file), &new_file_stat) == -1) return false; for (int i = 0; i < max_file_idx; i++) { struct stat loaded_file_stat; if (fstat(fileno(foz_db->file[i]), &loaded_file_stat) == -1) continue; if ((loaded_file_stat.st_dev == new_file_stat.st_dev) && (loaded_file_stat.st_ino == new_file_stat.st_ino)) return true; } return false; } static bool load_from_list_file(struct foz_db *foz_db, const char *foz_dbs_list_filename) { uint8_t file_idx; char list_entry[PATH_MAX]; /* Find the first empty file idx slot */ for (file_idx = 0; file_idx < FOZ_MAX_DBS; file_idx++) { if (!foz_db->file[file_idx]) break; } if (file_idx >= FOZ_MAX_DBS) return false; FILE *foz_dbs_list_file = fopen(foz_dbs_list_filename, "rb"); if (!foz_dbs_list_file) return false; while (fgets(list_entry, sizeof(list_entry), foz_dbs_list_file)) { char *db_filename = NULL; char *idx_filename = NULL; FILE *db_file = NULL; FILE *idx_file = NULL; list_entry[strcspn(list_entry, "\n")] = '\0'; if (!create_foz_db_filenames(foz_db->cache_path, list_entry, &db_filename, &idx_filename)) continue; db_file = fopen(db_filename, "rb"); idx_file = fopen(idx_filename, "rb"); free(db_filename); free(idx_filename); if (!check_files_opened_successfully(db_file, idx_file)) continue; if (check_file_already_loaded(foz_db, db_file, file_idx)) { fclose(db_file); fclose(idx_file); continue; } /* Must be set before calling load_foz_dbs() */ foz_db->file[file_idx] = db_file; if (!load_foz_dbs(foz_db, idx_file, file_idx, true)) { fclose(db_file); fclose(idx_file); foz_db->file[file_idx] = NULL; continue; } fclose(idx_file); file_idx++; if (file_idx >= FOZ_MAX_DBS) break; } fclose(foz_dbs_list_file); return true; } static int foz_dbs_list_updater_thrd(void *data) { char buf[10 * (sizeof(struct inotify_event) + NAME_MAX + 1)]; struct foz_db *foz_db = data; struct foz_dbs_list_updater *updater = &foz_db->updater; while (1) { int len = read(updater->inotify_fd, buf, sizeof(buf)); if (len == -1 && errno != EAGAIN) return errno; int i = 0; while (i < len) { struct inotify_event *event = (struct inotify_event *)&buf[i]; i += sizeof(struct inotify_event) + event->len; if (event->mask & IN_CLOSE_WRITE) load_from_list_file(foz_db, foz_db->updater.list_filename); /* List file deleted or watch removed by foz destroy */ if ((event->mask & IN_DELETE_SELF) || (event->mask & IN_IGNORED)) return 0; } } return 0; } static bool foz_dbs_list_updater_init(struct foz_db *foz_db, char *list_filename) { struct foz_dbs_list_updater *updater = &foz_db->updater; /* Initial load */ if (!load_from_list_file(foz_db, list_filename)) return false; updater->list_filename = list_filename; int fd = inotify_init1(IN_CLOEXEC); if (fd < 0) return false; int wd = inotify_add_watch(fd, foz_db->updater.list_filename, IN_CLOSE_WRITE | IN_DELETE_SELF); if (wd < 0) { close(fd); return false; } updater->inotify_fd = fd; updater->inotify_wd = wd; if (thrd_create(&updater->thrd, foz_dbs_list_updater_thrd, foz_db)) { inotify_rm_watch(fd, wd); close(fd); return false; } return true; } #endif /* Here we open mesa cache foz dbs files. If the files exist we load the index * db into a hash table. The index db contains the offsets needed to later * read cache entries from the foz db containing the actual cache entries. */ bool foz_prepare(struct foz_db *foz_db, char *cache_path) { char *filename = NULL; char *idx_filename = NULL; simple_mtx_init(&foz_db->mtx, mtx_plain); simple_mtx_init(&foz_db->flock_mtx, mtx_plain); foz_db->mem_ctx = ralloc_context(NULL); foz_db->index_db = _mesa_hash_table_u64_create(NULL); foz_db->cache_path = cache_path; /* Open the default foz dbs for read/write. If the files didn't already exist * create them. */ if (debug_get_bool_option("MESA_DISK_CACHE_SINGLE_FILE", false)) { if (!create_foz_db_filenames(cache_path, "foz_cache", &filename, &idx_filename)) goto fail; foz_db->file[0] = fopen(filename, "a+b"); foz_db->db_idx = fopen(idx_filename, "a+b"); free(filename); free(idx_filename); if (!check_files_opened_successfully(foz_db->file[0], foz_db->db_idx)) goto fail; if (!load_foz_dbs(foz_db, foz_db->db_idx, 0, false)) goto fail; } char *foz_dbs_ro = getenv("MESA_DISK_CACHE_READ_ONLY_FOZ_DBS"); if (foz_dbs_ro) load_foz_dbs_ro(foz_db, foz_dbs_ro); #ifdef FOZ_DB_UTIL_DYNAMIC_LIST char *foz_dbs_list = getenv("MESA_DISK_CACHE_READ_ONLY_FOZ_DBS_DYNAMIC_LIST"); if (foz_dbs_list) foz_dbs_list_updater_init(foz_db, foz_dbs_list); #endif return true; fail: foz_destroy(foz_db); return false; } void foz_destroy(struct foz_db *foz_db) { #ifdef FOZ_DB_UTIL_DYNAMIC_LIST struct foz_dbs_list_updater *updater = &foz_db->updater; if (updater->thrd) { inotify_rm_watch(updater->inotify_fd, updater->inotify_wd); /* inotify_rm_watch() triggers the IN_IGNORE event for the thread * to exit. */ thrd_join(updater->thrd, NULL); close(updater->inotify_fd); } #endif if (foz_db->db_idx) fclose(foz_db->db_idx); for (unsigned i = 0; i < FOZ_MAX_DBS; i++) { if (foz_db->file[i]) fclose(foz_db->file[i]); } if (foz_db->mem_ctx) { _mesa_hash_table_u64_destroy(foz_db->index_db); ralloc_free(foz_db->mem_ctx); simple_mtx_destroy(&foz_db->flock_mtx); simple_mtx_destroy(&foz_db->mtx); } memset(foz_db, 0, sizeof(*foz_db)); } /* Here we lookup a cache entry in the index hash table. If an entry is found * we use the retrieved offset to read the cache entry from disk. */ void * foz_read_entry(struct foz_db *foz_db, const uint8_t *cache_key_160bit, size_t *size) { uint64_t hash = truncate_hash_to_64bits(cache_key_160bit); void *data = NULL; if (!foz_db->alive) return NULL; simple_mtx_lock(&foz_db->mtx); struct foz_db_entry *entry = _mesa_hash_table_u64_search(foz_db->index_db, hash); if (!entry && foz_db->db_idx) { update_foz_index(foz_db, foz_db->db_idx, 0); entry = _mesa_hash_table_u64_search(foz_db->index_db, hash); } if (!entry) { simple_mtx_unlock(&foz_db->mtx); return NULL; } uint8_t file_idx = entry->file_idx; if (fseek(foz_db->file[file_idx], entry->offset, SEEK_SET) < 0) goto fail; uint32_t header_size = sizeof(struct foz_payload_header); if (fread(&entry->header, 1, header_size, foz_db->file[file_idx]) != header_size) goto fail; /* Check for collision using full 160bit hash for increased assurance * against potential collisions. */ for (int i = 0; i < 20; i++) { if (cache_key_160bit[i] != entry->key[i]) goto fail; } uint32_t data_sz = entry->header.payload_size; data = malloc(data_sz); if (fread(data, 1, data_sz, foz_db->file[file_idx]) != data_sz) goto fail; /* verify checksum */ if (entry->header.crc != 0) { if (util_hash_crc32(data, data_sz) != entry->header.crc) goto fail; } simple_mtx_unlock(&foz_db->mtx); if (size) *size = data_sz; return data; fail: free(data); /* reading db entry failed. reset the file offset */ simple_mtx_unlock(&foz_db->mtx); return NULL; } /* Here we write the cache entry to disk and store its offset in the index db. */ bool foz_write_entry(struct foz_db *foz_db, const uint8_t *cache_key_160bit, const void *blob, size_t blob_size) { uint64_t hash = truncate_hash_to_64bits(cache_key_160bit); if (!foz_db->alive || !foz_db->file[0]) return false; /* The flock is per-fd, not per thread, we do it outside of the main mutex to avoid having to * wait in the mutex potentially blocking reads. We use the secondary flock_mtx to stop race * conditions between the write threads sharing the same file descriptor. */ simple_mtx_lock(&foz_db->flock_mtx); /* Wait for 1 second. This is done outside of the main mutex as I believe there is more potential * for file contention than mtx contention of significant length. */ int err = lock_file_with_timeout(foz_db->file[0], 1000000000); if (err == -1) goto fail_file; simple_mtx_lock(&foz_db->mtx); update_foz_index(foz_db, foz_db->db_idx, 0); struct foz_db_entry *entry = _mesa_hash_table_u64_search(foz_db->index_db, hash); if (entry) { simple_mtx_unlock(&foz_db->mtx); flock(fileno(foz_db->file[0]), LOCK_UN); simple_mtx_unlock(&foz_db->flock_mtx); return NULL; } /* Prepare db entry header and blob ready for writing */ struct foz_payload_header header; header.uncompressed_size = blob_size; header.format = FOSSILIZE_COMPRESSION_NONE; header.payload_size = blob_size; header.crc = util_hash_crc32(blob, blob_size); fseek(foz_db->file[0], 0, SEEK_END); /* Write hash header to db */ char hash_str[FOSSILIZE_BLOB_HASH_LENGTH + 1]; /* 40 digits + null */ _mesa_sha1_format(hash_str, cache_key_160bit); if (fwrite(hash_str, 1, FOSSILIZE_BLOB_HASH_LENGTH, foz_db->file[0]) != FOSSILIZE_BLOB_HASH_LENGTH) goto fail; off_t offset = ftell(foz_db->file[0]); /* Write db entry header */ if (fwrite(&header, 1, sizeof(header), foz_db->file[0]) != sizeof(header)) goto fail; /* Now write the db entry blob */ if (fwrite(blob, 1, blob_size, foz_db->file[0]) != blob_size) goto fail; /* Flush everything to file to reduce chance of cache corruption */ fflush(foz_db->file[0]); /* Write hash header to index db */ if (fwrite(hash_str, 1, FOSSILIZE_BLOB_HASH_LENGTH, foz_db->db_idx) != FOSSILIZE_BLOB_HASH_LENGTH) goto fail; header.uncompressed_size = sizeof(uint64_t); header.format = FOSSILIZE_COMPRESSION_NONE; header.payload_size = sizeof(uint64_t); header.crc = 0; if (fwrite(&header, 1, sizeof(header), foz_db->db_idx) != sizeof(header)) goto fail; if (fwrite(&offset, 1, sizeof(uint64_t), foz_db->db_idx) != sizeof(uint64_t)) goto fail; /* Flush everything to file to reduce chance of cache corruption */ fflush(foz_db->db_idx); entry = ralloc(foz_db->mem_ctx, struct foz_db_entry); entry->header = header; entry->offset = offset; entry->file_idx = 0; _mesa_sha1_hex_to_sha1(entry->key, hash_str); _mesa_hash_table_u64_insert(foz_db->index_db, hash, entry); simple_mtx_unlock(&foz_db->mtx); flock(fileno(foz_db->file[0]), LOCK_UN); simple_mtx_unlock(&foz_db->flock_mtx); return true; fail: simple_mtx_unlock(&foz_db->mtx); fail_file: flock(fileno(foz_db->file[0]), LOCK_UN); simple_mtx_unlock(&foz_db->flock_mtx); return false; } #else bool foz_prepare(struct foz_db *foz_db, char *filename) { fprintf(stderr, "Warning: Mesa single file cache selected but Mesa wasn't " "built with single cache file support. Shader cache will be disabled" "!\n"); return false; } void foz_destroy(struct foz_db *foz_db) { } void * foz_read_entry(struct foz_db *foz_db, const uint8_t *cache_key_160bit, size_t *size) { return false; } bool foz_write_entry(struct foz_db *foz_db, const uint8_t *cache_key_160bit, const void *blob, size_t size) { return false; } #endif