Author: Fred Riss Date: 2023-01-19T14:07:34-08:00 New Revision: cf12709222a4699ff5a4bb257cb891b55b5f6fba
URL: https://github.com/llvm/llvm-project/commit/cf12709222a4699ff5a4bb257cb891b55b5f6fba DIFF: https://github.com/llvm/llvm-project/commit/cf12709222a4699ff5a4bb257cb891b55b5f6fba.diff LOG: Revert "[Clang] Give Clang the ability to use a shared stat cache" This reverts commit c5abe893120b115907376359a5809229a9f9608a. This reverts commit a033dbbe5c43247b60869b008e67ed86ed230eaa. This broke the build with -DLLVM_LINK_LLVM_DYLIB=ON. Reverting while I investigate. Added: Modified: clang/include/clang/Basic/DiagnosticFrontendKinds.td clang/include/clang/Driver/Options.td clang/include/clang/Frontend/CompilerInvocation.h clang/include/clang/Lex/HeaderSearchOptions.h clang/lib/Frontend/ASTUnit.cpp clang/lib/Frontend/CompilerInvocation.cpp clang/test/CMakeLists.txt clang/tools/CMakeLists.txt llvm/lib/Support/CMakeLists.txt llvm/unittests/Support/VirtualFileSystemTest.cpp Removed: clang/test/Driver/vfsstatcache.c clang/test/clang-stat-cache/cache-effects.c clang/test/clang-stat-cache/errors.test clang/tools/clang-stat-cache/CMakeLists.txt clang/tools/clang-stat-cache/clang-stat-cache.cpp llvm/include/llvm/Support/StatCacheFileSystem.h llvm/lib/Support/StatCacheFileSystem.cpp ################################################################################ diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td index e106858688ac7..d0f672ae5a1bd 100644 --- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -256,11 +256,6 @@ def err_test_module_file_extension_version : Error< "test module file extension '%0' has diff erent version (%1.%2) than expected " "(%3.%4)">; -def err_missing_vfs_stat_cache_file : Error< - "stat cache file '%0' not found">, DefaultFatal; -def err_invalid_vfs_stat_cache : Error< - "invalid stat cache file '%0'">, DefaultFatal; - def err_missing_vfs_overlay_file : Error< "virtual filesystem overlay file '%0' not found">, DefaultFatal; def err_invalid_vfs_overlay : Error< diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 9334e6319d57b..ba49b335cf287 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -3357,8 +3357,6 @@ def iwithsysroot : JoinedOrSeparate<["-"], "iwithsysroot">, Group<clang_i_Group> HelpText<"Add directory to SYSTEM include search path, " "absolute paths are relative to -isysroot">, MetaVarName<"<directory>">, Flags<[CC1Option]>; -def ivfsstatcache : JoinedOrSeparate<["-"], "ivfsstatcache">, Group<clang_i_Group>, Flags<[CC1Option]>, - HelpText<"Use the stat data cached in file instead of doing filesystem syscalls. See clang-stat-cache utility.">; def ivfsoverlay : JoinedOrSeparate<["-"], "ivfsoverlay">, Group<clang_i_Group>, Flags<[CC1Option]>, HelpText<"Overlay the virtual filesystem described by file over the real file system">; def imultilib : Separate<["-"], "imultilib">, Group<gfortran_Group>; diff --git a/clang/include/clang/Frontend/CompilerInvocation.h b/clang/include/clang/Frontend/CompilerInvocation.h index 9cc6aa5c4d8a1..254f048ed3c7e 100644 --- a/clang/include/clang/Frontend/CompilerInvocation.h +++ b/clang/include/clang/Frontend/CompilerInvocation.h @@ -296,7 +296,6 @@ IntrusiveRefCntPtr<llvm::vfs::FileSystem> createVFSFromCompilerInvocation( IntrusiveRefCntPtr<llvm::vfs::FileSystem> createVFSFromOverlayFiles(ArrayRef<std::string> VFSOverlayFiles, - ArrayRef<std::string> VFSStatCacheFiles, DiagnosticsEngine &Diags, IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS); diff --git a/clang/include/clang/Lex/HeaderSearchOptions.h b/clang/include/clang/Lex/HeaderSearchOptions.h index 548f7d4493de4..6436a9b3bde20 100644 --- a/clang/include/clang/Lex/HeaderSearchOptions.h +++ b/clang/include/clang/Lex/HeaderSearchOptions.h @@ -181,9 +181,6 @@ class HeaderSearchOptions { /// of computing the module hash. llvm::SmallSetVector<llvm::CachedHashString, 16> ModulesIgnoreMacros; - /// The set of user-provided stat cache files. - std::vector<std::string> VFSStatCacheFiles; - /// The set of user-provided virtual filesystem overlay files. std::vector<std::string> VFSOverlayFiles; @@ -253,10 +250,6 @@ class HeaderSearchOptions { SystemHeaderPrefixes.emplace_back(Prefix, IsSystemHeader); } - void AddVFSStatCacheFile(StringRef Name) { - VFSStatCacheFiles.push_back(std::string(Name)); - } - void AddVFSOverlayFile(StringRef Name) { VFSOverlayFiles.push_back(std::string(Name)); } diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp index dbf55d95746e9..3b4f25182ac95 100644 --- a/clang/lib/Frontend/ASTUnit.cpp +++ b/clang/lib/Frontend/ASTUnit.cpp @@ -574,7 +574,7 @@ class ASTInfoCollector : public ASTReaderListener { // performs the initialization too late (once both target and language // options are read). PP.getFileManager().setVirtualFileSystem(createVFSFromOverlayFiles( - HSOpts.VFSOverlayFiles, HSOpts.VFSStatCacheFiles, PP.getDiagnostics(), + HSOpts.VFSOverlayFiles, PP.getDiagnostics(), PP.getFileManager().getVirtualFileSystemPtr())); InitializedHeaderSearchPaths = true; diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index b0ef37fad2227..0bb9c8c83c63b 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -83,7 +83,6 @@ #include "llvm/Support/Path.h" #include "llvm/Support/Process.h" #include "llvm/Support/Regex.h" -#include "llvm/Support/StatCacheFileSystem.h" #include "llvm/Support/VersionTuple.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/raw_ostream.h" @@ -3085,9 +3084,6 @@ static void GenerateHeaderSearchArgs(HeaderSearchOptions &Opts, GenerateArg(Args, Opt, P.Prefix, SA); } - for (const std::string &F : Opts.VFSStatCacheFiles) - GenerateArg(Args, OPT_ivfsstatcache, F, SA); - for (const std::string &F : Opts.VFSOverlayFiles) GenerateArg(Args, OPT_ivfsoverlay, F, SA); } @@ -3221,9 +3217,6 @@ static bool ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args, Opts.AddSystemHeaderPrefix( A->getValue(), A->getOption().matches(OPT_system_header_prefix)); - for (const auto *A : Args.filtered(OPT_ivfsstatcache)) - Opts.AddVFSStatCacheFile(A->getValue()); - for (const auto *A : Args.filtered(OPT_ivfsoverlay)) Opts.AddVFSOverlayFile(A->getValue()); @@ -4754,31 +4747,12 @@ clang::createVFSFromCompilerInvocation( const CompilerInvocation &CI, DiagnosticsEngine &Diags, IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS) { return createVFSFromOverlayFiles(CI.getHeaderSearchOpts().VFSOverlayFiles, - CI.getHeaderSearchOpts().VFSStatCacheFiles, Diags, std::move(BaseFS)); } IntrusiveRefCntPtr<llvm::vfs::FileSystem> clang::createVFSFromOverlayFiles( - ArrayRef<std::string> VFSOverlayFiles, - ArrayRef<std::string> VFSStatCacheFiles, DiagnosticsEngine &Diags, + ArrayRef<std::string> VFSOverlayFiles, DiagnosticsEngine &Diags, IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS) { - for (const auto &File : VFSStatCacheFiles) { - llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer = - BaseFS->getBufferForFile(File); - if (!Buffer) { - Diags.Report(diag::err_missing_vfs_stat_cache_file) << File; - continue; - } - - auto StatCache = - llvm::vfs::StatCacheFileSystem::create(std::move(*Buffer), BaseFS); - - if (errorToBool(StatCache.takeError())) - Diags.Report(diag::err_invalid_vfs_stat_cache) << File; - else - BaseFS = std::move(*StatCache); - } - if (VFSOverlayFiles.empty()) return BaseFS; diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt index cd3775b55a559..1d6377b5f2d80 100644 --- a/clang/test/CMakeLists.txt +++ b/clang/test/CMakeLists.txt @@ -71,7 +71,6 @@ list(APPEND CLANG_TEST_DEPS clang-refactor clang- diff clang-scan-deps - clang-stat-cache diagtool hmaptool ) diff --git a/clang/test/Driver/vfsstatcache.c b/clang/test/Driver/vfsstatcache.c deleted file mode 100644 index ec3c279d53d08..0000000000000 --- a/clang/test/Driver/vfsstatcache.c +++ /dev/null @@ -1,5 +0,0 @@ -// RUN: %clang -ivfsstatcache foo.h -### %s 2>&1 | FileCheck %s -// CHECK: "-ivfsstatcache" "foo.h" - -// RUN: not %clang -ivfsstatcache foo.h %s 2>&1 | FileCheck -check-prefix=CHECK-MISSING %s -// CHECK-MISSING: stat cache file 'foo.h' not found diff --git a/clang/test/clang-stat-cache/cache-effects.c b/clang/test/clang-stat-cache/cache-effects.c deleted file mode 100644 index bf2e2db447e2a..0000000000000 --- a/clang/test/clang-stat-cache/cache-effects.c +++ /dev/null @@ -1,63 +0,0 @@ -#include "foo.h" - -// Testing the effects of a cache is tricky, because it's just supposed to speed -// things up, not change the behavior. In this test, we are using an outdated -// cache to trick HeaderSearch into finding the wrong module and show that it is -// being used. - -// Clear the module cache. -// RUN: rm -rf %t -// RUN: mkdir -p %t/Inputs -// RUN: mkdir -p %t/Inputs/Foo1 -// RUN: mkdir -p %t/Inputs/Foo2 -// RUN: mkdir -p %t/modules-to-compare - -// === -// Create a Foo module in the Foo1 direcotry. -// RUN: echo 'void meow(void);' > %t/Inputs/Foo1/foo.h -// RUN: echo 'module Foo { header "foo.h" }' > %t/Inputs/Foo1/module.map - -// === -// Compile the module. Note that the compiler has 2 header search paths: -// Foo2 and Foo1 in that order. The module has been created in Foo1, and -// it is the only version available now. -// RUN: %clang_cc1 -cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -fmodules-cache-path=%t/modules-cache -fsyntax-only -I %t/Inputs/Foo2 -I %t/Inputs/Foo1 -Rmodule-build %s 2>&1 -// RUN: cp %t/modules-cache/Foo.pcm %t/modules-to-compare/Foo-before.pcm - -// === -// Create a stat cache for our inputs directory -// RUN: clang-stat-cache %t/Inputs -o %t/stat.cache - -// === -// As a sanity check, re-run the same compilation with the cache and check that -// the module does not change. -// RUN: %clang_cc1 -cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -fmodules-cache-path=%t/modules-cache -fsyntax-only -I %t/Inputs/Foo2 -I %t/Inputs/Foo1 -ivfsstatcache %t/stat.cache %s -Rmodule-build 2>&1 -// RUN: cp %t/modules-cache/Foo.pcm %t/modules-to-compare/Foo-after.pcm - -// RUN: diff %t/modules-to-compare/Foo-before.pcm %t/modules-to-compare/Foo-after.pcm - -// === -// Now introduce a diff erent Foo module in the Foo2 directory which is before -// Foo1 in the search paths. -// RUN: echo 'void meow2(void);' > %t/Inputs/Foo2/foo.h -// RUN: echo 'module Foo { header "foo.h" }' > %t/Inputs/Foo2/module.map - -// === -// Because we're using the (now-outdated) stat cache, this compilation -// should still be using the first module. It will not see the new one -// which is earlier in the search paths. -// RUN: %clang_cc1 -cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -fmodules-cache-path=%t/modules-cache -fsyntax-only -I %t/Inputs/Foo2 -I %t/Inputs/Foo1 -ivfsstatcache %t/stat.cache -Rmodule-build -Rmodule-import %s 2>&1 -// RUN: cp %t/modules-cache/Foo.pcm %t/modules-to-compare/Foo-after.pcm - -// RUN: diff %t/modules-to-compare/Foo-before.pcm %t/modules-to-compare/Foo-after.pcm - -// === -// Regenerate the stat cache for our Inputs directory -// RUN: clang-stat-cache -f %t/Inputs -o %t/stat.cache 2>&1 - -// === -// Use the module and now see that we are recompiling the new one. -// RUN: %clang_cc1 -cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -fmodules-cache-path=%t/modules-cache -fsyntax-only -I %t/Inputs/Foo2 -I %t/Inputs/Foo1 -ivfsstatcache %t/stat.cache -Rmodule-build %s 2>&1 -// RUN: cp %t/modules-cache/Foo.pcm %t/modules-to-compare/Foo-after.pcm - -// RUN: not diff %t/modules-to-compare/Foo-before.pcm %t/modules-to-compare/Foo-after.pcm diff --git a/clang/test/clang-stat-cache/errors.test b/clang/test/clang-stat-cache/errors.test deleted file mode 100644 index ab73a1348f10e..0000000000000 --- a/clang/test/clang-stat-cache/errors.test +++ /dev/null @@ -1,42 +0,0 @@ -RUN: rm -rf %t -RUN: mkdir -p %t - -RUN: not clang-stat-cache %t/not-there -o %t/stat.cache 2>&1 | FileCheck --check-prefix=NO-SUCH-DIR %s -NO-SUCH-DIR: Failed to stat the target directory: {{[Nn]}}o such file or directory - -RUN: not clang-stat-cache %t -o %t/not-there/stat.cache 2>&1 | FileCheck --check-prefix=NO-SUCH-FILE %s -NO-SUCH-FILE: Failed to open cache file: '{{.*}}': {{[Nn]}}o such file or directory - -# Use mixed-case directories to exercise the case insensitive implementation. -RUN: mkdir -p %t/Dir -RUN: mkdir -p %t/Dir2 - -# Try to overwrite a few invalid caches -RUN: echo "Not a stat cache" > %t/stat.cache -RUN: not clang-stat-cache %t/Dir -o %t/stat.cache 2>&1 | FileCheck --check-prefix=INVALID-CACHE %s -RUN: echo "Not a stat cache, but bigger than the stat cache header" > %t/stat.cache -RUN: not clang-stat-cache %t/Dir -o %t/stat.cache 2>&1 | FileCheck --check-prefix=INVALID-CACHE %s -RUN: echo "STAT. This has the correct MAGIC and is bigger than the header." > %t/stat.cache -RUN: not clang-stat-cache %t/Dir -o %t/stat.cache 2>&1 | FileCheck --check-prefix=INVALID-CACHE %s - -INVALID-CACHE: The output cache file exists and is not a valid stat cache. Aborting. - -# Test the force flag -RUN: echo "STAT. This has the correct MAGIC and is bigger than the header." > %t/stat.cache -RUN: clang-stat-cache %t/Dir -f -o %t/stat.cache 2>&1 | FileCheck --check-prefix=INVALID-CACHE-FORCE %s -INVALID-CACHE-FORCE: The output cache file exists and is not a valid stat cache. Forced update. - -# Generate a valid cache for dir -RUN: rm %t/stat.cache -RUN: clang-stat-cache %t/Dir -o %t/stat.cache -RUN: cp %t/stat.cache %t/stat.cache.save - -# Try with same base direcotry but with extraneous separators -RUN: clang-stat-cache %t/Dir/// -v -o %t/stat.cache | FileCheck --check-prefix=EXTRA-SEP %s -EXTRA-SEP-NOT: Existing cache has diff erent directory. Regenerating... -EXTRA-SEP: Cache up-to-date, exiting - -# Rewrite the cache with a diff erent base directory -RUN: clang-stat-cache %t/Dir2 -o %t/stat.cache 2>&1 | FileCheck --check-prefix=OTHER-DIR %s -OTHER-DIR: Existing cache has diff erent directory. Regenerating... - diff --git a/clang/tools/CMakeLists.txt b/clang/tools/CMakeLists.txt index 147555d87dfc5..f60db6ef0ba34 100644 --- a/clang/tools/CMakeLists.txt +++ b/clang/tools/CMakeLists.txt @@ -15,7 +15,6 @@ add_clang_subdirectory(clang-scan-deps) if(HAVE_CLANG_REPL_SUPPORT) add_clang_subdirectory(clang-repl) endif() -add_clang_subdirectory(clang-stat-cache) add_clang_subdirectory(c-index-test) diff --git a/clang/tools/clang-stat-cache/CMakeLists.txt b/clang/tools/clang-stat-cache/CMakeLists.txt deleted file mode 100644 index ab93d8b3d0f0e..0000000000000 --- a/clang/tools/clang-stat-cache/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -set(LLVM_LINK_COMPONENTS - Core - Support - ) - -add_clang_tool(clang-stat-cache - clang-stat-cache.cpp - ) - -if(APPLE) -set(CLANG_STAT_CACHE_LIB_DEPS - "-framework CoreServices" - ) -endif() - -clang_target_link_libraries(clang-stat-cache - PRIVATE - ${CLANG_STAT_CACHE_LIB_DEPS} - ) diff --git a/clang/tools/clang-stat-cache/clang-stat-cache.cpp b/clang/tools/clang-stat-cache/clang-stat-cache.cpp deleted file mode 100644 index 183d0d79fadd8..0000000000000 --- a/clang/tools/clang-stat-cache/clang-stat-cache.cpp +++ /dev/null @@ -1,318 +0,0 @@ -//===- clang-stat-cache.cpp -----------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/StatCacheFileSystem.h" -#include "llvm/Support/Timer.h" -#include "llvm/Support/raw_ostream.h" - -#include <assert.h> - -#ifdef __APPLE__ -#include <CoreServices/CoreServices.h> - -#include <sys/mount.h> -#include <sys/param.h> -#endif // __APPLE__ - -// The clang-stat-cache utility creates an on-disk cache for the stat data -// of a file-system tree which is expected to be immutable during a build. - -using namespace llvm; -using llvm::vfs::StatCacheFileSystem; - -cl::OptionCategory StatCacheCategory("clang-stat-cache options"); - -cl::opt<std::string> OutputFilename("o", cl::Required, - cl::desc("Specify output filename"), - cl::value_desc("filename"), - cl::cat(StatCacheCategory)); - -cl::opt<std::string> TargetDirectory(cl::Positional, cl::Required, - cl::value_desc("dirname"), - cl::cat(StatCacheCategory)); - -cl::opt<bool> Verbose("v", cl::desc("More verbose output")); -cl::opt<bool> Force("f", cl::desc("Force cache generation")); - -#if __APPLE__ -// Used by checkContentsValidity. See below. -struct CallbackInfo { - bool SeenChanges = false; -}; - -// Used by checkContentsValidity. See below. -static void FSEventsCallback(ConstFSEventStreamRef streamRef, void *CtxInfo, - size_t numEvents, void *eventPaths, - const FSEventStreamEventFlags *eventFlags, - const FSEventStreamEventId *eventIds) { - CallbackInfo *Info = static_cast<CallbackInfo *>(CtxInfo); - for (size_t i = 0; i < numEvents; ++i) { - // The kFSEventStreamEventFlagHistoryDone is set on the last 'historical' - // event passed to the callback. This means it is passed after the callback - // all the relevant activity between the StartEvent of the stream and the - // point the stream was created. - // If the callback didn't see any other event, it means there haven't been - // any alterations to the target directory hierarchy and the cache contents - // is still up-to-date. - if (eventFlags[i] & kFSEventStreamEventFlagHistoryDone) { - // Let's stop the main queue and go back to our non-queue code. - CFRunLoopStop(CFRunLoopGetCurrent()); - break; - } - - // If we see any event outisde of the kFSEventStreamEventFlagHistoryDone - // one, there have been changes to the target directory. - Info->SeenChanges = true; - } -} - -// FSEvents-based check for cache contents validity. We store the latest -// FSEventStreamEventId in the cache as a ValidityToken and check if any -// file system events affected the base directory since the cache was -// generated. -static bool checkContentsValidity(uint64_t &ValidityToken) { - CFStringRef TargetDir = CFStringCreateWithCStringNoCopy( - kCFAllocatorDefault, TargetDirectory.c_str(), kCFStringEncodingASCII, - kCFAllocatorNull); - CFArrayRef PathsToWatch = - CFArrayCreate(nullptr, (const void **)&TargetDir, 1, nullptr); - CallbackInfo Info; - FSEventStreamContext Ctx = {0, &Info, nullptr, nullptr, nullptr}; - FSEventStreamRef Stream; - CFAbsoluteTime Latency = 0; // Latency in seconds. Do not wait. - - // Start at the latest event stored in the cache. - FSEventStreamEventId StartEvent = ValidityToken; - // Update the Validity token with the current latest event. - ValidityToken = FSEventsGetCurrentEventId(); - - // Create the stream - Stream = - FSEventStreamCreate(NULL, &FSEventsCallback, &Ctx, PathsToWatch, - StartEvent, Latency, kFSEventStreamCreateFlagNone); - - // Associate the stream with the main queue. - FSEventStreamSetDispatchQueue(Stream, dispatch_get_main_queue()); - // Start the stream (needs the queue to run to do anything). - if (!FSEventStreamStart(Stream)) { - errs() << "Failed to create FS event stream. " - << "Considering the cache up-to-date.\n"; - return true; - } - - // Start the main queue. It will be exited by our callback when it got - // confirmed it processed all events. - CFRunLoopRun(); - - return !Info.SeenChanges; -} - -#else // __APPLE__ - -// There is no cross-platform way to implement a validity check. If this -// platform doesn't support it, just consider the cache contents always -// valid. When that's the case, the tool running cache generation needs -// to have the knowledge to do it only when needed. -static bool checkContentsValidity(uint64_t &ValidityToken) { return true; } - -#endif // __APPLE__ - -// Populate Generator with the stat cache data for the filesystem tree -// rooted at BasePath. -static std::error_code -populateHashTable(StringRef BasePath, - StatCacheFileSystem::StatCacheWriter &Generator) { - using namespace llvm; - using namespace sys::fs; - - std::error_code ErrorCode; - - // Just loop over the target directory using a recursive iterator. - // This invocation follows symlinks, so we are going to potentially - // store the status of the same file multiple times with diff erent - // names. - for (recursive_directory_iterator I(BasePath, ErrorCode), E; - I != E && !ErrorCode; I.increment(ErrorCode)) { - StringRef Path = I->path(); - sys::fs::file_status s; - // This can fail (broken symlink) and leave the file_status with - // its default values. The reader knows this. - status(Path, s); - - Generator.addEntry(Path, s); - } - - return ErrorCode; -} - -static bool checkCacheValid(int FD, raw_fd_ostream &Out, - uint64_t &ValidityToken) { - sys::fs::file_status Status; - auto EC = sys::fs::status(FD, Status); - if (EC) { - llvm::errs() << "fstat failed: " - << llvm::toString(llvm::errorCodeToError(EC)) << "\n"; - return false; - } - - auto Size = Status.getSize(); - if (Size == 0) { - // New file. -#ifdef __APPLE__ - // Get the current (global) FSEvent id and use this as ValidityToken. - ValidityToken = FSEventsGetCurrentEventId(); -#endif - return false; - } - - auto ErrorOrBuffer = MemoryBuffer::getOpenFile( - sys::fs::convertFDToNativeFile(FD), OutputFilename, Status.getSize()); - - // Refuse to write to this cache file if it exists but its contents do - // not look like a valid cache file. - StringRef BaseDir; - bool IsCaseSensitive; - bool VersionMatch; - if (auto E = StatCacheFileSystem::validateCacheFile( - (*ErrorOrBuffer)->getMemBufferRef(), BaseDir, IsCaseSensitive, - VersionMatch, ValidityToken)) { - llvm::errs() << "The output cache file exists and is not a valid stat " - "cache."; - if (!Force) { - llvm::errs() << " Aborting.\n"; - exit(1); - } - - consumeError(std::move(E)); - llvm::errs() << " Forced update.\n"; - return false; - } - - if (BaseDir != TargetDirectory && - (IsCaseSensitive || !BaseDir.equals_insensitive(TargetDirectory))) { - llvm::errs() << "Existing cache has diff erent directory. Regenerating...\n"; - return false; - } - - if (!VersionMatch) { - llvm::errs() - << "Exisitng cache has diff erent version number. Regenerating...\n"; - return false; - } - - // Basic structure checks have passed. Lets see if we can prove that the cache - // contents are still valid. - bool IsValid = checkContentsValidity(ValidityToken); - if (IsValid) { - // The cache is valid, but we might have gotten an updated ValidityToken. - // Update the cache with it as clang-stat-cache is just going to exit after - // returning from this function. - StatCacheFileSystem::updateValidityToken(Out, ValidityToken); - } - return IsValid && !Force; -} - -int main(int argc, char *argv[]) { - cl::ParseCommandLineOptions(argc, argv); - - llvm::SmallString<128> CanonicalDirectory = StringRef(TargetDirectory); - - // Remove extraneous separators from the end of the basename. - while (!CanonicalDirectory.empty() && - sys::path::is_separator(CanonicalDirectory.back())) - CanonicalDirectory.pop_back(); - // Canonicalize separators on Windows - llvm::sys::path::make_preferred(CanonicalDirectory); - TargetDirectory = std::string(CanonicalDirectory); - - StringRef Dirname(TargetDirectory); - - std::error_code EC; - int FD; - EC = sys::fs::openFileForReadWrite( - OutputFilename, FD, llvm::sys::fs::CD_OpenAlways, llvm::sys::fs::OF_None); - if (EC) { - llvm::errs() << "Failed to open cache file: " - << toString(llvm::createFileError(OutputFilename, EC)) << "\n"; - return 1; - } - - raw_fd_ostream Out(FD, /* ShouldClose=*/true); - - uint64_t ValidityToken = 0; - // Check if the cache is valid and up-to-date. - if (checkCacheValid(FD, Out, ValidityToken)) { - if (Verbose) - outs() << "Cache up-to-date, exiting\n"; - return 0; - } - - if (Verbose) - outs() << "Building a stat cache for '" << TargetDirectory << "' into '" - << OutputFilename << "'\n"; - - // Do not generate a cache for NFS. Iterating huge directory hierarchies - // over NFS will be very slow. Better to let the compiler search only the - // pieces that it needs than use a cache that takes ages to populate. - bool IsLocal; - EC = sys::fs::is_local(Dirname, IsLocal); - if (EC) { - errs() << "Failed to stat the target directory: " - << llvm::toString(llvm::errorCodeToError(EC)) << "\n"; - return 1; - } - - if (!IsLocal && !Force) { - errs() << "Target directory is not a local filesystem. " - << "Not populating the cache.\n"; - return 0; - } - - sys::fs::file_status BaseDirStatus; - if (std::error_code EC = status(Dirname, BaseDirStatus)) { - errs() << "Failed to stat the target directory: " - << llvm::toString(llvm::errorCodeToError(EC)) << "\n"; - return 1; - } - - // Check if the filesystem hosting the target directory is case sensitive. - bool IsCaseSensitive = true; -#ifdef _PC_CASE_SENSITIVE - IsCaseSensitive = - ::pathconf(TargetDirectory.c_str(), _PC_CASE_SENSITIVE) == 1; -#endif - StatCacheFileSystem::StatCacheWriter Generator( - Dirname, BaseDirStatus, IsCaseSensitive, ValidityToken); - - // Populate the cache. - auto startTime = llvm::TimeRecord::getCurrentTime(); - populateHashTable(Dirname, Generator); - auto duration = llvm::TimeRecord::getCurrentTime(); - duration -= startTime; - - if (Verbose) - errs() << "populateHashTable took: " << duration.getWallTime() << "s\n"; - - // Write the cache to disk. - startTime = llvm::TimeRecord::getCurrentTime(); - int Size = Generator.writeStatCache(Out); - duration = llvm::TimeRecord::getCurrentTime(); - duration -= startTime; - - if (Verbose) - errs() << "writeStatCache took: " << duration.getWallTime() << "s\n"; - - // We might have opened a pre-exising cache which was bigger. - llvm::sys::fs::resize_file(FD, Size); - - return 0; -} diff --git a/llvm/include/llvm/Support/StatCacheFileSystem.h b/llvm/include/llvm/Support/StatCacheFileSystem.h deleted file mode 100644 index cf2e06768acd2..0000000000000 --- a/llvm/include/llvm/Support/StatCacheFileSystem.h +++ /dev/null @@ -1,110 +0,0 @@ -//===- StatCacheFileSystem.h - Status Caching Proxy File System -*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_SUPPORT_STATCACHEFILESYSTEM_H -#define LLVM_SUPPORT_STATCACHEFILESYSTEM_H - -#include "llvm/Support/VirtualFileSystem.h" - -#include <list> - -namespace llvm { -template <typename T> class OnDiskIterableChainedHashTable; -template <typename T> class OnDiskChainedHashTableGenerator; - -namespace vfs { - -/// A ProxyFileSystem using cached information for status() rather than going to -/// the underlying filesystem. -/// -/// When dealing with a huge tree of (mostly) immutable filesystem content -/// like an SDK, it can be very costly to ask the underlying filesystem for -/// `stat` data. Even when caching the `stat`s internally, having many -/// concurrent Clangs accessing the same tree in a similar way causes -/// contention. As SDK files are mostly immutable, we can pre-compute the status -/// information using clang-stat-cache and use that information directly without -/// accessing the real filesystem until Clang needs to open a file. This can -/// speed up module verification and HeaderSearch by significant amounts. -class StatCacheFileSystem : public llvm::vfs::ProxyFileSystem { - class StatCacheLookupInfo; - using StatCacheType = - llvm::OnDiskIterableChainedHashTable<StatCacheLookupInfo>; - - class StatCacheGenerationInfo; - using StatCacheGeneratorType = - llvm::OnDiskChainedHashTableGenerator<StatCacheGenerationInfo>; - - explicit StatCacheFileSystem(std::unique_ptr<llvm::MemoryBuffer> CacheFile, - IntrusiveRefCntPtr<FileSystem> FS, - bool IsCaseSensitive); - -public: - /// Create a StatCacheFileSystem from the passed \a CacheBuffer, a - /// MemoryBuffer representing the contents of the \a CacheFilename file. The - /// returned filesystem will be overlaid on top of \a FS. - static Expected<IntrusiveRefCntPtr<StatCacheFileSystem>> - create(std::unique_ptr<llvm::MemoryBuffer> CacheBuffer, - IntrusiveRefCntPtr<FileSystem> FS); - - /// The status override which will consult the cache if \a Path is in the - /// cached filesystem tree. - llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override; - -public: - /// A helper class to generate stat caches. - class StatCacheWriter { - llvm::SmallString<128> BaseDir; - bool IsCaseSensitive; - uint64_t ValidityToken; - StatCacheGeneratorType *Generator; - std::list<std::string> PathStorage; - - public: - /// Create a StatCacheWriter - /// - /// \param BaseDir The base directory for the path. Every filename passed to - /// addEntry() needs to start with this base directory. - /// \param Status The status entry for the base directory. - /// \param IsCaseSensitive Whether the cache is case sensitive. - /// \param ValidityToken A 64 bits token that gets embedded in the cache and - /// can be used by generator tools to check for the - /// cache validity in a platform-specific way. - StatCacheWriter(StringRef BaseDir, const sys::fs::file_status &Status, - bool IsCaseSensitive, uint64_t ValidityToken = 0); - ~StatCacheWriter(); - - /// Add a cache entry storing \a Status for the file at \a Path. - void addEntry(StringRef Path, const sys::fs::file_status &Status); - - /// Write the cache file to \a Out. - size_t writeStatCache(raw_fd_ostream &Out); - }; - -public: - /// Validate that the file content in \a Buffer is a valid stat cache file. - /// \a BaseDir, \a IsCaseSensitive and \a ValidityToken are output parameters - /// that get populated by this call. - static Error validateCacheFile(llvm::MemoryBufferRef Buffer, - StringRef &BaseDir, bool &IsCaseSensitive, - bool &VersionMatch, uint64_t &ValidityToken); - - /// Update the ValidityToken data in \a CacheFile. - static void updateValidityToken(raw_fd_ostream &CacheFile, - uint64_t ValidityToken); - -private: - std::unique_ptr<llvm::MemoryBuffer> StatCacheFile; - llvm::StringRef StatCachePrefix; - std::unique_ptr<StatCacheType> StatCache; - bool IsCaseSensitive = true; -}; - -} // namespace vfs -} // namespace llvm - -#endif // LLVM_SUPPORT_STATCACHEFILESYSTEM_H diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt index 16531c4ec78d2..9b5402fa54f0f 100644 --- a/llvm/lib/Support/CMakeLists.txt +++ b/llvm/lib/Support/CMakeLists.txt @@ -212,7 +212,6 @@ add_llvm_component_library(LLVMSupport SmallVector.cpp SourceMgr.cpp SpecialCaseList.cpp - StatCacheFileSystem.cpp Statistic.cpp StringExtras.cpp StringMap.cpp diff --git a/llvm/lib/Support/StatCacheFileSystem.cpp b/llvm/lib/Support/StatCacheFileSystem.cpp deleted file mode 100644 index 4633a0d885d98..0000000000000 --- a/llvm/lib/Support/StatCacheFileSystem.cpp +++ /dev/null @@ -1,306 +0,0 @@ -//===- StatCacheFileSystem.cpp - Status Caching Proxy File System ---------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "llvm/Support/StatCacheFileSystem.h" - -#include "llvm/ADT/IntrusiveRefCntPtr.h" -#include "llvm/Support/ErrorOr.h" -#include "llvm/Support/OnDiskHashTable.h" - -namespace llvm { -namespace vfs { - -class StatCacheFileSystem::StatCacheLookupInfo { -public: - typedef StringRef external_key_type; - typedef StringRef internal_key_type; - typedef llvm::sys::fs::file_status data_type; - typedef uint32_t hash_value_type; - typedef uint32_t offset_type; - - static bool EqualKey(const internal_key_type &a, const internal_key_type &b) { - return a == b; - } - - static hash_value_type ComputeHash(const internal_key_type &a) { - return hash_value(a); - } - - static std::pair<unsigned, unsigned> - ReadKeyDataLength(const unsigned char *&d) { - using namespace llvm::support; - unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d); - unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d); - return std::make_pair(KeyLen, DataLen); - } - - static const internal_key_type &GetInternalKey(const external_key_type &x) { - return x; - } - - static const external_key_type &GetExternalKey(const internal_key_type &x) { - return x; - } - - static internal_key_type ReadKey(const unsigned char *d, unsigned n) { - return StringRef((const char *)d, n); - } - - static data_type ReadData(const internal_key_type &k, const unsigned char *d, - unsigned DataLen) { - data_type Result; - memcpy(&Result, d, sizeof(Result)); - return Result; - } -}; - -class StatCacheFileSystem::StatCacheGenerationInfo { -public: - typedef StringRef key_type; - typedef const StringRef &key_type_ref; - typedef sys::fs::file_status data_type; - typedef const sys::fs::file_status &data_type_ref; - typedef uint32_t hash_value_type; - typedef uint32_t offset_type; - - /// Calculate the hash for Key - static hash_value_type ComputeHash(key_type_ref Key) { - return static_cast<size_t>(hash_value(Key)); - } - - /// Return the lengths, in bytes, of the given Key/Data pair. - static std::pair<unsigned, unsigned> - EmitKeyDataLength(raw_ostream &Out, key_type_ref Key, data_type_ref Data) { - using namespace llvm::support; - endian::Writer LE(Out, little); - unsigned KeyLen = Key.size(); - unsigned DataLen = sizeof(Data); - LE.write<uint16_t>(KeyLen); - LE.write<uint16_t>(DataLen); - return std::make_pair(KeyLen, DataLen); - } - - static void EmitKey(raw_ostream &Out, key_type_ref Key, unsigned KeyLen) { - Out.write(Key.data(), KeyLen); - } - - /// Write Data to Out. DataLen is the length from EmitKeyDataLength. - static void EmitData(raw_ostream &Out, key_type_ref Key, data_type_ref Data, - unsigned Len) { - Out.write((const char *)&Data, Len); - } - - static bool EqualKey(key_type_ref Key1, key_type_ref Key2) { - return Key1 == Key2; - } -}; - -// The format of the stat cache is (pseudo-code): -// struct stat_cache { -// char Magic[4]; // "STAT" or "Stat" -// uint32_t BucketOffset; // See BucketOffset in OnDiskHashTable.h -// uint64_t ValidityToken; // Platofrm specific data allowing to check -// // whether the cache is up-to-date. -// uint32_t Version; // The stat cache format version. -// char BaseDir[N]; // Zero terminated path to the base directory -// < OnDiskHashtable Data > // Data for the has table. The keys are the -// // relative paths under BaseDir. The data is -// // llvm::sys::fs::file_status structures. -// }; - -#define MAGIC_CASE_SENSITIVE "Stat" -#define MAGIC_CASE_INSENSITIVE "STAT" -#define STAT_CACHE_VERSION 1 - -namespace { -struct StatCacheHeader { - char Magic[4]; - uint32_t BucketOffset; - uint64_t ValidityToken; - uint32_t Version; - char BaseDir[1]; -}; -} // namespace - -StatCacheFileSystem::StatCacheFileSystem( - std::unique_ptr<MemoryBuffer> CacheFile, IntrusiveRefCntPtr<FileSystem> FS, - bool IsCaseSensitive) - : ProxyFileSystem(std::move(FS)), StatCacheFile(std::move(CacheFile)), - IsCaseSensitive(IsCaseSensitive) { - const char *CacheFileStart = StatCacheFile->getBufferStart(); - auto *Header = reinterpret_cast<const StatCacheHeader *>(CacheFileStart); - - uint32_t BucketOffset = Header->BucketOffset; - StatCachePrefix = StringRef(Header->BaseDir); - // HashTableStart points at the beginning of the data emitted by the - // OnDiskHashTable. - const unsigned char *HashTableStart = (const unsigned char *)CacheFileStart + - StatCachePrefix.size() + - sizeof(StatCacheHeader); - StatCache.reset(StatCacheType::Create( - (const unsigned char *)CacheFileStart + BucketOffset, HashTableStart, - (const unsigned char *)CacheFileStart)); -} - -Expected<IntrusiveRefCntPtr<StatCacheFileSystem>> -StatCacheFileSystem::create(std::unique_ptr<MemoryBuffer> CacheBuffer, - IntrusiveRefCntPtr<FileSystem> FS) { - StringRef BaseDir; - bool IsCaseSensitive; - bool VersionMatch; - uint64_t ValidityToken; - if (auto E = validateCacheFile(*CacheBuffer, BaseDir, IsCaseSensitive, - VersionMatch, ValidityToken)) - return std::move(E); - if (!VersionMatch) { - return createStringError(inconvertibleErrorCode(), - CacheBuffer->getBufferIdentifier() + - ": Mismatched cache file version"); - } - return new StatCacheFileSystem(std::move(CacheBuffer), FS, IsCaseSensitive); -} - -ErrorOr<Status> StatCacheFileSystem::status(const Twine &Path) { - SmallString<180> StringPath; - Path.toVector(StringPath); - // If the cache is not case sensitive, do all operations on lower-cased paths. - if (!IsCaseSensitive) - std::transform(StringPath.begin(), StringPath.end(), StringPath.begin(), - toLower); - - // Canonicalize the path. This removes single dot path components, - // but it also gets rid of repeating separators. - llvm::sys::path::remove_dots(StringPath); - - // If on Windows, canonicalize separators. - llvm::sys::path::make_preferred(StringPath); - - // Check if the requested path falls into the cache. - StringRef SuffixPath(StringPath); - if (!SuffixPath.consume_front(StatCachePrefix)) - return ProxyFileSystem::status(Path); - - auto It = StatCache->find(SuffixPath); - if (It == StatCache->end()) { - // We didn't find the file in the cache even though it started with the - // cache prefix. It could be that the file doesn't exist, or the spelling - // the path is diff erent. `remove_dots` canonicalizes the path by removing - // `.` and excess separators, but leaves `..` since it isn't semantically - // preserving to remove them in the presence of symlinks. If the path - // does not contain '..' we can safely say it doesn't exist. - if (std::find(sys::path::begin(SuffixPath), sys::path::end(SuffixPath), - "..") == sys::path::end(SuffixPath)) { - return llvm::errc::no_such_file_or_directory; - } - return ProxyFileSystem::status(Path); - } - - // clang-stat-cache will record entries for broken symlnks with a default- - // constructed Status. This will have a default-constructed UinqueID. - if ((*It).getUniqueID() == llvm::sys::fs::UniqueID()) - return llvm::errc::no_such_file_or_directory; - - return llvm::vfs::Status::copyWithNewName(*It, Path); -} - -StatCacheFileSystem::StatCacheWriter::StatCacheWriter( - StringRef BaseDir, const sys::fs::file_status &Status, bool IsCaseSensitive, - uint64_t ValidityToken) - : BaseDir(IsCaseSensitive ? BaseDir.str() : BaseDir.lower()), - IsCaseSensitive(IsCaseSensitive), ValidityToken(ValidityToken), - Generator(new StatCacheGeneratorType()) { - addEntry(BaseDir, Status); - // If on Windows, canonicalize separators. - llvm::sys::path::make_preferred(this->BaseDir); -} - -StatCacheFileSystem::StatCacheWriter::~StatCacheWriter() { delete Generator; } - -void StatCacheFileSystem::StatCacheWriter::addEntry( - StringRef Path, const sys::fs::file_status &Status) { - llvm::SmallString<128> StoredPath; - -#if defined(_WIN32) - StoredPath = Path; - llvm::sys::path::make_preferred(StoredPath); - Path = StoredPath; -#endif - - if (!IsCaseSensitive) { - StoredPath = Path.lower(); - Path = StoredPath; - } - - LLVM_ATTRIBUTE_UNUSED bool Consumed = Path.consume_front(BaseDir); - assert(Consumed && "Path does not start with expected prefix."); - - PathStorage.emplace_back(Path.str()); - Generator->insert(PathStorage.back(), Status); -} - -size_t -StatCacheFileSystem::StatCacheWriter::writeStatCache(raw_fd_ostream &Out) { - const uint32_t Version = STAT_CACHE_VERSION; - // Magic value. - if (IsCaseSensitive) - Out.write(MAGIC_CASE_SENSITIVE, 4); - else - Out.write(MAGIC_CASE_INSENSITIVE, 4); - // Placeholder for BucketOffset, filled in below. - Out.write("\0\0\0\0", 4); - // Write out the validity token. - Out.write((const char *)&ValidityToken, sizeof(ValidityToken)); - // Write out the version. - Out.write((const char *)&Version, sizeof(Version)); - // Write out the base directory for the cache. - Out.write(BaseDir.c_str(), BaseDir.size() + 1); - // Write out the hashtable data. - uint32_t BucketOffset = Generator->Emit(Out); - int Size = Out.tell(); - // Move back to right after the Magic to insert BucketOffset - Out.seek(4); - Out.write((const char *)&BucketOffset, sizeof(BucketOffset)); - return Size; -} - -Error StatCacheFileSystem::validateCacheFile(MemoryBufferRef Buffer, - StringRef &BaseDir, - bool &IsCaseSensitive, - bool &VersionMatch, - uint64_t &ValidityToken) { - auto *Header = - reinterpret_cast<const StatCacheHeader *>(Buffer.getBufferStart()); - if (Buffer.getBufferSize() < sizeof(StatCacheHeader) || - (memcmp(Header->Magic, MAGIC_CASE_INSENSITIVE, sizeof(Header->Magic)) && - memcmp(Header->Magic, MAGIC_CASE_SENSITIVE, sizeof(Header->Magic))) || - Header->BucketOffset > Buffer.getBufferSize()) - return createStringError(inconvertibleErrorCode(), "Invalid cache file"); - - auto PathLen = - strnlen(Header->BaseDir, - Buffer.getBufferSize() - offsetof(StatCacheHeader, BaseDir)); - if (Header->BaseDir[PathLen] != 0) - return createStringError(inconvertibleErrorCode(), "Invalid cache file"); - - IsCaseSensitive = Header->Magic[1] == MAGIC_CASE_SENSITIVE[1]; - VersionMatch = Header->Version == STAT_CACHE_VERSION; - BaseDir = StringRef(Header->BaseDir, PathLen); - ValidityToken = Header->ValidityToken; - - return ErrorSuccess(); -} - -void StatCacheFileSystem::updateValidityToken(raw_fd_ostream &CacheFile, - uint64_t ValidityToken) { - CacheFile.pwrite(reinterpret_cast<char *>(&ValidityToken), - sizeof(ValidityToken), - offsetof(StatCacheHeader, ValidityToken)); -} - -} // namespace vfs -} // namespace llvm diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp index 89fd0aac17f2f..242bb76865b2c 100644 --- a/llvm/unittests/Support/VirtualFileSystemTest.cpp +++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp @@ -14,11 +14,9 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/SourceMgr.h" -#include "llvm/Support/StatCacheFileSystem.h" #include "llvm/Testing/Support/SupportHelpers.h" #include "gmock/gmock.h" #include "gtest/gtest.h" -#include <list> #include <map> #include <string> @@ -3230,306 +3228,3 @@ TEST(RedirectingFileSystemTest, PrintOutput) { " DummyFileSystem (RecursiveContents)\n", Output); } - -class StatCacheFileSystemTest : public ::testing::Test { -public: - void SetUp() override {} - - template <typename StringCollection> - void createStatCacheFileSystem( - StringRef OutputFile, StringRef BaseDir, bool IsCaseSensitive, - IntrusiveRefCntPtr<vfs::StatCacheFileSystem> &Result, - StringCollection &Filenames, - IntrusiveRefCntPtr<vfs::FileSystem> Lower = new ErrorDummyFileSystem(), - uint64_t ValidityToken = 0) { - sys::fs::file_status s; - status(BaseDir, s); - vfs::StatCacheFileSystem::StatCacheWriter Generator( - BaseDir, s, IsCaseSensitive, ValidityToken); - std::error_code ErrorCode; - - Result.reset(); - - // Base path should be present in the stat cache. - Filenames.push_back(std::string(BaseDir)); - - for (sys::fs::recursive_directory_iterator I(BaseDir, ErrorCode), E; - I != E && !ErrorCode; I.increment(ErrorCode)) { - Filenames.push_back(I->path()); - StringRef Path(Filenames.back().c_str()); - status(Path, s); - Generator.addEntry(Path, s); - } - - { - raw_fd_ostream StatCacheFile(OutputFile, ErrorCode); - ASSERT_FALSE(ErrorCode); - Generator.writeStatCache(StatCacheFile); - } - - loadCacheFile(OutputFile, ValidityToken, Lower, Result); - } - - void loadCacheFile(StringRef OutputFile, uint64_t ExpectedValidityToken, - IntrusiveRefCntPtr<vfs::FileSystem> Lower, - IntrusiveRefCntPtr<vfs::StatCacheFileSystem> &Result) { - auto ErrorOrBuffer = MemoryBuffer::getFile(OutputFile); - EXPECT_TRUE(ErrorOrBuffer); - StringRef CacheBaseDir; - bool IsCaseSensitive; - bool VersionMatch; - uint64_t FileValidityToken; - auto E = vfs::StatCacheFileSystem::validateCacheFile( - (*ErrorOrBuffer)->getMemBufferRef(), CacheBaseDir, IsCaseSensitive, - VersionMatch, FileValidityToken); - ASSERT_FALSE(E); - EXPECT_TRUE(VersionMatch); - EXPECT_EQ(FileValidityToken, ExpectedValidityToken); - auto ExpectedCache = - vfs::StatCacheFileSystem::create(std::move(*ErrorOrBuffer), Lower); - ASSERT_FALSE(ExpectedCache.takeError()); - Result = *ExpectedCache; - } - - template <typename StringCollection> - void - compareStatCacheToRealFS(IntrusiveRefCntPtr<vfs::StatCacheFileSystem> CacheFS, - const StringCollection &Files) { - IntrusiveRefCntPtr<vfs::FileSystem> RealFS = vfs::getRealFileSystem(); - - for (auto &File : Files) { - auto ErrorOrStatus1 = RealFS->status(File); - auto ErrorOrStatus2 = CacheFS->status(File); - - EXPECT_EQ((bool)ErrorOrStatus1, (bool)ErrorOrStatus2); - if (!ErrorOrStatus1 || !ErrorOrStatus2) - continue; - - vfs::Status s1 = *ErrorOrStatus1, s2 = *ErrorOrStatus2; - EXPECT_EQ(s1.getName(), s2.getName()); - EXPECT_EQ(s1.getType(), s2.getType()); - EXPECT_EQ(s1.getPermissions(), s2.getPermissions()); - EXPECT_EQ(s1.getLastModificationTime(), s2.getLastModificationTime()); - EXPECT_EQ(s1.getUniqueID(), s2.getUniqueID()); - EXPECT_EQ(s1.getUser(), s2.getUser()); - EXPECT_EQ(s1.getGroup(), s2.getGroup()); - EXPECT_EQ(s1.getSize(), s2.getSize()); - } - } -}; - -TEST_F(StatCacheFileSystemTest, Basic) { - TempDir TestDirectory("virtual-file-system-test", /*Unique*/ true); - TempDir _a(TestDirectory.path("a")); - TempFile _ab(TestDirectory.path("a/b")); - TempDir _ac(TestDirectory.path("a/c")); - TempFile _acd(TestDirectory.path("a/c/d"), "", "Dummy contents"); - TempFile _ace(TestDirectory.path("a/c/e")); - TempFile _acf(TestDirectory.path("a/c/f"), "", "More dummy contents"); - TempDir _ag(TestDirectory.path("a/g")); - TempFile _agh(TestDirectory.path("a/g/h")); - - StringRef BaseDir(_a.path()); - - SmallVector<std::string, 10> Filenames; - IntrusiveRefCntPtr<vfs::StatCacheFileSystem> StatCacheFS; - createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir, - /* IsCaseSensitive= */ true, StatCacheFS, - Filenames); - ASSERT_TRUE(StatCacheFS); - compareStatCacheToRealFS(StatCacheFS, Filenames); -} - -TEST_F(StatCacheFileSystemTest, CaseSensitivity) { - TempDir TestDirectory("virtual-file-system-test", /*Unique*/ true); - TempDir _a(TestDirectory.path("a")); - TempDir _ac(TestDirectory.path("a/c")); - TempFile _acd(TestDirectory.path("a/c/d"), "", "Dummy contents"); - TempDir _b(TestDirectory.path("B")); - TempDir _bc(TestDirectory.path("B/c")); - TempFile _bcd(TestDirectory.path("B/c/D"), "", "Dummy contents"); - - StringRef BaseDir(TestDirectory.path()); - SmallVector<std::string, 10> Filenames; - IntrusiveRefCntPtr<vfs::StatCacheFileSystem> StatCacheFS; - createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir, - /* IsCaseSensitive= */ true, StatCacheFS, - Filenames); - ASSERT_TRUE(StatCacheFS); - - auto ErrorOrStatus = StatCacheFS->status(_acd.path()); - EXPECT_TRUE(ErrorOrStatus); - ErrorOrStatus = StatCacheFS->status(_bcd.path()); - EXPECT_TRUE(ErrorOrStatus); - ErrorOrStatus = StatCacheFS->status(TestDirectory.path("a/C/d")); - EXPECT_FALSE(ErrorOrStatus); - ErrorOrStatus = StatCacheFS->status(TestDirectory.path("A/C/d")); - EXPECT_FALSE(ErrorOrStatus); - ErrorOrStatus = StatCacheFS->status(TestDirectory.path("a/c/D")); - EXPECT_FALSE(ErrorOrStatus); - ErrorOrStatus = StatCacheFS->status(TestDirectory.path("b/c/d")); - EXPECT_FALSE(ErrorOrStatus); - ErrorOrStatus = StatCacheFS->status(TestDirectory.path("b/C/d")); - EXPECT_FALSE(ErrorOrStatus); - ErrorOrStatus = StatCacheFS->status(TestDirectory.path("B/C/D")); - EXPECT_FALSE(ErrorOrStatus); - - createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir, - /* IsCaseSensitive= */ false, StatCacheFS, - Filenames); - ASSERT_TRUE(StatCacheFS); - ErrorOrStatus = StatCacheFS->status(_acd.path()); - EXPECT_TRUE(ErrorOrStatus); - ErrorOrStatus = StatCacheFS->status(_bcd.path()); - EXPECT_TRUE(ErrorOrStatus); - ErrorOrStatus = StatCacheFS->status(TestDirectory.path("a/C/d")); - EXPECT_TRUE(ErrorOrStatus); - ErrorOrStatus = StatCacheFS->status(TestDirectory.path("A/C/d")); - EXPECT_TRUE(ErrorOrStatus); - ErrorOrStatus = StatCacheFS->status(TestDirectory.path("a/c/D")); - EXPECT_TRUE(ErrorOrStatus); - ErrorOrStatus = StatCacheFS->status(TestDirectory.path("b/c/d")); - EXPECT_TRUE(ErrorOrStatus); - ErrorOrStatus = StatCacheFS->status(TestDirectory.path("b/C/d")); - EXPECT_TRUE(ErrorOrStatus); - ErrorOrStatus = StatCacheFS->status(TestDirectory.path("B/C/D")); - EXPECT_TRUE(ErrorOrStatus); -} - -TEST_F(StatCacheFileSystemTest, DotDot) { - TempDir TestDirectory("virtual-file-system-test", /*Unique*/ true); - TempDir _a(TestDirectory.path("a")); - TempDir _ab(TestDirectory.path("a/b")); - TempFile _abd(TestDirectory.path("a/b/d")); - TempDir _ac(TestDirectory.path("a/c")); - TempFile _acd(TestDirectory.path("a/c/d")); - - StringRef BaseDir(_a.path()); - SmallVector<std::string, 10> Filenames; - IntrusiveRefCntPtr<vfs::StatCacheFileSystem> StatCacheFS; - auto RealFS = vfs::getRealFileSystem(); - createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir, - /* IsCaseSensitive= */ true, StatCacheFS, Filenames, - RealFS); - ASSERT_TRUE(StatCacheFS); - - // Create a file in the cached prefix after the cache was created. - TempFile _abe(TestDirectory.path("a/b/e")); - // Verify the cache is kicking in. - ASSERT_FALSE(StatCacheFS->status(_abe.path())); - // We can access the new file using a ".." because the StatCache will - // just pass that request to the FileSystem below it. - const SmallString<128> PathsToTest[] = { - TestDirectory.path("a/b/../e"), - TestDirectory.path("a/b/../c/d"), - TestDirectory.path("a/b/.."), - }; - compareStatCacheToRealFS(StatCacheFS, PathsToTest); -} - -#ifdef LLVM_ON_UNIX -TEST_F(StatCacheFileSystemTest, Links) { - TempDir TestDirectory("virtual-file-system-test", /*Unique*/ true); - TempDir _a(TestDirectory.path("a")); - TempLink _ab("d", TestDirectory.path("a/b")); - TempFile _ac(TestDirectory.path("a/c")); - TempDir _ad(TestDirectory.path("a/d")); - TempFile _add(TestDirectory.path("a/d/d"), "", "Dummy contents"); - TempFile _ade(TestDirectory.path("a/d/e")); - TempFile _adf(TestDirectory.path("a/d/f"), "", "More dummy contents"); - TempLink _adg(_ad.path(), TestDirectory.path("a/d/g")); - TempDir _ah(TestDirectory.path("a/h")); - TempLink _ahi(_ad.path(), TestDirectory.path("a/h/i")); - TempLink _ahj("no_such_file", TestDirectory.path("a/h/j")); - - StringRef BaseDir(_a.path()); - - SmallVector<std::string, 10> Filenames; - IntrusiveRefCntPtr<vfs::StatCacheFileSystem> StatCacheFS; - createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir, - /* IsCaseSensitive= */ true, StatCacheFS, - Filenames); - ASSERT_TRUE(StatCacheFS); - EXPECT_NE(std::find(Filenames.begin(), Filenames.end(), - TestDirectory.path("a/d/g/g")), - Filenames.end()); - EXPECT_NE(std::find(Filenames.begin(), Filenames.end(), - TestDirectory.path("a/b/e")), - Filenames.end()); - EXPECT_NE(std::find(Filenames.begin(), Filenames.end(), - TestDirectory.path("a/h/i/f")), - Filenames.end()); - EXPECT_NE(std::find(Filenames.begin(), Filenames.end(), - TestDirectory.path("a/h/j")), - Filenames.end()); - compareStatCacheToRealFS(StatCacheFS, Filenames); - - createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir, - /* IsCaseSensitive= */ true, StatCacheFS, Filenames, - vfs::getRealFileSystem()); - const SmallString<128> PathsToTest[] = { - TestDirectory.path("a/h/i/../c"), - TestDirectory.path("a/b/../d"), - TestDirectory.path("a/g/g/../c"), - TestDirectory.path("a/b/.."), - }; - compareStatCacheToRealFS(StatCacheFS, PathsToTest); -} -#endif - -TEST_F(StatCacheFileSystemTest, Canonical) { - TempDir TestDirectory("virtual-file-system-test", /*Unique*/ true); - TempDir _a(TestDirectory.path("a")); - TempFile _ab(TestDirectory.path("a/b")); - TempDir _ac(TestDirectory.path("a/c")); - TempFile _acd(TestDirectory.path("a/c/d"), "", "Dummy contents"); - - StringRef BaseDir(_a.path()); - SmallVector<std::string, 10> Filenames; - IntrusiveRefCntPtr<vfs::StatCacheFileSystem> StatCacheFS; - createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir, - /* IsCaseSensitive= */ true, StatCacheFS, - Filenames); - ASSERT_TRUE(StatCacheFS); - - const SmallString<128> PathsToTest[] = { - TestDirectory.path("./a/b"), TestDirectory.path("a//./b"), - TestDirectory.path("a///b"), TestDirectory.path("a//c//d"), - TestDirectory.path("a//c/./d"), TestDirectory.path("a/./././b"), - TestDirectory.path("a/.//.//.//b"), - }; - compareStatCacheToRealFS(StatCacheFS, PathsToTest); -} - -TEST_F(StatCacheFileSystemTest, ValidityToken) { - TempDir TestDirectory("virtual-file-system-test", /*Unique*/ true); - TempDir _a(TestDirectory.path("a")); - TempFile _ab(TestDirectory.path("a/b")); - TempDir _ac(TestDirectory.path("a/c")); - TempFile _acd(TestDirectory.path("a/c/d"), "", "Dummy contents"); - - StringRef BaseDir(_a.path()); - IntrusiveRefCntPtr<vfs::StatCacheFileSystem> StatCacheFS; - { - SmallVector<std::string, 10> Filenames; - uint64_t ValidityToken = 0x1234567890abcfef; - createStatCacheFileSystem(TestDirectory.path("stat.cache"), BaseDir, - /* IsCaseSensitive= */ true, StatCacheFS, - Filenames, new DummyFileSystem(), ValidityToken); - ASSERT_TRUE(StatCacheFS); - } - - uint64_t UpdatedValidityToken = 0xabcdef0123456789; - { - std::error_code EC; - raw_fd_ostream CacheFile(TestDirectory.path("stat.cache"), EC, - sys::fs::CD_OpenAlways); - ASSERT_FALSE(EC); - vfs::StatCacheFileSystem::updateValidityToken(CacheFile, - UpdatedValidityToken); - } - - loadCacheFile(TestDirectory.path("stat.cache"), UpdatedValidityToken, - new DummyFileSystem(), StatCacheFS); - EXPECT_TRUE(StatCacheFS); -} _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
