This change leaks memory: http://lab.llvm.org:8011/builders/sanitizer-x86_64-linux-fast/builds/5453/steps/check-clang%20asan/logs/stdio
On Mon, Jun 5, 2017 at 11:10 AM, Richard Smith via cfe-commits <cfe-commits@lists.llvm.org> wrote: > Author: rsmith > Date: Mon Jun 5 13:10:11 2017 > New Revision: 304726 > > URL: http://llvm.org/viewvc/llvm-project?rev=304726&view=rev > Log: > Rather than rejecting attempts to run preprocessor-only actions on AST files, > replay the steps taken to create the AST file with the preprocessor-only > action > installed to produce preprocessed output. > > This can be used to produce the preprocessed text for an existing .pch or .pcm > file. > > Modified: > cfe/trunk/include/clang/Basic/LangOptions.h > cfe/trunk/include/clang/Basic/SourceManager.h > cfe/trunk/include/clang/Frontend/ASTUnit.h > cfe/trunk/include/clang/Frontend/FrontendAction.h > cfe/trunk/lib/Basic/SourceManager.cpp > cfe/trunk/lib/Frontend/ASTUnit.cpp > cfe/trunk/lib/Frontend/FrontendAction.cpp > cfe/trunk/lib/Serialization/ASTReader.cpp > cfe/trunk/test/Modules/preprocess-module.cpp > > Modified: cfe/trunk/include/clang/Basic/LangOptions.h > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/LangOptions.h?rev=304726&r1=304725&r2=304726&view=diff > ============================================================================== > --- cfe/trunk/include/clang/Basic/LangOptions.h (original) > +++ cfe/trunk/include/clang/Basic/LangOptions.h Mon Jun 5 13:10:11 2017 > @@ -58,6 +58,7 @@ public: > SOB_Trapping // -ftrapv > }; > > + // FIXME: Unify with TUKind. > enum CompilingModuleKind { > CMK_None, ///< Not compiling a module interface at all. > CMK_ModuleMap, ///< Compiling a module from a module map. > > Modified: cfe/trunk/include/clang/Basic/SourceManager.h > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/SourceManager.h?rev=304726&r1=304725&r2=304726&view=diff > ============================================================================== > --- cfe/trunk/include/clang/Basic/SourceManager.h (original) > +++ cfe/trunk/include/clang/Basic/SourceManager.h Mon Jun 5 13:10:11 2017 > @@ -722,6 +722,10 @@ public: > > void clearIDTables(); > > + /// Initialize this source manager suitably to replay the compilation > + /// described by \p Old. Requires that \p Old outlive \p *this. > + void initializeForReplay(const SourceManager &Old); > + > DiagnosticsEngine &getDiagnostics() const { return Diag; } > > FileManager &getFileManager() const { return FileMgr; } > > Modified: cfe/trunk/include/clang/Frontend/ASTUnit.h > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Frontend/ASTUnit.h?rev=304726&r1=304725&r2=304726&view=diff > ============================================================================== > --- cfe/trunk/include/clang/Frontend/ASTUnit.h (original) > +++ cfe/trunk/include/clang/Frontend/ASTUnit.h Mon Jun 5 13:10:11 2017 > @@ -51,6 +51,7 @@ class DiagnosticsEngine; > class FileEntry; > class FileManager; > class HeaderSearch; > +class InputKind; > class MemoryBufferCache; > class Preprocessor; > class PCHContainerOperations; > @@ -305,9 +306,6 @@ private: > /// (likely to change while trying to use them). > bool UserFilesAreVolatile : 1; > > - /// \brief The language options used when we load an AST file. > - LangOptions ASTFileLangOpts; > - > static void ConfigureDiags(IntrusiveRefCntPtr<DiagnosticsEngine> Diags, > ASTUnit &AST, bool CaptureDiagnostics); > > @@ -702,6 +700,9 @@ public: > /// \brief Determine what kind of translation unit this AST represents. > TranslationUnitKind getTranslationUnitKind() const { return TUKind; } > > + /// \brief Determine the input kind this AST unit represents. > + InputKind getInputKind() const; > + > /// \brief A mapping from a file name to the memory buffer that stores the > /// remapped contents of that file. > typedef std::pair<std::string, llvm::MemoryBuffer *> RemappedFile; > > Modified: cfe/trunk/include/clang/Frontend/FrontendAction.h > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Frontend/FrontendAction.h?rev=304726&r1=304725&r2=304726&view=diff > ============================================================================== > --- cfe/trunk/include/clang/Frontend/FrontendAction.h (original) > +++ cfe/trunk/include/clang/Frontend/FrontendAction.h Mon Jun 5 13:10:11 2017 > @@ -176,10 +176,10 @@ public: > virtual TranslationUnitKind getTranslationUnitKind() { return TU_Complete; > } > > /// \brief Does this action support use with PCH? > - virtual bool hasPCHSupport() const { return !usesPreprocessorOnly(); } > + virtual bool hasPCHSupport() const { return true; } > > /// \brief Does this action support use with AST files? > - virtual bool hasASTFileSupport() const { return !usesPreprocessorOnly(); } > + virtual bool hasASTFileSupport() const { return true; } > > /// \brief Does this action support use with IR files? > virtual bool hasIRSupport() const { return false; } > > Modified: cfe/trunk/lib/Basic/SourceManager.cpp > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Basic/SourceManager.cpp?rev=304726&r1=304725&r2=304726&view=diff > ============================================================================== > --- cfe/trunk/lib/Basic/SourceManager.cpp (original) > +++ cfe/trunk/lib/Basic/SourceManager.cpp Mon Jun 5 13:10:11 2017 > @@ -345,6 +345,41 @@ void SourceManager::clearIDTables() { > createExpansionLoc(SourceLocation(),SourceLocation(),SourceLocation(), 1); > } > > +void SourceManager::initializeForReplay(const SourceManager &Old) { > + assert(MainFileID.isInvalid() && "expected uninitialized SourceManager"); > + > + auto CloneContentCache = [&](const ContentCache *Cache) -> ContentCache * { > + auto *Clone = new (ContentCacheAlloc.Allocate<ContentCache>()) > ContentCache; > + Clone->OrigEntry = Cache->OrigEntry; > + Clone->ContentsEntry = Cache->ContentsEntry; > + Clone->BufferOverridden = Cache->BufferOverridden; > + Clone->IsSystemFile = Cache->IsSystemFile; > + Clone->IsTransient = Cache->IsTransient; > + Clone->replaceBuffer(Cache->getRawBuffer(), /*DoNotFree*/true); > + return Clone; > + }; > + > + // Set up our main file ID as a copy of the old source manager's main file. > + const SLocEntry &OldMainFile = Old.getSLocEntry(Old.getMainFileID()); > + assert(OldMainFile.isFile() && "main file is macro expansion?"); > + setMainFileID(createFileID( > + CloneContentCache(OldMainFile.getFile().getContentCache()), > + SourceLocation(), OldMainFile.getFile().getFileCharacteristic(), 0, > 0)); > + > + // Ensure all SLocEntries are loaded from the external source. > + for (unsigned I = 0, N = Old.LoadedSLocEntryTable.size(); I != N; ++I) > + if (!Old.SLocEntryLoaded[I]) > + Old.loadSLocEntry(I, nullptr); > + > + // Inherit any content cache data from the old source manager. > + for (auto &FileInfo : Old.FileInfos) { > + SrcMgr::ContentCache *&Slot = FileInfos[FileInfo.first]; > + if (Slot) > + continue; > + Slot = CloneContentCache(FileInfo.second); > + } > +} > + > /// getOrCreateContentCache - Create or return a cached ContentCache for the > /// specified file. > const ContentCache * > > Modified: cfe/trunk/lib/Frontend/ASTUnit.cpp > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/ASTUnit.cpp?rev=304726&r1=304725&r2=304726&view=diff > ============================================================================== > --- cfe/trunk/lib/Frontend/ASTUnit.cpp (original) > +++ cfe/trunk/lib/Frontend/ASTUnit.cpp Mon Jun 5 13:10:11 2017 > @@ -667,6 +667,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr > > ConfigureDiags(Diags, *AST, CaptureDiagnostics); > > + AST->LangOpts = std::make_shared<LangOptions>(); > AST->OnlyLocalDecls = OnlyLocalDecls; > AST->CaptureDiagnostics = CaptureDiagnostics; > AST->Diagnostics = Diags; > @@ -682,7 +683,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr > AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts, > AST->getSourceManager(), > AST->getDiagnostics(), > - AST->ASTFileLangOpts, > + AST->getLangOpts(), > /*Target=*/nullptr)); > > auto PPOpts = std::make_shared<PreprocessorOptions>(); > @@ -696,13 +697,13 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr > unsigned Counter; > > AST->PP = std::make_shared<Preprocessor>( > - std::move(PPOpts), AST->getDiagnostics(), AST->ASTFileLangOpts, > + std::move(PPOpts), AST->getDiagnostics(), *AST->LangOpts, > AST->getSourceManager(), *AST->PCMCache, HeaderInfo, *AST, > /*IILookup=*/nullptr, > /*OwnsHeaderSearch=*/false); > Preprocessor &PP = *AST->PP; > > - AST->Ctx = new ASTContext(AST->ASTFileLangOpts, AST->getSourceManager(), > + AST->Ctx = new ASTContext(*AST->LangOpts, AST->getSourceManager(), > PP.getIdentifierTable(), PP.getSelectorTable(), > PP.getBuiltinInfo()); > ASTContext &Context = *AST->Ctx; > @@ -716,7 +717,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr > AllowPCHWithCompilerErrors); > > AST->Reader->setListener(llvm::make_unique<ASTInfoCollector>( > - *AST->PP, Context, AST->ASTFileLangOpts, AST->TargetOpts, AST->Target, > + *AST->PP, Context, *AST->LangOpts, AST->TargetOpts, AST->Target, > Counter)); > > // Attach the AST reader to the AST context as an external AST > @@ -2879,7 +2880,32 @@ const FileEntry *ASTUnit::getPCHFile() { > } > > bool ASTUnit::isModuleFile() { > - return isMainFileAST() && ASTFileLangOpts.isCompilingModule(); > + return isMainFileAST() && getLangOpts().isCompilingModule(); > +} > + > +InputKind ASTUnit::getInputKind() const { > + auto &LangOpts = getLangOpts(); > + > + InputKind::Language Lang; > + if (LangOpts.OpenCL) > + Lang = InputKind::OpenCL; > + else if (LangOpts.CUDA) > + Lang = InputKind::CUDA; > + else if (LangOpts.RenderScript) > + Lang = InputKind::RenderScript; > + else if (LangOpts.CPlusPlus) > + Lang = LangOpts.ObjC1 ? InputKind::ObjCXX : InputKind::CXX; > + else > + Lang = LangOpts.ObjC1 ? InputKind::ObjC : InputKind::C; > + > + InputKind::Format Fmt = InputKind::Source; > + if (LangOpts.getCompilingModule() == LangOptions::CMK_ModuleMap) > + Fmt = InputKind::ModuleMap; > + > + // We don't know if input was preprocessed. Assume not. > + bool PP = false; > + > + return InputKind(Lang, Fmt, PP); > } > > void ASTUnit::PreambleData::countLines() const { > > Modified: cfe/trunk/lib/Frontend/FrontendAction.cpp > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/FrontendAction.cpp?rev=304726&r1=304725&r2=304726&view=diff > ============================================================================== > --- cfe/trunk/lib/Frontend/FrontendAction.cpp (original) > +++ cfe/trunk/lib/Frontend/FrontendAction.cpp Mon Jun 5 13:10:11 2017 > @@ -387,8 +387,7 @@ static std::error_code collectModuleHead > return std::error_code(); > } > > -static bool loadModuleMapForModuleBuild(CompilerInstance &CI, > - StringRef Filename, bool IsSystem, > +static bool loadModuleMapForModuleBuild(CompilerInstance &CI, bool IsSystem, > bool IsPreprocessed, > std::string &PresumedModuleMapFile, > unsigned &Offset) { > @@ -523,7 +522,8 @@ getInputBufferForModule(CompilerInstance > } > > bool FrontendAction::BeginSourceFile(CompilerInstance &CI, > - const FrontendInputFile &Input) { > + const FrontendInputFile &RealInput) { > + FrontendInputFile Input(RealInput); > assert(!Instance && "Already processing a source file!"); > assert(!Input.isEmpty() && "Unexpected empty filename!"); > setCurrentInput(Input); > @@ -531,15 +531,69 @@ bool FrontendAction::BeginSourceFile(Com > > StringRef InputFile = Input.getFile(); > bool HasBegunSourceFile = false; > + bool ReplayASTFile = Input.getKind().getFormat() == InputKind::Precompiled > && > + usesPreprocessorOnly(); > if (!BeginInvocation(CI)) > goto failure; > > + // If we're replaying the build of an AST file, import it and set up > + // the initial state from its build. > + if (ReplayASTFile) { > + IntrusiveRefCntPtr<DiagnosticsEngine> Diags(&CI.getDiagnostics()); > + > + // The AST unit populates its own diagnostics engine rather than ours. > + IntrusiveRefCntPtr<DiagnosticsEngine> ASTDiags( > + new DiagnosticsEngine(Diags->getDiagnosticIDs(), > + &Diags->getDiagnosticOptions())); > + ASTDiags->setClient(Diags->getClient(), /*OwnsClient*/false); > + > + std::unique_ptr<ASTUnit> AST = ASTUnit::LoadFromASTFile( > + InputFile, CI.getPCHContainerReader(), ASTDiags, > CI.getFileSystemOpts(), > + CI.getCodeGenOpts().DebugTypeExtRefs); > + if (!AST) > + goto failure; > + > + // Options relating to how we treat the input (but not what we do with > it) > + // are inherited from the AST unit. > + CI.getLangOpts() = AST->getLangOpts(); > + > + // Preload all the module files loaded transitively by the AST unit. > + if (auto ASTReader = AST->getASTReader()) { > + auto &MM = ASTReader->getModuleManager(); > + for (ModuleFile &MF : MM) > + if (&MF != &MM.getPrimaryModule()) > + CI.getFrontendOpts().ModuleFiles.push_back(MF.FileName); > + } > + > + // Set the shared objects, these are reset when we finish processing the > + // file, otherwise the CompilerInstance will happily destroy them. > + CI.setFileManager(&AST->getFileManager()); > + CI.createSourceManager(CI.getFileManager()); > + CI.getSourceManager().initializeForReplay(AST->getSourceManager()); > + CI.createPreprocessor(getTranslationUnitKind()); > + > + // Set up the input file for replay purposes. > + auto Kind = AST->getInputKind(); > + if (Kind.getFormat() == InputKind::ModuleMap) { > + Module *ASTModule = > + AST->getPreprocessor().getHeaderSearchInfo().lookupModule( > + AST->getLangOpts().CurrentModule, /*AllowSearch*/ false); > + Input = FrontendInputFile(ASTModule->PresumedModuleMapFile, Kind); > + } else { > + auto &SM = CI.getSourceManager(); > + FileID ID = SM.getMainFileID(); > + if (auto *File = SM.getFileEntryForID(ID)) > + Input = FrontendInputFile(File->getName(), Kind); > + else > + Input = FrontendInputFile(SM.getBuffer(ID), Kind); > + } > + setCurrentInput(Input, std::move(AST)); > + } > + > // AST files follow a very different path, since they share objects via the > // AST unit. > if (Input.getKind().getFormat() == InputKind::Precompiled) { > - // FIXME: We should not be asserting on bad command-line arguments. > - assert(!usesPreprocessorOnly() && > - "Attempt to pass AST file to preprocessor only action!"); > + assert(!usesPreprocessorOnly() && "this case was handled above"); > assert(hasASTFileSupport() && > "This action does not have AST file support!"); > > @@ -680,7 +734,7 @@ bool FrontendAction::BeginSourceFile(Com > > std::string PresumedModuleMapFile; > unsigned OffsetToContents; > - if (loadModuleMapForModuleBuild(CI, Input.getFile(), Input.isSystem(), > + if (loadModuleMapForModuleBuild(CI, Input.isSystem(), > Input.isPreprocessed(), > PresumedModuleMapFile, OffsetToContents)) > goto failure; > @@ -829,14 +883,7 @@ bool FrontendAction::BeginSourceFile(Com > > // If we failed, reset state since the client will not end up calling the > // matching EndSourceFile(). > - failure: > - if (isCurrentFileAST()) { > - CI.setASTContext(nullptr); > - CI.setPreprocessor(nullptr); > - CI.setSourceManager(nullptr); > - CI.setFileManager(nullptr); > - } > - > +failure: > if (HasBegunSourceFile) > CI.getDiagnosticClient().EndSourceFile(); > CI.clearOutputFiles(/*EraseFiles=*/true); > @@ -914,6 +961,7 @@ void FrontendAction::EndSourceFile() { > CI.resetAndLeakPreprocessor(); > CI.resetAndLeakSourceManager(); > CI.resetAndLeakFileManager(); > + BuryPointer(CurrentASTUnit.release()); > } else { > CI.setPreprocessor(nullptr); > CI.setSourceManager(nullptr); > > Modified: cfe/trunk/lib/Serialization/ASTReader.cpp > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Serialization/ASTReader.cpp?rev=304726&r1=304725&r2=304726&view=diff > ============================================================================== > --- cfe/trunk/lib/Serialization/ASTReader.cpp (original) > +++ cfe/trunk/lib/Serialization/ASTReader.cpp Mon Jun 5 13:10:11 2017 > @@ -4918,6 +4918,7 @@ ASTReader::ReadSubmoduleBlock(ModuleFile > } > > CurrentModule->setASTFile(F.File); > + CurrentModule->PresumedModuleMapFile = F.ModuleMapPath; > } > > CurrentModule->Kind = ModuleKind; > > Modified: cfe/trunk/test/Modules/preprocess-module.cpp > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Modules/preprocess-module.cpp?rev=304726&r1=304725&r2=304726&view=diff > ============================================================================== > --- cfe/trunk/test/Modules/preprocess-module.cpp (original) > +++ cfe/trunk/test/Modules/preprocess-module.cpp Mon Jun 5 13:10:11 2017 > @@ -34,10 +34,15 @@ > // RUN: rm %t/fwd.h %t/file.h %t/file2.h %t/module.modulemap > // RUN: %clang_cc1 -fmodules -fmodule-name=file -fmodule-file=%t/fwd.pcm -x > c++-module-map-cpp-output %t/copy.ii -emit-module -o %t/copy.pcm > > -// Finally, check that our module contains correct mapping information for > the headers. > +// Check that our module contains correct mapping information for the > headers. > // RUN: cp %S/Inputs/preprocess/fwd.h %S/Inputs/preprocess/file.h > %S/Inputs/preprocess/file2.h %S/Inputs/preprocess/module.modulemap %t > // RUN: %clang_cc1 -fmodules -fmodule-file=%t/copy.pcm %s -I%t -verify > -fno-modules-error-recovery -DCOPY -DINCLUDE > > +// Check that we can preprocess from a .pcm file and that we get the same > result as preprocessing from the original sources. > +// RUN: %clang_cc1 -fmodules -fmodule-name=file -fmodule-file=%t/fwd.pcm > -I%S/Inputs/preprocess -x c++-module-map > %S/Inputs/preprocess/module.modulemap -emit-module -o %t/file.pcm > +// RUN: %clang_cc1 -fmodules -fmodule-name=file -fmodule-file=%t/fwd.pcm > -I%S/Inputs/preprocess %t/file.pcm -E -frewrite-includes -o %t/file.rewrite.ii > +// RUN: cmp %t/rewrite.ii %t/file.rewrite.ii > + > // == module map > // CHECK: # 1 "{{.*}}module.modulemap" > // CHECK: module file { > > > _______________________________________________ > cfe-commits mailing list > cfe-commits@lists.llvm.org > http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
