//===- VirtualFileSystem.cpp - Virtual File System Layer ------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file implements the VirtualFileSystem interface. // //===----------------------------------------------------------------------===// #include "llvm/Support/VirtualFileSystem.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" #include "llvm/ADT/Twine.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Config/llvm-config.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Chrono.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Errc.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/FileSystem/UniqueID.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/SMLoc.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/YAMLParser.h" #include "llvm/Support/raw_ostream.h" #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace llvm; using namespace llvm::vfs; using llvm::sys::fs::file_t; using llvm::sys::fs::file_status; using llvm::sys::fs::file_type; using llvm::sys::fs::kInvalidFile; using llvm::sys::fs::perms; using llvm::sys::fs::UniqueID; Status::Status(const file_status &Status) : UID(Status.getUniqueID()), MTime(Status.getLastModificationTime()), User(Status.getUser()), Group(Status.getGroup()), Size(Status.getSize()), Type(Status.type()), Perms(Status.permissions()) {} Status::Status(const Twine &Name, UniqueID UID, sys::TimePoint<> MTime, uint32_t User, uint32_t Group, uint64_t Size, file_type Type, perms Perms) : Name(Name.str()), UID(UID), MTime(MTime), User(User), Group(Group), Size(Size), Type(Type), Perms(Perms) {} Status Status::copyWithNewSize(const Status &In, uint64_t NewSize) { return Status(In.getName(), In.getUniqueID(), In.getLastModificationTime(), In.getUser(), In.getGroup(), NewSize, In.getType(), In.getPermissions()); } Status Status::copyWithNewName(const Status &In, const Twine &NewName) { return Status(NewName, In.getUniqueID(), In.getLastModificationTime(), In.getUser(), In.getGroup(), In.getSize(), In.getType(), In.getPermissions()); } Status Status::copyWithNewName(const file_status &In, const Twine &NewName) { return Status(NewName, In.getUniqueID(), In.getLastModificationTime(), In.getUser(), In.getGroup(), In.getSize(), In.type(), In.permissions()); } bool Status::equivalent(const Status &Other) const { assert(isStatusKnown() && Other.isStatusKnown()); return getUniqueID() == Other.getUniqueID(); } bool Status::isDirectory() const { return Type == file_type::directory_file; } bool Status::isRegularFile() const { return Type == file_type::regular_file; } bool Status::isOther() const { return exists() && !isRegularFile() && !isDirectory() && !isSymlink(); } bool Status::isSymlink() const { return Type == file_type::symlink_file; } bool Status::isStatusKnown() const { return Type != file_type::status_error; } bool Status::exists() const { return isStatusKnown() && Type != file_type::file_not_found; } File::~File() = default; FileSystem::~FileSystem() = default; ErrorOr> FileSystem::getBufferForFile(const llvm::Twine &Name, int64_t FileSize, bool RequiresNullTerminator, bool IsVolatile) { auto F = openFileForRead(Name); if (!F) return F.getError(); return (*F)->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile); } std::error_code FileSystem::makeAbsolute(SmallVectorImpl &Path) const { if (llvm::sys::path::is_absolute(Path)) return {}; auto WorkingDir = getCurrentWorkingDirectory(); if (!WorkingDir) return WorkingDir.getError(); llvm::sys::fs::make_absolute(WorkingDir.get(), Path); return {}; } std::error_code FileSystem::getRealPath(const Twine &Path, SmallVectorImpl &Output) { return errc::operation_not_permitted; } std::error_code FileSystem::isLocal(const Twine &Path, bool &Result) { return errc::operation_not_permitted; } bool FileSystem::exists(const Twine &Path) { auto Status = status(Path); return Status && Status->exists(); } llvm::ErrorOr FileSystem::equivalent(const Twine &A, const Twine &B) { auto StatusA = status(A); if (!StatusA) return StatusA.getError(); auto StatusB = status(B); if (!StatusB) return StatusB.getError(); return StatusA->equivalent(*StatusB); } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void FileSystem::dump() const { print(dbgs(), PrintType::RecursiveContents); } #endif #ifndef NDEBUG static bool isTraversalComponent(StringRef Component) { return Component == ".." || Component == "."; } static bool pathHasTraversal(StringRef Path) { using namespace llvm::sys; for (StringRef Comp : llvm::make_range(path::begin(Path), path::end(Path))) if (isTraversalComponent(Comp)) return true; return false; } #endif //===-----------------------------------------------------------------------===/ // RealFileSystem implementation //===-----------------------------------------------------------------------===/ namespace { /// Wrapper around a raw file descriptor. class RealFile : public File { friend class RealFileSystem; file_t FD; Status S; std::string RealName; RealFile(file_t RawFD, StringRef NewName, StringRef NewRealPathName) : FD(RawFD), S(NewName, {}, {}, {}, {}, {}, llvm::sys::fs::file_type::status_error, {}), RealName(NewRealPathName.str()) { assert(FD != kInvalidFile && "Invalid or inactive file descriptor"); } public: ~RealFile() override; ErrorOr status() override; ErrorOr getName() override; ErrorOr> getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, bool IsVolatile) override; std::error_code close() override; void setPath(const Twine &Path) override; }; } // namespace RealFile::~RealFile() { close(); } ErrorOr RealFile::status() { assert(FD != kInvalidFile && "cannot stat closed file"); if (!S.isStatusKnown()) { file_status RealStatus; if (std::error_code EC = sys::fs::status(FD, RealStatus)) return EC; S = Status::copyWithNewName(RealStatus, S.getName()); } return S; } ErrorOr RealFile::getName() { return RealName.empty() ? S.getName().str() : RealName; } ErrorOr> RealFile::getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, bool IsVolatile) { assert(FD != kInvalidFile && "cannot get buffer for closed file"); return MemoryBuffer::getOpenFile(FD, Name, FileSize, RequiresNullTerminator, IsVolatile); } std::error_code RealFile::close() { std::error_code EC = sys::fs::closeFile(FD); FD = kInvalidFile; return EC; } void RealFile::setPath(const Twine &Path) { RealName = Path.str(); if (auto Status = status()) S = Status.get().copyWithNewName(Status.get(), Path); } namespace { /// A file system according to your operating system. /// This may be linked to the process's working directory, or maintain its own. /// /// Currently, its own working directory is emulated by storing the path and /// sending absolute paths to llvm::sys::fs:: functions. /// A more principled approach would be to push this down a level, modelling /// the working dir as an llvm::sys::fs::WorkingDir or similar. /// This would enable the use of openat()-style functions on some platforms. class RealFileSystem : public FileSystem { public: explicit RealFileSystem(bool LinkCWDToProcess) { if (!LinkCWDToProcess) { SmallString<128> PWD, RealPWD; if (std::error_code EC = llvm::sys::fs::current_path(PWD)) WD = EC; else if (llvm::sys::fs::real_path(PWD, RealPWD)) WD = WorkingDirectory{PWD, PWD}; else WD = WorkingDirectory{PWD, RealPWD}; } } ErrorOr status(const Twine &Path) override; ErrorOr> openFileForRead(const Twine &Path) override; directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override; llvm::ErrorOr getCurrentWorkingDirectory() const override; std::error_code setCurrentWorkingDirectory(const Twine &Path) override; std::error_code isLocal(const Twine &Path, bool &Result) override; std::error_code getRealPath(const Twine &Path, SmallVectorImpl &Output) override; protected: void printImpl(raw_ostream &OS, PrintType Type, unsigned IndentLevel) const override; private: // If this FS has its own working dir, use it to make Path absolute. // The returned twine is safe to use as long as both Storage and Path live. Twine adjustPath(const Twine &Path, SmallVectorImpl &Storage) const { if (!WD || !*WD) return Path; Path.toVector(Storage); sys::fs::make_absolute(WD->get().Resolved, Storage); return Storage; } struct WorkingDirectory { // The current working directory, without symlinks resolved. (echo $PWD). SmallString<128> Specified; // The current working directory, with links resolved. (readlink .). SmallString<128> Resolved; }; std::optional> WD; }; } // namespace ErrorOr RealFileSystem::status(const Twine &Path) { SmallString<256> Storage; sys::fs::file_status RealStatus; if (std::error_code EC = sys::fs::status(adjustPath(Path, Storage), RealStatus)) return EC; return Status::copyWithNewName(RealStatus, Path); } ErrorOr> RealFileSystem::openFileForRead(const Twine &Name) { SmallString<256> RealName, Storage; Expected FDOrErr = sys::fs::openNativeFileForRead( adjustPath(Name, Storage), sys::fs::OF_None, &RealName); if (!FDOrErr) return errorToErrorCode(FDOrErr.takeError()); return std::unique_ptr( new RealFile(*FDOrErr, Name.str(), RealName.str())); } llvm::ErrorOr RealFileSystem::getCurrentWorkingDirectory() const { if (WD && *WD) return std::string(WD->get().Specified); if (WD) return WD->getError(); SmallString<128> Dir; if (std::error_code EC = llvm::sys::fs::current_path(Dir)) return EC; return std::string(Dir); } std::error_code RealFileSystem::setCurrentWorkingDirectory(const Twine &Path) { if (!WD) return llvm::sys::fs::set_current_path(Path); SmallString<128> Absolute, Resolved, Storage; adjustPath(Path, Storage).toVector(Absolute); bool IsDir; if (auto Err = llvm::sys::fs::is_directory(Absolute, IsDir)) return Err; if (!IsDir) return std::make_error_code(std::errc::not_a_directory); if (auto Err = llvm::sys::fs::real_path(Absolute, Resolved)) return Err; WD = WorkingDirectory{Absolute, Resolved}; return std::error_code(); } std::error_code RealFileSystem::isLocal(const Twine &Path, bool &Result) { SmallString<256> Storage; return llvm::sys::fs::is_local(adjustPath(Path, Storage), Result); } std::error_code RealFileSystem::getRealPath(const Twine &Path, SmallVectorImpl &Output) { SmallString<256> Storage; return llvm::sys::fs::real_path(adjustPath(Path, Storage), Output); } void RealFileSystem::printImpl(raw_ostream &OS, PrintType Type, unsigned IndentLevel) const { printIndent(OS, IndentLevel); OS << "RealFileSystem using "; if (WD) OS << "own"; else OS << "process"; OS << " CWD\n"; } IntrusiveRefCntPtr vfs::getRealFileSystem() { static IntrusiveRefCntPtr FS(new RealFileSystem(true)); return FS; } std::unique_ptr vfs::createPhysicalFileSystem() { return std::make_unique(false); } namespace { class RealFSDirIter : public llvm::vfs::detail::DirIterImpl { llvm::sys::fs::directory_iterator Iter; public: RealFSDirIter(const Twine &Path, std::error_code &EC) : Iter(Path, EC) { if (Iter != llvm::sys::fs::directory_iterator()) CurrentEntry = directory_entry(Iter->path(), Iter->type()); } std::error_code increment() override { std::error_code EC; Iter.increment(EC); CurrentEntry = (Iter == llvm::sys::fs::directory_iterator()) ? directory_entry() : directory_entry(Iter->path(), Iter->type()); return EC; } }; } // namespace directory_iterator RealFileSystem::dir_begin(const Twine &Dir, std::error_code &EC) { SmallString<128> Storage; return directory_iterator( std::make_shared(adjustPath(Dir, Storage), EC)); } //===-----------------------------------------------------------------------===/ // OverlayFileSystem implementation //===-----------------------------------------------------------------------===/ OverlayFileSystem::OverlayFileSystem(IntrusiveRefCntPtr BaseFS) { FSList.push_back(std::move(BaseFS)); } void OverlayFileSystem::pushOverlay(IntrusiveRefCntPtr FS) { FSList.push_back(FS); // Synchronize added file systems by duplicating the working directory from // the first one in the list. FS->setCurrentWorkingDirectory(getCurrentWorkingDirectory().get()); } ErrorOr OverlayFileSystem::status(const Twine &Path) { // FIXME: handle symlinks that cross file systems for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) { ErrorOr Status = (*I)->status(Path); if (Status || Status.getError() != llvm::errc::no_such_file_or_directory) return Status; } return make_error_code(llvm::errc::no_such_file_or_directory); } bool OverlayFileSystem::exists(const Twine &Path) { // FIXME: handle symlinks that cross file systems for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) { if ((*I)->exists(Path)) return true; } return false; } ErrorOr> OverlayFileSystem::openFileForRead(const llvm::Twine &Path) { // FIXME: handle symlinks that cross file systems for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) { auto Result = (*I)->openFileForRead(Path); if (Result || Result.getError() != llvm::errc::no_such_file_or_directory) return Result; } return make_error_code(llvm::errc::no_such_file_or_directory); } llvm::ErrorOr OverlayFileSystem::getCurrentWorkingDirectory() const { // All file systems are synchronized, just take the first working directory. return FSList.front()->getCurrentWorkingDirectory(); } std::error_code OverlayFileSystem::setCurrentWorkingDirectory(const Twine &Path) { for (auto &FS : FSList) if (std::error_code EC = FS->setCurrentWorkingDirectory(Path)) return EC; return {}; } std::error_code OverlayFileSystem::isLocal(const Twine &Path, bool &Result) { for (auto &FS : FSList) if (FS->exists(Path)) return FS->isLocal(Path, Result); return errc::no_such_file_or_directory; } std::error_code OverlayFileSystem::getRealPath(const Twine &Path, SmallVectorImpl &Output) { for (const auto &FS : FSList) if (FS->exists(Path)) return FS->getRealPath(Path, Output); return errc::no_such_file_or_directory; } void OverlayFileSystem::visitChildFileSystems(VisitCallbackTy Callback) { for (IntrusiveRefCntPtr FS : overlays_range()) { Callback(*FS); FS->visitChildFileSystems(Callback); } } void OverlayFileSystem::printImpl(raw_ostream &OS, PrintType Type, unsigned IndentLevel) const { printIndent(OS, IndentLevel); OS << "OverlayFileSystem\n"; if (Type == PrintType::Summary) return; if (Type == PrintType::Contents) Type = PrintType::Summary; for (const auto &FS : overlays_range()) FS->print(OS, Type, IndentLevel + 1); } llvm::vfs::detail::DirIterImpl::~DirIterImpl() = default; namespace { /// Combines and deduplicates directory entries across multiple file systems. class CombiningDirIterImpl : public llvm::vfs::detail::DirIterImpl { using FileSystemPtr = llvm::IntrusiveRefCntPtr; /// Iterators to combine, processed in reverse order. SmallVector IterList; /// The iterator currently being traversed. directory_iterator CurrentDirIter; /// The set of names already returned as entries. llvm::StringSet<> SeenNames; /// Sets \c CurrentDirIter to the next iterator in the list, or leaves it as /// is (at its end position) if we've already gone through them all. std::error_code incrementIter(bool IsFirstTime) { while (!IterList.empty()) { CurrentDirIter = IterList.back(); IterList.pop_back(); if (CurrentDirIter != directory_iterator()) break; // found } if (IsFirstTime && CurrentDirIter == directory_iterator()) return errc::no_such_file_or_directory; return {}; } std::error_code incrementDirIter(bool IsFirstTime) { assert((IsFirstTime || CurrentDirIter != directory_iterator()) && "incrementing past end"); std::error_code EC; if (!IsFirstTime) CurrentDirIter.increment(EC); if (!EC && CurrentDirIter == directory_iterator()) EC = incrementIter(IsFirstTime); return EC; } std::error_code incrementImpl(bool IsFirstTime) { while (true) { std::error_code EC = incrementDirIter(IsFirstTime); if (EC || CurrentDirIter == directory_iterator()) { CurrentEntry = directory_entry(); return EC; } CurrentEntry = *CurrentDirIter; StringRef Name = llvm::sys::path::filename(CurrentEntry.path()); if (SeenNames.insert(Name).second) return EC; // name not seen before } llvm_unreachable("returned above"); } public: CombiningDirIterImpl(ArrayRef FileSystems, std::string Dir, std::error_code &EC) { for (const auto &FS : FileSystems) { std::error_code FEC; directory_iterator Iter = FS->dir_begin(Dir, FEC); if (FEC && FEC != errc::no_such_file_or_directory) { EC = FEC; return; } if (!FEC) IterList.push_back(Iter); } EC = incrementImpl(true); } CombiningDirIterImpl(ArrayRef DirIters, std::error_code &EC) : IterList(DirIters.begin(), DirIters.end()) { EC = incrementImpl(true); } std::error_code increment() override { return incrementImpl(false); } }; } // namespace directory_iterator OverlayFileSystem::dir_begin(const Twine &Dir, std::error_code &EC) { directory_iterator Combined = directory_iterator( std::make_shared(FSList, Dir.str(), EC)); if (EC) return {}; return Combined; } void ProxyFileSystem::anchor() {} namespace llvm { namespace vfs { namespace detail { enum InMemoryNodeKind { IME_File, IME_Directory, IME_HardLink, IME_SymbolicLink, }; /// The in memory file system is a tree of Nodes. Every node can either be a /// file, symlink, hardlink or a directory. class InMemoryNode { InMemoryNodeKind Kind; std::string FileName; public: InMemoryNode(llvm::StringRef FileName, InMemoryNodeKind Kind) : Kind(Kind), FileName(std::string(llvm::sys::path::filename(FileName))) { } virtual ~InMemoryNode() = default; /// Return the \p Status for this node. \p RequestedName should be the name /// through which the caller referred to this node. It will override /// \p Status::Name in the return value, to mimic the behavior of \p RealFile. virtual Status getStatus(const Twine &RequestedName) const = 0; /// Get the filename of this node (the name without the directory part). StringRef getFileName() const { return FileName; } InMemoryNodeKind getKind() const { return Kind; } virtual std::string toString(unsigned Indent) const = 0; }; class InMemoryFile : public InMemoryNode { Status Stat; std::unique_ptr Buffer; public: InMemoryFile(Status Stat, std::unique_ptr Buffer) : InMemoryNode(Stat.getName(), IME_File), Stat(std::move(Stat)), Buffer(std::move(Buffer)) {} Status getStatus(const Twine &RequestedName) const override { return Status::copyWithNewName(Stat, RequestedName); } llvm::MemoryBuffer *getBuffer() const { return Buffer.get(); } std::string toString(unsigned Indent) const override { return (std::string(Indent, ' ') + Stat.getName() + "\n").str(); } static bool classof(const InMemoryNode *N) { return N->getKind() == IME_File; } }; namespace { class InMemoryHardLink : public InMemoryNode { const InMemoryFile &ResolvedFile; public: InMemoryHardLink(StringRef Path, const InMemoryFile &ResolvedFile) : InMemoryNode(Path, IME_HardLink), ResolvedFile(ResolvedFile) {} const InMemoryFile &getResolvedFile() const { return ResolvedFile; } Status getStatus(const Twine &RequestedName) const override { return ResolvedFile.getStatus(RequestedName); } std::string toString(unsigned Indent) const override { return std::string(Indent, ' ') + "HardLink to -> " + ResolvedFile.toString(0); } static bool classof(const InMemoryNode *N) { return N->getKind() == IME_HardLink; } }; class InMemorySymbolicLink : public InMemoryNode { std::string TargetPath; Status Stat; public: InMemorySymbolicLink(StringRef Path, StringRef TargetPath, Status Stat) : InMemoryNode(Path, IME_SymbolicLink), TargetPath(std::move(TargetPath)), Stat(Stat) {} std::string toString(unsigned Indent) const override { return std::string(Indent, ' ') + "SymbolicLink to -> " + TargetPath; } Status getStatus(const Twine &RequestedName) const override { return Status::copyWithNewName(Stat, RequestedName); } StringRef getTargetPath() const { return TargetPath; } static bool classof(const InMemoryNode *N) { return N->getKind() == IME_SymbolicLink; } }; /// Adapt a InMemoryFile for VFS' File interface. The goal is to make /// \p InMemoryFileAdaptor mimic as much as possible the behavior of /// \p RealFile. class InMemoryFileAdaptor : public File { const InMemoryFile &Node; /// The name to use when returning a Status for this file. std::string RequestedName; public: explicit InMemoryFileAdaptor(const InMemoryFile &Node, std::string RequestedName) : Node(Node), RequestedName(std::move(RequestedName)) {} llvm::ErrorOr status() override { return Node.getStatus(RequestedName); } llvm::ErrorOr> getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, bool IsVolatile) override { llvm::MemoryBuffer *Buf = Node.getBuffer(); return llvm::MemoryBuffer::getMemBuffer( Buf->getBuffer(), Buf->getBufferIdentifier(), RequiresNullTerminator); } std::error_code close() override { return {}; } void setPath(const Twine &Path) override { RequestedName = Path.str(); } }; } // namespace class InMemoryDirectory : public InMemoryNode { Status Stat; std::map> Entries; public: InMemoryDirectory(Status Stat) : InMemoryNode(Stat.getName(), IME_Directory), Stat(std::move(Stat)) {} /// Return the \p Status for this node. \p RequestedName should be the name /// through which the caller referred to this node. It will override /// \p Status::Name in the return value, to mimic the behavior of \p RealFile. Status getStatus(const Twine &RequestedName) const override { return Status::copyWithNewName(Stat, RequestedName); } UniqueID getUniqueID() const { return Stat.getUniqueID(); } InMemoryNode *getChild(StringRef Name) const { auto I = Entries.find(Name.str()); if (I != Entries.end()) return I->second.get(); return nullptr; } InMemoryNode *addChild(StringRef Name, std::unique_ptr Child) { return Entries.emplace(Name, std::move(Child)).first->second.get(); } using const_iterator = decltype(Entries)::const_iterator; const_iterator begin() const { return Entries.begin(); } const_iterator end() const { return Entries.end(); } std::string toString(unsigned Indent) const override { std::string Result = (std::string(Indent, ' ') + Stat.getName() + "\n").str(); for (const auto &Entry : Entries) Result += Entry.second->toString(Indent + 2); return Result; } static bool classof(const InMemoryNode *N) { return N->getKind() == IME_Directory; } }; } // namespace detail // The UniqueID of in-memory files is derived from path and content. // This avoids difficulties in creating exactly equivalent in-memory FSes, // as often needed in multithreaded programs. static sys::fs::UniqueID getUniqueID(hash_code Hash) { return sys::fs::UniqueID(std::numeric_limits::max(), uint64_t(size_t(Hash))); } static sys::fs::UniqueID getFileID(sys::fs::UniqueID Parent, llvm::StringRef Name, llvm::StringRef Contents) { return getUniqueID(llvm::hash_combine(Parent.getFile(), Name, Contents)); } static sys::fs::UniqueID getDirectoryID(sys::fs::UniqueID Parent, llvm::StringRef Name) { return getUniqueID(llvm::hash_combine(Parent.getFile(), Name)); } Status detail::NewInMemoryNodeInfo::makeStatus() const { UniqueID UID = (Type == sys::fs::file_type::directory_file) ? getDirectoryID(DirUID, Name) : getFileID(DirUID, Name, Buffer ? Buffer->getBuffer() : ""); return Status(Path, UID, llvm::sys::toTimePoint(ModificationTime), User, Group, Buffer ? Buffer->getBufferSize() : 0, Type, Perms); } InMemoryFileSystem::InMemoryFileSystem(bool UseNormalizedPaths) : Root(new detail::InMemoryDirectory( Status("", getDirectoryID(llvm::sys::fs::UniqueID(), ""), llvm::sys::TimePoint<>(), 0, 0, 0, llvm::sys::fs::file_type::directory_file, llvm::sys::fs::perms::all_all))), UseNormalizedPaths(UseNormalizedPaths) {} InMemoryFileSystem::~InMemoryFileSystem() = default; std::string InMemoryFileSystem::toString() const { return Root->toString(/*Indent=*/0); } bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime, std::unique_ptr Buffer, std::optional User, std::optional Group, std::optional Type, std::optional Perms, MakeNodeFn MakeNode) { SmallString<128> Path; P.toVector(Path); // Fix up relative paths. This just prepends the current working directory. std::error_code EC = makeAbsolute(Path); assert(!EC); (void)EC; if (useNormalizedPaths()) llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true); if (Path.empty()) return false; detail::InMemoryDirectory *Dir = Root.get(); auto I = llvm::sys::path::begin(Path), E = sys::path::end(Path); const auto ResolvedUser = User.value_or(0); const auto ResolvedGroup = Group.value_or(0); const auto ResolvedType = Type.value_or(sys::fs::file_type::regular_file); const auto ResolvedPerms = Perms.value_or(sys::fs::all_all); // Any intermediate directories we create should be accessible by // the owner, even if Perms says otherwise for the final path. const auto NewDirectoryPerms = ResolvedPerms | sys::fs::owner_all; StringRef Name = *I; while (true) { Name = *I; ++I; if (I == E) break; detail::InMemoryNode *Node = Dir->getChild(Name); if (!Node) { // This isn't the last element, so we create a new directory. Status Stat( StringRef(Path.str().begin(), Name.end() - Path.str().begin()), getDirectoryID(Dir->getUniqueID(), Name), llvm::sys::toTimePoint(ModificationTime), ResolvedUser, ResolvedGroup, 0, sys::fs::file_type::directory_file, NewDirectoryPerms); Dir = cast(Dir->addChild( Name, std::make_unique(std::move(Stat)))); continue; } // Creating file under another file. if (!isa(Node)) return false; Dir = cast(Node); } detail::InMemoryNode *Node = Dir->getChild(Name); if (!Node) { Dir->addChild(Name, MakeNode({Dir->getUniqueID(), Path, Name, ModificationTime, std::move(Buffer), ResolvedUser, ResolvedGroup, ResolvedType, ResolvedPerms})); return true; } if (isa(Node)) return ResolvedType == sys::fs::file_type::directory_file; assert((isa(Node) || isa(Node)) && "Must be either file, hardlink or directory!"); // Return false only if the new file is different from the existing one. if (auto *Link = dyn_cast(Node)) { return Link->getResolvedFile().getBuffer()->getBuffer() == Buffer->getBuffer(); } return cast(Node)->getBuffer()->getBuffer() == Buffer->getBuffer(); } bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime, std::unique_ptr Buffer, std::optional User, std::optional Group, std::optional Type, std::optional Perms) { return addFile(P, ModificationTime, std::move(Buffer), User, Group, Type, Perms, [](detail::NewInMemoryNodeInfo NNI) -> std::unique_ptr { Status Stat = NNI.makeStatus(); if (Stat.getType() == sys::fs::file_type::directory_file) return std::make_unique(Stat); return std::make_unique( Stat, std::move(NNI.Buffer)); }); } bool InMemoryFileSystem::addFileNoOwn( const Twine &P, time_t ModificationTime, const llvm::MemoryBufferRef &Buffer, std::optional User, std::optional Group, std::optional Type, std::optional Perms) { return addFile(P, ModificationTime, llvm::MemoryBuffer::getMemBuffer(Buffer), std::move(User), std::move(Group), std::move(Type), std::move(Perms), [](detail::NewInMemoryNodeInfo NNI) -> std::unique_ptr { Status Stat = NNI.makeStatus(); if (Stat.getType() == sys::fs::file_type::directory_file) return std::make_unique(Stat); return std::make_unique( Stat, std::move(NNI.Buffer)); }); } detail::NamedNodeOrError InMemoryFileSystem::lookupNode(const Twine &P, bool FollowFinalSymlink, size_t SymlinkDepth) const { SmallString<128> Path; P.toVector(Path); // Fix up relative paths. This just prepends the current working directory. std::error_code EC = makeAbsolute(Path); assert(!EC); (void)EC; if (useNormalizedPaths()) llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true); const detail::InMemoryDirectory *Dir = Root.get(); if (Path.empty()) return detail::NamedNodeOrError(Path, Dir); auto I = llvm::sys::path::begin(Path), E = llvm::sys::path::end(Path); while (true) { detail::InMemoryNode *Node = Dir->getChild(*I); ++I; if (!Node) return errc::no_such_file_or_directory; if (auto Symlink = dyn_cast(Node)) { // If we're at the end of the path, and we're not following through // terminal symlinks, then we're done. if (I == E && !FollowFinalSymlink) return detail::NamedNodeOrError(Path, Symlink); if (SymlinkDepth > InMemoryFileSystem::MaxSymlinkDepth) return errc::no_such_file_or_directory; SmallString<128> TargetPath = Symlink->getTargetPath(); if (std::error_code EC = makeAbsolute(TargetPath)) return EC; // Keep going with the target. We always want to follow symlinks here // because we're either at the end of a path that we want to follow, or // not at the end of a path, in which case we need to follow the symlink // regardless. auto Target = lookupNode(TargetPath, /*FollowFinalSymlink=*/true, SymlinkDepth + 1); if (!Target || I == E) return Target; if (!isa(*Target)) return errc::no_such_file_or_directory; // Otherwise, continue on the search in the symlinked directory. Dir = cast(*Target); continue; } // Return the file if it's at the end of the path. if (auto File = dyn_cast(Node)) { if (I == E) return detail::NamedNodeOrError(Path, File); return errc::no_such_file_or_directory; } // If Node is HardLink then return the resolved file. if (auto File = dyn_cast(Node)) { if (I == E) return detail::NamedNodeOrError(Path, &File->getResolvedFile()); return errc::no_such_file_or_directory; } // Traverse directories. Dir = cast(Node); if (I == E) return detail::NamedNodeOrError(Path, Dir); } } bool InMemoryFileSystem::addHardLink(const Twine &NewLink, const Twine &Target) { auto NewLinkNode = lookupNode(NewLink, /*FollowFinalSymlink=*/false); // Whether symlinks in the hardlink target are followed is // implementation-defined in POSIX. // We're following symlinks here to be consistent with macOS. auto TargetNode = lookupNode(Target, /*FollowFinalSymlink=*/true); // FromPath must not have been added before. ToPath must have been added // before. Resolved ToPath must be a File. if (!TargetNode || NewLinkNode || !isa(*TargetNode)) return false; return addFile(NewLink, 0, nullptr, std::nullopt, std::nullopt, std::nullopt, std::nullopt, [&](detail::NewInMemoryNodeInfo NNI) { return std::make_unique( NNI.Path.str(), *cast(*TargetNode)); }); } bool InMemoryFileSystem::addSymbolicLink( const Twine &NewLink, const Twine &Target, time_t ModificationTime, std::optional User, std::optional Group, std::optional Perms) { auto NewLinkNode = lookupNode(NewLink, /*FollowFinalSymlink=*/false); if (NewLinkNode) return false; SmallString<128> NewLinkStr, TargetStr; NewLink.toVector(NewLinkStr); Target.toVector(TargetStr); return addFile(NewLinkStr, ModificationTime, nullptr, User, Group, sys::fs::file_type::symlink_file, Perms, [&](detail::NewInMemoryNodeInfo NNI) { return std::make_unique( NewLinkStr, TargetStr, NNI.makeStatus()); }); } llvm::ErrorOr InMemoryFileSystem::status(const Twine &Path) { auto Node = lookupNode(Path, /*FollowFinalSymlink=*/true); if (Node) return (*Node)->getStatus(Path); return Node.getError(); } llvm::ErrorOr> InMemoryFileSystem::openFileForRead(const Twine &Path) { auto Node = lookupNode(Path,/*FollowFinalSymlink=*/true); if (!Node) return Node.getError(); // When we have a file provide a heap-allocated wrapper for the memory buffer // to match the ownership semantics for File. if (auto *F = dyn_cast(*Node)) return std::unique_ptr( new detail::InMemoryFileAdaptor(*F, Path.str())); // FIXME: errc::not_a_file? return make_error_code(llvm::errc::invalid_argument); } /// Adaptor from InMemoryDir::iterator to directory_iterator. class InMemoryFileSystem::DirIterator : public llvm::vfs::detail::DirIterImpl { const InMemoryFileSystem *FS; detail::InMemoryDirectory::const_iterator I; detail::InMemoryDirectory::const_iterator E; std::string RequestedDirName; void setCurrentEntry() { if (I != E) { SmallString<256> Path(RequestedDirName); llvm::sys::path::append(Path, I->second->getFileName()); sys::fs::file_type Type = sys::fs::file_type::type_unknown; switch (I->second->getKind()) { case detail::IME_File: case detail::IME_HardLink: Type = sys::fs::file_type::regular_file; break; case detail::IME_Directory: Type = sys::fs::file_type::directory_file; break; case detail::IME_SymbolicLink: if (auto SymlinkTarget = FS->lookupNode(Path, /*FollowFinalSymlink=*/true)) { Path = SymlinkTarget.getName(); Type = (*SymlinkTarget)->getStatus(Path).getType(); } break; } CurrentEntry = directory_entry(std::string(Path), Type); } else { // When we're at the end, make CurrentEntry invalid and DirIterImpl will // do the rest. CurrentEntry = directory_entry(); } } public: DirIterator() = default; DirIterator(const InMemoryFileSystem *FS, const detail::InMemoryDirectory &Dir, std::string RequestedDirName) : FS(FS), I(Dir.begin()), E(Dir.end()), RequestedDirName(std::move(RequestedDirName)) { setCurrentEntry(); } std::error_code increment() override { ++I; setCurrentEntry(); return {}; } }; directory_iterator InMemoryFileSystem::dir_begin(const Twine &Dir, std::error_code &EC) { auto Node = lookupNode(Dir, /*FollowFinalSymlink=*/true); if (!Node) { EC = Node.getError(); return directory_iterator(std::make_shared()); } if (auto *DirNode = dyn_cast(*Node)) return directory_iterator( std::make_shared(this, *DirNode, Dir.str())); EC = make_error_code(llvm::errc::not_a_directory); return directory_iterator(std::make_shared()); } std::error_code InMemoryFileSystem::setCurrentWorkingDirectory(const Twine &P) { SmallString<128> Path; P.toVector(Path); // Fix up relative paths. This just prepends the current working directory. std::error_code EC = makeAbsolute(Path); assert(!EC); (void)EC; if (useNormalizedPaths()) llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true); if (!Path.empty()) WorkingDirectory = std::string(Path); return {}; } std::error_code InMemoryFileSystem::getRealPath(const Twine &Path, SmallVectorImpl &Output) { auto CWD = getCurrentWorkingDirectory(); if (!CWD || CWD->empty()) return errc::operation_not_permitted; Path.toVector(Output); if (auto EC = makeAbsolute(Output)) return EC; llvm::sys::path::remove_dots(Output, /*remove_dot_dot=*/true); return {}; } std::error_code InMemoryFileSystem::isLocal(const Twine &Path, bool &Result) { Result = false; return {}; } void InMemoryFileSystem::printImpl(raw_ostream &OS, PrintType PrintContents, unsigned IndentLevel) const { printIndent(OS, IndentLevel); OS << "InMemoryFileSystem\n"; } } // namespace vfs } // namespace llvm //===-----------------------------------------------------------------------===/ // RedirectingFileSystem implementation //===-----------------------------------------------------------------------===/ namespace { static llvm::sys::path::Style getExistingStyle(llvm::StringRef Path) { // Detect the path style in use by checking the first separator. llvm::sys::path::Style style = llvm::sys::path::Style::native; const size_t n = Path.find_first_of("/\\"); // Can't distinguish between posix and windows_slash here. if (n != static_cast(-1)) style = (Path[n] == '/') ? llvm::sys::path::Style::posix : llvm::sys::path::Style::windows_backslash; return style; } /// Removes leading "./" as well as path components like ".." and ".". static llvm::SmallString<256> canonicalize(llvm::StringRef Path) { // First detect the path style in use by checking the first separator. llvm::sys::path::Style style = getExistingStyle(Path); // Now remove the dots. Explicitly specifying the path style prevents the // direction of the slashes from changing. llvm::SmallString<256> result = llvm::sys::path::remove_leading_dotslash(Path, style); llvm::sys::path::remove_dots(result, /*remove_dot_dot=*/true, style); return result; } /// Whether the error and entry specify a file/directory that was not found. static bool isFileNotFound(std::error_code EC, RedirectingFileSystem::Entry *E = nullptr) { if (E && !isa(E)) return false; return EC == llvm::errc::no_such_file_or_directory; } } // anonymous namespace RedirectingFileSystem::RedirectingFileSystem(IntrusiveRefCntPtr FS) : ExternalFS(std::move(FS)) { if (ExternalFS) if (auto ExternalWorkingDirectory = ExternalFS->getCurrentWorkingDirectory()) { WorkingDirectory = *ExternalWorkingDirectory; } } /// Directory iterator implementation for \c RedirectingFileSystem's /// directory entries. class llvm::vfs::RedirectingFSDirIterImpl : public llvm::vfs::detail::DirIterImpl { std::string Dir; RedirectingFileSystem::DirectoryEntry::iterator Current, End; std::error_code incrementImpl(bool IsFirstTime) { assert((IsFirstTime || Current != End) && "cannot iterate past end"); if (!IsFirstTime) ++Current; if (Current != End) { SmallString<128> PathStr(Dir); llvm::sys::path::append(PathStr, (*Current)->getName()); sys::fs::file_type Type = sys::fs::file_type::type_unknown; switch ((*Current)->getKind()) { case RedirectingFileSystem::EK_Directory: [[fallthrough]]; case RedirectingFileSystem::EK_DirectoryRemap: Type = sys::fs::file_type::directory_file; break; case RedirectingFileSystem::EK_File: Type = sys::fs::file_type::regular_file; break; } CurrentEntry = directory_entry(std::string(PathStr), Type); } else { CurrentEntry = directory_entry(); } return {}; }; public: RedirectingFSDirIterImpl( const Twine &Path, RedirectingFileSystem::DirectoryEntry::iterator Begin, RedirectingFileSystem::DirectoryEntry::iterator End, std::error_code &EC) : Dir(Path.str()), Current(Begin), End(End) { EC = incrementImpl(/*IsFirstTime=*/true); } std::error_code increment() override { return incrementImpl(/*IsFirstTime=*/false); } }; namespace { /// Directory iterator implementation for \c RedirectingFileSystem's /// directory remap entries that maps the paths reported by the external /// file system's directory iterator back to the virtual directory's path. class RedirectingFSDirRemapIterImpl : public llvm::vfs::detail::DirIterImpl { std::string Dir; llvm::sys::path::Style DirStyle; llvm::vfs::directory_iterator ExternalIter; public: RedirectingFSDirRemapIterImpl(std::string DirPath, llvm::vfs::directory_iterator ExtIter) : Dir(std::move(DirPath)), DirStyle(getExistingStyle(Dir)), ExternalIter(ExtIter) { if (ExternalIter != llvm::vfs::directory_iterator()) setCurrentEntry(); } void setCurrentEntry() { StringRef ExternalPath = ExternalIter->path(); llvm::sys::path::Style ExternalStyle = getExistingStyle(ExternalPath); StringRef File = llvm::sys::path::filename(ExternalPath, ExternalStyle); SmallString<128> NewPath(Dir); llvm::sys::path::append(NewPath, DirStyle, File); CurrentEntry = directory_entry(std::string(NewPath), ExternalIter->type()); } std::error_code increment() override { std::error_code EC; ExternalIter.increment(EC); if (!EC && ExternalIter != llvm::vfs::directory_iterator()) setCurrentEntry(); else CurrentEntry = directory_entry(); return EC; } }; } // namespace llvm::ErrorOr RedirectingFileSystem::getCurrentWorkingDirectory() const { return WorkingDirectory; } std::error_code RedirectingFileSystem::setCurrentWorkingDirectory(const Twine &Path) { // Don't change the working directory if the path doesn't exist. if (!exists(Path)) return errc::no_such_file_or_directory; SmallString<128> AbsolutePath; Path.toVector(AbsolutePath); if (std::error_code EC = makeAbsolute(AbsolutePath)) return EC; WorkingDirectory = std::string(AbsolutePath); return {}; } std::error_code RedirectingFileSystem::isLocal(const Twine &Path_, bool &Result) { SmallString<256> Path; Path_.toVector(Path); if (makeAbsolute(Path)) return {}; return ExternalFS->isLocal(Path, Result); } std::error_code RedirectingFileSystem::makeAbsolute(SmallVectorImpl &Path) const { // is_absolute(..., Style::windows_*) accepts paths with both slash types. if (llvm::sys::path::is_absolute(Path, llvm::sys::path::Style::posix) || llvm::sys::path::is_absolute(Path, llvm::sys::path::Style::windows_backslash)) // This covers windows absolute path with forward slash as well, as the // forward slashes are treated as path separation in llvm::path // regardless of what path::Style is used. return {}; auto WorkingDir = getCurrentWorkingDirectory(); if (!WorkingDir) return WorkingDir.getError(); return makeAbsolute(WorkingDir.get(), Path); } std::error_code RedirectingFileSystem::makeAbsolute(StringRef WorkingDir, SmallVectorImpl &Path) const { // We can't use sys::fs::make_absolute because that assumes the path style // is native and there is no way to override that. Since we know WorkingDir // is absolute, we can use it to determine which style we actually have and // append Path ourselves. if (!WorkingDir.empty() && !sys::path::is_absolute(WorkingDir, sys::path::Style::posix) && !sys::path::is_absolute(WorkingDir, sys::path::Style::windows_backslash)) { return std::error_code(); } sys::path::Style style = sys::path::Style::windows_backslash; if (sys::path::is_absolute(WorkingDir, sys::path::Style::posix)) { style = sys::path::Style::posix; } else { // Distinguish between windows_backslash and windows_slash; getExistingStyle // returns posix for a path with windows_slash. if (getExistingStyle(WorkingDir) != sys::path::Style::windows_backslash) style = sys::path::Style::windows_slash; } std::string Result = std::string(WorkingDir); StringRef Dir(Result); if (!Dir.ends_with(sys::path::get_separator(style))) { Result += sys::path::get_separator(style); } // backslashes '\' are legit path charactors under POSIX. Windows APIs // like CreateFile accepts forward slashes '/' as path // separator (even when mixed with backslashes). Therefore, // `Path` should be directly appended to `WorkingDir` without converting // path separator. Result.append(Path.data(), Path.size()); Path.assign(Result.begin(), Result.end()); return {}; } directory_iterator RedirectingFileSystem::dir_begin(const Twine &Dir, std::error_code &EC) { SmallString<256> Path; Dir.toVector(Path); EC = makeAbsolute(Path); if (EC) return {}; ErrorOr Result = lookupPath(Path); if (!Result) { if (Redirection != RedirectKind::RedirectOnly && isFileNotFound(Result.getError())) return ExternalFS->dir_begin(Path, EC); EC = Result.getError(); return {}; } // Use status to make sure the path exists and refers to a directory. ErrorOr S = status(Path, Dir, *Result); if (!S) { if (Redirection != RedirectKind::RedirectOnly && isFileNotFound(S.getError(), Result->E)) return ExternalFS->dir_begin(Dir, EC); EC = S.getError(); return {}; } if (!S->isDirectory()) { EC = errc::not_a_directory; return {}; } // Create the appropriate directory iterator based on whether we found a // DirectoryRemapEntry or DirectoryEntry. directory_iterator RedirectIter; std::error_code RedirectEC; if (auto ExtRedirect = Result->getExternalRedirect()) { auto RE = cast(Result->E); RedirectIter = ExternalFS->dir_begin(*ExtRedirect, RedirectEC); if (!RE->useExternalName(UseExternalNames)) { // Update the paths in the results to use the virtual directory's path. RedirectIter = directory_iterator(std::make_shared( std::string(Path), RedirectIter)); } } else { auto DE = cast(Result->E); RedirectIter = directory_iterator(std::make_shared( Path, DE->contents_begin(), DE->contents_end(), RedirectEC)); } if (RedirectEC) { if (RedirectEC != errc::no_such_file_or_directory) { EC = RedirectEC; return {}; } RedirectIter = {}; } if (Redirection == RedirectKind::RedirectOnly) { EC = RedirectEC; return RedirectIter; } std::error_code ExternalEC; directory_iterator ExternalIter = ExternalFS->dir_begin(Path, ExternalEC); if (ExternalEC) { if (ExternalEC != errc::no_such_file_or_directory) { EC = ExternalEC; return {}; } ExternalIter = {}; } SmallVector Iters; switch (Redirection) { case RedirectKind::Fallthrough: Iters.push_back(ExternalIter); Iters.push_back(RedirectIter); break; case RedirectKind::Fallback: Iters.push_back(RedirectIter); Iters.push_back(ExternalIter); break; default: llvm_unreachable("unhandled RedirectKind"); } directory_iterator Combined{ std::make_shared(Iters, EC)}; if (EC) return {}; return Combined; } void RedirectingFileSystem::setOverlayFileDir(StringRef Dir) { OverlayFileDir = Dir.str(); } StringRef RedirectingFileSystem::getOverlayFileDir() const { return OverlayFileDir; } void RedirectingFileSystem::setFallthrough(bool Fallthrough) { if (Fallthrough) { Redirection = RedirectingFileSystem::RedirectKind::Fallthrough; } else { Redirection = RedirectingFileSystem::RedirectKind::RedirectOnly; } } void RedirectingFileSystem::setRedirection( RedirectingFileSystem::RedirectKind Kind) { Redirection = Kind; } std::vector RedirectingFileSystem::getRoots() const { std::vector R; R.reserve(Roots.size()); for (const auto &Root : Roots) R.push_back(Root->getName()); return R; } void RedirectingFileSystem::printImpl(raw_ostream &OS, PrintType Type, unsigned IndentLevel) const { printIndent(OS, IndentLevel); OS << "RedirectingFileSystem (UseExternalNames: " << (UseExternalNames ? "true" : "false") << ")\n"; if (Type == PrintType::Summary) return; for (const auto &Root : Roots) printEntry(OS, Root.get(), IndentLevel); printIndent(OS, IndentLevel); OS << "ExternalFS:\n"; ExternalFS->print(OS, Type == PrintType::Contents ? PrintType::Summary : Type, IndentLevel + 1); } void RedirectingFileSystem::printEntry(raw_ostream &OS, RedirectingFileSystem::Entry *E, unsigned IndentLevel) const { printIndent(OS, IndentLevel); OS << "'" << E->getName() << "'"; switch (E->getKind()) { case EK_Directory: { auto *DE = cast(E); OS << "\n"; for (std::unique_ptr &SubEntry : llvm::make_range(DE->contents_begin(), DE->contents_end())) printEntry(OS, SubEntry.get(), IndentLevel + 1); break; } case EK_DirectoryRemap: case EK_File: { auto *RE = cast(E); OS << " -> '" << RE->getExternalContentsPath() << "'"; switch (RE->getUseName()) { case NK_NotSet: break; case NK_External: OS << " (UseExternalName: true)"; break; case NK_Virtual: OS << " (UseExternalName: false)"; break; } OS << "\n"; break; } } } void RedirectingFileSystem::visitChildFileSystems(VisitCallbackTy Callback) { if (ExternalFS) { Callback(*ExternalFS); ExternalFS->visitChildFileSystems(Callback); } } /// A helper class to hold the common YAML parsing state. class llvm::vfs::RedirectingFileSystemParser { yaml::Stream &Stream; void error(yaml::Node *N, const Twine &Msg) { Stream.printError(N, Msg); } // false on error bool parseScalarString(yaml::Node *N, StringRef &Result, SmallVectorImpl &Storage) { const auto *S = dyn_cast(N); if (!S) { error(N, "expected string"); return false; } Result = S->getValue(Storage); return true; } // false on error bool parseScalarBool(yaml::Node *N, bool &Result) { SmallString<5> Storage; StringRef Value; if (!parseScalarString(N, Value, Storage)) return false; if (Value.equals_insensitive("true") || Value.equals_insensitive("on") || Value.equals_insensitive("yes") || Value == "1") { Result = true; return true; } else if (Value.equals_insensitive("false") || Value.equals_insensitive("off") || Value.equals_insensitive("no") || Value == "0") { Result = false; return true; } error(N, "expected boolean value"); return false; } std::optional parseRedirectKind(yaml::Node *N) { SmallString<12> Storage; StringRef Value; if (!parseScalarString(N, Value, Storage)) return std::nullopt; if (Value.equals_insensitive("fallthrough")) { return RedirectingFileSystem::RedirectKind::Fallthrough; } else if (Value.equals_insensitive("fallback")) { return RedirectingFileSystem::RedirectKind::Fallback; } else if (Value.equals_insensitive("redirect-only")) { return RedirectingFileSystem::RedirectKind::RedirectOnly; } return std::nullopt; } std::optional parseRootRelativeKind(yaml::Node *N) { SmallString<12> Storage; StringRef Value; if (!parseScalarString(N, Value, Storage)) return std::nullopt; if (Value.equals_insensitive("cwd")) { return RedirectingFileSystem::RootRelativeKind::CWD; } else if (Value.equals_insensitive("overlay-dir")) { return RedirectingFileSystem::RootRelativeKind::OverlayDir; } return std::nullopt; } struct KeyStatus { bool Required; bool Seen = false; KeyStatus(bool Required = false) : Required(Required) {} }; using KeyStatusPair = std::pair; // false on error bool checkDuplicateOrUnknownKey(yaml::Node *KeyNode, StringRef Key, DenseMap &Keys) { if (!Keys.count(Key)) { error(KeyNode, "unknown key"); return false; } KeyStatus &S = Keys[Key]; if (S.Seen) { error(KeyNode, Twine("duplicate key '") + Key + "'"); return false; } S.Seen = true; return true; } // false on error bool checkMissingKeys(yaml::Node *Obj, DenseMap &Keys) { for (const auto &I : Keys) { if (I.second.Required && !I.second.Seen) { error(Obj, Twine("missing key '") + I.first + "'"); return false; } } return true; } public: static RedirectingFileSystem::Entry * lookupOrCreateEntry(RedirectingFileSystem *FS, StringRef Name, RedirectingFileSystem::Entry *ParentEntry = nullptr) { if (!ParentEntry) { // Look for a existent root for (const auto &Root : FS->Roots) { if (Name == Root->getName()) { ParentEntry = Root.get(); return ParentEntry; } } } else { // Advance to the next component auto *DE = dyn_cast(ParentEntry); for (std::unique_ptr &Content : llvm::make_range(DE->contents_begin(), DE->contents_end())) { auto *DirContent = dyn_cast(Content.get()); if (DirContent && Name == Content->getName()) return DirContent; } } // ... or create a new one std::unique_ptr E = std::make_unique( Name, Status("", getNextVirtualUniqueID(), std::chrono::system_clock::now(), 0, 0, 0, file_type::directory_file, sys::fs::all_all)); if (!ParentEntry) { // Add a new root to the overlay FS->Roots.push_back(std::move(E)); ParentEntry = FS->Roots.back().get(); return ParentEntry; } auto *DE = cast(ParentEntry); DE->addContent(std::move(E)); return DE->getLastContent(); } private: void uniqueOverlayTree(RedirectingFileSystem *FS, RedirectingFileSystem::Entry *SrcE, RedirectingFileSystem::Entry *NewParentE = nullptr) { StringRef Name = SrcE->getName(); switch (SrcE->getKind()) { case RedirectingFileSystem::EK_Directory: { auto *DE = cast(SrcE); // Empty directories could be present in the YAML as a way to // describe a file for a current directory after some of its subdir // is parsed. This only leads to redundant walks, ignore it. if (!Name.empty()) NewParentE = lookupOrCreateEntry(FS, Name, NewParentE); for (std::unique_ptr &SubEntry : llvm::make_range(DE->contents_begin(), DE->contents_end())) uniqueOverlayTree(FS, SubEntry.get(), NewParentE); break; } case RedirectingFileSystem::EK_DirectoryRemap: { assert(NewParentE && "Parent entry must exist"); auto *DR = cast(SrcE); auto *DE = cast(NewParentE); DE->addContent( std::make_unique( Name, DR->getExternalContentsPath(), DR->getUseName())); break; } case RedirectingFileSystem::EK_File: { assert(NewParentE && "Parent entry must exist"); auto *FE = cast(SrcE); auto *DE = cast(NewParentE); DE->addContent(std::make_unique( Name, FE->getExternalContentsPath(), FE->getUseName())); break; } } } std::unique_ptr parseEntry(yaml::Node *N, RedirectingFileSystem *FS, bool IsRootEntry) { auto *M = dyn_cast(N); if (!M) { error(N, "expected mapping node for file or directory entry"); return nullptr; } KeyStatusPair Fields[] = { KeyStatusPair("name", true), KeyStatusPair("type", true), KeyStatusPair("contents", false), KeyStatusPair("external-contents", false), KeyStatusPair("use-external-name", false), }; DenseMap Keys(std::begin(Fields), std::end(Fields)); enum { CF_NotSet, CF_List, CF_External } ContentsField = CF_NotSet; std::vector> EntryArrayContents; SmallString<256> ExternalContentsPath; SmallString<256> Name; yaml::Node *NameValueNode = nullptr; auto UseExternalName = RedirectingFileSystem::NK_NotSet; RedirectingFileSystem::EntryKind Kind; for (auto &I : *M) { StringRef Key; // Reuse the buffer for key and value, since we don't look at key after // parsing value. SmallString<256> Buffer; if (!parseScalarString(I.getKey(), Key, Buffer)) return nullptr; if (!checkDuplicateOrUnknownKey(I.getKey(), Key, Keys)) return nullptr; StringRef Value; if (Key == "name") { if (!parseScalarString(I.getValue(), Value, Buffer)) return nullptr; NameValueNode = I.getValue(); // Guarantee that old YAML files containing paths with ".." and "." // are properly canonicalized before read into the VFS. Name = canonicalize(Value).str(); } else if (Key == "type") { if (!parseScalarString(I.getValue(), Value, Buffer)) return nullptr; if (Value == "file") Kind = RedirectingFileSystem::EK_File; else if (Value == "directory") Kind = RedirectingFileSystem::EK_Directory; else if (Value == "directory-remap") Kind = RedirectingFileSystem::EK_DirectoryRemap; else { error(I.getValue(), "unknown value for 'type'"); return nullptr; } } else if (Key == "contents") { if (ContentsField != CF_NotSet) { error(I.getKey(), "entry already has 'contents' or 'external-contents'"); return nullptr; } ContentsField = CF_List; auto *Contents = dyn_cast(I.getValue()); if (!Contents) { // FIXME: this is only for directories, what about files? error(I.getValue(), "expected array"); return nullptr; } for (auto &I : *Contents) { if (std::unique_ptr E = parseEntry(&I, FS, /*IsRootEntry*/ false)) EntryArrayContents.push_back(std::move(E)); else return nullptr; } } else if (Key == "external-contents") { if (ContentsField != CF_NotSet) { error(I.getKey(), "entry already has 'contents' or 'external-contents'"); return nullptr; } ContentsField = CF_External; if (!parseScalarString(I.getValue(), Value, Buffer)) return nullptr; SmallString<256> FullPath; if (FS->IsRelativeOverlay) { FullPath = FS->getOverlayFileDir(); assert(!FullPath.empty() && "External contents prefix directory must exist"); llvm::sys::path::append(FullPath, Value); } else { FullPath = Value; } // Guarantee that old YAML files containing paths with ".." and "." // are properly canonicalized before read into the VFS. FullPath = canonicalize(FullPath); ExternalContentsPath = FullPath.str(); } else if (Key == "use-external-name") { bool Val; if (!parseScalarBool(I.getValue(), Val)) return nullptr; UseExternalName = Val ? RedirectingFileSystem::NK_External : RedirectingFileSystem::NK_Virtual; } else { llvm_unreachable("key missing from Keys"); } } if (Stream.failed()) return nullptr; // check for missing keys if (ContentsField == CF_NotSet) { error(N, "missing key 'contents' or 'external-contents'"); return nullptr; } if (!checkMissingKeys(N, Keys)) return nullptr; // check invalid configuration if (Kind == RedirectingFileSystem::EK_Directory && UseExternalName != RedirectingFileSystem::NK_NotSet) { error(N, "'use-external-name' is not supported for 'directory' entries"); return nullptr; } if (Kind == RedirectingFileSystem::EK_DirectoryRemap && ContentsField == CF_List) { error(N, "'contents' is not supported for 'directory-remap' entries"); return nullptr; } sys::path::Style path_style = sys::path::Style::native; if (IsRootEntry) { // VFS root entries may be in either Posix or Windows style. Figure out // which style we have, and use it consistently. if (sys::path::is_absolute(Name, sys::path::Style::posix)) { path_style = sys::path::Style::posix; } else if (sys::path::is_absolute(Name, sys::path::Style::windows_backslash)) { path_style = sys::path::Style::windows_backslash; } else { // Relative VFS root entries are made absolute to either the overlay // directory, or the current working directory, then we can determine // the path style from that. std::error_code EC; if (FS->RootRelative == RedirectingFileSystem::RootRelativeKind::OverlayDir) { StringRef FullPath = FS->getOverlayFileDir(); assert(!FullPath.empty() && "Overlay file directory must exist"); EC = FS->makeAbsolute(FullPath, Name); Name = canonicalize(Name); } else { EC = sys::fs::make_absolute(Name); } if (EC) { assert(NameValueNode && "Name presence should be checked earlier"); error( NameValueNode, "entry with relative path at the root level is not discoverable"); return nullptr; } path_style = sys::path::is_absolute(Name, sys::path::Style::posix) ? sys::path::Style::posix : sys::path::Style::windows_backslash; } // is::path::is_absolute(Name, sys::path::Style::windows_backslash) will // return true even if `Name` is using forward slashes. Distinguish // between windows_backslash and windows_slash. if (path_style == sys::path::Style::windows_backslash && getExistingStyle(Name) != sys::path::Style::windows_backslash) path_style = sys::path::Style::windows_slash; } // Remove trailing slash(es), being careful not to remove the root path StringRef Trimmed = Name; size_t RootPathLen = sys::path::root_path(Trimmed, path_style).size(); while (Trimmed.size() > RootPathLen && sys::path::is_separator(Trimmed.back(), path_style)) Trimmed = Trimmed.slice(0, Trimmed.size() - 1); // Get the last component StringRef LastComponent = sys::path::filename(Trimmed, path_style); std::unique_ptr Result; switch (Kind) { case RedirectingFileSystem::EK_File: Result = std::make_unique( LastComponent, std::move(ExternalContentsPath), UseExternalName); break; case RedirectingFileSystem::EK_DirectoryRemap: Result = std::make_unique( LastComponent, std::move(ExternalContentsPath), UseExternalName); break; case RedirectingFileSystem::EK_Directory: Result = std::make_unique( LastComponent, std::move(EntryArrayContents), Status("", getNextVirtualUniqueID(), std::chrono::system_clock::now(), 0, 0, 0, file_type::directory_file, sys::fs::all_all)); break; } StringRef Parent = sys::path::parent_path(Trimmed, path_style); if (Parent.empty()) return Result; // if 'name' contains multiple components, create implicit directory entries for (sys::path::reverse_iterator I = sys::path::rbegin(Parent, path_style), E = sys::path::rend(Parent); I != E; ++I) { std::vector> Entries; Entries.push_back(std::move(Result)); Result = std::make_unique( *I, std::move(Entries), Status("", getNextVirtualUniqueID(), std::chrono::system_clock::now(), 0, 0, 0, file_type::directory_file, sys::fs::all_all)); } return Result; } public: RedirectingFileSystemParser(yaml::Stream &S) : Stream(S) {} // false on error bool parse(yaml::Node *Root, RedirectingFileSystem *FS) { auto *Top = dyn_cast(Root); if (!Top) { error(Root, "expected mapping node"); return false; } KeyStatusPair Fields[] = { KeyStatusPair("version", true), KeyStatusPair("case-sensitive", false), KeyStatusPair("use-external-names", false), KeyStatusPair("root-relative", false), KeyStatusPair("overlay-relative", false), KeyStatusPair("fallthrough", false), KeyStatusPair("redirecting-with", false), KeyStatusPair("roots", true), }; DenseMap Keys(std::begin(Fields), std::end(Fields)); std::vector> RootEntries; // Parse configuration and 'roots' for (auto &I : *Top) { SmallString<10> KeyBuffer; StringRef Key; if (!parseScalarString(I.getKey(), Key, KeyBuffer)) return false; if (!checkDuplicateOrUnknownKey(I.getKey(), Key, Keys)) return false; if (Key == "roots") { auto *Roots = dyn_cast(I.getValue()); if (!Roots) { error(I.getValue(), "expected array"); return false; } for (auto &I : *Roots) { if (std::unique_ptr E = parseEntry(&I, FS, /*IsRootEntry*/ true)) RootEntries.push_back(std::move(E)); else return false; } } else if (Key == "version") { StringRef VersionString; SmallString<4> Storage; if (!parseScalarString(I.getValue(), VersionString, Storage)) return false; int Version; if (VersionString.getAsInteger(10, Version)) { error(I.getValue(), "expected integer"); return false; } if (Version < 0) { error(I.getValue(), "invalid version number"); return false; } if (Version != 0) { error(I.getValue(), "version mismatch, expected 0"); return false; } } else if (Key == "case-sensitive") { if (!parseScalarBool(I.getValue(), FS->CaseSensitive)) return false; } else if (Key == "overlay-relative") { if (!parseScalarBool(I.getValue(), FS->IsRelativeOverlay)) return false; } else if (Key == "use-external-names") { if (!parseScalarBool(I.getValue(), FS->UseExternalNames)) return false; } else if (Key == "fallthrough") { if (Keys["redirecting-with"].Seen) { error(I.getValue(), "'fallthrough' and 'redirecting-with' are mutually exclusive"); return false; } bool ShouldFallthrough = false; if (!parseScalarBool(I.getValue(), ShouldFallthrough)) return false; if (ShouldFallthrough) { FS->Redirection = RedirectingFileSystem::RedirectKind::Fallthrough; } else { FS->Redirection = RedirectingFileSystem::RedirectKind::RedirectOnly; } } else if (Key == "redirecting-with") { if (Keys["fallthrough"].Seen) { error(I.getValue(), "'fallthrough' and 'redirecting-with' are mutually exclusive"); return false; } if (auto Kind = parseRedirectKind(I.getValue())) { FS->Redirection = *Kind; } else { error(I.getValue(), "expected valid redirect kind"); return false; } } else if (Key == "root-relative") { if (auto Kind = parseRootRelativeKind(I.getValue())) { FS->RootRelative = *Kind; } else { error(I.getValue(), "expected valid root-relative kind"); return false; } } else { llvm_unreachable("key missing from Keys"); } } if (Stream.failed()) return false; if (!checkMissingKeys(Top, Keys)) return false; // Now that we sucessefully parsed the YAML file, canonicalize the internal // representation to a proper directory tree so that we can search faster // inside the VFS. for (auto &E : RootEntries) uniqueOverlayTree(FS, E.get()); return true; } }; std::unique_ptr RedirectingFileSystem::create(std::unique_ptr Buffer, SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath, void *DiagContext, IntrusiveRefCntPtr ExternalFS) { SourceMgr SM; yaml::Stream Stream(Buffer->getMemBufferRef(), SM); SM.setDiagHandler(DiagHandler, DiagContext); yaml::document_iterator DI = Stream.begin(); yaml::Node *Root = DI->getRoot(); if (DI == Stream.end() || !Root) { SM.PrintMessage(SMLoc(), SourceMgr::DK_Error, "expected root node"); return nullptr; } RedirectingFileSystemParser P(Stream); std::unique_ptr FS( new RedirectingFileSystem(ExternalFS)); if (!YAMLFilePath.empty()) { // Use the YAML path from -ivfsoverlay to compute the dir to be prefixed // to each 'external-contents' path. // // Example: // -ivfsoverlay dummy.cache/vfs/vfs.yaml // yields: // FS->OverlayFileDir => //dummy.cache/vfs // SmallString<256> OverlayAbsDir = sys::path::parent_path(YAMLFilePath); std::error_code EC = llvm::sys::fs::make_absolute(OverlayAbsDir); assert(!EC && "Overlay dir final path must be absolute"); (void)EC; FS->setOverlayFileDir(OverlayAbsDir); } if (!P.parse(Root, FS.get())) return nullptr; return FS; } std::unique_ptr RedirectingFileSystem::create( ArrayRef> RemappedFiles, bool UseExternalNames, FileSystem &ExternalFS) { std::unique_ptr FS( new RedirectingFileSystem(&ExternalFS)); FS->UseExternalNames = UseExternalNames; StringMap Entries; for (auto &Mapping : llvm::reverse(RemappedFiles)) { SmallString<128> From = StringRef(Mapping.first); SmallString<128> To = StringRef(Mapping.second); { auto EC = ExternalFS.makeAbsolute(From); (void)EC; assert(!EC && "Could not make absolute path"); } // Check if we've already mapped this file. The first one we see (in the // reverse iteration) wins. RedirectingFileSystem::Entry *&ToEntry = Entries[From]; if (ToEntry) continue; // Add parent directories. RedirectingFileSystem::Entry *Parent = nullptr; StringRef FromDirectory = llvm::sys::path::parent_path(From); for (auto I = llvm::sys::path::begin(FromDirectory), E = llvm::sys::path::end(FromDirectory); I != E; ++I) { Parent = RedirectingFileSystemParser::lookupOrCreateEntry(FS.get(), *I, Parent); } assert(Parent && "File without a directory?"); { auto EC = ExternalFS.makeAbsolute(To); (void)EC; assert(!EC && "Could not make absolute path"); } // Add the file. auto NewFile = std::make_unique( llvm::sys::path::filename(From), To, UseExternalNames ? RedirectingFileSystem::NK_External : RedirectingFileSystem::NK_Virtual); ToEntry = NewFile.get(); cast(Parent)->addContent( std::move(NewFile)); } return FS; } RedirectingFileSystem::LookupResult::LookupResult( Entry *E, sys::path::const_iterator Start, sys::path::const_iterator End) : E(E) { assert(E != nullptr); // If the matched entry is a DirectoryRemapEntry, set ExternalRedirect to the // path of the directory it maps to in the external file system plus any // remaining path components in the provided iterator. if (auto *DRE = dyn_cast(E)) { SmallString<256> Redirect(DRE->getExternalContentsPath()); sys::path::append(Redirect, Start, End, getExistingStyle(DRE->getExternalContentsPath())); ExternalRedirect = std::string(Redirect); } } void RedirectingFileSystem::LookupResult::getPath( llvm::SmallVectorImpl &Result) const { Result.clear(); for (Entry *Parent : Parents) llvm::sys::path::append(Result, Parent->getName()); llvm::sys::path::append(Result, E->getName()); } std::error_code RedirectingFileSystem::makeCanonicalForLookup( SmallVectorImpl &Path) const { if (std::error_code EC = makeAbsolute(Path)) return EC; llvm::SmallString<256> CanonicalPath = canonicalize(StringRef(Path.data(), Path.size())); if (CanonicalPath.empty()) return make_error_code(llvm::errc::invalid_argument); Path.assign(CanonicalPath.begin(), CanonicalPath.end()); return {}; } ErrorOr RedirectingFileSystem::lookupPath(StringRef Path) const { llvm::SmallString<128> CanonicalPath(Path); if (std::error_code EC = makeCanonicalForLookup(CanonicalPath)) return EC; // RedirectOnly means the VFS is always used. if (UsageTrackingActive && Redirection == RedirectKind::RedirectOnly) HasBeenUsed = true; sys::path::const_iterator Start = sys::path::begin(CanonicalPath); sys::path::const_iterator End = sys::path::end(CanonicalPath); llvm::SmallVector Entries; for (const auto &Root : Roots) { ErrorOr Result = lookupPathImpl(Start, End, Root.get(), Entries); if (UsageTrackingActive && Result && isa(Result->E)) HasBeenUsed = true; if (Result || Result.getError() != llvm::errc::no_such_file_or_directory) { Result->Parents = std::move(Entries); return Result; } } return make_error_code(llvm::errc::no_such_file_or_directory); } ErrorOr RedirectingFileSystem::lookupPathImpl( sys::path::const_iterator Start, sys::path::const_iterator End, RedirectingFileSystem::Entry *From, llvm::SmallVectorImpl &Entries) const { assert(!isTraversalComponent(*Start) && !isTraversalComponent(From->getName()) && "Paths should not contain traversal components"); StringRef FromName = From->getName(); // Forward the search to the next component in case this is an empty one. if (!FromName.empty()) { if (!pathComponentMatches(*Start, FromName)) return make_error_code(llvm::errc::no_such_file_or_directory); ++Start; if (Start == End) { // Match! return LookupResult(From, Start, End); } } if (isa(From)) return make_error_code(llvm::errc::not_a_directory); if (isa(From)) return LookupResult(From, Start, End); auto *DE = cast(From); for (const std::unique_ptr &DirEntry : llvm::make_range(DE->contents_begin(), DE->contents_end())) { Entries.push_back(From); ErrorOr Result = lookupPathImpl(Start, End, DirEntry.get(), Entries); if (Result || Result.getError() != llvm::errc::no_such_file_or_directory) return Result; Entries.pop_back(); } return make_error_code(llvm::errc::no_such_file_or_directory); } static Status getRedirectedFileStatus(const Twine &OriginalPath, bool UseExternalNames, Status ExternalStatus) { // The path has been mapped by some nested VFS and exposes an external path, // don't override it with the original path. if (ExternalStatus.ExposesExternalVFSPath) return ExternalStatus; Status S = ExternalStatus; if (!UseExternalNames) S = Status::copyWithNewName(S, OriginalPath); else S.ExposesExternalVFSPath = true; return S; } ErrorOr RedirectingFileSystem::status( const Twine &LookupPath, const Twine &OriginalPath, const RedirectingFileSystem::LookupResult &Result) { if (std::optional ExtRedirect = Result.getExternalRedirect()) { SmallString<256> RemappedPath((*ExtRedirect).str()); if (std::error_code EC = makeAbsolute(RemappedPath)) return EC; ErrorOr S = ExternalFS->status(RemappedPath); if (!S) return S; S = Status::copyWithNewName(*S, *ExtRedirect); auto *RE = cast(Result.E); return getRedirectedFileStatus(OriginalPath, RE->useExternalName(UseExternalNames), *S); } auto *DE = cast(Result.E); return Status::copyWithNewName(DE->getStatus(), LookupPath); } ErrorOr RedirectingFileSystem::getExternalStatus(const Twine &LookupPath, const Twine &OriginalPath) const { auto Result = ExternalFS->status(LookupPath); // The path has been mapped by some nested VFS, don't override it with the // original path. if (!Result || Result->ExposesExternalVFSPath) return Result; return Status::copyWithNewName(Result.get(), OriginalPath); } ErrorOr RedirectingFileSystem::status(const Twine &OriginalPath) { SmallString<256> Path; OriginalPath.toVector(Path); if (std::error_code EC = makeAbsolute(Path)) return EC; if (Redirection == RedirectKind::Fallback) { // Attempt to find the original file first, only falling back to the // mapped file if that fails. ErrorOr S = getExternalStatus(Path, OriginalPath); if (S) return S; } ErrorOr Result = lookupPath(Path); if (!Result) { // Was not able to map file, fallthrough to using the original path if // that was the specified redirection type. if (Redirection == RedirectKind::Fallthrough && isFileNotFound(Result.getError())) return getExternalStatus(Path, OriginalPath); return Result.getError(); } ErrorOr S = status(Path, OriginalPath, *Result); if (!S && Redirection == RedirectKind::Fallthrough && isFileNotFound(S.getError(), Result->E)) { // Mapped the file but it wasn't found in the underlying filesystem, // fallthrough to using the original path if that was the specified // redirection type. return getExternalStatus(Path, OriginalPath); } return S; } bool RedirectingFileSystem::exists(const Twine &OriginalPath) { SmallString<256> Path; OriginalPath.toVector(Path); if (makeAbsolute(Path)) return false; if (Redirection == RedirectKind::Fallback) { // Attempt to find the original file first, only falling back to the // mapped file if that fails. if (ExternalFS->exists(Path)) return true; } ErrorOr Result = lookupPath(Path); if (!Result) { // Was not able to map file, fallthrough to using the original path if // that was the specified redirection type. if (Redirection == RedirectKind::Fallthrough && isFileNotFound(Result.getError())) return ExternalFS->exists(Path); return false; } std::optional ExtRedirect = Result->getExternalRedirect(); if (!ExtRedirect) { assert(isa(Result->E)); return true; } SmallString<256> RemappedPath((*ExtRedirect).str()); if (makeAbsolute(RemappedPath)) return false; if (ExternalFS->exists(RemappedPath)) return true; if (Redirection == RedirectKind::Fallthrough) { // Mapped the file but it wasn't found in the underlying filesystem, // fallthrough to using the original path if that was the specified // redirection type. return ExternalFS->exists(Path); } return false; } namespace { /// Provide a file wrapper with an overriden status. class FileWithFixedStatus : public File { std::unique_ptr InnerFile; Status S; public: FileWithFixedStatus(std::unique_ptr InnerFile, Status S) : InnerFile(std::move(InnerFile)), S(std::move(S)) {} ErrorOr status() override { return S; } ErrorOr> getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, bool IsVolatile) override { return InnerFile->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile); } std::error_code close() override { return InnerFile->close(); } void setPath(const Twine &Path) override { S = S.copyWithNewName(S, Path); } }; } // namespace ErrorOr> File::getWithPath(ErrorOr> Result, const Twine &P) { // See \c getRedirectedFileStatus - don't update path if it's exposing an // external path. if (!Result || (*Result)->status()->ExposesExternalVFSPath) return Result; ErrorOr> F = std::move(*Result); auto Name = F->get()->getName(); if (Name && Name.get() != P.str()) F->get()->setPath(P); return F; } ErrorOr> RedirectingFileSystem::openFileForRead(const Twine &OriginalPath) { SmallString<256> Path; OriginalPath.toVector(Path); if (std::error_code EC = makeAbsolute(Path)) return EC; if (Redirection == RedirectKind::Fallback) { // Attempt to find the original file first, only falling back to the // mapped file if that fails. auto F = File::getWithPath(ExternalFS->openFileForRead(Path), OriginalPath); if (F) return F; } ErrorOr Result = lookupPath(Path); if (!Result) { // Was not able to map file, fallthrough to using the original path if // that was the specified redirection type. if (Redirection == RedirectKind::Fallthrough && isFileNotFound(Result.getError())) return File::getWithPath(ExternalFS->openFileForRead(Path), OriginalPath); return Result.getError(); } if (!Result->getExternalRedirect()) // FIXME: errc::not_a_file? return make_error_code(llvm::errc::invalid_argument); StringRef ExtRedirect = *Result->getExternalRedirect(); SmallString<256> RemappedPath(ExtRedirect.str()); if (std::error_code EC = makeAbsolute(RemappedPath)) return EC; auto *RE = cast(Result->E); auto ExternalFile = File::getWithPath(ExternalFS->openFileForRead(RemappedPath), ExtRedirect); if (!ExternalFile) { if (Redirection == RedirectKind::Fallthrough && isFileNotFound(ExternalFile.getError(), Result->E)) { // Mapped the file but it wasn't found in the underlying filesystem, // fallthrough to using the original path if that was the specified // redirection type. return File::getWithPath(ExternalFS->openFileForRead(Path), OriginalPath); } return ExternalFile; } auto ExternalStatus = (*ExternalFile)->status(); if (!ExternalStatus) return ExternalStatus.getError(); // Otherwise, the file was successfully remapped. Mark it as such. Also // replace the underlying path if the external name is being used. Status S = getRedirectedFileStatus( OriginalPath, RE->useExternalName(UseExternalNames), *ExternalStatus); return std::unique_ptr( std::make_unique(std::move(*ExternalFile), S)); } std::error_code RedirectingFileSystem::getRealPath(const Twine &OriginalPath, SmallVectorImpl &Output) { SmallString<256> Path; OriginalPath.toVector(Path); if (std::error_code EC = makeAbsolute(Path)) return EC; if (Redirection == RedirectKind::Fallback) { // Attempt to find the original file first, only falling back to the // mapped file if that fails. std::error_code EC = ExternalFS->getRealPath(Path, Output); if (!EC) return EC; } ErrorOr Result = lookupPath(Path); if (!Result) { // Was not able to map file, fallthrough to using the original path if // that was the specified redirection type. if (Redirection == RedirectKind::Fallthrough && isFileNotFound(Result.getError())) return ExternalFS->getRealPath(Path, Output); return Result.getError(); } // If we found FileEntry or DirectoryRemapEntry, look up the mapped // path in the external file system. if (auto ExtRedirect = Result->getExternalRedirect()) { auto P = ExternalFS->getRealPath(*ExtRedirect, Output); if (P && Redirection == RedirectKind::Fallthrough && isFileNotFound(P, Result->E)) { // Mapped the file but it wasn't found in the underlying filesystem, // fallthrough to using the original path if that was the specified // redirection type. return ExternalFS->getRealPath(Path, Output); } return P; } // We found a DirectoryEntry, which does not have a single external contents // path. Use the canonical virtual path. if (Redirection == RedirectKind::Fallthrough) { Result->getPath(Output); return {}; } return llvm::errc::invalid_argument; } std::unique_ptr vfs::getVFSFromYAML(std::unique_ptr Buffer, SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath, void *DiagContext, IntrusiveRefCntPtr ExternalFS) { return RedirectingFileSystem::create(std::move(Buffer), DiagHandler, YAMLFilePath, DiagContext, std::move(ExternalFS)); } static void getVFSEntries(RedirectingFileSystem::Entry *SrcE, SmallVectorImpl &Path, SmallVectorImpl &Entries) { auto Kind = SrcE->getKind(); if (Kind == RedirectingFileSystem::EK_Directory) { auto *DE = dyn_cast(SrcE); assert(DE && "Must be a directory"); for (std::unique_ptr &SubEntry : llvm::make_range(DE->contents_begin(), DE->contents_end())) { Path.push_back(SubEntry->getName()); getVFSEntries(SubEntry.get(), Path, Entries); Path.pop_back(); } return; } if (Kind == RedirectingFileSystem::EK_DirectoryRemap) { auto *DR = dyn_cast(SrcE); assert(DR && "Must be a directory remap"); SmallString<128> VPath; for (auto &Comp : Path) llvm::sys::path::append(VPath, Comp); Entries.push_back( YAMLVFSEntry(VPath.c_str(), DR->getExternalContentsPath())); return; } assert(Kind == RedirectingFileSystem::EK_File && "Must be a EK_File"); auto *FE = dyn_cast(SrcE); assert(FE && "Must be a file"); SmallString<128> VPath; for (auto &Comp : Path) llvm::sys::path::append(VPath, Comp); Entries.push_back(YAMLVFSEntry(VPath.c_str(), FE->getExternalContentsPath())); } void vfs::collectVFSFromYAML(std::unique_ptr Buffer, SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath, SmallVectorImpl &CollectedEntries, void *DiagContext, IntrusiveRefCntPtr ExternalFS) { std::unique_ptr VFS = RedirectingFileSystem::create( std::move(Buffer), DiagHandler, YAMLFilePath, DiagContext, std::move(ExternalFS)); if (!VFS) return; ErrorOr RootResult = VFS->lookupPath("/"); if (!RootResult) return; SmallVector Components; Components.push_back("/"); getVFSEntries(RootResult->E, Components, CollectedEntries); } UniqueID vfs::getNextVirtualUniqueID() { static std::atomic UID; unsigned ID = ++UID; // The following assumes that uint64_t max will never collide with a real // dev_t value from the OS. return UniqueID(std::numeric_limits::max(), ID); } void YAMLVFSWriter::addEntry(StringRef VirtualPath, StringRef RealPath, bool IsDirectory) { assert(sys::path::is_absolute(VirtualPath) && "virtual path not absolute"); assert(sys::path::is_absolute(RealPath) && "real path not absolute"); assert(!pathHasTraversal(VirtualPath) && "path traversal is not supported"); Mappings.emplace_back(VirtualPath, RealPath, IsDirectory); } void YAMLVFSWriter::addFileMapping(StringRef VirtualPath, StringRef RealPath) { addEntry(VirtualPath, RealPath, /*IsDirectory=*/false); } void YAMLVFSWriter::addDirectoryMapping(StringRef VirtualPath, StringRef RealPath) { addEntry(VirtualPath, RealPath, /*IsDirectory=*/true); } namespace { class JSONWriter { llvm::raw_ostream &OS; SmallVector DirStack; unsigned getDirIndent() { return 4 * DirStack.size(); } unsigned getFileIndent() { return 4 * (DirStack.size() + 1); } bool containedIn(StringRef Parent, StringRef Path); StringRef containedPart(StringRef Parent, StringRef Path); void startDirectory(StringRef Path); void endDirectory(); void writeEntry(StringRef VPath, StringRef RPath); public: JSONWriter(llvm::raw_ostream &OS) : OS(OS) {} void write(ArrayRef Entries, std::optional UseExternalNames, std::optional IsCaseSensitive, std::optional IsOverlayRelative, StringRef OverlayDir); }; } // namespace bool JSONWriter::containedIn(StringRef Parent, StringRef Path) { using namespace llvm::sys; // Compare each path component. auto IParent = path::begin(Parent), EParent = path::end(Parent); for (auto IChild = path::begin(Path), EChild = path::end(Path); IParent != EParent && IChild != EChild; ++IParent, ++IChild) { if (*IParent != *IChild) return false; } // Have we exhausted the parent path? return IParent == EParent; } StringRef JSONWriter::containedPart(StringRef Parent, StringRef Path) { assert(!Parent.empty()); assert(containedIn(Parent, Path)); return Path.slice(Parent.size() + 1, StringRef::npos); } void JSONWriter::startDirectory(StringRef Path) { StringRef Name = DirStack.empty() ? Path : containedPart(DirStack.back(), Path); DirStack.push_back(Path); unsigned Indent = getDirIndent(); OS.indent(Indent) << "{\n"; OS.indent(Indent + 2) << "'type': 'directory',\n"; OS.indent(Indent + 2) << "'name': \"" << llvm::yaml::escape(Name) << "\",\n"; OS.indent(Indent + 2) << "'contents': [\n"; } void JSONWriter::endDirectory() { unsigned Indent = getDirIndent(); OS.indent(Indent + 2) << "]\n"; OS.indent(Indent) << "}"; DirStack.pop_back(); } void JSONWriter::writeEntry(StringRef VPath, StringRef RPath) { unsigned Indent = getFileIndent(); OS.indent(Indent) << "{\n"; OS.indent(Indent + 2) << "'type': 'file',\n"; OS.indent(Indent + 2) << "'name': \"" << llvm::yaml::escape(VPath) << "\",\n"; OS.indent(Indent + 2) << "'external-contents': \"" << llvm::yaml::escape(RPath) << "\"\n"; OS.indent(Indent) << "}"; } void JSONWriter::write(ArrayRef Entries, std::optional UseExternalNames, std::optional IsCaseSensitive, std::optional IsOverlayRelative, StringRef OverlayDir) { using namespace llvm::sys; OS << "{\n" " 'version': 0,\n"; if (IsCaseSensitive) OS << " 'case-sensitive': '" << (*IsCaseSensitive ? "true" : "false") << "',\n"; if (UseExternalNames) OS << " 'use-external-names': '" << (*UseExternalNames ? "true" : "false") << "',\n"; bool UseOverlayRelative = false; if (IsOverlayRelative) { UseOverlayRelative = *IsOverlayRelative; OS << " 'overlay-relative': '" << (UseOverlayRelative ? "true" : "false") << "',\n"; } OS << " 'roots': [\n"; if (!Entries.empty()) { const YAMLVFSEntry &Entry = Entries.front(); startDirectory( Entry.IsDirectory ? Entry.VPath : path::parent_path(Entry.VPath) ); StringRef RPath = Entry.RPath; if (UseOverlayRelative) { assert(RPath.starts_with(OverlayDir) && "Overlay dir must be contained in RPath"); RPath = RPath.slice(OverlayDir.size(), RPath.size()); } bool IsCurrentDirEmpty = true; if (!Entry.IsDirectory) { writeEntry(path::filename(Entry.VPath), RPath); IsCurrentDirEmpty = false; } for (const auto &Entry : Entries.slice(1)) { StringRef Dir = Entry.IsDirectory ? Entry.VPath : path::parent_path(Entry.VPath); if (Dir == DirStack.back()) { if (!IsCurrentDirEmpty) { OS << ",\n"; } } else { bool IsDirPoppedFromStack = false; while (!DirStack.empty() && !containedIn(DirStack.back(), Dir)) { OS << "\n"; endDirectory(); IsDirPoppedFromStack = true; } if (IsDirPoppedFromStack || !IsCurrentDirEmpty) { OS << ",\n"; } startDirectory(Dir); IsCurrentDirEmpty = true; } StringRef RPath = Entry.RPath; if (UseOverlayRelative) { assert(RPath.starts_with(OverlayDir) && "Overlay dir must be contained in RPath"); RPath = RPath.slice(OverlayDir.size(), RPath.size()); } if (!Entry.IsDirectory) { writeEntry(path::filename(Entry.VPath), RPath); IsCurrentDirEmpty = false; } } while (!DirStack.empty()) { OS << "\n"; endDirectory(); } OS << "\n"; } OS << " ]\n" << "}\n"; } void YAMLVFSWriter::write(llvm::raw_ostream &OS) { llvm::sort(Mappings, [](const YAMLVFSEntry &LHS, const YAMLVFSEntry &RHS) { return LHS.VPath < RHS.VPath; }); JSONWriter(OS).write(Mappings, UseExternalNames, IsCaseSensitive, IsOverlayRelative, OverlayDir); } vfs::recursive_directory_iterator::recursive_directory_iterator( FileSystem &FS_, const Twine &Path, std::error_code &EC) : FS(&FS_) { directory_iterator I = FS->dir_begin(Path, EC); if (I != directory_iterator()) { State = std::make_shared(); State->Stack.push_back(I); } } vfs::recursive_directory_iterator & recursive_directory_iterator::increment(std::error_code &EC) { assert(FS && State && !State->Stack.empty() && "incrementing past end"); assert(!State->Stack.back()->path().empty() && "non-canonical end iterator"); vfs::directory_iterator End; if (State->HasNoPushRequest) State->HasNoPushRequest = false; else { if (State->Stack.back()->type() == sys::fs::file_type::directory_file) { vfs::directory_iterator I = FS->dir_begin(State->Stack.back()->path(), EC); if (I != End) { State->Stack.push_back(I); return *this; } } } while (!State->Stack.empty() && State->Stack.back().increment(EC) == End) State->Stack.pop_back(); if (State->Stack.empty()) State.reset(); // end iterator return *this; } const char FileSystem::ID = 0; const char OverlayFileSystem::ID = 0; const char ProxyFileSystem::ID = 0; const char InMemoryFileSystem::ID = 0; const char RedirectingFileSystem::ID = 0;