From a132af5b78914e4dadcd66a9e8837de10b3a3c9f Mon Sep 17 00:00:00 2001 From: Andrew Woloszyn Date: Mon, 7 Mar 2016 13:23:09 -0500 Subject: [PATCH] Updated the includer interface to allow relative includes. This plumbs both the current file path and the include depth back up to the includer. This allows the includer to properly support relative paths. This also replaces the string copy that was done during include with a zero-copy method of accomplishing the same thing. This prevents extra copies of entire files. --- StandAlone/StandAlone.cpp | 3 +- glslang/MachineIndependent/Scan.h | 53 +++++++++-- glslang/MachineIndependent/ShaderLang.cpp | 19 ++-- .../MachineIndependent/preprocessor/Pp.cpp | 26 +++--- .../preprocessor/PpContext.cpp | 6 +- .../preprocessor/PpContext.h | 87 +++++++++++++++---- glslang/Public/ShaderLang.h | 80 +++++++++++++++-- 7 files changed, 220 insertions(+), 54 deletions(-) diff --git a/StandAlone/StandAlone.cpp b/StandAlone/StandAlone.cpp index cc59d389d..6ebdb7f5c 100644 --- a/StandAlone/StandAlone.cpp +++ b/StandAlone/StandAlone.cpp @@ -699,8 +699,9 @@ void CompileAndLinkShaderUnits(std::vector compUnits) if (Options & EOptionOutputPreprocessed) { std::string str; + glslang::TShader::ForbidInclude includer; if (shader->preprocess(&Resources, defaultVersion, ENoProfile, false, false, - messages, &str, glslang::TShader::ForbidInclude())) { + messages, &str, includer)) { PutsIfNonEmpty(str.c_str()); } else { CompileFailed = true; diff --git a/glslang/MachineIndependent/Scan.h b/glslang/MachineIndependent/Scan.h index 936b99f11..f3102fbf9 100644 --- a/glslang/MachineIndependent/Scan.h +++ b/glslang/MachineIndependent/Scan.h @@ -51,10 +51,10 @@ const int EndOfInput = -1; // class TInputScanner { public: - TInputScanner(int n, const char* const s[], size_t L[], const char* const* names = nullptr, int b = 0, int f = 0) : + TInputScanner(int n, const char* const s[], size_t L[], const char* const* names = nullptr, int b = 0, int f = 0, bool single = false) : numSources(n), sources(reinterpret_cast(s)), // up to this point, common usage is "char*", but now we need positive 8-bit characters - lengths(L), currentSource(0), currentChar(0), stringBias(b), finale(f) + lengths(L), currentSource(0), currentChar(0), stringBias(b), finale(f), singleLogical(single) { loc = new TSourceLoc[numSources]; for (int i = 0; i < numSources; ++i) { @@ -67,6 +67,10 @@ public: loc[currentSource].string = -stringBias; loc[currentSource].line = 1; loc[currentSource].column = 0; + logicalSourceLoc.string = 0; + logicalSourceLoc.line = 1; + logicalSourceLoc.column = 0; + logicalSourceLoc.name = loc[0].name; } virtual ~TInputScanner() @@ -82,8 +86,11 @@ public: int ret = peek(); ++loc[currentSource].column; + ++logicalSourceLoc.column; if (ret == '\n') { ++loc[currentSource].line; + ++logicalSourceLoc.line; + logicalSourceLoc.column = 0; loc[currentSource].column = 0; } advance(); @@ -118,6 +125,7 @@ public: if (currentChar > 0) { --currentChar; --loc[currentSource].column; + --logicalSourceLoc.column; if (loc[currentSource].column < 0) { // We've moved back past a new line. Find the // previous newline (or start of the file) to compute @@ -129,6 +137,7 @@ public: } --chIndex; } + logicalSourceLoc.column = (int)(currentChar - chIndex); loc[currentSource].column = (int)(currentChar - chIndex); } } else { @@ -141,23 +150,49 @@ public: } else currentChar = lengths[currentSource] - 1; } - if (peek() == '\n') + if (peek() == '\n') { --loc[currentSource].line; + --logicalSourceLoc.line; + } } // for #line override - void setLine(int newLine) { loc[getLastValidSourceIndex()].line = newLine; } - void setFile(const char* filename) { loc[getLastValidSourceIndex()].name = filename; } + void setLine(int newLine) + { + logicalSourceLoc.line = newLine; + loc[getLastValidSourceIndex()].line = newLine; + } + + // for #line override in filename based parsing + void setFile(const char* filename) + { + logicalSourceLoc.name = filename; + loc[getLastValidSourceIndex()].name = filename; + } + void setString(int newString) { + logicalSourceLoc.string = newString; loc[getLastValidSourceIndex()].string = newString; + logicalSourceLoc.name = nullptr; loc[getLastValidSourceIndex()].name = nullptr; } // for #include content indentation - void setColumn(int col) { loc[getLastValidSourceIndex()].column = col; } + void setColumn(int col) + { + logicalSourceLoc.column = col; + loc[getLastValidSourceIndex()].column = col; + } - const TSourceLoc& getSourceLoc() const { return loc[std::max(0, std::min(currentSource, numSources - finale - 1))]; } + const TSourceLoc& getSourceLoc() const + { + if (singleLogical) { + return logicalSourceLoc; + } else { + return loc[std::max(0, std::min(currentSource, numSources - finale - 1))]; + } + } // Returns the index (starting from 0) of the most recent valid source string we are reading from. int getLastValidSourceIndex() const { return std::min(currentSource, numSources - 1); } @@ -204,6 +239,10 @@ protected: int stringBias; // the first string that is the user's string number 0 int finale; // number of internal strings after user's last string + + TSourceLoc logicalSourceLoc; + bool singleLogical; // treats the strings as a single logical string. + // locations will be reported from the first string. }; } // end namespace glslang diff --git a/glslang/MachineIndependent/ShaderLang.cpp b/glslang/MachineIndependent/ShaderLang.cpp index 1f8355390..2a6e1e198 100644 --- a/glslang/MachineIndependent/ShaderLang.cpp +++ b/glslang/MachineIndependent/ShaderLang.cpp @@ -131,7 +131,8 @@ bool InitializeSymbolTable(const TString& builtIns, int version, EProfile profil TIntermediate intermediate(language, version, profile); TParseContext parseContext(symbolTable, intermediate, true, version, profile, spv, vulkan, language, infoSink); - TPpContext ppContext(parseContext, TShader::ForbidInclude()); + TShader::ForbidInclude includer; + TPpContext ppContext(parseContext, "", includer); TScanContext scanContext(parseContext); parseContext.setScanContext(&scanContext); parseContext.setPpContext(&ppContext); @@ -482,7 +483,7 @@ bool ProcessDeferred( TIntermediate& intermediate, // returned tree, etc. ProcessingContext& processingContext, bool requireNonempty, - const TShader::Includer& includer + TShader::Includer& includer ) { if (! InitThread()) @@ -550,7 +551,6 @@ bool ProcessDeferred( version = defaultVersion; profile = defaultProfile; } - int spv = (messages & EShMsgSpvRules) ? 100 : 0; // TODO find path to get real version number here, for now non-0 is what matters bool goodVersion = DeduceVersionProfile(compiler->infoSink, compiler->getLanguage(), versionNotFirst, defaultVersion, version, profile, spv); bool versionWillBeError = (versionNotFound || (profile == EEsProfile && version >= 300 && versionNotFirst)); @@ -590,7 +590,7 @@ bool ProcessDeferred( TParseContext parseContext(symbolTable, intermediate, false, version, profile, spv, vulkan, compiler->getLanguage(), compiler->infoSink, forwardCompatible, messages); glslang::TScanContext scanContext(parseContext); - TPpContext ppContext(parseContext, includer); + TPpContext ppContext(parseContext, names[numPre]? names[numPre]: "", includer); parseContext.setScanContext(&scanContext); parseContext.setPpContext(&ppContext); parseContext.setLimits(*resources); @@ -863,7 +863,7 @@ bool PreprocessDeferred( bool forceDefaultVersionAndProfile, bool forwardCompatible, // give errors for use of deprecated features EShMessages messages, // warnings/errors/AST; things to print out - const TShader::Includer& includer, + TShader::Includer& includer, TIntermediate& intermediate, // returned tree, etc. std::string* outputString) { @@ -902,7 +902,7 @@ bool CompileDeferred( bool forwardCompatible, // give errors for use of deprecated features EShMessages messages, // warnings/errors/AST; things to print out TIntermediate& intermediate,// returned tree, etc. - const TShader::Includer& includer) + TShader::Includer& includer) { DoFullParse parser; return ProcessDeferred(compiler, shaderStrings, numStrings, inputLengths, stringNames, @@ -1051,9 +1051,10 @@ int ShCompile( compiler->infoSink.debug.erase(); TIntermediate intermediate(compiler->getLanguage()); + TShader::ForbidInclude includer; bool success = CompileDeferred(compiler, shaderStrings, numStrings, inputLengths, nullptr, "", optLevel, resources, defaultVersion, ENoProfile, false, - forwardCompatible, messages, intermediate, TShader::ForbidInclude()); + forwardCompatible, messages, intermediate, includer); // // Call the machine dependent compiler @@ -1361,7 +1362,7 @@ void TShader::setStringsWithLengthsAndNames( // Returns true for success. // bool TShader::parse(const TBuiltInResource* builtInResources, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile, - bool forwardCompatible, EShMessages messages, const Includer& includer) + bool forwardCompatible, EShMessages messages, Includer& includer) { if (! InitThread()) return false; @@ -1389,7 +1390,7 @@ bool TShader::preprocess(const TBuiltInResource* builtInResources, bool forceDefaultVersionAndProfile, bool forwardCompatible, EShMessages message, std::string* output_string, - const TShader::Includer& includer) + Includer& includer) { if (! InitThread()) return false; diff --git a/glslang/MachineIndependent/preprocessor/Pp.cpp b/glslang/MachineIndependent/preprocessor/Pp.cpp index 9a33a776f..ed5543591 100644 --- a/glslang/MachineIndependent/preprocessor/Pp.cpp +++ b/glslang/MachineIndependent/preprocessor/Pp.cpp @@ -611,24 +611,28 @@ int TPpContext::CPPinclude(TPpToken* ppToken) if (token != '\n' && token != EndOfInput) { parseContext.ppError(ppToken->loc, "extra content after file designation", "#include", ""); } else { - auto include = includer.include(filename.c_str()); - std::string sourceName = include.first; - std::string replacement = include.second; - if (!sourceName.empty()) { - if (!replacement.empty()) { + TShader::Includer::IncludeResult* res = includer.include(filename.c_str(), TShader::Includer::EIncludeRelative, currentSourceFile.c_str(), includeStack.size() + 1); + if (res && !res->file_name.empty()) { + if (res->file_data && res->file_length) { const bool forNextLine = parseContext.lineDirectiveShouldSetNextLine(); - std::ostringstream content; - content << "#line " << forNextLine << " " << "\"" << sourceName << "\"\n"; - content << replacement << (replacement.back() == '\n' ? "" : "\n"); - content << "#line " << directiveLoc.line + forNextLine << " " << directiveLoc.getStringNameOrNum() << "\n"; - pushInput(new TokenizableString(directiveLoc, content.str(), this)); + std::ostringstream prologue; + std::ostringstream epilogue; + prologue << "#line " << forNextLine << " " << "\"" << res->file_name << "\"\n"; + epilogue << (res->file_data[res->file_length - 1] == '\n'? "" : "\n") << "#line " << directiveLoc.line + forNextLine << " " << directiveLoc.getStringNameOrNum() << "\n"; + pushInput(new TokenizableIncludeFile(directiveLoc, prologue.str(), res, epilogue.str(), this)); } // At EOF, there's no "current" location anymore. if (token != EndOfInput) parseContext.setCurrentColumn(0); // Don't accidentally return EndOfInput, which will end all preprocessing. return '\n'; } else { - parseContext.ppError(directiveLoc, replacement.c_str(), "#include", ""); + std::string message = + res ? std::string(res->file_data, res->file_length) + : std::string("Could not process include directive"); + parseContext.ppError(directiveLoc, message.c_str(), "#include", ""); + if (res) { + includer.releaseInclude(res); + } } } } diff --git a/glslang/MachineIndependent/preprocessor/PpContext.cpp b/glslang/MachineIndependent/preprocessor/PpContext.cpp index b8d2c7371..37c2a5f2d 100644 --- a/glslang/MachineIndependent/preprocessor/PpContext.cpp +++ b/glslang/MachineIndependent/preprocessor/PpContext.cpp @@ -83,8 +83,10 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace glslang { -TPpContext::TPpContext(TParseContext& pc, const TShader::Includer& inclr) : - preamble(0), strings(0), parseContext(pc), includer(inclr), inComment(false) +TPpContext::TPpContext(TParseContext& pc, const std::string& rootFileName, TShader::Includer& inclr) : + preamble(0), strings(0), parseContext(pc), includer(inclr), inComment(false), + rootFileName(rootFileName), + currentSourceFile(rootFileName) { InitAtomTable(); InitScanner(); diff --git a/glslang/MachineIndependent/preprocessor/PpContext.h b/glslang/MachineIndependent/preprocessor/PpContext.h index f1d546916..113215bd2 100644 --- a/glslang/MachineIndependent/preprocessor/PpContext.h +++ b/glslang/MachineIndependent/preprocessor/PpContext.h @@ -78,6 +78,7 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef PPCONTEXT_H #define PPCONTEXT_H +#include #include #include "../ParseHelper.h" @@ -121,7 +122,7 @@ class TInputScanner; // Don't expect too much in terms of OO design. class TPpContext { public: - TPpContext(TParseContext&, const TShader::Includer&); + TPpContext(TParseContext&, const std::string& rootFileName, TShader::Includer&); virtual ~TPpContext(); void setPreamble(const char* preamble, size_t length); @@ -290,7 +291,7 @@ protected: // // Used to obtain #include content. - const TShader::Includer& includer; + TShader::Includer& includer; int InitCPP(); int CPPdefine(TPpToken * ppToken); @@ -430,16 +431,30 @@ protected: TInputScanner* input; }; - // Holds a string that can be tokenized via the tInput interface. - class TokenizableString : public tInput { + // Holds a reference to included file data, as well as a + // prologue and an epilogue string. This can be scanned using the tInput + // interface and acts as a single source string. + class TokenizableIncludeFile : public tInput { public: - // Copies str, which must be non-empty. - TokenizableString(const TSourceLoc& startLoc, const std::string& str, TPpContext* pp) + // Copies prologue and epilogue. The includedFile must remain valid + // until this TokenizableIncludeFile is no longer used. + TokenizableIncludeFile(const TSourceLoc& startLoc, + const std::string& prologue, + TShader::Includer::IncludeResult* includedFile, + const std::string& epilogue, + TPpContext* pp) : tInput(pp), - str_(str), - strings(str_.data()), - length(str_.size()), - scanner(1, &strings, &length), + prologue_(prologue), + includedFile_(includedFile), + epilogue_(epilogue), + strings({prologue_.data(), includedFile_->file_data, epilogue_.data()}), + lengths({prologue_.size(), includedFile_->file_length, epilogue_.size()}), + names({ + includedFile_->file_name.c_str(), + includedFile_->file_name.c_str(), + includedFile_->file_name.c_str() + }), + scanner(3, strings, lengths, names, 0, 0, true), prevScanner(nullptr), stringInput(pp, scanner) { scanner.setLine(startLoc.line); @@ -456,16 +471,34 @@ protected: { prevScanner = pp->parseContext.getScanner(); pp->parseContext.setScanner(&scanner); + pp->push_include(includedFile_); + } + + void notifyDeleted() override + { + pp->parseContext.setScanner(prevScanner); + pp->pop_include(); } - void notifyDeleted() override { pp->parseContext.setScanner(prevScanner); } private: - // Stores the titular string. - const std::string str_; - // Will point to str_[0] and be passed to scanner constructor. - const char* const strings; + // Stores the prologue for this string. + const std::string prologue_; + + // Stores the epilogue for this string. + const std::string epilogue_; + + // Points to the IncludeResult that this TokenizableIncludeFile represents. + TShader::Includer::IncludeResult* includedFile_; + + // Will point to prologue_, includedFile_->file_data and epilogue_ + // This is passed to scanner constructor. + // These do not own the storage and it must remain valid until this + // object has been destroyed. + const char* strings[3]; // Length of str_, passed to scanner constructor. - size_t length; + size_t lengths[3]; + // String names + const char* names[3]; // Scans over str_. TInputScanner scanner; // The previous effective scanner before the scanner in this instance @@ -480,6 +513,24 @@ protected: void missingEndifCheck(); int lFloatConst(int len, int ch, TPpToken* ppToken); + void push_include(TShader::Includer::IncludeResult* result) + { + currentSourceFile = result->file_name; + includeStack.push(result); + } + + void pop_include() + { + TShader::Includer::IncludeResult* include = includeStack.top(); + includeStack.pop(); + includer.releaseInclude(include); + if (includeStack.empty()) { + currentSourceFile = rootFileName; + } else { + currentSourceFile = includeStack.top()->file_name; + } + } + bool inComment; // @@ -487,8 +538,12 @@ protected: // typedef TUnorderedMap TAtomMap; typedef TVector TStringMap; + TAtomMap atomMap; TStringMap stringMap; + std::stack includeStack; + std::string currentSourceFile; + std::string rootFileName; int nextAtom; void InitAtomTable(); void AddAtomFixed(const char* s, int atom); diff --git a/glslang/Public/ShaderLang.h b/glslang/Public/ShaderLang.h index 702b66f47..0771d1943 100644 --- a/glslang/Public/ShaderLang.h +++ b/glslang/Public/ShaderLang.h @@ -292,25 +292,89 @@ public: void setPreamble(const char* s) { preamble = s; } // Interface to #include handlers. + // + // To support #include, a client of Glslang does the following: + // 1. Call setStringsWithNames to set the source strings and associated + // names. For example, the names could be the names of the files + // containing the shader sources. + // 2. Call parse with an Includer. + // + // When the Glslang parser encounters an #include directive, it calls + // the Includer's include method with the the requested include name + // together with the current string name. The returned IncludeResult + // contains the fully resolved name of the included source, together + // with the source text that should replace the #include directive + // in the source stream. After parsing that source, Glslang will + // release the IncludeResult object. class Includer { public: - // On success, returns the full path and content of the file with the given - // filename that replaces "#include filename". On failure, returns an empty - // string and an error message. - virtual std::pair include(const char* filename) const = 0; + typedef enum { + EIncludeRelative, // For #include "something" + EIncludeStandard // Reserved. For #include + } IncludeType; + + // An IncludeResult contains the resolved name and content of a source + // inclusion. + struct IncludeResult { + // For a successful inclusion, the fully resolved name of the requested + // include. For example, in a filesystem-based includer, full resolution + // should convert a relative path name into an absolute path name. + // For a failed inclusion, this is an empty string. + std::string file_name; + // The content and byte length of the requested inclusion. The + // Includer producing this IncludeResult retains ownership of the + // storage. + // For a failed inclusion, the file_data + // field points to a string containing error details. + const char* file_data; + const size_t file_length; + // Include resolver's context. + void* user_data; + }; + + // Resolves an inclusion request by name, type, current source name, + // and include depth. + // On success, returns an IncludeResult containing the resolved name + // and content of the include. On failure, returns an IncludeResult + // with an empty string for the file_name and error details in the + // file_data field. The Includer retains ownership of the contents + // of the returned IncludeResult value, and those contents must + // remain valid until the releaseInclude method is called on that + // IncludeResult object. + virtual IncludeResult* include(const char* requested_source, + IncludeType type, + const char* requesting_source, + size_t inclusion_depth) = 0; + // Signals that the parser will no longer use the contents of the + // specified IncludeResult. + virtual void releaseInclude(IncludeResult* result) = 0; }; // Returns an error message for any #include directive. class ForbidInclude : public Includer { public: - std::pair include(const char* /*filename*/) const override + IncludeResult* include(const char*, IncludeType, const char*, size_t) override { - return std::make_pair("", "unexpected include directive"); + static const char unexpected_include[] = + "unexpected include directive"; + return new IncludeResult( + {"", unexpected_include, sizeof(unexpected_include) - 1, nullptr}); + } + virtual void releaseInclude(IncludeResult* result) override + { + delete result; } }; + bool parse(const TBuiltInResource* res, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile, + bool forwardCompatible, EShMessages messages) + { + TShader::ForbidInclude includer; + return parse(res, defaultVersion, defaultProfile, forceDefaultVersionAndProfile, forwardCompatible, messages, includer); + } + bool parse(const TBuiltInResource*, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile, - bool forwardCompatible, EShMessages, const Includer& = ForbidInclude()); + bool forwardCompatible, EShMessages, Includer&); // Equivalent to parse() without a default profile and without forcing defaults. // Provided for backwards compatibility. @@ -318,7 +382,7 @@ public: bool preprocess(const TBuiltInResource* builtInResources, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile, bool forwardCompatible, EShMessages message, std::string* outputString, - const TShader::Includer& includer); + Includer& includer); const char* getInfoLog(); const char* getInfoDebugLog();