Updated the includer interface to allow relative includes.

This plumbs both the current file path and the include depth
back up to the includer. This allows the includer to properly
support relative paths.

This also replaces the string copy that was done during include
with a zero-copy method of accomplishing the same thing. This
prevents extra copies of entire files.

StandAlone/StandAlone.cpp

@@ -699,8 +699,9 @@ void CompileAndLinkShaderUnits(std::vector<ShaderCompUnit> compUnits)
 
         if (Options & EOptionOutputPreprocessed) {
             std::string str;
+            glslang::TShader::ForbidInclude includer;
             if (shader->preprocess(&Resources, defaultVersion, ENoProfile, false, false,
-                                   messages, &str, glslang::TShader::ForbidInclude())) {
+                                   messages, &str, includer)) {
                 PutsIfNonEmpty(str.c_str());
             } else {
                 CompileFailed = true;

glslang/MachineIndependent/Scan.h

@@ -51,10 +51,10 @@ const int EndOfInput = -1;
 //
 class TInputScanner {
 public:
-    TInputScanner(int n, const char* const s[], size_t L[], const char* const* names = nullptr, int b = 0, int f = 0) :
+    TInputScanner(int n, const char* const s[], size_t L[], const char* const* names = nullptr, int b = 0, int f = 0, bool single = false) :
         numSources(n),
         sources(reinterpret_cast<const unsigned char* const *>(s)), // up to this point, common usage is "char*", but now we need positive 8-bit characters
-        lengths(L), currentSource(0), currentChar(0), stringBias(b), finale(f)
+        lengths(L), currentSource(0), currentChar(0), stringBias(b), finale(f), singleLogical(single)
     {
         loc = new TSourceLoc[numSources];
         for (int i = 0; i < numSources; ++i) {
@@ -67,6 +67,10 @@ public:
         loc[currentSource].string = -stringBias;
         loc[currentSource].line = 1;
         loc[currentSource].column = 0;
+        logicalSourceLoc.string = 0;
+        logicalSourceLoc.line = 1;
+        logicalSourceLoc.column = 0;
+        logicalSourceLoc.name = loc[0].name;
     }
 
     virtual ~TInputScanner()
@@ -82,8 +86,11 @@ public:
 
         int ret = peek();
         ++loc[currentSource].column;
+        ++logicalSourceLoc.column;
         if (ret == '\n') {
             ++loc[currentSource].line;
+            ++logicalSourceLoc.line;
+            logicalSourceLoc.column = 0;
             loc[currentSource].column = 0;
         }
         advance();
@@ -118,6 +125,7 @@ public:
         if (currentChar > 0) {
             --currentChar;
             --loc[currentSource].column;
+            --logicalSourceLoc.column;
             if (loc[currentSource].column < 0) {
                 // We've moved back past a new line. Find the
                 // previous newline (or start of the file) to compute
@@ -129,6 +137,7 @@ public:
                     }
                     --chIndex;
                 }
+                logicalSourceLoc.column = (int)(currentChar - chIndex);
                 loc[currentSource].column = (int)(currentChar - chIndex);
             }
         } else {
@@ -141,23 +150,49 @@ public:
             } else
                 currentChar = lengths[currentSource] - 1;
         }
-        if (peek() == '\n')
+        if (peek() == '\n') {
             --loc[currentSource].line;
+            --logicalSourceLoc.line;
+        }
     }
 
     // for #line override
-    void setLine(int newLine) { loc[getLastValidSourceIndex()].line = newLine; }
+    void setLine(int newLine)
-    void setFile(const char* filename) { loc[getLastValidSourceIndex()].name = filename; }
+    {
+        logicalSourceLoc.line = newLine;
+        loc[getLastValidSourceIndex()].line = newLine;
+    }
+
+    // for #line override in filename based parsing
+    void setFile(const char* filename)
+    {
+        logicalSourceLoc.name = filename;
+        loc[getLastValidSourceIndex()].name = filename;
+    }
+
     void setString(int newString)
     {
+        logicalSourceLoc.string = newString;
         loc[getLastValidSourceIndex()].string = newString;
+        logicalSourceLoc.name = nullptr;
         loc[getLastValidSourceIndex()].name = nullptr;
     }
 
     // for #include content indentation
-    void setColumn(int col) { loc[getLastValidSourceIndex()].column = col; }
+    void setColumn(int col)
+    {
+        logicalSourceLoc.column = col;
+        loc[getLastValidSourceIndex()].column = col;
+    }
 
-    const TSourceLoc& getSourceLoc() const { return loc[std::max(0, std::min(currentSource, numSources - finale - 1))]; }
+    const TSourceLoc& getSourceLoc() const
+    {
+        if (singleLogical) {
+            return logicalSourceLoc;
+        } else {
+            return loc[std::max(0, std::min(currentSource, numSources - finale - 1))];
+        }
+    }
     // Returns the index (starting from 0) of the most recent valid source string we are reading from.
     int getLastValidSourceIndex() const { return std::min(currentSource, numSources - 1); }
 
@@ -204,6 +239,10 @@ protected:
 
     int stringBias;   // the first string that is the user's string number 0
     int finale;       // number of internal strings after user's last string
+
+    TSourceLoc logicalSourceLoc;
+    bool singleLogical; // treats the strings as a single logical string.
+                        // locations will be reported from the first string.
 };
 
 } // end namespace glslang

glslang/MachineIndependent/ShaderLang.cpp

@@ -131,7 +131,8 @@ bool InitializeSymbolTable(const TString& builtIns, int version, EProfile profil
     TIntermediate intermediate(language, version, profile);
     
     TParseContext parseContext(symbolTable, intermediate, true, version, profile, spv, vulkan, language, infoSink);
-    TPpContext ppContext(parseContext, TShader::ForbidInclude());
+    TShader::ForbidInclude includer;
+    TPpContext ppContext(parseContext, "", includer);
     TScanContext scanContext(parseContext);
     parseContext.setScanContext(&scanContext);
     parseContext.setPpContext(&ppContext);
@@ -482,7 +483,7 @@ bool ProcessDeferred(
     TIntermediate& intermediate, // returned tree, etc.
     ProcessingContext& processingContext,
     bool requireNonempty,
-    const TShader::Includer& includer
+    TShader::Includer& includer
     )
 {
     if (! InitThread())
@@ -550,7 +551,6 @@ bool ProcessDeferred(
         version = defaultVersion;
         profile = defaultProfile;
     }
-
     int spv = (messages & EShMsgSpvRules) ? 100 : 0;         // TODO find path to get real version number here, for now non-0 is what matters
     bool goodVersion = DeduceVersionProfile(compiler->infoSink, compiler->getLanguage(), versionNotFirst, defaultVersion, version, profile, spv);
     bool versionWillBeError = (versionNotFound || (profile == EEsProfile && version >= 300 && versionNotFirst));
@@ -590,7 +590,7 @@ bool ProcessDeferred(
 
     TParseContext parseContext(symbolTable, intermediate, false, version, profile, spv, vulkan, compiler->getLanguage(), compiler->infoSink, forwardCompatible, messages);
     glslang::TScanContext scanContext(parseContext);
-    TPpContext ppContext(parseContext, includer);
+    TPpContext ppContext(parseContext, names[numPre]? names[numPre]: "", includer);
     parseContext.setScanContext(&scanContext);
     parseContext.setPpContext(&ppContext);
     parseContext.setLimits(*resources);
@@ -863,7 +863,7 @@ bool PreprocessDeferred(
     bool forceDefaultVersionAndProfile,
     bool forwardCompatible,     // give errors for use of deprecated features
     EShMessages messages,       // warnings/errors/AST; things to print out
-    const TShader::Includer& includer,
+    TShader::Includer& includer,
     TIntermediate& intermediate, // returned tree, etc.
     std::string* outputString)
 {
@@ -902,7 +902,7 @@ bool CompileDeferred(
     bool forwardCompatible,     // give errors for use of deprecated features
     EShMessages messages,       // warnings/errors/AST; things to print out
     TIntermediate& intermediate,// returned tree, etc.
-    const TShader::Includer& includer)
+    TShader::Includer& includer)
 {
     DoFullParse parser;
     return ProcessDeferred(compiler, shaderStrings, numStrings, inputLengths, stringNames,
@@ -1051,9 +1051,10 @@ int ShCompile(
     compiler->infoSink.debug.erase();
 
     TIntermediate intermediate(compiler->getLanguage());
+    TShader::ForbidInclude includer;
     bool success = CompileDeferred(compiler, shaderStrings, numStrings, inputLengths, nullptr,
                                    "", optLevel, resources, defaultVersion, ENoProfile, false,
-                                   forwardCompatible, messages, intermediate, TShader::ForbidInclude());
+                                   forwardCompatible, messages, intermediate, includer);
 
     //
     // Call the machine dependent compiler
@@ -1361,7 +1362,7 @@ void TShader::setStringsWithLengthsAndNames(
 // Returns true for success.
 //
 bool TShader::parse(const TBuiltInResource* builtInResources, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
-                    bool forwardCompatible, EShMessages messages, const Includer& includer)
+                    bool forwardCompatible, EShMessages messages, Includer& includer)
 {
     if (! InitThread())
         return false;
@@ -1389,7 +1390,7 @@ bool TShader::preprocess(const TBuiltInResource* builtInResources,
                          bool forceDefaultVersionAndProfile,
                          bool forwardCompatible, EShMessages message,
                          std::string* output_string,
-                         const TShader::Includer& includer)
+                         Includer& includer)
 {
     if (! InitThread())
         return false;

glslang/MachineIndependent/preprocessor/Pp.cpp

@@ -611,24 +611,28 @@ int TPpContext::CPPinclude(TPpToken* ppToken)
         if (token != '\n' && token != EndOfInput) {
             parseContext.ppError(ppToken->loc, "extra content after file designation", "#include", "");
         } else {
-            auto include = includer.include(filename.c_str());
+            TShader::Includer::IncludeResult* res = includer.include(filename.c_str(), TShader::Includer::EIncludeRelative, currentSourceFile.c_str(), includeStack.size() + 1);
-            std::string sourceName = include.first;
+            if (res && !res->file_name.empty()) {
-            std::string replacement = include.second;
+                if (res->file_data && res->file_length) {
-            if (!sourceName.empty()) {
-                if (!replacement.empty()) {
                     const bool forNextLine = parseContext.lineDirectiveShouldSetNextLine();
-                    std::ostringstream content;
+                    std::ostringstream prologue;
-                    content << "#line " << forNextLine << " " << "\"" << sourceName << "\"\n";
+                    std::ostringstream epilogue;
-                    content << replacement << (replacement.back() == '\n' ? "" : "\n");
+                    prologue << "#line " << forNextLine << " " << "\"" << res->file_name << "\"\n";
-                    content << "#line " << directiveLoc.line + forNextLine << " " << directiveLoc.getStringNameOrNum() << "\n";
+                    epilogue << (res->file_data[res->file_length - 1] == '\n'? "" : "\n") << "#line " << directiveLoc.line + forNextLine << " " << directiveLoc.getStringNameOrNum() << "\n";
-                    pushInput(new TokenizableString(directiveLoc, content.str(), this));
+                    pushInput(new TokenizableIncludeFile(directiveLoc, prologue.str(), res, epilogue.str(), this));
                 }
                 // At EOF, there's no "current" location anymore.
                 if (token != EndOfInput) parseContext.setCurrentColumn(0);
                 // Don't accidentally return EndOfInput, which will end all preprocessing.
                 return '\n';
             } else {
-                parseContext.ppError(directiveLoc, replacement.c_str(), "#include", "");
+                std::string message =
+                    res ? std::string(res->file_data, res->file_length)
+                        : std::string("Could not process include directive");
+                parseContext.ppError(directiveLoc, message.c_str(), "#include", "");
+                if (res) {
+                    includer.releaseInclude(res);
+                }
             }
         }
     }

glslang/MachineIndependent/preprocessor/PpContext.cpp

@@ -83,8 +83,10 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace glslang {
 
-TPpContext::TPpContext(TParseContext& pc, const TShader::Includer& inclr) : 
+TPpContext::TPpContext(TParseContext& pc, const std::string& rootFileName, TShader::Includer& inclr) : 
-    preamble(0), strings(0), parseContext(pc), includer(inclr), inComment(false)
+    preamble(0), strings(0), parseContext(pc), includer(inclr), inComment(false),
+    rootFileName(rootFileName),
+    currentSourceFile(rootFileName)
 {
     InitAtomTable();
     InitScanner();

glslang/MachineIndependent/preprocessor/PpContext.h

@@ -78,6 +78,7 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifndef PPCONTEXT_H
 #define PPCONTEXT_H
 
+#include <stack>
 #include <unordered_map>
 
 #include "../ParseHelper.h"
@@ -121,7 +122,7 @@ class TInputScanner;
 // Don't expect too much in terms of OO design.
 class TPpContext {
 public:
-    TPpContext(TParseContext&, const TShader::Includer&);
+    TPpContext(TParseContext&, const std::string& rootFileName, TShader::Includer&);
     virtual ~TPpContext();
 
     void setPreamble(const char* preamble, size_t length);
@@ -290,7 +291,7 @@ protected:
     //
 
     // Used to obtain #include content.
-    const TShader::Includer& includer;
+    TShader::Includer& includer;
 
     int InitCPP();
     int CPPdefine(TPpToken * ppToken);
@@ -430,16 +431,30 @@ protected:
         TInputScanner* input;
     };
 
-    // Holds a string that can be tokenized via the tInput interface.
+    // Holds a reference to included file data, as well as a 
-    class TokenizableString : public tInput {
+    // prologue and an epilogue string. This can be scanned using the tInput
+    // interface and acts as a single source string.
+    class TokenizableIncludeFile : public tInput {
     public:
-        // Copies str, which must be non-empty.
+        // Copies prologue and epilogue. The includedFile must remain valid
-        TokenizableString(const TSourceLoc& startLoc, const std::string& str, TPpContext* pp)
+        // until this TokenizableIncludeFile is no longer used.
+        TokenizableIncludeFile(const TSourceLoc& startLoc,
+                          const std::string& prologue,
+                          TShader::Includer::IncludeResult* includedFile,
+                          const std::string& epilogue,
+                          TPpContext* pp)
             : tInput(pp),
-              str_(str),
+              prologue_(prologue),
-              strings(str_.data()),
+              includedFile_(includedFile),
-              length(str_.size()),
+              epilogue_(epilogue),
-              scanner(1, &strings, &length),
+              strings({prologue_.data(), includedFile_->file_data, epilogue_.data()}),
+              lengths({prologue_.size(), includedFile_->file_length, epilogue_.size()}),
+              names({
+                      includedFile_->file_name.c_str(),
+                      includedFile_->file_name.c_str(),
+                      includedFile_->file_name.c_str()
+              }),
+              scanner(3, strings, lengths, names, 0, 0, true),
               prevScanner(nullptr),
               stringInput(pp, scanner) {
                   scanner.setLine(startLoc.line);
@@ -456,16 +471,34 @@ protected:
         {
             prevScanner = pp->parseContext.getScanner();
             pp->parseContext.setScanner(&scanner);
+            pp->push_include(includedFile_);
+        }
+
+        void notifyDeleted() override
+        {
+            pp->parseContext.setScanner(prevScanner);
+            pp->pop_include();
         }
-        void notifyDeleted() override { pp->parseContext.setScanner(prevScanner); }
 
     private:
-        // Stores the titular string.
+        // Stores the prologue for this string.
-        const std::string str_;
+        const std::string prologue_;
-        // Will point to str_[0] and be passed to scanner constructor.
+
-        const char* const strings;
+        // Stores the epilogue for this string.
+        const std::string epilogue_;
+
+        // Points to the IncludeResult that this TokenizableIncludeFile represents.
+        TShader::Includer::IncludeResult* includedFile_;
+
+        // Will point to prologue_, includedFile_->file_data and epilogue_
+        // This is passed to scanner constructor.
+        // These do not own the storage and it must remain valid until this
+        // object has been destroyed.
+        const char* strings[3];
         // Length of str_, passed to scanner constructor.
-        size_t length;
+        size_t lengths[3];
+        // String names
+        const char* names[3];
         // Scans over str_.
         TInputScanner scanner;
         // The previous effective scanner before the scanner in this instance
@@ -480,6 +513,24 @@ protected:
     void missingEndifCheck();
     int lFloatConst(int len, int ch, TPpToken* ppToken);
 
+    void push_include(TShader::Includer::IncludeResult* result)
+    {
+        currentSourceFile = result->file_name;
+        includeStack.push(result);
+    }
+
+    void pop_include()
+    {
+        TShader::Includer::IncludeResult* include = includeStack.top();
+        includeStack.pop();
+        includer.releaseInclude(include);
+        if (includeStack.empty()) {
+            currentSourceFile = rootFileName;
+        } else {
+            currentSourceFile = includeStack.top()->file_name;
+        }
+    }
+
     bool inComment;
 
     //
@@ -487,8 +538,12 @@ protected:
     //
     typedef TUnorderedMap<TString, int> TAtomMap;
     typedef TVector<const TString*> TStringMap;
+
     TAtomMap atomMap;
     TStringMap stringMap;
+    std::stack<TShader::Includer::IncludeResult*> includeStack;
+    std::string currentSourceFile;
+    std::string rootFileName;
     int nextAtom;
     void InitAtomTable();
     void AddAtomFixed(const char* s, int atom);

glslang/Public/ShaderLang.h

@@ -292,25 +292,89 @@ public:
     void setPreamble(const char* s) { preamble = s; }
 
     // Interface to #include handlers.
+    //
+    // To support #include, a client of Glslang does the following:
+    // 1. Call setStringsWithNames to set the source strings and associated
+    //    names.  For example, the names could be the names of the files
+    //    containing the shader sources.
+    // 2. Call parse with an Includer.
+    //
+    // When the Glslang parser encounters an #include directive, it calls
+    // the Includer's include method with the the requested include name
+    // together with the current string name.  The returned IncludeResult
+    // contains the fully resolved name of the included source, together
+    // with the source text that should replace the #include directive
+    // in the source stream.  After parsing that source, Glslang will
+    // release the IncludeResult object.
     class Includer {
     public:
-        // On success, returns the full path and content of the file with the given
+        typedef enum {
-        // filename that replaces "#include filename". On failure, returns an empty
+          EIncludeRelative, // For #include "something"
-        // string and an error message.
+          EIncludeStandard  // Reserved. For #include <something>
-        virtual std::pair<std::string, std::string> include(const char* filename) const = 0;
+        } IncludeType;
+
+        // An IncludeResult contains the resolved name and content of a source
+        // inclusion.
+        struct IncludeResult {
+            // For a successful inclusion, the fully resolved name of the requested
+            // include.  For example, in a filesystem-based includer, full resolution
+            // should convert a relative path name into an absolute path name.
+            // For a failed inclusion, this is an empty string.
+            std::string file_name;
+            // The content and byte length of the requested inclusion.  The
+            // Includer producing this IncludeResult retains ownership of the
+            // storage.
+            // For a failed inclusion, the file_data
+            // field points to a string containing error details.
+            const char* file_data;
+            const size_t file_length;
+            // Include resolver's context.
+            void* user_data;
+        };
+
+        // Resolves an inclusion request by name, type, current source name,
+        // and include depth.
+        // On success, returns an IncludeResult containing the resolved name
+        // and content of the include.  On failure, returns an IncludeResult
+        // with an empty string for the file_name and error details in the
+        // file_data field.  The Includer retains ownership of the contents
+        // of the returned IncludeResult value, and those contents must
+        // remain valid until the releaseInclude method is called on that
+        // IncludeResult object.
+        virtual IncludeResult* include(const char* requested_source,
+                                      IncludeType type,
+                                      const char* requesting_source,
+                                      size_t inclusion_depth) = 0;
+        // Signals that the parser will no longer use the contents of the
+        // specified IncludeResult.
+        virtual void releaseInclude(IncludeResult* result) = 0;
     };
 
     // Returns an error message for any #include directive.
     class ForbidInclude : public Includer {
     public:
-        std::pair<std::string, std::string> include(const char* /*filename*/) const override
+        IncludeResult* include(const char*, IncludeType, const char*, size_t) override
         {
-            return std::make_pair<std::string, std::string>("", "unexpected include directive");
+            static const char unexpected_include[] =
+                "unexpected include directive";
+            return new IncludeResult(
+                {"", unexpected_include, sizeof(unexpected_include) - 1, nullptr});
+        }
+        virtual void releaseInclude(IncludeResult* result) override
+        {
+            delete result;
         }
     };
 
+    bool parse(const TBuiltInResource* res, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
+               bool forwardCompatible, EShMessages messages)
+    {
+        TShader::ForbidInclude includer;
+        return parse(res, defaultVersion, defaultProfile, forceDefaultVersionAndProfile, forwardCompatible, messages, includer);
+    }
+
     bool parse(const TBuiltInResource*, int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
-               bool forwardCompatible, EShMessages, const Includer& = ForbidInclude());
+               bool forwardCompatible, EShMessages, Includer&);
 
     // Equivalent to parse() without a default profile and without forcing defaults.
     // Provided for backwards compatibility.
@@ -318,7 +382,7 @@ public:
     bool preprocess(const TBuiltInResource* builtInResources,
                     int defaultVersion, EProfile defaultProfile, bool forceDefaultVersionAndProfile,
                     bool forwardCompatible, EShMessages message, std::string* outputString,
-                    const TShader::Includer& includer);
+                    Includer& includer);
 
     const char* getInfoLog();
     const char* getInfoDebugLog();