HLSL: Separate out token stream handling from grammar recognition.

hlsl/CMakeLists.txt

@@ -3,12 +3,14 @@ cmake_minimum_required(VERSION 2.8)
 set(SOURCES
     hlslParseHelper.cpp
     hlslScanContext.cpp
+    hlslTokenStream.cpp
     hlslGrammar.cpp)
 
 set(HEADERS
     hlslParseHelper.h
     hlslTokens.h
     hlslScanContext.h
+    hlslTokenStream.h
     hlslGrammar.h)
 
 add_library(HLSL STATIC ${SOURCES} ${HEADERS})

hlsl/hlslGrammar.cpp

@@ -68,31 +68,6 @@ void HlslGrammar::expected(const char* syntax)
     parseContext.error(token.loc, "Expected", syntax, "");
 }
 
-// Load 'token' with the next token in the stream of tokens.
-void HlslGrammar::advanceToken()
-{
-    scanner.tokenize(token);
-}
-
-// Return true and advance to the next token if the current token is the
-// expected (passed in) token class.
-bool HlslGrammar::acceptTokenClass(EHlslTokenClass tokenClass)
-{
-    if (token.tokenClass == tokenClass) {
-        advanceToken();
-        return true;
-    }
-
-    return false;
-}
-
-// Return true, without advancing to the next token, if the current token is
-// the expected (passed in) token class.
-bool HlslGrammar::peekTokenClass(EHlslTokenClass tokenClass)
-{
-    return token.tokenClass == tokenClass;
-}
-
 // Only process the next token if it is an identifier.
 // Return true if it was an identifier.
 bool HlslGrammar::acceptIdentifier(HlslToken& idToken)
@@ -113,7 +88,7 @@ bool HlslGrammar::acceptCompilationUnit()
 {
     TIntermNode* unitNode = nullptr;
 
-    while (token.tokenClass != EHTokNone) {
+    while (! peekTokenClass(EHTokNone)) {
         // externalDeclaration
         TIntermNode* declarationNode;
         if (! acceptDeclaration(declarationNode))
@@ -215,7 +190,7 @@ bool HlslGrammar::acceptFullySpecifiedType(TType& type)
 // qualifier.  Otherwise, return false, and don't advance.
 void HlslGrammar::acceptQualifier(TQualifier& qualifier)
 {
-    switch (token.tokenClass) {
+    switch (peek()) {
     case EHTokUniform:
         qualifier.storage = EvqUniform;
         break;
@@ -237,7 +212,7 @@ bool HlslGrammar::acceptType(TType& type)
     if (! token.isType)
         return false;
 
-    switch (token.tokenClass) {
+    switch (peek()) {
     case EHTokInt:
     case EHTokInt1:
     case EHTokDword:

hlsl/hlslGrammar.h

@@ -36,29 +36,25 @@
 #ifndef HLSLGRAMMAR_H_
 #define HLSLGRAMMAR_H_
 
-#include "hlslScanContext.h"
 #include "hlslParseHelper.h"
+#include "hlslTokenStream.h"
 
 namespace glslang {
 
     // Should just be the grammar aspect of HLSL.
     // Described in more detail in hlslGrammar.cpp.
 
-    class HlslGrammar {
+    class HlslGrammar : public HlslTokenStream {
     public:
         HlslGrammar(HlslScanContext& scanner, HlslParseContext& parseContext)
-            : scanner(scanner), parseContext(parseContext), intermediate(parseContext.intermediate) { }
+            : HlslTokenStream(scanner), parseContext(parseContext), intermediate(parseContext.intermediate) { }
         virtual ~HlslGrammar() { }
 
         bool parse();
 
     protected:
         void expected(const char*);
-        void advanceToken();
-        bool acceptTokenClass(EHlslTokenClass);
-        bool peekTokenClass(EHlslTokenClass);
         bool acceptIdentifier(HlslToken&);
-
         bool acceptCompilationUnit();
         bool acceptDeclaration(TIntermNode*& node);
         bool acceptFullySpecifiedType(TType&);
@@ -76,11 +72,8 @@ namespace glslang {
         bool acceptStatement(TIntermNode*&);
         bool acceptSemantic();
 
-        HlslScanContext& scanner;        // lexical scanner, to get next token
         HlslParseContext& parseContext;  // state of parsing and helper functions for building the intermediate
         TIntermediate& intermediate;     // the final product, the intermediate representation, includes the AST
-
-        HlslToken token;                 // the current token we are processing
     };
 
 } // end namespace glslang

hlsl/hlslTokenStream.cpp

@@ -0,0 +1,71 @@
+//
+//Copyright (C) 2016 Google, Inc.
+//
+//All rights reserved.
+//
+//Redistribution and use in source and binary forms, with or without
+//modification, are permitted provided that the following conditions
+//are met:
+//
+//    Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//
+//    Redistributions in binary form must reproduce the above
+//    copyright notice, this list of conditions and the following
+//    disclaimer in the documentation and/or other materials provided
+//    with the distribution.
+//
+//    Neither the name of Google, Inc., nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+//THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+//"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+//LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+//FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+//COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+//INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+//BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+//LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+//CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+//LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+//ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+//POSSIBILITY OF SUCH DAMAGE.
+//
+
+#include "hlslTokenStream.h"
+
+namespace glslang {
+
+// Load 'token' with the next token in the stream of tokens.
+void HlslTokenStream::advanceToken()
+{
+    scanner.tokenize(token);
+}
+
+// Return the current token class.
+EHlslTokenClass HlslTokenStream::peek() const
+{
+    return token.tokenClass;
+}
+
+// Return true, without advancing to the next token, if the current token is
+// the expected (passed in) token class.
+bool HlslTokenStream::peekTokenClass(EHlslTokenClass tokenClass) const
+{
+    return peek() == tokenClass;
+}
+
+// Return true and advance to the next token if the current token is the
+// expected (passed in) token class.
+bool HlslTokenStream::acceptTokenClass(EHlslTokenClass tokenClass)
+{
+    if (peekTokenClass(tokenClass)) {
+        advanceToken();
+        return true;
+    }
+
+    return false;
+}
+
+} // end namespace glslang

hlsl/hlslTokenStream.h

@@ -0,0 +1,64 @@
+//
+//Copyright (C) 2016 Google, Inc.
+//
+//All rights reserved.
+//
+//Redistribution and use in source and binary forms, with or without
+//modification, are permitted provided that the following conditions
+//are met:
+//
+//    Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+//
+//    Redistributions in binary form must reproduce the above
+//    copyright notice, this list of conditions and the following
+//    disclaimer in the documentation and/or other materials provided
+//    with the distribution.
+//
+//    Neither the name of Google, Inc., nor the names of its
+//    contributors may be used to endorse or promote products derived
+//    from this software without specific prior written permission.
+//
+//THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+//"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+//LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+//FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+//COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+//INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+//BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+//LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+//CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+//LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+//ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+//POSSIBILITY OF SUCH DAMAGE.
+//
+
+#ifndef HLSLTOKENSTREAM_H_
+#define HLSLTOKENSTREAM_H_
+
+#include "hlslScanContext.h"
+
+namespace glslang {
+
+    class HlslTokenStream {
+    public:
+        HlslTokenStream(HlslScanContext& scanner)
+            : scanner(scanner) { }
+        virtual ~HlslTokenStream() { }
+
+    public:
+        void advanceToken();
+        bool acceptTokenClass(EHlslTokenClass);
+        EHlslTokenClass peek() const;
+        bool peekTokenClass(EHlslTokenClass) const;
+
+    protected:
+        HlslToken token;                 // the current token we are processing
+    
+    private:
+        HlslScanContext& scanner;        // lexical scanner, to get next token
+    };
+
+} // end namespace glslang
+
+#endif // HLSLTOKENSTREAM_H_