From fcb4ed054cf4a02742e9659fef25a980aceb31f7 Mon Sep 17 00:00:00 2001 From: John Kessenich Date: Mon, 30 Dec 2013 20:34:28 +0000 Subject: [PATCH] Preprocessor: Rationalize, simplify, and correct the stack of input sources and their tokenization. This included - consistently dealing with EOF and its effect on error recovery (bug 11444, #1) - turning a simulated OO hierarchy of function pointers and typecasting into a real C++ class hierarchy - correctly handling '\' everywhere, in all classes of tokens, as a layer before preprocessing itself - conditionally handling '\n' in macro expansion input, depending on whether it is in a macro expression or not - delete some unused "memory cleanup" code git-svn-id: https://cvs.khronos.org/svn/repos/ogl/trunk/ecosystem/public/sdk/tools/glslang@24626 e7fa87d3-cd2b-0410-9028-fcbf551c1848 --- Test/baseResults/cppComplexExpr.vert.out | 22 +- Test/baseResults/cppIndent.vert.out | 6 +- Test/baseResults/cppNest.vert.out | 9 +- Test/baseResults/cppSimple.vert.out | 23 +- Test/baseResults/lineContinuation.vert.out | 100 +++++- Test/cppIndent.vert | 4 + Test/cppNest.vert | 5 + Test/cppSimple.vert | 20 ++ Test/lineContinuation.vert | 93 ++++- glslang/Include/revision.h | 4 +- .../MachineIndependent/preprocessor/Pp.cpp | 278 +++++++-------- .../preprocessor/PpContext.cpp | 17 +- .../preprocessor/PpContext.h | 142 ++++++-- .../preprocessor/PpMemory.cpp | 38 +-- .../preprocessor/PpScanner.cpp | 323 +++++++++--------- .../preprocessor/PpTokens.cpp | 145 ++++---- 16 files changed, 750 insertions(+), 479 deletions(-) diff --git a/Test/baseResults/cppComplexExpr.vert.out b/Test/baseResults/cppComplexExpr.vert.out index 11e7c4e97..4671f207d 100644 --- a/Test/baseResults/cppComplexExpr.vert.out +++ b/Test/baseResults/cppComplexExpr.vert.out @@ -5,14 +5,22 @@ ERROR: 0:66: '#define' : Macro redefined; different substitutions: BIG ERROR: 0:81: 'preprocessor evaluation' : bad expression ERROR: 0:81: '#if' : unexpected tokens following directive ERROR: 0:82: '#error' : good macro -ERROR: 0:89: 'macro expansion' : unexpected '#' foobar -ERROR: 0:90: 'preprocessor evaluation' : bad expression -ERROR: 0:94: 'macro expansion' : unexpected '#' foobar -ERROR: 0:95: 'preprocessor evaluation' : bad expression +ERROR: 0:87: 'macro expansion' : end of line in macro substitution: foobar +ERROR: 0:88: 'preprocessor evaluation' : can't evaluate expression +ERROR: 0:88: 'preprocessor evaluation' : bad expression +ERROR: 0:88: '#if' : unexpected tokens following directive +ERROR: 0:92: 'macro expansion' : end of line in macro substitution: foobar +ERROR: 0:93: 'preprocessor evaluation' : can't evaluate expression +ERROR: 0:93: 'preprocessor evaluation' : bad expression +ERROR: 0:93: '#if' : unexpected tokens following directive +ERROR: 0:99: 'macro expansion' : end of line in macro substitution: foobar +ERROR: 0:100: 'preprocessor evaluation' : can't evaluate expression ERROR: 0:100: 'preprocessor evaluation' : bad expression ERROR: 0:100: '#if' : unexpected tokens following directive -ERROR: 0:102: 'macro expansion' : unexpected '#' foobar -ERROR: 0:102: 'preprocessor evaluation' : undefined macro in expression not allowed in es profile endif +ERROR: 0:101: 'macro expansion' : end of line in macro substitution: foobar +ERROR: 0:102: 'preprocessor evaluation' : can't evaluate expression +ERROR: 0:102: 'preprocessor evaluation' : bad expression +ERROR: 0:102: '#if' : unexpected tokens following directive ERROR: 0:108: 'preprocessor evaluation' : undefined macro in expression not allowed in es profile UNDEF ERROR: 0:111: '#error' : good 0 ERROR: 0:115: '#error' : good 1 @@ -26,7 +34,7 @@ ERROR: 0:153: 'preprocessor evaluation' : undefined macro in expression not allo ERROR: 0:156: 'preprocessor evaluation' : undefined macro in expression not allowed in es profile UNDEF2 ERROR: 0:159: 'preprocessor evaluation' : undefined macro in expression not allowed in es profile UNDEF2 ERROR: 0:10002: '' : missing #endif -ERROR: 27 compilation errors. No code generated. +ERROR: 35 compilation errors. No code generated. ERROR: node is still EOpNull! diff --git a/Test/baseResults/cppIndent.vert.out b/Test/baseResults/cppIndent.vert.out index f6d8d0f70..d2d43fb74 100644 --- a/Test/baseResults/cppIndent.vert.out +++ b/Test/baseResults/cppIndent.vert.out @@ -1,6 +1,10 @@ cppIndent.vert +ERROR: 0:61: 'macro expansion' : Too few args in Macro FUNC +ERROR: 0:61: '' : syntax error +ERROR: 2 compilation errors. No code generated. -0:? Sequence + +ERROR: node is still EOpNull! 0:5 Sequence 0:5 move second child to first child (float) 0:5 'sum' (float) diff --git a/Test/baseResults/cppNest.vert.out b/Test/baseResults/cppNest.vert.out index 641e2ce5e..3a1d03596 100644 --- a/Test/baseResults/cppNest.vert.out +++ b/Test/baseResults/cppNest.vert.out @@ -3,7 +3,9 @@ ERROR: 0:144: '#elif' : #elif after #else ERROR: 0:152: '#else' : #else after #else ERROR: 0:161: '#elif' : #elif after #else ERROR: 0:169: '#else' : #else after #else -ERROR: 4 compilation errors. No code generated. +ERROR: 0:177: 'macro expansion' : EOF in macro FUNC +ERROR: 0:178: '' : syntax error +ERROR: 6 compilation errors. No code generated. ERROR: node is still EOpNull! @@ -58,6 +60,11 @@ ERROR: node is still EOpNull! 0:133 'selected3' (int) 0:133 Constant: 0:133 3 (const int) +0:175 Function Definition: foo985( (void) +0:175 Function Parameters: +0:175 Sequence +0:175 Constant: +0:175 6 (const int) 0:? Linker Objects 0:? 'sum' (float) 0:? 'selected4' (int) diff --git a/Test/baseResults/cppSimple.vert.out b/Test/baseResults/cppSimple.vert.out index 2f542d637..bcd1e7781 100644 --- a/Test/baseResults/cppSimple.vert.out +++ b/Test/baseResults/cppSimple.vert.out @@ -6,15 +6,15 @@ ERROR: 0:85: '#error' : good3 ERROR: 0:89: '#error' : good4 ERROR: 0:93: '#error' : good5 ERROR: 0:97: '#error' : good6 -ERROR: 0:101: 'preprocessor evaluation' : expected ')' +ERROR: 0:100: 'preprocessor evaluation' : expected ')' ERROR: 0:101: '#error' : bad1 ERROR: 0:104: '#if' : unexpected tokens following directive ERROR: 0:105: '#error' : bad2 -ERROR: 0:109: 'preprocessor evaluation' : expected ')' +ERROR: 0:108: 'preprocessor evaluation' : expected ')' ERROR: 0:109: '#error' : bad3 ERROR: 0:112: '#if' : unexpected tokens following directive ERROR: 0:113: '#error' : bad4 -ERROR: 0:117: 'preprocessor evaluation' : expected ')' +ERROR: 0:116: 'preprocessor evaluation' : expected ')' ERROR: 0:117: '#error' : bad5 ERROR: 0:120: '#if' : unexpected tokens following directive ERROR: 0:121: '#error' : bad6 @@ -74,8 +74,16 @@ ERROR: 12:20021: '#error' : line should be 20021 ERROR: 12:20046: '#define' : Macro redefined; different substitutions: SPACE_IN_MIDDLE ERROR: 12:20052: '#error' : good evaluation 1 ERROR: 12:20056: '#error' : good evaluation 2 +ERROR: 12:9001: 'preprocessor evaluation' : expected ')' +ERROR: 12:9003: '#if' : unexpected tokens following directive +ERROR: 12:9015: 'macro expansion' : expected '(' following FOOOM +ERROR: 12:9015: 'FOOOM' : undeclared identifier +ERROR: 12:9015: '=' : cannot convert from 'float' to 'int' +ERROR: 12:9016: 'macro expansion' : expected '(' following FOOOM +ERROR: 12:9017: 'preprocessor evaluation' : can't evaluate expression +ERROR: 12:9017: 'preprocessor evaluation' : bad expression ERROR: 12:10003: '' : missing #endif -ERROR: 75 compilation errors. No code generated. +ERROR: 83 compilation errors. No code generated. ERROR: node is still EOpNull! @@ -165,6 +173,10 @@ ERROR: node is still EOpNull! 12:20034 6.000000 12:20034 6.000000 12:20034 6.000000 +12:9012 Sequence +12:9012 move second child to first child (int) +12:9012 'R1' (int) +12:9012 'RECURSE' (int) 0:? Linker Objects 0:? 'sum' (float) 0:? 'linenumber' (int) @@ -174,6 +186,9 @@ ERROR: node is still EOpNull! 0:? 'a' (int) 0:? 'n' (int) 0:? 'f' (double) +0:? 'RECURSE' (int) +0:? 'R1' (int) +0:? 'aoeua' (int) 0:? 'gl_VertexID' (gl_VertexId int) 0:? 'gl_InstanceID' (gl_InstanceId int) diff --git a/Test/baseResults/lineContinuation.vert.out b/Test/baseResults/lineContinuation.vert.out index c41ee1b99..7f64cf14d 100644 --- a/Test/baseResults/lineContinuation.vert.out +++ b/Test/baseResults/lineContinuation.vert.out @@ -12,7 +12,15 @@ ERROR: 0:48: '\' : illegal use of escape character ERROR: 0:49: '$' : unexpected token ERROR: 0:50: '@' : unexpected token ERROR: 0:55: '#error' : good continuation -ERROR: 12 compilation errors. No code generated. +WARNING: 0:62: 'line continuation' : used at end of comment; the following line is still part of the comment +ERROR: 0:111: 'macro expansion' : end of line in macro substitution: FOOM +ERROR: 0:112: 'preprocessor evaluation' : can't evaluate expression +ERROR: 0:112: '#if' : unexpected tokens following directive +ERROR: 0:117: 'macro expansion' : end of line in macro substitution: FOOM +ERROR: 0:118: 'preprocessor evaluation' : can't evaluate expression +ERROR: 0:118: '#if' : unexpected tokens following directive +ERROR: 0:147: '' : syntax error +ERROR: 19 compilation errors. No code generated. ERROR: node is still EOpNull! @@ -72,6 +80,84 @@ ERROR: node is still EOpNull! 0:50 'q4' (highp int) 0:50 Constant: 0:50 1 (const int) +0:65 Sequence +0:65 move second child to first child (highp int) +0:65 'abdece' (highp int) +0:65 Constant: +0:65 10 (const int) +0:66 Sequence +0:66 move second child to first child (highp int) +0:66 'aoeuntaoehu' (highp int) +0:66 'abdece' (highp int) +0:74 Sequence +0:74 move second child to first child (highp float) +0:74 'funkyf' (highp float) +0:75 Constant: +0:75 12300000000000000.000000 +0:85 Sequence +0:84 move second child to first child (highp int) +0:84 'funkyh' (highp int) +0:86 Constant: +0:86 244 (const int) +0:91 Sequence +0:91 move second child to first child (highp int) +0:91 'funkyo' (highp int) +0:92 Constant: +0:92 34 (const int) +0:96 Sequence +0:96 move second child to first child (highp int) +0:96 'c' (highp int) +0:97 Constant: +0:97 11 (const int) +0:98 Sequence +0:98 move second child to first child (highp int) +0:98 'd' (highp int) +0:98 Constant: +0:98 12 (const int) +0:107 Sequence +0:107 move second child to first child (highp int) +0:107 'bar103' (highp int) +0:107 Constant: +0:107 17 (const int) +0:113 Sequence +0:113 move second child to first child (highp int) +0:113 'bar104' (highp int) +0:113 Constant: +0:113 19 (const int) +0:119 Sequence +0:119 move second child to first child (highp int) +0:119 'bar105' (highp int) +0:119 Constant: +0:119 19 (const int) +0:122 Sequence +0:122 move second child to first child (highp int) +0:122 'bar106' (highp int) +0:122 Constant: +0:122 12 (const int) +0:123 Sequence +0:123 move second child to first child (highp int) +0:123 'bar107' (highp int) +0:128 Constant: +0:128 5 (const int) +0:131 Function Definition: foo203209409( (void) +0:131 Function Parameters: +0:134 Sequence +0:134 add second child into first child (highp int) +0:133 'bar107' (highp int) +0:134 Constant: +0:134 37 (const int) +0:135 multiply second child into first child (highp int) +0:135 'bar107' (highp int) +0:136 Constant: +0:136 38 (const int) +0:137 divide second child into first child (highp int) +0:137 'bar107' (highp int) +0:138 Constant: +0:138 39 (const int) +0:139 add (highp int) +0:139 'bar107' (highp int) +0:140 Constant: +0:140 41 (const int) 0:? Linker Objects 0:? 'foo' (highp float) 0:? 'goodDecl' (highp int) @@ -83,6 +169,18 @@ ERROR: node is still EOpNull! 0:? 'q2' (highp int) 0:? 'q3' (highp int) 0:? 'q4' (highp int) +0:? 'abdece' (highp int) +0:? 'aoeuntaoehu' (highp int) +0:? 'funkyf' (highp float) +0:? 'funkyh' (highp int) +0:? 'funkyo' (highp int) +0:? 'c' (highp int) +0:? 'd' (highp int) +0:? 'bar103' (highp int) +0:? 'bar104' (highp int) +0:? 'bar105' (highp int) +0:? 'bar106' (highp int) +0:? 'bar107' (highp int) 0:? 'gl_VertexID' (gl_VertexId highp int) 0:? 'gl_InstanceID' (gl_InstanceId highp int) diff --git a/Test/cppIndent.vert b/Test/cppIndent.vert index 078776a3b..49ec8ba3d 100644 --- a/Test/cppIndent.vert +++ b/Test/cppIndent.vert @@ -55,3 +55,7 @@ sum += 900000000.0; // sum should be 980600301.0 gl_Position = vec4(sum); } + +#define FUNC(a,b) a+b +// needs to be last test in file due to syntax error +void foo986(){ FUNC( (((2)))), 4); } // ERROR, too many ) diff --git a/Test/cppNest.vert b/Test/cppNest.vert index c4bcb70e0..26038ea96 100644 --- a/Test/cppNest.vert +++ b/Test/cppNest.vert @@ -170,3 +170,8 @@ int; int; #endif #endif + +#define FUNC(a,b) a+b +void foo985(){ FUNC( (((2))), ((3),4)); } +// needs to be last test in file +void foo987(){ FUNC(((); } // ERROR, EOF in argument diff --git a/Test/cppSimple.vert b/Test/cppSimple.vert index 006d9ed7e..bb79568ae 100644 --- a/Test/cppSimple.vert +++ b/Test/cppSimple.vert @@ -306,6 +306,26 @@ void foo234() #error good evaluation 2 #endif +// ERRORS... +#line 9000 +#if defined(OUNH +#endif +#if defined OUNH) +#endif + +// recursion (okay) +#define RECURSE RECURSE +int RECURSE; +#define R2 R1 +#define R1 R2 +#undef RECURSE +int R1 = RECURSE; + +#define FOOOM(a,b) a + b +int aoeua = FOOOM; +#if FOOOM +#endif + #line 10000 #if 1 #else diff --git a/Test/lineContinuation.vert b/Test/lineContinuation.vert index 7c0e29b13..42affcf28 100644 --- a/Test/lineContinuation.vert +++ b/Test/lineContinuation.vert @@ -54,4 +54,95 @@ D; # \ error good continuation -#define AA a \ b +#define AA1 a \ b +#define AA2 a \\ b +#define AA3 a \\\ b +#define AA4 a \\\\ b + +// anoetuh nonaetu \\\\\\ +still in comment + +int abdece = 10; +int aoeuntaoehu = abd\ +\ +\ +\ +\ +\ +ece; + +float funkyf = \ +.\ +1\ +2\ +3\ +e\ ++\ +1\ +7\ +;\ +int funkyh\ +=\ +0\ +x\ +f\ +4\ +; +int funkyo =\ +0\ +4\ +2\ +; +int c = \ +11; +int d = 1\ +2; + +#define FOOM(a,b) a + b + +#if FO\ +OM(2\ +,\ +3) +int bar103 = 17; +#endif + +// ERROR +#if FOOM(2, +3) +int bar104 = 19; +#endif + +// ERROR +#if FOOM( +2,3) +int bar105 = 19; +#endif + +int bar106 = FOOM(5,7); +int bar107 = FOOM // okay + ( + 2 + , + 3 + ) + ; + +void foo203209409() +{ + bar107 \ ++= 37; + bar107 *\ += 38; + bar107 /=\ +39; + bar107 +\ +41; +} + +void foo230920394() +{ + // syntax error + bar107 +\ + = 42; +} diff --git a/glslang/Include/revision.h b/glslang/Include/revision.h index 8a78e7f7c..ae1ff7f3b 100644 --- a/glslang/Include/revision.h +++ b/glslang/Include/revision.h @@ -9,5 +9,5 @@ // source have to figure out how to create revision.h just to get a build // going. However, if it is not updated, it can be a version behind. -#define GLSLANG_REVISION "24569" -#define GLSLANG_DATE "2013/12/18 11:47:12" +#define GLSLANG_REVISION "24594" +#define GLSLANG_DATE "2013/12/20 11:36:27" diff --git a/glslang/MachineIndependent/preprocessor/Pp.cpp b/glslang/MachineIndependent/preprocessor/Pp.cpp index 7cb5a38b6..754ca0b8f 100644 --- a/glslang/MachineIndependent/preprocessor/Pp.cpp +++ b/glslang/MachineIndependent/preprocessor/Pp.cpp @@ -132,7 +132,7 @@ int TPpContext::CPPdefine(TPpToken* ppToken) Symbol *symb; // get macro name - int token = currentInput->scan(this, currentInput, ppToken); + int token = scanToken(ppToken); if (token != CPP_IDENTIFIER) { parseContext.error(ppToken->loc, "must be followed by macro name", "#define", ""); return token; @@ -145,12 +145,12 @@ int TPpContext::CPPdefine(TPpToken* ppToken) } // gather parameters to the macro, between (...) - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); if (token == '(' && ! ppToken->space) { int argc = 0; int args[maxMacroArgs]; do { - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); if (argc == 0 && token == ')') break; if (token != CPP_IDENTIFIER) { @@ -173,7 +173,7 @@ int TPpContext::CPPdefine(TPpToken* ppToken) else parseContext.error(ppToken->loc, "too many macro parameters", "#define", ""); } - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); } while (token == ','); if (token != ')') { parseContext.error(ppToken->loc, "missing parenthesis", "#define", ""); @@ -183,7 +183,7 @@ int TPpContext::CPPdefine(TPpToken* ppToken) mac.argc = argc; mac.args = (int*)mem_Alloc(pool, argc * sizeof(int)); memcpy(mac.args, args, argc * sizeof(int)); - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); } // record the definition of the macro @@ -191,7 +191,7 @@ int TPpContext::CPPdefine(TPpToken* ppToken) mac.body = new TokenStream; while (token != '\n') { RecordToken(mac.body, token, ppToken); - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); if (token != '\n' && ppToken->space) RecordToken(mac.body, ' ', ppToken); } @@ -238,7 +238,7 @@ int TPpContext::CPPdefine(TPpToken* ppToken) // Handle #undef int TPpContext::CPPundef(TPpToken* ppToken) { - int token = currentInput->scan(this, currentInput, ppToken); + int token = scanToken(ppToken); Symbol *symb; if (token != CPP_IDENTIFIER) { parseContext.error(ppToken->loc, "must be followed by macro name", "#undef", ""); @@ -246,14 +246,14 @@ int TPpContext::CPPundef(TPpToken* ppToken) return token; } - const char* name = GetAtomString(ppToken->atom); // TODO preprocessor simplification: the token text should have been built into the ppToken during currentInput->scan() + const char* name = GetAtomString(ppToken->atom); // TODO preprocessor simplification: the token text should have been built into the ppToken during scanToken() parseContext.reservedPpErrorCheck(ppToken->loc, name, "#undef"); symb = LookUpSymbol(ppToken->atom); if (symb) { symb->mac.undef = 1; } - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); if (token != '\n') parseContext.error(ppToken->loc, "can only be followed by a single macro name", "#undef", ""); @@ -269,21 +269,21 @@ int TPpContext::CPPelse(int matchelse, TPpToken* ppToken) { int atom; int depth = 0; - int token = currentInput->scan(this, currentInput, ppToken); + int token = scanToken(ppToken); while (token != EOF) { if (token != '#') { while (token != '\n' && token != EOF) - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); if (token == EOF) return EOF; - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); continue; } - if ((token = currentInput->scan(this, currentInput, ppToken)) != CPP_IDENTIFIER) + if ((token = scanToken(ppToken)) != CPP_IDENTIFIER) continue; atom = ppToken->atom; @@ -292,7 +292,7 @@ int TPpContext::CPPelse(int matchelse, TPpToken* ppToken) ifdepth++; elsetracker++; } else if (atom == endifAtom) { - token = extraTokenCheck(atom, ppToken, currentInput->scan(this, currentInput, ppToken)); + token = extraTokenCheck(atom, ppToken, scanToken(ppToken)); elseSeen[elsetracker] = false; --elsetracker; if (depth == 0) { @@ -306,7 +306,7 @@ int TPpContext::CPPelse(int matchelse, TPpToken* ppToken) } else if (matchelse && depth == 0) { if (atom == elseAtom) { elseSeen[elsetracker] = true; - token = extraTokenCheck(atom, ppToken, currentInput->scan(this, currentInput, ppToken)); + token = extraTokenCheck(atom, ppToken, scanToken(ppToken)); // found the #else we are looking for break; } else if (atom == elifAtom) { @@ -327,7 +327,7 @@ int TPpContext::CPPelse(int matchelse, TPpToken* ppToken) parseContext.error(ppToken->loc, "#else after #else", "#else", ""); else elseSeen[elsetracker] = true; - token = extraTokenCheck(atom, ppToken, currentInput->scan(this, currentInput, ppToken)); + token = extraTokenCheck(atom, ppToken, scanToken(ppToken)); } else if (atom == elifAtom) { if (elseSeen[elsetracker]) parseContext.error(ppToken->loc, "#elif after #else", "#elif", ""); @@ -363,7 +363,7 @@ int TPpContext::extraTokenCheck(int atom, TPpToken* ppToken, int token) parseContext.error(ppToken->loc, message, label, ""); while (token != '\n') - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); } return token; @@ -438,16 +438,17 @@ struct TUnop { int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, bool& err, TPpToken* ppToken) { + TSourceLoc loc = ppToken->loc; // because we sometimes read the newline before reporting the error if (token == CPP_IDENTIFIER) { if (ppToken->atom == definedAtom) { bool needclose = 0; - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); if (token == '(') { needclose = true; - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); } if (token != CPP_IDENTIFIER) { - parseContext.error(ppToken->loc, "incorrect directive, expected identifier", "preprocessor evaluation", ""); + parseContext.error(loc, "incorrect directive, expected identifier", "preprocessor evaluation", ""); err = true; res = 0; @@ -455,16 +456,16 @@ int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, boo } Symbol* s; res = (s = LookUpSymbol(ppToken->atom)) ? !s->mac.undef : 0; - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); if (needclose) { if (token != ')') { - parseContext.error(ppToken->loc, "#else after #else", "preprocessor evaluation", ""); + parseContext.error(loc, "expected ')'", "preprocessor evaluation", ""); err = true; res = 0; return token; } - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); } } else { token = evalToToken(token, shortCircuit, res, err, ppToken); @@ -472,19 +473,19 @@ int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, boo } } else if (token == CPP_INTCONSTANT) { res = ppToken->ival; - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); } else if (token == '(') { - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); token = eval(token, MIN_PRECEDENCE, shortCircuit, res, err, ppToken); if (! err) { if (token != ')') { - parseContext.error(ppToken->loc, "expected ')'", "preprocessor evaluation", ""); + parseContext.error(loc, "expected ')'", "preprocessor evaluation", ""); err = true; res = 0; return token; } - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); } } else { int op; @@ -493,11 +494,11 @@ int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, boo break; } if (op >= 0) { - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); token = eval(token, UNARY, shortCircuit, res, err, ppToken); res = unop[op].op(res); } else { - parseContext.error(ppToken->loc, "bad expression", "preprocessor evaluation", ""); + parseContext.error(loc, "bad expression", "preprocessor evaluation", ""); err = true; res = 0; @@ -528,7 +529,7 @@ int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, boo shortCircuit = true; } - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); token = eval(token, binop[op].precedence, shortCircuit, res, err, ppToken); res = binop[op].op(leftSide, res); } @@ -540,11 +541,12 @@ int TPpContext::eval(int token, int precedence, bool shortCircuit, int& res, boo int TPpContext::evalToToken(int token, bool shortCircuit, int& res, bool& err, TPpToken* ppToken) { while (token == CPP_IDENTIFIER && ppToken->atom != definedAtom) { - int macroReturn = MacroExpand(ppToken->atom, ppToken, true); + int macroReturn = MacroExpand(ppToken->atom, ppToken, true, false); if (macroReturn == 0) { parseContext.error(ppToken->loc, "can't evaluate expression", "preprocessor evaluation", ""); err = true; res = 0; + token = scanToken(ppToken); break; } if (macroReturn == -1) { @@ -557,7 +559,7 @@ int TPpContext::evalToToken(int token, bool shortCircuit, int& res, bool& err, T parseContext.error(ppToken->loc, message, "preprocessor evaluation", name); } } - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); } return token; @@ -566,7 +568,7 @@ int TPpContext::evalToToken(int token, bool shortCircuit, int& res, bool& err, T // Handle #if int TPpContext::CPPif(TPpToken* ppToken) { - int token = currentInput->scan(this, currentInput, ppToken); + int token = scanToken(ppToken); elsetracker++; if (! ifdepth++) ifloc = ppToken->loc; @@ -587,7 +589,7 @@ int TPpContext::CPPif(TPpToken* ppToken) // Handle #ifdef int TPpContext::CPPifdef(int defined, TPpToken* ppToken) { - int token = currentInput->scan(this, currentInput, ppToken); + int token = scanToken(ppToken); int name = ppToken->atom; if (++ifdepth > maxIfNesting) { parseContext.error(ppToken->loc, "maximum nesting depth exceeded", "#ifdef", ""); @@ -601,11 +603,11 @@ int TPpContext::CPPifdef(int defined, TPpToken* ppToken) parseContext.error(ppToken->loc, "must be followed by macro name", "#ifndef", ""); } else { Symbol *s = LookUpSymbol(name); - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); if (token != '\n') { parseContext.error(ppToken->loc, "unexpected tokens following #ifdef directive - expected a newline", "#ifdef", ""); while (token != '\n') - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); } if (((s && !s->mac.undef) ? 1 : 0) != defined) token = CPPelse(1, ppToken); @@ -621,7 +623,7 @@ int TPpContext::CPPline(TPpToken* ppToken) // "#line line // "#line line source-string-number" - int token = currentInput->scan(this, currentInput, ppToken); + int token = scanToken(ppToken); if (token == '\n') { parseContext.error(ppToken->loc, "must by followed by an integral literal", "#line", ""); return token; @@ -651,7 +653,7 @@ int TPpContext::CPPline(TPpToken* ppToken) // Handle #error int TPpContext::CPPerror(TPpToken* ppToken) { - int token = currentInput->scan(this, currentInput, ppToken); + int token = scanToken(ppToken); std::string message; TSourceLoc loc = ppToken->loc; @@ -665,7 +667,7 @@ int TPpContext::CPPerror(TPpToken* ppToken) message.append(GetAtomString(token)); } message.append(" "); - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); } //store this msg into the shader's information log..set the Compile Error flag!!!! parseContext.error(loc, message.c_str(), "#error", ""); @@ -683,7 +685,7 @@ int TPpContext::CPPpragma(TPpToken* ppToken) TVector tokens; TSourceLoc loc = ppToken->loc; // because we go to the next line before processing - int token = currentInput->scan(this, currentInput, ppToken); + int token = scanToken(ppToken); while (token != '\n' && token != EOF) { switch (token) { case CPP_IDENTIFIER: @@ -702,7 +704,7 @@ int TPpContext::CPPpragma(TPpToken* ppToken) SrcStrName[1] = '\0'; tokens.push_back(SrcStrName); } - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); } if (token == EOF) @@ -716,7 +718,7 @@ int TPpContext::CPPpragma(TPpToken* ppToken) // #version: This is just for error checking: the version and profile are decided before preprocessing starts int TPpContext::CPPversion(TPpToken* ppToken) { - int token = currentInput->scan(this, currentInput, ppToken); + int token = scanToken(ppToken); if (errorOnVersion || versionSeen) parseContext.error(ppToken->loc, "must occur first in shader", "#version", ""); @@ -733,7 +735,7 @@ int TPpContext::CPPversion(TPpToken* ppToken) ppToken->ival = atoi(ppToken->name); - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); if (token == '\n') return token; @@ -743,7 +745,7 @@ int TPpContext::CPPversion(TPpToken* ppToken) ppToken->atom != esAtom) parseContext.error(ppToken->loc, "bad profile name; use es, core, or compatibility", "#version", ""); - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); if (token == '\n') return token; @@ -757,7 +759,7 @@ int TPpContext::CPPversion(TPpToken* ppToken) // Handle #extension int TPpContext::CPPextension(TPpToken* ppToken) { - int token = currentInput->scan(this, currentInput, ppToken); + int token = scanToken(ppToken); char extensionName[80]; if (token=='\n') { @@ -770,13 +772,13 @@ int TPpContext::CPPextension(TPpToken* ppToken) strcpy(extensionName, GetAtomString(ppToken->atom)); - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); if (token != ':') { parseContext.error(ppToken->loc, "':' missing after extension name", "#extension", ""); return token; } - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); if (token != CPP_IDENTIFIER) { parseContext.error(ppToken->loc, "behavior for extension not specified", "#extension", ""); return token; @@ -784,7 +786,7 @@ int TPpContext::CPPextension(TPpToken* ppToken) parseContext.updateExtensionBehavior(extensionName, GetAtomString(ppToken->atom)); - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); if (token == '\n') return token; else @@ -795,7 +797,7 @@ int TPpContext::CPPextension(TPpToken* ppToken) int TPpContext::readCPPline(TPpToken* ppToken) { - int token = currentInput->scan(this, currentInput, ppToken); + int token = scanToken(ppToken); bool isVersion = false; if (token == CPP_IDENTIFIER) { @@ -807,7 +809,7 @@ int TPpContext::readCPPline(TPpToken* ppToken) elsetracker[elseSeen] = true; if (! ifdepth) parseContext.error(ppToken->loc, "mismatched statements", "#else", ""); - token = extraTokenCheck(elseAtom, ppToken, currentInput->scan(this, currentInput, ppToken)); + token = extraTokenCheck(elseAtom, ppToken, scanToken(ppToken)); token = CPPelse(0, ppToken); } else if (ppToken->atom == elifAtom) { if (! ifdepth) @@ -815,9 +817,9 @@ int TPpContext::readCPPline(TPpToken* ppToken) if (elseSeen[elsetracker]) parseContext.error(ppToken->loc, "#elif after #else", "#elif", ""); // this token is really a dont care, but we still need to eat the tokens - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); while (token != '\n') - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); token = CPPelse(0, ppToken); } else if (ppToken->atom == endifAtom) { elseSeen[elsetracker] = false; @@ -826,7 +828,7 @@ int TPpContext::readCPPline(TPpToken* ppToken) parseContext.error(ppToken->loc, "mismatched statements", "#endif", ""); else --ifdepth; - token = extraTokenCheck(endifAtom, ppToken, currentInput->scan(this, currentInput, ppToken)); + token = extraTokenCheck(endifAtom, ppToken, scanToken(ppToken)); } else if (ppToken->atom == ifAtom) { token = CPPif (ppToken); } else if (ppToken->atom == ifdefAtom) { @@ -849,38 +851,16 @@ int TPpContext::readCPPline(TPpToken* ppToken) } else { parseContext.error(ppToken->loc, "invalid directive:", "#", GetAtomString(ppToken->atom)); } - } else if (token != '\n' && token > 0) + } else if (token != '\n' && token != EOF) parseContext.error(ppToken->loc, "invalid directive", "#", ""); while (token != '\n' && token != 0 && token != EOF) - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); return token; } -int eof_scan(TPpContext*, TPpContext::InputSrc* in, TPpToken* ppToken) { return -1; } -void noop(TPpContext*, TPpContext::InputSrc* in, int ch, TPpToken* ppToken) { } - -void TPpContext::PushEofSrc() -{ - InputSrc *in = new InputSrc; - in->scan = eof_scan; - in->getch = eof_scan; - in->ungetch = noop; - in->prev = currentInput; - currentInput = in; -} - -void TPpContext::PopEofSrc() -{ - if (currentInput->scan == eof_scan) { - InputSrc *in = currentInput; - currentInput = in->prev; - delete in; - } -} - -TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream *a, TPpToken* ppToken) +TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* a, TPpToken* ppToken, bool newLineOkay) { int token; TokenStream *n; @@ -889,89 +869,79 @@ TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream *a, TPpToken* p token = ReadToken(a, ppToken); if (token == CPP_IDENTIFIER && LookUpSymbol(ppToken->atom)) break; - } while (token != EOF); - if (token == EOF) + } while (token != tInput::endOfInput); + + if (token == tInput::endOfInput) return a; + n = new TokenStream; - PushEofSrc(); - ReadFromTokenStream(a, 0, 0); - while ((token = currentInput->scan(this, currentInput, ppToken)) > 0) { - if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, false) == 1) + pushInput(new tMarkerInput(this)); + pushTokenStreamInput(a, 0); + while ((token = scanToken(ppToken)) != tMarkerInput::marker) { + if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, false, newLineOkay) != 0) continue; RecordToken(n, token, ppToken); } - PopEofSrc(); + popInput(); delete a; return n; } +// +// Return the next token for a macro expansion, handling macro args. // -// These are called through function pointers -// - -/* -** return the next token for a macro expansion, handling macro args -*/ -int TPpContext::macro_scan(TPpContext* pp, InputSrc* inInput, TPpToken* ppToken) +int TPpContext::tMacroInput::scan(TPpToken* ppToken) { - MacroInputSrc* in = (TPpContext::MacroInputSrc*)inInput; - - int i; int token; do { - token = pp->ReadToken(in->mac->body, ppToken); + token = pp->ReadToken(mac->body, ppToken); } while (token == ' '); // handle white space in macro + // TODO: preprocessor: properly handle whitespace (or lack of it) between tokens when expanding if (token == CPP_IDENTIFIER) { - for (i = in->mac->argc-1; i>=0; i--) - if (in->mac->args[i] == ppToken->atom) + int i; + for (i = mac->argc - 1; i >= 0; i--) + if (mac->args[i] == ppToken->atom) break; if (i >= 0) { - pp->ReadFromTokenStream(in->args[i], ppToken->atom, 0); + pp->pushTokenStreamInput(args[i], ppToken->atom); - return pp->currentInput->scan(pp, pp->currentInput, ppToken); + return pp->scanToken(ppToken); } } - if (token != EOF) - return token; - - in->mac->busy = 0; - pp->currentInput = in->prev; - delete in; - - return pp->currentInput->scan(pp, pp->currentInput, ppToken); + if (token == endOfInput) + mac->busy = 0; + + return token; } // return a zero, for scanning a macro that was never defined -int TPpContext::zero_scan(TPpContext* pp, InputSrc *inInput, TPpToken* ppToken) +int TPpContext::tZeroInput::scan(TPpToken* ppToken) { - MacroInputSrc* in = (MacroInputSrc*)inInput; + if (done) + return endOfInput; strcpy(ppToken->name, "0"); ppToken->ival = 0; ppToken->space = false; - - // pop input - pp->currentInput = in->prev; - delete in; + done = true; return CPP_INTCONSTANT; } -/* -** Check an identifier (atom) to see if it is a macro that should be expanded. -** If it is, and defined, push an InputSrc that will produce the appropriate expansion -** and return 1. -** If it is, but undefined, and expandUndef is requested, push an InputSrc that will -** expand to 0 and return -1. -** Otherwise, return 0. -*/ -int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef) +// +// Check an identifier (atom) to see if it is a macro that should be expanded. +// If it is, and defined, push an tInput that will produce the appropriate expansion +// and return 1. +// If it is, but undefined, and expandUndef is requested, push an tInput that will +// expand to 0 and return -1. +// Otherwise, return 0 to indicate no expansion, which is not necessarily an error. +// +int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool newLineOkay) { Symbol *sym = LookUpSymbol(atom); - MacroInputSrc *in; int token; int depth = 0; @@ -1004,30 +974,32 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef) if (sym && sym->mac.busy) return 0; - // not expanding of undefined symbols + // not expanding undefined macros if ((! sym || sym->mac.undef) && ! expandUndef) return 0; - in = new MacroInputSrc; - + // 0 is the value of an undefined macro if ((! sym || sym->mac.undef) && expandUndef) { - // push input - in->scan = zero_scan; - in->prev = currentInput; - currentInput = in; - + pushInput(new tZeroInput(this)); return -1; } + tMacroInput *in = new tMacroInput(this); + TSourceLoc loc = ppToken->loc; // in case we go to the next line before discovering the error - in->scan = macro_scan; in->mac = &sym->mac; if (sym->mac.args) { - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); + if (newLineOkay) { + while (token == '\n') + token = scanToken(ppToken); + } if (token != '(') { + parseContext.error(loc, "expected '(' following", "macro expansion", GetAtomString(atom)); UngetToken(token, ppToken); ppToken->atom = atom; + delete in; return 0; } in->args.resize(in->mac->argc); @@ -1038,23 +1010,24 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef) do { depth = 0; while (1) { - token = currentInput->scan(this, currentInput, ppToken); - if (token <= 0) { + token = scanToken(ppToken); + if (token == EOF) { parseContext.error(loc, "EOF in macro", "macro expansion", GetAtomString(atom)); - return 1; + delete in; + return 0; } if (token == '\n') { - // TODO: Preprocessor functionality: Correctly handle new line and escaped new line, for expansions that are both in and not in another preprocessor directive - - //if (in a pp line) { - // parseContext.error(loc, "missing ')':", "macro expansion", GetAtomString(atom)); - // return 1; - //} + if (! newLineOkay) { + parseContext.error(loc, "end of line in macro substitution:", "macro expansion", GetAtomString(atom)); + delete in; + return 0; + } continue; } if (token == '#') { parseContext.error(ppToken->loc, "unexpected '#'", "macro expansion", GetAtomString(atom)); - return 1; + delete in; + return 0; } if (in->mac->argc == 0 && token != ')') break; @@ -1080,31 +1053,30 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef) parseContext.error(loc, "Too few args in Macro", "macro expansion", GetAtomString(atom)); else if (token != ')') { depth=0; - while (token >= 0 && (depth > 0 || token != ')')) { + while (token != EOF && (depth > 0 || token != ')')) { if (token == ')') depth--; - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); if (token == '(') depth++; } - if (token <= 0) { + if (token == EOF) { parseContext.error(loc, "EOF in macro", "macro expansion", GetAtomString(atom)); - return 1; + delete in; + return 0; } parseContext.error(loc, "Too many args in macro", "macro expansion", GetAtomString(atom)); } for (int i = 0; i < in->mac->argc; i++) - in->args[i] = PrescanMacroArg(in->args[i], ppToken); + in->args[i] = PrescanMacroArg(in->args[i], ppToken, newLineOkay); } - /*retain the input source*/ - in->prev = currentInput; + pushInput(in); sym->mac.busy = 1; RewindTokenStream(sym->mac.body); - currentInput = in; return 1; } -} // end namespace glslang \ No newline at end of file +} // end namespace glslang diff --git a/glslang/MachineIndependent/preprocessor/PpContext.cpp b/glslang/MachineIndependent/preprocessor/PpContext.cpp index f7b7a7517..862086ec6 100644 --- a/glslang/MachineIndependent/preprocessor/PpContext.cpp +++ b/glslang/MachineIndependent/preprocessor/PpContext.cpp @@ -84,7 +84,7 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace glslang { TPpContext::TPpContext(TParseContext& pc) : - preamble(0), strings(0), parseContext(pc) + preamble(0), strings(0), parseContext(pc), inComment(false) { InitAtomTable(); InitScanner(this); @@ -143,17 +143,18 @@ TPpContext::~TPpContext() delete it->second->mac.body; mem_FreePool(pool); delete [] preamble; + + // free up the inputStack + while (! inputStack.empty()) + popInput(); } void TPpContext::setInput(TInputScanner& input, bool versionWillBeError) { - StringInputSrc* in = new StringInputSrc; - in->input = &input; - in->scan = sourceScan; - in->getch = (int (*)(TPpContext*, InputSrc *, TPpToken *))sourceGetCh; - in->ungetch = (void (*)(TPpContext*, InputSrc *, int, TPpToken *))sourceUngetCh; - in->prev = currentInput; - currentInput = in; + assert(inputStack.size() == 0); + + pushInput(new tStringInput(this, input)); + errorOnVersion = versionWillBeError; versionSeen = false; } diff --git a/glslang/MachineIndependent/preprocessor/PpContext.h b/glslang/MachineIndependent/preprocessor/PpContext.h index 81dbe9c03..0c91ce65c 100644 --- a/glslang/MachineIndependent/preprocessor/PpContext.h +++ b/glslang/MachineIndependent/preprocessor/PpContext.h @@ -121,19 +121,37 @@ public: virtual ~TPpContext(); void setPreamble(const char* preamble, size_t length); - void setInput(TInputScanner& input, bool versionWillBeError); const char* tokenize(TPpToken* ppToken); - // TODO: preprocessor simplification: this should be a base class, not a set of function pointers - struct InputSrc { - InputSrc() : prev(0), scan(0), getch(0), ungetch(0) { } - struct InputSrc *prev; - int (*scan)(TPpContext*, struct InputSrc *, TPpToken *); - int (*getch)(TPpContext*, struct InputSrc *, TPpToken *); - void (*ungetch)(TPpContext*, struct InputSrc *, int, TPpToken *); + class tInput { + public: + tInput(TPpContext* p) : done(false), pp(p) { } + virtual ~tInput() { } + + virtual int scan(TPpToken*) = 0; + virtual int getch() = 0; + virtual void ungetch() = 0; + + static const int endOfInput = -2; + + protected: + bool done; + TPpContext* pp; }; + void setInput(TInputScanner& input, bool versionWillBeError); + + void TPpContext::pushInput(tInput* in) + { + inputStack.push_back(in); + } + void TPpContext::popInput() + { + delete inputStack.back(); + inputStack.pop_back(); + } + struct TokenStream { TokenStream() : current(0) { } TVector data; @@ -145,7 +163,6 @@ public: uintptr_t free, end; size_t chunksize; uintptr_t alignmask; - struct cleanup *cleanup; }; // @@ -184,10 +201,31 @@ protected: int currentString; // which string we're currently parsing (-1 for preamble) // Scanner data: - int mostRecentToken; // Most recent token seen by the scanner int previous_token; TParseContext& parseContext; + // Get the next token from *stack* of input sources, popping input sources + // that are out of tokens, down until an input sources is found that has a token. + // Return EOF when there are no more tokens to be found by doing this. + int scanToken(TPpToken* ppToken) + { + int token = EOF; + + while (! inputStack.empty()) { + token = inputStack.back()->scan(ppToken); + if (token != tInput::endOfInput) + break; + popInput(); + } + + if (token == tInput::endOfInput) + return EOF; + + return token; + } + int getChar() { return inputStack.back()->getch(); } + void ungetChar() { inputStack.back()->ungetch(); } + static const int maxMacroArgs = 64; static const int maxIfNesting = 64; @@ -196,18 +234,47 @@ protected: int elsetracker; // #if-#else and #endif constructs...Counter. const char *ErrMsg; - struct MacroInputSrc : public InputSrc { - MacroInputSrc() : mac(0) { } - virtual ~MacroInputSrc() + class tMacroInput : public tInput { + public: + tMacroInput(TPpContext* pp) : tInput(pp) { } + virtual ~tMacroInput() { for (size_t i = 0; i < args.size(); ++i) delete args[i]; } + + virtual int scan(TPpToken*); + virtual int getch() { assert(0); return endOfInput; } + virtual void ungetch() { assert(0); } MacroSymbol *mac; TVector args; }; - InputSrc *currentInput; + class tMarkerInput : public tInput { + public: + tMarkerInput(TPpContext* pp) : tInput(pp) { } + virtual int scan(TPpToken*) + { + if (done) + return endOfInput; + done = true; + + return marker; + } + virtual int getch() { assert(0); return endOfInput; } + virtual void ungetch() { assert(0); } + static const int marker = -3; + }; + + class tZeroInput : public tInput { + public: + tZeroInput(TPpContext* pp) : tInput(pp) { } + virtual int scan(TPpToken*); + virtual int getch() { assert(0); return endOfInput; } + virtual void ungetch() { assert(0); } + }; + + std::vector inputStack; bool errorOnVersion; bool versionSeen; @@ -256,12 +323,8 @@ protected: int CPPversion(TPpToken * ppToken); int CPPextension(TPpToken * ppToken); int readCPPline(TPpToken * ppToken); - void PushEofSrc(); - void PopEofSrc(); - TokenStream* PrescanMacroArg(TokenStream *a, TPpToken * ppToken); - static int macro_scan(TPpContext* pp, InputSrc *inInput, TPpToken * ppToken); - static int zero_scan(TPpContext* pp, InputSrc *inInput, TPpToken * ppToken); - int MacroExpand(int atom, TPpToken* ppToken, bool expandUndef); + TokenStream* PrescanMacroArg(TokenStream *a, TPpToken * ppToken, bool newLineOkay); + int MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool newLineOkay); // // from PpSymbols.cpp @@ -278,32 +341,49 @@ protected: void RecordToken(TokenStream* pTok, int token, TPpToken* ppToken); void RewindTokenStream(TokenStream *pTok); int ReadToken(TokenStream* pTok, TPpToken* ppToken); - int ReadFromTokenStream(TokenStream *ts, int name, int (*final)(TPpContext *)); + void pushTokenStreamInput(TokenStream *ts, int name); void UngetToken(int token, TPpToken* ppToken); - struct TokenInputSrc : public InputSrc { + + class tTokenInput : public tInput { + public: + tTokenInput(TPpContext* pp, TokenStream* t) : tInput(pp), tokens(t) { } + virtual int scan(TPpToken *); + virtual int getch() { assert(0); return endOfInput; } + virtual void ungetch() { assert(0); } + protected: TokenStream *tokens; - int (*final)(TPpContext *); }; - static int scan_token(TPpContext*, TokenInputSrc *in, TPpToken * ppToken); - struct UngotToken : public InputSrc { + + class tUngotTokenInput : public tInput { + public: + tUngotTokenInput(TPpContext* pp, int t, TPpToken* p) : tInput(pp), token(t), lval(*p) { } + virtual int scan(TPpToken *); + virtual int getch() { assert(0); return endOfInput; } + virtual void ungetch() { assert(0); } + protected: int token; TPpToken lval; }; - static int reget_token(TPpContext *, UngotToken *t, TPpToken * ppToken); // // From PpScanner.cpp // - struct StringInputSrc : public InputSrc { + class tStringInput : public tInput { + public: + tStringInput(TPpContext* pp, TInputScanner& i) : tInput(pp), input(&i) { } + virtual int scan(TPpToken *); + virtual int getch(); + virtual void ungetch(); + protected: TInputScanner* input; }; + int InitScanner(TPpContext *cpp); - static int sourceGetCh(TPpContext*, StringInputSrc *in); - static void sourceUngetCh(TPpContext*, StringInputSrc *in, int ch, TPpToken *type); int ScanFromString(char *s); - bool check_EOF(int token); + void missingEndifCheck(); int lFloatConst(char *str, int len, int ch, TPpToken * ppToken); - static int sourceScan(TPpContext*, InputSrc *in, TPpToken * ppToken); + + bool inComment; // // From PpAtom.cpp diff --git a/glslang/MachineIndependent/preprocessor/PpMemory.cpp b/glslang/MachineIndependent/preprocessor/PpMemory.cpp index d51147edf..b71d2cf50 100644 --- a/glslang/MachineIndependent/preprocessor/PpMemory.cpp +++ b/glslang/MachineIndependent/preprocessor/PpMemory.cpp @@ -75,7 +75,7 @@ NVIDIA SOFTWARE, HOWEVER CAUSED AND WHETHER UNDER THEORY OF CONTRACT, TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, EVEN IF NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \****************************************************************************/ -// + #include #include #include @@ -93,13 +93,6 @@ struct chunk { struct chunk *next; }; -struct cleanup { - struct cleanup *next; - void (*fn)(void *, void *); - void *arg1; - void *arg2; -}; - TPpContext::MemoryPool* TPpContext::mem_CreatePool(size_t chunksize, unsigned int align) { MemoryPool *pool; @@ -108,32 +101,28 @@ TPpContext::MemoryPool* TPpContext::mem_CreatePool(size_t chunksize, unsigned in align = ALIGN; if (chunksize == 0) chunksize = CHUNKSIZE; - if (align & (align-1)) + if (align & (align - 1)) return 0; if (chunksize < sizeof(MemoryPool)) return 0; - if (chunksize & (align-1)) + if (chunksize & (align - 1)) return 0; if (!(pool = (MemoryPool*)malloc(chunksize))) return 0; pool->next = 0; pool->chunksize = chunksize; - pool->alignmask = (uintptr_t)(align)-1; + pool->alignmask = (uintptr_t)(align) - 1; pool->free = ((uintptr_t)(pool + 1) + pool->alignmask) & ~pool->alignmask; pool->end = (uintptr_t)pool + chunksize; - pool->cleanup = 0; + return pool; } void TPpContext::mem_FreePool(MemoryPool *pool) { - struct cleanup *cleanup; - struct chunk *p, *next; + struct chunk *p, *next; - for (cleanup = pool->cleanup; cleanup; cleanup = cleanup->next) { - cleanup->fn(cleanup->arg1, cleanup->arg2); - } for (p = (struct chunk *)pool; p; p = next) { next = p->next; free(p); @@ -168,19 +157,4 @@ void* TPpContext::mem_Alloc(MemoryPool *pool, size_t size) return rv; } -int TPpContext::mem_AddCleanup(MemoryPool *pool, void (*fn)(void *, void*), void* arg1, void* arg2) -{ - struct cleanup *cleanup; - - pool->free = (pool->free + sizeof(void *) - 1) & ~(sizeof(void *)-1); - cleanup = (struct cleanup *)(mem_Alloc(pool, sizeof(struct cleanup))); - if (!cleanup) return -1; - cleanup->next = pool->cleanup; - cleanup->fn = fn; - cleanup->arg1 = arg1; - cleanup->arg2 = arg2; - pool->cleanup = cleanup; - return 0; -} - } // end namespace glslang diff --git a/glslang/MachineIndependent/preprocessor/PpScanner.cpp b/glslang/MachineIndependent/preprocessor/PpScanner.cpp index 82eaf26c7..4e68f343d 100644 --- a/glslang/MachineIndependent/preprocessor/PpScanner.cpp +++ b/glslang/MachineIndependent/preprocessor/PpScanner.cpp @@ -98,28 +98,11 @@ int TPpContext::InitScanner(TPpContext *cpp) if (!InitCPP()) return 0; - mostRecentToken = 0; - currentInput = 0; previous_token = '\n'; return 1; } -int TPpContext::sourceGetCh(TPpContext* pp, StringInputSrc *in) -{ - int ch = in->input->get(); - - if (ch == EOF) - delete in; - - return ch; -} - -void TPpContext::sourceUngetCh(TPpContext* pp, StringInputSrc *in, int ch, TPpToken *type) -{ - in->input->unget(); -} - /////////////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////// Floating point constants: ///////////////////////////////// /////////////////////////////////////////////////////////////////////////////////////////////// @@ -144,7 +127,7 @@ int TPpContext::lFloatConst(char* str, int len, int ch, TPpToken* ppToken) if (ch == '.') { HasDecimalOrExponent = true; str[len++]=ch; - ch = currentInput->getch(this, currentInput, ppToken); + ch = getChar(); while (ch >= '0' && ch <= '9') { if (len < TPpToken::maxTokenLength) { declen++; @@ -152,7 +135,7 @@ int TPpContext::lFloatConst(char* str, int len, int ch, TPpToken* ppToken) str[len] = ch; len++;str_len++; } - ch = currentInput->getch(this, currentInput, ppToken); + ch = getChar(); } else { parseContext.error(ppToken->loc, "float literal too long", "", ""); len = 1,str_len=1; @@ -170,21 +153,21 @@ int TPpContext::lFloatConst(char* str, int len, int ch, TPpToken* ppToken) } else { ExpSign = 1; str[len++]=ch; - ch = currentInput->getch(this, currentInput, ppToken); + ch = getChar(); if (ch == '+') { str[len++]=ch; - ch = currentInput->getch(this, currentInput, ppToken); + ch = getChar(); } else if (ch == '-') { ExpSign = -1; str[len++]=ch; - ch = currentInput->getch(this, currentInput, ppToken); + ch = getChar(); } if (ch >= '0' && ch <= '9') { while (ch >= '0' && ch <= '9') { if (len < TPpToken::maxTokenLength) { exp = exp*10 + ch - '0'; str[len++]=ch; - ch = currentInput->getch(this, currentInput, ppToken); + ch = getChar(); } else { parseContext.error(ppToken->loc, "float literal too long", "", ""); len = 1,str_len=1; @@ -205,10 +188,10 @@ int TPpContext::lFloatConst(char* str, int len, int ch, TPpToken* ppToken) parseContext.doubleCheck(ppToken->loc, "double floating-point suffix"); if (! HasDecimalOrExponent) parseContext.error(ppToken->loc, "float literal needs a decimal point or exponent", "", ""); - int ch2 = currentInput->getch(this, currentInput, ppToken); + int ch2 = getChar(); if (ch2 != 'f' && ch2 != 'F') { - currentInput->ungetch(this, currentInput, ch2, ppToken); - currentInput->ungetch(this, currentInput, ch, ppToken); + ungetChar(); + ungetChar(); } else { if (len < TPpToken::maxTokenLength) { str[len++] = ch; @@ -231,7 +214,7 @@ int TPpContext::lFloatConst(char* str, int len, int ch, TPpToken* ppToken) len = 1,str_len=1; } } else - currentInput->ungetch(this, currentInput, ch, ppToken); + ungetChar(); str[len]='\0'; @@ -249,7 +232,7 @@ int TPpContext::lFloatConst(char* str, int len, int ch, TPpToken* ppToken) // // Scanner used to tokenize source stream. // -int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken) +int TPpContext::tStringInput::scan(TPpToken* ppToken) { char tokenText[TPpToken::maxTokenLength + 1]; int AlreadyComplained = 0; @@ -258,18 +241,22 @@ int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken) ppToken->ival = 0; ppToken->space = false; - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); for (;;) { - while (ch == ' ' || ch == '\t' || ch == '\r') { + while (ch == ' ' || ch == '\t') { ppToken->space = true; - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); } ppToken->loc = pp->parseContext.getCurrentLoc(); len = 0; switch (ch) { default: - return ch; // Single character token + return ch; // Single character token, including '\' (escaped newlines are handled at a lower level, so this is just a '\' token) + + case EOF: + return endOfInput; + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': @@ -281,67 +268,45 @@ int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken) case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': - case 'z': case '\\': + case 'z': do { - if (ch == '\\') { - // escaped character - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); - if (ch == '\r' || ch == '\n') { - pp->parseContext.lineContinuationCheck(ppToken->loc, false); - int nextch = pp->currentInput->getch(pp, pp->currentInput, ppToken); - if (ch == '\r' && nextch == '\n') - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); - else - ch = nextch; - } else { - // Not an escaped newline. - // Put back whatever it was - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); - // If not in the middle of an identifier, the \ is our token - if (len == 0) - return '\\'; - // Otherwise, put back the \ character, leave it for the next call - ch = '\\'; // for the upcoming unget(...ch...); - break; - } - } else if (len < TPpToken::maxTokenLength) { + if (len < TPpToken::maxTokenLength) { tokenText[len++] = ch; - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); } else { if (! AlreadyComplained) { pp->parseContext.error(ppToken->loc, "name too long", "", ""); AlreadyComplained = 1; } - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); } } while ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || - ch == '_' || - ch == '\\'); + ch == '_'); // line continuation with no token before or after makes len == 0, and need to start over skipping white space, etc. if (len == 0) continue; tokenText[len] = '\0'; - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); ppToken->atom = pp->LookUpAddString(tokenText); return CPP_IDENTIFIER; case '0': ppToken->name[len++] = ch; - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == 'x' || ch == 'X') { // must be hexidecimal bool isUnsigned = false; ppToken->name[len++] = ch; - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F') || - (ch >= 'a' && ch <= 'f')) - { + (ch >= 'a' && ch <= 'f')) { + ival = 0; do { if (ival <= 0x0fffffff) { @@ -362,7 +327,7 @@ int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken) } ival = 0xffffffff; } - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); } while ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F') || (ch >= 'a' && ch <= 'f')); @@ -374,7 +339,7 @@ int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken) ppToken->name[len++] = ch; isUnsigned = true; } else - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); ppToken->name[len] = '\0'; ppToken->ival = (int)ival; @@ -403,7 +368,7 @@ int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken) ival = (ival << 3) | ii; } else octalOverflow = true; - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); } // could be part of a float... @@ -416,7 +381,7 @@ int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken) pp->parseContext.error(ppToken->loc, "numeric literal too long", "", ""); AlreadyComplained = 1; } - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); } while (ch >= '0' && ch <= '9'); } if (ch == '.' || ch == 'e' || ch == 'f' || ch == 'E' || ch == 'F' || ch == 'l' || ch == 'L') @@ -431,7 +396,7 @@ int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken) ppToken->name[len++] = ch; isUnsigned = true; } else - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); ppToken->name[len] = '\0'; if (octalOverflow) @@ -456,7 +421,7 @@ int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken) pp->parseContext.error(ppToken->loc, "numeric literal too long", "", ""); AlreadyComplained = 1; } - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); } while (ch >= '0' && ch <= '9'); if (ch == '.' || ch == 'e' || ch == 'f' || ch == 'E' || ch == 'F' || ch == 'l' || ch == 'L') { return pp->lFloatConst(ppToken->name, len, ch, ppToken); @@ -469,7 +434,7 @@ int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken) ppToken->name[len++] = ch; uint = 1; } else - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); ppToken->name[len] = '\0'; ival = 0; @@ -491,112 +456,112 @@ int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken) } break; case '-': - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '-') { return CPP_DEC_OP; } else if (ch == '=') { return CPP_SUB_ASSIGN; } else { - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return '-'; } case '+': - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '+') { return CPP_INC_OP; } else if (ch == '=') { return CPP_ADD_ASSIGN; } else { - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return '+'; } case '*': - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '=') { return CPP_MUL_ASSIGN; } else { - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return '*'; } case '%': - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '=') { return CPP_MOD_ASSIGN; } else if (ch == '>'){ return CPP_RIGHT_BRACE; } else { - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return '%'; } case ':': - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '>') { return CPP_RIGHT_BRACKET; } else { - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return ':'; } case '^': - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '^') { return CPP_XOR_OP; } else { if (ch == '=') return CPP_XOR_ASSIGN; else{ - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return '^'; } } case '=': - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '=') { return CPP_EQ_OP; } else { - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return '='; } case '!': - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '=') { return CPP_NE_OP; } else { - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return '!'; } case '|': - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '|') { return CPP_OR_OP; } else { if (ch == '=') return CPP_OR_ASSIGN; else{ - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return '|'; } } case '&': - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '&') { return CPP_AND_OP; } else { if (ch == '=') return CPP_AND_ASSIGN; else{ - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return '&'; } } case '<': - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '<') { - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '=') return CPP_LEFT_ASSIGN; else{ - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return CPP_LEFT_OP; } } else { @@ -608,78 +573,66 @@ int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken) else if (ch == ':') return CPP_LEFT_BRACKET; else{ - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return '<'; } } } case '>': - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '>') { - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '=') return CPP_RIGHT_ASSIGN; else{ - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return CPP_RIGHT_OP; } } else { if (ch == '=') { return CPP_GE_OP; } else { - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return '>'; } } case '.': - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch >= '0' && ch <= '9') { - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return pp->lFloatConst(ppToken->name, 0, '.', ppToken); } else { - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return '.'; } case '/': - // TODO: preprocessor simplification: use the Scan.cpp comment scanner - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == '/') { + pp->inComment = true; do { - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); - if (ch == '\\') { - // allow an escaped newline, otherwise escapes in comments are meaningless - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); - if (ch == '\r' || ch == '\n') { - if (! pp->parseContext.lineContinuationCheck(ppToken->loc, true)) - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); - else { - int nextch = pp->currentInput->getch(pp, pp->currentInput, ppToken); - if (ch == '\r' && nextch == '\n') - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); - else - ch = nextch; - } - } - } + ch = pp->getChar(); } while (ch != '\n' && ch != EOF); ppToken->space = true; + pp->inComment = false; + + if (ch == EOF) + return endOfInput; + return ch; } else if (ch == '*') { - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); do { while (ch != '*') { if (ch == EOF) { pp->parseContext.error(ppToken->loc, "EOF in comment", "comment", ""); - - return EOF; + return endOfInput; } - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); } - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); if (ch == EOF) { pp->parseContext.error(ppToken->loc, "EOF in comment", "comment", ""); - - return EOF; + return endOfInput; } } while (ch != '/'); ppToken->space = true; @@ -688,23 +641,17 @@ int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken) } else if (ch == '=') { return CPP_DIV_ASSIGN; } else { - pp->currentInput->ungetch(pp, pp->currentInput, ch, ppToken); + pp->ungetChar(); return '/'; } break; case '"': - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); while (ch != '"' && ch != '\n' && ch != EOF) { - if (ch == '\\') { - pp->parseContext.lineContinuationCheck(ppToken->loc, false); - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); - if (ch == '\n' || ch == '\r' || ch == EOF) - break; - } if (len < TPpToken::maxTokenLength) { tokenText[len] = ch; len++; - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); } else break; }; @@ -718,10 +665,84 @@ int TPpContext::sourceScan(TPpContext* pp, InputSrc*, TPpToken* ppToken) } } - ch = pp->currentInput->getch(pp, pp->currentInput, ppToken); + ch = pp->getChar(); } } +// +// Scanner used to get source stream characters. +// - Escaped newlines are handled here, invisibly to the caller. +// - All forms of newline are handled, and turned into just a '\n'. +// +int TPpContext::tStringInput::getch() +{ + int ch = input->get(); + + if (ch == '\\') { + // Move past escaped newlines, as many as sequentially exist + do { + if (input->peek() == '\r' || input->peek() == '\n') { + bool allowed = pp->parseContext.lineContinuationCheck(input->getSourceLoc(), pp->inComment); + if (! allowed && pp->inComment) + return '\\'; + + // escape one newline now + ch = input->get(); + int nextch = input->get(); + if (ch == '\r' && nextch == '\n') + ch = input->get(); + else + ch = nextch; + } else + return '\\'; + } while (ch == '\\'); + } + + // handle any non-escaped newline + if (ch == '\r' || ch == '\n') { + if (ch == '\r' && input->peek() == '\n') + ch = input->get(); + return '\n'; + } + + return ch; +} + +// +// Scanner used to backup the source stream characters. Newlines are +// handled here, invisibly to the caller, meaning have to undo exactly +// what getch() above does (e.g., don't leave things in the middle of a +// sequence of escaped newlines). +// +void TPpContext::tStringInput::ungetch() +{ + input->unget(); + + do { + int ch = input->peek(); + if (ch == '\r' || ch == '\n') { + if (ch == '\n') { + // correct for two-character newline + input->unget(); + if (input->peek() != '\r') + input->get(); + } + // now in front of a complete newline, move past an escape character + input->unget(); + if (input->peek() == '\\') + input->unget(); + else { + input->get(); + break; + } + } else + break; + } while (true); +} + +// +// The main functional entry-point into the preprocessor, which will +// scan the source strings to figure out and return the next processing token. // // Return string pointer to next token. // Return 0 when no more tokens. @@ -732,15 +753,19 @@ const char* TPpContext::tokenize(TPpToken* ppToken) for(;;) { const char* tokenString = 0; - token = currentInput->scan(this, currentInput, ppToken); + token = scanToken(ppToken); ppToken->token = token; - if (check_EOF(token)) + if (token == EOF) { + missingEndifCheck(); return 0; + } if (token == '#') { - if (previous_token == '\n' || previous_token == 0) { + if (previous_token == '\n') { token = readCPPline(ppToken); - if (check_EOF(token)) + if (token == EOF) { + missingEndifCheck(); return 0; + } continue; } else { parseContext.error(ppToken->loc, "preprocessor directive cannot be preceded by another token", "#", ""); @@ -753,7 +778,7 @@ const char* TPpContext::tokenize(TPpToken* ppToken) continue; // expand macros - if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, false) == 1) + if (token == CPP_IDENTIFIER && MacroExpand(ppToken->atom, ppToken, false, true) != 0) continue; if (token == CPP_IDENTIFIER) @@ -779,17 +804,11 @@ const char* TPpContext::tokenize(TPpToken* ppToken) } } -//Checks if the token just read is EOF or not. -bool TPpContext::check_EOF(int token) +// Checks if we've seen balanced #if...#endif +void TPpContext::missingEndifCheck() { - if (token == EOF) { - if (ifdepth > 0) - parseContext.error(parseContext.getCurrentLoc(), "missing #endif", "", ""); - - return true; - } - - return false; + if (ifdepth > 0) + parseContext.error(parseContext.getCurrentLoc(), "missing #endif", "", ""); } } // end namespace glslang diff --git a/glslang/MachineIndependent/preprocessor/PpTokens.cpp b/glslang/MachineIndependent/preprocessor/PpTokens.cpp index ee27cbd2f..98228a671 100644 --- a/glslang/MachineIndependent/preprocessor/PpTokens.cpp +++ b/glslang/MachineIndependent/preprocessor/PpTokens.cpp @@ -109,7 +109,7 @@ int TPpContext::lReadByte(TokenStream *pTok) if (pTok->current < pTok->data.size()) return pTok->data[pTok->current++]; else - return -1; + return tInput::endOfInput; } /* @@ -124,6 +124,7 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken* ppToken) lAddByte(pTok, (unsigned char)((token & 0x7f) + 0x80)); else lAddByte(pTok, (unsigned char)(token & 0x7f)); + switch (token) { case CPP_IDENTIFIER: case CPP_STRCONSTANT: @@ -154,11 +155,6 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken* ppToken) void TPpContext::RewindTokenStream(TokenStream *pTok) { pTok->current = 0; - - //if (pTok->head) { - // pTok->current = pTok->head; - // pTok->current->current = 0; - //} } /* @@ -172,105 +168,82 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken) ltoken = lReadByte(pTok); ppToken->loc = parseContext.getCurrentLoc(); - if (ltoken >= 0) { - if (ltoken > 127) - ltoken += 128; - switch (ltoken) { - case CPP_STRCONSTANT: - case CPP_IDENTIFIER: - case CPP_FLOATCONSTANT: - case CPP_DOUBLECONSTANT: - case CPP_INTCONSTANT: - case CPP_UINTCONSTANT: - len = 0; - ch = lReadByte(pTok); - while (ch != 0) { - if (len < TPpToken::maxTokenLength) { - tokenText[len] = ch; - len++; - ch = lReadByte(pTok); - } else { - parseContext.error(ppToken->loc, "token too long", "", ""); - break; - } - } - tokenText[len] = 0; - - switch (ltoken) { - case CPP_IDENTIFIER: - case CPP_STRCONSTANT: - ppToken->atom = LookUpAddString(tokenText); - break; - case CPP_FLOATCONSTANT: - case CPP_DOUBLECONSTANT: - strcpy(ppToken->name, tokenText); - ppToken->dval = atof(ppToken->name); - break; - case CPP_INTCONSTANT: - case CPP_UINTCONSTANT: - strcpy(ppToken->name, tokenText); - if (len > 0 && tokenText[0] == '0') { - if (len > 1 && tokenText[1] == 'x' || tokenText[1] == 'X') - ppToken->ival = strtol(ppToken->name, 0, 16); - else - ppToken->ival = strtol(ppToken->name, 0, 8); - } else - ppToken->ival = atoi(ppToken->name); + if (ltoken > 127) + ltoken += 128; + switch (ltoken) { + case CPP_STRCONSTANT: + case CPP_IDENTIFIER: + case CPP_FLOATCONSTANT: + case CPP_DOUBLECONSTANT: + case CPP_INTCONSTANT: + case CPP_UINTCONSTANT: + len = 0; + ch = lReadByte(pTok); + while (ch != 0) { + if (len < TPpToken::maxTokenLength) { + tokenText[len] = ch; + len++; + ch = lReadByte(pTok); + } else { + parseContext.error(ppToken->loc, "token too long", "", ""); break; } } - return ltoken; + tokenText[len] = 0; + + switch (ltoken) { + case CPP_IDENTIFIER: + case CPP_STRCONSTANT: + ppToken->atom = LookUpAddString(tokenText); + break; + case CPP_FLOATCONSTANT: + case CPP_DOUBLECONSTANT: + strcpy(ppToken->name, tokenText); + ppToken->dval = atof(ppToken->name); + break; + case CPP_INTCONSTANT: + case CPP_UINTCONSTANT: + strcpy(ppToken->name, tokenText); + if (len > 0 && tokenText[0] == '0') { + if (len > 1 && tokenText[1] == 'x' || tokenText[1] == 'X') + ppToken->ival = strtol(ppToken->name, 0, 16); + else + ppToken->ival = strtol(ppToken->name, 0, 8); + } else + ppToken->ival = atoi(ppToken->name); + break; + } } - return EOF; + + return ltoken; } -int TPpContext::scan_token(TPpContext* pp, TokenInputSrc *in, TPpToken * ppToken) +int TPpContext::tTokenInput::scan(TPpToken* ppToken) { - int token = pp->ReadToken(in->tokens, ppToken); - int (*final)(TPpContext *); - if (token > 0) - return token; - - pp->currentInput = in->prev; - final = in->final; - delete in; - if (final && !final(pp)) - return -1; - - return pp->currentInput->scan(pp, pp->currentInput, ppToken); + return pp->ReadToken(tokens, ppToken); } -int TPpContext::ReadFromTokenStream(TokenStream *ts, int name, int (*final)(TPpContext *)) +void TPpContext::pushTokenStreamInput(TokenStream* ts, int name) { - TokenInputSrc* in = new TokenInputSrc; - in->prev = currentInput; - in->scan = (int (*)(TPpContext*, InputSrc*, TPpToken*))scan_token; - in->tokens = ts; - in->final = final; + pushInput(new tTokenInput(this, ts)); RewindTokenStream(ts); - currentInput = in; - - return 1; } -int TPpContext::reget_token(TPpContext* pp, UngotToken *t, TPpToken * ppToken) +int TPpContext::tUngotTokenInput::scan(TPpToken* ppToken) { - int token = t->token; - *ppToken = t->lval; - pp->currentInput = t->prev; - delete t; + if (done) + return endOfInput; - return token; + int ret = token; + *ppToken = lval; + done = true; + + return ret; } void TPpContext::UngetToken(int token, TPpToken* ppToken) { - UngotToken *t = new UngotToken; - t->token = token; - t->lval = *ppToken; - t->scan = (int(*)(TPpContext*, struct InputSrc *, TPpToken *))reget_token; - t->prev = currentInput; - currentInput = t; + pushInput(new tUngotTokenInput(this, token, ppToken)); } } // end namespace glslang