PP: Implement token pasting for PP identifiers.

Implement token pasting as per the C++ specification, within the current
style of the PP code.
Non-identifiers (turning 12 ## 10 into the numeral 1210) is not yet covered;
they should be a simple incremental change built on this one.
Addresses issue #255.
This commit is contained in:
John Kessenich 2016-12-19 09:19:43 -07:00
parent abf5057948
commit d485e0b710
10 changed files with 304 additions and 34 deletions

View File

@ -77,9 +77,8 @@ ERROR: 0:192: 'gl_ClipDistance' : left of '[' is not of type array, matrix, or
ERROR: 0:192: 'assign' : l-value required (can't modify a const)
ERROR: 0:195: 'gl_ModelViewMatrix' : identifiers starting with "gl_" are reserved
ERROR: 0:200: 'token pasting (##)' : not supported for this version or the enabled extensions
ERROR: 0:200: '##' : token pasting not implemented (internal error)
ERROR: 0:200: '' : syntax error
ERROR: 80 compilation errors. No code generated.
ERROR: 0:203: 'token pasting (##)' : not supported for this version or the enabled extensions
ERROR: 79 compilation errors. No code generated.
Shader version: 120
@ -427,7 +426,8 @@ ERROR: node is still EOpNull!
0:? 'c2D' (in 2-component vector of float)
0:? 'c3D' (in 3-component vector of float)
0:? 'v4' (uniform 4-component vector of float)
0:? 'abc' (global int)
0:? 'abcdef' (global int)
0:? 'qrstuv' (global int)
Linked vertex stage:
@ -499,5 +499,6 @@ ERROR: node is still EOpNull!
0:? 'c2D' (in 2-component vector of float)
0:? 'c3D' (in 3-component vector of float)
0:? 'v4' (uniform 4-component vector of float)
0:? 'abc' (global int)
0:? 'abcdef' (global int)
0:? 'qrstuv' (global int)

View File

@ -3,9 +3,7 @@ ERROR: 0:59: 'gl_InstanceID' : undeclared identifier
ERROR: 0:59: '=' : cannot convert from 'temp float' to 'temp int'
ERROR: 0:61: 'texelFetch' : no matching overloaded function found
ERROR: 0:61: 'assign' : cannot convert from 'const float' to 'temp int'
ERROR: 0:75: '##' : token pasting not implemented (internal error)
ERROR: 0:75: '' : syntax error
ERROR: 6 compilation errors. No code generated.
ERROR: 4 compilation errors. No code generated.
Shader version: 130
@ -149,7 +147,8 @@ ERROR: node is still EOpNull!
0:? 'v4' (uniform 4-component vector of float)
0:? 'gl_ClipDistance' (smooth out implicitly-sized array of float ClipDistance)
0:? 'gl_TexCoord' (smooth out implicitly-sized array of 4-component vector of float TexCoord)
0:? 'abc' (global int)
0:? 'abcdef' (global int)
0:? 'qrstuv' (global int)
0:? 'gl_VertexID' (gl_VertexId int VertexId)
@ -281,6 +280,7 @@ ERROR: node is still EOpNull!
0:? 'v4' (uniform 4-component vector of float)
0:? 'gl_ClipDistance' (smooth out 2-element array of float ClipDistance)
0:? 'gl_TexCoord' (smooth out 1-element array of 4-component vector of float TexCoord)
0:? 'abc' (global int)
0:? 'abcdef' (global int)
0:? 'qrstuv' (global int)
0:? 'gl_VertexID' (gl_VertexId int VertexId)

View File

@ -0,0 +1,60 @@
tokenPaste.vert
Warning, version 450 is not yet complete; most version-specific features are present, but some are missing.
ERROR: 0:38: '##' : unexpected location
ERROR: 0:40: '##' : unexpected location; end of replacement list
ERROR: 0:49: '##' : combined tokens are too long
ERROR: 0:52: '##' : only supported for preprocessing identifiers
ERROR: 4 compilation errors. No code generated.
Shader version: 450
ERROR: node is still EOpNull!
0:52 Sequence
0:52 move second child to first child (temp int)
0:52 'a' (global int)
0:52 Constant:
0:52 11 (const int)
0:? Linker Objects
0:? 'SecondExpansion' (global int)
0:? 'PostPasteExpansion' (global int)
0:? 'foo27' (global float)
0:? 'foo155' (uniform float)
0:? 'foo719' (global float)
0:? 'barfoo' (uniform float)
0:? 'argless' (global float)
0:? 'dc1' (global float)
0:? 'dc2' (global float)
0:? 'foo875' (uniform float)
0:? 'ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123451234' (global float)
0:? 'a' (global int)
0:? 'gl_VertexID' (gl_VertexId int VertexId)
0:? 'gl_InstanceID' (gl_InstanceId int InstanceId)
Linked vertex stage:
ERROR: Linking vertex stage: Missing entry point: Each stage requires one entry point
Shader version: 450
ERROR: node is still EOpNull!
0:52 Sequence
0:52 move second child to first child (temp int)
0:52 'a' (global int)
0:52 Constant:
0:52 11 (const int)
0:? Linker Objects
0:? 'SecondExpansion' (global int)
0:? 'PostPasteExpansion' (global int)
0:? 'foo27' (global float)
0:? 'foo155' (uniform float)
0:? 'foo719' (global float)
0:? 'barfoo' (uniform float)
0:? 'argless' (global float)
0:? 'dc1' (global float)
0:? 'dc2' (global float)
0:? 'foo875' (uniform float)
0:? 'ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123451234' (global float)
0:? 'a' (global int)
0:? 'gl_VertexID' (gl_VertexId int VertexId)
0:? 'gl_InstanceID' (gl_InstanceId int InstanceId)

52
Test/tokenPaste.vert Normal file
View File

@ -0,0 +1,52 @@
#version 450
// side test verifies multiple rounds of argument expansion
#define bear SecondExpansion
#define mmmB bear
#define mmmA(a) a
int mmmA(mmmB); // mmmB -> bear, and then in mmmA(), bear -> SecondExpansion
// pasting skips the first round of expansion
#define mmcatmmdog PostPasteExpansion
#define mmcat cat
#define mmdog dog
#define mmp(a,b) a## b
int mmp(mmcat, mmdog); // mmcat/mmdog not expanded, mmcatmmdog -> PostPasteExpansion
// multi-token pre
#define mmtokpastepre(a) a##27
mmtokpastepre(float foo); // should declare "float foo27;"
// multi-token post
#define mmtokpastepost(a) uni ##a
mmtokpastepost(form float foo155); // should declare "uniform float foo155;"
// non-first argument
#define foo ShouldntExpandToThis
#define semi ;
#define bothpaste(a,b) a##b
float bothpaste(foo, 719); // should declare "float foo719;"
#define secpaste(a,b) a bar ## b
secpaste(uniform float, foo semi) // should declare "uniform float barfoo;"
// no args
#define noArg fl##oat
noArg argless;
// bad location
#define bad1 ## float
bad1 dc1;
#define bad2 float ##
bad2 dc2;
// multiple ##
#define multiPaste(a, b, c) a##or##b flo##at foo##c
multiPaste(unif, m, 875);
// too long
#define simplePaste(a,b) a##b
// 1020 + 5 characters
float simplePaste(ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF012345, 12345);
// non-identifiers
int a = simplePaste(11,12);

View File

@ -2,5 +2,5 @@
// For the version, it uses the latest git tag followed by the number of commits.
// For the date, it uses the current date (when then script is run).
#define GLSLANG_REVISION "Overload400-PrecQual.1695"
#define GLSLANG_DATE "16-Dec-2016"
#define GLSLANG_REVISION "Overload400-PrecQual.1696"
#define GLSLANG_DATE "19-Dec-2016"

View File

@ -933,36 +933,38 @@ int TPpContext::readCPPline(TPpToken* ppToken)
return token;
}
TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* a, TPpToken* ppToken, bool newLineOkay)
// Macro-expand a macro argument 'arg' to create 'expandedArg'.
// Does not replace 'arg'.
// Returns nullptr if no expanded argument is created.
TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream* arg, TPpToken* ppToken, bool newLineOkay)
{
int token;
TokenStream *n;
RewindTokenStream(a);
RewindTokenStream(arg);
do {
token = ReadToken(a, ppToken);
token = ReadToken(arg, ppToken);
if (token == PpAtomIdentifier && LookUpSymbol(ppToken->atom))
break;
} while (token != EndOfInput);
if (token == EndOfInput)
return a;
return nullptr;
n = new TokenStream;
TokenStream* expandedArg = new TokenStream;
pushInput(new tMarkerInput(this));
pushTokenStreamInput(a);
pushTokenStreamInput(arg);
while ((token = scanToken(ppToken)) != tMarkerInput::marker) {
if (token == PpAtomIdentifier && MacroExpand(ppToken->atom, ppToken, false, newLineOkay) != 0)
continue;
RecordToken(n, token, ppToken);
RecordToken(expandedArg, token, ppToken);
}
popInput();
delete a;
return n;
return expandedArg;
}
//
// Return the next token for a macro expansion, handling macro args.
// Return the next token for a macro expansion, handling macro arguments,
// whose semantics are dependent on being adjacent to ##.
//
int TPpContext::tMacroInput::scan(TPpToken* ppToken)
{
@ -971,6 +973,39 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken)
token = pp->ReadToken(mac->body, ppToken);
} while (token == ' '); // handle white space in macro
// Hash operators basically turn off a round of macro substitution
// (the round done on the argument before the round done on the RHS of the
// macro definition):
//
// "A parameter in the replacement list, unless preceded by a # or ##
// preprocessing token or followed by a ## preprocessing token (see below),
// is replaced by the corresponding argument after all macros contained
// therein have been expanded."
//
// "If, in the replacement list, a parameter is immediately preceded or
// followed by a ## preprocessing token, the parameter is replaced by the
// corresponding argument's preprocessing token sequence."
bool pasting = false;
if (postpaste) {
// don't expand next token
pasting = true;
postpaste = false;
}
if (prepaste) {
// already know we should be on a ##, verify
assert(token == PpAtomPaste);
prepaste = false;
postpaste = true;
}
// see if are preceding a ##
if (peekMacPasting()) {
prepaste = true;
pasting = true;
}
// TODO: preprocessor: properly handle whitespace (or lack of it) between tokens when expanding
if (token == PpAtomIdentifier) {
int i;
@ -978,7 +1013,10 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken)
if (mac->args[i] == ppToken->atom)
break;
if (i >= 0) {
pp->pushTokenStreamInput(args[i]);
TokenStream* arg = expandedArgs[i];
if (arg == nullptr || pasting)
arg = args[i];
pp->pushTokenStreamInput(arg, prepaste);
return pp->scanToken(ppToken);
}
@ -990,6 +1028,31 @@ int TPpContext::tMacroInput::scan(TPpToken* ppToken)
return token;
}
// See if the next non-white-space token in the macro is ##
bool TPpContext::tMacroInput::peekMacPasting()
{
// don't return early, have to restore this
size_t savePos = mac->body->current;
// skip white-space
int ltoken;
do {
ltoken = pp->lReadByte(mac->body);
} while (ltoken == ' ');
// check for ##
bool pasting = false;
if (ltoken == '#') {
ltoken = pp->lReadByte(mac->body);
if (ltoken == '#')
pasting = true;
}
mac->body->current = savePos;
return pasting;
}
// return a textual zero, for scanning a macro that was never defined
int TPpContext::tZeroInput::scan(TPpToken* ppToken)
{
@ -1080,6 +1143,9 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool
in->args.resize(in->mac->argc);
for (int i = 0; i < in->mac->argc; i++)
in->args[i] = new TokenStream;
in->expandedArgs.resize(in->mac->argc);
for (int i = 0; i < in->mac->argc; i++)
in->expandedArgs[i] = nullptr;
int arg = 0;
bool tokenRecorded = false;
do {
@ -1143,8 +1209,11 @@ int TPpContext::MacroExpand(int atom, TPpToken* ppToken, bool expandUndef, bool
}
parseContext.ppError(loc, "Too many args in macro", "macro expansion", GetAtomString(atom));
}
// We need both expanded and non-expanded forms of the argument, for whether or
// not token pasting is in play.
for (int i = 0; i < in->mac->argc; i++)
in->args[i] = PrescanMacroArg(in->args[i], ppToken, newLineOkay);
in->expandedArgs[i] = PrescanMacroArg(in->args[i], ppToken, newLineOkay);
}
pushInput(in);

View File

@ -129,6 +129,7 @@ public:
void setPreamble(const char* preamble, size_t length);
const char* tokenize(TPpToken* ppToken);
int tokenPaste(TPpToken&);
class tInput {
public:
@ -138,6 +139,8 @@ public:
virtual int scan(TPpToken*) = 0;
virtual int getch() = 0;
virtual void ungetch() = 0;
virtual bool peekPasting() { return false; } // true when about to see ##
virtual bool endOfReplacementList() { return false; } // true when at the end of a macro replacement list (RHS of #define)
// Will be called when we start reading tokens from this instance
virtual void notifyActivated() {}
@ -235,6 +238,8 @@ protected:
}
int getChar() { return inputStack.back()->getch(); }
void ungetChar() { inputStack.back()->ungetch(); }
bool peekPasting() { return !inputStack.empty() && inputStack.back()->peekPasting(); }
bool endOfReplacementList() { return inputStack.empty() || inputStack.back()->endOfReplacementList(); }
static const int maxMacroArgs = 64;
static const int maxIfNesting = 64;
@ -245,18 +250,29 @@ protected:
class tMacroInput : public tInput {
public:
tMacroInput(TPpContext* pp) : tInput(pp) { }
tMacroInput(TPpContext* pp) : tInput(pp), prepaste(false), postpaste(false) { }
virtual ~tMacroInput()
{
for (size_t i = 0; i < args.size(); ++i)
delete args[i];
for (size_t i = 0; i < expandedArgs.size(); ++i)
delete expandedArgs[i];
}
virtual int scan(TPpToken*);
virtual int getch() { assert(0); return EndOfInput; }
virtual void ungetch() { assert(0); }
bool peekPasting() override { return prepaste; }
bool endOfReplacementList() override { return mac->body->current >= mac->body->data.size(); }
MacroSymbol *mac;
TVector<TokenStream*> args;
TVector<TokenStream*> expandedArgs;
protected:
bool peekMacPasting();
bool prepaste; // true if we are just before ##
bool postpaste; // true if we are right after ##
};
class tMarkerInput : public tInput {
@ -329,17 +345,19 @@ protected:
void RecordToken(TokenStream* pTok, int token, TPpToken* ppToken);
void RewindTokenStream(TokenStream *pTok);
int ReadToken(TokenStream* pTok, TPpToken* ppToken);
void pushTokenStreamInput(TokenStream *ts);
void pushTokenStreamInput(TokenStream *ts, bool pasting = false);
void UngetToken(int token, TPpToken* ppToken);
class tTokenInput : public tInput {
public:
tTokenInput(TPpContext* pp, TokenStream* t) : tInput(pp), tokens(t) { }
tTokenInput(TPpContext* pp, TokenStream* t, bool prepasting) : tInput(pp), tokens(t), lastTokenPastes(prepasting) { }
virtual int scan(TPpToken *);
virtual int getch() { assert(0); return EndOfInput; }
virtual void ungetch() { assert(0); }
virtual bool peekPasting() override;
protected:
TokenStream *tokens;
bool lastTokenPastes; // true if the last token in the input is to be pasted, rather than consumed as a token
};
class tUngotTokenInput : public tInput {

View File

@ -739,6 +739,11 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
for(;;) {
token = scanToken(ppToken);
ppToken->token = token;
// Handle token-pasting logic
token = tokenPaste(*ppToken);
ppToken->token = token;
if (token == EndOfInput) {
missingEndifCheck();
return nullptr;
@ -800,6 +805,47 @@ const char* TPpContext::tokenize(TPpToken* ppToken)
}
}
//
// Do all token-pasting related combining of two pasted tokens when getting a
// stream of tokens from a replacement list. Degenerates to no processing if a
// replacement list is not the source of the token stream.
//
int TPpContext::tokenPaste(TPpToken& ppToken)
{
// starting with ## is illegal, skip to next token
if (ppToken.token == PpAtomPaste) {
parseContext.ppError(ppToken.loc, "unexpected location", "##", "");
ppToken.token = scanToken(&ppToken);
}
// ## can be chained, process all in the chain at once
while (peekPasting()) {
TPpToken pastedPpToken;
// next token has to be ##
pastedPpToken.token = scanToken(&pastedPpToken);
assert(pastedPpToken.token == PpAtomPaste);
if (endOfReplacementList()) {
parseContext.ppError(ppToken.loc, "unexpected location; end of replacement list", "##", "");
break;
}
// get the token after the ##
scanToken(&pastedPpToken);
// combine the tokens
if (strlen(ppToken.name) + strlen(pastedPpToken.name) > MaxTokenLength)
parseContext.ppError(ppToken.loc, "combined tokens are too long", "##", "");
strncat(ppToken.name, pastedPpToken.name, MaxTokenLength - strlen(ppToken.name));
ppToken.atom = LookUpAddString(ppToken.name);
if (ppToken.token != PpAtomIdentifier)
parseContext.ppError(ppToken.loc, "only supported for preprocessing identifiers", "##", "");
}
return ppToken.token;
}
// Checks if we've seen balanced #if...#endif
void TPpContext::missingEndifCheck()
{

View File

@ -160,7 +160,7 @@ void TPpContext::RecordToken(TokenStream *pTok, int token, TPpToken* ppToken)
}
/*
* Reset a token stream in preperation for reading.
* Reset a token stream in preparation for reading.
*/
void TPpContext::RewindTokenStream(TokenStream *pTok)
{
@ -187,9 +187,7 @@ int TPpContext::ReadToken(TokenStream *pTok, TPpToken *ppToken)
if (lReadByte(pTok) == '#') {
parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)");
parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)");
parseContext.error(ppToken->loc, "token pasting not implemented (internal error)", "##", "");
//return PpAtomPaste;
return ReadToken(pTok, ppToken);
ltoken = PpAtomPaste;
} else
lUnreadByte(pTok);
}
@ -279,9 +277,34 @@ int TPpContext::tTokenInput::scan(TPpToken* ppToken)
return pp->ReadToken(tokens, ppToken);
}
void TPpContext::pushTokenStreamInput(TokenStream* ts)
// We are pasting if the entire macro is preceding a pasting operator
// (lastTokenPastes) and we are also on the last token.
bool TPpContext::tTokenInput::peekPasting()
{
pushInput(new tTokenInput(this, ts));
if (! lastTokenPastes)
return false;
// Getting here means the last token will be pasted.
// Are we at the last non-whitespace token?
size_t savePos = tokens->current;
bool moreTokens = false;
do {
int byte = pp->lReadByte(tokens);
if (byte == EndOfInput)
break;
if (byte != ' ') {
moreTokens = true;
break;
}
} while (true);
tokens->current = savePos;
return !moreTokens;
}
void TPpContext::pushTokenStreamInput(TokenStream* ts, bool prepasting)
{
pushInput(new tTokenInput(this, ts, prepasting));
RewindTokenStream(ts);
}

View File

@ -178,6 +178,7 @@ INSTANTIATE_TEST_CASE_P(
"syntaxError.frag",
"test.frag",
"texture.frag",
"tokenPaste.vert",
"types.frag",
"uniformArray.frag",
"variableArrayIndex.frag",