mirror of
https://github.com/KhronosGroup/glslang
synced 2024-09-20 04:40:01 +00:00
PP: Non-functional: Remove the sub-tokenization of macro stream record.
This has been a continually fragile area. Switching to a vector of real objects, instead of a linearized stream of characters, removes a bunch of code and generally makes this area more robust.
This commit is contained in:
parent
bce1f51096
commit
5cdf3c5a23
@ -243,24 +243,45 @@ public:
|
||||
// From PpTokens.cpp
|
||||
//
|
||||
|
||||
// Capture the needed parts of a token stream for macro recording/playback.
|
||||
class TokenStream {
|
||||
public:
|
||||
TokenStream() : current(0) { }
|
||||
// Manage a stream of these 'Token', which capture the relevant parts
|
||||
// of a TPpToken, plus its atom.
|
||||
class Token {
|
||||
public:
|
||||
Token(int atom, const TPpToken& ppToken) :
|
||||
atom(atom),
|
||||
i64val(ppToken.i64val),
|
||||
name(ppToken.name) { }
|
||||
int get(TPpToken& ppToken)
|
||||
{
|
||||
ppToken.clear();
|
||||
ppToken.i64val = i64val;
|
||||
snprintf(ppToken.name, sizeof(ppToken.name), "%s", name.c_str());
|
||||
return atom;
|
||||
}
|
||||
bool isAtom(int a) { return atom == a; }
|
||||
protected:
|
||||
Token() {}
|
||||
int atom;
|
||||
long long i64val;
|
||||
TString name;
|
||||
};
|
||||
|
||||
TokenStream() : currentPos(0) { }
|
||||
|
||||
void putToken(int token, TPpToken* ppToken);
|
||||
bool peekToken(int atom) { return !atEnd() && stream[currentPos].isAtom(atom); }
|
||||
int getToken(TParseContextBase&, TPpToken*);
|
||||
bool atEnd() { return current >= data.size(); }
|
||||
bool atEnd() { return currentPos >= stream.size(); }
|
||||
bool peekTokenizedPasting(bool lastTokenPastes);
|
||||
bool peekUntokenizedPasting();
|
||||
void reset() { current = 0; }
|
||||
void reset() { currentPos = 0; }
|
||||
|
||||
protected:
|
||||
void putSubtoken(char);
|
||||
int getSubtoken();
|
||||
void ungetSubtoken();
|
||||
|
||||
TVector<unsigned char> data;
|
||||
size_t current;
|
||||
TVector<Token> stream;
|
||||
size_t currentPos;
|
||||
};
|
||||
|
||||
//
|
||||
|
@ -99,151 +99,33 @@ NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
namespace glslang {
|
||||
|
||||
|
||||
namespace {
|
||||
|
||||
// When recording (and playing back) should the backing name string
|
||||
// be saved (restored)?
|
||||
bool SaveName(int atom)
|
||||
{
|
||||
switch (atom) {
|
||||
case PpAtomIdentifier:
|
||||
case PpAtomConstString:
|
||||
case PpAtomConstInt:
|
||||
case PpAtomConstUint:
|
||||
case PpAtomConstInt64:
|
||||
case PpAtomConstUint64:
|
||||
#ifdef AMD_EXTENSIONS
|
||||
case PpAtomConstInt16:
|
||||
case PpAtomConstUint16:
|
||||
#endif
|
||||
case PpAtomConstFloat:
|
||||
case PpAtomConstDouble:
|
||||
case PpAtomConstFloat16:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// When recording (and playing back) should the numeric value
|
||||
// be saved (restored)?
|
||||
bool SaveValue(int atom)
|
||||
{
|
||||
switch (atom) {
|
||||
case PpAtomConstInt:
|
||||
case PpAtomConstUint:
|
||||
case PpAtomConstInt64:
|
||||
case PpAtomConstUint64:
|
||||
#ifdef AMD_EXTENSIONS
|
||||
case PpAtomConstInt16:
|
||||
case PpAtomConstUint16:
|
||||
#endif
|
||||
case PpAtomConstFloat:
|
||||
case PpAtomConstDouble:
|
||||
case PpAtomConstFloat16:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// push onto back of stream
|
||||
void TPpContext::TokenStream::putSubtoken(char subtoken)
|
||||
{
|
||||
data.push_back(static_cast<unsigned char>(subtoken));
|
||||
}
|
||||
|
||||
// get the next token in stream
|
||||
int TPpContext::TokenStream::getSubtoken()
|
||||
{
|
||||
if (current < data.size())
|
||||
return data[current++];
|
||||
else
|
||||
return EndOfInput;
|
||||
}
|
||||
|
||||
// back up one position in the stream
|
||||
void TPpContext::TokenStream::ungetSubtoken()
|
||||
{
|
||||
if (current > 0)
|
||||
--current;
|
||||
}
|
||||
|
||||
// Add a complete token (including backing string) to the end of a list
|
||||
// for later playback.
|
||||
// Add a token (including backing string) to the end of a macro
|
||||
// token stream, for later playback.
|
||||
void TPpContext::TokenStream::putToken(int atom, TPpToken* ppToken)
|
||||
{
|
||||
// save the atom
|
||||
assert((atom & ~0xff) == 0);
|
||||
putSubtoken(static_cast<char>(atom));
|
||||
|
||||
// save the backing name string
|
||||
if (SaveName(atom)) {
|
||||
const char* s = ppToken->name;
|
||||
while (*s)
|
||||
putSubtoken(*s++);
|
||||
putSubtoken(0);
|
||||
TokenStream::Token streamToken(atom, *ppToken);
|
||||
stream.push_back(streamToken);
|
||||
}
|
||||
|
||||
// save the numeric value
|
||||
if (SaveValue(atom)) {
|
||||
const char* n = reinterpret_cast<const char*>(&ppToken->i64val);
|
||||
for (size_t i = 0; i < sizeof(ppToken->i64val); ++i)
|
||||
putSubtoken(*n++);
|
||||
}
|
||||
}
|
||||
|
||||
// Read the next token from a token stream.
|
||||
// (Not the source stream, but a stream used to hold a tokenized macro).
|
||||
// Read the next token from a macro token stream.
|
||||
int TPpContext::TokenStream::getToken(TParseContextBase& parseContext, TPpToken *ppToken)
|
||||
{
|
||||
// get the atom
|
||||
int atom = getSubtoken();
|
||||
if (atom == EndOfInput)
|
||||
return atom;
|
||||
if (atEnd())
|
||||
return EndOfInput;
|
||||
|
||||
// init the token
|
||||
ppToken->clear();
|
||||
int atom = stream[currentPos++].get(*ppToken);
|
||||
ppToken->loc = parseContext.getCurrentLoc();
|
||||
|
||||
// get the backing name string
|
||||
if (SaveName(atom)) {
|
||||
int ch = getSubtoken();
|
||||
int len = 0;
|
||||
while (ch != 0 && ch != EndOfInput) {
|
||||
if (len < MaxTokenLength) {
|
||||
ppToken->name[len] = (char)ch;
|
||||
len++;
|
||||
ch = getSubtoken();
|
||||
} else {
|
||||
parseContext.error(ppToken->loc, "token too long", "", "");
|
||||
break;
|
||||
}
|
||||
}
|
||||
ppToken->name[len] = 0;
|
||||
}
|
||||
|
||||
// Check for ##, unless the current # is the last character
|
||||
if (atom == '#') {
|
||||
if (current < data.size()) {
|
||||
if (getSubtoken() == '#') {
|
||||
if (peekToken('#')) {
|
||||
parseContext.requireProfile(ppToken->loc, ~EEsProfile, "token pasting (##)");
|
||||
parseContext.profileRequires(ppToken->loc, ~EEsProfile, 130, 0, "token pasting (##)");
|
||||
currentPos++;
|
||||
atom = PpAtomPaste;
|
||||
} else
|
||||
ungetSubtoken();
|
||||
}
|
||||
}
|
||||
|
||||
// get the numeric value
|
||||
if (SaveValue(atom)) {
|
||||
char* n = reinterpret_cast<char*>(&ppToken->i64val);
|
||||
for (size_t i = 0; i < sizeof(ppToken->i64val); ++i)
|
||||
*n++ = (char)getSubtoken();
|
||||
}
|
||||
|
||||
return atom;
|
||||
}
|
||||
|
||||
@ -256,15 +138,14 @@ bool TPpContext::TokenStream::peekTokenizedPasting(bool lastTokenPastes)
|
||||
{
|
||||
// 1. preceding ##?
|
||||
|
||||
size_t savePos = current;
|
||||
int subtoken;
|
||||
size_t savePos = currentPos;
|
||||
// skip white space
|
||||
do {
|
||||
subtoken = getSubtoken();
|
||||
} while (subtoken == ' ');
|
||||
current = savePos;
|
||||
if (subtoken == PpAtomPaste)
|
||||
while (peekToken(' '))
|
||||
++currentPos;
|
||||
if (peekToken(PpAtomPaste)) {
|
||||
currentPos = savePos;
|
||||
return true;
|
||||
}
|
||||
|
||||
// 2. last token and we've been told after this there will be a ##
|
||||
|
||||
@ -273,18 +154,18 @@ bool TPpContext::TokenStream::peekTokenizedPasting(bool lastTokenPastes)
|
||||
// Getting here means the last token will be pasted, after this
|
||||
|
||||
// Are we at the last non-whitespace token?
|
||||
savePos = current;
|
||||
savePos = currentPos;
|
||||
bool moreTokens = false;
|
||||
do {
|
||||
subtoken = getSubtoken();
|
||||
if (subtoken == EndOfInput)
|
||||
if (atEnd())
|
||||
break;
|
||||
if (subtoken != ' ') {
|
||||
if (!peekToken(' ')) {
|
||||
moreTokens = true;
|
||||
break;
|
||||
}
|
||||
++currentPos;
|
||||
} while (true);
|
||||
current = savePos;
|
||||
currentPos = savePos;
|
||||
|
||||
return !moreTokens;
|
||||
}
|
||||
@ -293,23 +174,21 @@ bool TPpContext::TokenStream::peekTokenizedPasting(bool lastTokenPastes)
|
||||
bool TPpContext::TokenStream::peekUntokenizedPasting()
|
||||
{
|
||||
// don't return early, have to restore this
|
||||
size_t savePos = current;
|
||||
size_t savePos = currentPos;
|
||||
|
||||
// skip white-space
|
||||
int subtoken;
|
||||
do {
|
||||
subtoken = getSubtoken();
|
||||
} while (subtoken == ' ');
|
||||
while (peekToken(' '))
|
||||
++currentPos;
|
||||
|
||||
// check for ##
|
||||
bool pasting = false;
|
||||
if (subtoken == '#') {
|
||||
subtoken = getSubtoken();
|
||||
if (subtoken == '#')
|
||||
if (peekToken('#')) {
|
||||
++currentPos;
|
||||
if (peekToken('#'))
|
||||
pasting = true;
|
||||
}
|
||||
|
||||
current = savePos;
|
||||
currentPos = savePos;
|
||||
|
||||
return pasting;
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user