qdoc: Bring qdoc's QML parser up to date

qdoc's QML parser has fallen behind the QML: parser in
QtDeclarative. Bring it up to date.

Change-Id: I12a688873564762434852960350c56655004e460
Task-number: QTBUG-44868
Reviewed-by: Martin Smith <martin.smith@digia.com>

src/tools/qdoc/qmlparser/qqmljsastfwd_p.h

@@ -56,7 +56,7 @@ namespace QQmlJS { namespace AST {
 class SourceLocation
 {
 public:
-    SourceLocation(quint32 offset = 0, quint32 length = 0, quint32 line = 0, quint32 column = 0)
+    explicit SourceLocation(quint32 offset = 0, quint32 length = 0, quint32 line = 0, quint32 column = 0)
         : offset(offset), length(length),
           startLine(line), startColumn(column)
     { }

src/tools/qdoc/qmlparser/qqmljsengine_p.cpp

@@ -114,7 +114,7 @@ double integerFromString(const QString &str, int radix)
 
 
 Engine::Engine()
-    : _lexer(0)
+    : _lexer(0), _directives(0)
 { }
 
 Engine::~Engine()
@@ -135,6 +135,12 @@ Lexer *Engine::lexer() const
 void Engine::setLexer(Lexer *lexer)
 { _lexer = lexer; }
 
+Directives *Engine::directives() const
+{ return _directives; }
+
+void Engine::setDirectives(Directives *directives)
+{ _directives = directives; }
+
 MemoryPool *Engine::pool()
 { return &_pool; }
 

src/tools/qdoc/qmlparser/qqmljsengine_p.h

@@ -57,6 +57,7 @@ QT_QML_BEGIN_NAMESPACE
 namespace QQmlJS {
 
 class Lexer;
+class Directives;
 class MemoryPool;
 
 class QML_PARSER_EXPORT DiagnosticMessage
@@ -84,6 +85,7 @@ public:
 class QML_PARSER_EXPORT Engine
 {
     Lexer *_lexer;
+    Directives *_directives;
     MemoryPool _pool;
     QList<AST::SourceLocation> _comments;
     QString _extraCode;
@@ -102,6 +104,9 @@ public:
     Lexer *lexer() const;
     void setLexer(Lexer *lexer);
 
+    Directives *directives() const;
+    void setDirectives(Directives *directives);
+
     MemoryPool *pool();
 
     inline QStringRef midRef(int position, int size) { return _code.midRef(position, size); }

src/tools/qdoc/qmlparser/qqmljsglobal_p.h

@@ -33,17 +33,6 @@
 #ifndef QQMLJSGLOBAL_P_H
 #define QQMLJSGLOBAL_P_H
 
-//
-//  W A R N I N G
-//  -------------
-//
-// This file is not part of the Qt API.  It exists purely as an
-// implementation detail.  This header file may change from version to
-// version without notice, or even be removed.
-//
-// We mean it.
-//
-
 #include <QtCore/qglobal.h>
 
 #ifdef QT_CREATOR
@@ -65,9 +54,9 @@
      // QmlDevTools is a static library
 #    define QML_PARSER_EXPORT
 #  elif defined(QT_BUILD_QML_LIB)
-#    define QML_PARSER_EXPORT Q_AUTOTEST_EXPORT
+#    define QML_PARSER_EXPORT Q_DECL_EXPORT
 #  else
-#    define QML_PARSER_EXPORT
+#    define QML_PARSER_EXPORT Q_DECL_IMPORT
 #  endif
 #endif // QT_CREATOR
 

src/tools/qdoc/qmlparser/qqmljslexer.cpp

@@ -887,8 +887,7 @@ again:
 int Lexer::scanNumber(QChar ch)
 {
     if (ch != QLatin1Char('0')) {
-        QByteArray buf;
+        QVarLengthArray<char, 64> buf;
-        buf.reserve(64);
         buf += ch.toLatin1();
 
         QChar n = _char;
@@ -1225,12 +1224,60 @@ bool Lexer::canInsertAutomaticSemicolon(int token) const
             || _followsClosingBrace;
 }
 
-bool Lexer::scanDirectives(Directives *directives)
+static const int uriTokens[] = {
+    QQmlJSGrammar::T_IDENTIFIER,
+    QQmlJSGrammar::T_PROPERTY,
+    QQmlJSGrammar::T_SIGNAL,
+    QQmlJSGrammar::T_READONLY,
+    QQmlJSGrammar::T_ON,
+    QQmlJSGrammar::T_BREAK,
+    QQmlJSGrammar::T_CASE,
+    QQmlJSGrammar::T_CATCH,
+    QQmlJSGrammar::T_CONTINUE,
+    QQmlJSGrammar::T_DEFAULT,
+    QQmlJSGrammar::T_DELETE,
+    QQmlJSGrammar::T_DO,
+    QQmlJSGrammar::T_ELSE,
+    QQmlJSGrammar::T_FALSE,
+    QQmlJSGrammar::T_FINALLY,
+    QQmlJSGrammar::T_FOR,
+    QQmlJSGrammar::T_FUNCTION,
+    QQmlJSGrammar::T_IF,
+    QQmlJSGrammar::T_IN,
+    QQmlJSGrammar::T_INSTANCEOF,
+    QQmlJSGrammar::T_NEW,
+    QQmlJSGrammar::T_NULL,
+    QQmlJSGrammar::T_RETURN,
+    QQmlJSGrammar::T_SWITCH,
+    QQmlJSGrammar::T_THIS,
+    QQmlJSGrammar::T_THROW,
+    QQmlJSGrammar::T_TRUE,
+    QQmlJSGrammar::T_TRY,
+    QQmlJSGrammar::T_TYPEOF,
+    QQmlJSGrammar::T_VAR,
+    QQmlJSGrammar::T_VOID,
+    QQmlJSGrammar::T_WHILE,
+    QQmlJSGrammar::T_CONST,
+    QQmlJSGrammar::T_DEBUGGER,
+    QQmlJSGrammar::T_RESERVED_WORD,
+    QQmlJSGrammar::T_WITH,
+
+    QQmlJSGrammar::EOF_SYMBOL
+};
+static inline bool isUriToken(int token)
 {
-    if (_qmlMode) {
+    const int *current = uriTokens;
-        // the directives are a Javascript-only extension.
+    while (*current != QQmlJSGrammar::EOF_SYMBOL) {
-        return false;
+        if (*current == token)
+            return true;
+        ++current;
     }
+    return false;
+}
+
+bool Lexer::scanDirectives(Directives *directives, DiagnosticMessage *error)
+{
+    Q_ASSERT(!_qmlMode);
 
     lex(); // fetch the first token
 
@@ -1238,24 +1285,33 @@ bool Lexer::scanDirectives(Directives *directives)
         return true;
 
     do {
+        const int lineNumber = tokenStartLine();
+        const int column = tokenStartColumn();
+
         lex(); // skip T_DOT
 
-        const int lineNumber = tokenStartLine();
-
         if (! (_tokenKind == T_IDENTIFIER || _tokenKind == T_RESERVED_WORD))
-            return false; // expected a valid QML/JS directive
+            return true; // expected a valid QML/JS directive
 
         const QString directiveName = tokenText();
 
         if (! (directiveName == QLatin1String("pragma") ||
-               directiveName == QLatin1String("import")))
+               directiveName == QLatin1String("import"))) {
+            error->message = QCoreApplication::translate("QQmlParser", "Syntax error");
+            error->loc.startLine = tokenStartLine();
+            error->loc.startColumn = tokenStartColumn();
             return false; // not a valid directive name
+        }
 
         // it must be a pragma or an import directive.
         if (directiveName == QLatin1String("pragma")) {
             // .pragma library
-            if (! (lex() == T_IDENTIFIER && tokenText() == QLatin1String("library")))
+            if (! (lex() == T_IDENTIFIER && tokenText() == QLatin1String("library"))) {
+                error->message = QCoreApplication::translate("QQmlParser", "Syntax error");
+                error->loc.startLine = tokenStartLine();
+                error->loc.startColumn = tokenStartColumn();
                 return false; // expected `library
+            }
 
             // we found a .pragma library directive
             directives->pragmaLibrary();
@@ -1274,22 +1330,53 @@ bool Lexer::scanDirectives(Directives *directives)
                 fileImport = true;
                 pathOrUri = tokenText();
 
+                if (!pathOrUri.endsWith(QLatin1String("js"))) {
+                    error->message = QCoreApplication::translate("QQmlParser","Imported file must be a script");
+                    error->loc.startLine = tokenStartLine();
+                    error->loc.startColumn = tokenStartColumn();
+                    return false;
+                }
+
             } else if (_tokenKind == T_IDENTIFIER) {
                 // .import T_IDENTIFIER (. T_IDENTIFIER)* T_NUMERIC_LITERAL as T_IDENTIFIER
 
-                pathOrUri = tokenText();
+                while (true) {
-
+                    if (!isUriToken(_tokenKind)) {
-                lex(); // skip the first T_IDENTIFIER
+                        error->message = QCoreApplication::translate("QQmlParser","Invalid module URI");
-                for (; _tokenKind == T_DOT; lex()) {
+                        error->loc.startLine = tokenStartLine();
-                    if (lex() != T_IDENTIFIER)
+                        error->loc.startColumn = tokenStartColumn();
                         return false;
-
-                    pathOrUri += QLatin1Char('.');
-                    pathOrUri += tokenText();
                     }
 
-                if (_tokenKind != T_NUMERIC_LITERAL)
+                    pathOrUri.append(tokenText());
+
+                    lex();
+                    if (tokenStartLine() != lineNumber) {
+                        error->message = QCoreApplication::translate("QQmlParser","Invalid module URI");
+                        error->loc.startLine = tokenStartLine();
+                        error->loc.startColumn = tokenStartColumn();
+                        return false;
+                    }
+                    if (_tokenKind != QQmlJSGrammar::T_DOT)
+                        break;
+
+                    pathOrUri.append(QLatin1Char('.'));
+
+                    lex();
+                    if (tokenStartLine() != lineNumber) {
+                        error->message = QCoreApplication::translate("QQmlParser","Invalid module URI");
+                        error->loc.startLine = tokenStartLine();
+                        error->loc.startColumn = tokenStartColumn();
+                        return false;
+                    }
+                }
+
+                if (_tokenKind != T_NUMERIC_LITERAL) {
+                    error->message = QCoreApplication::translate("QQmlParser","Module import requires a version");
+                    error->loc.startLine = tokenStartLine();
+                    error->loc.startColumn = tokenStartColumn();
                     return false; // expected the module version number
+                }
 
                 version = tokenText();
             }
@@ -1297,22 +1384,51 @@ bool Lexer::scanDirectives(Directives *directives)
             //
             // recognize the mandatory `as' followed by the module name
             //
-            if (! (lex() == T_IDENTIFIER && tokenText() == QLatin1String("as")))
+            if (! (lex() == T_IDENTIFIER && tokenText() == QLatin1String("as") && tokenStartLine() == lineNumber)) {
-                return false; // expected `as'
-
-            if (lex() != T_IDENTIFIER)
-                return false; // expected module name
-
-            const QString module = tokenText();
-
                 if (fileImport)
-                directives->importFile(pathOrUri, module);
+                    error->message = QCoreApplication::translate("QQmlParser", "File import requires a qualifier");
                 else
-                directives->importModule(pathOrUri, version, module);
+                    error->message = QCoreApplication::translate("QQmlParser", "Module import requires a qualifier");
+                if (tokenStartLine() != lineNumber) {
+                    error->loc.startLine = lineNumber;
+                    error->loc.startColumn = column;
+                } else {
+                    error->loc.startLine = tokenStartLine();
+                    error->loc.startColumn = tokenStartColumn();
+                }
+                return false; // expected `as'
             }
 
-        if (tokenStartLine() != lineNumber)
+            if (lex() != T_IDENTIFIER || tokenStartLine() != lineNumber) {
+                if (fileImport)
+                    error->message = QCoreApplication::translate("QQmlParser", "File import requires a qualifier");
+                else
+                    error->message = QCoreApplication::translate("QQmlParser", "Module import requires a qualifier");
+                error->loc.startLine = tokenStartLine();
+                error->loc.startColumn = tokenStartColumn();
+                return false; // expected module name
+            }
+
+            const QString module = tokenText();
+            if (!module.at(0).isUpper()) {
+                error->message = QCoreApplication::translate("QQmlParser","Invalid import qualifier");
+                error->loc.startLine = tokenStartLine();
+                error->loc.startColumn = tokenStartColumn();
+                return false;
+            }
+
+            if (fileImport)
+                directives->importFile(pathOrUri, module, lineNumber, column);
+            else
+                directives->importModule(pathOrUri, version, module, lineNumber, column);
+        }
+
+        if (tokenStartLine() != lineNumber) {
+            error->message = QCoreApplication::translate("QQmlParser", "Syntax error");
+            error->loc.startLine = tokenStartLine();
+            error->loc.startColumn = tokenStartColumn();
             return false; // the directives cannot span over multiple lines
+        }
 
         // fetch the first token after the .pragma/.import directive
         lex();

src/tools/qdoc/qmlparser/qqmljslexer_p.h

@@ -55,6 +55,7 @@ QT_QML_BEGIN_NAMESPACE
 namespace QQmlJS {
 
 class Engine;
+class DiagnosticMessage;
 
 class QML_PARSER_EXPORT Directives {
 public:
@@ -64,17 +65,21 @@ public:
     {
     }
 
-    virtual void importFile(const QString &jsfile, const QString &module)
+    virtual void importFile(const QString &jsfile, const QString &module, int line, int column)
     {
         Q_UNUSED(jsfile);
         Q_UNUSED(module);
+        Q_UNUSED(line);
+        Q_UNUSED(column);
     }
 
-    virtual void importModule(const QString &uri, const QString &version, const QString &module)
+    virtual void importModule(const QString &uri, const QString &version, const QString &module, int line, int column)
     {
         Q_UNUSED(uri);
         Q_UNUSED(version);
         Q_UNUSED(module);
+        Q_UNUSED(line);
+        Q_UNUSED(column);
     }
 };
 
@@ -146,7 +151,7 @@ public:
     int lex();
 
     bool scanRegExp(RegExpBodyPrefix prefix = NoPrefix);
-    bool scanDirectives(Directives *directives);
+    bool scanDirectives(Directives *directives, DiagnosticMessage *error);
 
     int regExpFlags() const { return _patternFlags; }
     QString regExpPattern() const { return _tokenText; }

src/tools/qdoc/qmlparser/qqmljsmemorypool_p.h

@@ -57,6 +57,8 @@ QT_QML_BEGIN_NAMESPACE
 
 namespace QQmlJS {
 
+class Managed;
+
 class QML_PARSER_EXPORT MemoryPool : public QSharedData
 {
     MemoryPool(const MemoryPool &other);
@@ -100,6 +102,30 @@ public:
         _ptr = _end = 0;
     }
 
+    template <typename _Tp> _Tp *New() { return new (this->allocate(sizeof(_Tp))) _Tp(); }
+
+    template <typename PoolContentType, typename Visitor>
+    void visitManagedPool(Visitor &visitor)
+    {
+        for (int i = 0; i <= _blockCount; ++i) {
+            char *p = _blocks[i];
+            char *end = p + BLOCK_SIZE;
+            if (i == _blockCount) {
+                Q_ASSERT(_ptr <= end);
+                end = _ptr;
+            }
+
+            Q_ASSERT(p <= end);
+
+            const qptrdiff increment = (sizeof(PoolContentType) + 7) & ~7;
+
+            while (p + increment <= end) {
+                visitor(reinterpret_cast<PoolContentType*>(p));
+                p += increment;
+            }
+        }
+    }
+
 private:
     void *allocate_helper(size_t size)
     {

src/tools/qdoc/qmlparser/qqmljsparser.cpp

@@ -161,7 +161,24 @@ bool Parser::parse(int startToken)
 
     token_buffer[0].token = startToken;
     first_token = &token_buffer[0];
+    if (startToken == T_FEED_JS_PROGRAM && !lexer->qmlMode()) {
+        Directives ignoreDirectives;
+        Directives *directives = driver->directives();
+        if (!directives)
+            directives = &ignoreDirectives;
+        DiagnosticMessage error;
+        if (!lexer->scanDirectives(directives, &error)) {
+            diagnostic_messages.append(error);
+            return false;
+        }
+        token_buffer[1].token = lexer->tokenKind();
+        token_buffer[1].dval = lexer->tokenValue();
+        token_buffer[1].loc = location(lexer);
+        token_buffer[1].spell = lexer->tokenSpell();
+        last_token = &token_buffer[2];
+    } else {
         last_token = &token_buffer[1];
+    }
 
     tos = -1;
     program = 0;