qt5base-lts/util/cmake/qmake_parser.py
Alexandru Croitor 4ed483b0e2 CMake: pro2cmake: Fix errors reported by flake8 and mypy
Change-Id: I842c0f8d7ec1d173fa1dbd74e769e2c370e70518
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Cristian Adam <cristian.adam@qt.io>
2020-07-31 12:55:33 +02:00

398 lines
14 KiB
Python

#!/usr/bin/env python3
#############################################################################
##
## Copyright (C) 2018 The Qt Company Ltd.
## Contact: https://www.qt.io/licensing/
##
## This file is part of the plugins of the Qt Toolkit.
##
## $QT_BEGIN_LICENSE:GPL-EXCEPT$
## Commercial License Usage
## Licensees holding valid commercial Qt licenses may use this file in
## accordance with the commercial license agreement provided with the
## Software or, alternatively, in accordance with the terms contained in
## a written agreement between you and The Qt Company. For licensing terms
## and conditions see https://www.qt.io/terms-conditions. For further
## information use the contact form at https://www.qt.io/contact-us.
##
## GNU General Public License Usage
## Alternatively, this file may be used under the terms of the GNU
## General Public License version 3 as published by the Free Software
## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
## included in the packaging of this file. Please review the following
## information to ensure the GNU General Public License requirements will
## be met: https://www.gnu.org/licenses/gpl-3.0.html.
##
## $QT_END_LICENSE$
##
#############################################################################
import collections
import os
import re
from itertools import chain
from typing import Tuple
import pyparsing as pp # type: ignore
from helper import _set_up_py_parsing_nicer_debug_output
_set_up_py_parsing_nicer_debug_output(pp)
def fixup_linecontinuation(contents: str) -> str:
# Remove all line continuations, aka a backslash followed by
# a newline character with an arbitrary amount of whitespace
# between the backslash and the newline.
# This greatly simplifies the qmake parsing grammar.
contents = re.sub(r"([^\t ])\\[ \t]*\n", "\\1 ", contents)
contents = re.sub(r"\\[ \t]*\n", "", contents)
return contents
def fixup_comments(contents: str) -> str:
# Get rid of completely commented out lines.
# So any line which starts with a '#' char and ends with a new line
# will be replaced by a single new line.
#
# This is needed because qmake syntax is weird. In a multi line
# assignment (separated by backslashes and newlines aka
# # \\\n ), if any of the lines are completely commented out, in
# principle the assignment should fail.
#
# It should fail because you would have a new line separating
# the previous value from the next value, and the next value would
# not be interpreted as a value, but as a new token / operation.
# qmake is lenient though, and accepts that, so we need to take
# care of it as well, as if the commented line didn't exist in the
# first place.
contents = re.sub(r"\n#[^\n]*?\n", "\n", contents, re.DOTALL)
return contents
def flatten_list(input_list):
""" Flattens an irregular nested list into a simple list."""
for el in input_list:
if isinstance(el, collections.abc.Iterable) and not isinstance(el, (str, bytes)):
yield from flatten_list(el)
else:
yield el
def handle_function_value(group: pp.ParseResults):
function_name = group[0]
function_args = group[1]
if function_name == "qtLibraryTarget":
if len(function_args) > 1:
raise RuntimeError(
"Don't know what to with more than one function argument "
"for $$qtLibraryTarget()."
)
return str(function_args[0])
if function_name == "quote":
# Do nothing, just return a string result
return str(group)
if function_name == "files":
return str(function_args[0])
if function_name == "basename":
if len(function_args) != 1:
print(f"XXXX basename with more than one argument")
if function_args[0] == "_PRO_FILE_PWD_":
return os.path.basename(os.getcwd())
print(f"XXXX basename with value other than _PRO_FILE_PWD_")
return os.path.basename(str(function_args[0]))
if isinstance(function_args, pp.ParseResults):
function_args = list(flatten_list(function_args.asList()))
# For other functions, return the whole expression as a string.
return f"$${function_name}({' '.join(function_args)})"
class QmakeParser:
def __init__(self, *, debug: bool = False) -> None:
self.debug = debug
self._Grammar = self._generate_grammar()
def _generate_grammar(self):
# Define grammar:
pp.ParserElement.setDefaultWhitespaceChars(" \t")
def add_element(name: str, value: pp.ParserElement):
nonlocal self
if self.debug:
value.setName(name)
value.setDebug()
return value
EOL = add_element("EOL", pp.Suppress(pp.LineEnd()))
Else = add_element("Else", pp.Keyword("else"))
Identifier = add_element(
"Identifier", pp.Word(f"{pp.alphas}_", bodyChars=pp.alphanums + "_-./")
)
BracedValue = add_element(
"BracedValue",
pp.nestedExpr(
ignoreExpr=pp.quotedString
| pp.QuotedString(
quoteChar="$(", endQuoteChar=")", escQuote="\\", unquoteResults=False
)
).setParseAction(lambda s, l, t: ["(", *t[0], ")"]),
)
Substitution = add_element(
"Substitution",
pp.Combine(
pp.Literal("$")
+ (
(
(pp.Literal("$") + Identifier + pp.Optional(pp.nestedExpr()))
| (pp.Literal("(") + Identifier + pp.Literal(")"))
| (pp.Literal("{") + Identifier + pp.Literal("}"))
| (
pp.Literal("$")
+ pp.Literal("{")
+ Identifier
+ pp.Optional(pp.nestedExpr())
+ pp.Literal("}")
)
| (pp.Literal("$") + pp.Literal("[") + Identifier + pp.Literal("]"))
)
)
),
)
LiteralValuePart = add_element(
"LiteralValuePart", pp.Word(pp.printables, excludeChars="$#{}()")
)
SubstitutionValue = add_element(
"SubstitutionValue",
pp.Combine(pp.OneOrMore(Substitution | LiteralValuePart | pp.Literal("$"))),
)
FunctionValue = add_element(
"FunctionValue",
pp.Group(
pp.Suppress(pp.Literal("$") + pp.Literal("$"))
+ Identifier
+ pp.nestedExpr() # .setParseAction(lambda s, l, t: ['(', *t[0], ')'])
).setParseAction(lambda s, l, t: handle_function_value(*t)),
)
Value = add_element(
"Value",
pp.NotAny(Else | pp.Literal("}") | EOL)
+ (
pp.QuotedString(quoteChar='"', escChar="\\")
| FunctionValue
| SubstitutionValue
| BracedValue
),
)
Values = add_element("Values", pp.ZeroOrMore(Value)("value"))
Op = add_element(
"OP",
pp.Literal("=")
| pp.Literal("-=")
| pp.Literal("+=")
| pp.Literal("*=")
| pp.Literal("~="),
)
Key = add_element("Key", Identifier)
Operation = add_element(
"Operation", Key("key") + pp.locatedExpr(Op)("operation") + Values("value")
)
CallArgs = add_element("CallArgs", pp.nestedExpr())
def parse_call_args(results):
out = ""
for item in chain(*results):
if isinstance(item, str):
out += item
else:
out += "(" + parse_call_args(item) + ")"
return out
CallArgs.setParseAction(parse_call_args)
Load = add_element("Load", pp.Keyword("load") + CallArgs("loaded"))
Include = add_element(
"Include", pp.Keyword("include") + pp.locatedExpr(CallArgs)("included")
)
Option = add_element("Option", pp.Keyword("option") + CallArgs("option"))
RequiresCondition = add_element("RequiresCondition", pp.originalTextFor(pp.nestedExpr()))
def parse_requires_condition(s, l_unused, t):
# The following expression unwraps the condition via the additional info
# set by originalTextFor.
condition_without_parentheses = s[t._original_start + 1 : t._original_end - 1]
# And this replaces the colons with '&&' similar how it's done for 'Condition'.
condition_without_parentheses = (
condition_without_parentheses.strip().replace(":", " && ").strip(" && ")
)
return condition_without_parentheses
RequiresCondition.setParseAction(parse_requires_condition)
Requires = add_element(
"Requires", pp.Keyword("requires") + RequiresCondition("project_required_condition")
)
FunctionArgumentsAsString = add_element(
"FunctionArgumentsAsString", pp.originalTextFor(pp.nestedExpr())
)
QtNoMakeTools = add_element(
"QtNoMakeTools",
pp.Keyword("qtNomakeTools") + FunctionArgumentsAsString("qt_no_make_tools_arguments"),
)
# ignore the whole thing...
DefineTestDefinition = add_element(
"DefineTestDefinition",
pp.Suppress(
pp.Keyword("defineTest")
+ CallArgs
+ pp.nestedExpr(opener="{", closer="}", ignoreExpr=pp.LineEnd())
),
)
# ignore the whole thing...
ForLoop = add_element(
"ForLoop",
pp.Suppress(
pp.Keyword("for")
+ CallArgs
+ pp.nestedExpr(opener="{", closer="}", ignoreExpr=pp.LineEnd())
),
)
# ignore the whole thing...
ForLoopSingleLine = add_element(
"ForLoopSingleLine",
pp.Suppress(pp.Keyword("for") + CallArgs + pp.Literal(":") + pp.SkipTo(EOL)),
)
# ignore the whole thing...
FunctionCall = add_element("FunctionCall", pp.Suppress(Identifier + pp.nestedExpr()))
Scope = add_element("Scope", pp.Forward())
Statement = add_element(
"Statement",
pp.Group(
Load
| Include
| Option
| Requires
| QtNoMakeTools
| ForLoop
| ForLoopSingleLine
| DefineTestDefinition
| FunctionCall
| Operation
),
)
StatementLine = add_element("StatementLine", Statement + (EOL | pp.FollowedBy("}")))
StatementGroup = add_element(
"StatementGroup", pp.ZeroOrMore(StatementLine | Scope | pp.Suppress(EOL))
)
Block = add_element(
"Block",
pp.Suppress("{")
+ pp.Optional(EOL)
+ StatementGroup
+ pp.Optional(EOL)
+ pp.Suppress("}")
+ pp.Optional(EOL),
)
ConditionEnd = add_element(
"ConditionEnd",
pp.FollowedBy(
(pp.Optional(pp.White()) + (pp.Literal(":") | pp.Literal("{") | pp.Literal("|")))
),
)
ConditionPart1 = add_element(
"ConditionPart1", (pp.Optional("!") + Identifier + pp.Optional(BracedValue))
)
ConditionPart2 = add_element("ConditionPart2", pp.CharsNotIn("#{}|:=\\\n"))
ConditionPart = add_element(
"ConditionPart", (ConditionPart1 ^ ConditionPart2) + ConditionEnd
)
ConditionOp = add_element("ConditionOp", pp.Literal("|") ^ pp.Literal(":"))
ConditionWhiteSpace = add_element(
"ConditionWhiteSpace", pp.Suppress(pp.Optional(pp.White(" ")))
)
ConditionRepeated = add_element(
"ConditionRepeated", pp.ZeroOrMore(ConditionOp + ConditionWhiteSpace + ConditionPart)
)
Condition = add_element("Condition", pp.Combine(ConditionPart + ConditionRepeated))
Condition.setParseAction(lambda x: " ".join(x).strip().replace(":", " && ").strip(" && "))
# Weird thing like write_file(a)|error() where error() is the alternative condition
# which happens to be a function call. In this case there is no scope, but our code expects
# a scope with a list of statements, so create a fake empty statement.
ConditionEndingInFunctionCall = add_element(
"ConditionEndingInFunctionCall",
pp.Suppress(ConditionOp)
+ FunctionCall
+ pp.Empty().setParseAction(lambda x: [[]]).setResultsName("statements"),
)
SingleLineScope = add_element(
"SingleLineScope",
pp.Suppress(pp.Literal(":")) + pp.Group(Block | (Statement + EOL))("statements"),
)
MultiLineScope = add_element("MultiLineScope", Block("statements"))
SingleLineElse = add_element(
"SingleLineElse",
pp.Suppress(pp.Literal(":")) + (Scope | Block | (Statement + pp.Optional(EOL))),
)
MultiLineElse = add_element("MultiLineElse", Block)
ElseBranch = add_element("ElseBranch", pp.Suppress(Else) + (SingleLineElse | MultiLineElse))
# Scope is already add_element'ed in the forward declaration above.
Scope <<= pp.Group(
Condition("condition")
+ (SingleLineScope | MultiLineScope | ConditionEndingInFunctionCall)
+ pp.Optional(ElseBranch)("else_statements")
)
Grammar = StatementGroup("statements")
Grammar.ignore(pp.pythonStyleComment())
return Grammar
def parseFile(self, file: str) -> Tuple[pp.ParseResults, str]:
print(f'Parsing "{file}"...')
try:
with open(file, "r") as file_fd:
contents = file_fd.read()
# old_contents = contents
contents = fixup_comments(contents)
contents = fixup_linecontinuation(contents)
result = self._Grammar.parseString(contents, parseAll=True)
except pp.ParseException as pe:
print(pe.line)
print(f"{' ' * (pe.col-1)}^")
print(pe)
raise pe
return result, contents
def parseProFile(file: str, *, debug=False) -> Tuple[pp.ParseResults, str]:
parser = QmakeParser(debug=debug)
return parser.parseFile(file)