qt5base-lts/util/locale_database/ldml.py
Edward Welbourne 963931550d Check all matches for each XPath when searching
Previously, if we found one element with required attributes, we would
search into it and ignore any later elements also with those required
attributes. This meant that, if the first didn't contain the child
elements we were looking for, we'd fail to find what we sought, if it
was in a later matching element (e.g. with some ignored attributes).
We would then go on to look for a match in a later file, where there
might have been a match we should have found in the earlier file.

Check all matches, rather than only the first match in each file.  Do
the search in each file "in parallel" to save reparsing the XPath.
This clears the search code of rather hard-to-follow break/else
handling in loops; and currently makes no change to the generated
data.

Change-Id: I86b010e65b9a1fc1b79e5fdd45a5aeff1ed5d5d5
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
2020-04-02 19:43:43 +01:00

589 lines
24 KiB
Python

#############################################################################
##
## Copyright (C) 2020 The Qt Company Ltd.
## Contact: https://www.qt.io/licensing/
##
## This file is part of the test suite of the Qt Toolkit.
##
## $QT_BEGIN_LICENSE:GPL-EXCEPT$
## Commercial License Usage
## Licensees holding valid commercial Qt licenses may use this file in
## accordance with the commercial license agreement provided with the
## Software or, alternatively, in accordance with the terms contained in
## a written agreement between you and The Qt Company. For licensing terms
## and conditions see https://www.qt.io/terms-conditions. For further
## information use the contact form at https://www.qt.io/contact-us.
##
## GNU General Public License Usage
## Alternatively, this file may be used under the terms of the GNU
## General Public License version 3 as published by the Free Software
## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
## included in the packaging of this file. Please review the following
## information to ensure the GNU General Public License requirements will
## be met: https://www.gnu.org/licenses/gpl-3.0.html.
##
## $QT_END_LICENSE$
##
#############################################################################
"""Parsing the Locale Data Markup Language
It's an XML format, so the raw parsing of XML is, of course, delegated
to xml.dom.minidom; but it has its own specific schemata and some
funky rules for combining data from various files (inheritance between
locales). The use of it we're interested in is extraction of CLDR's
data, so some of the material here is specific to CLDR; see cldr.py
for how it is mainly used.
Provides various classes to wrap xml.dom's objects, specifically those
returned by minidom.parse() and their child-nodes:
Node -- wraps any node in the DOM tree
XmlScanner -- wraps the root element of a stand-alone XML file
Supplement -- specializes XmlScanner for supplemental data files
LocaleScanner -- wraps a locale's inheritance-chain of file roots
See individual classes for further detail.
"""
from localetools import Error
from dateconverter import convert_date
class Node (object):
"""Wrapper for an arbitrary DOM node.
Provides various ways to select chldren of a node. Selected child
nodes are returned wrapped as Node objects. A Node exposes the
raw DOM node it wraps via its .dom attribute."""
def __init__(self, elt, dullAttrs = None, draft = 0):
"""Wraps a DOM node for ease of access.
First argument, elt, is the DOM node to wrap.
Optional second argument, dullAttrs, should either be None or
map each LDML tag name to a list of the names of
non-distinguishing attributes for nodes with the given tag
name. If None is given, no distinguishing attribute checks are
performed.
(Optional third argument, draft, should only be supplied by
this class's creation of child nodes; it is the maximum draft
score of any ancestor of the new node.)"""
self.dom, self.__dull = elt, dullAttrs
try:
attr = elt.attributes['draft'].nodeValue
except KeyError:
self.draft = draft
else:
self.draft = max(draft, self.draftScore(attr))
def findAllChildren(self, tag, wanted = None, allDull = False):
"""All children that do have the given tag and attributes.
First argument is the tag: children with any other tag are
ignored.
Optional second argument, wanted, should either be None or map
attribute names to the values they must have. Only child nodes
with thes attributes set to the given values are yielded.
By default, nodes that have distinguishing attributes, other
than those specified in wanted, are ignored. Pass the allDull
parameter a true value to suppress this check."""
if self.__dull is None:
allDull = True
dull = () if allDull else self.__dull[tag]
for child in self.dom.childNodes:
if child.nodeType != child.ELEMENT_NODE:
continue
if child.nodeName != tag:
continue
if wanted:
try:
if any(child.attributes[k].nodeValue != v
for k, v in wanted.items()):
continue
except KeyError: # Some wanted attribute is missing
continue
if not (allDull or all(k in dull or k in wanted
for k in child.attributes.keys())):
continue
elif not (allDull or all(k in dull
for k in child.attributes.keys())):
continue
yield Node(child, self.__dull, self.draft)
def findUniqueChild(self, tag):
"""Returns the single child with the given nodeName.
Raises Error if there is no such child or there is more than
one."""
seq = self.findAllChildren(tag)
try:
node = seq.next()
except StopIteration:
raise Error('No child found where one was expected', tag)
for it in seq:
raise Error('Many children found where only one was expected', tag)
return node
@classmethod
def draftScore(cls, level):
"""Maps draft level names to numeric scores.
Single parameter, level, is the least sure value of the draft
attribute on a node that you're willing to accept; returns a
numeric value (lower is less drafty).
Tempting as it is to insist on low draft scores, there are
many locales in which pretty much every leaf is
unconfirmed. It may make sense to actually check each
XmlScanner object, or each node in each LocaleScanner's nodes
list, to see what its distribution of draft level looks like,
so as to set the acceptable draft score for its elements
accordingly. However, for the moment, we mostly just accept
all elements, regardless of draft values (the one exception is
am/pm indicators)."""
return cls.__draftScores.get(level, 5) if level else 0
# Implementation details:
__draftScores = dict(true = 4, unconfirmed = 3, provisional = 2,
contributed = 1, approved = 0, false = 0)
def _parseXPath(selector):
# Split "tag[attr=val][...]" into tag-name and attribute mapping
attrs = selector.split('[')
name = attrs.pop(0)
if attrs:
attrs = [x.strip() for x in attrs]
assert all(x.endswith(']') for x in attrs)
attrs = [x[:-1].split('=') for x in attrs]
assert all(len(x) in (1, 2) for x in attrs)
attrs = (('type', x[0]) if len(x) == 1 else x for x in attrs)
return name, dict(attrs)
def _iterateEach(iters):
# Flatten a two-layer iterator.
for it in iters:
for item in it:
yield item
class XmlScanner (object):
"""Wrap an XML file to enable XPath access to its nodes.
"""
def __init__(self, node):
self.root = node
def findNodes(self, xpath):
"""Return all nodes under self.root matching this xpath.
Ignores any excess attributes."""
elts = (self.root,)
for selector in xpath.split('/'):
tag, attrs = _parseXPath(selector)
elts = tuple(_iterateEach(e.findAllChildren(tag, attrs) for e in elts))
if not elts:
break
return elts
class Supplement (XmlScanner):
def find(self, xpath):
elts = self.findNodes(xpath)
for elt in _iterateEach(e.dom.childNodes if e.dom.childNodes else (e.dom,)
for e in elts):
if elt.attributes:
yield (elt.nodeName,
dict((k, v if isinstance(v, basestring) else v.nodeValue)
for k, v in elt.attributes.items()))
class LocaleScanner (object):
def __init__(self, name, nodes, root):
self.name, self.nodes, self.base = name, nodes, root
def find(self, xpath, default = None, draft = None):
"""XPath search for the content of an element.
Required argument, xpath, is the XPath to search for. Optional
second argument is a default value to use, if no such node is
found. Optional third argument is a draft score (see
Node.draftScore() for details); if given, leaf elements with
higher draft scores are ignored."""
try:
for elt in self.__find(xpath):
try:
if draft is None or elt.draft <= draft:
return elt.dom.firstChild.nodeValue
except (AttributeError, KeyError):
pass
except Error as e:
if default is None:
raise
return default
def tagCodes(self):
"""Yields four tag codes
The tag codes are language, script, country and variant; an
empty value for any of them indicates that no value was
provided. The values are obtained from the primary file's
top-level <identity> element. An Error is raised if any
top-level <alias> element of this file has a non-empty source
attribute; that attribute value is mentioned in the error's
message."""
root = self.nodes[0]
for alias in root.findAllChildren('alias', allDull=True):
try:
source = alias.dom.attributes['source'].nodeValue
except (KeyError, AttributeError):
pass
else:
raise Error('Alias to {}'.format(source))
ids = root.findUniqueChild('identity')
for code in ('language', 'script', 'territory', 'variant'):
for node in ids.findAllChildren(code, allDull=True):
try:
yield node.dom.attributes['type'].nodeValue
except (KeyError, AttributeError):
pass
else:
break # only want one value for each code
else: # No value for this code, use empty
yield ''
def currencyData(self, isoCode):
"""Fetches currency data for this locale.
Single argument, isoCode, is the ISO currency code for the
currency in use in the country. See also numericData, which
includes some currency formats.
"""
if isoCode:
stem = 'numbers/currencies/currency[{}]/'.format(isoCode)
symbol = self.find(stem + 'symbol', '')
name = ';'.join(
self.find(stem + 'displayName' + tail, '')
for tail in ('',) + tuple(
'[count={}]'.format(x) for x in ('zero', 'one', 'two', 'few', 'many', 'other')
)) + ';'
else:
symbol = name = ''
yield 'currencySymbol', symbol
yield 'currencyDisplayName', name
def numericData(self, lookup, complain = lambda text: None):
"""Generate assorted numeric data for the locale.
First argument, lookup, is a callable that maps a numbering
system's name to certain data about the system, as a mapping;
we expect this to have u'digits' as a key.
"""
system = self.find('numbers/defaultNumberingSystem')
stem = 'numbers/symbols[numberSystem={}]/'.format(system)
decimal = self.find(stem + 'decimal')
group = self.find(stem + 'group')
assert decimal != group, (self.name, system, decimal)
yield 'decimal', decimal
yield 'group', group
yield 'percent', self.find(stem + 'percentSign')
yield 'list', self.find(stem + 'list')
# FIXME: don't lower-case:
yield 'exp', self.find(stem + 'exponential').lower()
digits = lookup(system)['digits']
assert len(digits) == 10
zero = digits[0]
# Qt's number-formatting code assumes digits are consecutive:
assert all(ord(c) == i for i, c in enumerate(digits, ord(zero)))
yield 'zero', zero
plus = self.find(stem + 'plusSign')
minus = self.find(stem + 'minusSign')
yield 'plus', plus
yield 'minus', minus
# Currency formatting:
xpath = 'numbers/currencyFormats/currencyFormatLength/currencyFormat[accounting]/pattern'
try:
money = self.find(xpath.replace('Formats/',
'Formats[numberSystem={}]/'.format(system)))
except Error:
money = self.find(xpath)
money = self.__currencyFormats(money, plus, minus)
yield 'currencyFormat', money.next()
neg = ''
for it in money:
assert not neg, 'There should be at most one more pattern'
neg = it
yield 'currencyNegativeFormat', neg
def textPatternData(self):
for key in ('quotationStart', 'alternateQuotationEnd',
'quotationEnd', 'alternateQuotationStart'):
yield key, self.find('delimiters/' + key)
for key in ('start', 'middle', 'end'):
yield ('listPatternPart' + key.capitalize(),
self.__fromLdmlListPattern(self.find(
'listPatterns/listPattern/listPatternPart[{}]'.format(key))))
yield ('listPatternPartTwo',
self.__fromLdmlListPattern(self.find(
'listPatterns/listPattern/listPatternPart[2]')))
stem = 'dates/calendars/calendar[gregorian]/'
# TODO: is wide really the right width to use here ?
# abbreviated might be an option ... or try both ?
meridiem = stem + 'dayPeriods/dayPeriodContext[format]/dayPeriodWidth[wide]/'
for key in ('am', 'pm'):
yield key, self.find(meridiem + 'dayPeriod[{}]'.format(key),
draft = Node.draftScore('contributed'))
for pair in (('long', 'full'), ('short', 'short')):
for key in ('time', 'date'):
yield (pair[0] + key.capitalize() + 'Format',
convert_date(self.find(
stem + '{}Formats/{}FormatLength[{}]/{}Format/pattern'.format(
key, key, pair[1], key))))
def endonyms(self, language, script, country, variant):
# TODO: take variant into account ?
for seq in ((language, script, country),
(language, script), (language, country), (language,)):
if not all(seq):
continue
try:
yield ('languageEndonym',
self.find('localeDisplayNames/languages/language[{}]'
.format('_'.join(seq))))
except Error:
pass
else:
break
else:
# grumble(failed to find endonym for language)
yield 'languageEndonym', ''
yield ('countryEndonym',
self.find('localeDisplayNames/territories/territory[{}]'
.format(country), ''))
def unitData(self):
yield ('byte_unit',
self.find('units/unitLength[long]/unit[digital-byte]/displayName',
'bytes'))
unit = self.__findUnit('', 'B')
cache = [] # Populated by the SI call, to give hints to the IEC call
yield ('byte_si_quantified',
';'.join(self.__unitCount('', unit, cache)))
# IEC 60027-2
# http://physics.nist.gov/cuu/Units/binary.html
yield ('byte_iec_quantified',
';'.join(self.__unitCount('bi', 'iB', cache)))
def calendarNames(self, calendars):
namings = self.__nameForms
for cal in calendars:
stem = 'dates/calendars/calendar[' + cal + ']/months/'
for key, mode, size in namings:
prop = 'monthContext[' + mode + ']/monthWidth[' + size + ']/'
yield (key + 'Months_' + cal,
';'.join(self.find(stem + prop + 'month[{}]'.format(i))
for i in range(1, 13)) + ';')
# Day data (for Gregorian, at least):
stem = 'dates/calendars/calendar[gregorian]/days/'
days = ('sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat')
for (key, mode, size) in namings:
prop = 'dayContext[' + mode + ']/dayWidth[' + size + ']/day'
yield (key + 'Days',
';'.join(self.find(stem + prop + '[' + day + ']')
for day in days) + ';')
# Implementation details
__nameForms = (
('standaloneLong', 'stand-alone', 'wide'),
('standaloneShort', 'stand-alone', 'abbreviated'),
('standaloneNarrow', 'stand-alone', 'narrow'),
('long', 'format', 'wide'),
('short', 'format', 'abbreviated'),
('narrow', 'format', 'narrow'),
) # Used for month and day names
def __find(self, xpath):
retries = [ xpath.split('/') ]
while retries:
tags, elts, roots = retries.pop(), self.nodes, (self.base.root,)
for selector in tags:
tag, attrs = _parseXPath(selector)
elts = tuple(_iterateEach(e.findAllChildren(tag, attrs) for e in elts))
if not elts:
break
else: # Found matching elements
# Possibly filter elts to prefer the least drafty ?
for elt in elts:
yield elt
# Process roots separately: otherwise the alias-processing
# is excessive.
for i, selector in enumerate(tags):
tag, attrs = _parseXPath(selector)
for alias in tuple(_iterateEach(r.findAllChildren('alias', allDull=True)
for r in roots)):
if alias.dom.attributes['source'].nodeValue == 'locale':
replace = alias.dom.attributes['path'].nodeValue.split('/')
retries.append(self.__xpathJoin(tags[:i], replace, tags[i:]))
roots = tuple(_iterateEach(r.findAllChildren(tag, attrs) for r in roots))
if not roots:
if retries: # Let outer loop fall back on an alias path:
break
sought = '/'.join(tags)
if sought != xpath:
sought += ' (for {})'.format(xpath)
raise Error('All lack child {} for {} in {}'.format(
selector, sought, self.name))
else: # Found matching elements
for elt in roots:
yield elt
sought = '/'.join(tags)
if sought != xpath:
sought += ' (for {})'.format(xpath)
raise Error('No {} in {}'.format(sought, self.name))
def __findUnit(self, keySuffix, quantify, fallback=''):
# The displayName for a quantified unit in en.xml is kByte
# (even for unitLength[narrow]) instead of kB (etc.), so
# prefer any unitPattern provided, but prune its placeholder:
for size in ('short', 'narrow'): # TODO: reverse order ?
stem = 'units/unitLength[{}]/unit[digital-{}byte]/'.format(size + keySuffix, quantify)
for count in ('many', 'few', 'two', 'other', 'zero', 'one'):
try:
ans = self.find(stem + 'unitPattern[count={}]'.format(count))
except Error:
continue
# TODO: do count-handling, instead of discarding placeholders
if False: # TODO: do it this way, instead !
ans = ans.replace('{0}', '').strip()
elif ans.startswith('{0}'):
ans = ans[3:].lstrip()
if ans:
return ans
try:
return self.find(stem + 'displayName')
except Error:
pass
return fallback
def __unitCount(self, keySuffix, suffix, cache,
# Stop at exa/exbi: 16 exbi = 2^{64} < zetta =
# 1000^7 < zebi = 2^{70}, the next quantifiers up:
siQuantifiers = ('kilo', 'mega', 'giga', 'tera', 'peta', 'exa')):
"""Work out the unit quantifiers.
Unfortunately, the CLDR data only go up to terabytes and we
want all the way to exabytes; but we can recognize the SI
quantifiers as prefixes, strip and identify the tail as the
localized translation for 'B' (e.g. French has 'octet' for
'byte' and uses ko, Mo, Go, To from which we can extrapolate
Po, Eo).
Should be called first for the SI quantifiers, with suffix =
'B', then for the IEC ones, with suffix = 'iB'; the list cache
(initially empty before first call) is used to let the second
call know what the first learned about the localized unit.
"""
if suffix == 'iB': # second call, re-using first's cache
if cache:
byte = cache.pop()
if all(byte == k for k in cache):
suffix = 'i' + byte
for q in siQuantifiers:
# Those don't (yet, v36) exist in CLDR, so we always get the fall-back:
yield self.__findUnit(keySuffix, q[:2], q[0].upper() + suffix)
else: # first call
tail = suffix = suffix or 'B'
for q in siQuantifiers:
it = self.__findUnit(keySuffix, q)
# kB for kilobyte, in contrast with KiB for IEC:
q = q[0] if q == 'kilo' else q[0].upper()
if not it:
it = q + tail
elif it.startswith(q):
rest = it[1:]
tail = rest if all(rest == k for k in cache) else suffix
cache.append(rest)
yield it
@staticmethod
def __currencyFormats(patterns, plus, minus):
for p in patterns.split(';'):
p = p.replace('0', '#').replace(',', '').replace('.', '')
try:
cut = p.find('#') + 1
except ValueError:
pass
else:
p = p[:cut] + p[cut:].replace('#', '')
p = p.replace('#', "%1")
# According to http://www.unicode.org/reports/tr35/#Number_Format_Patterns
# there can be doubled or trippled currency sign, however none of the
# locales use that.
p = p.replace(u'\xa4', "%2")
# Single quote goes away, but double goes to single:
p = p.replace("''", '###').replace("'", '').replace('###', "'")
# Use number system's signs:
p = p.replace('+', plus).replace('-', minus)
yield p
@staticmethod
def __fromLdmlListPattern(pattern):
# This is a very limited parsing of the format for list pattern part only.
return pattern.replace('{0}', '%1').replace('{1}', '%2').replace('{2}', '%3')
@staticmethod
def __fromLdmlPath(seq): # tool function for __xpathJoin()
"""Convert LDML's [@name='value'] to our [name=value] form."""
for it in seq:
# First dismember it:
attrs = it.split('[')
tag = attrs.pop(0)
if not attrs: # Short-cut the easy case:
yield it
continue
assert all(x.endswith(']') for x in attrs)
attrs = [x[:-1].split('=') for x in attrs]
# Then fix each attribute specification in it:
attrs = [(x[0][1:] if x[0].startswith('@') else x[0],
x[1][1:-1] if x[1].startswith("'") and x[1].endswith("'") else x[1])
for x in attrs]
# Finally, put it all back together:
attrs = ['='.join(x) + ']' for x in attrs]
attrs.insert(0, tag)
yield '['.join(attrs)
@classmethod
def __xpathJoin(cls, head, insert, tail):
"""Join three lists of XPath selectors.
Each of head, insert and tail is a sequence of selectors but
insert may start with some uses of '..', that we want to
resolve away, and may use LDML's attribute format, that we
want to convert to our format."""
while insert and insert[0] == '..':
insert.pop(0)
head.pop()
return head + list(cls.__fromLdmlPath(insert)) + tail