81cf23c7a7
When doing XPATH searches, child nodes that have distinguished
attributes that were not asked for should be skipped. This is part of
the LDML spec and matters when resolving locale inheritance. Scan the
LDML DTD (previously only scanned for the CLDR version) to find which
attributes of which tags are ignorable - all others are distinguished
- and take the result into account when performing XPATH searches.
The XPath we were using for currency formats wasn't excluding
currencyFormatLength elements with type="short" and patterns specific
to thousands (and larger multiples); this is fixed by taking
distinguished attributes into account. However, the XPATH also wasn't
specifying the always distinguished attribute type="standard" that
was, in practice, used for nearly all locales that weren't (wrongly)
using short-forms for thousands; so type="standard" is now made
explicit, so as to minimize the diff.
This leaves only twenty-one locales with a negative currency formats.
A later commit shall switch to using accounting by default (it falls
back via an alias to standard, in any case), thereby restoring the two
mentioned below that were using it by accident, but the present change
gives the minimal diff here.
Thousands-specific formats replaced with sensible ones:
* zh_Hant_{HK,MO} (Traditional Mandarin, Hong Kong and Macau)
* eo_001 (Esperanto)
* fr_CA (Canadian French)
* ha_* (Hausa, when not written in Arabic)
* es_{GT,MX,US} (Spanish - Guatemala, Mexico, USA)
* sw_KE (Swahili, Kenya)
* yi_001 (Yiddish)
* mfe_MU (Morisyen, Mauritius)
* lag_TZ (Langi, Tanzania)
* mgh_MZ (Makhuwa Meetto, Mozambique)
* wae_CH (Walser, Switzerland)
* kkj_CM (Kako, Cameroon)
* lkt_US (Lakota, USA)
* pa_Arab_PK (Punjabi, in Arabic script, as used in Pakistan; uses
arabext number system, whose currency falls back to latn's, for
which pa_Arab over-rides the thousands-format).
Format changed from an over-ridden type="accounting" to standard (so
these lost a negative-specific form) in:
* en_SI (English, Slovenia)
* es_DO (Spanish, Dominican Republic; same)
For some locales we were picking up over-rides of narrow or short list
formats, or formats for or-lists or unit-lists rather than and-lists,
in place of the standard list format, that these locales don't
over-ride, provided by a parent locale. This changed list formats for:
* en_CA, en_IN (dropped "Oxford" comma before "and")
* qu_* (Quechua; dropped "utaq", presumably meaning "and")
* ur_IN (Urdu, India; was using unit-list formats)
[ChangeLog][QtCore][QLocale] Data used for currency formats in several
locales and list patterns in some locales have changed due to now
parsing the CLDR data more faithfully.
Fixes: QTBUG-81344
Change-Id: I6b95c6c37db92df167153767c1b103becfb0ac98
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
724 lines
30 KiB
Python
724 lines
30 KiB
Python
#############################################################################
|
|
##
|
|
## Copyright (C) 2020 The Qt Company Ltd.
|
|
## Contact: https://www.qt.io/licensing/
|
|
##
|
|
## This file is part of the test suite of the Qt Toolkit.
|
|
##
|
|
## $QT_BEGIN_LICENSE:GPL-EXCEPT$
|
|
## Commercial License Usage
|
|
## Licensees holding valid commercial Qt licenses may use this file in
|
|
## accordance with the commercial license agreement provided with the
|
|
## Software or, alternatively, in accordance with the terms contained in
|
|
## a written agreement between you and The Qt Company. For licensing terms
|
|
## and conditions see https://www.qt.io/terms-conditions. For further
|
|
## information use the contact form at https://www.qt.io/contact-us.
|
|
##
|
|
## GNU General Public License Usage
|
|
## Alternatively, this file may be used under the terms of the GNU
|
|
## General Public License version 3 as published by the Free Software
|
|
## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
|
|
## included in the packaging of this file. Please review the following
|
|
## information to ensure the GNU General Public License requirements will
|
|
## be met: https://www.gnu.org/licenses/gpl-3.0.html.
|
|
##
|
|
## $QT_END_LICENSE$
|
|
##
|
|
#############################################################################
|
|
"""Digesting the CLDR's data.
|
|
|
|
Provides two classes:
|
|
CldrReader -- driver for reading CLDR data
|
|
CldrAccess -- used by the reader to access the tree of data files
|
|
|
|
The former should normally be all you need to access.
|
|
See individual classes for further detail.
|
|
"""
|
|
|
|
from xml.dom import minidom
|
|
from weakref import WeakValueDictionary as CacheDict
|
|
import os
|
|
|
|
from ldml import Error, Node, XmlScanner, Supplement, LocaleScanner
|
|
from qlocalexml import Locale
|
|
|
|
class CldrReader (object):
|
|
def __init__(self, root, grumble = lambda msg: None, whitter = lambda msg: None):
|
|
"""Set up a reader object for reading CLDR data.
|
|
|
|
Single parameter, root, is the file-system path to the root of
|
|
the unpacked CLDR archive; its common/ sub-directory should
|
|
contain dtd/, main/ and supplemental/ sub-directories.
|
|
|
|
Optional second argument, grumble, is a callable that logs
|
|
warnings and complaints, e.g. sys.stderr.write would be a
|
|
suitable callable. The default is a no-op that ignores its
|
|
single argument. Optional third argument is similar, used for
|
|
less interesting output; pass sys.stderr.write for it for
|
|
verbose output."""
|
|
self.root = CldrAccess(root)
|
|
self.whitter, self.grumble = whitter, grumble
|
|
|
|
def likelySubTags(self):
|
|
"""Generator for likely subtag information.
|
|
|
|
Yields pairs (have, give) of 4-tuples; if what you have
|
|
matches the left member, giving the right member is probably
|
|
sensible. Each 4-tuple's entries are the full names of a
|
|
language, a script, a country (strictly territory) and a
|
|
variant (currently ignored)."""
|
|
skips = []
|
|
for got, use in self.root.likelySubTags():
|
|
try:
|
|
have = self.__parseTags(got)
|
|
give = self.__parseTags(use)
|
|
except Error as e:
|
|
if ((use.startswith(got) or got.startswith('und_'))
|
|
and e.message.startswith('Unknown ') and ' code ' in e.message):
|
|
skips.append(use)
|
|
else:
|
|
self.grumble('Skipping likelySubtag "{}" -> "{}" ({})\n'.format(got, use, e.message))
|
|
continue
|
|
if all(code.startswith('Any') and code[3].isupper() for code in have[:-1]):
|
|
continue
|
|
|
|
give = (give[0],
|
|
# Substitute according to http://www.unicode.org/reports/tr35/#Likely_Subtags
|
|
have[1] if give[1] == 'AnyScript' else give[1],
|
|
have[2] if give[2] == 'AnyCountry' else give[2],
|
|
give[3]) # AnyVariant similarly ?
|
|
|
|
yield have, give
|
|
|
|
if skips:
|
|
# TODO: look at LDML's reserved locale tag names; they
|
|
# show up a lot in this, and may be grounds for filtering
|
|
# more out.
|
|
pass # self.__wrapped(self.whitter, 'Skipping likelySubtags (for unknown codes): ', skips)
|
|
|
|
def readLocales(self, calendars = ('gregorian',)):
|
|
locales = tuple(self.__allLocales(calendars))
|
|
return dict(((k.language_id, k.script_id, k.country_id, k.variant_code),
|
|
k) for k in locales)
|
|
|
|
def __allLocales(self, calendars):
|
|
def skip(locale, reason):
|
|
return 'Skipping defaultContent locale "{}" ({})\n'.format(locale, reason)
|
|
|
|
for locale in self.root.defaultContentLocales:
|
|
try:
|
|
language, script, country, variant = self.__splitLocale(locale)
|
|
except ValueError:
|
|
self.whitter(skip(locale, 'only language tag'))
|
|
continue
|
|
|
|
if not (script or country):
|
|
self.grumble(skip(locale, 'second tag is neither script nor territory'))
|
|
continue
|
|
|
|
if not (language and country):
|
|
continue
|
|
|
|
try:
|
|
yield self.__getLocaleData(self.root.locale(locale), calendars,
|
|
language, script, country, variant)
|
|
except Error as e:
|
|
self.grumble(skip(locale, e.message))
|
|
|
|
for locale in self.root.fileLocales:
|
|
try:
|
|
chain = self.root.locale(locale)
|
|
language, script, country, variant = chain.tagCodes()
|
|
assert language
|
|
# TODO: this skip should probably be based on likely
|
|
# sub-tags, instead of empty country: if locale has a
|
|
# likely-subtag expansion, that's what QLocale uses,
|
|
# and we'll be saving its data for the expanded locale
|
|
# anyway, so don't need to record it for itself.
|
|
# See also QLocaleXmlReader.loadLocaleMap's grumble.
|
|
if not country:
|
|
continue
|
|
yield self.__getLocaleData(chain, calendars, language, script, country, variant)
|
|
except Error as e:
|
|
self.grumble('Skipping file locale "{}" ({})\n'.format(locale, e.message))
|
|
|
|
import textwrap
|
|
@staticmethod
|
|
def __wrapped(writer, prefix, tokens, wrap = textwrap.wrap):
|
|
writer('\n'.join(wrap(prefix + ', '.join(tokens),
|
|
subsequent_indent=' ', width=80)) + '\n')
|
|
del textwrap
|
|
|
|
def __parseTags(self, locale):
|
|
tags = self.__splitLocale(locale)
|
|
language = tags.next()
|
|
script = country = variant = ''
|
|
try:
|
|
script, country, variant = tags
|
|
except ValueError:
|
|
pass
|
|
return tuple(p[1] for p in self.root.codesToIdName(language, script, country, variant))
|
|
|
|
def __splitLocale(self, name):
|
|
"""Generate (language, script, territory, variant) from a locale name
|
|
|
|
Ignores any trailing fields (with a warning), leaves script (a
|
|
capitalised four-letter token), territory (either a number or
|
|
an all-uppercase token) or variant (upper case and digits)
|
|
empty if unspecified. Only generates one entry if name is a
|
|
single tag (i.e. contains no underscores). Always yields 1 or
|
|
4 values, never 2 or 3."""
|
|
tags = iter(name.split('_'))
|
|
yield tags.next() # Language
|
|
tag = tags.next() # may raise StopIteration
|
|
|
|
# Script is always four letters, always capitalised:
|
|
if len(tag) == 4 and tag[0].isupper() and tag[1:].islower():
|
|
yield tag
|
|
try:
|
|
tag = tags.next()
|
|
except StopIteration:
|
|
tag = ''
|
|
else:
|
|
yield ''
|
|
|
|
# Territory is upper-case or numeric:
|
|
if tag and tag.isupper() or tag.isdigit():
|
|
yield tag
|
|
try:
|
|
tag = tags.next()
|
|
except StopIteration:
|
|
tag = ''
|
|
else:
|
|
yield ''
|
|
|
|
# Variant can be any mixture of upper-case and digits.
|
|
if tag and all(c.isupper() or c.isdigit() for c in tag):
|
|
yield tag
|
|
tag = ''
|
|
else:
|
|
yield ''
|
|
|
|
# If nothing is left, StopIteration will avoid the warning:
|
|
if not tag:
|
|
tag = tags.next()
|
|
self.grumble('Ignoring unparsed cruft {} in {}\n'.format('_'.join(tag + tuple(tags)), name))
|
|
|
|
def __getLocaleData(self, scan, calendars, language, script, country, variant):
|
|
ids, names = zip(*self.root.codesToIdName(language, script, country, variant))
|
|
assert ids[0] > 0 and ids[2] > 0, (language, script, country, variant)
|
|
locale = Locale(
|
|
language = names[0], language_code = language, language_id = ids[0],
|
|
script = names[1], script_code = script, script_id = ids[1],
|
|
country = names[2], country_code = country, country_id = ids[2],
|
|
variant_code = variant)
|
|
|
|
firstDay, weStart, weEnd = self.root.weekData(country)
|
|
assert all(day in ('mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun')
|
|
for day in (firstDay, weStart, weEnd))
|
|
|
|
locale.update(firstDayOfWeek = firstDay,
|
|
weekendStart = weStart,
|
|
weekendEnd = weEnd)
|
|
|
|
iso, digits, rounding = self.root.currencyData(country)
|
|
locale.update(currencyIsoCode = iso,
|
|
currencyDigits = int(digits),
|
|
currencyRounding = int(rounding))
|
|
|
|
locale.update(scan.currencyData(iso))
|
|
locale.update(scan.numericData(self.root.numberSystem, self.whitter))
|
|
locale.update(scan.textPatternData())
|
|
locale.update(scan.endonyms(language, script, country, variant))
|
|
locale.update(scan.unitData()) # byte, kB, MB, GB, ..., KiB, MiB, GiB, ...
|
|
locale.update(scan.calendarNames(calendars)) # Names of days and months
|
|
|
|
return locale
|
|
|
|
# Note: various caches assume this class is a singleton, so the
|
|
# "default" value for a parameter no caller should pass can serve as
|
|
# the cache. If a process were to instantiate this class with distinct
|
|
# roots, each cache would be filled by the first to need it !
|
|
class CldrAccess (object):
|
|
def __init__(self, root):
|
|
"""Set up a master object for accessing CLDR data.
|
|
|
|
Single parameter, root, is the file-system path to the root of
|
|
the unpacked CLDR archive; its common/ sub-directory should
|
|
contain dtd/, main/ and supplemental/ sub-directories."""
|
|
self.root = root
|
|
|
|
def xml(self, *path):
|
|
"""Load a single XML file and return its root element as an XmlScanner.
|
|
|
|
The path is interpreted relative to self.root"""
|
|
return XmlScanner(Node(self.__xml(path)))
|
|
|
|
def supplement(self, name):
|
|
"""Loads supplemental data as a Supplement object.
|
|
|
|
The name should be that of a file in common/supplemental/, without path.
|
|
"""
|
|
return Supplement(Node(self.__xml(('common', 'supplemental', name))))
|
|
|
|
def locale(self, name):
|
|
"""Loads all data for a locale as a LocaleScanner object.
|
|
|
|
The name should be a locale name; adding suffix '.xml' to it
|
|
should usually yield a file in common/main/. The returned
|
|
LocaleScanner object packages this file along with all those
|
|
from which it inherits; its methods know how to handle that
|
|
inheritance, where relevant."""
|
|
return LocaleScanner(name, self.__localeRoots(name), self.__rootLocale)
|
|
|
|
@property
|
|
def fileLocales(self, joinPath = os.path.join, listDirectory = os.listdir,
|
|
splitExtension = os.path.splitext):
|
|
"""Generator for locale IDs seen in file-names.
|
|
|
|
All *.xml other than root.xml in common/main/ are assumed to
|
|
identify locales."""
|
|
for name in listDirectory(joinPath(self.root, 'common', 'main')):
|
|
stem, ext = splitExtension(name)
|
|
if ext == '.xml' and stem != 'root':
|
|
yield stem
|
|
|
|
@property
|
|
def defaultContentLocales(self):
|
|
"""Generator for the default content locales."""
|
|
for name, attrs in self.supplement('supplementalMetadata.xml').find('metadata/defaultContent'):
|
|
try:
|
|
locales = attrs['locales']
|
|
except KeyError:
|
|
pass
|
|
else:
|
|
for locale in locales.split():
|
|
yield locale
|
|
|
|
def likelySubTags(self):
|
|
for ignore, attrs in self.supplement('likelySubtags.xml').find('likelySubtags'):
|
|
yield attrs['from'], attrs['to']
|
|
|
|
def numberSystem(self, system):
|
|
"""Get a description of a numbering system.
|
|
|
|
Returns a mapping, with keys u'digits', u'type' and u'id'; the
|
|
value for this last is system. Raises KeyError for unknown
|
|
number system, ldml.Error on failure to load data."""
|
|
try:
|
|
return self.__numberSystems[system]
|
|
except KeyError:
|
|
raise Error('Unsupported number system: {}'.format(system))
|
|
|
|
def weekData(self, country):
|
|
"""Data on the weekly cycle.
|
|
|
|
Returns a triple (W, S, E) of en's short names for week-days;
|
|
W is the first day of the week, S the start of the week-end
|
|
and E the end of the week-end. Where data for a country is
|
|
unavailable, the data for CLDR's territory 001 (The World) is
|
|
used."""
|
|
try:
|
|
return self.__weekData[country]
|
|
except KeyError:
|
|
return self.__weekData['001']
|
|
|
|
def currencyData(self, country):
|
|
"""Returns currency data for the given country code.
|
|
|
|
Return value is a tuple (ISO4217 code, digit count, rounding
|
|
mode). If CLDR provides no data for this country, ('', 2, 1)
|
|
is the default result.
|
|
"""
|
|
try:
|
|
return self.__currencyData[country]
|
|
except KeyError:
|
|
return '', 2, 1
|
|
|
|
def codesToIdName(self, language, script, country, variant = ''):
|
|
"""Maps each code to the appropriate ID and name.
|
|
|
|
Returns a 4-tuple of (ID, name) pairs corresponding to the
|
|
language, script, country and variant given. Raises a
|
|
suitable error if any of them is unknown, indicating all that
|
|
are unknown plus suitable names for any that could sensibly be
|
|
added to enumdata.py to make them known.
|
|
|
|
Until we implement variant support (QTBUG-81051), the fourth
|
|
member of the returned tuple is always 0 paired with a string
|
|
that should not be used."""
|
|
enum = self.__enumMap
|
|
try:
|
|
return (enum('language')[language],
|
|
enum('script')[script],
|
|
enum('country')[country],
|
|
enum('variant')[variant])
|
|
except KeyError:
|
|
pass
|
|
|
|
parts, values = [], [language, script, country, variant]
|
|
for index, key in enumerate(('language', 'script', 'country', 'variant')):
|
|
naming, enums = self.__codeMap(key), enum(key)
|
|
value = values[index]
|
|
if value not in enums:
|
|
text = '{} code {}'.format(key, value)
|
|
name = naming.get(value)
|
|
if name and value != 'POSIX':
|
|
text += u' (could add {})'.format(name)
|
|
parts.append(text)
|
|
if len(parts) > 1:
|
|
parts[-1] = 'and ' + parts[-1]
|
|
assert parts
|
|
raise Error('Unknown ' + ', '.join(parts),
|
|
language, script, country, variant)
|
|
|
|
def readWindowsTimeZones(self, lookup): # For use by cldr2qtimezone.py
|
|
"""Digest CLDR's MS-Win time-zone name mapping.
|
|
|
|
MS-Win have their own eccentric names for time-zones. CLDR
|
|
helpfully provides a translation to more orthodox names.
|
|
|
|
Singe argument, lookup, is a mapping from known MS-Win names
|
|
for locales to a unique integer index (starting at 1).
|
|
|
|
The XML structure we read has the form:
|
|
|
|
<supplementalData>
|
|
<windowsZones>
|
|
<mapTimezones otherVersion="..." typeVersion="...">
|
|
<!-- (UTC-08:00) Pacific Time (US & Canada) -->
|
|
<mapZone other="Pacific Standard Time" territory="001" type="America/Los_Angeles"/>
|
|
<mapZone other="Pacific Standard Time" territory="CA" type="America/Vancouver America/Dawson America/Whitehorse"/>
|
|
<mapZone other="Pacific Standard Time" territory="US" type="America/Los_Angeles America/Metlakatla"/>
|
|
<mapZone other="Pacific Standard Time" territory="ZZ" type="PST8PDT"/>
|
|
</mapTimezones>
|
|
</windowsZones>
|
|
</supplementalData>
|
|
"""
|
|
zones = self.supplement('windowsZones.xml')
|
|
enum = self.__enumMap('country')
|
|
badZones, unLands, defaults, windows = set(), set(), {}, {}
|
|
|
|
for name, attrs in zones.find('windowsZones/mapTimezones'):
|
|
if name != 'mapZone':
|
|
continue
|
|
|
|
wid, code = attrs['other'], attrs['territory']
|
|
data = dict(windowsId = wid,
|
|
countryCode = code,
|
|
ianaList = attrs['type'])
|
|
|
|
try:
|
|
key = lookup[wid]
|
|
except KeyError:
|
|
badZones.add(wid)
|
|
key = 0
|
|
data['windowsKey'] = key
|
|
|
|
if code == u'001':
|
|
defaults[key] = data['ianaList']
|
|
else:
|
|
try:
|
|
cid, name = enum[code]
|
|
except KeyError:
|
|
unLands.append(code)
|
|
continue
|
|
data.update(countryId = cid, country = name)
|
|
windows[key, cid] = data
|
|
|
|
if unLands:
|
|
raise Error('Unknown country codes, please add to enumdata.py: '
|
|
+ ', '.join(sorted(unLands)))
|
|
|
|
if badZones:
|
|
raise Error('Unknown Windows IDs, please add to cldr2qtimezone.py: '
|
|
+ ', '.join(sorted(badZones)))
|
|
|
|
return self.cldrVersion, defaults, windows
|
|
|
|
@property
|
|
def cldrVersion(self):
|
|
# Evaluate so as to ensure __cldrVersion is set:
|
|
self.__unDistinguishedAttributes
|
|
return self.__cldrVersion
|
|
|
|
# Implementation details
|
|
def __xml(self, path, cache = CacheDict(), read = minidom.parse, joinPath = os.path.join):
|
|
try:
|
|
doc = cache[path]
|
|
except KeyError:
|
|
cache[path] = doc = read(joinPath(self.root, *path)).documentElement
|
|
return doc
|
|
|
|
def __open(self, path, joinPath=os.path.join):
|
|
return open(joinPath(self.root, *path))
|
|
|
|
@property
|
|
def __rootLocale(self, cache = []):
|
|
if not cache:
|
|
cache.append(self.xml('common', 'main', 'root.xml'))
|
|
return cache[0]
|
|
|
|
@property
|
|
def __supplementalData(self, cache = []):
|
|
if not cache:
|
|
cache.append(self.supplement('supplementalData.xml'))
|
|
return cache[0]
|
|
|
|
@property
|
|
def __numberSystems(self, cache = {}, joinPath=os.path.join):
|
|
if not cache:
|
|
for ignore, attrs in self.supplement('numberingSystems.xml').find('numberingSystems'):
|
|
if ord(attrs.get('digits', u'\x10000')[0]) > 0xffff:
|
|
# FIXME, QTBUG-69324: make this redundant:
|
|
# omit number system if zero doesn't fit in single-char16 UTF-16 :-(
|
|
continue
|
|
|
|
cache[attrs['id']] = attrs
|
|
assert cache
|
|
return cache
|
|
|
|
@property
|
|
def __weekData(self, cache = {}):
|
|
if not cache:
|
|
firstDay, weStart, weEnd = self.__getWeekData()
|
|
# Massage those into an easily-consulted form:
|
|
# World defaults given for code '001':
|
|
mon, sat, sun = firstDay['001'], weStart['001'], weEnd['001']
|
|
lands = set(firstDay) | set(weStart) | set(weEnd)
|
|
cache.update((land,
|
|
(firstDay.get(land, mon), weStart.get(land, sat), weEnd.get(land, sun)))
|
|
for land in lands)
|
|
assert cache
|
|
return cache
|
|
|
|
def __getWeekData(self):
|
|
"""Scan for data on the weekly cycle.
|
|
|
|
Yields three mappings from locales to en's short names for
|
|
week-days; if a locale isn't a key of a given mapping, it
|
|
should use the '001' (world) locale's value. The first mapping
|
|
gives the day on which the week starts, the second gives the
|
|
day on which the week-end starts, the third gives the last day
|
|
of the week-end."""
|
|
source = self.__supplementalData
|
|
for key in ('firstDay', 'weekendStart', 'weekendEnd'):
|
|
result = {}
|
|
for ignore, attrs in source.find('weekData/' + key):
|
|
assert ignore == key
|
|
day = attrs['day']
|
|
assert day in ('mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'), day
|
|
if 'alt' in attrs:
|
|
continue
|
|
for loc in attrs.get('territories', '').split():
|
|
result[loc] = day
|
|
yield result
|
|
|
|
@property
|
|
def __currencyData(self, cache = {}):
|
|
if not cache:
|
|
source = self.__supplementalData
|
|
for elt in source.findNodes('currencyData/region'):
|
|
iso, digits, rounding = '', 2, 1
|
|
try:
|
|
country = elt.dom.attributes['iso3166'].nodeValue
|
|
except KeyError:
|
|
continue
|
|
for child in elt.findAllChildren('currency'):
|
|
try:
|
|
if child.dom.attributes['tender'].nodeValue == 'false':
|
|
continue
|
|
except KeyError:
|
|
pass
|
|
try:
|
|
child.dom.attributes['to'] # Is set if this element has gone out of date.
|
|
except KeyError:
|
|
iso = child.dom.attributes['iso4217'].nodeValue
|
|
break
|
|
if iso:
|
|
for tag, data in source.find(
|
|
'currencyData/fractions/info[iso4217={}]'.format(iso)):
|
|
digits = data['digits']
|
|
rounding = data['rounding']
|
|
cache[country] = iso, digits, rounding
|
|
assert cache
|
|
|
|
return cache
|
|
|
|
@property
|
|
def __unDistinguishedAttributes(self, cache = {}, joinPath = os.path.join):
|
|
"""Mapping from tag names to lists of attributes.
|
|
|
|
LDML defines some attributes as 'distinguishing': if a node
|
|
has distinguishing attributes that weren't specified in an
|
|
XPath, a search on that XPath should exclude the node's
|
|
children.
|
|
|
|
This property is a mapping from tag names to tuples of
|
|
attribute names that *aren't* distinguishing for that tag.
|
|
Its value is cached (so its costly computation isonly done
|
|
once) and there's a side-effect of populating its cache: it
|
|
sets self.__cldrVersion to the value found in ldml.dtd, during
|
|
parsing."""
|
|
if not cache:
|
|
cache.update(self.__scanLdmlDtd())
|
|
assert cache
|
|
|
|
return cache
|
|
|
|
def __scanLdmlDtd(self, joinPath = os.path.join):
|
|
"""Scan the LDML DTD, record CLDR version
|
|
|
|
Yields (tag, attrs) pairs: on elements with a given tag,
|
|
attributes named in its attrs (a tuple) may be ignored in an
|
|
XPath search; other attributes are distinguished attributes,
|
|
in the terminology of LDML's locale-inheritance rules.
|
|
|
|
Sets self.__cldrVersion as a side-effect, since this
|
|
information is found in the same file."""
|
|
with self.__open(('common', 'dtd', 'ldml.dtd')) as dtd:
|
|
tag, ignored, last = None, None, None
|
|
|
|
for line in dtd:
|
|
if line.startswith('<!ELEMENT '):
|
|
if ignored:
|
|
assert tag
|
|
yield tag, tuple(ignored)
|
|
tag, ignored, last = line.split()[1], [], None
|
|
continue
|
|
|
|
if line.startswith('<!ATTLIST '):
|
|
assert tag is not None
|
|
parts = line.split()
|
|
assert parts[1] == tag
|
|
last = parts[2]
|
|
if parts[1:5] == ['version', 'cldrVersion', 'CDATA', '#FIXED']:
|
|
# parts[5] is the version, in quotes, although the final > might be stuck on its end:
|
|
self.__cldrVersion = parts[5].split('"')[1]
|
|
continue
|
|
|
|
# <!ELEMENT...>s can also be @METADATA, but not @VALUE:
|
|
if '<!--@VALUE-->' in line or (last and '<!--@METADATA-->' in line):
|
|
assert last is not None
|
|
assert ignored is not None
|
|
assert tag is not None
|
|
ignored.append(last)
|
|
last = None # No attribute is both value and metadata
|
|
|
|
if tag and ignored:
|
|
yield tag, tuple(ignored)
|
|
|
|
def __enumMap(self, key, cache = {}):
|
|
if not cache:
|
|
cache['variant'] = {'': (0, 'This should never be seen outside ldml.py')}
|
|
# They're not actually lists: mappings from numeric value
|
|
# to pairs of full name and short code. What we want, in
|
|
# each case, is a mapping from code to the other two.
|
|
from enumdata import language_list, script_list, country_list
|
|
for form, book, empty in (('language', language_list, 'AnyLanguage'),
|
|
('script', script_list, 'AnyScript'),
|
|
('country', country_list, 'AnyCountry')):
|
|
cache[form] = dict((pair[1], (num, pair[0]))
|
|
for num, pair in book.items() if pair[0] != 'C')
|
|
# (Have to filter out the C locale, as we give it the
|
|
# same (all space) code as AnyLanguage, whose code
|
|
# should probably be 'und' instead.)
|
|
|
|
# Map empty to zero and the any value:
|
|
cache[form][''] = (0, empty)
|
|
# and map language code 'und' also to (0, any):
|
|
cache['language']['und'] = (0, 'AnyLanguage')
|
|
|
|
return cache[key]
|
|
|
|
def __codeMap(self, key, cache = {},
|
|
# Maps our name for it to CLDR's name:
|
|
naming = {'language': 'languages', 'script': 'scripts',
|
|
'country': 'territories', 'variant': 'variants'}):
|
|
if not cache:
|
|
root = self.xml('common', 'main', 'en.xml').root.findUniqueChild('localeDisplayNames')
|
|
for dst, src in naming.items():
|
|
cache[dst] = dict(self.__codeMapScan(root.findUniqueChild(src)))
|
|
assert cache
|
|
|
|
return cache[key]
|
|
|
|
def __codeMapScan(self, node):
|
|
"""Get mapping from codes to element values.
|
|
|
|
Passed in node is a <languages>, <scripts>, <territories> or
|
|
<variants> node, each child of which is a <language>,
|
|
<script>, <territory> or <variant> node as appropriate, whose
|
|
type is a code (of the appropriate flavour) and content is its
|
|
full name. In some cases, two child nodes have the same type;
|
|
in these cases, one always has an alt attribute and we should
|
|
prefer the other. Yields all such type, content pairs found
|
|
in node's children (skipping any with an alt attribute, if
|
|
their type has been seen previously)."""
|
|
seen = set()
|
|
for elt in node.dom.childNodes:
|
|
try:
|
|
key, value = elt.attributes['type'].nodeValue, elt.childNodes[0].wholeText
|
|
except (KeyError, ValueError, TypeError):
|
|
pass
|
|
else:
|
|
if key not in seen or not elt.attributes.has_key('alt'):
|
|
yield key, value
|
|
seen.add(key)
|
|
|
|
# CLDR uses inheritance between locales to save repetition:
|
|
def __parentLocale(self, name, cache = {}):
|
|
# see http://www.unicode.org/reports/tr35/#Parent_Locales
|
|
if not cache:
|
|
for tag, attrs in self.__supplementalData.find('parentLocales'):
|
|
parent = attrs.get('parent', '')
|
|
for child in attrs['locales'].split():
|
|
cache[child] = parent
|
|
assert cache
|
|
|
|
return cache[name]
|
|
|
|
def __localeAsDoc(self, name, aliasFor = None,
|
|
joinPath = os.path.join, exists = os.path.isfile):
|
|
path = ('common', 'main', name + '.xml')
|
|
if exists(joinPath(self.root, *path)):
|
|
elt = self.__xml(path)
|
|
for child in Node(elt).findAllChildren('alias'):
|
|
try:
|
|
alias = child.dom.attributes['source'].nodeValue
|
|
except (KeyError, AttributeError):
|
|
pass
|
|
else:
|
|
return self.__localeAsDoc(alias, aliasFor or name)
|
|
# No alias child with a source:
|
|
return elt
|
|
|
|
if aliasFor:
|
|
raise Error('Fatal error: found an alias "{}" -> "{}", but found no file for the alias'
|
|
.format(aliasFor, name))
|
|
|
|
def __scanLocaleRoots(self, name):
|
|
while name and name != 'root':
|
|
doc = self.__localeAsDoc(name)
|
|
if doc is not None:
|
|
yield Node(doc, self.__unDistinguishedAttributes)
|
|
|
|
try:
|
|
name = self.__parentLocale(name)
|
|
except KeyError:
|
|
try:
|
|
name, tail = name.rsplit('_', 1)
|
|
except ValueError: # No tail to discard: we're done
|
|
break
|
|
|
|
class __Seq (list): pass # No weakref for tuple and list, but list sub-class is ok.
|
|
def __localeRoots(self, name, cache = CacheDict()):
|
|
try:
|
|
chain = cache[name]
|
|
except KeyError:
|
|
cache[name] = chain = self.__Seq(self.__scanLocaleRoots(name))
|
|
return chain
|
|
|
|
# Unpolute the namespace: we don't need to export these.
|
|
del minidom, CacheDict, os
|