2021-07-05 15:45:26 +00:00
|
|
|
#!/usr/bin/env python3
|
2022-05-10 10:06:48 +00:00
|
|
|
# Copyright (C) 2021 The Qt Company Ltd.
|
|
|
|
# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
|
2021-07-07 10:41:10 +00:00
|
|
|
"""Script to generate C++ code from CLDR data in QLocaleXML form
|
2017-05-23 13:24:35 +00:00
|
|
|
|
2021-07-07 10:41:10 +00:00
|
|
|
See ``cldr2qlocalexml.py`` for how to generate the QLocaleXML data itself.
|
2022-10-17 08:58:31 +00:00
|
|
|
Pass the output file from that as first parameter to this script; pass the ISO
|
|
|
|
639-3 data file as second parameter. You can optionally pass the root of the
|
|
|
|
qtbase check-out as third parameter; it defaults to the root of the qtbase
|
|
|
|
check-out containing this script.
|
2021-11-22 14:56:53 +00:00
|
|
|
|
|
|
|
The ISO 639-3 data file can be downloaded from the SIL website:
|
|
|
|
|
|
|
|
https://iso639-3.sil.org/sites/iso639-3/files/downloads/iso-639-3.tab
|
2017-05-23 13:24:35 +00:00
|
|
|
"""
|
2011-04-27 10:05:43 +00:00
|
|
|
|
|
|
|
import datetime
|
2021-07-07 10:41:10 +00:00
|
|
|
import argparse
|
2021-07-09 13:34:40 +00:00
|
|
|
from pathlib import Path
|
2021-11-22 14:56:53 +00:00
|
|
|
from typing import Optional
|
2011-04-27 10:05:43 +00:00
|
|
|
|
2020-02-25 11:30:06 +00:00
|
|
|
from qlocalexml import QLocaleXmlReader
|
2022-10-17 08:58:31 +00:00
|
|
|
from localetools import unicode2hex, wrap_list, Error, Transcriber, SourceFileEditor, qtbase_root
|
2021-11-22 14:56:53 +00:00
|
|
|
from iso639_3 import LanguageCodeData
|
2017-05-30 13:50:47 +00:00
|
|
|
|
2021-07-06 11:26:29 +00:00
|
|
|
class LocaleKeySorter:
|
|
|
|
"""Sort-ordering representation of a locale key.
|
2021-07-06 10:33:05 +00:00
|
|
|
|
2021-07-06 11:26:29 +00:00
|
|
|
This is for passing to a sorting algorithm as key-function, that
|
|
|
|
it applies to each entry in the list to decide which belong
|
|
|
|
earlier. It adds an entry to the (language, script, territory)
|
|
|
|
triple, just before script, that sorts earlier if the territory is
|
|
|
|
the default for the given language and script, later otherwise.
|
|
|
|
"""
|
2021-07-06 10:33:05 +00:00
|
|
|
|
|
|
|
# TODO: study the relationship between this and CLDR's likely
|
|
|
|
# sub-tags algorithm. Work out how locale sort-order impacts
|
|
|
|
# QLocale's likely sub-tag matching algorithms. Make sure this is
|
|
|
|
# sorting in an order compatible with those algorithms.
|
2017-06-08 10:19:23 +00:00
|
|
|
|
2021-07-06 11:26:29 +00:00
|
|
|
def __init__(self, defaults):
|
|
|
|
self.map = dict(defaults)
|
|
|
|
def foreign(self, key):
|
|
|
|
default = self.map.get(key[:2])
|
|
|
|
return default is None or default != key[2]
|
|
|
|
def __call__(self, key):
|
|
|
|
# TODO: should we compare territory before or after script ?
|
|
|
|
return (key[0], self.foreign(key)) + key[1:]
|
2011-04-27 10:05:43 +00:00
|
|
|
|
|
|
|
class StringDataToken:
|
2020-01-09 13:48:21 +00:00
|
|
|
def __init__(self, index, length, bits):
|
|
|
|
if index > 0xffff:
|
2021-07-06 14:22:07 +00:00
|
|
|
raise ValueError(f'Start-index ({index}) exceeds the uint16 range!')
|
2020-01-09 13:48:21 +00:00
|
|
|
if length >= (1 << bits):
|
2021-07-06 14:22:07 +00:00
|
|
|
raise ValueError(f'Data size ({length}) exceeds the {bits}-bit range!')
|
2020-01-09 13:48:21 +00:00
|
|
|
|
2011-04-27 10:05:43 +00:00
|
|
|
self.index = index
|
|
|
|
self.length = length
|
|
|
|
|
|
|
|
class StringData:
|
2017-05-31 14:17:54 +00:00
|
|
|
def __init__(self, name):
|
2011-04-27 10:05:43 +00:00
|
|
|
self.data = []
|
|
|
|
self.hash = {}
|
2017-05-31 14:17:54 +00:00
|
|
|
self.name = name
|
2020-01-09 19:47:23 +00:00
|
|
|
self.text = '' # Used in quick-search for matches in data
|
2017-01-14 16:53:31 +00:00
|
|
|
|
2020-04-06 23:00:12 +00:00
|
|
|
def append(self, s, bits = 8):
|
2011-04-27 10:05:43 +00:00
|
|
|
try:
|
2020-01-09 19:47:23 +00:00
|
|
|
token = self.hash[s]
|
|
|
|
except KeyError:
|
2020-01-09 13:48:21 +00:00
|
|
|
token = self.__store(s, bits)
|
2020-01-09 19:47:23 +00:00
|
|
|
self.hash[s] = token
|
2011-04-27 10:05:43 +00:00
|
|
|
return token
|
|
|
|
|
2020-01-09 13:48:21 +00:00
|
|
|
def __store(self, s, bits):
|
2020-01-09 19:47:23 +00:00
|
|
|
"""Add string s to known data.
|
|
|
|
|
|
|
|
Seeks to avoid duplication, where possible.
|
|
|
|
For example, short-forms may be prefixes of long-forms.
|
|
|
|
"""
|
|
|
|
if not s:
|
2020-01-09 13:48:21 +00:00
|
|
|
return StringDataToken(0, 0, bits)
|
2020-01-09 19:47:23 +00:00
|
|
|
ucs2 = unicode2hex(s)
|
|
|
|
try:
|
|
|
|
index = self.text.index(s) - 1
|
|
|
|
matched = 0
|
|
|
|
while matched < len(ucs2):
|
|
|
|
index, matched = self.data.index(ucs2[0], index + 1), 1
|
|
|
|
if index + len(ucs2) >= len(self.data):
|
|
|
|
raise ValueError # not found after all !
|
|
|
|
while matched < len(ucs2) and self.data[index + matched] == ucs2[matched]:
|
|
|
|
matched += 1
|
|
|
|
except ValueError:
|
|
|
|
index = len(self.data)
|
|
|
|
self.data += ucs2
|
|
|
|
self.text += s
|
|
|
|
|
|
|
|
assert index >= 0
|
|
|
|
try:
|
2020-01-09 13:48:21 +00:00
|
|
|
return StringDataToken(index, len(ucs2), bits)
|
2020-01-09 19:47:23 +00:00
|
|
|
except ValueError as e:
|
|
|
|
e.args += (self.name, s)
|
|
|
|
raise
|
|
|
|
|
2017-01-14 16:53:31 +00:00
|
|
|
def write(self, fd):
|
2020-01-09 13:48:21 +00:00
|
|
|
if len(self.data) > 0xffff:
|
2021-07-06 14:22:07 +00:00
|
|
|
raise ValueError(f'Data is too big ({len(self.data)}) for quint16 index to its end!',
|
2020-01-09 13:48:21 +00:00
|
|
|
self.name)
|
2022-05-24 04:32:02 +00:00
|
|
|
fd.write(f"\nstatic constexpr char16_t {self.name}[] = {{\n")
|
2023-07-28 08:10:49 +00:00
|
|
|
fd.write(wrap_list(self.data, 12)) # 12 == 100 // len('0xhhhh, ')
|
2017-01-14 16:53:31 +00:00
|
|
|
fd.write("\n};\n")
|
|
|
|
|
2011-04-27 10:05:43 +00:00
|
|
|
def currencyIsoCodeData(s):
|
|
|
|
if s:
|
2017-05-12 10:00:55 +00:00
|
|
|
return '{' + ",".join(str(ord(x)) for x in s) + '}'
|
|
|
|
return "{0,0,0}"
|
2011-04-27 10:05:43 +00:00
|
|
|
|
2020-02-19 17:22:25 +00:00
|
|
|
class LocaleSourceEditor (SourceFileEditor):
|
2021-07-09 13:34:40 +00:00
|
|
|
def __init__(self, path: Path, temp: Path, version: str):
|
2021-07-06 07:20:56 +00:00
|
|
|
super().__init__(path, temp)
|
2021-07-12 12:11:39 +00:00
|
|
|
self.version = version
|
|
|
|
|
|
|
|
def onEnter(self) -> None:
|
|
|
|
super().onEnter()
|
2021-07-06 14:22:07 +00:00
|
|
|
self.writer.write(f"""
|
2020-02-19 17:22:25 +00:00
|
|
|
/*
|
2021-07-06 14:22:07 +00:00
|
|
|
This part of the file was generated on {datetime.date.today()} from the
|
2021-07-12 12:11:39 +00:00
|
|
|
Common Locale Data Repository v{self.version}
|
2011-04-27 10:05:43 +00:00
|
|
|
|
2020-02-19 17:22:25 +00:00
|
|
|
http://www.unicode.org/cldr/
|
|
|
|
|
|
|
|
Do not edit this section: instead regenerate it using
|
|
|
|
cldr2qlocalexml.py and qlocalexml2cpp.py on updated (or
|
|
|
|
edited) CLDR data; see qtbase/util/locale_database/.
|
|
|
|
*/
|
2011-04-27 10:05:43 +00:00
|
|
|
|
2021-07-06 14:22:07 +00:00
|
|
|
""")
|
2020-02-19 17:22:25 +00:00
|
|
|
|
|
|
|
class LocaleDataWriter (LocaleSourceEditor):
|
|
|
|
def likelySubtags(self, likely):
|
2020-10-12 11:12:48 +00:00
|
|
|
# First sort likely, so that we can use binary search in C++
|
|
|
|
# code. Although the entries are (lang, script, region), sort
|
|
|
|
# as (lang, region, script) and sort 0 after all non-zero
|
|
|
|
# values. This ensures that, when several mappings partially
|
|
|
|
# match a requested locale, the one we should prefer to use
|
|
|
|
# appears first.
|
|
|
|
huge = 0x10000 # > any ushort; all tag values are ushort
|
|
|
|
def keyLikely(entry):
|
|
|
|
have = entry[1] # Numeric id triple
|
|
|
|
return have[0] or huge, have[2] or huge, have[1] or huge # language, region, script
|
2021-07-05 15:45:26 +00:00
|
|
|
likely = sorted(likely, key=keyLikely)
|
2020-10-12 11:12:48 +00:00
|
|
|
|
|
|
|
i = 0
|
2022-05-24 04:32:02 +00:00
|
|
|
self.writer.write('static constexpr QLocaleId likely_subtags[] = {\n')
|
2020-10-12 11:12:48 +00:00
|
|
|
for had, have, got, give in likely:
|
|
|
|
i += 1
|
2020-02-19 17:22:25 +00:00
|
|
|
self.writer.write(' {{ {:3d}, {:3d}, {:3d} }}'.format(*have))
|
|
|
|
self.writer.write(', {{ {:3d}, {:3d}, {:3d} }}'.format(*give))
|
2020-10-12 11:12:48 +00:00
|
|
|
self.writer.write(' ' if i == len(likely) else ',')
|
2021-07-06 14:22:07 +00:00
|
|
|
self.writer.write(f' // {had} -> {got}\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
self.writer.write('};\n\n')
|
|
|
|
|
|
|
|
def localeIndex(self, indices):
|
2022-05-24 04:32:02 +00:00
|
|
|
self.writer.write('static constexpr quint16 locale_index[] = {\n')
|
2021-07-06 14:22:07 +00:00
|
|
|
for index, name in indices:
|
|
|
|
self.writer.write(f'{index:6d}, // {name}\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
self.writer.write(' 0 // trailing 0\n')
|
|
|
|
self.writer.write('};\n\n')
|
|
|
|
|
|
|
|
def localeData(self, locales, names):
|
|
|
|
list_pattern_part_data = StringData('list_pattern_part_data')
|
2020-04-06 23:00:12 +00:00
|
|
|
single_character_data = StringData('single_character_data')
|
2020-02-19 17:22:25 +00:00
|
|
|
date_format_data = StringData('date_format_data')
|
|
|
|
time_format_data = StringData('time_format_data')
|
|
|
|
days_data = StringData('days_data')
|
|
|
|
am_data = StringData('am_data')
|
|
|
|
pm_data = StringData('pm_data')
|
|
|
|
byte_unit_data = StringData('byte_unit_data')
|
|
|
|
currency_symbol_data = StringData('currency_symbol_data')
|
|
|
|
currency_display_name_data = StringData('currency_display_name_data')
|
|
|
|
currency_format_data = StringData('currency_format_data')
|
|
|
|
endonyms_data = StringData('endonyms_data')
|
|
|
|
|
|
|
|
# Locale data
|
2022-05-24 04:32:02 +00:00
|
|
|
self.writer.write('static constexpr QLocaleData locale_data[] = {\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
# Table headings: keep each label centred in its field, matching line_format:
|
|
|
|
self.writer.write(' // '
|
|
|
|
# Width 6 + comma
|
|
|
|
' lang ' # IDs
|
|
|
|
'script '
|
|
|
|
' terr '
|
2020-04-06 23:00:12 +00:00
|
|
|
|
|
|
|
# Range entries (all start-indices, then all sizes)
|
|
|
|
# Width 5 + comma
|
|
|
|
'lStrt ' # List pattern
|
|
|
|
'lpMid '
|
|
|
|
'lpEnd '
|
|
|
|
'lPair '
|
|
|
|
'lDelm ' # List delimiter
|
|
|
|
# Representing numbers
|
|
|
|
' dec '
|
|
|
|
'group '
|
|
|
|
'prcnt '
|
|
|
|
' zero '
|
|
|
|
'minus '
|
|
|
|
'plus '
|
|
|
|
' exp '
|
|
|
|
# Quotation marks
|
|
|
|
'qtOpn '
|
|
|
|
'qtEnd '
|
|
|
|
'altQO '
|
|
|
|
'altQE '
|
|
|
|
'lDFmt ' # Date format
|
|
|
|
'sDFmt '
|
|
|
|
'lTFmt ' # Time format
|
|
|
|
'sTFmt '
|
|
|
|
'slDay ' # Day names
|
|
|
|
'lDays '
|
|
|
|
'ssDys '
|
|
|
|
'sDays '
|
|
|
|
'snDay '
|
|
|
|
'nDays '
|
|
|
|
' am ' # am/pm indicators
|
|
|
|
' pm '
|
|
|
|
' byte '
|
|
|
|
'siQnt '
|
|
|
|
'iecQn '
|
|
|
|
'crSym ' # Currency formatting
|
|
|
|
'crDsp '
|
|
|
|
'crFmt '
|
|
|
|
'crFNg '
|
|
|
|
'ntLng ' # Name of language in itself, and of territory
|
|
|
|
'ntTer '
|
|
|
|
# Width 3 + comma for each size; no header
|
|
|
|
+ ' ' * 37 +
|
|
|
|
|
|
|
|
# Strays (char array, bit-fields):
|
|
|
|
# Width 10 + 2 spaces + comma
|
2020-02-19 17:22:25 +00:00
|
|
|
' currISO '
|
|
|
|
# Width 6 + comma
|
2020-04-06 23:00:12 +00:00
|
|
|
'curDgt ' # Currency digits
|
|
|
|
'curRnd ' # Currencty rounding (unused: QTBUG-81343)
|
2020-02-19 17:22:25 +00:00
|
|
|
'dow1st ' # First day of week
|
|
|
|
' wknd+ ' # Week-end start/end days
|
2020-01-17 10:00:24 +00:00
|
|
|
' wknd- '
|
|
|
|
'grpTop '
|
|
|
|
'grpMid '
|
|
|
|
'grpEnd'
|
2020-02-19 17:22:25 +00:00
|
|
|
# No trailing space on last entry (be sure to
|
|
|
|
# pad before adding anything after it).
|
|
|
|
'\n')
|
|
|
|
|
|
|
|
formatLine = ''.join((
|
|
|
|
' {{ ',
|
|
|
|
# Locale-identifier
|
|
|
|
'{:6d},' * 3,
|
2020-04-06 23:00:12 +00:00
|
|
|
# List patterns, date/time formats, day names, am/pm
|
2020-02-19 17:22:25 +00:00
|
|
|
# SI/IEC byte-unit abbreviations
|
2020-04-06 23:00:12 +00:00
|
|
|
# Currency and endonyms
|
|
|
|
# Range starts
|
|
|
|
'{:5d},' * 37,
|
|
|
|
# Range sizes
|
|
|
|
'{:3d},' * 37,
|
|
|
|
|
2020-02-19 17:22:25 +00:00
|
|
|
# Currency ISO code
|
|
|
|
' {:>10s}, ',
|
|
|
|
# Currency formatting
|
|
|
|
'{:6d},{:6d}',
|
|
|
|
# Day of week and week-end
|
|
|
|
',{:6d}' * 3,
|
2020-01-17 10:00:24 +00:00
|
|
|
# Number group sizes
|
|
|
|
',{:6d}' * 3,
|
2020-02-19 17:22:25 +00:00
|
|
|
' }}')).format
|
|
|
|
for key in names:
|
|
|
|
locale = locales[key]
|
2020-04-06 23:00:12 +00:00
|
|
|
# Sequence of StringDataToken:
|
|
|
|
ranges = (tuple(list_pattern_part_data.append(p) for p in # 5 entries:
|
|
|
|
(locale.listPatternPartStart, locale.listPatternPartMiddle,
|
|
|
|
locale.listPatternPartEnd, locale.listPatternPartTwo,
|
|
|
|
locale.listDelim)) +
|
|
|
|
tuple(single_character_data.append(p) for p in # 11 entries
|
|
|
|
(locale.decimal, locale.group, locale.percent, locale.zero,
|
|
|
|
locale.minus, locale.plus, locale.exp,
|
|
|
|
locale.quotationStart, locale.quotationEnd,
|
|
|
|
locale.alternateQuotationStart, locale.alternateQuotationEnd)) +
|
|
|
|
tuple (date_format_data.append(f) for f in # 2 entries:
|
|
|
|
(locale.longDateFormat, locale.shortDateFormat)) +
|
|
|
|
tuple(time_format_data.append(f) for f in # 2 entries:
|
|
|
|
(locale.longTimeFormat, locale.shortTimeFormat)) +
|
|
|
|
tuple(days_data.append(d) for d in # 6 entries:
|
|
|
|
(locale.standaloneLongDays, locale.longDays,
|
|
|
|
locale.standaloneShortDays, locale.shortDays,
|
|
|
|
locale.standaloneNarrowDays, locale.narrowDays)) +
|
|
|
|
(am_data.append(locale.am), pm_data.append(locale.pm)) + # 2 entries
|
|
|
|
tuple(byte_unit_data.append(b) for b in # 3 entries:
|
|
|
|
(locale.byte_unit,
|
|
|
|
locale.byte_si_quantified,
|
|
|
|
locale.byte_iec_quantified)) +
|
|
|
|
(currency_symbol_data.append(locale.currencySymbol),
|
|
|
|
currency_display_name_data.append(locale.currencyDisplayName),
|
|
|
|
currency_format_data.append(locale.currencyFormat),
|
|
|
|
currency_format_data.append(locale.currencyNegativeFormat),
|
|
|
|
endonyms_data.append(locale.languageEndonym),
|
2021-05-04 11:20:32 +00:00
|
|
|
endonyms_data.append(locale.territoryEndonym)) # 6 entries
|
2020-04-06 23:00:12 +00:00
|
|
|
) # Total: 37 entries
|
|
|
|
assert len(ranges) == 37
|
|
|
|
|
|
|
|
self.writer.write(formatLine(*(
|
|
|
|
key +
|
|
|
|
tuple(r.index for r in ranges) +
|
|
|
|
tuple(r.length for r in ranges) +
|
|
|
|
(currencyIsoCodeData(locale.currencyIsoCode),
|
|
|
|
locale.currencyDigits,
|
|
|
|
locale.currencyRounding, # unused (QTBUG-81343)
|
2020-01-17 10:00:24 +00:00
|
|
|
locale.firstDayOfWeek, locale.weekendStart, locale.weekendEnd,
|
|
|
|
locale.groupTop, locale.groupHigher, locale.groupLeast) ))
|
2021-07-06 14:22:07 +00:00
|
|
|
+ f', // {locale.language}/{locale.script}/{locale.territory}\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
self.writer.write(formatLine(*( # All zeros, matching the format:
|
2020-04-06 23:00:12 +00:00
|
|
|
(0,) * 3 + (0,) * 37 * 2
|
2020-02-19 17:22:25 +00:00
|
|
|
+ (currencyIsoCodeData(0),)
|
2020-01-17 10:00:24 +00:00
|
|
|
+ (0,) * 8 ))
|
2020-02-19 17:22:25 +00:00
|
|
|
+ ' // trailing zeros\n')
|
|
|
|
self.writer.write('};\n')
|
|
|
|
|
|
|
|
# StringData tables:
|
2020-04-06 23:00:12 +00:00
|
|
|
for data in (list_pattern_part_data, single_character_data,
|
|
|
|
date_format_data, time_format_data, days_data,
|
2020-02-19 17:22:25 +00:00
|
|
|
byte_unit_data, am_data, pm_data, currency_symbol_data,
|
|
|
|
currency_display_name_data, currency_format_data,
|
|
|
|
endonyms_data):
|
|
|
|
data.write(self.writer)
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def __writeNameData(out, book, form):
|
2022-05-24 04:32:02 +00:00
|
|
|
out(f'static constexpr char {form}_name_list[] =\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
out('"Default\\0"\n')
|
|
|
|
for key, value in book.items():
|
|
|
|
if key == 0:
|
|
|
|
continue
|
2021-07-06 14:22:07 +00:00
|
|
|
out(f'"{value[0]}\\0"\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
out(';\n\n')
|
|
|
|
|
2022-05-24 04:32:02 +00:00
|
|
|
out(f'static constexpr quint16 {form}_name_index[] = {{\n')
|
2021-07-06 14:22:07 +00:00
|
|
|
out(f' 0, // Any{form.capitalize()}\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
index = 8
|
|
|
|
for key, value in book.items():
|
|
|
|
if key == 0:
|
|
|
|
continue
|
|
|
|
name = value[0]
|
2021-07-06 14:22:07 +00:00
|
|
|
out(f'{index:6d}, // {name}\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
index += len(name) + 1
|
|
|
|
out('};\n\n')
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def __writeCodeList(out, book, form, width):
|
2022-05-24 04:32:02 +00:00
|
|
|
out(f'static constexpr unsigned char {form}_code_list[] =\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
for key, value in book.items():
|
|
|
|
code = value[1]
|
|
|
|
code += r'\0' * max(width - len(code), 0)
|
2021-07-06 14:22:07 +00:00
|
|
|
out(f'"{code}" // {value[0]}\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
out(';\n\n')
|
|
|
|
|
|
|
|
def languageNames(self, languages):
|
|
|
|
self.__writeNameData(self.writer.write, languages, 'language')
|
|
|
|
|
|
|
|
def scriptNames(self, scripts):
|
|
|
|
self.__writeNameData(self.writer.write, scripts, 'script')
|
|
|
|
|
2021-05-04 11:20:32 +00:00
|
|
|
def territoryNames(self, territories):
|
|
|
|
self.__writeNameData(self.writer.write, territories, 'territory')
|
2020-02-19 17:22:25 +00:00
|
|
|
|
|
|
|
# TODO: unify these next three into the previous three; kept
|
|
|
|
# separate for now to verify we're not changing data.
|
|
|
|
|
2021-11-22 14:56:53 +00:00
|
|
|
def languageCodes(self, languages, code_data: LanguageCodeData):
|
|
|
|
out = self.writer.write
|
|
|
|
|
|
|
|
out(f'constexpr std::array<LanguageCodeEntry, {len(languages)}> languageCodeList {{\n')
|
|
|
|
|
|
|
|
def q(val: Optional[str], size: int) -> str:
|
|
|
|
"""Quote the value and adjust the result for tabular view."""
|
2023-02-17 16:59:12 +00:00
|
|
|
s = '' if val is None else ', '.join(f"'{c}'" for c in val)
|
|
|
|
return f'{{{s}}}' if size == 0 else f'{{{s}}},'.ljust(size * 5 + 2)
|
2021-11-22 14:56:53 +00:00
|
|
|
|
|
|
|
for key, value in languages.items():
|
|
|
|
code = value[1]
|
|
|
|
if key < 2:
|
|
|
|
result = code_data.query('und')
|
|
|
|
else:
|
|
|
|
result = code_data.query(code)
|
|
|
|
assert code == result.id()
|
|
|
|
assert result is not None
|
|
|
|
|
|
|
|
codeString = q(result.part1Code, 2)
|
|
|
|
codeString += q(result.part2BCode, 3)
|
|
|
|
codeString += q(result.part2TCode, 3)
|
|
|
|
codeString += q(result.part3Code, 0)
|
|
|
|
out(f' LanguageCodeEntry {{{codeString}}}, // {value[0]}\n')
|
|
|
|
|
|
|
|
out('};\n\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
|
|
|
|
def scriptCodes(self, scripts):
|
|
|
|
self.__writeCodeList(self.writer.write, scripts, 'script', 4)
|
|
|
|
|
2021-05-04 11:20:32 +00:00
|
|
|
def territoryCodes(self, territories): # TODO: unify with territoryNames()
|
|
|
|
self.__writeCodeList(self.writer.write, territories, 'territory', 3)
|
2020-02-19 17:22:25 +00:00
|
|
|
|
|
|
|
class CalendarDataWriter (LocaleSourceEditor):
|
2020-04-06 23:00:12 +00:00
|
|
|
formatCalendar = (
|
|
|
|
' {{'
|
|
|
|
+ ','.join(('{:6d}',) * 3 + ('{:5d}',) * 6 + ('{:3d}',) * 6)
|
|
|
|
+ ' }},').format
|
2020-02-19 17:22:25 +00:00
|
|
|
def write(self, calendar, locales, names):
|
|
|
|
months_data = StringData('months_data')
|
2011-04-27 10:05:43 +00:00
|
|
|
|
2022-05-24 04:32:02 +00:00
|
|
|
self.writer.write('static constexpr QCalendarLocale locale_data[] = {\n')
|
2020-04-06 23:00:12 +00:00
|
|
|
self.writer.write(
|
|
|
|
' //'
|
|
|
|
# IDs, width 7 (6 + comma)
|
|
|
|
' lang '
|
|
|
|
' script'
|
|
|
|
' terr '
|
|
|
|
# Month-name start-indices, width 6 (5 + comma)
|
|
|
|
'sLong '
|
|
|
|
' long '
|
|
|
|
'sShrt '
|
|
|
|
'short '
|
|
|
|
'sNarw '
|
|
|
|
'narow '
|
|
|
|
# No individual headers for the sizes.
|
|
|
|
'Sizes...'
|
|
|
|
'\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
for key in names:
|
|
|
|
locale = locales[key]
|
2020-04-06 23:00:12 +00:00
|
|
|
# Sequence of StringDataToken:
|
|
|
|
try:
|
|
|
|
# Twelve long month names can add up to more than 256 (e.g. kde_TZ: 264)
|
|
|
|
ranges = (tuple(months_data.append(m[calendar], 16) for m in
|
|
|
|
(locale.standaloneLongMonths, locale.longMonths)) +
|
|
|
|
tuple(months_data.append(m[calendar]) for m in
|
|
|
|
(locale.standaloneShortMonths, locale.shortMonths,
|
|
|
|
locale.standaloneNarrowMonths, locale.narrowMonths)))
|
|
|
|
except ValueError as e:
|
2021-07-06 12:50:57 +00:00
|
|
|
e.args += (locale.language, locale.script, locale.territory)
|
2020-04-06 23:00:12 +00:00
|
|
|
raise
|
2011-04-27 10:05:43 +00:00
|
|
|
|
2020-02-19 17:22:25 +00:00
|
|
|
self.writer.write(
|
2020-04-06 23:00:12 +00:00
|
|
|
self.formatCalendar(*(
|
|
|
|
key +
|
|
|
|
tuple(r.index for r in ranges) +
|
|
|
|
tuple(r.length for r in ranges) ))
|
2021-07-06 14:22:07 +00:00
|
|
|
+ f'// {locale.language}/{locale.script}/{locale.territory}\n')
|
2020-04-06 23:00:12 +00:00
|
|
|
self.writer.write(self.formatCalendar(*( (0,) * (3 + 6 * 2) ))
|
2020-02-19 17:22:25 +00:00
|
|
|
+ '// trailing zeros\n')
|
|
|
|
self.writer.write('};\n')
|
|
|
|
months_data.write(self.writer)
|
|
|
|
|
|
|
|
class LocaleHeaderWriter (SourceFileEditor):
|
|
|
|
def __init__(self, path, temp, dupes):
|
2021-07-06 07:20:56 +00:00
|
|
|
super().__init__(path, temp)
|
2020-02-19 17:22:25 +00:00
|
|
|
self.__dupes = dupes
|
|
|
|
|
|
|
|
def languages(self, languages):
|
|
|
|
self.__enum('Language', languages, self.__language)
|
|
|
|
self.writer.write('\n')
|
|
|
|
|
2021-05-04 11:20:32 +00:00
|
|
|
def territories(self, territories):
|
2021-03-09 08:19:54 +00:00
|
|
|
self.writer.write(" // ### Qt 7: Rename to Territory\n")
|
2021-05-04 11:20:32 +00:00
|
|
|
self.__enum('Country', territories, self.__territory, 'Territory')
|
2020-02-19 17:22:25 +00:00
|
|
|
|
|
|
|
def scripts(self, scripts):
|
|
|
|
self.__enum('Script', scripts, self.__script)
|
|
|
|
self.writer.write('\n')
|
|
|
|
|
|
|
|
# Implementation details
|
|
|
|
from enumdata import (language_aliases as __language,
|
2021-05-04 11:20:32 +00:00
|
|
|
territory_aliases as __territory,
|
2020-02-19 17:22:25 +00:00
|
|
|
script_aliases as __script)
|
|
|
|
|
2021-03-09 08:19:54 +00:00
|
|
|
def __enum(self, name, book, alias, suffix = None):
|
2020-02-19 17:22:25 +00:00
|
|
|
assert book
|
2021-03-09 08:19:54 +00:00
|
|
|
|
|
|
|
if suffix is None:
|
|
|
|
suffix = name
|
|
|
|
|
2020-02-19 17:22:25 +00:00
|
|
|
out, dupes = self.writer.write, self.__dupes
|
2021-07-06 14:22:07 +00:00
|
|
|
out(f' enum {name} : ushort {{\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
for key, value in book.items():
|
2020-10-19 13:13:21 +00:00
|
|
|
member = value[0].replace('-', ' ')
|
2020-02-19 17:22:25 +00:00
|
|
|
if name == 'Script':
|
|
|
|
# Don't .capitalize() as some names are already camel-case (see enumdata.py):
|
|
|
|
member = ''.join(word[0].upper() + word[1:] for word in member.split())
|
|
|
|
if not member.endswith('Script'):
|
|
|
|
member += 'Script'
|
|
|
|
if member in dupes:
|
2021-07-06 14:22:07 +00:00
|
|
|
raise Error(f'The script name "{member}" is messy')
|
2020-02-19 17:22:25 +00:00
|
|
|
else:
|
|
|
|
member = ''.join(member.split())
|
2021-03-09 08:19:54 +00:00
|
|
|
member = member + suffix if member in dupes else member
|
2021-07-06 14:22:07 +00:00
|
|
|
out(f' {member} = {key},\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
|
|
|
|
out('\n '
|
2021-07-06 14:22:07 +00:00
|
|
|
+ ',\n '.join(f'{k} = {v}' for k, v in sorted(alias.items()))
|
|
|
|
+ f',\n\n Last{suffix} = {member}')
|
2021-03-09 08:19:54 +00:00
|
|
|
|
|
|
|
# for "LastCountry = LastTerritory"
|
|
|
|
# ### Qt 7: Remove
|
|
|
|
if suffix != name:
|
2021-07-06 14:22:07 +00:00
|
|
|
out(f',\n Last{name} = Last{suffix}')
|
2021-03-09 08:19:54 +00:00
|
|
|
|
|
|
|
out('\n };\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
|
|
|
|
|
2021-07-07 10:41:10 +00:00
|
|
|
def main(out, err):
|
|
|
|
calendars_map = {
|
2022-10-17 08:58:31 +00:00
|
|
|
# CLDR name: Qt file name fragment
|
2021-07-07 10:41:10 +00:00
|
|
|
'gregorian': 'roman',
|
|
|
|
'persian': 'jalali',
|
|
|
|
'islamic': 'hijri',
|
|
|
|
# 'hebrew': 'hebrew'
|
|
|
|
}
|
|
|
|
all_calendars = list(calendars_map.keys())
|
|
|
|
|
|
|
|
parser = argparse.ArgumentParser(
|
|
|
|
description='Generate C++ code from CLDR data in QLocaleXML form.',
|
|
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
|
|
parser.add_argument('input_file', help='input XML file name',
|
|
|
|
metavar='input-file.xml')
|
2021-11-22 14:56:53 +00:00
|
|
|
parser.add_argument('iso_path', help='path to the ISO 639-3 data file',
|
|
|
|
metavar='iso-639-3.tab')
|
2022-10-17 08:58:31 +00:00
|
|
|
parser.add_argument('qtbase_path', help='path to the root of the qtbase source tree',
|
|
|
|
nargs='?', default=qtbase_root)
|
2021-07-07 10:41:10 +00:00
|
|
|
parser.add_argument('--calendars', help='select calendars to emit data for',
|
|
|
|
nargs='+', metavar='CALENDAR',
|
|
|
|
choices=all_calendars, default=all_calendars)
|
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
|
|
qlocalexml = args.input_file
|
2021-07-09 13:34:40 +00:00
|
|
|
qtsrcdir = Path(args.qtbase_path)
|
2021-07-07 10:41:10 +00:00
|
|
|
calendars = {cal: calendars_map[cal] for cal in args.calendars}
|
2011-04-27 10:05:43 +00:00
|
|
|
|
2021-07-09 13:34:40 +00:00
|
|
|
if not (qtsrcdir.is_dir()
|
|
|
|
and all(qtsrcdir.joinpath('src/corelib/text', leaf).is_file()
|
2017-05-31 19:42:11 +00:00
|
|
|
for leaf in ('qlocale_data_p.h', 'qlocale.h', 'qlocale.qdoc'))):
|
2021-07-07 10:41:10 +00:00
|
|
|
parser.error(f'Missing expected files under qtbase source root {qtsrcdir}')
|
2011-04-27 10:05:43 +00:00
|
|
|
|
2020-02-25 11:30:06 +00:00
|
|
|
reader = QLocaleXmlReader(qlocalexml)
|
Rework cldr2qlocalexml.py's reading of CLDR data
Move the code out to a CldrReader class in cldr.py, expand CldrAccess
with facilities that needs, expand ldml.py to include support for more
features, finally making xpathlite.py redundant. This initial commit
aims, though, to be bug-for-bug compatible with xpathlite in its
reading of the CLDR data.
It turns out we've been using draftier data than we were aware of
(which might not be a bad thing). The xpathlite code appeared to check
for draft attributes, but these only appear on leaf nodes and most
data were fetched by finding a parent and then scanning its children
without the draft check; only am/pm data was actually being excluded
based on draft values. (We allowed contributed, for am/pm, in
addition to approved, which is all the xpathlite code allows
otherwise.) There are also some less equivocal bugs; I'll deal with
these in later commits.
Simplified number-system data look-ups; the old get_number_in_system()
was taking care of old LDML versions' placement of the number system
attribute; this is no longer needed. (It was also being used for a
currency value to which it was not appropriate, which is now handled
separately; this is one of the bugs mentioned above.) Ditched a
fall-back to nativeZeroDigit, which no longer exists in CLDR.
Change the command-line to take the root of the CLDR data tree, rather
than its common/main/ sub-directory. Support naming the file to which
to write output, as a second command-line argument, instead of always
writing to stdout (which remains the default) and leaving whoever runs
the script to redirect stdout.
Support (internally for now, while adding TODOs to give main() more
command-line options) separating the stderr output into its more and
less interesting parts; for now, continue producing both, but suppress
the least interesting entirely.
Task-number: QTBUG-81344
Change-Id: Ie611b47403a9452b51feaeeaaa0fbc8f7e84dc71
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>
2020-02-27 12:58:58 +00:00
|
|
|
locale_map = dict(reader.loadLocaleMap(calendars, err.write))
|
2021-07-06 11:26:29 +00:00
|
|
|
locale_keys = sorted(locale_map.keys(), key=LocaleKeySorter(reader.defaultMap()))
|
2011-04-27 10:05:43 +00:00
|
|
|
|
2021-11-22 14:56:53 +00:00
|
|
|
code_data = LanguageCodeData(args.iso_path)
|
|
|
|
|
2020-02-19 17:22:25 +00:00
|
|
|
try:
|
2021-07-12 12:11:39 +00:00
|
|
|
with LocaleDataWriter(qtsrcdir.joinpath('src/corelib/text/qlocale_data_p.h'),
|
|
|
|
qtsrcdir, reader.cldrVersion) as writer:
|
|
|
|
writer.likelySubtags(reader.likelyMap())
|
|
|
|
writer.localeIndex(reader.languageIndices(tuple(k[0] for k in locale_map)))
|
|
|
|
writer.localeData(locale_map, locale_keys)
|
|
|
|
writer.writer.write('\n')
|
|
|
|
writer.languageNames(reader.languages)
|
|
|
|
writer.scriptNames(reader.scripts)
|
|
|
|
writer.territoryNames(reader.territories)
|
|
|
|
# TODO: merge the next three into the previous three
|
2021-11-22 14:56:53 +00:00
|
|
|
writer.languageCodes(reader.languages, code_data)
|
2021-07-12 12:11:39 +00:00
|
|
|
writer.scriptCodes(reader.scripts)
|
|
|
|
writer.territoryCodes(reader.territories)
|
|
|
|
except Exception as e:
|
2021-07-06 14:22:07 +00:00
|
|
|
err.write(f'\nError updating locale data: {e}\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
return 1
|
|
|
|
|
2017-01-14 16:53:31 +00:00
|
|
|
# Generate calendar data
|
|
|
|
for calendar, stem in calendars.items():
|
2020-02-19 17:22:25 +00:00
|
|
|
try:
|
2021-07-12 12:11:39 +00:00
|
|
|
with CalendarDataWriter(
|
|
|
|
qtsrcdir.joinpath(f'src/corelib/time/q{stem}calendar_data_p.h'),
|
|
|
|
qtsrcdir, reader.cldrVersion) as writer:
|
|
|
|
writer.write(calendar, locale_map, locale_keys)
|
|
|
|
except Exception as e:
|
2021-07-06 14:22:07 +00:00
|
|
|
err.write(f'\nError updating {calendar} locale data: {e}\n')
|
2017-01-14 16:53:31 +00:00
|
|
|
|
2011-04-27 10:05:43 +00:00
|
|
|
# qlocale.h
|
2020-02-19 17:22:25 +00:00
|
|
|
try:
|
2021-07-12 12:11:39 +00:00
|
|
|
with LocaleHeaderWriter(qtsrcdir.joinpath('src/corelib/text/qlocale.h'),
|
|
|
|
qtsrcdir, reader.dupes) as writer:
|
|
|
|
writer.languages(reader.languages)
|
|
|
|
writer.scripts(reader.scripts)
|
|
|
|
writer.territories(reader.territories)
|
|
|
|
except Exception as e:
|
2021-07-06 14:22:07 +00:00
|
|
|
err.write(f'\nError updating qlocale.h: {e}\n')
|
2011-04-27 10:05:43 +00:00
|
|
|
|
|
|
|
# qlocale.qdoc
|
2020-02-19 17:22:25 +00:00
|
|
|
try:
|
2021-07-12 12:11:39 +00:00
|
|
|
with Transcriber(qtsrcdir.joinpath('src/corelib/text/qlocale.qdoc'), qtsrcdir) as qdoc:
|
|
|
|
DOCSTRING = " QLocale's data is based on Common Locale Data Repository "
|
|
|
|
for line in qdoc.reader:
|
|
|
|
if DOCSTRING in line:
|
|
|
|
qdoc.writer.write(f'{DOCSTRING}v{reader.cldrVersion}.\n')
|
|
|
|
else:
|
|
|
|
qdoc.writer.write(line)
|
|
|
|
except Exception as e:
|
|
|
|
err.write(f'\nError updating qlocale.h: {e}\n')
|
2020-02-19 17:22:25 +00:00
|
|
|
return 1
|
|
|
|
|
|
|
|
return 0
|
2011-04-27 10:05:43 +00:00
|
|
|
|
|
|
|
if __name__ == "__main__":
|
2020-02-19 17:22:25 +00:00
|
|
|
import sys
|
2021-07-07 10:41:10 +00:00
|
|
|
sys.exit(main(sys.stdout, sys.stderr))
|