AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

locale_database: Use pathlib to manipulate paths in Python code

Browse Source 
pathlib's API is more modern and easier to use than os.path. It
also allows to distinguish between paths and other strings in type
annotations.

Task-number: QTBUG-83488
Pick-to: 6.2
Change-Id: Ie6d9b4e35596f7f6befa4c9635f4a65ea3b20025
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
This commit is contained in:
Ievgenii Meshcheriakov 2021-07-09 15:34:40 +02:00
parent e095fa7f9c
commit 2d0c2c9f1c
5 changed files with 76 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
util/locale_database/cldr.py
View File

@ -36,15 +36,16 @@ The former should normally be all you need to access.
See individual classes for further detail.
"""
from typing import Iterable, TextIO
from xml.dom import minidom
from weakref import WeakValueDictionary as CacheDict
import os
from pathlib import Path
from ldml import Error, Node, XmlScanner, Supplement, LocaleScanner
from qlocalexml import Locale
class CldrReader (object):
def __init__(self, root, grumble = lambda msg: None, whitter = lambda msg: None):
def __init__(self, root: Path, grumble = lambda msg: None, whitter = lambda msg: None):
"""Set up a reader object for reading CLDR data.
Single parameter, root, is the file-system path to the root of
@ -247,7 +248,7 @@ class CldrReader (object):
# the cache. If a process were to instantiate this class with distinct
# roots, each cache would be filled by the first to need it !
class CldrAccess (object):
def __init__(self, root):
def __init__(self, root: Path):
"""Set up a master object for accessing CLDR data.
Single parameter, root, is the file-system path to the root of
@ -255,18 +256,18 @@ class CldrAccess (object):
contain dtd/, main/ and supplemental/ sub-directories."""
self.root = root
def xml(self, *path):
def xml(self, relative_path: str):
"""Load a single XML file and return its root element as an XmlScanner.
The path is interpreted relative to self.root"""
return XmlScanner(Node(self.__xml(path)))
return XmlScanner(Node(self.__xml(relative_path)))
def supplement(self, name):
"""Loads supplemental data as a Supplement object.
The name should be that of a file in common/supplemental/, without path.
"""
return Supplement(Node(self.__xml(('common', 'supplemental', name))))
return Supplement(Node(self.__xml(f'common/supplemental/{name}')))
def locale(self, name):
"""Loads all data for a locale as a LocaleScanner object.
@ -279,16 +280,14 @@ class CldrAccess (object):
return LocaleScanner(name, self.__localeRoots(name), self.__rootLocale)
@property
def fileLocales(self, joinPath = os.path.join, listDirectory = os.listdir,
splitExtension = os.path.splitext):
def fileLocales(self) -> Iterable[str]:
"""Generator for locale IDs seen in file-names.
All *.xml other than root.xml in common/main/ are assumed to
identify locales."""
for name in listDirectory(joinPath(self.root, 'common', 'main')):
stem, ext = splitExtension(name)
if ext == '.xml' and stem != 'root':
yield stem
for path in self.root.joinpath('common/main').glob('*.xml'):
if path.stem != 'root':
yield path.stem
@property
def defaultContentLocales(self):
@ -512,20 +511,20 @@ enumdata.py (keeping the old name as an alias):
return self.__cldrVersion
# Implementation details
def __xml(self, path, cache = CacheDict(), read = minidom.parse, joinPath = os.path.join):
def __xml(self, relative_path: str, cache = CacheDict(), read = minidom.parse):
try:
doc = cache[path]
doc = cache[relative_path]
except KeyError:
cache[path] = doc = read(joinPath(self.root, *path)).documentElement
cache[relative_path] = doc = read(str(self.root.joinpath(relative_path))).documentElement
return doc
def __open(self, path, joinPath=os.path.join):
return open(joinPath(self.root, *path))
def __open(self, relative_path: str) -> TextIO:
return self.root.joinpath(relative_path).open()
@property
def __rootLocale(self, cache = []):
if not cache:
cache.append(self.xml('common', 'main', 'root.xml'))
cache.append(self.xml('common/main/root.xml'))
return cache[0]
@property
@ -535,7 +534,7 @@ enumdata.py (keeping the old name as an alias):
return cache[0]
@property
def __numberSystems(self, cache = {}, joinPath=os.path.join):
def __numberSystems(self, cache = {}):
if not cache:
for ignore, attrs in self.supplement('numberingSystems.xml').find('numberingSystems'):
cache[attrs['id']] = attrs
@ -610,7 +609,7 @@ enumdata.py (keeping the old name as an alias):
return cache
@property
def __unDistinguishedAttributes(self, cache = {}, joinPath = os.path.join):
def __unDistinguishedAttributes(self, cache = {}):
"""Mapping from tag names to lists of attributes.
LDML defines some attributes as 'distinguishing': if a node
@ -630,7 +629,7 @@ enumdata.py (keeping the old name as an alias):
return cache
def __scanLdmlDtd(self, joinPath = os.path.join):
def __scanLdmlDtd(self):
"""Scan the LDML DTD, record CLDR version
Yields (tag, attrs) pairs: on elements with a given tag,
@ -640,7 +639,7 @@ enumdata.py (keeping the old name as an alias):
Sets self.__cldrVersion as a side-effect, since this
information is found in the same file."""
with self.__open(('common', 'dtd', 'ldml.dtd')) as dtd:
with self.__open('common/dtd/ldml.dtd') as dtd:
tag, ignored, last = None, None, None
for line in dtd:
@ -700,7 +699,7 @@ enumdata.py (keeping the old name as an alias):
naming = {'language': 'languages', 'script': 'scripts',
'territory': 'territories', 'variant': 'variants'}):
if not cache:
root = self.xml('common', 'main', 'en.xml').root.findUniqueChild('localeDisplayNames')
root = self.xml('common/main/en.xml').root.findUniqueChild('localeDisplayNames')
for dst, src in naming.items():
cache[dst] = dict(self.__codeMapScan(root.findUniqueChild(src)))
assert cache
@ -743,10 +742,9 @@ enumdata.py (keeping the old name as an alias):
return cache
def __localeAsDoc(self, name, aliasFor = None,
joinPath = os.path.join, exists = os.path.isfile):
path = ('common', 'main', f'{name}.xml')
if exists(joinPath(self.root, *path)):
def __localeAsDoc(self, name: str, aliasFor = None):
path = f'common/main/{name}.xml'
if self.root.joinpath(path).exists():
elt = self.__xml(path)
for child in Node(elt).findAllChildren('alias'):
try:
@ -785,4 +783,4 @@ enumdata.py (keeping the old name as an alias):
return chain
# Unpolute the namespace: we don't need to export these.
del minidom, CacheDict, os
del minidom, CacheDict

10
util/locale_database/cldr2qlocalexml.py
View File

@ -56,7 +56,7 @@ All the scripts mentioned support --help to tell you how to use them.
.. _CLDR: ftp://unicode.org/Public/cldr/
"""
import os
from pathlib import Path
import sys
import argparse
@ -79,10 +79,12 @@ def main(out, err):
args = parser.parse_args()
root = args.cldr_path
if not os.path.exists(os.path.join(root, 'common', 'main', 'root.xml')):
root = Path(args.cldr_path)
root_xml_path = 'common/main/root.xml'
if not root.joinpath(root_xml_path).exists():
parser.error('First argument is the root of the CLDR tree: '
f'found no common/main/root.xml under {root}')
f'found no {root_xml_path} under {root}')
xml = args.out_file
if not xml or xml == '-':

15
util/locale_database/cldr2qtimezone.py
View File

@ -36,8 +36,8 @@ shall update qtbase's src/corelib/time/qtimezoneprivate_data_p.h ready
for use.
"""
import os
import datetime
from pathlib import Path
import textwrap
import argparse
@ -341,17 +341,18 @@ def main(out, err):
args = parser.parse_args()
cldrPath = args.cldr_path
qtPath = args.qtbase_path
cldrPath = Path(args.cldr_path)
qtPath = Path(args.qtbase_path)
if not os.path.isdir(qtPath):
if not qtPath.is_dir():
parser.error(f"No such Qt directory: {qtPath}")
if not os.path.isdir(cldrPath):
if not cldrPath.is_dir():
parser.error(f"No such CLDR directory: {cldrPath}")
dataFilePath = os.path.join(qtPath, 'src', 'corelib', 'time', 'qtimezoneprivate_data_p.h')
if not os.path.isfile(dataFilePath):
dataFilePath = qtPath.joinpath('src/corelib/time/qtimezoneprivate_data_p.h')
if not dataFilePath.is_file():
parser.error(f'No such file: {dataFilePath}')
try:

38
util/locale_database/localetools.py
View File

@ -38,6 +38,7 @@ Classes:
"""
import os
from pathlib import Path
import tempfile
class Error (Exception):
@ -81,29 +82,40 @@ class Transcriber (object):
Callers should call close() on success or cleanup() on failure (to
clear away the temporary file).
"""
def __init__(self, path, temp):
def __init__(self, path: Path, temp_dir: Path):
# Open the old file
self.reader = open(path)
# Create a temp file to write the new data into
temp, tempPath = tempfile.mkstemp(os.path.split(path)[1], dir = temp)
self.__names = path, tempPath
temp, tempPath = tempfile.mkstemp(path.name, dir=temp_dir)
self.path = path
self.tempPath = Path(tempPath)
self.writer = os.fdopen(temp, "w")
def close(self):
def close(self) -> None:
self.reader.close()
self.writer.close()
self.reader = self.writer = None
source, temp = self.__names
os.remove(source)
os.rename(temp, source)
def cleanup(self):
if self.__names:
# Move the modified file to the original location
self.path.unlink()
self.tempPath.rename(self.path)
self.tempPath = None
def cleanup(self) -> None:
if self.reader:
self.reader.close()
self.reader = None
if self.writer:
self.writer.close()
self.writer = None
if self.tempPath:
# Remove temp-file:
os.remove(self.__names[1])
self.__names = ()
self.tempPath.unlink(missing_ok=True)
self.tempPath = None
class SourceFileEditor (Transcriber):
"""Transcriber with transcription of code around a gnerated block.
@ -125,14 +137,14 @@ class SourceFileEditor (Transcriber):
Callers should call close() on success or cleanup() on failure (to
clear away the temporary file); see Transcriber.
"""
def __init__(self, path, temp):
def __init__(self, path: Path, temp_dir: Path):
"""Set up the source file editor.
Requires two arguments: the path to the source file to be read
and, on success, replaced with a new version; and the
directory in which to store the temporary file during the
rewrite."""
super().__init__(path, temp)
super().__init__(path, temp_dir)
self.__copyPrelude()
def close(self):

24
util/locale_database/qlocalexml2cpp.py
View File

@ -33,9 +33,9 @@ Pass the output file from that as first parameter to this script; pass
the root of the qtbase check-out as second parameter.
"""
import os
import datetime
import argparse
from pathlib import Path
from qlocalexml import QLocaleXmlReader
from localetools import unicode2hex, wrap_list, Error, Transcriber, SourceFileEditor
@ -133,7 +133,7 @@ def currencyIsoCodeData(s):
return "{0,0,0}"
class LocaleSourceEditor (SourceFileEditor):
def __init__(self, path, temp, version):
def __init__(self, path: Path, temp: Path, version: str):
super().__init__(path, temp)
self.writer.write(f"""
/*
@ -522,11 +522,11 @@ def main(out, err):
args = parser.parse_args()
qlocalexml = args.input_file
qtsrcdir = args.qtbase_path
qtsrcdir = Path(args.qtbase_path)
calendars = {cal: calendars_map[cal] for cal in args.calendars}
if not (os.path.isdir(qtsrcdir)
and all(os.path.isfile(os.path.join(qtsrcdir, 'src', 'corelib', 'text', leaf))
if not (qtsrcdir.is_dir()
and all(qtsrcdir.joinpath('src/corelib/text', leaf).is_file()
for leaf in ('qlocale_data_p.h', 'qlocale.h', 'qlocale.qdoc'))):
parser.error(f'Missing expected files under qtbase source root {qtsrcdir}')
@ -535,8 +535,7 @@ def main(out, err):
locale_keys = sorted(locale_map.keys(), key=LocaleKeySorter(reader.defaultMap()))
try:
writer = LocaleDataWriter(os.path.join(qtsrcdir, 'src', 'corelib', 'text',
'qlocale_data_p.h'),
writer = LocaleDataWriter(qtsrcdir.joinpath('src/corelib/text/qlocale_data_p.h'),
qtsrcdir, reader.cldrVersion)
except IOError as e:
err.write(f'Failed to open files to transcribe locale data: {e}')
@ -564,9 +563,9 @@ def main(out, err):
# Generate calendar data
for calendar, stem in calendars.items():
try:
writer = CalendarDataWriter(os.path.join(qtsrcdir, 'src', 'corelib', 'time',
f'q{stem}calendar_data_p.h'),
qtsrcdir, reader.cldrVersion)
writer = CalendarDataWriter(
qtsrcdir.joinpath(f'src/corelib/time/q{stem}calendar_data_p.h'),
qtsrcdir, reader.cldrVersion)
except IOError as e:
err.write(f'Failed to open files to transcribe {calendar} data {e}')
return 1
@ -582,7 +581,7 @@ def main(out, err):
# qlocale.h
try:
writer = LocaleHeaderWriter(os.path.join(qtsrcdir, 'src', 'corelib', 'text', 'qlocale.h'),
writer = LocaleHeaderWriter(qtsrcdir.joinpath('src/corelib/text/qlocale.h'),
qtsrcdir, reader.dupes)
except IOError as e:
err.write(f'Failed to open files to transcribe qlocale.h: {e}')
@ -601,8 +600,7 @@ def main(out, err):
# qlocale.qdoc
try:
writer = Transcriber(os.path.join(qtsrcdir, 'src', 'corelib', 'text', 'qlocale.qdoc'),
qtsrcdir)
writer = Transcriber(qtsrcdir.joinpath('src/corelib/text/qlocale.qdoc'), qtsrcdir)
except IOError as e:
err.write(f'Failed to open files to transcribe qlocale.qdoc: {e}')
return 1