mirror of
https://sourceware.org/git/glibc.git
synced 2024-11-21 12:30:06 +00:00
Unicode 7.0.0 update; added generator scripts.
for localedata/ChangeLog [BZ #17588] [BZ #13064] [BZ #14094] [BZ #17998] * unicode-gen/Makefile: New. * unicode-gen/unicode-license.txt: New, from Unicode. * unicode-gen/UnicodeData.txt: New, from Unicode. * unicode-gen/DerivedCoreProperties.txt: New, from Unicode. * unicode-gen/EastAsianWidth.txt: New, from Unicode. * unicode-gen/gen_unicode_ctype.py: New generator, from Mike FABIAN <mfabian@redhat.com>. * unicode-gen/ctype_compatibility.py: New verifier, from Pravin Satpute <psatpute@redhat.com> and Mike FABIAN. * unicode-gen/ctype_compatibility_test_cases.py: New verifier module, from Mike FABIAN. * unicode-gen/utf8_gen.py: New generator, from Pravin Satpute and Mike FABIAN. * unicode-gen/utf8_compatibility.py: New verifier, from Pravin Satpute and Mike FABIAN. * charmaps/UTF-8: Update. * locales/i18n: Update. * gen-unicode-ctype.c: Remove. * tst-ctype-de_DE.ISO-8859-1.in: Adjust, islower now returns true for ordinal indicators.
This commit is contained in:
Alexandre Oliva
parent
e4a399dc3d
commit
4a4839c94a
11
NEWS
11
NEWS
@ -9,8 +9,15 @@ Version 2.22
|
||||
|
||||
* The following bugs are resolved with this release:
|
||||
|
||||
4719, 15319, 15467, 15790, 16560, 17569, 17792, 17912, 17932, 17944,
|
||||
17949, 17964, 17965, 17967, 17969, 17978, 17987, 17991, 17996, 17999.
|
||||
4719, 13064, 14094, 15319, 15467, 15790, 16560, 17569, 17588, 17792,
|
||||
17912, 17932, 17944, 17949, 17964, 17965, 17967, 17969, 17978, 17987,
|
||||
17991, 17996, 17998, 17999.
|
||||
|
||||
* Character encoding and ctype tables were updated to Unicode 7.0.0, using
|
||||
new generator scripts contributed by Pravin Satpute and Mike FABIAN (Red
|
||||
Hat). These updates cause user visible changes, such as the fix for bug
|
||||
17998.
|
||||
|
||||
|
||||
Version 2.21
|
||||
|
||||
|
||||
@ -1,3 +1,30 @@
|
||||
2015-02-20 Alexandre Oliva <aoliva@redhat.com>
|
||||
|
||||
[BZ #17588]
|
||||
[BZ #13064]
|
||||
[BZ #14094]
|
||||
[BZ #17998]
|
||||
* unicode-gen/Makefile: New.
|
||||
* unicode-gen/unicode-license.txt: New, from Unicode.
|
||||
* unicode-gen/UnicodeData.txt: New, from Unicode.
|
||||
* unicode-gen/DerivedCoreProperties.txt: New, from Unicode.
|
||||
* unicode-gen/EastAsianWidth.txt: New, from Unicode.
|
||||
* unicode-gen/gen_unicode_ctype.py: New generator, from Mike
|
||||
FABIAN <mfabian@redhat.com>.
|
||||
* unicode-gen/ctype_compatibility.py: New verifier, from
|
||||
Pravin Satpute <psatpute@redhat.com> and Mike FABIAN.
|
||||
* unicode-gen/ctype_compatibility_test_cases.py: New verifier
|
||||
module, from Mike FABIAN.
|
||||
* unicode-gen/utf8_gen.py: New generator, from Pravin Satpute
|
||||
and Mike FABIAN.
|
||||
* unicode-gen/utf8_compatibility.py: New verifier, from Pravin
|
||||
Satpute and Mike FABIAN.
|
||||
* charmaps/UTF-8: Update.
|
||||
* locales/i18n: Update.
|
||||
* gen-unicode-ctype.c: Remove.
|
||||
* tst-ctype-de_DE.ISO-8859-1.in: Adjust, islower now returns
|
||||
true for ordinal indicators.
|
||||
|
||||
2015-01-21 Marek Polacek <polacek@redhat.com>
|
||||
|
||||
* tests-mbwc/tst_wcscpy.c (tst_wcscpy): Fix condition.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,784 +0,0 @@
|
||||
/* Generate a Unicode conforming LC_CTYPE category from a UnicodeData file.
|
||||
Copyright (C) 2000-2015 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
Contributed by Bruno Haible <haible@clisp.cons.org>, 2000.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
/* Usage example:
|
||||
$ gen-unicode /usr/local/share/Unidata/UnicodeData.txt 3.1
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
/* This structure represents one line in the UnicodeData.txt file. */
|
||||
struct unicode_attribute
|
||||
{
|
||||
const char *name; /* Character name */
|
||||
const char *category; /* General category */
|
||||
const char *combining; /* Canonical combining classes */
|
||||
const char *bidi; /* Bidirectional category */
|
||||
const char *decomposition; /* Character decomposition mapping */
|
||||
const char *decdigit; /* Decimal digit value */
|
||||
const char *digit; /* Digit value */
|
||||
const char *numeric; /* Numeric value */
|
||||
int mirrored; /* mirrored */
|
||||
const char *oldname; /* Old Unicode 1.0 name */
|
||||
const char *comment; /* Comment */
|
||||
unsigned int upper; /* Uppercase mapping */
|
||||
unsigned int lower; /* Lowercase mapping */
|
||||
unsigned int title; /* Titlecase mapping */
|
||||
};
|
||||
|
||||
/* Missing fields are represented with "" for strings, and NONE for
|
||||
characters. */
|
||||
#define NONE (~(unsigned int)0)
|
||||
|
||||
/* The entire contents of the UnicodeData.txt file. */
|
||||
struct unicode_attribute unicode_attributes [0x110000];
|
||||
|
||||
/* Stores in unicode_attributes[i] the values from the given fields. */
|
||||
static void
|
||||
fill_attribute (unsigned int i,
|
||||
const char *field1, const char *field2,
|
||||
const char *field3, const char *field4,
|
||||
const char *field5, const char *field6,
|
||||
const char *field7, const char *field8,
|
||||
const char *field9, const char *field10,
|
||||
const char *field11, const char *field12,
|
||||
const char *field13, const char *field14)
|
||||
{
|
||||
struct unicode_attribute * uni;
|
||||
|
||||
if (i >= 0x110000)
|
||||
{
|
||||
fprintf (stderr, "index too large\n");
|
||||
exit (1);
|
||||
}
|
||||
if (strcmp (field2, "Cs") == 0)
|
||||
/* Surrogates are UTF-16 artefacts, not real characters. Ignore them. */
|
||||
return;
|
||||
uni = &unicode_attributes[i];
|
||||
/* Copy the strings. */
|
||||
uni->name = strdup (field1);
|
||||
uni->category = (field2[0] == '\0' ? "" : strdup (field2));
|
||||
uni->combining = (field3[0] == '\0' ? "" : strdup (field3));
|
||||
uni->bidi = (field4[0] == '\0' ? "" : strdup (field4));
|
||||
uni->decomposition = (field5[0] == '\0' ? "" : strdup (field5));
|
||||
uni->decdigit = (field6[0] == '\0' ? "" : strdup (field6));
|
||||
uni->digit = (field7[0] == '\0' ? "" : strdup (field7));
|
||||
uni->numeric = (field8[0] == '\0' ? "" : strdup (field8));
|
||||
uni->mirrored = (field9[0] == 'Y');
|
||||
uni->oldname = (field10[0] == '\0' ? "" : strdup (field10));
|
||||
uni->comment = (field11[0] == '\0' ? "" : strdup (field11));
|
||||
uni->upper = (field12[0] =='\0' ? NONE : strtoul (field12, NULL, 16));
|
||||
uni->lower = (field13[0] =='\0' ? NONE : strtoul (field13, NULL, 16));
|
||||
uni->title = (field14[0] =='\0' ? NONE : strtoul (field14, NULL, 16));
|
||||
}
|
||||
|
||||
/* Maximum length of a field in the UnicodeData.txt file. */
|
||||
#define FIELDLEN 120
|
||||
|
||||
/* Reads the next field from STREAM. The buffer BUFFER has size FIELDLEN.
|
||||
Reads up to (but excluding) DELIM.
|
||||
Returns 1 when a field was successfully read, otherwise 0. */
|
||||
static int
|
||||
getfield (FILE *stream, char *buffer, int delim)
|
||||
{
|
||||
int count = 0;
|
||||
int c;
|
||||
|
||||
for (; (c = getc (stream)), (c != EOF && c != delim); )
|
||||
{
|
||||
/* The original unicode.org UnicodeData.txt file happens to have
|
||||
CR/LF line terminators. Silently convert to LF. */
|
||||
if (c == '\r')
|
||||
continue;
|
||||
|
||||
/* Put c into the buffer. */
|
||||
if (++count >= FIELDLEN - 1)
|
||||
{
|
||||
fprintf (stderr, "field too long\n");
|
||||
exit (1);
|
||||
}
|
||||
*buffer++ = c;
|
||||
}
|
||||
|
||||
if (c == EOF)
|
||||
return 0;
|
||||
|
||||
*buffer = '\0';
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Stores in unicode_attributes[] the entire contents of the UnicodeData.txt
|
||||
file. */
|
||||
static void
|
||||
fill_attributes (const char *unicodedata_filename)
|
||||
{
|
||||
unsigned int i, j;
|
||||
FILE *stream;
|
||||
char field0[FIELDLEN];
|
||||
char field1[FIELDLEN];
|
||||
char field2[FIELDLEN];
|
||||
char field3[FIELDLEN];
|
||||
char field4[FIELDLEN];
|
||||
char field5[FIELDLEN];
|
||||
char field6[FIELDLEN];
|
||||
char field7[FIELDLEN];
|
||||
char field8[FIELDLEN];
|
||||
char field9[FIELDLEN];
|
||||
char field10[FIELDLEN];
|
||||
char field11[FIELDLEN];
|
||||
char field12[FIELDLEN];
|
||||
char field13[FIELDLEN];
|
||||
char field14[FIELDLEN];
|
||||
int lineno = 0;
|
||||
|
||||
for (i = 0; i < 0x110000; i++)
|
||||
unicode_attributes[i].name = NULL;
|
||||
|
||||
stream = fopen (unicodedata_filename, "r");
|
||||
if (stream == NULL)
|
||||
{
|
||||
fprintf (stderr, "error during fopen of '%s'\n", unicodedata_filename);
|
||||
exit (1);
|
||||
}
|
||||
|
||||
for (;;)
|
||||
{
|
||||
int n;
|
||||
|
||||
lineno++;
|
||||
n = getfield (stream, field0, ';');
|
||||
n += getfield (stream, field1, ';');
|
||||
n += getfield (stream, field2, ';');
|
||||
n += getfield (stream, field3, ';');
|
||||
n += getfield (stream, field4, ';');
|
||||
n += getfield (stream, field5, ';');
|
||||
n += getfield (stream, field6, ';');
|
||||
n += getfield (stream, field7, ';');
|
||||
n += getfield (stream, field8, ';');
|
||||
n += getfield (stream, field9, ';');
|
||||
n += getfield (stream, field10, ';');
|
||||
n += getfield (stream, field11, ';');
|
||||
n += getfield (stream, field12, ';');
|
||||
n += getfield (stream, field13, ';');
|
||||
n += getfield (stream, field14, '\n');
|
||||
if (n == 0)
|
||||
break;
|
||||
if (n != 15)
|
||||
{
|
||||
fprintf (stderr, "short line in'%s':%d\n",
|
||||
unicodedata_filename, lineno);
|
||||
exit (1);
|
||||
}
|
||||
i = strtoul (field0, NULL, 16);
|
||||
if (field1[0] == '<'
|
||||
&& strlen (field1) >= 9
|
||||
&& !strcmp (field1 + strlen(field1) - 8, ", First>"))
|
||||
{
|
||||
/* Deal with a range. */
|
||||
lineno++;
|
||||
n = getfield (stream, field0, ';');
|
||||
n += getfield (stream, field1, ';');
|
||||
n += getfield (stream, field2, ';');
|
||||
n += getfield (stream, field3, ';');
|
||||
n += getfield (stream, field4, ';');
|
||||
n += getfield (stream, field5, ';');
|
||||
n += getfield (stream, field6, ';');
|
||||
n += getfield (stream, field7, ';');
|
||||
n += getfield (stream, field8, ';');
|
||||
n += getfield (stream, field9, ';');
|
||||
n += getfield (stream, field10, ';');
|
||||
n += getfield (stream, field11, ';');
|
||||
n += getfield (stream, field12, ';');
|
||||
n += getfield (stream, field13, ';');
|
||||
n += getfield (stream, field14, '\n');
|
||||
if (n != 15)
|
||||
{
|
||||
fprintf (stderr, "missing end range in '%s':%d\n",
|
||||
unicodedata_filename, lineno);
|
||||
exit (1);
|
||||
}
|
||||
if (!(field1[0] == '<'
|
||||
&& strlen (field1) >= 8
|
||||
&& !strcmp (field1 + strlen (field1) - 7, ", Last>")))
|
||||
{
|
||||
fprintf (stderr, "missing end range in '%s':%d\n",
|
||||
unicodedata_filename, lineno);
|
||||
exit (1);
|
||||
}
|
||||
field1[strlen (field1) - 7] = '\0';
|
||||
j = strtoul (field0, NULL, 16);
|
||||
for (; i <= j; i++)
|
||||
fill_attribute (i, field1+1, field2, field3, field4, field5,
|
||||
field6, field7, field8, field9, field10,
|
||||
field11, field12, field13, field14);
|
||||
}
|
||||
else
|
||||
{
|
||||
/* Single character line */
|
||||
fill_attribute (i, field1, field2, field3, field4, field5,
|
||||
field6, field7, field8, field9, field10,
|
||||
field11, field12, field13, field14);
|
||||
}
|
||||
}
|
||||
if (ferror (stream) || fclose (stream))
|
||||
{
|
||||
fprintf (stderr, "error reading from '%s'\n", unicodedata_filename);
|
||||
exit (1);
|
||||
}
|
||||
}
|
||||
|
||||
/* Character mappings. */
|
||||
|
||||
static unsigned int
|
||||
to_upper (unsigned int ch)
|
||||
{
|
||||
if (unicode_attributes[ch].name != NULL
|
||||
&& unicode_attributes[ch].upper != NONE)
|
||||
return unicode_attributes[ch].upper;
|
||||
else
|
||||
return ch;
|
||||
}
|
||||
|
||||
static unsigned int
|
||||
to_lower (unsigned int ch)
|
||||
{
|
||||
if (unicode_attributes[ch].name != NULL
|
||||
&& unicode_attributes[ch].lower != NONE)
|
||||
return unicode_attributes[ch].lower;
|
||||
else
|
||||
return ch;
|
||||
}
|
||||
|
||||
static unsigned int
|
||||
to_title (unsigned int ch)
|
||||
{
|
||||
if (unicode_attributes[ch].name != NULL
|
||||
&& unicode_attributes[ch].title != NONE)
|
||||
return unicode_attributes[ch].title;
|
||||
else
|
||||
return ch;
|
||||
}
|
||||
|
||||
/* Character class properties. */
|
||||
|
||||
static bool
|
||||
is_upper (unsigned int ch)
|
||||
{
|
||||
return (to_lower (ch) != ch);
|
||||
}
|
||||
|
||||
static bool
|
||||
is_lower (unsigned int ch)
|
||||
{
|
||||
return (to_upper (ch) != ch)
|
||||
/* <U00DF> is lowercase, but without simple to_upper mapping. */
|
||||
|| (ch == 0x00DF);
|
||||
}
|
||||
|
||||
static bool
|
||||
is_alpha (unsigned int ch)
|
||||
{
|
||||
return (unicode_attributes[ch].name != NULL
|
||||
&& ((unicode_attributes[ch].category[0] == 'L'
|
||||
/* Theppitak Karoonboonyanan <thep@links.nectec.or.th> says
|
||||
<U0E2F>, <U0E46> should belong to is_punct. */
|
||||
&& (ch != 0x0E2F) && (ch != 0x0E46))
|
||||
/* Theppitak Karoonboonyanan <thep@links.nectec.or.th> says
|
||||
<U0E31>, <U0E34>..<U0E3A>, <U0E47>..<U0E4E> are is_alpha. */
|
||||
|| (ch == 0x0E31)
|
||||
|| (ch >= 0x0E34 && ch <= 0x0E3A)
|
||||
|| (ch >= 0x0E47 && ch <= 0x0E4E)
|
||||
/* Avoid warning for <U0345>. */
|
||||
|| (ch == 0x0345)
|
||||
/* Avoid warnings for <U2160>..<U217F>. */
|
||||
|| (unicode_attributes[ch].category[0] == 'N'
|
||||
&& unicode_attributes[ch].category[1] == 'l')
|
||||
/* Avoid warnings for <U24B6>..<U24E9>. */
|
||||
|| (unicode_attributes[ch].category[0] == 'S'
|
||||
&& unicode_attributes[ch].category[1] == 'o'
|
||||
&& strstr (unicode_attributes[ch].name, " LETTER ")
|
||||
!= NULL)
|
||||
/* Consider all the non-ASCII digits as alphabetic.
|
||||
ISO C 99 forbids us to have them in category "digit",
|
||||
but we want iswalnum to return true on them. */
|
||||
|| (unicode_attributes[ch].category[0] == 'N'
|
||||
&& unicode_attributes[ch].category[1] == 'd'
|
||||
&& !(ch >= 0x0030 && ch <= 0x0039))));
|
||||
}
|
||||
|
||||
static bool
|
||||
is_digit (unsigned int ch)
|
||||
{
|
||||
#if 0
|
||||
return (unicode_attributes[ch].name != NULL
|
||||
&& unicode_attributes[ch].category[0] == 'N'
|
||||
&& unicode_attributes[ch].category[1] == 'd');
|
||||
/* Note: U+0BE7..U+0BEF and U+1369..U+1371 are digit systems without
|
||||
a zero. Must add <0> in front of them by hand. */
|
||||
#else
|
||||
/* SUSV2 gives us some freedom for the "digit" category, but ISO C 99
|
||||
takes it away:
|
||||
7.25.2.1.5:
|
||||
The iswdigit function tests for any wide character that corresponds
|
||||
to a decimal-digit character (as defined in 5.2.1).
|
||||
5.2.1:
|
||||
the 10 decimal digits 0 1 2 3 4 5 6 7 8 9
|
||||
*/
|
||||
return (ch >= 0x0030 && ch <= 0x0039);
|
||||
#endif
|
||||
}
|
||||
|
||||
static bool
|
||||
is_outdigit (unsigned int ch)
|
||||
{
|
||||
return (ch >= 0x0030 && ch <= 0x0039);
|
||||
}
|
||||
|
||||
static bool
|
||||
is_blank (unsigned int ch)
|
||||
{
|
||||
return (ch == 0x0009 /* '\t' */
|
||||
/* Category Zs without mention of "<noBreak>" */
|
||||
|| (unicode_attributes[ch].name != NULL
|
||||
&& unicode_attributes[ch].category[0] == 'Z'
|
||||
&& unicode_attributes[ch].category[1] == 's'
|
||||
&& !strstr (unicode_attributes[ch].decomposition, "<noBreak>")));
|
||||
}
|
||||
|
||||
static bool
|
||||
is_space (unsigned int ch)
|
||||
{
|
||||
/* Don't make U+00A0 a space. Non-breaking space means that all programs
|
||||
should treat it like a punctuation character, not like a space. */
|
||||
return (ch == 0x0020 /* ' ' */
|
||||
|| ch == 0x000C /* '\f' */
|
||||
|| ch == 0x000A /* '\n' */
|
||||
|| ch == 0x000D /* '\r' */
|
||||
|| ch == 0x0009 /* '\t' */
|
||||
|| ch == 0x000B /* '\v' */
|
||||
/* Categories Zl, Zp, and Zs without mention of "<noBreak>" */
|
||||
|| (unicode_attributes[ch].name != NULL
|
||||
&& unicode_attributes[ch].category[0] == 'Z'
|
||||
&& (unicode_attributes[ch].category[1] == 'l'
|
||||
|| unicode_attributes[ch].category[1] == 'p'
|
||||
|| (unicode_attributes[ch].category[1] == 's'
|
||||
&& !strstr (unicode_attributes[ch].decomposition,
|
||||
"<noBreak>")))));
|
||||
}
|
||||
|
||||
static bool
|
||||
is_cntrl (unsigned int ch)
|
||||
{
|
||||
return (unicode_attributes[ch].name != NULL
|
||||
&& (!strcmp (unicode_attributes[ch].name, "<control>")
|
||||
/* Categories Zl and Zp */
|
||||
|| (unicode_attributes[ch].category[0] == 'Z'
|
||||
&& (unicode_attributes[ch].category[1] == 'l'
|
||||
|| unicode_attributes[ch].category[1] == 'p'))));
|
||||
}
|
||||
|
||||
static bool
|
||||
is_xdigit (unsigned int ch)
|
||||
{
|
||||
#if 0
|
||||
return is_digit (ch)
|
||||
|| (ch >= 0x0041 && ch <= 0x0046)
|
||||
|| (ch >= 0x0061 && ch <= 0x0066);
|
||||
#else
|
||||
/* SUSV2 gives us some freedom for the "xdigit" category, but ISO C 99
|
||||
takes it away:
|
||||
7.25.2.1.12:
|
||||
The iswxdigit function tests for any wide character that corresponds
|
||||
to a hexadecimal-digit character (as defined in 6.4.4.1).
|
||||
6.4.4.1:
|
||||
hexadecimal-digit: one of 0 1 2 3 4 5 6 7 8 9 a b c d e f A B C D E F
|
||||
*/
|
||||
return (ch >= 0x0030 && ch <= 0x0039)
|
||||
|| (ch >= 0x0041 && ch <= 0x0046)
|
||||
|| (ch >= 0x0061 && ch <= 0x0066);
|
||||
#endif
|
||||
}
|
||||
|
||||
static bool
|
||||
is_graph (unsigned int ch)
|
||||
{
|
||||
return (unicode_attributes[ch].name != NULL
|
||||
&& strcmp (unicode_attributes[ch].name, "<control>")
|
||||
&& !is_space (ch));
|
||||
}
|
||||
|
||||
static bool
|
||||
is_print (unsigned int ch)
|
||||
{
|
||||
return (unicode_attributes[ch].name != NULL
|
||||
&& strcmp (unicode_attributes[ch].name, "<control>")
|
||||
/* Categories Zl and Zp */
|
||||
&& !(unicode_attributes[ch].name != NULL
|
||||
&& unicode_attributes[ch].category[0] == 'Z'
|
||||
&& (unicode_attributes[ch].category[1] == 'l'
|
||||
|| unicode_attributes[ch].category[1] == 'p')));
|
||||
}
|
||||
|
||||
static bool
|
||||
is_punct (unsigned int ch)
|
||||
{
|
||||
#if 0
|
||||
return (unicode_attributes[ch].name != NULL
|
||||
&& unicode_attributes[ch].category[0] == 'P');
|
||||
#else
|
||||
/* The traditional POSIX definition of punctuation is every graphic,
|
||||
non-alphanumeric character. */
|
||||
return (is_graph (ch) && !is_alpha (ch) && !is_digit (ch));
|
||||
#endif
|
||||
}
|
||||
|
||||
static bool
|
||||
is_combining (unsigned int ch)
|
||||
{
|
||||
/* Up to Unicode 3.0.1 we took the Combining property from the PropList.txt
|
||||
file. In 3.0.1 it was identical to the union of the general categories
|
||||
"Mn", "Mc", "Me". In Unicode 3.1 this property has been dropped from the
|
||||
PropList.txt file, so we take the latter definition. */
|
||||
return (unicode_attributes[ch].name != NULL
|
||||
&& unicode_attributes[ch].category[0] == 'M'
|
||||
&& (unicode_attributes[ch].category[1] == 'n'
|
||||
|| unicode_attributes[ch].category[1] == 'c'
|
||||
|| unicode_attributes[ch].category[1] == 'e'));
|
||||
}
|
||||
|
||||
static bool
|
||||
is_combining_level3 (unsigned int ch)
|
||||
{
|
||||
return is_combining (ch)
|
||||
&& !(unicode_attributes[ch].combining[0] != '\0'
|
||||
&& unicode_attributes[ch].combining[0] != '0'
|
||||
&& strtoul (unicode_attributes[ch].combining, NULL, 10) >= 200);
|
||||
}
|
||||
|
||||
/* Return the UCS symbol string for a Unicode character. */
|
||||
static const char *
|
||||
ucs_symbol (unsigned int i)
|
||||
{
|
||||
static char buf[11+1];
|
||||
|
||||
sprintf (buf, (i < 0x10000 ? "<U%04X>" : "<U%08X>"), i);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* Return the UCS symbol range string for a Unicode characters interval. */
|
||||
static const char *
|
||||
ucs_symbol_range (unsigned int low, unsigned int high)
|
||||
{
|
||||
static char buf[24+1];
|
||||
|
||||
strcpy (buf, ucs_symbol (low));
|
||||
strcat (buf, "..");
|
||||
strcat (buf, ucs_symbol (high));
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* Output a character class (= property) table. */
|
||||
|
||||
static void
|
||||
output_charclass (FILE *stream, const char *classname,
|
||||
bool (*func) (unsigned int))
|
||||
{
|
||||
char table[0x110000];
|
||||
unsigned int i;
|
||||
bool need_semicolon;
|
||||
const int max_column = 75;
|
||||
int column;
|
||||
|
||||
for (i = 0; i < 0x110000; i++)
|
||||
table[i] = (int) func (i);
|
||||
|
||||
fprintf (stream, "%s ", classname);
|
||||
need_semicolon = false;
|
||||
column = 1000;
|
||||
for (i = 0; i < 0x110000; )
|
||||
{
|
||||
if (!table[i])
|
||||
i++;
|
||||
else
|
||||
{
|
||||
unsigned int low, high;
|
||||
char buf[25];
|
||||
|
||||
low = i;
|
||||
do
|
||||
i++;
|
||||
while (i < 0x110000 && table[i]);
|
||||
high = i - 1;
|
||||
|
||||
if (low == high)
|
||||
strcpy (buf, ucs_symbol (low));
|
||||
else
|
||||
strcpy (buf, ucs_symbol_range (low, high));
|
||||
|
||||
if (need_semicolon)
|
||||
{
|
||||
fprintf (stream, ";");
|
||||
column++;
|
||||
}
|
||||
|
||||
if (column + strlen (buf) > max_column)
|
||||
{
|
||||
fprintf (stream, "/\n ");
|
||||
column = 3;
|
||||
}
|
||||
|
||||
fprintf (stream, "%s", buf);
|
||||
column += strlen (buf);
|
||||
need_semicolon = true;
|
||||
}
|
||||
}
|
||||
fprintf (stream, "\n");
|
||||
}
|
||||
|
||||
/* Output a character mapping table. */
|
||||
|
||||
static void
|
||||
output_charmap (FILE *stream, const char *mapname,
|
||||
unsigned int (*func) (unsigned int))
|
||||
{
|
||||
char table[0x110000];
|
||||
unsigned int i;
|
||||
bool need_semicolon;
|
||||
const int max_column = 75;
|
||||
int column;
|
||||
|
||||
for (i = 0; i < 0x110000; i++)
|
||||
table[i] = (func (i) != i);
|
||||
|
||||
fprintf (stream, "%s ", mapname);
|
||||
need_semicolon = false;
|
||||
column = 1000;
|
||||
for (i = 0; i < 0x110000; i++)
|
||||
if (table[i])
|
||||
{
|
||||
char buf[25+1];
|
||||
|
||||
strcpy (buf, "(");
|
||||
strcat (buf, ucs_symbol (i));
|
||||
strcat (buf, ",");
|
||||
strcat (buf, ucs_symbol (func (i)));
|
||||
strcat (buf, ")");
|
||||
|
||||
if (need_semicolon)
|
||||
{
|
||||
fprintf (stream, ";");
|
||||
column++;
|
||||
}
|
||||
|
||||
if (column + strlen (buf) > max_column)
|
||||
{
|
||||
fprintf (stream, "/\n ");
|
||||
column = 3;
|
||||
}
|
||||
|
||||
fprintf (stream, "%s", buf);
|
||||
column += strlen (buf);
|
||||
need_semicolon = true;
|
||||
}
|
||||
fprintf (stream, "\n");
|
||||
}
|
||||
|
||||
/* Output the width table. */
|
||||
|
||||
static void
|
||||
output_widthmap (FILE *stream)
|
||||
{
|
||||
}
|
||||
|
||||
/* Output the tables to the given file. */
|
||||
|
||||
static void
|
||||
output_tables (const char *filename, const char *version)
|
||||
{
|
||||
FILE *stream;
|
||||
unsigned int ch;
|
||||
|
||||
stream = fopen (filename, "w");
|
||||
if (stream == NULL)
|
||||
{
|
||||
fprintf (stderr, "cannot open '%s' for writing\n", filename);
|
||||
exit (1);
|
||||
}
|
||||
|
||||
fprintf (stream, "escape_char /\n");
|
||||
fprintf (stream, "comment_char %%\n");
|
||||
fprintf (stream, "\n");
|
||||
fprintf (stream, "%% Generated automatically by gen-unicode-ctype for Unicode %s.\n",
|
||||
version);
|
||||
fprintf (stream, "\n");
|
||||
|
||||
fprintf (stream, "LC_IDENTIFICATION\n");
|
||||
fprintf (stream, "title \"Unicode %s FDCC-set\"\n", version);
|
||||
fprintf (stream, "source \"UnicodeData.txt, PropList.txt\"\n");
|
||||
fprintf (stream, "address \"\"\n");
|
||||
fprintf (stream, "contact \"\"\n");
|
||||
fprintf (stream, "email \"bug-glibc-locales@gnu.org\"\n");
|
||||
fprintf (stream, "tel \"\"\n");
|
||||
fprintf (stream, "fax \"\"\n");
|
||||
fprintf (stream, "language \"\"\n");
|
||||
fprintf (stream, "territory \"Earth\"\n");
|
||||
fprintf (stream, "revision \"%s\"\n", version);
|
||||
{
|
||||
time_t now;
|
||||
char date[11];
|
||||
now = time (NULL);
|
||||
strftime (date, sizeof (date), "%Y-%m-%d", gmtime (&now));
|
||||
fprintf (stream, "date \"%s\"\n", date);
|
||||
}
|
||||
fprintf (stream, "category \"unicode:2001\";LC_CTYPE\n");
|
||||
fprintf (stream, "END LC_IDENTIFICATION\n");
|
||||
fprintf (stream, "\n");
|
||||
|
||||
/* Verifications. */
|
||||
for (ch = 0; ch < 0x110000; ch++)
|
||||
{
|
||||
/* toupper restriction: "Only characters specified for the keywords
|
||||
lower and upper shall be specified. */
|
||||
if (to_upper (ch) != ch && !(is_lower (ch) || is_upper (ch)))
|
||||
fprintf (stderr,
|
||||
"%s is not upper|lower but toupper(0x%04X) = 0x%04X\n",
|
||||
ucs_symbol (ch), ch, to_upper (ch));
|
||||
|
||||
/* tolower restriction: "Only characters specified for the keywords
|
||||
lower and upper shall be specified. */
|
||||
if (to_lower (ch) != ch && !(is_lower (ch) || is_upper (ch)))
|
||||
fprintf (stderr,
|
||||
"%s is not upper|lower but tolower(0x%04X) = 0x%04X\n",
|
||||
ucs_symbol (ch), ch, to_lower (ch));
|
||||
|
||||
/* alpha restriction: "Characters classified as either upper or lower
|
||||
shall automatically belong to this class. */
|
||||
if ((is_lower (ch) || is_upper (ch)) && !is_alpha (ch))
|
||||
fprintf (stderr, "%s is upper|lower but not alpha\n", ucs_symbol (ch));
|
||||
|
||||
/* alpha restriction: "No character specified for the keywords cntrl,
|
||||
digit, punct or space shall be specified." */
|
||||
if (is_alpha (ch) && is_cntrl (ch))
|
||||
fprintf (stderr, "%s is alpha and cntrl\n", ucs_symbol (ch));
|
||||
if (is_alpha (ch) && is_digit (ch))
|
||||
fprintf (stderr, "%s is alpha and digit\n", ucs_symbol (ch));
|
||||
if (is_alpha (ch) && is_punct (ch))
|
||||
fprintf (stderr, "%s is alpha and punct\n", ucs_symbol (ch));
|
||||
if (is_alpha (ch) && is_space (ch))
|
||||
fprintf (stderr, "%s is alpha and space\n", ucs_symbol (ch));
|
||||
|
||||
/* space restriction: "No character specified for the keywords upper,
|
||||
lower, alpha, digit, graph or xdigit shall be specified."
|
||||
upper, lower, alpha already checked above. */
|
||||
if (is_space (ch) && is_digit (ch))
|
||||
fprintf (stderr, "%s is space and digit\n", ucs_symbol (ch));
|
||||
if (is_space (ch) && is_graph (ch))
|
||||
fprintf (stderr, "%s is space and graph\n", ucs_symbol (ch));
|
||||
if (is_space (ch) && is_xdigit (ch))
|
||||
fprintf (stderr, "%s is space and xdigit\n", ucs_symbol (ch));
|
||||
|
||||
/* cntrl restriction: "No character specified for the keywords upper,
|
||||
lower, alpha, digit, punct, graph, print or xdigit shall be
|
||||
specified." upper, lower, alpha already checked above. */
|
||||
if (is_cntrl (ch) && is_digit (ch))
|
||||
fprintf (stderr, "%s is cntrl and digit\n", ucs_symbol (ch));
|
||||
if (is_cntrl (ch) && is_punct (ch))
|
||||
fprintf (stderr, "%s is cntrl and punct\n", ucs_symbol (ch));
|
||||
if (is_cntrl (ch) && is_graph (ch))
|
||||
fprintf (stderr, "%s is cntrl and graph\n", ucs_symbol (ch));
|
||||
if (is_cntrl (ch) && is_print (ch))
|
||||
fprintf (stderr, "%s is cntrl and print\n", ucs_symbol (ch));
|
||||
if (is_cntrl (ch) && is_xdigit (ch))
|
||||
fprintf (stderr, "%s is cntrl and xdigit\n", ucs_symbol (ch));
|
||||
|
||||
/* punct restriction: "No character specified for the keywords upper,
|
||||
lower, alpha, digit, cntrl, xdigit or as the <space> character shall
|
||||
be specified." upper, lower, alpha, cntrl already checked above. */
|
||||
if (is_punct (ch) && is_digit (ch))
|
||||
fprintf (stderr, "%s is punct and digit\n", ucs_symbol (ch));
|
||||
if (is_punct (ch) && is_xdigit (ch))
|
||||
fprintf (stderr, "%s is punct and xdigit\n", ucs_symbol (ch));
|
||||
if (is_punct (ch) && (ch == 0x0020))
|
||||
fprintf (stderr, "%s is punct\n", ucs_symbol (ch));
|
||||
|
||||
/* graph restriction: "No character specified for the keyword cntrl
|
||||
shall be specified." Already checked above. */
|
||||
|
||||
/* print restriction: "No character specified for the keyword cntrl
|
||||
shall be specified." Already checked above. */
|
||||
|
||||
/* graph - print relation: differ only in the <space> character.
|
||||
How is this possible if there are more than one space character?!
|
||||
I think susv2/xbd/locale.html should speak of "space characters",
|
||||
not "space character". */
|
||||
if (is_print (ch) && !(is_graph (ch) || /* ch == 0x0020 */ is_space (ch)))
|
||||
fprintf (stderr,
|
||||
"%s is print but not graph|<space>\n", ucs_symbol (ch));
|
||||
if (!is_print (ch) && (is_graph (ch) || ch == 0x0020))
|
||||
fprintf (stderr,
|
||||
"%s is graph|<space> but not print\n", ucs_symbol (ch));
|
||||
}
|
||||
|
||||
fprintf (stream, "LC_CTYPE\n");
|
||||
output_charclass (stream, "upper", is_upper);
|
||||
output_charclass (stream, "lower", is_lower);
|
||||
output_charclass (stream, "alpha", is_alpha);
|
||||
output_charclass (stream, "digit", is_digit);
|
||||
output_charclass (stream, "outdigit", is_outdigit);
|
||||
output_charclass (stream, "blank", is_blank);
|
||||
output_charclass (stream, "space", is_space);
|
||||
output_charclass (stream, "cntrl", is_cntrl);
|
||||
output_charclass (stream, "punct", is_punct);
|
||||
output_charclass (stream, "xdigit", is_xdigit);
|
||||
output_charclass (stream, "graph", is_graph);
|
||||
output_charclass (stream, "print", is_print);
|
||||
output_charclass (stream, "class \"combining\";", is_combining);
|
||||
output_charclass (stream, "class \"combining_level3\";", is_combining_level3);
|
||||
output_charmap (stream, "toupper", to_upper);
|
||||
output_charmap (stream, "tolower", to_lower);
|
||||
output_charmap (stream, "map \"totitle\";", to_title);
|
||||
output_widthmap (stream);
|
||||
fprintf (stream, "END LC_CTYPE\n");
|
||||
|
||||
if (ferror (stream) || fclose (stream))
|
||||
{
|
||||
fprintf (stderr, "error writing to '%s'\n", filename);
|
||||
exit (1);
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
main (int argc, char * argv[])
|
||||
{
|
||||
if (argc != 3)
|
||||
{
|
||||
fprintf (stderr, "Usage: %s UnicodeData.txt version\n", argv[0]);
|
||||
exit (1);
|
||||
}
|
||||
|
||||
fill_attributes (argv[1]);
|
||||
|
||||
output_tables ("unicode", argv[2]);
|
||||
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,5 +1,5 @@
|
||||
lower 嵗╯丰戍貝物洎悖停眾斯須號獄播噶擱藏霰匸<E99CB0>帊昅恘
|
||||
000000000000000000000100000000000000000000000000
|
||||
000000000010000000000100001000000000000000000000
|
||||
lower 倳眑婭笫崷窙嗲睧颬睼麧緗鴇膹擨闀貘覷鏷禴矙𡜍𦶠<F0A19C8D>
|
||||
000000000000000111111111111111111111111011111111
|
||||
upper 嵗╯丰戍貝物洎悖停眾斯須號獄播噶擱藏霰匸<E99CB0>帊昅恘
|
||||
|
||||
10794
localedata/unicode-gen/DerivedCoreProperties.txt
Normal file
10794
localedata/unicode-gen/DerivedCoreProperties.txt
Normal file
File diff suppressed because it is too large
Load Diff
2121
localedata/unicode-gen/EastAsianWidth.txt
Normal file
2121
localedata/unicode-gen/EastAsianWidth.txt
Normal file
File diff suppressed because it is too large
Load Diff
99
localedata/unicode-gen/Makefile
Normal file
99
localedata/unicode-gen/Makefile
Normal file
@ -0,0 +1,99 @@
|
||||
# Copyright (C) 2015 Free Software Foundation, Inc.
|
||||
# This file is part of the GNU C Library.
|
||||
|
||||
# The GNU C Library is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU Lesser General Public
|
||||
# License as published by the Free Software Foundation; either
|
||||
# version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
# The GNU C Library is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Lesser General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with the GNU C Library; if not, see
|
||||
# <http://www.gnu.org/licenses/>.
|
||||
|
||||
# Makefile for generating and updating Unicode-extracted files.
|
||||
|
||||
|
||||
# This Makefile is NOT used as part of the GNU libc build. It needs
|
||||
# to be run manually, within the source tree, at Unicode upgrades
|
||||
# (change UNICODE_VERSION below), to update ../locales/i18n ctype
|
||||
# information (part of the file is preserved, so don't wipe it all
|
||||
# out), and ../charmaps/UTF-8.
|
||||
|
||||
# Use make all to generate the files used in the glibc build out of
|
||||
# the original Unicode files; make check to verify that they are what
|
||||
# we expect; make install to copy them to the location expected by the
|
||||
# glibc build; and make clean to remove all generated files.
|
||||
|
||||
# We keep a local copy of the downloaded Unicode files, to avoid
|
||||
# running afoul of the LGPL corresponding sources requirements, even
|
||||
# though it's not clear that they are preferred over the generated
|
||||
# files for making modifications.
|
||||
|
||||
|
||||
UNICODE_VERSION = 7.0.0
|
||||
|
||||
PYTHON3 = python3
|
||||
WGET = wget
|
||||
|
||||
DOWNLOADS = UnicodeData.txt DerivedCoreProperties.txt EastAsianWidth.txt
|
||||
GENERATED = i18n UTF-8
|
||||
REPORTS = i18n-report UTF-8-report
|
||||
|
||||
all: $(GENERATED)
|
||||
|
||||
check: check-i18n check-UTF-8
|
||||
|
||||
install:
|
||||
cp -p i18n ../locales/i18n
|
||||
cp -p UTF-8 ../charmaps/UTF-8
|
||||
|
||||
clean: mostlyclean
|
||||
-rm -rf __pycache__
|
||||
mostlyclean:
|
||||
-rm -f $(REPORTS) $(GENERATED)
|
||||
|
||||
.PHONY: all check clean mostlyclean install
|
||||
|
||||
i18n: UnicodeData.txt DerivedCoreProperties.txt
|
||||
i18n: ../locales/i18n # Preserve non-ctype information.
|
||||
i18n: gen_unicode_ctype.py
|
||||
$(PYTHON3) gen_unicode_ctype.py -u UnicodeData.txt \
|
||||
-d DerivedCoreProperties.txt -i ../locales/i18n -o $@ \
|
||||
--unicode_version $(UNICODE_VERSION)
|
||||
|
||||
i18n-report: i18n ../locales/i18n
|
||||
i18n-report: ctype_compatibility.py ctype_compatibility_test_cases.py
|
||||
$(PYTHON3) ./ctype_compatibility.py -o ../locales/i18n \
|
||||
-n i18n -a -m > $@
|
||||
|
||||
check-i18n: i18n-report
|
||||
@if grep '\(Missing\|Added\) [^0]\|^Number of errors[^=]* = [^0]' \
|
||||
i18n-report; \
|
||||
then echo manual verification required; false; else true; fi
|
||||
|
||||
UTF-8: UnicodeData.txt EastAsianWidth.txt
|
||||
UTF-8: utf8_gen.py
|
||||
$(PYTHON3) utf8_gen.py UnicodeData.txt EastAsianWidth.txt
|
||||
|
||||
UTF-8-report: UTF-8 ../charmaps/UTF-8
|
||||
UTF-8-report: utf8_compatibility.py
|
||||
$(PYTHON3) ./utf8_compatibility.py -o ../charmaps/UTF-8 \
|
||||
-n UTF-8 -a -m > $@
|
||||
|
||||
check-UTF-8: UTF-8-report
|
||||
@if grep '^Total.*: [^0]' UTF-8-report; \
|
||||
then echo manual verification required; false; else true; fi
|
||||
|
||||
|
||||
.PHONY: downloads clean-downloads
|
||||
downloads: $(DOWNLOADS)
|
||||
clean-downloads:
|
||||
-rm -f $(DOWNLOADS)
|
||||
|
||||
$(DOWNLOADS):
|
||||
$(WGET) http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/$@
|
||||
27268
localedata/unicode-gen/UnicodeData.txt
Normal file
27268
localedata/unicode-gen/UnicodeData.txt
Normal file
File diff suppressed because it is too large
Load Diff
546
localedata/unicode-gen/ctype_compatibility.py
Executable file
546
localedata/unicode-gen/ctype_compatibility.py
Executable file
@ -0,0 +1,546 @@
|
||||
#!/usr/bin/python3
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (C) 2014, 2015 Free Software Foundation, Inc.
|
||||
# This file is part of the GNU C Library.
|
||||
#
|
||||
# The GNU C Library is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU Lesser General Public
|
||||
# License as published by the Free Software Foundation; either
|
||||
# version 2.1 of the License, or (at your option) any later version.
|
||||
#
|
||||
# The GNU C Library is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with the GNU C Library; if not, see
|
||||
# <http://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
This script is useful for checking the differences between
|
||||
an old LC_CTYPE file /usr/share/i18n/locale/i18n and a
|
||||
new one generated by gen_unicode_ctype.py
|
||||
|
||||
To see how it is used, call it with the “-h” option:
|
||||
|
||||
$ ./ctype_compatibility.py -h
|
||||
… prints usage message …
|
||||
'''
|
||||
|
||||
import sys
|
||||
import re
|
||||
import unicodedata
|
||||
import argparse
|
||||
|
||||
from ctype_compatibility_test_cases import TEST_CASES
|
||||
|
||||
def get_lines_from_file(filename):
|
||||
'''Get all non-comment lines from a i18n file
|
||||
|
||||
Also merge all lines which are continued on the next line because
|
||||
they end in “/” into a single line.
|
||||
'''
|
||||
with open(filename) as i18n_file:
|
||||
current_line = ''
|
||||
for line in i18n_file:
|
||||
line = line.strip('\n')
|
||||
if '%' in line:
|
||||
if line.endswith('/'):
|
||||
line = line[0:line.find('%')] + '/'
|
||||
else:
|
||||
line = line[0:line.find('%')]
|
||||
line = line.strip()
|
||||
if line.endswith('/'):
|
||||
current_line += line[:-1]
|
||||
else:
|
||||
yield current_line + line
|
||||
current_line = ''
|
||||
if current_line: # file ends with a continuation line
|
||||
yield current_line
|
||||
|
||||
def extract_character_classes(filename):
|
||||
'''Get all Unicode code points for each character class from a file
|
||||
|
||||
Store these code points in a dictionary using the character classes
|
||||
as keys and the list of code points in this character class as values.
|
||||
|
||||
In case of the character classes “toupper”, “tolower”, and “totitle”,
|
||||
these area actually pairs of code points
|
||||
'''
|
||||
ctype_dict = {}
|
||||
for line in get_lines_from_file(filename):
|
||||
for char_class in [
|
||||
'upper',
|
||||
'lower',
|
||||
'alpha',
|
||||
'digit',
|
||||
'outdigit',
|
||||
'space',
|
||||
'cntrl',
|
||||
'punct',
|
||||
'graph',
|
||||
'print',
|
||||
'xdigit',
|
||||
'blank',
|
||||
'combining',
|
||||
'combining_level3',
|
||||
'toupper',
|
||||
'tolower',
|
||||
'totitle']:
|
||||
match = re.match(r'^('
|
||||
+'(?:(?:class|map)\s+")'
|
||||
+re.escape(char_class)+
|
||||
'(?:";)\s+'
|
||||
+'|'
|
||||
+re.escape(char_class)+'\s+'
|
||||
+')', line)
|
||||
if match:
|
||||
if char_class not in ctype_dict:
|
||||
ctype_dict[char_class] = []
|
||||
process_chars(
|
||||
ctype_dict[char_class],
|
||||
line[match.end():])
|
||||
return ctype_dict
|
||||
|
||||
def process_chars(char_class_list, code_point_line):
|
||||
'''
|
||||
Extract Unicode values from code_point_line
|
||||
and add to the list of code points in a character class
|
||||
'''
|
||||
for code_points in code_point_line.split(';'):
|
||||
code_points = code_points.strip()
|
||||
match = re.match(r'^<U(?P<codepoint>[0-9A-F]{4,8})>$', code_points)
|
||||
if match: # <Uxxxx>
|
||||
char_class_list.append(
|
||||
int(match.group('codepoint'), 16))
|
||||
continue
|
||||
match = re.match(
|
||||
r'^<U(?P<codepoint1>[0-9A-F]{4,8})>'
|
||||
+'\.\.'+
|
||||
'<U(?P<codepoint2>[0-9A-F]{4,8})>$',
|
||||
code_points)
|
||||
if match: # <Uxxxx>..<Uxxxx>
|
||||
for codepoint in range(
|
||||
int(match.group('codepoint1'), 16),
|
||||
int(match.group('codepoint2'), 16) + 1):
|
||||
char_class_list.append(codepoint)
|
||||
continue
|
||||
match = re.match(
|
||||
r'^<U(?P<codepoint1>[0-9A-F]{4,8})>'
|
||||
+'\.\.\(2\)\.\.'+
|
||||
'<U(?P<codepoint2>[0-9A-F]{4,8})>$',
|
||||
code_points)
|
||||
if match: # <Uxxxx>..(2)..<Uxxxx>
|
||||
for codepoint in range(
|
||||
int(match.group('codepoint1'), 16),
|
||||
int(match.group('codepoint2'), 16) + 1,
|
||||
2):
|
||||
char_class_list.append(codepoint)
|
||||
continue
|
||||
match = re.match(
|
||||
r'^\('
|
||||
+'<U(?P<codepoint1>[0-9A-F]{4,8})>'
|
||||
+','+
|
||||
'<U(?P<codepoint2>[0-9A-F]{4,8})>'
|
||||
+'\)$',
|
||||
code_points)
|
||||
if match: # (<Uxxxx>,<Uxxxx>)
|
||||
char_class_list.append((
|
||||
int(match.group('codepoint1'), 16),
|
||||
int(match.group('codepoint2'), 16)))
|
||||
continue
|
||||
sys.stderr.write(
|
||||
('None of the regexps matched '
|
||||
+ 'code_points=%(cp)s in code_point_line=%(cpl)s\n') %{
|
||||
'cp': code_points,
|
||||
'cpl': code_point_line
|
||||
})
|
||||
exit(1)
|
||||
|
||||
def compare_lists(old_ctype_dict, new_ctype_dict):
|
||||
'''Compare character classes in the old and the new LC_CTYPE'''
|
||||
print('****************************************************')
|
||||
print('Character classes which are only in the new '
|
||||
+ 'or only in the old file:')
|
||||
for char_class in sorted(old_ctype_dict):
|
||||
if char_class not in new_ctype_dict:
|
||||
print('Character class %s is in old ctype but not in new ctype'
|
||||
%char_class)
|
||||
for char_class in sorted(new_ctype_dict):
|
||||
if char_class not in old_ctype_dict:
|
||||
print('Character class %s is in new ctype but not in old ctype'
|
||||
%char_class)
|
||||
for char_class in sorted(old_ctype_dict):
|
||||
print("****************************************************")
|
||||
print("%s: %d chars in old ctype and %d chars in new ctype" %(
|
||||
char_class,
|
||||
len(old_ctype_dict[char_class]),
|
||||
len(new_ctype_dict[char_class])))
|
||||
print("----------------------------------------------------")
|
||||
report(char_class,
|
||||
old_ctype_dict[char_class],
|
||||
new_ctype_dict[char_class])
|
||||
|
||||
def report_code_points(char_class, code_point_list, text=''):
|
||||
'''Report all code points which have been added to or removed from a
|
||||
character class.
|
||||
'''
|
||||
for code_point in sorted(code_point_list):
|
||||
if type(code_point) == type(int()):
|
||||
print('%(char_class)s: %(text)s: %(char)s %(code_point)s %(name)s'
|
||||
%{'text': text,
|
||||
'char': chr(code_point),
|
||||
'char_class': char_class,
|
||||
'code_point': hex(code_point),
|
||||
'name': unicodedata.name(chr(code_point), 'name unknown')})
|
||||
else:
|
||||
print(('%(char_class)s: %(text)s: '
|
||||
+ '%(char0)s → %(char1)s '
|
||||
+ '%(code_point0)s → %(code_point1)s '
|
||||
+ '%(name0)s → %(name1)s') %{
|
||||
'text': text,
|
||||
'char_class': char_class,
|
||||
'char0': chr(code_point[0]),
|
||||
'code_point0': hex(code_point[0]),
|
||||
'name0': unicodedata.name(chr(code_point[0]), 'name unknown'),
|
||||
'char1': chr(code_point[1]),
|
||||
'code_point1': hex(code_point[1]),
|
||||
'name1': unicodedata.name(chr(code_point[1]), 'name unknown')
|
||||
})
|
||||
|
||||
def report(char_class, old_list, new_list):
|
||||
'''Report the differences for a certain LC_CTYPE character class
|
||||
between the old and the newly generated state
|
||||
'''
|
||||
missing_chars = list(set(old_list)-set(new_list))
|
||||
print(('%(char_class)s: Missing %(number)d characters '
|
||||
+ 'of old ctype in new ctype ')
|
||||
%{'char_class': char_class, 'number': len(missing_chars)})
|
||||
if ARGS.show_missing_characters:
|
||||
report_code_points(char_class, missing_chars, 'Missing')
|
||||
added_chars = list(set(new_list)-set(old_list))
|
||||
print(('%(char_class)s: Added %(number)d characters '
|
||||
+ 'in new ctype which were not in old ctype')
|
||||
%{'char_class': char_class, 'number': len(added_chars)})
|
||||
if ARGS.show_added_characters:
|
||||
report_code_points(char_class, added_chars, 'Added')
|
||||
|
||||
|
||||
def cperror(error_message, errorcounter=0):
|
||||
'''Increase number of errors by one and print an error message'''
|
||||
print(error_message)
|
||||
return errorcounter + 1
|
||||
|
||||
def cpcheck(ctype_dict, code_point_list_with_ranges, char_classes, reason='',
|
||||
errorcounter=0):
|
||||
'''The parameter “code_point_list_with_ranges” is a list of
|
||||
integers or pairs of integers, for example:
|
||||
|
||||
[0x0E31, (0x0E34, 0x0E3A), (0x0E47, 0x0E4E)]
|
||||
|
||||
where the pairs of integers stand for all the code points in the range
|
||||
of the two integers given, including the two integers of the pair.
|
||||
|
||||
'''
|
||||
for code_point_range in code_point_list_with_ranges:
|
||||
for code_point in ([code_point_range]
|
||||
if type(code_point_range) == type(int())
|
||||
else range(code_point_range[0],
|
||||
code_point_range[1]+1)):
|
||||
for char_class_tuple in char_classes:
|
||||
char_class = char_class_tuple[0]
|
||||
in_char_class = char_class_tuple[1]
|
||||
if (code_point in ctype_dict[char_class]) != in_char_class:
|
||||
errorcounter = cperror(
|
||||
('error: %(code_point)s %(char)s '
|
||||
+ '%(char_class)s %(in)s: %(reason)s') %{
|
||||
'code_point': hex(code_point),
|
||||
'char': chr(code_point),
|
||||
'char_class': char_class,
|
||||
'in': not in_char_class,
|
||||
'reason': reason},
|
||||
errorcounter)
|
||||
return errorcounter
|
||||
|
||||
def tests(ctype_dict, errorcounter = 0):
|
||||
'''Test a LC_CTYPE character class dictionary for known errors'''
|
||||
# copy the information from ctype_dict (which contains lists) in
|
||||
# a new dictionary ctype_dict2 (which contains dictionaries).
|
||||
# The checks below are easier with that type of data structure.
|
||||
|
||||
ctype_dict2 = {}
|
||||
for key in ctype_dict:
|
||||
ctype_dict2[key] = {}
|
||||
if ctype_dict[key]:
|
||||
if type(ctype_dict[key][0]) == type(int()):
|
||||
for value in ctype_dict[key]:
|
||||
ctype_dict2[key][value] = 1
|
||||
else: # key is 'toupper', 'tolower', or 'totitle'
|
||||
for value in ctype_dict[key]:
|
||||
ctype_dict2[key][value[0]] = value[1]
|
||||
|
||||
for test_case in TEST_CASES:
|
||||
errorcounter = cpcheck(ctype_dict2,
|
||||
test_case[0],
|
||||
test_case[1],
|
||||
test_case[2],
|
||||
errorcounter = errorcounter)
|
||||
|
||||
for code_point in range(0, 0x110000):
|
||||
# toupper restriction: "Only characters specified for the keywords
|
||||
# lower and upper shall be specified.
|
||||
if (code_point in ctype_dict2['toupper']
|
||||
and code_point != ctype_dict2['toupper'][code_point]
|
||||
and not (code_point in ctype_dict2['lower']
|
||||
or code_point in ctype_dict2['upper'])):
|
||||
errorcounter = cperror(
|
||||
('error: %(char1)s is not upper|lower '
|
||||
+ 'but toupper(%(cp1)s)=%(cp2)s (%(char2)s)') %{
|
||||
'char1': chr(code_point),
|
||||
'cp1': hex(code_point),
|
||||
'cp2': hex(ctype_dict2['toupper'][code_point]),
|
||||
'char2': chr(ctype_dict2['toupper'][code_point])
|
||||
},
|
||||
errorcounter)
|
||||
# tolower restriction: "Only characters specified for the keywords
|
||||
# lower and upper shall be specified.
|
||||
if (code_point in ctype_dict2['tolower']
|
||||
and code_point != ctype_dict2['tolower'][code_point]
|
||||
and not (code_point in ctype_dict2['lower']
|
||||
or code_point in ctype_dict2['upper'])):
|
||||
errorcounter = cperror(
|
||||
('error: %(char1)s is not upper|lower '
|
||||
+ 'but tolower(%(cp1)s)=%(cp2)s (%(char2)s)') %{
|
||||
'char1': chr(code_point),
|
||||
'cp1': hex(code_point),
|
||||
'cp2': hex(ctype_dict2['tolower'][code_point]),
|
||||
'char2': chr(ctype_dict2['tolower'][code_point])
|
||||
},
|
||||
errorcounter)
|
||||
# alpha restriction: "Characters classified as either upper or lower
|
||||
# shall automatically belong to this class.
|
||||
if ((code_point in ctype_dict2['lower']
|
||||
or code_point in ctype_dict2['upper'])
|
||||
and code_point not in ctype_dict2['alpha']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is upper|lower but not alpha' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
# alpha restriction: "No character specified for the keywords cntrl,
|
||||
# digit, punct or space shall be specified."
|
||||
if (code_point in ctype_dict2['alpha']
|
||||
and code_point in ctype_dict2['cntrl']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is alpha and cntrl' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
if (code_point in ctype_dict2['alpha']
|
||||
and code_point in ctype_dict2['digit']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is alpha and digit' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
if (code_point in ctype_dict2['alpha']
|
||||
and code_point in ctype_dict2['punct']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is alpha and punct' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
if (code_point in ctype_dict2['alpha']
|
||||
and code_point in ctype_dict2['space']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is alpha and space' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
# space restriction: "No character specified for the keywords upper,
|
||||
# lower, alpha, digit, graph or xdigit shall be specified."
|
||||
# upper, lower, alpha already checked above.
|
||||
if (code_point in ctype_dict2['space']
|
||||
and code_point in ctype_dict2['digit']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is space and digit' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
if (code_point in ctype_dict2['space']
|
||||
and code_point in ctype_dict2['graph']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is space and graph' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
if (code_point in ctype_dict2['space']
|
||||
and code_point in ctype_dict2['xdigit']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is space and xdigit' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
# cntrl restriction: "No character specified for the keywords upper,
|
||||
# lower, alpha, digit, punct, graph, print or xdigit shall be
|
||||
# specified." upper, lower, alpha already checked above.
|
||||
if (code_point in ctype_dict2['cntrl']
|
||||
and code_point in ctype_dict2['digit']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is cntrl and digit' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
if (code_point in ctype_dict2['cntrl']
|
||||
and code_point in ctype_dict2['punct']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is cntrl and punct' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
if (code_point in ctype_dict2['cntrl']
|
||||
and code_point in ctype_dict2['graph']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is cntrl and graph' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
if (code_point in ctype_dict2['cntrl']
|
||||
and code_point in ctype_dict2['print']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is cntrl and print' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
if (code_point in ctype_dict2['cntrl']
|
||||
and code_point in ctype_dict2['xdigit']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is cntrl and xdigit' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
# punct restriction: "No character specified for the keywords upper,
|
||||
# lower, alpha, digit, cntrl, xdigit or as the <space> character shall
|
||||
# be specified." upper, lower, alpha, cntrl already checked above.
|
||||
if (code_point in ctype_dict2['punct']
|
||||
and code_point in ctype_dict2['digit']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is punct and digit' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
if (code_point in ctype_dict2['punct']
|
||||
and code_point in ctype_dict2['xdigit']):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is punct and xdigit' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
if (code_point in ctype_dict2['punct']
|
||||
and code_point == 0x0020):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is punct.' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
# graph restriction: "No character specified for the keyword cntrl
|
||||
# shall be specified." Already checked above.
|
||||
|
||||
# print restriction: "No character specified for the keyword cntrl
|
||||
# shall be specified." Already checked above.
|
||||
|
||||
# graph - print relation: differ only in the <space> character.
|
||||
# How is this possible if there are more than one space character?!
|
||||
# I think susv2/xbd/locale.html should speak of "space characters",
|
||||
# not "space character".
|
||||
if (code_point in ctype_dict2['print']
|
||||
and not (code_point in ctype_dict2['graph']
|
||||
or code_point in ctype_dict2['space'])):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s is print but not graph|space' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
if (code_point not in ctype_dict2['print']
|
||||
and (code_point in ctype_dict2['graph']
|
||||
or code_point == 0x0020)):
|
||||
errorcounter = cperror(
|
||||
'error: %(char)s %(cp)s graph|space but not print' %{
|
||||
'char': chr(code_point),
|
||||
'cp': hex(code_point)
|
||||
},
|
||||
errorcounter)
|
||||
return errorcounter
|
||||
|
||||
if __name__ == "__main__":
|
||||
PARSER = argparse.ArgumentParser(
|
||||
description='''
|
||||
Compare the contents of LC_CTYPE in two files and check for errors.
|
||||
''')
|
||||
PARSER.add_argument(
|
||||
'-o', '--old_ctype_file',
|
||||
nargs='?',
|
||||
type=str,
|
||||
default='i18n',
|
||||
help='The old ctype file, default: %(default)s')
|
||||
PARSER.add_argument(
|
||||
'-n', '--new_ctype_file',
|
||||
nargs='?',
|
||||
type=str,
|
||||
default='unicode-ctype',
|
||||
help='The new ctype file, default: %(default)s')
|
||||
PARSER.add_argument(
|
||||
'-a', '--show_added_characters',
|
||||
action='store_true',
|
||||
help=('Show characters which were added to each '
|
||||
+ 'character class in detail.'))
|
||||
PARSER.add_argument(
|
||||
'-m', '--show_missing_characters',
|
||||
action='store_true',
|
||||
help=('Show characters which were removed from each '
|
||||
+ 'character class in detail.'))
|
||||
ARGS = PARSER.parse_args()
|
||||
|
||||
OLD_CTYPE_DICT = extract_character_classes(
|
||||
ARGS.old_ctype_file)
|
||||
NEW_CTYPE_DICT = extract_character_classes(
|
||||
ARGS.new_ctype_file)
|
||||
compare_lists(OLD_CTYPE_DICT, NEW_CTYPE_DICT)
|
||||
print('============================================================')
|
||||
print('Checking for errors in old ctype file: %s' %ARGS.old_ctype_file)
|
||||
print('------------------------------------------------------------')
|
||||
NUMBER_OF_ERRORS_IN_OLD_FILE = tests(OLD_CTYPE_DICT, errorcounter = 0)
|
||||
print('------------------------------------------------------------')
|
||||
print('Old file = %s' %ARGS.old_ctype_file)
|
||||
print('Number of errors in old file = %s' %NUMBER_OF_ERRORS_IN_OLD_FILE)
|
||||
print('------------------------------------------------------------')
|
||||
print('============================================================')
|
||||
print('Checking for errors in new ctype file: %s' %ARGS.new_ctype_file)
|
||||
print('------------------------------------------------------------')
|
||||
NUMBER_OF_ERRORS_IN_NEW_FILE = tests(NEW_CTYPE_DICT, errorcounter = 0)
|
||||
print('------------------------------------------------------------')
|
||||
print('New file = %s' %ARGS.new_ctype_file)
|
||||
print('Number of errors in new file = %s' %NUMBER_OF_ERRORS_IN_NEW_FILE)
|
||||
print('------------------------------------------------------------')
|
||||
if NUMBER_OF_ERRORS_IN_NEW_FILE > 0:
|
||||
exit(1)
|
||||
else:
|
||||
exit(0)
|
||||
951
localedata/unicode-gen/ctype_compatibility_test_cases.py
Normal file
951
localedata/unicode-gen/ctype_compatibility_test_cases.py
Normal file
@ -0,0 +1,951 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (C) 2014, 2015 Free Software Foundation, Inc.
|
||||
# This file is part of the GNU C Library.
|
||||
#
|
||||
# The GNU C Library is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU Lesser General Public
|
||||
# License as published by the Free Software Foundation; either
|
||||
# version 2.1 of the License, or (at your option) any later version.
|
||||
#
|
||||
# The GNU C Library is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with the GNU C Library; if not, see
|
||||
# <http://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
This file contains a list of test cases used by
|
||||
the ctype_compatibility.py script.
|
||||
'''
|
||||
|
||||
TEST_CASES = [
|
||||
[[0x0E2F, 0x0E46], [('alpha', True), ('punct', False)],
|
||||
'''Theppitak Karoonboonyanan <thep@links.nectec.or.th> says
|
||||
<U0E2F>, <U0E46> should belong to punct. DerivedCoreProperties.txt
|
||||
says it is alpha. We trust DerivedCoreProperties.txt.'''
|
||||
],
|
||||
[[0x0E31, (0x0E34, 0x0E3A)], [('alpha', True)],
|
||||
'''gen-unicode-ctype.c: Theppitak Karoonboonyanan
|
||||
<thep@links.nectec.or.th> says <U0E31>, <U0E34>..<U0E3A>
|
||||
are alpha. DerivedCoreProperties.txt agrees.'''
|
||||
],
|
||||
[[(0x0E47, 0x0E4C), 0x0E4E], [('alpha', False)],
|
||||
'''gen-unicode-ctype.c: Theppitak Karoonboonyanan
|
||||
<thep@links.nectec.or.th> says <U0E47>..<U0E4E> are
|
||||
is_alpha. DerivedCoreProperties does says *only* <U0E4D>
|
||||
in that range is alphabetic, the others are *not*. We
|
||||
trust DerivedCoreProperties.txt.'''
|
||||
],
|
||||
[[0x0E4D], [('alpha', True)],
|
||||
'''gen-unicode-ctype.c: Theppitak Karoonboonyanan
|
||||
<thep@links.nectec.or.th> says <U0E47>..<U0E4E> are
|
||||
is_alpha. DerivedCoreProperties does says *only* <U0E4D>
|
||||
in that range is alphabetic, the others are *not*. We
|
||||
trust DerivedCoreProperties.txt.
|
||||
'''
|
||||
],
|
||||
[[0x0345], [('alpha', True), ('lower', True)],
|
||||
'''COMBINING GREEK YPOGEGRAMMENI
|
||||
According to DerivedCoreProperties.txt, this is “Alphabetic”
|
||||
and “Lowercase”.'''
|
||||
],
|
||||
[[(0x2160, 0x2188)], [('alpha', True)],
|
||||
'''Roman Numerals are “Alphabetic” according to
|
||||
DerivedCoreProperties.txt'''
|
||||
],
|
||||
[[(0x24B6, 0x24E9)], [('alpha', True)],
|
||||
'''Circled Latin letters are “Alphabetic” according to
|
||||
DerivedCoreProperties.txt'''
|
||||
],
|
||||
[[0x661], [('alpha', True), ('digit', False)],
|
||||
'''gen-unicode-ctype.c: All non-ASCII digits should be alphabetic.
|
||||
ISO C 99 forbids us to have them in category "digit", but we
|
||||
want iswalnum to return true on them. Don’t forget to
|
||||
have a look at all the other digits, 0x661 is just one
|
||||
example tested here.'''
|
||||
],
|
||||
[[(0x0030, 0x0039)], [('digit', True)],
|
||||
'''gen-unicode-ctype.c: All ASCII digits should be digits.'''
|
||||
],
|
||||
[[0x0009], [('blank', True)],
|
||||
'''gen-unicode-ctype.c: CHARACTER TABULATION'''
|
||||
],
|
||||
[[0x2007], [('blank', False), ('space', False)],
|
||||
'''gen-unicode-ctype.c: FIGURE SPACE, because it has <noBreak>
|
||||
in the description.'''
|
||||
],
|
||||
[[0x0009, 0x000A, 0x000B, 0x000C, 0x000D], [('space', True)],
|
||||
'''gen-unicode-ctype.c: CHARACTER TABULATION, LINE FEED (LF), LINE
|
||||
TABULATION, ;FORM FEED (FF), CARRIAGE RETURN (CR)'''
|
||||
],
|
||||
[[0x2028, 0x2029], [('cntrl', True)],
|
||||
'''gen-unicode-ctype.c: LINE SEPARATOR and PARAGRAPH SEPARATOR
|
||||
should be cntrl.'''
|
||||
],
|
||||
[[(0x0030, 0x0039), (0x0041, 0x0046), (0x0061, 0x0066)],
|
||||
[('xdigit', True)],
|
||||
'''gen-unicode-ctype.c: ISO C 99 says (6.4.4.1): hexadecimal-digit:
|
||||
one of 0 1 2 3 4 5 6 7 8 9 a b c d e f A B C D E F (nothing else
|
||||
should be considered as a hexadecimal-digit)'''
|
||||
],
|
||||
[[0x0330], [('combining', True), ('combining_level3', False)],
|
||||
'''gen-unicode-ctype.c: COMBINING TILDE BELOW, canonical combining
|
||||
class value >= 200, should be in combining but not in
|
||||
combining_level3'''
|
||||
],
|
||||
[[0x0250, 0x0251, 0x0271], [('lower', True)],
|
||||
'''Should be lower in Unicode 7.0.0 (was not lower in
|
||||
Unicode 5.0.0).
|
||||
'''
|
||||
],
|
||||
[[0x2184], [('lower', True)],
|
||||
'''Should be lower both in Unicode 5.0.0 and 7.0.0'''
|
||||
],
|
||||
[[0xA67F], [('punct', False), ('alpha', True)],
|
||||
'''0xa67f CYRILLIC PAYEROK. Not in Unicode 5.0.0. In Unicode
|
||||
7.0.0. General category Lm (Letter
|
||||
modifier). DerivedCoreProperties.txt says it is
|
||||
“Alphabetic”. Apparently added manually to punct by mistake in
|
||||
glibc’s old LC_CTYPE.'''
|
||||
],
|
||||
[[0xA60C], [('punct', False), ('alpha', True)],
|
||||
'''0xa60c VAI SYLLABLE LENGTHENER. Not in Unicode 5.0.0.
|
||||
In Unicode 7.0.0. General category Lm (Letter
|
||||
modifier). DerivedCoreProperties.txt says it is
|
||||
“Alphabetic”. Apparently added manually to punct by mistake in
|
||||
glibc’s old LC_CTYPE.'''
|
||||
],
|
||||
[[0x2E2F], [('punct', False), ('alpha', True)],
|
||||
'''0x2E2F VERTICAL TILDE. Not in Unicode 5.0.0. In Unicode
|
||||
7.0.0. General category Lm (Letter
|
||||
modifier). DerivedCoreProperties.txt says it is
|
||||
“Alphabetic”. Apparently added manually to punct by mistake in
|
||||
glibc’s old LC_CTYPE.'''
|
||||
],
|
||||
[[(0x1090, 0x1099)], [('punct', False), ('alpha', True)],
|
||||
'''MYANMAR SHAN DIGIT ZERO - MYANMAR SHAN DIGIT NINE.
|
||||
These are digits, but because ISO C 99 forbids to
|
||||
put them into digit they should go into alpha.'''
|
||||
],
|
||||
[[0x103F], [('punct', False), ('alpha', True)],
|
||||
'''0x103F MYANMAR LETTER GREAT SA. Not in Unicode 5.0.0.
|
||||
In Unicode 7.0.0. General category Lo
|
||||
(Other_Letter). DerivedCoreProperties.txt says it is
|
||||
“Alphabetic”. Apparently added manually to punct by
|
||||
mistake in glibc’s old LC_CTYPE.'''
|
||||
],
|
||||
[[0x0374], [('punct', False), ('alpha', True)],
|
||||
'''0x0374 GREEK NUMERAL SIGN. Unicode 5.0.0: general category
|
||||
Sk. Unicode 7.0.0: General category Lm
|
||||
(Modifier_Letter). DerivedCoreProperties.txt says it is
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x02EC], [('punct', False), ('alpha', True)],
|
||||
'''0x02EC MODIFIER LETTER VOICING. Unicode 5.0.0: general category
|
||||
Sk. Unicode 7.0.0: General category Lm
|
||||
(Modifier_Letter). DerivedCoreProperties.txt says it is
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x180E], [('space', False), ('blank', False)],
|
||||
'''0x180e MONGOLIAN VOWEL SEPARATOR. Unicode 5.0.0: General
|
||||
category Zs (Space_Separator) Unicode 7.0.0: General category Cf
|
||||
(Format).'''
|
||||
],
|
||||
[[0x1E9C, 0x1E9D, 0x1E9F],
|
||||
[('lower', True), ('upper', False), ('tolower', False),
|
||||
('toupper', False), ('totitle', False)],
|
||||
'''ẜ 0x1e9c LATIN SMALL LETTER LONG S WITH DIAGONAL STROKE,
|
||||
ẝ 0x1e9d LATIN SMALL LETTER LONG S WITH HIGH STROKE,
|
||||
ẟ 0x1e9f LATIN SMALL LETTER DELTA. These are “Lowercase”
|
||||
according to DerivedCoreProperties.txt but no upper case versions
|
||||
exist.'''
|
||||
],
|
||||
[[0x1E9E],
|
||||
[('lower', False), ('upper', True), ('tolower', True),
|
||||
('toupper', False), ('totitle', False)],
|
||||
'''0x1E9E ẞ LATIN CAPITAL LETTER SHARP S This is “Uppercase”
|
||||
according to DerivedCoreProperties.txt and the lower case
|
||||
version is 0x00DF ß LATIN SMALL LETTER SHARP S.'''
|
||||
],
|
||||
[[0x2188],
|
||||
[('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''0x2188 ROMAN NUMERAL ONE HUNDRED THOUSAND. This is “Alphabetic”
|
||||
according to DerivedCoreProperties.txt. In glibc’s old
|
||||
LC_CTYPE, it was in “lower”, which seems to be a
|
||||
mistake. It is not “Lowercase” in
|
||||
DerivedCoreProperties.txt and does not have case mappings
|
||||
in UnicodeData.txt either.'''
|
||||
],
|
||||
[[0x2C71, 0x2C74, (0x2C77, 0x2C7A)],
|
||||
[('alpha', True), ('lower', True), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''These are Latin small letters which were not in Unicode 5.0.0
|
||||
but are in Unicode 7.0.0. According to
|
||||
DerivedCoreProperties.txt they are “Lowercase”. But no
|
||||
uppercase versions exist. They have apparently been added
|
||||
manually to glibc’s old LC_CTYPE.'''
|
||||
],
|
||||
[[0xA730, 0xA731],
|
||||
[('alpha', True), ('lower', True), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''These are Latin small “capital” letters which were not in
|
||||
Unicode 5.0.0 but are in Unicode 7.0.0. According to
|
||||
DerivedCoreProperties.txt they are “Lowercase”. But no
|
||||
uppercase versions exist. They have apparently been added
|
||||
manually to glibc’s old LC_CTYPE.'''
|
||||
],
|
||||
[[(0xA771, 0xA778)],
|
||||
[('alpha', True), ('lower', True), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''These are Latin small (or small “capital”) letters which
|
||||
were not in Unicodee 5.0.0 but are in Unicode 7.0.0. According to
|
||||
DerivedCoreProperties.txt they are “Lowercase”. But no
|
||||
uppercase versions exist. They have apparently been added
|
||||
manually to glibc’s old LC_CTYPE.'''
|
||||
],
|
||||
[[0x0375],
|
||||
[('combining', False), ('combining_level3', False),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''“0375;GREEK LOWER NUMERAL SIGN;Sk;0;ON;;;;;N;;;;;”. Has
|
||||
apparently been added manually to glibc’s old LC_CTYPE as
|
||||
“combining_level3”. That seems wrong, it is no combining
|
||||
character because it does not have one of the general
|
||||
categories Mn, Mc, or Me. According to
|
||||
DerivedCoreProperties.txt it is not “Alphabetic”.'''
|
||||
],
|
||||
[[0x108D],
|
||||
[('combining', True), ('combining_level3', False),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''“108D;MYANMAR SIGN SHAN COUNCIL EMPHATIC
|
||||
TONE;Mn;220;NSM;;;;;N;;;;;”. Has apparently been added
|
||||
manually to glibc’s old LC_CTYPE as
|
||||
“combining_level3”. That seems wrong, although it is a
|
||||
combining character because it has the general category
|
||||
Mn, it is not “combining_level3” because the canonical
|
||||
combining class value is 220 which is >= 200. According to
|
||||
gen-unicode-ctype.c, “combining_level3” needs a
|
||||
canonical combining class value < 200. According to
|
||||
DerivedCoreProperties.txt it is not “Alphabetic”.'''
|
||||
],
|
||||
[[0x06DE],
|
||||
[('combining', False), ('combining_level3', False),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
''' UnicodeData.txt 5.0.0: “06DE;ARABIC START OF RUB EL
|
||||
HIZB;Me;0;NSM;;;;;N;;;;;”; UnicodeData.txt 7.0.0:
|
||||
“06DE;ARABIC START OF RUB EL
|
||||
HIZB;So;0;ON;;;;;N;;;;;”. I.e. this used to be a
|
||||
combining character in Unicode 5.0.0 but not anymore in
|
||||
7.0.0. According to DerivedCoreProperties.txt it is not
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0BD0],
|
||||
[('combining', False), ('combining_level3', False),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''Not in UnicodeData.txt 5.0.0. UnicodeData.txt 7.0.0:
|
||||
“0BD0;TAMIL OM;Lo;0;L;;;;;N;;;;;”. Apparently manually added to
|
||||
“combining” and “combining_level3” in glibc’s old
|
||||
LC_CTYPE. That seems wrong. According to
|
||||
DerivedCoreProperties.txt it is “Alphabetic”.'''
|
||||
],
|
||||
[[0x103F],
|
||||
[('combining', False), ('combining_level3', False),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''Not in UnicodeData.txt 5.0.0. UnicodeData.txt 7.0.0:
|
||||
“103F;MYANMAR LETTER GREAT SA;Lo;0;L;;;;;N;;;;;”.
|
||||
Apparently manually added to “combining” and
|
||||
“combining_level3” in glibc’s old LC_CTYPE. That seems
|
||||
wrong. According to DerivedCoreProperties.txt it is
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0901, 0x0903)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''These have general category “Mn” i.e. these are combining
|
||||
characters (both in UnicodeData.txt 5.0.0 and 7.0.0):
|
||||
“0901;DEVANAGARI SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;;”,
|
||||
”0902;DEVANAGARI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;;”,
|
||||
“0903;DEVANAGARI SIGN VISARGA;Mc;0;L;;;;;N;;;;;”.
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x093C],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''UnicodeData.txt (5.0.0 and 7.0.0): “093C;DEVANAGARI SIGN
|
||||
NUKTA;Mn;7;NSM;;;;;N;;;;;” According to
|
||||
DerivedCoreProperties.txt (7.0.0) this is *not*
|
||||
“Alphabetic”. glibc’s old LC_TYPE has this in “alpha”.'''
|
||||
],
|
||||
[[(0x093E, 0x093F)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''These have general category “Mc” i.e. these are combining
|
||||
characters (both in UnicodeData.txt 5.0.0 and 7.0.0):
|
||||
“093E;DEVANAGARI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;;”
|
||||
“093F;DEVANAGARI VOWEL SIGN I;Mc;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0940, 0x094C)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''These are all combining
|
||||
characters (“Mc” or “Mn” both in UnicodeData.txt 5.0.0 and 7.0.0).
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x094D],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''Combining character, both in UnicodeData.txt 5.0.0 and 7.0.0.
|
||||
“094D;DEVANAGARI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) it is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0951, 0x0954)],
|
||||
[('combining', True), ('combining_level3', False),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''Combining characters, both in UnicodeData.txt 5.0.0 and 7.0.0.
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0962, 0x0963), (0x0981, 0x0983)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''Combining characters, both in UnicodeData.txt 5.0.0 and 7.0.0.
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x09BC],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“09BC;BENGALI SIGN NUKTA;Mn;7;NSM;;;;;N;;;;;”
|
||||
Combining character, both in UnicodeData.txt 5.0.0 and 7.0.0.
|
||||
According to DerivedCoreProperties.txt (7.0.0) it is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x09BE, 0x09BF), (0x09C0, 0x09C4), (0x09C7, 0x09C8),
|
||||
(0x09CB, 0x09CC)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“09BE;BENGALI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;;”
|
||||
“09BF;BENGALI VOWEL SIGN I;Mc;0;L;;;;;N;;;;;”
|
||||
“09C0;BENGALI VOWEL SIGN II;Mc;0;L;;;;;N;;;;;”
|
||||
“09C1;BENGALI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;;”
|
||||
“09C2;BENGALI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;;”
|
||||
“09C3;BENGALI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;;”
|
||||
“09C4;BENGALI VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;;”
|
||||
“09C7;BENGALI VOWEL SIGN E;Mc;0;L;;;;;N;;;;;”
|
||||
“09C8;BENGALI VOWEL SIGN AI;Mc;0;L;;;;;N;;;;;”
|
||||
“09CB;BENGALI VOWEL SIGN O;Mc;0;L;09C7 09BE;;;;N;;;;;”
|
||||
“09CC;BENGALI VOWEL SIGN AU;Mc;0;L;09C7 09D7;;;;N;;;;;”
|
||||
Combining characters, both in UnicodeData.txt 5.0.0 and 7.0.0.
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x09CD],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“09CD;BENGALI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;;”
|
||||
Combining character, both in UnicodeData.txt 5.0.0 and 7.0.0.
|
||||
According to DerivedCoreProperties.txt (7.0.0) it is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x09D7, (0x09E2, 0x09E3)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''Combining characters, both in UnicodeData.txt 5.0.0 and 7.0.0.
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x09F2, 0x09F3],
|
||||
[('combining', False), ('combining_level3', False),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“09F2;BENGALI RUPEE MARK;Sc;0;ET;;;;;N;;;;;”
|
||||
“09F3;BENGALI RUPEE SIGN;Sc;0;ET;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x09F4, 0x09FA)],
|
||||
[('combining', False), ('combining_level3', False),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“09F4;BENGALI CURRENCY NUMERATOR ONE;No;0;L;;;;1/16;N;;;;;”
|
||||
“09F5;BENGALI CURRENCY NUMERATOR TWO;No;0;L;;;;1/8;N;;;;;”
|
||||
“09F6;BENGALI CURRENCY NUMERATOR THREE;No;0;L;;;;3/16;N;;;;;”
|
||||
“09F7;BENGALI CURRENCY NUMERATOR FOUR;No;0;L;;;;1/4;N;;;;;”
|
||||
“09F8;BENGALI CURRENCY NUMERATOR ONE LESS THAN THE DENOMINATOR;
|
||||
No;0;L;;;;3/4;N;;;;;”
|
||||
“09F9;BENGALI CURRENCY DENOMINATOR SIXTEEN;No;0;L;;;;16;N;;;;;”
|
||||
“09FA;BENGALI ISSHAR;So;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0A01, 0x0A03)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0A01;GURMUKHI SIGN ADAK BINDI;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0A02;GURMUKHI SIGN BINDI;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0A03;GURMUKHI SIGN VISARGA;Mc;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0A3C],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0A3C;GURMUKHI SIGN NUKTA;Mn;7;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) this is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0A3E, 0x0A40), (0x0A41, 0x0A42), (0x0A47, 0x0A48),
|
||||
(0x0A4B, 0x0A4C)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0A3E;GURMUKHI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;;”
|
||||
“0A3F;GURMUKHI VOWEL SIGN I;Mc;0;L;;;;;N;;;;;”
|
||||
“0A40;GURMUKHI VOWEL SIGN II;Mc;0;L;;;;;N;;;;;”
|
||||
“0A41;GURMUKHI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0A42;GURMUKHI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0A47;GURMUKHI VOWEL SIGN EE;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0A48;GURMUKHI VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0A4B;GURMUKHI VOWEL SIGN OO;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0A4C;GURMUKHI VOWEL SIGN AU;Mn;0;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0A4D],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0A4D;GURMUKHI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) this is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0A51, (0x0A70, 0x0A71), 0x0A75, (0x0A81, 0x0A83)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0A4D;GURMUKHI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;;”
|
||||
“0A70;GURMUKHI TIPPI;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0A71;GURMUKHI ADDAK;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0A75;GURMUKHI SIGN YAKASH;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0A81;GUJARATI SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0A82;GUJARATI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0A83;GUJARATI SIGN VISARGA;Mc;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0ABC],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0ABC;GUJARATI SIGN NUKTA;Mn;7;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) this is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0ABE, 0x0AC5), (0x0AC7, 0x0AC9), (0x0ACB, 0x0ACC)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0ABE;GUJARATI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;;”
|
||||
“0ABF;GUJARATI VOWEL SIGN I;Mc;0;L;;;;;N;;;;;”
|
||||
“0AC0;GUJARATI VOWEL SIGN II;Mc;0;L;;;;;N;;;;;”
|
||||
“0AC1;GUJARATI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0AC2;GUJARATI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0AC3;GUJARATI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0AC4;GUJARATI VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0AC5;GUJARATI VOWEL SIGN CANDRA E;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0AC7;GUJARATI VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0AC8;GUJARATI VOWEL SIGN AI;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0AC9;GUJARATI VOWEL SIGN CANDRA O;Mc;0;L;;;;;N;;;;;”
|
||||
“0ACB;GUJARATI VOWEL SIGN O;Mc;0;L;;;;;N;;;;;”
|
||||
“0ACC;GUJARATI VOWEL SIGN AU;Mc;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0ACD],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0ACD;GUJARATI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) this is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0AE2, 0x0AE3)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0AE2;GUJARATI VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0AE3;GUJARATI VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0AF1],
|
||||
[('combining', False), ('combining_level3', False),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0AF1;GUJARATI RUPEE SIGN;Sc;0;ET;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) this is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0B01, 0x0B03)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0B01;ORIYA SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0B02;ORIYA SIGN ANUSVARA;Mc;0;L;;;;;N;;;;;”
|
||||
“0B03;ORIYA SIGN VISARGA;Mc;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0B3C],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0B3C;ORIYA SIGN NUKTA;Mn;7;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) this is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0B3E, 0x0B44), (0x0B47, 0x0B48), (0x0B4B, 0x0B4C)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0B3E;ORIYA VOWEL SIGN AA;Mc;0;L;;;;;N;;;;;”
|
||||
“0B3F;ORIYA VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0B40;ORIYA VOWEL SIGN II;Mc;0;L;;;;;N;;;;;”
|
||||
“0B41;ORIYA VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0B42;ORIYA VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0B43;ORIYA VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0B44;ORIYA VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0B47;ORIYA VOWEL SIGN E;Mc;0;L;;;;;N;;;;;”
|
||||
“0B48;ORIYA VOWEL SIGN AI;Mc;0;L;0B47 0B56;;;;N;;;;;”
|
||||
“0B4B;ORIYA VOWEL SIGN O;Mc;0;L;0B47 0B3E;;;;N;;;;;”
|
||||
“0B4C;ORIYA VOWEL SIGN AU;Mc;0;L;0B47 0B57;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0B4D],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0B4D;ORIYA SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) this is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0B56, 0x0B57), (0x0B62, 0x0B63)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0B56;ORIYA AI LENGTH MARK;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0B57;ORIYA AU LENGTH MARK;Mc;0;L;;;;;N;;;;;”
|
||||
“0B62;ORIYA VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0B63;ORIYA VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0B70],
|
||||
[('combining', False), ('combining_level3', False),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0B70;ORIYA ISSHAR;So;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) this is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0B82],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0B82;TAMIL SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) this is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0BBE, 0x0BC2), (0x0BC6, 0x0BC8), (0x0BCA, 0x0BCC)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0BBE;TAMIL VOWEL SIGN AA;Mc;0;L;;;;;N;;;;;”
|
||||
“0BBF;TAMIL VOWEL SIGN I;Mc;0;L;;;;;N;;;;;”
|
||||
“0BC0;TAMIL VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0BC1;TAMIL VOWEL SIGN U;Mc;0;L;;;;;N;;;;;”
|
||||
“0BC2;TAMIL VOWEL SIGN UU;Mc;0;L;;;;;N;;;;;”
|
||||
“0BC6;TAMIL VOWEL SIGN E;Mc;0;L;;;;;N;;;;;”
|
||||
“0BC7;TAMIL VOWEL SIGN EE;Mc;0;L;;;;;N;;;;;”
|
||||
“0BC8;TAMIL VOWEL SIGN AI;Mc;0;L;;;;;N;;;;;”
|
||||
“0BCA;TAMIL VOWEL SIGN O;Mc;0;L;0BC6 0BBE;;;;N;;;;;”
|
||||
“0BCB;TAMIL VOWEL SIGN OO;Mc;0;L;0BC7 0BBE;;;;N;;;;;”
|
||||
“0BCC;TAMIL VOWEL SIGN AU;Mc;0;L;0BC6 0BD7;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0BCD],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0BCD;TAMIL SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) this is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0BD7],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0BD7;TAMIL AU LENGTH MARK;Mc;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) this is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0BF0, 0x0BFA)],
|
||||
[('combining', False), ('combining_level3', False),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0BF0;TAMIL NUMBER TEN;No;0;L;;;;10;N;;;;;”
|
||||
“0BF1;TAMIL NUMBER ONE HUNDRED;No;0;L;;;;100;N;;;;;”
|
||||
“0BF2;TAMIL NUMBER ONE THOUSAND;No;0;L;;;;1000;N;;;;;”
|
||||
“0BF3;TAMIL DAY SIGN;So;0;ON;;;;;N;;;;;”
|
||||
“0BF4;TAMIL MONTH SIGN;So;0;ON;;;;;N;;;;;”
|
||||
“0BF5;TAMIL YEAR SIGN;So;0;ON;;;;;N;;;;;”
|
||||
“0BF6;TAMIL DEBIT SIGN;So;0;ON;;;;;N;;;;;”
|
||||
“0BF7;TAMIL CREDIT SIGN;So;0;ON;;;;;N;;;;;”
|
||||
“0BF8;TAMIL AS ABOVE SIGN;So;0;ON;;;;;N;;;;;”
|
||||
“0BF9;TAMIL RUPEE SIGN;Sc;0;ET;;;;;N;;;;;”
|
||||
“0BFA;TAMIL NUMBER SIGN;So;0;ON;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) this is *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0C01, 0x0C03)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0C01;TELUGU SIGN CANDRABINDU;Mc;0;L;;;;;N;;;;;”
|
||||
“0C02;TELUGU SIGN ANUSVARA;Mc;0;L;;;;;N;;;;;”
|
||||
“0C03;TELUGU SIGN VISARGA;Mc;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0C3E, 0x0C44), (0x0C46, 0x0C48), (0x0C4A, 0x0C4C)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0C3E;TELUGU VOWEL SIGN AA;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0C3F;TELUGU VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0C40;TELUGU VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0C41;TELUGU VOWEL SIGN U;Mc;0;L;;;;;N;;;;;”
|
||||
“0C42;TELUGU VOWEL SIGN UU;Mc;0;L;;;;;N;;;;;”
|
||||
“0C43;TELUGU VOWEL SIGN VOCALIC R;Mc;0;L;;;;;N;;;;;”
|
||||
“0C44;TELUGU VOWEL SIGN VOCALIC RR;Mc;0;L;;;;;N;;;;;”
|
||||
“0C46;TELUGU VOWEL SIGN E;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0C47;TELUGU VOWEL SIGN EE;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0C48;TELUGU VOWEL SIGN AI;Mn;0;NSM;0C46 0C56;;;;N;;;;;”
|
||||
“0C4A;TELUGU VOWEL SIGN O;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0C4B;TELUGU VOWEL SIGN OO;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0C4C;TELUGU VOWEL SIGN AU;Mn;0;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0C4D],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0C4D;TELUGU SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0C55, 0x0C56), (0x0C62, 0x0C63)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0C55;TELUGU LENGTH MARK;Mn;84;NSM;;;;;N;;;;;”
|
||||
“0C56;TELUGU AI LENGTH MARK;Mn;91;NSM;;;;;N;;;;;”
|
||||
“0C62;TELUGU VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0C63;TELUGU VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0C78, 0x0C7F)],
|
||||
[('combining', False), ('combining_level3', False),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0C78;TELUGU FRACTION DIGIT ZERO FOR ODD POWERS OF FOUR;
|
||||
No;0;ON;;;;0;N;;;;;”
|
||||
“0C79;TELUGU FRACTION DIGIT ONE FOR ODD POWERS OF FOUR;
|
||||
No;0;ON;;;;1;N;;;;;”
|
||||
“0C7A;TELUGU FRACTION DIGIT TWO FOR ODD POWERS OF FOUR;
|
||||
No;0;ON;;;;2;N;;;;;”
|
||||
“0C7B;TELUGU FRACTION DIGIT THREE FOR ODD POWERS OF FOUR;
|
||||
No;0;ON;;;;3;N;;;;;”
|
||||
“0C7C;TELUGU FRACTION DIGIT ONE FOR EVEN POWERS OF FOUR;
|
||||
No;0;ON;;;;1;N;;;;;”
|
||||
“0C7D;TELUGU FRACTION DIGIT TWO FOR EVEN POWERS OF FOUR;
|
||||
No;0;ON;;;;2;N;;;;;”
|
||||
“0C7E;TELUGU FRACTION DIGIT THREE FOR EVEN POWERS OF FOUR;
|
||||
No;0;ON;;;;3;N;;;;;”
|
||||
“0C7F;TELUGU SIGN TUUMU;So;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0C82, 0x0C83)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0C81;KANNADA SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0C82;KANNADA SIGN ANUSVARA;Mc;0;L;;;;;N;;;;;”
|
||||
“0C83;KANNADA SIGN VISARGA;Mc;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0CBC],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0CBC;KANNADA SIGN NUKTA;Mn;7;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0CBE, 0x0CC4), (0x0CC6, 0x0CC8), (0x0CCA, 0x0CCC)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0CBE;KANNADA VOWEL SIGN AA;Mc;0;L;;;;;N;;;;;”
|
||||
“0CBF;KANNADA VOWEL SIGN I;Mn;0;L;;;;;N;;;;;”
|
||||
“0CC0;KANNADA VOWEL SIGN II;Mc;0;L;0CBF 0CD5;;;;N;;;;;”
|
||||
“0CC1;KANNADA VOWEL SIGN U;Mc;0;L;;;;;N;;;;;”
|
||||
“0CC2;KANNADA VOWEL SIGN UU;Mc;0;L;;;;;N;;;;;”
|
||||
“0CC3;KANNADA VOWEL SIGN VOCALIC R;Mc;0;L;;;;;N;;;;;”
|
||||
“0CC4;KANNADA VOWEL SIGN VOCALIC RR;Mc;0;L;;;;;N;;;;;”
|
||||
“0CC6;KANNADA VOWEL SIGN E;Mn;0;L;;;;;N;;;;;”
|
||||
“0CC7;KANNADA VOWEL SIGN EE;Mc;0;L;0CC6 0CD5;;;;N;;;;;”
|
||||
“0CC8;KANNADA VOWEL SIGN AI;Mc;0;L;0CC6 0CD6;;;;N;;;;;”
|
||||
“0CCA;KANNADA VOWEL SIGN O;Mc;0;L;0CC6 0CC2;;;;N;;;;;”
|
||||
“0CCB;KANNADA VOWEL SIGN OO;Mc;0;L;0CCA 0CD5;;;;N;;;;;”
|
||||
“0CCC;KANNADA VOWEL SIGN AU;Mn;0;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0CCD],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0CCD;KANNADA SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0CD5, 0x0CD6), (0x0CE2, 0x0CE3)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
0CD5;KANNADA LENGTH MARK;Mc;0;L;;;;;N;;;;;
|
||||
0CD6;KANNADA AI LENGTH MARK;Mc;0;L;;;;;N;;;;;
|
||||
0CE2;KANNADA VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;;
|
||||
0CE3;KANNADA VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;;
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0D02, 0x0D03), (0x0D3E, 0x0D44), (0x0D46, 0x0D48),
|
||||
(0x0D4A, 0x0D4C)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0D02;MALAYALAM SIGN ANUSVARA;Mc;0;L;;;;;N;;;;;”
|
||||
“0D03;MALAYALAM SIGN VISARGA;Mc;0;L;;;;;N;;;;;”
|
||||
“0D3E;MALAYALAM VOWEL SIGN AA;Mc;0;L;;;;;N;;;;;”
|
||||
“0D3F;MALAYALAM VOWEL SIGN I;Mc;0;L;;;;;N;;;;;”
|
||||
“0D40;MALAYALAM VOWEL SIGN II;Mc;0;L;;;;;N;;;;;”
|
||||
“0D41;MALAYALAM VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0D42;MALAYALAM VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0D43;MALAYALAM VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0D44;MALAYALAM VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0D46;MALAYALAM VOWEL SIGN E;Mc;0;L;;;;;N;;;;;”
|
||||
“0D47;MALAYALAM VOWEL SIGN EE;Mc;0;L;;;;;N;;;;;”
|
||||
“0D48;MALAYALAM VOWEL SIGN AI;Mc;0;L;;;;;N;;;;;”
|
||||
“0D4A;MALAYALAM VOWEL SIGN O;Mc;0;L;0D46 0D3E;;;;N;;;;;”
|
||||
“0D4B;MALAYALAM VOWEL SIGN OO;Mc;0;L;0D47 0D3E;;;;N;;;;;”
|
||||
“0D4C;MALAYALAM VOWEL SIGN AU;Mc;0;L;0D46 0D57;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0D4D],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0D4D;MALAYALAM SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0D57, (0x0D62, 0x0D63)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0D57;MALAYALAM AU LENGTH MARK;Mc;0;L;;;;;N;;;;;”
|
||||
“0D62;MALAYALAM VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0D63;MALAYALAM VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0D70, 0x0D79)],
|
||||
[('combining', False), ('combining_level3', False),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0D70;MALAYALAM NUMBER TEN;No;0;L;;;;10;N;;;;;”
|
||||
“0D71;MALAYALAM NUMBER ONE HUNDRED;No;0;L;;;;100;N;;;;;”
|
||||
“0D72;MALAYALAM NUMBER ONE THOUSAND;No;0;L;;;;1000;N;;;;;”
|
||||
“0D73;MALAYALAM FRACTION ONE QUARTER;No;0;L;;;;1/4;N;;;;;”
|
||||
“0D74;MALAYALAM FRACTION ONE HALF;No;0;L;;;;1/2;N;;;;;”
|
||||
“0D75;MALAYALAM FRACTION THREE QUARTERS;No;0;L;;;;3/4;N;;;;;”
|
||||
“0D79;MALAYALAM DATE MARK;So;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0D82, 0x0D83)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0D82;SINHALA SIGN ANUSVARAYA;Mc;0;L;;;;;N;;;;;”
|
||||
“0D83;SINHALA SIGN VISARGAYA;Mc;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0DCA],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0DCA;SINHALA SIGN AL-LAKUNA;Mn;9;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0x0DCF, 0x0DD4), 0x0DD6, (0x0DD8, 0x0DDF), (0x0DF2, 0x0DF3)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0DCF;SINHALA VOWEL SIGN AELA-PILLA;Mc;0;L;;;;;N;;;;;”
|
||||
“0DD0;SINHALA VOWEL SIGN KETTI AEDA-PILLA;Mc;0;L;;;;;N;;;;;”
|
||||
“0DD1;SINHALA VOWEL SIGN DIGA AEDA-PILLA;Mc;0;L;;;;;N;;;;;”
|
||||
“0DD2;SINHALA VOWEL SIGN KETTI IS-PILLA;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0DD3;SINHALA VOWEL SIGN DIGA IS-PILLA;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0DD4;SINHALA VOWEL SIGN KETTI PAA-PILLA;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0DD6;SINHALA VOWEL SIGN DIGA PAA-PILLA;Mn;0;NSM;;;;;N;;;;;”
|
||||
“0DD8;SINHALA VOWEL SIGN GAETTA-PILLA;Mc;0;L;;;;;N;;;;;”
|
||||
“0DD9;SINHALA VOWEL SIGN KOMBUVA;Mc;0;L;;;;;N;;;;;”
|
||||
“0DDA;SINHALA VOWEL SIGN DIGA KOMBUVA;Mc;0;L;0DD9 0DCA;;;;N;;;;;”
|
||||
“0DDB;SINHALA VOWEL SIGN KOMBU DEKA;Mc;0;L;;;;;N;;;;;”
|
||||
“0DDC;SINHALA VOWEL SIGN KOMBUVA HAA AELA-PILLA;
|
||||
Mc;0;L;0DD9 0DCF;;;;N;;;;;”
|
||||
“0DDD;SINHALA VOWEL SIGN KOMBUVA HAA DIGA AELA-PILLA;
|
||||
Mc;0;L;0DDC 0DCA;;;;N;;;;;”
|
||||
“0DDE;SINHALA VOWEL SIGN KOMBUVA HAA GAYANUKITTA;
|
||||
Mc;0;L;0DD9 0DDF;;;;N;;;;;”
|
||||
“0DDF;SINHALA VOWEL SIGN GAYANUKITTA;Mc;0;L;;;;;N;;;;;”
|
||||
“0DF2;SINHALA VOWEL SIGN DIGA GAETTA-PILLA;Mc;0;L;;;;;N;;;;;”
|
||||
“0DF3;SINHALA VOWEL SIGN DIGA GAYANUKITTA;Mc;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[0x0DF4],
|
||||
[('combining', False), ('combining_level3', False),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“0DF4;SINHALA PUNCTUATION KUNDDALIYA;Po;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0xA789, 0xA78A)],
|
||||
[('combining', False), ('combining_level3', False),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“A789;MODIFIER LETTER COLON;Sk;0;L;;;;;N;;;;;”
|
||||
“A78A;MODIFIER LETTER SHORT EQUALS SIGN;Sk;0;L;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are *not*
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0xA926, 0xA92A)],
|
||||
[('combining', True), ('combining_level3', True),
|
||||
('alpha', True), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“A926;KAYAH LI VOWEL UE;Mn;0;NSM;;;;;N;;;;;”
|
||||
“A927;KAYAH LI VOWEL E;Mn;0;NSM;;;;;N;;;;;”
|
||||
“A928;KAYAH LI VOWEL U;Mn;0;NSM;;;;;N;;;;;”
|
||||
“A929;KAYAH LI VOWEL EE;Mn;0;NSM;;;;;N;;;;;”
|
||||
“A92A;KAYAH LI VOWEL O;Mn;0;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are
|
||||
“Alphabetic”.'''
|
||||
],
|
||||
[[(0xA92B, 0xA92D)],
|
||||
[('combining', True), ('combining_level3', False),
|
||||
('alpha', False), ('lower', False), ('upper', False),
|
||||
('tolower', False), ('toupper', False), ('totitle', False)],
|
||||
'''
|
||||
“A92B;KAYAH LI TONE PLOPHU;Mn;220;NSM;;;;;N;;;;;”
|
||||
“A92C;KAYAH LI TONE CALYA;Mn;220;NSM;;;;;N;;;;;”
|
||||
“A92D;KAYAH LI TONE CALYA PLOPHU;Mn;220;NSM;;;;;N;;;;;”
|
||||
According to DerivedCoreProperties.txt (7.0.0) these are *not*
|
||||
“Alphabetic”.'''
|
||||
]
|
||||
]
|
||||
751
localedata/unicode-gen/gen_unicode_ctype.py
Executable file
751
localedata/unicode-gen/gen_unicode_ctype.py
Executable file
@ -0,0 +1,751 @@
|
||||
#!/usr/bin/python3
|
||||
#
|
||||
# Generate a Unicode conforming LC_CTYPE category from a UnicodeData file.
|
||||
# Copyright (C) 2014, 2015 Free Software Foundation, Inc.
|
||||
# This file is part of the GNU C Library.
|
||||
# Based on gen-unicode-ctype.c by Bruno Haible <haible@clisp.cons.org>, 2000.
|
||||
#
|
||||
# The GNU C Library is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU Lesser General Public
|
||||
# License as published by the Free Software Foundation; either
|
||||
# version 2.1 of the License, or (at your option) any later version.
|
||||
#
|
||||
# The GNU C Library is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with the GNU C Library; if not, see
|
||||
# <http://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Generate a Unicode conforming LC_CTYPE category from UnicodeData.txt and
|
||||
DerivedCoreProperties.txt files.
|
||||
|
||||
To see how this script is used, call it with the “-h” option:
|
||||
|
||||
$ ./gen_unicode_ctype.py -h
|
||||
… prints usage message …
|
||||
'''
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import time
|
||||
import re
|
||||
|
||||
# Dictionary holding the entire contents of the UnicodeData.txt file
|
||||
#
|
||||
# Contents of this dictionary look like this:
|
||||
#
|
||||
# {0: {'category': 'Cc',
|
||||
# 'title': None,
|
||||
# 'digit': '',
|
||||
# 'name': '<control>',
|
||||
# 'bidi': 'BN',
|
||||
# 'combining': '0',
|
||||
# 'comment': '',
|
||||
# 'oldname': 'NULL',
|
||||
# 'decomposition': '',
|
||||
# 'upper': None,
|
||||
# 'mirrored': 'N',
|
||||
# 'lower': None,
|
||||
# 'decdigit': '',
|
||||
# 'numeric': ''},
|
||||
# …
|
||||
# }
|
||||
UNICODE_ATTRIBUTES = {}
|
||||
|
||||
# Dictionary holding the entire contents of the DerivedCoreProperties.txt file
|
||||
#
|
||||
# Contents of this dictionary look like this:
|
||||
#
|
||||
# {917504: ['Default_Ignorable_Code_Point'],
|
||||
# 917505: ['Case_Ignorable', 'Default_Ignorable_Code_Point'],
|
||||
# …
|
||||
# }
|
||||
DERIVED_CORE_PROPERTIES = {}
|
||||
|
||||
def fill_attribute(code_point, fields):
|
||||
'''Stores in UNICODE_ATTRIBUTES[code_point] the values from the fields.
|
||||
|
||||
One entry in the UNICODE_ATTRIBUTES dictionary represents one line
|
||||
in the UnicodeData.txt file.
|
||||
|
||||
'''
|
||||
UNICODE_ATTRIBUTES[code_point] = {
|
||||
'name': fields[1], # Character name
|
||||
'category': fields[2], # General category
|
||||
'combining': fields[3], # Canonical combining classes
|
||||
'bidi': fields[4], # Bidirectional category
|
||||
'decomposition': fields[5], # Character decomposition mapping
|
||||
'decdigit': fields[6], # Decimal digit value
|
||||
'digit': fields[7], # Digit value
|
||||
'numeric': fields[8], # Numeric value
|
||||
'mirrored': fields[9], # mirrored
|
||||
'oldname': fields[10], # Old Unicode 1.0 name
|
||||
'comment': fields[11], # comment
|
||||
# Uppercase mapping
|
||||
'upper': int(fields[12], 16) if fields[12] else None,
|
||||
# Lowercase mapping
|
||||
'lower': int(fields[13], 16) if fields[13] else None,
|
||||
# Titlecase mapping
|
||||
'title': int(fields[14], 16) if fields[14] else None,
|
||||
}
|
||||
|
||||
def fill_attributes(filename):
|
||||
'''Stores the entire contents of the UnicodeData.txt file
|
||||
in the UNICODE_ATTRIBUTES dictionary.
|
||||
|
||||
A typical line for a single code point in UnicodeData.txt looks
|
||||
like this:
|
||||
|
||||
0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;
|
||||
|
||||
Code point ranges are indicated by pairs of lines like this:
|
||||
|
||||
4E00;<CJK Ideograph, First>;Lo;0;L;;;;;N;;;;;
|
||||
9FCC;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;;
|
||||
'''
|
||||
with open(filename, mode='r') as unicode_data_file:
|
||||
fields_start = []
|
||||
for line in unicode_data_file:
|
||||
fields = line.strip().split(';')
|
||||
if len(fields) != 15:
|
||||
sys.stderr.write(
|
||||
'short line in file "%(f)s": %(l)s\n' %{
|
||||
'f': filename, 'l': line})
|
||||
exit(1)
|
||||
if fields[2] == 'Cs':
|
||||
# Surrogates are UTF-16 artefacts,
|
||||
# not real characters. Ignore them.
|
||||
fields_start = []
|
||||
continue
|
||||
if fields[1].endswith(', First>'):
|
||||
fields_start = fields
|
||||
fields_start[1] = fields_start[1].split(',')[0][1:]
|
||||
continue
|
||||
if fields[1].endswith(', Last>'):
|
||||
fields[1] = fields[1].split(',')[0][1:]
|
||||
if fields[1:] != fields_start[1:]:
|
||||
sys.stderr.write(
|
||||
'broken code point range in file "%(f)s": %(l)s\n' %{
|
||||
'f': filename, 'l': line})
|
||||
exit(1)
|
||||
for code_point in range(
|
||||
int(fields_start[0], 16),
|
||||
int(fields[0], 16)+1):
|
||||
fill_attribute(code_point, fields)
|
||||
fields_start = []
|
||||
continue
|
||||
fill_attribute(int(fields[0], 16), fields)
|
||||
fields_start = []
|
||||
|
||||
def fill_derived_core_properties(filename):
|
||||
'''Stores the entire contents of the DerivedCoreProperties.txt file
|
||||
in the DERIVED_CORE_PROPERTIES dictionary.
|
||||
|
||||
Lines in DerivedCoreProperties.txt are either a code point range like
|
||||
this:
|
||||
|
||||
0061..007A ; Lowercase # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z
|
||||
|
||||
or a single code point like this:
|
||||
|
||||
00AA ; Lowercase # Lo FEMININE ORDINAL INDICATOR
|
||||
|
||||
'''
|
||||
with open(filename, mode='r') as derived_core_properties_file:
|
||||
for line in derived_core_properties_file:
|
||||
match = re.match(
|
||||
r'^(?P<codepoint1>[0-9A-F]{4,6})'
|
||||
+ r'(?:\.\.(?P<codepoint2>[0-9A-F]{4,6}))?'
|
||||
+ r'\s*;\s*(?P<property>[a-zA-Z_]+)',
|
||||
line)
|
||||
if not match:
|
||||
continue
|
||||
start = match.group('codepoint1')
|
||||
end = match.group('codepoint2')
|
||||
if not end:
|
||||
end = start
|
||||
for code_point in range(int(start, 16), int(end, 16)+1):
|
||||
prop = match.group('property')
|
||||
if code_point in DERIVED_CORE_PROPERTIES:
|
||||
DERIVED_CORE_PROPERTIES[code_point].append(prop)
|
||||
else:
|
||||
DERIVED_CORE_PROPERTIES[code_point] = [prop]
|
||||
|
||||
def to_upper(code_point):
|
||||
'''Returns the code point of the uppercase version
|
||||
of the given code point'''
|
||||
if (UNICODE_ATTRIBUTES[code_point]['name']
|
||||
and UNICODE_ATTRIBUTES[code_point]['upper']):
|
||||
return UNICODE_ATTRIBUTES[code_point]['upper']
|
||||
else:
|
||||
return code_point
|
||||
|
||||
def to_lower(code_point):
|
||||
'''Returns the code point of the lowercase version
|
||||
of the given code point'''
|
||||
if (UNICODE_ATTRIBUTES[code_point]['name']
|
||||
and UNICODE_ATTRIBUTES[code_point]['lower']):
|
||||
return UNICODE_ATTRIBUTES[code_point]['lower']
|
||||
else:
|
||||
return code_point
|
||||
|
||||
def to_title(code_point):
|
||||
'''Returns the code point of the titlecase version
|
||||
of the given code point'''
|
||||
if (UNICODE_ATTRIBUTES[code_point]['name']
|
||||
and UNICODE_ATTRIBUTES[code_point]['title']):
|
||||
return UNICODE_ATTRIBUTES[code_point]['title']
|
||||
else:
|
||||
return code_point
|
||||
|
||||
def is_upper(code_point):
|
||||
'''Checks whether the character with this code point is uppercase'''
|
||||
return (to_lower(code_point) != code_point
|
||||
or (code_point in DERIVED_CORE_PROPERTIES
|
||||
and 'Uppercase' in DERIVED_CORE_PROPERTIES[code_point]))
|
||||
|
||||
def is_lower(code_point):
|
||||
'''Checks whether the character with this code point is lowercase'''
|
||||
# Some characters are defined as “Lowercase” in
|
||||
# DerivedCoreProperties.txt but do not have a mapping to upper
|
||||
# case. For example, ꜰ U+A72F “LATIN LETTER SMALL CAPITAL F” is
|
||||
# one of these.
|
||||
return (to_upper(code_point) != code_point
|
||||
# <U00DF> is lowercase, but without simple to_upper mapping.
|
||||
or code_point == 0x00DF
|
||||
or (code_point in DERIVED_CORE_PROPERTIES
|
||||
and 'Lowercase' in DERIVED_CORE_PROPERTIES[code_point]))
|
||||
|
||||
def is_alpha(code_point):
|
||||
'''Checks whether the character with this code point is alphabetic'''
|
||||
return ((code_point in DERIVED_CORE_PROPERTIES
|
||||
and
|
||||
'Alphabetic' in DERIVED_CORE_PROPERTIES[code_point])
|
||||
or
|
||||
# Consider all the non-ASCII digits as alphabetic.
|
||||
# ISO C 99 forbids us to have them in category “digit”,
|
||||
# but we want iswalnum to return true on them.
|
||||
(UNICODE_ATTRIBUTES[code_point]['category'] == 'Nd'
|
||||
and not (code_point >= 0x0030 and code_point <= 0x0039)))
|
||||
|
||||
def is_digit(code_point):
|
||||
'''Checks whether the character with this code point is a digit'''
|
||||
if False:
|
||||
return (UNICODE_ATTRIBUTES[code_point]['name']
|
||||
and UNICODE_ATTRIBUTES[code_point]['category'] == 'Nd')
|
||||
# Note: U+0BE7..U+0BEF and U+1369..U+1371 are digit systems without
|
||||
# a zero. Must add <0> in front of them by hand.
|
||||
else:
|
||||
# SUSV2 gives us some freedom for the "digit" category, but ISO C 99
|
||||
# takes it away:
|
||||
# 7.25.2.1.5:
|
||||
# The iswdigit function tests for any wide character that
|
||||
# corresponds to a decimal-digit character (as defined in 5.2.1).
|
||||
# 5.2.1:
|
||||
# the 10 decimal digits 0 1 2 3 4 5 6 7 8 9
|
||||
return (code_point >= 0x0030 and code_point <= 0x0039)
|
||||
|
||||
def is_outdigit(code_point):
|
||||
'''Checks whether the character with this code point is outdigit'''
|
||||
return (code_point >= 0x0030 and code_point <= 0x0039)
|
||||
|
||||
def is_blank(code_point):
|
||||
'''Checks whether the character with this code point is blank'''
|
||||
return (code_point == 0x0009 # '\t'
|
||||
# Category Zs without mention of '<noBreak>'
|
||||
or (UNICODE_ATTRIBUTES[code_point]['name']
|
||||
and UNICODE_ATTRIBUTES[code_point]['category'] == 'Zs'
|
||||
and '<noBreak>' not in
|
||||
UNICODE_ATTRIBUTES[code_point]['decomposition']))
|
||||
|
||||
def is_space(code_point):
|
||||
'''Checks whether the character with this code point is a space'''
|
||||
# Don’t make U+00A0 a space. Non-breaking space means that all programs
|
||||
# should treat it like a punctuation character, not like a space.
|
||||
return (code_point == 0x0020 # ' '
|
||||
or code_point == 0x000C # '\f'
|
||||
or code_point == 0x000A # '\n'
|
||||
or code_point == 0x000D # '\r'
|
||||
or code_point == 0x0009 # '\t'
|
||||
or code_point == 0x000B # '\v'
|
||||
# Categories Zl, Zp, and Zs without mention of "<noBreak>"
|
||||
or (UNICODE_ATTRIBUTES[code_point]['name']
|
||||
and
|
||||
(UNICODE_ATTRIBUTES[code_point]['category'] in ['Zl', 'Zp']
|
||||
or
|
||||
(UNICODE_ATTRIBUTES[code_point]['category'] in ['Zs']
|
||||
and
|
||||
'<noBreak>' not in
|
||||
UNICODE_ATTRIBUTES[code_point]['decomposition']))))
|
||||
|
||||
def is_cntrl(code_point):
|
||||
'''Checks whether the character with this code point is
|
||||
a control character'''
|
||||
return (UNICODE_ATTRIBUTES[code_point]['name']
|
||||
and (UNICODE_ATTRIBUTES[code_point]['name'] == '<control>'
|
||||
or
|
||||
UNICODE_ATTRIBUTES[code_point]['category'] in ['Zl', 'Zp']))
|
||||
|
||||
def is_xdigit(code_point):
|
||||
'''Checks whether the character with this code point is
|
||||
a hexadecimal digit'''
|
||||
if False:
|
||||
return (is_digit(code_point)
|
||||
or (code_point >= 0x0041 and code_point <= 0x0046)
|
||||
or (code_point >= 0x0061 and code_point <= 0x0066))
|
||||
else:
|
||||
# SUSV2 gives us some freedom for the "xdigit" category, but ISO C 99
|
||||
# takes it away:
|
||||
# 7.25.2.1.12:
|
||||
# The iswxdigit function tests for any wide character that
|
||||
# corresponds to a hexadecimal-digit character (as defined
|
||||
# in 6.4.4.1).
|
||||
# 6.4.4.1:
|
||||
# hexadecimal-digit: one of
|
||||
# 0 1 2 3 4 5 6 7 8 9 a b c d e f A B C D E F
|
||||
return ((code_point >= 0x0030 and code_point <= 0x0039)
|
||||
or (code_point >= 0x0041 and code_point <= 0x0046)
|
||||
or (code_point >= 0x0061 and code_point <= 0x0066))
|
||||
|
||||
def is_graph(code_point):
|
||||
'''Checks whether the character with this code point is
|
||||
a graphical character'''
|
||||
return (UNICODE_ATTRIBUTES[code_point]['name']
|
||||
and UNICODE_ATTRIBUTES[code_point]['name'] != '<control>'
|
||||
and not is_space(code_point))
|
||||
|
||||
def is_print(code_point):
|
||||
'''Checks whether the character with this code point is printable'''
|
||||
return (UNICODE_ATTRIBUTES[code_point]['name']
|
||||
and UNICODE_ATTRIBUTES[code_point]['name'] != '<control>'
|
||||
and UNICODE_ATTRIBUTES[code_point]['category'] not in ['Zl', 'Zp'])
|
||||
|
||||
def is_punct(code_point):
|
||||
'''Checks whether the character with this code point is punctuation'''
|
||||
if False:
|
||||
return (UNICODE_ATTRIBUTES[code_point]['name']
|
||||
and UNICODE_ATTRIBUTES[code_point]['category'].startswith('P'))
|
||||
else:
|
||||
# The traditional POSIX definition of punctuation is every graphic,
|
||||
# non-alphanumeric character.
|
||||
return (is_graph(code_point)
|
||||
and not is_alpha(code_point)
|
||||
and not is_digit(code_point))
|
||||
|
||||
def is_combining(code_point):
|
||||
'''Checks whether the character with this code point is
|
||||
a combining character'''
|
||||
# Up to Unicode 3.0.1 we took the Combining property from the PropList.txt
|
||||
# file. In 3.0.1 it was identical to the union of the general categories
|
||||
# "Mn", "Mc", "Me". In Unicode 3.1 this property has been dropped from the
|
||||
# PropList.txt file, so we take the latter definition.
|
||||
return (UNICODE_ATTRIBUTES[code_point]['name']
|
||||
and
|
||||
UNICODE_ATTRIBUTES[code_point]['category'] in ['Mn', 'Mc', 'Me'])
|
||||
|
||||
def is_combining_level3(code_point):
|
||||
'''Checks whether the character with this code point is
|
||||
a combining level3 character'''
|
||||
return (is_combining(code_point)
|
||||
and
|
||||
int(UNICODE_ATTRIBUTES[code_point]['combining']) in range(0, 200))
|
||||
|
||||
def ucs_symbol(code_point):
|
||||
'''Return the UCS symbol string for a Unicode character.'''
|
||||
if code_point < 0x10000:
|
||||
return '<U{:04X}>'.format(code_point)
|
||||
else:
|
||||
return '<U{:08X}>'.format(code_point)
|
||||
|
||||
def ucs_symbol_range(code_point_low, code_point_high):
|
||||
'''Returns a string UCS symbol string for a code point range.
|
||||
|
||||
Example:
|
||||
|
||||
<U0041>..<U005A>
|
||||
'''
|
||||
return ucs_symbol(code_point_low) + '..' + ucs_symbol(code_point_high)
|
||||
|
||||
def code_point_ranges(is_class_function):
|
||||
'''Returns a list of ranges of code points for which is_class_function
|
||||
returns True.
|
||||
|
||||
Example:
|
||||
|
||||
[[65, 90], [192, 214], [216, 222], [256], … ]
|
||||
'''
|
||||
cp_ranges = []
|
||||
for code_point in sorted(UNICODE_ATTRIBUTES):
|
||||
if is_class_function(code_point):
|
||||
if (cp_ranges
|
||||
and cp_ranges[-1][-1] == code_point - 1):
|
||||
if len(cp_ranges[-1]) == 1:
|
||||
cp_ranges[-1].append(code_point)
|
||||
else:
|
||||
cp_ranges[-1][-1] = code_point
|
||||
else:
|
||||
cp_ranges.append([code_point])
|
||||
return cp_ranges
|
||||
|
||||
def output_charclass(i18n_file, class_name, is_class_function):
|
||||
'''Output a LC_CTYPE character class section
|
||||
|
||||
Example:
|
||||
|
||||
upper /
|
||||
<U0041>..<U005A>;<U00C0>..<U00D6>;<U00D8>..<U00DE>;<U0100>;<U0102>;/
|
||||
…
|
||||
<U0001D790>..<U0001D7A8>;<U0001D7CA>;<U0001F130>..<U0001F149>;/
|
||||
<U0001F150>..<U0001F169>;<U0001F170>..<U0001F189>
|
||||
'''
|
||||
cp_ranges = code_point_ranges(is_class_function)
|
||||
if cp_ranges:
|
||||
i18n_file.write('%s /\n' %class_name)
|
||||
max_column = 75
|
||||
prefix = ' '
|
||||
line = prefix
|
||||
range_string = ''
|
||||
for code_point_range in cp_ranges:
|
||||
if line.strip():
|
||||
line += ';'
|
||||
if len(code_point_range) == 1:
|
||||
range_string = ucs_symbol(code_point_range[0])
|
||||
else:
|
||||
range_string = ucs_symbol_range(
|
||||
code_point_range[0], code_point_range[-1])
|
||||
if len(line+range_string) > max_column:
|
||||
i18n_file.write(line+'/\n')
|
||||
line = prefix
|
||||
line += range_string
|
||||
if line.strip():
|
||||
i18n_file.write(line+'\n')
|
||||
i18n_file.write('\n')
|
||||
|
||||
def output_charmap(i18n_file, map_name, map_function):
|
||||
'''Output a LC_CTYPE character map section
|
||||
|
||||
Example:
|
||||
|
||||
toupper /
|
||||
(<U0061>,<U0041>);(<U0062>,<U0042>);(<U0063>,<U0043>);(<U0064>,<U0044>);/
|
||||
…
|
||||
(<U000118DC>,<U000118BC>);(<U000118DD>,<U000118BD>);/
|
||||
(<U000118DE>,<U000118BE>);(<U000118DF>,<U000118BF>)
|
||||
'''
|
||||
max_column = 75
|
||||
prefix = ' '
|
||||
line = prefix
|
||||
map_string = ''
|
||||
i18n_file.write('%s /\n' %map_name)
|
||||
for code_point in sorted(UNICODE_ATTRIBUTES):
|
||||
mapped = map_function(code_point)
|
||||
if code_point != mapped:
|
||||
if line.strip():
|
||||
line += ';'
|
||||
map_string = '(' \
|
||||
+ ucs_symbol(code_point) \
|
||||
+ ',' \
|
||||
+ ucs_symbol(mapped) \
|
||||
+ ')'
|
||||
if len(line+map_string) > max_column:
|
||||
i18n_file.write(line+'/\n')
|
||||
line = prefix
|
||||
line += map_string
|
||||
if line.strip():
|
||||
i18n_file.write(line+'\n')
|
||||
i18n_file.write('\n')
|
||||
|
||||
def verifications():
|
||||
'''Tests whether the is_* functions observe the known restrictions'''
|
||||
for code_point in sorted(UNICODE_ATTRIBUTES):
|
||||
# toupper restriction: "Only characters specified for the keywords
|
||||
# lower and upper shall be specified.
|
||||
if (to_upper(code_point) != code_point
|
||||
and not (is_lower(code_point) or is_upper(code_point))):
|
||||
sys.stderr.write(
|
||||
('%(sym)s is not upper|lower '
|
||||
+ 'but toupper(0x%(c)04X) = 0x%(uc)04X\n') %{
|
||||
'sym': ucs_symbol(code_point),
|
||||
'c': code_point,
|
||||
'uc': to_upper(code_point)})
|
||||
# tolower restriction: "Only characters specified for the keywords
|
||||
# lower and upper shall be specified.
|
||||
if (to_lower(code_point) != code_point
|
||||
and not (is_lower(code_point) or is_upper(code_point))):
|
||||
sys.stderr.write(
|
||||
('%(sym)s is not upper|lower '
|
||||
+ 'but tolower(0x%(c)04X) = 0x%(uc)04X\n') %{
|
||||
'sym': ucs_symbol(code_point),
|
||||
'c': code_point,
|
||||
'uc': to_lower(code_point)})
|
||||
# alpha restriction: "Characters classified as either upper or lower
|
||||
# shall automatically belong to this class.
|
||||
if ((is_lower(code_point) or is_upper(code_point))
|
||||
and not is_alpha(code_point)):
|
||||
sys.stderr.write('%(sym)s is upper|lower but not alpha\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
# alpha restriction: “No character specified for the keywords cntrl,
|
||||
# digit, punct or space shall be specified.”
|
||||
if (is_alpha(code_point) and is_cntrl(code_point)):
|
||||
sys.stderr.write('%(sym)s is alpha and cntrl\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
if (is_alpha(code_point) and is_digit(code_point)):
|
||||
sys.stderr.write('%(sym)s is alpha and digit\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
if (is_alpha(code_point) and is_punct(code_point)):
|
||||
sys.stderr.write('%(sym)s is alpha and punct\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
if (is_alpha(code_point) and is_space(code_point)):
|
||||
sys.stderr.write('%(sym)s is alpha and space\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
# space restriction: “No character specified for the keywords upper,
|
||||
# lower, alpha, digit, graph or xdigit shall be specified.”
|
||||
# upper, lower, alpha already checked above.
|
||||
if (is_space(code_point) and is_digit(code_point)):
|
||||
sys.stderr.write('%(sym)s is space and digit\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
if (is_space(code_point) and is_graph(code_point)):
|
||||
sys.stderr.write('%(sym)s is space and graph\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
if (is_space(code_point) and is_xdigit(code_point)):
|
||||
sys.stderr.write('%(sym)s is space and xdigit\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
# cntrl restriction: “No character specified for the keywords upper,
|
||||
# lower, alpha, digit, punct, graph, print or xdigit shall be
|
||||
# specified.” upper, lower, alpha already checked above.
|
||||
if (is_cntrl(code_point) and is_digit(code_point)):
|
||||
sys.stderr.write('%(sym)s is cntrl and digit\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
if (is_cntrl(code_point) and is_punct(code_point)):
|
||||
sys.stderr.write('%(sym)s is cntrl and punct\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
if (is_cntrl(code_point) and is_graph(code_point)):
|
||||
sys.stderr.write('%(sym)s is cntrl and graph\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
if (is_cntrl(code_point) and is_print(code_point)):
|
||||
sys.stderr.write('%(sym)s is cntrl and print\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
if (is_cntrl(code_point) and is_xdigit(code_point)):
|
||||
sys.stderr.write('%(sym)s is cntrl and xdigit\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
# punct restriction: “No character specified for the keywords upper,
|
||||
# lower, alpha, digit, cntrl, xdigit or as the <space> character shall
|
||||
# be specified.” upper, lower, alpha, cntrl already checked above.
|
||||
if (is_punct(code_point) and is_digit(code_point)):
|
||||
sys.stderr.write('%(sym)s is punct and digit\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
if (is_punct(code_point) and is_xdigit(code_point)):
|
||||
sys.stderr.write('%(sym)s is punct and xdigit\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
if (is_punct(code_point) and code_point == 0x0020):
|
||||
sys.stderr.write('%(sym)s is punct\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
# graph restriction: “No character specified for the keyword cntrl
|
||||
# shall be specified.” Already checked above.
|
||||
|
||||
# print restriction: “No character specified for the keyword cntrl
|
||||
# shall be specified.” Already checked above.
|
||||
|
||||
# graph - print relation: differ only in the <space> character.
|
||||
# How is this possible if there are more than one space character?!
|
||||
# I think susv2/xbd/locale.html should speak of “space characters”,
|
||||
# not “space character”.
|
||||
if (is_print(code_point)
|
||||
and not (is_graph(code_point) or is_space(code_point))):
|
||||
sys.stderr.write('%(sym)s is print but not graph|<space>\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
if (not is_print(code_point)
|
||||
and (is_graph(code_point) or code_point == 0x0020)):
|
||||
sys.stderr.write('%(sym)s is graph|<space> but not print\n' %{
|
||||
'sym': ucs_symbol(code_point)})
|
||||
|
||||
def read_input_file(filename):
|
||||
'''Reads the original glibc i18n file to get the original head
|
||||
and tail.
|
||||
|
||||
We want to replace only the character classes in LC_CTYPE, and the
|
||||
date stamp. All the rest of the i18n file should stay unchanged.
|
||||
To avoid having to cut and paste the generated data into the
|
||||
original file, it is helpful to read the original file here
|
||||
to be able to generate a complete result file.
|
||||
'''
|
||||
head = tail = ''
|
||||
with open(filename, mode='r') as i18n_file:
|
||||
for line in i18n_file:
|
||||
match = re.match(
|
||||
r'^(?P<key>date\s+)(?P<value>"[0-9]{4}-[0-9]{2}-[0-9]{2}")',
|
||||
line)
|
||||
if match:
|
||||
line = match.group('key') \
|
||||
+ '"{:s}"\n'.format(time.strftime('%Y-%m-%d'))
|
||||
head = head + line
|
||||
if line.startswith('LC_CTYPE'):
|
||||
break
|
||||
for line in i18n_file:
|
||||
if line.startswith('translit_start'):
|
||||
tail = line
|
||||
break
|
||||
for line in i18n_file:
|
||||
tail = tail + line
|
||||
return (head, tail)
|
||||
|
||||
def output_head(i18n_file, unicode_version, head=''):
|
||||
'''Write the header of the output file, i.e. the part of the file
|
||||
before the “LC_CTYPE” line.
|
||||
'''
|
||||
if ARGS.input_file and head:
|
||||
i18n_file.write(head)
|
||||
else:
|
||||
i18n_file.write('escape_char /\n')
|
||||
i18n_file.write('comment_char %\n')
|
||||
i18n_file.write('\n')
|
||||
i18n_file.write('% Generated automatically by '
|
||||
+ 'gen_unicode_ctype.py '
|
||||
+ 'for Unicode {:s}.\n'.format(unicode_version))
|
||||
i18n_file.write('\n')
|
||||
i18n_file.write('LC_IDENTIFICATION\n')
|
||||
i18n_file.write('title "Unicode {:s} FDCC-set"\n'.format(
|
||||
unicode_version))
|
||||
i18n_file.write('source "UnicodeData.txt, '
|
||||
+ 'DerivedCoreProperties.txt"\n')
|
||||
i18n_file.write('address ""\n')
|
||||
i18n_file.write('contact ""\n')
|
||||
i18n_file.write('email "bug-glibc-locales@gnu.org"\n')
|
||||
i18n_file.write('tel ""\n')
|
||||
i18n_file.write('fax ""\n')
|
||||
i18n_file.write('language ""\n')
|
||||
i18n_file.write('territory "Earth"\n')
|
||||
i18n_file.write('revision "{:s}"\n'.format(unicode_version))
|
||||
i18n_file.write('date "{:s}"\n'.format(
|
||||
time.strftime('%Y-%m-%d')))
|
||||
i18n_file.write('category "unicode:2014";LC_CTYPE\n')
|
||||
i18n_file.write('END LC_IDENTIFICATION\n')
|
||||
i18n_file.write('\n')
|
||||
i18n_file.write('LC_CTYPE\n')
|
||||
|
||||
def output_tail(i18n_file, tail=''):
|
||||
'''Write the tail of the output file, i.e. the part of the file
|
||||
after the last “LC_CTYPE” character class.
|
||||
'''
|
||||
if ARGS.input_file and tail:
|
||||
i18n_file.write(tail)
|
||||
else:
|
||||
i18n_file.write('END LC_CTYPE\n')
|
||||
|
||||
def output_tables(i18n_file, unicode_version):
|
||||
'''Write the new LC_CTYPE character classes to the output file'''
|
||||
i18n_file.write('% The following is the 14652 i18n fdcc-set '
|
||||
+ 'LC_CTYPE category.\n')
|
||||
i18n_file.write('% It covers Unicode version {:s}.\n'.format(
|
||||
unicode_version))
|
||||
i18n_file.write('% The character classes and mapping tables were '
|
||||
+ 'automatically\n')
|
||||
i18n_file.write('% generated using the gen_unicode_ctype.py '
|
||||
+ 'program.\n\n')
|
||||
i18n_file.write('% The "upper" class reflects the uppercase '
|
||||
+ 'characters of class "alpha"\n')
|
||||
output_charclass(i18n_file, 'upper', is_upper)
|
||||
i18n_file.write('% The "lower" class reflects the lowercase '
|
||||
+ 'characters of class "alpha"\n')
|
||||
output_charclass(i18n_file, 'lower', is_lower)
|
||||
i18n_file.write('% The "alpha" class of the "i18n" FDCC-set is '
|
||||
+ 'reflecting\n')
|
||||
i18n_file.write('% the recommendations in TR 10176 annex A\n')
|
||||
output_charclass(i18n_file, 'alpha', is_alpha)
|
||||
i18n_file.write('% The "digit" class must only contain the '
|
||||
+ 'BASIC LATIN digits, says ISO C 99\n')
|
||||
i18n_file.write('% (sections 7.25.2.1.5 and 5.2.1).\n')
|
||||
output_charclass(i18n_file, 'digit', is_digit)
|
||||
i18n_file.write('% The "outdigit" information is by default '
|
||||
+ '"0" to "9". We don\'t have to\n')
|
||||
i18n_file.write('% provide it here since localedef will fill '
|
||||
+ 'in the bits and it would\n')
|
||||
i18n_file.write('% prevent locales copying this file define '
|
||||
+ 'their own values.\n')
|
||||
i18n_file.write('% outdigit /\n')
|
||||
i18n_file.write('% <U0030>..<U0039>\n\n')
|
||||
# output_charclass(i18n_file, 'outdigit', is_outdigit)
|
||||
output_charclass(i18n_file, 'space', is_space)
|
||||
output_charclass(i18n_file, 'cntrl', is_cntrl)
|
||||
output_charclass(i18n_file, 'punct', is_punct)
|
||||
output_charclass(i18n_file, 'graph', is_graph)
|
||||
output_charclass(i18n_file, 'print', is_print)
|
||||
i18n_file.write('% The "xdigit" class must only contain the '
|
||||
+ 'BASIC LATIN digits and A-F, a-f,\n')
|
||||
i18n_file.write('% says ISO C 99 '
|
||||
+ '(sections 7.25.2.1.12 and 6.4.4.1).\n')
|
||||
output_charclass(i18n_file, 'xdigit', is_xdigit)
|
||||
output_charclass(i18n_file, 'blank', is_blank)
|
||||
output_charmap(i18n_file, 'toupper', to_upper)
|
||||
output_charmap(i18n_file, 'tolower', to_lower)
|
||||
output_charmap(i18n_file, 'map "totitle";', to_title)
|
||||
i18n_file.write('% The "combining" class reflects ISO/IEC 10646-1 '
|
||||
+ 'annex B.1\n')
|
||||
i18n_file.write('% That is, all combining characters (level 2+3).\n')
|
||||
output_charclass(i18n_file, 'class "combining";', is_combining)
|
||||
i18n_file.write('% The "combining_level3" class reflects '
|
||||
+ 'ISO/IEC 10646-1 annex B.2\n')
|
||||
i18n_file.write('% That is, combining characters of level 3.\n')
|
||||
output_charclass(i18n_file,
|
||||
'class "combining_level3";', is_combining_level3)
|
||||
|
||||
if __name__ == "__main__":
|
||||
PARSER = argparse.ArgumentParser(
|
||||
description='''
|
||||
Generate a Unicode conforming LC_CTYPE category from
|
||||
UnicodeData.txt and DerivedCoreProperties.txt files.
|
||||
''')
|
||||
PARSER.add_argument(
|
||||
'-u', '--unicode_data_file',
|
||||
nargs='?',
|
||||
type=str,
|
||||
default='UnicodeData.txt',
|
||||
help=('The UnicodeData.txt file to read, '
|
||||
+ 'default: %(default)s'))
|
||||
PARSER.add_argument(
|
||||
'-d', '--derived_core_properties_file',
|
||||
nargs='?',
|
||||
type=str,
|
||||
default='DerivedCoreProperties.txt',
|
||||
help=('The DerivedCoreProperties.txt file to read, '
|
||||
+ 'default: %(default)s'))
|
||||
PARSER.add_argument(
|
||||
'-i', '--input_file',
|
||||
nargs='?',
|
||||
type=str,
|
||||
help='''The original glibc/localedata/locales/i18n file.''')
|
||||
PARSER.add_argument(
|
||||
'-o', '--output_file',
|
||||
nargs='?',
|
||||
type=str,
|
||||
default='i18n.new',
|
||||
help='''The file which shall contain the generated LC_CTYPE category,
|
||||
default: %(default)s. If the original
|
||||
glibc/localedata/locales/i18n has been given
|
||||
as an option, all data from the original file
|
||||
except the newly generated LC_CTYPE character
|
||||
classes and the date stamp in
|
||||
LC_IDENTIFICATION will be copied unchanged
|
||||
into the output file. ''')
|
||||
PARSER.add_argument(
|
||||
'--unicode_version',
|
||||
nargs='?',
|
||||
required=True,
|
||||
type=str,
|
||||
help='The Unicode version of the input files used.')
|
||||
ARGS = PARSER.parse_args()
|
||||
|
||||
fill_attributes(ARGS.unicode_data_file)
|
||||
fill_derived_core_properties(ARGS.derived_core_properties_file)
|
||||
verifications()
|
||||
HEAD = TAIL = ''
|
||||
if ARGS.input_file:
|
||||
(HEAD, TAIL) = read_input_file(ARGS.input_file)
|
||||
with open(ARGS.output_file, mode='w') as I18N_FILE:
|
||||
output_head(I18N_FILE, ARGS.unicode_version, head=HEAD)
|
||||
output_tables(I18N_FILE, ARGS.unicode_version)
|
||||
output_tail(I18N_FILE, tail=TAIL)
|
||||
50
localedata/unicode-gen/unicode-license.txt
Normal file
50
localedata/unicode-gen/unicode-license.txt
Normal file
@ -0,0 +1,50 @@
|
||||
UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
|
||||
|
||||
Unicode Data Files include all data files under the directories
|
||||
http://www.unicode.org/Public/, http://www.unicode.org/reports/, and
|
||||
http://www.unicode.org/cldr/data/. Unicode Data Files do not include PDF
|
||||
online code charts under the directory http://www.unicode.org/Public/.
|
||||
Software includes any source code published in the Unicode Standard or under
|
||||
the directories http://www.unicode.org/Public/,
|
||||
http://www.unicode.org/reports/, and http://www.unicode.org/cldr/data/.
|
||||
|
||||
NOTICE TO USER: Carefully read the following legal agreement. BY
|
||||
DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S DATA FILES
|
||||
("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"), YOU UNEQUIVOCALLY ACCEPT, AND
|
||||
AGREE TO BE BOUND BY, ALL OF THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF
|
||||
YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA
|
||||
FILES OR SOFTWARE.
|
||||
|
||||
COPYRIGHT AND PERMISSION NOTICE
|
||||
|
||||
Copyright © 1991-2013 Unicode, Inc. All rights reserved. Distributed under
|
||||
the Terms of Use in http://www.unicode.org/copyright.html.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a
|
||||
copy of the Unicode data files and any associated documentation (the "Data
|
||||
Files") or Unicode software and any associated documentation (the "Software")
|
||||
to deal in the Data Files or Software without restriction, including without
|
||||
limitation the rights to use, copy, modify, merge, publish, distribute, and/or
|
||||
sell copies of the Data Files or Software, and to permit persons to whom the
|
||||
Data Files or Software are furnished to do so, provided that (a) the above
|
||||
copyright notice(s) and this permission notice appear with all copies of the
|
||||
Data Files or Software, (b) both the above copyright notice(s) and this
|
||||
permission notice appear in associated documentation, and (c) there is clear
|
||||
notice in each modified Data File or in the Software as well as in the
|
||||
documentation associated with the Data File(s) or Software that the data or
|
||||
software has been modified.
|
||||
|
||||
THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
|
||||
KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD
|
||||
PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN
|
||||
THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
|
||||
DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
|
||||
PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
|
||||
ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE
|
||||
DATA FILES OR SOFTWARE.
|
||||
|
||||
Except as contained in this notice, the name of a copyright holder shall
|
||||
not be used in advertising or otherwise to promote the sale, use or other
|
||||
dealings in these Data Files or Software without prior written authorization
|
||||
of the copyright holder.
|
||||
399
localedata/unicode-gen/utf8_compatibility.py
Executable file
399
localedata/unicode-gen/utf8_compatibility.py
Executable file
@ -0,0 +1,399 @@
|
||||
#!/usr/bin/python3
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (C) 2014, 2015 Free Software Foundation, Inc.
|
||||
# This file is part of the GNU C Library.
|
||||
#
|
||||
# The GNU C Library is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU Lesser General Public
|
||||
# License as published by the Free Software Foundation; either
|
||||
# version 2.1 of the License, or (at your option) any later version.
|
||||
#
|
||||
# The GNU C Library is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with the GNU C Library; if not, see
|
||||
# <http://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
This script is useful for checking backward compatibility of newly
|
||||
generated UTF-8 file from utf8_gen.py script
|
||||
|
||||
To see how this script is used, call it with the “-h” option:
|
||||
|
||||
$ ./utf8_compatibility.py -h
|
||||
… prints usage message …
|
||||
'''
|
||||
|
||||
import sys
|
||||
import re
|
||||
import argparse
|
||||
|
||||
# Dictionary holding the entire contents of the UnicodeData.txt file
|
||||
#
|
||||
# Contents of this dictionary look like this:
|
||||
#
|
||||
# {0: {'category': 'Cc',
|
||||
# 'title': None,
|
||||
# 'digit': '',
|
||||
# 'name': '<control>',
|
||||
# 'bidi': 'BN',
|
||||
# 'combining': '0',
|
||||
# 'comment': '',
|
||||
# 'oldname': 'NULL',
|
||||
# 'decomposition': '',
|
||||
# 'upper': None,
|
||||
# 'mirrored': 'N',
|
||||
# 'lower': None,
|
||||
# 'decdigit': '',
|
||||
# 'numeric': ''},
|
||||
# …
|
||||
# }
|
||||
UNICODE_ATTRIBUTES = {}
|
||||
|
||||
# Dictionary holding the entire contents of the EastAsianWidths.txt file
|
||||
#
|
||||
# Contents of this dictionary look like this:
|
||||
#
|
||||
# {0: 'N', … , 45430: 'W', …}
|
||||
EAST_ASIAN_WIDTHS = {}
|
||||
|
||||
def fill_attribute(code_point, fields):
|
||||
'''Stores in UNICODE_ATTRIBUTES[code_point] the values from the fields.
|
||||
|
||||
One entry in the UNICODE_ATTRIBUTES dictionary represents one line
|
||||
in the UnicodeData.txt file.
|
||||
|
||||
'''
|
||||
UNICODE_ATTRIBUTES[code_point] = {
|
||||
'name': fields[1], # Character name
|
||||
'category': fields[2], # General category
|
||||
'combining': fields[3], # Canonical combining classes
|
||||
'bidi': fields[4], # Bidirectional category
|
||||
'decomposition': fields[5], # Character decomposition mapping
|
||||
'decdigit': fields[6], # Decimal digit value
|
||||
'digit': fields[7], # Digit value
|
||||
'numeric': fields[8], # Numeric value
|
||||
'mirrored': fields[9], # mirrored
|
||||
'oldname': fields[10], # Old Unicode 1.0 name
|
||||
'comment': fields[11], # comment
|
||||
# Uppercase mapping
|
||||
'upper': int(fields[12], 16) if fields[12] else None,
|
||||
# Lowercase mapping
|
||||
'lower': int(fields[13], 16) if fields[13] else None,
|
||||
# Titlecase mapping
|
||||
'title': int(fields[14], 16) if fields[14] else None,
|
||||
}
|
||||
|
||||
def fill_attributes(filename):
|
||||
'''Stores the entire contents of the UnicodeData.txt file
|
||||
in the UNICODE_ATTRIBUTES dictionary.
|
||||
|
||||
A typical line for a single code point in UnicodeData.txt looks
|
||||
like this:
|
||||
|
||||
0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;
|
||||
|
||||
Code point ranges are indicated by pairs of lines like this:
|
||||
|
||||
4E00;<CJK Ideograph, First>;Lo;0;L;;;;;N;;;;;
|
||||
9FCC;<CJK Ideograph, Last>;Lo;0;L;;;;;N;;;;;
|
||||
'''
|
||||
with open(filename, mode='r') as unicode_data_file:
|
||||
fields_start = []
|
||||
for line in unicode_data_file:
|
||||
fields = line.strip().split(';')
|
||||
if len(fields) != 15:
|
||||
sys.stderr.write(
|
||||
'short line in file "%(f)s": %(l)s\n' %{
|
||||
'f': filename, 'l': line})
|
||||
exit(1)
|
||||
if fields[2] == 'Cs':
|
||||
# Surrogates are UTF-16 artefacts,
|
||||
# not real characters. Ignore them.
|
||||
fields_start = []
|
||||
continue
|
||||
if fields[1].endswith(', First>'):
|
||||
fields_start = fields
|
||||
fields_start[1] = fields_start[1].split(',')[0][1:]
|
||||
continue
|
||||
if fields[1].endswith(', Last>'):
|
||||
fields[1] = fields[1].split(',')[0][1:]
|
||||
if fields[1:] != fields_start[1:]:
|
||||
sys.stderr.write(
|
||||
'broken code point range in file "%(f)s": %(l)s\n' %{
|
||||
'f': filename, 'l': line})
|
||||
exit(1)
|
||||
for code_point in range(
|
||||
int(fields_start[0], 16),
|
||||
int(fields[0], 16)+1):
|
||||
fill_attribute(code_point, fields)
|
||||
fields_start = []
|
||||
continue
|
||||
fill_attribute(int(fields[0], 16), fields)
|
||||
fields_start = []
|
||||
|
||||
def fill_east_asian_widths(filename):
|
||||
'''Stores the entire contents of the EastAsianWidths.txt file
|
||||
in the EAST_ASIAN_WIDTHS dictionary.
|
||||
|
||||
Lines in EastAsianWidths.txt are either a code point range like
|
||||
this:
|
||||
|
||||
9FCD..9FFF;W # Cn [51] <reserved-9FCD>..<reserved-9FFF>
|
||||
|
||||
or a single code point like this:
|
||||
|
||||
A015;W # Lm YI SYLLABLE WU
|
||||
'''
|
||||
with open(filename, mode='r') as east_asian_widths_file:
|
||||
for line in east_asian_widths_file:
|
||||
match = re.match(
|
||||
r'^(?P<codepoint1>[0-9A-F]{4,6})'
|
||||
+r'(?:\.\.(?P<codepoint2>[0-9A-F]{4,6}))?'
|
||||
+r'\s*;\s*(?P<property>[a-zA-Z]+)',
|
||||
line)
|
||||
if not match:
|
||||
continue
|
||||
start = match.group('codepoint1')
|
||||
end = match.group('codepoint2')
|
||||
if not end:
|
||||
end = start
|
||||
for code_point in range(int(start, 16), int(end, 16)+1):
|
||||
EAST_ASIAN_WIDTHS[code_point] = match.group('property')
|
||||
|
||||
def ucs_symbol(code_point):
|
||||
'''Return the UCS symbol string for a Unicode character.'''
|
||||
if code_point < 0x10000:
|
||||
return '<U{:04X}>'.format(code_point)
|
||||
else:
|
||||
return '<U{:08X}>'.format(code_point)
|
||||
|
||||
def create_charmap_dictionary(file_name):
|
||||
'''Create a dictionary for all code points found in the CHARMAP
|
||||
section of a file
|
||||
'''
|
||||
with open(file_name, mode='r') as utf8_file:
|
||||
charmap_dictionary = {}
|
||||
for line in utf8_file:
|
||||
if line.startswith('CHARMAP'):
|
||||
break
|
||||
for line in utf8_file:
|
||||
if line.startswith('END CHARMAP'):
|
||||
return charmap_dictionary
|
||||
if line.startswith('%'):
|
||||
continue
|
||||
match = re.match(
|
||||
r'^<U(?P<codepoint1>[0-9A-F]{4,8})>'
|
||||
+r'(:?\.\.<U(?P<codepoint2>[0-9-A-F]{4,8})>)?'
|
||||
+r'\s+(?P<hexutf8>(/x[0-9a-f]{2}){1,4})',
|
||||
line)
|
||||
if not match:
|
||||
continue
|
||||
codepoint1 = match.group('codepoint1')
|
||||
codepoint2 = match.group('codepoint2')
|
||||
if not codepoint2:
|
||||
codepoint2 = codepoint1
|
||||
for i in range(int(codepoint1, 16),
|
||||
int(codepoint2, 16) + 1):
|
||||
charmap_dictionary[i] = match.group('hexutf8')
|
||||
sys.stderr.write('No “CHARMAP” or no “END CHARMAP” found in %s\n'
|
||||
%file_name)
|
||||
exit(1)
|
||||
|
||||
def check_charmap(original_file_name, new_file_name):
|
||||
'''Report differences in the CHARMAP section between the old and the
|
||||
new file
|
||||
'''
|
||||
print('************************************************************')
|
||||
print('Report on CHARMAP:')
|
||||
ocharmap = create_charmap_dictionary(original_file_name)
|
||||
ncharmap = create_charmap_dictionary(new_file_name)
|
||||
print('------------------------------------------------------------')
|
||||
print('Total removed characters in newly generated CHARMAP: %d'
|
||||
%len(set(ocharmap)-set(ncharmap)))
|
||||
if ARGS.show_missing_characters:
|
||||
for key in sorted(set(ocharmap)-set(ncharmap)):
|
||||
print('removed: {:s} {:s} {:s}'.format(
|
||||
ucs_symbol(key),
|
||||
ocharmap[key],
|
||||
UNICODE_ATTRIBUTES[key]['name'] \
|
||||
if key in UNICODE_ATTRIBUTES else None))
|
||||
print('------------------------------------------------------------')
|
||||
changed_charmap = {}
|
||||
for key in set(ocharmap).intersection(set(ncharmap)):
|
||||
if ocharmap[key] != ncharmap[key]:
|
||||
changed_charmap[key] = (ocharmap[key], ncharmap[key])
|
||||
print('Total changed characters in newly generated CHARMAP: %d'
|
||||
%len(changed_charmap))
|
||||
if ARGS.show_changed_characters:
|
||||
for key in sorted(changed_charmap):
|
||||
print('changed: {:s} {:s}->{:s} {:s}'.format(
|
||||
ucs_symbol(key),
|
||||
changed_charmap[key][0],
|
||||
changed_charmap[key][1],
|
||||
UNICODE_ATTRIBUTES[key]['name'] \
|
||||
if key in UNICODE_ATTRIBUTES else None))
|
||||
print('------------------------------------------------------------')
|
||||
print('Total added characters in newly generated CHARMAP: %d'
|
||||
%len(set(ncharmap)-set(ocharmap)))
|
||||
if ARGS.show_added_characters:
|
||||
for key in sorted(set(ncharmap)-set(ocharmap)):
|
||||
print('added: {:s} {:s} {:s}'.format(
|
||||
ucs_symbol(key),
|
||||
ncharmap[key],
|
||||
UNICODE_ATTRIBUTES[key]['name'] \
|
||||
if key in UNICODE_ATTRIBUTES else None))
|
||||
|
||||
def create_width_dictionary(file_name):
|
||||
'''Create a dictionary for all code points found in the WIDTH
|
||||
section of a file
|
||||
'''
|
||||
with open(file_name, mode='r') as utf8_file:
|
||||
width_dictionary = {}
|
||||
for line in utf8_file:
|
||||
if line.startswith('WIDTH'):
|
||||
break
|
||||
for line in utf8_file:
|
||||
if line.startswith('END WIDTH'):
|
||||
return width_dictionary
|
||||
match = re.match(
|
||||
r'^<U(?P<codepoint1>[0-9A-F]{4,8})>'
|
||||
+r'(:?\.\.\.<U(?P<codepoint2>[0-9-A-F]{4,8})>)?'
|
||||
+r'\s+(?P<width>[02])',
|
||||
line)
|
||||
if not match:
|
||||
continue
|
||||
codepoint1 = match.group('codepoint1')
|
||||
codepoint2 = match.group('codepoint2')
|
||||
if not codepoint2:
|
||||
codepoint2 = codepoint1
|
||||
for i in range(int(codepoint1, 16),
|
||||
int(codepoint2, 16) + 1):
|
||||
width_dictionary[i] = int(match.group('width'))
|
||||
sys.stderr.write('No “WIDTH” or no “END WIDTH” found in %s\n' %file)
|
||||
|
||||
def check_width(original_file_name, new_file_name):
|
||||
'''Report differences in the WIDTH section between the old and the new
|
||||
file
|
||||
'''
|
||||
print('************************************************************')
|
||||
print('Report on WIDTH:')
|
||||
owidth = create_width_dictionary(original_file_name)
|
||||
nwidth = create_width_dictionary(new_file_name)
|
||||
print('------------------------------------------------------------')
|
||||
print('Total removed characters in newly generated WIDTH: %d'
|
||||
%len(set(owidth)-set(nwidth)))
|
||||
print('(Characters not in WIDTH get width 1 by default, '
|
||||
+ 'i.e. these have width 1 now.)')
|
||||
if ARGS.show_missing_characters:
|
||||
for key in sorted(set(owidth)-set(nwidth)):
|
||||
print('removed: {:s} '.format(ucs_symbol(key))
|
||||
+ '{:d} : '.format(owidth[key])
|
||||
+ 'eaw={:s} '.format(
|
||||
EAST_ASIAN_WIDTHS[key]
|
||||
if key in EAST_ASIAN_WIDTHS else None)
|
||||
+ 'category={:2s} '.format(
|
||||
UNICODE_ATTRIBUTES[key]['category']
|
||||
if key in UNICODE_ATTRIBUTES else None)
|
||||
+ 'bidi={:3s} '.format(
|
||||
UNICODE_ATTRIBUTES[key]['bidi']
|
||||
if key in UNICODE_ATTRIBUTES else None)
|
||||
+ 'name={:s}'.format(
|
||||
UNICODE_ATTRIBUTES[key]['name']
|
||||
if key in UNICODE_ATTRIBUTES else None))
|
||||
print('------------------------------------------------------------')
|
||||
changed_width = {}
|
||||
for key in set(owidth).intersection(set(nwidth)):
|
||||
if owidth[key] != nwidth[key]:
|
||||
changed_width[key] = (owidth[key], nwidth[key])
|
||||
print('Total changed characters in newly generated WIDTH: %d'
|
||||
%len(changed_width))
|
||||
if ARGS.show_changed_characters:
|
||||
for key in sorted(changed_width):
|
||||
print('changed width: {:s} '.format(ucs_symbol(key))
|
||||
+ '{:d}->{:d} : '.format(changed_width[key][0],
|
||||
changed_width[key][1])
|
||||
+ 'eaw={:s} '.format(
|
||||
EAST_ASIAN_WIDTHS[key]
|
||||
if key in EAST_ASIAN_WIDTHS else None)
|
||||
+ 'category={:2s} '.format(
|
||||
UNICODE_ATTRIBUTES[key]['category']
|
||||
if key in UNICODE_ATTRIBUTES else None)
|
||||
+ 'bidi={:3s} '.format(
|
||||
UNICODE_ATTRIBUTES[key]['bidi']
|
||||
if key in UNICODE_ATTRIBUTES else None)
|
||||
+ 'name={:s}'.format(
|
||||
UNICODE_ATTRIBUTES[key]['name']
|
||||
if key in UNICODE_ATTRIBUTES else None))
|
||||
print('------------------------------------------------------------')
|
||||
print('Total added characters in newly generated WIDTH: %d'
|
||||
%len(set(nwidth)-set(owidth)))
|
||||
print('(Characters not in WIDTH get width 1 by default, '
|
||||
+ 'i.e. these had width 1 before.)')
|
||||
if ARGS.show_added_characters:
|
||||
for key in sorted(set(nwidth)-set(owidth)):
|
||||
print('added: {:s} '.format(ucs_symbol(key))
|
||||
+ '{:d} : '.format(nwidth[key])
|
||||
+ 'eaw={:s} '.format(
|
||||
EAST_ASIAN_WIDTHS[key]
|
||||
if key in EAST_ASIAN_WIDTHS else None)
|
||||
+ 'category={:2s} '.format(
|
||||
UNICODE_ATTRIBUTES[key]['category']
|
||||
if key in UNICODE_ATTRIBUTES else None)
|
||||
+ 'bidi={:3s} '.format(
|
||||
UNICODE_ATTRIBUTES[key]['bidi']
|
||||
if key in UNICODE_ATTRIBUTES else None)
|
||||
+ 'name={:s}'.format(
|
||||
UNICODE_ATTRIBUTES[key]['name']
|
||||
if key in UNICODE_ATTRIBUTES else None))
|
||||
|
||||
if __name__ == "__main__":
|
||||
PARSER = argparse.ArgumentParser(
|
||||
description='''
|
||||
Compare the contents of LC_CTYPE in two files and check for errors.
|
||||
''')
|
||||
PARSER.add_argument(
|
||||
'-o', '--old_utf8_file',
|
||||
nargs='?',
|
||||
required=True,
|
||||
type=str,
|
||||
help='The old UTF-8 file.')
|
||||
PARSER.add_argument(
|
||||
'-n', '--new_utf8_file',
|
||||
nargs='?',
|
||||
required=True,
|
||||
type=str,
|
||||
help='The new UTF-8 file.')
|
||||
PARSER.add_argument(
|
||||
'-u', '--unicode_data_file',
|
||||
nargs='?',
|
||||
type=str,
|
||||
help='The UnicodeData.txt file to read.')
|
||||
PARSER.add_argument(
|
||||
'-e', '--east_asian_width_file',
|
||||
nargs='?',
|
||||
type=str,
|
||||
help='The EastAsianWidth.txt file to read.')
|
||||
PARSER.add_argument(
|
||||
'-a', '--show_added_characters',
|
||||
action='store_true',
|
||||
help='Show characters which were added in detail.')
|
||||
PARSER.add_argument(
|
||||
'-m', '--show_missing_characters',
|
||||
action='store_true',
|
||||
help='Show characters which were removed in detail.')
|
||||
PARSER.add_argument(
|
||||
'-c', '--show_changed_characters',
|
||||
action='store_true',
|
||||
help='Show characters whose width was changed in detail.')
|
||||
ARGS = PARSER.parse_args()
|
||||
|
||||
if ARGS.unicode_data_file:
|
||||
fill_attributes(ARGS.unicode_data_file)
|
||||
if ARGS.east_asian_width_file:
|
||||
fill_east_asian_widths(ARGS.east_asian_width_file)
|
||||
check_charmap(ARGS.old_utf8_file, ARGS.new_utf8_file)
|
||||
check_width(ARGS.old_utf8_file, ARGS.new_utf8_file)
|
||||
286
localedata/unicode-gen/utf8_gen.py
Executable file
286
localedata/unicode-gen/utf8_gen.py
Executable file
@ -0,0 +1,286 @@
|
||||
#!/usr/bin/python3
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (C) 2014, 2015 Free Software Foundation, Inc.
|
||||
# This file is part of the GNU C Library.
|
||||
#
|
||||
# The GNU C Library is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU Lesser General Public
|
||||
# License as published by the Free Software Foundation; either
|
||||
# version 2.1 of the License, or (at your option) any later version.
|
||||
#
|
||||
# The GNU C Library is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with the GNU C Library; if not, see
|
||||
# <http://www.gnu.org/licenses/>.
|
||||
|
||||
'''glibc/localedata/charmaps/UTF-8 file generator script
|
||||
|
||||
This script generates a glibc/localedata/charmaps/UTF-8 file
|
||||
from Unicode data.
|
||||
|
||||
Usage: python3 utf8_gen.py UnicodeData.txt EastAsianWidth.txt
|
||||
|
||||
It will output UTF-8 file
|
||||
'''
|
||||
|
||||
import sys
|
||||
import re
|
||||
|
||||
# Auxiliary tables for Hangul syllable names, see the Unicode 3.0 book,
|
||||
# sections 3.11 and 4.4.
|
||||
|
||||
jamo_initial_short_name = [
|
||||
'G', 'GG', 'N', 'D', 'DD', 'R', 'M', 'B', 'BB', 'S', 'SS', '', 'J', 'JJ',
|
||||
'C', 'K', 'T', 'P', 'H'
|
||||
]
|
||||
|
||||
jamo_medial_short_name = [
|
||||
'A', 'AE', 'YA', 'YAE', 'EO', 'E', 'YEO', 'YE', 'O', 'WA', 'WAE', 'OE',
|
||||
'YO', 'U', 'WEO', 'WE', 'WI', 'YU', 'EU', 'YI', 'I'
|
||||
]
|
||||
|
||||
jamo_final_short_name = [
|
||||
'', 'G', 'GG', 'GS', 'N', 'NI', 'NH', 'D', 'L', 'LG', 'LM', 'LB', 'LS',
|
||||
'LT', 'LP', 'LH', 'M', 'B', 'BS', 'S', 'SS', 'NG', 'J', 'C', 'K', 'T',
|
||||
'P', 'H'
|
||||
]
|
||||
|
||||
def ucs_symbol(code_point):
|
||||
'''Return the UCS symbol string for a Unicode character.'''
|
||||
if code_point < 0x10000:
|
||||
return '<U{:04X}>'.format(code_point)
|
||||
else:
|
||||
return '<U{:08X}>'.format(code_point)
|
||||
|
||||
def process_range(start, end, outfile, name):
|
||||
'''Writes a range of code points into the CHARMAP section of the
|
||||
output file
|
||||
|
||||
'''
|
||||
if 'Hangul Syllable' in name:
|
||||
# from glibc/localedata/ChangeLog:
|
||||
#
|
||||
# 2000-09-24 Bruno Haible <haible@clisp.cons.org>
|
||||
# * charmaps/UTF-8: Expand <Hangul Syllable> and <Private Use> ranges,
|
||||
# so they become printable and carry a width. Comment out surrogate
|
||||
# ranges. Add a WIDTH table
|
||||
#
|
||||
# So we expand the Hangul Syllables here:
|
||||
for i in range(int(start, 16), int(end, 16)+1 ):
|
||||
index2, index3 = divmod(i - 0xaC00, 28)
|
||||
index1, index2 = divmod(index2, 21)
|
||||
hangul_syllable_name = 'HANGUL SYLLABLE ' \
|
||||
+ jamo_initial_short_name[index1] \
|
||||
+ jamo_medial_short_name[index2] \
|
||||
+ jamo_final_short_name[index3]
|
||||
outfile.write('{:<11s} {:<12s} {:s}\n'.format(
|
||||
ucs_symbol(i), convert_to_hex(i),
|
||||
hangul_syllable_name))
|
||||
return
|
||||
# UnicodeData.txt file has contains code point ranges like this:
|
||||
#
|
||||
# 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
|
||||
# 4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
|
||||
#
|
||||
# The glibc UTF-8 file splits ranges like these into shorter
|
||||
# ranges of 64 code points each:
|
||||
#
|
||||
# <U3400>..<U343F> /xe3/x90/x80 <CJK Ideograph Extension A>
|
||||
# …
|
||||
# <U4D80>..<U4DB5> /xe4/xb6/x80 <CJK Ideograph Extension A>
|
||||
for i in range(int(start, 16), int(end, 16), 64 ):
|
||||
if i > (int(end, 16)-64):
|
||||
outfile.write('{:s}..{:s} {:<12s} {:s}\n'.format(
|
||||
ucs_symbol(i),
|
||||
ucs_symbol(int(end,16)),
|
||||
convert_to_hex(i),
|
||||
name))
|
||||
break
|
||||
outfile.write('{:s}..{:s} {:<12s} {:s}\n'.format(
|
||||
ucs_symbol(i),
|
||||
ucs_symbol(i+63),
|
||||
convert_to_hex(i),
|
||||
name))
|
||||
|
||||
def process_charmap(flines, outfile):
|
||||
'''This function takes an array which contains *all* lines of
|
||||
of UnicodeData.txt and write lines to outfile as used in the
|
||||
|
||||
CHARMAP
|
||||
…
|
||||
END CHARMAP
|
||||
|
||||
section of the UTF-8 file in glibc/localedata/charmaps/UTF-8.
|
||||
|
||||
Samples for input lines:
|
||||
|
||||
0010;<control>;Cc;0;BN;;;;;N;DATA LINK ESCAPE;;;;
|
||||
3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
|
||||
4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
|
||||
D800;<Non Private Use High Surrogate, First>;Cs;0;L;;;;;N;;;;;
|
||||
DB7F;<Non Private Use High Surrogate, Last>;Cs;0;L;;;;;N;;;;;
|
||||
100000;<Plane 16 Private Use, First>;Co;0;L;;;;;N;;;;;
|
||||
10FFFD;<Plane 16 Private Use, Last>;Co;0;L;;;;;N;;;;;
|
||||
|
||||
Samples for output lines (Unicode-Value UTF-8-HEX Unicode-Char-Name):
|
||||
|
||||
<U0010> /x10 DATA LINK ESCAPE
|
||||
<U3400>..<U343F> /xe3/x90/x80 <CJK Ideograph Extension A>
|
||||
%<UD800> /xed/xa0/x80 <Non Private Use High Surrogate, First>
|
||||
%<UDB7F> /xed/xad/xbf <Non Private Use High Surrogate, Last>
|
||||
<U0010FFC0>..<U0010FFFD> /xf4/x8f/xbf/x80 <Plane 16 Private Use>
|
||||
|
||||
'''
|
||||
fields_start = []
|
||||
for line in flines:
|
||||
fields = line.split(";")
|
||||
# Some characters have “<control>” as their name. We try to
|
||||
# use the “Unicode 1.0 Name” (10th field in
|
||||
# UnicodeData.txt) for them.
|
||||
#
|
||||
# The Characters U+0080, U+0081, U+0084 and U+0099 have
|
||||
# “<control>” as their name but do not even have aa
|
||||
# ”Unicode 1.0 Name”. We could write code to take their
|
||||
# alternate names from NameAliases.txt.
|
||||
if fields[1] == "<control>" and fields[10]:
|
||||
fields[1] = fields[10]
|
||||
# Handling code point ranges like:
|
||||
#
|
||||
# 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
|
||||
# 4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
|
||||
if fields[1].endswith(', First>') and not 'Surrogate,' in fields[1]:
|
||||
fields_start = fields
|
||||
continue
|
||||
if fields[1].endswith(', Last>') and not 'Surrogate,' in fields[1]:
|
||||
process_range(fields_start[0], fields[0],
|
||||
outfile, fields[1][:-7]+'>')
|
||||
fields_start = []
|
||||
continue
|
||||
fields_start = []
|
||||
if 'Surrogate,' in fields[1]:
|
||||
# Comment out the surrogates in the UTF-8 file.
|
||||
# One could of course skip them completely but
|
||||
# the original UTF-8 file in glibc had them as
|
||||
# comments, so we keep these comment lines.
|
||||
outfile.write('%')
|
||||
outfile.write('{:<11s} {:<12s} {:s}\n'.format(
|
||||
ucs_symbol(int(fields[0], 16)),
|
||||
convert_to_hex(int(fields[0], 16)),
|
||||
fields[1]))
|
||||
|
||||
def convert_to_hex(code_point):
|
||||
'''Converts a code point to a hexadecimal UTF-8 representation
|
||||
like /x**/x**/x**.'''
|
||||
# Getting UTF8 of Unicode characters.
|
||||
# In Python3, .encode('UTF-8') does not work for
|
||||
# surrogates. Therefore, we use this conversion table
|
||||
surrogates = {
|
||||
0xD800: '/xed/xa0/x80',
|
||||
0xDB7F: '/xed/xad/xbf',
|
||||
0xDB80: '/xed/xae/x80',
|
||||
0xDBFF: '/xed/xaf/xbf',
|
||||
0xDC00: '/xed/xb0/x80',
|
||||
0xDFFF: '/xed/xbf/xbf',
|
||||
}
|
||||
if code_point in surrogates:
|
||||
return surrogates[code_point]
|
||||
return ''.join([
|
||||
'/x{:02x}'.format(c) for c in chr(code_point).encode('UTF-8')
|
||||
])
|
||||
|
||||
def write_header_charmap(outfile):
|
||||
'''Write the header on top of the CHARMAP section to the output file'''
|
||||
outfile.write("<code_set_name> UTF-8\n")
|
||||
outfile.write("<comment_char> %\n")
|
||||
outfile.write("<escape_char> /\n")
|
||||
outfile.write("<mb_cur_min> 1\n")
|
||||
outfile.write("<mb_cur_max> 6\n\n")
|
||||
outfile.write("% CHARMAP generated using utf8_gen.py\n")
|
||||
outfile.write("% alias ISO-10646/UTF-8\n")
|
||||
outfile.write("CHARMAP\n")
|
||||
|
||||
def write_header_width(outfile):
|
||||
'''Writes the header on top of the WIDTH section to the output file'''
|
||||
outfile.write('% Character width according to Unicode 7.0.0.\n')
|
||||
outfile.write('% - Default width is 1.\n')
|
||||
outfile.write('% - Double-width characters have width 2; generated from\n')
|
||||
outfile.write('% "grep \'^[^;]*;[WF]\' EastAsianWidth.txt"\n')
|
||||
outfile.write('% - Non-spacing characters have width 0; '
|
||||
+ 'generated from PropList.txt or\n')
|
||||
outfile.write('% "grep \'^[^;]*;[^;]*;[^;]*;[^;]*;NSM;\' '
|
||||
+ 'UnicodeData.txt"\n')
|
||||
outfile.write('% - Format control characters have width 0; '
|
||||
+ 'generated from\n')
|
||||
outfile.write("% \"grep '^[^;]*;[^;]*;Cf;' UnicodeData.txt\"\n")
|
||||
# Not needed covered by Cf
|
||||
# outfile.write("% - Zero width characters have width 0; generated from\n")
|
||||
# outfile.write("% \"grep '^[^;]*;ZERO WIDTH ' UnicodeData.txt\"\n")
|
||||
outfile.write("WIDTH\n")
|
||||
|
||||
def process_width(outfile, ulines, elines):
|
||||
'''ulines are lines from UnicodeData.txt, elines are lines from
|
||||
EastAsianWidth.txt
|
||||
|
||||
'''
|
||||
width_dict = {}
|
||||
for line in ulines:
|
||||
fields = line.split(";")
|
||||
if fields[4] == "NSM" or fields[2] == "Cf":
|
||||
width_dict[int(fields[0], 16)] = ucs_symbol(
|
||||
int(fields[0], 16)) + '\t0'
|
||||
|
||||
for line in elines:
|
||||
# If an entry in EastAsianWidth.txt is found, it overrides entries in
|
||||
# UnicodeData.txt:
|
||||
fields = line.split(";")
|
||||
if not '..' in fields[0]:
|
||||
width_dict[int(fields[0], 16)] = ucs_symbol(
|
||||
int(fields[0], 16)) + '\t2'
|
||||
else:
|
||||
code_points = fields[0].split("..")
|
||||
for key in range(int(code_points[0], 16),
|
||||
int(code_points[1], 16)+1):
|
||||
if key in width_dict:
|
||||
del width_dict[key]
|
||||
width_dict[int(code_points[0], 16)] = '{:s}...{:s}\t2'.format(
|
||||
ucs_symbol(int(code_points[0], 16)),
|
||||
ucs_symbol(int(code_points[1], 16)))
|
||||
|
||||
for key in sorted(width_dict):
|
||||
outfile.write(width_dict[key]+'\n')
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 3:
|
||||
print("USAGE: python3 utf8_gen.py UnicodeData.txt EastAsianWidth.txt")
|
||||
else:
|
||||
with open(sys.argv[1], mode='r') as UNIDATA_FILE:
|
||||
UNICODE_DATA_LINES = UNIDATA_FILE.readlines()
|
||||
with open(sys.argv[2], mode='r') as EAST_ASIAN_WIDTH_FILE:
|
||||
EAST_ASIAN_WIDTH_LINES = []
|
||||
for LINE in EAST_ASIAN_WIDTH_FILE:
|
||||
# If characters from EastAasianWidth.txt which are from
|
||||
# from reserved ranges (i.e. not yet assigned code points)
|
||||
# are added to the WIDTH section of the UTF-8 file, then
|
||||
# “make check” produces “Unknown Character” errors for
|
||||
# these code points because such unassigned code points
|
||||
# are not in the CHARMAP section of the UTF-8 file.
|
||||
#
|
||||
# Therefore, we skip all reserved code points when reading
|
||||
# the EastAsianWidth.txt file.
|
||||
if re.match(r'.*<reserved-.+>\.\.<reserved-.+>.*', LINE):
|
||||
continue
|
||||
if re.match(r'^[^;]*;[WF]', LINE):
|
||||
EAST_ASIAN_WIDTH_LINES.append(LINE.strip())
|
||||
with open('UTF-8', mode='w') as OUTFILE:
|
||||
# Processing UnicodeData.txt and write CHARMAP to UTF-8 file
|
||||
write_header_charmap(OUTFILE)
|
||||
process_charmap(UNICODE_DATA_LINES, OUTFILE)
|
||||
OUTFILE.write("END CHARMAP\n\n")
|
||||
# Processing EastAsianWidth.txt and write WIDTH to UTF-8 file
|
||||
write_header_width(OUTFILE)
|
||||
process_width(OUTFILE, UNICODE_DATA_LINES, EAST_ASIAN_WIDTH_LINES)
|
||||
OUTFILE.write("END WIDTH\n")
|
||||
Loading…
Reference in New Issue
Block a user