Initial checkin; creates ICU rules from ICU4J rules
X-SVN-Rev: 1442
This commit is contained in:
Alan Liu
parent
721dca33fc
commit
4576173828
238
icu4j/src/com/ibm/icu/dev/tool/translit/dumpICUrules.bat
Executable file
238
icu4j/src/com/ibm/icu/dev/tool/translit/dumpICUrules.bat
Executable file
@ -0,0 +1,238 @@
|
|||||||
|
@rem = '--*-Perl-*--
|
||||||
|
@echo off
|
||||||
|
if "%OS%" == "Windows_NT" goto WinNT
|
||||||
|
perl -x -S "%0" %1 %2 %3 %4 %5 %6 %7 %8 %9
|
||||||
|
goto endofperl
|
||||||
|
:WinNT
|
||||||
|
perl -x -S "%0" %*
|
||||||
|
if NOT "%COMSPEC%" == "%SystemRoot%\system32\cmd.exe" goto endofperl
|
||||||
|
if %errorlevel% == 9009 echo You do not have Perl in your PATH.
|
||||||
|
goto endofperl
|
||||||
|
@rem ';
|
||||||
|
#!perl
|
||||||
|
#line 14
|
||||||
|
|
||||||
|
# This perl script creates ICU transliterator data files, that live
|
||||||
|
# in icu/data, from ICU4J Java transliterator data files, in
|
||||||
|
# icu4j/src/com/ibm/text/resources.
|
||||||
|
#
|
||||||
|
# The transformation that is done is very minimal. The script assumes
|
||||||
|
# that the Java input files use only // comments (no /**/ comments)
|
||||||
|
# and that they follow a rigid format. Leading or trailing '+' (but not both)
|
||||||
|
# concatenation operators are stripped from each line.
|
||||||
|
#
|
||||||
|
# The output files are named according to ICU conventions (see NAME_MAP
|
||||||
|
# below) and created in the current directory. They should be manually
|
||||||
|
# checked and then copied into the icu/data directory. An ICU build must
|
||||||
|
# then be initiated, and the standard suite of ICU transliterator tests
|
||||||
|
# should be run after that.
|
||||||
|
#
|
||||||
|
# Alan Liu 5/19/00
|
||||||
|
|
||||||
|
if (scalar @ARGV != 1) {
|
||||||
|
usage();
|
||||||
|
}
|
||||||
|
$DIR = shift;
|
||||||
|
if (! -d $DIR) {
|
||||||
|
usage();
|
||||||
|
}
|
||||||
|
|
||||||
|
sub usage {
|
||||||
|
my $me = $0;
|
||||||
|
$me =~ s|.+[/\\]||;
|
||||||
|
print "Usage: $me <dir>\n";
|
||||||
|
print " where <dir> contains the TransliteratorRule_*.java\n";
|
||||||
|
print " files.\n";
|
||||||
|
print "e.g., $me F:/icu4j/src/com/ibm/text/resources\n";
|
||||||
|
die;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Mapping from Java IDs to ICU file names
|
||||||
|
# Copied from icu/data/translit_index.txt, with long lines folded into 1 line
|
||||||
|
$NAME_MAP = <<'END';
|
||||||
|
{ "Fullwidth-Halfwidth", "Halfwidth-Fullwidth", "fullhalf" }
|
||||||
|
{ "Latin-Arabic", "Arabic-Latin", "larabic" }
|
||||||
|
{ "Latin-Cyrillic", "Cyrillic-Latin", "lcyril" }
|
||||||
|
{ "Latin-Devanagari", "Devanagari-Latin", "ldevan" }
|
||||||
|
{ "Latin-Greek", "Greek-Latin", "lgreek" }
|
||||||
|
{ "Latin-Hebrew", "Hebrew-Latin", "lhebrew" }
|
||||||
|
{ "Latin-Jamo", "Jamo-Latin", "ljamo" }
|
||||||
|
{ "Latin-Kana", "Kana-Latin", "lkana" }
|
||||||
|
|
||||||
|
// Other miscellaneous rules
|
||||||
|
{ "StraightQuotes-CurlyQuotes", "CurlyQuotes-StraightQuotes", "quotes" }
|
||||||
|
{ "KeyboardEscape-Latin1", "", "kbdescl1" }
|
||||||
|
{ "UnicodeName-UnicodeChar", "", "ucname" }
|
||||||
|
END
|
||||||
|
|
||||||
|
foreach (split(/\n/, $NAME_MAP)) {
|
||||||
|
s|//.+||;
|
||||||
|
if (m|\"(.+)\".+\"(.*)\".+\"(.+)\".+|) {
|
||||||
|
$NAME_MAP{$1} = $3;
|
||||||
|
} elsif (/\S/) {
|
||||||
|
print STDERR "Ignoring $_\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Header blocks of text written at start of ICU output files
|
||||||
|
$HEADER1 = <<END;
|
||||||
|
//--------------------------------------------------------------------
|
||||||
|
// Copyright (c) 1999-2000, International Business Machines
|
||||||
|
// Corporation and others. All Rights Reserved.
|
||||||
|
//--------------------------------------------------------------------
|
||||||
|
// THIS IS A MACHINE-GENERATED FILE
|
||||||
|
END
|
||||||
|
$HEADER2 = <<END;
|
||||||
|
//--------------------------------------------------------------------
|
||||||
|
END
|
||||||
|
|
||||||
|
$TOOL = $0;
|
||||||
|
|
||||||
|
# Iterate over all Java RBT resource files. Process those with a mapping to
|
||||||
|
# an ICU name.
|
||||||
|
foreach (<$DIR/TransliterationRule_*.java>) {
|
||||||
|
next if (/~$/);
|
||||||
|
my $id;
|
||||||
|
if (m|TransliterationRule_(.+)\.java$|) {
|
||||||
|
$id = $1;
|
||||||
|
} else { die; }
|
||||||
|
$id =~ s/_/-/g;
|
||||||
|
if (!exists $NAME_MAP{$id}) {
|
||||||
|
print STDERR "$id: skipping, no ICU file name\n";
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
file($id, $_, $NAME_MAP{$id});
|
||||||
|
}
|
||||||
|
|
||||||
|
# Process one file
|
||||||
|
# Param: ID, e.g. Fullwidth-Halfwidth
|
||||||
|
# Param: Java input file name, e.g.
|
||||||
|
# f:/icu4j/src/com/ibm/text/resources/TransliterationRule_Fullwidth_Halfwidth.java
|
||||||
|
# Param: ICU output file name, e.g. fullhalf
|
||||||
|
sub file {
|
||||||
|
my $id = shift;
|
||||||
|
my $IN = shift;
|
||||||
|
my $out = shift;
|
||||||
|
|
||||||
|
my $OUT = "$out.txt";
|
||||||
|
|
||||||
|
# Show input size. Show output size later -- useful for quick sanity check.
|
||||||
|
print "$id (", -s $IN, ") -> $OUT (";
|
||||||
|
|
||||||
|
# Write output file header
|
||||||
|
open(OUT, ">$OUT") or die;
|
||||||
|
print OUT $HEADER1;
|
||||||
|
print OUT "// Tool: $TOOL\n// Source: $IN\n";
|
||||||
|
print OUT "// Date: ", scalar localtime, "\n";
|
||||||
|
print OUT $HEADER2;
|
||||||
|
print OUT "\n";
|
||||||
|
print OUT "// $id\n";
|
||||||
|
print OUT "\n";
|
||||||
|
print OUT "$out {\n";
|
||||||
|
print OUT " Rule {\n";
|
||||||
|
|
||||||
|
# Open input file and skip over everything before "Rule" RB key
|
||||||
|
open(IN, $IN) or die;
|
||||||
|
while (<IN>) {
|
||||||
|
last if (/\"Rule\"/);
|
||||||
|
}
|
||||||
|
|
||||||
|
# Process each line by deleting leading or trailing '+' (but not both)
|
||||||
|
# and by normalizing leading space.
|
||||||
|
|
||||||
|
# Recognize these kinds of lines:
|
||||||
|
# "9>\u0669;"+ // optional comment
|
||||||
|
# +"9>\u0669;" // optional comment
|
||||||
|
# // comment
|
||||||
|
# + "Zh>$ZH;" + "Zh<$ZH}$lower;"
|
||||||
|
# "'account of%'>\u2100", -- this occurs in a String[] resource
|
||||||
|
while (<IN>) {
|
||||||
|
last if (/^\s*\}/); # Any line starting with '}' ends the rule set
|
||||||
|
|
||||||
|
# NOTE: We have to handle a rule like this:
|
||||||
|
# "a", "b", "c",
|
||||||
|
# that fails to terminate statements with separators.
|
||||||
|
|
||||||
|
# Trim leading and trailing space
|
||||||
|
s|^\s+||;
|
||||||
|
s|\s+$||;
|
||||||
|
|
||||||
|
my $raw = $_;
|
||||||
|
|
||||||
|
# Transform escaped characters
|
||||||
|
hideEscapes();
|
||||||
|
|
||||||
|
# Process double-quoted strings
|
||||||
|
my $q;
|
||||||
|
for (;;) {
|
||||||
|
if (s|^\s*,\s*||) { # Trim leading ','
|
||||||
|
# Add separator between comma-separated rules
|
||||||
|
# if it isn't there already:
|
||||||
|
# "a>b", "c>d" -> "a>b;" "c>d"
|
||||||
|
print STDERR "Error: Can't parse \"$raw\"" unless ($q);
|
||||||
|
$q =~ s|\"$|;\"| unless ($q =~ m|;\"$|);
|
||||||
|
} else {
|
||||||
|
s|^\s*\+\s*||; # Trim leading '+'
|
||||||
|
}
|
||||||
|
if (s|^(\".*?\")||) {
|
||||||
|
$q .= ' ' if ($q);
|
||||||
|
$q .= $1;
|
||||||
|
} else {
|
||||||
|
last;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (s|^\s*,\s*||) { # Trim final ','
|
||||||
|
print STDERR "Error: Can't parse \"$raw\"" unless ($q);
|
||||||
|
$q =~ s|\"$|;\"| unless ($q =~ m|;\"$|);
|
||||||
|
} else {
|
||||||
|
s|^\s*\+\s*||; # Trim final '+'
|
||||||
|
}
|
||||||
|
|
||||||
|
# Remove and save trailing // comment
|
||||||
|
my $cmt;
|
||||||
|
if (s|^\s*(//.*)$||) {
|
||||||
|
$cmt = ' ' if ($q);
|
||||||
|
$cmt .= $1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (/\S/) {
|
||||||
|
chomp($raw);
|
||||||
|
print STDERR "Error: left over \"$_\" in \"$raw\"\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
$_ = " " . $q . $cmt . "\n";
|
||||||
|
|
||||||
|
# Restore escaped characters
|
||||||
|
restoreEscapes();
|
||||||
|
|
||||||
|
print OUT;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Finish up
|
||||||
|
close(IN);
|
||||||
|
print OUT " }\n";
|
||||||
|
print OUT "}\n";
|
||||||
|
close(OUT);
|
||||||
|
|
||||||
|
# Write output file size for sanity check
|
||||||
|
print -s $OUT, ")\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
sub hideEscapes {
|
||||||
|
# Transform escaped characters
|
||||||
|
s|\\u([a-zA-Z0-9]{4})|<<u$1>>|g; # Transform Unicode escapes
|
||||||
|
s|\\\"|<<dq>>|; # Transform backslash double quote
|
||||||
|
s|\\(.)|<<q$1>>|; # Transform backslash escapes
|
||||||
|
}
|
||||||
|
|
||||||
|
sub restoreEscapes {
|
||||||
|
# Restore escaped characters
|
||||||
|
s|<<dq>>|\\\"|g;
|
||||||
|
s|<<q(.)>>|\\$1|g;
|
||||||
|
s|<<u0000>>|\\\\u0000|g; # Double escape U+0000
|
||||||
|
s|<<u(....)>>|\\u$1|g;
|
||||||
|
}
|
||||||
|
|
||||||
|
__END__
|
||||||
|
:endofperl
|
||||||
238
icu4j/src/com/ibm/tools/translit/dumpICUrules.bat
Executable file
238
icu4j/src/com/ibm/tools/translit/dumpICUrules.bat
Executable file
@ -0,0 +1,238 @@
|
|||||||
|
@rem = '--*-Perl-*--
|
||||||
|
@echo off
|
||||||
|
if "%OS%" == "Windows_NT" goto WinNT
|
||||||
|
perl -x -S "%0" %1 %2 %3 %4 %5 %6 %7 %8 %9
|
||||||
|
goto endofperl
|
||||||
|
:WinNT
|
||||||
|
perl -x -S "%0" %*
|
||||||
|
if NOT "%COMSPEC%" == "%SystemRoot%\system32\cmd.exe" goto endofperl
|
||||||
|
if %errorlevel% == 9009 echo You do not have Perl in your PATH.
|
||||||
|
goto endofperl
|
||||||
|
@rem ';
|
||||||
|
#!perl
|
||||||
|
#line 14
|
||||||
|
|
||||||
|
# This perl script creates ICU transliterator data files, that live
|
||||||
|
# in icu/data, from ICU4J Java transliterator data files, in
|
||||||
|
# icu4j/src/com/ibm/text/resources.
|
||||||
|
#
|
||||||
|
# The transformation that is done is very minimal. The script assumes
|
||||||
|
# that the Java input files use only // comments (no /**/ comments)
|
||||||
|
# and that they follow a rigid format. Leading or trailing '+' (but not both)
|
||||||
|
# concatenation operators are stripped from each line.
|
||||||
|
#
|
||||||
|
# The output files are named according to ICU conventions (see NAME_MAP
|
||||||
|
# below) and created in the current directory. They should be manually
|
||||||
|
# checked and then copied into the icu/data directory. An ICU build must
|
||||||
|
# then be initiated, and the standard suite of ICU transliterator tests
|
||||||
|
# should be run after that.
|
||||||
|
#
|
||||||
|
# Alan Liu 5/19/00
|
||||||
|
|
||||||
|
if (scalar @ARGV != 1) {
|
||||||
|
usage();
|
||||||
|
}
|
||||||
|
$DIR = shift;
|
||||||
|
if (! -d $DIR) {
|
||||||
|
usage();
|
||||||
|
}
|
||||||
|
|
||||||
|
sub usage {
|
||||||
|
my $me = $0;
|
||||||
|
$me =~ s|.+[/\\]||;
|
||||||
|
print "Usage: $me <dir>\n";
|
||||||
|
print " where <dir> contains the TransliteratorRule_*.java\n";
|
||||||
|
print " files.\n";
|
||||||
|
print "e.g., $me F:/icu4j/src/com/ibm/text/resources\n";
|
||||||
|
die;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Mapping from Java IDs to ICU file names
|
||||||
|
# Copied from icu/data/translit_index.txt, with long lines folded into 1 line
|
||||||
|
$NAME_MAP = <<'END';
|
||||||
|
{ "Fullwidth-Halfwidth", "Halfwidth-Fullwidth", "fullhalf" }
|
||||||
|
{ "Latin-Arabic", "Arabic-Latin", "larabic" }
|
||||||
|
{ "Latin-Cyrillic", "Cyrillic-Latin", "lcyril" }
|
||||||
|
{ "Latin-Devanagari", "Devanagari-Latin", "ldevan" }
|
||||||
|
{ "Latin-Greek", "Greek-Latin", "lgreek" }
|
||||||
|
{ "Latin-Hebrew", "Hebrew-Latin", "lhebrew" }
|
||||||
|
{ "Latin-Jamo", "Jamo-Latin", "ljamo" }
|
||||||
|
{ "Latin-Kana", "Kana-Latin", "lkana" }
|
||||||
|
|
||||||
|
// Other miscellaneous rules
|
||||||
|
{ "StraightQuotes-CurlyQuotes", "CurlyQuotes-StraightQuotes", "quotes" }
|
||||||
|
{ "KeyboardEscape-Latin1", "", "kbdescl1" }
|
||||||
|
{ "UnicodeName-UnicodeChar", "", "ucname" }
|
||||||
|
END
|
||||||
|
|
||||||
|
foreach (split(/\n/, $NAME_MAP)) {
|
||||||
|
s|//.+||;
|
||||||
|
if (m|\"(.+)\".+\"(.*)\".+\"(.+)\".+|) {
|
||||||
|
$NAME_MAP{$1} = $3;
|
||||||
|
} elsif (/\S/) {
|
||||||
|
print STDERR "Ignoring $_\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Header blocks of text written at start of ICU output files
|
||||||
|
$HEADER1 = <<END;
|
||||||
|
//--------------------------------------------------------------------
|
||||||
|
// Copyright (c) 1999-2000, International Business Machines
|
||||||
|
// Corporation and others. All Rights Reserved.
|
||||||
|
//--------------------------------------------------------------------
|
||||||
|
// THIS IS A MACHINE-GENERATED FILE
|
||||||
|
END
|
||||||
|
$HEADER2 = <<END;
|
||||||
|
//--------------------------------------------------------------------
|
||||||
|
END
|
||||||
|
|
||||||
|
$TOOL = $0;
|
||||||
|
|
||||||
|
# Iterate over all Java RBT resource files. Process those with a mapping to
|
||||||
|
# an ICU name.
|
||||||
|
foreach (<$DIR/TransliterationRule_*.java>) {
|
||||||
|
next if (/~$/);
|
||||||
|
my $id;
|
||||||
|
if (m|TransliterationRule_(.+)\.java$|) {
|
||||||
|
$id = $1;
|
||||||
|
} else { die; }
|
||||||
|
$id =~ s/_/-/g;
|
||||||
|
if (!exists $NAME_MAP{$id}) {
|
||||||
|
print STDERR "$id: skipping, no ICU file name\n";
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
file($id, $_, $NAME_MAP{$id});
|
||||||
|
}
|
||||||
|
|
||||||
|
# Process one file
|
||||||
|
# Param: ID, e.g. Fullwidth-Halfwidth
|
||||||
|
# Param: Java input file name, e.g.
|
||||||
|
# f:/icu4j/src/com/ibm/text/resources/TransliterationRule_Fullwidth_Halfwidth.java
|
||||||
|
# Param: ICU output file name, e.g. fullhalf
|
||||||
|
sub file {
|
||||||
|
my $id = shift;
|
||||||
|
my $IN = shift;
|
||||||
|
my $out = shift;
|
||||||
|
|
||||||
|
my $OUT = "$out.txt";
|
||||||
|
|
||||||
|
# Show input size. Show output size later -- useful for quick sanity check.
|
||||||
|
print "$id (", -s $IN, ") -> $OUT (";
|
||||||
|
|
||||||
|
# Write output file header
|
||||||
|
open(OUT, ">$OUT") or die;
|
||||||
|
print OUT $HEADER1;
|
||||||
|
print OUT "// Tool: $TOOL\n// Source: $IN\n";
|
||||||
|
print OUT "// Date: ", scalar localtime, "\n";
|
||||||
|
print OUT $HEADER2;
|
||||||
|
print OUT "\n";
|
||||||
|
print OUT "// $id\n";
|
||||||
|
print OUT "\n";
|
||||||
|
print OUT "$out {\n";
|
||||||
|
print OUT " Rule {\n";
|
||||||
|
|
||||||
|
# Open input file and skip over everything before "Rule" RB key
|
||||||
|
open(IN, $IN) or die;
|
||||||
|
while (<IN>) {
|
||||||
|
last if (/\"Rule\"/);
|
||||||
|
}
|
||||||
|
|
||||||
|
# Process each line by deleting leading or trailing '+' (but not both)
|
||||||
|
# and by normalizing leading space.
|
||||||
|
|
||||||
|
# Recognize these kinds of lines:
|
||||||
|
# "9>\u0669;"+ // optional comment
|
||||||
|
# +"9>\u0669;" // optional comment
|
||||||
|
# // comment
|
||||||
|
# + "Zh>$ZH;" + "Zh<$ZH}$lower;"
|
||||||
|
# "'account of%'>\u2100", -- this occurs in a String[] resource
|
||||||
|
while (<IN>) {
|
||||||
|
last if (/^\s*\}/); # Any line starting with '}' ends the rule set
|
||||||
|
|
||||||
|
# NOTE: We have to handle a rule like this:
|
||||||
|
# "a", "b", "c",
|
||||||
|
# that fails to terminate statements with separators.
|
||||||
|
|
||||||
|
# Trim leading and trailing space
|
||||||
|
s|^\s+||;
|
||||||
|
s|\s+$||;
|
||||||
|
|
||||||
|
my $raw = $_;
|
||||||
|
|
||||||
|
# Transform escaped characters
|
||||||
|
hideEscapes();
|
||||||
|
|
||||||
|
# Process double-quoted strings
|
||||||
|
my $q;
|
||||||
|
for (;;) {
|
||||||
|
if (s|^\s*,\s*||) { # Trim leading ','
|
||||||
|
# Add separator between comma-separated rules
|
||||||
|
# if it isn't there already:
|
||||||
|
# "a>b", "c>d" -> "a>b;" "c>d"
|
||||||
|
print STDERR "Error: Can't parse \"$raw\"" unless ($q);
|
||||||
|
$q =~ s|\"$|;\"| unless ($q =~ m|;\"$|);
|
||||||
|
} else {
|
||||||
|
s|^\s*\+\s*||; # Trim leading '+'
|
||||||
|
}
|
||||||
|
if (s|^(\".*?\")||) {
|
||||||
|
$q .= ' ' if ($q);
|
||||||
|
$q .= $1;
|
||||||
|
} else {
|
||||||
|
last;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (s|^\s*,\s*||) { # Trim final ','
|
||||||
|
print STDERR "Error: Can't parse \"$raw\"" unless ($q);
|
||||||
|
$q =~ s|\"$|;\"| unless ($q =~ m|;\"$|);
|
||||||
|
} else {
|
||||||
|
s|^\s*\+\s*||; # Trim final '+'
|
||||||
|
}
|
||||||
|
|
||||||
|
# Remove and save trailing // comment
|
||||||
|
my $cmt;
|
||||||
|
if (s|^\s*(//.*)$||) {
|
||||||
|
$cmt = ' ' if ($q);
|
||||||
|
$cmt .= $1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (/\S/) {
|
||||||
|
chomp($raw);
|
||||||
|
print STDERR "Error: left over \"$_\" in \"$raw\"\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
$_ = " " . $q . $cmt . "\n";
|
||||||
|
|
||||||
|
# Restore escaped characters
|
||||||
|
restoreEscapes();
|
||||||
|
|
||||||
|
print OUT;
|
||||||
|
}
|
||||||
|
|
||||||
|
# Finish up
|
||||||
|
close(IN);
|
||||||
|
print OUT " }\n";
|
||||||
|
print OUT "}\n";
|
||||||
|
close(OUT);
|
||||||
|
|
||||||
|
# Write output file size for sanity check
|
||||||
|
print -s $OUT, ")\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
sub hideEscapes {
|
||||||
|
# Transform escaped characters
|
||||||
|
s|\\u([a-zA-Z0-9]{4})|<<u$1>>|g; # Transform Unicode escapes
|
||||||
|
s|\\\"|<<dq>>|; # Transform backslash double quote
|
||||||
|
s|\\(.)|<<q$1>>|; # Transform backslash escapes
|
||||||
|
}
|
||||||
|
|
||||||
|
sub restoreEscapes {
|
||||||
|
# Restore escaped characters
|
||||||
|
s|<<dq>>|\\\"|g;
|
||||||
|
s|<<q(.)>>|\\$1|g;
|
||||||
|
s|<<u0000>>|\\\\u0000|g; # Double escape U+0000
|
||||||
|
s|<<u(....)>>|\\u$1|g;
|
||||||
|
}
|
||||||
|
|
||||||
|
__END__
|
||||||
|
:endofperl
|
||||||
Loading…
Reference in New Issue
Block a user