388 lines
15 KiB
Java
388 lines
15 KiB
Java
/**
|
|
*******************************************************************************
|
|
* Copyright (C) 1996-2001, International Business Machines Corporation and *
|
|
* others. All Rights Reserved. *
|
|
*******************************************************************************
|
|
*
|
|
* $Source: /xsrl/Nsvn/icu/unicodetools/com/ibm/text/UCD/Compare14652.java,v $
|
|
* $Date: 2004/02/07 01:01:16 $
|
|
* $Revision: 1.3 $
|
|
*
|
|
*******************************************************************************
|
|
*/
|
|
|
|
package com.ibm.text.UCD;
|
|
|
|
import java.util.*;
|
|
import java.io.*;
|
|
|
|
import com.ibm.text.utility.*;
|
|
import com.ibm.icu.text.UTF16;
|
|
import com.ibm.icu.text.UnicodeSet;
|
|
|
|
// quick and dirty function for grabbing contents of ISO 14652 file
|
|
|
|
public class Compare14652 implements UCD_Types {
|
|
|
|
static final boolean oldVersion = false;
|
|
|
|
public static UnicodeSet getSet(int prop, byte propValue) {
|
|
return UnifiedBinaryProperty.make(prop | propValue).getSet();
|
|
}
|
|
|
|
static UnicodeSet
|
|
titleSet = getSet(CATEGORY, Lt),
|
|
combiningSet = getSet(CATEGORY, Mc)
|
|
.addAll(getSet(CATEGORY, Me))
|
|
.addAll(getSet(CATEGORY, Mn)),
|
|
zSet = getSet(CATEGORY, Zs)
|
|
.addAll(getSet(CATEGORY, Zl))
|
|
.addAll(getSet(CATEGORY, Zp)),
|
|
pSet = getSet(CATEGORY, Pd)
|
|
.addAll(getSet(CATEGORY, Ps))
|
|
.addAll(getSet(CATEGORY, Pe))
|
|
.addAll(getSet(CATEGORY, Pc))
|
|
.addAll(getSet(CATEGORY, Po))
|
|
.addAll(getSet(CATEGORY, Pi))
|
|
.addAll(getSet(CATEGORY, Pf)),
|
|
sSet = getSet(CATEGORY, Sm)
|
|
.addAll(getSet(CATEGORY, Sc))
|
|
.addAll(getSet(CATEGORY, Sk))
|
|
.addAll(getSet(CATEGORY, So)),
|
|
noSet = getSet(CATEGORY, No),
|
|
csSet = getSet(CATEGORY, Cs),
|
|
cfSet = getSet(CATEGORY, Cf),
|
|
cnSet = getSet(CATEGORY, Cn),
|
|
circled = getSet(DECOMPOSITION_TYPE, COMPAT_CIRCLE),
|
|
whitespaceSet = getSet(BINARY_PROPERTIES, White_space),
|
|
alphaSet = getSet(DERIVED, PropAlphabetic).addAll(combiningSet),
|
|
lowerSet = getSet(DERIVED, PropLowercase).addAll(titleSet).removeAll(circled),
|
|
upperSet = getSet(DERIVED, PropUppercase).addAll(titleSet).removeAll(circled),
|
|
digitSet = getSet(CATEGORY, Nd),
|
|
xdigitSet = new UnicodeSet("[a-fA-F\uFF21-\uFF26\uFF41-\uFF46]").addAll(digitSet),
|
|
spaceSet = whitespaceSet.size() == 0 ? zSet : whitespaceSet,
|
|
controlSet = getSet(CATEGORY, Cc),
|
|
punctSet = new UnicodeSet(pSet).addAll(sSet),
|
|
graphSet = new UnicodeSet(0,0x10ffff)
|
|
.removeAll(controlSet)
|
|
//.removeAll(getSet(CATEGORY, Cf))
|
|
.removeAll(csSet)
|
|
.removeAll(cnSet)
|
|
.removeAll(zSet),
|
|
// Cc, Cf, Cs, Cn, Z
|
|
blankSet = new UnicodeSet(spaceSet).removeAll(new UnicodeSet("[\\u000A-\\u000D\\u0085]"))
|
|
.removeAll(getSet(CATEGORY, Zl))
|
|
.removeAll(getSet(CATEGORY, Zp));
|
|
|
|
|
|
static class Prop {
|
|
String name;
|
|
UnicodeSet contents = new UnicodeSet();
|
|
String guess = "???";
|
|
UnicodeSet guessContents = new UnicodeSet();
|
|
|
|
String wsname = whitespaceSet.size() == 0 ? "gc=Z" : "Whitespace";
|
|
|
|
Prop(String name) {
|
|
this.name = name;
|
|
if (name.equals("alpha")) {
|
|
guess = "Alphabetic + gc=M";
|
|
guessContents = alphaSet;
|
|
} else if (name.equals("lower")) {
|
|
guess = "Lowercase + gc=Lt - dt=circle";
|
|
guessContents = lowerSet;
|
|
} else if (name.equals("upper")) {
|
|
guess = "Uppercase + gc=Lt - dt=circle";
|
|
guessContents = upperSet;
|
|
} else if (name.equals("digit")) {
|
|
guess = "gc=Nd";
|
|
guessContents = digitSet;
|
|
} else if (name.equals("xdigit")) {
|
|
guess = "gc=Nd+a..f (upper/lower,normal/fullwidth)";
|
|
guessContents = xdigitSet;
|
|
} else if (name.equals("space")) {
|
|
guess = wsname;
|
|
guessContents = spaceSet;
|
|
//Utility.showSetNames("Whitespace", spaceSet, true, Default.ucd);
|
|
} else if (name.equals("cntrl")) {
|
|
guess = "gc=Cc";
|
|
guessContents = controlSet;
|
|
} else if (name.equals("punct")) {
|
|
guess = "gc=P,S";
|
|
guessContents = punctSet;
|
|
} else if (name.equals("graph")) {
|
|
guess = "All - gc=Cc, Cs, Cn, or Z";
|
|
guessContents = graphSet;
|
|
} else if (name.equals("blank")) {
|
|
guess = wsname + " - (LF,VT,FF,CR,NEL + gc=Zl,Zp)";
|
|
guessContents = blankSet;
|
|
} else if (name.equals("ISO_14652_class \"combining\"")) {
|
|
guess = "gc=M";
|
|
guessContents = combiningSet;
|
|
}
|
|
|
|
|
|
/*upper
|
|
lower
|
|
alpha
|
|
digit
|
|
outdigit
|
|
space
|
|
cntrl
|
|
punct
|
|
graph
|
|
xdigit
|
|
blank
|
|
toupper
|
|
tolower
|
|
*/
|
|
}
|
|
|
|
void show(PrintWriter pw) {
|
|
if (name.equals("ISO_14652_LC_CTYPE")) return;
|
|
if (name.equals("ISO_14652_toupper")) return;
|
|
if (name.equals("ISO_14652_tolower")) return;
|
|
if (name.equals("ISO_14652_outdigit")) return;
|
|
if (name.equals("ISO_14652_outdigit")) return;
|
|
if (name.startsWith("ISO_14652_class")) return;
|
|
|
|
pw.println();
|
|
pw.println("**************************************************");
|
|
pw.println(name);
|
|
pw.println("**************************************************");
|
|
Utility.showSetDifferences(pw, name, contents, guess, guessContents, false, true, null, Default.ucd());
|
|
//pw.println(props[i].contents);
|
|
}
|
|
}
|
|
|
|
static Prop[] props = new Prop[100];
|
|
static int propCount = 0;
|
|
|
|
public static void main(String[] args) throws IOException {
|
|
|
|
String version = Default.ucd().getVersion();
|
|
PrintWriter log = Utility.openPrintWriter("Diff14652_" + version + ".txt", Utility.UTF8_WINDOWS);
|
|
try {
|
|
log.write('\uFEFF');
|
|
log.print("Version: " + version);
|
|
|
|
if (false) {
|
|
UnicodeSet ID = getSet(DERIVED, ID_Start).addAll(getSet(DERIVED, ID_Continue_NO_Cf));
|
|
UnicodeSet XID = getSet(DERIVED, Mod_ID_Start).addAll(getSet(DERIVED, Mod_ID_Continue_NO_Cf));
|
|
UnicodeSet alphanumSet = new UnicodeSet(alphaSet).addAll(digitSet).addAll(getSet(CATEGORY, Pc));
|
|
|
|
Utility.showSetDifferences("ID", ID, "XID", XID, false, Default.ucd());
|
|
Utility.showSetDifferences("ID", ID, "Alphabetic+Digit+Pc", alphanumSet, false, Default.ucd());
|
|
}
|
|
|
|
BufferedReader br = Utility.openReadFile("C:\\DATA\\ISO14652_CTYPE.txt", Utility.LATIN1);
|
|
while (true) {
|
|
String line = br.readLine();
|
|
if (line == null) break;
|
|
line = line.trim();
|
|
if (line.length() == 0) continue;
|
|
if (line.charAt(line.length() - 1) == '/') {
|
|
line = line.substring(0, line.length() - 1);
|
|
}
|
|
line = line.trim();
|
|
if (line.length() == 0) continue;
|
|
|
|
char ch = line.charAt(0);
|
|
if (ch == '%') continue;
|
|
if (ch == '(') continue;
|
|
if (ch == '<') {
|
|
addItems(line, props[propCount-1].contents);
|
|
} else {
|
|
// new property
|
|
System.out.println(line);
|
|
if (line.equals("width")) break;
|
|
props[propCount] = new Prop(line);
|
|
props[propCount].name = "ISO_14652_" + line;
|
|
props[propCount].contents = new UnicodeSet();
|
|
propCount++;
|
|
}
|
|
}
|
|
|
|
for (int i = 0; i < propCount; ++i) props[i].show(log);
|
|
|
|
log.println();
|
|
log.println("**************************************************");
|
|
log.println("Checking POSIX requirements for inclusion and disjointness.");
|
|
log.println("**************************************************");
|
|
log.println();
|
|
/*
|
|
alpha, digit, punct, cntrl are all disjoint
|
|
space, cntrl, blank are pairwise disjoint with any of alpha, digit, xdigit
|
|
alpha includes upper, lower
|
|
graph includes alpha, digit, punct
|
|
print includes graph
|
|
xdigit includes digit
|
|
*/
|
|
Prop
|
|
alpha = getProp("ISO_14652_alpha"),
|
|
upper = getProp("ISO_14652_upper"),
|
|
lower = getProp("ISO_14652_lower"),
|
|
graph = getProp("ISO_14652_graph"),
|
|
//print = getProp("ISO_14652_print"),
|
|
punct = getProp("ISO_14652_punct"),
|
|
digit = getProp("ISO_14652_digit"),
|
|
xdigit = getProp("ISO_14652_xdigit"),
|
|
space = getProp("ISO_14652_space"),
|
|
blank = getProp("ISO_14652_blank"),
|
|
cntrl = getProp("ISO_14652_cntrl");
|
|
|
|
checkDisjoint(log, new Prop[] {alpha, digit, punct, cntrl});
|
|
|
|
Prop [] l1 = new Prop[] {space, cntrl, blank};
|
|
Prop [] l2 = new Prop[] {alpha, digit, xdigit};
|
|
for (int i = 0; i < l1.length; ++i) {
|
|
for (int j = i + 1; j < l2.length; ++j) {
|
|
checkDisjoint(log, l1[i], l2[j]);
|
|
}
|
|
}
|
|
checkIncludes(log, alpha, upper);
|
|
checkIncludes(log, alpha, lower);
|
|
checkIncludes(log, graph, alpha);
|
|
checkIncludes(log, graph, digit);
|
|
checkIncludes(log, graph, punct);
|
|
//checkIncludes(log, print, graph);
|
|
checkIncludes(log, xdigit, digit);
|
|
|
|
|
|
// possibly alpha, digit, punct, cntrl, space cover the !(Cn,Cs)
|
|
|
|
UnicodeSet trRemainder = new UnicodeSet(cnSet)
|
|
.complement()
|
|
.removeAll(csSet)
|
|
.removeAll(digit.contents)
|
|
.removeAll(punct.contents)
|
|
.removeAll(alpha.contents)
|
|
.removeAll(cntrl.contents)
|
|
.removeAll(space.contents);
|
|
Utility.showSetNames(log, "TR Remainder: ", trRemainder, false, false, Default.ucd());
|
|
|
|
UnicodeSet propRemainder = new UnicodeSet(cnSet)
|
|
.complement()
|
|
.removeAll(csSet)
|
|
//.removeAll(noSet)
|
|
//.removeAll(cfSet)
|
|
.removeAll(digit.guessContents)
|
|
.removeAll(punct.guessContents)
|
|
.removeAll(alpha.guessContents)
|
|
.removeAll(cntrl.guessContents)
|
|
.removeAll(space.guessContents);
|
|
Utility.showSetNames(log, "Prop Remainder: ", propRemainder, false, false, Default.ucd());
|
|
|
|
/*
|
|
checkDisjoint(new Prop[] {alpha, digit, punct, cntrl});
|
|
UnicodeSet remainder = cnSet.complement();
|
|
UnicodeSet guessRemainder = new UnicodeSet(remainder);
|
|
for (int i = 0; i < list.length; ++i) {
|
|
for (int j = i + 1; j < list.length; ++j) {
|
|
compare(log, list[i].name, list[i].contents, list[j].name, list[j].contents);
|
|
compare(log, list[i].guess, list[i].guessContents, list[j].guess, list[j].guessContents);
|
|
}
|
|
remainder.removeAll(list[i].contents);
|
|
guessRemainder.removeAll(list[i].guessContents);
|
|
}
|
|
if (remainder.size() != 0) {
|
|
log.println();
|
|
log.println("Incomplete (TR): " + remainder);
|
|
}
|
|
if (guessRemainder.size() != 0) {
|
|
log.println();
|
|
log.println("Incomplete (Prop): " + guessRemainder);
|
|
}
|
|
*/
|
|
|
|
} finally {
|
|
log.close();
|
|
}
|
|
}
|
|
|
|
static void checkDisjoint(PrintWriter log, Prop[] list) {
|
|
for (int i = 0; i < list.length; ++i) {
|
|
for (int j = i + 1; j < list.length; ++j) {
|
|
checkDisjoint(log, list[i], list[j]);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void checkDisjoint(PrintWriter log, Prop prop1, Prop prop2) {
|
|
checkDisjoint(log, prop1.name, prop1.contents, prop2.name, prop2.contents);
|
|
checkDisjoint(log, prop1.guess, prop1.guessContents, prop2.guess, prop2.guessContents);
|
|
}
|
|
|
|
static void checkDisjoint(PrintWriter log, String name, UnicodeSet set, String name2, UnicodeSet set2) {
|
|
if (set.containsSome(set2)) {
|
|
log.println();
|
|
log.println("Fails test: " + name + " disjoint-with " + name2);
|
|
UnicodeSet diff = new UnicodeSet(set).retainAll(set2);
|
|
Utility.showSetNames(log, "", diff, false, false, Default.ucd());
|
|
}
|
|
}
|
|
|
|
static void checkIncludes(PrintWriter log, Prop prop1, Prop prop2) {
|
|
checkIncludes(log, prop1.name, prop1.contents, prop2.name, prop2.contents);
|
|
checkIncludes(log, prop1.guess, prop1.guessContents, prop2.guess, prop2.guessContents);
|
|
}
|
|
|
|
static void checkIncludes(PrintWriter log, String name, UnicodeSet set, String name2, UnicodeSet set2) {
|
|
if (!set.containsAll(set2)) {
|
|
log.println();
|
|
log.println("Fails test:" + name + " includes " + name2);
|
|
UnicodeSet diff = new UnicodeSet(set2).removeAll(set);
|
|
Utility.showSetNames(log, "", diff, false, false, Default.ucd());
|
|
}
|
|
}
|
|
|
|
static String[] pieces = new String[100];
|
|
|
|
// example: <U1F48>..<U1F4D>;<U1F59>;<U1F5B>;<U1F5D>;<U1F5F>;<U1F68>..<U1F6F>;/
|
|
static void addItems(String line, UnicodeSet contents) {
|
|
int len = Utility.split(line, ';', pieces);
|
|
for (int i = 0; i < len; ++i) {
|
|
String piece = pieces[i].trim();
|
|
if (piece.length() == 0) continue;
|
|
if (piece.equals("<0>")) continue;
|
|
int start, end;
|
|
int rangePoint = piece.indexOf("..");
|
|
if (rangePoint >= 0) {
|
|
start = parse(piece.substring(0,rangePoint));
|
|
end = parse(piece.substring(rangePoint+2));
|
|
} else {
|
|
start = end = parse(piece);
|
|
}
|
|
contents.add(start, end);
|
|
}
|
|
}
|
|
|
|
static int parse(String piece) {
|
|
if (!piece.startsWith("<U") || !piece.endsWith(">")) {
|
|
throw new IllegalArgumentException("Bogus code point: " + piece);
|
|
}
|
|
return Integer.parseInt(piece.substring(2,piece.length()-1), 16);
|
|
}
|
|
|
|
static Prop getProp(String name) {
|
|
//System.out.println("Searching for: " + name);
|
|
for (int i = 0; i < propCount; ++i) {
|
|
//System.out.println("Checking: " + props[i].name);
|
|
if (props[i].name.equals(name)) {
|
|
return props[i];
|
|
}
|
|
}
|
|
//System.out.println("Missed");
|
|
return null;
|
|
}
|
|
|
|
// oddities:
|
|
// extra space after ';' <U0300>..<U036F>; <U20D0>..<U20FF>; <UFE20>..<UFE2F>;/
|
|
// <0>?? <0>;<U0BE7>..<U0BEF>;/
|
|
// <U202C>; <U202D>;<U202E>; <UFEFF> : 0;/
|
|
// % "print" is by default "graph", and the <space> character
|
|
// print is odd, since it includes space but not other spaces.
|
|
// alnum not defined.
|
|
|
|
}
|