/**
*******************************************************************************
* Copyright (C) 1996-2001, International Business Machines Corporation and *
* others. All Rights Reserved. *
*******************************************************************************
*
* $Source: /xsrl/Nsvn/icu/unicodetools/com/ibm/text/UCA/WriteCharts.java,v $
* $Date: 2003/08/22 16:51:21 $
* $Revision: 1.17 $
*
*******************************************************************************
*/
package com.ibm.text.UCA;
import java.util.*;
import java.io.*;
import com.ibm.text.UCD.*;
import com.ibm.text.utility.*;
import com.ibm.icu.text.UTF16;
import com.ibm.icu.text.UnicodeSetIterator;
import com.ibm.icu.text.Transliterator;
import com.ibm.icu.text.UnicodeSet;
import java.text.NumberFormat;
import java.text.SimpleDateFormat;
public class WriteCharts implements UCD_Types {
static boolean HACK_KANA = false;
static public void special() {
Default.setUCD();
for (int i = 0xE000; i < 0x10000; ++i) {
if (!Default.ucd.isRepresented(i)) continue;
if (!Default.nfkc.isNormalized(i)) continue;
System.out.println(Default.ucd.getCodeAndName(i));
}
}
static public void collationChart(UCA uca) throws IOException {
Default.setUCD(uca.getUCDVersion());
HACK_KANA = true;
uca.setAlternate(UCA.NON_IGNORABLE);
//Normalizer nfd = new Normalizer(Normalizer.NFD);
//Normalizer nfc = new Normalizer(Normalizer.NFC);
UCA.UCAContents cc = uca.getContents(UCA.FIXED_CE, null); // nfd instead of null if skipping decomps
cc.enableSamples();
Set set = new TreeSet();
while (true) {
String x = cc.next();
if (x == null) break;
if (x.equals("\u2F00")) {
System.out.println("debug");
}
set.add(new Pair(uca.getSortKey(x), x));
}
PrintWriter output = null;
Iterator it = set.iterator();
byte oldScript = -127;
int[] scriptCount = new int[128];
int counter = 0;
String lastSortKey = "\u0000";
int high = uca.getSortKey("a").charAt(0);
int variable = UCA.getPrimary(uca.getVariableHigh());
int columnCount = 0;
String[] replacement = new String[] {"%%%", "Collation Charts"};
String folder = "charts\\uca\\";
Utility.copyTextFile("index.html", Utility.UTF8, folder + "index.html", replacement);
Utility.copyTextFile("charts.css", Utility.LATIN1, folder + "charts.css");
Utility.copyTextFile("help.html", Utility.UTF8, folder + "help.html");
indexFile = Utility.openPrintWriter(folder + "index_list.html", Utility.UTF8_WINDOWS);
Utility.appendFile("index_header.html", Utility.UTF8, indexFile, replacement);
/*
indexFile.println("UCA Default Collation Table ");
indexFile.println("UCA Default Collation Table ");
indexFile.println("Help ");
*/
while (it.hasNext()) {
Utility.dot(counter);
Pair p = (Pair) it.next();
String sortKey = (String) p.first;
String s = (String) p.second;
int cp = UTF16.charAt(s,0);
byte script = Default.ucd.getScript(cp);
// get first non-zero primary
int currentPrimary = getFirstPrimary(sortKey);
int primary = currentPrimary >>> 16;
if (sortKey.length() < 4) script = NULL_ORDER;
else if (primary == 0) script = IGNORABLE_ORDER;
else if (primary <= variable) script = VARIABLE_ORDER;
else if (primary < high) script = COMMON_SCRIPT;
else if (UCA.isImplicitLeadPrimary(primary)) {
if (primary < UCA_Types.UNSUPPORTED_CJK_AB_BASE) script = CJK;
else if (primary < UCA_Types.UNSUPPORTED_OTHER_BASE) script = CJK_AB;
else script = UNSUPPORTED;
}
if (script == KATAKANA_SCRIPT) script = HIRAGANA_SCRIPT;
else if ((script == INHERITED_SCRIPT || script == COMMON_SCRIPT) && oldScript >= 0) script = oldScript;
if (script != oldScript
// && (script != COMMON_SCRIPT && script != INHERITED_SCRIPT)
) {
closeFile(output);
output = null;
oldScript = script;
}
if (output == null) {
++scriptCount[script+3];
if (scriptCount[script+3] > 1) {
System.out.println("\t\tFAIL: " + scriptCount[script+3] + ", " +
getChunkName(script, LONG) + ", " + Default.ucd.getCodeAndName(s));
}
output = openFile(scriptCount[script+3], folder, script);
}
boolean firstPrimaryEquals = currentPrimary == getFirstPrimary(lastSortKey);
int strength = uca.strengthDifference(sortKey, lastSortKey);
if (strength < 0) strength = -strength;
lastSortKey = sortKey;
// find out if this is an expansion: more than one primary weight
int primaryCount = 0;
for (int i = 0; i < sortKey.length(); ++i) {
char w = sortKey.charAt(i);
if (w == 0) break;
if (UCA.isImplicitLeadPrimary(w)) {
++i; // skip next
}
++ primaryCount;
}
String breaker = "";
if (columnCount > 10 || !firstPrimaryEquals) {
columnCount = 0;
if (!firstPrimaryEquals || script == UNSUPPORTED) breaker = "
";
else {
breaker = " "
+ Utility.hex(s)
//+ " "
+ (script == UNSUPPORTED
? "" + Utility.quoteXML(name, true) + " UCA Default Collation Table ");
indexFile.println("UCA Default Collation Table ");
indexFile.println("Help ");
*/
while (it.hasNext()) {
Utility.dot(counter);
Pair p = (Pair) it.next();
int script = ((Integer) p.first).intValue();
int cp = ((Integer)((Pair) p.second).second).intValue();
if (script != oldScript
// && (script != COMMON_SCRIPT && script != INHERITED_SCRIPT)
) {
closeFile(output);
output = null;
oldScript = script;
}
if (output == null) {
output = openFile(0, folder, script);
output.println("
Code C D KC KD ");
String prefix;
String code = UTF16.valueOf(cp);
String c = Default.nfc.normalize(cp);
String d = Default.nfd.normalize(cp);
String kc = Default.nfkc.normalize(cp);
String kd = Default.nfkd.normalize(cp);
showCell(output, code, "");
}
closeFile(output);
closeIndexFile(indexFile, "", NORMALIZATION);
}
static public void caseChart() throws IOException {
Default.setUCD();
HACK_KANA = false;
Set set = new TreeSet();
for (int i = 0; i <= 0x10FFFF; ++i) {
if (!Default.ucd.isRepresented(i)) continue;
byte cat = Default.ucd.getCategory(i);
if (cat == Cs || cat == Co) continue;
String code = UTF16.valueOf(i);
String lower = Default.ucd.getCase(i, FULL, LOWER);
String title = Default.ucd.getCase(i, FULL, TITLE);
String upper = Default.ucd.getCase(i, FULL, UPPER);
String fold = Default.ucd.getCase(i, FULL, FOLD);
String decomp = Default.nfkd.normalize(i);
int script = 0;
if (lower.equals(code) && upper.equals(code) && fold.equals(code) && title.equals(code)) {
if (!containsCase(decomp)) continue;
script = NO_CASE_MAPPING;
}
if (script == 0) script = getBestScript(decomp);
set.add(new Pair(new Integer(script == COMMON_SCRIPT ? cat + CAT_OFFSET : script),
new Pair(Default.ucd.getCase(decomp, FULL, FOLD),
new Integer(i))));
}
PrintWriter output = null;
Iterator it = set.iterator();
int oldScript = -127;
int counter = 0;
String[] replacement = new String[] {"%%%", "Case Charts"};
String folder = "charts\\case\\";
Utility.copyTextFile("index.html", Utility.UTF8, folder + "index.html", replacement);
Utility.copyTextFile("charts.css", Utility.LATIN1, folder + "charts.css");
Utility.copyTextFile("case_help.html", Utility.UTF8, folder + "help.html");
indexFile = Utility.openPrintWriter(folder + "index_list.html", Utility.UTF8_WINDOWS);
Utility.appendFile("index_header.html", Utility.UTF8, indexFile, replacement);
/*
indexFile.println("UCA Default Collation Table ");
indexFile.println("UCA Default Collation Table ");
indexFile.println("Help ");
*/
int columnCount = 0;
while (it.hasNext()) {
Utility.dot(counter);
Pair p = (Pair) it.next();
int script = ((Integer) p.first).intValue();
int cp = ((Integer)((Pair) p.second).second).intValue();
if (script != oldScript
// && (script != COMMON_SCRIPT && script != INHERITED_SCRIPT)
) {
closeFile(output);
output = null;
oldScript = script;
}
if (output == null) {
output = openFile(0, folder, script);
if (script == NO_CASE_MAPPING) output.println("
");
else output.println(" Code Lower Title "
+"Upper Fold ");
columnCount = 0;
}
showCell(output, UTF16.valueOf(cp), "");
String prefix;
String code = UTF16.valueOf(cp);
String lower = Default.ucd.getCase(cp, FULL, LOWER);
String title = Default.ucd.getCase(cp, FULL, TITLE);
String upper = Default.ucd.getCase(cp, FULL, UPPER);
String fold = Default.ucd.getCase(cp, FULL, FOLD);
showCell(output, code, " ");
}
closeFile(output);
closeIndexFile(indexFile, "", CASE);
}
static public void scriptChart() throws IOException {
Default.setUCD();
HACK_KANA = false;
Set set = new TreeSet();
for (int i = 0; i <= 0x10FFFF; ++i) {
if (!Default.ucd.isRepresented(i)) continue;
byte cat = Default.ucd.getCategory(i);
if (cat == Cs || cat == Co || cat == Cn) continue;
String code = UTF16.valueOf(i);
String decomp = Default.nfkd.normalize(i);
int script = getBestScript(decomp);
set.add(new Pair(new Integer(script == COMMON_SCRIPT ? cat + CAT_OFFSET : script),
new Pair(decomp,
new Integer(i))));
}
PrintWriter output = null;
Iterator it = set.iterator();
int oldScript = -127;
int counter = 0;
String[] replacement = new String[] {"%%%", "Script Charts"};
String folder = "charts\\script\\";
Utility.copyTextFile("index.html", Utility.UTF8, folder + "index.html", replacement);
Utility.copyTextFile("charts.css", Utility.LATIN1, folder + "charts.css");
Utility.copyTextFile("script_help.html", Utility.UTF8, folder + "help.html");
indexFile = Utility.openPrintWriter(folder + "index_list.html", Utility.UTF8_WINDOWS);
Utility.appendFile("script_index_header.html", Utility.UTF8, indexFile, replacement);
/*
indexFile.println("UCA Default Collation Table ");
indexFile.println("UCA Default Collation Table ");
indexFile.println("Help ");
*/
int columnCount = 0;
while (it.hasNext()) {
Utility.dot(counter);
Pair p = (Pair) it.next();
int script = ((Integer) p.first).intValue();
int cp = ((Integer)((Pair)p.second).second).intValue();
if (script != oldScript
// && (script != COMMON_SCRIPT && script != INHERITED_SCRIPT)
) {
closeFile(output);
output = null;
oldScript = script;
columnCount = 0;
}
if (output == null) {
output = openFile(0, folder, script);
}
if (columnCount > 10) {
output.println("
");
columnCount = 0;
}
showCell(output, UTF16.valueOf(cp), "" + key + " ");
columnCount = 1;
Iterator sublist = chars.iterator();
while (sublist.hasNext()) {
String ch = (String) sublist.next();
if (columnCount > 10) {
output.println(" ");
}
closeFile(output);
closeIndexFile(indexFile, "", CASE);
}
static void showCell(PrintWriter output, String s, String prefix, String extra, boolean skipName) {
String name = Default.ucd.getName(s);
String comp = Default.nfc.normalize(s);
String outline = prefix
+ (skipName ? "" : " title='" + Utility.quoteXML(name, true) + "'")
+ extra + ">"
+ Utility.quoteXML(comp, true)
+ ""
+ Utility.hex(s)
//+ " ";
output.println(outline);
}
static byte getBestScript(String s) {
int cp;
byte result = COMMON_SCRIPT;
for (int i = 0; i < s.length(); i += UTF16.getCharCount(cp)) {
cp = UTF16.charAt(s, i);
result = Default.ucd.getScript(cp);
if (result != COMMON_SCRIPT && result != INHERITED_SCRIPT) return result;
}
return COMMON_SCRIPT;
}
static int getFirstPrimary(String sortKey) {
int result = sortKey.charAt(0);
if (UCA.isImplicitLeadPrimary(result)) {
return (result << 16) | sortKey.charAt(1);
}
return (result << 16);
}
static final String[] CLASSNAME = {
"" + scriptName + "");
String title = "UCA: " + scriptName;
output.println("" + title + " ");
output.println("" + scriptName + " ");
output.println("");
return output;
}
static PrintWriter openFile2(int count, String directory, String name) throws IOException {
String fileName = "chart_" + name + (count > 1 ? count + "" : "") + ".html";
PrintWriter output = Utility.openPrintWriter(directory + fileName, Utility.UTF8_WINDOWS);
Utility.fixDot();
System.out.println("Writing: " + name);
indexFile.println(" " + name + " ");
String title = name;
output.println("" + title + " ");
output.println("");
return output;
}
static final int
NULL_ORDER = -3,
IGNORABLE_ORDER = -2,
VARIABLE_ORDER = -1,
// scripts in here
CJK = 120,
CJK_AB = 121,
UNSUPPORTED = 122,
CAT_OFFSET = 128,
// categories in here
NO_CASE_MAPPING = 200;
static String getChunkName(int script, byte length) {
switch(script) {
case NO_CASE_MAPPING: return "NoCaseMapping";
case NULL_ORDER: return "Null";
case IGNORABLE_ORDER: return "Ignorable";
case VARIABLE_ORDER: return "Variable";
case CJK: return "CJK";
case CJK_AB: return "CJK-Extensions";
case UNSUPPORTED: return "Unsupported";
default:
if (script >= CAT_OFFSET) return Default.ucd.getCategoryID_fromIndex((byte)(script - CAT_OFFSET), length);
else if (script == HIRAGANA_SCRIPT && HACK_KANA) return length == SHORT ? "Kata-Hira" : "Katakana-Hiragana";
else return Default.ucd.getCase(Default.ucd.getScriptID_fromIndex((byte)script, length), FULL, TITLE);
}
}
static void closeFile(PrintWriter output) {
if (output == null) return;
output.println("
");
output.close();
}
static final byte COLLATION = 0, NORMALIZATION = 1, CASE = 2;
static void closeIndexFile(PrintWriter indexFile, String extra, byte choice) {
SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss");
df.setTimeZone(TimeZone.getTimeZone("GMT"));
indexFile.println("");
boolean gotOne = false;
if (choice != COLLATION) {
indexFile.println("Collation Charts ");
gotOne = true;
}
if (choice != NORMALIZATION) {
if (gotOne) indexFile.println("Normalization Charts ");
gotOne = true;
}
if (choice != CASE) {
if (gotOne) indexFile.println("Case Charts ");
gotOne = true;
}
indexFile.println("
");
indexFile.println("UCD: " + Default.ucd.getVersion() + extra);
indexFile.println("MED ");
indexFile.println("
");
indexFile.close();
}
static boolean containsCase(String s) {
int cp;
for (int i = 0; i < s.length(); i += UTF16.getCharCount(cp)) {
cp = UTF16.charAt(s, i);
// contains Lu, Lo, Lt, or Lowercase or Uppercase
byte cat = Default.ucd.getCategory(cp);
if (cat == Lu || cat == Ll || cat == Lt) return true;
if (Default.ucd.getBinaryProperty(cp, Other_Lowercase)) return true;
if (Default.ucd.getBinaryProperty(cp, Other_Uppercase)) return true;
}
return false;
}
static final Transliterator addCircle = Transliterator.createFromRules(
"any-addCircle", "([[:Mn:][:Me:]]) > \u25CC $1", Transliterator.FORWARD);
public static void writeCompositionChart() throws IOException {
Default.setUCD();
UCA uca = new UCA(null,"");
Set letters = new TreeSet();
Set marks = new TreeSet(uca);
Set totalMarks = new TreeSet(uca);
Map decomposes = new HashMap();
Set notPrinted = new TreeSet(new UTF16.StringComparator());
Set printed = new HashSet();
// UnicodeSet latin = new UnicodeSet("[:latin:]");
PrintWriter out = Utility.openPrintWriter("composition_chart.html", Utility.UTF8_WINDOWS);
try {
out.println("Composites ");
UnicodeSetIterator it = new UnicodeSetIterator();
for (byte script = 0; script < UCD_Types.LIMIT_SCRIPT; ++script) {
String scriptName = "";
try {
scriptName = Default.ucd.getScriptID_fromIndex(script);
Utility.fixDot();
System.out.println(scriptName);
} catch (IllegalArgumentException e) {
System.out.println("Failed to create transliterator for: " + scriptName + "(" + script + ")");
continue;
}
letters.clear();
letters.add(""); // header row
marks.clear();
notPrinted.clear();
printed.clear();
for (int cp = 0; cp < 0x10FFFF; ++cp) {
byte type = Default.ucd.getCategory(cp);
if (type == Default.ucd.UNASSIGNED || type == Default.ucd.PRIVATE_USE) continue; // skip chaff
Utility.dot(cp);
byte newScript = Default.ucd.getScript(cp);
if (newScript != script) continue;
String source = UTF16.valueOf(cp);
String decomp = Default.nfd.normalize(source);
if (decomp.equals(source)) continue;
// pick up all decompositions
int count = UTF16.getCharCount(UTF16.charAt(decomp, 0));
if (count == decomp.length()) {
notPrinted.add(source);
continue; // skip unless marks
}
if (UCD.isHangulSyllable(cp)) count = 2;
String first = decomp.substring(0, count);
String second = decomp.substring(count);
//if (!markSet.containsAll(second)) continue; // skip unless marks
letters.add(first);
marks.add(second);
Utility.addToSet(decomposes, decomp, source);
notPrinted.add(source);
if (source.equals("\u212b")) System.out.println("A-RING!");
}
if (marks.size() != 0) {
totalMarks.addAll(marks);
out.println("");
out.println("" + scriptName + " ");
Iterator it2 = letters.iterator();
while (it2.hasNext()) {
String let = (String)it2.next();
out.println("" + showCell(Default.nfc.normalize(let), "class='h'"));
Iterator it3 = marks.iterator();
while (it3.hasNext()) {
String mark = (String)it3.next();
String merge = let + mark;
if (let.length() != 0 && decomposes.get(merge) == null) {
out.println(" ");
continue;
}
String comp;
try {
comp = Default.nfc.normalize(merge);
} catch (Exception e) {
System.out.println("Failed when trying to compose <" + Utility.hex(e) + ">");
continue;
}
// skip unless single char or header
/*if (let.length() != 0
&& (UTF16.countCodePoint(comp) != 1 || comp.equals(merge))) {
out.println(" ");
continue;
}
*/
Set decomps = (Set) decomposes.get(merge);
if (let.length() == 0) {
printed.add(comp);
out.println(showCell(comp, "class='h'"));
} else if (decomps.contains(comp)) {
printed.add(comp);
out.println(showCell(comp, "class='w'"));
} else {
comp = (String) new ArrayList(decomps).get(0);
printed.add(comp);
out.println(showCell(comp, "class='r'"));
}
}
out.println(" ");
}
out.println("
Other Letters Other Marks ");
//tabulate(out, atomics.iterator(),16);
//out.println(" ");
//out.println("
");
}
notPrinted.removeAll(printed);
if (notPrinted.size() != 0) {
tabulate(out, scriptName + " Excluded", notPrinted.iterator(), 24, "class='r'");
out.println("");
if (caption != null && caption.length() != 0) {
out.println("" + caption + " ");
}
while (it2.hasNext()) {
if (++count > limit) {
out.println(" ");
count = 1;
}
out.println(showCell((String)it2.next(), classType));
}
out.println("
");
}
public static String showCell(String comp, String classType) {
if (comp == null) {
return " ";
}
return "" + addCircle.transliterate(comp)
+ "" + Utility.hex(comp) + " ";
}
public static void writeAllocation() throws IOException {
Default.setUCD();
String[] names = new String[300]; // HACK, 300 is plenty for now. Fix if it ever gets larger
int[] starts = new int[names.length];
int[] ends = new int[names.length];
UCD.BlockData blockData = new UCD.BlockData();
int counter = 0;
int blockId = 0;
while (Default.ucd.getBlockData(blockId++, blockData)) {
names[counter] = blockData.name;
starts[counter] = blockData.start;
ends[counter] = blockData.end;
//System.out.println(names[counter] + ", " + values[counter]);
++counter;
// HACK
if (blockData.name.equals("Tags")) {
names[counter] = "reserved default ignorable ";
starts[counter] = 0xE0080;
ends[counter] = 0xE0FFF;
++counter;
}
}
/*
Graphic
Format
Control
Private Use
Surrogate
Noncharacter
Reserved (default ignorable)
Reserved (other)
*/
PrintWriter out = Utility.openPrintWriter("allocation.html", Utility.UTF8_WINDOWS);
try {
out.println("Unicode Allocation ");
out.println("");
for (int textOnly = 0; textOnly < 2; ++textOnly) {
out.println(""); // width='100%'
if (textOnly == 0) {
out.println("Graphic Version Start Block Name Size Textual Version (decimal) Block Name Start Total Assigned reserved ", starts[i] - lastEnd + 1, 0, "#000000", "#000000", textOnly);
}
int total = ends[i] - starts[i] + 1;
int alloc = 0;
for (int j = starts[i]; j <= ends[i]; ++j) {
if (Default.ucd.isAllocated(j)) ++alloc;
}
//System.out.println(names[i] + "\t" + alloc + "\t" + total);
String color = names[i].indexOf("Surrogates") >= 0 ? "#FF0000"
: names[i].indexOf("Private") >= 0 ? "#0000FF"
: "#00FF00";
String colorReserved = names[i].indexOf("reserved default ignorable") >= 0 ? "#CCCCCC"
: "#000000";
drawAllocation(out, starts[i], names[i], total, alloc, color, colorReserved, textOnly);
lastEnd = ends[i];
}
out.println("
");
}
out.println("Key
"
+ "Green Graphic, Control, Format, Noncharacter* code points Red Surrogate code points Blue Private Use code points Gray Reserved (default ignorable) code points Black Reserved (other) code points
code points are not necessarily assigned characters .)"
+ "");
out.println("");
} finally {
out.close();
}
}
static double longestBar = 1000;
static int longestBlock = 722402;
static NumberFormat nf = NumberFormat.getNumberInstance(Locale.US);
static {nf.setMaximumFractionDigits(0);}
static void drawAllocation(PrintWriter out, int start, String title, int total, int alloc, String color, String colorReserved, int textOnly) {
if (textOnly == 0) {
int unalloc = total - alloc;
double totalWidth = longestBar*(Math.sqrt(total) / Math.sqrt(longestBlock));
double allocWidth = alloc * totalWidth / total;
double unallocWidth = totalWidth - allocWidth;
out.println("" + Utility.hex(start)
+ "" + title
+ " ");
if (alloc != 0) out.println(" ");
if (unalloc != 0) out.println(" ");
out.println("
" + title + " " + start + " " + total + " " + alloc + "