2401 lines
92 KiB
Java
2401 lines
92 KiB
Java
/**
|
||
*******************************************************************************
|
||
* Copyright (C) 1996-2001, International Business Machines Corporation and *
|
||
* others. All Rights Reserved. *
|
||
*******************************************************************************
|
||
*
|
||
* $Source: /xsrl/Nsvn/icu/unicodetools/com/ibm/text/UCD/GenerateHanTransliterator.java,v $
|
||
* $Date: 2004/06/26 00:26:16 $
|
||
* $Revision: 1.16 $
|
||
*
|
||
*******************************************************************************
|
||
*/
|
||
|
||
package com.ibm.text.UCD;
|
||
import java.io.*;
|
||
|
||
import com.ibm.text.utility.*;
|
||
|
||
import com.ibm.icu.dev.test.util.BagFormatter;
|
||
import com.ibm.icu.dev.test.util.UnicodeMap;
|
||
import com.ibm.icu.text.Collator;
|
||
import com.ibm.icu.text.RuleBasedCollator;
|
||
import com.ibm.icu.text.Transliterator;
|
||
import com.ibm.icu.text.UnicodeSet;
|
||
import com.ibm.icu.text.UTF16;
|
||
import com.ibm.icu.text.Replaceable;
|
||
import com.ibm.icu.text.ReplaceableString;
|
||
import com.ibm.icu.text.UnicodeMatcher;
|
||
import com.ibm.icu.text.UnicodeSetIterator;
|
||
import com.ibm.icu.util.ULocale;
|
||
|
||
|
||
import java.util.*;
|
||
|
||
|
||
public final class GenerateHanTransliterator implements UCD_Types {
|
||
|
||
static final boolean DISAMBIG = false;
|
||
static final boolean DEBUG = false;
|
||
|
||
static class HanInfo {
|
||
int count = 0;
|
||
int minLen = Integer.MAX_VALUE;
|
||
int maxLen = Integer.MIN_VALUE;
|
||
int sampleLen = 0;
|
||
Set samples = new TreeSet();
|
||
Map map = new TreeMap();
|
||
}
|
||
|
||
public static void readUnihan() throws java.io.IOException {
|
||
|
||
log = Utility.openPrintWriter("Unihan_log.html", Utility.UTF8_WINDOWS);
|
||
log.println("<body>");
|
||
log.println("<head>");
|
||
log.println("<meta http-equiv='Content-Type' content='text/html; charset=utf-8'>");
|
||
log.println("<title>Unihan check</title>");
|
||
log.println("</head>");
|
||
|
||
BufferedReader in = Utility.openUnicodeFile("Unihan", Default.ucdVersion(), true, Utility.UTF8);
|
||
|
||
Map properties = new TreeMap();
|
||
|
||
Integer integerCode = new Integer(0);
|
||
int lineCounter = 0;
|
||
|
||
while (true) {
|
||
Utility.dot(++lineCounter);
|
||
|
||
String line = in.readLine();
|
||
if (line == null) break;
|
||
if (line.length() < 6) continue;
|
||
if (line.charAt(0) == '#') continue;
|
||
line = line.trim();
|
||
|
||
int tabPos = line.indexOf('\t');
|
||
String scode = line.substring(2, tabPos).trim();
|
||
|
||
int code = Integer.parseInt(scode, 16);
|
||
if (code != integerCode.intValue()) {
|
||
integerCode = new Integer(code);
|
||
}
|
||
|
||
int tabPos2 = line.indexOf('\t', tabPos+1);
|
||
String property = line.substring(tabPos+1, tabPos2).trim();
|
||
|
||
String propertyValue = line.substring(tabPos2+1).trim();
|
||
if (propertyValue.indexOf("U+") >= 0) propertyValue = fromHexUnicode.transliterate(propertyValue);
|
||
|
||
HanInfo values = (HanInfo) properties.get(property);
|
||
if (values == null) {
|
||
values = new HanInfo();
|
||
properties.put(property, values);
|
||
Utility.fixDot();
|
||
System.out.println("Property: " + property);
|
||
}
|
||
++values.count;
|
||
if (values.minLen > propertyValue.length()) values.minLen = propertyValue.length();
|
||
if (values.maxLen < propertyValue.length()) values.maxLen = propertyValue.length();
|
||
if (values.sampleLen < 150) {
|
||
String temp = scode + ":" + propertyValue;
|
||
values.sampleLen += temp.length() + 2;
|
||
values.samples.add(temp);
|
||
}
|
||
if (property.endsWith("Variant")
|
||
|| property.endsWith("Numeric")
|
||
|| property.startsWith("kRS")
|
||
|| property.equals("kTotalStrokes")) {
|
||
values.map.put(integerCode, propertyValue);
|
||
}
|
||
}
|
||
|
||
Set props = properties.keySet();
|
||
/*
|
||
log.println("Properties");
|
||
log.print(" ");
|
||
Utility.print(log, props, "\r\n ");
|
||
log.println();
|
||
log.println();
|
||
|
||
log.println("Sample Values");
|
||
*/
|
||
Iterator it = props.iterator();
|
||
log.println("<ol>");
|
||
while (it.hasNext()) {
|
||
String property = (String)it.next();
|
||
HanInfo values = (HanInfo) properties.get(property);
|
||
log.println("<li><b>" + property + "</b><ul><li>");
|
||
log.println("count: " + values.count
|
||
+ ", min length: " + values.minLen
|
||
+ ", max length: " + values.maxLen);
|
||
log.println("</li><li>samples:");
|
||
Utility.print(log, values.samples, "; ");
|
||
log.println("</li></ul></li>");
|
||
}
|
||
log.println("</ol>");
|
||
|
||
String[] list = {"kRSJapanese", "kRSKanWa", "kRSKangXi", "kRSKorean"};
|
||
Map kRSUnicodeMap = ((HanInfo) properties.get("kRSUnicode")).map;
|
||
Set redundants = new HashSet();
|
||
int unequalCount = 0;
|
||
for (int j = 0; j < list.length; ++j) {
|
||
unequalCount = 0;
|
||
log.println("<p><b>Checking Redundants for " + list[j] + "</b></p><blockquote>");
|
||
redundants.clear();
|
||
Map otherInfo = ((HanInfo) properties.get(list[j])).map;
|
||
it = otherInfo.keySet().iterator();
|
||
while (it.hasNext()) {
|
||
Integer key = (Integer) it.next();
|
||
Object ovalue = otherInfo.get(key);
|
||
Object uvalue = kRSUnicodeMap.get(key);
|
||
if (ovalue.equals(uvalue)) {
|
||
redundants.add(key);
|
||
} else if (++unequalCount < 5) {
|
||
log.println("<p>" + Integer.toString(key.intValue(),16)
|
||
+ ": <b>" + ovalue + "</b>, " + uvalue + "</p>");
|
||
}
|
||
}
|
||
log.println("</p>Total Unique: " + (otherInfo.size() - redundants.size())
|
||
+ "(out of" + otherInfo.size() + ")</p></blockquote>");
|
||
}
|
||
|
||
log.println("<p><b>Checking Redundants for kTotalStrokes</b></p><blockquote>");
|
||
|
||
// pass through first to get a count for the radicals
|
||
Map kTotalStrokesMap = ((HanInfo) properties.get("kTotalStrokes")).map;
|
||
int[] radCount = new int[512];
|
||
it = kRSUnicodeMap.keySet().iterator();
|
||
while(it.hasNext()) {
|
||
Integer key = (Integer) it.next();
|
||
String uvalue = (String) kRSUnicodeMap.get(key);
|
||
if (uvalue.endsWith(".0")) {
|
||
String tvalue = (String) kTotalStrokesMap.get(key);
|
||
if (tvalue == null) continue;
|
||
int rs = getRadicalStroke(uvalue);
|
||
radCount[rs>>8] = Integer.parseInt(tvalue);
|
||
}
|
||
}
|
||
|
||
// now compare the computed value against the real value
|
||
it = kTotalStrokesMap.keySet().iterator();
|
||
unequalCount = 0;
|
||
redundants.clear();
|
||
while(it.hasNext()) {
|
||
Integer key = (Integer) it.next();
|
||
String uvalue = (String) kRSUnicodeMap.get(key);
|
||
int rs = getRadicalStroke(uvalue);
|
||
String tvalue = (String) kTotalStrokesMap.get(key);
|
||
int t = Integer.parseInt(tvalue);
|
||
int projected = radCount[rs>>8] + (rs & 0xFF);
|
||
if (t == projected) {
|
||
redundants.add(key);
|
||
} else if (++unequalCount < 5) {
|
||
log.println("<p>" + Integer.toString(key.intValue(),16)
|
||
+ ": <b>" + t + "</b>, " + projected + "</p>");
|
||
}
|
||
}
|
||
log.println("</p>Total Unique: " + (kTotalStrokesMap.size() - redundants.size())
|
||
+ "(out of" + kTotalStrokesMap.size() + ")</p></blockquote>");
|
||
|
||
log.println("</body>");
|
||
in.close();
|
||
log.close();
|
||
}
|
||
|
||
static int getRadicalStroke(String s) {
|
||
int dotPos = s.indexOf('.');
|
||
int strokes = Integer.parseInt(s.substring(dotPos+1));
|
||
int radical = 0;
|
||
if (s.charAt(dotPos - 1) == '\'') {
|
||
radical = 256;
|
||
--dotPos;
|
||
}
|
||
radical += Integer.parseInt(s.substring(0,dotPos));
|
||
return (radical << 8) + strokes;
|
||
}
|
||
|
||
static Transliterator fromHexUnicode = Transliterator.getInstance("hex-any/unicode");
|
||
|
||
static Transliterator toHexUnicode = Transliterator.getInstance("any-hex/unicode");
|
||
|
||
/*
|
||
static String convertUPlus(String other) {
|
||
int pos1 = other.indexOf("U+");
|
||
if (pos1 < 0) return other;
|
||
return fromHexUnicode(
|
||
pos1 += 2;
|
||
|
||
StringBuffer result = new StringBuffer();
|
||
while (pos1 < other.length()) {
|
||
int end = getHexEnd(s, pos1);
|
||
result.append(UTF16.valueOf(Integer.parseInt(other.substring(pos1, end), 16)));
|
||
pos1 = other.indexOf("U+", pos1);
|
||
if (pos2 < 0) pos2 = other.length();
|
||
pos1 = pos2;
|
||
}
|
||
return result.toString();
|
||
}
|
||
|
||
static int getHexEnd(String s, int start) {
|
||
int i= start;
|
||
for (; i < s.length; ++i) {
|
||
char c = s.charAt(i);
|
||
if ('0' <= c && c <= '9') continue;
|
||
if ('A' <= c && c <= 'F') continue;
|
||
if ('a' <= c && c <= 'f') continue;
|
||
break;
|
||
}
|
||
return i;
|
||
}
|
||
*/
|
||
|
||
static final boolean TESTING = false;
|
||
static int type;
|
||
|
||
static final int CHINESE = 2, JAPANESE = 1, DEFINITION = 0;
|
||
|
||
static final boolean DO_SIMPLE = true;
|
||
static final boolean SKIP_OVERRIDES = true;
|
||
|
||
static PrintWriter out2;
|
||
|
||
public static void fixedMandarin() throws IOException {
|
||
UnicodeMap kMandarin = Default.ucd().getHanValue("kMandarin");
|
||
UnicodeMap kHanyuPinlu = Default.ucd().getHanValue("kHanyuPinlu");
|
||
UnicodeSet gotMandarin = kMandarin.getSet(null).complement();
|
||
UnicodeSet gotHanyu = kHanyuPinlu.getSet(null).complement();
|
||
UnicodeSet gotAtLeastOne = new UnicodeSet(gotMandarin).addAll(gotHanyu);
|
||
Map outmap = new TreeMap(Collator.getInstance(new ULocale("zh")));
|
||
for (UnicodeSetIterator it = new UnicodeSetIterator(gotAtLeastOne); it.next(); ) {
|
||
//String code = UTF16.valueOf(it.codepoint);
|
||
String hanyu = (String) kHanyuPinlu.getValue(it.codepoint);
|
||
String mandarin = (String) kMandarin.getValue(it.codepoint);
|
||
String hPinyin = hanyu == null ? null : digitPinyin_accentPinyin.transliterate(getUpTo(hanyu,'('));
|
||
String mPinyin = mandarin == null ? null : digitPinyin_accentPinyin.transliterate(getUpTo(mandarin.toLowerCase(),' '));
|
||
String uPinyin = hPinyin != null ? hPinyin : mPinyin;
|
||
UnicodeSet s = (UnicodeSet) outmap.get(uPinyin);
|
||
if (s == null) {
|
||
s = new UnicodeSet();
|
||
outmap.put(uPinyin, s);
|
||
}
|
||
s.add(it.codepoint);
|
||
}
|
||
String filename = "Raw_Transliterator_Han_Latin.txt";
|
||
PrintWriter out = BagFormatter.openUTF8Writer(UCD_Types.GEN_DIR, filename);
|
||
for (Iterator it = outmap.keySet().iterator(); it.hasNext();) {
|
||
String pinyin = (String) it.next();
|
||
UnicodeSet uset = (UnicodeSet) outmap.get(pinyin);
|
||
if (uset.size() == 1) {
|
||
UnicodeSetIterator usi = new UnicodeSetIterator(uset);
|
||
usi.next();
|
||
out.println(UTF16.valueOf(usi.codepoint) + ">" + pinyin + ";");
|
||
} else {
|
||
out.println(uset.toPattern(false) + ">" + pinyin + ";");
|
||
}
|
||
}
|
||
out.close();
|
||
}
|
||
|
||
public static class PairComparator implements Comparator {
|
||
Comparator first;
|
||
Comparator second;
|
||
PairComparator(Comparator first, Comparator second) {
|
||
this.first = first;
|
||
this.second = second;
|
||
}
|
||
public int compare(Object o1, Object o2) {
|
||
Pair p1 = (Pair)o1;
|
||
Pair p2 = (Pair)o2;
|
||
int result = first.compare(p1.first, p2.first);
|
||
if (result != 0) return result;
|
||
return second.compare(p1.second, p2.second);
|
||
}
|
||
}
|
||
|
||
public static void quickMandarin() throws Exception {
|
||
UnicodeMap gcl = new UnicodeMap();
|
||
addField("C:\\DATA\\dict\\", "gcl_icu.txt", 2, 3, gcl);
|
||
addField("C:\\DATA\\dict\\", "gcl_other.txt", 2, 5, gcl);
|
||
Transliterator icuPinyin = Transliterator.getInstance("han-latin");
|
||
UnicodeMap kMandarin = Default.ucd().getHanValue("kMandarin");
|
||
UnicodeMap kHanyuPinlu = Default.ucd().getHanValue("kHanyuPinlu");
|
||
UnicodeSet gotMandarin = kMandarin.getSet(null).complement();
|
||
UnicodeSet gotHanyu = kHanyuPinlu.getSet(null).complement();
|
||
UnicodeSet gotAtLeastOne = new UnicodeSet(gotMandarin).addAll(gotHanyu);
|
||
int counter = 0;
|
||
int hCount = 0;
|
||
log = Utility.openPrintWriter("Mandarin_First.txt", Utility.UTF8_WINDOWS);
|
||
log.println("N\tCode\tChar\tUnihan\tICU\tGCL\tkHanyuPinlu / kMandarin");
|
||
UnicodeMap reformed = new UnicodeMap();
|
||
for (UnicodeSetIterator it = new UnicodeSetIterator(gotAtLeastOne); it.next(); ) {
|
||
String code = UTF16.valueOf(it.codepoint);
|
||
String hanyu = (String) kHanyuPinlu.getValue(it.codepoint);
|
||
String mandarin = (String) kMandarin.getValue(it.codepoint);
|
||
String hPinyin = hanyu == null ? null : digitPinyin_accentPinyin.transliterate(getUpTo(hanyu,'('));
|
||
String mPinyin = mandarin == null ? null : digitPinyin_accentPinyin.transliterate(getUpTo(mandarin.toLowerCase(),' '));
|
||
String uPinyin = hPinyin != null ? hPinyin : mPinyin;
|
||
|
||
String iPinyin = icuPinyin.transliterate(code).trim();
|
||
if (iPinyin.equals(code)) iPinyin = null;
|
||
String gPinyin = (String) gcl.getValue(it.codepoint);
|
||
|
||
if (hPinyin != null) reformed.put(it.codepoint, hPinyin);
|
||
else if (gPinyin != null) reformed.put(it.codepoint, gPinyin);
|
||
else if (mPinyin != null) reformed.put(it.codepoint, mPinyin);
|
||
else if (iPinyin != null) reformed.put(it.codepoint, iPinyin);
|
||
|
||
if (gPinyin != null && !gPinyin.equals(uPinyin)) {
|
||
log.println((++counter) + "\t" + Utility.hex(it.codepoint) + "\t" + code
|
||
+ "\t" + (uPinyin == null ? "" : uPinyin)
|
||
+ "\t" + (iPinyin == null ? "" : iPinyin.equals(gPinyin) ? "" : iPinyin)
|
||
+ "\t" + (gPinyin == null ? "" : gPinyin)
|
||
+ "\t" + (hanyu == null ? "" : hanyu + " / ")
|
||
+ (mandarin == null ? "" : mandarin)
|
||
);
|
||
if (hanyu != null) hCount++;
|
||
continue;
|
||
}
|
||
if (true) continue;
|
||
if (isEqualOrNull(uPinyin, iPinyin)) continue;
|
||
log.println((++counter) + "\t" + Utility.hex(it.codepoint) + "\t" + code
|
||
+ "\t" + (uPinyin == null ? "" : uPinyin)
|
||
+ "\t" + (iPinyin == null ? "" : iPinyin)
|
||
+ "\t" + (gPinyin == null ? "" : gPinyin)
|
||
+ "\t" + (hanyu == null ? "" : hanyu + " / ")
|
||
+ (mandarin == null ? "" : mandarin)
|
||
);
|
||
}
|
||
log.println("kHanyuPinlu count: " + hCount);
|
||
|
||
Collator col = Collator.getInstance(new Locale("zh","","PINYIN"));
|
||
UnicodeSet tailored = col.getTailoredSet().addAll(gotAtLeastOne);
|
||
Collator pinyinCollator = new RuleBasedCollator(
|
||
"&[before 1] a < \u0101 <<< \u0100 << \u00E1 <<< \u00C1 << \u01CE <<< \u01CD << \u00E0 <<< \u00C0 << a <<< A" +
|
||
"&[before 1] e < \u0113 <<< \u0112 << \u00E9 <<< \u00C9 << \u011B <<< \u011A << \u00E8 <<< \u00C8 << e <<< A" +
|
||
"&[before 1] i < \u012B <<< \u012A << \u00ED <<< \u00CD << \u01D0 <<< \u01CF << \u00EC <<< \u00CC << i <<< I" +
|
||
"&[before 1] o < \u014D <<< \u014C << \u00F3 <<< \u00D3 << \u01D2 <<< \u01D1 << \u00F2 <<< \u00D2 << o <<< O" +
|
||
"&[before 1] u < \u016B <<< \u016A << \u00FA <<< \u00DA << \u01D4 <<< \u01D3 << \u00F9 <<< \u00D9 << u <<< U" +
|
||
" << \u01D6 <<< \u01D5 << \u01D8 <<< \u01D7 << \u01DA <<< \u01D9 << \u01DC <<< \u01DB << \u00FC");
|
||
printSortedChars("ICU_Pinyin_Sort.txt", col, tailored, reformed, kHanyuPinlu, kMandarin, pinyinCollator);
|
||
/*
|
||
MultiComparator mcol = new MultiComparator(new Comparator[] {
|
||
new UnicodeMapComparator(reformed, pinyinCollator), col});
|
||
printSortedChars("ICU_Pinyin_Sort2.txt", mcol, tailored);
|
||
*/
|
||
log.close();
|
||
}
|
||
|
||
static class UnicodeMapComparator implements Comparator {
|
||
UnicodeMap map;
|
||
Comparator comp;
|
||
UnicodeMapComparator(UnicodeMap map, Comparator comp) {
|
||
this.map = map;
|
||
this.comp = comp;
|
||
}
|
||
public int compare(Object o1, Object o2) {
|
||
int c1 = UTF16.charAt((String) o1,0);
|
||
int c2 = UTF16.charAt((String) o2,0);
|
||
Object v1 = map.getValue(c1);
|
||
Object v2 = map.getValue(c2);
|
||
if (v1 == null) {
|
||
if (v2 == null) return 0;
|
||
return -1;
|
||
} else if (v2 == null) return 1;
|
||
return comp.compare(v1, v2);
|
||
}
|
||
}
|
||
|
||
static class MultiComparator implements Comparator {
|
||
private Comparator[] comparators;
|
||
|
||
public MultiComparator (Comparator[] comparators) {
|
||
this.comparators = comparators;
|
||
}
|
||
|
||
/* Lexigraphic compare. Returns the first difference
|
||
* @return zero if equal. Otherwise +/- (i+1)
|
||
* where i is the index of the first comparator finding a difference
|
||
* @see java.util.Comparator#compare(java.lang.Object, java.lang.Object)
|
||
*/
|
||
public int compare(Object arg0, Object arg1) {
|
||
for (int i = 0; i < comparators.length; ++i) {
|
||
int result = comparators[i].compare(arg0, arg1);
|
||
if (result == 0) continue;
|
||
if (result > 0) return i+1;
|
||
return -(i+1);
|
||
}
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
private static void printSortedChars(String file, Comparator col, UnicodeSet tailored,
|
||
UnicodeMap map, UnicodeMap hanyu, UnicodeMap mand, Comparator p2)
|
||
throws IOException {
|
||
Set set = new TreeSet(col);
|
||
PrintWriter pw = Utility.openPrintWriter(file, Utility.UTF8_WINDOWS);
|
||
for (UnicodeSetIterator it = new UnicodeSetIterator(tailored); it.next(); ) {
|
||
set.add(UTF16.valueOf(it.codepoint));
|
||
}
|
||
String lastm = "";
|
||
String lasts = "";
|
||
for (Iterator it2 = set.iterator(); it2.hasNext(); ) {
|
||
String s = (String)it2.next();
|
||
String m = map == null ? null : (String) map.getValue(UTF16.charAt(s,0));
|
||
if (m == null) m = "";
|
||
String info = m;
|
||
if (p2.compare(lastm,m) > 0) {
|
||
info = info + "\t" + lastm + " > " + m + "\t";
|
||
Object temp;
|
||
temp = hanyu.getValue(UTF16.charAt(lasts,0));
|
||
if (temp != null) info += "[" + temp + "]";
|
||
temp = mand.getValue(UTF16.charAt(lasts,0));
|
||
if (temp != null) info += "[" + temp + "]";
|
||
info += " > ";
|
||
temp = hanyu.getValue(UTF16.charAt(s,0));
|
||
if (temp != null) info += "[" + temp + "]";
|
||
temp = mand.getValue(UTF16.charAt(s,0));
|
||
if (temp != null) info += "[" + temp + "]";
|
||
}
|
||
pw.println(Utility.hex(s) + "\t" + s + "\t" + info);
|
||
lastm = m;
|
||
lasts = s;
|
||
}
|
||
pw.close();
|
||
}
|
||
|
||
static void addField(String dir, String file, int hexCodeFieldNumber, int valueNumber, UnicodeMap result) throws IOException {
|
||
BufferedReader br = BagFormatter.openUTF8Reader(dir, file);
|
||
while (true) {
|
||
String line = br.readLine();
|
||
if (line == null) break;
|
||
line = line.trim();
|
||
if (line.length() == 0) continue;
|
||
if (line.startsWith("\uFEFF")) line = line.substring(1);
|
||
if (line.startsWith("#") || line.length() == 0) continue;
|
||
String[] pieces = Utility.split(line,'\t');
|
||
result.put(Integer.parseInt(pieces[hexCodeFieldNumber], 16), pieces[valueNumber]);
|
||
}
|
||
br.close();
|
||
}
|
||
|
||
static boolean isEqualOrNull(String a, String b) {
|
||
if (a == null || b == null) return true;
|
||
return a.equals(b);
|
||
}
|
||
public static String getUpTo(String s, char ch) {
|
||
int pos = s.indexOf(ch);
|
||
if (pos < 0) return s;
|
||
return s.substring(0,pos);
|
||
}
|
||
|
||
public static void main(int typeIn) throws IOException {
|
||
if (typeIn == CHINESE) {
|
||
fixedMandarin();
|
||
return;
|
||
}
|
||
type = typeIn;
|
||
|
||
try {
|
||
System.out.println("Starting");
|
||
System.out.println("Quoting: " + quoteNonLetters.toRules(true));
|
||
System.out.println("Quoting: " + quoteNonLetters.toRules(true));
|
||
|
||
|
||
String key; // kMandarin, kKorean, kJapaneseKun, kJapaneseOn
|
||
String filename;
|
||
|
||
switch (type) {
|
||
case DEFINITION:
|
||
key = "kDefinition"; // kMandarin, kKorean, kJapaneseKun, kJapaneseOn
|
||
filename = "Raw_Transliterator_Han_Latin_Definition";
|
||
break;
|
||
case JAPANESE:
|
||
key = "kJapaneseOn";
|
||
filename = "Raw_Transliterator_ja_Latin";
|
||
break;
|
||
case CHINESE:
|
||
key = "kMandarin";
|
||
filename = "Raw_Transliterator_Han_Latin";
|
||
break;
|
||
default: throw new IllegalArgumentException("Unexpected option: must be 0..2");
|
||
}
|
||
filename += Default.ucd().getVersion() + ".txt";
|
||
|
||
err = Utility.openPrintWriter("Transliterate_err.txt", Utility.UTF8_WINDOWS);
|
||
log = Utility.openPrintWriter("Transliterate_log.txt", Utility.UTF8_WINDOWS);
|
||
log.print('\uFEFF');
|
||
|
||
if (false /*!SKIP_OVERRIDES*/) {
|
||
log.println();
|
||
log.println("@*Override Data");
|
||
log.println();
|
||
readOverrides(type);
|
||
|
||
log.println();
|
||
log.println("@*DICT Data");
|
||
log.println();
|
||
readCDICTDefinitions(type);
|
||
}
|
||
|
||
log.println();
|
||
log.println("@Unihan Data");
|
||
log.println();
|
||
out2 = BagFormatter.openUTF8Writer(GEN_DIR, "unihan_kmandarinDump.txt");
|
||
|
||
readUnihanData(key);
|
||
|
||
out2.close();
|
||
|
||
if (false) {
|
||
readCDICT();
|
||
compareUnihanWithCEDICT();
|
||
}
|
||
|
||
readFrequencyData(type);
|
||
|
||
Iterator it = fullPinyin.iterator();
|
||
while (it.hasNext()) {
|
||
String s = (String) it.next();
|
||
if (!isValidPinyin2(s)) {
|
||
err.println("?Valid Pinyin: " + s);
|
||
}
|
||
}
|
||
|
||
|
||
it = unihanMap.keySet().iterator();
|
||
Map badPinyin = new TreeMap();
|
||
PrintWriter out2 = Utility.openPrintWriter("Raw_mapping.txt", Utility.UTF8_WINDOWS);
|
||
try {
|
||
while (it.hasNext()) {
|
||
String keyChar = (String) it.next();
|
||
String def = (String) unihanMap.get(keyChar);
|
||
if (!isValidPinyin(def)) {
|
||
String fixedDef = fixPinyin(def);
|
||
err.println(Default.ucd().getCode(keyChar) + "\t" + keyChar + "\t" + fixedDef + "\t#" + def
|
||
+ (fixedDef.equals(def) ? " FAIL" : ""));
|
||
Utility.addToSet(badPinyin, def, keyChar);
|
||
}
|
||
// check both ways
|
||
String digitDef = accentPinyin_digitPinyin.transliterate(def);
|
||
String accentDef = digitPinyin_accentPinyin.transliterate(digitDef);
|
||
if (!accentDef.equals(def)) {
|
||
err.println("Failed Digit Pinyin: "
|
||
+ Default.ucd().getCode(keyChar) + "\t" + keyChar + "\t"
|
||
+ def + " => " + digitDef + " => " + accentDef);
|
||
}
|
||
|
||
out2.println(toHexUnicode.transliterate(keyChar)
|
||
+ "\tkMandarin\t" + digitDef.toUpperCase() + "\t# " + keyChar + ";\t" + def);
|
||
}
|
||
err.println();
|
||
err.println("Summary of Bad syllables");
|
||
Utility.printMapOfCollection(err, badPinyin, "\r\n", ":\t", ", ");
|
||
} finally {
|
||
out2.close();
|
||
}
|
||
|
||
out = Utility.openPrintWriter(filename, Utility.UTF8_WINDOWS);
|
||
out.println("# Start RAW data for converting CJK characters");
|
||
/*
|
||
out.println("# Note: adds space between them and letters.");
|
||
out.println("{ ([:Han:]) } [:L:] > | $1 ' ';");
|
||
out.println("[\\.\\,\\?\\!\uFF0E\uFF0C\uFF1F\uFF01\u3001\u3002[:Pe:][:Pf:]] { } [:L:] > ' ';");
|
||
out.println("[:L:] { } [[:Han:][:Ps:][:Pi:]]> ' ';");
|
||
|
||
if (type == JAPANESE) {
|
||
out.println("$kata = [[\uFF9E\uFF9F\uFF70\u30FC][:katakana:]];");
|
||
out.println("$kata { } [[:L:]-$kata]> ' ';");
|
||
out.println("[[:L:]-$kata] { } $kata > ' ';");
|
||
out.println("[:hiragana:] { } [[:L:]-[:hiragana:]] > ' ';");
|
||
out.println("[[:L:]-[:hiragana:]] { } [:hiragana:]> ' ';");
|
||
}
|
||
*/
|
||
|
||
Set gotAlready = new HashSet();
|
||
Set lenSet = new TreeSet();
|
||
Set backSet = new TreeSet();
|
||
int rank = 0;
|
||
Map definitionCount = new HashMap();
|
||
|
||
it = rankList.iterator();
|
||
while (it.hasNext()) {
|
||
String keyChar = (String) it.next();
|
||
String def = (String) unihanMap.get(keyChar);
|
||
if (def == null) continue; // skipping
|
||
// sort longer definitions first!
|
||
|
||
Integer countInteger = (Integer) definitionCount.get(def);
|
||
int defCount = (countInteger == null) ? 0 : countInteger.intValue();
|
||
String oldDef = def;
|
||
if (DISAMBIG && (defCount != 0 || def.indexOf(' ') >= 0)) {
|
||
def += " " + toSub.transliterate(String.valueOf(defCount));
|
||
}
|
||
|
||
lenSet.add(new Pair(
|
||
new Pair(new Integer(-UTF16.countCodePoint(keyChar)),
|
||
new Pair(new Integer(-def.length()), new Integer(rank++))),
|
||
new Pair(keyChar, def)));
|
||
backSet.add(new Pair(
|
||
new Pair(new Integer(-def.toString().length()), new Integer(rank++)),
|
||
new Pair(keyChar, def)));
|
||
|
||
definitionCount.put(oldDef, new Integer(defCount+1));
|
||
gotAlready.add(keyChar);
|
||
}
|
||
|
||
// add the ones that are not ranked!
|
||
it = unihanMap.keySet().iterator();
|
||
while (it.hasNext()) {
|
||
String keyChar = (String) it.next();
|
||
if (gotAlready.contains(keyChar)) continue;
|
||
|
||
String def = (String) unihanMap.get(keyChar);
|
||
|
||
Integer countInteger = (Integer) definitionCount.get(def);
|
||
int defCount = (countInteger == null) ? 0 : countInteger.intValue();
|
||
String oldDef = def;
|
||
if (DISAMBIG && (defCount != 0 || def.indexOf(' ') >= 0)) {
|
||
def += " " + toSub.transliterate(String.valueOf(defCount));
|
||
}
|
||
|
||
lenSet.add(new Pair(
|
||
new Pair(new Integer(-UTF16.countCodePoint(keyChar)),
|
||
new Pair(new Integer(-def.toString().length()), new Integer(rank++))),
|
||
new Pair(keyChar, def)));
|
||
backSet.add(new Pair(
|
||
new Pair(new Integer(-def.toString().length()), new Integer(rank++)),
|
||
new Pair(keyChar, def)));
|
||
|
||
definitionCount.put(oldDef, new Integer(defCount+1));
|
||
}
|
||
|
||
// First, find the ones that we want a definition for, based on the ranking
|
||
// We might have a situation where the definitions are masked.
|
||
// In that case, write forwards and backwards separately
|
||
|
||
Set doReverse = new HashSet();
|
||
Set gotIt = new HashSet();
|
||
|
||
if (!DO_SIMPLE) {
|
||
it = backSet.iterator();
|
||
while (it.hasNext()) {
|
||
Pair p = (Pair) it.next();
|
||
p = (Pair) p.second;
|
||
|
||
String keyChar = (String) p.first;
|
||
String def = (String) p.second;
|
||
if (!gotIt.contains(def)) {
|
||
if (unihanNonSingular) {
|
||
out.println(quoteNonLetters.transliterate(keyChar)
|
||
+ " < " + quoteNonLetters.transliterate(def) + ";");
|
||
} else {
|
||
doReverse.add(keyChar);
|
||
}
|
||
}
|
||
gotIt.add(def);
|
||
}
|
||
}
|
||
|
||
|
||
it = lenSet.iterator();
|
||
while (it.hasNext()) {
|
||
Pair p = (Pair) it.next();
|
||
p = (Pair) p.second;
|
||
|
||
String keyChar = (String) p.first;
|
||
String def = (String) p.second;
|
||
String rel = !DO_SIMPLE && doReverse.contains(keyChar) ? "<>" : ">";
|
||
|
||
out.println(quoteNonLetters.transliterate(keyChar) + rel
|
||
+ quoteNonLetters.transliterate(def) + "|\\ ;");
|
||
//if (TESTING) System.out.println("# " + code + " > " + definition);
|
||
}
|
||
|
||
out.println("\u3002 <> '.';");
|
||
out.println("# End RAW data for converting CJK characters");
|
||
|
||
/*
|
||
if (type == JAPANESE) {
|
||
out.println(":: katakana-latin;");
|
||
out.println(":: hiragana-latin;");
|
||
}
|
||
out.println(":: fullwidth-halfwidth ();");
|
||
*/
|
||
|
||
|
||
System.out.println("Total: " + totalCount);
|
||
System.out.println("Defined Count: " + count);
|
||
|
||
log.println();
|
||
log.println("@Duplicates (Frequency Order");
|
||
log.println();
|
||
it = rankList.iterator();
|
||
while (it.hasNext()) {
|
||
String word = (String) it.next();
|
||
Collection dups = (Collection) duplicates.get(word);
|
||
if (dups == null) continue;
|
||
log.print(hex.transliterate(word) + "\t" + word + "\t");
|
||
Iterator it2 = dups.iterator();
|
||
boolean gotFirst = false;
|
||
while (it2.hasNext()) {
|
||
if (!gotFirst) gotFirst = true;
|
||
else log.print(", ");
|
||
log.print(it2.next());
|
||
}
|
||
if (overrideSet.contains(word)) log.print(" *override*");
|
||
log.println();
|
||
}
|
||
|
||
log.println();
|
||
log.println("@Duplicates (Character Order)");
|
||
log.println();
|
||
it = duplicates.keySet().iterator();
|
||
while (it.hasNext()) {
|
||
String word = (String) it.next();
|
||
log.print(hex.transliterate(word) + "\t" + word + "\t");
|
||
Collection dups = (Collection) duplicates.get(word);
|
||
Iterator it2 = dups.iterator();
|
||
boolean gotFirst = false;
|
||
while (it2.hasNext()) {
|
||
if (!gotFirst) gotFirst = true;
|
||
else log.print(", ");
|
||
log.print(it2.next());
|
||
}
|
||
if (overrideSet.contains(word)) log.print(" *override*");
|
||
log.println();
|
||
}
|
||
|
||
} catch (Exception e) {
|
||
System.out.println("Exception: " + e);
|
||
} finally {
|
||
if (log != null) log.close();
|
||
if (err != null) err.close();
|
||
if (out != null) out.close();
|
||
}
|
||
}
|
||
|
||
//http://fog.ccsf.cc.ca.us/~jliou/phonetic.htm
|
||
// longer ones must be AFTER!
|
||
// longer ones must be AFTER!
|
||
static final String[] initialPinyin = {
|
||
"",
|
||
"b", "p", "m", "f",
|
||
"d", "t", "n", "l",
|
||
"z", "c", "s",
|
||
"zh", "ch", "sh", "r",
|
||
"j", "q", "x",
|
||
"g", "k", "h",
|
||
"y", "w"}; // added to make checking simpler
|
||
|
||
static final String[] finalPinyin = {
|
||
"a", "ai", "ao", "an", "ang",
|
||
"o", "ou", "ong",
|
||
"e", "ei", "er", "en", "eng",
|
||
"i", "ia", "iao", "ie", "iu", "ian", "in", "iang", "ing", "iong",
|
||
"u", "ua", "uo", "uai", "ui", "uan", "un", "uang", "ueng",
|
||
"ü", "üe", "üan", "ün"
|
||
};
|
||
// Don't bother with the following rules; just add w,y to initials
|
||
// When “i” stands alone, a “y” will be added before it as “yi”.
|
||
// If “i” is the first letter of the syllable it will be changed to “y”.
|
||
// When “u” stands alone, a “w” will be added before it as “wu”.
|
||
// If “u” is the first letter of the syllable it will be changed to “w”. e.g. “uang -> wang”.
|
||
// When “ü” stands alone, a “y” will be added before it and “ü” will be changed to “u” as “yu”.
|
||
// If “ü” is the first letter of the syllable, then the spelling will be changed to “yu”. e.g. “üan -> yuan”.
|
||
//Note: The nasal final “ueng” never occurs after an initial but always form a syllable by itself.
|
||
// The “o” in “iou” is hidden, so it will be wrote as “iu”. But, don’t forget to pronounce it.
|
||
// The “e” in “uei” is hidden, so it will be wrote as “ui”. But, don’t forget to pronounce it.
|
||
|
||
|
||
public static final String[] pinyin_bopomofo = {
|
||
"a", "\u311a",
|
||
"ai", "\u311e",
|
||
"an", "\u3122",
|
||
"ang", "\u3124",
|
||
"ao", "\u3120",
|
||
"ba", "\u3105\u311a",
|
||
"bai", "\u3105\u311e",
|
||
"ban", "\u3105\u3122",
|
||
"bang", "\u3105\u3124",
|
||
"bao", "\u3105\u3120",
|
||
"bei", "\u3105\u311f",
|
||
"ben", "\u3105\u3123",
|
||
"beng", "\u3105\u3125",
|
||
"bi", "\u3105\u3127",
|
||
"bian", "\u3105\u3127\u3122",
|
||
"biao", "\u3105\u3127\u3120",
|
||
"bie", "\u3105\u3127\u311d",
|
||
"bin", "\u3105\u3127\u3123",
|
||
"bing", "\u3105\u3127\u3125",
|
||
"bo", "\u3105\u311b",
|
||
"bu", "\u3105\u3128",
|
||
"ca", "\u3118\u311a",
|
||
"cai", "\u3118\u311e",
|
||
"can", "\u3118\u3122",
|
||
"cang", "\u3118\u3124",
|
||
"cao", "\u3118\u3120",
|
||
"ce", "\u3118",
|
||
"cen", "\u3118\u3123",
|
||
"ceng", "\u3118\u3125",
|
||
"cha", "\u3114\u311a",
|
||
"chai", "\u3114\u311e",
|
||
"chan", "\u3114\u3122",
|
||
"chang", "\u3114\u3124",
|
||
"chao", "\u3114\u3120",
|
||
"che", "\u3114\u311c",
|
||
"chen", "\u3114\u3123",
|
||
"cheng", "\u3114\u3125",
|
||
"chi", "\u3114",
|
||
"chong", "\u3114\u3121\u3125",
|
||
"chou", "\u3114\u3121",
|
||
"chu", "\u3114\u3128",
|
||
//"chua", "XXX",
|
||
"chuai", "\u3114\u3128\u311e",
|
||
"chuan", "\u3114\u3128\u3122",
|
||
"chuang", "\u3114\u3128\u3124",
|
||
"chui", "\u3114\u3128\u311f",
|
||
"chun", "\u3114\u3128\u3123",
|
||
"chuo", "\u3114\u3128\u311b",
|
||
"ci", "\u3118",
|
||
"cong", "\u3118\u3128\u3125",
|
||
"cou", "\u3118\u3121",
|
||
"cu", "\u3118\u3128",
|
||
"cuan", "\u3118\u3128\u3122",
|
||
"cui", "\u3118\u3128\u311f",
|
||
"cun", "\u3118\u3128\u3123",
|
||
"cuo", "\u3118\u3128\u311b",
|
||
"da", "\u3109\u311a",
|
||
"dai", "\u3109\u311e",
|
||
"dan", "\u3109\u3122",
|
||
"dang", "\u3109\u3124",
|
||
"dao", "\u3109\u3120",
|
||
"de", "\u3109\u311c",
|
||
"dei", "\u3109\u311f",
|
||
"den", "\u3109\u3123",
|
||
"deng", "\u3109\u3125",
|
||
"di", "\u3109\u3127",
|
||
"dia", "\u3109\u3127\u311a",
|
||
"dian", "\u3109\u3127\u3122",
|
||
"diao", "\u3109\u3127\u3120",
|
||
"die", "\u3109\u3127\u311d",
|
||
"ding", "\u3109\u3127\u3125",
|
||
"diu", "\u3109\u3127\u3121",
|
||
"dong", "\u3109\u3128\u3125",
|
||
"dou", "\u3109\u3121",
|
||
"du", "\u3109\u3128",
|
||
"duan", "\u3109\u3128\u3122",
|
||
"dui", "\u3109\u3128\u311f",
|
||
"dun", "\u3109\u3128\u3123",
|
||
"duo", "\u3109\u3128\u311b",
|
||
"e", "\u311c",
|
||
"ei", "\u311f",
|
||
"en", "\u3123",
|
||
"eng", "\u3125",
|
||
"er", "\u3126",
|
||
"fa", "\u3108\u311a",
|
||
"fan", "\u3108\u3122",
|
||
"fang", "\u3108\u3124",
|
||
"fei", "\u3108\u311f",
|
||
"fen", "\u3108\u3123",
|
||
"feng", "\u3108\u3125",
|
||
"fo", "\u3108\u311b",
|
||
"fou", "\u3108\u3121",
|
||
"fu", "\u3108\u3128",
|
||
"ga", "\u310d\u311a",
|
||
"gai", "\u310d\u311e",
|
||
"gan", "\u310d\u3122",
|
||
"gang", "\u310d\u3124",
|
||
"gao", "\u310d\u3120",
|
||
"ge", "\u310d\u311c",
|
||
"gei", "\u310d\u311f",
|
||
"gen", "\u310d\u3123",
|
||
"geng", "\u310d\u3125",
|
||
"gong", "\u310d\u3128\u3125",
|
||
"gou", "\u310d\u3121",
|
||
"gu", "\u310d\u3128",
|
||
"gua", "\u310d\u3128\u311a",
|
||
"guai", "\u310d\u3128\u311e",
|
||
"guan", "\u310d\u3128\u3122",
|
||
"guang", "\u310d\u3128\u3124",
|
||
"gui", "\u310d\u3128\u311f",
|
||
"gun", "\u310d\u3128\u3123",
|
||
"guo", "\u310d\u3128\u311b",
|
||
"ha", "\u310f\u311a",
|
||
"hai", "\u310f\u311e",
|
||
"han", "\u310f\u3122",
|
||
"hang", "\u310f\u3124",
|
||
"hao", "\u310f\u3120",
|
||
"he", "\u310f\u311c",
|
||
"hei", "\u310f\u311f",
|
||
"hen", "\u310f\u3123",
|
||
"heng", "\u310f\u3125",
|
||
"hm", "\u310f\u3107",
|
||
"hng", "\u310f\u312b", // 'dialect of n'
|
||
"hong", "\u310f\u3128\u3125",
|
||
"hou", "\u310f\u3121",
|
||
"hu", "\u310f\u3128",
|
||
"hua", "\u310f\u3128\u311a",
|
||
"huai", "\u310f\u3128\u311e",
|
||
"huan", "\u310f\u3128\u3122",
|
||
"huang", "\u310f\u3128\u3124",
|
||
"hui", "\u310f\u3128\u311f",
|
||
"hun", "\u310f\u3128\u3123",
|
||
"huo", "\u310f\u3128\u311b",
|
||
"ji", "\u3110\u3127",
|
||
"jia", "\u3110\u3127\u311a",
|
||
"jian", "\u3110\u3127\u3122",
|
||
"jiang", "\u3110\u3127\u3124",
|
||
"jiao", "\u3110\u3127\u3120",
|
||
"jie", "\u3110\u3127\u311d",
|
||
"jin", "\u3110\u3127\u3123",
|
||
"jing", "\u3110\u3127\u3125",
|
||
"jiong", "\u3110\u3129\u3125",
|
||
"jiu", "\u3110\u3127\u3121",
|
||
"ju", "\u3110\u3129",
|
||
"juan", "\u3110\u3129\u3122",
|
||
"jue", "\u3110\u3129\u311d",
|
||
"jun", "\u3110\u3129\u3123",
|
||
"ka", "\u310e\u311a",
|
||
"kai", "\u310e\u311e",
|
||
"kan", "\u310e\u3122",
|
||
"kang", "\u310e\u3124",
|
||
"kao", "\u310e\u3120",
|
||
"ke", "\u310e\u311c",
|
||
"kei", "\u310e\u311f",
|
||
"ken", "\u310e\u3123",
|
||
"keng", "\u310e\u3125",
|
||
"kong", "\u310e\u3128\u3125",
|
||
"kou", "\u310e\u3121",
|
||
"ku", "\u310e\u3128",
|
||
"kua", "\u310e\u3128\u311a",
|
||
"kuai", "\u310e\u3128\u311e",
|
||
"kuan", "\u310e\u3128\u3122",
|
||
"kuang", "\u310e\u3128\u3124",
|
||
"kui", "\u310e\u3128\u311f",
|
||
"kun", "\u310e\u3128\u3123",
|
||
"kuo", "\u310e\u3128\u311b",
|
||
"la", "\u310c\u311a",
|
||
"lai", "\u310c\u311e",
|
||
"lan", "\u310c\u3122",
|
||
"lang", "\u310c\u3124",
|
||
"lao", "\u310c\u3120",
|
||
"le", "\u310c\u311c",
|
||
"lei", "\u310c\u311f",
|
||
"leng", "\u310c\u3125",
|
||
"li", "\u310c\u3127",
|
||
"lia", "\u310c\u3127\u311a",
|
||
"lian", "\u310c\u3127\u3122",
|
||
"liang", "\u310c\u3127\u3124",
|
||
"liao", "\u310c\u3127\u3120",
|
||
"lie", "\u310c\u3127\u311d",
|
||
"lin", "\u310c\u3127\u3123",
|
||
"ling", "\u310c\u3127\u3125",
|
||
"liu", "\u310c\u3127\u3121",
|
||
"lo", "\u310c\u311b",
|
||
"long", "\u310c\u3128\u3125",
|
||
"lou", "\u310c\u3121",
|
||
"lu", "\u310c\u3128",
|
||
"lü", "\u310c\u3129",
|
||
"luan", "\u310c\u3128\u3122",
|
||
"lüe", "\u310c\u3129\u311d",
|
||
"lun", "\u310c\u3128\u3123",
|
||
"luo", "\u310c\u3128\u311b",
|
||
"m", "\u3107",
|
||
"ma", "\u3107\u311a",
|
||
"mai", "\u3107\u311e",
|
||
"man", "\u3107\u3122",
|
||
"mang", "\u3107\u3124",
|
||
"mao", "\u3107\u3120",
|
||
"me", "\u3107\u311c",
|
||
"mei", "\u3107\u311f",
|
||
"men", "\u3107\u3123",
|
||
"meng", "\u3107\u3125",
|
||
"mi", "\u3107\u3127",
|
||
"mian", "\u3107\u3127\u3122",
|
||
"miao", "\u3107\u3127\u3120",
|
||
"mie", "\u3107\u3127\u311d",
|
||
"min", "\u3107\u3127\u3123",
|
||
"ming", "\u3107\u3127\u3125",
|
||
"miu", "\u3107\u3127\u3121",
|
||
"mo", "\u3107\u311b",
|
||
"mou", "\u3107\u3121",
|
||
"mu", "\u3107\u3128",
|
||
"n", "\u310b",
|
||
"na", "\u310b\u311a",
|
||
"nai", "\u310b\u311e",
|
||
"nan", "\u310b\u3122",
|
||
"nang", "\u310b\u3124",
|
||
"nao", "\u310b\u3120",
|
||
"ne", "\u310b\u311c",
|
||
"nei", "\u310b\u311f",
|
||
"nen", "\u310b\u3123",
|
||
"neng", "\u310b\u3125",
|
||
"ng", "\u312b",
|
||
"ni", "\u310b\u3127",
|
||
"nian", "\u310b\u3127\u3122",
|
||
"niang", "\u310b\u3127\u3124",
|
||
"niao", "\u310b\u3127\u3120",
|
||
"nie", "\u310b\u3127\u311d",
|
||
"nin", "\u310b\u3127\u3123",
|
||
"ning", "\u310b\u3127\u3125",
|
||
"niu", "\u310b\u3127\u3121",
|
||
"nong", "\u310b\u3128\u3125",
|
||
"nou", "\u310b\u3121",
|
||
"nu", "\u310b\u3128",
|
||
"nü", "\u310b\u3129",
|
||
"nuan", "\u310b\u3128\u3122",
|
||
"nüe", "\u310b\u3129\u311d",
|
||
"nuo", "\u310b\u3128\u311b",
|
||
"o", "\u311b",
|
||
"ou", "\u3121",
|
||
"pa", "\u3106\u311a",
|
||
"pai", "\u3106\u311e",
|
||
"pan", "\u3106\u3122",
|
||
"pang", "\u3106\u3124",
|
||
"pao", "\u3106\u3120",
|
||
"pei", "\u3106\u311f",
|
||
"pen", "\u3106\u3123",
|
||
"peng", "\u3106\u3125",
|
||
"pi", "\u3106\u3127",
|
||
"pian", "\u3106\u3127\u3122",
|
||
"piao", "\u3106\u3127\u3120",
|
||
"pie", "\u3106\u3127\u311d",
|
||
"pin", "\u3106\u3127\u3123",
|
||
"ping", "\u3106\u3127\u3125",
|
||
"po", "\u3106\u311b",
|
||
"pou", "\u3106\u3121",
|
||
"pu", "\u3106\u3128",
|
||
"qi", "\u3111",
|
||
"qia", "\u3111\u3127\u311a",
|
||
"qian", "\u3111\u3127\u3122",
|
||
"qiang", "\u3111\u3127\u3124",
|
||
"qiao", "\u3111\u3127\u3120",
|
||
"qie", "\u3111\u3127\u311d",
|
||
"qin", "\u3111\u3127\u3123",
|
||
"qing", "\u3111\u3127\u3125",
|
||
"qiong", "\u3111\u3129\u3125",
|
||
"qiu", "\u3111\u3129\u3121",
|
||
"qu", "\u3111\u3129",
|
||
"quan", "\u3111\u3129\u3122",
|
||
"que", "\u3111\u3129\u311d",
|
||
"qun", "\u3111\u3129\u3123",
|
||
"ran", "\u3116\u3122",
|
||
"rang", "\u3116\u3124",
|
||
"rao", "\u3116\u3120",
|
||
"re", "\u3116\u311c",
|
||
"ren", "\u3116\u3123",
|
||
"reng", "\u3116\u3125",
|
||
"ri", "\u3116",
|
||
"rong", "\u3116\u3128\u3125",
|
||
"rou", "\u3116\u3121",
|
||
"ru", "\u3116\u3128",
|
||
"ruan", "\u3116\u3128\u3122",
|
||
"rui", "\u3116\u3128\u311f",
|
||
"run", "\u3116\u3128\u3123",
|
||
"ruo", "\u3116\u3128\u311b",
|
||
"sa", "\u3119\u311a",
|
||
"sai", "\u3119\u311e",
|
||
"san", "\u3119\u3122",
|
||
"sang", "\u3119\u3124",
|
||
"sao", "\u3119\u3120",
|
||
"se", "\u3119\u311c",
|
||
"sen", "\u3119\u3123",
|
||
"seng", "\u3119\u3125",
|
||
"sha", "\u3115\u311a",
|
||
"shai", "\u3115\u311e",
|
||
"shan", "\u3115\u3122",
|
||
"shang", "\u3115\u3124",
|
||
"shao", "\u3115\u3120",
|
||
"she", "\u3115\u311c",
|
||
"shei", "\u3115\u311f",
|
||
"shen", "\u3115\u3123",
|
||
"sheng", "\u3115\u3125",
|
||
"shi", "\u3115",
|
||
"shou", "\u3115\u3121",
|
||
"shu", "\u3115\u3128",
|
||
"shua", "\u3115\u3128\u311a",
|
||
"shuai", "\u3115\u3128\u311e",
|
||
"shuan", "\u3115\u3128\u3122",
|
||
"shuang", "\u3115\u3128\u3124",
|
||
"shui", "\u3115\u3128\u311f",
|
||
"shun", "\u3115\u3128\u3123",
|
||
"shuo", "\u3115\u3128\u311b",
|
||
"si", "\u3119",
|
||
"song", "\u3119\u3128\u3125",
|
||
"sou", "\u3119\u3121",
|
||
"su", "\u3119\u3128",
|
||
"suan", "\u3119\u3128\u3122",
|
||
"sui", "\u3119\u3128\u311f",
|
||
"sun", "\u3119\u3128\u3123",
|
||
"suo", "\u3119\u3128\u311b",
|
||
"ta", "\u310a\u311a",
|
||
"tai", "\u310a\u311e",
|
||
"tan", "\u310a\u3122",
|
||
"tang", "\u310a\u3124",
|
||
"tao", "\u310a\u3120",
|
||
"te", "\u310a\u311c",
|
||
"teng", "\u310a\u3125",
|
||
"ti", "\u310a\u3127",
|
||
"tian", "\u310a\u3127\u3122",
|
||
"tiao", "\u310a\u3127\u3120",
|
||
"tie", "\u310a\u3127\u311d",
|
||
"ting", "\u310a\u3127\u3125",
|
||
"tong", "\u310a\u3128\u3125",
|
||
"tou", "\u310a\u3121",
|
||
"tu", "\u310a\u3128",
|
||
"tuan", "\u310a\u3128\u3122",
|
||
"tui", "\u310a\u3128\u311f",
|
||
"tun", "\u310a\u3128\u3123",
|
||
"tuo", "\u310a\u3128\u311b",
|
||
"wa", "\u3128\u311a",
|
||
"wai", "\u3128\u311e",
|
||
"wan", "\u3128\u3122",
|
||
"wang", "\u3128\u3124",
|
||
"wei", "\u3128\u311f",
|
||
"wen", "\u3128\u3123",
|
||
"weng", "\u3128\u3125",
|
||
"wo", "\u3128\u311b",
|
||
"wu", "\u3128",
|
||
"xi", "\u3112\u3127",
|
||
"xia", "\u3112\u3127\u311a",
|
||
"xian", "\u3112\u3127\u3122",
|
||
"xiang", "\u3112\u3127\u3124",
|
||
"xiao", "\u3112\u3127\u3120",
|
||
"xie", "\u3112\u3127\u311d",
|
||
"xin", "\u3112\u3127\u3123",
|
||
"xing", "\u3112\u3127\u3125",
|
||
"xiong", "\u3112\u3129\u3125",
|
||
"xiu", "\u3112\u3127\u3121",
|
||
"xu", "\u3112\u3129",
|
||
"xuan", "\u3112\u3129\u3122",
|
||
"xue", "\u3112\u3129\u311d",
|
||
"xun", "\u3112\u3129\u3123",
|
||
"ya", "\u3127\u311a",
|
||
"yai", "\u3127\u311e", // not in xinhua zidian index, but listed as alternate pronunciation
|
||
"yan", "\u3127\u3122",
|
||
"yang", "\u3127\u3124",
|
||
"yao", "\u3127\u3120",
|
||
"ye", "\u3127\u311d",
|
||
"yi", "\u3127",
|
||
"yin", "\u3127\u3123",
|
||
"ying", "\u3127\u3125",
|
||
"yo", "\u3127\u311b",
|
||
"yong", "\u3129\u3125",
|
||
"you", "\u3127\u3121",
|
||
"yu", "\u3129",
|
||
"yuan", "\u3129\u3122",
|
||
"yue", "\u3129\u311d",
|
||
"yun", "\u3129\u3123",
|
||
"za", "\u3117\u311a",
|
||
"zai", "\u3117\u311e",
|
||
"zan", "\u3117\u3122",
|
||
"zang", "\u3117\u3124",
|
||
"zao", "\u3117\u3120",
|
||
"ze", "\u3117",
|
||
"zei", "\u3117\u311f",
|
||
"zen", "\u3117\u3123",
|
||
"zeng", "\u3117\u3125",
|
||
"zha", "\u3113\u311a",
|
||
"zhai", "\u3113\u311e",
|
||
"zhan", "\u3113\u3122",
|
||
"zhang", "\u3113\u3124",
|
||
"zhao", "\u3113\u3120",
|
||
"zhe", "\u3113\u311d",
|
||
"zhei", "\u3113\u311f",
|
||
"zhen", "\u3113\u3123",
|
||
"zheng", "\u3113\u3125",
|
||
"zhi", "\u3113",
|
||
"zhong", "\u3113\u3128\u3125",
|
||
"zhou", "\u3113\u3121",
|
||
"zhu", "\u3113\u3128",
|
||
"zhua", "\u3113\u3128\u311a",
|
||
"zhuai", "\u3113\u3128\u311e",
|
||
"zhuan", "\u3113\u3128\u3122",
|
||
"zhuang", "\u3113\u3128\u3124",
|
||
"zhui", "\u3113\u3128\u311f",
|
||
"zhun", "\u3113\u3128\u3123",
|
||
"zhuo", "\u3113\u3128\u311b",
|
||
"zi", "\u3117",
|
||
"zong", "\u3117\u3128\u3125",
|
||
"zou", "\u3117\u3121",
|
||
"zu", "\u3117\u3128",
|
||
"zuan", "\u3117\u3128\u3122",
|
||
"zui", "\u3117\u3128\u311f",
|
||
"zun", "\u3117\u3128\u3123",
|
||
"zuo", "\u3117\u3128\u311b",
|
||
};
|
||
|
||
static final Set fullPinyin = new TreeSet();
|
||
static {
|
||
for (int i = 0; i < pinyin_bopomofo.length; i+= 2) {
|
||
fullPinyin.add(pinyin_bopomofo[i]);
|
||
}
|
||
}
|
||
|
||
static boolean isValidPinyin(String s) {
|
||
s = dropTones.transliterate(s);
|
||
if (fullPinyin.contains(s)) return true;
|
||
return false;
|
||
}
|
||
|
||
static boolean isValidPinyin2(String s) {
|
||
s = dropTones.transliterate(s);
|
||
for (int i = initialPinyin.length-1; i >= 0; --i) {
|
||
if (s.startsWith(initialPinyin[i])) {
|
||
String end = s.substring(initialPinyin[i].length());
|
||
for (int j = finalPinyin.length-1; j >= 0; --j) {
|
||
if (end.equals(finalPinyin[j])) return true;
|
||
}
|
||
return false;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/*
|
||
U+347C · liù #lyuè
|
||
U+3500 · lüè #lvè
|
||
U+3527 · liù #lyù
|
||
U+3729 · ào #àu
|
||
U+380E · jí #jjí
|
||
U+3825 · l· #lv·
|
||
U+3A3C · lüè #luè
|
||
U+3B5A · li· #ly· *** lü?
|
||
U+3CB6 · l· #lv·
|
||
U+3D56 · niù #nyù *** nü?
|
||
U+3D88 · li·ng #li·ng
|
||
U+3EF2 · li· #ly·*** lü?
|
||
U+3F94 · li· #ly·*** lü?
|
||
U+4071 · ào #àu
|
||
U+40AE · liù #lyuè *** lüe?
|
||
U+430E · liù #lyuè *** lüe?
|
||
U+451E · liù #lyù *** lü?
|
||
U+4588 · nüè #nuè
|
||
U+458B · nüè #nuè
|
||
U+45A1 · niù #nyù *** nü?
|
||
U+4610 · niù #nyù *** nü?
|
||
U+46BC · niù #nyù *** nü?
|
||
U+46DA · liù #lyuè *** lüe?
|
||
U+4896 · liù #lyù *** lü?
|
||
U+4923 · liù #lyuè *** lüe?
|
||
U+4968 · liù #lyù *** lü?
|
||
U+4A0B · niù #nyuè *** nüe?
|
||
U+4AC4 · chuò #chuà
|
||
U+4D08 · ·o #·u
|
||
U+4D8A · niù #nyù *** nü?
|
||
U+51CA · qíng #qýng
|
||
U+51D6 · zhu·n #zhu·n *** this is probably zh·n
|
||
U+5481 · gàn #gèm
|
||
U+5838 · féng #fúng
|
||
U+639F · lü· #lu· *** this pronunciation surprises me, but I don't know...
|
||
U+66D5 · yàn #yiàn
|
||
U+6B3B · chu· #chu· *** chua _is_ ok after all, my table missed an entry
|
||
U+6B56 · chu· #chu· *** chua
|
||
U+6C7C · ni· #ni·u
|
||
U+6E6D · qiú #qióu
|
||
U+6F71 · y· #yi·
|
||
U+7493 · xiù #xiòu
|
||
U+7607 · zh·ng #zh·ng *** I suspect zh·ng
|
||
U+7674 · luán #lüán
|
||
U+7867 · y·ng #i·ng
|
||
U+7878 · nüè #nuè
|
||
*/
|
||
|
||
static Transliterator fixTypos = Transliterator.createFromRules("fix_typos",
|
||
"$cons=[bcdfghjklmnpqrstvwxyz];"
|
||
+"$nlet=[^[:Letter:][:Mark:]];"
|
||
+"$cons{iou}$nlet > iu;"
|
||
+"$cons{em}$nlet > an;"
|
||
+"$cons{uen}$nlet > ueng;"
|
||
+"$cons{ve}$nlet > üe;"
|
||
+"$cons{v}$nlet > ü;"
|
||
+"$cons{yue}$nlet > iu;"
|
||
+"$cons{yng}$nlet > ing;"
|
||
+"$cons{yu}$nlet > iu;"
|
||
//+"$cons{ue} > üe;"
|
||
+"jj > j;"
|
||
//+"$nlet{ng}$nlet > eng;"
|
||
//+"$nlet{n}$nlet > en;"
|
||
//+"$nlet{m}$nlet > en;"
|
||
+"$nlet{au}$nlet > ao;"
|
||
|
||
// new fixes
|
||
+"zhueng}$nlet > zhong;"
|
||
+"zhuen}$nlet > zhuan;"
|
||
+"lue > lüe;"
|
||
+"liong > liang;"
|
||
+"nue > nüe;"
|
||
+"chua > chuo;"
|
||
+"yian > yan;"
|
||
+"yie > ye;"
|
||
+"lüan > luan;"
|
||
+"iong > yong;"
|
||
, Transliterator.FORWARD);
|
||
|
||
|
||
static String fixPinyin(String s) {
|
||
String original = s;
|
||
//err.println("Source: " + s);
|
||
s = accentPinyin_digitPinyin.transliterate(s);
|
||
//err.println("Digit: " + s);
|
||
s = fixTypos.transliterate(s);
|
||
//err.println("fixed: " + s);
|
||
s = digitPinyin_accentPinyin.transliterate(s);
|
||
//err.println("Result: " + s);
|
||
if (isValidPinyin(s)) return s;
|
||
return original;
|
||
}
|
||
|
||
static PrintWriter log;
|
||
static PrintWriter out;
|
||
static PrintWriter err;
|
||
|
||
static int count;
|
||
static int totalCount;
|
||
static int oldLine;
|
||
|
||
static void readFrequencyData(int type) throws java.io.IOException {
|
||
String line = "";
|
||
try {
|
||
|
||
// chinese_frequency.txt
|
||
// 1 çš„ 1588561 1588561 3.5008%
|
||
// japanese_frequency.txt
|
||
// 1 ? 17176
|
||
|
||
Set combinedRank = new TreeSet();
|
||
BufferedReader br;
|
||
int counter = 0;
|
||
Iterator it;
|
||
|
||
if (type == CHINESE) {
|
||
System.out.println("Reading chinese_frequency.txt");
|
||
br = Utility.openReadFile(BASE_DIR + "dict\\chinese_frequency.txt", Utility.UTF8);
|
||
counter = 0;
|
||
while (true) {
|
||
line = Utility.readDataLine(br);
|
||
if (line == null) break;
|
||
if (line.length() == 0) continue;
|
||
Utility.dot(counter++);
|
||
int tabPos = line.indexOf('\t');
|
||
int rank = Integer.parseInt(line.substring(0,tabPos));
|
||
int cp = line.charAt(tabPos+1);
|
||
//if ((rank % 100) == 0) System.out.println(rank + ", " + Utility.hex(cp));
|
||
combinedRank.add(new Pair(new Integer(rank), UTF16.valueOf(cp)));
|
||
}
|
||
br.close();
|
||
}
|
||
|
||
if (type == JAPANESE) {
|
||
System.out.println("Reading japanese_frequency.txt");
|
||
|
||
br = Utility.openReadFile( BASE_DIR + "dict\\japanese_frequency.txt", Utility.UTF8);
|
||
Map japaneseMap = new HashMap();
|
||
while (true) {
|
||
line = Utility.readDataLine(br);
|
||
if (line == null) break;
|
||
if (line.length() == 0) continue;
|
||
Utility.dot(counter++);
|
||
int tabPos = line.indexOf(' ');
|
||
|
||
int tabPos2 = line.indexOf(' ', tabPos+1);
|
||
int freq = Integer.parseInt(line.substring(tabPos2+1));
|
||
|
||
for (int i = tabPos+1; i < tabPos2; ++i) {
|
||
int cp = line.charAt(i);
|
||
int script = Default.ucd().getScript(cp);
|
||
if (script != HAN_SCRIPT) {
|
||
if (script != HIRAGANA_SCRIPT && script != KATAKANA_SCRIPT
|
||
&& cp != 0x30FB && cp != 0x30FC) {
|
||
System.out.println("Huh: " + Default.ucd().getCodeAndName(cp));
|
||
}
|
||
continue;
|
||
}
|
||
// if ((rank % 100) == 0) System.out.println(rank + ", " + Utility.hex(cp));
|
||
Utility.addCount(japaneseMap, UTF16.valueOf(cp), -freq);
|
||
}
|
||
}
|
||
br.close();
|
||
// get rank order japanese
|
||
it = japaneseMap.keySet().iterator();
|
||
int countJapanese = 0;
|
||
while (it.hasNext()) {
|
||
Comparable key = (Comparable) it.next();
|
||
Comparable val = (Comparable) japaneseMap.get(key);
|
||
combinedRank.add(new Pair(new Integer(++countJapanese), key));
|
||
}
|
||
|
||
}
|
||
|
||
|
||
int overallRank = 0;
|
||
it = combinedRank.iterator();
|
||
|
||
boolean showFrequency = false;
|
||
|
||
if (showFrequency) {
|
||
log.println();
|
||
log.println("@Frequency data: Rank of Character");
|
||
log.println();
|
||
}
|
||
|
||
// make up rankMap, rankList
|
||
|
||
while(it.hasNext()) {
|
||
Pair p = (Pair) it.next();
|
||
if (showFrequency) log.println(p.first + ", " + p.second);
|
||
Object rank = rankMap.get(p.second);
|
||
if (rank == null) {
|
||
rankMap.put(p.second, new Integer(++overallRank));
|
||
rankList.add(p.second);
|
||
}
|
||
}
|
||
|
||
if (showFrequency) {
|
||
log.println();
|
||
log.println("@Frequency data: Character to Rank");
|
||
log.println();
|
||
|
||
// get full order
|
||
it = rankList.iterator();
|
||
while (it.hasNext()) {
|
||
Comparable key = (Comparable) it.next();
|
||
Comparable val = (Comparable) rankMap.get(key);
|
||
log.println(key + ", " + val);
|
||
}
|
||
}
|
||
|
||
} catch (Exception e) {
|
||
throw new ChainException("Line \"{0}\"", new String[] {line}, e);
|
||
}
|
||
}
|
||
|
||
static void compareUnihanWithCEDICT() {
|
||
System.out.println("@Comparing CEDICT to Unihan");
|
||
log.println("@Comparing CEDICT to Unihan");
|
||
Iterator it = unihanMap.keySet().iterator();
|
||
List inCEDICT = new ArrayList();
|
||
List inUnihan = new ArrayList();
|
||
List inBoth = new ArrayList();
|
||
UnicodeSet noPinyin = new UnicodeSet();
|
||
UnicodeSet kPinyin = new UnicodeSet();
|
||
UnicodeSet tPinyin = new UnicodeSet();
|
||
UnicodeSet sPinyin = new UnicodeSet();
|
||
|
||
for (int i = 0; i < 0x10FFFF; ++i) {
|
||
if (!Default.ucd().isAllocated(i)) continue;
|
||
if (Default.ucd().getScript(i) != HAN_SCRIPT) continue;
|
||
Utility.dot(i);
|
||
|
||
String ch = UTF16.valueOf(i);
|
||
|
||
String pinyin = (String) unihanMap.get(ch);
|
||
if (pinyin == null) {
|
||
String ch2 = Default.nfkd().normalize(ch);
|
||
pinyin = (String) unihanMap.get(ch2);
|
||
if (pinyin != null) {
|
||
addCheck(ch, pinyin, "n/a");
|
||
kPinyin.add(i);
|
||
} else {
|
||
String trial = (String) simplifiedToTraditional.get(ch2);
|
||
if (trial != null) {
|
||
pinyin = (String) unihanMap.get(trial);
|
||
if (pinyin != null) {
|
||
addCheck(ch, pinyin, "n/a");
|
||
tPinyin.add(i);
|
||
} else {
|
||
trial = (String) traditionalToSimplified.get(ch2);
|
||
if (trial != null) {
|
||
pinyin = (String) unihanMap.get(trial);
|
||
if (pinyin != null) {
|
||
addCheck(ch, pinyin, "n/a");
|
||
sPinyin.add(i);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
Map pinyinSet = (Map) cdict.get(ch);
|
||
if (pinyin == null) {
|
||
if (pinyinSet != null) inCEDICT.add(ch + " => " + pinyinSet);
|
||
noPinyin.add(i);
|
||
} else if (pinyinSet == null) {
|
||
inUnihan.add(ch + " => " + pinyin);
|
||
} else {
|
||
Object temp = pinyinSet.get(pinyin);
|
||
if (temp == null) {
|
||
inBoth.add(ch + " => " + pinyin + "; " + pinyinSet);
|
||
}
|
||
}
|
||
}
|
||
|
||
log.println("@In CEDICT but not Unihan: ");
|
||
printCollection(log, inCEDICT);
|
||
|
||
log.println("@In Unihan but not CEDICT: ");
|
||
printCollection(log, inUnihan);
|
||
|
||
log.println("@In Unihan and CEDICT, but different: ");
|
||
printCollection(log, inBoth);
|
||
|
||
log.println("@Missing from Unihan: ");
|
||
log.println(noPinyin.toPattern(true));
|
||
|
||
log.println("@Has mapping if we NFKD it: ");
|
||
log.println(kPinyin.toPattern(true));
|
||
|
||
log.println("@Has mapping if we NFKC & simp-trad it: ");
|
||
log.println(tPinyin.toPattern(true));
|
||
|
||
log.println("@Has mapping if we NFKC & trad-simp it: ");
|
||
log.println(sPinyin.toPattern(true));
|
||
|
||
log.println("@Done comparison");
|
||
}
|
||
|
||
static void printCollection(PrintWriter p, Collection c) {
|
||
Iterator it = c.iterator();
|
||
int count = 0;
|
||
while (it.hasNext()) {
|
||
p.println((++count) + "\t" + it.next());
|
||
}
|
||
}
|
||
|
||
|
||
static Map rankMap = new TreeMap(); // maps from single char strings to overall rank
|
||
static List rankList = new ArrayList(10000);
|
||
|
||
// form: ???? [ai4 wu1 ji2 wu1] /love me/love my dog/
|
||
|
||
static void readCDICTDefinitions(int type) throws IOException {
|
||
String fname = "cdict.txt";
|
||
if (type == JAPANESE) fname = "edict.txt";
|
||
|
||
System.out.println("Reading " + fname);
|
||
BufferedReader br = Utility.openReadFile(BASE_DIR + "dict\\" + fname, Utility.UTF8);
|
||
int counter = 0;
|
||
String[] pieces = new String[50];
|
||
String line = "";
|
||
String definition;
|
||
try {
|
||
while (true) {
|
||
line = Utility.readDataLine(br);
|
||
if (line == null) break;
|
||
if (line.length() == 0) continue;
|
||
Utility.dot(counter++);
|
||
|
||
|
||
int pinyinStart = line.indexOf('[');
|
||
int pinyinEnd = line.indexOf(']', pinyinStart+1);
|
||
int defStart = line.indexOf('/', pinyinEnd+1);
|
||
int defEnd = line.indexOf('/', defStart+1);
|
||
|
||
int firstData = pinyinStart >= 0 ? pinyinStart : defStart;
|
||
|
||
String word = line.substring(0,firstData).trim();
|
||
|
||
if (type == DEFINITION) {
|
||
definition = fixDefinition(line.substring(defStart+1, defEnd), line);
|
||
addCheck(word, definition, line);
|
||
} else if (pinyinStart >= 0) {
|
||
definition = line.substring(pinyinStart+1, pinyinEnd).trim();
|
||
if (type == JAPANESE) {
|
||
processEdict(word, definition, line);
|
||
} else {
|
||
definition = digitToPinyin(definition, line);
|
||
//definition = Utility.replace(definition, " ", "\\ ");
|
||
addCheck(word, definition, line);
|
||
}
|
||
}
|
||
}
|
||
br.close();
|
||
} catch (Exception e) {
|
||
throw new ChainException("{0} Failed at {1}" , new Object []{new Integer(counter), line}, e);
|
||
}
|
||
}
|
||
|
||
static void readOverrides(int type) throws IOException {
|
||
if (type != CHINESE) return;
|
||
String fname = "Chinese_override.txt";
|
||
|
||
System.out.println("Reading " + fname);
|
||
BufferedReader br = Utility.openReadFile(BASE_DIR + "dict\\" + fname, Utility.UTF8);
|
||
int counter = 0;
|
||
String[] pieces = new String[50];
|
||
String line = "";
|
||
boolean noOverrideFailure = true;
|
||
try {
|
||
while (true) {
|
||
line = Utility.readDataLine(br);
|
||
if (line == null) break;
|
||
if (line.length() == 0) continue;
|
||
Utility.dot(counter++);
|
||
//System.out.println(line);
|
||
|
||
// skip code
|
||
line=line.toLowerCase();
|
||
|
||
int wordStart = line.indexOf('\t') + 1;
|
||
int wordEnd = line.indexOf('\t', wordStart);
|
||
String word = line.substring(wordStart, wordEnd);
|
||
String definition = fixPinyin(line.substring(wordEnd+1));
|
||
String old = (String) unihanMap.get(word);
|
||
if (old != null) {
|
||
if (!old.equals(definition)) {
|
||
if (noOverrideFailure) {
|
||
System.out.println("Overriding Failure");
|
||
noOverrideFailure = false;
|
||
}
|
||
err.println("Overriding Failure: " + word
|
||
+ "\t" + old + " " + toHexUnicode.transliterate(old)
|
||
+ "\t" + definition + " " + toHexUnicode.transliterate(definition));
|
||
}
|
||
} else {
|
||
addCheck(word, definition, line);
|
||
overrideSet.add(word);
|
||
}
|
||
}
|
||
br.close();
|
||
} catch (Exception e) {
|
||
throw new ChainException("{0} Failed at {1}" , new Object []{new Integer(counter), line}, e);
|
||
}
|
||
}
|
||
|
||
|
||
/*
|
||
@Unihan Data
|
||
|
||
Bad pinyin data: \u4E7F ? LE
|
||
\u7684 ? de, de, dí, dì
|
||
*/
|
||
|
||
static void fixChineseOverrides() throws IOException {
|
||
|
||
log = Utility.openPrintWriter("Transliterate_log.txt", Utility.UTF8_WINDOWS);
|
||
out = Utility.openPrintWriter("new_Chinese_override.txt", Utility.UTF8_WINDOWS);
|
||
try {
|
||
|
||
String fname = "fixed_Chinese_transliterate_log.txt";
|
||
|
||
int counter = 0;
|
||
String line = "";
|
||
String pinyinPrefix = "Bad pinyin data: ";
|
||
|
||
System.out.println("Reading " + fname);
|
||
BufferedReader br = Utility.openReadFile(BASE_DIR + "dict\\" + fname, Utility.UTF8);
|
||
try {
|
||
while (true) {
|
||
line = Utility.readDataLine(br);
|
||
if (line == null) break;
|
||
if (line.length() == 0) continue;
|
||
if (line.charAt(0) == 0xFEFF) {
|
||
line = line.substring(1); // remove BOM
|
||
if (line.length() == 0) continue;
|
||
}
|
||
Utility.dot(counter++);
|
||
|
||
|
||
if (line.charAt(0) == '@') continue;
|
||
if (line.startsWith(pinyinPrefix)) {
|
||
line = line.substring(pinyinPrefix.length());
|
||
}
|
||
line = line.toLowerCase();
|
||
|
||
//System.out.println(Default.ucd.getCode(line));
|
||
// skip code
|
||
int wordStart = line.indexOf('\t') + 1;
|
||
int wordEnd = line.indexOf('\t', wordStart);
|
||
String word = line.substring(wordStart, wordEnd).trim();
|
||
|
||
int defStart = wordEnd+1;
|
||
int defEnd = line.indexOf(',', defStart);
|
||
if (defEnd < 0) defEnd = line.length();
|
||
|
||
String definition = fixCircumflex.transliterate(line.substring(defStart, defEnd).trim());
|
||
|
||
String notones = dropTones.transliterate(definition);
|
||
if (definition.equals(notones)) {
|
||
definition = digitPinyin_accentPinyin.transliterate(definition + "1");
|
||
if (definition == null) {
|
||
System.out.println("Huh? " + notones);
|
||
}
|
||
log.println("Fixing: " + notones + " => " + definition + "; " + line);
|
||
}
|
||
|
||
out.println(hex.transliterate(word) + "\t" + word + "\t" + definition);
|
||
}
|
||
} catch (Exception e) {
|
||
throw new ChainException("{0} Failed at {1}" , new Object []{new Integer(counter), line}, e);
|
||
} finally {
|
||
br.close();
|
||
}
|
||
} finally {
|
||
out.close();
|
||
}
|
||
}
|
||
|
||
|
||
|
||
static Set overrideSet = new HashSet();
|
||
|
||
static void processEdict(String word, String definition, String line) {
|
||
// We have a situation where we have words of the form CCCHHHKKKCCHHCCH > HHHHHHKKKHHHHHHHH
|
||
// C = CJK, H = Hiragana, K = katakana
|
||
|
||
// We want to break those up into the following rules.
|
||
// { CCC } HHHKKKCCCHH => HHH
|
||
// CCCHHHKKK { CC } HHCCH => HH
|
||
// CCCHHHKKKCCHH { CC } H => HH
|
||
|
||
int[] offset = {0};
|
||
int[] offset2 = {0};
|
||
int[][] pairList = new int[50][2];
|
||
int pairCount = 0;
|
||
|
||
// first gather the information as to where the CJK blocks are
|
||
// do this all at once, so we can refer to stuff ahead of us
|
||
while (true) {
|
||
// find next CJK block
|
||
// where CJK really means anything but kana
|
||
int type = find(word, kana, offset, offset2, word.length(), false, false);
|
||
if (type == UnicodeMatcher.U_MISMATCH) break; // we are done.
|
||
pairList[pairCount][0] = offset[0];
|
||
pairList[pairCount++][1] = offset2[0];
|
||
offset[0] = offset2[0]; // get ready for the next one
|
||
}
|
||
|
||
// IF we only got one CJK block, and it goes from the start to the end, then just do it.
|
||
|
||
if (pairCount == 1 && pairList[0][0] == 0 && pairList[0][1] == word.length()) {
|
||
addCheck(word, kanaToLatin.transliterate(definition), line);
|
||
return;
|
||
}
|
||
|
||
// IF we didn't find any Kanji, bail.
|
||
|
||
if (pairCount < 1) {
|
||
System.out.println("No Kanji on line, skipping");
|
||
System.out.println(hex.transliterate(word) + " > " + hex.transliterate(definition)
|
||
+ ", " + kanaToLatin.transliterate(definition));
|
||
return;
|
||
}
|
||
|
||
// Now generate the rules
|
||
|
||
|
||
if (DEBUG && pairCount > 1) {
|
||
System.out.println("Paircount: " + pairCount);
|
||
System.out.println("\t" + hex.transliterate(word) + " > " + hex.transliterate(definition) + ", " + kanaToLatin.transliterate(definition));
|
||
}
|
||
|
||
pairList[pairCount][0] = word.length(); // to make the algorithm easier, we add a termination
|
||
int delta = 0; // the current difference in positions between the definition and the word
|
||
|
||
for (int i = 0; i < pairCount; ++i) {
|
||
int start = pairList[i][0];
|
||
int limit = pairList[i][1];
|
||
if (DEBUG && pairCount > 1) System.out.println(start + ", " + limit + ", " + delta);
|
||
|
||
// that part was easy. the hard part is figuring out where this corresponds to in the definition.
|
||
// For now, we use a simple mechanism.
|
||
|
||
// The word and the definition should match to this point, so we just use the start (offset by delta)
|
||
// We'll check just to be sure.
|
||
|
||
int lastLimit = i == 0 ? 0 : pairList[i-1][1];
|
||
|
||
int defStart = start + delta;
|
||
|
||
String defPrefix = definition.substring(0, defStart);
|
||
String wordInfix = word.substring(lastLimit, start);
|
||
|
||
boolean firstGood = defPrefix.endsWith(wordInfix);
|
||
if (!firstGood) {
|
||
String wordInfix2 = katakanatoHiragana.transliterate(wordInfix);
|
||
firstGood = defPrefix.endsWith(wordInfix2);
|
||
}
|
||
if (!firstGood) {
|
||
// Houston, we have a problem.
|
||
Utility.fixDot();
|
||
System.out.println("Suspect line: " + hex.transliterate(word) + " > " + hex.transliterate(definition)
|
||
+ ", " + kanaToLatin.transliterate(definition));
|
||
System.out.println("\tNo match for " + hex.transliterate(word.substring(lastLimit, start))
|
||
+ " at end of " + hex.transliterate(definition.substring(0, defStart)));
|
||
break; // BAIL
|
||
}
|
||
|
||
// For the limit of the defintion, we get the intermediate portion of the word
|
||
// then search for it in the definition.
|
||
// We could get tripped up if the end of the transliteration of the Kanji matched the start.
|
||
// If so, we should find out on the next pass.
|
||
|
||
int defLimit;
|
||
if (limit == word.length()) {
|
||
defLimit = definition.length();
|
||
} else {
|
||
String afterPart = word.substring(limit, pairList[i+1][0]);
|
||
defLimit = definition.indexOf(afterPart, defStart+1); // we assume the CJK is at least one!
|
||
if (defLimit < 0) {
|
||
String afterPart2 = katakanatoHiragana.transliterate(afterPart);
|
||
defLimit = definition.indexOf(afterPart2, defStart+1); // we assume the CJK is at least one!
|
||
}
|
||
|
||
if (defLimit < 0) {
|
||
// Houston, we have a problem.
|
||
Utility.fixDot();
|
||
System.out.println("Suspect line: " + hex.transliterate(word) + " > " + hex.transliterate(definition)
|
||
+ ", " + kanaToLatin.transliterate(definition));
|
||
System.out.println("\tNo match for " + hex.transliterate(afterPart)
|
||
+ " in " + hex.transliterate(definition.substring(0, defStart+1)));
|
||
}
|
||
break;
|
||
}
|
||
|
||
String defPart = definition.substring(defStart, defLimit);
|
||
defPart = kanaToLatin.transliterate(defPart);
|
||
|
||
// FOR NOW, JUNK the context before!!
|
||
// String contextWord = word.substring(0, start) + "{" + word.substring(start, limit) + "}" + word.substring(limit);
|
||
String contextWord = word.substring(start, limit);
|
||
if (limit != word.length()) contextWord += "}" + word.substring(limit);
|
||
|
||
addCheck(contextWord, defPart, line);
|
||
if (DEBUG && pairCount > 1) System.out.println("\t" + hex.transliterate(contextWord) + " > " + hex.transliterate(defPart));
|
||
|
||
delta = defLimit - limit;
|
||
}
|
||
|
||
}
|
||
|
||
// Useful Utilities?
|
||
|
||
/**
|
||
* Returns the start of the first substring that matches m.
|
||
* Most arguments are the same as UnicodeMatcher.matches, except for offset[]
|
||
* @positive Use true if you want the first point that matches, and false if you want the first point that doesn't match.
|
||
* @offset On input, the starting position. On output, the start of the match position (not the end!!)
|
||
*/
|
||
static int find(Replaceable s, UnicodeMatcher m, int[] offset, int limit, boolean incremental, boolean positive) {
|
||
int direction = offset[0] <= limit ? 1 : -1;
|
||
|
||
|
||
while (offset[0] != limit) {
|
||
int original = offset[0];
|
||
int type = m.matches(s, offset, limit, incremental); // if successful, changes offset.
|
||
if (type == UnicodeMatcher.U_MISMATCH) {
|
||
if (!positive) {
|
||
return UnicodeMatcher.U_MATCH;
|
||
}
|
||
offset[0] += direction; // used to skip to next code unit, in the positive case
|
||
// !! This should be safe, and saves checking the length of the code point
|
||
} else if (positive) {
|
||
offset[0] = original; // reset to the start position!!!
|
||
return type;
|
||
}
|
||
}
|
||
return UnicodeMatcher.U_MISMATCH;
|
||
}
|
||
|
||
/**
|
||
* Returns the start/limit of the first substring that matches m. Most arguments are the same as find().<br>
|
||
* <b>Warning:</b> if the search is backwards, then substringEnd will contain the <i>start</i> of the substring
|
||
* and offset will contain the </i>limit</i> of the substring.
|
||
*/
|
||
static int find(Replaceable s, UnicodeMatcher m, int[] offset, int[] offset2, int limit, boolean incremental, boolean positive) {
|
||
int type = find(s, m, offset, limit, incremental, positive);
|
||
if (type == UnicodeMatcher.U_MISMATCH) return type;
|
||
offset2[0] = offset[0];
|
||
int type2 = find(s, m, offset2, limit, incremental, !positive);
|
||
return type;
|
||
}
|
||
|
||
static int find(String ss, UnicodeMatcher m, int[] offset, int limit, boolean incremental, boolean positive) {
|
||
// UGLY that we have to create a wrapper!
|
||
return find(new ReplaceableString(ss), m, offset, limit, incremental, positive);
|
||
}
|
||
|
||
static int find(String ss, UnicodeMatcher m, int[] offset, int[] offset2, int limit, boolean incremental, boolean positive) {
|
||
// UGLY that we have to create a wrapper!
|
||
return find(new ReplaceableString(ss), m, offset, offset2, limit, incremental, positive);
|
||
}
|
||
|
||
static UnicodeSet pua = new UnicodeSet("[:private use:]");
|
||
static UnicodeSet numbers = new UnicodeSet("[0-9]");
|
||
|
||
static void addCheck(String word, String definition, String line) {
|
||
int lastSlash = 0;
|
||
while (lastSlash < word.length()) {
|
||
int wordSlash = word.indexOf('/', lastSlash);
|
||
if (wordSlash < 0) wordSlash = word.length();
|
||
addCheck2(word.substring(lastSlash, wordSlash), definition, line);
|
||
lastSlash = wordSlash + 1;
|
||
}
|
||
}
|
||
|
||
static void addCheck2(String word, String definition, String line) {
|
||
definition = Default.nfc().normalize(definition);
|
||
word = Default.nfc().normalize(word);
|
||
if (DO_SIMPLE && UTF16.countCodePoint(word) > 1) return;
|
||
|
||
if (pua.containsSome(word) ) {
|
||
Utility.fixDot();
|
||
System.out.println("PUA on: " + line);
|
||
} else if (numbers.containsAll(definition) ) {
|
||
Utility.fixDot();
|
||
System.out.println("Only numbers on: " + line);
|
||
} else {
|
||
Object alreadyThere = unihanMap.get(word);
|
||
if (alreadyThere == null) {
|
||
unihanMap.put(word, definition);
|
||
} else if (!definition.equals(alreadyThere)) {
|
||
Utility.addToList(duplicates, word, alreadyThere, true);
|
||
Utility.addToList(duplicates, word, definition, true);
|
||
}
|
||
}
|
||
if (UTF16.countCodePoint(word) > 1) unihanNonSingular = true;
|
||
}
|
||
|
||
static void readCDICT() throws IOException {
|
||
System.out.println("Reading cdict.txt");
|
||
String fname = "cdict.txt";
|
||
|
||
BufferedReader br = Utility.openReadFile(BASE_DIR + "dict\\" + fname, Utility.UTF8);
|
||
int counter = 0;
|
||
String[] pieces = new String[50];
|
||
String line = "";
|
||
|
||
try {
|
||
while (true) {
|
||
line = Utility.readDataLine(br);
|
||
if (line == null) break;
|
||
if (line.length() == 0) continue;
|
||
Utility.dot(counter++);
|
||
int tabPos = line.indexOf('[');
|
||
String word = line.substring(0,tabPos).trim();
|
||
word = Utility.replace(word, "\uFE4D", "");
|
||
word = Utility.replace(word, ".", "");
|
||
word = Utility.replace(word, "/", "");
|
||
word = Utility.replace(word, "(", "");
|
||
word = Utility.replace(word, ")", "");
|
||
|
||
|
||
int tab2Pos = line.indexOf(']', tabPos+1);
|
||
String pinyins = line.substring(tabPos+1, tab2Pos);
|
||
int len = Utility.split(pinyins, ' ', pieces);
|
||
if (word.length() != len) {
|
||
log.println("Len mismatch: " + line);
|
||
continue;
|
||
}
|
||
for (int i = 0; i < len; ++i) {
|
||
String chr = word.substring(i, i+1);
|
||
|
||
String piece = digitToPinyin(pieces[i], line);
|
||
|
||
Map oldMap = (Map) cdict.get(chr);
|
||
if (oldMap == null) {
|
||
oldMap = new TreeMap();
|
||
cdict.put(chr, oldMap);
|
||
}
|
||
/*&& !oldMap.equals(piece)) {
|
||
log.println("Variant for '" + chr + "', new: '" + piece + "', old: '" + oldMap + "'");
|
||
}
|
||
*/
|
||
Utility.addCount(oldMap, piece, 1);
|
||
}
|
||
}
|
||
br.close();
|
||
|
||
Iterator it = cdict.keySet().iterator();
|
||
Set tempSet = new TreeSet();
|
||
while (it.hasNext()) {
|
||
Object key = it.next();
|
||
Map val = (Map) cdict.get(key);
|
||
log.print(key + ": ");
|
||
Iterator it2 = val.keySet().iterator();
|
||
tempSet.clear();
|
||
while (it2.hasNext()) {
|
||
Comparable key2 = (Comparable) it2.next();
|
||
Comparable count = (Comparable) val.get(key2);
|
||
Pair p = new Pair(count, key2);
|
||
tempSet.add(p); // reverse the order
|
||
}
|
||
it2 = tempSet.iterator();
|
||
int counter2 = 0;
|
||
while (it2.hasNext()) {
|
||
if (counter2++ != 0) log.print("/");
|
||
log.print(it2.next());
|
||
}
|
||
log.println();
|
||
}
|
||
|
||
} catch (Exception e) {
|
||
throw new ChainException("{0} Failed at {1}" , new Object []{new Integer(counter), line}, e);
|
||
}
|
||
}
|
||
|
||
static String digitToPinyin(String source, String line) {
|
||
if (source.indexOf('5') >= 0) log.println("Pinyin Tone5 at: " + line);
|
||
return digitPinyin_accentPinyin.transliterate(source);
|
||
}
|
||
|
||
static Map cdict = new TreeMap();
|
||
static Map simplifiedToTraditional = new HashMap();
|
||
static Map traditionalToSimplified = new HashMap();
|
||
|
||
static UnicodeMap kHanyuPinlu = new UnicodeMap();
|
||
|
||
static void readUnihanData(String key) throws java.io.IOException {
|
||
|
||
BufferedReader in = Utility.openUnicodeFile("Unihan", Default.ucdVersion(), true, Utility.UTF8);
|
||
|
||
int count = 0;
|
||
int lineCounter = 0;
|
||
|
||
while (true) {
|
||
Utility.dot(++lineCounter);
|
||
|
||
String line = in.readLine();
|
||
if (line == null) break;
|
||
if (line.length() < 6) continue;
|
||
if (line.charAt(0) == '#') continue;
|
||
line = line.trim();
|
||
|
||
int tabPos = line.indexOf('\t');
|
||
int tabPos2 = line.indexOf('\t', tabPos+1);
|
||
|
||
String scode = line.substring(2, tabPos).trim();
|
||
|
||
int code = Integer.parseInt(scode, 16);
|
||
String property = line.substring(tabPos+1, tabPos2).trim();
|
||
|
||
String propertyValue = line.substring(tabPos2+1).trim();
|
||
if (propertyValue.indexOf("U+") >= 0) propertyValue = fromHexUnicode.transliterate(propertyValue);
|
||
|
||
// gather traditional mapping
|
||
if (property.equals("kTraditionalVariant")) {
|
||
simplifiedToTraditional.put(UTF16.valueOf(code), propertyValue);
|
||
}
|
||
|
||
if (property.equals("kSimplifiedVariant")) {
|
||
traditionalToSimplified.put(UTF16.valueOf(code), propertyValue);
|
||
}
|
||
|
||
if (key.equals("kMandarin") && property.equals("kHanyuPinlu")) {
|
||
// U+64D4 kHanyuPinlu dan1(297), dan4(61), dan5(36)
|
||
String[] piece = Utility.split(propertyValue,'(');
|
||
String pinyin = digitToPinyin(piece[0], line);
|
||
log.println(scode + "\t" + pinyin + "\t" + line);
|
||
kHanyuPinlu.put(Integer.parseInt(scode,16), pinyin);
|
||
}
|
||
if (property.equals(key)
|
||
|| key.equals("kJapaneseOn") && property.equals("kJapaneseKun")
|
||
) {
|
||
storeDef(out, code, propertyValue, line);
|
||
}
|
||
}
|
||
|
||
in.close();
|
||
}
|
||
|
||
static void storeDef(PrintWriter out, int cp, String rawDefinition, String line) {
|
||
// skip spaces & numbers at start
|
||
int start;
|
||
for (start = 0;start < rawDefinition.length(); ++start) {
|
||
char ch = rawDefinition.charAt(start);
|
||
if (ch != ' ' && ch != '\t' && (ch < '0' || ch > '9')) break;
|
||
}
|
||
|
||
// go up to comma or semicolon, whichever is earlier
|
||
int end = rawDefinition.indexOf(";", start);
|
||
if (end < 0) end = rawDefinition.length();
|
||
|
||
int end2 = rawDefinition.indexOf(",", start);
|
||
if (end2 < 0) end2 = rawDefinition.length();
|
||
if (end > end2) end = end2;
|
||
|
||
// IF CHINESE or JAPANESE, stop at first space!!!
|
||
rawDefinition = rawDefinition.substring(start,end);
|
||
|
||
if (type == DEFINITION) {
|
||
storeDef2(out, cp, rawDefinition, line);
|
||
} else {
|
||
if (rawDefinition.indexOf(' ') < 0) storeDef2(out, cp, rawDefinition, line);
|
||
else {
|
||
String [] pieces = Utility.split(rawDefinition, ' ');
|
||
for (int i = 0; i < pieces.length; ++i) {
|
||
storeDef2(out, cp, pieces[i], line);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
static void storeDef2(PrintWriter out, int cp, String definition, String line) {
|
||
if (type == CHINESE) {
|
||
// since data are messed up, terminate after first digit
|
||
int end3 = findInString(definition, "12345")+1;
|
||
if (end3 == 0) {
|
||
log.println("Bad pinyin data: " + hex.transliterate(UTF16.valueOf(cp))
|
||
+ "\t" + UTF16.valueOf(cp) + "\t" + definition);
|
||
end3 = definition.length();
|
||
}
|
||
definition = definition.substring(0, end3);
|
||
|
||
definition = digitToPinyin(definition, line);
|
||
out2.println(Utility.hex(cp) + '\t' + UTF16.valueOf(cp) + "\t" + definition.toLowerCase());
|
||
}
|
||
if (type == DEFINITION) {
|
||
definition = removeMatched(definition,'(', ')', line);
|
||
definition = removeMatched(definition,'[', ']', line);
|
||
definition = fixDefinition(definition, line);
|
||
}
|
||
definition = definition.trim();
|
||
definition = Default.ucd().getCase(definition, FULL, LOWER);
|
||
|
||
if (definition.length() == 0) {
|
||
Utility.fixDot();
|
||
err.println("Zero value for " + Default.ucd().getCode(cp) + " on: " + hex.transliterate(line));
|
||
} else {
|
||
addCheck(UTF16.valueOf(cp), definition, line);
|
||
}
|
||
/*
|
||
String key = (String) unihanMap.get(definition);
|
||
if (key == null) {
|
||
unihanMap.put(definition, cp);
|
||
}
|
||
out.println(cp + (key == null ? " <> " : " > ") + Default.ucd.getCase(definition, FULL, TITLE) + ";");
|
||
if (TESTING) System.out.println("# " + code + " > " + definition);
|
||
*/
|
||
}
|
||
|
||
static String fixDefinition(String definition, String rawDefinition) {
|
||
definition = definition.trim();
|
||
definition = Utility.replace(definition, " ", " ");
|
||
definition = Utility.replace(definition, " ", "-");
|
||
definition = Default.ucd().getCase(definition, FULL, LOWER);
|
||
return definition;
|
||
}
|
||
|
||
|
||
// WARNING not supplemenatary-safe!
|
||
|
||
static int findInString(String source, String chars) {
|
||
for (int i = 0; i < source.length(); ++i) {
|
||
if (chars.indexOf(source.charAt(i)) >= 0) return i;
|
||
}
|
||
return -1;
|
||
}
|
||
|
||
// WARNING not supplemenatary-safe!
|
||
|
||
static String removeMatched(String source, char start, char end, String originalLine) {
|
||
while (true) {
|
||
int pos = source.indexOf(start);
|
||
if (pos < 0) break;
|
||
int epos = source.indexOf(end, pos+1);
|
||
if (epos < 0) {
|
||
epos = source.length()-1;
|
||
log.println("Mismatches with " + start + ", " + end + ": " + originalLine);
|
||
}
|
||
source = source.substring(0,pos) + source.substring(epos+1);
|
||
}
|
||
return source;
|
||
}
|
||
|
||
static Map unihanMap = new TreeMap(); // could be hashmap
|
||
static Map duplicates = new TreeMap();
|
||
|
||
static boolean unihanNonSingular = false;
|
||
|
||
static StringBuffer handlePinyinTemp = new StringBuffer();
|
||
|
||
static final Transliterator hex = Transliterator.getInstance("[^\\u0020-\\u007F] hex");
|
||
static final Transliterator quoteNonLetters = Transliterator.createFromRules("any-quotenonletters",
|
||
"([[\\u0020-\\u007E]-[:L:]-[\\'\\{\\}]-[0-9]]) > \\u005C $1; "
|
||
+ "\\' > \\'\\';",
|
||
Transliterator.FORWARD);
|
||
static final Transliterator toSub = Transliterator.createFromRules("any-subscript",
|
||
" 0 > \u2080; "
|
||
+ " 1 > \u2081; "
|
||
+ " 2 > \u2082; "
|
||
+ " 3 > \u2084; "
|
||
+ " 4 > \u2084; "
|
||
+ " 5 > \u2085; "
|
||
+ " 6 > \u2086; "
|
||
+ " 7 > \u2087; "
|
||
+ " 8 > \u2088; "
|
||
+ " 9 > \u2089; ",
|
||
Transliterator.FORWARD);
|
||
|
||
static final Transliterator kanaToLatin = Transliterator.createFromRules("any-subscript",
|
||
" $kata = [[:katakana:]\u30FC]; "
|
||
+ "[:hiragana:] {} [:^hiragana:] > ' '; "
|
||
+ "$kata {} [^[:hiragana:]$kata] > ' '; "
|
||
+ "::Katakana-Latin; "
|
||
+ "::Hiragana-Latin;",
|
||
Transliterator.FORWARD);
|
||
|
||
static final Transliterator katakanatoHiragana = Transliterator.getInstance("katakana-hiragana");
|
||
|
||
static final UnicodeSet kana = new UnicodeSet("[[:hiragana:][:katakana:]\u30FC]");
|
||
// since we are working in NFC, we don't worry about the combining marks.
|
||
|
||
// ADD Factory since otherwise getInverse blows out
|
||
static class DummyFactory implements Transliterator.Factory {
|
||
static DummyFactory singleton = new DummyFactory();
|
||
static HashMap m = new HashMap();
|
||
|
||
// Since Transliterators are immutable, we don't have to clone on set & get
|
||
static void add(String ID, Transliterator t) {
|
||
m.put(ID, t);
|
||
System.out.println("Registering: " + ID + ", " + t.toRules(true));
|
||
Transliterator.registerFactory(ID, singleton);
|
||
}
|
||
public Transliterator getInstance(String ID) {
|
||
return (Transliterator) m.get(ID);
|
||
}
|
||
}
|
||
|
||
static Transliterator digitPinyin_accentPinyin;
|
||
|
||
static Transliterator accentPinyin_digitPinyin = Transliterator.createFromRules("accentPinyin_digitPinyin",
|
||
"::NFD; "
|
||
+ " ([\u0304\u0301\u030C\u0300\u0306]) ([[:Mark:][:Letter:]]+) > $2 | $1;"
|
||
+ "\u0304 > '1'; \u0301 > '2'; \u030C > '3'; \u0300 > '4'; \u0306 > '3';"
|
||
+ " ::NFC;", Transliterator.FORWARD);
|
||
|
||
static Transliterator fixCircumflex = Transliterator.createFromRules("fix_circumflex",
|
||
"::NFD; \u0306 > \u030C; ::NFC;", Transliterator.FORWARD);
|
||
|
||
static Transliterator dropTones = Transliterator.createFromRules("drop_tones",
|
||
"::NFD; \u0304 > ; \u0301 > ; \u030C > ; \u0300 > ; \u0306 > ; ::NFC;", Transliterator.FORWARD);
|
||
|
||
static {
|
||
String dt = "1 > \u0304;\n"
|
||
+ "2 <> \u0301;\n"
|
||
+ "3 <> \u030C;\n"
|
||
+ "4 <> \u0300;\n"
|
||
+ "5 <> ;";
|
||
|
||
String dp = "# syllable is ...vowel+ consonant* number\n"
|
||
+ "# 'a', 'e' are the preferred bases\n"
|
||
+ "# otherwise 'o'\n"
|
||
+ "# otherwise last vowel\n"
|
||
+ "::NFC;\n"
|
||
+ "$vowel = [aAeEiIoOuUüÜ];\n"
|
||
+ "$consonant = [[a-z A-Z] - [$vowel]];\n"
|
||
+ "$digit = [1-5];\n"
|
||
+ "([aAeE]) ($vowel* $consonant*) ($digit) > $1 &digit-tone($3) $2;\n"
|
||
+ "([oO]) ([$vowel-[aeAE]]* $consonant*) ($digit) > $1 &digit-tone($3) $2;\n"
|
||
+ "($vowel) ($consonant*) ($digit) > $1 &digit-tone($3) $2;\n"
|
||
+ "($digit) > &digit-tone($1);\n"
|
||
+ "::NFC;\n";
|
||
|
||
Transliterator at = Transliterator.createFromRules("digit-tone", dt, Transliterator.FORWARD);
|
||
System.out.println(at.transliterate("a1a2a3a4a5"));
|
||
DummyFactory.add(at.getID(), at);
|
||
|
||
digitPinyin_accentPinyin = Transliterator.createFromRules("digit-pinyin", dp, Transliterator.FORWARD);
|
||
System.out.println(digitPinyin_accentPinyin.transliterate("an2 aon2 oan2 ion2 oin2 uin2 iun2"));
|
||
|
||
}
|
||
/*
|
||
|
||
static String convertTones(String source, String debugLine) {
|
||
try {
|
||
result = new StringBuffer();
|
||
main:
|
||
for (int i = 0; i < source.length(); ++i) {
|
||
ch = source.charAt(i);
|
||
switch (ch) {
|
||
case ':':
|
||
if (i > 0) {
|
||
char last = result.charAt(result.length()-1);
|
||
if (last == 'u') {
|
||
result.setCharAt(result.length()-1, 'ü');
|
||
continue main;
|
||
} else if (last == 'U') {
|
||
result.setCharAt(result.length()-1, 'Ü');
|
||
continue main;
|
||
}
|
||
}
|
||
break;
|
||
case '1': break; // skip character
|
||
case '2': case '3': case '4': case '5':
|
||
applyToPrecedingBase(result, ch-'0');
|
||
break;
|
||
default:
|
||
result.append(ch);
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
|
||
source = source.trim();
|
||
char ch = source.charAt(source.length()-1);
|
||
int num = (int)(ch-'1');
|
||
if (num < 0 || num > 5) throw new Exception("none");
|
||
handlePinyinTemp.setLength(0);
|
||
boolean gotIt = false;
|
||
boolean messageIfNoGotIt = true;
|
||
|
||
for (int i = source.length()-2; i >= 0; --i) {
|
||
ch = source.charAt(i);
|
||
if (ch == ':') {
|
||
ch = 'Ü';
|
||
--i;
|
||
}
|
||
if ('0' <= ch && ch <= '9') break;
|
||
if (ch != 'Ü' && (ch < 'A' || ch > 'Z')) {
|
||
Utility.fixDot();
|
||
System.out.println("Warning: non-ASCII in " + hex.transliterate(source) + " (" + hex.transliterate(debugLine) + ")");
|
||
break;
|
||
}
|
||
if (!gotIt) switch (ch) {
|
||
case 'A': ch = "AÁ\u0102À\u0100".charAt(num); gotIt = true; break;
|
||
case 'E': ch = "EÉ\u0114È\u0112".charAt(num); gotIt = true; break;
|
||
case 'I': ch = "IÍ\u012CÌ\u012A".charAt(num); gotIt = true; break;
|
||
case 'O': ch = "OÓ\u014EÒ\u014C".charAt(num); gotIt = true; break;
|
||
case 'U': ch = "UÚ\u016CÙ\u016A".charAt(num); gotIt = true; break;
|
||
case 'Ü': ch = "Ü\u01D7\u01D9\u01DB\u01D5".charAt(num); gotIt = true; break;
|
||
}
|
||
handlePinyinTemp.insert(0,ch);
|
||
}
|
||
if (!gotIt && num > 0) {
|
||
handlePinyinTemp.append(" \u0301\u0306\u0300\u0304".charAt(num));
|
||
if (messageIfNoGotIt) {
|
||
err.println("Missing vowel?: " + debugLine + " -> " + handlePinyinTemp
|
||
.toString());
|
||
}
|
||
}
|
||
source = handlePinyinTemp.toString().toLowerCase();
|
||
} catch (Exception e) {
|
||
log.println("Bad line: " + debugLine);
|
||
}
|
||
return source;
|
||
}
|
||
|
||
/*
|
||
A and e trump all other vowels and always take the tone mark.
|
||
There are no Mandarin syllables that contain both a and e.
|
||
In the combination ou, o takes the mark.
|
||
In all other cases, the final vowel takes the mark.
|
||
*/
|
||
/*
|
||
static String applyToPrecedingBase(StringBuffer result, int tone) {
|
||
for (int i = result.length()-1; i >= 0; --i) {
|
||
char ch = result.charAt(i);
|
||
switch (ch) {
|
||
case 'a': case 'e': case 'A': case 'E':
|
||
result.setCharAt(i, mapTone(ch, tone));
|
||
return;
|
||
case 'o': case 'O': bestSoFar = i; break;
|
||
case 'i': case 'I': case 'u': case 'U': case '
|
||
if (tone == 1) return String.valueOf(ch);
|
||
return Default.nfc.normalize(ch + mapTone[tone]);
|
||
}
|
||
|
||
static final char[] MAP_TONE = {"\u0301", "\u0306", "\u0300", "\u0304"};
|
||
*/
|
||
} |