/**
*******************************************************************************
* Copyright (C) 1996-2001, International Business Machines Corporation and    *
* others. All Rights Reserved.                                                *
*******************************************************************************
*
* $Source: /xsrl/Nsvn/icu/unicodetools/com/ibm/text/UCD/GenerateData.java,v $
* $Date: 2005/03/26 05:40:04 $
* $Revision: 1.38 $
*
*******************************************************************************
*/

package com.ibm.text.UCD;

import java.util.*;
import java.io.*;

import com.ibm.text.utility.*;
import com.ibm.icu.dev.test.util.UnicodeProperty;
import com.ibm.icu.text.UTF16;
import com.ibm.icu.text.UnicodeSet;


public class GenerateData implements UCD_Types {
    
    static final boolean DEBUG = false;
    
    static final String HORIZONTAL_LINE = "# ================================================";
    
    static final void genSplit () {
        UnicodeSet split = new UnicodeSet();
        UnicodeSet reordrant = new UnicodeSet(
            "[\u093F\u09BF\u09c7\u09c8\u0abf\u0abf\u0b47\u0bc6\u0bc7\u0bc8"
            + "\u0d46\u0d47\u0d48\u0dd9\u0dda\u0ddb\u1031\u17be\u17c1\u17c2\u17c3]");
        UnicodeSet subjoined = new UnicodeSet();
        for (int i = 0; i <= 0x10FFFF; ++i) {
            if (!Default.ucd().isAssigned(i)) continue;
            Utility.dot(i);
            int cat = Default.ucd().getCategory(i);
            if (cat != Mc && cat != Mn && cat != Me) continue;
            if (Default.ucd().getName(i).indexOf("SUBJOINED") >= 0) {
                System.out.print('*');
                subjoined.add(i);
                continue;
            }
            String decomp = Default.nfd().normalize(i);
            //int count = countTypes(decomp, Mc);
            if (UTF16.countCodePoint(decomp) > 1) split.add(i);
        }
        Utility.fixDot();
        System.out.println("Split: " + split.size());
        Utility.showSetNames("", split, false, Default.ucd());
        
        System.out.println("Reordrant: " + reordrant.size());
        Utility.showSetNames("", reordrant, false, Default.ucd());
        
        System.out.println("Subjoined: " + subjoined.size());
        Utility.showSetNames("", subjoined, false, Default.ucd());
    }
    
    static int countTypes(String s, int filter) {
        int count = 0;
        int cp;
        for (int i = 0; i < s.length(); i+= UTF16.getCharCount(cp)) {
            cp = UTF16.charAt(s, i);
            int cat = Default.ucd().getCategory(i);
            if (cat == filter) count++;
        }
        return count;
    }

    //static UnifiedBinaryProperty ubp
        
    public static void checkHoffman(String test) {
        String result = Default.nfkc().normalize(test);
        System.out.println(Utility.hex(test) + " => " + Utility.hex(result));
        System.out.println();
        show(test, 0);
        System.out.println();
        show(result, 0);
    }

    public static void show(String s, int indent) {
        int cp;
        for (int i = 0; i < s.length(); i += UTF32.count16(cp)) {
            cp = UTF32.char32At(s, i);
            String cc = " " + Default.ucd().getCombiningClass(cp);
            cc = Utility.repeat(" ", 4 - cc.length()) + cc;
            System.out.println(Utility.repeat(" ", indent) + Default.ucd().getCode(cp) + cc + " " + Default.ucd().getName(cp));
            String decomp = Default.nfkc().normalize(cp);
            if (!decomp.equals(UTF32.valueOf32(cp))) {
                show(decomp, indent + 4);
            }
        }
    }


    static final int HEADER_EXTEND = 0, HEADER_DERIVED = 1, HEADER_SCRIPTS = 2;

    public static void doHeader(String fileName, PrintWriter output, int headerChoice) {
        output.println("# " + fileName);
        output.println(UnicodeDataFile.generateDateLine());
        output.println("#");
        if (headerChoice == HEADER_SCRIPTS) {
        } else if (headerChoice == HEADER_EXTEND) {
            output.println("# Unicode Character Database: Extended Properties");
        } else {
            output.println("# Unicode Character Database: Derived Property Data");
            output.println("# Generated algorithmically from the Unicode Character Database");
        }
        output.println("# For documentation, see UCD.html");
        //output.println("# Note: Unassigned and Noncharacter codepoints may be omitted");
        //output.println("#       if they have default property values.");
        output.println(HORIZONTAL_LINE);
        output.println();
    }

    public static void checkDifferences (String targetVersion) throws IOException {
        System.out.println("Checking Differences");
        UCD target = UCD.make(targetVersion);
        
        PrintWriter log1 = Utility.openPrintWriter("Log1.xml", Utility.LATIN1_UNIX);
        log1.println("<diff version='" + target.getVersion() + "'>");

        PrintWriter log2 = Utility.openPrintWriter("Log2.xml", Utility.LATIN1_UNIX);
        log2.println("<diff version='" + Default.ucd().getVersion() + "'>");
        
        for (int i = 0; i <= 0x10FFFF; ++i) {
            if (!target.isAllocated(i)) continue;
            Utility.dot(i);
            UData t = target.get(i, true);
            UData current = Default.ucd().get(i, true);
            if (i == 0x5E) {
                System.out.println(target.getDecompositionTypeID(i) 
                    + ", " + Utility.hex(target.getDecompositionMapping(i)));
                System.out.println(Default.ucd().getDecompositionTypeID(i) 
                    + ", " + Utility.hex(Default.ucd().getDecompositionMapping(i)));
            }
            if (t.equals(current)) continue;
            
            // print both for comparison
            log1.println(t.toString(target, UData.ABBREVIATED));
            log2.println(current.toString(Default.ucd(), UData.ABBREVIATED));
        }
        log1.println("</diff>");
        log2.println("</diff>");
        log1.close();
        log2.close();
    }
    
    public static void generateDerived (byte type, boolean checkTypeAndStandard, int headerChoice, String directory, String fileName) throws IOException {

        
        String newFile = directory + fileName + UnicodeDataFile.getFileSuffix(true);
        System.out.println("New File: " + newFile);
        PrintWriter output = Utility.openPrintWriter(newFile, Utility.LATIN1_UNIX);
        String[] batName = {""};
        String mostRecent = UnicodeDataFile.generateBat(directory, fileName, UnicodeDataFile.getFileSuffix(true), batName);
        System.out.println("Most recent: " + mostRecent);
        
        doHeader(fileName + UnicodeDataFile.getFileSuffix(false), output, headerChoice);
        for (int i = 0; i < DERIVED_PROPERTY_LIMIT; ++i) {
            UCDProperty up = DerivedProperty.make(i, Default.ucd());
            if (up == null) continue;
            boolean keepGoing = true;
            if (!up.isStandard()) keepGoing = false;
            if ((up.getType() & type) == 0) keepGoing = false;
            
            if (checkTypeAndStandard != keepGoing) continue;
            //if ((bitMask & (1L<<i)) == 0) continue;
            
            System.out.print('.');
            output.println(HORIZONTAL_LINE);
            output.println();
            new DerivedPropertyLister(Default.ucd(), i, output).print();
            output.flush();
        }
        output.close();
        Utility.renameIdentical(mostRecent, Utility.getOutputName(newFile), batName[0]);
    }

    /*
    public static void listStrings(String file, int type, int subtype) throws IOException {
        Default.ucd = UCD.make("3.1.0");
        UCD ucd30 = UCD.make("3.0.0");
        PrintStream output = new PrintStream(new FileOutputStream(GEN_DIR + file));

        for (int i = 0; i < 0x10FFFF; ++i) {
            if ((i & 0xFFF) == 0) System.out.println("# " + i);
            if (!Default.ucd.isRepresented(i)) continue;
            if (ucd30.isRepresented(i)) continue;
            String string = "";
            switch(type) {
                case 0: string = Default.ucd.getSimpleLowercase(i);
            }
            if (UTF32.length32(string) == 1 && UTF32.char32At(string,0) == i) continue;
            output.println(Utility.hex(i) + "; C; " + Utility.hex(string) + "; # " + Default.ucd.getName(i));
        }
        output.close();
    }
    */

    public static void generateCompExclusions() throws IOException {
        
        String newFile = "DerivedData/CompositionExclusions" + UnicodeDataFile.getFileSuffix(true);
        PrintWriter output = Utility.openPrintWriter(newFile, Utility.LATIN1_UNIX);
        String[] batName = {""};
        String mostRecent = UnicodeDataFile.generateBat("DerivedData/", "CompositionExclusions", UnicodeDataFile.getFileSuffix(true), batName);
        
        output.println("# CompositionExclusions" + UnicodeDataFile.getFileSuffix(false));
        output.println(UnicodeDataFile.generateDateLine());
        output.println("#");
        output.println("# This file lists the characters from the UAX #15 Composition Exclusion Table.");
        output.println("#");
        if (Default.ucd().getVersion().equals("3.2.0")) {
            output.println("# The format of the comments in this file has been updated since the last version,");
            output.println("# CompositionExclusions-3.txt. The only substantive change to this file between that");
            output.println("# version and this one is the addition of U+2ADC FORKING.");
            output.println("#");
        }
        output.println("# For more information, see");
        output.println("# http://www.unicode.org/unicode/reports/tr15/#Primary Exclusion List Table");
        output.println(HORIZONTAL_LINE);
        output.println();
        output.println("# (1) Script Specifics");
        output.println("# This list of characters cannot be derived from the UnicodeData file.");
        output.println(HORIZONTAL_LINE);
        output.println();
        
        new CompLister(output, 1).print();
        
        output.println(HORIZONTAL_LINE);
        output.println("# (2) Post Composition Version precomposed characters");
        output.println("# These characters cannot be derived solely from the UnicodeData.txt file");
        output.println("# in this version of Unicode.");
        output.println(HORIZONTAL_LINE);
        output.println();
        
        new CompLister(output, 2).print();

        output.println(HORIZONTAL_LINE);
        output.println("# (3) Singleton Decompositions");
        output.println("# These characters can be derived from the UnicodeData file");
        output.println("# by including all characters whose canonical decomposition");
        output.println("# consists of a single character.");
        output.println("# These characters are simply quoted here for reference.");
        output.println(HORIZONTAL_LINE);
        output.println();

        new CompLister(output, 3).print();
        
        output.println(HORIZONTAL_LINE);
        output.println("# (4) Non-Starter Decompositions");
        output.println("# These characters can be derived from the UnicodeData file");
        output.println("# by including all characters whose canonical decomposition consists");
        output.println("# of a sequence of characters, the first of which has a non-zero");
        output.println("# combining class.");
        output.println("# These characters are simply quoted here for reference.");        
        output.println(HORIZONTAL_LINE);
        output.println();
        new CompLister(output, 4).print();
        
        output.close();
        Utility.renameIdentical(mostRecent, Utility.getOutputName(newFile), batName[0]);
    }
    
    static class CompLister extends PropertyLister {
        UCD oldUCD;
        int type;

        public CompLister(PrintWriter output, int type) {
            this.output = output;
            ucdData = Default.ucd();
            oldUCD = UCD.make("3.0.0");
            // showOnConsole = true;
            alwaysBreaks = type <= 2; // CHANGE LATER
            commentOut = type > 2;
            this.type = type;
        }

        public String optionalComment(int cp) { return ""; }
        /*
        public String valueName(int cp) {
            return UTF32.length32(ucdData.getDecompositionMapping(cp)) + "";
        }
        */
        public byte status(int cp) {
            if (getType(cp) == type) return INCLUDE;
            return EXCLUDE;
        }
        
        public int getType(int cp) {
            if (!ucdData.isAssigned(cp)) return -1;
            if (ucdData.getDecompositionType(cp) != CANONICAL) return -1;
            
            if (oldUCD.getBinaryProperty(cp, CompositionExclusion)) return 1;
            if (cp == 0xFB1D) return 1; // special
            
            String decomp = ucdData.getDecompositionMapping(cp);
            int len = UTF32.length32(decomp);
            if (len == 1) return 3;
            int first = UTF32.char32At(decomp,0);
            if (ucdData.getCombiningClass(first) != 0) return 4;
            
            if (oldUCD.getDecompositionType(cp) == CANONICAL) return -1;
            if (ucdData.getDecompositionType(cp) == CANONICAL) return 2;
            
            return -1;
        }
    }

    public static void generatePropertyAliases() throws IOException {
        
        String prop = "";
        String propAbb = "";
        String value = "";
        String valueAbb = "";
        
        Map duplicates = new TreeMap();
        Set sorted = new TreeSet(java.text.Collator.getInstance());
        Set accumulation = new TreeSet(java.text.Collator.getInstance());
        
        for (int k = 0; k < UCD_Names.NON_ENUMERATED_NAMES.length; ++k) {
            propAbb = Utility.getUnskeleton(UCD_Names.NON_ENUMERATED_NAMES[k][0], false);
            prop = Utility.getUnskeleton(UCD_Names.NON_ENUMERATED_NAMES[k][1], true);
            
            byte type = STRING_PROP;
            if (propAbb.equals("nv")) {
                type = NUMERIC_PROP;
            } else if (propAbb.equals("age")) {
                type = CATALOG_PROP;
            } else if (propAbb.equals("blk")) {
                type = CATALOG_PROP;
            } else if (propAbb.equals("na")) {
                type = MISC_PROP;
            } else if (propAbb.equals("na1")) {
                type = MISC_PROP;
            } else if (propAbb.equals("isc")) {
                type = MISC_PROP;
            }
            addLine(sorted, UCD_Names.PROP_TYPE_NAMES[type][1], propAbb, prop, null);
            checkDuplicate(duplicates, accumulation, propAbb, prop);
            if (!prop.equals(propAbb)) checkDuplicate(duplicates, accumulation, prop, prop);
        }
        addLine(sorted, UCD_Names.PROP_TYPE_NAMES[MISC_PROP][1], "URS", "Unicode_Radical_Stroke", null);
        // TODO: merge above
 
        for (int k = 0; k < UCD_Names.SUPER_CATEGORIES.length; ++k) {
            valueAbb = Utility.getUnskeleton(UCD_Names.SUPER_CATEGORIES[k][0], false);
            value = Utility.getUnskeleton(UCD_Names.SUPER_CATEGORIES[k][1], true);
            String extra = Utility.getUnskeleton(UCD_Names.SUPER_CATEGORIES[k][1], true);
            addLine(sorted, "gc", valueAbb, value, extra, "# " + UCD_Names.SUPER_CATEGORIES[k][2]);
            checkDuplicate(duplicates, accumulation, value, "General_Category=" + value);
            if (!value.equals(valueAbb)) checkDuplicate(duplicates, accumulation, valueAbb, "General_Category=" + value);
            if (extra != null) checkDuplicate(duplicates, accumulation, extra, "General_Category=" + value);
        }
        
        /*
        addLine(sorted, "xx; T         ; True");
        checkDuplicate(duplicates, accumulation, "T", "xx=True");
        addLine(sorted, "xx; F         ; False");
        checkDuplicate(duplicates, accumulation, "F", "xx=False");
        */
        addLine(sorted, "qc", UCD_Names.YN_TABLE[1], UCD_Names.YN_TABLE_LONG[1], null);
        checkDuplicate(duplicates, accumulation, UCD_Names.YN_TABLE[1], "qc=" + UCD_Names.YN_TABLE_LONG[1]);
        addLine(sorted, "qc", UCD_Names.YN_TABLE[0], UCD_Names.YN_TABLE_LONG[0], null);
        checkDuplicate(duplicates, accumulation, UCD_Names.YN_TABLE[0], "qc=" + UCD_Names.YN_TABLE_LONG[0]);
        addLine(sorted, "qc", "M", "Maybe", null);
        checkDuplicate(duplicates, accumulation, "M", "qc=Maybe");
        
        addLine(sorted, "blk", "n/a", Utility.getUnskeleton("no block", true), null);
        
        for (int i = 0; i < LIMIT_ENUM; ++i) {
            int type = i & 0xFF00;
            if (type == AGE) continue;
            if (i == (BINARY_PROPERTIES | CaseFoldTurkishI)) continue;
            if (i == (BINARY_PROPERTIES | Non_break)) continue;
            if (i == (BINARY_PROPERTIES | Case_Sensitive)) continue;
            
            if (type == NUMERIC_TYPE) {
                //System.out.println("debug");
            }
            
            UCDProperty up = UnifiedBinaryProperty.make(i, Default.ucd());
            if (up == null) continue;
            if (!up.isStandard()) continue;
            
            // System.out.println("At" + Utility.hex(i));
            
            // Save the Type Name, under BB for binary
            
            if (type == i || type == BINARY_PROPERTIES || type == DERIVED) {
                if (propAbb.equals("") || propAbb.equals(UCD_Names.YN_TABLE[1])) {
                    System.out.println("WHOOPS: " + Utility.hex(i));
                }
                propAbb = Utility.getUnskeleton(up.getPropertyName(SHORT), false);
                prop = Utility.getUnskeleton(up.getPropertyName(LONG), true);
                addLine(sorted, 
                    type == SCRIPT
                    ? UCD_Names.PROP_TYPE_NAMES[CATALOG_PROP][1]
                    : type != DERIVED && type != BINARY_PROPERTIES 
                    ? UCD_Names.PROP_TYPE_NAMES[ENUMERATED_PROP][1] 
                    : UCD_Names.PROP_TYPE_NAMES[up.getValueType()][1], 
                    propAbb, prop, null);
                checkDuplicate(duplicates, accumulation, propAbb, prop);
                if (!prop.equals(propAbb)) checkDuplicate(duplicates, accumulation, prop, prop);
            }
            
            if (up.getValueType() < BINARY_PROP) continue;
            value = up.getValue(LONG);
            if (value.length() == 0) value = "none";
            else if (value.equals(UnicodeProperty.UNUSED)) continue;

            if (type != DECOMPOSITION_TYPE) {
                value = Utility.getUnskeleton(value, true);
            }
            
            //if (type == DERIVED) {
                //System.out.println("Derived " + up.getProperty());        
            //}
            
            
            if (type == SCRIPT) {
                value = Default.ucd().getCase(value, FULL, TITLE);
            }
            
            valueAbb = up.getValue(SHORT);
            valueAbb = Utility.getUnskeleton(valueAbb, false);
            if (valueAbb.length() == 0) valueAbb = "n/a";
            //else if (valueAbb.equals(value)) valueAbb = "n/a";

            
            if (type == COMBINING_CLASS) {
                if (value.charAt(0) <= '9') { continue; }
            }
            
            
            if (type == JOINING_GROUP) {
                valueAbb = "n/a";
            }
            
            /*
            String elide = "";
            if (type == CATEGORY || type == SCRIPT || type == BINARY_PROPERTIES) elide = "\\p{"
                + valueAbb
                + "}";
            String abb = "";
            if (type != BINARY_PROPERTIES) abb = "\\p{"
                + UCD_Names.ABB_UNIFIED_PROPERTIES[i>>8]
                + "="
                + valueAbb
                + "}";
            String norm = "";
            if (type != BINARY_PROPERTIES) norm = "\\p{"
                + UCD_Names.SHORT_UNIFIED_PROPERTIES[i>>8]
                + "="
                + value
                + "}";
            System.out.println("<tr><td>" + elide + "</td><td>" + abb + "</td><td>" + norm + "</td></tr>");
            */
            
            /*
            if (type == BINARY_PROPERTIES || type == DERIVED) {
                //if (value.equals(YN_TABLE_LONG[1])) continue;
                addLine(sorted, PROP_TYPE_NAMES[BINARY][1], valueAbb, value);
                checkDuplicate(duplicates, accumulation, value, value);
                if (!value.equalsIgnoreCase(valueAbb)) checkDuplicate(duplicates, accumulation, valueAbb, value);
                continue;
            }
            */
            
            if (type == COMBINING_CLASS) {
                String num = up.getValue(NUMBER);
                num = "; " + Utility.repeat(" ", 3-num.length()) + num;
                addLine(sorted, propAbb + num, valueAbb, value, null);
            } else if (!valueAbb.equals(UCD_Names.YN_TABLE[1])) {
                addLine(sorted, propAbb, valueAbb, value, null);
            }
            checkDuplicate(duplicates, accumulation, value, prop + "=" + value);
            if (!value.equalsIgnoreCase(valueAbb) && !valueAbb.equals("n/a")) {
                checkDuplicate(duplicates, accumulation, valueAbb, prop + "=" + value);
            }
        }
        
        Iterator blockIterator = Default.ucd().getBlockNames().iterator();
        while (blockIterator.hasNext()) {
            addLine(sorted, "blk", "n/a", (String)blockIterator.next(), null);
        }
        /*
        UCD.BlockData blockData = new UCD.BlockData();
        
        int blockId = 0;
        while (Default.ucd().getBlockData(blockId++, blockData)) {
            addLine(sorted, "blk", "n/a", blockData.name);
        }
        */
        
        String filename = "PropertyAliases";
        String newFile = "DerivedData/" + filename + UnicodeDataFile.getFileSuffix(true);
        PrintWriter log = Utility.openPrintWriter(newFile, Utility.LATIN1_UNIX);
        String[] batName = {""};
        String mostRecent = UnicodeDataFile.generateBat("DerivedData/", filename, UnicodeDataFile.getFileSuffix(true), batName);
        
        log.println("# " + filename + UnicodeDataFile.getFileSuffix(false));
        log.println(UnicodeDataFile.generateDateLine());
        log.println("#");
        Utility.appendFile("PropertyAliasesHeader.txt", Utility.LATIN1, log);
        log.println(HORIZONTAL_LINE);
        log.println();
        int count = Utility.print(log, sorted, "\r\n", new MyBreaker(true));
        log.println();
        log.println();
        log.println(HORIZONTAL_LINE);
        log.println("# Total: \t" + count);
        log.println();
        log.close();
        Utility.renameIdentical(mostRecent, Utility.getOutputName(newFile), batName[0]);
        
        filename = "PropertyValueAliases";
        newFile = "DerivedData/" + filename + UnicodeDataFile.getFileSuffix(true);
        log = Utility.openPrintWriter(newFile, Utility.LATIN1_UNIX);
        mostRecent = UnicodeDataFile.generateBat("DerivedData/", filename, UnicodeDataFile.getFileSuffix(true), batName);
        
        log.println("# " + filename + UnicodeDataFile.getFileSuffix(false));
        log.println(UnicodeDataFile.generateDateLine());
        log.println("#");
        Utility.appendFile("PropertyValueAliasesHeader.txt", Utility.LATIN1, log);
        log.println(HORIZONTAL_LINE);
        log.println();
        Utility.print(log, sorted, "\r\n", new MyBreaker(false));
        log.println();
        log.close();
        Utility.renameIdentical(mostRecent, Utility.getOutputName(newFile), batName[0]);
        
        filename = "PropertyAliasSummary";
        newFile = "OtherData/" + filename + UnicodeDataFile.getFileSuffix(true);
        log = Utility.openPrintWriter(newFile, Utility.LATIN1_UNIX);
        mostRecent = UnicodeDataFile.generateBat("OtherData/", filename, UnicodeDataFile.getFileSuffix(true), batName);
        
        log.println();
        log.println(HORIZONTAL_LINE);
        log.println();
        log.println("# Non-Unique names: the same name (under either an exact or loose match)");
        log.println("# occurs as a property name or property value name");
        log.println("# Note: no two property names can be the same,");
        log.println("# nor can two property value names for the same property be the same.");
        log.println();
        Utility.print(log, accumulation, "\r\n", new MyBreaker(false));
        log.println();
        log.close();
        Utility.renameIdentical(mostRecent, Utility.getOutputName(newFile), batName[0]);
    }
    
    static void addLine(Set sorted, String f1, String f2, String f3, String f4) {
        addLine(sorted, f1, f2, f3, f4, null);
    }
    
    static void addLine(Set sorted, String f1, String f2, String f3, String f4, String comment) {
        //System.out.println("Adding: " + line);
        f1 += Utility.repeat(" ", 3 - f1.length());
        f1 += "; " + f2;
        f1 += Utility.repeat(" ", 15 - f1.length());
        f1 += "; " + f3;
        if (f4 != null) {
            f1 += Utility.repeat(" ", 30 - f1.length());
            f1 += f4;
        }
        if (comment != null) {
            f1 += Utility.repeat(" ", 50 - f1.length());
            f1 += comment;
        }
        sorted.add(f1);
    }
    
    static class MyBreaker implements Utility.Breaker {
        boolean status;
        int count;
        
        public MyBreaker(boolean status) {
            this.status = status;
        }
        
        public byte getType (String c) {
            for (byte i = 0; i <= BINARY_PROP; ++i) {
                if (c.startsWith(UCD_Names.PROP_TYPE_NAMES[i][1])) return i;
            }
            return UNKNOWN_PROP;
        }
        
        public boolean filter(Object current) {
            String c = current.toString();
            byte type = getType(c);
            if (type != UNKNOWN_PROP) return status;
            return !status;
        }
        
        public String get(Object current, Object old) {
            if (old == null) {
                old = "  ";
            }
            String c = current.toString();
            String o = old.toString();
            String sep = "";
            if (!c.substring(0,2).equals(o.substring(0,2))) {
                sep = "\r\n";
                if (status) {
                    byte type = getType(c);
                    sep = sep + HORIZONTAL_LINE + sep + "# " + UCD_Names.PROP_TYPE_NAMES[type][0] + " Properties" + sep + HORIZONTAL_LINE + sep;
                }
            }
            if (status) {
                int pos = c.indexOf(';');
                c = c.substring(pos+1).trim();
            }
            return sep + c;
        }
    }
    
    static void checkDuplicate(Map m, Set accumulation, String toCheck, String originalComment) {
        toCheck = Utility.getSkeleton(toCheck);
        String comment = "{" + originalComment + "}";
        
        Set result = (Set) m.get(toCheck);
        if (result != null) {
            // Warn on serious problem: two property-names collide
            // or two property names & values collide.
            // examples:
            // if (1) "c" stood for both "General_Category" and "Combining_Class"
            // or if (2) "X=cc" stood for "X=control" and "X=compatibility"
            // 1: comment doesn't contain "=", and something in the results doesn't contain "="
            // 2: comment does contain "X=", and something else in results contains "X="
            
            int equalPos = comment.indexOf('=');
            if (equalPos < 0) { // #1
                String conflict = Utility.findSubstring("=", result, false);
                if (conflict != null) {
                    System.out.println("Property Name Conflict " + toCheck);
                    System.out.println("  With " + comment);
                    System.out.println("  And  " + conflict);
                }
            } else {    // #2
                String trial = comment.substring(0,equalPos+1);
                String conflict = Utility.findSubstring(trial, result, true);
                if (conflict != null) {
                    System.out.println("Property Value Name Conflict " + toCheck);
                    System.out.println("  With " + comment);
                    System.out.println("  And  " + conflict);
                }
            }
            
            // accumulate differences
            /*
            String acc = (String)accumulation.get(toCheck);
            if (acc == null) {
                acc = "# \"" + toCheck + "\":\t" + originalComment;
            }
            acc += ";\t" + result;
            */
            result.add(comment);
            accumulation.add("# " + result.toString() + ":\t" + toCheck);
        } else {
            result = new TreeSet();
            result.add(comment);
            m.put(toCheck, result);
        }
    }
    
    public static void generateVerticalSlice(int startEnum, int endEnum,
            int headerChoice, String directory, String file) throws IOException {

        
        String newFile = directory + file + UnicodeDataFile.getFileSuffix(true);
        PrintWriter output = Utility.openPrintWriter(newFile, Utility.LATIN1_UNIX);
        String[] batName = {""};
        String mostRecent = UnicodeDataFile.generateBat(directory, file, UnicodeDataFile.getFileSuffix(true), batName);
        
        doHeader(file + UnicodeDataFile.getFileSuffix(false), output, headerChoice);
        int last = -1;
        for (int i = startEnum; i < endEnum; ++i) {
            UCDProperty up = UnifiedBinaryProperty.make(i, Default.ucd());
            if (up == null) continue;
            if (up.skipInDerivedListing()) continue;
            
            /*
            if (i == DECOMPOSITION_TYPE || i == NUMERIC_TYPE
                || i == (BINARY_PROPERTIES | Non_break)
                || i == (BINARY_PROPERTIES | CaseFoldTurkishI)
                || i == (HANGUL_SYLLABLE_TYPE | NA)
                || i == (JOINING_TYPE | JT_U)
                || i == (JOINING_GROUP | NO_SHAPING)
                ) continue; // skip zero case
            */
            /*if (skipSpecial == SKIP_SPECIAL
                    && i >= (BINARY_PROPERTIES | CompositionExclusion)
                    && i < (AGE + NEXT_ENUM)) continue;
                    */
            if ((last & 0xFF00) != (i & 0xFF00) && (i <= BINARY_PROPERTIES || i >= SCRIPT)) {
                output.println();
                output.println(HORIZONTAL_LINE);
                output.println("# " + up.getHeader());
                output.println(HORIZONTAL_LINE);
                output.println();
                System.out.println();
                System.out.println(up.getHeader());
                last = i;
            } else {
                output.println(HORIZONTAL_LINE);
                output.println();
            }
            System.out.print(".");
            if (DEBUG) System.out.println(i);
            new MyPropertyLister(Default.ucd(), i, output).print();
            output.flush();
        }
        if (endEnum == LIMIT_ENUM) {
            output.println();
                output.println(HORIZONTAL_LINE);
            output.println("# Numeric Values (from UnicodeData.txt, field 6/7/8)");
                output.println(HORIZONTAL_LINE);
            output.println();
            System.out.println();
            System.out.println("@NUMERIC VALUES");

            Set numericValueSet = new TreeSet();
            for (int i = 0; i < 0x10FFFF; ++i) {
                double nv = Default.ucd().getNumericValue(i);
                if (Double.isNaN(nv)) continue;
                numericValueSet.add(new Double(nv));
            }
            Iterator it = numericValueSet.iterator();
            while(it.hasNext()) {
                new MyFloatLister(Default.ucd(), ((Double)it.next()).doubleValue(), output).print();
                output.println();
                System.out.print(".");
            }
            output.flush();
        }
        output.close();
        //System.out.println("HERE");
        Utility.renameIdentical(mostRecent, Utility.getOutputName(newFile), batName[0]);
        System.out.println();
    }
    
    static public void writeNormalizerTestSuite(String directory, String fileName) throws IOException {
        
    	UnicodeDataFile fc = UnicodeDataFile.openAndWriteHeader(directory, fileName);
        PrintWriter log = fc.out;
        
        String newFile = directory + fileName + UnicodeDataFile.getFileSuffix(true);
        //PrintWriter log = Utility.openPrintWriter(newFile, Utility.UTF8_UNIX);
        //String[] batName = {""};
        //String mostRecent = UnicodeDataFile.generateBat(directory, fileName, UnicodeDataFile.getFileSuffix(true), batName);

        String[] example = new String[256];

        //log.println("# " + fileName + UnicodeDataFile.getFileSuffix(false));
        //log.println(UnicodeDataFile.generateDateLine());
        /*log.println("#");
        log.println("# Normalization Test Suite");
        log.println("# Format:");
        log.println("#");
        log.println("#   Columns (c1, c2,...) are separated by semicolons");
        log.println("#   Comments are indicated with hash marks");
        log.println("#");
        log.println("# CONFORMANCE:");
        log.println("# 1. The following invariants must be true for all conformant implementations");
        log.println("#");
        log.println("#    NFC");
        log.println("#      c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3)");
        log.println("#      c4 ==  NFC(c4) ==  NFC(c5)");
        log.println("#");
        log.println("#    NFD");
        log.println("#      c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3)");
        log.println("#      c5 ==  NFD(c4) ==  NFD(c5)");
        log.println("#");
        log.println("#    NFKC");
        log.println("#      c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)");
        log.println("#");
        log.println("#    NFKD");
        log.println("#      c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)");
        log.println("#");
        log.println("# 2. For every code point X assigned in this version of Unicode that is not specifically");
        log.println("#    listed in Part 1, the following invariants must be true for all conformant");
        log.println("#    implementations:");
        log.println("#");
        log.println("#      X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X)");

        System.out.println("Writing Part 1");

        log.println("#");
        log.println("@Part0 # Specific cases");
        log.println("#");*/

        for (int j = 0; j < testSuiteCases.length; ++j) {
            writeLine(testSuiteCases[j], log, false);
        }

        System.out.println("Writing Part 2");

        log.println("#");
        log.println("@Part1 # Character by character test");
        log.println("# All characters not explicitly occurring in c1 of Part 1 have identical NFC, D, KC, KD forms.");
        log.println("#");
        
        for (int ch = 0; ch < 0x10FFFF; ++ch) {
            Utility.dot(ch);
            if (!Default.ucd().isAssigned(ch)) continue;
            if (Default.ucd().isPUA(ch)) continue;
            String cc = UTF32.valueOf32(ch);
            writeLine(cc,log, true);
        }
        Utility.fixDot();

        System.out.println("Finding Examples");

        for (int ch = 0; ch < 0x10FFFF; ++ch) {
            Utility.dot(ch);
            if (!Default.ucd().isAssigned(ch)) continue;
            if (Default.ucd().isPUA(ch)) continue;
            int cc = Default.ucd().getCombiningClass(ch);
            if (example[cc] == null) example[cc] = UTF32.valueOf32(ch);
        }

        Utility.fixDot();
        System.out.println("Writing Part 2");

        log.println("#");
        log.println("@Part2 # Canonical Order Test");
        log.println("#");

        for (int ch = 0; ch < 0x10FFFF; ++ch) {

            Utility.dot(ch);
            if (!Default.ucd().isAssigned(ch)) continue;
            if (Default.ucd().isPUA(ch)) continue;
            short c = Default.ucd().getCombiningClass(ch);
            if (c == 0) continue;

            // add character with higher class, same class, lower class

            String sample = "";
            for (int i = c+1; i < example.length; ++i) {
                if (example[i] == null) continue;
                sample += example[i];
                break;
            }
            sample += example[c];
            for (int i = c-1; i > 0; --i) {
                if (example[i] == null) continue;
                sample += example[i];
                break;
            }

            writeLine("a" + sample + UTF32.valueOf32(ch) + "b", log, false);
            writeLine("a" + UTF32.valueOf32(ch) + sample + "b", log, false);
        }
        
        System.out.println("Writing Part 3");
        log.println("#");
        log.println("@Part3 # PRI #29 Test");
        log.println("#");
        
        Set prilist = new TreeSet();
        
        for (int ch = 0; ch < 0x10FFFF; ++ch) {
            Utility.dot(ch);
            if (!Default.ucd().isAssigned(ch)) continue;
            if (Default.ucd().isPUA(ch)) continue;
            if (0xAC00 <= ch && ch <= 0xD7FF) { // skip most
            	if (((ch - 0xAC00) % 91) != 0) continue;
            }
           // also gather data for pri29 test
            if (ch == 0x09CB) {
            	System.out.println("debug");
            }
            if (Default.ucd().getDecompositionType(ch) != CANONICAL) continue;
            //if (!Default.nfc().isNormalized(ch)) continue;
            String s = Default.ucd().getDecompositionMapping(ch);
            if (UTF16.hasMoreCodePointsThan(s, 2)) continue;
            if (!UTF16.hasMoreCodePointsThan(s, 1)) continue;
            int c1 = UTF16.charAt(s, 0);
            int c2 = UTF16.charAt(s, UTF16.getCharCount(c1));
            if (Default.ucd().getCombiningClass(c1) != 0) continue;
            if (Default.ucd().getCombiningClass(c2) != 0) continue;
            prilist.add(UTF16.valueOf(c1) + '\u0334' + UTF16.valueOf(c2));
        }
        Utility.fixDot();
        
        for (Iterator it = prilist.iterator(); it.hasNext();) {
            writeLine((String)it.next(),log, false);
        }

        Utility.fixDot();
        log.println("#");
        log.println("# END OF FILE");
        fc.close();
        //Utility.renameIdentical(mostRecent, Utility.getOutputName(newFile), batName[0]);
    }
    
    static void handleIdentical() throws IOException {
        DirectoryIterator target = new DirectoryIterator(GEN_DIR + File.separator + "DerivedData");
        DirectoryIterator.RootFileFilter filter = new DirectoryIterator.RootFileFilter("");
        DirectoryIterator recent = new DirectoryIterator(UCD_DIR, filter);
        while (true) {
            File targetFile = target.next();
            if (targetFile == null) break;
            recent.reset();
            filter.setRoot(DirectoryIterator.getRoot(targetFile));
            File lastFile = recent.next();
            if (lastFile == null) break;
            System.out.println("Target: " + targetFile);
            System.out.println("Last: " + lastFile);
            if (!DirectoryIterator.isAlmostIdentical(targetFile, lastFile, true)) continue;
            System.out.println("Almost Identical");
        }
    }

    static void writeLine(String cc, PrintWriter log, boolean check) {
        String c = Default.nfc().normalize(cc);
        String d = Default.nfd().normalize(cc);
        String kc = Default.nfkc().normalize(cc);
        String kd = Default.nfkd().normalize(cc);
        if (check & cc.equals(c) && cc.equals(d) && cc.equals(kc) && cc.equals(kd)) return;

        // consistency check
        String dc = Default.nfd().normalize(c);
        String dkc = Default.nfd().normalize(kc);
        if (!dc.equals(d) || !dkc.equals(kd)) {
            System.out.println("Danger Will Robinson!");
            Normalizer.SHOW_PROGRESS = true;
            d = Default.nfd().normalize(cc);
        }

        // printout
        log.println(
            Utility.hex(cc," ") + ";" + Utility.hex(c," ") + ";" + Utility.hex(d," ") + ";"
            + Utility.hex(kc," ") + ";" + Utility.hex(kd," ")
            + "; # ("
            + comma(cc) + "; " + comma(c) + "; " + comma(d) + "; " + comma(kc) + "; " + comma(kd) + "; "
            + ") " + Default.ucd().getName(cc));
    }

    static StringBuffer commaResult = new StringBuffer();

    // not recursive!!!
    static final String comma(String s) {
    	//if (true) return s;
        commaResult.setLength(0);
        int cp;
        for (int i = 0; i < s.length(); i += UTF16.getCharCount(cp)) {
            cp = UTF16.charAt(s, i);
            if (Default.ucd().getCategory(cp) == Mn) commaResult.append('\u25CC');
            UTF16.append(commaResult, cp);
        }
        return commaResult.toString();
    }

    static final String[] testSuiteCases = {
        "\u1E0A",
        "\u1E0C",
        "\u1E0A\u0323",
        "\u1E0C\u0307",
        "D\u0307\u0323",
        "D\u0323\u0307",
        "\u1E0A\u031B",
        "\u1E0C\u031B",
        "\u1E0A\u031B\u0323",
        "\u1E0C\u031B\u0307",
        "D\u031B\u0307\u0323",
        "D\u031B\u0323\u0307",
        "\u00C8",
        "\u0112",
        "E\u0300",
        "E\u0304",
        "\u1E14",
        "\u0112\u0300",
        "\u1E14\u0304",
        "E\u0304\u0300",
        "E\u0300\u0304",
        "\u05B8\u05B9\u05B1\u0591\u05C3\u05B0\u05AC\u059F",
        "\u0592\u05B7\u05BC\u05A5\u05B0\u05C0\u05C4\u05AD"

    };
    
    static final void backwardsCompat(String directory, String filename, int[] list) throws IOException {
    
        
        String newFile = directory + filename + UnicodeDataFile.getFileSuffix(true);
        PrintWriter log = Utility.openPrintWriter(newFile, Utility.LATIN1_UNIX);
        String[] batName = {""};
        String mostRecent = UnicodeDataFile.generateBat(directory, filename, UnicodeDataFile.getFileSuffix(true), batName);
        DiffPropertyLister dpl;
        UnicodeSet cummulative = new UnicodeSet();
        
        try {
        	for (int i = 0; i < list.length; ++i) {
        		int prop = list[i];
            	log.println();
            	log.println(HORIZONTAL_LINE);
        		log.println("###### " + DerivedProperty.make(prop, Default.ucd()).getName());
            	//log.println();
            	//log.println(HORIZONTAL_LINE);
            	//new DiffPropertyLister("3.2.0", "1.1.0", log, prop).print();
            	log.println();
            	log.println(HORIZONTAL_LINE);
            	
            	log.println();
            	dpl = new DiffPropertyLister("3.2.0", "2.0.0", log, prop);
            	dpl.print();
            	cummulative.addAll(dpl.getSet());
            	log.println(HORIZONTAL_LINE);
            	
            	log.println();
            	dpl = new DiffPropertyLister("3.2.0", "2.1.2", log, prop);
            	dpl.print();
            	cummulative.addAll(dpl.getSet());
            	log.println(HORIZONTAL_LINE);
            	
            	log.println();
            	dpl = new DiffPropertyLister("3.2.0", "2.1.5", log, prop);
            	dpl.print();
            	cummulative.addAll(dpl.getSet());
            	log.println(HORIZONTAL_LINE);
            	
            	log.println();
            	dpl = new DiffPropertyLister("3.2.0", "2.1.8", log, prop);
            	dpl.print();
            	cummulative.addAll(dpl.getSet());
            	log.println(HORIZONTAL_LINE);
            	
            	log.println();
            	dpl = new DiffPropertyLister("3.2.0", "3.0.0", log, prop);
            	dpl.print();
            	cummulative.addAll(dpl.getSet());
            	log.println(HORIZONTAL_LINE);
            	
            	log.println();
            	dpl = new DiffPropertyLister("3.2.0", "3.0.1", log, prop);
            	dpl.print();
            	cummulative.addAll(dpl.getSet());
            	log.println(HORIZONTAL_LINE);
            	
            	log.println();
            	dpl = new DiffPropertyLister("3.2.0", "3.1.0", log, prop);
            	dpl.print();
            	cummulative.addAll(dpl.getSet());
            	log.println(HORIZONTAL_LINE);
            	
            	log.println();
            	dpl = new DiffPropertyLister("3.2.0", "3.1.1", log, prop);
            	dpl.print();
            	cummulative.addAll(dpl.getSet());
            	log.println(HORIZONTAL_LINE);
            	
            	log.println();
            	log.println("Cummulative differences");
            	UCDProperty up = DerivedProperty.make(prop, Default.ucd());
            	UnicodeSet newProp = up.getSet();
            	Utility.showSetNames(log, "", cummulative.removeAll(newProp), false, false, Default.ucd());
            }
        } finally {
            if (log != null) {
                log.close();
                Utility.renameIdentical(mostRecent, Utility.getOutputName(newFile), batName[0]);
            }
        }
    }
    
    static final void generateAge(String directory, String filename) throws IOException {
        
        String newFile = directory + filename + UnicodeDataFile.getFileSuffix(true);
        PrintWriter log = Utility.openPrintWriter(newFile, Utility.LATIN1_UNIX);
        String[] batName = {""};
        String mostRecent = UnicodeDataFile.generateBat(directory, filename, UnicodeDataFile.getFileSuffix(true), batName);
        try {
            log.println("# " + filename + UnicodeDataFile.getFileSuffix(false));
            log.println(UnicodeDataFile.generateDateLine());
            log.println("#");
            log.println("# Unicode Character Database: Derived Property Data");
            log.println("# This file shows when various code points were designated in Unicode");
            log.println("# Notes:");
            log.println("# - The term 'designated' means that a previously reserved code point was specified");
            log.println("#   to be a noncharacter or surrogate, or assigned as a character,");
            log.println("#   control or format code.");
            log.println("# - Versions are only tracked from 1.1 onwards, since version 1.0");
            log.println("#   predated changes required by the ISO 10646 merger.");
            log.println("# - The Hangul Syllables that were removed from 2.0 are not included in the 1.1 listing.");
            log.println("# - The supplementary private use code points and the non-character code points");
            log.println("#   were designated in version 2.0, but not specifically listed in the UCD");
            log.println("#   until versions 3.0 and 3.1 respectively.");
            log.println("#");
            log.println("# For details on the contents of each version, see");
            log.println("#   http://www.unicode.org/versions/enumeratedversions.html.");
            
            // http://www.unicode.org/versions/enumeratedversions.html
            
            log.println(HORIZONTAL_LINE);
            log.println();
            new DiffPropertyLister(null, "1.1.0", log).print();
            log.println(HORIZONTAL_LINE);
            log.println();
            new DiffPropertyLister("1.1.0", "2.0.0", log).print();
            log.println(HORIZONTAL_LINE);
            log.println();
            new DiffPropertyLister("2.0.0", "2.1.2", log).print();
            log.println(HORIZONTAL_LINE);
            log.println();
            new DiffPropertyLister("2.1.2", "3.0.0", log).print();
            log.println(HORIZONTAL_LINE);
            log.println();
            new DiffPropertyLister("3.0.0", "3.1.0", log).print();
            log.println(HORIZONTAL_LINE);
            log.println();
            new DiffPropertyLister("3.1.0", "3.2.0", log).print();
            log.println(HORIZONTAL_LINE);
            log.println();
            new DiffPropertyLister("3.2.0", "4.0.0", log).print();
            /*
            printDiff("110", "200");
	        UnicodeSet u11 = fromFile(BASE_DIR + "UnicodeData\\Versions\\UnicodeData-1.1.txt", false);
	        UnicodeSet u20 = fromFile(BASE_DIR + "UnicodeData\\Versions\\UnicodeData-2.0.txt", false);
	        UnicodeSet u21 = fromFile(BASE_DIR + "UnicodeData\\Versions\\UnicodeData-2.1.txt", false);
	        UnicodeSet u30 = fromFile(BASE_DIR + "UnicodeData\\Versions\\UnicodeData-3.0.txt", false);
	        UnicodeSet u31 = fromFile(BASE_DIR + "UnicodeData\\Versions\\UnicodeData-3.1.txt", false);

            log.println();
            log.println("# Code points assigned in Unicode 1.1 (minus Hangul Syllables): "
                + n.format(u11.count()));
            log.println();
            u11.print(log, false, false, "1.1");

            UnicodeSet u20m = new UnicodeSet(u20).remove(u11);
            log.println();
            log.println("# Code points assigned in Unicode 2.0 (minus Unicode 1.1): "
                + n.format(u20m.count()));
            log.println();
            u20m.print(log, false, false, "2.0");

            UnicodeSet u21m = new UnicodeSet(u21).remove(u20);
            log.println();
            log.println("# Code points assigned in Unicode 2.1 (minus Unicode 2.0): "
                + n.format(u21m.count()));
            log.println();
            u21m.print(log, false, false, "2.1");

            UnicodeSet u30m = new UnicodeSet(u30).remove(u21);
            log.println();
            log.println("# Code points assigned in Unicode 3.0 (minus Unicode 2.1): "
                + n.format(u30m.count()));
            log.println();
            u30m.print(log, false, false, "3.0");

            UnicodeSet u31m = new UnicodeSet(u31).remove(u30);
            log.println();
            log.println("# Code points assigned in Unicode 3.1 (minus Unicode 3.0): "
                + n.format(u31m.count()));
            log.println();
            u31m.print(log, false, false, "3.1");
            */
        } finally {
            if (log != null) {
                log.close();
                Utility.renameIdentical(mostRecent, Utility.getOutputName(newFile), batName[0]);
            }
        }

    }
    
    public static void listCombiningAccents() throws IOException {
        
        PrintWriter log = Utility.openPrintWriter("ListAccents" + UnicodeDataFile.getFileSuffix(true), Utility.LATIN1_UNIX);
        Set set = new TreeSet();
        Set set2 = new TreeSet();
        
        for (int i = 0; i < 0x10FFFF; ++i) {
            Utility.dot(i);
            if (!Default.ucd().isRepresented(i)) continue;
            
            if (Default.nfd().isNormalized(i)) {
                if (Default.ucd().getScript(i) == LATIN_SCRIPT) {
                    int cp = i;
                    String hex = "u" + Utility.hex(cp, 4);
                    set.add("# yyy $x <> \\" + hex + " ; # " + Default.ucd().getName(cp));
                }
                continue;
            }
            
            String decomp = Default.nfd().normalize(i);
            int j;
            for (j = 0; j < decomp.length(); j += UTF16.getCharCount(i)) {
                int cp = UTF16.charAt(decomp, j);
                byte cat = Default.ucd().getCategory(cp);
                if (cat != Mn) continue;
                String hex = "u" + Utility.hex(cp, 4);
                set.add("# xxx $x <> \\" + hex + " ; # " + Default.ucd().getName(cp));
            }
        }
        
        Iterator it = set.iterator();
        while (it.hasNext()) {
            log.println(it.next());
        }
        log.close();
    }
    
    public static void listGreekVowels() throws IOException {
        
        PrintWriter log = Utility.openPrintWriter("ListGreekVowels" + UnicodeDataFile.getFileSuffix(true), Utility.LATIN1_UNIX);
        Set set = new TreeSet();
        Set set2 = new TreeSet();
        
        String vowels = "\u03B1\u03B5\u03B7\u03B9\u03BF\u03C5\u03C9\u0391\u0395\u0397\u0399\u039F\u03A5\u03A9";
        String diphthongEnd = "\u03B9\u03C5\u0399\u03A5";
        String diphthongStart = "\u03B1\u03B5\u03B7\u03BF\u03C5\u0391\u0395\u0397\u039F\u03A5";
        String etas = "\u03B7\u0397";
        String iotas = "\u03B9\u0399";
        
        for (char i = 0; i < 0xFFFF; ++i) {
            Utility.dot(i);
            if (!Default.ucd().isRepresented(i)) continue;
            if (Default.ucd().getScript(i) != GREEK_SCRIPT) continue;
            String decomp = Default.nfd().normalize(i);
            
            if (decomp.indexOf('\u0306') >= 0) continue; // skip breve
            if (decomp.indexOf('\u0304') >= 0) continue; // skip macron
            
            String comp = Default.nfc().normalize(decomp);
            if (!comp.equals(String.valueOf(i))) continue; // skip compats
            
            char first = decomp.charAt(0);
            
            if (vowels.indexOf(first) < 0) continue;
            
            String h = "";
            if (decomp.indexOf('\u0314') >= 0) h = "\uFFFF";
            
            if (diphthongEnd.indexOf(first) >= 0) {
                for (int j = 0; j < diphthongStart.length(); ++j) {
                    String v = diphthongStart.substring(j, j+1);
                    char vc = v.charAt(0);
                    if (Default.ucd().getCategory(vc) == Ll && Default.ucd().getCategory(first) == Lu) continue;
                    if (etas.indexOf(vc) >= 0 && iotas.indexOf(first) >= 0) continue;
                    set.add(new Pair(h + v + first, new Pair(v + decomp, v + i)));
                }
            }
            set.add(new Pair(h+first, new Pair(decomp, String.valueOf(i))));
        }
        
        Iterator it = set.iterator();
        Object last = "";
        while (it.hasNext()) {
            Pair p = (Pair) it.next();
            if (!last.equals(p.first)) {
                log.println();
                last = p.first;
            } else {
                log.print(", ");
            }
            p = (Pair) p.second;
            log.print(p.second);
        }
        log.close();
    }
    
    public static void listKatakana() throws IOException {
        
        
        for (char i = 'a'; i <= 'z'; ++i) {
            doKana(String.valueOf(i));
            if (i == 'c') doKana("ch");
            if (i == 's') doKana("sh");
            if (i == 'd') {
                doKana("dz");
                doKana("dj");
            }
        }
        
        System.out.println();
    }
    
    public static void doKana(String i) {
        
        String vowels = "aeiou";
        System.out.println();
        System.out.print(i + " " + i + i);
        System.out.println();
        for (int j = 0; j < vowels.length(); ++j) {
            char c = vowels.charAt(j);
            System.out.print(" " + i + c);
        }

        System.out.println();
        for (int j = 0; j < vowels.length(); ++j) {
            char c = vowels.charAt(j);
            System.out.print(" " + i + "y" + c);
        }
    }
    
    public static void genTrailingZeros() {
        
        UnicodeSet result = new UnicodeSet();
        for (int i = 0; i < 0x10FFFF; ++i) {
            if ((i & 0xFFF) == 0) System.out.println("# " + i);
            if (!Default.ucd().isAssigned(i)) continue;
            if (Default.nfd().isNormalized(i)) continue;
            String decomp = Default.nfd().normalize(i);
            int cp;
            for (int j = 0; j < decomp.length(); j += UTF16.getCharCount(cp)) {
                cp = UTF16.charAt(decomp,j);
                if (j == 0) continue; // skip first
                if (Default.ucd().getCombiningClass(cp) == 0) {
                    result.add(cp);
                }
            }
        }
        int rangeCount = result.getRangeCount();
        for (int k = 0; k < rangeCount; ++k) {
            int start = result.getRangeStart(k);
            int end = result.getRangeEnd(k);
            System.out.println(
                Utility.hex(start)
                + (start != end ? ".." + Utility.hex(end) : "")
                + "; "
                + Default.ucd().getName(start)
                + (start != end ? ".." + Default.ucd().getName(end) : ""));
        }
        System.out.println("TrailingZero count: " + result.size());
    }
}