From fbfe74d4c33a083fba69940583fcb23c20bab707 Mon Sep 17 00:00:00 2001 From: Eric Mader Date: Wed, 19 Jul 2000 21:17:18 +0000 Subject: [PATCH] New syntax, copied most recent default rules. X-SVN-Rev: 1952 --- .../rbbi/BreakIteratorRules_en_US_TEST.java | 285 ++++++++++++------ .../rbbi/BreakIteratorRules_en_US_TEST.java | 285 ++++++++++++------ 2 files changed, 394 insertions(+), 176 deletions(-) diff --git a/icu4j/src/com/ibm/icu/dev/test/rbbi/BreakIteratorRules_en_US_TEST.java b/icu4j/src/com/ibm/icu/dev/test/rbbi/BreakIteratorRules_en_US_TEST.java index 12fbbb667e..71694915b3 100755 --- a/icu4j/src/com/ibm/icu/dev/test/rbbi/BreakIteratorRules_en_US_TEST.java +++ b/icu4j/src/com/ibm/icu/dev/test/rbbi/BreakIteratorRules_en_US_TEST.java @@ -5,8 +5,8 @@ ******************************************************************************* * * $Source: /xsrl/Nsvn/icu/icu4j/src/com/ibm/icu/dev/test/rbbi/BreakIteratorRules_en_US_TEST.java,v $ - * $Date: 2000/03/10 03:47:46 $ - * $Revision: 1.3 $ + * $Date: 2000/07/19 21:17:18 $ + * $Revision: 1.4 $ * ***************************************************************************************** */ @@ -22,92 +22,201 @@ import java.net.URL; * with good resource data (and a good dictionary file) for Thai */ public class BreakIteratorRules_en_US_TEST extends ListResourceBundle { + private static final URL url = + BreakIteratorRules_en_US_TEST.class.getResource("/com/ibm/test/rbbi/english.dict"); + public Object[][] getContents() { - URL url = getClass().getResource("english.dict"); - - // if dictionary wasn't found, then this resource bundle doesn't have - // much to contribute... - if (url == null) { - return new Object[0][0]; - } - return new Object[][] { - // names of classes to instantiate for the different kinds of break - // iterator. Notice we're now using DictionaryBasedBreakIterator - // for word and line breaking. - { "BreakIteratorClasses", - new String[] { "RuleBasedBreakIterator", // character-break iterator class - "DictionaryBasedBreakIterator", // word-break iterator class - "DictionaryBasedBreakIterator", // line-break iterator class - "RuleBasedBreakIterator" } // sentence-break iterator class - }, - - // These are the same word-breaking rules as are specified in the default - // resource, except that the Latin letters, apostrophe, and hyphen are - // specified as dictionary characters - { "WordBreakRules", - "$ignore=[[:Mn:][:Me:][:Cf:]];" - + "dictionary=[a-zA-z\\'\\-];" - + "kanji=[\u3005\u4e00-\u9fa5\uf900-\ufa2d];" - + "kata=[\u30a1-\u30fa];" - + "hira=[\u3041-\u3094];" - + "cjk-diacrit=[\u3099-\u309c];" - + "let=[[:L:]^[{kanji}{kata}{hira}{cjk-diacrit}{dictionary}]];" - + "dgt=[[:N:]];" - + "mid-word=[[:Pd:]\u00ad\u2027\\\"\\\'\\.];" - + "mid-num=[\\\"\\\'\\,\u066b\\.];" - + "pre-num=[[:Sc:]\\#\\.^\u00a2];" - + "post-num=[\\%\\&\u00a2\u066a\u2030\u2031];" - + "ls=[\n\u000c\u2028\u2029];" - + "ws=[[:Zs:]\t];" - + "word=({let}{let}*({mid-word}{let}{let}*)*|[a-zA-Z][a-z\\'\\-]*);" - + "number=({dgt}{dgt}*({mid-num}{dgt}{dgt}*)*);" - + ".;" - + "{{word}}({number}{word})*{{number}{{post-num}}};" - + "{pre-num}({number}{word})*{{number}{{post-num}}};" - + "{ws}*{\r}{{ls}};" - + "[{kata}{cjk-diacrit}]*;" - + "[{hira}{cjk-diacrit}]*;" - + "{kanji}*;" }, - - // These are the same line-breaking rules as are specified in the default - // resource, except that the Latin letters, apostrophe, and hyphen are - // specified as dictionary characters - { "LineBreakRules", - "=[:Mn::Me::Cf:];" - + "=[a-zA-z\\'\\-];" - + "=[\u0003\t\n\f\u2028\u2029];" - + "=[\u00a0\u2007\u2011\ufeff];" - + "=[:Zs::Cc:^[\r]];" - + "=[:Pd:\u00ad^];" - + "=[:Sc::Ps:^\u00a2];" - + "=[:Pe:\\!\\%\\.\\,\\:\\;\\?\u00a2\u00b0\u066a\u2030-\u2034\u2103" - + "\u2105\u2109\u3001\u3002\u3005\u3041\u3043\u3045\u3047\u3049\u3063" - + "\u3083\u3085\u3087\u308e\u3099-\u309e\u30a1\u30a3\u30a5\u30a7\u30a9" - + "\u30c3\u30e3\u30e5\u30e7\u30ee\u30f5\u30f6\u30fc-\u30fe\uff01\uff0e" - + "\uff1f];" - + "=[\u4e00-\u9fa5\uf900-\ufa2d\u3041-\u3094\u30a1-\u30fa^[]];" - + "=[:Nd::No:];" - + "=[\\.\\,];" - + "=[^[\r]];" - + "=([]**(*)*);" - + "=(*|||[a-zA-Z][a-z\\'\\-]*);" - + "=((*|*)*);" - + "=(*);" - + "(*)*{\r}{};" }, - - // these two resources specify the pathnames of the dictionary files to - // use for word breaking and line breaking. Both currently refer to - // a file called english.dict placed in com\ibm\text\resources - // somewhere in the class path. It's important to note that - // english.dict was created for testing purposes only, and doesn't - // come anywhere close to being an exhaustive dictionary of English - // words (basically, it contains all the words in the Declaration of - // Independence, and the Revised Standard Version of the book of Genesis, - // plus a few other words thrown in to show more interesting cases). - // { "WordBreakDictionary", "com\\ibm\\text\\resources\\english.dict" }, - // { "LineBreakDictionary", "com\\ibm\\text\\resources\\english.dict" } - { "WordBreakDictionary", url }, - { "LineBreakDictionary", url } - }; + return contents; } + + static final Object[][] contents = { + // names of classes to instantiate for the different kinds of break + // iterator. Notice we're now using DictionaryBasedBreakIterator + // for word and line breaking. + { "BreakIteratorClasses", + new String[] { "RuleBasedBreakIterator", // character-break iterator class + "DictionaryBasedBreakIterator", // word-break iterator class + "DictionaryBasedBreakIterator", // line-break iterator class + "RuleBasedBreakIterator" } // sentence-break iterator class + }, + + // These are the same word-breaking rules as are specified in the default + // resource, except that the Latin letters, apostrophe, and hyphen are + // specified as dictionary characters + { "WordBreakRules", + // ignore non-spacing marks, enclosing marks, and format characters, + // all of which should not influence the algorithm + "$ignore=[[:Mn:][:Me:][:Cf:]];" + + // lower and upper case Roman letters, apostrophy and dash are + // in the English dictionary + + "$dictionary=[a-zA-Z\\'\\-];" + + // Hindi phrase separator, kanji, katakana, hiragana, CJK diacriticals, + // other letters, and digits + + "danda=[\u0964\u0965];" + + "kanji=[\u3005\u4e00-\u9fa5\uf900-\ufa2d];" + + "kata=[\u3099-\u309c\u30a1-\u30fe];" + + "hira=[\u3041-\u309e\u30fc];" + + "let=[[[:L:][:Mc:]]-[{kanji}{kata}{hira}]];" + + "dgt=[:N:];" + + // punctuation that can occur in the middle of a word: currently + // dashes, apostrophes, and quotation marks + + "mid-word=[[:Pd:]\u00ad\u2027\\\"\\\'];" + + // punctuation that can occur in the middle of a number: currently + // apostrophes, qoutation marks, periods, commas, and the Arabic + // decimal point + + "mid-num=[\\\"\\\'\\,\u066b\\.];" + + // punctuation that can occur at the beginning of a number: currently + // the period, the number sign, and all currency symbols except the cents sign + + "pre-num=[[[:Sc:]-[\u00a2]]\\#\\.];" + + // punctuation that can occur at the end of a number: currently + // the percent, per-thousand, per-ten-thousand, and Arabic percent + // signs, the cents sign, and the ampersand + + "post-num=[\\%\\&\u00a2\u066a\u2030\u2031];" + + // line separators: currently LF, FF, PS, and LS + + "ls=[\n\u000c\u2028\u2029];" + + // whitespace: all space separators and the tab character + + "ws=[[:Zs:]\t];" + + // a word is a sequence of letters that may contain internal + // punctuation, as long as it begins and ends with a letter and + // never contains two punctuation marks in a row + + "word=({let}+({mid-word}{let}+)*{danda}?);" + + // a number is a sequence of digits that may contain internal + // punctuation, as long as it begins and ends with a digit and + // never contains two punctuation marks in a row. + + "number=({dgt}+({mid-num}{dgt}+)*);" + + // break after every character, with the following exceptions + // (this will cause punctuation marks that aren't considered + // part of words or numbers to be treated as words unto themselves) + + ".;" + + // keep together any sequence of contiguous words and numbers + // (including just one of either), plus an optional trailing + // number-suffix character + + "{word}?({number}{word})*({number}{post-num}?)?;" + + // keep together and sequence of contiguous words and numbers + // that starts with a number-prefix character and a number, + // and may end with a number-suffix character + + "{pre-num}({number}{word})*({number}{post-num}?)?;" + + // keep together runs of whitespace (optionally with a single trailing + // line separator or CRLF sequence) + + "{ws}*\r?{ls}?;" + + // keep together runs of Katakana + + "{kata}*;" + + // keep together runs of Hiragana + + "{hira}*;" + + // keep together runs of Kanji + + "{kanji}*;"}, + + // These are the same line-breaking rules as are specified in the default + // resource, except that the Latin letters, apostrophe, and hyphen are + // specified as dictionary characters + { "LineBreakRules", + // ignore non-spacing marks, enclosing marks, and format characters + "$ignore=[[:Mn:][:Me:][:Cf:]];" + + // lower and upper case Roman letters, apostrophy and dash + // are in the English dictionary + + "$dictionary=[a-zA-Z\\'\\-];" + + // Hindi phrase separators + + "danda=[\u0964\u0965];" + + // characters that always cause a break: ETX, tab, LF, FF, LS, and PS + + "break=[\u0003\t\n\f\u2028\u2029];" + + // characters that always prevent a break: the non-breaking space + // and similar characters + + "nbsp=[\u00a0\u2007\u2011\ufeff];" + + // whitespace: space separators and control characters, except for + // CR and the other characters mentioned above + + "space=[[[:Zs:][:Cc:]]-[{nbsp}{break}\r]];" + + // dashes: dash punctuation and the discretionary hyphen, except for + // non-breaking hyphens + + "dash=[[[:Pd:]\u00ad]-[{nbsp}]];" + + // characters that stick to a word if they precede it: currency symbols + // (except the cents sign) and starting punctuation + + "pre-word=[[[:Sc:]-[\u00a2]][:Ps:]\\\"\\\'];" + + // characters that stick to a word if they follow it: ending punctuation, + // other punctuation that usually occurs at the end of a sentence, + // small Kana characters, some CJK diacritics, etc. + + "post-word=[[:Pe:]\\!\\\"\\\'\\%\\.\\,\\:\\;\\?\u00a2\u00b0\u066a\u2030-\u2034" + + "\u2103\u2105\u2109\u3001\u3002\u3005\u3041\u3043\u3045\u3047\u3049\u3063" + + "\u3083\u3085\u3087\u308e\u3099-\u309e\u30a1\u30a3\u30a5\u30a7\u30a9" + + "\u30c3\u30e3\u30e5\u30e7\u30ee\u30f5\u30f6\u30fc-\u30fe\uff01\uff0c" + + "\uff0e\uff1f];" + + // Kanji: actually includes both Kanji and Kana, except for small Kana and + // CJK diacritics + + "kanji=[[\u4e00-\u9fa5\uf900-\ufa2d\u3041-\u3094\u30a1-\u30fa]-[{post-word}{$ignore}]];" + + // digits + + "digit=[[:Nd:][:No:]];" + + // punctuation that can occur in the middle of a number: periods and commas + + "mid-num=[\\.\\,];" + + // everything not mentioned above, plus the quote marks (which are both + // , , and ) + + "char=[^{break}{space}{dash}{kanji}{nbsp}{$ignore}{pre-word}{post-word}{mid-num}{danda}\r\\\"\\\'];" + + // a "number" is a run of prefix characters and dashes, followed by one or + // more digits with isolated number-punctuation characters interspersed + + "number=([{pre-word}{dash}]*{digit}+({mid-num}{digit}+)*);" + + // the basic core of a word can be either a "number" as defined above, a single + // "Kanji" character, or a run of any number of not-explicitly-mentioned + // characters (this includes Latin letters) + + "word-core=([{pre-word}{char}]*|{kanji}|{number});" + + // a word may end with an optional suffix that be either a run of one or + // more dashes or a run of word-suffix characters, followed by an optional + // run of whitespace + + "word-suffix=(({dash}+|{post-word}*){space}*);" + + // a word, thus, is an optional run of word-prefix characters, followed by + // a word core and a word suffix (the syntax of and + // actually allows either of them to match the empty string, putting a break + // between things like ")(" or "aaa(aaa" + + "word=({pre-word}*{word-core}{word-suffix});" + + // finally, the rule that does the work: Keep together any run of words that + // are joined by runs of one of more non-spacing mark. Also keep a trailing + // line-break character or CRLF combination with the word. (line separators + // "win" over nbsp's) + + "{word}({nbsp}+{word})*\r?{break}?;" }, + + // these two resources specify the pathnames of the dictionary files to + // use for word breaking and line breaking. Both currently refer to + // a file called english.dict placed in com\ibm\text\resources + // somewhere in the class path. It's important to note that + // english.dict was created for testing purposes only, and doesn't + // come anywhere close to being an exhaustive dictionary of English + // words (basically, it contains all the words in the Declaration of + // Independence, and the Revised Standard Version of the book of Genesis, + // plus a few other words thrown in to show more interesting cases). + // { "WordBreakDictionary", "com\\ibm\\text\\resources\\english.dict" }, + // { "LineBreakDictionary", "com\\ibm\\text\\resources\\english.dict" } + { "WordBreakDictionary", url }, + { "LineBreakDictionary", url } + }; } diff --git a/icu4j/src/com/ibm/test/rbbi/BreakIteratorRules_en_US_TEST.java b/icu4j/src/com/ibm/test/rbbi/BreakIteratorRules_en_US_TEST.java index cae8e3f93b..09eb419ac6 100755 --- a/icu4j/src/com/ibm/test/rbbi/BreakIteratorRules_en_US_TEST.java +++ b/icu4j/src/com/ibm/test/rbbi/BreakIteratorRules_en_US_TEST.java @@ -5,8 +5,8 @@ ******************************************************************************* * * $Source: /xsrl/Nsvn/icu/icu4j/src/com/ibm/test/rbbi/Attic/BreakIteratorRules_en_US_TEST.java,v $ - * $Date: 2000/03/10 03:47:46 $ - * $Revision: 1.3 $ + * $Date: 2000/07/19 21:17:18 $ + * $Revision: 1.4 $ * ***************************************************************************************** */ @@ -22,92 +22,201 @@ import java.net.URL; * with good resource data (and a good dictionary file) for Thai */ public class BreakIteratorRules_en_US_TEST extends ListResourceBundle { + private static final URL url = + BreakIteratorRules_en_US_TEST.class.getResource("/com/ibm/test/rbbi/english.dict"); + public Object[][] getContents() { - URL url = getClass().getResource("english.dict"); - - // if dictionary wasn't found, then this resource bundle doesn't have - // much to contribute... - if (url == null) { - return new Object[0][0]; - } - return new Object[][] { - // names of classes to instantiate for the different kinds of break - // iterator. Notice we're now using DictionaryBasedBreakIterator - // for word and line breaking. - { "BreakIteratorClasses", - new String[] { "RuleBasedBreakIterator", // character-break iterator class - "DictionaryBasedBreakIterator", // word-break iterator class - "DictionaryBasedBreakIterator", // line-break iterator class - "RuleBasedBreakIterator" } // sentence-break iterator class - }, - - // These are the same word-breaking rules as are specified in the default - // resource, except that the Latin letters, apostrophe, and hyphen are - // specified as dictionary characters - { "WordBreakRules", - "$ignore=[[:Mn:][:Me:][:Cf:]];" - + "dictionary=[a-zA-z\\'\\-];" - + "kanji=[\u3005\u4e00-\u9fa5\uf900-\ufa2d];" - + "kata=[\u30a1-\u30fa];" - + "hira=[\u3041-\u3094];" - + "cjk-diacrit=[\u3099-\u309c];" - + "let=[[:L:]^[{kanji}{kata}{hira}{cjk-diacrit}{dictionary}]];" - + "dgt=[[:N:]];" - + "mid-word=[[:Pd:]\u00ad\u2027\\\"\\\'\\.];" - + "mid-num=[\\\"\\\'\\,\u066b\\.];" - + "pre-num=[[:Sc:]\\#\\.^\u00a2];" - + "post-num=[\\%\\&\u00a2\u066a\u2030\u2031];" - + "ls=[\n\u000c\u2028\u2029];" - + "ws=[[:Zs:]\t];" - + "word=({let}{let}*({mid-word}{let}{let}*)*|[a-zA-Z][a-z\\'\\-]*);" - + "number=({dgt}{dgt}*({mid-num}{dgt}{dgt}*)*);" - + ".;" - + "{{word}}({number}{word})*{{number}{{post-num}}};" - + "{pre-num}({number}{word})*{{number}{{post-num}}};" - + "{ws}*{\r}{{ls}};" - + "[{kata}{cjk-diacrit}]*;" - + "[{hira}{cjk-diacrit}]*;" - + "{kanji}*;" }, - - // These are the same line-breaking rules as are specified in the default - // resource, except that the Latin letters, apostrophe, and hyphen are - // specified as dictionary characters - { "LineBreakRules", - "=[:Mn::Me::Cf:];" - + "=[a-zA-z\\'\\-];" - + "=[\u0003\t\n\f\u2028\u2029];" - + "=[\u00a0\u2007\u2011\ufeff];" - + "=[:Zs::Cc:^[\r]];" - + "=[:Pd:\u00ad^];" - + "=[:Sc::Ps:^\u00a2];" - + "=[:Pe:\\!\\%\\.\\,\\:\\;\\?\u00a2\u00b0\u066a\u2030-\u2034\u2103" - + "\u2105\u2109\u3001\u3002\u3005\u3041\u3043\u3045\u3047\u3049\u3063" - + "\u3083\u3085\u3087\u308e\u3099-\u309e\u30a1\u30a3\u30a5\u30a7\u30a9" - + "\u30c3\u30e3\u30e5\u30e7\u30ee\u30f5\u30f6\u30fc-\u30fe\uff01\uff0e" - + "\uff1f];" - + "=[\u4e00-\u9fa5\uf900-\ufa2d\u3041-\u3094\u30a1-\u30fa^[]];" - + "=[:Nd::No:];" - + "=[\\.\\,];" - + "=[^[\r]];" - + "=([]**(*)*);" - + "=(*|||[a-zA-Z][a-z\\'\\-]*);" - + "=((*|*)*);" - + "=(*);" - + "(*)*{\r}{};" }, - - // these two resources specify the pathnames of the dictionary files to - // use for word breaking and line breaking. Both currently refer to - // a file called english.dict placed in com\ibm\text\resources - // somewhere in the class path. It's important to note that - // english.dict was created for testing purposes only, and doesn't - // come anywhere close to being an exhaustive dictionary of English - // words (basically, it contains all the words in the Declaration of - // Independence, and the Revised Standard Version of the book of Genesis, - // plus a few other words thrown in to show more interesting cases). - // { "WordBreakDictionary", "com\\ibm\\text\\resources\\english.dict" }, - // { "LineBreakDictionary", "com\\ibm\\text\\resources\\english.dict" } - { "WordBreakDictionary", url }, - { "LineBreakDictionary", url } - }; + return contents; } + + static final Object[][] contents = { + // names of classes to instantiate for the different kinds of break + // iterator. Notice we're now using DictionaryBasedBreakIterator + // for word and line breaking. + { "BreakIteratorClasses", + new String[] { "RuleBasedBreakIterator", // character-break iterator class + "DictionaryBasedBreakIterator", // word-break iterator class + "DictionaryBasedBreakIterator", // line-break iterator class + "RuleBasedBreakIterator" } // sentence-break iterator class + }, + + // These are the same word-breaking rules as are specified in the default + // resource, except that the Latin letters, apostrophe, and hyphen are + // specified as dictionary characters + { "WordBreakRules", + // ignore non-spacing marks, enclosing marks, and format characters, + // all of which should not influence the algorithm + "$ignore=[[:Mn:][:Me:][:Cf:]];" + + // lower and upper case Roman letters, apostrophy and dash are + // in the English dictionary + + "$dictionary=[a-zA-Z\\'\\-];" + + // Hindi phrase separator, kanji, katakana, hiragana, CJK diacriticals, + // other letters, and digits + + "danda=[\u0964\u0965];" + + "kanji=[\u3005\u4e00-\u9fa5\uf900-\ufa2d];" + + "kata=[\u3099-\u309c\u30a1-\u30fe];" + + "hira=[\u3041-\u309e\u30fc];" + + "let=[[[:L:][:Mc:]]-[{kanji}{kata}{hira}]];" + + "dgt=[:N:];" + + // punctuation that can occur in the middle of a word: currently + // dashes, apostrophes, and quotation marks + + "mid-word=[[:Pd:]\u00ad\u2027\\\"\\\'];" + + // punctuation that can occur in the middle of a number: currently + // apostrophes, qoutation marks, periods, commas, and the Arabic + // decimal point + + "mid-num=[\\\"\\\'\\,\u066b\\.];" + + // punctuation that can occur at the beginning of a number: currently + // the period, the number sign, and all currency symbols except the cents sign + + "pre-num=[[[:Sc:]-[\u00a2]]\\#\\.];" + + // punctuation that can occur at the end of a number: currently + // the percent, per-thousand, per-ten-thousand, and Arabic percent + // signs, the cents sign, and the ampersand + + "post-num=[\\%\\&\u00a2\u066a\u2030\u2031];" + + // line separators: currently LF, FF, PS, and LS + + "ls=[\n\u000c\u2028\u2029];" + + // whitespace: all space separators and the tab character + + "ws=[[:Zs:]\t];" + + // a word is a sequence of letters that may contain internal + // punctuation, as long as it begins and ends with a letter and + // never contains two punctuation marks in a row + + "word=({let}+({mid-word}{let}+)*{danda}?);" + + // a number is a sequence of digits that may contain internal + // punctuation, as long as it begins and ends with a digit and + // never contains two punctuation marks in a row. + + "number=({dgt}+({mid-num}{dgt}+)*);" + + // break after every character, with the following exceptions + // (this will cause punctuation marks that aren't considered + // part of words or numbers to be treated as words unto themselves) + + ".;" + + // keep together any sequence of contiguous words and numbers + // (including just one of either), plus an optional trailing + // number-suffix character + + "{word}?({number}{word})*({number}{post-num}?)?;" + + // keep together and sequence of contiguous words and numbers + // that starts with a number-prefix character and a number, + // and may end with a number-suffix character + + "{pre-num}({number}{word})*({number}{post-num}?)?;" + + // keep together runs of whitespace (optionally with a single trailing + // line separator or CRLF sequence) + + "{ws}*\r?{ls}?;" + + // keep together runs of Katakana + + "{kata}*;" + + // keep together runs of Hiragana + + "{hira}*;" + + // keep together runs of Kanji + + "{kanji}*;"}, + + // These are the same line-breaking rules as are specified in the default + // resource, except that the Latin letters, apostrophe, and hyphen are + // specified as dictionary characters + { "LineBreakRules", + // ignore non-spacing marks, enclosing marks, and format characters + "$ignore=[[:Mn:][:Me:][:Cf:]];" + + // lower and upper case Roman letters, apostrophy and dash + // are in the English dictionary + + "$dictionary=[a-zA-Z\\'\\-];" + + // Hindi phrase separators + + "danda=[\u0964\u0965];" + + // characters that always cause a break: ETX, tab, LF, FF, LS, and PS + + "break=[\u0003\t\n\f\u2028\u2029];" + + // characters that always prevent a break: the non-breaking space + // and similar characters + + "nbsp=[\u00a0\u2007\u2011\ufeff];" + + // whitespace: space separators and control characters, except for + // CR and the other characters mentioned above + + "space=[[[:Zs:][:Cc:]]-[{nbsp}{break}\r]];" + + // dashes: dash punctuation and the discretionary hyphen, except for + // non-breaking hyphens + + "dash=[[[:Pd:]\u00ad]-[{nbsp}]];" + + // characters that stick to a word if they precede it: currency symbols + // (except the cents sign) and starting punctuation + + "pre-word=[[[:Sc:]-[\u00a2]][:Ps:]\\\"\\\'];" + + // characters that stick to a word if they follow it: ending punctuation, + // other punctuation that usually occurs at the end of a sentence, + // small Kana characters, some CJK diacritics, etc. + + "post-word=[[:Pe:]\\!\\\"\\\'\\%\\.\\,\\:\\;\\?\u00a2\u00b0\u066a\u2030-\u2034" + + "\u2103\u2105\u2109\u3001\u3002\u3005\u3041\u3043\u3045\u3047\u3049\u3063" + + "\u3083\u3085\u3087\u308e\u3099-\u309e\u30a1\u30a3\u30a5\u30a7\u30a9" + + "\u30c3\u30e3\u30e5\u30e7\u30ee\u30f5\u30f6\u30fc-\u30fe\uff01\uff0c" + + "\uff0e\uff1f];" + + // Kanji: actually includes both Kanji and Kana, except for small Kana and + // CJK diacritics + + "kanji=[[\u4e00-\u9fa5\uf900-\ufa2d\u3041-\u3094\u30a1-\u30fa]-[{post-word}{$ignore}]];" + + // digits + + "digit=[[:Nd:][:No:]];" + + // punctuation that can occur in the middle of a number: periods and commas + + "mid-num=[\\.\\,];" + + // everything not mentioned above, plus the quote marks (which are both + // , , and ) + + "char=[^{break}{space}{dash}{kanji}{nbsp}{$ignore}{pre-word}{post-word}{mid-num}{danda}\r\\\"\\\'];" + + // a "number" is a run of prefix characters and dashes, followed by one or + // more digits with isolated number-punctuation characters interspersed + + "number=([{pre-word}{dash}]*{digit}+({mid-num}{digit}+)*);" + + // the basic core of a word can be either a "number" as defined above, a single + // "Kanji" character, or a run of any number of not-explicitly-mentioned + // characters (this includes Latin letters) + + "word-core=([{pre-word}{char}]*|{kanji}|{number});" + + // a word may end with an optional suffix that be either a run of one or + // more dashes or a run of word-suffix characters, followed by an optional + // run of whitespace + + "word-suffix=(({dash}+|{post-word}*){space}*);" + + // a word, thus, is an optional run of word-prefix characters, followed by + // a word core and a word suffix (the syntax of and + // actually allows either of them to match the empty string, putting a break + // between things like ")(" or "aaa(aaa" + + "word=({pre-word}*{word-core}{word-suffix});" + + // finally, the rule that does the work: Keep together any run of words that + // are joined by runs of one of more non-spacing mark. Also keep a trailing + // line-break character or CRLF combination with the word. (line separators + // "win" over nbsp's) + + "{word}({nbsp}+{word})*\r?{break}?;" }, + + // these two resources specify the pathnames of the dictionary files to + // use for word breaking and line breaking. Both currently refer to + // a file called english.dict placed in com\ibm\text\resources + // somewhere in the class path. It's important to note that + // english.dict was created for testing purposes only, and doesn't + // come anywhere close to being an exhaustive dictionary of English + // words (basically, it contains all the words in the Declaration of + // Independence, and the Revised Standard Version of the book of Genesis, + // plus a few other words thrown in to show more interesting cases). + // { "WordBreakDictionary", "com\\ibm\\text\\resources\\english.dict" }, + // { "LineBreakDictionary", "com\\ibm\\text\\resources\\english.dict" } + { "WordBreakDictionary", url }, + { "LineBreakDictionary", url } + }; }