ICU-50 RBBI Rules for Surrogates
X-SVN-Rev: 5579
This commit is contained in:
Andy Heninger
parent
4bc449ea45
commit
72ed312676
@ -5,8 +5,8 @@
|
||||
*******************************************************************************
|
||||
*
|
||||
* $Source: /xsrl/Nsvn/icu/icu4j/src/com/ibm/icu/impl/data/BreakIteratorRules.java,v $
|
||||
* $Date: 2001/02/06 22:37:45 $
|
||||
* $Revision: 1.5 $
|
||||
* $Date: 2001/08/24 17:36:24 $
|
||||
* $Revision: 1.6 $
|
||||
*
|
||||
*****************************************************************************************
|
||||
*/
|
||||
@ -97,17 +97,25 @@ public class BreakIteratorRules extends ListResourceBundle {
|
||||
|
||||
// default rules for finding word boundaries
|
||||
{ "WordBreakRules",
|
||||
|
||||
// Surrogates. Until better support is available, ignore low surrogates
|
||||
// and classify high surrogates according to the characters within the block.
|
||||
"$surr_lo=[\udc00-\udfff];"
|
||||
+ "$surr_hi_let=[\ud800\ud801\ud834\ud835];" // Hi Surrogates for Old Italic, Gothic, Deseret, Music, Math
|
||||
+ "$surr_hi_ideo=[\ud840-\ud880];" // Hi Surrogates for CJK
|
||||
+ "$surr_hi_misc=[\udb40-\udbff];" // Hi Surrogates for Tags, Private Use.
|
||||
|
||||
// ignore non-spacing marks, enclosing marks, and format characters,
|
||||
// all of which should not influence the algorithm
|
||||
"$_ignore_=[[:Mn:][:Me:][:Cf:]];"
|
||||
+ "$_ignore_=[[:Mn:][:Me:][:Cf:]$surr_lo$surr_hi_misc];"
|
||||
|
||||
// Hindi phrase separator, kanji, katakana, hiragana, CJK diacriticals,
|
||||
// other letters, and digits
|
||||
+ "$danda=[\u0964\u0965];"
|
||||
+ "$kanji=[\u3005\u4e00-\u9fa5\uf900-\ufa2d];"
|
||||
+ "$kanji=[\u3005\u4e00-\u9fa5\uf900-\ufa2d$surr_hi_ideo];"
|
||||
+ "$kata=[\u3099-\u309c\u30a1-\u30fe];"
|
||||
+ "$hira=[\u3041-\u309e\u30fc];"
|
||||
+ "$let=[[[:L:][:Mc:]]-[$kanji$kata$hira]];"
|
||||
+ "$let=[[[:L:][:Mc:]$surr_hi_let]-[$kanji$kata$hira]];"
|
||||
+ "$dgt=[:N:];"
|
||||
|
||||
// punctuation that can occur in the middle of a word: currently
|
||||
@ -175,8 +183,15 @@ public class BreakIteratorRules extends ListResourceBundle {
|
||||
|
||||
// default rules for determining legal line-breaking positions
|
||||
{ "LineBreakRules",
|
||||
// Surrogates. Until better support is available, ignore low surrogates
|
||||
// and classify high surrogates according to the characters within the block.
|
||||
"$surr_lo=[\udc00-\udfff];"
|
||||
+ "$surr_hi_let=[\ud800\ud801\ud834\ud835];" // Hi Surrogates for Old Italic, Gothic, Deseret, Music, Math
|
||||
+ "$surr_hi_ideo=[\ud840-\ud880];" // Hi Surrogates for CJK
|
||||
+ "$surr_hi_misc=[\udb40-\udbff];" // Hi Surrogates for Tags, Private Use.
|
||||
|
||||
// ignore non-spacing marks, enclosing marks, and format characters
|
||||
"$_ignore_=[[:Mn:][:Me:][:Cf:]];"
|
||||
+ "$_ignore_=[[:Mn:][:Me:][:Cf:]$surr_lo$surr_hi_misc];"
|
||||
|
||||
// Hindi phrase separators
|
||||
+ "$danda=[\u0964\u0965];"
|
||||
@ -211,7 +226,7 @@ public class BreakIteratorRules extends ListResourceBundle {
|
||||
|
||||
// Kanji: actually includes both Kanji and Kana, except for small Kana and
|
||||
// CJK diacritics
|
||||
+ "$kanji=[[\u4e00-\u9fa5\uf900-\ufa2d\u3041-\u3094\u30a1-\u30fa]-[$post_word$_ignore_]];"
|
||||
+ "$kanji=[[\u4e00-\u9fa5\uf900-\ufa2d\u3041-\u3094\u30a1-\u30fa$surr_hi_ideo]-[$post_word$_ignore_]];"
|
||||
|
||||
// digits
|
||||
+ "$digit=[[:Nd:][:No:]];"
|
||||
@ -252,8 +267,15 @@ public class BreakIteratorRules extends ListResourceBundle {
|
||||
|
||||
// default rules for finding sentence boundaries
|
||||
{ "SentenceBreakRules",
|
||||
// Surrogates. Until better support is available, ignore low surrogates
|
||||
// and classify high surrogates according to the characters within the block.
|
||||
"$surr_lo=[\udc00-\udfff];"
|
||||
+ "$surr_hi_let=[\ud800\ud801\ud834\ud835];" // Hi Surrogates for Old Italic, Gothic, Deseret, Music, Math
|
||||
+ "$surr_hi_ideo=[\ud840-\ud880];" // Hi Surrogates for CJK
|
||||
+ "$surr_hi_misc=[\udb40-\udbff];" // Hi Surrogates for Tags, Private Use.
|
||||
|
||||
// ignore non-spacing marks, enclosing marks, and format characters
|
||||
"$_ignore_=[[:Mn:][:Me:][:Cf:]];"
|
||||
+ "$_ignore_=[[:Mn:][:Me:][:Cf:]$surr_lo$surr_hi_misc];"
|
||||
|
||||
// lowercase letters
|
||||
+ "$lc=[:Ll:];"
|
||||
@ -283,7 +305,7 @@ public class BreakIteratorRules extends ListResourceBundle {
|
||||
|
||||
// characters that may occur at the beginning of a sentence: basically anything
|
||||
// not mentioned above (lowercase letters and digits are specifically excluded)
|
||||
+ "$sent_start=[^$lc$ucLatin$space$start$end$digit$term$period\u2029$_ignore_];"
|
||||
+ "$sent_start=[^$lc$ucLatin$space$start$end$digit$term$period\u2029$surr_hi_let$_ignore_];"
|
||||
|
||||
// Hindi phrase separator
|
||||
+ "$danda=[\u0964\u0965];"
|
||||
|
||||
@ -5,8 +5,8 @@
|
||||
*******************************************************************************
|
||||
*
|
||||
* $Source: /xsrl/Nsvn/icu/icu4j/src/com/ibm/text/resources/Attic/BreakIteratorRules.java,v $
|
||||
* $Date: 2001/02/06 22:37:45 $
|
||||
* $Revision: 1.5 $
|
||||
* $Date: 2001/08/24 17:36:24 $
|
||||
* $Revision: 1.6 $
|
||||
*
|
||||
*****************************************************************************************
|
||||
*/
|
||||
@ -97,17 +97,25 @@ public class BreakIteratorRules extends ListResourceBundle {
|
||||
|
||||
// default rules for finding word boundaries
|
||||
{ "WordBreakRules",
|
||||
|
||||
// Surrogates. Until better support is available, ignore low surrogates
|
||||
// and classify high surrogates according to the characters within the block.
|
||||
"$surr_lo=[\udc00-\udfff];"
|
||||
+ "$surr_hi_let=[\ud800\ud801\ud834\ud835];" // Hi Surrogates for Old Italic, Gothic, Deseret, Music, Math
|
||||
+ "$surr_hi_ideo=[\ud840-\ud880];" // Hi Surrogates for CJK
|
||||
+ "$surr_hi_misc=[\udb40-\udbff];" // Hi Surrogates for Tags, Private Use.
|
||||
|
||||
// ignore non-spacing marks, enclosing marks, and format characters,
|
||||
// all of which should not influence the algorithm
|
||||
"$_ignore_=[[:Mn:][:Me:][:Cf:]];"
|
||||
+ "$_ignore_=[[:Mn:][:Me:][:Cf:]$surr_lo$surr_hi_misc];"
|
||||
|
||||
// Hindi phrase separator, kanji, katakana, hiragana, CJK diacriticals,
|
||||
// other letters, and digits
|
||||
+ "$danda=[\u0964\u0965];"
|
||||
+ "$kanji=[\u3005\u4e00-\u9fa5\uf900-\ufa2d];"
|
||||
+ "$kanji=[\u3005\u4e00-\u9fa5\uf900-\ufa2d$surr_hi_ideo];"
|
||||
+ "$kata=[\u3099-\u309c\u30a1-\u30fe];"
|
||||
+ "$hira=[\u3041-\u309e\u30fc];"
|
||||
+ "$let=[[[:L:][:Mc:]]-[$kanji$kata$hira]];"
|
||||
+ "$let=[[[:L:][:Mc:]$surr_hi_let]-[$kanji$kata$hira]];"
|
||||
+ "$dgt=[:N:];"
|
||||
|
||||
// punctuation that can occur in the middle of a word: currently
|
||||
@ -175,8 +183,15 @@ public class BreakIteratorRules extends ListResourceBundle {
|
||||
|
||||
// default rules for determining legal line-breaking positions
|
||||
{ "LineBreakRules",
|
||||
// Surrogates. Until better support is available, ignore low surrogates
|
||||
// and classify high surrogates according to the characters within the block.
|
||||
"$surr_lo=[\udc00-\udfff];"
|
||||
+ "$surr_hi_let=[\ud800\ud801\ud834\ud835];" // Hi Surrogates for Old Italic, Gothic, Deseret, Music, Math
|
||||
+ "$surr_hi_ideo=[\ud840-\ud880];" // Hi Surrogates for CJK
|
||||
+ "$surr_hi_misc=[\udb40-\udbff];" // Hi Surrogates for Tags, Private Use.
|
||||
|
||||
// ignore non-spacing marks, enclosing marks, and format characters
|
||||
"$_ignore_=[[:Mn:][:Me:][:Cf:]];"
|
||||
+ "$_ignore_=[[:Mn:][:Me:][:Cf:]$surr_lo$surr_hi_misc];"
|
||||
|
||||
// Hindi phrase separators
|
||||
+ "$danda=[\u0964\u0965];"
|
||||
@ -211,7 +226,7 @@ public class BreakIteratorRules extends ListResourceBundle {
|
||||
|
||||
// Kanji: actually includes both Kanji and Kana, except for small Kana and
|
||||
// CJK diacritics
|
||||
+ "$kanji=[[\u4e00-\u9fa5\uf900-\ufa2d\u3041-\u3094\u30a1-\u30fa]-[$post_word$_ignore_]];"
|
||||
+ "$kanji=[[\u4e00-\u9fa5\uf900-\ufa2d\u3041-\u3094\u30a1-\u30fa$surr_hi_ideo]-[$post_word$_ignore_]];"
|
||||
|
||||
// digits
|
||||
+ "$digit=[[:Nd:][:No:]];"
|
||||
@ -252,8 +267,15 @@ public class BreakIteratorRules extends ListResourceBundle {
|
||||
|
||||
// default rules for finding sentence boundaries
|
||||
{ "SentenceBreakRules",
|
||||
// Surrogates. Until better support is available, ignore low surrogates
|
||||
// and classify high surrogates according to the characters within the block.
|
||||
"$surr_lo=[\udc00-\udfff];"
|
||||
+ "$surr_hi_let=[\ud800\ud801\ud834\ud835];" // Hi Surrogates for Old Italic, Gothic, Deseret, Music, Math
|
||||
+ "$surr_hi_ideo=[\ud840-\ud880];" // Hi Surrogates for CJK
|
||||
+ "$surr_hi_misc=[\udb40-\udbff];" // Hi Surrogates for Tags, Private Use.
|
||||
|
||||
// ignore non-spacing marks, enclosing marks, and format characters
|
||||
"$_ignore_=[[:Mn:][:Me:][:Cf:]];"
|
||||
+ "$_ignore_=[[:Mn:][:Me:][:Cf:]$surr_lo$surr_hi_misc];"
|
||||
|
||||
// lowercase letters
|
||||
+ "$lc=[:Ll:];"
|
||||
@ -283,7 +305,7 @@ public class BreakIteratorRules extends ListResourceBundle {
|
||||
|
||||
// characters that may occur at the beginning of a sentence: basically anything
|
||||
// not mentioned above (lowercase letters and digits are specifically excluded)
|
||||
+ "$sent_start=[^$lc$ucLatin$space$start$end$digit$term$period\u2029$_ignore_];"
|
||||
+ "$sent_start=[^$lc$ucLatin$space$start$end$digit$term$period\u2029$surr_hi_let$_ignore_];"
|
||||
|
||||
// Hindi phrase separator
|
||||
+ "$danda=[\u0964\u0965];"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user