150 lines
5.7 KiB
Plaintext
150 lines
5.7 KiB
Plaintext
#
|
|
# Copyright (C) 2002-2003, International Business Machines Corporation and others.
|
|
# All Rights Reserved.
|
|
#
|
|
# file: word.txt
|
|
#
|
|
# ICU Word Break Rules
|
|
# See Unicode Standard Annex #29.
|
|
# These rules are based on Version 4.0.0, dated 2003-04-17
|
|
#
|
|
|
|
|
|
|
|
####################################################################################
|
|
#
|
|
# Character class definitions from TR 29
|
|
#
|
|
####################################################################################
|
|
$Katakana = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
|
|
[:name = HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
|
|
[:name = HALFWIDTH KATAKANA VOICED SOUND MARK:]
|
|
[:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]];
|
|
|
|
|
|
$ALetter = [[:Alphabetic:] [:name= HEBREW PUNCTUATION GERESH:]
|
|
- [:Ideographic:]
|
|
- $Katakana
|
|
- [:Script = Thai:]
|
|
- [:Script = Lao:]
|
|
- [:Script = Hiragana:]];
|
|
|
|
$MidLetter = [[:name = APOSTROPHE:] [:name = MIDDLE DOT:] [:name = HEBREW PUNCTUATION GERSHAYIM:]
|
|
[:name = RIGHT SINGLE QUOTATION MARK:] [:name = HYPHENATION POINT:]];
|
|
|
|
$MidNumLet = [[:name = FULL STOP:] [:name = COLON:]];
|
|
|
|
$MidNum = [[:LineBreak = Infix_Numeric:] - $MidNumLet];
|
|
$Numeric = [:LineBreak = Numeric:];
|
|
|
|
|
|
#
|
|
# Character Class Definitions.
|
|
# The names are those from TR29.
|
|
#
|
|
$CR = \u000d;
|
|
$LF = \u000a;
|
|
$Control = [[:Zl:] [:Zp:] [:Cc:] [:Cf:]];
|
|
$Extend = [[:Grapheme_Extend = TRUE:]];
|
|
|
|
|
|
|
|
|
|
####################################################################################
|
|
#
|
|
# Word Break Rules. Definitions and Rules specific to word break begin Here.
|
|
#
|
|
####################################################################################
|
|
|
|
$Format = [[:Cf:]];
|
|
|
|
# ALetter2 - There are some characters, e.g. \u0fa9, that are both combining marks ($Extend)
|
|
# and alphabetic (ALetter). $ALetter2 is ALetter from the Unicode TR, less all such chars.
|
|
# We need this because of sequences of the form
|
|
# <Letter> <MidLetter> <alpha combining mark> <Numeric>
|
|
# Rule 3 says treat graphme clusters as a unit, as their first character.
|
|
# The <MidLetter> <alpha combining mark> thus should be treated as just <MidLetter>
|
|
# Rules for this are awkward, because the sequence
|
|
# <Letter> <MidLetter> <Letter> <Numeric>
|
|
# should not break, but the sequence
|
|
# <Letter> <MidLetter> <Numeric>
|
|
# should break after the <Letter>.
|
|
$ALetter2 = [$ALetter - $Extend];
|
|
|
|
# Rule 3: Treat a grapheme cluster as if it were a single character.
|
|
# Hangul Syllables are easier to deal with here than they are in Grapheme Clusters
|
|
# because we don't need to find the boundaries between adjacent syllables -
|
|
# they won't be word boundaries.
|
|
#
|
|
|
|
|
|
#
|
|
# "Extended" definitions. Grapheme Cluster + Format Chars, treated like the base char.
|
|
#
|
|
$ALetterEx = $ALetter $Extend*;
|
|
$ALetter2Ex = $ALetter2 $Extend*;
|
|
$NumericEx = $Numeric $Extend*;
|
|
$MidNumEx = $MidNum $Extend*;
|
|
$MidNumLetEx = $MidNumLet $Extend*;
|
|
$MidLetterEx = $MidLetter $Extend*;
|
|
$KatakanaEx = $Katakana $Extend*;
|
|
|
|
|
|
#
|
|
# Numbers. Rules 8, 11, 12 form the TR.
|
|
#
|
|
$NumberSequence = $NumericEx ($Format* ($MidNumEx | $MidNumLetEx)? $Format* $NumericEx)*;
|
|
$NumberSequence {100};
|
|
|
|
#
|
|
# Words. Alpha-numerics. Rule 5, 6, 7, 9, 10
|
|
# - must include at least one letter.
|
|
# - may include both letters and numbers.
|
|
# - may include MideLetter, MidNumber punctuation.
|
|
#
|
|
#$LetterSequence = $ALetterEx ($Format* ($MidLetterEx | $MidNumLetEx)? $Format* $ALetterEx)*; # rules #6, #7
|
|
$WordGlue = $MidLetterEx | $MidNumLetEx;
|
|
$MidWordFragment = ($WordGlue $ALetter2Ex | $WordGlue $Format+ $ALetterEx);
|
|
$WordSequence = $ALetterEx ($Format* ($ALetterEx | $MidWordFragment))*;
|
|
$WordSequence2 = $ALetter2Ex ($Format* ($ALetterEx | $MidWordFragment))*;
|
|
$WordTail = ($Format* $NumberSequence $Format+ $WordSequence) | ($Format* $NumberSequence $WordSequence2?);
|
|
($NumberSequence $Format+)? $WordSequence $WordTail* {200};
|
|
($NumberSequence)? $WordSequence2 $WordTail* {200};
|
|
|
|
#
|
|
# Do not break between Katakana. Rule #13.
|
|
#
|
|
$KatakanaEx ($Format* $KatakanaEx)* {300};
|
|
[:Hiragana:] $Extend* {300};
|
|
|
|
#
|
|
# Ideographic Characters. Stand by themselves as words.
|
|
# Separated from the "Everything Else" rule, below, only so that they
|
|
# can be tagged with a return value. TODO: is this what we want?
|
|
#
|
|
[:IDEOGRAPHIC:] $Extend* {400};
|
|
|
|
#
|
|
# Everything Else, with no tag.
|
|
# Non-Control chars combine with $Extend (combining) chars.
|
|
# Controls are do not.
|
|
#
|
|
[^$Control [:Ideographic:]] $Extend*;
|
|
$CR $LF;
|
|
|
|
#
|
|
# Reverse Rules. Back up over any of the chars that can group together.
|
|
# (Reverse rules do not need to be exact; they can back up too far,
|
|
# but must back up at least enough, and must stop on a boundary.)
|
|
#
|
|
|
|
# NonStarters are the set of all characters that can appear at the 2nd - nth position of
|
|
# a word. (They may also be the first.) The reverse rule skips over these, until it
|
|
# reaches something that can only be the start (and probably only) char in a "word".
|
|
# A space or punctuation meets the test.
|
|
#
|
|
$NonStarters = [$Numeric $ALetter $Katakana $MidLetter $MidNum $MidNumLet $Extend $Format];
|
|
|
|
#!.*;
|
|
! ($NonStarters* | \n \r) .;
|