ICU-10993 BRS: ensure correct line-endings and svn property
X-SVN-Rev: 36437
This commit is contained in:
Scott Russell
parent
509d9757b0
commit
b6bfbcce5b
6
.gitattributes
vendored
6
.gitattributes
vendored
@ -262,11 +262,6 @@ icu4j/main/classes/core/.settings/edu.umd.cs.findbugs.core.prefs -text
|
||||
icu4j/main/classes/core/.settings/org.eclipse.core.resources.prefs -text
|
||||
icu4j/main/classes/core/.settings/org.eclipse.jdt.core.prefs -text
|
||||
icu4j/main/classes/core/manifest.stub -text
|
||||
icu4j/main/classes/core/src/com/ibm/icu/impl/SimpleFilteredSentenceBreakIterator.java -text
|
||||
icu4j/main/classes/core/src/com/ibm/icu/impl/TZDBTimeZoneNames.java -text
|
||||
icu4j/main/classes/core/src/com/ibm/icu/impl/locale/KeyTypeData.java -text
|
||||
icu4j/main/classes/core/src/com/ibm/icu/text/BurmeseBreakEngine.java -text
|
||||
icu4j/main/classes/core/src/com/ibm/icu/text/FilteredBreakIteratorBuilder.java -text
|
||||
icu4j/main/classes/currdata/.externalToolBuilders/copy-data-currdata.launch -text
|
||||
icu4j/main/classes/currdata/.settings/org.eclipse.core.resources.prefs -text
|
||||
icu4j/main/classes/currdata/.settings/org.eclipse.jdt.core.prefs -text
|
||||
@ -327,7 +322,6 @@ icu4j/main/tests/core/manifest.stub -text
|
||||
icu4j/main/tests/core/src/com/ibm/icu/dev/data/rbbi/english.dict -text
|
||||
icu4j/main/tests/core/src/com/ibm/icu/dev/data/resources/testmessages.properties -text
|
||||
icu4j/main/tests/core/src/com/ibm/icu/dev/data/thai6.ucs -text
|
||||
icu4j/main/tests/core/src/com/ibm/icu/dev/test/format/NumberFormatSpecificationTest.java -text
|
||||
icu4j/main/tests/core/src/com/ibm/icu/dev/test/serializable/data/ICU_3.6/com.ibm.icu.impl.OlsonTimeZone.dat -text
|
||||
icu4j/main/tests/core/src/com/ibm/icu/dev/test/serializable/data/ICU_3.6/com.ibm.icu.impl.TimeZoneAdapter.dat -text
|
||||
icu4j/main/tests/core/src/com/ibm/icu/dev/test/serializable/data/ICU_3.6/com.ibm.icu.math.BigDecimal.dat -text
|
||||
|
||||
@ -1,325 +1,325 @@
|
||||
/*
|
||||
*******************************************************************************
|
||||
* Copyright (C) 2014, International Business Machines Corporation and
|
||||
* others. All Rights Reserved.
|
||||
*******************************************************************************
|
||||
*/
|
||||
package com.ibm.icu.impl;
|
||||
|
||||
import java.text.CharacterIterator;
|
||||
import java.util.HashSet;
|
||||
|
||||
import com.ibm.icu.text.BreakIterator;
|
||||
import com.ibm.icu.text.FilteredBreakIteratorBuilder;
|
||||
import com.ibm.icu.text.UCharacterIterator;
|
||||
import com.ibm.icu.util.BytesTrie;
|
||||
import com.ibm.icu.util.CharsTrie;
|
||||
import com.ibm.icu.util.CharsTrieBuilder;
|
||||
import com.ibm.icu.util.StringTrieBuilder;
|
||||
import com.ibm.icu.util.ULocale;
|
||||
import com.ibm.icu.util.UResourceBundle;
|
||||
|
||||
/**
|
||||
* @author tomzhang
|
||||
*/
|
||||
public class SimpleFilteredSentenceBreakIterator extends BreakIterator {
|
||||
|
||||
private BreakIterator delegate;
|
||||
private UCharacterIterator text; // TODO(Tom): suffice to move into the local scope in next() ?
|
||||
private CharsTrie backwardsTrie; // i.e. ".srM" for Mrs.
|
||||
private CharsTrie forwardsPartialTrie; // Has ".a" for "a.M."
|
||||
|
||||
/**
|
||||
* @param adoptBreakIterator
|
||||
* break iterator to adopt
|
||||
* @param forwardsPartialTrie
|
||||
* forward & partial char trie to adopt
|
||||
* @param backwardsTrie
|
||||
* backward trie to adopt
|
||||
*/
|
||||
public SimpleFilteredSentenceBreakIterator(BreakIterator adoptBreakIterator, CharsTrie forwardsPartialTrie,
|
||||
CharsTrie backwardsTrie) {
|
||||
this.delegate = adoptBreakIterator;
|
||||
this.forwardsPartialTrie = forwardsPartialTrie;
|
||||
this.backwardsTrie = backwardsTrie;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int next() {
|
||||
int n = delegate.next();
|
||||
if (n == BreakIterator.DONE || // at end or
|
||||
backwardsTrie == null) { // .. no backwards table loaded == no exceptions
|
||||
return n;
|
||||
}
|
||||
// UCharacterIterator text;
|
||||
text = UCharacterIterator.getInstance((CharacterIterator) delegate.getText().clone());
|
||||
do { // outer loop runs once per underlying break (from fDelegate).
|
||||
// loops while 'n' points to an exception.
|
||||
text.setIndex(n);
|
||||
backwardsTrie.reset();
|
||||
int uch;
|
||||
|
||||
// Assume a space is following the '.' (so we handle the case: "Mr. /Brown")
|
||||
if ((uch = text.previousCodePoint()) == ' ') { // TODO: skip a class of chars here??
|
||||
// TODO only do this the 1st time?
|
||||
} else {
|
||||
uch = text.nextCodePoint();
|
||||
}
|
||||
|
||||
BytesTrie.Result r = BytesTrie.Result.INTERMEDIATE_VALUE;
|
||||
|
||||
int bestPosn = -1;
|
||||
int bestValue = -1;
|
||||
|
||||
while ((uch = text.previousCodePoint()) != BreakIterator.DONE && // more to consume backwards and..
|
||||
((r = backwardsTrie.nextForCodePoint(uch)).hasNext())) {// more in the trie
|
||||
if (r.hasValue()) { // remember the best match so far
|
||||
bestPosn = text.getIndex();
|
||||
bestValue = backwardsTrie.getValue();
|
||||
}
|
||||
}
|
||||
|
||||
if (r.matches()) { // exact match?
|
||||
bestValue = backwardsTrie.getValue();
|
||||
bestPosn = text.getIndex();
|
||||
}
|
||||
|
||||
if (bestPosn >= 0) {
|
||||
if (bestValue == Builder.MATCH) { // exact match!
|
||||
n = delegate.next(); // skip this one. Find the next lowerlevel break.
|
||||
if (n == BreakIterator.DONE)
|
||||
return n;
|
||||
continue; // See if the next is another exception.
|
||||
} else if (bestValue == Builder.PARTIAL && forwardsPartialTrie != null) {
|
||||
// make sure there's a forward trie
|
||||
// We matched the "Ph." in "Ph.D." - now we need to run everything through the forwards trie
|
||||
// to see if it matches something going forward.
|
||||
forwardsPartialTrie.reset();
|
||||
|
||||
BytesTrie.Result rfwd = BytesTrie.Result.INTERMEDIATE_VALUE;
|
||||
text.setIndex(bestPosn); // hope that's close ..
|
||||
while ((uch = text.nextCodePoint()) != BreakIterator.DONE
|
||||
&& ((rfwd = forwardsPartialTrie.nextForCodePoint(uch)).hasNext())) {
|
||||
}
|
||||
if (rfwd.matches()) {
|
||||
// only full matches here, nothing to check
|
||||
// skip the next:
|
||||
n = delegate.next();
|
||||
if (n == BreakIterator.DONE)
|
||||
return n;
|
||||
continue;
|
||||
} else {
|
||||
// no match (no exception) -return the 'underlying' break
|
||||
return n;
|
||||
}
|
||||
} else {
|
||||
return n; // internal error and/or no forwards trie
|
||||
}
|
||||
} else {
|
||||
return n; // No match - so exit. Not an exception.
|
||||
}
|
||||
} while (n != BreakIterator.DONE);
|
||||
return n;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (obj == null)
|
||||
return false;
|
||||
if (this == obj)
|
||||
return true;
|
||||
if (getClass() != obj.getClass())
|
||||
return false;
|
||||
SimpleFilteredSentenceBreakIterator other = (SimpleFilteredSentenceBreakIterator) obj;
|
||||
return delegate.equals(other.delegate) && text.equals(other.text) && backwardsTrie.equals(other.backwardsTrie)
|
||||
&& forwardsPartialTrie.equals(other.forwardsPartialTrie);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object clone() {
|
||||
SimpleFilteredSentenceBreakIterator other = (SimpleFilteredSentenceBreakIterator) super.clone();
|
||||
return other;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int first() {
|
||||
return delegate.first();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int last() {
|
||||
return delegate.last();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int next(int n) {
|
||||
// TODO
|
||||
throw new UnsupportedOperationException("next(int) is not yet implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public int previous() {
|
||||
// TODO
|
||||
throw new UnsupportedOperationException("previous() is not yet implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public int following(int offset) {
|
||||
// TODO
|
||||
throw new UnsupportedOperationException("following(int) is not yet implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public int current() {
|
||||
return delegate.current();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int preceding(int offset) {
|
||||
// TODO
|
||||
throw new UnsupportedOperationException("preceding(int) is not yet implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public CharacterIterator getText() {
|
||||
return delegate.getText();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setText(CharacterIterator newText) {
|
||||
delegate.setText(newText);
|
||||
}
|
||||
|
||||
public static class Builder extends FilteredBreakIteratorBuilder {
|
||||
/**
|
||||
* filter set to store all exceptions
|
||||
*/
|
||||
private HashSet<String> filterSet;
|
||||
|
||||
static final int PARTIAL = (1 << 0); // < partial - need to run through forward trie
|
||||
static final int MATCH = (1 << 1); // < exact match - skip this one.
|
||||
static final int SuppressInReverse = (1 << 0);
|
||||
static final int AddToForward = (1 << 1);
|
||||
|
||||
/**
|
||||
* Create SimpleFilteredBreakIteratorBuilder using given locale
|
||||
* @param loc the locale to get filtered iterators
|
||||
*/
|
||||
public Builder(ULocale loc) {
|
||||
ICUResourceBundle rb = (ICUResourceBundle) UResourceBundle.getBundleInstance(
|
||||
ICUResourceBundle.ICU_BRKITR_BASE_NAME, loc);
|
||||
ICUResourceBundle exceptions = rb.findWithFallback("exceptions");
|
||||
ICUResourceBundle breaks = exceptions.findWithFallback("SentenceBreak");
|
||||
|
||||
filterSet = new HashSet<String>();
|
||||
if (breaks != null) {
|
||||
for (int index = 0, size = breaks.getSize(); index < size; ++index) {
|
||||
ICUResourceBundle b = (ICUResourceBundle) breaks.get(index);
|
||||
String br = b.getString();
|
||||
filterSet.add(br);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create SimpleFilteredBreakIteratorBuilder with no exception
|
||||
*/
|
||||
public Builder() {
|
||||
filterSet = new HashSet<String>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean suppressBreakAfter(String str) {
|
||||
if (filterSet == null) {
|
||||
filterSet = new HashSet<String>();
|
||||
}
|
||||
return filterSet.add(str);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean unsuppressBreakAfter(String str) {
|
||||
if (filterSet == null) {
|
||||
return false;
|
||||
} else {
|
||||
return filterSet.remove(str);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public BreakIterator build(BreakIterator adoptBreakIterator) {
|
||||
CharsTrieBuilder builder = new CharsTrieBuilder();
|
||||
CharsTrieBuilder builder2 = new CharsTrieBuilder();
|
||||
|
||||
int revCount = 0;
|
||||
int fwdCount = 0;
|
||||
|
||||
int subCount = filterSet.size();
|
||||
String[] ustrs = new String[subCount];
|
||||
int[] partials = new int[subCount];
|
||||
|
||||
CharsTrie backwardsTrie = null; // i.e. ".srM" for Mrs.
|
||||
CharsTrie forwardsPartialTrie = null; // Has ".a" for "a.M."
|
||||
|
||||
int i = 0;
|
||||
for (String s : filterSet) {
|
||||
ustrs[i] = s; // copy by value?
|
||||
partials[i] = 0; // default: no partial
|
||||
i++;
|
||||
}
|
||||
|
||||
for (i = 0; i < subCount; i++) {
|
||||
int nn = ustrs[i].indexOf('.'); // TODO: non-'.' abbreviations
|
||||
if (nn > -1 && (nn + 1) != ustrs[i].length()) {
|
||||
// is partial.
|
||||
// is it unique?
|
||||
int sameAs = -1;
|
||||
for (int j = 0; j < subCount; j++) {
|
||||
if (j == i)
|
||||
continue;
|
||||
if (ustrs[i].regionMatches(0, ustrs[j], 0, nn + 1)) {
|
||||
if (partials[j] == 0) { // hasn't been processed yet
|
||||
partials[j] = SuppressInReverse | AddToForward;
|
||||
} else if ((partials[j] & SuppressInReverse) != 0) {
|
||||
sameAs = j; // the other entry is already in the reverse table.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ((sameAs == -1) && (partials[i] == 0)) {
|
||||
StringBuilder prefix = new StringBuilder(ustrs[i].substring(0, nn + 1));
|
||||
// first one - add the prefix to the reverse table.
|
||||
prefix.reverse();
|
||||
builder.add(prefix, PARTIAL);
|
||||
revCount++;
|
||||
partials[i] = SuppressInReverse | AddToForward;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (i = 0; i < subCount; i++) {
|
||||
if (partials[i] == 0) {
|
||||
StringBuilder reversed = new StringBuilder(ustrs[i]).reverse();
|
||||
builder.add(reversed, MATCH);
|
||||
revCount++;
|
||||
} else {
|
||||
// an optimization would be to only add the portion after the '.'
|
||||
// for example, for "Ph.D." we store ".hP" in the reverse table. We could just store "D." in the
|
||||
// forward,
|
||||
// instead of "Ph.D." since we already know the "Ph." part is a match.
|
||||
// would need the trie to be able to hold 0-length strings, though.
|
||||
builder2.add(ustrs[i], MATCH); // forward
|
||||
fwdCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if (revCount > 0) {
|
||||
backwardsTrie = builder.build(StringTrieBuilder.Option.FAST);
|
||||
}
|
||||
|
||||
if (fwdCount > 0) {
|
||||
forwardsPartialTrie = builder2.build(StringTrieBuilder.Option.FAST);
|
||||
}
|
||||
return new SimpleFilteredSentenceBreakIterator(adoptBreakIterator, forwardsPartialTrie, backwardsTrie);
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
*******************************************************************************
|
||||
* Copyright (C) 2014, International Business Machines Corporation and
|
||||
* others. All Rights Reserved.
|
||||
*******************************************************************************
|
||||
*/
|
||||
package com.ibm.icu.impl;
|
||||
|
||||
import java.text.CharacterIterator;
|
||||
import java.util.HashSet;
|
||||
|
||||
import com.ibm.icu.text.BreakIterator;
|
||||
import com.ibm.icu.text.FilteredBreakIteratorBuilder;
|
||||
import com.ibm.icu.text.UCharacterIterator;
|
||||
import com.ibm.icu.util.BytesTrie;
|
||||
import com.ibm.icu.util.CharsTrie;
|
||||
import com.ibm.icu.util.CharsTrieBuilder;
|
||||
import com.ibm.icu.util.StringTrieBuilder;
|
||||
import com.ibm.icu.util.ULocale;
|
||||
import com.ibm.icu.util.UResourceBundle;
|
||||
|
||||
/**
|
||||
* @author tomzhang
|
||||
*/
|
||||
public class SimpleFilteredSentenceBreakIterator extends BreakIterator {
|
||||
|
||||
private BreakIterator delegate;
|
||||
private UCharacterIterator text; // TODO(Tom): suffice to move into the local scope in next() ?
|
||||
private CharsTrie backwardsTrie; // i.e. ".srM" for Mrs.
|
||||
private CharsTrie forwardsPartialTrie; // Has ".a" for "a.M."
|
||||
|
||||
/**
|
||||
* @param adoptBreakIterator
|
||||
* break iterator to adopt
|
||||
* @param forwardsPartialTrie
|
||||
* forward & partial char trie to adopt
|
||||
* @param backwardsTrie
|
||||
* backward trie to adopt
|
||||
*/
|
||||
public SimpleFilteredSentenceBreakIterator(BreakIterator adoptBreakIterator, CharsTrie forwardsPartialTrie,
|
||||
CharsTrie backwardsTrie) {
|
||||
this.delegate = adoptBreakIterator;
|
||||
this.forwardsPartialTrie = forwardsPartialTrie;
|
||||
this.backwardsTrie = backwardsTrie;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int next() {
|
||||
int n = delegate.next();
|
||||
if (n == BreakIterator.DONE || // at end or
|
||||
backwardsTrie == null) { // .. no backwards table loaded == no exceptions
|
||||
return n;
|
||||
}
|
||||
// UCharacterIterator text;
|
||||
text = UCharacterIterator.getInstance((CharacterIterator) delegate.getText().clone());
|
||||
do { // outer loop runs once per underlying break (from fDelegate).
|
||||
// loops while 'n' points to an exception.
|
||||
text.setIndex(n);
|
||||
backwardsTrie.reset();
|
||||
int uch;
|
||||
|
||||
// Assume a space is following the '.' (so we handle the case: "Mr. /Brown")
|
||||
if ((uch = text.previousCodePoint()) == ' ') { // TODO: skip a class of chars here??
|
||||
// TODO only do this the 1st time?
|
||||
} else {
|
||||
uch = text.nextCodePoint();
|
||||
}
|
||||
|
||||
BytesTrie.Result r = BytesTrie.Result.INTERMEDIATE_VALUE;
|
||||
|
||||
int bestPosn = -1;
|
||||
int bestValue = -1;
|
||||
|
||||
while ((uch = text.previousCodePoint()) != BreakIterator.DONE && // more to consume backwards and..
|
||||
((r = backwardsTrie.nextForCodePoint(uch)).hasNext())) {// more in the trie
|
||||
if (r.hasValue()) { // remember the best match so far
|
||||
bestPosn = text.getIndex();
|
||||
bestValue = backwardsTrie.getValue();
|
||||
}
|
||||
}
|
||||
|
||||
if (r.matches()) { // exact match?
|
||||
bestValue = backwardsTrie.getValue();
|
||||
bestPosn = text.getIndex();
|
||||
}
|
||||
|
||||
if (bestPosn >= 0) {
|
||||
if (bestValue == Builder.MATCH) { // exact match!
|
||||
n = delegate.next(); // skip this one. Find the next lowerlevel break.
|
||||
if (n == BreakIterator.DONE)
|
||||
return n;
|
||||
continue; // See if the next is another exception.
|
||||
} else if (bestValue == Builder.PARTIAL && forwardsPartialTrie != null) {
|
||||
// make sure there's a forward trie
|
||||
// We matched the "Ph." in "Ph.D." - now we need to run everything through the forwards trie
|
||||
// to see if it matches something going forward.
|
||||
forwardsPartialTrie.reset();
|
||||
|
||||
BytesTrie.Result rfwd = BytesTrie.Result.INTERMEDIATE_VALUE;
|
||||
text.setIndex(bestPosn); // hope that's close ..
|
||||
while ((uch = text.nextCodePoint()) != BreakIterator.DONE
|
||||
&& ((rfwd = forwardsPartialTrie.nextForCodePoint(uch)).hasNext())) {
|
||||
}
|
||||
if (rfwd.matches()) {
|
||||
// only full matches here, nothing to check
|
||||
// skip the next:
|
||||
n = delegate.next();
|
||||
if (n == BreakIterator.DONE)
|
||||
return n;
|
||||
continue;
|
||||
} else {
|
||||
// no match (no exception) -return the 'underlying' break
|
||||
return n;
|
||||
}
|
||||
} else {
|
||||
return n; // internal error and/or no forwards trie
|
||||
}
|
||||
} else {
|
||||
return n; // No match - so exit. Not an exception.
|
||||
}
|
||||
} while (n != BreakIterator.DONE);
|
||||
return n;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (obj == null)
|
||||
return false;
|
||||
if (this == obj)
|
||||
return true;
|
||||
if (getClass() != obj.getClass())
|
||||
return false;
|
||||
SimpleFilteredSentenceBreakIterator other = (SimpleFilteredSentenceBreakIterator) obj;
|
||||
return delegate.equals(other.delegate) && text.equals(other.text) && backwardsTrie.equals(other.backwardsTrie)
|
||||
&& forwardsPartialTrie.equals(other.forwardsPartialTrie);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object clone() {
|
||||
SimpleFilteredSentenceBreakIterator other = (SimpleFilteredSentenceBreakIterator) super.clone();
|
||||
return other;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int first() {
|
||||
return delegate.first();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int last() {
|
||||
return delegate.last();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int next(int n) {
|
||||
// TODO
|
||||
throw new UnsupportedOperationException("next(int) is not yet implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public int previous() {
|
||||
// TODO
|
||||
throw new UnsupportedOperationException("previous() is not yet implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public int following(int offset) {
|
||||
// TODO
|
||||
throw new UnsupportedOperationException("following(int) is not yet implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public int current() {
|
||||
return delegate.current();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int preceding(int offset) {
|
||||
// TODO
|
||||
throw new UnsupportedOperationException("preceding(int) is not yet implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public CharacterIterator getText() {
|
||||
return delegate.getText();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setText(CharacterIterator newText) {
|
||||
delegate.setText(newText);
|
||||
}
|
||||
|
||||
public static class Builder extends FilteredBreakIteratorBuilder {
|
||||
/**
|
||||
* filter set to store all exceptions
|
||||
*/
|
||||
private HashSet<String> filterSet;
|
||||
|
||||
static final int PARTIAL = (1 << 0); // < partial - need to run through forward trie
|
||||
static final int MATCH = (1 << 1); // < exact match - skip this one.
|
||||
static final int SuppressInReverse = (1 << 0);
|
||||
static final int AddToForward = (1 << 1);
|
||||
|
||||
/**
|
||||
* Create SimpleFilteredBreakIteratorBuilder using given locale
|
||||
* @param loc the locale to get filtered iterators
|
||||
*/
|
||||
public Builder(ULocale loc) {
|
||||
ICUResourceBundle rb = (ICUResourceBundle) UResourceBundle.getBundleInstance(
|
||||
ICUResourceBundle.ICU_BRKITR_BASE_NAME, loc);
|
||||
ICUResourceBundle exceptions = rb.findWithFallback("exceptions");
|
||||
ICUResourceBundle breaks = exceptions.findWithFallback("SentenceBreak");
|
||||
|
||||
filterSet = new HashSet<String>();
|
||||
if (breaks != null) {
|
||||
for (int index = 0, size = breaks.getSize(); index < size; ++index) {
|
||||
ICUResourceBundle b = (ICUResourceBundle) breaks.get(index);
|
||||
String br = b.getString();
|
||||
filterSet.add(br);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create SimpleFilteredBreakIteratorBuilder with no exception
|
||||
*/
|
||||
public Builder() {
|
||||
filterSet = new HashSet<String>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean suppressBreakAfter(String str) {
|
||||
if (filterSet == null) {
|
||||
filterSet = new HashSet<String>();
|
||||
}
|
||||
return filterSet.add(str);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean unsuppressBreakAfter(String str) {
|
||||
if (filterSet == null) {
|
||||
return false;
|
||||
} else {
|
||||
return filterSet.remove(str);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public BreakIterator build(BreakIterator adoptBreakIterator) {
|
||||
CharsTrieBuilder builder = new CharsTrieBuilder();
|
||||
CharsTrieBuilder builder2 = new CharsTrieBuilder();
|
||||
|
||||
int revCount = 0;
|
||||
int fwdCount = 0;
|
||||
|
||||
int subCount = filterSet.size();
|
||||
String[] ustrs = new String[subCount];
|
||||
int[] partials = new int[subCount];
|
||||
|
||||
CharsTrie backwardsTrie = null; // i.e. ".srM" for Mrs.
|
||||
CharsTrie forwardsPartialTrie = null; // Has ".a" for "a.M."
|
||||
|
||||
int i = 0;
|
||||
for (String s : filterSet) {
|
||||
ustrs[i] = s; // copy by value?
|
||||
partials[i] = 0; // default: no partial
|
||||
i++;
|
||||
}
|
||||
|
||||
for (i = 0; i < subCount; i++) {
|
||||
int nn = ustrs[i].indexOf('.'); // TODO: non-'.' abbreviations
|
||||
if (nn > -1 && (nn + 1) != ustrs[i].length()) {
|
||||
// is partial.
|
||||
// is it unique?
|
||||
int sameAs = -1;
|
||||
for (int j = 0; j < subCount; j++) {
|
||||
if (j == i)
|
||||
continue;
|
||||
if (ustrs[i].regionMatches(0, ustrs[j], 0, nn + 1)) {
|
||||
if (partials[j] == 0) { // hasn't been processed yet
|
||||
partials[j] = SuppressInReverse | AddToForward;
|
||||
} else if ((partials[j] & SuppressInReverse) != 0) {
|
||||
sameAs = j; // the other entry is already in the reverse table.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ((sameAs == -1) && (partials[i] == 0)) {
|
||||
StringBuilder prefix = new StringBuilder(ustrs[i].substring(0, nn + 1));
|
||||
// first one - add the prefix to the reverse table.
|
||||
prefix.reverse();
|
||||
builder.add(prefix, PARTIAL);
|
||||
revCount++;
|
||||
partials[i] = SuppressInReverse | AddToForward;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (i = 0; i < subCount; i++) {
|
||||
if (partials[i] == 0) {
|
||||
StringBuilder reversed = new StringBuilder(ustrs[i]).reverse();
|
||||
builder.add(reversed, MATCH);
|
||||
revCount++;
|
||||
} else {
|
||||
// an optimization would be to only add the portion after the '.'
|
||||
// for example, for "Ph.D." we store ".hP" in the reverse table. We could just store "D." in the
|
||||
// forward,
|
||||
// instead of "Ph.D." since we already know the "Ph." part is a match.
|
||||
// would need the trie to be able to hold 0-length strings, though.
|
||||
builder2.add(ustrs[i], MATCH); // forward
|
||||
fwdCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if (revCount > 0) {
|
||||
backwardsTrie = builder.build(StringTrieBuilder.Option.FAST);
|
||||
}
|
||||
|
||||
if (fwdCount > 0) {
|
||||
forwardsPartialTrie = builder2.build(StringTrieBuilder.Option.FAST);
|
||||
}
|
||||
return new SimpleFilteredSentenceBreakIterator(adoptBreakIterator, forwardsPartialTrie, backwardsTrie);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,121 +1,121 @@
|
||||
/*
|
||||
*******************************************************************************
|
||||
* Copyright (C) 2014, International Business Machines Corporation and
|
||||
* others. All Rights Reserved.
|
||||
*******************************************************************************
|
||||
*/
|
||||
package com.ibm.icu.text;
|
||||
|
||||
import com.ibm.icu.impl.SimpleFilteredSentenceBreakIterator;
|
||||
import com.ibm.icu.util.ULocale;
|
||||
|
||||
/**
|
||||
* The BreakIteratorFilter is used to modify the behavior of a BreakIterator
|
||||
* by constructing a new BreakIterator which suppresses certain segment boundaries.
|
||||
* See http://www.unicode.org/reports/tr35/tr35-general.html#Segmentation_Exceptions .
|
||||
* For example, a typical English Sentence Break Iterator would break on the space
|
||||
* in the string "Mr. Smith" (resulting in two segments),
|
||||
* but with "Mr." as an exception, a filtered break iterator
|
||||
* would consider the string "Mr. Smith" to be a single segment.
|
||||
*
|
||||
* <p><b>Note:</b> An instance of {@link BreakIterator} returned by this builder
|
||||
* class currently does not support following operations in this technology preview
|
||||
* version:
|
||||
* <ul>
|
||||
* <li>{@link BreakIterator#next(int) next(int n)}</li>
|
||||
* <li>{@link BreakIterator#previous() previous()}</li>
|
||||
* <li>{@link BreakIterator#following(int) following(int offset)}</li>
|
||||
* <li>{@link BreakIterator#preceding(int) preceding(int offset)}</li>
|
||||
* </ul>
|
||||
* When one of above methods is called, {@link UnsupportedOperationException} will be
|
||||
* thrown.
|
||||
*
|
||||
* @author tomzhang
|
||||
*
|
||||
* @internal ICU 54 technology preview
|
||||
* @deprecated This API might change or be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
public abstract class FilteredBreakIteratorBuilder {
|
||||
|
||||
/**
|
||||
* Construct a FilteredBreakIteratorBuilder based on rules in a locale.
|
||||
* The rules are taken from CLDR exception data for the locale,
|
||||
* see http://www.unicode.org/reports/tr35/tr35-general.html#Segmentation_Exceptions
|
||||
* This is the equivalent of calling createInstance(UErrorCode&)
|
||||
* and then repeatedly calling addNoBreakAfter(...) with the contents
|
||||
* of the CLDR exception data.
|
||||
* @param where the locale.
|
||||
* @return the new builder
|
||||
* @internal ICU 54 technology preview
|
||||
* @deprecated This API might change or be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
public static FilteredBreakIteratorBuilder createInstance(ULocale where) {
|
||||
FilteredBreakIteratorBuilder ret = new SimpleFilteredSentenceBreakIterator.Builder(where);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct an empty FilteredBreakIteratorBuilder.
|
||||
* In this state, it will not suppress any segment boundaries.
|
||||
* @return the new builder
|
||||
* @internal ICU 54 technology preview
|
||||
* @deprecated This API might change or be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
public static FilteredBreakIteratorBuilder createInstance() {
|
||||
FilteredBreakIteratorBuilder ret = new SimpleFilteredSentenceBreakIterator.Builder();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Suppress a certain string from being the end of a segment.
|
||||
* For example, suppressing "Mr.", then segments ending in "Mr." will not be returned
|
||||
* by the iterator.
|
||||
* @param str the string to suppress, such as "Mr."
|
||||
* @return returns true if the string was not present and now added,
|
||||
* false if the call was a no-op because the string was already being suppressed.
|
||||
* @internal ICU 54 technology preview
|
||||
* @deprecated This API might change or be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
public abstract boolean suppressBreakAfter(String str);
|
||||
|
||||
/**
|
||||
* Stop suppressing a certain string from being the end of the segment.
|
||||
* This function does not create any new segment boundaries, but only serves to un-do
|
||||
* the effect of earlier calls to suppressBreakAfter, or to un-do the effect of
|
||||
* locale data which may be suppressing certain strings.
|
||||
* @param str the str the string to unsuppress, such as "Mr."
|
||||
* @return returns true if the string was present and now removed,
|
||||
* false if the call was a no-op because the string was not being suppressed.
|
||||
* @internal ICU 54 technology preview
|
||||
* @deprecated This API might change or be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
public abstract boolean unsuppressBreakAfter(String str);
|
||||
|
||||
/**
|
||||
* Wrap (adopt) an existing break iterator in a new filtered instance.
|
||||
* The resulting BreakIterator is owned by the caller.
|
||||
* The BreakIteratorFilter may be destroyed before the BreakIterator is destroyed.
|
||||
* Note that the adoptBreakIterator is adopted by the new BreakIterator
|
||||
* and should no longer be used by the caller.
|
||||
* The FilteredBreakIteratorBuilder may be reused.
|
||||
* @param adoptBreakIterator the break iterator to adopt
|
||||
* @return the new BreakIterator, owned by the caller.
|
||||
* @internal ICU 54 technology preview
|
||||
* @deprecated This API might change or be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
public abstract BreakIterator build(BreakIterator adoptBreakIterator);
|
||||
|
||||
/**
|
||||
* For subclass use
|
||||
* @internal ICU 54 technology preview
|
||||
* @deprecated This API might change or be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
protected FilteredBreakIteratorBuilder() {}
|
||||
/*
|
||||
*******************************************************************************
|
||||
* Copyright (C) 2014, International Business Machines Corporation and
|
||||
* others. All Rights Reserved.
|
||||
*******************************************************************************
|
||||
*/
|
||||
package com.ibm.icu.text;
|
||||
|
||||
import com.ibm.icu.impl.SimpleFilteredSentenceBreakIterator;
|
||||
import com.ibm.icu.util.ULocale;
|
||||
|
||||
/**
|
||||
* The BreakIteratorFilter is used to modify the behavior of a BreakIterator
|
||||
* by constructing a new BreakIterator which suppresses certain segment boundaries.
|
||||
* See http://www.unicode.org/reports/tr35/tr35-general.html#Segmentation_Exceptions .
|
||||
* For example, a typical English Sentence Break Iterator would break on the space
|
||||
* in the string "Mr. Smith" (resulting in two segments),
|
||||
* but with "Mr." as an exception, a filtered break iterator
|
||||
* would consider the string "Mr. Smith" to be a single segment.
|
||||
*
|
||||
* <p><b>Note:</b> An instance of {@link BreakIterator} returned by this builder
|
||||
* class currently does not support following operations in this technology preview
|
||||
* version:
|
||||
* <ul>
|
||||
* <li>{@link BreakIterator#next(int) next(int n)}</li>
|
||||
* <li>{@link BreakIterator#previous() previous()}</li>
|
||||
* <li>{@link BreakIterator#following(int) following(int offset)}</li>
|
||||
* <li>{@link BreakIterator#preceding(int) preceding(int offset)}</li>
|
||||
* </ul>
|
||||
* When one of above methods is called, {@link UnsupportedOperationException} will be
|
||||
* thrown.
|
||||
*
|
||||
* @author tomzhang
|
||||
*
|
||||
* @internal ICU 54 technology preview
|
||||
* @deprecated This API might change or be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
public abstract class FilteredBreakIteratorBuilder {
|
||||
|
||||
/**
|
||||
* Construct a FilteredBreakIteratorBuilder based on rules in a locale.
|
||||
* The rules are taken from CLDR exception data for the locale,
|
||||
* see http://www.unicode.org/reports/tr35/tr35-general.html#Segmentation_Exceptions
|
||||
* This is the equivalent of calling createInstance(UErrorCode&)
|
||||
* and then repeatedly calling addNoBreakAfter(...) with the contents
|
||||
* of the CLDR exception data.
|
||||
* @param where the locale.
|
||||
* @return the new builder
|
||||
* @internal ICU 54 technology preview
|
||||
* @deprecated This API might change or be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
public static FilteredBreakIteratorBuilder createInstance(ULocale where) {
|
||||
FilteredBreakIteratorBuilder ret = new SimpleFilteredSentenceBreakIterator.Builder(where);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct an empty FilteredBreakIteratorBuilder.
|
||||
* In this state, it will not suppress any segment boundaries.
|
||||
* @return the new builder
|
||||
* @internal ICU 54 technology preview
|
||||
* @deprecated This API might change or be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
public static FilteredBreakIteratorBuilder createInstance() {
|
||||
FilteredBreakIteratorBuilder ret = new SimpleFilteredSentenceBreakIterator.Builder();
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Suppress a certain string from being the end of a segment.
|
||||
* For example, suppressing "Mr.", then segments ending in "Mr." will not be returned
|
||||
* by the iterator.
|
||||
* @param str the string to suppress, such as "Mr."
|
||||
* @return returns true if the string was not present and now added,
|
||||
* false if the call was a no-op because the string was already being suppressed.
|
||||
* @internal ICU 54 technology preview
|
||||
* @deprecated This API might change or be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
public abstract boolean suppressBreakAfter(String str);
|
||||
|
||||
/**
|
||||
* Stop suppressing a certain string from being the end of the segment.
|
||||
* This function does not create any new segment boundaries, but only serves to un-do
|
||||
* the effect of earlier calls to suppressBreakAfter, or to un-do the effect of
|
||||
* locale data which may be suppressing certain strings.
|
||||
* @param str the str the string to unsuppress, such as "Mr."
|
||||
* @return returns true if the string was present and now removed,
|
||||
* false if the call was a no-op because the string was not being suppressed.
|
||||
* @internal ICU 54 technology preview
|
||||
* @deprecated This API might change or be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
public abstract boolean unsuppressBreakAfter(String str);
|
||||
|
||||
/**
|
||||
* Wrap (adopt) an existing break iterator in a new filtered instance.
|
||||
* The resulting BreakIterator is owned by the caller.
|
||||
* The BreakIteratorFilter may be destroyed before the BreakIterator is destroyed.
|
||||
* Note that the adoptBreakIterator is adopted by the new BreakIterator
|
||||
* and should no longer be used by the caller.
|
||||
* The FilteredBreakIteratorBuilder may be reused.
|
||||
* @param adoptBreakIterator the break iterator to adopt
|
||||
* @return the new BreakIterator, owned by the caller.
|
||||
* @internal ICU 54 technology preview
|
||||
* @deprecated This API might change or be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
public abstract BreakIterator build(BreakIterator adoptBreakIterator);
|
||||
|
||||
/**
|
||||
* For subclass use
|
||||
* @internal ICU 54 technology preview
|
||||
* @deprecated This API might change or be removed in a future release.
|
||||
*/
|
||||
@Deprecated
|
||||
protected FilteredBreakIteratorBuilder() {}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user