scuffed-code/icu4c/source/test/intltest/rbbitst.cpp

/********************************************************************
 * COPYRIGHT: 
 * Copyright (c) 1999-2000, International Business Machines Corporation and
 * others. All Rights Reserved.
 ********************************************************************/
/************************************************************************
*   Date        Name        Description
*   12/15/99    Madhu        Creation.
*   01/12/2000  Madhu        Updated for changed API and added new tests
************************************************************************/


#include "intltest.h"
#include "unicode/brkiter.h"
#include "unicode/rbbi.h"
#include "unicode/unicode.h"
#include <stdio.h>
#include "unicode/utypes.h"
#include "rbbitst.h"
#include <string.h>
#include "unicode/schriter.h"

//--------------------------------------------------------------------------------------
/**
 * "Vector" class for holding test tables
 * (this class is actually a linked list, but we use the name and API of the
 * java.util.Vector class to keep as much of our test code as possible the same.)
 */
class Enumeration { // text enumeration
public:
  virtual UBool hasMoreElements() = 0;
  virtual UnicodeString nextElement() = 0;
};

class Vector { // text vector
public:

  class TextLink {
  public:
    TextLink() : fLink(0), fText() {}
    TextLink(TextLink* link, UnicodeString text) : fLink(link), fText(text) {}

    TextLink* fLink;
    UnicodeString fText;
  };

public:
  TextLink fBase;
  TextLink* fEnd;
  int32_t fSize;

public:
  class VectorEnumeration : public Enumeration {
  public:
    VectorEnumeration(Vector* vector) : fVector(vector), fPos(&vector->fBase) {}
    
    UBool hasMoreElements() { return fPos->fLink != &fVector->fBase; }
    UnicodeString nextElement() { fPos = fPos->fLink; return fPos->fText; }

    Vector* fVector;
    TextLink* fPos;
  };

  Vector() : fBase(), fEnd(&fBase), fSize(0) { fBase.fLink = &fBase; }

  ~Vector() { 
    while (fBase.fLink != &fBase) { 
        TextLink* link = fBase.fLink;
        fBase.fLink = link->fLink;
        delete link;
        }
  }

  void addElement(UnicodeString text) { fEnd->fLink = new TextLink(&fBase, text); fEnd = fEnd->fLink; ++fSize; }
  void insertElementAt(UnicodeString text, int pos) { 
      if(pos >= fSize || pos < 0)
          ;
      else if(pos == 0){
          TextLink* insert = new TextLink(&fBase, text);
          insert->fLink=fBase.fLink;
          ++fSize;
          fBase.fLink=insert;
      }
      else{
          TextLink* link = fBase.fLink; 
          while(--pos > 0)
             link=link->fLink;
          TextLink* insert = new TextLink(&fBase, text);
          insert->fLink =link->fLink;
          link->fLink=insert;
          ++fSize;

      }

  }
    UnicodeString elementAt(int32_t pos) {
        if (pos >= fSize)
          return UnicodeString();

    TextLink* link = fBase.fLink; 
    while (pos-- > 0) link = link->fLink; 
    return link->fText;
  }
  UnicodeString lastElement() { return fEnd == &fBase ? UnicodeString() : fEnd->fText; }
  int32_t size() { return fSize; }

  Enumeration* elements() { return new VectorEnumeration(this); }

};

//--------------------------------------------------------------------------------------
/**
 * RBBITest is medium top level test class RuleBasedBreakIterator
 */

const UnicodeString  halfNA  = CharsToUnicodeString("\\u0928\\u094d\\u200d");  //halfform NA = devanigiri NA + virama(supresses inherent vowel)+ zero width joiner   
const UnicodeString  halfSA  = CharsToUnicodeString("\\u0938\\u094d\\u200d"); 
const UnicodeString  halfCHA = CharsToUnicodeString("\\u091a\\u094d\\u200d"); 
const UnicodeString  halfKA  = CharsToUnicodeString("\\u0915\\u094d\\u200d"); 
const UnicodeString  deadTA  = CharsToUnicodeString("\\u0924\\u094d");
//--------------------------------------------------------------------
//tests default rules based character iteration
//--------------------------------------------------------------------
void RBBITest::TestDefaultRuleBasedCharacterIteration()
{
   //   RuleBasedBreakIterator* rbbi=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createCharacterInstance();
      logln((UnicodeString)"Testing the RBBI for character iteration by using default rules");
  //fetch the rules used to create the above RuleBasedBreakIterator
    //    UnicodeString defaultRules=rbbi->getRules();
    //     RuleBasedCharacterIterator charIterDefault = new RuleBasedBreakIterator(defaultRules); 

      UErrorCode status=U_ZERO_ERROR;
      RuleBasedBreakIterator* charIterDefault=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createCharacterInstance(Locale::getDefault(), status);
      if(U_FAILURE(status)){
          errln("FAIL : in construction");
          return;
      }

      Vector *chardata = new Vector();
      chardata->addElement("H");
      chardata->addElement("e");
      chardata->addElement("l");
      chardata->addElement("l");
      chardata->addElement("o");
      chardata->addElement(CharsToUnicodeString("e\\u0301"));                   //acuteE
      chardata->addElement("&");
      chardata->addElement(CharsToUnicodeString("e\\u0303"));                   //tildaE
      //devanagiri characters for Hindi support
      chardata->addElement(CharsToUnicodeString("\\u0906"));                    //devanagiri AA
      chardata->addElement(CharsToUnicodeString("\\u093e\\u0901"));              //devanagiri vowelsign AA+ chandrabindhu
      chardata->addElement(CharsToUnicodeString("\\u0916\\u0947"));              //devanagiri KHA+vowelsign E
      chardata->addElement(CharsToUnicodeString("\\u0938\\u0941\\u0902"));        //devanagiri SA+vowelsign U + anusvara(bindu)
      chardata->addElement(CharsToUnicodeString("\\u0926"));                    //devanagiri consonant DA
      chardata->addElement(CharsToUnicodeString("\\u0930"));                    //devanagiri consonant RA
      chardata->addElement(CharsToUnicodeString("\\u0939\\u094c"));              //devanagiri HA+vowel sign AI
      chardata->addElement(CharsToUnicodeString("\\u0964"));                    //devanagiri danda
      //end hindi characters      
      chardata->addElement(CharsToUnicodeString("A\\u0302"));                   //circumflexA 
      chardata->addElement(CharsToUnicodeString("i\\u0301"));                   //acuteBelowI   
      // conjoining jamo->.. 
      chardata->addElement(CharsToUnicodeString("\\u1109\\u1161\\u11bc"));
      chardata->addElement(CharsToUnicodeString("\\u1112\\u1161\\u11bc"));
      chardata->addElement("\n");
      chardata->addElement("\r\n");                      //keep CRLF sequences together  
      chardata->addElement(CharsToUnicodeString("S\\u0300"));                   //graveS
      chardata->addElement(CharsToUnicodeString("i\\u0301"));                   //acuteBelowI
      chardata->addElement("!");

    // What follows is a string of Korean characters (I found it in the Yellow Pages
    // ad for the Korean Presbyterian Church of San Francisco, and I hope I transcribed
    // it correctly), first as precomposed syllables, and then as conjoining jamo.
    // Both sequences should be semantically identical and break the same way.
    // precomposed syllables...
      chardata->addElement(CharsToUnicodeString("\\uc0c1"));
      chardata->addElement(CharsToUnicodeString("\\ud56d"));
      chardata->addElement(" ");
      chardata->addElement(CharsToUnicodeString("\\ud55c"));
      chardata->addElement(CharsToUnicodeString("\\uc778"));
      chardata->addElement(" ");
      chardata->addElement(CharsToUnicodeString("\\uc5f0"));
      chardata->addElement(CharsToUnicodeString("\\ud569"));
      chardata->addElement(" ");
      chardata->addElement(CharsToUnicodeString("\\uc7a5"));
      chardata->addElement(CharsToUnicodeString("\\ub85c"));
      chardata->addElement(CharsToUnicodeString("\\uad50"));
      chardata->addElement(CharsToUnicodeString("\\ud68c"));
      chardata->addElement(" ");
    // conjoining jamo...
      chardata->addElement(CharsToUnicodeString("\\u1109\\u1161\\u11bc"));
      chardata->addElement(CharsToUnicodeString("\\u1112\\u1161\\u11bc"));
      chardata->addElement(" ");
      chardata->addElement(CharsToUnicodeString("\\u1112\\u1161\\u11ab"));
      chardata->addElement(CharsToUnicodeString("\\u110b\\u1175\\u11ab"));
      chardata->addElement(" ");
      chardata->addElement(CharsToUnicodeString("\\u110b\\u1167\\u11ab"));
      chardata->addElement(CharsToUnicodeString("\\u1112\\u1161\\u11b8"));
      chardata->addElement(" ");
      chardata->addElement(CharsToUnicodeString("\\u110c\\u1161\\u11bc"));
      chardata->addElement(CharsToUnicodeString("\\u1105\\u1169"));
      chardata->addElement(CharsToUnicodeString("\\u1100\\u116d"));
      chardata->addElement(CharsToUnicodeString("\\u1112\\u116c"));
      generalIteratorTest(*charIterDefault, chardata);

      delete charIterDefault;
      delete chardata;
//      delete rbbi;

}
//--------------------------------------------------------------------
//tests default rules based word iteration
//--------------------------------------------------------------------
void RBBITest::TestDefaultRuleBasedWordIteration()
{
      logln((UnicodeString)"Testing the RBBI for word iteration using default rules");
//      RuleBasedBreakIterator *rbbi=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createWordInstance();
//      fetch the rules used to create the above RuleBasedBreakIterator
//      UnicodeString defaultRules=rbbi->getRules();
//      RuleBasedBreakIterator wordIterDefault = new RuleBasedBreakIterator(defaultRules); 

      UErrorCode status=U_ZERO_ERROR; 
      RuleBasedBreakIterator* wordIterDefault=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createWordInstance(Locale::getDefault(), status);
      if(U_FAILURE(status)){
          errln("FAIL : in construction");
          return;
      }

      Vector *worddata = new Vector();
      worddata->addElement ("Write");
      worddata->addElement (" ");
      worddata->addElement ("wordrules");
      worddata->addElement (".");
      worddata->addElement(" ");
      worddata->addElement("alpha-beta-gamma");
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u092f\\u0939"));
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u0939\\u093f") + halfNA + CharsToUnicodeString("\\u0926\\u0940"));
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u0939\\u0948"));   
      //worddata->addElement("\\u0964");   //danda followed by a space "\u0964->danda: hindi phrase seperator"
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u0905\\u093e\\u092a"));
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u0938\\u093f\\u0916\\u094b\\u0917\\u0947"));
      worddata->addElement("?");
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u0968\\u0966.\\u0969\\u096f"));            //hindi numbers
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u0967\\u0966\\u0966.\\u0966\\u0966%"));     //postnumeric
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u20a8\\u0967,\\u0967\\u0966\\u0966.\\u0966\\u0966")); //pre-number India currency symbol Rs->\\u20aD
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u0905\\u092e\\u091c"));
      worddata->addElement("\n");
      worddata->addElement(halfSA+CharsToUnicodeString("\\u0935\\u0924\\u0902")+deadTA+CharsToUnicodeString("\\u0930"));
      worddata->addElement("\r");
      worddata->addElement("It's");
      worddata->addElement(" ");
      worddata->addElement("$30.10");
      worddata->addElement(" ");  
      worddata->addElement(CharsToUnicodeString("\\u00A2")); //cent sign
      worddata->addElement(CharsToUnicodeString("\\u00A3")); //pound sign
      worddata->addElement(CharsToUnicodeString("\\u00A4")); //currency sign
      worddata->addElement(CharsToUnicodeString("\\u00A5")); //yen sign
      worddata->addElement("alpha-beta-gamma");
      worddata->addElement(" ");
      worddata->addElement("Badges");
      worddata->addElement("?");
      worddata->addElement(" ");
      worddata->addElement("BADGES");
      worddata->addElement("!");
      worddata->addElement("1000,233,456.000");
      worddata->addElement(" ");
      worddata->addElement("1,23.322%");
      worddata->addElement(" ");
      worddata->addElement("123.1222");
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u0024123,000.20"));
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("179.01\\u0025"));
      worddata->addElement("X");
      worddata->addElement(" ");
      worddata->addElement("Now");
      worddata->addElement("\r");
      worddata->addElement("is");
      worddata->addElement("\n");
      worddata->addElement("the");
      worddata->addElement("\r\n");
      worddata->addElement("time");
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\uc5f0\\ud569"));
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\uc7a5\\ub85c\\uad50\\ud68c"));
      worddata->addElement(" ");
      // conjoining jamo...
      worddata->addElement(CharsToUnicodeString("\\u1109\\u1161\\u11bc\\u1112\\u1161\\u11bc"));
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u1112\\u1161\\u11ab\\u110b\\u1175\\u11ab"));
      worddata->addElement(" ");
      worddata->addElement("Hello");
      worddata->addElement(",");
      worddata->addElement(" ");
      worddata->addElement("how");
      worddata->addElement(" ");
      worddata->addElement("are");
      worddata->addElement(" ");
      worddata->addElement("you");
      worddata->addElement(" ");
      generalIteratorTest(*wordIterDefault, worddata);

      delete wordIterDefault;
      delete worddata;
//      delete rbbi;
}
//--------------------------------------------------------------------
//tests default rules based word iteration
//--------------------------------------------------------------------
const UnicodeString kParagraphSeparator = CharsToUnicodeString("\\u2029");
const UnicodeString kLineSeparator = CharsToUnicodeString("\\u2028");

void RBBITest::TestDefaultRuleBasedSentenceIteration()
{
      logln((UnicodeString)"Testing the RBBI for sentence iteration using default rules");
     // RuleBasedBreakIterator *rbbi=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createSentenceInstance();
      //fetch the rules used to create the above RuleBasedBreakIterator
    //  UnicodeString defaultRules=rbbi->getRules();
    //  RuleBasedBreakIterator sentIterDefault = new RuleBasedBreakIterator(defaultRules); 
      UErrorCode status=U_ZERO_ERROR;
      RuleBasedBreakIterator* sentIterDefault=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createSentenceInstance(Locale::getDefault(), status);
      if(U_FAILURE(status)){
          errln("FAIL : in construction");
          return;
      }
      Vector *sentdata = new Vector();
      sentdata->addElement("(This is it.) ");
      sentdata->addElement("Testing the sentence iterator. ");
      sentdata->addElement("\"This isn\'t it.\" ");
      sentdata->addElement("Hi! ");
      //sentdata = new Vector();
      sentdata->addElement("This is a simple sample sentence. ");
      sentdata->addElement("(This is it.) ");
      sentdata->addElement("This is a simple sample sentence. ");
      sentdata->addElement("\"This isn\'t it.\" ");
      sentdata->addElement("Hi! ");
      sentdata->addElement("This is a simple sample sentence. ");
      sentdata->addElement("It does not have to make any sense as you can see. ");
      sentdata->addElement("Nel mezzo del cammin di nostra vita, mi ritrovai in una selva oscura. ");
      sentdata->addElement("Che la dritta via aveo smarrita. ");
      sentdata->addElement("He said, that I said, that you said!! ");

      sentdata->addElement("Don't rock the boat." + kParagraphSeparator);

      sentdata->addElement("Because I am the daddy, that is why. ");
      sentdata->addElement("Not on my time (el timo.)! ");

      sentdata->addElement("So what!!" + kParagraphSeparator);

      sentdata->addElement("\"But now,\" he said, \"I know!\" ");
      sentdata->addElement("Harris thumbed down several, including \"Away We Go\" (which became the huge success Oklahoma!). ");
      sentdata->addElement("One species, B. anthracis, is highly virulent.\n");
      sentdata->addElement("Wolf said about Sounder:\"Beautifully thought-out and directed.\" ");
      sentdata->addElement("Have you ever said, \"This is where\tI shall live\"? ");
      sentdata->addElement("He answered, \"You may not!\" ");
      sentdata->addElement("Another popular saying is: \"How do you do?\". ");
      sentdata->addElement("Yet another popular saying is: \'I\'m fine thanks.\' ");
      sentdata->addElement("What is the proper use of the abbreviation pp.? ");
      sentdata->addElement("Yes, I am definatelly 12\" tall!!");
      // test for bug #4113835: \n and \r count as spaces, not as paragraph breaks
      sentdata->addElement(CharsToUnicodeString("Now\ris\nthe\r\ntime\n\rfor\r\rall\\u2029"));

    // test that it doesn't break sentences at the boundary between CJK
    // and other letters
      sentdata->addElement(CharsToUnicodeString("\\u5487\\u67ff\\ue591\\u5017\\u61b3\\u60a1\\u9510\\u8165:\"JAVA\\u821c")
        + CharsToUnicodeString("\\u8165\\u7fc8\\u51ce\\u306d,\\u2494\\u56d8\\u4ec0\\u60b1\\u8560\\u51ba")
        + CharsToUnicodeString("\\u611d\\u57b6\\u2510\\u5d46\".\\u2029"));
      sentdata->addElement(CharsToUnicodeString("\\u5487\\u67ff\\ue591\\u5017\\u61b3\\u60a1\\u9510\\u8165\\u9de8")
        + CharsToUnicodeString("\\u97e4JAVA\\u821c\\u8165\\u7fc8\\u51ce\\u306d\\ue30b\\u2494\\u56d8\\u4ec0")
        + CharsToUnicodeString("\\u60b1\\u8560\\u51ba\\u611d\\u57b6\\u2510\\u5d46\\u97e5\\u7751\\u2029"));
      sentdata->addElement(CharsToUnicodeString("\\u5487\\u67ff\\ue591\\u5017\\u61b3\\u60a1\\u9510\\u8165\\u9de8\\u97e4")
        + CharsToUnicodeString("\\u6470\\u8790JAVA\\u821c\\u8165\\u7fc8\\u51ce\\u306d\\ue30b\\u2494\\u56d8")
        + CharsToUnicodeString("\\u4ec0\\u60b1\\u8560\\u51ba\\u611d\\u57b6\\u2510\\u5d46\\u97e5\\u7751\\u2029"));
      sentdata->addElement(CharsToUnicodeString("He said, \"I can go there.\"\\u2029"));

      // Treat fullwidth variants of .!? the same as their
      // normal counterparts
      sentdata->addElement(CharsToUnicodeString("I know I'm right\\uff0e "));
      sentdata->addElement(CharsToUnicodeString("Right\\uff1f "));
      sentdata->addElement(CharsToUnicodeString("Right\\uff01 "));

      // Don't break sentences at boundary between CJK and digits
      sentdata->addElement(CharsToUnicodeString("\\u5487\\u67ff\\ue591\\u5017\\u61b3\\u60a1\\u9510\\u8165\\u9de8")
                + CharsToUnicodeString("\\u97e48888\\u821c\\u8165\\u7fc8\\u51ce\\u306d\\ue30b\\u2494\\u56d8\\u4ec0")
                + CharsToUnicodeString("\\u60b1\\u8560\\u51ba\\u611d\\u57b6\\u2510\\u5d46\\u97e5\\u7751\\u2029"));

      // Break sentence between a sentence terminator and
      // opening punctuation
      sentdata->addElement("How do you do?");
      sentdata->addElement("(fine).");
      //sentence breaks for hindi which used Devanagari script
      //make sure there is sentence break after ?,danda(hindi phrase separator),fullstop followed by space and no break after \n \r 
      sentdata->addElement(CharsToUnicodeString("\\u0928\\u092e")+halfSA
                                   + CharsToUnicodeString("\\u0924\\u0947 ")  
                                   + CharsToUnicodeString("\\u0930\\u092e\\u0947\\u0936, ") 
                                   + CharsToUnicodeString("\\u0905\\u093e\\u092a")
                                   + CharsToUnicodeString("\\u0915\\u0948\\u0938\\u0947 ")
                                   + CharsToUnicodeString("\\u0939\\u0948?"));   
      sentdata->addElement(CharsToUnicodeString("\\u092e\\u0948 \\u0905")
                                   + halfCHA
                                   +CharsToUnicodeString("\\u091b\\u093e \\u0939\\u0942\\u0901\\u0964 "));   
      sentdata->addElement(CharsToUnicodeString("\\u0905\\u093e\\u092a\r\n \\u0915\\u0948\\u0938\\u0947 \\u0939\\u0948?"));   
      sentdata->addElement(CharsToUnicodeString("\\u0935\\u0939 ")
                                   + halfKA
                                   +CharsToUnicodeString("\\u092f\\u093e\n \\u0939\\u0948?"));   
      sentdata->addElement(CharsToUnicodeString("\\u092f\\u0939 \\u0905\\u093e\\u092e \\u0939\\u0948. "));   
      sentdata->addElement(CharsToUnicodeString("\\u092f\\u0939 means \"this\". ")); 
      sentdata->addElement(CharsToUnicodeString("\"\\u092a\\u095d\\u093e\\u0908\" meaning \"education\" or \"studies\". "));
      sentdata->addElement(CharsToUnicodeString("\\u0905\\u093e\\u091c") 
                                   + CharsToUnicodeString("(")
                                   + halfSA
                                   + CharsToUnicodeString("\\u0935\\u0924\\u0902")
                                   + deadTA+ CharsToUnicodeString("\\u0930 ")
                                   + CharsToUnicodeString("\\u0926\\u093f\\u0935\\u093e\\u0938) ")
                                   + CharsToUnicodeString("\\u0939\\u0948\\u0964 "));
      sentdata->addElement("Let's end here. ");
      generalIteratorTest(*sentIterDefault, sentdata);

      delete sentIterDefault;
      delete sentdata;
//      delete rbbi;
}
//--------------------------------------------------------------------
//tests default rules based line iteration
//--------------------------------------------------------------------
void RBBITest::TestDefaultRuleBasedLineIteration()
{
    UErrorCode status= U_ZERO_ERROR;
    RuleBasedBreakIterator* lineIterDefault=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createLineInstance(Locale::getDefault(), status);
    if(U_FAILURE(status)){
          errln("FAIL : in construction");
          return;
      }
    Vector *linedata = new Vector();
      linedata->addElement("Multi-");
      linedata->addElement("Level ");
      linedata->addElement("example ");
      linedata->addElement("of ");
      linedata->addElement("a ");
      linedata->addElement("semi-");
      linedata->addElement("idiotic ");
      linedata->addElement("non-");
      linedata->addElement("sensical ");
      linedata->addElement("(non-");
      linedata->addElement("important) ");
      linedata->addElement("sentence. ");

      linedata->addElement("Hi  ");
      linedata->addElement("Hello ");
      linedata->addElement("How\n");
      linedata->addElement("are\r");
      linedata->addElement("you" + kLineSeparator);
      linedata->addElement("fine.\t");
      linedata->addElement("good.  ");

      linedata->addElement("Now\r");
      linedata->addElement("is\n");
      linedata->addElement("the\r\n");
      linedata->addElement("time\n");
      linedata->addElement("\r");
      linedata->addElement("for\r");
      linedata->addElement("\r");
      linedata->addElement("all");

    // to test for bug #4068133
      linedata->addElement(CharsToUnicodeString("\\u96f6"));
      linedata->addElement(CharsToUnicodeString("\\u4e00\\u3002"));
      linedata->addElement(CharsToUnicodeString("\\u4e8c\\u3001"));
      linedata->addElement(CharsToUnicodeString("\\u4e09\\u3002\\u3001"));
      linedata->addElement(CharsToUnicodeString("\\u56db\\u3001\\u3002\\u3001"));
      linedata->addElement(CharsToUnicodeString("\\u4e94,"));
      linedata->addElement(CharsToUnicodeString("\\u516d."));
      linedata->addElement(CharsToUnicodeString("\\u4e03.\\u3001,\\u3002"));
      linedata->addElement(CharsToUnicodeString("\\u516b"));

    // to test for bug #4086052
      linedata->addElement(CharsToUnicodeString("foo\\u00a0bar "));
//          linedata->addElement("foo\\ufeffbar");

    // to test for bug #4097920
      linedata->addElement("dog,");
      linedata->addElement("cat,");
      linedata->addElement("mouse ");
      linedata->addElement("(one)");
      linedata->addElement("(two)\n");

    // to test for bug #4035266
      linedata->addElement("The ");
      linedata->addElement("balance ");
      linedata->addElement("is ");
      linedata->addElement("$-23,456.78, ");
      linedata->addElement("not ");
      linedata->addElement("-$32,456.78!\n");

    // to test for bug #4098467
    // What follows is a string of Korean characters (I found it in the Yellow Pages
    // ad for the Korean Presbyterian Church of San Francisco, and I hope I transcribed
    // it correctly), first as precomposed syllables, and then as conjoining jamo.
    // Both sequences should be semantically identical and break the same way.
    // precomposed syllables...
      linedata->addElement(CharsToUnicodeString("\\uc0c1\\ud56d "));
      linedata->addElement(CharsToUnicodeString("\\ud55c\\uc778 "));
      linedata->addElement(CharsToUnicodeString("\\uc5f0\\ud569 "));
      linedata->addElement(CharsToUnicodeString("\\uc7a5\\ub85c\\uad50\\ud68c "));
    // conjoining jamo...
      linedata->addElement(CharsToUnicodeString("\\u1109\\u1161\\u11bc\\u1112\\u1161\\u11bc "));
      linedata->addElement(CharsToUnicodeString("\\u1112\\u1161\\u11ab\\u110b\\u1175\\u11ab "));
      linedata->addElement(CharsToUnicodeString("\\u110b\\u1167\\u11ab\\u1112\\u1161\\u11b8 "));
      linedata->addElement(CharsToUnicodeString("\\u110c\\u1161\\u11bc\\u1105\\u1169\\u1100\\u116d\\u1112\\u116c"));

    // to test for bug #4117554: Fullwidth .!? should be treated as postJwrd
      linedata->addElement(CharsToUnicodeString("\\u4e01\\uff0e"));
      linedata->addElement(CharsToUnicodeString("\\u4e02\\uff01"));
      linedata->addElement(CharsToUnicodeString("\\u4e03\\uff1f"));

    generalIteratorTest(*lineIterDefault, linedata);

    delete lineIterDefault;
    delete linedata;

}
//--------------------------------------------------------------------
//Testing the BreakIterator for devanagari script
//--------------------------------------------------------------------
 
const UnicodeString deadRA  = CharsToUnicodeString("\\u0930\\u094d");         /*deadform RA = devanagari RA + virama*/
const UnicodeString deadPHA = CharsToUnicodeString("\\u092b\\u094d");         /*deadform PHA = devanagari PHA + virama*/
const UnicodeString deadTTHA= CharsToUnicodeString("\\u0920\\u094d");
const UnicodeString deadPA  = CharsToUnicodeString("\\u092a\\u094d");
const UnicodeString deadSA  = CharsToUnicodeString("\\u0938\\u094d");
const UnicodeString visarga = CharsToUnicodeString("\\u0903");              /*devanagari visarga looks like a english colon*/

void RBBITest::TestHindiCharacterBreak()
{
      Vector *hindicharData = new Vector();
      //devanagari characters for Hindi support
      hindicharData->addElement(CharsToUnicodeString("\\u0906"));                    //devanagari AA
      //hindi character break should make sure that it 
      // doesn't break in-between a vowelsign and a chandrabindu
      hindicharData->addElement(CharsToUnicodeString("\\u093e\\u0901"));              //devanagari vowelsign AA+ chandrabindu
      hindicharData->addElement(CharsToUnicodeString("\\u0916\\u0947"));              //devanagari KHA+vowelsign E
      hindicharData->addElement(CharsToUnicodeString("\\u0938\\u0941\\u0902"));        //devanagari SA+vowelsign U + anusvara(bindu)
      hindicharData->addElement(CharsToUnicodeString("\\u0926"));                    //devanagari consonant DA
      hindicharData->addElement(CharsToUnicodeString("\\u0930"));                    //devanagari consonant RA
      hindicharData->addElement(CharsToUnicodeString("\\u0939\\u094c"));              //devanagari consonant HA+dependent vowel sign AI
      hindicharData->addElement(CharsToUnicodeString("\\u0964"));                    //devanagari danda
      hindicharData->addElement(CharsToUnicodeString("\\u0950"));                    //devanagari OM 
      hindicharData->addElement(CharsToUnicodeString("\\u0915\\u0943"));              //devanagari KA+dependent vowel RI->KRI

      //dependent half-forms
      hindicharData->addElement(halfSA+CharsToUnicodeString("\\u0924"));             //halfSA+base consonant TA->STA
      hindicharData->addElement(halfSA+CharsToUnicodeString("\\u0925"));             //halfSA+base consonant THA->STHA
      hindicharData->addElement(halfSA+CharsToUnicodeString("\\u092e"));             //halfSA+base consonant MA->SMA
      hindicharData->addElement(halfCHA+CharsToUnicodeString("\\u091b"));            //halfCHA+base consonant CHHA->CHHHA
      hindicharData->addElement(halfNA+CharsToUnicodeString("\\u0917"));             //halfNA+base consonant GA->NGA
      hindicharData->addElement(CharsToUnicodeString("\\u092a\\u094d\\u200d\\u092f"));   //halfPA(PA+virama+zerowidthjoiner+base consonant YA->PYA


      //consonant RA rules ----------
      //if the dead consonant RA precedes either a consonant or an independent vowel,
      //then it is replaced by its superscript non-spacing mark
      hindicharData->addElement(deadRA+ CharsToUnicodeString("\\u0915"));             //deadRA+devanagari consonant KA->KA+superRA 
      hindicharData->addElement(deadRA+ CharsToUnicodeString("\\u0923"));             //deadRA+devanagari consonant NNA->NNA+superRA
      hindicharData->addElement(deadRA+ CharsToUnicodeString("\\u0917"));             //deadRA+devanagari consonant GA->GA+superRA
    //  hindicharData->addElement(deadRA+ CharsToUnicodeString("\\u0960"));           //deadRA+devanagari cosonant RRI->RRI+superRA
      
      //if any dead consonant(other than dead RA)precedes the consonant RA, then
      //it is replaced with its nominal forma nd RA is replaced by the subscript non-spacing mark.
      hindicharData->addElement(deadPHA+ CharsToUnicodeString("\\u0930"));            //deadPHA+devanagari consonant RA->PHA+subRA
      hindicharData->addElement(deadPA+ CharsToUnicodeString("\\u0930"));             //deadPA+devanagari consonant RA->PA+subRA
      hindicharData->addElement(deadTTHA+ CharsToUnicodeString("\\u0930"));           //deadTTHA+devanagari consonant RA->TTHA+subRA
      hindicharData->addElement(deadTA+ CharsToUnicodeString("\\u0930"));             //deadTA+RA->TRA 
      hindicharData->addElement(CharsToUnicodeString("\\u0936\\u094d\\u0930"));         //deadSHA(SHA+virama)+RA->SHRA 

      //conjuct ligatures
      hindicharData->addElement(CharsToUnicodeString("\\u0915\\u094d\\u0937"));         //deadKA(KA+virama) followed by SSHA wraps up into a single character KSSHA
      hindicharData->addElement(deadTA+CharsToUnicodeString("\\u0924"));              //deadTA+TA wraps up into glyph TTHA
      hindicharData->addElement(CharsToUnicodeString("\\u0926\\u094d\\u0935"));         //deadDA(DA+virama)+VA wraps up into DVA
      hindicharData->addElement(CharsToUnicodeString("\\u091c\\u094d\\u091e"));         //deadJA(JA+virama)+NYA wraps up into JNYA
     
      UErrorCode status= U_ZERO_ERROR;
      RuleBasedBreakIterator *e=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createCharacterInstance(Locale::getDefault(), status);
      if(U_FAILURE(status)){
          errln("FAIL : in construction");
          return;
      }
      generalIteratorTest(*e, hindicharData);
      delete e;
      delete hindicharData;
}
void RBBITest::TestHindiWordBreak()
   {
     Vector *hindiWordData = new Vector();

      //hindi
     hindiWordData->addElement(CharsToUnicodeString("\\u0917\\u092a-\\u0936\\u092a"));
     hindiWordData->addElement("!");
     hindiWordData->addElement(CharsToUnicodeString("\\u092f\\u0939"));
     hindiWordData->addElement(" ");
     hindiWordData->addElement(CharsToUnicodeString("\\u0939\\u093f") + halfNA + CharsToUnicodeString("\\u0926\\u0940"));
     hindiWordData->addElement(" ");
     hindiWordData->addElement(CharsToUnicodeString("\\u0939\\u0948"));
     //danda is similar to full stop. danda is a hindi phrase seperator
     //Make sure it breaks before danda and after danda when it is followed by a space
     //hindiWordData->addElement(CharsToUnicodeString("\\u0964"));   //fails here doesn't break at danda
     hindiWordData->addElement(" ");
     hindiWordData->addElement(CharsToUnicodeString("\\u0905\\u093e\\u092a"));
     hindiWordData->addElement(" ");
     hindiWordData->addElement(CharsToUnicodeString("\\u0938\\u093f\\u0916\\u094b\\u0917\\u0947"));
     hindiWordData->addElement("?");
     hindiWordData->addElement("\n"); 
     hindiWordData->addElement(":");
     hindiWordData->addElement(deadPA+CharsToUnicodeString("\\u0930\\u093e\\u092f")+visarga);    //no break before visarga
     hindiWordData->addElement(" ");

     hindiWordData->addElement(CharsToUnicodeString("\\u0935") + deadRA+ CharsToUnicodeString("\\u0937\\u093e"));
     hindiWordData->addElement("\r\n");
     hindiWordData->addElement(deadPA+ CharsToUnicodeString("\\u0930\\u0915\\u093e\\u0936"));     //deadPA+RA+KA+vowel AA+SHA -> prakash
     hindiWordData->addElement(","); 
     hindiWordData->addElement(CharsToUnicodeString("\\u0924\\u0941\\u092e\\u093e\\u0930\\u094b")); 
     hindiWordData->addElement(" ");
     hindiWordData->addElement(CharsToUnicodeString("\\u092e\\u093f")+ deadTA+ CharsToUnicodeString("\\u0930"));       //MA+vowel I+ deadTA + RA 
     hindiWordData->addElement(" ");
     hindiWordData->addElement(CharsToUnicodeString("\\u0915\\u093e"));
     hindiWordData->addElement(" ");
     hindiWordData->addElement(CharsToUnicodeString("\\u092a")+ deadTA + CharsToUnicodeString("\\u0930"));            //PA + deadTA + RA
     hindiWordData->addElement(" ");
     hindiWordData->addElement(CharsToUnicodeString("\\u092a\\u095d\\u094b"));
    // hindiWordData->addElement(CharsToUnicodeString("\\u0964")); //fails here doesn't break at danda
     hindiWordData->addElement(" ");
     hindiWordData->addElement(deadSA + deadTA + CharsToUnicodeString("\\u0930\\u093f"));       //deadSA+deadTA+RA+vowel I->sthri
     hindiWordData->addElement(".");
     hindiWordData->addElement(" ");
     hindiWordData->addElement(CharsToUnicodeString("\\u0968\\u0966.\\u0969\\u096f"));            //hindi numbers
     hindiWordData->addElement(" ");
     hindiWordData->addElement(CharsToUnicodeString("\\u0967\\u0966\\u0966.\\u0966\\u0966%"));     //postnumeric
     hindiWordData->addElement(" ");
     hindiWordData->addElement(CharsToUnicodeString("\\u20a8\\u0967,\\u0967\\u0966\\u0966.\\u0966\\u0966")); //pre-number India currency symbol Rs.\\u20aD
     hindiWordData->addElement(" ");
     hindiWordData->addElement(CharsToUnicodeString("\\u0905\\u092e\\u091c"));
     hindiWordData->addElement("\n");
     hindiWordData->addElement(halfSA+CharsToUnicodeString("\\u0935\\u0924\\u0902")+deadTA+CharsToUnicodeString("\\u0930"));
     hindiWordData->addElement("\r");

     UErrorCode status=U_ZERO_ERROR;
     RuleBasedBreakIterator *e=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createWordInstance(Locale::getDefault(), status);
      if(U_FAILURE(status)){
          errln("FAIL : in construction");
          return;
      }
      generalIteratorTest(*e, hindiWordData);
      delete e;
      delete hindiWordData;
}
/*
//Bug: if there is no word break before and after danda when it is followed by a space
void RBBITest::TestDanda()
{
      Vector *hindiWordData = new Vector();
      //hindi
      hindiWordData->addElement(CharsToUnicodeString("\\u092f\\u0939"));
      hindiWordData->addElement(" ");
      //Danda is similar to full stop, danda is a hindi phrase seperator.
      //Make sure there is a word break before and after danda when it is followed by a space
     //following fail----
      hindiWordData->addElement(CharsToUnicodeString("\\u0939\\u0948"));
    //  hindiWordData->addElement(CharsToUnicodeString("\\u0964"));         // devanagari danda
      hindiWordData->addElement(" ");
      hindiWordData->addElement(CharsToUnicodeString("\\u092f\\u0939"));
  //    hindiWordData->addElement(CharsToUnicodeString("\\u0965"));         //devanagari double danda
      hindiWordData->addElement(" ");

      RuleBasedBreakIterator* e=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createWordInstance();
      generalIteratorTest(*e, hindiWordData);
      delete e;
      delete hindiWordData;
}
//Make sure the character wrapping is done correctly
void RBBITest::TestHindiCharacterWrapping()
{
      Vector *hindicharData = new Vector();
      //if the dead consonant RA precedes either a consonant or an independent vowel,
      //then it is replaced by its superscript non-spacing mark
      hindicharData->addElement(deadRA+ CharsToUnicodeString("\\u0917")); //deadRA+devanagari consonant GA->GA+superRA
      //following fail----
     // hindicharData->addElement(deadRA+ CharsToUnicodeString("\\u0960"));   //deadRA+devanagari RRI->RRI+superRA

      RuleBasedBreakIterator* e=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createCharacterInstance();
      generalIteratorTest(*e, hindicharData);
      delete e;
      delete hindicharData;

}*/






//----------------------------------------------------------------------------------
//adds rules for telugu support and tests the behaviour of chracterIterator of RBBI 
//----------------------------------------------------------------------------------
/*void RBBITest::TestTeluguRuleBasedCharacterIteration()
{
     logln((UnicodeString)"Testing the RBBI by adding rules for Telugu(Indian Language) Support");
     //get the default rules
     RuleBasedBreakIterator *rb= (RuleBasedBreakIterator*)BreakIterator::createCharacterInstance();
     //additional rules for Telugu(Indian Language) support
     UnicodeString crules1 = rb->getRules()                                                 +  //default rules +
                      "<telvirama>=[\\u0c4d];"                                               +  //telugu virama
                      "<telVowelSign>=[\\u0c3e-\\u0c44\\u0c46\\u0c47\\u0c48\\u0c4a\\u0c4b\\u0c4c];" +  //telugu dependent vowel signs
                      "<telConsonant>=[\\u0c15-\\u0c28\\u0c2a-\\u0c33\\u0c35-\\u0c39];"           +  //telugu consonants
                      "<telCharEnd>=[\\u0c02\\u0c03\\u0c55\\u0c56];"                            +  //to create half forms and dead forms
                      "<telConjunct>=({<telConsonant><telvirama>{<zwj>}}<telConsonant>);"   +
                      "<telConjunct>{<telVowelSign>}{<telCharEnd>};";
      RuleBasedBreakIterator charIter=null;
      charIter   = new RuleBasedBreakIterator(crules1); 

      Vector *chardata = new Vector();
      //behaviour of telugu characters from specified rules
      chardata->addElement(CharsToUnicodeString("\\u0c15"));                    //telugu consonant KA
      chardata->addElement(CharsToUnicodeString("\\u0c30\\u0c47"));              //telugu consonant RA+telugu dependent vowel EE
      chardata->addElement(CharsToUnicodeString("\\u0c1b\\u0c3e"));              //telugu consonant CHA+telegu depenednt vowel AA
      chardata->addElement(CharsToUnicodeString("\\u0c17\\u0c48"));              //telegu consonant GA+teleugu dependent vowel AI
      chardata->addElement(CharsToUnicodeString("\\u0c17\\u0c46\\u0c56"));        //telugu consonant GA+telugu dependent vowel sign E+telugu AI length mark
      chardata->addElement(CharsToUnicodeString("\\u0c28\\u0c4d\\u200d\\u0c28"));  //telugu consonant NA+telugu virama+zwj=>halfNA+NA->NNA(dependent half-form)
      chardata->addElement(CharsToUnicodeString("\\u0c17\\u0c4d\\u0c30"));        //GA+deadRA(RA+telvirama)->GA+subRA->GRA
      chardata->addElement(CharsToUnicodeString("\\u0c66"));                    //telugu digit
      chardata->addElement(CharsToUnicodeString("\\u0c37\\u0c4d\\u0c15"));        //deadSSA(SSA+telvirama)+KA+subSSA->KSHA
      //behaviour of other characters from default rules
      chardata->addElement("h");
      chardata->addElement(CharsToUnicodeString("A\\u0302"));                   // circumflexA 
      chardata->addElement(CharsToUnicodeString("i\\u0301"));                   // acuteBelowI 
      chardata->addElement(CharsToUnicodeString("\\u1109\\u1161\\u11bc"));
      chardata->addElement(CharsToUnicodeString("\\u1112\\u1161\\u11bc"));
      chardata->addElement("\n");
      chardata->addElement("\r\n");    

      generalIteratorTest(charIter, chardata);

      delete charIter;
      delete charData;
      delete rb;
}

//--------------------------------------------------------------------
//tests the behaviour of character iteration of RBBI with custom rules
//--------------------------------------------------------------------
 
void RBBITest::TestCustomRuleBasedCharacterIteration()
{
      logln((UnicodeString)"Testing the RBBI by using custom rules for character iteration");

      UnicodeString crules2="<ignore>=[e];"                         + //ignore the character "e"
                     ".;"                                           + 
                     "<devVowelSign>=[\\u093e-\\u094c\\u0962\\u0963];"  +  //devanagiri vowel = \\u093e tp \\u094c and \\u0962.\\u0963
                     "<devConsonant>=[\\u0915-\\u0939];"              +  //devanagiri consonant = \\u0915 to \\u0939
                     "<devConsonant>{<devVowelSign>};" ;               //break at all places except the  following 
                                                                       //devanagiri consonants+ devanagiri vowelsign

      RuleBasedCharacterIterator charIterCustom   = new RuleBasedBreakIterator(crules2);
      Vector *chardata = new Vector();
      chardata->addElement("He");              //ignores 'e'
      chardata->addElement("l");                
      chardata->addElement("l");
      chardata->addElement("oe");              //ignores 'e' hence wraps it into 'o' instead of wrapping with
      chardata->addElement(CharsToUnicodeString("\\u0301"));          //'\\u0301' to form 'acuteE '
      chardata->addElement("&e");              //ignores 'e' hence wraps it into '&' instead of wrapping with
      chardata->addElement(CharsToUnicodeString("\\u0303"));          //'\\u0303 to form 'tildaE'
      //devanagiri characters 
      chardata->addElement(CharsToUnicodeString("\\u0906"));          //devanagiri AA
      chardata->addElement(CharsToUnicodeString("\\u093e"));          //devanagiri vowelsign AA:--breaks at \\u0901 which is devanagiri 
      chardata->addElement(CharsToUnicodeString("\\u0901"));          //chandra bindhu since it is not mentioned in the rules
      chardata->addElement(CharsToUnicodeString("\\u0916\\u0947"));    //devanagiri KHA+vowelsign E
      chardata->addElement(CharsToUnicodeString("\\u0938\\u0941"));    //devanagiri SA+vowelsign U : - breaks at
      chardata->addElement(CharsToUnicodeString("\\u0902"));          //\\u0902 devanagiri anusvara since it is not mentioned in the rules
      chardata->addElement(CharsToUnicodeString("\\u0926"));          //devanagiri consonant DA
      chardata->addElement(CharsToUnicodeString("\\u0930"));          //devanagiri consonant RA
      chardata->addElement(CharsToUnicodeString("\\u0939\\u094c"));    //devanagiri HA+vowel sign AI
      chardata->addElement(CharsToUnicodeString("\\u0964"));          //devanagiri danda
      // devanagiri chracters end
      chardata->addElement("A");               //breaks in between since it is not mentioned in the rules
      chardata->addElement(CharsToUnicodeString("\\u0302"));          // circumflexA    
      chardata->addElement("i");               //breaks in between since not mentioned in the rules
      chardata->addElement(CharsToUnicodeString("\\u0301"));          // acuteBelowI   
      //Rules don't support conjoining jamo->->..
      chardata->addElement(CharsToUnicodeString("\\u1109"));          //break at every character since rules
      chardata->addElement(CharsToUnicodeString("\\u1161"));          //don't support conjoining jamo
      chardata->addElement(CharsToUnicodeString("\\u11bc"));
      chardata->addElement(CharsToUnicodeString("\\u1112"));
      chardata->addElement(CharsToUnicodeString("\\u1161"));
      chardata->addElement(CharsToUnicodeString("\\u11bc"));
      chardata->addElement("\n");
      chardata->addElement("\r");             //doesn't keep CRLGF together since rules do not mention it
      chardata->addElement("\n");
      chardata->addElement("S");              //graveS
      chardata->addElement(CharsToUnicodeString("\\u0300"));         //breaks in between since it is not mentioned in the rules
      chardata->addElement("i");              //acuteBelowI
      chardata->addElement(CharsToUnicodeString("\\u0301"));         //breaks in between since it is not mentioned in the rules
      generalIteratorTest(charIterCustom, chardata);

      delete charIterCustom;
      delete chardata;
}*/
/*//--------------------------------------------------------------------
//tests custom rules based word iteration
//--------------------------------------------------------------------
void RBBITest::TestCustomRuleBasedWordIteration(){
      logln("(UnicodeString)Testing the RBBI by using custom rules for word iteration");
      UnicodeString wrules1="<ignore>=[:Mn::Me::Cf:];"                  + //ignore non-spacing marks, enclosing marks, and format characters,
                      "<danda>=[\\u0964\\u0965];"                       + //Hindi Phrase seperator
                      "<let>=[:L::Mc:];"                                + //uppercase(Lu), lowercase(Ll), titlecase(Lt), modifier(Lm) letters, Mc-combining space mark
                      "<mid-word>=[:Pd:\\\"\\\'\\.];"                   + //dashes, quotation, apostraphes, period
                      "<ls>=[\\n\\u000c\\u2028\\u2029];"                + //line separators:  LF, FF, PS, and LS
                      "<ws>=[:Zs:\\t];"                                 + //all space separators and the tab character
                      "<word>=((<let><let>*(<mid-word><let><let>*)*));" +  
                      ".;"                                              + //break after every character, with the following exceptions
                      "{<word>};"                                       + 
                      "<ws>*{\\r}{<ls>}{<danda>};" ;

      RuleBasedBreakIterator wordIterCustom   = new RuleBasedBreakIterator(wrules1); 
      Vector *worddata = new Vector();
      worddata->addElement("Write");
      worddata->addElement(" ");
      worddata->addElement("wordrules");
      worddata->addElement(".");
      worddata->addElement(" ");
      //play with hindi
      worddata->addElement(CharsToUnicodeString("\\u092f\\u0939"));
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u0939\\u093f") + halfNA + CharsToUnicodeString("\\u0926\\u0940"));
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u0939\\u0948"));
      worddata->addElement(CharsToUnicodeString("\\u0964"));   //Danda is similar to full stop-> Danda followed by a space
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u0905\\u093e\\u092a"));
      worddata->addElement(" ");
      worddata->addElement(CharsToUnicodeString("\\u0938\\u093f\\u0916\\u094b\\u0917\\u0947"));
      worddata->addElement("?");
      worddata->addElement(" ");
      worddata->addElement("It's");
      worddata->addElement(" ");
      worddata->addElement("$");
      worddata->addElement("3");
      worddata->addElement("0");
      worddata->addElement(".");
      worddata->addElement("1");
      worddata->addElement("0");
      worddata->addElement(" ");
      // worddata->addElement(" ");
      generalIteratorTest(wordIterCustom, worddata); 

      delete wordIterCustom;
      delete worddata;
}
//-------------------------------------------------------------------------------
//adds extra rules to deal with abbrevations(limited) and test the word Iteration
//-------------------------------------------------------------------------------
void RBBITest::TestAbbrRuleBasedWordIteration()
{
      logln((UnicodeString)"Testing the RBBI for word iteration by adding rules to support abbreviation");
      RuleBasedBreakIterator *rb =(RuleBasedBreakIterator*)BreakIterator::createWordInstance();
      
      UnicodeString wrules2="<abbr>=((Mr.)|(Mrs.)|(Ms.)|(Dr.)|(U.S.));" + // abbreviations. 
                     rb->getRules()                               +
                     "{(<abbr><ws>)*<word>};";
      RuleBasedBreakIterator wordIter=null;
      //try{
      wordIter   = new RuleBasedBreakIterator(wrules2); 
    //  }catch(IllegalArgumentException iae){
   //       errln("ERROR: failed construction illegal rules");
   //   }
      Vector *worddata = new Vector();
      worddata->addElement("Mr. George");
      worddata->addElement(" ");
      worddata->addElement("is");
      worddata->addElement(" ");
      worddata->addElement("from");
      worddata->addElement(" ");
      worddata->addElement("U.S. Navy");
      worddata->addElement(".");
      worddata->addElement(" ");
      worddata->addElement("His");
      worddata->addElement("\n");
      worddata->addElement("friend");
      worddata->addElement("\t");
      worddata->addElement("Dr. Steven");
      worddata->addElement(" ");
      worddata->addElement("married");
      worddata->addElement(" ");
      worddata->addElement("Ms. Benneth");
      worddata->addElement("!");
      worddata->addElement(" ");
      worddata->addElement("Mrs. Johnson");
      worddata->addElement("\r\n");
      worddata->addElement("paid");
      worddata->addElement(" ");
      worddata->addElement("$2,400.00");
      generalIteratorTest(wordIter, worddata);

      delete wordIter;
      delete worddata;
      delete rb;
} */  

//---------------------------------------------
// runIndexedTest
//---------------------------------------------

void RBBITest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* par )
{
    if (exec) logln("TestSuite RuleBasedBreakIterator: ");
    switch (index) {
    
        case 0: name = "TestDefaultRuleBasedCharacterIteration"; 
            if(exec) TestDefaultRuleBasedCharacterIteration(); break;
        case 1: name = "TestDefaultRuleBasedWordIteration";      
            if(exec) TestDefaultRuleBasedWordIteration();      break;
        case 2: name = "TestDefaultRuleBasedSentenceIteration";  
            if(exec) TestDefaultRuleBasedSentenceIteration();  break;
        case 3: name = "TestDefaulRuleBasedLineIteration()";
            if(exec) TestDefaultRuleBasedLineIteration();      break;
        case 4: name = "TestHindiCharacterBreak()";
            if(exec) TestHindiCharacterBreak();                break;
        case 5: name = "TestHindiWordBreak()";
            if(exec) TestHindiWordBreak();                     break;
        case 6: name = "TestHindiWordBreak()";
            if(exec) TestHindiWordBreak();                     break;
//      case 7: name = "TestDanda()";
//           if(exec) TestDanda();                              break;
//       case 8: name = "TestHindiCharacterWrapping()";
//           if(exec) TestHindiCharacterWrapping();             break;

//      case 9: name = "TestCustomRuleBasedWordIteration";       
//          if(exec) TestCustomRuleBasedWordIteration();       break;
//      case 10: name = "TestAbbrRuleBasedWordIteration";         
//          if(exec) TestAbbrRuleBasedWordIteration();         break;
//      case 11: name = "TestTeluguRuleBasedCharacterIteration";  
//          if(exec) TestTeluguRuleBasedCharacterIteration();  break;
//      case 12: name = "TestCustomRuleBasedCharacterIteration";  
//          if(exec) TestCustomRuleBasedCharacterIteration();  break;


        default: name = ""; break; //needed to end loop
    }
}

//---------------------------------------------
// Test implementation routines
//---------------------------------------------
UnicodeString RBBITest::createTestData(Enumeration* e)
{
  UnicodeString result = "";

  while (e->hasMoreElements()) {
    result += e->nextElement();
  }
  return result;
}
// general test Implementation subroutines
void RBBITest::generalIteratorTest(RuleBasedBreakIterator& bi, Vector* expectedResult) 
{
    Enumeration *elems = expectedResult->elements();
    UnicodeString text = createTestData(elems);
    delete elems;

    bi.setText(&text);

    Vector *nextResults = testFirstAndNext(bi, text);
    Vector *previousResults = testLastAndPrevious(bi, text);

    logln("comparing forward and backward...");
    int errs = getErrors();
    UnicodeString str1="forward iteration";
    UnicodeString str2="backward iteration";
    compareFragmentLists(str1, str2, nextResults,
                    previousResults);
    if (getErrors() == errs) {
        logln("comparing expected and actual...");
        str1="expected result";
        str2="actual result";
        compareFragmentLists(str1, str2, expectedResult,
                        nextResults);
    }

    int32_t *boundaries = new int32_t[expectedResult->size() + 3];
    boundaries[0] = RuleBasedBreakIterator::DONE;
    boundaries[1] = 0;
    for (int i = 0; i < expectedResult->size(); i++)
        boundaries[i + 2] = boundaries[i + 1] + ((UnicodeString)expectedResult->elementAt(i)).
                        length();

    int len = expectedResult->size() + 3 -1;
    boundaries[len] = RuleBasedBreakIterator::DONE;

    testFollowing(bi, text, boundaries);
    testPreceding(bi, text, boundaries);
    testIsBoundary(bi, text, boundaries);

    doMultipleSelectionTest(bi, text);

    delete[] boundaries;
    delete nextResults;
    delete previousResults;
}

Vector* RBBITest::testFirstAndNext(RuleBasedBreakIterator& bi, UnicodeString& text) 
{
    int32_t p = bi.first();
    int32_t lastP = p;
    Vector *result = new Vector();
    UnicodeString selection;

    if (p != 0)
        errln((UnicodeString)"first() returned " + p + (UnicodeString)" instead of 0");
    while (p != RuleBasedBreakIterator::DONE) {
        p = bi.next();
        if (p != RuleBasedBreakIterator::DONE) {
            if (p <= lastP)
                errln((UnicodeString)"next() failed to move forward: next() on position "
                                + lastP + (UnicodeString)" yielded " + p);

            text.extractBetween(lastP, p, selection);  
            result->addElement(selection);
        }
        else {
            if (lastP != text.length())
                errln((UnicodeString)"next() returned DONE prematurely: offset was "
                                + lastP + (UnicodeString)" instead of " + text.length());
        }
        lastP = p;
    }
    return result;
}
Vector* RBBITest::testLastAndPrevious(RuleBasedBreakIterator& bi, UnicodeString& text) 
{
    int32_t p = bi.last();
    int32_t lastP = p;
    Vector *result = new Vector();
    UnicodeString selection;

    if (p != text.length())
        errln((UnicodeString)"last() returned " + p + (UnicodeString)" instead of " + text.length());
    while (p != RuleBasedBreakIterator::DONE) {
        p = bi.previous();
        if (p != RuleBasedBreakIterator::DONE) {
            if (p >= lastP)
                errln((UnicodeString)"previous() failed to move backward: previous() on position "
                                + lastP + (UnicodeString)" yielded " + p);
            text.extractBetween(p, lastP, selection);
            result->insertElementAt(selection, 0);
        }
        else {
            if (lastP != 0)
                errln((UnicodeString)"previous() returned DONE prematurely: offset was "
                                + lastP + (UnicodeString)" instead of 0");
        }
        lastP = p;
    }
    return result;
}

void RBBITest::compareFragmentLists(UnicodeString& f1Name, UnicodeString& f2Name, Vector* f1, Vector* f2) 
{
    int32_t p1 = 0;
    int32_t p2 = 0;
    UnicodeString s1;
    UnicodeString s2;
    int32_t t1 = 0;
    int32_t t2 = 0;


    while (p1 < f1->size() && p2 < f2->size()) {
        s1 = (UnicodeString)f1->elementAt(p1);
        s2 = (UnicodeString)f2->elementAt(p2);
        t1 += s1.length();
        t2 += s2.length();

        if (s1.compare(s2) == 0) {
            logln(prettify((UnicodeString)"   >" + s1 + (UnicodeString)"<"));
            ++p1;
            ++p2;
        }
        else {
            int32_t tempT1 = t1;
            int32_t tempT2 = t2;
            int32_t tempP1 = p1;
            int32_t tempP2 = p2;

            while (tempT1 != tempT2 && tempP1 < f1->size() && tempP2 < f2->size()) {
                while (tempT1 < tempT2 && tempP1 < f1->size()) {
                    tempT1 += ((UnicodeString)f1->elementAt(tempP1)).length();
                    ++tempP1;
                }
                while (tempT2 < tempT1 && tempP2 < f2->size()) {
                    tempT2 += ((UnicodeString)f2->elementAt(tempP2)).length();
                    ++tempP2;
                }
            }
            logln((UnicodeString)"*** " + f1Name + (UnicodeString)" has:");
            while (p1 <= tempP1 && p1 < f1->size()) {
                s1 = (UnicodeString)f1->elementAt(p1);
                t1 += s1.length();
                logln(prettify((UnicodeString)" *** >" + s1 + (UnicodeString)"<"));
                ++p1;
            }
            logln("***** " + f2Name + " has:");
            while (p2 <= tempP2 && p2 < f2->size()) {
                s2 = (UnicodeString)f2->elementAt(p2);
                t2 += s2.length();
                logln(prettify(" ***** >" + s2 + "<"));
                ++p2;
            }
            errln((UnicodeString)"Discrepancy between " + f1Name + (UnicodeString)" and " + f2Name);
        }
    }
}

void RBBITest::testFollowing(RuleBasedBreakIterator& bi, UnicodeString& text, int32_t *boundaries) 
{
    logln("testFollowing():");
    int p = 2;
    for (int i = 0; i <= text.length(); i++) {
        if (i == boundaries[p])
            ++p;

        int32_t b = bi.following(i);
        logln((UnicodeString)"bi.following(" + i + ") -> " + b);
        if (b != boundaries[p])
            errln((UnicodeString)"Wrong result from following() for " + i + (UnicodeString)": expected " + boundaries[p]
                            + (UnicodeString)", got " + b);
    }
}

void RBBITest::testPreceding(RuleBasedBreakIterator& bi, UnicodeString& text, int32_t *boundaries) {
    logln("testPreceding():");
    int p = 0;
    for (int i = 0; i <= text.length(); i++) {
        int32_t b = bi.preceding(i);
        logln((UnicodeString)"bi.preceding(" + i + ") -> " + b);
        if (b != boundaries[p])
            errln((UnicodeString)"Wrong result from preceding() for " + i + (UnicodeString)": expected " + boundaries[p]
                            + (UnicodeString)", got " + b);

        if (i == boundaries[p + 1])
            ++p;
    }
}

void RBBITest::testIsBoundary(RuleBasedBreakIterator& bi, UnicodeString& text, int32_t *boundaries) {
    logln("testIsBoundary():");
    int p = 1;
    UBool isB;
    for (int i = 0; i < text.length(); i++) {
        isB = bi.isBoundary(i);
        logln((UnicodeString)"bi.isBoundary(" + i + ") -> " + isB);

        if (i == boundaries[p]) {
            if (!isB)
                errln((UnicodeString)"Wrong result from isBoundary() for " + i + (UnicodeString)": expected true, got false");
            p++;
        }
        else {
            if (isB)
                errln((UnicodeString)"Wrong result from isBoundary() for " + i + (UnicodeString)": expected false, got true");
        }
    }
}

void RBBITest::doMultipleSelectionTest(RuleBasedBreakIterator& iterator,
                                                  UnicodeString& testText)
{
    iterator.setText(&testText);

    RuleBasedBreakIterator* testIterator =(RuleBasedBreakIterator*)iterator.clone();
    int32_t offset = iterator.first();
    int32_t testOffset;
    int32_t count = 0;

    logln("doMultipleSelectionTest text of length: "+testText.length());

    if (*testIterator != iterator)
        errln("clone() or operator!= failed: two clones compared unequal");

    do {
        testOffset = testIterator->first();
        testOffset = testIterator->next(count);
        if (offset != testOffset)
            errln(UnicodeString("next(n) and next() not returning consistent results: for step ") + count + ", next(n) returned " + testOffset + " and next() had " + offset);

        if (offset != RuleBasedBreakIterator::DONE) {
            count++;
            offset = iterator.next();

            if (offset != RuleBasedBreakIterator::DONE && *testIterator == iterator)
                errln("operator== failed: Two unequal iterators compared equal.");
        }
    } while (offset != RuleBasedBreakIterator::DONE);

    // now do it backwards...
    offset = iterator.last();
    count = 0;

    do {
        testOffset = testIterator->last();
        testOffset = testIterator->next(count);
        if (offset != testOffset)
            errln(UnicodeString("next(n) and next() not returning consistent results: for step ") + count + ", next(n) returned " + testOffset + " and next() had " + offset);

        if (offset != RuleBasedBreakIterator::DONE) {
            count--;
            offset = iterator.previous();
        }
    } while (offset != RuleBasedBreakIterator::DONE);

    delete testIterator;
}