scuffed-code/icu4c/source/test/cintltst/ncnvtst.c

/********************************************************************
 * COPYRIGHT: 
 * Copyright (c) 1997-1999, International Business Machines Corporation and
 * others. All Rights Reserved.
 ********************************************************************/
/********************************************************************************
*
* File CCONVTST.C
*
* Modification History:
*        Name                     Description
*   Madhu Katragadda              7/7/2000        Converter Tests for extended code coverage
*********************************************************************************
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include "unicode/uloc.h"
#include "unicode/ucnv.h"
#include "cintltst.h"
#include "unicode/utypes.h"
#include "unicode/ustring.h"

#define MAX_LENGTH 99
#ifdef TEST_BUFFER_SIZE
#undef TEST_BUFFER_SIZE
#endif
#define TEST_BUFFER_SIZE 5555555

#define UNICODE_LIMIT 0x10FFFF

static int32_t  gInBufferSize = 0;
static int32_t  gOutBufferSize = 0;
static char     gNuConvTestName[1024];

#define nct_min(x,y)  ((x<y) ? x : y)

static void printSeq(const unsigned char* a, int len);
static void printSeqErr(const unsigned char* a, int len);
static void printUSeq(const UChar* a, int len);
static void printUSeqErr(const UChar* a, int len);
static UBool convertFromU( const UChar *source, int sourceLen,  const uint8_t *expect, int expectLen, 
                const char *codepage, int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus);
static UBool convertToU( const uint8_t *source, int sourceLen, const UChar *expect, int expectLen, 
               const char *codepage, int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus);

static UBool testConvertFromU( const UChar *source, int sourceLen,  const uint8_t *expect, int expectLen, 
                const char *codepage, UConverterFromUCallback callback, int32_t *expectOffsets, UBool testReset);
static UBool testConvertToU( const uint8_t *source, int sourcelen, const UChar *expect, int expectlen, 
               const char *codepage, UConverterToUCallback callback, int32_t *expectOffsets, UBool testReset);

static void setNuConvTestName(const char *codepage, const char *direction)
{
  sprintf(gNuConvTestName, "[Testing %s %s Unicode, InputBufSiz=%d, OutputBufSiz=%d]",
      codepage,
      direction,
      gInBufferSize,
      gOutBufferSize);
}


static void TestSurrogateBehaviour(void);
static void TestErrorBehaviour(void);
static void TestToUnicodeErrorBehaviour(void);
static void TestGetNextErrorBehaviour(void);
static void TestRegressionUTF8(void);
static void TestRegressionUTF32(void);
static void TestAvailableConverters(void);
static void TestFlushInternalBuffer(void);  /*for improved code coverage in ucnv_cnv.c*/
static void TestResetBehaviour(void);

static void TestWithBufferSize(int32_t osize, int32_t isize);


static void printSeq(const unsigned char* a, int len)
{
    int i=0;
    log_verbose("\n{");
    while (i<len)
        log_verbose("0x%02X ", a[i++]);
    log_verbose("}\n");
}

static void printUSeq(const UChar* a, int len)
{
    int i=0;
    log_verbose("\n{");
    while (i<len)
        log_verbose("%0x04X ", a[i++]);
    log_verbose("}\n");
}

static void printSeqErr(const unsigned char* a, int len)
{
    int i=0;
    fprintf(stderr, "\n{");
    while (i<len)  fprintf(stderr, "0x%02X ", a[i++]);
    fprintf(stderr, "}\n");
}

static void printUSeqErr(const UChar* a, int len)
{
    int i=0;
    fprintf(stderr, "\n{");
    while (i<len)
        fprintf(stderr, "0x%04X ", a[i++]);
    fprintf(stderr,"}\n");
}

void addExtraTests(TestNode** root)
{
     addTest(root, &TestSurrogateBehaviour,         "tsconv/ncnvtst/TestSurrogateBehaviour");
     addTest(root, &TestErrorBehaviour,             "tsconv/ncnvtst/TestErrorBehaviour");
     addTest(root, &TestToUnicodeErrorBehaviour,    "tsconv/ncnvtst/ToUnicodeErrorBehaviour");
     addTest(root, &TestGetNextErrorBehaviour,      "tsconv/ncnvtst/TestGetNextErrorBehaviour");
     addTest(root, &TestRegressionUTF8,             "tsconv/ncnvtst/TestRegressionUTF8");
     addTest(root, &TestRegressionUTF32,             "tsconv/ncnvtst/TestRegressionUTF32");
     addTest(root, &TestAvailableConverters,        "tsconv/ncnvtst/TestAvailableConverters");
     addTest(root, &TestFlushInternalBuffer,        "tsconv/ncnvtst/TestFlushInternalBuffer");
     addTest(root, &TestResetBehaviour,              "tsconv/ncnvtst/TestResetBehaviour");

}

/*test surrogate behaviour*/
static void TestSurrogateBehaviour(){
    log_verbose("Testing for SBCS and LATIN_1\n");
    {
        UChar sampleText[] = {0x0031, 0xd801, 0xdc01, 0x0032};
        const uint8_t expected[] = {0x31, 0x1a, 0x32};
        /*SBCS*/
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-920", 0 , TRUE, U_ZERO_ERROR))
            log_err("u-> ibm-920 [UCNV_SBCS] not match.\n");

        /*LATIN_1*/
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "LATIN_1", 0, TRUE, U_ZERO_ERROR ))
            log_err("u-> LATIN_1 not match.\n");

    }
    log_verbose("Testing for DBCS and MBCS\n");
    {
        UChar sampleText[]       = {0x00a1, 0xd801, 0xdc01, 0x00a4};
        const uint8_t expected[] = {0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
        int32_t offsets[]        = {0x00, 0x00, 0x01, 0x01, 0x03, 0x03 };

        /*DBCS*/
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-1362", 0 , TRUE, U_ZERO_ERROR))
            log_err("u-> ibm-1362 [UCNV_DBCS] not match.\n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-1362", offsets , TRUE, U_ZERO_ERROR))
            log_err("u-> ibm-1362 [UCNV_DBCS] not match.\n");
        /*MBCS*/
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-1363", 0 , TRUE, U_ZERO_ERROR))
            log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-1363", offsets, TRUE, U_ZERO_ERROR))
            log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
    }
    log_verbose("Testing for ISO-2022-jp\n");
    {
        UChar    sampleText[] =   { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};

        const uint8_t expected[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
                                    0x31,0x1A, 0x32};


        int32_t offsets[] = {0,0,0,0,0,1,1,2,2,2,2,3,5 };

        /*iso-2022-jp*/
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-jp", 0 , TRUE, U_ZERO_ERROR))
            log_err("u-> not match.\n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-jp", offsets , TRUE, U_ZERO_ERROR))
            log_err("u->  not match.\n");
    }
    log_verbose("Testing for ISO-2022-cn\n");
    {
        UChar    sampleText[] =   { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};

        const uint8_t expected[] = {
                                    0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B, 
                                    0x36, 0x21,
                                    0x1B, 0x24, 0x29, 0x47, 0x1B, 0x4E, 0x24, 0x22, 
                                    0x0f, 0x1A, 
                                    0x0f, 0x32
                                    };
        

        int32_t offsets[] = {
                                    0,    0,    0,    0,    0,    0,    0,      
                                    1,    1,
                                    2,    2,    2,    2,    2,    2,    2,    2,
                                    3,    3,
                                    5,    5,  };

        /*iso-2022-CN*/
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-cn", 0 , TRUE, U_ZERO_ERROR))
            log_err("u-> not match.\n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-cn", offsets , TRUE, U_ZERO_ERROR))
            log_err("u-> not match.\n");
    }
        log_verbose("Testing for ISO-2022-kr\n");
    {
        UChar    sampleText[] =   { 0x4e00,0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};

        const uint8_t expected[] = {0x1B, 0x24, 0x29, 0x43, 
                                    0x0E, 0x6C, 0x69, 
                                    0x0f, 0x1A, 
                                    0x0e, 0x6F, 0x4B, 
                                    0x0F, 0x31, 
                                    0x1A, 
                                    0x32 };        

        int32_t offsets[] = {-1, -1, -1, -1,
                              0, 0, 0, 
                              1, 1,  
                              3, 3, 3, 
                              4, 4, 
                              5, 
                              7,
                            };

        /*iso-2022-kr*/
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-kr", 0 , TRUE, U_ZERO_ERROR))
            log_err("u-> ibm-1362 [UCNV_DBCS] not match.\n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-kr", offsets , TRUE, U_ZERO_ERROR))
            log_err("u-> ibm-1362 [UCNV_DBCS] not match.\n");
    }
        log_verbose("Testing for HZ\n");
    {
        UChar    sampleText[] =   { 0x4e00, 0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};

        const uint8_t expected[] = {0x7E, 0x7B, 0x52, 0x3B,
                                    0x7E, 0x7D, 0x1A, 
                                    0x7E, 0x7B, 0x36, 0x21, 
                                    0x7E, 0x7D, 0x31, 
                                    0x1A,
                                    0x32 };


        int32_t offsets[] = {0,0,0,0,
                             1,1,1,
                             3,3,3,3,
                             4,4,4,
                             5,
                             7,};

        /*hz*/
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "HZ", 0 , TRUE, U_ZERO_ERROR))
            log_err("u->  not match.\n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "HZ", offsets , TRUE, U_ZERO_ERROR))
            log_err("u->  not match.\n");
    }
    /*UTF-8*/
     log_verbose("Testing for UTF8\n");
    {
        UChar    sampleText[] =   { 0x4e00, 0x0701, 0x0031, 0xbfc1, 0xd801, 0xdc01, 0x0032};
        int32_t offsets[]={0x00, 0x00, 0x00, 0x01, 0x01, 0x02,
                           0x03, 0x03, 0x03, 0x04, 0x04, 0x04,
                           0x04, 0x06 };
        const uint8_t expected[] = {0xe4, 0xb8, 0x80, 0xdc, 0x81, 0x31, 
            0xeb, 0xbf, 0x81, 0xF0, 0x90, 0x90, 0x81, 0x32};


        int32_t fromOffsets[] = { 0x0000, 0x0003, 0x0005, 0x0006, 0x0009, 0x0009, 0x000D }; 
        /*UTF-8*/
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
            expected, sizeof(expected), "UTF8", offsets, TRUE, U_ZERO_ERROR ))
            log_err("u-> UTF8 with offsets and flush true did not match.\n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
            expected, sizeof(expected), "UTF8", 0, TRUE, U_ZERO_ERROR ))
            log_err("u-> UTF8 with offsets and flush true did not match.\n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
            expected, sizeof(expected), "UTF8", offsets, FALSE, U_ZERO_ERROR ))
            log_err("u-> UTF8 with offsets and flush true did not match.\n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
            expected, sizeof(expected), "UTF8", 0, FALSE, U_ZERO_ERROR ))
            log_err("u-> UTF8 with offsets and flush true did not match.\n");

        if(!convertToU(expected, sizeof(expected), 
            sampleText, sizeof(sampleText)/sizeof(sampleText[0]), "UTF8", 0, TRUE, U_ZERO_ERROR ))
            log_err("UTF8 -> did not match.\n");
        if(!convertToU(expected, sizeof(expected), 
            sampleText, sizeof(sampleText)/sizeof(sampleText[0]), "UTF8", 0, FALSE, U_ZERO_ERROR ))
            log_err("UTF8 -> did not match.\n");
        if(!convertToU(expected, sizeof(expected), 
            sampleText, sizeof(sampleText)/sizeof(sampleText[0]), "UTF8", fromOffsets, TRUE, U_ZERO_ERROR ))
            log_err("UTF8 -> did not match.\n");
        if(!convertToU(expected, sizeof(expected), 
            sampleText, sizeof(sampleText)/sizeof(sampleText[0]), "UTF8", fromOffsets, FALSE, U_ZERO_ERROR ))
            log_err("UTF8 -> did not match.\n");

    }



}

/*test various error behaviours*/
static void TestErrorBehaviour(){
    log_verbose("Testing for SBCS and LATIN_1\n");
    {
        UChar    sampleText[] =   { 0x0031, 0xd801};
        UChar    sampleText2[] =   { 0x0031, 0xd801, 0x0032};
        const uint8_t expected[] =          { 0x31};
        const uint8_t expected2[] =         { 0x31, 0x1a, 0x32};

        /*SBCS*/
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-920", 0, TRUE, U_TRUNCATED_CHAR_FOUND))
            log_err("u-> ibm-920 [UCNV_SBCS] \n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-920", 0, FALSE, U_ZERO_ERROR))
            log_err("u-> ibm-920 [UCNV_SBCS] \n");
        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "ibm-920", 0, TRUE, U_ZERO_ERROR))
            log_err("u-> ibm-920 [UCNV_SBCS] did not match\n");


        /*LATIN_1*/
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "LATIN_1", 0, TRUE, U_TRUNCATED_CHAR_FOUND))
            log_err("u-> LATIN_1 is supposed to fail\n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "LATIN_1", 0, FALSE, U_ZERO_ERROR))
            log_err("u-> LATIN_1 is supposed to fail\n");

        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "LATIN_1", 0, TRUE, U_ZERO_ERROR))
            log_err("u-> LATIN_1 did not match\n");
    }


    log_verbose("Testing for DBCS and MBCS\n");
    {
        UChar    sampleText[]    = { 0x00a1, 0xd801};
        const uint8_t expected[] = { 0xa2, 0xae};
        int32_t offsets[]        = { 0x00, 0x00, 0x01, 0x01};

        UChar       sampleText2[] = { 0x00a1, 0xd801, 0x00a4};
        const uint8_t expected2[] = { 0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
        int32_t offsets2[]        = { 0x00, 0x00, 0x01, 0x01, 0x02, 0x02};

        UChar       sampleText3MBCS[] = { 0x0001, 0x00a4, 0xdc01};
        const uint8_t expected3MBCS[] = { 0x01, 0xa2, 0xb4, 0xa1, 0xe0};
        int32_t offsets3MBCS[]        = { 0x00, 0x01, 0x01, 0x02, 0x02};

        UChar       sampleText4MBCS[] = { 0x0061, 0x00a6, 0xdc01};
        const uint8_t expected4MBCS[] = { 0x61, 0x8f, 0xa2, 0xc3, 0xf4, 0xfe};
        int32_t offsets4MBCS[]        = { 0x00, 0x01, 0x01, 0x01, 0x02, 0x02 };





        /*DBCS*/
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-1362", 0, TRUE, U_TRUNCATED_CHAR_FOUND))
            log_err("u-> ibm-1362 [UCNV_DBCS] is supposed to fail\n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-1362", 0, FALSE, U_ZERO_ERROR))
            log_err("u-> ibm-1362 [UCNV_DBCS] is supposed to fail\n");
       
        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "ibm-1362", 0, TRUE, U_ZERO_ERROR))
            log_err("u-> ibm-1362 [UCNV_DBCS] did not match \n");
        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "ibm-1362", offsets2, TRUE, U_ZERO_ERROR))
            log_err("u-> ibm-1362 [UCNV_DBCS] did not match \n");

        /*MBCS*/
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-1363", 0, TRUE, U_TRUNCATED_CHAR_FOUND))
            log_err("u-> ibm-1363 [UCNV_MBCS] \n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-1363", 0, FALSE, U_ZERO_ERROR))
            log_err("u-> ibm-1363 [UCNV_MBCS] \n");

        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
            log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "ibm-1363", 0, FALSE, U_ZERO_ERROR))
            log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "ibm-1363", offsets2, FALSE, U_ZERO_ERROR))
            log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");

        if(!convertFromU(sampleText3MBCS, sizeof(sampleText3MBCS)/sizeof(sampleText3MBCS[0]),
                expected3MBCS, sizeof(expected3MBCS), "ibm-1363", offsets3MBCS, TRUE, U_ZERO_ERROR))
            log_err("u-> ibm-1363 [UCNV_MBCS] \n");
        if(!convertFromU(sampleText3MBCS, sizeof(sampleText3MBCS)/sizeof(sampleText3MBCS[0]),
                expected3MBCS, sizeof(expected3MBCS), "ibm-1363", offsets3MBCS, FALSE, U_ZERO_ERROR))
            log_err("u-> ibm-1363 [UCNV_MBCS] \n");

        if(!convertFromU(sampleText4MBCS, sizeof(sampleText4MBCS)/sizeof(sampleText4MBCS[0]),
                expected4MBCS, sizeof(expected4MBCS), "euc-jp", offsets4MBCS, TRUE, U_ZERO_ERROR))
            log_err("u-> euc-jp [UCNV_MBCS] \n");
        if(!convertFromU(sampleText4MBCS, sizeof(sampleText4MBCS)/sizeof(sampleText4MBCS[0]),
                expected4MBCS, sizeof(expected4MBCS), "euc-jp", offsets4MBCS, FALSE, U_ZERO_ERROR))
            log_err("u-> euc-jp [UCNV_MBCS] \n");
    }
    /*iso-2022-jp*/
    log_verbose("Testing for iso-2022-jp\n");
    {
        UChar    sampleText[]    = { 0x0031, 0xd801};
        const uint8_t expected[] = {  0x31};
        int32_t offsets[]        = { 0x00};

        UChar       sampleText2[] = { 0x0031, 0xd801, 0x0032};
        const uint8_t expected2[] = {  0x31,0x1A,0x32};
        int32_t offsets2[]        = { 0x00,0x01,0x02};

        UChar       sampleText3MBCS[] = { 0x3000, 0x0050, 0xdc01,0x3001};
        const uint8_t expected3MBCS[] = { 0x1B, 0x24, 0x42, 0x21, 0x21, 0x1B, 0x28, 0x42, 0x50, 0x1A, 0x1B, 0x24, 0x42, 0x21, 0x22,};
        int32_t offsets3MBCS[]        = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03,};

        UChar       sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
        const uint8_t expected4MBCS[] = { 0x61, 0x1b, 0x24, 0x42, 0x30, 0x6c,0x1b,0x28,0x42,0x1a};
        int32_t offsets4MBCS[]        = { 0x00, 0x01, 0x01 ,0x01, 0x01, 0x01,0x02,0x02,0x02,0x02 };
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-jp", offsets, TRUE, U_TRUNCATED_CHAR_FOUND))
            log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-jp", offsets, FALSE, U_ZERO_ERROR))
            log_err("u-> ibm-1363 [UCNV_MBCS] \n");

        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "iso-2022-jp", offsets2, TRUE, U_ZERO_ERROR))
            log_err("u->iso-2022-jp[UCNV_DBCS] did not match\n");
        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "iso-2022-jp", offsets2, FALSE, U_ZERO_ERROR))
            log_err("u-> iso-2022-jp [UCNV_DBCS] did not match\n");
        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "iso-2022-jp", offsets2, FALSE, U_ZERO_ERROR))
            log_err("u-> iso-2022-jp [UCNV_DBCS] did not match\n");

        if(!convertFromU(sampleText3MBCS, sizeof(sampleText3MBCS)/sizeof(sampleText3MBCS[0]),
                expected3MBCS, sizeof(expected3MBCS), "iso-2022-jp", offsets3MBCS, TRUE, U_ZERO_ERROR))
            log_err("u->iso-2022-jp [UCNV_MBCS] \n");
        if(!convertFromU(sampleText3MBCS, sizeof(sampleText3MBCS)/sizeof(sampleText3MBCS[0]),
                expected3MBCS, sizeof(expected3MBCS), "iso-2022-jp", offsets3MBCS, FALSE, U_ZERO_ERROR))
            log_err("u-> iso-2022-jp[UCNV_MBCS] \n");

        if(!convertFromU(sampleText4MBCS, sizeof(sampleText4MBCS)/sizeof(sampleText4MBCS[0]),
                expected4MBCS, sizeof(expected4MBCS), "iso-2022-jp", offsets4MBCS, TRUE, U_ZERO_ERROR))
            log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
        if(!convertFromU(sampleText4MBCS, sizeof(sampleText4MBCS)/sizeof(sampleText4MBCS[0]),
                expected4MBCS, sizeof(expected4MBCS), "iso-2022-jp", offsets4MBCS, FALSE, U_ZERO_ERROR))
            log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
    }
    /*iso-2022-cn*/
    log_verbose("Testing for iso-2022-cn\n");
    {
        UChar    sampleText[]    = { 0x0031, 0xd801};
        const uint8_t expected[] = { 0x0f, 0x31};
        int32_t offsets[]        = { 0x00, 0x00};

        UChar       sampleText2[] = { 0x0031, 0xd801, 0x0032};
        const uint8_t expected2[] = { 0x0f, 0x31, 0x1A,0x0f,0x32};
        int32_t offsets2[]        = { 0x00, 0x00, 0x01,0x02,0x02};

        UChar       sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
        const uint8_t expected3MBCS[] = {0x0f, 0x51, 0x50, 0x1A};
        int32_t offsets3MBCS[]        = { 0x00, 0x00, 0x01, 0x02 };

        UChar       sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
        const uint8_t expected4MBCS[] = { 0x0f, 0x61, 0x1b, 0x24, 0x29, 0x41, 0x0e, 0x52, 0x3b, 0x0f, 0x1a };
        int32_t offsets4MBCS[]        = { 0x00, 0x00, 0x01, 0x01 ,0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02 };
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-cn", offsets, TRUE, U_TRUNCATED_CHAR_FOUND))
            log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-cn", offsets, FALSE, U_ZERO_ERROR))
            log_err("u-> ibm-1363 [UCNV_MBCS] \n");

        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "iso-2022-cn", offsets2, TRUE, U_ZERO_ERROR))
            log_err("u->iso-2022-cn[UCNV_DBCS] did not match\n");
        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "iso-2022-cn", offsets2, FALSE, U_ZERO_ERROR))
            log_err("u-> iso-2022-cn [UCNV_DBCS] did not match\n");
        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "iso-2022-cn", offsets2, FALSE, U_ZERO_ERROR))
            log_err("u-> iso-2022-cn [UCNV_DBCS] did not match\n");

        if(!convertFromU(sampleText3MBCS, sizeof(sampleText3MBCS)/sizeof(sampleText3MBCS[0]),
                expected3MBCS, sizeof(expected3MBCS), "iso-2022-cn", offsets3MBCS, TRUE, U_ZERO_ERROR))
            log_err("u->iso-2022-cn [UCNV_MBCS] \n");
        if(!convertFromU(sampleText3MBCS, sizeof(sampleText3MBCS)/sizeof(sampleText3MBCS[0]),
                expected3MBCS, sizeof(expected3MBCS), "iso-2022-cn", offsets3MBCS, FALSE, U_ZERO_ERROR))
            log_err("u-> iso-2022-cn[UCNV_MBCS] \n");

        if(!convertFromU(sampleText4MBCS, sizeof(sampleText4MBCS)/sizeof(sampleText4MBCS[0]),
                expected4MBCS, sizeof(expected4MBCS), "iso-2022-cn", offsets4MBCS, TRUE, U_ZERO_ERROR))
            log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
        if(!convertFromU(sampleText4MBCS, sizeof(sampleText4MBCS)/sizeof(sampleText4MBCS[0]),
                expected4MBCS, sizeof(expected4MBCS), "iso-2022-cn", offsets4MBCS, FALSE, U_ZERO_ERROR))
            log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
    }
    /*iso-2022-kr*/
    log_verbose("Testing for iso-2022-kr\n");
    {
        UChar    sampleText[]    = { 0x0031, 0xd801};
        const uint8_t expected[] = { 0x1b, 0x24, 0x29, 0x43, 0x31};
        int32_t offsets[]        = { -1,   -1,   -1,   -1,   0x00};

        UChar       sampleText2[] = { 0x0031, 0xd801, 0x0032};
        const uint8_t expected2[] = { 0x1b, 0x24, 0x29, 0x43, 0x31, 0x1A, 0x32};
        int32_t offsets2[]        = { -1,   -1,   -1,   -1,   0x00, 0x01, 0x02};

        UChar       sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
        const uint8_t expected3MBCS[] = { 0x1b, 0x24, 0x29, 0x43,  0x51, 0x50, 0x1A };
        int32_t offsets3MBCS[]        = { -1,   -1,   -1,   -1,    0x00, 0x01, 0x02, 0x02 };

        UChar       sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01,0x4e00};
        const uint8_t expected4MBCS[] = { 0x1b, 0x24, 0x29, 0x43,
                                          0x61, 
                                          0x0e, 0x6c, 0x69, 
                                          0x0f, 0x1a,
                                          0x0e, 0x6c, 0x69,};
        int32_t offsets4MBCS[]        = { -1,   -1,   -1,   -1, 0x00, 0x01 ,0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x03 };
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-kr", offsets, TRUE, U_TRUNCATED_CHAR_FOUND))
            log_err("u-> iso-2022-kr [UCNV_MBCS] \n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-kr", offsets, FALSE, U_ZERO_ERROR))
            log_err("u-> ibm-1363 [UCNV_MBCS] \n");

        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "iso-2022-kr", offsets2, TRUE, U_ZERO_ERROR))
            log_err("u->iso-2022-kr[UCNV_DBCS] did not match\n");
        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "iso-2022-kr", offsets2, FALSE, U_ZERO_ERROR))
            log_err("u-> iso-2022-kr [UCNV_DBCS] did not match\n");
        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "iso-2022-kr", offsets2, FALSE, U_ZERO_ERROR))
            log_err("u-> iso-2022-kr [UCNV_DBCS] did not match\n");

        if(!convertFromU(sampleText3MBCS, sizeof(sampleText3MBCS)/sizeof(sampleText3MBCS[0]),
                expected3MBCS, sizeof(expected3MBCS), "iso-2022-kr", offsets3MBCS, TRUE, U_ZERO_ERROR))
            log_err("u->iso-2022-kr [UCNV_MBCS] \n");
        if(!convertFromU(sampleText3MBCS, sizeof(sampleText3MBCS)/sizeof(sampleText3MBCS[0]),
                expected3MBCS, sizeof(expected3MBCS), "iso-2022-kr", offsets3MBCS, FALSE, U_ZERO_ERROR))
            log_err("u-> iso-2022-kr[UCNV_MBCS] \n");

        if(!convertFromU(sampleText4MBCS, sizeof(sampleText4MBCS)/sizeof(sampleText4MBCS[0]),
                expected4MBCS, sizeof(expected4MBCS), "iso-2022-kr", offsets4MBCS, TRUE, U_ZERO_ERROR))
            log_err("u-> iso-2022-kr [UCNV_MBCS] \n");
        if(!convertFromU(sampleText4MBCS, sizeof(sampleText4MBCS)/sizeof(sampleText4MBCS[0]),
                expected4MBCS, sizeof(expected4MBCS), "iso-2022-kr", offsets4MBCS, FALSE, U_ZERO_ERROR))
            log_err("u-> iso-2022-kr [UCNV_MBCS] \n");
    }

    /*HZ*/
    log_verbose("Testing for HZ\n");
    {
        UChar    sampleText[]    = { 0x0031, 0xd801};
        const uint8_t expected[] = { 0x7e, 0x7d, 0x31};
        int32_t offsets[]        = { 0x00, 0x00, 0x00};

        UChar       sampleText2[] = { 0x0031, 0xd801, 0x0032};
        const uint8_t expected2[] = { 0x7e, 0x7d, 0x31,  0x1A,  0x32 };
        int32_t offsets2[]        = { 0x00, 0x00, 0x00, 0x01,  0x02 };

        UChar       sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
        const uint8_t expected3MBCS[] = { 0x7e, 0x7d, 0x51, 0x50,  0x1A };
        int32_t offsets3MBCS[]        = { 0x00, 0x00, 0x00, 0x01, 0x02};

        UChar       sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
        const uint8_t expected4MBCS[] = { 0x7e, 0x7d, 0x61, 0x7e, 0x7b, 0x52, 0x3b, 0x7e, 0x7d, 0x1a };
        int32_t offsets4MBCS[]        = { 0x00, 0x00, 0x00, 0x01, 0x01, 0x01 ,0x01, 0x02, 0x02, 0x02 };
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "HZ", offsets, TRUE, U_TRUNCATED_CHAR_FOUND))
            log_err("u-> HZ [UCNV_MBCS] \n");
        if(!convertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "HZ", offsets, FALSE, U_ZERO_ERROR))
            log_err("u-> ibm-1363 [UCNV_MBCS] \n");

        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "HZ", offsets2, TRUE, U_ZERO_ERROR))
            log_err("u->HZ[UCNV_DBCS] did not match\n");
        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "HZ", offsets2, FALSE, U_ZERO_ERROR))
            log_err("u-> HZ [UCNV_DBCS] did not match\n");
        if(!convertFromU(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
                expected2, sizeof(expected2), "HZ", offsets2, FALSE, U_ZERO_ERROR))
            log_err("u-> HZ [UCNV_DBCS] did not match\n");

        if(!convertFromU(sampleText3MBCS, sizeof(sampleText3MBCS)/sizeof(sampleText3MBCS[0]),
                expected3MBCS, sizeof(expected3MBCS), "HZ", offsets3MBCS, TRUE, U_ZERO_ERROR))
            log_err("u->HZ [UCNV_MBCS] \n");
        if(!convertFromU(sampleText3MBCS, sizeof(sampleText3MBCS)/sizeof(sampleText3MBCS[0]),
                expected3MBCS, sizeof(expected3MBCS), "HZ", offsets3MBCS, FALSE, U_ZERO_ERROR))
            log_err("u-> HZ[UCNV_MBCS] \n");

        if(!convertFromU(sampleText4MBCS, sizeof(sampleText4MBCS)/sizeof(sampleText4MBCS[0]),
                expected4MBCS, sizeof(expected4MBCS), "HZ", offsets4MBCS, TRUE, U_ZERO_ERROR))
            log_err("u-> HZ [UCNV_MBCS] \n");
        if(!convertFromU(sampleText4MBCS, sizeof(sampleText4MBCS)/sizeof(sampleText4MBCS[0]),
                expected4MBCS, sizeof(expected4MBCS), "HZ", offsets4MBCS, FALSE, U_ZERO_ERROR))
            log_err("u-> HZ [UCNV_MBCS] \n");
    }


}

/*test different convertToUnicode error behaviours*/
static void TestToUnicodeErrorBehaviour()
{
    log_verbose("Testing error conditions for DBCS\n");
    {
        uint8_t sampleText[] = { 0xa2, 0xae, 0x03, 0x04};
        const UChar expected[] = { 0x00a1 };
        
        uint8_t sampleText2[] = { 0xa2, 0xae, 0xa2};
        const UChar expected2[] = { 0x00a1 };

        if(!convertToU(sampleText, sizeof(sampleText), 
                expected, sizeof(expected)/sizeof(expected[0]), "ibm-1362", 0, TRUE, U_ZERO_ERROR ))
            log_err("DBCS (ibm-1362)->Unicode  did not match.\n");
        if(!convertToU(sampleText, sizeof(sampleText), 
                expected, sizeof(expected)/sizeof(expected[0]), "ibm-1362", 0, FALSE, U_ZERO_ERROR ))
            log_err("DBCS (ibm-1362)->Unicode  with flush = false did not match.\n");
       
        if(!convertToU(sampleText2, sizeof(sampleText2), 
                expected2, sizeof(expected2)/sizeof(expected2[0]), "ibm-1362", 0, TRUE, U_TRUNCATED_CHAR_FOUND ))
            log_err("DBCS (ibm-1362)->Unicode with TRUNCATED CHARACTER  did not match.\n");
       
    
    }
    log_verbose("Testing error conditions for SBCS\n");
    {
        uint8_t sampleText[] = { 0xa2, 0xFF};
        const UChar expected[] = { 0x00c2 };

      /*  uint8_t sampleText2[] = { 0xa2, 0x70 };
        const UChar expected2[] = { 0x0073 };*/

        if(!convertToU(sampleText, sizeof(sampleText), 
                expected, sizeof(expected)/sizeof(expected[0]), "ibm-1051", 0, TRUE, U_ZERO_ERROR ))
            log_err("SBCS (ibm-1051)->Unicode  did not match.\n");
        if(!convertToU(sampleText, sizeof(sampleText), 
                expected, sizeof(expected)/sizeof(expected[0]), "ibm-1051", 0, FALSE, U_ZERO_ERROR ))
            log_err("SBCS (ibm-1051)->Unicode  with flush = false did not match.\n");

    }
    log_verbose("Testing error conditions for UTF8\n");
    {
        const uint8_t sampleText[] = { 0x31, 0xe4, 0xba, 0x8c, 0xe4, 0xb8 };
        UChar    expectedUTF8[] = {  0x0031, 0x4e8c};
        int32_t offsets[] = {   0x0000, 0x0001};

        const uint8_t sampleText2[] = { 0x31, 0xff, 0xe4, 0xba, 0x8c, 
            0xe0, 0x80, 0x61};
        UChar    expected2UTF8[] = {  0x0031, 0xfffd, 0x4e8c, 0xfffd, 0x0061};
        int32_t offsets2[] = {   0x0000, 0x0001, 0x0002, 0x0005, 0x0007};

        const uint8_t sampleText3[] = { 0x31, 0xfb, 0xbf, 0xbf, 0xbf, 0xbf, 
            0x61};
        UChar    expected3UTF8[] = {  0x0031, 0xfffd, 0x0061};
        int32_t offsets3[] = {   0x0000, 0x0001, 0x0006};

        if(!convertToU(sampleText, sizeof(sampleText), 
                expectedUTF8, sizeof(expectedUTF8)/sizeof(expectedUTF8[0]), "utf-8", 0, TRUE, U_TRUNCATED_CHAR_FOUND ))
            log_err("utf-8->Unicode  did not match.\n");
        if(!convertToU(sampleText, sizeof(sampleText), 
                expectedUTF8, sizeof(expectedUTF8)/sizeof(expectedUTF8[0]), "utf-8", 0, FALSE, U_ZERO_ERROR ))
            log_err("utf-8->Unicode  did not match.\n");
        if(!convertToU(sampleText, sizeof(sampleText), 
                expectedUTF8, sizeof(expectedUTF8)/sizeof(expectedUTF8[0]), "utf-8", offsets, TRUE, U_TRUNCATED_CHAR_FOUND ))
            log_err("utf-8->Unicode  did not match.\n");
        if(!convertToU(sampleText, sizeof(sampleText), 
                expectedUTF8, sizeof(expectedUTF8)/sizeof(expectedUTF8[0]), "utf-8", offsets, FALSE, U_ZERO_ERROR ))
            log_err("utf-8->Unicode  did not match.\n");

        if(!convertToU(sampleText2, sizeof(sampleText2), 
                expected2UTF8, sizeof(expected2UTF8)/sizeof(expected2UTF8[0]), "utf-8", 0, TRUE, U_ZERO_ERROR ))
            log_err("utf-8->Unicode  did not match.\n");
        if(!convertToU(sampleText2, sizeof(sampleText2), 
                expected2UTF8, sizeof(expected2UTF8)/sizeof(expected2UTF8[0]), "utf-8", 0, FALSE, U_ZERO_ERROR ))
            log_err("utf-8->Unicode  did not match.\n");
        if(!convertToU(sampleText2, sizeof(sampleText2), 
                expected2UTF8, sizeof(expected2UTF8)/sizeof(expected2UTF8[0]), "utf-8", offsets2, TRUE, U_ZERO_ERROR ))
            log_err("utf-8->Unicode  did not match.\n");
        if(!convertToU(sampleText2, sizeof(sampleText2), 
                expected2UTF8, sizeof(expected2UTF8)/sizeof(expected2UTF8[0]), "utf-8", offsets2, FALSE, U_ZERO_ERROR ))
            log_err("utf-8->Unicode  did not match.\n");

        if(!convertToU(sampleText3, sizeof(sampleText3), 
                expected3UTF8, sizeof(expected3UTF8)/sizeof(expected3UTF8[0]), "utf-8", offsets3, TRUE, U_ZERO_ERROR ))
            log_err("utf-8->Unicode  did not match.\n");
        if(!convertToU(sampleText3, sizeof(sampleText3), 
                expected3UTF8, sizeof(expected3UTF8)/sizeof(expected3UTF8[0]), "utf-8", offsets3, FALSE, U_ZERO_ERROR ))
            log_err("utf-8->Unicode  did not match with flush false.\n");

    }

}

static void TestGetNextErrorBehaviour(){
   /*Test for unassigned character*/
    static const char input1[]={ 0x70 };
    const char* source=(const char*)input1;
    UErrorCode err=U_ZERO_ERROR;
    UChar32 c=0;
    UConverter *cnv=ucnv_open("ibm-1159", &err);
    if(U_FAILURE(err)) {
        log_err("Unable to open a SBCS(ibm-1159) converter: %s\n", u_errorName(err));
        return;
    }
    c=ucnv_getNextUChar(cnv, &source, source+sizeof(source), &err);
    if(err != U_INVALID_CHAR_FOUND && c!=0xfffd){
        log_err("FAIL in TestGetNextErrorBehaviour(unassigned): Expected: U_INVALID_CHAR_ERROR or 0xfffd ----Got:%s and 0x%lx\n",  myErrorName(err), c);
    }
    ucnv_close(cnv);



}

/*Regression test for utf8 converter*/
static void TestRegressionUTF8(){
    uint8_t *buffer=0;
    UChar32 c;
    char *targ;
    char *targetLimit;
    UChar *source;
    const UChar *src=0;
    const UChar *sourceLimit=0;
    UChar *extractedTargetBuffer=0;
    UChar *extractedTarget=0;
    UChar *target=0;
    UChar *limit=0;
    int32_t count=0;

    int32_t offset=0;
    int32_t i=0;
    UErrorCode status=U_ZERO_ERROR;
    UConverter *conv=ucnv_open("utf8", &status);
    if(U_FAILURE(status)) {
        log_err("Unable to open a utf-8 converter: %s\n", u_errorName(status));
    }
    source=(UChar*)malloc(sizeof(UChar) * TEST_BUFFER_SIZE);
    extractedTargetBuffer=(UChar*)malloc(sizeof(UChar) * TEST_BUFFER_SIZE);
    buffer=(uint8_t*)malloc(sizeof(uint8_t) * TEST_BUFFER_SIZE);

    for(i =0; i< TEST_BUFFER_SIZE; i++){
        buffer[i]=0xF0;
        source[i]=(UChar)0xFFFE;
        extractedTargetBuffer[i]=(UChar)0xFFFE;
    }
    for(c=0x0000; c <= UNICODE_LIMIT; c++){
        UTF16_APPEND_CHAR_SAFE(source, offset, TEST_BUFFER_SIZE, c);
        count++;
    }
    log_verbose("\nThe the No: of code units=%ld,  The no: of Code Points=%d\n", offset, count);
    
    src=source;
    sourceLimit=src+(offset);
    targ=(char *)buffer;
    targetLimit=targ+(TEST_BUFFER_SIZE);
    ucnv_fromUnicode (conv,
                  &targ,
                  targetLimit,
                  &src,
                  sourceLimit,
                  NULL,
                  TRUE, 
                  &status);
    if(U_FAILURE(status)){
        log_err("FAILED: error= %s at offset=0x%04X and target=0x%02X\n", myErrorName(status), (UNICODE_LIMIT-(sourceLimit-src)/sizeof(UChar)) , *targ);
    }

    log_verbose("The No: of bytes in target buffer =%ld\n", (const char *)(targ-(const char *)buffer));

    targetLimit=targ;
    targ=(char *)buffer;

    extractedTarget=extractedTargetBuffer;
    limit=extractedTarget+TEST_BUFFER_SIZE;
    ucnv_toUnicode(conv, 
                   &extractedTarget, 
                   limit, 
                   (const char **) &targ, 
                   targetLimit, 
                   NULL, 
                   TRUE, 
                   &status);

    /*if(memcmp(source, extractedTarget, extractedTarget-extractedTargetBuffer) != 0){
        log_err("FAILED\n");
    }*/
    src=source;
    target=extractedTargetBuffer;
    while(target < extractedTarget){
        if(*src != *target){
            log_err("FAILED: comparision failed at source=0x%04X, extracted=0x%04X\n", *src, *target);
            break;
        }
        src++;
        target++;

    }
    if((extractedTarget-extractedTargetBuffer) != (sourceLimit-source)){
        log_err("The conversion didn't go through the whole range: Expected= %d, Got=%d\n", (sourceLimit-source), (extractedTarget-extractedTargetBuffer));

    }
    ucnv_close(conv);
    free(source);
    free(extractedTargetBuffer);
    free(buffer);

}

static void TestRegressionUTF32(){
    uint32_t *buffer=0;
    UChar32 c;
    char *targ;
    char *targetLimit;
    UChar *source;
    const UChar *src=0;
    const UChar *sourceLimit=0;
    UChar *extractedTargetBuffer=0;
    UChar *extractedTarget=0;
    UChar *target=0;
    UChar *limit=0;
    int32_t count=0;

    int32_t offset=0;
    int32_t i=0;
    UErrorCode status=U_ZERO_ERROR;
    UConverter *conv=ucnv_open("utf32", &status);
    if(U_FAILURE(status)) {
        log_err("Unable to open a utf-32 converter: %s\n", u_errorName(status));
    }
    source=(UChar*)malloc(sizeof(UChar) * TEST_BUFFER_SIZE);
    extractedTargetBuffer=(UChar*)malloc(sizeof(UChar) * TEST_BUFFER_SIZE);
    buffer=(uint32_t*)malloc(sizeof(uint32_t) * TEST_BUFFER_SIZE);

    for(i =0; i< TEST_BUFFER_SIZE; i++){
        buffer[i]=0xF0;
        source[i]=(UChar)0xFFFE;
        extractedTargetBuffer[i]=(UChar)0xFFFE;
    }
    for(c=0x0000; c <= UNICODE_LIMIT; c++){
        UTF16_APPEND_CHAR_SAFE(source, offset, TEST_BUFFER_SIZE, c);
        count++;
    }
    log_verbose("\nThe the No: of code units=%ld,  The no: of Code Points=%d\n", offset, count);
    
    src=source;
    sourceLimit=src+(offset);
    targ=(char *)buffer;
    targetLimit=targ+(TEST_BUFFER_SIZE);
    ucnv_fromUnicode (conv,
                  &targ,
                  targetLimit,
                  &src,
                  sourceLimit,
                  NULL,
                  TRUE, 
                  &status);
    if(U_FAILURE(status)){
        log_err("FAILED: error= %s at offset=0x%04X and target=0x%02X\n", myErrorName(status), (UNICODE_LIMIT-(sourceLimit-src)/sizeof(UChar)) , *targ);
    }

    log_verbose("The No: of bytes in target buffer =%ld\n", (const char *)(targ-(const char *)buffer));

    for(c=0x0000; c <= UNICODE_LIMIT && buffer[c] == c; c++){
    }

    if (c == 0xDBFF && buffer[c] == 0x10FC00) {
        /* skip the surrogate and compare with rest of UTF-32 skipping DC00 in UTF-16 */
        for(c++; c < UNICODE_LIMIT && buffer[c] == c+1; c++){
        }
        if (c < UNICODE_LIMIT) {
            log_err("FAILED: at offset=0x%04X and target=0x%02X\n", c, buffer[c]);
        }
    }
    else {
        log_err("FAILED: surrogate not found at offset=0x%04X and target=0x%02X\n", c, buffer[c]);
    }

    targetLimit=targ;
    targ=(char *)buffer;

    extractedTarget=extractedTargetBuffer;
    limit=extractedTarget+TEST_BUFFER_SIZE;
    ucnv_toUnicode(conv, 
                   &extractedTarget, 
                   limit, 
                   (const char **) &targ, 
                   targetLimit, 
                   NULL, 
                   TRUE, 
                   &status);

    /*if(memcmp(source, extractedTarget, extractedTarget-extractedTargetBuffer) != 0){
        log_err("FAILED\n");
    }*/
    src=source;
    target=extractedTargetBuffer;
    while(target < extractedTarget){
        if(*src != *target){
            log_err("FAILED: comparision failed at source=0x%04X, extracted=0x%04X\n", *src, *target);
            break;
        }
        src++;
        target++;

    }
    if((extractedTarget-extractedTargetBuffer) != (sourceLimit-source)){
        log_err("The conversion didn't go through the whole range: Expected= %d, Got=%d\n", (sourceLimit-source), (extractedTarget-extractedTargetBuffer));

    }
    ucnv_close(conv);
    free(source);
    free(extractedTargetBuffer);
    free(buffer);

}

/*Walk through the available converters*/
static void TestAvailableConverters(){
    UErrorCode status=U_ZERO_ERROR;
    UConverter *conv=NULL;
    int32_t i=0;
    for(i=0; i < ucnv_countAvailable(); i++){
        status=U_ZERO_ERROR;
        conv=ucnv_open(ucnv_getAvailableName(i), &status);
        if(U_FAILURE(status)){
            log_err("ERROR: converter creation failed. Failure in alias table or the data table for \n converter=%s. Error=%s\n", 
                        ucnv_getAvailableName(i), myErrorName(status));
            continue;
        }
        ucnv_close(conv);
    }

}

static void TestFlushInternalBuffer(){
    TestWithBufferSize(MAX_LENGTH, 1);
    TestWithBufferSize(1, 1);
    TestWithBufferSize(1, MAX_LENGTH);
    TestWithBufferSize(MAX_LENGTH, MAX_LENGTH);
}

static void TestWithBufferSize(int32_t insize, int32_t outsize){

    gInBufferSize =insize;
    gOutBufferSize = outsize;

     log_verbose("Testing fromUnicode for UTF-8 with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
    {
        UChar    sampleText[] = 
            { 0x0031, 0x0032, 0x0033, 0x0000, 0x4e00, 0x4e8c, 0x4e09,  0x002E  };
        const uint8_t expectedUTF8[] = 
            { 0x31, 0x32, 0x33, 0x00, 0xe4, 0xb8, 0x80, 0xe4, 0xba, 0x8c, 0xe4, 0xb8, 0x89, 0x2E };
        int32_t  toUTF8Offs[] = 
            { 0x00, 0x01, 0x02, 0x03, 0x04, 0x04, 0x04, 0x05, 0x05, 0x05, 0x06, 0x06, 0x06, 0x07};
       /* int32_t fmUTF8Offs[] = 
            { 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0007, 0x000a, 0x000d };*/

        /*UTF-8*/
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
            expectedUTF8, sizeof(expectedUTF8), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE, toUTF8Offs ,FALSE))
             log_err("u-> UTF8 did not match.\n");
    }

     log_verbose("Testing fromUnicode with UCNV_FROM_U_CALLBACK_ESCAPE  \n");
    {
        UChar inputTest[] = { 0x0061, 0xd801, 0xdc01, 0xd801, 0x0061 };
        const uint8_t toIBM943[]= { 0x61, 
            0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
            0x25, 0x55, 0x44, 0x43, 0x30, 0x31,
            0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
            0x61 };
        int32_t offset[]= {0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 4};

        if(!testConvertFromU(inputTest, sizeof(inputTest)/sizeof(inputTest[0]),
                toIBM943, sizeof(toIBM943), "ibm-943",
                (UConverterFromUCallback)UCNV_FROM_U_CALLBACK_ESCAPE, offset,FALSE))
            log_err("u-> ibm-943 with subst with value did not match.\n");
    }

     log_verbose("Testing fromUnicode for UTF-8 with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
    {
        const uint8_t sampleText1[] = { 0x31, 0xe4, 0xba, 0x8c, 
            0xe0, 0x80,  0x61};
        UChar    expected1[] = {  0x0031, 0x4e8c, 0xfffd, 0x0061};
        int32_t offsets1[] = {   0x0000, 0x0001, 0x0004, 0x0006};

        if(!testConvertToU(sampleText1, sizeof(sampleText1),
                 expected1, sizeof(expected1)/sizeof(expected1[0]),"utf8", UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets1,FALSE))
            log_err("utf8->u with substitute did not match.\n");;
    }


    log_verbose("Testing toUnicode with UCNV_TO_U_CALLBACK_ESCAPE \n");
    /*to Unicode*/
    {
        const uint8_t sampleTxtToU[]= { 0x00, 0x9f, 0xaf, 
            0x81, 0xad, /*unassigned*/
            0x89, 0xd3 };
        UChar IBM_943toUnicode[] = { 0x0000, 0x6D63, 
            0x25, 0x58, 0x38, 0x31, 0x25, 0x58, 0x41, 0x44,
            0x7B87};
        int32_t  fromIBM943Offs [] =    { 0, 1, 3, 3, 3, 3, 3, 3, 3, 3, 5};

        if(!testConvertToU(sampleTxtToU, sizeof(sampleTxtToU),
                 IBM_943toUnicode, sizeof(IBM_943toUnicode)/sizeof(IBM_943toUnicode[0]),"ibm-943",
                (UConverterToUCallback)UCNV_TO_U_CALLBACK_ESCAPE, fromIBM943Offs,FALSE))
            log_err("ibm-943->u with substitute with value did not match.\n");

    }

}

static UBool convertFromU( const UChar *source, int sourceLen,  const uint8_t *expect, int expectLen, 
                const char *codepage, int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus)
{

    int32_t i=0;
    uint8_t *p=0;
    const UChar *src;
    uint8_t buffer[MAX_LENGTH];
    int32_t offsetBuffer[MAX_LENGTH];
    int32_t *offs=0;
    uint8_t *targ;
    uint8_t *targetLimit;
    UChar *sourceLimit=0;
    UErrorCode status = U_ZERO_ERROR;
    UConverter *conv = 0;
    conv = ucnv_open(codepage, &status);
    if(U_FAILURE(status))
    {
        log_err("Couldn't open converter %s\n",codepage);    
        return FALSE;
    }
    log_verbose("Converter %s opened..\n", ucnv_getName(conv, &status));

    for(i=0; i<MAX_LENGTH; i++){
        buffer[i]=0xF0;
        offsetBuffer[i]=0xFF;
    }

    src=source;
    sourceLimit=(UChar*)src+(sourceLen);
    targ=buffer;
    targetLimit=targ+MAX_LENGTH;
    offs=offsetBuffer;
    ucnv_fromUnicode (conv,
                  (char **)&targ,
                  (const char *)targetLimit,
                  &src,
                  sourceLimit,
                  expectOffsets ? offs : NULL,
                  doFlush, 
                  &status);
    ucnv_close(conv);
    if(status != expectedStatus){
          log_err("ucnv_fromUnicode() failed for codepage=%s. Error =%s Expected=%s\n", codepage, myErrorName(status), myErrorName(expectedStatus));
          return FALSE;
    }

    log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
        sourceLen, targ-buffer);

    if(expectLen != targ-buffer)
    {
        log_err("Expected %d chars out, got %d FROM Unicode to %s\n", expectLen, targ-buffer, codepage);
        log_verbose("Expected %d chars out, got %d FROM Unicode to %s\n", expectLen, targ-buffer, codepage);
        printSeqErr((const unsigned char *)buffer, targ-buffer);
        printSeqErr((const unsigned char*)expect, expectLen);
        return FALSE;
    }

    if(memcmp(buffer, expect, expectLen)){
        log_err("String does not match. FROM Unicode to codePage%s\n", codepage);
        printf("\nGot:");
        printSeqErr((const unsigned char *)buffer, expectLen);
        printf("\nExpected:");
        printSeqErr((const unsigned char *)expect, expectLen);
        return FALSE;
    }
    else {    
        log_verbose("Matches!\n");
    }

    if (expectOffsets != 0){
        log_verbose("comparing %d offsets..\n", targ-buffer);
        if(memcmp(offsetBuffer,expectOffsets,(targ-buffer) * sizeof(int32_t) )){
            log_err("did not get the expected offsets. for FROM Unicode to %s\n", codepage);
            printf("\nGot  : ");
            printSeqErr((const unsigned char*)buffer, targ-buffer);
            for(p=buffer;p<targ;p++)
                printf("%d, ", offsetBuffer[p-buffer]); 
            printf("\nExpected: ");
            for(i=0; i< (targ-buffer); i++)
                printf("%d,", expectOffsets[i]);
        }
    }

    return TRUE;    
}


static UBool convertToU( const uint8_t *source, int sourceLen, const UChar *expect, int expectLen, 
               const char *codepage, int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus)
{
    UErrorCode status = U_ZERO_ERROR;
    UConverter *conv = 0;
    int32_t i=0;
    UChar *p=0;
    const uint8_t* src;
    UChar buffer[MAX_LENGTH];
    int32_t offsetBuffer[MAX_LENGTH];
    int32_t *offs=0;
    UChar *targ;
    UChar *targetLimit;
    uint8_t *sourceLimit=0;



    conv = ucnv_open(codepage, &status);
    if(U_FAILURE(status))
    {
        log_err("Couldn't open converter %s\n",codepage);    
        return FALSE;
    }
    log_verbose("Converter %s opened..\n", ucnv_getName(conv, &status));



    for(i=0; i<MAX_LENGTH; i++){
        buffer[i]=0xFFFE;
        offsetBuffer[i]=-1;
    }

    src=source;
    sourceLimit=(uint8_t*)(src+(sourceLen));
    targ=buffer;
    targetLimit=targ+MAX_LENGTH;
    offs=offsetBuffer;



    ucnv_toUnicode (conv,
                &targ,
                targetLimit,
                (const char **)&src,
                (const char *)sourceLimit,
                expectOffsets ? offs : NULL,
                doFlush,
                &status);

    ucnv_close(conv);
    if(status != expectedStatus){
          log_err("ucnv_fromUnicode() failed for codepage=%s. Error =%s Expected=%s\n", codepage, myErrorName(status), myErrorName(expectedStatus));
          return FALSE;
    }
    log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
        sourceLen, targ-buffer);




    log_verbose("comparing %d uchars (%d bytes)..\n",expectLen,expectLen*2);

    if (expectOffsets != 0) {
        if(memcmp(offsetBuffer, expectOffsets, (targ-buffer) * sizeof(int32_t))){

            log_err("did not get the expected offsets from %s To UNICODE\n", codepage);
            printf("\nGot : ");
            for(p=buffer;p<targ;p++)
                printf("%d, ", offsetBuffer[p-buffer]); 
            printf("\nExpected: ");
            for(i=0; i<(targ-buffer); i++)
                printf("%d, ", expectOffsets[i]);
            printf("\nGot result:");
            for(i=0; i<(targ-buffer); i++)
                printf("0x%04X,", buffer[i]);
            printf("\nFrom Input:");
            for(i=0; i<(src-source); i++)
                printf("0x%02X,", (unsigned char)source[i]);
            puts("\n");
        }
    }
    if(memcmp(buffer, expect, expectLen*2)){
        log_err("String does not match. from codePage %s TO Unicode\n", codepage);
        printf("\nGot:");
        printUSeqErr(buffer, expectLen);
        printf("\nExpected:");
        printUSeqErr(expect, expectLen);
        return FALSE;
    }
    else {
        log_verbose("Matches!\n");
    }

    return TRUE;
}


static UBool testConvertFromU( const UChar *source, int sourceLen,  const uint8_t *expect, int expectLen, 
                const char *codepage, UConverterFromUCallback callback , int32_t *expectOffsets, UBool testReset)
{
    UErrorCode status = U_ZERO_ERROR;
    UConverter *conv = 0;
    uint8_t    junkout[MAX_LENGTH]; /* FIX */
    int32_t    junokout[MAX_LENGTH]; /* FIX */
    uint8_t *p;
    const UChar *src;
    uint8_t *end;
    uint8_t *targ;
    int32_t *offs;
    int i;
    int32_t   realBufferSize;
    uint8_t *realBufferEnd;
    const UChar *realSourceEnd;
    const UChar *sourceLimit;
    UBool checkOffsets = TRUE;
    UBool doFlush;

    UConverterFromUCallback oldAction = NULL;
    void* oldContext = NULL;

    for(i=0;i<MAX_LENGTH;i++)
        junkout[i] = 0xF0;
    for(i=0;i<MAX_LENGTH;i++)
        junokout[i] = 0xFF;

    setNuConvTestName(codepage, "FROM");

    log_verbose("\n=========  %s\n", gNuConvTestName);

    conv = ucnv_open(codepage, &status);
    if(U_FAILURE(status))
    {
        log_err("Couldn't open converter %s\n",codepage);    
        return FALSE;
    }

    log_verbose("Converter opened..\n");
    /*----setting the callback routine----*/
    ucnv_setFromUCallBack (conv, callback, NULL, &oldAction, &oldContext, &status);
    if (U_FAILURE(status)) { 
        log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));  
    }
    /*------------------------*/

    src = source;
    targ = junkout;
    offs = junokout;

    realBufferSize = (sizeof(junkout)/sizeof(junkout[0]));
    realBufferEnd = junkout + realBufferSize;
    realSourceEnd = source + sourceLen;

    if ( gOutBufferSize != realBufferSize )
      checkOffsets = FALSE;

    if( gInBufferSize != MAX_LENGTH )
      checkOffsets = FALSE;

    do
      {
        end = nct_min(targ + gOutBufferSize, realBufferEnd);
        sourceLimit = nct_min(src + gInBufferSize, realSourceEnd);

        doFlush = (UBool)(sourceLimit == realSourceEnd);

        if(targ == realBufferEnd)
          {
        log_err("Error, overflowed the real buffer while about to call fromUnicode! targ=%08lx %s", targ, gNuConvTestName);
        return FALSE;
          }
        log_verbose("calling fromUnicode @ SOURCE:%08lx to %08lx  TARGET: %08lx to %08lx, flush=%s\n", src,sourceLimit, targ,end, doFlush?"TRUE":"FALSE");


        status = U_ZERO_ERROR;
        if(gInBufferSize ==999 && gOutBufferSize==999)
            doFlush = FALSE;
        ucnv_fromUnicode (conv,
                  (char **)&targ,
                  (const char *)end,
                  &src,
                  sourceLimit,
                  offs,
                  doFlush, /* flush if we're at the end of the input data */
                  &status);
        if(testReset) 
            ucnv_resetToUnicode(conv);
        if(gInBufferSize ==999 && gOutBufferSize==999)
            ucnv_resetToUnicode(conv);

      } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && sourceLimit < realSourceEnd) );

    if(U_FAILURE(status)) {
        log_err("Problem doing fromUnicode to %s, errcode %s %s\n", codepage, myErrorName(status), gNuConvTestName);
        return FALSE;
      }

    log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
        sourceLen, targ-junkout);
    if(VERBOSITY)
    {
        char junk[999];
        char offset_str[999];
        uint8_t *ptr;

        junk[0] = 0;
        offset_str[0] = 0;
        for(ptr = junkout;ptr<targ;ptr++)
        {
            sprintf(junk + strlen(junk), "0x%02x, ", (0xFF) & (unsigned int)*ptr);
            sprintf(offset_str + strlen(offset_str), "0x%02x, ", (0xFF) & (unsigned int)junokout[ptr-junkout]);
        }
        
        log_verbose(junk);
        printSeq((const unsigned char *)expect, expectLen);
        if ( checkOffsets )
          {
            log_verbose("\nOffsets:");
            log_verbose(offset_str);
          }
        log_verbose("\n");
    }
    ucnv_close(conv);


    if(expectLen != targ-junkout)
    {
        log_err("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
        log_verbose("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
        printf("\nGot:");
        printSeqErr((const unsigned char*)junkout, targ-junkout);
        printf("\nExpected:");
        printSeqErr((const unsigned char*)expect, expectLen);
        return FALSE;
    }

    if (checkOffsets && (expectOffsets != 0) )
    {
        log_verbose("comparing %d offsets..\n", targ-junkout);
        if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t) )){
            log_err("did not get the expected offsets. %s", gNuConvTestName);
            log_err("Got  : ");
            printSeqErr((const unsigned char*)junkout, targ-junkout);
            for(p=junkout;p<targ;p++)
                log_err("%d, ", junokout[p-junkout]); 
            log_err("\nExpected: ");
            for(i=0; i<(targ-junkout); i++)
                log_err("%d,", expectOffsets[i]);
        }
    }

    log_verbose("comparing..\n");
    if(!memcmp(junkout, expect, expectLen))
    {
        log_verbose("Matches!\n");
        return TRUE;
    }
    else
    {
        log_err("String does not match. %s\n", gNuConvTestName);
        printUSeqErr(source, sourceLen);
        printf("\nGot:");
        printSeqErr((const unsigned char *)junkout, expectLen);
        printf("\nExpected:");
        printSeqErr((const unsigned char *)expect, expectLen);

        return FALSE;
    }
}

static UBool testConvertToU( const uint8_t *source, int sourcelen, const UChar *expect, int expectlen, 
               const char *codepage, UConverterToUCallback callback, int32_t *expectOffsets, UBool testReset)
{
    UErrorCode status = U_ZERO_ERROR;
    UConverter *conv = 0;
    UChar    junkout[MAX_LENGTH]; /* FIX */
    int32_t    junokout[MAX_LENGTH]; /* FIX */
    const uint8_t *src;
    const uint8_t *realSourceEnd;
    const uint8_t *srcLimit;
    UChar *p;
    UChar *targ;
    UChar *end;
    int32_t *offs;
    int i;
    UBool   checkOffsets = TRUE;
    int32_t   realBufferSize;
    UChar *realBufferEnd;
    UBool doFlush;

    UConverterToUCallback oldAction = NULL;
    void* oldContext = NULL;
    

    for(i=0;i<MAX_LENGTH;i++)
        junkout[i] = 0xFFFE;

    for(i=0;i<MAX_LENGTH;i++)
        junokout[i] = -1;

    setNuConvTestName(codepage, "TO");

    log_verbose("\n=========  %s\n", gNuConvTestName);

    conv = ucnv_open(codepage, &status);
    if(U_FAILURE(status))
    {
        log_err("Couldn't open converter %s\n",gNuConvTestName);
        return FALSE;
    }

    log_verbose("Converter opened..\n");
     /*----setting the callback routine----*/
    ucnv_setToUCallBack (conv, callback, NULL, &oldAction, &oldContext, &status);
    if (U_FAILURE(status)) { 
        log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));  
    }
    /*-------------------------------------*/

    src = source;
    targ = junkout;
    offs = junokout;
    
    realBufferSize = (sizeof(junkout)/sizeof(junkout[0]));
    realBufferEnd = junkout + realBufferSize;
    realSourceEnd = src + sourcelen;

    if ( gOutBufferSize != realBufferSize )
      checkOffsets = FALSE;

    if( gInBufferSize != MAX_LENGTH )
      checkOffsets = FALSE;

    do
      {
        end = nct_min( targ + gOutBufferSize, realBufferEnd);
        srcLimit = nct_min(realSourceEnd, src + gInBufferSize);

        if(targ == realBufferEnd)
          {
        log_err("Error, the end would overflow the real output buffer while about to call toUnicode! tarjey=%08lx %s",targ,gNuConvTestName);
        return FALSE;
          }
        log_verbose("calling toUnicode @ %08lx to %08lx\n", targ,end);

        /* oldTarg = targ; */

        status = U_ZERO_ERROR;
        doFlush=(gInBufferSize ==999 && gOutBufferSize==999)?(UBool)(srcLimit == realSourceEnd) : FALSE;
            
        ucnv_toUnicode (conv,
                &targ,
                end,
                (const char **)&src,
                (const char *)srcLimit,
                offs,
                doFlush, /* flush if we're at the end of hte source data */
                &status);
        if(testReset) 
            ucnv_resetFromUnicode(conv);
        if(gInBufferSize ==999 && gOutBufferSize==999)
            ucnv_resetToUnicode(conv);    
        /*        offs += (targ-oldTarg); */

      } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && (srcLimit < realSourceEnd)) ); /* while we just need another buffer */

    if(U_FAILURE(status))
    {
        log_err("Problem doing %s toUnicode, errcode %s %s\n", codepage, myErrorName(status), gNuConvTestName);
        return FALSE;
    }

    log_verbose("\nConversion done. %d bytes -> %d chars.\nResult :",
        sourcelen, targ-junkout);
    if(VERBOSITY)
    {
        char junk[999];
        char offset_str[999];
    
        UChar *ptr;
        
        junk[0] = 0;
        offset_str[0] = 0;

        for(ptr = junkout;ptr<targ;ptr++)
        {
            sprintf(junk + strlen(junk), "0x%04x, ", (0xFFFF) & (unsigned int)*ptr);
            sprintf(offset_str + strlen(offset_str), "0x%04x, ", (0xFFFF) & (unsigned int)junokout[ptr-junkout]);
        }
        
        log_verbose(junk);

        if ( checkOffsets )
          {
            log_verbose("\nOffsets:");
            log_verbose(offset_str);
          }
        log_verbose("\n");
    }
    ucnv_close(conv);

    log_verbose("comparing %d uchars (%d bytes)..\n",expectlen,expectlen*2);

    if (checkOffsets && (expectOffsets != 0))
    {
        if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t))){
            
            log_err("did not get the expected offsets. %s",gNuConvTestName);
            for(p=junkout;p<targ;p++)
                log_err("%d, ", junokout[p-junkout]); 
            log_err("\nExpected: ");
            for(i=0; i<(targ-junkout); i++)
                log_err("%d,", expectOffsets[i]);
            log_err("");
            for(i=0; i<(targ-junkout); i++)
                log_err("%X,", junkout[i]);
            log_err("");
            for(i=0; i<(src-source); i++)
                log_err("%X,", (unsigned char)source[i]);
        }
    }

    if(!memcmp(junkout, expect, expectlen*2))
    {
        log_verbose("Matches!\n");
        return TRUE;
    }
    else
    {
        log_err("String does not match. %s\n", gNuConvTestName);
        log_verbose("String does not match. %s\n", gNuConvTestName);
        printf("\nGot:");
        printUSeq(junkout, expectlen);
        printf("\nExpected:");
        printUSeq(expect, expectlen); 
        return FALSE;
    }
}


static void TestResetBehaviour(void){
    log_verbose("Testing  Reset for SBCS and LATIN_1\n");
    {
        UChar sampleText[] = {0x0031, 0xd801, 0xdc01, 0x0032};
        const uint8_t expected[] = {0x31, 0x1a, 0x32};
        int32_t offsets[] =  { 0,1,3};

        UChar sampleText1[] = {0x0031, 0x0033, 0x0034, 0x0032};
        const uint8_t expected1[] = {0x31, 0x33,0x34, 0x32};
        int32_t offsets1[] =  { 0,1,2,3};

        /*SBCS*/
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-920", UCNV_FROM_U_CALLBACK_SUBSTITUTE , offsets, TRUE))
            log_err("u-> ibm-920 [UCNV_SBCS] not match.\n");
        if(!testConvertToU(expected1, sizeof(expected1), 
                sampleText1, sizeof(sampleText1)/sizeof(sampleText1[0]), "ibm-920",UCNV_TO_U_CALLBACK_SUBSTITUTE , 
                offsets1, TRUE))
           log_err("ibm -920 -> did not match.\n");

        /*LATIN_1*/
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "LATIN1", UCNV_FROM_U_CALLBACK_SUBSTITUTE , offsets, TRUE))
            log_err("u-> LATIN_1 not match.\n");
        if(!testConvertToU(expected1, sizeof(expected1), 
                sampleText1, sizeof(sampleText1)/sizeof(sampleText1[0]), "LATIN1",UCNV_TO_U_CALLBACK_SUBSTITUTE , 
                offsets1, TRUE))
           log_err("LATIN1 -> did not match.\n");
        


    }
    log_verbose("Testing Reset for DBCS and MBCS\n");
    {
        UChar sampleText[]       = {0x00a1, 0xd801, 0xdc01, 0x00a4};
        const uint8_t expected[] = {0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
        int32_t offsets[]        = {0x00, 0x00, 0x01, 0x01, 0x03, 0x03 };

        
        UChar sampleText1[] = {0x00a1, 0x00a4, 0x00a7, 0x00a8};
        const uint8_t expected1[] = {0xa2, 0xae,0xA2,0xB4,0xA1,0xD7,0xA1,0xA7};
        int32_t offsets1[] =  { 0,2,4,6};

        /*DBCS*/
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-1362", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
            log_err("u-> ibm-1362 [UCNV_DBCS] not match.\n");
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-1362", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
            log_err("u-> ibm-1362 [UCNV_DBCS] not match.\n");
       
        if(!testConvertToU(expected1, sizeof(expected1), 
                sampleText1, sizeof(sampleText1)/sizeof(sampleText1[0]), "ibm-1362",UCNV_TO_U_CALLBACK_SUBSTITUTE , 
                offsets1, TRUE))
           log_err("ibm-1362 -> did not match.\n");
        /*MBCS*/
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
            log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
            log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
      
        if(!testConvertToU(expected1, sizeof(expected1), 
                sampleText1, sizeof(sampleText1)/sizeof(sampleText1[0]), "ibm-1363",UCNV_TO_U_CALLBACK_SUBSTITUTE , 
                offsets1, TRUE))
           log_err("ibm-1363 -> did not match.\n");

    }
    log_verbose("Testing Reset for ISO-2022-jp\n");
    {
        UChar    sampleText[] =   { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};

        const uint8_t expected[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
                                    0x31,0x1A, 0x32};


        int32_t offsets[] = {0,0,0,0,0,1,1,2,2,2,2,3,5 };

        
        UChar sampleText1[] = {0x4e00, 0x04e01, 0x0031,0x001A, 0x0032};
        const uint8_t expected1[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
                                    0x31,0x1A, 0x32};
        int32_t offsets1[] =  { 3,5,10,11,12};

        /*iso-2022-jp*/
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-jp",  UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
            log_err("u-> not match.\n");
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-jp", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
            log_err("u->  not match.\n");
        
        if(!testConvertToU(expected1, sizeof(expected1), 
                sampleText1, sizeof(sampleText1)/sizeof(sampleText1[0]), "iso-2022-jp",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
                offsets1, TRUE))
           log_err("iso-2022-jp -> did not match.\n");

    }
    log_verbose("Testing Reset for ISO-2022-cn\n");
    {
        UChar    sampleText[] =   { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};

        const uint8_t expected[] = {
                                    0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B, 
                                    0x36, 0x21,
                                    0x1B, 0x24, 0x29, 0x47, 0x1B, 0x4E, 0x24, 0x22, 
                                    0x0f, 0x1A, 
                                    0x0f, 0x32
                                    };
        

        int32_t offsets[] = {
                                    0,    0,    0,    0,    0,    0,    0,      
                                    1,    1,
                                    2,    2,    2,    2,    2,    2,    2,    2,
                                    3,    3,
                                    5,    5,  };
        
        UChar sampleText1[] = {0x4e00, 0x04e01, 0x0031,0x001A, 0x0032};
        const uint8_t expected1[] = {
                                    0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B, 
                                    0x36, 0x21,
                                    0x1B, 0x24, 0x29, 0x47, 0x1B, 0x4E, 0x24, 0x22, 
                                    0x0f, 0x1A, 
                                    0x32
                                    };
        int32_t offsets1[] =  { 5,7,15,18,19};

        /*iso-2022-CN*/
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-cn", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
            log_err("u-> not match.\n");
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-cn", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
            log_err("u-> not match.\n");

        if(!testConvertToU(expected1, sizeof(expected1), 
                sampleText1, sizeof(sampleText1)/sizeof(sampleText1[0]), "iso-2022-cn",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
                offsets1, TRUE))
           log_err("iso-2022-cn -> did not match.\n");
    }
        log_verbose("Testing Reset for ISO-2022-kr\n");
    {
        UChar    sampleText[] =   { 0x4e00,0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};

        const uint8_t expected[] = {0x1B, 0x24, 0x29, 0x43, 
                                    0x0E, 0x6C, 0x69, 
                                    0x0f, 0x1A, 
                                    0x0e, 0x6F, 0x4B, 
                                    0x0F, 0x31, 
                                    0x1A, 
                                    0x32 };        

        int32_t offsets[] = {-1, -1, -1, -1,
                              0, 0, 0, 
                              1, 1,  
                              3, 3, 3, 
                              4, 4, 
                              5, 
                              7,
                            };
        UChar    sampleText1[] =   { 0x4e00,0x0041, 0x04e01, 0x0031, 0x0042, 0x0032};

        const uint8_t expected1[] = {0x1B, 0x24, 0x29, 0x43, 
                                    0x0E, 0x6C, 0x69, 
                                    0x0f, 0x41, 
                                    0x0e, 0x6F, 0x4B, 
                                    0x0F, 0x31, 
                                    0x42, 
                                    0x32 };        

        int32_t offsets1[] = {
                              5, 8, 10, 
                              13, 14, 15  
                      
                            };
        /*iso-2022-kr*/
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-kr",  UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
            log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "iso-2022-kr",  UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
            log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
        if(!testConvertToU(expected1, sizeof(expected1), 
                sampleText1, sizeof(sampleText1)/sizeof(sampleText1[0]), "iso-2022-kr",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
                offsets1, TRUE))
           log_err("iso-2022-kr -> did not match.\n");
    }
        log_verbose("Testing Reset for HZ\n");
    {
        UChar    sampleText[] =   { 0x4e00, 0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};

        const uint8_t expected[] = {0x7E, 0x7B, 0x52, 0x3B,
                                    0x7E, 0x7D, 0x1A, 
                                    0x7E, 0x7B, 0x36, 0x21, 
                                    0x7E, 0x7D, 0x31, 
                                    0x1A,
                                    0x32 };


        int32_t offsets[] = {0,0,0,0,
                             1,1,1,
                             3,3,3,3,
                             4,4,4,
                             5,
                             7,};
        UChar    sampleText1[] =   { 0x4e00, 0x0035, 0x04e01, 0x0031, 0x0041, 0x0032};

        const uint8_t expected1[] = {0x7E, 0x7B, 0x52, 0x3B,
                                    0x7E, 0x7D, 0x35, 
                                    0x7E, 0x7B, 0x36, 0x21, 
                                    0x7E, 0x7D, 0x31, 
                                    0x41,
                                    0x32 };


        int32_t offsets1[] = {2,6,9,13,14,15
                            };

        /*hz*/
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "HZ", UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
            log_err("u->  not match.\n");
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
                expected, sizeof(expected), "HZ", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
            log_err("u->  not match.\n");
        if(!testConvertToU(expected1, sizeof(expected1), 
                sampleText1, sizeof(sampleText1)/sizeof(sampleText1[0]), "hz",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
                offsets1, TRUE))
           log_err("hz -> did not match.\n");
    }
    /*UTF-8*/
     log_verbose("Testing for UTF8\n");
    {
        UChar    sampleText[] =   { 0x4e00, 0x0701, 0x0031, 0xbfc1, 0xd801, 0xdc01, 0x0032};
        int32_t offsets[]={0x00, 0x00, 0x00, 0x01, 0x01, 0x02,
                           0x03, 0x03, 0x03, 0x04, 0x04, 0x04,
                           0x04, 0x06 };
        const uint8_t expected[] = {0xe4, 0xb8, 0x80, 0xdc, 0x81, 0x31, 
            0xeb, 0xbf, 0x81, 0xF0, 0x90, 0x90, 0x81, 0x32};


        int32_t fromOffsets[] = { 0x0000, 0x0003, 0x0005, 0x0006, 0x0009, 0x0009, 0x000D }; 
        /*UTF-8*/
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
            expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
            log_err("u-> UTF8 with offsets and flush true did not match.\n");
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
            expected, sizeof(expected), "UTF8",  UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
            log_err("u-> UTF8 with offsets and flush true did not match.\n");
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
            expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
            log_err("u-> UTF8 with offsets and flush true did not match.\n");
        if(!testConvertFromU(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
            expected, sizeof(expected), "UTF8",  UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
            log_err("u-> UTF8 with offsets and flush true did not match.\n");
        if(!testConvertToU(expected, sizeof(expected), 
            sampleText, sizeof(sampleText)/sizeof(sampleText[0]), "UTF8",UCNV_TO_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
            log_err("UTF8 -> did not match.\n");
        if(!testConvertToU(expected, sizeof(expected), 
            sampleText, sizeof(sampleText)/sizeof(sampleText[0]), "UTF8", UCNV_TO_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
            log_err("UTF8 -> did not match.\n");
        if(!testConvertToU(expected, sizeof(expected), 
            sampleText, sizeof(sampleText)/sizeof(sampleText[0]), "UTF8",UCNV_TO_U_CALLBACK_SUBSTITUTE , fromOffsets, TRUE))
            log_err("UTF8 -> did not match.\n");
        if(!testConvertToU(expected, sizeof(expected), 
            sampleText, sizeof(sampleText)/sizeof(sampleText[0]), "UTF8", UCNV_TO_U_CALLBACK_SUBSTITUTE , fromOffsets, TRUE))
            log_err("UTF8 -> did not match.\n");

    }

}