scuffed-code/icu4c/source/tools/makeconv/makeconv.c

/*
 ********************************************************************************
 *
 *   Copyright (C) 1998-2001, International Business Machines
 *   Corporation and others.  All Rights Reserved.
 *
 ********************************************************************************
 *
 *
 *  makeconv.c:
 *  tool creating a binary (compressed) representation of the conversion mapping
 *  table (IBM NLTC ucmap format).
 *
 *  05/04/2000    helena     Added fallback mapping into the picture...
 *  06/29/2000  helena      Major rewrite of the callback APIs.
 */

#include <stdio.h>
#include "unicode/putil.h"
#include "ucnv_io.h"
#include "unicode/ucnv_err.h"
#include "ucnv_bld.h"
#include "ucnv_imp.h"
#include "ucnv_cnv.h"
#include "cstring.h"
#include "cmemory.h"
#include "filestrm.h"
#include "toolutil.h"
#include "uoptions.h"
#include "unicode/udata.h"
#include "unewdata.h"
#include "ucmpwrit.h"
#include "makeconv.h"
#include "genmbcs.h"

#define DEBUG 0

/*
 * from ucnvstat.c - static prototypes of data-based converters
 */
extern const UConverterStaticData * ucnv_converterStaticData[UCNV_NUMBER_OF_SUPPORTED_CONVERTER_TYPES];

/*
 * Global - verbosity
 */
UBool VERBOSE = FALSE;
UBool TOUCHFILE = FALSE;

/*Reads the header of the table file and fills in basic knowledge about the converter
 *in "converter"
 */
static void readHeaderFromFile(UConverterSharedData* myConverter, FileStream* convFile, const char* converterName, UErrorCode* err);

/*Reads the rest of the file, and fills up the shared objects if necessary
Returns the UConverterTable. */
static void loadTableFromFile(FileStream* convFile, UConverterSharedData* sharedData, UErrorCode* err);

/* creates a UConverterSharedData from a mapping file.
 * Fills in:  *staticData, *table.  Converter is NOT otherwise useful.
 */
static UConverterSharedData* createConverterFromTableFile(const char* realName, UErrorCode* err);

/*
 * Set up the UNewData and write the converter..
 */
void writeConverterData(UConverterSharedData *mySharedData, const char *cnvName, const char *cnvDir, UErrorCode *status);

static const char NLTC_SEPARATORS[9] = { '\r', '\n', '\t', ' ', '<', '>' ,'"' , 'U', '\0' };
static const char FALLBACK_SEPARATOR = '|';
static const char CODEPOINT_SEPARATORS[8] = {  '\r', '>', '\\', 'x', '\n', ' ', '\t', '\0' };
static const char UNICODE_CODEPOINT_SEPARATORS[6] = {  '<', '>', 'U', ' ', '\t', '\0' };

static const char *
skipWhitespace(const char *s) {
    while(*s==' ' || *s=='\t') {
        ++s;
    }
    return s;
}

static int32_t
parseCodepageBytes(const char *s, uint32_t *pBytes, const char **pEnd) {
    char *end;
    int32_t length=0;
    uint32_t bytes=0, value;

    while(s[0]=='\\' && s[1]=='x') {
        if(length==4) {
            return -1;
        }
        value=uprv_strtoul(s+2, &end, 16);
        s+=4;
        if(end!=s) {
            return -1;
        }
        bytes=(bytes<<8)|value;
        ++length;
    }
    if(length==0) {
        return -1;
    }
    if(pEnd!=NULL) {
        *pEnd=s;
    }
    *pBytes=bytes;
    return length;
}

/* Remove all characters followed by '#'. There is an exception if there
 * is a fallback sign '|' after the comment and the comment does not
 * start in column 0. In this case, we just blank from '#' to just
 * before the '|' in order to support the fact that IBM official .ucm
 * files have the fallback information in comments!
 */
static char *
  removeComments (char *line)
{
  char *pound;

  line = (char*)skipWhitespace(line);
  pound = uprv_strchr (line, '#');
  if (pound != NULL)
  {
      char *fallback = pound == line ? 0 : uprv_strchr(pound + 1, '|');
      if (fallback != NULL)
      {
          uprv_memset(pound, ' ', fallback-pound);
      }
      else
      {
          *pound = '\0';
      }
  }
  return line;
}

/* Returns true in c is a in set 'setOfChars', false otherwise
 */
static UBool
  isInSet (char c, const char *setOfChars)
{
  uint8_t i = 0;

  while (setOfChars[i] != '\0')
    {
      if (c == setOfChars[i++])
        return TRUE;
    }

  return FALSE;
}

/* Returns pointer to the next non-whitespace (or non-separator)
 */
static int32_t
  nextTokenOffset (const char *line, const char *separators)
{
    int32_t i = 0;

    while (line[i] && isInSet(line[i], separators))
        i++;

    return i;
}

/* Returns pointer to the next token based on the set of separators
 */
static char *
  getToken (char *token, char *line, const char *separators)
{
    int32_t i = nextTokenOffset (line, separators);
    int8_t j = 0;

    while (line[i] && (!isInSet(line[i], separators)))
        token[j++] = line[i++];
    token[j] = '\0';

    return line + i;
}

UBool haveCopyright=TRUE;

static UDataInfo dataInfo={
    sizeof(UDataInfo),
    0,

    U_IS_BIG_ENDIAN,
    U_CHARSET_FAMILY,
    sizeof(UChar),
    0,

    {0x63, 0x6e, 0x76, 0x74},     /* dataFormat="cnvt" */
    {6, 2, 0, 0},                 /* formatVersion */
    {0, 0, 0, 0}                  /* dataVersion (calculated at runtime) */
};

void writeConverterData(UConverterSharedData *mySharedData,
                        const char *cnvName,
                        const char *cnvDir,
                        UErrorCode *status)
{
    UNewDataMemory *mem = NULL;
    uint32_t sz2;
    uint32_t size = 0;

    if(U_FAILURE(*status))
      {
        return;
      }

    mem = udata_create(cnvDir, "cnv", cnvName, &dataInfo, haveCopyright ? U_COPYRIGHT_STRING : NULL, status);

    if(U_FAILURE(*status))
      {
        fprintf(stderr, "Couldn't create the udata %s.%s: %s\n",
                cnvName,
                "cnv",
                u_errorName(*status));
        return;
      }

    if(VERBOSE)
      {
        fprintf(stderr, "- Opened udata %s.%s\n", cnvName, "cnv");
      }

    /* all read only, clean, platform independent data.  Mmmm. :)  */
    udata_writeBlock(mem, mySharedData->staticData, sizeof(UConverterStaticData));
    size += sizeof(UConverterStaticData); /* Is 4-aligned  - by size */
    /* Now, write the table */
    size += ((NewConverter *)mySharedData->table)->write((NewConverter *)mySharedData->table, mySharedData->staticData, mem);

    sz2 = udata_finish(mem, status);
    if(size != sz2)
    {
        fprintf(stderr, "error: wrote %d bytes to the .cnv file but counted %d bytes\n", sz2, size);
        *status=U_INTERNAL_PROGRAM_ERROR;
    }
    if(VERBOSE)
    {
      fprintf(stderr, "- Wrote %d bytes to the udata.\n", sz2);
    }
}

static UOption options[]={
    UOPTION_HELP_H,              /* 0  Numbers for those who*/
    UOPTION_HELP_QUESTION_MARK,  /* 1   can't count. */
    UOPTION_COPYRIGHT,           /* 2 */
    UOPTION_VERSION,             /* 3 */
    UOPTION_DESTDIR,             /* 4 */
    UOPTION_VERBOSE,             /* 5 */
    UOPTION_PACKAGE_NAME,        /* 6 */
	UOPTION_DEF( "touchfile", 't', UOPT_NO_ARG) /* 7 */
};

int main(int argc, char* argv[])
{
    UConverterSharedData* mySharedData = NULL;
    UErrorCode err = U_ZERO_ERROR;
    char outFileName[UCNV_MAX_FULL_FILE_NAME_LENGTH];
    char touchFileName[UCNV_MAX_FULL_FILE_NAME_LENGTH];
    const char* destdir, *arg;
    const char *pkgName = NULL;
    size_t destdirlen;
    char* dot = NULL, *outBasename;
    char cnvName[UCNV_MAX_FULL_FILE_NAME_LENGTH];
    char cnvNameWithPkg[UCNV_MAX_FULL_FILE_NAME_LENGTH];
    UVersionInfo icuVersion;

    U_MAIN_INIT_ARGS(argc, argv);

    /* Set up the ICU version number */
    u_getVersion(icuVersion);
    uprv_memcpy(&dataInfo.dataVersion, &icuVersion, sizeof(UVersionInfo));

    /* preset then read command line options */
    options[4].value=u_getDataDirectory();
    argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);

    /* error handling, printing usage message */
    if(argc<0) {
        fprintf(stderr,
            "error in command line argument \"%s\"\n",
            argv[-argc]);
    } else if(argc<2) {
        argc=-1;
    }
    if(argc<0 || options[0].doesOccur || options[1].doesOccur) {
        fprintf(stderr,
            "usage: %s [-options] files...\n"
            "\tread .ucm codepage mapping files and write .cnv files\n"
            "options:\n"
            "\t-h or -? or --help  this usage text\n"
            "\t-V or --version     show a version message\n"
            "\t-c or --copyright   include a copyright notice\n"
            "\t-d or --destdir     destination directory, followed by the path\n"
            "\t-v or --verbose     Turn on verbose output\n",
            "\t-p or --pkgname     sets the 'package' name for output files.  If ICUDATA, then the default\n"
                "\t                    icu package name will be used.\n"
			"\t-t or --touchfile   Generate additional small file without packagename, for nmake\n",
            argv[0]);
        return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
    }

    if(options[3].doesOccur) {
      fprintf(stderr,"makeconv version %hu.%hu, ICU tool to read .ucm codepage mapping files and write .cnv files\n",
            dataInfo.formatVersion[0], dataInfo.formatVersion[1]);
      fprintf(stderr, "Copyright (C) 1998-2000, International Business Machines\n");
      fprintf(stderr,"Corporation and others.  All Rights Reserved.\n");
        exit(0);
    }

   TOUCHFILE = options[7].doesOccur;

   if(!options[6].doesOccur)
    {
        fprintf(stderr, "%s :  option -p (package name) is required.\n", 
                argv[0]);
        exit(1);
    }
    else
    {
        pkgName =options[6].value;
        if(!strcmp(pkgName, "ICUDATA"))
        {
            pkgName = U_ICUDATA_NAME;
        }
        if(pkgName[0] == 0)
        {
            pkgName = NULL;

			if(TOUCHFILE)
			{
				fprintf(stderr, "%s: Don't use touchfile option with an empty packagename.\n",
							argv[0]);
				exit(1);
			}
        }
    }

    /* get the options values */
    haveCopyright = options[2].doesOccur;
    destdir = options[4].value;
    VERBOSE = options[5].doesOccur;

    if (destdir != NULL && *destdir != 0) {
        uprv_strcpy(outFileName, destdir);
        destdirlen = uprv_strlen(destdir);
        outBasename = outFileName + destdirlen;
        if (*(outBasename - 1) != U_FILE_SEP_CHAR) {
            *outBasename++ = U_FILE_SEP_CHAR;
            ++destdirlen;
        }
    } else {
        destdirlen = 0;
        outBasename = outFileName;
    }

#if DEBUG
    {
      int i;
      printf("makeconv: processing %d files...\n", argc - 1);
      for(i=1; i<argc; ++i) {
        printf("%s ", argv[i]);
      }
      printf("\n");
      fflush(stdout);
    }
#endif

  for (++argv; --argc; ++argv)
    {
      err = U_ZERO_ERROR;
      arg = getLongPathname(*argv);

      /*produces the right destination path for display*/
      if (destdirlen != 0)
        {
          const char *basename;

          /* find the last file sepator */
          basename = uprv_strrchr(arg, U_FILE_SEP_CHAR);
          if (basename == NULL) {
              basename = arg;
          } else {
              ++basename;
          }

          uprv_strcpy(outBasename, basename);
        }
      else
        {
          uprv_strcpy(outFileName, arg);
        }

      /*removes the extension if any is found*/
      dot = uprv_strrchr(outBasename, '.');
      if (dot)
        {
          *dot = '\0';
        }

      /* the basename without extension is the converter name */
      uprv_strcpy(cnvName, outBasename);

      if(TOUCHFILE)
	  {
		  uprv_strcpy(touchFileName, outBasename);
		  uprv_strcat(touchFileName, ".cnv");
	  }

      if(pkgName != NULL)
      {
          /* changes both baename and filename */
          uprv_strcpy(outBasename, pkgName);
          uprv_strcat(outBasename, "_");
          uprv_strcat(outBasename, cnvName);
      }
                      

      /*Adds the target extension*/
      uprv_strcat(outBasename, CONVERTER_FILE_EXTENSION);

#if DEBUG
        printf("makeconv: processing %s  ...\n", arg);
        fflush(stdout);
#endif
      mySharedData = createConverterFromTableFile(arg, &err);

      if (U_FAILURE(err) || (mySharedData == NULL))
        {
          /* if an error is found, print out an error msg and keep going */
          fprintf(stderr, "Error creating converter for \"%s\" file for \"%s\" (error code %d - %s)\n", outFileName, arg, err,
                        u_errorName(err));
          err = U_ZERO_ERROR;
        }
      else
        {
          /* Make the static data name equal to the file name */
          if( /*VERBOSE &&  */ uprv_stricmp(cnvName,mySharedData->staticData->name))
          {
            fprintf(stderr, "Warning: %s%s claims to be '%s'\n",
                    cnvName,
                    CONVERTER_FILE_EXTENSION,
                    mySharedData->staticData->name);
          }

          uprv_strcpy((char*)mySharedData->staticData->name, cnvName);

          if(pkgName == NULL)
          {
              uprv_strcpy(cnvNameWithPkg, cnvName);
          }
          else
          {
              uprv_strcpy(cnvNameWithPkg, pkgName);
              uprv_strcat(cnvNameWithPkg, "_");
              uprv_strcat(cnvNameWithPkg, cnvName);
          }

          writeConverterData(mySharedData, cnvNameWithPkg, destdir, &err);
          ((NewConverter *)mySharedData->table)->close((NewConverter *)mySharedData->table);
		  if(TOUCHFILE)
		  {
			  FileStream *q;
			  char msg[1024];
			  
			  sprintf(msg, "This empty file tells nmake that %s in package %s has been updated.\n",
					cnvName, pkgName);

			  q = T_FileStream_open(touchFileName, "w");
			  if(q == NULL)
			  {
				  fprintf(stderr, "Error writing touchfile \"%s\"\n", touchFileName);
				  err = U_FILE_ACCESS_ERROR;
			  }

			  else
			  {
				  T_FileStream_write(q, msg, uprv_strlen(msg));
				  T_FileStream_close(q);
			  }
		  }

    /* write the information data */
          uprv_free((UConverterStaticData *)mySharedData->staticData);
          uprv_free(mySharedData);

          if(U_FAILURE(err))
          {
                  /* if an error is found, print out an error msg and keep going*/
            fprintf(stderr, "Error writing \"%s\" file for \"%s\" (error code %d - %s)\n", outFileName, arg, err,
                    u_errorName(err));
          }
          else
          {
              puts(outFileName);
          }
        }
      fflush(stdout);
      fflush(stderr);
    }

  return err;
}

static void
getPlatformAndCCSIDFromName(const char *name, int8_t *pPlatform, int32_t *pCCSID) {
    if( (name[0]=='i' || name[0]=='I') &&
        (name[1]=='b' || name[1]=='B') &&
        (name[2]=='m' || name[2]=='M')
    ) {
        name+=3;
        if(*name=='-') {
            ++name;
        }
        *pPlatform=UCNV_IBM;
        *pCCSID=(int32_t)uprv_strtoul(name, NULL, 10);
    } else {
        *pPlatform=UCNV_UNKNOWN;
        *pCCSID=0;
    }
}

/*Reads the header of the table file and fills in basic knowledge about the converter in "converter"*/
void readHeaderFromFile(UConverterSharedData* mySharedData,
                        FileStream* convFile,
                        const char* converterName,
                        UErrorCode *pErrorCode)
{
    char line[200];
    char *s, *end, *key, *value;
    UConverterStaticData *staticData;
    char c;

    if(U_FAILURE(*pErrorCode)) {
        return;
    }

    staticData=(UConverterStaticData *)mySharedData->staticData;
    staticData->conversionType=UCNV_UNSUPPORTED_CONVERTER;
    staticData->platform=UCNV_IBM;
    staticData->subCharLen=0;

    while(T_FileStream_readLine(convFile, line, sizeof(line))) {
        /* remove comments and trailing CR and LF and remove whitespace from the end */
        for(end=line; (c=*end)!=0; ++end) {
            if(c=='#' || c=='\r' || c=='\n') {
                break;
            }
        }
        while(end>line && (*(end-1)==' ' || *(end-1)=='\t')) {
            --end;
        }
        *end=0;

        /* skip leading white space and ignore empty lines */
        s=(char *)skipWhitespace(line);
        if(*s==0) {
            continue;
        }

        /* stop at the beginning of the mapping section */
        if(uprv_memcmp(s, "CHARMAP", 7)==0) {
            break;
        }

        /* get the key name, bracketed in <> */
        if(*s!='<') {
            fprintf(stderr, "error: no header field <key> in line \"%s\"\n", line);
            *pErrorCode=U_INVALID_TABLE_FORMAT;
            return;
        }
        key=++s;
        while(*s!='>') {
            if(*s==0) {
                fprintf(stderr, "error: incomplete header field <key> in line \"%s\"\n", line);
                *pErrorCode=U_INVALID_TABLE_FORMAT;
                return;
            }
            ++s;
        }
        *s=0;

        /* get the value string, possibly quoted */
        s=(char *)skipWhitespace(s+1);
        if(*s!='"') {
            value=s;
        } else {
            /* remove the quotes */
            value=s+1;
            if(end>value && *(end-1)=='"') {
                *--end=0;
            }
        }

        /* collect the information from the header field, ignore unknown keys */
        if(uprv_strcmp(key, "code_set_name")==0) {
            if(*value!=0) {
                uprv_strcpy((char*)staticData->name, value);
                getPlatformAndCCSIDFromName(value, &staticData->platform, &staticData->codepage);
            }
        } else if(uprv_strcmp(key, "uconv_class")==0) {
            const UConverterStaticData *prototype;

            if(uprv_strcmp(value, "DBCS")==0) {
                staticData->conversionType=UCNV_DBCS;
            } else if(uprv_strcmp(value, "SBCS")==0) {
                staticData->conversionType = UCNV_SBCS;
            } else if(uprv_strcmp(value, "MBCS")==0) {
                staticData->conversionType = UCNV_MBCS;
            } else if(uprv_strcmp(value, "EBCDIC_STATEFUL")==0) {
                staticData->conversionType = UCNV_EBCDIC_STATEFUL;
            } else {
                fprintf(stderr, "error: unknown <uconv_class> %s\n", value);
                *pErrorCode=U_INVALID_TABLE_FORMAT;
                return;
            }

            /* Now that we know the type, copy any 'default' values from the table. */
            prototype=ucnv_converterStaticData[staticData->conversionType];
            if(prototype!=NULL) {
                if(staticData->name[0]==0) {
                    uprv_strcpy((char*)staticData->name, prototype->name);
                }

                if(staticData->codepage==0) {
                    staticData->codepage = prototype->codepage;
                }

                if(staticData->platform==0) {
                    staticData->platform = prototype->platform;
                }

                if(staticData->minBytesPerChar==0) {
                    staticData->minBytesPerChar = prototype->minBytesPerChar;
                }

                if(staticData->maxBytesPerChar==0) {
                    staticData->maxBytesPerChar = prototype->maxBytesPerChar;
                }

                if(staticData->subCharLen==0) {
                    staticData->subCharLen=prototype->subCharLen;
                    if(prototype->subCharLen>0) {
                        uprv_memcpy(staticData->subChar, prototype->subChar, prototype->subCharLen);
                    }
                }
            }
        } else if(uprv_strcmp(key, "mb_cur_max")==0) {
            if('1'<=*value && *value<='4' && value[1]==0) {
                staticData->maxBytesPerChar=(int8_t)(*value-'0');
            } else {
                fprintf(stderr, "error: illegal <mb_cur_max> %s\n", value);
                *pErrorCode=U_INVALID_TABLE_FORMAT;
                return;
            }
        } else if(uprv_strcmp(key, "mb_cur_min")==0) {
            if('1'<=*value && *value<='4' && value[1]==0) {
                staticData->minBytesPerChar=(int8_t)(*value-'0');
            } else {
                fprintf(stderr, "error: illegal <mb_cur_min> %s\n", value);
                *pErrorCode=U_INVALID_TABLE_FORMAT;
                return;
            }
        } else if(uprv_strcmp(key, "subchar")==0) {
            uint32_t bytes;
            int32_t length;

            length=parseCodepageBytes(value, &bytes, (const char **)&end);
            if(length>0 && *end==0) {
                staticData->subCharLen=(int8_t)length;
                do {
                    staticData->subChar[--length]=(uint8_t)bytes;
                    bytes>>=8;
                } while(length>0);
            } else {
                fprintf(stderr, "error: illegal <subchar> %s\n", value);
                *pErrorCode=U_INVALID_TABLE_FORMAT;
                return;
            }
        } else if(uprv_strcmp(key, "subchar1")==0) {
            uint32_t bytes;

            if(1==parseCodepageBytes(value, &bytes, (const char **)&end) && *end==0) {
                staticData->subChar1=(uint8_t)bytes;
            } else {
                fprintf(stderr, "error: illegal <subchar1> %s\n", value);
                *pErrorCode=U_INVALID_TABLE_FORMAT;
                return;
            }
        } else if(uprv_strcmp(key, "icu:state")==0) {
            /* if an SBCS/DBCS/EBCDIC_STATEFUL converter has icu:state, then turn it into MBCS */
            switch(staticData->conversionType) {
            case UCNV_SBCS:
            case UCNV_DBCS:
            case UCNV_EBCDIC_STATEFUL:
                staticData->conversionType = UCNV_MBCS;
                break;
            case UCNV_MBCS:
                break;
            default:
                fprintf(stderr, "error: <icu:state> entry for non-MBCS table or before the <uconv_class> line\n");
                *pErrorCode=U_INVALID_TABLE_FORMAT;
                return;
            }

            if(staticData->maxBytesPerChar==0) {
                fprintf(stderr, "error: <icu:state> before the <mb_cur_max> line\n");
                *pErrorCode=U_INVALID_TABLE_FORMAT;
                return;
            }
            if(mySharedData->table==NULL) {
                mySharedData->table=(UConverterTable *)MBCSOpen(staticData->maxBytesPerChar);
                if(mySharedData->table==NULL) {
                    *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
                    return;
                }
            }
            if(!MBCSAddState((NewConverter *)mySharedData->table, value)) {
                *pErrorCode=U_INVALID_TABLE_FORMAT;
                return;
            }
        }
    }

    if(staticData->conversionType==UCNV_UNSUPPORTED_CONVERTER) {
        *pErrorCode=U_INVALID_TABLE_FORMAT;
    } else if(staticData->conversionType==UCNV_MBCS && mySharedData->table==NULL) {
        fprintf(stderr, "error: missing state table information (<icu:state>) for MBCS\n");
        *pErrorCode=U_INVALID_TABLE_FORMAT;
    }
}

void loadTableFromFile(FileStream* convFile, UConverterSharedData* sharedData, UErrorCode* err)
{
    char storageLine[200];
    char* line = NULL;
    UConverterStaticData *staticData=(UConverterStaticData *)sharedData->staticData;
    NewConverter *cnvData = (NewConverter *)sharedData->table;
    UChar32 unicodeValue, codepageValue;
    uint8_t mbcsBytes[8];
    int32_t mbcsLength;
    char codepointBytes[20];
    UBool isOK = TRUE;
    uint8_t precisionMask = 0, unicodeMask = 0;
    char endOfLine;

    if(cnvData->startMappings!=NULL)
    {
        if(!cnvData->startMappings(cnvData)) {
            *err = U_INVALID_TABLE_FORMAT;
            return;
        }
    }

    if(cnvData->isValid!=NULL)
    {
        const uint8_t *p = staticData->subChar;
        codepageValue = 0;
        switch(staticData->subCharLen) {
        case 4:     codepageValue = (codepageValue << 8) | *p++;
        case 3:     codepageValue = (codepageValue << 8) | *p++;
        case 2:     codepageValue = (codepageValue << 8) | *p++;
        case 1:     codepageValue = (codepageValue << 8) | *p;
        default:    break; /* must never occur */
        }
        if(!cnvData->isValid(cnvData, staticData->subChar, staticData->subCharLen, codepageValue)) {
            fprintf(stderr, "       the substitution character byte sequence is illegal in this codepage structure!\n");
            *err = U_INVALID_TABLE_FORMAT;
            isOK = FALSE;
        }
    }

    staticData->hasFromUnicodeFallback = staticData->hasToUnicodeFallback = FALSE;

    while (T_FileStream_readLine(convFile, storageLine, sizeof(storageLine)))
    {
        removeComments(storageLine);
        line = storageLine;
        if (line[nextTokenOffset(line, NLTC_SEPARATORS)] != '\0')
        {
            /* get the Unicode code point */
            line = getToken(codepointBytes, line, UNICODE_CODEPOINT_SEPARATORS);
            if (uprv_strcmp(codepointBytes, "END") == 0)
            {
                break;
            }
            unicodeValue = (UChar32)T_CString_stringToInteger(codepointBytes, 16);

            /* get the codepage bytes */
            codepageValue = 0;
            mbcsLength = 0;
            do
            {
                line = getToken(codepointBytes, line, CODEPOINT_SEPARATORS);
                mbcsBytes[mbcsLength] = (uint8_t)T_CString_stringToInteger(codepointBytes, 16);
                codepageValue = codepageValue << 8 | mbcsBytes[mbcsLength++];

                /* End of line could be \0 or | (if fallback) */
                endOfLine= line[nextTokenOffset(line, CODEPOINT_SEPARATORS)];
            } while((endOfLine != '\0') && (endOfLine != FALLBACK_SEPARATOR));

            if(unicodeValue>=0x10000) {
                unicodeMask|=UCNV_HAS_SUPPLEMENTARY;    /* there are supplementary code points */
            } else if(UTF_IS_SURROGATE(unicodeValue)) {
                unicodeMask|=UCNV_HAS_SURROGATES;       /* there are single surrogates */
            }

            if((uint32_t)unicodeValue > 0x10ffff)
            {
                fprintf(stderr, "error: Unicode code point > U+10ffff in '%s'\n", storageLine);
                isOK = FALSE;
            }
            else if(endOfLine == FALLBACK_SEPARATOR)
            {
                /* we know that there is a fallback separator */
                precisionMask |= 1;
                line = uprv_strchr(line, FALLBACK_SEPARATOR) + 1;
                switch(*line)
                {
                case '0':
                    /* set roundtrip mappings */
                    isOK &= cnvData->addToUnicode(cnvData, mbcsBytes, mbcsLength, unicodeValue, codepageValue, 0) &&
                            cnvData->addFromUnicode(cnvData, mbcsBytes, mbcsLength, unicodeValue, codepageValue, 0);
                    break;
                case '1':
                    /* set only a fallback mapping from Unicode to codepage */
                    staticData->hasFromUnicodeFallback = TRUE;
                    isOK &= cnvData->addFromUnicode(cnvData, mbcsBytes, mbcsLength, unicodeValue, codepageValue, 1);
                    break;
                case '2':
                    /* skip subchar mappings */
                    break;
                case '3':
                    /* set only a fallback mapping from codepage to Unicode */
                    staticData->hasToUnicodeFallback = TRUE;
                    isOK &= cnvData->addToUnicode(cnvData, mbcsBytes, mbcsLength, unicodeValue, codepageValue, 1);
                    break;
                default:
                    fprintf(stderr, "error: illegal fallback indicator '%s' in '%s'\n", line - 1, storageLine);
                    *err = U_INVALID_TABLE_FORMAT;
                    break;
                }
            }
            else
            {
                precisionMask |= 2;
                /* set the mappings */
                isOK &= cnvData->addToUnicode(cnvData, mbcsBytes, mbcsLength, unicodeValue, codepageValue, -1) &&
                        cnvData->addFromUnicode(cnvData, mbcsBytes, mbcsLength, unicodeValue, codepageValue, -1);
            }
        }
    }

    if(unicodeMask == 3)
    {
        fprintf(stderr, "warning: contains mappings to both supplementary code points and single surrogates\n");
    }
    staticData->unicodeMask = unicodeMask;

    if(cnvData->finishMappings!=NULL)
    {
        cnvData->finishMappings(cnvData, staticData);
    }

    if(!isOK)
    {
        *err = U_INVALID_TABLE_FORMAT;
    }
    else if(precisionMask == 3)
    {
        fprintf(stderr, "error: some entries have the mapping precision (with '|'), some do not\n");
        *err = U_INVALID_TABLE_FORMAT;
    }
}

/*creates a UConverterStaticData, fills in necessary links to it the appropriate function pointers*/
UConverterSharedData* createConverterFromTableFile(const char* converterName, UErrorCode* err)
{
  FileStream* convFile = NULL;
  UConverterSharedData* mySharedData = NULL;
  UConverterStaticData* myStaticData = NULL;

  if (U_FAILURE(*err)) return NULL;

  convFile = T_FileStream_open(converterName, "r");
  if (convFile == NULL)
    {
      *err = U_FILE_ACCESS_ERROR;
      return NULL;
    }


  mySharedData = (UConverterSharedData*) uprv_malloc(sizeof(UConverterSharedData));
  if (mySharedData == NULL)
    {
      *err = U_MEMORY_ALLOCATION_ERROR;
      T_FileStream_close(convFile);
      return NULL;
    }

  uprv_memset(mySharedData, 0, sizeof(UConverterSharedData));

  mySharedData->structSize = sizeof(UConverterSharedData);

  myStaticData =  (UConverterStaticData*) uprv_malloc(sizeof(UConverterStaticData));
  if (myStaticData == NULL)
    {
      *err = U_MEMORY_ALLOCATION_ERROR;
      T_FileStream_close(convFile);
      return NULL;
    }
  uprv_memset(myStaticData, 0, sizeof(UConverterStaticData));
  mySharedData->staticData = myStaticData;
  myStaticData->structSize = sizeof(UConverterStaticData);
/*  mySharedData->staticDataOwned = FALSE; */ /* not owned if in udata */
  mySharedData->sharedDataCached = FALSE;

  mySharedData->dataMemory = NULL; /* for init */

  readHeaderFromFile(mySharedData, convFile, converterName, err);

  if (U_FAILURE(*err)) return NULL;

  switch (myStaticData->conversionType)
    {
    case UCNV_SBCS:
      {
        /* SBCS: use MBCS data structure with a default state table */
        if(mySharedData->staticData->maxBytesPerChar!=1) {
            fprintf(stderr, "error: SBCS codepage with max bytes/char!=1\n");
            *err = U_INVALID_TABLE_FORMAT;
            break;
        }
        myStaticData->conversionType = UCNV_MBCS;
        if(mySharedData->table == NULL) {
            NewConverter *sharedDataTable = MBCSOpen(1);
            if(sharedDataTable != NULL) {
                if(!MBCSAddState(sharedDataTable, "0-ff")) {
                    *err = U_INVALID_TABLE_FORMAT;
                    sharedDataTable->close(sharedDataTable);
                } else {
                    mySharedData->table = (UConverterTable *)sharedDataTable;
                }
            } else {
                *err = U_MEMORY_ALLOCATION_ERROR;
            }
        }
        break;
      }
    case UCNV_MBCS:
      {
        /* MBCSOpen() was called by readHeaderFromFile() */
        break;
      }
    case UCNV_EBCDIC_STATEFUL:
      {
        /* EBCDIC_STATEFUL: use MBCS data structure with a default state table */
        if(mySharedData->staticData->maxBytesPerChar!=2) {
            fprintf(stderr, "error: DBCS codepage with max bytes/char!=2\n");
            *err = U_INVALID_TABLE_FORMAT;
            break;
        }
        myStaticData->conversionType = UCNV_MBCS;
        if(mySharedData->table == NULL) {
            NewConverter *sharedDataTable = MBCSOpen(2);
            if(sharedDataTable != NULL) {
                if( !MBCSAddState(sharedDataTable, "0-ff, e:1.s, f:0.s") ||
                    !MBCSAddState(sharedDataTable, "initial, 0-3f:4, e:1.s, f:0.s, 40:3, 41-fe:2, ff:4") ||
                    !MBCSAddState(sharedDataTable, "0-40:1.i, 41-fe:1., ff:1.i") ||
                    !MBCSAddState(sharedDataTable, "0-ff:1.i, 40:1.") ||
                    !MBCSAddState(sharedDataTable, "0-ff:1.i")
                ) {
                    *err = U_INVALID_TABLE_FORMAT;
                    sharedDataTable->close(sharedDataTable);
                } else {
                    mySharedData->table = (UConverterTable *)sharedDataTable;
                }
            } else {
                *err = U_MEMORY_ALLOCATION_ERROR;
            }
        }
        break;
      }
    case UCNV_DBCS:
      {
        /* DBCS: use MBCS data structure with a default state table */
        if(mySharedData->staticData->maxBytesPerChar!=2) {
            fprintf(stderr, "error: DBCS codepage with max bytes/char!=2\n");
            *err = U_INVALID_TABLE_FORMAT;
            break;
        }
        myStaticData->conversionType = UCNV_MBCS;
        if(mySharedData->table == NULL) {
            NewConverter *sharedDataTable = MBCSOpen(2);
            if(sharedDataTable != NULL) {
                if( !MBCSAddState(sharedDataTable, "0-3f:3, 40:2, 41-fe:1, ff:3") ||
                    !MBCSAddState(sharedDataTable, "41-fe") ||
                    !MBCSAddState(sharedDataTable, "40") ||
                    !MBCSAddState(sharedDataTable, "")
                ) {
                    *err = U_INVALID_TABLE_FORMAT;
                    sharedDataTable->close(sharedDataTable);
                } else {
                    mySharedData->table = (UConverterTable *)sharedDataTable;
                }
            } else {
                *err = U_MEMORY_ALLOCATION_ERROR;
            }
        }
        break;
      }

    default :
      fprintf(stderr, "error: <uconv_class> omitted\n");
      *err = U_INVALID_TABLE_FORMAT;
      mySharedData->table = NULL;
      break;
    };

    if(U_SUCCESS(*err) && mySharedData->table != NULL)
    {
        loadTableFromFile(convFile, mySharedData, err);
    }

  T_FileStream_close(convFile);

  return mySharedData;
}

/*
 * Hey, Emacs, please set the following:
 *
 * Local Variables:
 * indent-tabs-mode: nil
 * End:
 *
 */