1762 lines
57 KiB
C++
1762 lines
57 KiB
C++
/********************************************************************
|
|
* COPYRIGHT:
|
|
* Copyright (C) 2001-2011 IBM, Inc. All Rights Reserved.
|
|
*
|
|
********************************************************************/
|
|
/********************************************************************************
|
|
*
|
|
* File CALLCOLL.C
|
|
*
|
|
* Modification History:
|
|
* Name Description
|
|
* Andy Heninger First Version
|
|
*
|
|
*********************************************************************************
|
|
*/
|
|
|
|
//
|
|
// This program tests string collation and sort key generation performance.
|
|
// Three APIs can be teste: ICU C , Unix strcoll, strxfrm and Windows LCMapString
|
|
// A file of names is required as input, one per line. It must be in utf-8 or utf-16 format,
|
|
// and include a byte order mark. Either LE or BE format is OK.
|
|
//
|
|
|
|
const char gUsageString[] =
|
|
"usage: collperf options...\n"
|
|
"-help Display this message.\n"
|
|
"-file file_name utf-16 format file of names.\n"
|
|
"-locale name ICU locale to use. Default is en_US\n"
|
|
"-rules file_name Collation rules file (overrides locale)\n"
|
|
"-langid 0x1234 Windows Language ID number. Default to value for -locale option\n"
|
|
" see http://msdn.microsoft.com/library/psdk/winbase/nls_8xo3.htm\n"
|
|
"-win Run test using Windows native services. (ICU is default)\n"
|
|
"-unix Run test using Unix strxfrm, strcoll services.\n"
|
|
"-uselen Use API with string lengths. Default is null-terminated strings\n"
|
|
"-usekeys Run tests using sortkeys rather than strcoll\n"
|
|
"-strcmp Run tests using u_strcmp rather than strcoll\n"
|
|
"-strcmpCPO Run tests using u_strcmpCodePointOrder rather than strcoll\n"
|
|
"-loop nnnn Loopcount for test. Adjust for reasonable total running time.\n"
|
|
"-iloop n Inner Loop Count. Default = 1. Number of calls to function\n"
|
|
" under test at each call point. For measuring test overhead.\n"
|
|
"-terse Terse numbers-only output. Intended for use by scripts.\n"
|
|
"-french French accent ordering\n"
|
|
"-frenchoff No French accent ordering (for use with French locales.)\n"
|
|
"-norm Normalizing mode on\n"
|
|
"-shifted Shifted mode\n"
|
|
"-lower Lower case first\n"
|
|
"-upper Upper case first\n"
|
|
"-case Enable separate case level\n"
|
|
"-level n Sort level, 1 to 5, for Primary, Secndary, Tertiary, Quaternary, Identical\n"
|
|
"-keyhist Produce a table sort key size vs. string length\n"
|
|
"-binsearch Binary Search timing test\n"
|
|
"-keygen Sort Key Generation timing test\n"
|
|
"-qsort Quicksort timing test\n"
|
|
"-iter Iteration Performance Test\n"
|
|
"-dump Display strings, sort keys and CEs.\n"
|
|
;
|
|
|
|
|
|
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
#include <stdlib.h>
|
|
#include <math.h>
|
|
#include <locale.h>
|
|
#include <errno.h>
|
|
|
|
#include <unicode/utypes.h>
|
|
#include <unicode/ucol.h>
|
|
#include <unicode/ucoleitr.h>
|
|
#include <unicode/uloc.h>
|
|
#include <unicode/ustring.h>
|
|
#include <unicode/ures.h>
|
|
#include <unicode/uchar.h>
|
|
#include <unicode/ucnv.h>
|
|
#include <unicode/utf8.h>
|
|
|
|
#ifdef WIN32
|
|
#include <windows.h>
|
|
#else
|
|
//
|
|
// Stubs for Windows API functions when building on UNIXes.
|
|
//
|
|
typedef int DWORD;
|
|
inline int CompareStringW(DWORD, DWORD, UChar *, int, UChar *, int) {return 0;}
|
|
#include <sys/time.h>
|
|
unsigned long timeGetTime() {
|
|
struct timeval t;
|
|
gettimeofday(&t, 0);
|
|
unsigned long val = t.tv_sec * 1000; // Let it overflow. Who cares.
|
|
val += t.tv_usec / 1000;
|
|
return val;
|
|
}
|
|
inline int LCMapStringW(DWORD, DWORD, UChar *, int, UChar *, int) {return 0;}
|
|
const int LCMAP_SORTKEY = 0;
|
|
#define MAKELCID(a,b) 0
|
|
const int SORT_DEFAULT = 0;
|
|
#endif
|
|
|
|
|
|
|
|
//
|
|
// Command line option variables
|
|
// These global variables are set according to the options specified
|
|
// on the command line by the user.
|
|
char * opt_fName = 0;
|
|
const char * opt_locale = "en_US";
|
|
int opt_langid = 0; // Defaults to value corresponding to opt_locale.
|
|
char * opt_rules = 0;
|
|
UBool opt_help = FALSE;
|
|
int opt_loopCount = 1;
|
|
int opt_iLoopCount = 1;
|
|
UBool opt_terse = FALSE;
|
|
UBool opt_qsort = FALSE;
|
|
UBool opt_binsearch = FALSE;
|
|
UBool opt_icu = TRUE;
|
|
UBool opt_win = FALSE; // Run with Windows native functions.
|
|
UBool opt_unix = FALSE; // Run with UNIX strcoll, strxfrm functions.
|
|
UBool opt_uselen = FALSE;
|
|
UBool opt_usekeys = FALSE;
|
|
UBool opt_strcmp = FALSE;
|
|
UBool opt_strcmpCPO = FALSE;
|
|
UBool opt_norm = FALSE;
|
|
UBool opt_keygen = FALSE;
|
|
UBool opt_french = FALSE;
|
|
UBool opt_frenchoff = FALSE;
|
|
UBool opt_shifted = FALSE;
|
|
UBool opt_lower = FALSE;
|
|
UBool opt_upper = FALSE;
|
|
UBool opt_case = FALSE;
|
|
int opt_level = 0;
|
|
UBool opt_keyhist = FALSE;
|
|
UBool opt_itertest = FALSE;
|
|
UBool opt_dump = FALSE;
|
|
|
|
|
|
|
|
//
|
|
// Definitions for the command line options
|
|
//
|
|
struct OptSpec {
|
|
const char *name;
|
|
enum {FLAG, NUM, STRING} type;
|
|
void *pVar;
|
|
};
|
|
|
|
OptSpec opts[] = {
|
|
{"-file", OptSpec::STRING, &opt_fName},
|
|
{"-locale", OptSpec::STRING, &opt_locale},
|
|
{"-langid", OptSpec::NUM, &opt_langid},
|
|
{"-rules", OptSpec::STRING, &opt_rules},
|
|
{"-qsort", OptSpec::FLAG, &opt_qsort},
|
|
{"-binsearch", OptSpec::FLAG, &opt_binsearch},
|
|
{"-iter", OptSpec::FLAG, &opt_itertest},
|
|
{"-win", OptSpec::FLAG, &opt_win},
|
|
{"-unix", OptSpec::FLAG, &opt_unix},
|
|
{"-uselen", OptSpec::FLAG, &opt_uselen},
|
|
{"-usekeys", OptSpec::FLAG, &opt_usekeys},
|
|
{"-strcmp", OptSpec::FLAG, &opt_strcmp},
|
|
{"-strcmpCPO", OptSpec::FLAG, &opt_strcmpCPO},
|
|
{"-norm", OptSpec::FLAG, &opt_norm},
|
|
{"-french", OptSpec::FLAG, &opt_french},
|
|
{"-frenchoff", OptSpec::FLAG, &opt_frenchoff},
|
|
{"-shifted", OptSpec::FLAG, &opt_shifted},
|
|
{"-lower", OptSpec::FLAG, &opt_lower},
|
|
{"-upper", OptSpec::FLAG, &opt_upper},
|
|
{"-case", OptSpec::FLAG, &opt_case},
|
|
{"-level", OptSpec::NUM, &opt_level},
|
|
{"-keyhist", OptSpec::FLAG, &opt_keyhist},
|
|
{"-keygen", OptSpec::FLAG, &opt_keygen},
|
|
{"-loop", OptSpec::NUM, &opt_loopCount},
|
|
{"-iloop", OptSpec::NUM, &opt_iLoopCount},
|
|
{"-terse", OptSpec::FLAG, &opt_terse},
|
|
{"-dump", OptSpec::FLAG, &opt_dump},
|
|
{"-help", OptSpec::FLAG, &opt_help},
|
|
{"-?", OptSpec::FLAG, &opt_help},
|
|
{0, OptSpec::FLAG, 0}
|
|
};
|
|
|
|
|
|
//---------------------------------------------------------------------------
|
|
//
|
|
// Global variables pointing to and describing the test file
|
|
//
|
|
//---------------------------------------------------------------------------
|
|
|
|
//
|
|
// struct Line
|
|
//
|
|
// Each line from the source file (containing a name, presumably) gets
|
|
// one of these structs.
|
|
//
|
|
struct Line {
|
|
UChar *name;
|
|
int len;
|
|
char *winSortKey;
|
|
char *icuSortKey;
|
|
char *unixSortKey;
|
|
char *unixName;
|
|
};
|
|
|
|
|
|
|
|
Line *gFileLines; // Ptr to array of Line structs, one per line in the file.
|
|
int gNumFileLines;
|
|
UCollator *gCol;
|
|
DWORD gWinLCID;
|
|
|
|
Line **gSortedLines;
|
|
Line **gRandomLines;
|
|
int gCount;
|
|
|
|
|
|
|
|
//---------------------------------------------------------------------------
|
|
//
|
|
// ProcessOptions() Function to read the command line options.
|
|
//
|
|
//---------------------------------------------------------------------------
|
|
UBool ProcessOptions(int argc, const char **argv, OptSpec opts[])
|
|
{
|
|
int i;
|
|
int argNum;
|
|
const char *pArgName;
|
|
OptSpec *pOpt;
|
|
|
|
for (argNum=1; argNum<argc; argNum++) {
|
|
pArgName = argv[argNum];
|
|
for (pOpt = opts; pOpt->name != 0; pOpt++) {
|
|
if (strcmp(pOpt->name, pArgName) == 0) {
|
|
switch (pOpt->type) {
|
|
case OptSpec::FLAG:
|
|
*(UBool *)(pOpt->pVar) = TRUE;
|
|
break;
|
|
case OptSpec::STRING:
|
|
argNum ++;
|
|
if (argNum >= argc) {
|
|
fprintf(stderr, "value expected for \"%s\" option.\n", pOpt->name);
|
|
return FALSE;
|
|
}
|
|
*(const char **)(pOpt->pVar) = argv[argNum];
|
|
break;
|
|
case OptSpec::NUM:
|
|
argNum ++;
|
|
if (argNum >= argc) {
|
|
fprintf(stderr, "value expected for \"%s\" option.\n", pOpt->name);
|
|
return FALSE;
|
|
}
|
|
char *endp;
|
|
i = strtol(argv[argNum], &endp, 0);
|
|
if (endp == argv[argNum]) {
|
|
fprintf(stderr, "integer value expected for \"%s\" option.\n", pOpt->name);
|
|
return FALSE;
|
|
}
|
|
*(int *)(pOpt->pVar) = i;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
if (pOpt->name == 0)
|
|
{
|
|
fprintf(stderr, "Unrecognized option \"%s\"\n", pArgName);
|
|
return FALSE;
|
|
}
|
|
}
|
|
return TRUE;
|
|
}
|
|
|
|
//---------------------------------------------------------------------------------------
|
|
//
|
|
// Comparison functions for use by qsort.
|
|
//
|
|
// Six flavors, ICU or Windows, SortKey or String Compare, Strings with length
|
|
// or null terminated.
|
|
//
|
|
//---------------------------------------------------------------------------------------
|
|
int ICUstrcmpK(const void *a, const void *b) {
|
|
gCount++;
|
|
int t = strcmp((*(Line **)a)->icuSortKey, (*(Line **)b)->icuSortKey);
|
|
return t;
|
|
}
|
|
|
|
|
|
int ICUstrcmpL(const void *a, const void *b) {
|
|
gCount++;
|
|
UCollationResult t;
|
|
t = ucol_strcoll(gCol, (*(Line **)a)->name, (*(Line **)a)->len, (*(Line **)b)->name, (*(Line **)b)->len);
|
|
if (t == UCOL_LESS) return -1;
|
|
if (t == UCOL_GREATER) return +1;
|
|
return 0;
|
|
}
|
|
|
|
|
|
int ICUstrcmp(const void *a, const void *b) {
|
|
gCount++;
|
|
UCollationResult t;
|
|
t = ucol_strcoll(gCol, (*(Line **)a)->name, -1, (*(Line **)b)->name, -1);
|
|
if (t == UCOL_LESS) return -1;
|
|
if (t == UCOL_GREATER) return +1;
|
|
return 0;
|
|
}
|
|
|
|
|
|
int Winstrcmp(const void *a, const void *b) {
|
|
gCount++;
|
|
int t;
|
|
t = CompareStringW(gWinLCID, 0, (*(Line **)a)->name, -1, (*(Line **)b)->name, -1);
|
|
return t-2;
|
|
}
|
|
|
|
|
|
int UNIXstrcmp(const void *a, const void *b) {
|
|
gCount++;
|
|
int t;
|
|
t = strcoll((*(Line **)a)->unixName, (*(Line **)b)->unixName);
|
|
return t;
|
|
}
|
|
|
|
|
|
int WinstrcmpL(const void *a, const void *b) {
|
|
gCount++;
|
|
int t;
|
|
t = CompareStringW(gWinLCID, 0, (*(Line **)a)->name, (*(Line **)a)->len, (*(Line **)b)->name, (*(Line **)b)->len);
|
|
return t-2;
|
|
}
|
|
|
|
|
|
int WinstrcmpK(const void *a, const void *b) {
|
|
gCount++;
|
|
int t = strcmp((*(Line **)a)->winSortKey, (*(Line **)b)->winSortKey);
|
|
return t;
|
|
}
|
|
|
|
|
|
//---------------------------------------------------------------------------------------
|
|
//
|
|
// Function for sorting the names (lines) into a random order.
|
|
// Order is based on a hash of the ICU Sort key for the lines
|
|
// The randomized order is used as input for the sorting timing tests.
|
|
//
|
|
//---------------------------------------------------------------------------------------
|
|
int ICURandomCmp(const void *a, const void *b) {
|
|
char *ask = (*(Line **)a)->icuSortKey;
|
|
char *bsk = (*(Line **)b)->icuSortKey;
|
|
int aVal = 0;
|
|
int bVal = 0;
|
|
int retVal;
|
|
while (*ask != 0) {
|
|
aVal += aVal*37 + *ask++;
|
|
}
|
|
while (*bsk != 0) {
|
|
bVal += bVal*37 + *bsk++;
|
|
}
|
|
retVal = -1;
|
|
if (aVal == bVal) {
|
|
retVal = 0;
|
|
}
|
|
else if (aVal > bVal) {
|
|
retVal = 1;
|
|
}
|
|
return retVal;
|
|
}
|
|
|
|
//---------------------------------------------------------------------------------------
|
|
//
|
|
// doKeyGen() Key Generation Timing Test
|
|
//
|
|
//---------------------------------------------------------------------------------------
|
|
void doKeyGen()
|
|
{
|
|
int line;
|
|
int loops = 0;
|
|
int iLoop;
|
|
int t;
|
|
int len=-1;
|
|
|
|
// Adjust loop count to compensate for file size. Should be order n
|
|
double dLoopCount = double(opt_loopCount) * (1000. / double(gNumFileLines));
|
|
int adj_loopCount = int(dLoopCount);
|
|
if (adj_loopCount < 1) adj_loopCount = 1;
|
|
|
|
|
|
unsigned long startTime = timeGetTime();
|
|
|
|
if (opt_win) {
|
|
for (loops=0; loops<adj_loopCount; loops++) {
|
|
for (line=0; line < gNumFileLines; line++) {
|
|
if (opt_uselen) {
|
|
len = gFileLines[line].len;
|
|
}
|
|
for (iLoop=0; iLoop < opt_iLoopCount; iLoop++) {
|
|
t=LCMapStringW(gWinLCID, LCMAP_SORTKEY,
|
|
gFileLines[line].name, len,
|
|
(unsigned short *)gFileLines[line].winSortKey, 5000); // TODO something with length.
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else if (opt_icu)
|
|
{
|
|
for (loops=0; loops<adj_loopCount; loops++) {
|
|
for (line=0; line < gNumFileLines; line++) {
|
|
if (opt_uselen) {
|
|
len = gFileLines[line].len;
|
|
}
|
|
for (iLoop=0; iLoop < opt_iLoopCount; iLoop++) {
|
|
t = ucol_getSortKey(gCol, gFileLines[line].name, len, (unsigned char *)gFileLines[line].icuSortKey, 5000);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else if (opt_unix)
|
|
{
|
|
for (loops=0; loops<adj_loopCount; loops++) {
|
|
for (line=0; line < gNumFileLines; line++) {
|
|
for (iLoop=0; iLoop < opt_iLoopCount; iLoop++) {
|
|
t = strxfrm(gFileLines[line].unixSortKey, gFileLines[line].unixName, 5000);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
unsigned long elapsedTime = timeGetTime() - startTime;
|
|
int ns = (int)(float(1000000) * (float)elapsedTime / (float)(adj_loopCount*gNumFileLines));
|
|
|
|
if (opt_terse == FALSE) {
|
|
printf("Sort Key Generation: total # of keys = %d\n", loops*gNumFileLines);
|
|
printf("Sort Key Generation: time per key = %d ns\n", ns);
|
|
}
|
|
else {
|
|
printf("%d, ", ns);
|
|
}
|
|
|
|
int totalKeyLen = 0;
|
|
int totalChars = 0;
|
|
for (line=0; line<gNumFileLines; line++) {
|
|
totalChars += u_strlen(gFileLines[line].name);
|
|
if (opt_win) {
|
|
totalKeyLen += strlen(gFileLines[line].winSortKey);
|
|
}
|
|
else if (opt_icu) {
|
|
totalKeyLen += strlen(gFileLines[line].icuSortKey);
|
|
}
|
|
else if (opt_unix) {
|
|
totalKeyLen += strlen(gFileLines[line].unixSortKey);
|
|
}
|
|
|
|
}
|
|
if (opt_terse == FALSE) {
|
|
printf("Key Length / character = %f\n", (float)totalKeyLen / (float)totalChars);
|
|
} else {
|
|
printf("%f, ", (float)totalKeyLen / (float)totalChars);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
//---------------------------------------------------------------------------------------
|
|
//
|
|
// doBinarySearch() Binary Search timing test. Each name from the list
|
|
// is looked up in the full sorted list of names.
|
|
//
|
|
//---------------------------------------------------------------------------------------
|
|
void doBinarySearch()
|
|
{
|
|
|
|
gCount = 0;
|
|
int line;
|
|
int loops = 0;
|
|
int iLoop = 0;
|
|
unsigned long elapsedTime = 0;
|
|
|
|
// Adjust loop count to compensate for file size. Should be order n (lookups) * log n (compares/lookup)
|
|
// Accurate timings do not depend on this being perfect. The correction is just to try to
|
|
// get total running times of about the right order, so the that user doesn't need to
|
|
// manually adjust the loop count for every different file size.
|
|
double dLoopCount = double(opt_loopCount) * 3000. / (log10((double)gNumFileLines) * double(gNumFileLines));
|
|
if (opt_usekeys) dLoopCount *= 5;
|
|
int adj_loopCount = int(dLoopCount);
|
|
if (adj_loopCount < 1) adj_loopCount = 1;
|
|
|
|
|
|
for (;;) { // not really a loop, just allows "break" to work, to simplify
|
|
// inadvertantly running more than one test through here.
|
|
if (opt_strcmp || opt_strcmpCPO)
|
|
{
|
|
unsigned long startTime = timeGetTime();
|
|
typedef int32_t (U_EXPORT2 *PF)(const UChar *, const UChar *);
|
|
PF pf = u_strcmp;
|
|
if (opt_strcmpCPO) {pf = u_strcmpCodePointOrder;}
|
|
//if (opt_strcmp && opt_win) {pf = (PF)wcscmp;} // Damn the difference between int32_t and int
|
|
// which forces the use of a cast here.
|
|
|
|
int r = 0;
|
|
for (loops=0; loops<adj_loopCount; loops++) {
|
|
|
|
for (line=0; line < gNumFileLines; line++) {
|
|
int hi = gNumFileLines-1;
|
|
int lo = 0;
|
|
int guess = -1;
|
|
for (;;) {
|
|
int newGuess = (hi + lo) / 2;
|
|
if (newGuess == guess)
|
|
break;
|
|
guess = newGuess;
|
|
for (iLoop=0; iLoop < opt_iLoopCount; iLoop++) {
|
|
r = (*pf)((gSortedLines[line])->name, (gSortedLines[guess])->name);
|
|
}
|
|
gCount++;
|
|
if (r== 0)
|
|
break;
|
|
if (r < 0)
|
|
hi = guess;
|
|
else
|
|
lo = guess;
|
|
}
|
|
}
|
|
}
|
|
elapsedTime = timeGetTime() - startTime;
|
|
break;
|
|
}
|
|
|
|
|
|
if (opt_icu)
|
|
{
|
|
unsigned long startTime = timeGetTime();
|
|
UCollationResult r = UCOL_EQUAL;
|
|
for (loops=0; loops<adj_loopCount; loops++) {
|
|
|
|
for (line=0; line < gNumFileLines; line++) {
|
|
int lineLen = -1;
|
|
int guessLen = -1;
|
|
if (opt_uselen) {
|
|
lineLen = (gSortedLines[line])->len;
|
|
}
|
|
int hi = gNumFileLines-1;
|
|
int lo = 0;
|
|
int guess = -1;
|
|
for (;;) {
|
|
int newGuess = (hi + lo) / 2;
|
|
if (newGuess == guess)
|
|
break;
|
|
guess = newGuess;
|
|
int ri = 0;
|
|
if (opt_usekeys) {
|
|
for (iLoop=0; iLoop < opt_iLoopCount; iLoop++) {
|
|
ri = strcmp((gSortedLines[line])->icuSortKey, (gSortedLines[guess])->icuSortKey);
|
|
}
|
|
gCount++;
|
|
r=UCOL_GREATER; if(ri<0) {r=UCOL_LESS;} else if (ri==0) {r=UCOL_EQUAL;}
|
|
}
|
|
else
|
|
{
|
|
if (opt_uselen) {
|
|
guessLen = (gSortedLines[guess])->len;
|
|
}
|
|
for (iLoop=0; iLoop < opt_iLoopCount; iLoop++) {
|
|
r = ucol_strcoll(gCol, (gSortedLines[line])->name, lineLen, (gSortedLines[guess])->name, guessLen);
|
|
}
|
|
gCount++;
|
|
}
|
|
if (r== UCOL_EQUAL)
|
|
break;
|
|
if (r == UCOL_LESS)
|
|
hi = guess;
|
|
else
|
|
lo = guess;
|
|
}
|
|
}
|
|
}
|
|
elapsedTime = timeGetTime() - startTime;
|
|
break;
|
|
}
|
|
|
|
if (opt_win)
|
|
{
|
|
unsigned long startTime = timeGetTime();
|
|
int r = 0;
|
|
for (loops=0; loops<adj_loopCount; loops++) {
|
|
|
|
for (line=0; line < gNumFileLines; line++) {
|
|
int lineLen = -1;
|
|
int guessLen = -1;
|
|
if (opt_uselen) {
|
|
lineLen = (gSortedLines[line])->len;
|
|
}
|
|
int hi = gNumFileLines-1;
|
|
int lo = 0;
|
|
int guess = -1;
|
|
for (;;) {
|
|
int newGuess = (hi + lo) / 2;
|
|
if (newGuess == guess)
|
|
break;
|
|
guess = newGuess;
|
|
if (opt_usekeys) {
|
|
for (iLoop=0; iLoop < opt_iLoopCount; iLoop++) {
|
|
r = strcmp((gSortedLines[line])->winSortKey, (gSortedLines[guess])->winSortKey);
|
|
}
|
|
gCount++;
|
|
r+=2;
|
|
}
|
|
else
|
|
{
|
|
if (opt_uselen) {
|
|
guessLen = (gSortedLines[guess])->len;
|
|
}
|
|
for (iLoop=0; iLoop < opt_iLoopCount; iLoop++) {
|
|
r = CompareStringW(gWinLCID, 0, (gSortedLines[line])->name, lineLen, (gSortedLines[guess])->name, guessLen);
|
|
}
|
|
if (r == 0) {
|
|
if (opt_terse == FALSE) {
|
|
fprintf(stderr, "Error returned from Windows CompareStringW.\n");
|
|
}
|
|
exit(-1);
|
|
}
|
|
gCount++;
|
|
}
|
|
if (r== 2) // strings ==
|
|
break;
|
|
if (r == 1) // line < guess
|
|
hi = guess;
|
|
else // line > guess
|
|
lo = guess;
|
|
}
|
|
}
|
|
}
|
|
elapsedTime = timeGetTime() - startTime;
|
|
break;
|
|
}
|
|
|
|
if (opt_unix)
|
|
{
|
|
unsigned long startTime = timeGetTime();
|
|
int r = 0;
|
|
for (loops=0; loops<adj_loopCount; loops++) {
|
|
|
|
for (line=0; line < gNumFileLines; line++) {
|
|
int hi = gNumFileLines-1;
|
|
int lo = 0;
|
|
int guess = -1;
|
|
for (;;) {
|
|
int newGuess = (hi + lo) / 2;
|
|
if (newGuess == guess)
|
|
break;
|
|
guess = newGuess;
|
|
if (opt_usekeys) {
|
|
for (iLoop=0; iLoop < opt_iLoopCount; iLoop++) {
|
|
r = strcmp((gSortedLines[line])->unixSortKey, (gSortedLines[guess])->unixSortKey);
|
|
}
|
|
gCount++;
|
|
}
|
|
else
|
|
{
|
|
for (iLoop=0; iLoop < opt_iLoopCount; iLoop++) {
|
|
r = strcoll((gSortedLines[line])->unixName, (gSortedLines[guess])->unixName);
|
|
}
|
|
errno = 0;
|
|
if (errno != 0) {
|
|
fprintf(stderr, "Error %d returned from strcoll.\n", errno);
|
|
exit(-1);
|
|
}
|
|
gCount++;
|
|
}
|
|
if (r == 0) // strings ==
|
|
break;
|
|
if (r < 0) // line < guess
|
|
hi = guess;
|
|
else // line > guess
|
|
lo = guess;
|
|
}
|
|
}
|
|
}
|
|
elapsedTime = timeGetTime() - startTime;
|
|
break;
|
|
}
|
|
break;
|
|
}
|
|
|
|
int ns = (int)(float(1000000) * (float)elapsedTime / (float)gCount);
|
|
if (opt_terse == FALSE) {
|
|
printf("binary search: total # of string compares = %d\n", gCount);
|
|
printf("binary search: compares per loop = %d\n", gCount / loops);
|
|
printf("binary search: time per compare = %d ns\n", ns);
|
|
} else {
|
|
printf("%d, ", ns);
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
//---------------------------------------------------------------------------------------
|
|
//
|
|
// doQSort() The quick sort timing test. Uses the C library qsort function.
|
|
//
|
|
//---------------------------------------------------------------------------------------
|
|
void doQSort() {
|
|
int i;
|
|
Line **sortBuf = new Line *[gNumFileLines];
|
|
|
|
// Adjust loop count to compensate for file size. QSort should be n log(n)
|
|
double dLoopCount = double(opt_loopCount) * 3000. / (log10((double)gNumFileLines) * double(gNumFileLines));
|
|
if (opt_usekeys) dLoopCount *= 5;
|
|
int adj_loopCount = int(dLoopCount);
|
|
if (adj_loopCount < 1) adj_loopCount = 1;
|
|
|
|
|
|
gCount = 0;
|
|
unsigned long startTime = timeGetTime();
|
|
if (opt_win && opt_usekeys) {
|
|
for (i=0; i<opt_loopCount; i++) {
|
|
memcpy(sortBuf, gRandomLines, gNumFileLines * sizeof(Line *));
|
|
qsort(sortBuf, gNumFileLines, sizeof(Line *), WinstrcmpK);
|
|
}
|
|
}
|
|
|
|
else if (opt_win && opt_uselen) {
|
|
for (i=0; i<adj_loopCount; i++) {
|
|
memcpy(sortBuf, gRandomLines, gNumFileLines * sizeof(Line *));
|
|
qsort(sortBuf, gNumFileLines, sizeof(Line *), WinstrcmpL);
|
|
}
|
|
}
|
|
|
|
|
|
else if (opt_win && !opt_uselen) {
|
|
for (i=0; i<adj_loopCount; i++) {
|
|
memcpy(sortBuf, gRandomLines, gNumFileLines * sizeof(Line *));
|
|
qsort(sortBuf, gNumFileLines, sizeof(Line *), Winstrcmp);
|
|
}
|
|
}
|
|
|
|
else if (opt_icu && opt_usekeys) {
|
|
for (i=0; i<adj_loopCount; i++) {
|
|
memcpy(sortBuf, gRandomLines, gNumFileLines * sizeof(Line *));
|
|
qsort(sortBuf, gNumFileLines, sizeof(Line *), ICUstrcmpK);
|
|
}
|
|
}
|
|
|
|
else if (opt_icu && opt_uselen) {
|
|
for (i=0; i<adj_loopCount; i++) {
|
|
memcpy(sortBuf, gRandomLines, gNumFileLines * sizeof(Line *));
|
|
qsort(sortBuf, gNumFileLines, sizeof(Line *), ICUstrcmpL);
|
|
}
|
|
}
|
|
|
|
|
|
else if (opt_icu && !opt_uselen) {
|
|
for (i=0; i<adj_loopCount; i++) {
|
|
memcpy(sortBuf, gRandomLines, gNumFileLines * sizeof(Line *));
|
|
qsort(sortBuf, gNumFileLines, sizeof(Line *), ICUstrcmp);
|
|
}
|
|
}
|
|
|
|
else if (opt_unix && !opt_usekeys) {
|
|
for (i=0; i<adj_loopCount; i++) {
|
|
memcpy(sortBuf, gRandomLines, gNumFileLines * sizeof(Line *));
|
|
qsort(sortBuf, gNumFileLines, sizeof(Line *), UNIXstrcmp);
|
|
}
|
|
}
|
|
|
|
unsigned long elapsedTime = timeGetTime() - startTime;
|
|
int ns = (int)(float(1000000) * (float)elapsedTime / (float)gCount);
|
|
if (opt_terse == FALSE) {
|
|
printf("qsort: total # of string compares = %d\n", gCount);
|
|
printf("qsort: time per compare = %d ns\n", ns);
|
|
} else {
|
|
printf("%d, ", ns);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
//---------------------------------------------------------------------------------------
|
|
//
|
|
// doKeyHist() Output a table of data for
|
|
// average sort key size vs. string length.
|
|
//
|
|
//---------------------------------------------------------------------------------------
|
|
void doKeyHist() {
|
|
int i;
|
|
int maxLen = 0;
|
|
|
|
// Find the maximum string length
|
|
for (i=0; i<gNumFileLines; i++) {
|
|
if (gFileLines[i].len > maxLen) maxLen = gFileLines[i].len;
|
|
}
|
|
|
|
// Allocate arrays to hold the histogram data
|
|
int *accumulatedLen = new int[maxLen+1];
|
|
int *numKeysOfSize = new int[maxLen+1];
|
|
for (i=0; i<=maxLen; i++) {
|
|
accumulatedLen[i] = 0;
|
|
numKeysOfSize[i] = 0;
|
|
}
|
|
|
|
// Fill the arrays...
|
|
for (i=0; i<gNumFileLines; i++) {
|
|
int len = gFileLines[i].len;
|
|
accumulatedLen[len] += strlen(gFileLines[i].icuSortKey);
|
|
numKeysOfSize[len] += 1;
|
|
}
|
|
|
|
// And write out averages
|
|
printf("String Length, Avg Key Length, Avg Key Len per char\n");
|
|
for (i=1; i<=maxLen; i++) {
|
|
if (numKeysOfSize[i] > 0) {
|
|
printf("%d, %f, %f\n", i, (float)accumulatedLen[i] / (float)numKeysOfSize[i],
|
|
(float)accumulatedLen[i] / (float)(numKeysOfSize[i] * i));
|
|
}
|
|
}
|
|
delete []accumulatedLen;
|
|
delete []numKeysOfSize ;
|
|
}
|
|
|
|
//---------------------------------------------------------------------------------------
|
|
//
|
|
// doForwardIterTest(UBool) Forward iteration test
|
|
// argument null-terminated string used
|
|
//
|
|
//---------------------------------------------------------------------------------------
|
|
void doForwardIterTest(UBool haslen) {
|
|
int count = 0;
|
|
|
|
UErrorCode error = U_ZERO_ERROR;
|
|
printf("\n\nPerforming forward iteration performance test with ");
|
|
|
|
if (haslen) {
|
|
printf("non-null terminated data -----------\n");
|
|
}
|
|
else {
|
|
printf("null terminated data -----------\n");
|
|
}
|
|
printf("performance test on strings from file -----------\n");
|
|
|
|
UChar dummytext[] = {0, 0};
|
|
UCollationElements *iter = ucol_openElements(gCol, NULL, 0, &error);
|
|
ucol_setText(iter, dummytext, 1, &error);
|
|
|
|
gCount = 0;
|
|
unsigned long startTime = timeGetTime();
|
|
while (count < opt_loopCount) {
|
|
int linecount = 0;
|
|
while (linecount < gNumFileLines) {
|
|
UChar *str = gFileLines[linecount].name;
|
|
int strlen = haslen?gFileLines[linecount].len:-1;
|
|
ucol_setText(iter, str, strlen, &error);
|
|
while (ucol_next(iter, &error) != UCOL_NULLORDER) {
|
|
gCount++;
|
|
}
|
|
|
|
linecount ++;
|
|
}
|
|
count ++;
|
|
}
|
|
unsigned long elapsedTime = timeGetTime() - startTime;
|
|
printf("elapsedTime %ld\n", elapsedTime);
|
|
|
|
// empty loop recalculation
|
|
count = 0;
|
|
startTime = timeGetTime();
|
|
while (count < opt_loopCount) {
|
|
int linecount = 0;
|
|
while (linecount < gNumFileLines) {
|
|
UChar *str = gFileLines[linecount].name;
|
|
int strlen = haslen?gFileLines[linecount].len:-1;
|
|
ucol_setText(iter, str, strlen, &error);
|
|
linecount ++;
|
|
}
|
|
count ++;
|
|
}
|
|
elapsedTime -= (timeGetTime() - startTime);
|
|
printf("elapsedTime %ld\n", elapsedTime);
|
|
|
|
ucol_closeElements(iter);
|
|
|
|
int ns = (int)(float(1000000) * (float)elapsedTime / (float)gCount);
|
|
printf("Total number of strings compared %d in %d loops\n", gNumFileLines,
|
|
opt_loopCount);
|
|
printf("Average time per ucol_next() nano seconds %d\n", ns);
|
|
|
|
printf("performance test on skipped-5 concatenated strings from file -----------\n");
|
|
|
|
UChar *str;
|
|
int strlen = 0;
|
|
// appending all the strings
|
|
int linecount = 0;
|
|
while (linecount < gNumFileLines) {
|
|
strlen += haslen?gFileLines[linecount].len:
|
|
u_strlen(gFileLines[linecount].name);
|
|
linecount ++;
|
|
}
|
|
str = (UChar *)malloc(sizeof(UChar) * strlen);
|
|
int strindex = 0;
|
|
linecount = 0;
|
|
while (strindex < strlen) {
|
|
int len = 0;
|
|
len += haslen?gFileLines[linecount].len:
|
|
u_strlen(gFileLines[linecount].name);
|
|
memcpy(str + strindex, gFileLines[linecount].name,
|
|
sizeof(UChar) * len);
|
|
strindex += len;
|
|
linecount ++;
|
|
}
|
|
|
|
printf("Total size of strings %d\n", strlen);
|
|
|
|
gCount = 0;
|
|
count = 0;
|
|
|
|
if (!haslen) {
|
|
strlen = -1;
|
|
}
|
|
iter = ucol_openElements(gCol, str, strlen, &error);
|
|
if (!haslen) {
|
|
strlen = u_strlen(str);
|
|
}
|
|
strlen -= 5; // any left over characters are not iterated,
|
|
// this is to ensure the backwards and forwards iterators
|
|
// gets the same position
|
|
startTime = timeGetTime();
|
|
while (count < opt_loopCount) {
|
|
int count5 = 5;
|
|
strindex = 0;
|
|
ucol_setOffset(iter, strindex, &error);
|
|
while (TRUE) {
|
|
if (ucol_next(iter, &error) == UCOL_NULLORDER) {
|
|
break;
|
|
}
|
|
gCount++;
|
|
count5 --;
|
|
if (count5 == 0) {
|
|
strindex += 10;
|
|
if (strindex > strlen) {
|
|
break;
|
|
}
|
|
ucol_setOffset(iter, strindex, &error);
|
|
count5 = 5;
|
|
}
|
|
}
|
|
count ++;
|
|
}
|
|
|
|
elapsedTime = timeGetTime() - startTime;
|
|
printf("elapsedTime %ld\n", elapsedTime);
|
|
|
|
// empty loop recalculation
|
|
int tempgCount = 0;
|
|
count = 0;
|
|
startTime = timeGetTime();
|
|
while (count < opt_loopCount) {
|
|
int count5 = 5;
|
|
strindex = 0;
|
|
ucol_setOffset(iter, strindex, &error);
|
|
while (TRUE) {
|
|
tempgCount ++;
|
|
count5 --;
|
|
if (count5 == 0) {
|
|
strindex += 10;
|
|
if (strindex > strlen) {
|
|
break;
|
|
}
|
|
ucol_setOffset(iter, strindex, &error);
|
|
count5 = 5;
|
|
}
|
|
}
|
|
count ++;
|
|
}
|
|
elapsedTime -= (timeGetTime() - startTime);
|
|
printf("elapsedTime %ld\n", elapsedTime);
|
|
|
|
ucol_closeElements(iter);
|
|
|
|
printf("gCount %d\n", gCount);
|
|
ns = (int)(float(1000000) * (float)elapsedTime / (float)gCount);
|
|
printf("Average time per ucol_next() nano seconds %d\n", ns);
|
|
}
|
|
|
|
//---------------------------------------------------------------------------------------
|
|
//
|
|
// doBackwardIterTest(UBool) Backwards iteration test
|
|
// argument null-terminated string used
|
|
//
|
|
//---------------------------------------------------------------------------------------
|
|
void doBackwardIterTest(UBool haslen) {
|
|
int count = 0;
|
|
UErrorCode error = U_ZERO_ERROR;
|
|
printf("\n\nPerforming backward iteration performance test with ");
|
|
|
|
if (haslen) {
|
|
printf("non-null terminated data -----------\n");
|
|
}
|
|
else {
|
|
printf("null terminated data -----------\n");
|
|
}
|
|
|
|
printf("performance test on strings from file -----------\n");
|
|
|
|
UCollationElements *iter = ucol_openElements(gCol, NULL, 0, &error);
|
|
UChar dummytext[] = {0, 0};
|
|
ucol_setText(iter, dummytext, 1, &error);
|
|
|
|
gCount = 0;
|
|
unsigned long startTime = timeGetTime();
|
|
while (count < opt_loopCount) {
|
|
int linecount = 0;
|
|
while (linecount < gNumFileLines) {
|
|
UChar *str = gFileLines[linecount].name;
|
|
int strlen = haslen?gFileLines[linecount].len:-1;
|
|
ucol_setText(iter, str, strlen, &error);
|
|
while (ucol_previous(iter, &error) != UCOL_NULLORDER) {
|
|
gCount ++;
|
|
}
|
|
|
|
linecount ++;
|
|
}
|
|
count ++;
|
|
}
|
|
unsigned long elapsedTime = timeGetTime() - startTime;
|
|
|
|
printf("elapsedTime %ld\n", elapsedTime);
|
|
|
|
// empty loop recalculation
|
|
count = 0;
|
|
startTime = timeGetTime();
|
|
while (count < opt_loopCount) {
|
|
int linecount = 0;
|
|
while (linecount < gNumFileLines) {
|
|
UChar *str = gFileLines[linecount].name;
|
|
int strlen = haslen?gFileLines[linecount].len:-1;
|
|
ucol_setText(iter, str, strlen, &error);
|
|
linecount ++;
|
|
}
|
|
count ++;
|
|
}
|
|
elapsedTime -= (timeGetTime() - startTime);
|
|
|
|
printf("elapsedTime %ld\n", elapsedTime);
|
|
ucol_closeElements(iter);
|
|
|
|
int ns = (int)(float(1000000) * (float)elapsedTime / (float)gCount);
|
|
printf("Total number of strings compared %d in %d loops\n", gNumFileLines,
|
|
opt_loopCount);
|
|
printf("Average time per ucol_previous() nano seconds %d\n", ns);
|
|
|
|
printf("performance test on skipped-5 concatenated strings from file -----------\n");
|
|
|
|
UChar *str;
|
|
int strlen = 0;
|
|
// appending all the strings
|
|
int linecount = 0;
|
|
while (linecount < gNumFileLines) {
|
|
strlen += haslen?gFileLines[linecount].len:
|
|
u_strlen(gFileLines[linecount].name);
|
|
linecount ++;
|
|
}
|
|
str = (UChar *)malloc(sizeof(UChar) * strlen);
|
|
int strindex = 0;
|
|
linecount = 0;
|
|
while (strindex < strlen) {
|
|
int len = 0;
|
|
len += haslen?gFileLines[linecount].len:
|
|
u_strlen(gFileLines[linecount].name);
|
|
memcpy(str + strindex, gFileLines[linecount].name,
|
|
sizeof(UChar) * len);
|
|
strindex += len;
|
|
linecount ++;
|
|
}
|
|
|
|
printf("Total size of strings %d\n", strlen);
|
|
|
|
gCount = 0;
|
|
count = 0;
|
|
|
|
if (!haslen) {
|
|
strlen = -1;
|
|
}
|
|
|
|
iter = ucol_openElements(gCol, str, strlen, &error);
|
|
if (!haslen) {
|
|
strlen = u_strlen(str);
|
|
}
|
|
|
|
startTime = timeGetTime();
|
|
while (count < opt_loopCount) {
|
|
int count5 = 5;
|
|
strindex = 5;
|
|
ucol_setOffset(iter, strindex, &error);
|
|
while (TRUE) {
|
|
if (ucol_previous(iter, &error) == UCOL_NULLORDER) {
|
|
break;
|
|
}
|
|
gCount ++;
|
|
count5 --;
|
|
if (count5 == 0) {
|
|
strindex += 10;
|
|
if (strindex > strlen) {
|
|
break;
|
|
}
|
|
ucol_setOffset(iter, strindex, &error);
|
|
count5 = 5;
|
|
}
|
|
}
|
|
count ++;
|
|
}
|
|
|
|
elapsedTime = timeGetTime() - startTime;
|
|
printf("elapsedTime %ld\n", elapsedTime);
|
|
|
|
// empty loop recalculation
|
|
count = 0;
|
|
int tempgCount = 0;
|
|
startTime = timeGetTime();
|
|
while (count < opt_loopCount) {
|
|
int count5 = 5;
|
|
strindex = 5;
|
|
ucol_setOffset(iter, strindex, &error);
|
|
while (TRUE) {
|
|
tempgCount ++;
|
|
count5 --;
|
|
if (count5 == 0) {
|
|
strindex += 10;
|
|
if (strindex > strlen) {
|
|
break;
|
|
}
|
|
ucol_setOffset(iter, strindex, &error);
|
|
count5 = 5;
|
|
}
|
|
}
|
|
count ++;
|
|
}
|
|
elapsedTime -= (timeGetTime() - startTime);
|
|
printf("elapsedTime %ld\n", elapsedTime);
|
|
ucol_closeElements(iter);
|
|
|
|
printf("gCount %d\n", gCount);
|
|
ns = (int)(float(1000000) * (float)elapsedTime / (float)gCount);
|
|
printf("Average time per ucol_previous() nano seconds %d\n", ns);
|
|
}
|
|
|
|
//---------------------------------------------------------------------------------------
|
|
//
|
|
// doIterTest() Iteration test
|
|
//
|
|
//---------------------------------------------------------------------------------------
|
|
void doIterTest() {
|
|
doForwardIterTest(opt_uselen);
|
|
doBackwardIterTest(opt_uselen);
|
|
}
|
|
|
|
|
|
//----------------------------------------------------------------------------------------
|
|
//
|
|
// UnixConvert -- Convert the lines of the file to the encoding for UNIX
|
|
// Since it appears that Unicode support is going in the general
|
|
// direction of the use of UTF-8 locales, that is the approach
|
|
// that is used here.
|
|
//
|
|
//----------------------------------------------------------------------------------------
|
|
void UnixConvert() {
|
|
int line;
|
|
|
|
UConverter *cvrtr; // An ICU code page converter.
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
|
|
|
|
cvrtr = ucnv_open("utf-8", &status); // we are just doing UTF-8 locales for now.
|
|
if (U_FAILURE(status)) {
|
|
fprintf(stderr, "ICU Converter open failed.: %s\n", u_errorName(status));
|
|
exit(-1);
|
|
}
|
|
|
|
for (line=0; line < gNumFileLines; line++) {
|
|
int sizeNeeded = ucnv_fromUChars(cvrtr,
|
|
0, // ptr to target buffer.
|
|
0, // length of target buffer.
|
|
gFileLines[line].name,
|
|
-1, // source is null terminated
|
|
&status);
|
|
if (status != U_BUFFER_OVERFLOW_ERROR && status != U_ZERO_ERROR) {
|
|
//fprintf(stderr, "Conversion from Unicode, something is wrong.\n");
|
|
//exit(-1);
|
|
}
|
|
status = U_ZERO_ERROR;
|
|
gFileLines[line].unixName = new char[sizeNeeded+1];
|
|
sizeNeeded = ucnv_fromUChars(cvrtr,
|
|
gFileLines[line].unixName, // ptr to target buffer.
|
|
sizeNeeded+1, // length of target buffer.
|
|
gFileLines[line].name,
|
|
-1, // source is null terminated
|
|
&status);
|
|
if (U_FAILURE(status)) {
|
|
fprintf(stderr, "ICU Conversion Failed.: %d\n", status);
|
|
exit(-1);
|
|
}
|
|
gFileLines[line].unixName[sizeNeeded] = 0;
|
|
};
|
|
ucnv_close(cvrtr);
|
|
}
|
|
|
|
|
|
//----------------------------------------------------------------------------------------
|
|
//
|
|
// class UCharFile Class to hide all the gorp to read a file in
|
|
// and produce a stream of UChars.
|
|
//
|
|
//----------------------------------------------------------------------------------------
|
|
class UCharFile {
|
|
public:
|
|
UCharFile(const char *fileName);
|
|
~UCharFile();
|
|
UChar get();
|
|
UBool eof() {return fEof;};
|
|
UBool error() {return fError;};
|
|
|
|
private:
|
|
UCharFile (const UCharFile & /*other*/) {}; // No copy constructor.
|
|
UCharFile & operator = (const UCharFile &/*other*/) {return *this;}; // No assignment op
|
|
|
|
FILE *fFile;
|
|
const char *fName;
|
|
UBool fEof;
|
|
UBool fError;
|
|
UChar fPending2ndSurrogate;
|
|
|
|
enum {UTF16LE, UTF16BE, UTF8} fEncoding;
|
|
};
|
|
|
|
UCharFile::UCharFile(const char * fileName) {
|
|
fEof = FALSE;
|
|
fError = FALSE;
|
|
fName = fileName;
|
|
fFile = fopen(fName, "rb");
|
|
fPending2ndSurrogate = 0;
|
|
if (fFile == NULL) {
|
|
fprintf(stderr, "Can not open file \"%s\"\n", opt_fName);
|
|
fError = TRUE;
|
|
return;
|
|
}
|
|
//
|
|
// Look for the byte order mark at the start of the file.
|
|
//
|
|
int BOMC1, BOMC2, BOMC3;
|
|
BOMC1 = fgetc(fFile);
|
|
BOMC2 = fgetc(fFile);
|
|
|
|
if (BOMC1 == 0xff && BOMC2 == 0xfe) {
|
|
fEncoding = UTF16LE; }
|
|
else if (BOMC1 == 0xfe && BOMC2 == 0xff) {
|
|
fEncoding = UTF16BE; }
|
|
else if (BOMC1 == 0xEF && BOMC2 == 0xBB && (BOMC3 = fgetc(fFile)) == 0xBF ) {
|
|
fEncoding = UTF8; }
|
|
else
|
|
{
|
|
fprintf(stderr, "collperf: file \"%s\" encoding must be UTF-8 or UTF-16, and "
|
|
"must include a BOM.\n", fileName);
|
|
fError = true;
|
|
return;
|
|
}
|
|
}
|
|
|
|
|
|
UCharFile::~UCharFile() {
|
|
fclose(fFile);
|
|
}
|
|
|
|
|
|
|
|
UChar UCharFile::get() {
|
|
UChar c;
|
|
switch (fEncoding) {
|
|
case UTF16LE:
|
|
{
|
|
int cL, cH;
|
|
cL = fgetc(fFile);
|
|
cH = fgetc(fFile);
|
|
c = cL | (cH << 8);
|
|
if (cH == EOF) {
|
|
c = 0;
|
|
fEof = TRUE;
|
|
}
|
|
break;
|
|
}
|
|
case UTF16BE:
|
|
{
|
|
int cL, cH;
|
|
cH = fgetc(fFile);
|
|
cL = fgetc(fFile);
|
|
c = cL | (cH << 8);
|
|
if (cL == EOF) {
|
|
c = 0;
|
|
fEof = TRUE;
|
|
}
|
|
break;
|
|
}
|
|
case UTF8:
|
|
{
|
|
if (fPending2ndSurrogate != 0) {
|
|
c = fPending2ndSurrogate;
|
|
fPending2ndSurrogate = 0;
|
|
break;
|
|
}
|
|
|
|
int ch = fgetc(fFile); // Note: c and ch are separate cause eof test doesn't work on UChar type.
|
|
if (ch == EOF) {
|
|
c = 0;
|
|
fEof = TRUE;
|
|
break;
|
|
}
|
|
|
|
if (ch <= 0x7f) {
|
|
// It's ascii. No further utf-8 conversion.
|
|
c = ch;
|
|
break;
|
|
}
|
|
|
|
// Figure out the lenght of the char and read the rest of the bytes
|
|
// into a temp array.
|
|
int nBytes;
|
|
if (ch >= 0xF0) {nBytes=4;}
|
|
else if (ch >= 0xE0) {nBytes=3;}
|
|
else if (ch >= 0xC0) {nBytes=2;}
|
|
else {
|
|
fprintf(stderr, "utf-8 encoded file contains corrupt data.\n");
|
|
fError = TRUE;
|
|
return 0;
|
|
}
|
|
|
|
unsigned char bytes[10];
|
|
bytes[0] = (unsigned char)ch;
|
|
int i;
|
|
for (i=1; i<nBytes; i++) {
|
|
bytes[i] = fgetc(fFile);
|
|
if (bytes[i] < 0x80 || bytes[i] >= 0xc0) {
|
|
fprintf(stderr, "utf-8 encoded file contains corrupt data.\n");
|
|
fError = TRUE;
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
// Convert the bytes from the temp array to a Unicode char.
|
|
i = 0;
|
|
uint32_t cp;
|
|
UTF8_NEXT_CHAR_UNSAFE(bytes, i, cp);
|
|
c = (UChar)cp;
|
|
|
|
if (cp >= 0x10000) {
|
|
// The code point needs to be broken up into a utf-16 surrogate pair.
|
|
// Process first half this time through the main loop, and
|
|
// remember the other half for the next time through.
|
|
UChar utf16Buf[3];
|
|
i = 0;
|
|
UTF16_APPEND_CHAR_UNSAFE(utf16Buf, i, cp);
|
|
fPending2ndSurrogate = utf16Buf[1];
|
|
c = utf16Buf[0];
|
|
}
|
|
break;
|
|
};
|
|
default:
|
|
c = 0xFFFD; /* Error, unspecified codepage*/
|
|
fprintf(stderr, "UCharFile: Error: unknown fEncoding\n");
|
|
exit(1);
|
|
}
|
|
return c;
|
|
}
|
|
|
|
//----------------------------------------------------------------------------------------
|
|
//
|
|
// openRulesCollator - Command line specified a rules file. Read it in
|
|
// and open a collator with it.
|
|
//
|
|
//----------------------------------------------------------------------------------------
|
|
UCollator *openRulesCollator() {
|
|
UCharFile f(opt_rules);
|
|
if (f.error()) {
|
|
return 0;
|
|
}
|
|
|
|
int bufLen = 10000;
|
|
UChar *buf = (UChar *)malloc(bufLen * sizeof(UChar));
|
|
UChar *tmp;
|
|
int i = 0;
|
|
|
|
for(;;) {
|
|
buf[i] = f.get();
|
|
if (f.eof()) {
|
|
break;
|
|
}
|
|
if (f.error()) {
|
|
return 0;
|
|
}
|
|
i++;
|
|
if (i >= bufLen) {
|
|
tmp = buf;
|
|
bufLen += 10000;
|
|
buf = (UChar *)realloc(buf, bufLen);
|
|
if (buf == NULL) {
|
|
free(tmp);
|
|
return 0;
|
|
}
|
|
}
|
|
}
|
|
buf[i] = 0;
|
|
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
UCollator *coll = ucol_openRules(buf, u_strlen(buf), UCOL_OFF,
|
|
UCOL_DEFAULT_STRENGTH, NULL, &status);
|
|
if (U_FAILURE(status)) {
|
|
fprintf(stderr, "ICU ucol_openRules() open failed.: %d\n", status);
|
|
return 0;
|
|
}
|
|
free(buf);
|
|
return coll;
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
//----------------------------------------------------------------------------------------
|
|
//
|
|
// Main -- process command line, read in and pre-process the test file,
|
|
// call other functions to do the actual tests.
|
|
//
|
|
//----------------------------------------------------------------------------------------
|
|
int main(int argc, const char** argv) {
|
|
if (ProcessOptions(argc, argv, opts) != TRUE || opt_help || opt_fName == 0) {
|
|
printf(gUsageString);
|
|
exit (1);
|
|
}
|
|
|
|
// Make sure that we've only got one API selected.
|
|
if (opt_unix || opt_win) opt_icu = FALSE;
|
|
if (opt_unix) opt_win = FALSE;
|
|
|
|
//
|
|
// Set up an ICU collator
|
|
//
|
|
UErrorCode status = U_ZERO_ERROR;
|
|
|
|
if (opt_rules != 0) {
|
|
gCol = openRulesCollator();
|
|
if (gCol == 0) {return -1;}
|
|
}
|
|
else {
|
|
gCol = ucol_open(opt_locale, &status);
|
|
if (U_FAILURE(status)) {
|
|
fprintf(stderr, "Collator creation failed.: %d\n", status);
|
|
return -1;
|
|
}
|
|
}
|
|
if (status==U_USING_DEFAULT_WARNING && opt_terse==FALSE) {
|
|
fprintf(stderr, "Warning, U_USING_DEFAULT_WARNING for %s\n", opt_locale);
|
|
}
|
|
if (status==U_USING_FALLBACK_WARNING && opt_terse==FALSE) {
|
|
fprintf(stderr, "Warning, U_USING_FALLBACK_ERROR for %s\n", opt_locale);
|
|
}
|
|
|
|
if (opt_norm) {
|
|
ucol_setAttribute(gCol, UCOL_NORMALIZATION_MODE, UCOL_ON, &status);
|
|
}
|
|
if (opt_french && opt_frenchoff) {
|
|
fprintf(stderr, "collperf: Error, specified both -french and -frenchoff options.");
|
|
exit(-1);
|
|
}
|
|
if (opt_french) {
|
|
ucol_setAttribute(gCol, UCOL_FRENCH_COLLATION, UCOL_ON, &status);
|
|
}
|
|
if (opt_frenchoff) {
|
|
ucol_setAttribute(gCol, UCOL_FRENCH_COLLATION, UCOL_OFF, &status);
|
|
}
|
|
if (opt_lower) {
|
|
ucol_setAttribute(gCol, UCOL_CASE_FIRST, UCOL_LOWER_FIRST, &status);
|
|
}
|
|
if (opt_upper) {
|
|
ucol_setAttribute(gCol, UCOL_CASE_FIRST, UCOL_UPPER_FIRST, &status);
|
|
}
|
|
if (opt_case) {
|
|
ucol_setAttribute(gCol, UCOL_CASE_LEVEL, UCOL_ON, &status);
|
|
}
|
|
if (opt_shifted) {
|
|
ucol_setAttribute(gCol, UCOL_ALTERNATE_HANDLING, UCOL_SHIFTED, &status);
|
|
}
|
|
if (opt_level != 0) {
|
|
switch (opt_level) {
|
|
case 1:
|
|
ucol_setAttribute(gCol, UCOL_STRENGTH, UCOL_PRIMARY, &status);
|
|
break;
|
|
case 2:
|
|
ucol_setAttribute(gCol, UCOL_STRENGTH, UCOL_SECONDARY, &status);
|
|
break;
|
|
case 3:
|
|
ucol_setAttribute(gCol, UCOL_STRENGTH, UCOL_TERTIARY, &status);
|
|
break;
|
|
case 4:
|
|
ucol_setAttribute(gCol, UCOL_STRENGTH, UCOL_QUATERNARY, &status);
|
|
break;
|
|
case 5:
|
|
ucol_setAttribute(gCol, UCOL_STRENGTH, UCOL_IDENTICAL, &status);
|
|
break;
|
|
default:
|
|
fprintf(stderr, "-level param must be between 1 and 5\n");
|
|
exit(-1);
|
|
}
|
|
}
|
|
|
|
if (U_FAILURE(status)) {
|
|
fprintf(stderr, "Collator attribute setting failed.: %d\n", status);
|
|
return -1;
|
|
}
|
|
|
|
|
|
//
|
|
// Set up a Windows LCID
|
|
//
|
|
if (opt_langid != 0) {
|
|
gWinLCID = MAKELCID(opt_langid, SORT_DEFAULT);
|
|
}
|
|
else {
|
|
gWinLCID = uloc_getLCID(opt_locale);
|
|
}
|
|
|
|
|
|
//
|
|
// Set the UNIX locale
|
|
//
|
|
if (opt_unix) {
|
|
if (setlocale(LC_ALL, opt_locale) == 0) {
|
|
fprintf(stderr, "setlocale(LC_ALL, %s) failed.\n", opt_locale);
|
|
exit(-1);
|
|
}
|
|
}
|
|
|
|
// Read in the input file.
|
|
// File assumed to be utf-16.
|
|
// Lines go onto heap buffers. Global index array to line starts is created.
|
|
// Lines themselves are null terminated.
|
|
//
|
|
|
|
UCharFile f(opt_fName);
|
|
if (f.error()) {
|
|
exit(-1);
|
|
}
|
|
|
|
const int MAXLINES = 100000;
|
|
gFileLines = new Line[MAXLINES];
|
|
UChar buf[1024];
|
|
int column = 0;
|
|
|
|
// Read the file, split into lines, and save in memory.
|
|
// Loop runs once per utf-16 value from the input file,
|
|
// (The number of bytes read from file per loop iteration depends on external encoding.)
|
|
for (;;) {
|
|
|
|
UChar c = f.get();
|
|
if (f.error()){
|
|
exit(-1);
|
|
}
|
|
|
|
|
|
// We now have a good UTF-16 value in c.
|
|
|
|
// Watch for CR, LF, EOF; these finish off a line.
|
|
if (c == 0xd) {
|
|
continue;
|
|
}
|
|
|
|
if (f.eof() || c == 0x0a || c==0x2028) { // Unipad inserts 2028 line separators!
|
|
buf[column++] = 0;
|
|
if (column > 1) {
|
|
gFileLines[gNumFileLines].name = new UChar[column];
|
|
gFileLines[gNumFileLines].len = column-1;
|
|
memcpy(gFileLines[gNumFileLines].name, buf, column * sizeof(UChar));
|
|
gNumFileLines++;
|
|
column = 0;
|
|
if (gNumFileLines >= MAXLINES) {
|
|
fprintf(stderr, "File too big. Max number of lines is %d\n", MAXLINES);
|
|
exit(-1);
|
|
}
|
|
|
|
}
|
|
if (c == 0xa || c == 0x2028)
|
|
continue;
|
|
else
|
|
break; // EOF
|
|
}
|
|
buf[column++] = c;
|
|
if (column >= 1023)
|
|
{
|
|
static UBool warnFlag = TRUE;
|
|
if (warnFlag) {
|
|
fprintf(stderr, "Warning - file line longer than 1023 chars truncated.\n");
|
|
warnFlag = FALSE;
|
|
}
|
|
column--;
|
|
}
|
|
}
|
|
|
|
if (opt_terse == FALSE) {
|
|
printf("file \"%s\", %d lines.\n", opt_fName, gNumFileLines);
|
|
}
|
|
|
|
|
|
// Convert the lines to the UNIX encoding.
|
|
if (opt_unix) {
|
|
UnixConvert();
|
|
}
|
|
|
|
//
|
|
// Pre-compute ICU sort keys for the lines of the file.
|
|
//
|
|
int line;
|
|
int32_t t;
|
|
|
|
for (line=0; line<gNumFileLines; line++) {
|
|
t = ucol_getSortKey(gCol, gFileLines[line].name, -1, (unsigned char *)buf, sizeof(buf));
|
|
gFileLines[line].icuSortKey = new char[t];
|
|
|
|
if (t > (int32_t)sizeof(buf)) {
|
|
t = ucol_getSortKey(gCol, gFileLines[line].name, -1, (unsigned char *)gFileLines[line].icuSortKey , t);
|
|
}
|
|
else
|
|
{
|
|
memcpy(gFileLines[line].icuSortKey, buf, t);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
//
|
|
// Pre-compute Windows sort keys for the lines of the file.
|
|
//
|
|
for (line=0; line<gNumFileLines; line++) {
|
|
t=LCMapStringW(gWinLCID, LCMAP_SORTKEY, gFileLines[line].name, -1, buf, sizeof(buf));
|
|
gFileLines[line].winSortKey = new char[t];
|
|
if (t > (int32_t)sizeof(buf)) {
|
|
t = LCMapStringW(gWinLCID, LCMAP_SORTKEY, gFileLines[line].name, -1, (unsigned short *)(gFileLines[line].winSortKey), t);
|
|
}
|
|
else
|
|
{
|
|
memcpy(gFileLines[line].winSortKey, buf, t);
|
|
}
|
|
}
|
|
|
|
//
|
|
// Pre-compute UNIX sort keys for the lines of the file.
|
|
//
|
|
if (opt_unix) {
|
|
for (line=0; line<gNumFileLines; line++) {
|
|
t=strxfrm((char *)buf, gFileLines[line].unixName, sizeof(buf));
|
|
gFileLines[line].unixSortKey = new char[t];
|
|
if (t > (int32_t)sizeof(buf)) {
|
|
t = strxfrm(gFileLines[line].unixSortKey, gFileLines[line].unixName, sizeof(buf));
|
|
}
|
|
else
|
|
{
|
|
memcpy(gFileLines[line].unixSortKey, buf, t);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
//
|
|
// Dump file lines, CEs, Sort Keys if requested.
|
|
//
|
|
if (opt_dump) {
|
|
int i;
|
|
for (line=0; line<gNumFileLines; line++) {
|
|
for (i=0;;i++) {
|
|
UChar c = gFileLines[line].name[i];
|
|
if (c == 0)
|
|
break;
|
|
if (c < 0x20 || c > 0x7e) {
|
|
printf("\\u%.4x", c);
|
|
}
|
|
else {
|
|
printf("%c", c);
|
|
}
|
|
}
|
|
printf("\n");
|
|
|
|
printf(" CEs: ");
|
|
UCollationElements *CEiter = ucol_openElements(gCol, gFileLines[line].name, -1, &status);
|
|
int32_t ce;
|
|
i = 0;
|
|
for (;;) {
|
|
ce = ucol_next(CEiter, &status);
|
|
if (ce == UCOL_NULLORDER) {
|
|
break;
|
|
}
|
|
printf(" %.8x", ce);
|
|
if (++i > 8) {
|
|
printf("\n ");
|
|
i = 0;
|
|
}
|
|
}
|
|
printf("\n");
|
|
ucol_closeElements(CEiter);
|
|
|
|
|
|
printf(" ICU Sort Key: ");
|
|
for (i=0; ; i++) {
|
|
unsigned char c = gFileLines[line].icuSortKey[i];
|
|
printf("%02x ", c);
|
|
if (c == 0) {
|
|
break;
|
|
}
|
|
if (i > 0 && i % 20 == 0) {
|
|
printf("\n ");
|
|
}
|
|
}
|
|
printf("\n");
|
|
}
|
|
}
|
|
|
|
|
|
//
|
|
// Pre-sort the lines.
|
|
//
|
|
int i;
|
|
gSortedLines = new Line *[gNumFileLines];
|
|
for (i=0; i<gNumFileLines; i++) {
|
|
gSortedLines[i] = &gFileLines[i];
|
|
}
|
|
|
|
if (opt_win) {
|
|
qsort(gSortedLines, gNumFileLines, sizeof(Line *), Winstrcmp);
|
|
}
|
|
else if (opt_unix) {
|
|
qsort(gSortedLines, gNumFileLines, sizeof(Line *), UNIXstrcmp);
|
|
}
|
|
else /* ICU */
|
|
{
|
|
qsort(gSortedLines, gNumFileLines, sizeof(Line *), ICUstrcmp);
|
|
}
|
|
|
|
|
|
//
|
|
// Make up a randomized order, will be used for sorting tests.
|
|
//
|
|
gRandomLines = new Line *[gNumFileLines];
|
|
for (i=0; i<gNumFileLines; i++) {
|
|
gRandomLines[i] = &gFileLines[i];
|
|
}
|
|
qsort(gRandomLines, gNumFileLines, sizeof(Line *), ICURandomCmp);
|
|
|
|
|
|
|
|
|
|
//
|
|
// We've got the file read into memory. Go do something with it.
|
|
//
|
|
|
|
if (opt_qsort) doQSort();
|
|
if (opt_binsearch) doBinarySearch();
|
|
if (opt_keygen) doKeyGen();
|
|
if (opt_keyhist) doKeyHist();
|
|
if (opt_itertest) doIterTest();
|
|
|
|
return 0;
|
|
|
|
}
|