454 lines
16 KiB
Perl
454 lines
16 KiB
Perl
#!/usr/local/bin/perl
|
|
# *******************************************************************************
|
|
# * Copyright (C) 2002-2012 International Business Machines Corporation and *
|
|
# * others. All Rights Reserved. *
|
|
# *******************************************************************************
|
|
|
|
use XML::LibXML;
|
|
|
|
# Assume we are running within the icu4j root directory
|
|
use lib 'src/com/ibm/icu/dev/test/perf';
|
|
use Dataset;
|
|
my $OS=$^O;
|
|
|
|
my $CLASSPATH;
|
|
if ($OS eq "linux" || $OS eq "darwin") {
|
|
$CLASSPATH="../icu4j.jar:../tools/misc/out/lib/icu4j-tools.jar:out/bin";
|
|
} else {
|
|
$CLASSPATH="../icu4j.jar;../tools/misc/out/lib/icu4j-tools.jar;out/bin";
|
|
}
|
|
#---------------------------------------------------------------------
|
|
|
|
# Methods to be tested. Each pair represents a test method and
|
|
# a baseline method which is used for comparison.
|
|
my @METHODS = (
|
|
['TestJDKConstruction', 'TestICUConstruction'],
|
|
['TestJDKParse', 'TestICUParse'],
|
|
['TestJDKFormat', 'TestICUFormat']
|
|
);
|
|
# Patterns which define the set of characters used for testing.
|
|
my @OPTIONS = (
|
|
# locale pattern date string
|
|
[ "en_US", "dddd MMM yyyy", "15 Jan 2007"],
|
|
[ "sw_KE", "dddd MMM yyyy", "15 Jan 2007"],
|
|
[ "en_US", "HH:mm", "13:13"],
|
|
[ "en_US", "HH:mm zzzz", "13:13 Pacific Standard Time"],
|
|
[ "en_US", "HH:mm z", "13:13 PST"],
|
|
[ "en_US", "HH:mm Z", "13:13 -0800"],
|
|
);
|
|
|
|
my $THREADS; # number of threads (input from command-line args)
|
|
my $CALIBRATE = 2; # duration in seconds for initial calibration
|
|
my $DURATION = 10; # duration in seconds for each pass
|
|
my $NUMPASSES = 4; # number of passes. If > 1 then the first pass
|
|
# is discarded as a JIT warm-up pass.
|
|
|
|
my $TABLEATTR = 'BORDER="1" CELLPADDING="4" CELLSPACING="0"';
|
|
|
|
my $PLUS_MINUS = "±";
|
|
|
|
if ($NUMPASSES < 3) {
|
|
die "Need at least 3 passes. One is discarded (JIT warmup) and need two to have 1 degree of freedom (t distribution).";
|
|
}
|
|
|
|
|
|
# run all tests with the specified number of threads from command-line input
|
|
# (if there is no arguments, use $THREADS = 1)
|
|
foreach my $arg ($#ARGV >= 0 ? @ARGV : "1") {
|
|
$THREADS = $arg;
|
|
main();
|
|
}
|
|
|
|
|
|
#---------------------------------------------------------------------
|
|
sub main {
|
|
|
|
#-----------DATE FORMAT PERFORMANCE TESTS-----------------------------
|
|
my $testclass = 'com.ibm.icu.dev.test.perf.DateFormatPerformanceTest';
|
|
#my $threads = ($THREADS > 1) ? "($THREADS threads)" : "";
|
|
|
|
my $doc = XML::LibXML::Document->new('1.0', 'utf-8');
|
|
my $root = $doc->createElement("perfTestResults");
|
|
|
|
# my $raw = "";
|
|
my @shortNames = ( "open" , "parse", "fmt");
|
|
my $index=0;
|
|
|
|
for my $methodPair (@METHODS) {
|
|
|
|
my $testMethod = $methodPair->[0];
|
|
my $baselineMethod = $methodPair->[1];
|
|
my $testname = $shortNames[$index];
|
|
$index++;
|
|
|
|
$OUT = '';
|
|
my $patternCounter=1;
|
|
|
|
for my $pat (@OPTIONS) {
|
|
|
|
# measure the test method
|
|
my $t = measure2($testclass, $testMethod, $pat, -$DURATION);
|
|
my $testResult = $t->getMean();
|
|
my $jdkElement = $doc->createElement("perfTestResult");
|
|
my $testName = "DateFmt-$testname-pat$patternCounter-JDK";
|
|
$jdkElement->setAttribute("test" => $testName);
|
|
$jdkElement->setAttribute("iterations" => 1);
|
|
$jdkElement->setAttribute("time" => $testResult);
|
|
$root->appendChild($jdkElement);
|
|
|
|
# measure baseline method
|
|
my $b = measure2($testclass, $baselineMethod, $pat, -$DURATION);
|
|
my $baseResult = $b->getMean();
|
|
my $icuElement = $doc->createElement("perfTestResult");
|
|
my $testName = "DateFmt-$testname-pat$patternCounter";
|
|
$patternCounter++;
|
|
$icuElement->setAttribute("test"=> $testName);
|
|
$icuElement->setAttribute("iterations" => 1);
|
|
$icuElement->setAttribute("time" => $baseResult);
|
|
$root->appendChild($icuElement);
|
|
|
|
}
|
|
}
|
|
|
|
#------------------DECIMAL FORMAT TESTS---------------------------------
|
|
|
|
my $testclass = 'com.ibm.icu.dev.test.perf.DecimalFormatPerformanceTest';
|
|
my @OPTIONS = (
|
|
# locale pattern date string
|
|
[ "en_US", "#,###.##", "1,234.56"],
|
|
[ "de_DE", "#,###.##", "1.234,56"],
|
|
);
|
|
my $index=0;
|
|
for my $methodPair (@METHODS) {
|
|
|
|
my $testMethod = $methodPair->[0];
|
|
my $baselineMethod = $methodPair->[1];
|
|
my $testname = $shortNames[$index];
|
|
$index++;
|
|
|
|
|
|
for my $pat (@OPTIONS) {
|
|
my $patternName = $pat->[0];
|
|
|
|
# measure the test method
|
|
my $t = measure2($testclass, $testMethod, $pat, -$DURATION);
|
|
my $testResult = $t->getMean();
|
|
my $jdkElement = $doc->createElement("perfTestResult");
|
|
my $testName = "NumFmt-$testname-$patternName-JDK";
|
|
$jdkElement->setAttribute("test" => $testName);
|
|
$jdkElement->setAttribute("iterations"=>1);
|
|
$jdkElement->setAttribute("time" => $testResult);
|
|
$root->appendChild($jdkElement);
|
|
|
|
# measure baseline method
|
|
my $b = measure2($testclass, $baselineMethod, $pat, -$DURATION);
|
|
my $baseResult = $b->getMean();
|
|
my $icuElement = $doc->createElement("perfTestResult");
|
|
my $testName = "NumFmt-$testname-$patternName";
|
|
$icuElement->setAttribute("test"=> $testName);
|
|
$icuElement->setAttribute("iterations"=>1);
|
|
$icuElement->setAttribute("time" => $baseResult);
|
|
$root->appendChild($icuElement);
|
|
}
|
|
}
|
|
|
|
#----------------COLLATION PERFORMANCE TESTS--------------------------_
|
|
|
|
%dataFiles = (
|
|
"en_US", "TestNames_Latin.txt",
|
|
"da_DK", "TestNames_Latin.txt",
|
|
"de_DE", "TestNames_Latin.txt",
|
|
"de__PHONEBOOK", "TestNames_Latin.txt",
|
|
"fr_FR", "TestNames_Latin.txt",
|
|
"ja_JP", "TestNames_Latin.txt TestNames_Japanese_h.txt TestNames_Japanese_k.txt TestNames_Asian.txt",
|
|
"zh_CN", "TestNames_Latin.txt TestNames_Chinese.txt",
|
|
"zh_TW", "TestNames_Latin.txt TestNames_Chinese.txt",
|
|
"zh__PINYIN", "TestNames_Latin.txt TestNames_Chinese.txt",
|
|
"ru_RU", "TestNames_Latin.txt TestNames_Russian.txt",
|
|
"th", "TestNames_Latin.txt TestNames_Thai.txt",
|
|
"ko_KR", "TestNames_Latin.txt TestNames_Korean.txt",
|
|
);
|
|
|
|
# Outer loop runs through the locales to test
|
|
# (Edit this list dirctly to make changes)
|
|
#
|
|
foreach $locale (
|
|
"en_US",
|
|
"da_DK",
|
|
"de_DE",
|
|
"de__PHONEBOOK",
|
|
"fr_FR",
|
|
"ja_JP",
|
|
"zh_CN",
|
|
"zh_TW",
|
|
"zh__PINYIN",
|
|
"ko_KR",
|
|
"ru_RU",
|
|
"th",
|
|
)
|
|
{
|
|
|
|
|
|
#
|
|
# Inner loop runs over the set of data files specified for each locale.
|
|
# (Edit the %datafiles initialization, above, to make changes.
|
|
#
|
|
$ff = $dataFiles{$locale};
|
|
@ff = split(/[\s]+/, $ff);
|
|
$counter = 1;
|
|
foreach $data (@ff) {
|
|
#
|
|
# Run ICU Test for this (locale, data file) pair.
|
|
#
|
|
$iStrCol = `java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch`;
|
|
print "java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch\n";
|
|
$iStrCol =~s/[,\s]*//g; # whack off the leading " ," in the returned result.
|
|
doKeyTimes("java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -keygen",
|
|
my $iKeyGen, my $iKeyLen);
|
|
|
|
#
|
|
# Run Windows test for this (locale, data file) pair. Only do if
|
|
# we are not on Windows 98/ME and we hava a windows langID
|
|
# for the locale.
|
|
#
|
|
$wStrCol = $wKeyGen = $wKeyLen = 0;
|
|
my $wStrCol = `java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch -java`;
|
|
$wStrCol =~s/[,\s]*//g; # whack off the leading " ," in the returned result.
|
|
doKeyTimes("java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -keygen -java",
|
|
$wKeyGen, $wKeyLen);
|
|
|
|
$collDiff = $keyGenDiff = $keyLenDiff = 0;
|
|
if ($wKeyLen > 0) {
|
|
$collDiff = (($wStrCol - $iStrCol) / $iStrCol) * 100;
|
|
$keyGenDiff = (($wKeyGen - $iKeyGen) / $iKeyGen) * 100;
|
|
$keyLenDiff = (($wKeyLen - $iKeyLen) / $iKeyLen) * 100;
|
|
}
|
|
|
|
my $ICU = $doc->createElement("perfTestResult");
|
|
my $testname = "Coll-$locale-data$counter-StrCol";
|
|
#write the results corresponding to this local,data pair
|
|
$ICU->setAttribute("test"=> $testname);
|
|
$ICU->setAttribute("iterations"=>1000);
|
|
$ICU->setAttribute("time"=> $iStrCol);
|
|
$root->appendChild($ICU);
|
|
|
|
my $Key = $doc->createElement("perfTestResult");
|
|
my $testname = "Coll-$locale-data$counter-keyGen";
|
|
$Key->setAttribute("test"=> $testname);
|
|
$Key->setAttribute("iterations"=>1000);
|
|
$Key->setAttribute("time"=>$iKeyGen);
|
|
$root->appendChild($Key);
|
|
|
|
my $JDK = $doc->createElement("perfTestResult");
|
|
my $testname = "Coll-$locale-data$counter-StrCol-JDK";
|
|
$JDK->setAttribute("test"=>$testname);
|
|
$JDK->setAttribute("iterations"=>1000);
|
|
$JDK->setAttribute("time"=>$wStrCol);
|
|
$root->appendChild($JDK);
|
|
|
|
my $Key = $doc->createElement("perfTestResult");
|
|
my $testname = "Coll-$locale-data$counter-keyGen-JDK";
|
|
$Key->setAttribute("test"=>$testname);
|
|
$Key->setAttribute("iterations"=>1000);
|
|
$Key->setAttribute("time"=>$wKeyGen);
|
|
$root->appendChild($Key);
|
|
$counter++;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
#----------WRITE RESULTS TO perf.xml-----------------------
|
|
$doc->setDocumentElement($root);
|
|
open my $out_fh, '>', "perf.xml";
|
|
print {$out_fh} $doc->toString;
|
|
}
|
|
|
|
|
|
#---------------------------------------------------------------------
|
|
# Append text to the global variable $OUT
|
|
sub out {
|
|
$OUT .= join('', @_);
|
|
}
|
|
|
|
|
|
#---------------------------------------------------------------------
|
|
# Measure a given test method with a give test pattern using the
|
|
# global run parameters.
|
|
#
|
|
# @param the method to run
|
|
# @param the pattern defining characters to test
|
|
# @param if >0 then the number of iterations per pass. If <0 then
|
|
# (negative of) the number of seconds per pass.
|
|
#
|
|
# @return a Dataset object, scaled by iterations per pass and
|
|
# events per iteration, to give time per event
|
|
#
|
|
sub measure2 {
|
|
my @data = measure1(@_);
|
|
my $iterPerPass = shift(@data);
|
|
my $eventPerIter = shift(@data);
|
|
|
|
shift(@data) if (@data > 1); # discard first run
|
|
|
|
my $ds = Dataset->new(@data);
|
|
$ds->setScale(1.0e-3 / ($iterPerPass * $eventPerIter));
|
|
$ds;
|
|
}
|
|
|
|
#---------------------------------------------------------------------
|
|
# Measure a given test method with a give test pattern using the
|
|
# global run parameters.
|
|
#
|
|
# @param the method to run
|
|
# @param the pattern defining characters to test
|
|
# @param if >0 then the number of iterations per pass. If <0 then
|
|
# (negative of) the number of seconds per pass.
|
|
#
|
|
# @return array of:
|
|
# [0] iterations per pass
|
|
# [1] events per iteration
|
|
# [2..] ms reported for each pass, in order
|
|
#
|
|
sub measure1 {
|
|
my $testclass = shift;
|
|
my $method = shift;
|
|
my $pat = shift;
|
|
my $iterCount = shift; # actually might be -seconds/pass
|
|
|
|
# is $iterCount actually -seconds/pass?
|
|
if ($iterCount < 0) {
|
|
|
|
# calibrate: estimate ms/iteration
|
|
print "Calibrating...";
|
|
my @t = callJava($testclass, $method, $pat, -$CALIBRATE, 1);
|
|
print "done.\n";
|
|
|
|
my @data = split(/\s+/, $t[0]->[2]);
|
|
$data[0] *= 1.0e+3;
|
|
|
|
my $timePerIter = 1.0e-3 * $data[0] / $data[1];
|
|
|
|
# determine iterations/pass
|
|
$iterCount = int(-$iterCount / $timePerIter + 0.5);
|
|
}
|
|
|
|
# run passes
|
|
print "Measuring $iterCount iterations x $NUMPASSES passes...";
|
|
my @t = callJava($testclass, $method, $pat, $iterCount, $NUMPASSES);
|
|
print "done.\n";
|
|
my @ms = ();
|
|
my @b; # scratch
|
|
for my $a (@t) {
|
|
# $a->[0]: method name, corresponds to $method
|
|
# $a->[1]: 'begin' data, == $iterCount
|
|
# $a->[2]: 'end' data, of the form <ms> <loops> <eventsPerIter>
|
|
# $a->[3...]: gc messages from JVM during pass
|
|
@b = split(/\s+/, $a->[2]);
|
|
push(@ms, $b[0] * 1.0e+3);
|
|
}
|
|
my $eventsPerIter = $b[2];
|
|
|
|
my @ms_str = @ms;
|
|
$ms_str[0] .= " (discarded)" if (@ms_str > 1);
|
|
|
|
($iterCount, $eventsPerIter, @ms);
|
|
}
|
|
|
|
#---------------------------------------------------------------------
|
|
# Invoke java to run $TESTCLASS, passing it the given parameters.
|
|
#
|
|
# @param the method to run
|
|
# @param the number of iterations, or if negative, the duration
|
|
# in seconds. If more than on pass is desired, pass in
|
|
# a string, e.g., "100 100 100".
|
|
# @param the pattern defining characters to test
|
|
#
|
|
# @return an array of results. Each result is an array REF
|
|
# describing one pass. The array REF contains:
|
|
# ->[0]: The method name as reported
|
|
# ->[1]: The params on the '= <meth> begin ...' line
|
|
# ->[2]: The params on the '= <meth> end ...' line
|
|
# ->[3..]: GC messages from the JVM, if any
|
|
#
|
|
sub callJava {
|
|
my $testclass = shift;
|
|
my $method = shift;
|
|
my $pat = shift;
|
|
my $n = shift;
|
|
my $passes = shift;
|
|
|
|
my $n = ($n < 0) ? "-t ".(-$n) : "-i ".$n;
|
|
|
|
my $cmd = "java -classpath $CLASSPATH $testclass $method $n -p $passes -L @$pat[0] \"@$pat[1]\" \"@$pat[2]\" -r $THREADS";
|
|
print "[$cmd]\n"; # for debugging
|
|
open(PIPE, "$cmd|") or die "Can't run \"$cmd\"";
|
|
my @out;
|
|
while (<PIPE>) {
|
|
push(@out, $_);
|
|
}
|
|
close(PIPE) or die "Java failed: \"$cmd\"";
|
|
|
|
@out = grep(!/^\#/, @out); # filter out comments
|
|
|
|
#print "[", join("\n", @out), "]\n";
|
|
|
|
my @results;
|
|
my $method = '';
|
|
my $data = [];
|
|
foreach (@out) {
|
|
next unless (/\S/);
|
|
|
|
if (/^=\s*(\w+)\s*(\w+)\s*(.*)/) {
|
|
my ($m, $state, $d) = ($1, $2, $3);
|
|
#print "$_ => [[$m $state $data]]\n";
|
|
if ($state eq 'begin') {
|
|
die "$method was begun but not finished" if ($method);
|
|
$method = $m;
|
|
push(@$data, $d);
|
|
push(@$data, ''); # placeholder for end data
|
|
} elsif ($state eq 'end') {
|
|
if ($m ne $method) {
|
|
die "$method end does not match: $_";
|
|
}
|
|
$data->[1] = $d; # insert end data at [1]
|
|
#print "#$method:", join(";",@$data), "\n";
|
|
unshift(@$data, $method); # add method to start
|
|
|
|
push(@results, $data);
|
|
$method = '';
|
|
$data = [];
|
|
} else {
|
|
die "Can't parse: $_";
|
|
}
|
|
}
|
|
elsif (/^\[/) {
|
|
if ($method) {
|
|
push(@$data, $_);
|
|
} else {
|
|
# ignore extraneous GC notices
|
|
}
|
|
}
|
|
else {
|
|
die "Can't parse: $_";
|
|
}
|
|
}
|
|
|
|
die "$method was begun but not finished" if ($method);
|
|
|
|
@results;
|
|
}
|
|
|
|
#-----------------------------------------------------------------------------------
|
|
# doKeyGenTimes($Command_to_run, $time, $key_length)
|
|
# Do a key-generation test and return the time and key length/char values.
|
|
#
|
|
sub doKeyTimes($$$) {
|
|
# print "$_[0]\n";
|
|
local($x) = `$_[0]`; # execute the collperf command.
|
|
($_[1], $_[2]) = split(/\,/, $x); # collperf returns "time, keylength" string.
|
|
}
|
|
|
|
|
|
#eof
|