ICU-2508 perl perf framework output revision + added charperf perl driver

X-SVN-Rev: 10313
This commit is contained in:
Vladimir Weinstein 2002-11-21 07:08:14 +00:00
parent a7cb8d8fe0
commit cb16c57b38
5 changed files with 269 additions and 93 deletions

View File

@ -0,0 +1,42 @@
#!/usr/bin/perl -w
use strict;
use lib '../perldriver';
use PerfFramework;
my $options = {
"title"=>"Character property performance: ICU vs. STDLib",
"headers"=>"StdLib ICU",
"operationIs"=>"code point",
"timePerOperationIs"=>"Time per code point",
"passes"=>"10",
"time"=>"5",
#"outputType"=>"HTML",
"dataDir"=>"c:/src/perf/data",
"outputDir"=>"../results"
};
# programs
# tests will be done for all the programs. Results will be stored and connected
my $p = "charperf.exe";
my $tests = {
"isAlpha", ["$p TestStdLibIsAlpha" , "$p TestIsAlpha" ],
"isUpper", ["$p TestStdLibIsUpper" , "$p TestIsUpper" ],
"isLower", ["$p TestStdLibIsLower" , "$p TestIsLower" ],
"isDigit", ["$p TestStdLibIsDigit" , "$p TestIsDigit" ],
"isSpace", ["$p TestStdLibIsSpace" , "$p TestIsSpace" ],
"isAlphaNumeric", ["$p TestStdLibIsAlphaNumeric" , "$p TestIsAlphaNumeric" ],
"isPrint", ["$p TestStdLibIsPrint" , "$p TestIsPrint" ],
"isControl", ["$p TestStdLibIsControl" , "$p TestIsControl" ],
"toLower", ["$p TestStdLibToLower" , "$p TestToLower" ],
"toUpper", ["$p TestStdLibToUpper" , "$p TestToUpper" ],
"isWhiteSpace", ["$p TestStdLibIsWhiteSpace" , "$p TestIsWhiteSpace" ],
};
my $dataFiles;
runTests($options, $tests, $dataFiles);

View File

@ -9,13 +9,12 @@ use PerfFramework;
my $options = {
"title"=>"Normalization performance: ICU vs. Win",
"headers"=>"ICU Win",
"operationIs"=>"File size in code points",
"timePerOperationIs"=>"Time per code point",
#"passes"=>"10",
#"time"=>"5",
"headers"=>"Win ICU",
"operationIs"=>"code point",
"passes"=>"10",
"time"=>"5",
#"outputType"=>"HTML",
"dataDir"=>"c:/src/perf/data",
"outputType"=>"HTML",
"outputDir"=>"../results"
};
@ -24,31 +23,31 @@ my $options = {
my $p = "normperf.exe -b -u";
my $tests = {
"NFC_NFD_Text", ["$p TestICU_NFC_NFD_Text", "$p TestWin_NFC_NFD_Text" ],
"NFC_NFC_Text", ["$p TestICU_NFC_NFC_Text", "$p TestWin_NFC_NFC_Text" ],
"NFC_Orig_Text", ["$p TestICU_NFC_Orig_Text", "$p TestWin_NFC_Orig_Text"],
"NFD_NFD_Text", ["$p TestICU_NFD_NFD_Text", "$p TestWin_NFD_NFD_Text" ],
"NFD_NFC_Text", ["$p TestICU_NFD_NFC_Text", "$p TestWin_NFD_NFC_Text" ],
"NFD_Orig_Text", ["$p TestICU_NFD_Orig_Text", "$p TestWin_NFD_Orig_Text"]
"NFC_NFD_Text", ["$p TestWin_NFC_NFD_Text" , "$p TestICU_NFC_NFD_Text" ],
"NFC_NFC_Text", ["$p TestWin_NFC_NFC_Text" , "$p TestICU_NFC_NFC_Text" ],
"NFC_Orig_Text", ["$p TestWin_NFC_Orig_Text" , "$p TestICU_NFC_Orig_Text"],
"NFD_NFD_Text", ["$p TestWin_NFD_NFD_Text" , "$p TestICU_NFD_NFD_Text" ],
"NFD_NFC_Text", ["$p TestWin_NFD_NFC_Text" , "$p TestICU_NFD_NFC_Text" ],
"NFD_Orig_Text", ["$p TestWin_NFD_Orig_Text" , "$p TestICU_NFD_Orig_Text"]
};
my $dataFiles = {
"",
[
# "TestNames_Asian.txt",
# "TestNames_Chinese.txt",
"TestNames_Asian.txt",
"TestNames_Chinese.txt",
"TestNames_Japanese.txt",
"TestNames_Japanese_h.txt",
"TestNames_Japanese_k.txt",
# "TestNames_Korean.txt",
# "TestNames_Latin.txt",
# "TestNames_SerbianSH.txt",
# "TestNames_SerbianSR.txt",
# "TestNames_Thai.txt",
# "Testnames_Russian.txt",
# "th18057.txt",
# "thesis.txt",
# "vfear11a.txt",
"TestNames_Korean.txt",
"TestNames_Latin.txt",
"TestNames_SerbianSH.txt",
"TestNames_SerbianSR.txt",
"TestNames_Thai.txt",
"Testnames_Russian.txt",
"th18057.txt",
"thesis.txt",
"vfear11a.txt",
]
};

View File

@ -2,15 +2,25 @@
use strict;
use Dataset;
my $TABLEATTR = 'BORDER="1" CELLPADDING="4" CELLSPACING="0"';
my $outType = "HTML";
my $html = "noName";
my $inTable;
my @headers;
my @timetypes = ("per iteration", "per operation", "events", "per event");
my @timetypes = ("mean per op", "error per op", "events", "per event");
my %raw;
my $current;
my $current = "";
my $exp = 0;
my $mult = 1e9; #use nanoseconds
my $perc = 100; #for percent
my $printEvents = 0;
my $legend = "<a name=\"Legend\">\n<h2>Table legend</h2></a><ul>";
my $legendDone = 0;
my %options;
my $operationIs = "operation";
my $eventIs = "event";
sub startTest {
$current = shift;
@ -18,22 +28,70 @@ sub startTest {
outputData($current);
}
sub printLeg {
if(!$legendDone) {
my $message;
foreach $message (@_) {
$legend .= "<li>".$message."</li>\n";
}
}
}
sub startTable {
my $printEvents = shift;
#my $printEvents = shift;
$inTable = 1;
my $i;
print HTML "<font size=3>";
print HTML "<table $TABLEATTR>\n";
if($#headers >= 0) {
my ($header, $i);
print HTML "<tr>";
print HTML "<th>Test Name</th>";
print HTML "<th>Operations</th>";
foreach $i (@timetypes) {
print HTML "<th><a href=\"#Test Name\">Test Name</a></th>";
print HTML "<th><a href=\"#Ops\">Ops</a></th>";
printLeg("<a name=\"Test Name\">Test Name</a> - name of the test as set by the test writer", "<a name=\"Ops\">Ops</a> - number of ".$operationIs."s per iteration");
if(!$printEvents) {
foreach $header (@headers) {
print HTML "<th>$header<br>$i</th>" unless ($i =~ /event/ && !$printEvents);
print HTML "<th><a href=\"#meanop $header\">$header<br>mean/op</a></th>";
print HTML "<th><a href=\"#errorop $header\">$header<br>error/op</a></th>";
printLeg("<a name=\"meanop $header\">$header mean/op</a> - mean time for $header per $operationIs");
printLeg("<a name=\"errorop $header\">$header error/op - error range for mean time");
}
}
for $i (1 .. $#headers) {
print HTML "<th><a href=\"#mean $i op\">ratio $i<br>mean/op</a></th>";
print HTML "<th><a href=\"#error $i op\">ratio $i<br>error/op</a></th>";
printLeg("<a name=\"mean $i op\">ratio $i mean/op</a> - ratio of per $operationIs time, calculated as: (($headers[0] - $headers[$i])/$headers[$i])*100%, mean value");
printLeg("<a name=\"error $i op\">ratio $i error/op</a> - error range of the above value");
}
if($printEvents) {
foreach $header (@headers) {
print HTML "<th><a href=\"#events $header\">$header<br>events</a></th>";
printLeg("<a name=\"events $header\">$header events</a> - number of ".$eventIs."s for $header per iteration");
}
foreach $header (@headers) {
print HTML "<th><a href=\"#mean ev $header\">$header<br>mean/ev</a></th>";
print HTML "<th><a href=\"#error ev $header\">$header<br>error/ev</a></th>";
printLeg("<a name=\"mean $header mean/op - mean time for $header per $eventIs");
printLeg("$header error/op - error range for mean time");
}
for $i (1 .. $#headers) {
print HTML "<th><a href=\"#mean $i ev\">ratio $i<br>mean/ev</a></th>";
print HTML "<th><a href=\"#error $i ev\">ratio $i<br>error/ev</a></th>";
printLeg("<a name=\"mean $i ev\">ratio $i mean/ev</a> - ratio of per $eventIs time, calculated as: (($headers[0] - $headers[$i])/$headers[$i])*100%, mean value");
printLeg("<a name=\"error $i ev\">ratio $i error/ev</a> - error range of the above value");
}
}
# foreach $i (@timetypes) {
# foreach $header (@headers) {
# print HTML "<th>$header<br>$i</th>" unless ($i =~ /event/ && !$printEvents);
# }
# }
print HTML "</tr>\n";
}
$legendDone = 1;
}
sub closeTable {
@ -41,6 +99,7 @@ sub closeTable {
undef $inTable;
print HTML "</tr>\n";
print HTML "</table>\n";
print HTML "</font>";
}
}
@ -54,14 +113,19 @@ sub newRow {
}
sub outputData {
my $message;
if($inTable) {
print HTML "<td>";
foreach $message (@_) {
print HTML "$message";
my $msg = shift;
my $align = shift;
print HTML "<td";
if($align) {
print HTML " align = $align>";
} else {
print HTML ">";
}
print HTML "$msg";
print HTML "</td>";
} else {
my $message;
foreach $message (@_) {
print HTML "$message";
}
@ -71,9 +135,15 @@ sub outputData {
sub setupOutput {
my $date = localtime;
my $options = shift;
my %options = %{ $options };
%options = %{ $options };
my $title = $options{ "title" };
my $headers = $options{ "headers" };
if($options{ "operationIs" }) {
$operationIs = $options{ "operationIs" };
}
if($options{ "eventIs" }) {
$eventIs = $options{ "eventIs" };
}
@headers = split(/ /, $headers);
my ($t, $rest);
($t, $rest) = split(/\.\w+/, $0);
@ -97,6 +167,14 @@ sub setupOutput {
<HTML>
<HEAD>
<TITLE>$title</TITLE>
<!--
<style>
<!--
td { text-align: "." }
td:before { content: "$" }
-->
</style>
-->
</HEAD>
<BODY>
EOF
@ -111,6 +189,8 @@ sub closeOutput {
if($inTable) {
closeTable;
}
$legend .= "</ul>\n";
print HTML $legend;
printRaw();
print HTML <<EOF;
</BODY>
@ -125,19 +205,55 @@ sub printRaw {
print HTML "<h2>Raw data</h2>";
my $key;
my $i;
my $j;
my $k;
print HTML "<table $TABLEATTR>\n";
for $key (sort keys %raw) {
print HTML $key;
foreach $i ( 0 .. $#{ $raw{$key} } ) {
print HTML " $i = $raw{$key}[$i]";
my $printkey = $key;
$printkey =~ s/\<br\>/ /;
if($printEvents) {
print HTML "<tr><td colspan = 7>$printkey</td></tr>\n"; # locale and data file
print HTML "<tr><th>test name</th><th>interesting arguments</th><th>iterations</th><th>operations</th><th>mean time (ns)</th><th>error (ns)</th><th>events</th></tr>\n";
} else {
print HTML "<tr><td colspan = 6>$printkey</td></tr>\n"; # locale and data file
print HTML "<tr><th>test name</th><th>interesting arguments</th><th>iterations</th><th>operations</th><th>mean time (ns)</th><th>error (ns)</th></tr>\n";
}
for $i ( $raw{$key} ) {
print HTML "<tr>";
for $j ( @$i ) {
my ($test, $args);
($test, $args) = split(/,/, shift(@$j));
print HTML "<td>".$test."</td>";
print HTML "<td>".$args."</td>";
#print HTML "<td>".shift(@$j)."</td>";
print HTML "<td align=\"right\">".shift(@$j)."</td>";
print HTML "<td align=\"right\">".shift(@$j)."</td>";
my @data = @{ shift(@$j) };
# for $k (@data) {
# print HTML "$k, ";
# }
my $ds = Dataset->new(@data);
print HTML "<td align=\"right\">".formatNumber(4, $mult, $ds->getMean)."</td><td align=\"right\">".formatNumber(4, $mult, $ds->getError)."</td>";
if($#{ $j } >= 0) {
print HTML "<td align=\"right\">".shift(@$j)."</td>";
}
print %raw;
print HTML "</tr>\n";
}
}
# print HTML "<br>\n";
}
# print %raw;
}
sub store {
$raw{$current}[$exp++] = [@_];
}
sub outputRow {
$raw{$current}[$exp++] = [@_];
#$raw{$current}[$exp++] = [@_];
my $testName = shift;
my @iterPerPass = @{shift(@_)};
my @noopers = @{shift(@_)};
@ -149,7 +265,8 @@ sub outputRow {
if(!$inTable) {
if(@noevents) {
debug("Have events header\n");
startTable(1);
$printEvents = 1;
startTable;
} else {
debug("No events header\n");
startTable;
@ -161,29 +278,43 @@ sub outputRow {
# Finished one row of results. Outputting
newRow;
outputData($testName);
outputData($testName, "LEFT");
#outputData($iterCount);
outputData($noopers[0]);
for $j ( 0 .. $#timedata ) {
my $perOperation = $timedata[$j]->divideByScalar($iterPerPass[$j]); # time per operation
#debug("Time per operation: ".formatSeconds(4, $perOperation->getMean, $perOperation->getError)."\n");
outputData(formatSeconds(4, $perOperation->getMean, $perOperation->getError));
}
outputData($noopers[0], "RIGHT");
if(!$printEvents) {
for $j ( 0 .. $#timedata ) {
my $perOperation = $timedata[$j]->divideByScalar($iterPerPass[$j]*$noopers[$j]); # time per operation
#debug("Time per operation: ".formatSeconds(4, $perOperation->getMean, $perOperation->getError)."\n");
outputData(formatSeconds(4, $perOperation->getMean, $perOperation->getError));
outputData(formatNumber(2, $mult, $perOperation->getMean), "RIGHT");
outputData(formatNumber(2, $mult, $perOperation->getError), "RIGHT");
}
if(@noevents) {
}
my $baseLinePO = $timedata[0]->divideByScalar($iterPerPass[0]*$noopers[0]);
for $j ( 1 .. $#timedata ) {
my $perOperation = $timedata[$j]->divideByScalar($iterPerPass[$j]*$noopers[$j]); # time per operation
my $ratio = $baseLinePO->subtract($perOperation);
$ratio = $ratio->divide($perOperation);
outputData(formatPercent(2, $ratio->getMean), "RIGHT");
outputData(formatPercent(2, $ratio->getError), "RIGHT");
}
if (@noevents) {
for $j ( 0 .. $#timedata ) {
outputData($noevents[$j]);
outputData($noevents[$j], "RIGHT");
}
for $j ( 0 .. $#timedata ) {
my $perEvent = $timedata[$j]->divideByScalar($iterPerPass[$j]*$noevents[$j]); # time per event
#debug("Time per operation: ".formatSeconds(4, $perEvent->getMean, $perEvent->getError)."\n");
outputData(formatSeconds(4, $perEvent->getMean, $perEvent->getError));
outputData(formatNumber(2, $mult, $perEvent->getMean), "RIGHT");
outputData(formatNumber(2, $mult, $perEvent->getError), "RIGHT");
}
my $baseLinePO = $timedata[0]->divideByScalar($iterPerPass[0]*$noevents[0]);
for $j ( 1 .. $#timedata ) {
my $perOperation = $timedata[$j]->divideByScalar($iterPerPass[$j]*$noevents[$j]); # time per operation
my $ratio = $baseLinePO->subtract($perOperation);
$ratio = $ratio->divide($perOperation);
outputData(formatPercent(2, $ratio->getMean), "RIGHT");
outputData(formatPercent(2, $ratio->getError), "RIGHT");
}
}
}

View File

@ -2,7 +2,7 @@
use strict;
use Dataset;
#use Dataset;
use Format;
use Output;
@ -108,7 +108,10 @@ sub compareLoop {
# first we calibrate. Use time from somewhere
# first test is used for calibration
($program, @argsAndTest) = split(/\ /, @{ $tests{$i} }[$j]);
my @res = measure1("$program -t $TIME -p $NUMPASSES $locAndData @argsAndTest");
my $commandLine = "$program -t $TIME -p $NUMPASSES $locAndData @argsAndTest";
#my $commandLine = "$program -i 5 -p $NUMPASSES $locAndData @argsAndTest";
my @res = measure1($commandLine);
store("$i, $program @argsAndTest", @res);
push(@iterPerPass, shift(@res));
push(@noopers, shift(@res));

View File

@ -7,14 +7,14 @@ use lib '../perldriver';
use PerfFramework;
my $options = {
"title"=>"BreakIterator performance: ICU 2.0 vs. ICU 2.4",
"headers"=>"ICU20 ICU24",
"operationIs"=>"File size in code points",
"timePerOperationIs"=>"Time per code point",
"passes"=>"3",
#"time"=>"1.1",
"title"=>"BreakIterator performance regression (ICU 2.0, 2.2 and 2.4)",
"headers"=>"ICU20 ICU22 ICU24",
"operationIs"=>"code point",
"eventIs"=>"break",
"passes"=>"10",
"time"=>"5",
#"outputType"=>"HTML",
"dataDir"=>"c:/src/perf/data",
"outputType"=>"HTML",
"outputDir"=>"../results"
};
@ -34,40 +34,41 @@ if(@_ >= 0) {
}
my $p1 = "ubrkperf20.exe";
my $p2 = "ubrkperf24.exe";
my $p2 = "ubrkperf22.exe";
my $p3 = "ubrkperf24.exe";
my $dataFiles = {
"en", ["thesis.txt",
# #"2drvb10.txt",
# #"ulyss10.txt",
# "nvsbl10.txt",
# "vfear11a.txt",
# "TestNames_Asian.txt",
# "TestNames_Chinese.txt",
#"2drvb10.txt",
#"ulyss10.txt",
"nvsbl10.txt",
"vfear11a.txt",
"TestNames_Asian.txt",
"TestNames_Chinese.txt",
"TestNames_Japanese.txt",
# "TestNames_Japanese_h.txt",
# "TestNames_Japanese_k.txt",
# "TestNames_Korean.txt",
"TestNames_Japanese_h.txt",
"TestNames_Japanese_k.txt",
"TestNames_Korean.txt",
"TestNames_Latin.txt",
# "TestNames_SerbianSH.txt",
# "TestNames_SerbianSR.txt",
# "TestNames_Thai.txt",
# "Testnames_Russian.txt",
"TestNames_SerbianSH.txt",
"TestNames_SerbianSR.txt",
"TestNames_Thai.txt",
"Testnames_Russian.txt",
],
#"th", ["TestNames_Thai.txt", "th18057.txt"]
};
my $tests = {
"TestForwardChar", ["$p1 $m1 TestICUForward", "$p2 $m1 TestICUForward"],
"TestForwardWord", ["$p1 $m2 TestICUForward", "$p2 $m2 TestICUForward"],
#"TestForwardLine", ["$p1 $m3 TestICUForward", "$p2 $m3 TestICUForward"],
#"TestForwardSentence", ["$p1 $m4 TestICUForward", "$p2 $m4 TestICUForward"],
"TestForwardChar", ["$p1 $m1 TestICUForward", "$p2 $m1 TestICUForward", "$p3 $m1 TestICUForward"],
"TestForwardWord", ["$p1 $m2 TestICUForward", "$p2 $m2 TestICUForward", "$p3 $m2 TestICUForward"],
"TestForwardLine", ["$p1 $m3 TestICUForward", "$p2 $m3 TestICUForward", "$p3 $m3 TestICUForward"],
"TestForwardSentence", ["$p1 $m4 TestICUForward", "$p2 $m4 TestICUForward", "$p3 $m4 TestICUForward"],
#"TestIsBoundChar", ["$p1 $m1 TestICUIsBound", "$p2 $m1 TestICUIsBound"],
#"TestIsBoundWord", ["$p1 $m2 TestICUIsBound", "$p2 $m2 TestICUIsBound"],
#"TestIsBoundLine", ["$p1 $m3 TestICUIsBound", "$p2 $m3 TestICUIsBound"],
#"TestIsBoundSentence", ["$p1 $m4 TestICUIsBound", "$p2 $m4 TestICUIsBound"],
"TestIsBoundChar", ["$p1 $m1 TestICUIsBound", "$p2 $m1 TestICUIsBound", "$p3 $m1 TestICUIsBound"],
"TestIsBoundWord", ["$p1 $m2 TestICUIsBound", "$p2 $m2 TestICUIsBound", "$p3 $m2 TestICUIsBound"],
"TestIsBoundLine", ["$p1 $m3 TestICUIsBound", "$p2 $m3 TestICUIsBound", "$p3 $m3 TestICUIsBound"],
"TestIsBoundSentence", ["$p1 $m4 TestICUIsBound", "$p2 $m4 TestICUIsBound", "$p3 $m4 TestICUIsBound"],
};