Skip to content

Commit a4e04a7

Browse files
dkechagdkechag
committed
v2.9 --duration option
1 parent 77e466e commit a4e04a7

File tree

7 files changed

+110
-78
lines changed

7 files changed

+110
-78
lines changed

‎.github/workflows/ci.yml

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ jobs:
1111
fail-fast: false
1212
matrix:
1313
perl-version:
14+
- '5.40'
1415
- '5.38'
1516
- '5.36'
1617
- '5.28'

‎Changes

+5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
Revision history for Benchmark-DKbench
22

3+
2.9 2024-07-29
4+
- Added --duration option.
5+
- Added min/max scores to the summary.
6+
- Updated version base for some modules.
7+
38
2.8 2024-06-24
49
- Added suite_calc.
510

‎README.md

+22-21
Original file line numberDiff line numberDiff line change
@@ -95,34 +95,35 @@ options to control number of threads, iterations, which benchmarks to run etc:
9595
dkbench [options]
9696

9797
Options:
98-
--threads <i>, -j <i> : Number of benchmark threads (default is 1).
99-
--multi, -m : Multi-threaded using all your CPU cores/threads.
100-
--max_threads <i> : Override the cpu detection to specify max cpu threads.
101-
--iter <i>, -i <i> : Number of suite iterations (with min/max/avg at the end).
102-
--stdev : Show relative standard deviation (for iter > 1).
103-
--include <regex> : Run only benchmarks that match regex.
104-
--exclude <regex> : Do not run benchmarks that match regex.
105-
--time, -t : Report time (sec) instead of score.
106-
--quick, -q : Quick benchmark run (implies -t).
107-
--no_mce : Do not run under MCE::Loop (implies -j 1).
108-
--scale <i>, -s <i> : Scale the bench workload by x times (incompatible with -q).
109-
--skip_bio : Skip BioPerl benchmarks.
110-
--skip_prove : Skip Moose prove benchmark.
111-
--time_piece : Run optional Time::Piece benchmark (see benchmark details).
112-
--bio_codons : Run optional BioPerl Codons benchmark (does not scale well).
113-
--sleep <i> : Sleep for <i> secs after each benchmark.
114-
--setup : Download the Genbank data to enable the BioPerl tests.
115-
--datapath <path> : Override the path where the expected benchmark data is found.
116-
--ver <num> : Skip benchmarks added after the specified version.
117-
--help -h : Show basic help and exit.
98+
--threads <i>, -j <i> : Number of benchmark threads (default is 1).
99+
--multi, -m : Multi-threaded using all your CPU cores/threads.
100+
--max_threads <i> : Override the cpu detection to specify max cpu threads.
101+
--iter <i>, -i <i> : Number of suite iterations (with min/max/avg at the end).
102+
--stdev : Show relative standard deviation (for iter > 1).
103+
--include <regex> : Run only benchmarks that match regex.
104+
--exclude <regex> : Do not run benchmarks that match regex.
105+
--time, -t : Report time (sec) instead of score.
106+
--quick, -q : Quick benchmark run (implies -t).
107+
--no_mce : Do not run under MCE::Loop (implies -j 1).
108+
--scale <i>, -s <i> : Scale the bench workload by x times (incompatible with -q).
109+
--skip_bio : Skip BioPerl benchmarks.
110+
--skip_prove : Skip Moose prove benchmark.
111+
--time_piece : Run optional Time::Piece benchmark (see benchmark details).
112+
--bio_codons : Run optional BioPerl Codons benchmark (does not scale well).
113+
--sleep <i> : Sleep for <i> secs after each benchmark.
114+
--duration <i>, -d <i> : Minimum duration in seconds for suite run.
115+
--setup : Download the Genbank data to enable the BioPerl tests.
116+
--datapath <path> : Override the path where the expected benchmark data is found.
117+
--ver <num> : Skip benchmarks added after the specified version.
118+
--help -h : Show basic help and exit.
118119

119120
The default run (no options) will run all the benchmarks both single-threaded and
120121
multi-threaded (using all detected CPU cores/hyperthreads) and show you scores and
121122
multi vs single threaded scalability.
122123

123124
The scores are calibrated such that a reference CPU (Intel Xeon Platinum 8481C -
124125
Sapphire Rapids) would achieve a score of 1000 in a single-core benchmark run using
125-
the default software configuration (Linux/Perl 5.36.0 built with multiplicity and
126+
the default software configuration (Linux/Perl 5.36+ built with multiplicity and
126127
threads, with reference CPAN module versions). Perl built without thread support and
127128
multi(plicity) will be a bit faster (usually in the order of ~3-4%), while older Perl
128129
versions will most likely be slower. Different CPAN module versions will also impact

‎dkbench

+25-23
Original file line numberDiff line numberDiff line change
@@ -17,26 +17,27 @@ See POD on the main module for info:
1717
dkbench [options]
1818
1919
Options:
20-
--threads <i>, -j <i> : Number of benchmark threads (default is 1).
21-
--multi, -m : Multi-threaded using all your CPU cores/threads.
22-
--max_threads <i> : Override the cpu detection to specify max cpu threads.
23-
--iter <i>, -i <i> : Number of suite iterations (with min/max/avg at the end).
24-
--stdev : Show relative standard deviation (for iter > 1).
25-
--include <regex> : Run only benchmarks that match regex.
26-
--exclude <regex> : Do not run benchmarks that match regex.
27-
--time, -t : Report time (sec) instead of score.
28-
--quick, -q : Quick benchmark run (implies -t).
29-
--no_mce : Do not run under MCE::Loop (implies -j 1).
30-
--scale <i>, -s <i> : Scale the bench workload by x times (incompatible with -q).
31-
--skip_bio : Skip BioPerl benchmarks.
32-
--skip_prove : Skip Moose prove benchmark.
33-
--time_piece : Run optional Time::Piece benchmark (see benchmark details).
34-
--bio_codons : Run optional BioPerl Codons benchmark (does not scale well).
35-
--sleep <i> : Sleep for <i> secs after each benchmark.
36-
--setup : Download the Genbank data to enable the BioPerl tests.
37-
--datapath <path> : Override the path where the expected benchmark data is found.
38-
--ver <num> : Skip benchmarks added after the specified version.
39-
--help -h : Show basic help and exit.
20+
--threads <i>, -j <i> : Number of benchmark threads (default is 1).
21+
--multi, -m : Multi-threaded using all your CPU cores/threads.
22+
--max_threads <i> : Override the cpu detection to specify max cpu threads.
23+
--iter <i>, -i <i> : Number of suite iterations (with min/max/avg at the end).
24+
--stdev : Show relative standard deviation (for iter > 1).
25+
--include <regex> : Run only benchmarks that match regex.
26+
--exclude <regex> : Do not run benchmarks that match regex.
27+
--time, -t : Report time (sec) instead of score.
28+
--quick, -q : Quick benchmark run (implies -t).
29+
--no_mce : Do not run under MCE::Loop (implies -j 1).
30+
--scale <i>, -s <i> : Scale the bench workload by x times (incompatible with -q).
31+
--skip_bio : Skip BioPerl benchmarks.
32+
--skip_prove : Skip Moose prove benchmark.
33+
--time_piece : Run optional Time::Piece benchmark (see benchmark details).
34+
--bio_codons : Run optional BioPerl Codons benchmark (does not scale well).
35+
--sleep <i> : Sleep for <i> secs after each benchmark.
36+
--duration <i>, -d <i> : Minimum duration in seconds for suite run.
37+
--setup : Download the Genbank data to enable the BioPerl tests.
38+
--datapath <path> : Override the path where the expected benchmark data is found.
39+
--ver <num> : Skip benchmarks added after the specified version.
40+
--help -h : Show basic help and exit.
4041
4142
The default run (no options) will run all the benchmarks both single-threaded and
4243
multi-threaded (using all detected CPU cores/hyperthreads) and show you scores and
@@ -79,6 +80,7 @@ GetOptions (
7980
'ver=s',
8081
'setup',
8182
'datapath=s',
83+
'duration|d=i',
8284
'help|h',
8385
);
8486

@@ -106,8 +108,8 @@ if ($opt{threads} || !$max_threads || $max_threads == 1) {
106108
}
107109

108110
sub check_kit {
109-
warn "! Perl version v5.36 recommended as a comparison base\n"
110-
unless $^V =~ /v5.36.\d/;
111+
warn "! Perl version v5.36 or later suggested as a comparison base\n"
112+
unless $] >= 5.036000;
111113

112114
my %mod_ver = Benchmark::DKbench::Setup::cpan_versions();
113115

@@ -116,7 +118,7 @@ sub check_kit {
116118
eval "use $module";
117119
my $ver = eval "\$${module}::VERSION" || 'none';
118120
unless ($ver eq $mod_ver{$module}) {
119-
my $msg = "! $module $mod_ver{$module} recommended as a comparison base ($ver found).";
121+
my $msg = "! $module $mod_ver{$module} suggested as a comparison base ($ver found).";
120122
$msg .= " Older versions may show benchmark failures."
121123
if $mod_ver{$module} cmp $ver; # $ver is less
122124
if ($module =~ /^Bio/ && $ver eq 'none') {

‎lib/Benchmark/DKbench.pm

+49-29
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ use Text::Levenshtein::XS;
3838

3939
my $mono_clock = $^O !~ /win/i || $Time::HiRes::VERSION >= 1.9764;
4040

41-
our $VERSION = '2.8';
41+
our $VERSION = '2.9';
4242
our @EXPORT = qw(system_identity suite_run calc_scalability suite_calc);
4343
our $datadir = dist_dir("Benchmark-DKbench");
4444

@@ -139,34 +139,35 @@ options to control number of threads, iterations, which benchmarks to run etc:
139139
dkbench [options]
140140
141141
Options:
142-
--threads <i>, -j <i> : Number of benchmark threads (default is 1).
143-
--multi, -m : Multi-threaded using all your CPU cores/threads.
144-
--max_threads <i> : Override the cpu detection to specify max cpu threads.
145-
--iter <i>, -i <i> : Number of suite iterations (with min/max/avg at the end).
146-
--stdev : Show relative standard deviation (for iter > 1).
147-
--include <regex> : Run only benchmarks that match regex.
148-
--exclude <regex> : Do not run benchmarks that match regex.
149-
--time, -t : Report time (sec) instead of score.
150-
--quick, -q : Quick benchmark run (implies -t).
151-
--no_mce : Do not run under MCE::Loop (implies -j 1).
152-
--scale <i>, -s <i> : Scale the bench workload by x times (incompatible with -q).
153-
--skip_bio : Skip BioPerl benchmarks.
154-
--skip_prove : Skip Moose prove benchmark.
155-
--time_piece : Run optional Time::Piece benchmark (see benchmark details).
156-
--bio_codons : Run optional BioPerl Codons benchmark (does not scale well).
157-
--sleep <i> : Sleep for <i> secs after each benchmark.
158-
--setup : Download the Genbank data to enable the BioPerl tests.
159-
--datapath <path> : Override the path where the expected benchmark data is found.
160-
--ver <num> : Skip benchmarks added after the specified version.
161-
--help -h : Show basic help and exit.
142+
--threads <i>, -j <i> : Number of benchmark threads (default is 1).
143+
--multi, -m : Multi-threaded using all your CPU cores/threads.
144+
--max_threads <i> : Override the cpu detection to specify max cpu threads.
145+
--iter <i>, -i <i> : Number of suite iterations (with min/max/avg at the end).
146+
--stdev : Show relative standard deviation (for iter > 1).
147+
--include <regex> : Run only benchmarks that match regex.
148+
--exclude <regex> : Do not run benchmarks that match regex.
149+
--time, -t : Report time (sec) instead of score.
150+
--quick, -q : Quick benchmark run (implies -t).
151+
--no_mce : Do not run under MCE::Loop (implies -j 1).
152+
--scale <i>, -s <i> : Scale the bench workload by x times (incompatible with -q).
153+
--skip_bio : Skip BioPerl benchmarks.
154+
--skip_prove : Skip Moose prove benchmark.
155+
--time_piece : Run optional Time::Piece benchmark (see benchmark details).
156+
--bio_codons : Run optional BioPerl Codons benchmark (does not scale well).
157+
--sleep <i> : Sleep for <i> secs after each benchmark.
158+
--duration <i>, -d <i> : Minimum duration in seconds for suite run.
159+
--setup : Download the Genbank data to enable the BioPerl tests.
160+
--datapath <path> : Override the path where the expected benchmark data is found.
161+
--ver <num> : Skip benchmarks added after the specified version.
162+
--help -h : Show basic help and exit.
162163
163164
The default run (no options) will run all the benchmarks both single-threaded and
164165
multi-threaded (using all detected CPU cores/hyperthreads) and show you scores and
165166
multi vs single threaded scalability.
166167
167168
The scores are calibrated such that a reference CPU (Intel Xeon Platinum 8481C -
168169
Sapphire Rapids) would achieve a score of 1000 in a single-core benchmark run using
169-
the default software configuration (Linux/Perl 5.36.0 built with multiplicity and
170+
the default software configuration (Linux/Perl 5.36+ built with multiplicity and
170171
threads, with reference CPAN module versions). Perl built without thread support and
171172
multi(plicity) will be a bit faster (usually in the order of ~3-4%), while older Perl
172173
versions will most likely be slower. Different CPAN module versions will also impact
@@ -570,9 +571,24 @@ sub suite_run {
570571
chunk_size => 1,
571572
} unless $opt->{no_mce};
572573

573-
foreach (1..$opt->{iter}) {
574-
print "Iteration $_ of $opt->{iter}...\n" if $opt->{iter} > 1;
575-
run_iteration($opt, \%stats);
574+
if ($opt->{duration}) {
575+
my $t0 = _get_time();
576+
my $cnt = 0;
577+
my $t = 0;
578+
while ($t < $opt->{duration}) {
579+
$cnt++;
580+
print "Iteration $cnt (".int($t+0.5)."s of $opt->{duration}s)...\n";
581+
run_iteration($opt, \%stats);
582+
$t = _get_time()-$t0;
583+
}
584+
$opt->{iter} = $cnt;
585+
$stats{_opt}->{iter} = $cnt;
586+
$opt->{duration} = 0;
587+
} else {
588+
foreach (1..$opt->{iter}) {
589+
print "Iteration $_ of $opt->{iter}...\n" if $opt->{iter} > 1;
590+
run_iteration($opt, \%stats);
591+
}
576592
}
577593

578594
total_stats($opt, \%stats) if $opt->{iter} > 1;
@@ -632,16 +648,20 @@ sub calc_scalability {
632648
}
633649
die "No bench times recorded" unless @perf;
634650
print (("-"x40)."\n");
635-
my $avg1 = min_max_avg($stats1->{_total}->{$display});
636-
my $avg2 = min_max_avg($stats2->{_total}->{$display});
651+
my @avg1 = min_max_avg($stats1->{_total}->{$display});
652+
my @avg2 = min_max_avg($stats2->{_total}->{$display});
637653
print "DKbench summary ($cnt benchmark";
638654
print "s" if $cnt > 1;
639655
print " x$opt->{scale} scale" if $opt->{scale} > 1;
640656
print ", $opt->{iter} iterations" if $opt->{iter} > 1;
641657
print ", $opt2->{threads} threads):\n";
642658
$opt->{f} .= "s" if $opt->{time};
643-
print pad_to("Single:").sprintf($opt->{f}, $avg1)."\n";
644-
print pad_to("Multi:").sprintf($opt->{f}, $avg2)."\n";
659+
my $f = $opt->{time} ? '%.3f' : '%.0f';
660+
$f = $opt->{iter} > 1 ? "$opt->{f}\t($f - $f)" : $opt->{f};
661+
@avg1 = $opt->{iter} > 1 ? ($avg1[2], $avg1[0], $avg1[1]) : ($avg1[2]);
662+
@avg2 = $opt->{iter} > 1 ? ($avg2[2], $avg2[0], $avg2[1]) : ($avg2[2]);
663+
print pad_to("Single:").sprintf($f, @avg1)."\n";
664+
print pad_to("Multi:").sprintf($f, @avg2)."\n";
645665
my @newperf = Benchmark::DKbench::drop_outliers(\@perf, -1);
646666
my @newscal = Benchmark::DKbench::drop_outliers(\@scal, -1);
647667
@perf = min_max_avg(\@newperf);

‎lib/Benchmark/DKbench/Setup.pm

+4-4
Original file line numberDiff line numberDiff line change
@@ -23,21 +23,21 @@ See POD on the main module for info.
2323

2424
sub cpan_packages {
2525
return qw(
26-
O/OA/OALDERS/HTML-Parser-3.81.tar.gz
26+
O/OA/OALDERS/HTML-Parser-3.82.tar.gz
2727
K/KE/KENTNL/HTML-Tree-5.07.tar.gz
2828
N/NI/NIGELM/HTML-Formatter-2.16.tar.gz
2929
K/KA/KAMELKEV/CSS-Inliner-4018.tar.gz
3030
C/CJ/CJFIELDS/BioPerl-1.7.8.tar.gz
31-
E/ET/ETHER/Moose-2.2206.tar.gz
31+
E/ET/ETHER/Moose-2.2207.tar.gz
3232
D/DR/DROLSKY/DateTime-TimeZone-2.62.tar.gz
3333
D/DR/DROLSKY/DateTime-1.65.tar.gz
3434
D/DK/DKECHAG/Astro-Coord-Precession-0.03.tar.gz
3535
D/DK/DKECHAG/Astro-Coord-Constellations-0.01.tar.gz
3636
D/DK/DKECHAG/Image-PHash-0.3.tar.gz
3737
D/DK/DKECHAG/Math-DCT-0.04.tar.gz
3838
D/DK/DKECHAG/SQL-Inserter-0.04.tar.gz
39-
M/MI/MIK/CryptX-0.078.tar.gz
40-
M/MI/MIK/Crypt-JWT-0.034.tar.gz
39+
M/MI/MIK/CryptX-0.080.tar.gz
40+
M/MI/MIK/Crypt-JWT-0.035.tar.gz
4141
M/ML/MLEHMANN/JSON-XS-4.03.tar.gz
4242
L/LE/LETO/Math-MatrixReal-2.13.tar.gz
4343
R/RI/RIBASUSHI/SQL-Abstract-Classic-1.91.tar.gz

‎t/simple.t

+4-1
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,8 @@ diag $std[0];
5656
%stats2 = %stats1;
5757
$stats2{_opt} = {%{$stats1{_opt}}};
5858
$stats2{_opt}->{threads} = 2;
59-
calc_scalability(\%stats1, \%stats2);
59+
@std = capture {calc_scalability(\%stats1, \%stats2)};
60+
like($std[0], qr/Single:\s*\d+\s*\(\d+ - \d+/, 'Min Max');
6061

6162
$stats1{_opt}->{iter} = 1;
6263
@std = capture {calc_scalability(\%stats1, \%stats2)};
@@ -92,12 +93,14 @@ my $datadir = dist_dir("Benchmark-DKbench");
9293
datapath => $datadir,
9394
time => 1,
9495
iter => 1,
96+
duration => 1,
9597
no_mce => 1,
9698
sleep => 1,
9799
include => 'prove',
98100
}
99101
)
100102
};
101103
like($std[0], qr/Overall Time/, 'Single');
104+
like($std[0], qr/0s of 1s/, 'Duration');
102105

103106
done_testing();

0 commit comments

Comments
 (0)