Agg script, pass_only

.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: CI
+
+on:
+  push:
+
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        perl-version:
+          - '5.40'
+          - '5.36'
+          - '5.32'
+          - '5.28'
+          - '5.24'
+          - '5.20'
+          - '5.16'
+          - '5.12'
+          - '5.8'
+
+    container:
+      image: perldocker/perl-tester:${{ matrix.perl-version }}
+
+    steps:
+      - uses: actions/checkout@main
+        with:
+            submodules: recursive
+      - run: perl -V
+      - run: cpanm --notest --installdeps --verbose .
+      - run: perl Makefile.PL
+      - run: make
+      - run: prove -wlvmb t

.gitignore

@@ -0,0 +1,19 @@
+MANIFEST.bak
+Makefile
+Makefile.old
+Build
+Build.bat
+META.*
+MYMETA.*
+.build/
+_build/
+cover_db/
+blib/
+inc/
+.lwpcookies
+.last_cover_stats
+nytprof.out
+pod2htm*.tmp
+pm_to_blib
+Test2-Aggregate-*
+Test2-Aggregate-*.tar.gz

Changes

@@ -1,5 +1,15 @@
 Revision history for Test2-Aggregate
 
+0.18    2024-07-03
+        agg helper script.
+        aggit helper script.
+        'pass_only' option.
+        'bail_on_fail' option.
+        Write stats per test file (repeat==1) to avoid losing them on SIG exit.
+        Test names without $root for stats, 'absolute' option for old behaviour.
+        Add './' to start of paths when required, 'no_dot' option for old behaviour.
+        'reverse' applied after 'sort'.
+
 0.17    2022-02-10
         Ignore empty lines in list files.
 

MANIFEST

@@ -1,5 +1,8 @@
+agg
+aggit
 Changes
 lib/Test2/Aggregate.pm
+lib/Test2/Aggregate/Helper.pm
 Makefile.PL
 MANIFEST			This list of files
 MANIFEST.SKIP

MANIFEST.SKIP

@@ -7,7 +7,9 @@ MYMETA.*
 _build/
 cover_db/
 blib/
+.github/
 .git/
+.gitignore/
 inc/
 .lwpcookies
 .last_cover_stats

Makefile.PL

@@ -19,6 +19,7 @@ my %WriteMakefileArgs = (
     CONFIGURE_REQUIRES => {
         'ExtUtils::MakeMaker' => '0',
     },
+    EXE_FILES     => ["agg", "aggit"],
     TEST_REQUIRES => {
         'Test::Output' => '0',
         'Test2::V0'    => '0'

README

@@ -23,12 +23,6 @@ You can also look for information at:
     RT, CPAN's request tracker (report bugs here)
         https://rt.cpan.org/NoAuth/Bugs.html?Dist=Test2-Aggregate
 
-    AnnoCPAN, Annotated CPAN documentation
-        http://annocpan.org/dist/Test2-Aggregate
-
-    CPAN Ratings
-        https://cpanratings.perl.org/d/Test2-Aggregate
-
     Search CPAN
         https://metacpan.org/release/Test2-Aggregate
 

README.pod

@@ -15,7 +15,7 @@ Test2::Aggregate - Aggregate tests for increased speed
 
 =head1 VERSION
 
-Version 0.17
+Version 0.18
 
 =head1 DESCRIPTION
 
@@ -64,6 +64,7 @@ have less issues with L<Test2::Suite> (see notes).
         override      => \%override,          # optional, requires Sub::Override
         stats_output  => $stats_output_path,  # optional
         extend_stats  => 0,                   # optional
+        pass_only     => 0,                   # optional
         test_warnings => 0,                   # optional
         allow_errors  => 0,                   # optional
         pre_eval      => $code_to_eval,       # optional
@@ -224,6 +225,12 @@ but still allow additions in future versions that will only be written with the
 C<extend_stats> option enabled.
 Additions with C<extend_stats> as of the current version:
 
+=item * C<pass_only> (optional)
+
+Modifies C<stats_output> by making it only print out a list of passing tests.
+Useful for creating lists of aggregateable tests.
+Has no effect if C<stats_output> is not defined.
+
 =over 4
 
 - starting date/time in ISO_8601.
@@ -315,47 +322,67 @@ disable warnings on redefines only for tests that run aggregated:
 Another idea is to make the test die when it is run under the aggregator, if, at
 design time, you know it is not supposed to run aggregated.
 
+=head2 agg helper script
+
+ agg [options] <file/dir ...>
+
+Pass a list of Perl test files/directories and they will run aggregated via yath
+(or prove if specified).
+
+ Options:
+ --out <s>,          -o <s> : Specify the test file to be created (tmp file by default).
+ --prove,            -p     : Force prove (default is yath if detected).
+ --verbose,          -v     : Verbose (passed to yath/prove)
+ --include <s>,      -I <s> : Library paths to include.
+ --test_warnings,    -w     : Fail tests on warnings.
+ --test_bundle <s>,  -t     : Test bundle (default: Test2::V0). Can be comma-separated list.
+ --pass_list <s>,    -l <s> : Output directory for list of 100% passing tests.
+ --stats_output <s>, -s <s> : Stats output directory (does not combine with pass_list).
+ --help              -h     : Show basic help and exit.
+
 =head2 Example aggregating strategy
 
 There are many approaches you could do to use C<Test2::Aggregate> with an existing
-test suite, so for example you can start by making a list of the test files you
-are trying to aggregate:
+test suite, usually involving an iterative process of trying to run several tests
+aggregated, seeing if you can fix the failing ones, otherwise you remove them from
+the aggregation etc.
 
- find t -name '*.t' > all.lst
+This process can be done with the help of the C<agg> script. For example, to try
+all tests under C<t/> aggregated and a list of passing tests put under the C<pass>
+directory you would do:
 
-If you have a substantial test suite, perhaps try with a portion of it (a subdir?)
-instead of the entire suite. In any case, try running them aggregated like this:
+ > agg -p pass t
 
- use Test2::Aggregate;
- use Test2::V0; # Or Test::More;
+If the run completes, you have a "starting point" - i.e. a .txt list that can run
+under the aggregator with the C<lists> option:
 
- my $stats = Test2::Aggregate::run_tests(
-    lists => ['all.lst'],
+ Test2::Aggregate::run_tests(
+     lists => ['pass/name_of_file.txt']
  );
 
- open OUT, ">pass.lst";
- foreach my $test (sort {$stats->{$a}->{test_no} <=> $stats->{$b}->{test_no}} keys %$stats) {
-     print OUT "$test\n" if $stats->{$test}->{pass_perc};
- }
- close OUT;
-
- done_testing();
+If the run does not complete (e.g. signal 11 on some test), try fewer tests by
+choosing just a subdirectory. If that's not possible, you'll probably have to go
+to the more manual method of getting a full list of your tests
+(C<find t -name '*.t' E<gt> all.lst>) then trying to run parts of it, again with
+the C<lists> option.
 
-Run the above with C<prove> or C<yath> in verbose mode, so that in case the run
-hangs (it can happen), you can see where it did so and edit C<all.lst> removing
-the offending test.
-
-If the run completes, you have a "starting point" - i.e. a list that can run under
-the aggregator in C<pass.lst>.
-You can try adding back some of the failed tests - test failures can be cascading,
-so some might be passing if added back, or have small issues you can address.
+After you have a starting point, you can try see if there is an obvious reason some
+tests fail and address it to add them back to the pass list. You can even try adding
+back some of the failed tests that were not among the first to fail - test failures
+can sometimes be cascading, so some might be passing if added back, or have small
+issues you can address.
 
 Try adding C<test_warnings =E<gt> 1> to C<run_tests> to fix warnings as well, unless
 it is common for your tests to have C<STDERR> output.
 
-To have your entire suite run aggregated tests together once and not repeat them
-along with the other, non-aggregated, tests, it is a good idea to use the
-C<--exclude-list> option of the C<Test2::Harness>.
+In the end, you will end up with part of your tests aggregated in (multiple if you
+want to run them in parallel) list files, with the rest of your tests to be run
+non-aggregated.
+
+You don't actually have to move and separate aggregated/non-aggregated files when
+using lists, you can still have your entire suite run the aggregated tests once and
+not repeat them along with the other, non-aggregated tests, by taking advantage of
+the C<--exclude-list> option of the C<Test2::Harness>.
 
 Hopefully your tests can run in parallel (C<prove/yath -j>), in which case you
 would split your aggregated tests into multiple lists to have them run in parallel.

agg

@@ -0,0 +1,85 @@
+#!/usr/bin/env perl
+
+=head1 NAME
+
+agg - Test2::Aggregate harness wrapper / helper script
+
+=head1 DESCRIPTION
+
+Pass a list of Perl test files/directories and they will run aggregated via yath
+(or prove if specified). It is not meant to be used for .t files that use Test2::Aggregate
+themselves.
+
+It is useful either to speed up a test run for tests you know can be aggregated,
+or to check whether specific tests can run successfully under Test2::Aggregate.
+
+=head1 SYNOPSIS
+
+ agg [options] <file/dir1 ...>
+
+ Options:
+ --out <s>,          -o <s>  : Specify the test file to be created (tmp file by default).
+ --lists,            -l      : Use files specified as lists.
+ --prove,            -p      : Force prove (default is yath/Test2 if installed).
+ --verbose,          -v      : Verbose (passed to yath/prove)
+ --absolute,         -a      : Use absolute paths in generated files. Disabled with -r.
+ --root <s>          -r <s>  : Define a custom root dir.
+ --include <s>,      -I <s>  : Library paths to include.
+ --test_warnings,    -w      : Fail tests on warnings.
+ --test_bundle <s>,  -t      : Test bundle (def: Test2::V0 or Test::More if -p). Can be comma-separated list.
+ --pass_output <s>,  -po <s> : Output directory for list of 100% passing tests.
+ --reverse                   : Reverse the test order.
+ --shuffle                   : Shuffle the test order.
+ --sort                      : Run the tests alphabetically.
+ --stats_output <s>, -so <s> : Stats output directory (does not combine with pass_output).
+ --help              -h      : Show basic help and exit.
+
+=cut
+
+use strict;
+use warnings;
+
+use lib 'lib';
+
+use Getopt::Long;
+use Path::Tiny;
+use Pod::Usage;
+use Test2::Aggregate::Helper;
+
+my %opt = (framework => 'Test2::V0');
+GetOptions(
+    \%opt,
+    'absolute|a',
+    'help|h',
+    'include|I=s',
+    'lists|l',
+    'out|o=s',
+    'pass_output|po=s',
+    'prove|p',
+    'reverse',
+    'root|r=s',
+    'shuffle',
+    'sort',
+    'stats_output|so=s',
+    'test_bundle|t=s',
+    'test_warnings|w',
+    'verbose|v',
+);
+
+pod2usage({-verbose => 1, -output => \*STDOUT, -noperldoc => 1})
+    if $opt{help} || !@ARGV;
+
+Test2::Aggregate::Helper::setup_options(\%opt);
+
+$opt{file} = $opt{out} ? path($opt{out}.'.t') : Path::Tiny->tempfile(
+    TEMPLATE => "aggXXXXXX",
+    SUFFIX   => '.t',
+    UNLINK   => 0
+);
+
+print "Writing output to $opt{file}\n" if $opt{verbose};
+
+my $code = Test2::Aggregate::Helper::generate_code(\%opt, @ARGV);
+$opt{file}->spew_utf8($code);
+
+exec(Test2::Aggregate::Helper::create_command(\%opt, $opt{file}));