Skip to content

Commit ed69537

Browse files
felixhandtefelixhandte
authored
Merge pull request #3623 from felixhandte/compile-out-compressors
Allow Build-Time Exclusion of Individual Compression Strategies
2 parents 7806d80 + 1b65803 commit ed69537

13 files changed

+426
-158
lines changed

‎.github/workflows/dev-short-tests.yml

+2
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,8 @@ jobs:
348348
make clean && make check MOREFLAGS="-Werror -DHUF_FORCE_DECOMPRESS_X2 -DZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG"
349349
make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
350350
make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
351+
make clean && make check ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP=1 MOREFLAGS="-Werror"
352+
make clean && make check ZSTD_LIB_EXCLUDE_COMPRESSORS_GREEDY_AND_UP=1 MOREFLAGS="-Werror"
351353
352354
dynamic-bmi2:
353355
runs-on: ubuntu-latest

‎lib/README.md

+9
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,15 @@ The file structure is designed to make this selection manually achievable for an
119119
binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and
120120
`ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` (implied by `ZSTD_LIB_MINIFY`).
121121

122+
On the compressor side, Zstd's compression levels map to several internal
123+
strategies. In environments where the higher compression levels aren't used,
124+
it is possible to exclude all but the fastest strategy with
125+
`ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP=1`. (Note that this will change
126+
the behavior of the default compression level.) Or if you want to retain the
127+
default compressor as well, you can set
128+
`ZSTD_LIB_EXCLUDE_COMPRESSORS_GREEDY_AND_UP=1`, at the cost of an additional
129+
~20KB or so.
130+
122131
For squeezing the last ounce of size out, you can also define
123132
`ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`,
124133
which removes the error messages that are otherwise returned by

‎lib/compress/zstd_compress.c

+117-45
Original file line numberDiff line numberDiff line change
@@ -1467,6 +1467,48 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
14671467
const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1);
14681468
assert(ZSTD_checkCParams(cPar)==0);
14691469

1470+
/* Cascade the selected strategy down to the next-highest one built into
1471+
* this binary. */
1472+
#ifdef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR
1473+
if (cPar.strategy == ZSTD_btultra2) {
1474+
cPar.strategy = ZSTD_btultra;
1475+
}
1476+
if (cPar.strategy == ZSTD_btultra) {
1477+
cPar.strategy = ZSTD_btopt;
1478+
}
1479+
#endif
1480+
#ifdef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
1481+
if (cPar.strategy == ZSTD_btopt) {
1482+
cPar.strategy = ZSTD_btlazy2;
1483+
}
1484+
#endif
1485+
#ifdef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR
1486+
if (cPar.strategy == ZSTD_btlazy2) {
1487+
cPar.strategy = ZSTD_lazy2;
1488+
}
1489+
#endif
1490+
#ifdef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR
1491+
if (cPar.strategy == ZSTD_lazy2) {
1492+
cPar.strategy = ZSTD_lazy;
1493+
}
1494+
#endif
1495+
#ifdef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR
1496+
if (cPar.strategy == ZSTD_lazy) {
1497+
cPar.strategy = ZSTD_greedy;
1498+
}
1499+
#endif
1500+
#ifdef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR
1501+
if (cPar.strategy == ZSTD_greedy) {
1502+
cPar.strategy = ZSTD_dfast;
1503+
}
1504+
#endif
1505+
#ifdef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
1506+
if (cPar.strategy == ZSTD_dfast) {
1507+
cPar.strategy = ZSTD_fast;
1508+
cPar.targetLength = 0;
1509+
}
1510+
#endif
1511+
14701512
switch (mode) {
14711513
case ZSTD_cpm_unknown:
14721514
case ZSTD_cpm_noAttachDict:
@@ -2992,40 +3034,43 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramS
29923034
static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = {
29933035
{ ZSTD_compressBlock_fast /* default for 0 */,
29943036
ZSTD_compressBlock_fast,
2995-
ZSTD_compressBlock_doubleFast,
2996-
ZSTD_compressBlock_greedy,
2997-
ZSTD_compressBlock_lazy,
2998-
ZSTD_compressBlock_lazy2,
2999-
ZSTD_compressBlock_btlazy2,
3000-
ZSTD_compressBlock_btopt,
3001-
ZSTD_compressBlock_btultra,
3002-
ZSTD_compressBlock_btultra2 },
3037+
ZSTD_COMPRESSBLOCK_DOUBLEFAST,
3038+
ZSTD_COMPRESSBLOCK_GREEDY,
3039+
ZSTD_COMPRESSBLOCK_LAZY,
3040+
ZSTD_COMPRESSBLOCK_LAZY2,
3041+
ZSTD_COMPRESSBLOCK_BTLAZY2,
3042+
ZSTD_COMPRESSBLOCK_BTOPT,
3043+
ZSTD_COMPRESSBLOCK_BTULTRA,
3044+
ZSTD_COMPRESSBLOCK_BTULTRA2
3045+
},
30033046
{ ZSTD_compressBlock_fast_extDict /* default for 0 */,
30043047
ZSTD_compressBlock_fast_extDict,
3005-
ZSTD_compressBlock_doubleFast_extDict,
3006-
ZSTD_compressBlock_greedy_extDict,
3007-
ZSTD_compressBlock_lazy_extDict,
3008-
ZSTD_compressBlock_lazy2_extDict,
3009-
ZSTD_compressBlock_btlazy2_extDict,
3010-
ZSTD_compressBlock_btopt_extDict,
3011-
ZSTD_compressBlock_btultra_extDict,
3012-
ZSTD_compressBlock_btultra_extDict },
3048+
ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT,
3049+
ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT,
3050+
ZSTD_COMPRESSBLOCK_LAZY_EXTDICT,
3051+
ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT,
3052+
ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT,
3053+
ZSTD_COMPRESSBLOCK_BTOPT_EXTDICT,
3054+
ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT,
3055+
ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT
3056+
},
30133057
{ ZSTD_compressBlock_fast_dictMatchState /* default for 0 */,
30143058
ZSTD_compressBlock_fast_dictMatchState,
3015-
ZSTD_compressBlock_doubleFast_dictMatchState,
3016-
ZSTD_compressBlock_greedy_dictMatchState,
3017-
ZSTD_compressBlock_lazy_dictMatchState,
3018-
ZSTD_compressBlock_lazy2_dictMatchState,
3019-
ZSTD_compressBlock_btlazy2_dictMatchState,
3020-
ZSTD_compressBlock_btopt_dictMatchState,
3021-
ZSTD_compressBlock_btultra_dictMatchState,
3022-
ZSTD_compressBlock_btultra_dictMatchState },
3059+
ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE,
3060+
ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE,
3061+
ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE,
3062+
ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE,
3063+
ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE,
3064+
ZSTD_COMPRESSBLOCK_BTOPT_DICTMATCHSTATE,
3065+
ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE,
3066+
ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE
3067+
},
30233068
{ NULL /* default for 0 */,
30243069
NULL,
30253070
NULL,
3026-
ZSTD_compressBlock_greedy_dedicatedDictSearch,
3027-
ZSTD_compressBlock_lazy_dedicatedDictSearch,
3028-
ZSTD_compressBlock_lazy2_dedicatedDictSearch,
3071+
ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH,
3072+
ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH,
3073+
ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH,
30293074
NULL,
30303075
NULL,
30313076
NULL,
@@ -3038,18 +3083,26 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramS
30383083
DEBUGLOG(4, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder);
30393084
if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) {
30403085
static const ZSTD_blockCompressor rowBasedBlockCompressors[4][3] = {
3041-
{ ZSTD_compressBlock_greedy_row,
3042-
ZSTD_compressBlock_lazy_row,
3043-
ZSTD_compressBlock_lazy2_row },
3044-
{ ZSTD_compressBlock_greedy_extDict_row,
3045-
ZSTD_compressBlock_lazy_extDict_row,
3046-
ZSTD_compressBlock_lazy2_extDict_row },
3047-
{ ZSTD_compressBlock_greedy_dictMatchState_row,
3048-
ZSTD_compressBlock_lazy_dictMatchState_row,
3049-
ZSTD_compressBlock_lazy2_dictMatchState_row },
3050-
{ ZSTD_compressBlock_greedy_dedicatedDictSearch_row,
3051-
ZSTD_compressBlock_lazy_dedicatedDictSearch_row,
3052-
ZSTD_compressBlock_lazy2_dedicatedDictSearch_row }
3086+
{
3087+
ZSTD_COMPRESSBLOCK_GREEDY_ROW,
3088+
ZSTD_COMPRESSBLOCK_LAZY_ROW,
3089+
ZSTD_COMPRESSBLOCK_LAZY2_ROW
3090+
},
3091+
{
3092+
ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW,
3093+
ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW,
3094+
ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW
3095+
},
3096+
{
3097+
ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW,
3098+
ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW,
3099+
ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW
3100+
},
3101+
{
3102+
ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW,
3103+
ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW,
3104+
ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW
3105+
}
30533106
};
30543107
DEBUGLOG(4, "Selecting a row-based matchfinder");
30553108
assert(useRowMatchFinder != ZSTD_ps_auto);
@@ -3280,9 +3333,11 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
32803333
}
32813334

32823335
/* Fallback to software matchfinder */
3283-
{ ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
3284-
zc->appliedParams.useRowMatchFinder,
3285-
dictMode);
3336+
{ ZSTD_blockCompressor const blockCompressor =
3337+
ZSTD_selectBlockCompressor(
3338+
zc->appliedParams.cParams.strategy,
3339+
zc->appliedParams.useRowMatchFinder,
3340+
dictMode);
32863341
ms->ldmSeqStore = NULL;
32873342
DEBUGLOG(
32883343
5,
@@ -3292,9 +3347,10 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
32923347
lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
32933348
} }
32943349
} else { /* not long range mode and no external matchfinder */
3295-
ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
3296-
zc->appliedParams.useRowMatchFinder,
3297-
dictMode);
3350+
ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(
3351+
zc->appliedParams.cParams.strategy,
3352+
zc->appliedParams.useRowMatchFinder,
3353+
dictMode);
32983354
ms->ldmSeqStore = NULL;
32993355
lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
33003356
}
@@ -4760,12 +4816,19 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
47604816
ZSTD_fillHashTable(ms, iend, dtlm, tfp);
47614817
break;
47624818
case ZSTD_dfast:
4819+
#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
47634820
ZSTD_fillDoubleHashTable(ms, iend, dtlm, tfp);
4821+
#else
4822+
assert(0); /* shouldn't be called: cparams should've been adjusted. */
4823+
#endif
47644824
break;
47654825

47664826
case ZSTD_greedy:
47674827
case ZSTD_lazy:
47684828
case ZSTD_lazy2:
4829+
#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \
4830+
|| !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \
4831+
|| !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR)
47694832
assert(srcSize >= HASH_READ_SIZE);
47704833
if (ms->dedicatedDictSearch) {
47714834
assert(ms->chainTable != NULL);
@@ -4782,14 +4845,23 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
47824845
DEBUGLOG(4, "Using chain-based hash table for lazy dict");
47834846
}
47844847
}
4848+
#else
4849+
assert(0); /* shouldn't be called: cparams should've been adjusted. */
4850+
#endif
47854851
break;
47864852

47874853
case ZSTD_btlazy2: /* we want the dictionary table fully sorted */
47884854
case ZSTD_btopt:
47894855
case ZSTD_btultra:
47904856
case ZSTD_btultra2:
4857+
#if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \
4858+
|| !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \
4859+
|| !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR)
47914860
assert(srcSize >= HASH_READ_SIZE);
47924861
ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend);
4862+
#else
4863+
assert(0); /* shouldn't be called: cparams should've been adjusted. */
4864+
#endif
47934865
break;
47944866

47954867
default:

‎lib/compress/zstd_double_fast.c

+4
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
#include "zstd_compress_internal.h"
1212
#include "zstd_double_fast.h"
1313

14+
#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
15+
1416
static void ZSTD_fillDoubleHashTableForCDict(ZSTD_matchState_t* ms,
1517
void const* end, ZSTD_dictTableLoadMethod_e dtlm)
1618
{
@@ -756,3 +758,5 @@ size_t ZSTD_compressBlock_doubleFast_extDict(
756758
return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize);
757759
}
758760
}
761+
762+
#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */

‎lib/compress/zstd_double_fast.h

+11
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,12 @@ extern "C" {
1818
#include "../common/mem.h" /* U32 */
1919
#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */
2020

21+
#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
22+
2123
void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
2224
void const* end, ZSTD_dictTableLoadMethod_e dtlm,
2325
ZSTD_tableFillPurpose_e tfp);
26+
2427
size_t ZSTD_compressBlock_doubleFast(
2528
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
2629
void const* src, size_t srcSize);
@@ -31,6 +34,14 @@ size_t ZSTD_compressBlock_doubleFast_extDict(
3134
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
3235
void const* src, size_t srcSize);
3336

37+
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST ZSTD_compressBlock_doubleFast
38+
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE ZSTD_compressBlock_doubleFast_dictMatchState
39+
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT ZSTD_compressBlock_doubleFast_extDict
40+
#else
41+
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST NULL
42+
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE NULL
43+
#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT NULL
44+
#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */
3445

3546
#if defined (__cplusplus)
3647
}

0 commit comments

Comments
 (0)