Skip to content

Commit 0b0bdec

Browse files
dopuskh3dopuskh3
authored
Merge pull request #109 from DataDog/dopuskh3/zstd-external-library
Add support for building zstd binding against an external libzstd library
2 parents 112dca3 + 1a7f3d7 commit 0b0bdec

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

87 files changed

+370
-10
lines changed

‎.circleci/config.yml

+58-6
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,45 @@
11
version: 2
22

33
jobs:
4-
"golang-1.14":
4+
"golang-1.16":
55
docker:
6-
- image: circleci/golang:1.14
6+
- image: circleci/golang:1.16
77
steps:
88
- checkout
99
- run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
1010
- run: 'unzip mr.zip'
1111
- run: 'go build'
1212
- run: 'PAYLOAD=`pwd`/mr go test -v'
1313
- run: 'PAYLOAD=`pwd`/mr go test -bench .'
14-
"golang-1.15":
14+
"golang-1.16-external-libzstd":
1515
docker:
16-
- image: circleci/golang:1.15
16+
- image: circleci/golang:1.16
1717
steps:
1818
- checkout
19+
- run: 'sudo apt update'
20+
- run: 'sudo apt install libzstd-dev'
21+
- run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
22+
- run: 'unzip mr.zip'
23+
- run: 'go build'
24+
- run: 'PAYLOAD=`pwd`/mr go test -v'
25+
- run: 'PAYLOAD=`pwd`/mr go test -bench .'
26+
"golang-1.17":
27+
docker:
28+
- image: circleci/golang:1.17
29+
steps:
30+
- checkout
31+
- run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
32+
- run: 'unzip mr.zip'
33+
- run: 'go build'
34+
- run: 'PAYLOAD=`pwd`/mr go test -v'
35+
- run: 'PAYLOAD=`pwd`/mr go test -bench .'
36+
"golang-1.17-external-libzstd":
37+
docker:
38+
- image: circleci/golang:1.17
39+
steps:
40+
- checkout
41+
- run: 'sudo apt update'
42+
- run: 'sudo apt install libzstd-dev'
1943
- run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
2044
- run: 'unzip mr.zip'
2145
- run: 'go build'
@@ -31,6 +55,18 @@ jobs:
3155
- run: 'go build'
3256
- run: 'PAYLOAD=`pwd`/mr go test -v'
3357
- run: 'PAYLOAD=`pwd`/mr go test -bench .'
58+
"golang-latest-external-libzstd":
59+
docker:
60+
- image: circleci/golang:latest
61+
steps:
62+
- checkout
63+
- run: 'sudo apt update'
64+
- run: 'sudo apt install libzstd-dev'
65+
- run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
66+
- run: 'unzip mr.zip'
67+
- run: 'go build -tags external_libzstd'
68+
- run: 'PAYLOAD=`pwd`/mr go test -tags external_libzstd -v'
69+
- run: 'PAYLOAD=`pwd`/mr go test -tags external_libzstd -bench .'
3470
"golang-efence":
3571
resource_class: xlarge
3672
docker:
@@ -41,6 +77,18 @@ jobs:
4177
- run: 'unzip mr.zip'
4278
- run: 'go build'
4379
- run: 'PAYLOAD=`pwd`/mr GODEBUG=efence=1 go test -v'
80+
"golang-efence-external-libzstd":
81+
resource_class: xlarge
82+
docker:
83+
- image: circleci/golang:latest
84+
steps:
85+
- checkout
86+
- run: 'sudo apt update'
87+
- run: 'sudo apt install libzstd-dev'
88+
- run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
89+
- run: 'unzip mr.zip'
90+
- run: 'go build -tags external_libzstd'
91+
- run: 'PAYLOAD=`pwd`/mr GODEBUG=efence=1 go test -tags external_libzstd -v'
4492
"golang-zstd-legacy-support":
4593
docker:
4694
- image: circleci/golang:latest
@@ -61,9 +109,13 @@ workflows:
61109
version: 2
62110
build:
63111
jobs:
64-
- "golang-1.14"
65-
- "golang-1.15"
112+
- "golang-1.16"
113+
- "golang-1.16-external-libzstd"
114+
- "golang-1.17"
115+
- "golang-1.17-external-libzstd"
66116
- "golang-latest"
117+
- "golang-latest-external-libzstd"
67118
- "golang-efence"
119+
- "golang-efence-external-libzstd"
68120
- "golang-i386"
69121
- "golang-zstd-legacy-support"

‎README.md

+15
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,21 @@ There are two main APIs:
1919
The compress/decompress APIs mirror that of lz4, while the streaming API was
2020
designed to be a drop-in replacement for zlib.
2121

22+
### Building against an external libzstd
23+
24+
By default, zstd source code is vendored in this repository and the binding will be built with
25+
the vendored source code bundled.
26+
27+
If you want to build this binding against an external static or shared libzstd library, you can
28+
use the `external_libzstd` build tag. This will look for the libzstd pkg-config file and extract
29+
build and linking parameters from that pkg-config file.
30+
31+
Note that it requires at least libzstd 1.4.0.
32+
33+
```bash
34+
go build -tags external_libzstd
35+
```
36+
2237
### Simple `Compress/Decompress`
2338

2439

‎bitstream.h

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/* ******************************************************************
23
* bitstream
34
* Part of FSE library
@@ -461,3 +462,5 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
461462
#endif
462463

463464
#endif /* BITSTREAM_H_MODULE */
465+
466+
#endif /* USE_EXTERNAL_ZSTD */

‎compiler.h

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/*
23
* Copyright (c) Yann Collet, Facebook, Inc.
34
* All rights reserved.
@@ -287,3 +288,5 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size);
287288
#endif
288289

289290
#endif /* ZSTD_COMPILER_H */
291+
292+
#endif /* USE_EXTERNAL_ZSTD */

‎cover.c

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/*
23
* Copyright (c) Yann Collet, Facebook, Inc.
34
* All rights reserved.
@@ -1244,3 +1245,5 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
12441245
return dictSize;
12451246
}
12461247
}
1248+
1249+
#endif /* USE_EXTERNAL_ZSTD */

‎cover.h

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/*
23
* Copyright (c) Facebook, Inc.
34
* All rights reserved.
@@ -156,3 +157,5 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection);
156157
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
157158
size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
158159
size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize);
160+
161+
#endif /* USE_EXTERNAL_ZSTD */

‎cpu.h

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/*
23
* Copyright (c) Facebook, Inc.
34
* All rights reserved.
@@ -211,3 +212,5 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
211212
#undef X
212213

213214
#endif /* ZSTD_COMMON_CPU_H */
215+
216+
#endif /* USE_EXTERNAL_ZSTD */

‎debug.c

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/* ******************************************************************
23
* debug
34
* Part of FSE library
@@ -22,3 +23,5 @@
2223
#include "debug.h"
2324

2425
int g_debuglevel = DEBUGLEVEL;
26+
27+
#endif /* USE_EXTERNAL_ZSTD */

‎debug.h

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/* ******************************************************************
23
* debug
34
* Part of FSE library
@@ -105,3 +106,5 @@ extern int g_debuglevel; /* the variable is only declared,
105106
#endif
106107

107108
#endif /* DEBUG_H_12987983217 */
109+
110+
#endif /* USE_EXTERNAL_ZSTD */

‎divsufsort.c

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/*
23
* divsufsort.c for libdivsufsort-lite
34
* Copyright (c) 2003-2008 Yuta Mori All Rights Reserved.
@@ -1911,3 +1912,5 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char *
19111912

19121913
return pidx;
19131914
}
1915+
1916+
#endif /* USE_EXTERNAL_ZSTD */

‎divsufsort.h

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/*
23
* divsufsort.h for libdivsufsort-lite
34
* Copyright (c) 2003-2008 Yuta Mori All Rights Reserved.
@@ -65,3 +66,5 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char *
6566
#endif /* __cplusplus */
6667

6768
#endif /* _DIVSUFSORT_H */
69+
70+
#endif /* USE_EXTERNAL_ZSTD */

‎entropy_common.c

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/* ******************************************************************
23
* Common functions of New Generation Entropy library
34
* Copyright (c) Yann Collet, Facebook, Inc.
@@ -360,3 +361,5 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
360361
(void)bmi2;
361362
return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
362363
}
364+
365+
#endif /* USE_EXTERNAL_ZSTD */

‎error_private.c

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/*
23
* Copyright (c) Yann Collet, Facebook, Inc.
34
* All rights reserved.
@@ -54,3 +55,5 @@ const char* ERR_getErrorString(ERR_enum code)
5455
}
5556
#endif
5657
}
58+
59+
#endif /* USE_EXTERNAL_ZSTD */

‎error_private.h

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/*
23
* Copyright (c) Yann Collet, Facebook, Inc.
34
* All rights reserved.
@@ -78,3 +79,5 @@ ERR_STATIC const char* ERR_getErrorName(size_t code)
7879
#endif
7980

8081
#endif /* ERROR_H_MODULE */
82+
83+
#endif /* USE_EXTERNAL_ZSTD */

‎errors.go

-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package zstd
22

33
/*
4-
#define ZSTD_STATIC_LINKING_ONLY
54
#include "zstd.h"
65
*/
76
import "C"

‎external_zstd.go

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
//go:build external_libzstd
2+
// +build external_libzstd
3+
4+
package zstd
5+
6+
// #cgo CFLAGS: -DUSE_EXTERNAL_ZSTD
7+
// #cgo pkg-config: libzstd
8+
/*
9+
#include<zstd.h>
10+
#if ZSTD_VERSION_NUMBER < 10400
11+
#error "ZSTD version >= 1.4 is required"
12+
#endif
13+
*/
14+
import "C"

‎fastcover.c

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/*
23
* Copyright (c) Facebook, Inc.
34
* All rights reserved.
@@ -757,3 +758,5 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
757758
}
758759

759760
}
761+
762+
#endif /* USE_EXTERNAL_ZSTD */

‎fse.h

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/* ******************************************************************
23
* FSE : Finite State Entropy codec
34
* Public Prototypes declaration
@@ -714,3 +715,5 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
714715
#if defined (__cplusplus)
715716
}
716717
#endif
718+
719+
#endif /* USE_EXTERNAL_ZSTD */

‎fse_compress.c

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/* ******************************************************************
23
* FSE : Finite State Entropy encoder
34
* Copyright (c) Yann Collet, Facebook, Inc.
@@ -703,3 +704,5 @@ size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcS
703704
#endif
704705

705706
#endif /* FSE_COMMONDEFS_ONLY */
707+
708+
#endif /* USE_EXTERNAL_ZSTD */

‎fse_decompress.c

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/* ******************************************************************
23
* FSE : Finite State Entropy decoder
34
* Copyright (c) Yann Collet, Facebook, Inc.
@@ -401,3 +402,5 @@ size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cS
401402

402403

403404
#endif /* FSE_COMMONDEFS_ONLY */
405+
406+
#endif /* USE_EXTERNAL_ZSTD */

‎hist.c

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/* ******************************************************************
23
* hist : Histogram functions
34
* part of Finite State Entropy project
@@ -179,3 +180,5 @@ size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
179180
return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters));
180181
}
181182
#endif
183+
184+
#endif /* USE_EXTERNAL_ZSTD */

‎hist.h

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/* ******************************************************************
23
* hist : Histogram functions
34
* part of Finite State Entropy project
@@ -73,3 +74,5 @@ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
7374
*/
7475
unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
7576
const void* src, size_t srcSize);
77+
78+
#endif /* USE_EXTERNAL_ZSTD */

‎huf.h

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/* ******************************************************************
23
* huff0 huffman codec,
34
* part of Finite State Entropy library
@@ -360,3 +361,5 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
360361
#if defined (__cplusplus)
361362
}
362363
#endif
364+
365+
#endif /* USE_EXTERNAL_ZSTD */

‎huf_compress.c

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/* ******************************************************************
23
* Huffman encoder, part of New Generation Entropy library
34
* Copyright (c) Yann Collet, Facebook, Inc.
@@ -935,3 +936,5 @@ size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSi
935936
return HUF_compress2(dst, maxDstSize, src, srcSize, 255, HUF_TABLELOG_DEFAULT);
936937
}
937938
#endif
939+
940+
#endif /* USE_EXTERNAL_ZSTD */

‎huf_decompress.c

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/* ******************************************************************
23
* huff0 huffman decoder,
34
* part of Finite State Entropy library
@@ -1349,3 +1350,5 @@ size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
13491350
workSpace, sizeof(workSpace));
13501351
}
13511352
#endif
1353+
1354+
#endif /* USE_EXTERNAL_ZSTD */

‎mem.h

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#ifndef USE_EXTERNAL_ZSTD
12
/*
23
* Copyright (c) Yann Collet, Facebook, Inc.
34
* All rights reserved.
@@ -422,3 +423,5 @@ MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (si
422423
#endif
423424

424425
#endif /* MEM_H_MODULE */
426+
427+
#endif /* USE_EXTERNAL_ZSTD */

0 commit comments

Comments
 (0)