Add support for linking to system libzstd

.circleci/config.yml

@@ -31,6 +31,18 @@ jobs:
       - run: 'go build'
       - run: 'PAYLOAD=`pwd`/mr go test -v'
       - run: 'PAYLOAD=`pwd`/mr go test -bench .'
+  "golang-libzstd":
+    docker:
+      - image: circleci/golang:latest
+    steps:
+      - checkout
+      - run: 'sudo apt-get update && sudo apt-get install libzstd-dev'
+      - run: 'wget https://github.com/DataDog/zstd/files/2246767/mr.zip'
+      - run: 'unzip mr.zip'
+      - run: 'go build -tags libzstd'
+      - run: 'go test -tags libzstd -c && ldd zstd.test | grep libzstd'
+      - run: 'PAYLOAD=`pwd`/mr go test -tags libzstd -v'
+      - run: 'PAYLOAD=`pwd`/mr go test -tags libzstd -bench .'
   "golang-efence":
     resource_class: xlarge
     docker:
@@ -64,6 +76,7 @@ workflows:
       - "golang-1.14"
       - "golang-1.15"
       - "golang-latest"
+      - "golang-libzstd"
       - "golang-efence"
       - "golang-i386"
       - "golang-zstd-legacy-support"

bitstream.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /* ******************************************************************
  * bitstream
  * Part of FSE library
@@ -461,3 +462,4 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
 #endif
 
 #endif /* BITSTREAM_H_MODULE */
+#endif /* USE_LIBZSTD */

compiler.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -286,3 +287,4 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size);
 #endif
 
 #endif /* ZSTD_COMPILER_H */
+#endif /* USE_LIBZSTD */

cover.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -1243,3 +1244,4 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover(
     return dictSize;
   }
 }
+#endif /* USE_LIBZSTD */

cover.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2017-2020, Facebook, Inc.
  * All rights reserved.
@@ -155,3 +156,4 @@ void COVER_dictSelectionFree(COVER_dictSelection_t selection);
  COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
                        size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
                        size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize);
+#endif /* USE_LIBZSTD */

cpu.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2018-2020, Facebook, Inc.
  * All rights reserved.
@@ -211,3 +212,4 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
 #undef X
 
 #endif /* ZSTD_COMMON_CPU_H */
+#endif /* USE_LIBZSTD */

debug.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /* ******************************************************************
  * debug
  * Part of FSE library
@@ -22,3 +23,4 @@
 #include "debug.h"
 
 int g_debuglevel = DEBUGLEVEL;
+#endif /* USE_LIBZSTD */

debug.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /* ******************************************************************
  * debug
  * Part of FSE library
@@ -105,3 +106,4 @@ extern int g_debuglevel; /* the variable is only declared,
 #endif
 
 #endif /* DEBUG_H_12987983217 */
+#endif /* USE_LIBZSTD */

divsufsort.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * divsufsort.c for libdivsufsort-lite
  * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved.
@@ -1911,3 +1912,4 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char *
 
   return pidx;
 }
+#endif /* USE_LIBZSTD */

divsufsort.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * divsufsort.h for libdivsufsort-lite
  * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved.
@@ -65,3 +66,4 @@ divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char *
 #endif /* __cplusplus */
 
 #endif /* _DIVSUFSORT_H */
+#endif /* USE_LIBZSTD */

entropy_common.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /* ******************************************************************
  * Common functions of New Generation Entropy library
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
@@ -360,3 +361,4 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
     (void)bmi2;
     return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
 }
+#endif /* USE_LIBZSTD */

error_private.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -54,3 +55,4 @@ const char* ERR_getErrorString(ERR_enum code)
     }
 #endif
 }
+#endif /* USE_LIBZSTD */

error_private.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -78,3 +79,4 @@ ERR_STATIC const char* ERR_getErrorName(size_t code)
 #endif
 
 #endif /* ERROR_H_MODULE */
+#endif /* USE_LIBZSTD */

fastcover.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2018-2020, Facebook, Inc.
  * All rights reserved.
@@ -756,3 +757,4 @@ ZDICT_optimizeTrainFromBuffer_fastCover(
     }
 
 }
+#endif /* USE_LIBZSTD */

fse.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /* ******************************************************************
  * FSE : Finite State Entropy codec
  * Public Prototypes declaration
@@ -714,3 +715,4 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
 #if defined (__cplusplus)
 }
 #endif
+#endif /* USE_LIBZSTD */

fse_compress.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /* ******************************************************************
  * FSE : Finite State Entropy encoder
  * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
@@ -703,3 +704,4 @@ size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcS
 #endif
 
 #endif   /* FSE_COMMONDEFS_ONLY */
+#endif /* USE_LIBZSTD */

fse_decompress.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /* ******************************************************************
  * FSE : Finite State Entropy decoder
  * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
@@ -391,3 +392,4 @@ size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cS
 
 
 #endif   /* FSE_COMMONDEFS_ONLY */
+#endif /* USE_LIBZSTD */

hist.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
@@ -179,3 +180,4 @@ size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
     return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters));
 }
 #endif
+#endif /* USE_LIBZSTD */

hist.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
@@ -73,3 +74,4 @@ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
  */
 unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
                            const void* src, size_t srcSize);
+#endif /* USE_LIBZSTD */

huf.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /* ******************************************************************
  * huff0 huffman codec,
  * part of Finite State Entropy library
@@ -359,3 +360,4 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
 #if defined (__cplusplus)
 }
 #endif
+#endif /* USE_LIBZSTD */

huf_compress.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /* ******************************************************************
  * Huffman encoder, part of New Generation Entropy library
  * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc.
@@ -912,3 +913,4 @@ size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSi
     return HUF_compress2(dst, maxDstSize, src, srcSize, 255, HUF_TABLELOG_DEFAULT);
 }
 #endif
+#endif /* USE_LIBZSTD */

huf_decompress.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /* ******************************************************************
  * huff0 huffman decoder,
  * part of Finite State Entropy library
@@ -1348,3 +1349,4 @@ size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
                                       workSpace, sizeof(workSpace));
 }
 #endif
+#endif /* USE_LIBZSTD */

insert_libzstd_ifdefs.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Run this after updating vendored zstd sources.
+
+for filename in *.h *c; do
+    if ! grep -F "#ifndef USE_LIBZSTD" "$filename" >/dev/null; then
+        if [[ $filename == "zstd.h" ]]; then
+            sed -i "$filename" \
+                -e '1 i\#ifndef USE_LIBZSTD' \
+                -e '$ a\#else /* USE_LIBZSTD */\n#undef ZSTD_STATIC_LINKING_ONLY\n#include_next <zstd.h>\n#endif /* USE_LIBZSTD */'
+        else
+            sed -i "$filename" \
+                -e '1 i\#ifndef USE_LIBZSTD' \
+                -e '$ a\#endif /* USE_LIBZSTD */'
+        fi
+    fi
+done

mem.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -424,3 +425,4 @@ MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (si
 #endif
 
 #endif /* MEM_H_MODULE */
+#endif /* USE_LIBZSTD */

pool.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -348,3 +349,4 @@ size_t POOL_sizeof(POOL_ctx* ctx) {
 }
 
 #endif  /* ZSTD_MULTITHREAD */
+#endif /* USE_LIBZSTD */

pool.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -82,3 +83,4 @@ int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
 #endif
 
 #endif
+#endif /* USE_LIBZSTD */

threading.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /**
  * Copyright (c) 2016 Tino Reichardt
  * All rights reserved.
@@ -120,3 +121,4 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond)
 }
 
 #endif
+#endif /* USE_LIBZSTD */

threading.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /**
  * Copyright (c) 2016 Tino Reichardt
  * All rights reserved.
@@ -153,3 +154,4 @@ typedef int ZSTD_pthread_cond_t;
 #endif
 
 #endif /* THREADING_H_938743 */
+#endif /* USE_LIBZSTD */

xxhash.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  *  xxHash - Fast Hash algorithm
  *  Copyright (c) 2012-2020, Yann Collet, Facebook, Inc.
@@ -824,3 +825,4 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src
 {
     return XXH_readBE64(src);
 }
+#endif /* USE_LIBZSTD */

xxhash.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * xxHash - Extremely Fast Hash algorithm
  * Header File
@@ -283,3 +284,4 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src
 #if defined (__cplusplus)
 }
 #endif
+#endif /* USE_LIBZSTD */

zbuff.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -212,3 +213,4 @@ ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_adv
 #if defined (__cplusplus)
 }
 #endif
+#endif /* USE_LIBZSTD */

zbuff_common.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -24,3 +25,4 @@ unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
 /*! ZBUFF_getErrorName() :
 *   provides error code string from function result (useful for debugging) */
 const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
+#endif /* USE_LIBZSTD */

zbuff_compress.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -145,3 +146,4 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
 ***************************************/
 size_t ZBUFF_recommendedCInSize(void)  { return ZSTD_CStreamInSize(); }
 size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_CStreamOutSize(); }
+#endif /* USE_LIBZSTD */

zbuff_decompress.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -73,3 +74,4 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
 ***************************************/
 size_t ZBUFF_recommendedDInSize(void)  { return ZSTD_DStreamInSize(); }
 size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_DStreamOutSize(); }
+#endif /* USE_LIBZSTD */

zdict.c

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -1132,3 +1133,4 @@ size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize
                                                      samplesBuffer, samplesSizes, nbSamples,
                                                      params);
 }
+#endif /* USE_LIBZSTD */

zdict.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -303,3 +304,4 @@ size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize
 #endif
 
 #endif   /* DICTBUILDER_H_001 */
+#endif /* USE_LIBZSTD */

zstd.h

@@ -1,3 +1,4 @@
+#ifndef USE_LIBZSTD
 /*
  * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -2389,3 +2390,7 @@ ZSTDLIB_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* blockStart,
 #if defined (__cplusplus)
 }
 #endif
+#else /* USE_LIBZSTD */
+#undef ZSTD_STATIC_LINKING_ONLY
+#include_next <zstd.h>
+#endif /* USE_LIBZSTD */