Update (google#620)

* add autotools build
* separate semantic and ABI version
* extract sources.lst (used by CMake and Automake)
* share pkgconfig templates (used by CMake and Automake)
* decoder: always set `total_out`
* encoder: fix `BROTLI_ENSURE_CAPACITY` macro (no-op after preprocessor)
* decoder/encoder: refine `free_func` contract

Author: eustas

CMakeLists.txt

@@ -72,8 +72,19 @@ string(REGEX REPLACE "^#define BROTLI_VERSION 0x([0-9a-fA-F]+)$" "\\1" _brotli_v
 hex_to_dec("${_brotli_version_hex}" _brotli_version)
 math(EXPR BROTLI_VERSION_MAJOR "${_brotli_version} >> 24")
 math(EXPR BROTLI_VERSION_MINOR "(${_brotli_version} >> 12) & 4095")
-math(EXPR BROTLI_VERSION_REVISION "${_brotli_version} & 4095")
-mark_as_advanced(BROTLI_VERSION_MAJOR BROTLI_VERSION_MINOR BROTLI_VERSION_REVISION)
+math(EXPR BROTLI_VERSION_PATCH "${_brotli_version} & 4095")
+set(BROTLI_VERSION "${BROTLI_VERSION_MAJOR}.${BROTLI_VERSION_MINOR}.${BROTLI_VERSION_PATCH}")
+mark_as_advanced(BROTLI_VERSION BROTLI_VERSION_MAJOR BROTLI_VERSION_MINOR BROTLI_VERSION_PATCH)
+
+# ABI Version information
+file(STRINGS "c/common/version.h" _brotli_abi_info_line REGEX "^#define BROTLI_ABI_VERSION (0x[0-9a-fA-F]+)$")
+string(REGEX REPLACE "^#define BROTLI_ABI_VERSION 0x([0-9a-fA-F]+)$" "\\1" _brotli_abi_info_hex "${_brotli_abi_info_line}")
+hex_to_dec("${_brotli_abi_info_hex}" _brotli_abi_info)
+math(EXPR BROTLI_ABI_CURRENT "${_brotli_abi_info} >> 24")
+math(EXPR BROTLI_ABI_REVISION "(${_brotli_abi_info} >> 12) & 4095")
+math(EXPR BROTLI_ABI_AGE "${_brotli_abi_info} & 4095")
+math(EXPR BROTLI_ABI_COMPATIBILITY "${BROTLI_ABI_CURRENT} - ${BROTLI_ABI_AGE}")
+mark_as_advanced(BROTLI_ABI_CURRENT BROTLI_ABI_REVISION BROTLI_ABI_AGE BROTLI_ABI_COMPATIBILITY)
 
 if (ENABLE_SANITIZER)
   set(CMAKE_C_FLAGS " ${CMAKE_C_FLAGS} -fsanitize=${ENABLE_SANITIZER}")
@@ -125,39 +136,24 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
   add_definitions(-DOS_MACOSX)
 endif()
 
-set(BROTLICOMMON_SOURCES
-  c/common/dictionary.c)
-set(BROTLIDEC_SOURCES
-  c/dec/bit_reader.c
-  c/dec/decode.c
-  c/dec/huffman.c
-  c/dec/state.c)
-set(BROTLIENC_SOURCES
-  c/enc/backward_references.c
-  c/enc/backward_references_hq.c
-  c/enc/bit_cost.c
-  c/enc/block_splitter.c
-  c/enc/brotli_bit_stream.c
-  c/enc/cluster.c
-  c/enc/compress_fragment.c
-  c/enc/compress_fragment_two_pass.c
-  c/enc/dictionary_hash.c
-  c/enc/encode.c
-  c/enc/entropy_encode.c
-  c/enc/histogram.c
-  c/enc/literal_cost.c
-  c/enc/memory.c
-  c/enc/metablock.c
-  c/enc/static_dict.c
-  c/enc/utf8_util.c)
-
-add_library(brotlicommon SHARED ${BROTLICOMMON_SOURCES})
-add_library(brotlidec SHARED ${BROTLIDEC_SOURCES})
-add_library(brotlienc SHARED ${BROTLIENC_SOURCES})
-
-add_library(brotlicommon-static STATIC ${BROTLICOMMON_SOURCES})
-add_library(brotlidec-static STATIC ${BROTLIDEC_SOURCES})
-add_library(brotlienc-static STATIC ${BROTLIENC_SOURCES})
+function(transform_sources_list INPUT_FILE OUTPUT_FILE)
+  file(READ ${INPUT_FILE} TEXT)
+  string(REGEX REPLACE "\\\\\n" "~continuation~" TEXT ${TEXT})
+  string(REGEX REPLACE "([a-zA-Z_][a-zA-Z0-9_]*)[\t ]*=[\t ]*([^\n]*)" "SET(\\1 \\2)" TEXT ${TEXT})
+  string(REPLACE "~continuation~" "\n" TEXT ${TEXT})
+  file(WRITE ${OUTPUT_FILE} ${TEXT})
+endfunction()
+
+transform_sources_list("scripts/sources.lst" "${CMAKE_CURRENT_BINARY_DIR}/sources.lst.cmake")
+include("${CMAKE_CURRENT_BINARY_DIR}/sources.lst.cmake")
+
+add_library(brotlicommon SHARED ${BROTLI_COMMON_C})
+add_library(brotlidec SHARED ${BROTLI_DEC_C})
+add_library(brotlienc SHARED ${BROTLI_ENC_C})
+
+add_library(brotlicommon-static STATIC ${BROTLI_COMMON_C})
+add_library(brotlidec-static STATIC ${BROTLI_DEC_C})
+add_library(brotlienc-static STATIC ${BROTLI_ENC_C})
 
 # Older CMake versions does not understand INCLUDE_DIRECTORIES property.
 include_directories(${BROTLI_INCLUDE_DIRS})
@@ -172,8 +168,8 @@ foreach(lib brotlicommon brotlidec brotlienc brotlicommon-static brotlidec-stati
   target_link_libraries(${lib} ${LIBM_LIBRARY})
   set_property(TARGET ${lib} APPEND PROPERTY INCLUDE_DIRECTORIES ${BROTLI_INCLUDE_DIRS})
   set_target_properties(${lib} PROPERTIES
-    SOVERSION "${BROTLI_VERSION_MAJOR}.${BROTLI_VERSION_MINOR}.${BROTLI_VERSION_REVISION}"
-    VERSION "${BROTLI_VERSION_MAJOR}.${BROTLI_VERSION_MINOR}.${BROTLI_VERSION_REVISION}"
+    VERSION "${BROTLI_ABI_COMPATIBILITY}.${BROTLI_ABI_AGE}.${BROTLI_ABI_REVISION}"
+    SOVERSION "${BROTLI_ABI_COMPATIBILITY}"
     POSITION_INDEPENDENT_CODE TRUE)
   set_property(TARGET ${lib} APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${BROTLI_INCLUDE_DIRS}")
 endforeach()
@@ -196,7 +192,7 @@ if(BROTLI_PARENT_DIRECTORY)
 endif()
 
 # Build the brotli executable
-add_executable(brotli c/tools/brotli.c)
+add_executable(brotli ${BROTLI_CLI_C})
 target_link_libraries(brotli ${BROTLI_LIBRARIES_STATIC})
 
 # Installation
@@ -287,9 +283,7 @@ if(NOT BROTLI_DISABLE_TESTS)
   endforeach()
 endif()
 
-# Generate a pkg-config file
-
-include(CMakeParseArguments)
+# Generate a pkg-config files
 
 function(generate_pkg_config_path outvar path)
   string(LENGTH "${path}" path_length)
@@ -334,103 +328,29 @@ function(generate_pkg_config_path outvar path)
   set("${outvar}" "${${outvar}}" PARENT_SCOPE)
 endfunction(generate_pkg_config_path)
 
-function(generate_pkg_config output_file)
-  set (options)
-  set (oneValueArgs NAME DESCRIPTION URL VERSION PREFIX LIBDIR INCLUDEDIR)
-  set (multiValueArgs DEPENDS_PRIVATE CFLAGS LIBRARIES)
-  cmake_parse_arguments(GEN_PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-  unset (options)
-  unset (oneValueArgs)
-  unset (multiValueArgs)
-
-  if(NOT GEN_PKG_PREFIX)
-    set(GEN_PKG_PREFIX "${CMAKE_INSTALL_PREFIX}")
-  endif()
+function(transform_pc_file INPUT_FILE OUTPUT_FILE VERSION)
+  file(READ ${INPUT_FILE} TEXT)
 
-  if(NOT GEN_PKG_LIBDIR)
-    set(GEN_PKG_LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}")
-  endif()
-  generate_pkg_config_path(GEN_PKG_LIBDIR "${GEN_PKG_LIBDIR}"
-    prefix "${GEN_PKG_PREFIX}")
+  set(PREFIX "${CMAKE_INSTALL_PREFIX}")
+  string(REGEX REPLACE "@prefix@" "${PREFIX}" TEXT ${TEXT})
+  string(REGEX REPLACE "@exec_prefix@" "${PREFIX}" TEXT ${TEXT})
 
-  if(NOT GEN_PKG_INCLUDEDIR)
-    set(GEN_PKG_INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}")
-  endif()
-  generate_pkg_config_path(GEN_PKG_INCLUDEDIR "${GEN_PKG_INCLUDEDIR}"
-    prefix "${GEN_PKG_PREFIX}")
+  generate_pkg_config_path(LIBDIR "${CMAKE_INSTALL_FULL_LIBDIR}" prefix "${PREFIX}")
+  string(REGEX REPLACE "@libdir@" "${LIBDIR}" TEXT ${TEXT})
 
-  file(WRITE  "${output_file}" "prefix=${GEN_PKG_PREFIX}\n")
-  file(APPEND "${output_file}" "libdir=${GEN_PKG_LIBDIR}\n")
-  file(APPEND "${output_file}" "includedir=${GEN_PKG_INCLUDEDIR}\n")
-  file(APPEND "${output_file}" "\n")
+  generate_pkg_config_path(INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}" prefix "${PREFIX}")
+  string(REGEX REPLACE "@includedir@" "${INCLUDEDIR}" TEXT ${TEXT})
 
-  if(GEN_PKG_NAME)
-    file(APPEND "${output_file}" "Name: ${GEN_PKG_NAME}\n")
-  else()
-    file(APPEND "${output_file}" "Name: ${CMAKE_PROJECT_NAME}\n")
-  endif()
+  string(REGEX REPLACE "@PACKAGE_VERSION@" "${VERSION}" TEXT ${TEXT})
 
-  if(GEN_PKG_DESCRIPTION)
-    file(APPEND "${output_file}" "Description: ${GEN_PKG_DESCRIPTION}\n")
-  endif()
+  file(WRITE ${OUTPUT_FILE} ${TEXT})
+endfunction()
 
-  if(GEN_PKG_URL)
-    file(APPEND "${output_file}" "URL: ${GEN_PKG_URL}\n")
-  endif()
+transform_pc_file("scripts/libbrotlicommon.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libbrotlicommon.pc" "${BROTLI_VERSION}")
 
-  if(GEN_PKG_VERSION)
-    file(APPEND "${output_file}" "Version: ${GEN_PKG_VERSION}\n")
-  endif()
-
-  if(GEN_PKG_DEPENDS_PRIVATE)
-    file(APPEND "${output_file}" "Requires.private:")
-    foreach(lib ${GEN_PKG_DEPENDS_PRIVATE})
-      file(APPEND "${output_file}" " ${lib}")
-    endforeach()
-    file(APPEND "${output_file}" "\n")
-  endif()
-
-  if(GEN_PKG_LIBRARIES)
-    set(libs)
+transform_pc_file("scripts/libbrotlidec.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libbrotlidec.pc" "${BROTLI_VERSION}")
 
-    file(APPEND "${output_file}" "Libs: -L\${libdir}")
-    foreach(lib ${GEN_PKG_LIBRARIES})
-      file(APPEND "${output_file}" " -l${lib}")
-    endforeach()
-    file(APPEND "${output_file}" "\n")
-  endif()
-
-  file(APPEND "${output_file}" "Cflags: -I\${includedir}")
-  if(GEN_PKG_CFLAGS)
-    foreach(cflag ${GEN_PKG_CFLAGS})
-      file(APPEND "${output_file}" " ${cflag}")
-    endforeach()
-  endif()
-  file(APPEND "${output_file}" "\n")
-endfunction(generate_pkg_config)
-
-generate_pkg_config ("${CMAKE_CURRENT_BINARY_DIR}/libbrotlicommon.pc"
-  NAME libbrotlicommon
-  DESCRIPTION "Shared data used by libbrotlienc and libbrotlidec libraries"
-  URL "https://github.com/google/brotli"
-  VERSION "${BROTLI_VERSION_MAJOR}.${BROTLI_VERSION_MINOR}.${BROTLI_VERSION_REVISION}"
-  LIBRARIES brotlicommon)
-
-generate_pkg_config ("${CMAKE_CURRENT_BINARY_DIR}/libbrotlidec.pc"
-  NAME libbrotlidec
-  DESCRIPTION "Brotli decoder library"
-  VERSION "${BROTLI_VERSION_MAJOR}.${BROTLI_VERSION_MINOR}.${BROTLI_VERSION_REVISION}"
-  URL "https://github.com/google/brotli"
-  DEPENDS_PRIVATE libbrotlicommon
-  LIBRARIES brotlidec)
-
-generate_pkg_config ("${CMAKE_CURRENT_BINARY_DIR}/libbrotlienc.pc"
-  NAME libbrotlienc
-  DESCRIPTION "Brotli encoder library"
-  VERSION "${BROTLI_VERSION_MAJOR}.${BROTLI_VERSION_MINOR}.${BROTLI_VERSION_REVISION}"
-  URL "https://github.com/google/brotli"
-  DEPENDS_PRIVATE libbrotlicommon
-  LIBRARIES brotlienc)
+transform_pc_file("scripts/libbrotlienc.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libbrotlienc.pc" "${BROTLI_VERSION}")
 
 if(NOT BROTLI_BUNDLED_MODE)
   install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libbrotlicommon.pc"

Makefile.am

@@ -0,0 +1,36 @@
+AUTOMAKE_OPTIONS = foreign nostdinc subdir-objects
+
+ACLOCAL_AMFLAGS = -I m4
+
+# Actual ABI version is substituted by bootstrap
+LIBBROTLI_VERSION_INFO = -version-info 0:0:0
+
+bin_PROGRAMS = brotli
+lib_LTLIBRARIES = libbrotlicommon.la libbrotlidec.la libbrotlienc.la
+
+include scripts/sources.lst
+
+brotliincludedir = $(includedir)/brotli
+brotliinclude_HEADERS = $(BROTLI_INCLUDE)
+
+AM_CFLAGS = -I$(top_srcdir)/c/include
+
+brotli_SOURCES = $(BROTLI_CLI_C)
+brotli_LDADD = libbrotlidec.la libbrotlienc.la libbrotlicommon.la -lm
+#brotli_LDFLAGS = -static
+
+libbrotlicommon_la_SOURCES = $(BROTLI_COMMON_C) $(BROTLI_COMMON_H)
+libbrotlicommon_la_LDFLAGS = $(AM_LDFLAGS) $(LIBBROTLI_VERSION_INFO) $(LDFLAGS)
+libbrotlidec_la_SOURCES = $(BROTLI_DEC_C) $(BROTLI_DEC_H)
+libbrotlidec_la_LDFLAGS = $(AM_LDFLAGS) $(LIBBROTLI_VERSION_INFO) $(LDFLAGS)
+libbrotlienc_la_SOURCES = $(BROTLI_ENC_C) $(BROTLI_ENC_H)
+libbrotlienc_la_LDFLAGS = $(AM_LDFLAGS) $(LIBBROTLI_VERSION_INFO) $(LDFLAGS)
+
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA = \
+  scripts/libbrotlicommon.pc \
+  scripts/libbrotlidec.pc \
+  scripts/libbrotlienc.pc
+pkgincludedir= $(brotliincludedir)
+
+dist_doc_DATA = README

README

@@ -0,0 +1,15 @@
+BROTLI DATA COMPRESSIOM LIBRARY
+
+Brotli is a generic-purpose lossless compression algorithm that compresses data
+using a combination of a modern variant of the LZ77 algorithm, Huffman coding
+and 2nd order context modeling, with a compression ratio comparable to the best
+currently available general-purpose compression methods. It is similar in speed
+with deflate but offers more dense compression.
+
+The specification of the Brotli Compressed Data Format is defined in RFC 7932
+https://tools.ietf.org/html/rfc7932
+
+Brotli is open-sourced under the MIT License, see the LICENSE file.
+
+Brotli mailing list:
+https://groups.google.com/forum/#!forum/brotli

bootstrap

@@ -0,0 +1,22 @@
+# !/bin/sh -e
+
+mkdir m4 2>/dev/null
+
+BROTLI_ABI_HEX=`sed -n 's/#define BROTLI_ABI_VERSION 0x//p' c/common/version.h`
+BROTLI_ABI_INT=`echo $((16#$BROTLI_ABI_HEX))`
+BROTLI_ABI_CURRENT=$(($BROTLI_ABI_INT >> 24))
+BROTLI_ABI_REVISION=$((($BROTLI_ABI_INT >> 12) & 4095))
+BROTLI_ABI_AGE=$(($BROTLI_ABI_INT & 4095))
+BROTLI_ABI_INFO="$BROTLI_ABI_CURRENT:$BROTLI_ABI_REVISION:$BROTLI_ABI_AGE"
+
+BROTLI_VERSION_HEX=`sed -n 's/#define BROTLI_VERSION 0x//p' c/common/version.h`
+BROTLI_VERSION_INT=`echo $((16#$BROTLI_VERSION_HEX))`
+BROTLI_VERSION_MAJOR=$(($BROTLI_VERSION_INT >> 24))
+BROTLI_VERSION_MINOR=$((($BROTLI_VERSION_INT >> 12) & 4095))
+BROTLI_VERSION_PATCH=$(($BROTLI_VERSION_INT & 4095))
+BROTLI_VERSION="$BROTLI_VERSION_MAJOR.$BROTLI_VERSION_MINOR.$BROTLI_VERSION_PATCH"
+
+sed -r "s/[0-9]+:[0-9]+:[0-9]+/$BROTLI_ABI_INFO/" Makefile.am -i
+sed -r "s/\[[0-9]+\.[0-9]+\.[0-9]+\]/[$BROTLI_VERSION]/" configure.ac -i
+
+autoreconf --install --force --symlink || exit $

c/common/version.h

@@ -14,6 +14,13 @@
    BrotliEncoderVersion methods. */
 
 /* Semantic version, calculated as (MAJOR << 24) | (MINOR << 12) | PATCH */
-#define BROTLI_VERSION 0x1000001
+#define BROTLI_VERSION 0x1000002
+
+/* This macro is used by build system to produce Libtool-friendly soname. See
+   https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html
+ */
+
+/* ABI version, calculated as (CURRENT << 24) | (REVISION << 12) | AGE */
+#define BROTLI_ABI_VERSION 0x1002000
 
 #endif  /* BROTLI_COMMON_VERSION_H_ */

c/dec/decode.c

@@ -1911,6 +1911,10 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
     size_t* available_out, uint8_t** next_out, size_t* total_out) {
   BrotliDecoderErrorCode result = BROTLI_DECODER_SUCCESS;
   BrotliBitReader* br = &s->br;
+  /* Ensure that *total_out is set, even if no data will ever be pushed out. */
+  if (total_out) {
+    *total_out = s->partial_pos_out;
+  }
   /* Do not try to process further in a case of unrecoverable error. */
   if ((int)s->error_code < 0) {
     return BROTLI_DECODER_RESULT_ERROR;

c/enc/port.h

@@ -167,13 +167,21 @@ TEMPLATE_(size_t) TEMPLATE_(uint32_t) TEMPLATE_(uint8_t)
   (A)[(J)] = __brotli_swap_tmp;   \
 }
 
+/*
+Dynamically grows array capacity to at least the requested size
+M: MemoryManager
+T: data type
+A: array
+C: capacity
+R: requested size
+*/
 #define BROTLI_ENSURE_CAPACITY(M, T, A, C, R) {  \
   if (C < (R)) {                                 \
     size_t _new_size = (C == 0) ? (R) : C;       \
     T* new_array;                                \
     while (_new_size < (R)) _new_size *= 2;      \
     new_array = BROTLI_ALLOC((M), T, _new_size); \
-    if (!BROTLI_IS_OOM(m) && C != 0)             \
+    if (!BROTLI_IS_OOM(M) && C != 0)             \
       memcpy(new_array, A, C * sizeof(T));       \
     BROTLI_FREE((M), A);                         \
     A = new_array;                               \

c/include/brotli/decode.h

@@ -159,10 +159,11 @@ BROTLI_DEC_API BROTLI_BOOL BrotliDecoderSetParameter(
  *
  * @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
  * case they are both zero, default memory allocators are used. @p opaque is
- * passed to @p alloc_func and @p free_func when they are called.
+ * passed to @p alloc_func and @p free_func when they are called. @p free_func
+ * should return without doing anything when asked to free a NULL pointer.
  *
  * @param alloc_func custom memory allocation function
- * @param free_func custom memory fee function
+ * @param free_func custom memory free function
  * @param opaque custom memory manager handle
  * @returns @c 0 if instance can not be allocated or initialized
  * @returns pointer to initialized ::BrotliDecoderState otherwise

c/include/brotli/encode.h

@@ -209,10 +209,11 @@ BROTLI_ENC_API BROTLI_BOOL BrotliEncoderSetParameter(
  *
  * @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
  * case they are both zero, default memory allocators are used. @p opaque is
- * passed to @p alloc_func and @p free_func when they are called.
+ * passed to @p alloc_func and @p free_func when they are called. @p free_func
+ * should return without doing anything when asked to free a NULL pointer.
  *
  * @param alloc_func custom memory allocation function
- * @param free_func custom memory fee function
+ * @param free_func custom memory free function
  * @param opaque custom memory manager handle
  * @returns @c 0 if instance can not be allocated or initialized
  * @returns pointer to initialized ::BrotliEncoderState otherwise

configure

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
-echo "Use Bazel, CMake or Premake5 to generate projects / build files."
+echo "Use Autotools, Bazel, CMake or Premake5 to generate projects / build files."
 echo "  Bazel: http://www.bazel.build/"
 echo "  CMake: https://cmake.org/"
 echo "  Premake5: https://premake.github.io/"
+echo "To generate Autotools 'configure' file run './bootstrap'."
+echo "Run './configure-cmake' for Autotools-like CMake configuration."
 echo "Or simply run 'make' to build and test command line tool."