Skip to content

Commit 77598fa

Browse files
committed
ARROW-233: Add visibility macros, add static build option
This also resolves ARROW-213. Builds off work done in PARQUET-489. I inserted a hack to deal with the fast the boost libs in apt won't statically link properly. We'll deal with that some other time. Author: Wes McKinney <wesm@apache.org> Closes #100 from wesm/ARROW-233 and squashes the following commits: 0253827 [Wes McKinney] Remove -Wno-unused-local-typedef 69b03b0 [Wes McKinney] - Add visibility macros. Hide boost symbols in arrow_io - Hack around Travis CI inability to use its boost static libraries - Use parquet_shared name - More informative verbose test logs - Fix some gtest-1.7.0 crankiness - Fix a valgrind shared_ptr possible memory leak stemming from static variable referenced at compile-time in libarrow_parquet - Fix a bunch of compiler warnings in release builds
1 parent fab4c82 commit 77598fa

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+439
-245
lines changed

ci/travis_install_conda.sh

-1
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,3 @@ conda install --yes conda-build jinja2 anaconda-client
2525

2626
# faster builds, please
2727
conda install -y nomkl
28-

ci/travis_script_cpp.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,6 @@ make lint
1616
# make check-clang-tidy
1717
# fi
1818

19-
ctest -L unittest
19+
ctest -VV -L unittest
2020

2121
popd

ci/travis_script_python.sh

+4-2
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ PYTHON_DIR=$TRAVIS_BUILD_DIR/python
77
# Re-use conda installation from C++
88
export MINICONDA=$TRAVIS_BUILD_DIR/miniconda
99
export PATH="$MINICONDA/bin:$PATH"
10-
export LD_LIBRARY_PATH="$MINICONDA/lib:$LD_LIBRARY_PATH"
1110
export PARQUET_HOME=$MINICONDA
1211

1312
# Share environment with C++
@@ -32,12 +31,15 @@ python_version_tests() {
3231
# Expensive dependencies install from Continuum package repo
3332
conda install -y pip numpy pandas cython
3433

34+
conda install -y parquet-cpp arrow-cpp -c apache/channel/dev
35+
3536
# Other stuff pip install
3637
pip install -r requirements.txt
3738

3839
export ARROW_HOME=$ARROW_CPP_INSTALL
3940

40-
python setup.py build_ext --inplace
41+
python setup.py build_ext \
42+
--inplace
4143

4244
python -m pytest -vv -r sxX pyarrow
4345
}

cpp/CMakeLists.txt

+118-99
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,22 @@ endif(CCACHE_FOUND)
4444

4545
# Top level cmake dir
4646
if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
47+
option(ARROW_BUILD_STATIC
48+
"Build the libarrow static libraries"
49+
ON)
50+
51+
option(ARROW_BUILD_SHARED
52+
"Build the libarrow shared libraries"
53+
ON)
54+
4755
option(ARROW_PARQUET
4856
"Build the Parquet adapter and link to libparquet"
4957
OFF)
58+
5059
option(ARROW_TEST_MEMCHECK
51-
"Run the test suite using valgrind --tool=memcheck"
52-
OFF)
60+
"Run the test suite using valgrind --tool=memcheck"
61+
OFF)
62+
5363
option(ARROW_BUILD_TESTS
5464
"Build the Arrow googletest unit tests"
5565
ON)
@@ -66,6 +76,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
6676
"Build the Arrow IO extensions for the Hadoop file system"
6777
OFF)
6878

79+
option(ARROW_BOOST_USE_SHARED
80+
"Rely on boost shared libraries where relevant"
81+
ON)
82+
6983
option(ARROW_SSE3
7084
"Build Arrow with SSE3"
7185
ON)
@@ -172,18 +186,6 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang")
172186
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CLANG_OPTIONS}")
173187
endif()
174188

175-
# Sanity check linking option.
176-
if (NOT ARROW_LINK)
177-
set(ARROW_LINK "d")
178-
elseif(NOT ("auto" MATCHES "^${ARROW_LINK}" OR
179-
"dynamic" MATCHES "^${ARROW_LINK}" OR
180-
"static" MATCHES "^${ARROW_LINK}"))
181-
message(FATAL_ERROR "Unknown value for ARROW_LINK, must be auto|dynamic|static")
182-
else()
183-
# Remove all but the first letter.
184-
string(SUBSTRING "${ARROW_LINK}" 0 1 ARROW_LINK)
185-
endif()
186-
187189
# ASAN / TSAN / UBSAN
188190
include(san-config)
189191

@@ -203,61 +205,11 @@ if ("${ARROW_GENERATE_COVERAGE}")
203205
# For coverage to work properly, we need to use static linkage. Otherwise,
204206
# __gcov_flush() doesn't properly flush coverage from every module.
205207
# See http://stackoverflow.com/questions/28164543/using-gcov-flush-within-a-library-doesnt-force-the-other-modules-to-yield-gc
206-
if("${ARROW_LINK}" STREQUAL "a")
207-
message("Using static linking for coverage build")
208-
set(ARROW_LINK "s")
209-
elseif("${ARROW_LINK}" STREQUAL "d")
210-
message(SEND_ERROR "Cannot use coverage with dynamic linking")
211-
endif()
212-
endif()
213-
214-
# If we still don't know what kind of linking to perform, choose based on
215-
# build type (developers like fast builds).
216-
if ("${ARROW_LINK}" STREQUAL "a")
217-
if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR
218-
"${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
219-
message("Using dynamic linking for ${CMAKE_BUILD_TYPE} builds")
220-
set(ARROW_LINK "d")
221-
else()
222-
message("Using static linking for ${CMAKE_BUILD_TYPE} builds")
223-
set(ARROW_LINK "s")
208+
if(NOT ARROW_BUILD_STATIC)
209+
message(SEND_ERROR "Coverage requires the static lib to be built")
224210
endif()
225211
endif()
226212

227-
# Are we using the gold linker? It doesn't work with dynamic linking as
228-
# weak symbols aren't properly overridden, causing tcmalloc to be omitted.
229-
# Let's flag this as an error in RELEASE builds (we shouldn't release a
230-
# product like this).
231-
#
232-
# See https://sourceware.org/bugzilla/show_bug.cgi?id=16979 for details.
233-
#
234-
# The gold linker is only for ELF binaries, which OSX doesn't use. We can
235-
# just skip.
236-
if (NOT APPLE)
237-
execute_process(COMMAND ${CMAKE_CXX_COMPILER} -Wl,--version OUTPUT_VARIABLE LINKER_OUTPUT)
238-
endif ()
239-
if (LINKER_OUTPUT MATCHES "gold")
240-
if ("${ARROW_LINK}" STREQUAL "d" AND
241-
"${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
242-
message(SEND_ERROR "Cannot use gold with dynamic linking in a RELEASE build "
243-
"as it would cause tcmalloc symbols to get dropped")
244-
else()
245-
message("Using gold linker")
246-
endif()
247-
set(ARROW_USING_GOLD 1)
248-
else()
249-
message("Using ld linker")
250-
endif()
251-
252-
# Having set ARROW_LINK due to build type and/or sanitizer, it's now safe to
253-
# act on its value.
254-
if ("${ARROW_LINK}" STREQUAL "d")
255-
set(BUILD_SHARED_LIBS ON)
256-
257-
# Position independent code is only necessary when producing shared objects.
258-
add_definitions(-fPIC)
259-
endif()
260-
261213
# set compile output directory
262214
string (TOLOWER ${CMAKE_BUILD_TYPE} BUILD_SUBDIR_NAME)
263215

@@ -290,6 +242,15 @@ set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
290242
set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
291243
include_directories(src)
292244

245+
############################################################
246+
# Visibility
247+
############################################################
248+
# For generate_export_header() and add_compiler_export_flags().
249+
include(GenerateExportHeader)
250+
251+
# Sets -fvisibility=hidden for gcc
252+
add_compiler_export_flags()
253+
293254
############################################################
294255
# Benchmarking
295256
############################################################
@@ -360,7 +321,7 @@ endfunction()
360321
#
361322
# Arguments after the test name will be passed to set_tests_properties().
362323
function(ADD_ARROW_TEST REL_TEST_NAME)
363-
if(NO_TESTS)
324+
if(NO_TESTS OR NOT ARROW_BUILD_STATIC)
364325
return()
365326
endif()
366327
get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
@@ -377,13 +338,13 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
377338
endif()
378339

379340
if (ARROW_TEST_MEMCHECK)
380-
SET_PROPERTY(TARGET ${TEST_NAME}
381-
APPEND_STRING PROPERTY
382-
COMPILE_FLAGS " -DARROW_VALGRIND")
383-
add_test(${TEST_NAME}
384-
valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH})
341+
SET_PROPERTY(TARGET ${TEST_NAME}
342+
APPEND_STRING PROPERTY
343+
COMPILE_FLAGS " -DARROW_VALGRIND")
344+
add_test(${TEST_NAME}
345+
valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH})
385346
else()
386-
add_test(${TEST_NAME}
347+
add_test(${TEST_NAME}
387348
${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
388349
endif()
389350
set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest")
@@ -427,19 +388,34 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
427388
message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
428389
endif()
429390

430-
if(("${ARROW_LINK}" STREQUAL "s" AND ARG_STATIC_LIB) OR (NOT ARG_SHARED_LIB))
391+
if(ARG_STATIC_LIB AND ARG_SHARED_LIB)
431392
if(NOT ARG_STATIC_LIB)
432393
message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
433394
endif()
395+
396+
SET(AUG_LIB_NAME "${LIB_NAME}_static")
397+
add_library(${AUG_LIB_NAME} STATIC IMPORTED)
398+
set_target_properties(${AUG_LIB_NAME}
399+
PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
400+
message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
401+
402+
SET(AUG_LIB_NAME "${LIB_NAME}_shared")
403+
add_library(${AUG_LIB_NAME} SHARED IMPORTED)
404+
set_target_properties(${AUG_LIB_NAME}
405+
PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
406+
message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
407+
elseif(ARG_STATIC_LIB)
434408
add_library(${LIB_NAME} STATIC IMPORTED)
435409
set_target_properties(${LIB_NAME}
436410
PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
437411
message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
438-
else()
412+
elseif(ARG_SHARED_LIB)
439413
add_library(${LIB_NAME} SHARED IMPORTED)
440414
set_target_properties(${LIB_NAME}
441415
PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
442416
message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
417+
else()
418+
message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
443419
endif()
444420

445421
if(ARG_DEPS)
@@ -538,9 +514,17 @@ endif()
538514
############################################################
539515
# Linker setup
540516
############################################################
541-
set(ARROW_MIN_TEST_LIBS arrow arrow_test_main ${ARROW_BASE_LIBS})
517+
set(ARROW_MIN_TEST_LIBS
518+
arrow_static
519+
arrow_test_main
520+
${ARROW_BASE_LIBS})
521+
542522
set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
543-
set(ARROW_BENCHMARK_LINK_LIBS arrow arrow_benchmark_main ${ARROW_BASE_LIBS})
523+
524+
set(ARROW_BENCHMARK_LINK_LIBS
525+
arrow_static
526+
arrow_benchmark_main
527+
${ARROW_BASE_LIBS})
544528

545529
############################################################
546530
# "make ctags" target
@@ -576,14 +560,14 @@ endif (UNIX)
576560
if (UNIX)
577561

578562
file(GLOB_RECURSE LINT_FILES
579-
"${CMAKE_CURRENT_SOURCE_DIR}/src/*.h"
580-
"${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc"
581-
)
563+
"${CMAKE_CURRENT_SOURCE_DIR}/src/*.h"
564+
"${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc"
565+
)
582566

583567
FOREACH(item ${LINT_FILES})
584-
IF(NOT (item MATCHES "_generated.h"))
568+
IF(NOT (item MATCHES "_generated.h"))
585569
LIST(APPEND FILTERED_LINT_FILES ${item})
586-
ENDIF()
570+
ENDIF()
587571
ENDFOREACH(item ${LINT_FILES})
588572

589573
# Full lint
@@ -628,7 +612,10 @@ endif()
628612
# Subdirectories
629613
############################################################
630614

631-
set(LIBARROW_LINK_LIBS
615+
set(ARROW_LINK_LIBS
616+
)
617+
618+
set(ARROW_PRIVATE_LINK_LIBS
632619
)
633620

634621
set(ARROW_SRCS
@@ -660,35 +647,67 @@ set(ARROW_SRCS
660647
src/arrow/util/status.cc
661648
)
662649

663-
set(LIBARROW_LINKAGE "SHARED")
664-
665-
add_library(arrow
666-
${LIBARROW_LINKAGE}
650+
add_library(arrow_objlib OBJECT
667651
${ARROW_SRCS}
668652
)
669653

654+
# Necessary to make static linking into other shared libraries work properly
655+
set_property(TARGET arrow_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
656+
657+
if(NOT APPLE)
658+
# Localize thirdparty symbols using a linker version script. This hides them
659+
# from the client application. The OS X linker does not support the
660+
# version-script option.
661+
set(SHARED_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/src/arrow/symbols.map")
662+
endif()
663+
664+
if (ARROW_BUILD_SHARED)
665+
add_library(arrow_shared SHARED $<TARGET_OBJECTS:arrow_objlib>)
666+
if(APPLE)
667+
set_target_properties(arrow_shared PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
668+
endif()
669+
set_target_properties(arrow_shared
670+
PROPERTIES
671+
LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
672+
LINK_FLAGS "${SHARED_LINK_FLAGS}"
673+
OUTPUT_NAME "arrow")
674+
target_link_libraries(arrow_shared
675+
LINK_PUBLIC ${ARROW_LINK_LIBS}
676+
LINK_PRIVATE ${ARROW_PRIVATE_LINK_LIBS})
677+
678+
install(TARGETS arrow_shared
679+
LIBRARY DESTINATION lib
680+
ARCHIVE DESTINATION lib)
681+
endif()
682+
683+
if (ARROW_BUILD_STATIC)
684+
add_library(arrow_static STATIC $<TARGET_OBJECTS:arrow_objlib>)
685+
set_target_properties(arrow_static
686+
PROPERTIES
687+
LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
688+
OUTPUT_NAME "arrow")
689+
690+
target_link_libraries(arrow_static
691+
LINK_PUBLIC ${ARROW_LINK_LIBS}
692+
LINK_PRIVATE ${ARROW_PRIVATE_LINK_LIBS})
693+
694+
install(TARGETS arrow_static
695+
LIBRARY DESTINATION lib
696+
ARCHIVE DESTINATION lib)
697+
endif()
698+
670699
if (APPLE)
671-
set_target_properties(arrow
700+
set_target_properties(arrow_shared
672701
PROPERTIES
673702
BUILD_WITH_INSTALL_RPATH ON
674703
INSTALL_NAME_DIR "@rpath")
675704
endif()
676705

677-
set_target_properties(arrow
678-
PROPERTIES
679-
LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
680-
)
681-
target_link_libraries(arrow ${LIBARROW_LINK_LIBS})
682-
683706
add_subdirectory(src/arrow)
684707
add_subdirectory(src/arrow/io)
685708
add_subdirectory(src/arrow/util)
686709
add_subdirectory(src/arrow/types)
687710

688-
install(TARGETS arrow
689-
LIBRARY DESTINATION lib
690-
ARCHIVE DESTINATION lib)
691-
692711
#----------------------------------------------------------------------
693712
# Parquet adapter library
694713

@@ -715,7 +734,7 @@ if(ARROW_IPC)
715734
include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
716735
add_library(flatbuffers STATIC IMPORTED)
717736
set_target_properties(flatbuffers PROPERTIES
718-
IMPORTED_LOCATION ${FLATBUFFERS_STATIC_LIB})
737+
IMPORTED_LOCATION ${FLATBUFFERS_STATIC_LIB})
719738

720739
add_subdirectory(src/arrow/ipc)
721740
endif()

0 commit comments

Comments
 (0)