resolve conflict

PaddlePaddle · Jun 7, 2023 · 1744ded · 1744ded
2 parents d8d19ff + 099b3d2
commit 1744ded
Show file tree

Hide file tree

Showing 106 changed files with 2,210 additions and 584 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -50,6 +50,18 @@
 	path = third_party/eigen3
 	url = https://gitlab.com/libeigen/eigen.git
 	ignore = dirty
+[submodule "third_party/snappy"]
+	path = third_party/snappy
+	url = https://github.com/google/snappy.git
+	ignore = dirty
+[submodule "third_party/cub"]
+	path = third_party/cub
+	url = https://github.com/NVIDIA/cub.git
+	ignore = dirty
+[submodule "third_party/cutlass"]
+	path = third_party/cutlass
+	url = https://github.com/NVIDIA/cutlass.git
+	ignore = dirty
 [submodule "third_party/mkldnn"]
 	path = third_party/mkldnn
 	url = https://github.com/oneapi-src/oneDNN.git

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -264,7 +264,6 @@ option(COVERALLS_UPLOAD "Package code coverage data to coveralls" OFF)
 option(WITH_PSLIB "Compile with pslib support" OFF)
 option(WITH_BOX_PS "Compile with box_ps support" OFF)
 option(WITH_XBYAK "Compile with xbyak support" ON)
-option(WITH_CONTRIB "Compile the third-party contributation" OFF)
 option(WITH_PSCORE "Compile with parameter server support" ${WITH_DISTRIBUTE})
 option(WITH_HETERPS "Compile with heterps" OFF)
 option(WITH_INFERENCE_API_TEST

diff --git a/cmake/external/cub.cmake b/cmake/external/cub.cmake
@@ -22,26 +22,27 @@ set(CUB_PATH
     CACHE STRING "A path setting for external_cub path.")
 set(CUB_PREFIX_DIR ${CUB_PATH})
 
-set(CUB_REPOSITORY ${GIT_URL}/NVlabs/cub.git)
+set(CUB_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/cub)
 
 if(${CMAKE_CUDA_COMPILER_VERSION} GREATER_EQUAL 11.6)
   # cuda_11.6/11.7/11.8‘s own cub is 1.15.0, which will cause compiling error in windows.
   set(CUB_TAG 1.16.0)
+  execute_process(COMMAND git --git-dir=${CUB_SOURCE_DIR}/.git
+                          --work-tree=${CUB_SOURCE_DIR} checkout ${CUB_TAG})
   # cub 1.16.0 is not compitable with current thrust version
   add_definitions(-DTHRUST_IGNORE_CUB_VERSION_CHECK)
 else()
   set(CUB_TAG 1.8.0)
 endif()
 
-set(CUB_INCLUDE_DIR ${CUB_PREFIX_DIR}/src/extern_cub)
+set(CUB_INCLUDE_DIR ${CUB_SOURCE_DIR})
 message("CUB_INCLUDE_DIR is ${CUB_INCLUDE_DIR}")
 include_directories(${CUB_INCLUDE_DIR})
 
 ExternalProject_Add(
   extern_cub
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${CUB_REPOSITORY}
-  GIT_TAG ${CUB_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${CUB_SOURCE_DIR}
   PREFIX ${CUB_PREFIX_DIR}
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""

diff --git a/cmake/external/cutlass.cmake b/cmake/external/cutlass.cmake
@@ -15,14 +15,12 @@
 include(ExternalProject)
 
 set(CUTLASS_PREFIX_DIR ${THIRD_PARTY_PATH}/cutlass)
-
-set(CUTLASS_REPOSITORY https://github.com/NVIDIA/cutlass.git)
 set(CUTLASS_TAG v2.11.0)
+set(CUTLASS_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/cutlass)
 
-include_directories("${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/")
-include_directories("${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/include/")
-include_directories(
-  "${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/tools/util/include/")
+include_directories("${CUTLASS_SOURCE_DIR}/")
+include_directories("${CUTLASS_SOURCE_DIR}/include/")
+include_directories("${CUTLASS_SOURCE_DIR}/tools/util/include/")
 
 add_definitions("-DPADDLE_WITH_CUTLASS")
 add_definitions("-DSPCONV_WITH_CUTLASS=0")
@@ -33,9 +31,8 @@ endif()
 
 ExternalProject_Add(
   extern_cutlass
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${CUTLASS_REPOSITORY}
-  GIT_TAG "${CUTLASS_TAG}"
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${CUTLASS_SOURCE_DIR}
   PREFIX ${CUTLASS_PREFIX_DIR}
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""
@@ -61,7 +58,7 @@ add_custom_target(
   COMMAND
     ${PYTHON_EXECUTABLE} -B
     ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_generator.py
-    "${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/tools/library/scripts/"
+    "${CUTLASS_SOURCE_DIR}/tools/library/scripts/"
     "${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator"
     "${CMAKE_CUDA_COMPILER_VERSION}"
   COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_gemm_operations_file}

diff --git a/cmake/external/lapack.cmake b/cmake/external/lapack.cmake
@@ -15,18 +15,19 @@
 include(ExternalProject)
 
 set(LAPACK_PREFIX_DIR ${THIRD_PARTY_PATH}/lapack)
-set(LAPACK_SOURCE_DIR ${THIRD_PARTY_PATH}/lapack/src/extern_lapack)
+set(LAPACK_DOWNLOAD_DIR
+    ${PADDLE_SOURCE_DIR}/third_party/lapack/${CMAKE_SYSTEM_NAME})
 set(LAPACK_INSTALL_DIR ${THIRD_PARTY_PATH}/install/lapack)
 set(LAPACK_LIB_DIR ${LAPACK_INSTALL_DIR}/lib)
 
 # Note(zhouwei): lapack need fortan compiler which many machines don't have, so use precompiled library.
 # use lapack tag v3.10.0 on 06/28/2021 https://github.com/Reference-LAPACK/lapack
 if(LINUX)
-  set(LAPACK_VER
-      "lapack_lnx_v3.10.0.20210628"
+  set(LAPACK_FILE
+      "lapack_lnx_v3.10.0.20210628.tar.gz"
       CACHE STRING "" FORCE)
   set(LAPACK_URL
-      "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}.tar.gz"
+      "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_FILE}"
       CACHE STRING "" FORCE)
   set(LAPACK_URL_MD5 71f8cc8237a8571692f3e07f9a4f25f6)
   set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath.so.0")
@@ -35,11 +36,11 @@ if(LINUX)
   set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.so.3")
 elseif(WIN32)
   # Refer to [lapack-for-windows] http://icl.cs.utk.edu/lapack-for-windows/lapack/#lapacke
-  set(LAPACK_VER
-      "lapack_win_v3.10.0.20210628"
+  set(LAPACK_FILE
+      "lapack_win_v3.10.0.20210628.zip"
       CACHE STRING "" FORCE)
   set(LAPACK_URL
-      "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}.zip"
+      "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_FILE}"
       CACHE STRING "" FORCE)
   set(LAPACK_URL_MD5 590d080392dcd5abbd5dca767a50b63a)
   set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath-0.dll")
@@ -48,11 +49,11 @@ elseif(WIN32)
   set(BLAS_LIB "${LAPACK_LIB_DIR}/libblas.dll")
   set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.dll")
 else()
-  set(LAPACK_VER
-      "lapack_mac_v3.10.0.20210628"
+  set(LAPACK_FILE
+      "lapack_mac_v3.10.0.20210628.tar.gz"
       CACHE STRING "" FORCE)
   set(LAPACK_URL
-      "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}.tar.gz"
+      "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_FILE}"
       CACHE STRING "" FORCE)
   set(LAPACK_URL_MD5 427aecf8dee8523de3566ca8e47944d7)
   set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath.0.dylib")
@@ -62,18 +63,53 @@ else()
   set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.3.dylib")
 endif()
 
+function(download_lapack)
+  message(
+    STATUS "Downloading ${LAPACK_URL} to ${LAPACK_DOWNLOAD_DIR}/${LAPACK_FILE}")
+  # NOTE: If the version is updated, consider emptying the folder; maybe add timeout
+  file(
+    DOWNLOAD ${LAPACK_URL} ${LAPACK_DOWNLOAD_DIR}/${LAPACK_FILE}
+    EXPECTED_MD5 ${LAPACK_URL_MD5}
+    STATUS ERR)
+  if(ERR EQUAL 0)
+    message(STATUS "Download ${LAPACK_FILE} success")
+  else()
+    message(
+      FATAL_ERROR
+        "Download failed, error: ${ERR}\n You can try downloading ${LAPACK_FILE} again"
+    )
+  endif()
+endfunction()
+
+find_file(
+  LOCAL_LAPACK_LIB_ZIP
+  NAMES ${LAPACK_FILE}
+  PATHS ${LAPACK_DOWNLOAD_DIR}
+  NO_DEFAULT_PATH)
+
+# Download and check lapack.
+if(LOCAL_LAPACK_LIB_ZIP)
+  file(MD5 ${LAPACK_DOWNLOAD_DIR}/${LAPACK_FILE} LAPACK_MD5)
+  if(NOT LAPACK_MD5 EQUAL LAPACK_URL_MD5)
+    download_lapack()
+  endif()
+else()
+  download_lapack()
+endif()
+
 ExternalProject_Add(
   extern_lapack
   ${EXTERNAL_PROJECT_LOG_ARGS}
-  URL ${LAPACK_URL}
+  URL ${LAPACK_DOWNLOAD_DIR}/${LAPACK_FILE}
   URL_MD5 ${LAPACK_URL_MD5}
+  DOWNLOAD_DIR ${LAPACK_DOWNLOAD_DIR}
+  SOURCE_DIR ${LAPACK_LIB_DIR}
   PREFIX ${LAPACK_PREFIX_DIR}
   DOWNLOAD_NO_PROGRESS 1
   PATCH_COMMAND ""
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""
   BUILD_COMMAND ""
-  INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory ${LAPACK_SOURCE_DIR}
-                  ${LAPACK_LIB_DIR}
+  INSTALL_COMMAND ""
   BUILD_BYPRODUCTS ${BLAS_LIB}
   BUILD_BYPRODUCTS ${LAPACK_LIB})
diff --git a/cmake/external/libmct.cmake b/cmake/external/libmct.cmake
@@ -23,13 +23,17 @@ if((NOT DEFINED LIBMCT_VER) OR (NOT DEFINED LIBMCT_URL))
   set(LIBMCT_NAME
       "libmct"
       CACHE STRING "" FORCE)
+  set(LIBMCT_DOWNLOAD_FILE
+      "${LIBMCT_NAME}.tar.gz"
+      CACHE STRING "" FORCE)
   set(LIBMCT_URL
-      "https://pslib.bj.bcebos.com/libmct/libmct.tar.gz"
+      "https://pslib.bj.bcebos.com/libmct/${LIBMCT_DOWNLOAD_FILE}"
       CACHE STRING "" FORCE)
 endif()
 message(STATUS "LIBMCT_NAME: ${LIBMCT_NAME}, LIBMCT_URL: ${LIBMCT_URL}")
 set(LIBMCT_PREFIX_DIR "${THIRD_PARTY_PATH}/libmct")
-set(LIBMCT_DOWNLOAD_DIR "${LIBMCT_PREFIX_DIR}/src/${LIBMCT_PROJECT}")
+set(LIBMCT_DOWNLOAD_DIR
+    ${PADDLE_SOURCE_DIR}/third_party/libmct/${CMAKE_SYSTEM_NAME})
 set(LIBMCT_DST_DIR "libmct")
 set(LIBMCT_INSTALL_ROOT "${THIRD_PARTY_PATH}/install")
 set(LIBMCT_INSTALL_DIR ${LIBMCT_INSTALL_ROOT}/${LIBMCT_DST_DIR})
@@ -39,22 +43,27 @@ set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${LIBMCT_ROOT}/lib")
 
 include_directories(${LIBMCT_INC_DIR})
 
+file(
+  DOWNLOAD ${LIBMCT_URL} ${LIBMCT_DOWNLOAD_DIR}/${LIBMCT_DOWNLOAD_FILE}
+  TLS_VERIFY OFF
+  STATUS ERR)
+
 file(
   WRITE ${LIBMCT_DOWNLOAD_DIR}/CMakeLists.txt
   "PROJECT(LIBMCT)\n" "cmake_minimum_required(VERSION 3.0)\n"
-  "install(DIRECTORY ${LIBMCT_NAME}/include ${LIBMCT_NAME}/lib \n"
+  "install(DIRECTORY ./include ./lib \n"
   "        DESTINATION ${LIBMCT_DST_DIR})\n")
 
 ExternalProject_Add(
   ${LIBMCT_PROJECT}
   ${EXTERNAL_PROJECT_LOG_ARGS}
+  URL ${LIBMCT_DOWNLOAD_DIR}/${LIBMCT_DOWNLOAD_FILE}
   PREFIX ${LIBMCT_PREFIX_DIR}
   DOWNLOAD_DIR ${LIBMCT_DOWNLOAD_DIR}
-  DOWNLOAD_COMMAND
-    wget --no-check-certificate ${LIBMCT_URL} -c -q -O ${LIBMCT_NAME}.tar.gz &&
-    tar --no-same-owner -zxvf ${LIBMCT_NAME}.tar.gz
-  DOWNLOAD_NO_PROGRESS 1
+  SOURCE_DIR ${LIBMCT_INSTALL_DIR}
   UPDATE_COMMAND ""
+  COMMAND ${CMAKE_COMMAND} -E copy ${LIBMCT_DOWNLOAD_DIR}/CMakeLists.txt
+          ${LIBMCT_INSTALL_DIR}
   CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBMCT_INSTALL_ROOT}
              -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
   CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${LIBMCT_INSTALL_ROOT}

diff --git a/cmake/external/mklml.cmake b/cmake/external/mklml.cmake
@@ -17,13 +17,15 @@ set(MKLML_INSTALL_DIR ${THIRD_PARTY_PATH}/install/mklml)
 set(MKLML_INC_DIR ${MKLML_INSTALL_DIR}/include)
 set(MKLML_LIB_DIR ${MKLML_INSTALL_DIR}/lib)
 set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${MKLML_LIB_DIR}")
+set(MKLML_DOWNLOAD_DIR
+    ${PADDLE_SOURCE_DIR}/third_party/mklml/${CMAKE_SYSTEM_NAME})
 
 if(WIN32)
-  set(MKLML_VER
-      "mklml_win_2019.0.5.20190502"
+  set(MKLML_FILE
+      "mklml_win_2019.0.5.20190502.zip"
       CACHE STRING "" FORCE)
   set(MKLML_URL
-      "https://paddlepaddledeps.bj.bcebos.com/${MKLML_VER}.zip"
+      "https://paddlepaddledeps.bj.bcebos.com/${MKLML_FILE}"
       CACHE STRING "" FORCE)
   set(MKLML_URL_MD5 ff8c5237570f03eea37377ccfc95a08a)
   set(MKLML_LIB ${MKLML_LIB_DIR}/mklml.lib)
@@ -34,11 +36,11 @@ else()
   #TODO(intel-huying):
   #  Now enable csrmm function in mklml library temporarily,
   #  it will be updated as offical version later.
-  set(MKLML_VER
-      "csrmm_mklml_lnx_2019.0.5"
+  set(MKLML_FILE
+      "csrmm_mklml_lnx_2019.0.5.tgz"
       CACHE STRING "" FORCE)
   set(MKLML_URL
-      "http://paddlepaddledeps.bj.bcebos.com/${MKLML_VER}.tgz"
+      "http://paddlepaddledeps.bj.bcebos.com/${MKLML_FILE}"
       CACHE STRING "" FORCE)
   set(MKLML_URL_MD5 bc6a7faea6a2a9ad31752386f3ae87da)
   set(MKLML_LIB ${MKLML_LIB_DIR}/libmklml_intel.so)
@@ -48,9 +50,42 @@ else()
 endif()
 
 set(MKLML_PROJECT "extern_mklml")
-message(STATUS "MKLML_VER: ${MKLML_VER}, MKLML_URL: ${MKLML_URL}")
+message(STATUS "MKLML_FILE: ${MKLML_FILE}, MKLML_URL: ${MKLML_URL}")
 set(MKLML_PREFIX_DIR ${THIRD_PARTY_PATH}/mklml)
-set(MKLML_SOURCE_DIR ${THIRD_PARTY_PATH}/mklml/src/extern_mklml)
+
+function(download_mklml)
+  message(
+    STATUS "Downloading ${MKLML_URL} to ${MKLML_DOWNLOAD_DIR}/${MKLML_FILE}")
+  # NOTE: If the version is updated, consider emptying the folder; maybe add timeout
+  file(
+    DOWNLOAD ${MKLML_URL} ${MKLML_DOWNLOAD_DIR}/${MKLML_FILE}
+    EXPECTED_MD5 ${MKLML_URL_MD5}
+    STATUS ERR)
+  if(ERR EQUAL 0)
+    message(STATUS "Download ${MKLML_FILE} success")
+  else()
+    message(
+      FATAL_ERROR
+        "Download failed, error: ${ERR}\n You can try downloading ${MKLML_FILE} again"
+    )
+  endif()
+endfunction()
+
+find_file(
+  LOCAL_MKLML_LIB_ZIP
+  NAMES ${MKLML_FILE}
+  PATHS ${MKLML_DOWNLOAD_DIR}
+  NO_DEFAULT_PATH)
+
+# Download and check mklml.
+if(LOCAL_MKLML_LIB_ZIP)
+  file(MD5 ${MKLML_DOWNLOAD_DIR}/${MKLML_FILE} MKLML_MD5)
+  if(NOT MKLML_MD5 EQUAL MKLML_URL_MD5)
+    download_mklml()
+  endif()
+else()
+  download_mklml()
+endif()
 
 # Ninja Generator can not establish the correct dependency relationship
 # between the imported library with target, the product file
@@ -60,17 +95,15 @@ set(MKLML_SOURCE_DIR ${THIRD_PARTY_PATH}/mklml/src/extern_mklml)
 ExternalProject_Add(
   ${MKLML_PROJECT}
   ${EXTERNAL_PROJECT_LOG_ARGS}
-  URL ${MKLML_URL}
+  URL ${MKLML_DOWNLOAD_DIR}/${MKLML_FILE}
   URL_MD5 ${MKLML_URL_MD5}
+  DOWNLOAD_DIR ${MKLML_DOWNLOAD_DIR}
+  SOURCE_DIR ${MKLML_INSTALL_DIR}
   PREFIX ${MKLML_PREFIX_DIR}
-  DOWNLOAD_NO_PROGRESS 1
   CONFIGURE_COMMAND ""
   BUILD_COMMAND ""
   UPDATE_COMMAND ""
-  INSTALL_COMMAND
-    ${CMAKE_COMMAND} -E copy_directory ${MKLML_SOURCE_DIR}/include
-    ${MKLML_INC_DIR} && ${CMAKE_COMMAND} -E copy_directory
-    ${MKLML_SOURCE_DIR}/lib ${MKLML_LIB_DIR}
+  INSTALL_COMMAND ""
   BUILD_BYPRODUCTS ${MKLML_LIB}
   BUILD_BYPRODUCTS ${MKLML_IOMP_LIB})