Merge pull request #630 from facebookresearch/release/v0.2.3

CompilerGym v0.2.3

Author: ChrisCummins

.github/actions/install-cmake-build-dependencies/action.yaml

@@ -9,12 +9,10 @@ runs:
         - name: Install CMake dependencies
           run: |
               if [ "$(uname)" = "Darwin" ]; then
-                  echo "CMake build for Darwin is unimplemented."
-                  exit 1
+                  brew install bazelisk ccache cmake zlib
               else
                   # Compiler
-                  sudo apt-get install clang++-9 lld-9 tar bzip2 ninja-build
-                  sudo apt-get install tar bzip2 ninja-build
+                  sudo apt-get install -y bzip2 clang-9 clang++-9 lld-9 tar bzip2 ninja-build tar ninja-build
                   # CMake
                   wget https://github.com/Kitware/CMake/releases/download/v3.20.5/cmake-3.20.5-linux-x86_64.sh -O /tmp/cmake.sh
                   sudo bash /tmp/cmake.sh --prefix=/usr/local --exclude-subdir --skip-license

.github/actions/install-runtime-dependencies/action.yaml

@@ -9,11 +9,5 @@ runs:
               if [ "$(uname)" = "Darwin" ]; then
                   brew install zlib
               fi
-          shell: bash
-
-        # DGL creates ~/.dgl on first run and I have found that this will fail
-        # if run from pytest / bazel. Suppress errors if DGL is not installed.
-        - name: Initialize DGL
-          run: |
-              python3 -c 'import dgl; print(dgl.__version__)' || true
+              python -m pip install -U pip wheel
           shell: bash

.github/workflows/build.yaml

@@ -94,4 +94,4 @@ jobs:
               uses: ./.github/actions/install-runtime-dependencies
 
             - name: Run the examples tests
-              run: make examples-test
+              run: make -C examples test

.github/workflows/ci.yaml

@@ -52,15 +52,30 @@ jobs:
             - name: Install build dependencies
               uses: ./.github/actions/install-cmake-build-dependencies
 
+            - name: Download LLVM 10.0.0 release
+              run: |
+                  if [ "$(uname)" = "Darwin" ]; then
+                    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/clang+llvm-10.0.0-x86_64-apple-darwin.tar.xz -O ~/llvm.tar.xz
+                  else
+                    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/clang+llvm-10.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz -O ~/llvm.tar.xz
+                  fi
+                  # TODO(cummins): Remove 'v' debugging flag:
+                  mkdir ~/llvm && tar xvf ~/llvm.tar.xz --strip-components 1 -C ~/llvm
+                  rm ~/llvm.tar.xz
+                  echo "Unpacked, testing for expected file:"
+                  test -d ~/llvm/lib/cmake/llvm
+
             - name: CMake Build
               run: |
                   cmake \
                       -GNinja \
-                      -DCMAKE_C_COMPILER=clang-9 \
-                      -DCMAKE_CXX_COMPILER=clang++-9 \
+                      -DCMAKE_C_COMPILER=clang \
+                      -DCMAKE_CXX_COMPILER=clang++ \
                       -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" \
                       -DCMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld" \
                       -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld" \
+                      -DCOMPILER_GYM_LLVM_PROVIDER=external \
+                      -DLLVM_DIR=$HOME/llvm/lib/cmake/llvm \
                       -DPython3_FIND_VIRTUALENV=FIRST \
                       -DCOMPILER_GYM_BUILD_TESTS=ON \
                       -DCOMPILER_GYM_BUILD_EXAMPLES=ON \
@@ -345,42 +360,6 @@ jobs:
               uses: codecov/codecov-action@v2
 
     test-gcc-env-macos:
-        needs: build-macos
-        runs-on: macos-latest
-        steps:
-            - uses: docker-practice/actions-setup-docker@master
-
-            - name: Confirm docker install
-              run: docker version
-
-            - uses: actions/checkout@v2
-
-            - name: Set up Python
-              uses: actions/setup-python@v2
-              with:
-                  python-version: 3.9
-
-            - name: Download Python wheel
-              uses: actions/download-artifact@v2
-              with:
-                  name: macos-wheel
-
-            - name: Install wheel
-              run: python -m pip install *.whl
-
-            - name: Install runtime dependencies
-              uses: ./.github/actions/install-runtime-dependencies
-
-            - name: Install test dependencies
-              run: python -m pip install -r tests/requirements.txt
-
-            - name: Run the test suite
-              run: make install-test-cov TEST_TARGET="tests/gcc"
-
-            - name: Upload coverage report to Codecov
-              uses: codecov/codecov-action@v2
-
-    test-gcc-env-macos-no-docker:
         needs: build-macos
         runs-on: macos-latest
         steps:
@@ -446,11 +425,6 @@ jobs:
         needs: build-macos
         runs-on: macos-latest
         steps:
-            - uses: docker-practice/actions-setup-docker@master
-
-            - name: Confirm docker install
-              run: docker version
-
             - uses: actions/checkout@v2
 
             - name: Set up Python
@@ -501,14 +475,14 @@ jobs:
             - name: Install wheel
               run: python -m pip install *.whl
 
-            - name: Install test dependencies
-              run: python -m pip install -r examples/requirements.txt -r tests/requirements.txt
-
             - name: Install runtime dependencies
               uses: ./.github/actions/install-runtime-dependencies
 
+            - name: Install test dependencies
+              run: make -C examples install
+
             - name: Run the test suite
-              run: make examples-test
+              run: make -C examples test
 
             - name: Upload coverage report to Codecov
               uses: codecov/codecov-action@v2
@@ -536,14 +510,14 @@ jobs:
             - name: Install wheel
               run: python -m pip install *.whl
 
-            - name: Install test dependencies
-              run: python -m pip install -r examples/requirements.txt -r tests/requirements.txt
-
             - name: Install runtime dependencies
               uses: ./.github/actions/install-runtime-dependencies
 
+            - name: Install test dependencies
+              run: make -C examples install
+
             - name: Run the test suite
-              run: make examples-test
+              run: make -C examples test
 
             - name: Upload coverage report to Codecov
               uses: codecov/codecov-action@v2
@@ -636,7 +610,7 @@ jobs:
             fail-fast: false
             matrix:
                 os: [ubuntu-latest, macos-latest]
-                node: [16]
+                node: [17]
         steps:
             - uses: actions/checkout@v2
 

.github/workflows/pip_install_test.yaml

@@ -1,8 +1,10 @@
 ---
-# This workflow pip install's teh compiler_gym package and checks that the
-# module can be imported. The goal of this workflow is to catch failures in the
-# installation process that can occur because of a breaking change in the
-# dependent packages.
+# This workflow pip install's the compiler_gym package and checks that the
+# module can be imported.
+#
+# The goal of this workflow is to catch failures in the installation process
+# that can occur because of a breaking change in the dependent packages, and to
+# test for any import-time error.
 name: Pip install test
 
 on:
@@ -26,7 +28,12 @@ jobs:
                   python-version: ${{ matrix.python }}
 
             - name: Install python wheel
-              run: python -m pip install compiler_gym
+              # We use the TestPyPI repository to install the compiler_gym
+              # package from because we use the install count on the main
+              # repository to track growth and usage of the project. We also
+              # provide the main PyPI repo as a fallback for installing
+              # dependencies.
+              run: python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ compiler_gym
 
             - name: Check package version
               run: python -c 'import compiler_gym; print(compiler_gym.__version__)'

.pre-commit-config.yaml

@@ -72,3 +72,7 @@ repos:
       rev: v1.0
       hooks:
           - id: dockerlint
+    - repo: https://github.com/BlankSpruce/gersemi
+      rev: 0.7.5
+      hooks:
+          - id: gersemi

CMakeLists.txt

@@ -6,67 +6,79 @@
 cmake_minimum_required(VERSION 3.20)
 
 if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
-  message(FATAL_ERROR "In-source builds are unsupported. Please, build out of the source tree.")
+    message(
+        FATAL_ERROR
+        "In-source builds are unsupported. Please, build out of the source tree."
+    )
 endif()
 
-if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-    set(DARWIN TRUE)
-endif()
-
-set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-
 project(compiler_gym ASM C CXX)
 
 set(CMAKE_C_STANDARD 11 CACHE STRING "C standard to be used.")
 set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to be used.")
 
 set_property(GLOBAL PROPERTY USE_FOLDERS ON)
 
-list(APPEND CMAKE_MODULE_PATH
-  ${CMAKE_CURRENT_LIST_DIR}/build_tools/cmake/
-)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/build_tools/cmake)
 
+# Project features.
 set(COMPILER_GYM_BUILD_TESTS OFF CACHE BOOL "Enable Compiler Gym tests.")
 set(COMPILER_GYM_BUILD_EXAMPLES OFF CACHE BOOL "Enable Comiler Gym examples.")
 
-include(cg_macros)
-include(cg_copts)
-include(cg_genrule)
-include(cg_cc_binary)
-include(cg_cc_library)
-include(cg_cc_test)
-include(cg_py_binary)
-include(cg_py_library)
-include(cg_py_test)
-include(cg_python)
-include(cg_add_all_subdirs)
-include(cg_filegroup)
-include(grpc)
-include(protobuf)
-
-set(COMPILER_GYM_PYTHONPATH "$ENV{PYTHONPATH}" CACHE STRING "PYTHONPATH environment variable during build step.")
-if (COMPILER_GYM_PYTHONPATH)
-  string(PREPEND COMPILER_GYM_PYTHONPATH ":")
+# Toolchain configuration.
+include(build_tools/cmake/FindAndEnableCcache.cmake)
+include(build_tools/cmake/FindAndEnableLld.cmake)
+
+include(build_tools/cmake/cg_macros.cmake)
+include(build_tools/cmake/cg_copts.cmake)
+include(build_tools/cmake/cg_genrule.cmake)
+include(build_tools/cmake/cg_cc_binary.cmake)
+include(build_tools/cmake/cg_cc_library.cmake)
+include(build_tools/cmake/cg_cc_test.cmake)
+include(build_tools/cmake/cg_py_binary.cmake)
+include(build_tools/cmake/cg_py_library.cmake)
+include(build_tools/cmake/cg_py_test.cmake)
+include(build_tools/cmake/cg_python.cmake)
+include(build_tools/cmake/cg_add_all_subdirs.cmake)
+include(build_tools/cmake/cg_filegroup.cmake)
+include(build_tools/cmake/grpc.cmake)
+include(build_tools/cmake/protobuf.cmake)
+
+# Set up the build Python path.
+set(COMPILER_GYM_BUILD_PYTHONPATH "$ENV{PYTHONPATH}"
+    CACHE STRING "PYTHONPATH environment variable during build step."
+)
+if(COMPILER_GYM_BUILD_PYTHONPATH)
+    string(PREPEND COMPILER_GYM_BUILD_PYTHONPATH ":")
 endif()
-string(PREPEND COMPILER_GYM_PYTHONPATH "${CMAKE_BINARY_DIR}")
-include(set_command_pythonpath)
+string(PREPEND COMPILER_GYM_BUILD_PYTHONPATH "${CMAKE_BINARY_DIR}")
+include(build_tools/cmake/set_command_pythonpath.cmake)
 
 set(DEFAULT_CMAKE_BUILD_TYPE "Release")
 if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
-  message(STATUS "No build type selected, default to ${DEFAULT_CMAKE_BUILD_TYPE}")
-  set(CMAKE_BUILD_TYPE "${DEFAULT_CMAKE_BUILD_TYPE}" CACHE STRING "Build type (default ${DEFAULT_CMAKE_BUILD_TYPE})" FORCE)
+    message(
+        STATUS
+        "No build type selected, default to ${DEFAULT_CMAKE_BUILD_TYPE}"
+    )
+    set(CMAKE_BUILD_TYPE "${DEFAULT_CMAKE_BUILD_TYPE}"
+        CACHE STRING "Build type (default ${DEFAULT_CMAKE_BUILD_TYPE})"
+        FORCE
+    )
 endif()
 
 set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
 
 find_package(Python3 REQUIRED COMPONENTS Interpreter)
 
 include(external/external.cmake)
+
 add_subdirectory(compiler_gym)
+
 if(COMPILER_GYM_BUILD_TESTS)
-  enable_testing()
-  add_subdirectory(tests)
+    enable_testing()
+    add_subdirectory(tests)
 endif()
+
 if(COMPILER_GYM_BUILD_EXAMPLES)
-  add_subdirectory(examples)
+    add_subdirectory(examples)
 endif()

INSTALL.md

@@ -54,7 +54,7 @@ with the required dependencies:
 
     conda create -y -n compiler_gym python=3.8
     conda activate compiler_gym
-    conda install -y -c conda-forge cmake pandoc patchelf
+    conda install -y -c conda-forge cmake doxygen pandoc patchelf
 
 Then clone the CompilerGym source code using:
 
@@ -122,24 +122,24 @@ By default most dependencies are built together with Compiler Gym. To search for
 * `COMPILER_GYM_GLOG_PROVIDER`
 * `COMPILER_GYM_GRPC_PROVIDER`
 * `COMPILER_GYM_GTEST_PROVIDER`
+* `COMPILER_GYM_LLVM_PROVIDER`
 * `COMPILER_GYM_NLOHMANN_JSON_PROVIDER`
 * `COMPILER_GYM_PROTOBUF_PROVIDER`
 
 ```bash
-cmake -GNinja \
+cmake \
   -DCMAKE_C_COMPILER=clang-9 \
   -DCMAKE_CXX_COMPILER=clang++-9 \
-  -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ # For faster rebuilds, can be removed
-  -DCMAKE_EXE_LINKER_FLAGS_INIT="-fuse-ld=lld" -DCMAKE_MODULE_LINKER_FLAGS_INIT="-fuse-ld=lld" -DCMAKE_SHARED_LINKER_FLAGS_INIT="-fuse-ld=lld" \ # For faster builds, can be removed
   -DPython3_FIND_VIRTUALENV=FIRST \
   -DCMAKE_BUILD_WITH_INSTALL_RPATH=true \
   -S "<path to source directory>" \
   -B "<path to build directory>"
 
-cmake  --build "<path to build directory>"
+cmake --build "<path to build directory>"
 
 pip install <path to build directory>/py_pkg/dist/compiler_gym*.whl --force-reinstall
 ```
+
 Additional optional configuration arguments:
 
 * Enables testing.
@@ -168,3 +168,18 @@ Additional optional configuration arguments:
     -DCMAKE_C_COMPILER_LAUNCHER=ccache
     -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
     ```
+
+By default, CompilerGym builds LLVM from source. This takes a long time and a
+lot of compute resources. To prevent this, download a pre-compiled clang+llvm
+release of LLVM 10.0.0 from the [llvm-project releases
+page](https://github.com/llvm/llvm-project/releases/tag/llvmorg-10.0.0), unpack
+it, and pass path of the `lib/cmake/llvm` subdirectory in the archive you just
+extracted to `LLVM_DIR`:
+
+```
+$ cmake ... \
+    -DCOMPILER_GYM_LLVM_PROVIDER=external \
+    -DLLVM_DIR=/path/to/llvm/lib/cmake/llvm
+```
+
+⚠️ CompilerGym requires exactly LLVM 10.0.0.