forked from facebookresearch/faiss
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathaction.yml
176 lines (166 loc) · 7.15 KB
/
action.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
name: Build cmake
inputs:
opt_level:
description: 'Compile options / optimization level.'
required: false
default: generic
gpu:
description: 'Enable GPU support.'
required: false
default: OFF
cuvs:
description: 'Enable cuVS support.'
required: false
default: OFF
rocm:
description: 'Enable ROCm support.'
required: false
default: OFF
runs:
using: composite
steps:
- name: Setup miniconda
uses: conda-incubator/setup-miniconda@v3
with:
python-version: '3.11'
miniconda-version: latest
- name: Configure build environment
shell: bash
run: |
# initialize Conda
conda config --set solver libmamba
conda update -y -q conda
echo "$CONDA/bin" >> $GITHUB_PATH
conda install -y -q python=3.11 cmake=3.26 make=4.2 swig=4.0 "numpy<2" scipy=1.14 pytest=7.4 gflags=2.2
# install base packages for ARM64
if [ "${{ runner.arch }}" = "ARM64" ]; then
conda install -y -q -c conda-forge openblas=0.3 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17
fi
# install base packages for X86_64
if [ "${{ runner.arch }}" = "X64" ]; then
# TODO: merge this with ARM64
conda install -y -q -c conda-forge gxx_linux-64=14.2 sysroot_linux-64=2.17
conda install -y -q mkl=2023 mkl-devel=2023
fi
# no CUDA needed for ROCm so skip this
if [ "${{ inputs.rocm }}" = "ON" ]; then
:
# regular CUDA for GPU builds
elif [ "${{ inputs.gpu }}" = "ON" ] && [ "${{ inputs.cuvs }}" = "OFF" ]; then
conda install -y -q cuda-toolkit=12.4 -c "nvidia/label/cuda-12.4.0"
# and CUDA from cuVS channel for cuVS builds
elif [ "${{ inputs.cuvs }}" = "ON" ]; then
conda install -y -q libcuvs=24.12 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai -c conda-forge
fi
# install test packages
if [ "${{ inputs.rocm }}" = "ON" ]; then
: # skip torch install via conda, we need to install via pip to get
# ROCm-enabled version until it's supported in conda by PyTorch
elif [ "${{ inputs.gpu }}" = "ON" ]; then
conda install -y -q "pytorch<2.5" pytorch-cuda=12.4 -c pytorch -c "nvidia/label/cuda-12.4.0"
else
conda install -y -q "pytorch<2.5" -c pytorch
fi
- name: ROCm - Install dependencies
if: inputs.rocm == 'ON'
shell: bash
run: |
# Update repos and install kmod, wget, gpg
sudo apt-get -qq update >/dev/null
sudo apt-get -qq install -y kmod wget gpg >/dev/null
# Get UBUNTU version name
UBUNTU_VERSION_NAME=`cat /etc/os-release | grep UBUNTU_CODENAME | awk -F= '{print $2}'`
# Set ROCm version
ROCM_VERSION="6.2"
# Download, prepare, and install the package signing key
mkdir --parents --mode=0755 /etc/apt/keyrings
wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
# Add rocm repository
wget -qO - http://repo.radeon.com/rocm/rocm.gpg.key | sudo apt-key add -
rocm_baseurl="http://repo.radeon.com/rocm/apt/${ROCM_VERSION}"
echo "deb [arch=amd64] ${rocm_baseurl} ${UBUNTU_VERSION_NAME} main" | sudo tee /etc/apt/sources.list.d/rocm.list
sudo apt-get -qq update --allow-insecure-repositories >/dev/null
sudo apt-get -qq install -y --allow-unauthenticated \
"rocm-dev${ROCM_VERSION}" "rocm-utils${ROCM_VERSION}" \
"rocm-libs${ROCM_VERSION}" >/dev/null
# Fake presence of MI200-class accelerators
echo "gfx90a" | sudo tee /opt/rocm/bin/target.lst
# Cleanup
sudo apt-get -qq autoclean >/dev/null
sudo apt-get -qq clean >/dev/null
sudo rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
- name: Symblink system dependencies
if: inputs.rocm == 'ON'
shell: bash
run: |
# symblink system libraries for HIP compiler
sudo ln -s /lib/x86_64-linux-gnu/libc.so.6 /lib64/libc.so.6
sudo ln -s /lib/x86_64-linux-gnu/libc_nonshared.a /usr/lib64/libc_nonshared.a
sudo ln -s /usr/lib/x86_64-linux-gnu/libpthread.so.0 /lib64/libpthread.so.0
sudo ln -s $HOME/miniconda3/x86_64-conda-linux-gnu/sysroot/usr/lib64/libpthread_nonshared.a /usr/lib64/libpthread_nonshared.a
- name: Build all targets
shell: bash
run: |
eval "$(conda shell.bash hook)"
conda activate
cmake -B build \
-DBUILD_TESTING=ON \
-DBUILD_SHARED_LIBS=ON \
-DFAISS_ENABLE_GPU=${{ inputs.gpu }} \
-DFAISS_ENABLE_CUVS=${{ inputs.cuvs }} \
-DFAISS_ENABLE_ROCM=${{ inputs.rocm }} \
-DFAISS_OPT_LEVEL=${{ inputs.opt_level }} \
-DFAISS_ENABLE_C_API=ON \
-DPYTHON_EXECUTABLE=$CONDA/bin/python \
-DCMAKE_BUILD_TYPE=Release \
-DBLA_VENDOR=${{ runner.arch == 'X64' && 'Intel10_64_dyn' || '' }} \
-DCMAKE_CUDA_FLAGS=${{ runner.arch == 'X64' && '"-gencode arch=compute_75,code=sm_75"' || '' }} \
.
make -k -C build -j$(nproc)
- name: C++ tests
shell: bash
run: |
export GTEST_OUTPUT="xml:$(realpath .)/test-results/googletest/"
make -C build test
- name: C++ perf benchmarks
shell: bash
if: inputs.rocm == 'OFF'
run: |
find ./build/perf_tests/ -executable -type f -name "bench*" -exec '{}' -v \;
- name: Install Python extension
shell: bash
working-directory: build/faiss/python
run: |
$CONDA/bin/python setup.py install
- name: ROCm - install ROCm-enabled torch via pip
if: inputs.rocm == 'ON'
shell: bash
run: |
pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.1
- name: Python tests (CPU only)
if: inputs.gpu == 'OFF'
shell: bash
run: |
pytest --junitxml=test-results/pytest/results.xml tests/test_*.py
pytest --junitxml=test-results/pytest/results-torch.xml tests/torch_*.py
- name: Python tests (CPU + GPU)
if: inputs.gpu == 'ON'
shell: bash
run: |
pytest --junitxml=test-results/pytest/results.xml tests/test_*.py
pytest --junitxml=test-results/pytest/results-torch.xml tests/torch_*.py
cp tests/common_faiss_tests.py faiss/gpu/test
pytest --junitxml=test-results/pytest/results-gpu.xml faiss/gpu/test/test_*.py
pytest --junitxml=test-results/pytest/results-gpu-torch.xml faiss/gpu/test/torch_*.py
- name: Test avx2 loading
if: inputs.opt_level == 'avx2'
shell: bash
run: |
FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs $CONDA/bin/python -c "import faiss" 2>&1 | grep faiss.so
LD_DEBUG=libs $CONDA/bin/python -c "import faiss" 2>&1 | grep faiss_avx2.so
- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
with:
name: test-results-arch=${{ runner.arch }}-opt=${{ inputs.opt_level }}-gpu=${{ inputs.gpu }}-cuvs=${{ inputs.cuvs }}-rocm=${{ inputs.rocm }}
path: test-results