From a0ce75f36c081c30ae1cbfbcf1926f2698c94420 Mon Sep 17 00:00:00 2001 From: Alberto Ibarrondo Date: Thu, 21 Sep 2023 00:08:15 +0200 Subject: [PATCH] v3.4.2: Added flip operator (^), fixed MacOs build using brew GCC --- .github/workflows/pytest-coverage.yml | 52 +++++++++++++++++++-------- Pyfhel/backend/palisade | 1 - Pyfhel/test/test_PyCtxt.py | 4 +-- README.md | 30 ++++++++++------ examples/Demo_2_Integer_BFV.py | 6 ++-- pyproject.toml | 14 ++++---- setup.py | 6 ++-- 7 files changed, 71 insertions(+), 42 deletions(-) delete mode 160000 Pyfhel/backend/palisade diff --git a/.github/workflows/pytest-coverage.yml b/.github/workflows/pytest-coverage.yml index fc6ef4f..6701357 100644 --- a/.github/workflows/pytest-coverage.yml +++ b/.github/workflows/pytest-coverage.yml @@ -1,4 +1,6 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For MacOs builds, GNU GCC is installed via Homebrew, and the CC/CXX environment variables are set to use it. +# The echo "CC=..." >> $GITHUB_ENV syntax is used to set environment variables for the next steps in the workflow. # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: CI @@ -25,40 +27,60 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} steps: + - name: Checkout repo uses: actions/checkout@v3 with: submodules: recursive - # brew install --with-toolchain llvm libomp ? - - name: (MAcOs only) Install GNU GCC + + - name: (MacOs only) Install GNU GCC. if: matrix.os == 'macos-latest' run: | brew install gcc libomp - - name: (MacOs only) Set gcc/gxx environment variables + + - name: (MacOs only) Set CC/CXX environment variables to GNU GCC if: matrix.os == 'macos-latest' run: | - echo "gcc=/usr/local/bin/$(ls /usr/local/bin | grep ^gcc-[0-9] | sort -V -r | head -n 1)" >> $GITHUB_ENV - echo "gxx=/usr/local/bin/$(ls /usr/local/bin | grep ^g++-[0-9] | sort -V -r | head -n 1)" >> $GITHUB_ENV + + # Check GCC installation path. Brew installs GCC in /opt/homebrew/bin on Apple Silicon and /usr/local/bin on Intel. + if [[ $(uname -m) = "arm64" ]]; then BREW_GCC_PATH="/opt/homebrew/bin"; else BREW_GCC_PATH="/usr/local/bin"; fi + echo "GCC installed at $BREW_GCC_PATH" + + # Set CC/CXX environment variables to GNU GCC + echo "CC=$BREW_GCC_PATH/$(ls $BREW_GCC_PATH | grep ^gcc-[0-9] | sort -V -r | head -n 1)" >> $GITHUB_ENV + echo "CXX=$BREW_GCC_PATH/$(ls $BREW_GCC_PATH | grep ^g++-[0-9] | sort -V -r | head -n 1)" >> $GITHUB_ENV + echo "CC=${{ env.CC }}" + echo "CXX=${{ env.CXX }}" + + # Set MACOSX_DEPLOYMENT_TARGET to avoid version mismatch warnings + echo "MACOSX_DEPLOYMENT_TARGET=$(sw_vers -productVersion)" >> $GITHUB_ENV + echo "MACOSX_DEPLOYMENT_TARGET=${{ env.MACOSX_DEPLOYMENT_TARGET }}" + - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Update pip - run: | - python -m pip install --upgrade pip - # Run interactive debugging session for debug_enabled manually - - name: Setup tmate session + cache: 'pip' # caching pip dependencies + + - name: (Manual) Setup tmate session for interactive debugging via SSH uses: mxschmitt/action-tmate@v3 if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - - name: Install package in test mode + + - name: Update pip + run: python -m pip install --upgrade pip + + - name: Install package verbosely run: | - touch .cov + + touch .cov # Create a .cov file to trigger Cython compilation with coverage support ls - python -m pip install .[test] + python -m pip install -v -v -v . + - name: Test with pytest run: | - python -m pip install cython==3.0.0a11 pytest-cov + python -m pip install cython==3.0.2 pytest-cov pytest --cov . + - name: Upload report to Codecov uses: codecov/codecov-action@v3.1.0 with: diff --git a/Pyfhel/backend/palisade b/Pyfhel/backend/palisade deleted file mode 160000 index 19e84ae..0000000 --- a/Pyfhel/backend/palisade +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 19e84ae53ecdd1ca3360c4cdbb6df2eb31f21195 diff --git a/Pyfhel/test/test_PyCtxt.py b/Pyfhel/test/test_PyCtxt.py index 3aaf83c..e06f6b0 100644 --- a/Pyfhel/test/test_PyCtxt.py +++ b/Pyfhel/test/test_PyCtxt.py @@ -198,8 +198,8 @@ def test_PyCtxt_rotate(self, HE): assert np.round(HE.decrypt(c1)[0])==1 # flip if c1._pyfhel.scheme == Scheme_t.bfv: - c1 |= 1 - c1 |= 1 + c1 ^= 1 + c1 ^= 1 assert np.round(HE.decrypt(c1)[0])==1 def test_PyCtxt_io(self, HE): diff --git a/README.md b/README.md index a677229..9bcc69c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -[![iCodecov](https://codecov.io/gh/ibarrond/Pyfhel/branch/dev/graph/badge.svg?token=S8J8Jlp1Fc)](https://codecov.io/gh/ibarrond/Pyfhel) +[![iCodecov](https://codecov.io/gh/ibarrond/Pyfhel/branch/master/graph/badge.svg?token=S8J8Jlp1Fc)](https://codecov.io/gh/ibarrond/Pyfhel) [![Documentation Status](https://readthedocs.org/projects/pyfhel/badge/?version=latest)](https://pyfhel.readthedocs.io/en/latest/?badge=latest) [![PyPI version](https://badge.fury.io/py/Pyfhel.svg)](https://badge.fury.io/py/Pyfhel) [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-brightgreen.svg)](https://github.com/ibarrond/Pyfhel/graphs/commit-activity) @@ -8,13 +8,13 @@ [![License: GPL v3](https://img.shields.io/badge/License-GPL%20v3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) -Python library for ADDITION, SUBSTRACTION, MULTIPLICATION and SCALAR PRODUCT over encrypted integers (BFV) and approximated floating point values (CKKS). This library acts as an optimized Python API for C++ Homomorphic Encryption libraries. +Python library for Addition, Subtraction, Multiplication and Scalar Product over *encrypted* integers (BFV/BGV schemes) and approximated floating point values (CKKS scheme). This library acts as an optimized Python API for C++ Homomorphic Encryption libraries. | | | |--------------------------------------------|--------------------------------------------------------------------------------------------| | :flags: **Language** | Python (3.7+), with Cython and C++ (:warning: _requires a [C++17 compiler][3]_ :warning:.) | | :computer: **OS** | Linux, Windows & MacOS. | -| :1234: **Version** | 3.4.1 (stable) | +| :1234: **Version** | 3.4.2 (stable) | | :books: **Docs** | In [readthedocs][1]! | | :pencil2: **Demos/Examples** | [In the docs][4] with the outputs, sources in the [`examples`][2] folder. | | :electric_plug: **Backends** | [SEAL][5], [OpenFHE (WIP)][6]. Shipped alongside Pyfhel. | @@ -56,28 +56,36 @@ If you wish to cite Pyfhel in your derived work, please use the following BibTeX ## Install & Uninstall To install `Pyfhel` from [PyPI](https://pypi.org/project/Pyfhel/), run (*WARNING! it takes several minutes to compile and install, be patient!*): ```bash -> pip install Pyfhel +pip install Pyfhel ``` To install the latest version, you can clone this repository with [all the submodules](https://stackoverflow.com/questions/3796927/how-to-git-clone-including-submodules) and install it by running: ```bash -> git clone --recursive https://github.com/ibarrond/Pyfhel.git -> pip install . +git clone --recursive https://github.com/ibarrond/Pyfhel.git +pip install . ``` To uninstall, just run: ```bash -> pip uninstall Pyfhel +pip uninstall Pyfhel ``` ### Installing a C/C++ Compiler `Pyfhel` requires a C/C++ compiler with C++17 support. We have tested: - *gcc6* to *gcc12* in Linux/MacOS/Windows WSL. To install: - Ubuntu: `sudo apt install gcc g++` - - MacOS: `brew install gcc`. MacOS users must set the environment variables `gcc` and `gxx` to the installed compilers, e.g.: + - MacOS: `brew install gcc`. MacOS users must also set several environment variables by running: ```bash - > export gcc=/usr/local/bin/gcc-12 - > export gxx=/usr/local/bin/g++-12 + # Brew installs GCC in /opt/homebrew/bin on Apple Silicon and /usr/local/bin on Intel. + if [[ $(uname -m) = "arm64" ]]; then BREW_GCC_PATH="/opt/homebrew/bin"; else BREW_GCC_PATH="/usr/local/bin"; fi + + # Set CC/CXX environment variables to the most recent GNU GCC + export CC="$BREW_GCC_PATH/$(ls $BREW_GCC_PATH | grep ^gcc-[0-9] | sort -V -r | head -n 1)" + export CXX="$BREW_GCC_PATH/$(ls $BREW_GCC_PATH | grep ^g++-[0-9] | sort -V -r | head -n 1)" + + # Set MACOSX_DEPLOYMENT_TARGET to avoid version mismatch warnings + echo "MACOSX_DEPLOYMENT_TARGET=$(sw_vers -productVersion)" >> $GITHUB_ENV + echo "MACOSX_DEPLOYMENT_TARGET=${{ env.MACOSX_DEPLOYMENT_TARGET }}" ``` - *MSVC2017* and *MSVC2019* in Windows. To install: - Install Visual C++ Build tools (Download [here](https://learn.microsoft.com/en-US/cpp/windows/latest-supported-vc-redist?view=msvc-170), guide in [here](https://stackoverflow.com/questions/40504552)) @@ -98,7 +106,7 @@ This repository contains: This is the standard process to develop/contribute: 1. _Code a new feature/fix a bug_. Using [Cython](https://cython.readthedocs.io/en/latest/) for the `.pyx` and `.pxd` extensions, C++ for `Afhel` or Python for examples/tests/other. -2. _Build/Install Pyfhel locally_. Use either `pip install .` or `python3 setup.py build` (for verbose output and fine control. Run `python3 setup.py --help` for further options). +2. _Build/Install Pyfhel locally_. Use `pip install -v -v -v .` for a verbose installation. 3. _Test changes (requires installing `pytest`)_. Run the tests locally by executing `pytest .` in the root directory, and make sure all tests pass. diff --git a/examples/Demo_2_Integer_BFV.py b/examples/Demo_2_Integer_BFV.py index be079e8..614a457 100755 --- a/examples/Demo_2_Integer_BFV.py +++ b/examples/Demo_2_Integer_BFV.py @@ -98,9 +98,9 @@ # to each of the rows! cRotL = ctxt1 << 2 # Calls HE.rotate(ctxt1, k=-2, in_new_ctxt=True) # `ctxt1 <<= 2` for inplace operation -cFlip = ctxt1 | 1 # Calls HE.flip(ctxt1, k=1, in_new_ctxt=True) +cFlip = ctxt1 ^ 1 # Calls HE.flip(ctxt1, k=1, in_new_ctxt=True) # `ctxt1 |= 1` for inplace operation -cCuAdd = (+ctxt1) # Calls HE.cumul_add(ctxt1, in_new_ctxt=True) +cCuAdd = (+ctxt1) # Calls HE.cumul_add(ctxt1, in_new_ctxt=True) # There is no equivalent for in-place operator, use # the above call with `in_new_ctxt=False` if required. @@ -189,7 +189,7 @@ print(" ->\tctxt1 >> 2 = cRotR --(decr)--> ", rcRotR) print(" ->\tctxt1 << 2 = cRotL --(decr)--> ", rcRotL) print(" ->\tctxt1 | 1 = cFlip --(decr)--> ", rcFlip) -print(" ->\t(+cCuAdd) = cCuAdd --(decr)--> ", cCuAdd) +print(" ->\t(+tctxt1) = cCuAdd -(decr)--> ", rcCuAdd) print(" Ciphertext-plaintext ops: ") print(" ->\tctxt1 + ptxt2 = cpSum --(decr)--> ", rcpSum) print(" ->\tctxt1 - ptxt2 = cpSub --(decr)--> ", rcpSub) diff --git a/pyproject.toml b/pyproject.toml index 92a1b17..5f8b346 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ repository = "https://github.com/ibarrond/github" requires = [ "setuptools<=60.9", "wheel", - "cython==3.0.0b3", + "cython>=3.0.0", "numpy>=1.21", "cmake>=3.15", "toml>=0.10" @@ -138,30 +138,32 @@ platforms = ["Linux", "Windows", "Darwin"] ] extra_compile_args = [ {Windows = ["/O2","/std:c++17","/openmp"]}, - {Darwin = ["-std=c++17","-O3","-fopenmp"]}, + {Darwin = ["-std=c++17","-O3","-Xpreprocessor","-fopenmp"]}, {Linux = ["-std=c++17","-O3","-fopenmp"]} ] extra_link_args = [ {Windows = []}, - {Darwin = ["-Wl,-rpath,@loader_path/.", "-Wl,-no_compact_unwind", "-fopenmp"]}, + {Darwin = ["-Wl,-rpath,@loader_path/.", "-fopenmp","-Wl,-no_compact_unwind"]}, {Linux = ["-Wl,-rpath=$ORIGIN/.", "-fopenmp"]} ] libraries = ['SEAL'] #----------------------------- CYTHON EXTENSIONS ------------------------------- [extensions.config] # Common compilation config for all extensions -include_dirs = [] +include_dirs = [ + {Darwin = ["/usr/local/include"]}, +] define_macros = [ ["NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION"], ["__PYX_ENUM_CLASS_DECL", "enum"], # Support enums in cython ] extra_compile_args = [ {Windows = ["/O2","/openmp","/std:c++17"]}, - {Darwin = ["-std=c++17","-O3","-fopenmp"]}, + {Darwin = ["-std=c++17","-O3","-Xpreprocessor","-fopenmp","-Wno-unreachable-code","-Wno-unqualified-std-cast-call"]}, {Linux = ["-std=c++17","-O3","-fopenmp"]}, ] extra_link_args = [ {Windows = []}, - {Darwin = ["-Wl,-rpath,@loader_path/.", "-Wl,-no_compact_unwind", "-fopenmp"]}, + {Darwin = ["-Wl,-rpath,@loader_path/.", "-fopenmp","-Wl,-no_compact_unwind"]}, {Linux = ["-Wl,-rpath=$ORIGIN/.", "-fopenmp"]}, ] libraries = [] # libraries to link with, cpplibraries above are added by default diff --git a/setup.py b/setup.py index 719f9bc..02754c4 100755 --- a/setup.py +++ b/setup.py @@ -28,9 +28,6 @@ # Get platform system platform_system = platform.system() -if platform_system == 'Darwin': - vars = sysconfig.get_config_vars() - vars["LDSHARED"] = (vars["LDSHARED"]+" -Wl,-no_fixup_chains,-x -undefined dynamic_lookup ").replace('-bundle', '-dynamiclib') # Read config file config = toml.load("pyproject.toml") @@ -55,7 +52,7 @@ def re_sub_file(regex: str, replace: str, filename: str): sub_file.write(re.sub(regex, '{}'.format(replace), file_string)) # Writing version in __init__.py and README.md -V_README_REGEX = r'(?<=\* \*\*_Version_\*\*: )[0-9]+\.[0-9]+\.[0-9a-z]+' +V_README_REGEX = r'(?<=\*\*Version\*\* \| )[0-9]+\.[0-9]+\.[0-9a-z]+' V_INIT_REGEX = r'(?<=__version__ = \")[0-9]+\.[0-9]+\.[0-9a-z]+(?=\")' re_sub_file(regex=V_README_REGEX, replace=VERSION, filename='README.md') re_sub_file(regex=V_INIT_REGEX, replace=VERSION, filename='Pyfhel/__init__.py') @@ -404,6 +401,7 @@ def build_shared_lib(self, lib_name, build_info): if platform_system == 'Darwin': build_info['extra_link_args'].append(f"-Wl,-install_name,@loader_path/{lib_file}") + self.compiler.linker_so = ['-dynamiclib' if val=='-bundle' else val for val in self.compiler.linker_so] self.compiler.link_shared_object( objects, lib_file,