group SWIG tests into one file (facebookresearch#3807)

Summary:
Pull Request resolved: facebookresearch#3807

This diff re-organizes the tests a bit:

* groups tests related to SWIG into a single file

* enable doxygen test conditionally

* removes a dep on platform.Version that does not exist on some envs

* move a few tests out of build_blocks to avoid it being a catch-all file for uncategorized tests

Differential Revision: D61959750

Author: mdouze

faiss/python/loader.py

@@ -3,13 +3,15 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-from packaging.version import Version
 import platform
 import subprocess
 import logging
 import os
 
 
+def Version(v):
+    return [int(x) for x in v.split('.')]
+
 def supported_instruction_sets():
     """
     Returns the set of supported CPU features, see

faiss/python/swigfaiss.swig

@@ -1245,10 +1245,13 @@ void * cast_integer_to_void_ptr (int64_t x) {
 %}
 
 %inline %{
-    void wait() {
-        // in gdb, use return to get out of this function
-        for(int i = 0; i == 0; i += 0);
-    }
- %}
+
+// the SWIG version is a 6-digit hex string, eg. version 3.2.1 is encoded as
+// 0x030201
+uint64_t swig_version() {
+    return SWIG_VERSION;
+}
+
+%}
 
 // End of file...

tests/test_build_blocks.py

@@ -66,43 +66,6 @@ def test_pca_epsilon(self):
         self.assertTrue(np.all(np.isfinite(y)))
 
 
-class TestRevSwigPtr(unittest.TestCase):
-
-    def test_rev_swig_ptr(self):
-
-        index = faiss.IndexFlatL2(4)
-        xb0 = np.vstack([
-            i * 10 + np.array([1, 2, 3, 4], dtype='float32')
-            for i in range(5)])
-        index.add(xb0)
-        xb = faiss.rev_swig_ptr(index.get_xb(), 4 * 5).reshape(5, 4)
-        self.assertEqual(np.abs(xb0 - xb).sum(), 0)
-
-
-class TestException(unittest.TestCase):
-
-    def test_exception(self):
-
-        index = faiss.IndexFlatL2(10)
-
-        a = np.zeros((5, 10), dtype='float32')
-        b = np.zeros(5, dtype='int64')
-
-        # an unsupported operation for IndexFlat
-        self.assertRaises(
-            RuntimeError,
-            index.add_with_ids, a, b
-        )
-        # assert 'add_with_ids not implemented' in str(e)
-
-    def test_exception_2(self):
-        self.assertRaises(
-            RuntimeError,
-            faiss.index_factory, 12, 'IVF256,Flat,PQ8'
-        )
-        #    assert 'could not parse' in str(e)
-
-
 class TestMapLong2Long(unittest.TestCase):
 
     def test_maplong2long(self):
@@ -380,7 +343,6 @@ def test_rand_vector(self):
         self.assertLess(ninter, 460)
 
 
-
 class TestPairwiseDis(unittest.TestCase):
 
     def test_L2(self):
@@ -418,142 +380,6 @@ def test_IP(self):
                 dis[i], np.dot(x[ix[i]], y[iy[i]]))
 
 
-class TestSWIGWrap(unittest.TestCase):
-    """ various regressions with the SWIG wrapper """
-
-    def test_size_t_ptr(self):
-        # issue 1064
-        index = faiss.IndexHNSWFlat(10, 32)
-
-        hnsw = index.hnsw
-        index.add(np.random.rand(100, 10).astype('float32'))
-        be = np.empty(2, 'uint64')
-        hnsw.neighbor_range(23, 0, faiss.swig_ptr(be), faiss.swig_ptr(be[1:]))
-
-    def test_id_map_at(self):
-        # issue 1020
-        n_features = 100
-        feature_dims = 10
-
-        features = np.random.random((n_features, feature_dims)).astype(np.float32)
-        idx = np.arange(n_features).astype(np.int64)
-
-        index = faiss.IndexFlatL2(feature_dims)
-        index = faiss.IndexIDMap2(index)
-        index.add_with_ids(features, idx)
-
-        [index.id_map.at(int(i)) for i in range(index.ntotal)]
-
-    def test_downcast_Refine(self):
-
-        index = faiss.IndexRefineFlat(
-            faiss.IndexScalarQuantizer(10, faiss.ScalarQuantizer.QT_8bit)
-        )
-
-        # serialize and deserialize
-        index2 = faiss.deserialize_index(
-            faiss.serialize_index(index)
-        )
-
-        assert isinstance(index2, faiss.IndexRefineFlat)
-
-    def do_test_array_type(self, dtype):
-        """ tests swig_ptr and rev_swig_ptr for this type of array """
-        a = np.arange(12).astype(dtype)
-        ptr = faiss.swig_ptr(a)
-        a2 = faiss.rev_swig_ptr(ptr, 12)
-        np.testing.assert_array_equal(a, a2)
-
-    def test_all_array_types(self):
-        self.do_test_array_type('float32')
-        self.do_test_array_type('float64')
-        self.do_test_array_type('int8')
-        self.do_test_array_type('uint8')
-        self.do_test_array_type('int16')
-        self.do_test_array_type('uint16')
-        self.do_test_array_type('int32')
-        self.do_test_array_type('uint32')
-        self.do_test_array_type('int64')
-        self.do_test_array_type('uint64')
-
-    def test_int64(self):
-        # see https://github.com/facebookresearch/faiss/issues/1529
-        v = faiss.Int64Vector()
-
-        for i in range(10):
-            v.push_back(i)
-        a = faiss.vector_to_array(v)
-        assert a.dtype == 'int64'
-        np.testing.assert_array_equal(a, np.arange(10, dtype='int64'))
-
-        # check if it works in an IDMap
-        idx = faiss.IndexIDMap(faiss.IndexFlatL2(32))
-        idx.add_with_ids(
-            np.random.rand(10, 32).astype('float32'),
-            np.random.randint(1000, size=10, dtype='int64')
-        )
-        faiss.vector_to_array(idx.id_map)
-
-
-class TestNNDescentKNNG(unittest.TestCase):
-
-    def test_knng_L2(self):
-        self.subtest(32, 10, faiss.METRIC_L2)
-
-    def test_knng_IP(self):
-        self.subtest(32, 10, faiss.METRIC_INNER_PRODUCT)
-
-    def subtest(self, d, K, metric):
-        metric_names = {faiss.METRIC_L1: 'L1',
-                        faiss.METRIC_L2: 'L2',
-                        faiss.METRIC_INNER_PRODUCT: 'IP'}
-
-        nb = 1000
-        _, xb, _ = get_dataset_2(d, 0, nb, 0)
-
-        _, knn = faiss.knn(xb, xb, K + 1, metric)
-        knn = knn[:, 1:]
-
-        index = faiss.IndexNNDescentFlat(d, K, metric)
-        index.nndescent.S = 10
-        index.nndescent.R = 32
-        index.nndescent.L = K + 20
-        index.nndescent.iter = 5
-        index.verbose = True
-
-        index.add(xb)
-        graph = index.nndescent.final_graph
-        graph = faiss.vector_to_array(graph)
-        graph = graph.reshape(nb, K)
-
-        recalls = 0
-        for i in range(nb):
-            for j in range(K):
-                for k in range(K):
-                    if graph[i, j] == knn[i, k]:
-                        recalls += 1
-                        break
-        recall = 1.0 * recalls / (nb * K)
-        assert recall > 0.99
-
-    def test_small_nndescent(self):
-        """ building a too small graph used to crash, make sure it raises
-        an exception instead.
-        TODO: build the exact knn graph for small cases
-        """
-        d = 32
-        K = 10
-        index = faiss.IndexNNDescentFlat(d, K, faiss.METRIC_L2)
-        index.nndescent.S = 10
-        index.nndescent.R = 32
-        index.nndescent.L = K + 20
-        index.nndescent.iter = 5
-        index.verbose = True
-
-        xb = np.zeros((78, d), dtype='float32')
-        self.assertRaises(RuntimeError, index.add, xb)
-
-
 class TestResultHeap(unittest.TestCase):
 
     def test_keep_min(self):
@@ -663,6 +489,7 @@ def test_bucket_sort_inplace_int64(self):
     def test_bucket_sort_inplace_parallel_int64(self):
         self.do_test_bucket_sort_inplace(4, dtype='int64')
 
+
 class TestMergeKNNResults(unittest.TestCase):
 
     def do_test(self, ismax, dtype):

tests/test_documentation.py

@@ -4,7 +4,6 @@
 # LICENSE file in the root directory of this source tree.
 
 import unittest
-
 import faiss
 
 

tests/test_doxygen_documentation.py

@@ -3,7 +3,7 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-""" a few tests for graph-based indices (HNSW and NSG)"""
+""" a few tests for graph-based indices (HNSW, nndescent and NSG)"""
 
 import numpy as np
 import unittest
@@ -506,3 +506,62 @@ def test_order(self):
         gt = np.arange(0, k)[np.newaxis, :]  # [1, k]
         gt = np.repeat(gt, nq, axis=0)  # [nq, k]
         np.testing.assert_array_equal(indices, gt)
+
+
+class TestNNDescentKNNG(unittest.TestCase):
+
+    def test_knng_L2(self):
+        self.subtest(32, 10, faiss.METRIC_L2)
+
+    def test_knng_IP(self):
+        self.subtest(32, 10, faiss.METRIC_INNER_PRODUCT)
+
+    def subtest(self, d, K, metric):
+        metric_names = {faiss.METRIC_L1: 'L1',
+                        faiss.METRIC_L2: 'L2',
+                        faiss.METRIC_INNER_PRODUCT: 'IP'}
+
+        nb = 1000
+        _, xb, _ = get_dataset_2(d, 0, nb, 0)
+
+        _, knn = faiss.knn(xb, xb, K + 1, metric)
+        knn = knn[:, 1:]
+
+        index = faiss.IndexNNDescentFlat(d, K, metric)
+        index.nndescent.S = 10
+        index.nndescent.R = 32
+        index.nndescent.L = K + 20
+        index.nndescent.iter = 5
+        index.verbose = True
+
+        index.add(xb)
+        graph = index.nndescent.final_graph
+        graph = faiss.vector_to_array(graph)
+        graph = graph.reshape(nb, K)
+
+        recalls = 0
+        for i in range(nb):
+            for j in range(K):
+                for k in range(K):
+                    if graph[i, j] == knn[i, k]:
+                        recalls += 1
+                        break
+        recall = 1.0 * recalls / (nb * K)
+        assert recall > 0.99
+
+    def test_small_nndescent(self):
+        """ building a too small graph used to crash, make sure it raises
+        an exception instead.
+        TODO: build the exact knn graph for small cases
+        """
+        d = 32
+        K = 10
+        index = faiss.IndexNNDescentFlat(d, K, faiss.METRIC_L2)
+        index.nndescent.S = 10
+        index.nndescent.R = 32
+        index.nndescent.L = K + 20
+        index.nndescent.iter = 5
+        index.verbose = True
+
+        xb = np.zeros((78, d), dtype='float32')
+        self.assertRaises(RuntimeError, index.add, xb)