Fixing unit tests

Author: Lucas-C

.github/workflows/continuous-integration-workflow.yml

@@ -24,7 +24,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install system dependencies ⚙️
         if: matrix.platform == 'ubuntu-latest'
-        run: sudo apt-get install libjpeg-dev
+        run: sudo apt-get install ghostscript libjpeg-dev
       - name: Install qpdf ⚙️
         if: matrix.platform == 'ubuntu-latest' && matrix.python-version != '3.9'  # This allows us to run the unit tests WITHOUT qpdf in just one parallel execution
         run: sudo apt-get install qpdf
@@ -35,9 +35,9 @@ jobs:
       - name: Statically checking code 🔎
         if: matrix.python-version == '3.10' && matrix.platform == 'ubuntu-latest'
         run: |
-            pylint fpdf test tutorial/tuto*.py
-            bandit -c .banditrc.yml -r contributors/ fpdf/ tutorial/
-            semgrep scan --config auto --lang=py --error --strict --exclude-rule=python.lang.security.insecure-hash-function.insecure-hash-function fpdf
+          pylint fpdf test tutorial/tuto*.py
+          bandit -c .banditrc.yml -r contributors/ fpdf/ tutorial/
+          semgrep scan --config auto --lang=py --error --strict --exclude-rule=python.lang.security.insecure-hash-function.insecure-hash-function fpdf
       - name: Ensure code has been autoformatted with black 🖌️
         if: matrix.python-version == '3.10' && matrix.platform == 'ubuntu-latest'
         run: black --check .
@@ -59,7 +59,7 @@ jobs:
           # Ensuring there is no `generate=True` left remaining in calls to assert_pdf_equal:
           grep -IRF generate=True test/ && exit 1
           # Executing all tests:
-          pytest -vv --final-memory-usage
+          pytest -vv --trace-memory-usage
       - name: Uploading coverage report to codecov.io ☑
         if: matrix.python-version == '3.10' && matrix.platform == 'ubuntu-latest'
         run: bash <(curl -s https://codecov.io/bash)

fpdf/util.py

@@ -97,11 +97,11 @@ def get_mem_usage(prefix) -> str:
     # heap_size, stack_size = get_process_heap_and_stack_sizes()
     # objs_size_sum = get_gc_managed_objs_total_size()
     pillow = get_pillow_allocated_memory()
+    # malloc_stats = "Malloc stats: " + get_pymalloc_allocated_over_total_size()
+    malloc_stats = ""
     if is_tracing():
-        malloc_stats = get_tracemalloc_traced_memory()
-    else:
-        malloc_stats = get_pymalloc_allocated_over_total_size()
-    return f"{prefix:<40} Malloc stats: {malloc_stats} | Pillow: {pillow} | Process RSS: {rss}"
+        malloc_stats = "Malloc stats: " + get_tracemalloc_traced_memory()
+    return f"{prefix:<40} {malloc_stats} | Pillow: {pillow} | Process RSS: {rss}"
 
 
 def get_process_rss() -> str:

setup.cfg

@@ -16,3 +16,5 @@ ignore = E203, E221, E241, E251, E701, E731, W503
 
 [tool:pytest]
 addopts = --cov=fpdf --cov-report=xml
+log_cli = 1
+log_cli_level= WARN

test/conftest.py

@@ -261,13 +261,20 @@ def handler(_, __):
 
 
 def ensure_rss_memory_below(max_in_mib):
+    """
+    Enure there is no unexpected / significant increase between
+    the process RSS memory BEFORE executing the test, and AFTER.
+    """
+
     def actual_decorator(test_func):
         @functools.wraps(test_func)
         def wrapper(*args, **kwargs):
+            start_rss_in_mib = get_process_rss_as_mib()
             test_func(*args, **kwargs)
-            rss_in_mib = get_process_rss_as_mib()
-            if rss_in_mib:
-                assert rss_in_mib < max_in_mib
+            if not start_rss_in_mib:
+                return  # not available under Windows
+            end_rss_in_mib = get_process_rss_as_mib()
+            assert (end_rss_in_mib - start_rss_in_mib) < max_in_mib
 
         return wrapper
 
@@ -278,9 +285,9 @@ def wrapper(*args, **kwargs):
 # so we require an opt-in through a CLI argument:
 def pytest_addoption(parser):
     parser.addoption(
-        "--final-memory-usage",
+        "--trace-memory-usage",
         action="store_true",
-        help="At the end of the tests execution, display the current memory usage",
+        help="Trace the memory usage during tests execution",
     )
     parser.addoption(
         "--pympler-summary",
@@ -294,15 +301,26 @@ def pytest_addoption(parser):
     )
 
 
+@pytest.fixture(scope="module", autouse=True)
+def module_memory_usage(request):
+    if request.config.getoption("trace_memory_usage"):
+        gc.collect()
+        capmanager = request.config.pluginmanager.getplugin("capturemanager")
+        with capmanager.global_and_fixture_disabled():
+            print("\n")
+            print_mem_usage("Memory usage:")
+    yield
+
+
 @pytest.fixture(scope="session", autouse=True)
 def final_memory_usage(request):
     yield
-    if request.config.getoption("final_memory_usage"):
+    if request.config.getoption("trace_memory_usage"):
         gc.collect()
         capmanager = request.config.pluginmanager.getplugin("capturemanager")
         with capmanager.global_and_fixture_disabled():
             print("\n")
-            print_mem_usage("Final memory usage:")
+            print_mem_usage("Memory usage:")
 
 
 @pytest.fixture(scope="session", autouse=True)

test/embed_file_all_optionals.pdf

@@ -42,7 +42,7 @@ def test_load_invalid_base64_data():
 
 
 # ensure memory usage does not get too high - this value depends on Python version:
-@ensure_rss_memory_below(max_in_mib=190)
+@ensure_rss_memory_below(max_in_mib=6)
 def test_share_images_cache(tmp_path):
     images_cache = {}
     icc_profiles_cache = {}

test/embed_file_self.pdf

@@ -7,7 +7,7 @@
 
 HERE = Path(__file__).resolve().parent
 IMAGE_PATH = HERE / "png_images/6c853ed9dacd5716bc54eb59cec30889.png"
-MAX_MEMORY_MB = 190  # memory usage depends on Python version
+MAX_MEMORY_MB = 12  # memory usage depends on Python version
 
 
 def test_oversized_images_warn(caplog):

test/file_attachment_annotation.pdf

@@ -1,7 +1,10 @@
 bandit
 black
-camelot-py[base]
+# camelot-py[base] : does not work due to https://github.com/vinayak-mehta/pdftopng/issues/11
+camelot-py
 endesive
+ghostscript  # required by camelot-py[base]: https://github.com/camelot-dev/camelot/blob/master/setup.py#L27
+opencv-python  # required by camelot-py[base]: https://github.com/camelot-dev/camelot/blob/master/setup.py#L27
 pylint
 pytest
 pytest-cov

test/html/html_customize_ul.pdf

@@ -14,10 +14,8 @@
 from test.table.test_table import TABLE_DATA
 
 HERE = Path(__file__).resolve().parent
-TABLE_DATA_AS_DF = DataFrame(TABLE_DATA)
-TABLE_DATA_AS_DF = DataFrame(
-    TABLE_DATA_AS_DF.values[1:], columns=TABLE_DATA_AS_DF.iloc[0]
-).astype({"Age": int})
+_TMP_DF = DataFrame(TABLE_DATA)
+TABLE_DATA_AS_DF = DataFrame(_TMP_DF.values[1:], columns=_TMP_DF.iloc[0])
 
 
 ###############################################################################
@@ -97,6 +95,7 @@ def test_tabula_extract_simple_table(filename):
     dataframes = tabula.read_pdf(HERE / filename, pages="all")
     assert len(dataframes) == 1
     for df in dataframes:
+        df = df.astype({"Age": str})
         assert_frame_equal(df, TABLE_DATA_AS_DF, check_names=False)
 
 
@@ -111,6 +110,7 @@ def test_tabula_extract_two_tables(filename):
     dataframes = tabula.read_pdf(HERE / filename, pages="all")
     assert len(dataframes) == 2
     for df in dataframes:
+        df = df.astype({"Age": str})
         assert_frame_equal(df, TABLE_DATA_AS_DF, check_names=False)
 
 

test/html/html_features.pdf

@@ -8,7 +8,7 @@
 HERE = Path(__file__).resolve().parent
 
 
-@ensure_rss_memory_below(max_in_mib=175)
+@ensure_rss_memory_below(max_in_mib=6)
 @pytest.mark.timeout(40)
 def test_intense_image_rendering():
     png_file_paths = []