New optional reset_page_indices parameter for insert_toc_placeholder() - partially fix issue #1343 (#1366)

Author: Lucas-C

.gitignore

@@ -1,10 +1,10 @@
-# my files
 .env
 .DS_Store
 
 # codecov.io
 coverage.xml
 
+__pycache__
 htmlcov
 public
 tutorial/invoice.pdf

CHANGELOG.md

@@ -26,6 +26,7 @@ This can also be enabled programmatically with `warnings.simplefilter('default',
 * documentation on [verifying provenance of `fpdf2` releases](https://py-pdf.github.io/fpdf2/#verifying-provenance)
 * documentation on [`fpdf2` internals](https://py-pdf.github.io/fpdf2/Internals.html)
 * support for adding TrueType fonts that are missing the `.notdef` glyph - [issue #1161](https://github.com/py-pdf/fpdf2/issues/1161)
+* new optional `reset_page_indices` parameter for [`insert_toc_placeholder()`](https://py-pdf.github.io/fpdf2/fpdf/fpdf.html#fpdf.fpdf.FPDF.insert_toc_placeholder)
 ### Fixed
 * [`FPDF.write_html()`](https://py-pdf.github.io/fpdf2/fpdf/fpdf.html#fpdf.fpdf.FPDF.write_html): Fixed rendering of content following `<a>` tags; now correctly resets emphasis style post `</a>` tag: hyperlink styling contained within the tag authority. - [Issue #1311](https://github.com/py-pdf/fpdf2/issues/1311)
 

docs/Development.md

@@ -212,8 +212,9 @@ Ask maintainers through comments if some errors in the pipeline seem obscure to
 [typos](https://github.com/crate-ci/typos) is a handy CLI tool to detect & auto-fix [typos](https://en.wikipedia.org/wiki/Typographical_error) in source files.
 Installation is relatively straightforward ([read the docs](https://github.com/crate-ci/typos?tab=readme-ov-file#install)).
 
-This tool is invoked in our CI pipeline.
+This tool is invoked in the [pre-commit hooks](#pre-commit-hook) and in our CI pipeline.
 If it fails, you should either:
+
 * auto-fix the errors detected by invoking `typos --write-changes`
 * add an exclusion rule to `.typos.toml`
 

docs/PageLabels.md

@@ -78,6 +78,8 @@ pdf.set_page_label(
 
 If you need to get the current page label, for example, to display it in a header or footer, you can use the `get_page_label()` method.
 
+!!! warning "In case of a [table of contents](DocumentOutlineAndTableOfContents.md) spanning **more than one page**, the page number returned by `get_page_label()` will be **incorrect**.<br>_cf._ [GitHub issue #1343](https://github.com/py-pdf/fpdf2/issues/1343)"
+
 ---
 
 ## Example Usage

fpdf/fpdf.py

@@ -167,6 +167,7 @@ class ToCPlaceholder(NamedTuple):
     y: int
     page_orientation: str
     pages: int = 1
+    reset_page_indices: bool = True
 
 
 # Disabling this check due to the "format" parameter below:
@@ -870,17 +871,29 @@ def alias_nb_pages(self, alias="{nb}"):
         """
         self.str_alias_nb_pages = alias
 
+    @check_page
     def set_page_label(
         self,
         label_style: Union[str, PageLabelStyle] = None,
         label_prefix: str = None,
         label_start: int = None,
     ):
-        current_page_label = (
-            None if self.page == 1 else self.pages[self.page - 1].get_page_label()
-        )
+        current_page_label = None
+        if self.page in self.pages:
+            current_page_label = self.pages[self.page].get_page_label()
+        elif self.page > 1:
+            current_page_label = self.pages[self.page - 1].get_page_label()
         new_page_label = None
         if label_style or label_prefix or label_start:
+            if current_page_label:
+                if label_style is None:
+                    label_style = current_page_label.get_style()
+                if label_prefix is None:
+                    label_prefix = current_page_label.get_prefix()
+                if label_start is None and not (
+                    self.toc_placeholder and self.toc_placeholder.reset_page_indices
+                ):
+                    label_start = current_page_label.get_start()
             label_style = (
                 PageLabelStyle.coerce(label_style, case_sensitive=True)
                 if label_style
@@ -4883,6 +4896,44 @@ def _insert_table_of_contents(self):
             self.in_footer = True
             self.footer()
             self.in_footer = False
+            # We need to reorder the pages, because some new pages have been inserted in the ToC,
+            # but they have been inserted at the end of self.pages:
+            new_pages = [
+                self.pages.pop(len(self.pages)) for _ in range(self._toc_inserted_pages)
+            ]
+            new_pages = list(reversed(new_pages))
+            indices_remap = {}
+            for page_index in range(
+                tocp.start_page + 1, self.pages_count + len(new_pages) + 1
+            ):
+                if page_index in self.pages:
+                    new_pages.append(self.pages.pop(page_index))
+                page = self.pages[page_index] = new_pages.pop(0)
+                # Fix page indices:
+                indices_remap[page.index()] = page_index
+                page.set_index(page_index)
+                # Fix page labels:
+                if tocp.reset_page_indices is False:
+                    page.get_page_label().st = page_index
+            assert len(new_pages) == 0, f"#new_pages: {len(new_pages)}"
+            # Fix outline:
+            for section in self._outline:
+                new_index = indices_remap.get(section.page_number)
+                if new_index is not None:
+                    section.dest = section.dest.replace(page=new_index)
+                    section.page_number = new_index
+                    if section.struct_elem:
+                        # pylint: disable=protected-access
+                        section.struct_elem._page_number = new_index
+            # Fix resource catalog:
+            new_resources_per_page = defaultdict(set)
+            for (
+                page_number,
+                resource_type,
+            ), resource in self._resource_catalog.resources_per_page.items():
+                key = (indices_remap.get(page_number, page_number), resource_type)
+                new_resources_per_page[key] = resource
+            self._resource_catalog.resources_per_page = new_resources_per_page
         self.page, self.y = prev_page, prev_y
 
     def file_id(self):  # pylint: disable=no-self-use
@@ -5186,6 +5237,7 @@ def insert_toc_placeholder(
         render_toc_function: Callable,
         pages: int = 1,
         allow_extra_pages: bool = False,
+        reset_page_indices: bool = True,
     ):
         """
         Configure Table Of Contents rendering at the end of the document generation,
@@ -5202,6 +5254,7 @@ def insert_toc_placeholder(
                 extra pages in the ToC, which may cause discrepancies with pre-rendered
                 page numbers. For consistent numbering, using page labels to create a
                 separate numbering style for the ToC is recommended.
+            reset_page_indices (bool): Whether to reset the pages indixes after the ToC. Default to True.
         """
         if not callable(render_toc_function):
             raise TypeError(
@@ -5213,7 +5266,12 @@ def insert_toc_placeholder(
                 f" on page {self.toc_placeholder.start_page}"
             )
         self.toc_placeholder = ToCPlaceholder(
-            render_toc_function, self.page, self.y, self.cur_orientation, pages
+            render_toc_function,
+            self.page,
+            self.y,
+            self.cur_orientation,
+            pages,
+            reset_page_indices,
         )
         self._toc_allow_page_insertion = allow_extra_pages
         for _ in range(pages):
@@ -5267,6 +5325,8 @@ def start_section(self, name, level=0, strict=True):
         Args:
             name (str): section name
             level (int): section level in the document outline. 0 means top-level.
+            strict (bool): whether to raise an exception if levels increase incorrectly,
+                for example with a level-3 section following a level-1 section.
         """
         if level < 0:
             raise ValueError('"level" mut be equal or greater than zero')

fpdf/outline.py

@@ -8,7 +8,8 @@
 in non-backward-compatible ways.
 """
 
-from typing import List, NamedTuple, Optional, TYPE_CHECKING
+from dataclasses import dataclass
+from typing import List, Optional, TYPE_CHECKING
 
 from .enums import Align, XPos, YPos
 from .fonts import TextStyle
@@ -19,12 +20,23 @@
     from .fpdf import FPDF
 
 
-class OutlineSection(NamedTuple):
+@dataclass
+class OutlineSection:
+    # RAM usage optimization
+    __slots__ = ("name", "level", "page_number", "dest", "struct_elem")
     name: str
     level: int
     page_number: int
     dest: Destination
-    struct_elem: Optional[StructElem] = None
+    struct_elem: Optional[StructElem]
+
+    # With __slots__ used, we need an __init__ method in order to define default values:
+    def __init__(self, name, level, page_number, dest, struct_elem=None):
+        self.name = name
+        self.level = level
+        self.page_number = page_number
+        self.dest = dest
+        self.struct_elem = struct_elem
 
 
 class OutlineItemDictionary(PDFObject):
@@ -58,6 +70,9 @@ def __init__(
         self.dest = dest
         self.struct_elem = struct_elem
 
+    def __str__(self):
+        return f"OutlineItemDictionary(title={self.title}, dest={self.dest})"
+
 
 class OutlineDictionary(PDFObject):
     __slots__ = ("_id", "type", "first", "last", "count")  # RAM usage optimization
@@ -69,6 +84,9 @@ def __init__(self, **kwargs):
         self.last = None
         self.count = 0
 
+    def __str__(self):
+        return f"OutlineDictionary(count={self.count})"
+
 
 def build_outline_objs(sections):
     """

fpdf/output.py

@@ -236,7 +236,7 @@ def __init__(
 
 
 class PDFPageLabel:
-    __slots__ = ["_style", "_prefix", "st"]
+    __slots__ = ("_style", "_prefix", "st")  # RAM usage optimization
 
     def __init__(
         self, label_style: PageLabelStyle, label_prefix: str, label_start: int
@@ -254,6 +254,9 @@ def p(self) -> PDFString:
         return PDFString(self._prefix) if self._prefix else None
 
     def __repr__(self):
+        return f"PDFPageLabel({self._style}, {self._prefix}, {self.st})"
+
+    def __str__(self):
         ret = self._prefix if self._prefix else ""
         if self._style:
             if self._style == PageLabelStyle.NUMBER:
@@ -266,7 +269,7 @@ def __repr__(self):
                 ret += int_to_letters(self.st - 1)
             elif self._style == PageLabelStyle.LOWER_LETTER:
                 ret += int_to_letters(self.st - 1).lower()
-        return None if ret == "" else ret
+        return ret
 
     def serialize(self) -> dict:
         return build_obj_dict({key: getattr(self, key) for key in dir(self)})
@@ -319,6 +322,7 @@ def __init__(
         self.struct_parents = None
         self.resources = None  # must always be set before calling .serialize()
         self.parent = None  # must always be set before calling .serialize()
+        # Useful properties that will not be serialized in the final PDF document:
         self._index = index
         self._width_pt, self._height_pt = None, None
         self._page_label: PDFPageLabel = None
@@ -327,6 +331,9 @@ def __init__(
     def index(self):
         return self._index
 
+    def set_index(self, i):
+        self._index = i
+
     def dimensions(self):
         "Return a pair (width, height) in the unit specified to FPDF constructor"
         return self._width_pt, self._height_pt
@@ -527,7 +534,7 @@ def bufferize(self):
         self.pdf_objs.append(xref)
 
         # 2. Plumbing - Inject all PDF object references required:
-        pages_root_obj.kids = PDFArray(self._reorder_page_objects(page_objs))
+        pages_root_obj.kids = PDFArray(page_objs)
         self._finalize_catalog(
             catalog_obj,
             pages_root_obj=pages_root_obj,
@@ -627,10 +634,19 @@ def _add_pages_root(self):
         self._add_pdf_obj(pages_root_obj)
         return pages_root_obj
 
+    def _iter_pages_in_order(self):
+        for page_index in range(1, self.fpdf.pages_count + 1):
+            page_obj = self.fpdf.pages[page_index]
+            # Defensive check:
+            assert (
+                page_obj.index() == page_index
+            ), f"{page_obj.index()=} != {page_index=}"
+            yield page_obj
+
     def _add_pages(self, _slice=slice(0, None)):
         fpdf = self.fpdf
         page_objs = []
-        for page_obj in list(fpdf.pages.values())[_slice]:
+        for page_obj in list(self._iter_pages_in_order())[_slice]:
             if fpdf.pdf_version > "1.3":
                 page_obj.group = pdf_dict(
                     {"/Type": "/Group", "/S": "/Transparency", "/CS": "/DeviceRGB"},
@@ -650,16 +666,6 @@ def _add_pages(self, _slice=slice(0, None)):
 
         return page_objs
 
-    def _reorder_page_objects(self, page_objs: list):
-        "Reorder page objects to move any Table of Contents pages generated at the end of the document to follow the ToC placeholder."
-        if not self.fpdf._toc_inserted_pages:
-            return page_objs
-        reordered = page_objs.copy()
-        for _ in range(self.fpdf._toc_inserted_pages):
-            last_page = reordered.pop()
-            reordered.insert(self.fpdf.toc_placeholder.start_page, last_page)
-        return reordered
-
     def _add_annotations_as_objects(self):
         sig_annotation_obj = None
         for page_obj in self.fpdf.pages.values():
@@ -1209,14 +1215,10 @@ def _finalize_catalog(
             catalog_obj.names = pdf_dict(
                 {"/EmbeddedFiles": pdf_dict({"/Names": pdf_list(file_spec_names)})}
             )
-        ordered_pages = list(fpdf.pages.items())
-        for _ in range(self.fpdf._toc_inserted_pages):
-            last_page = ordered_pages.pop()
-            ordered_pages.insert(self.fpdf.toc_placeholder.start_page, last_page)
         page_labels = [
-            f"{seq} {pdf_dict(page[1].get_page_label().serialize())}"
-            for (seq, page) in enumerate(ordered_pages)
-            if page[1].get_page_label()
+            f"{i} {pdf_dict(page.get_page_label().serialize())}"
+            for i, page in enumerate(self._iter_pages_in_order())
+            if page.get_page_label()
         ]
         if page_labels and not fpdf.pages[1].get_page_label():
             # If page labels are used, an entry for sequence 0 is mandatory

fpdf/syntax.py

@@ -377,3 +377,14 @@ def serialize(self, _security_handler=None, _obj_id=None):
         top = round(self.top, 2) if isinstance(self.top, float) else self.top
         assert self.page_ref
         return f"[{self.page_ref} /XYZ {left} {top} {self.zoom}]"
+
+    def replace(self, page=None, top=None, left=None, zoom=None):
+        assert (
+            not self.page_ref
+        ), "DestinationXYZ should not be copied after serialization"
+        return DestinationXYZ(
+            page=self.page_number if page is None else page,
+            top=self.top if top is None else top,
+            left=self.left if left is None else left,
+            zoom=self.zoom if zoom is None else zoom,
+        )

fpdf/util.py

@@ -133,6 +133,8 @@ def convert_unit(
 def int2roman(n):
     "Convert an integer to Roman numeral"
     result = ""
+    if n is None:
+        return result
     for numeral, integer in ROMAN_NUMERAL_MAP:
         while n >= integer:
             result += numeral