diff --git a/Cargo.toml b/Cargo.toml
index 921e551751c..30b394c344e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,6 +18,7 @@ rust-version = "1.63"
 cfg-if = "1.0"
 libc = "0.2.62"
 memoffset = "0.9"
+once_cell = "1"
 
 # ffi bindings to the python interpreter, split into a separate crate so they can be used independently
 pyo3-ffi = { path = "pyo3-ffi", version = "=0.22.0-dev" }
diff --git a/newsfragments/4178.changed.md b/newsfragments/4178.changed.md
new file mode 100644
index 00000000000..a97c1ec8f6e
--- /dev/null
+++ b/newsfragments/4178.changed.md
@@ -0,0 +1 @@
+The global reference pool (to track pending reference count decrements) is now initialized lazily to avoid the overhead of taking a mutex upon function entry when the functionality is not actually used.
diff --git a/src/gil.rs b/src/gil.rs
index 3e25c7cfb0f..ec20fc64c34 100644
--- a/src/gil.rs
+++ b/src/gil.rs
@@ -5,6 +5,8 @@ use crate::impl_::not_send::{NotSend, NOT_SEND};
 #[cfg(pyo3_disable_reference_pool)]
 use crate::impl_::panic::PanicTrap;
 use crate::{ffi, Python};
+#[cfg(not(pyo3_disable_reference_pool))]
+use once_cell::sync::Lazy;
 use std::cell::Cell;
 #[cfg(all(feature = "gil-refs", debug_assertions))]
 use std::cell::RefCell;
@@ -227,7 +229,9 @@ impl GILGuard {
         let pool = mem::ManuallyDrop::new(GILPool::new());
 
         #[cfg(not(pyo3_disable_reference_pool))]
-        POOL.update_counts(Python::assume_gil_acquired());
+        if let Some(pool) = Lazy::get(&POOL) {
+            pool.update_counts(Python::assume_gil_acquired());
+        }
         GILGuard::Ensured {
             gstate,
             #[cfg(feature = "gil-refs")]
@@ -240,7 +244,9 @@ impl GILGuard {
         increment_gil_count();
         let guard = GILGuard::Assumed;
         #[cfg(not(pyo3_disable_reference_pool))]
-        POOL.update_counts(guard.python());
+        if let Some(pool) = Lazy::get(&POOL) {
+            pool.update_counts(guard.python());
+        }
         guard
     }
 
@@ -307,11 +313,14 @@ impl ReferencePool {
     }
 }
 
+#[cfg(not(pyo3_disable_reference_pool))]
+unsafe impl Send for ReferencePool {}
+
 #[cfg(not(pyo3_disable_reference_pool))]
 unsafe impl Sync for ReferencePool {}
 
 #[cfg(not(pyo3_disable_reference_pool))]
-static POOL: ReferencePool = ReferencePool::new();
+static POOL: Lazy<ReferencePool> = Lazy::new(ReferencePool::new);
 
 /// A guard which can be used to temporarily release the GIL and restore on `Drop`.
 pub(crate) struct SuspendGIL {
@@ -336,7 +345,9 @@ impl Drop for SuspendGIL {
 
             // Update counts of PyObjects / Py that were cloned or dropped while the GIL was released.
             #[cfg(not(pyo3_disable_reference_pool))]
-            POOL.update_counts(Python::assume_gil_acquired());
+            if let Some(pool) = Lazy::get(&POOL) {
+                pool.update_counts(Python::assume_gil_acquired());
+            }
         }
     }
 }
@@ -409,7 +420,9 @@ impl GILPool {
     pub unsafe fn new() -> GILPool {
         // Update counts of PyObjects / Py that have been cloned or dropped since last acquisition
         #[cfg(not(pyo3_disable_reference_pool))]
-        POOL.update_counts(Python::assume_gil_acquired());
+        if let Some(pool) = Lazy::get(&POOL) {
+            pool.update_counts(Python::assume_gil_acquired());
+        }
         GILPool {
             start: OWNED_OBJECTS
                 .try_with(|owned_objects| {