diff --git a/scene/3d/lightmap_gi.cpp b/scene/3d/lightmap_gi.cpp
index 37380fd2b467..8f21260d573b 100644
--- a/scene/3d/lightmap_gi.cpp
+++ b/scene/3d/lightmap_gi.cpp
@@ -1491,11 +1491,17 @@ void LightmapGI::_notification(int p_what) {
 								"%s (%s): The directional lightmap textures are stored in a format that isn't supported anymore. Please bake lightmaps again to make lightmaps display from this node again.",
 								get_light_data()->get_path(), get_name()));
 
+				if (last_owner && last_owner != get_owner()) {
+					light_data->clear_users();
+				}
+
 				_assign_lightmaps();
 			}
 		} break;
 
 		case NOTIFICATION_EXIT_TREE: {
+			last_owner = get_owner();
+
 			if (light_data.is_valid()) {
 				_clear_lightmaps();
 			}
diff --git a/scene/3d/lightmap_gi.h b/scene/3d/lightmap_gi.h
index a48e599d6815..0848239fee0a 100644
--- a/scene/3d/lightmap_gi.h
+++ b/scene/3d/lightmap_gi.h
@@ -206,6 +206,7 @@ class LightmapGI : public VisualInstance3D {
 	Ref<CameraAttributes> camera_attributes;
 
 	Ref<LightmapGIData> light_data;
+	Node *last_owner = nullptr;
 
 	struct LightsFound {
 		Transform3D xform;