Skip to content

Commit 3b1d9e2

Browse files
LuBaolujoergroedel
authored andcommitted
iommu/vt-d: Add cache tag assignment interface
Caching tag is a combination of tags used by the hardware to cache various translations. Whenever a mapping in a domain is changed, the IOMMU driver should invalidate the caches with the caching tags. The VT-d specification describes caching tags in section 6.2.1, Tagging of Cached Translations. Add interface to assign caching tags to an IOMMU domain when attached to a RID or PASID, and unassign caching tags when a domain is detached from a RID or PASID. All caching tags are listed in the per-domain tag list and are protected by a dedicated lock. In addition to the basic IOTLB and devTLB caching tag types, NESTING_IOTLB and NESTING_DEVTLB tag types are also introduced. These tags are used for caches that store translations for DMA accesses through a nested user domain. They are affected by changes to mappings in the parent domain. Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com> Reviewed-by: Kevin Tian <kevin.tian@intel.com> Link: https://lore.kernel.org/r/20240416080656.60968-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel <jroedel@suse.de>
1 parent 304b3bd commit 3b1d9e2

File tree

6 files changed

+295
-9
lines changed

6 files changed

+295
-9
lines changed

drivers/iommu/intel/Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# SPDX-License-Identifier: GPL-2.0
22
obj-$(CONFIG_DMAR_TABLE) += dmar.o
3-
obj-$(CONFIG_INTEL_IOMMU) += iommu.o pasid.o nested.o
3+
obj-$(CONFIG_INTEL_IOMMU) += iommu.o pasid.o nested.o cache.o
44
obj-$(CONFIG_DMAR_TABLE) += trace.o cap_audit.o
55
obj-$(CONFIG_DMAR_PERF) += perf.o
66
obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += debugfs.o

drivers/iommu/intel/cache.c

+214
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
// SPDX-License-Identifier: GPL-2.0
2+
/*
3+
* cache.c - Intel VT-d cache invalidation
4+
*
5+
* Copyright (C) 2024 Intel Corporation
6+
*
7+
* Author: Lu Baolu <baolu.lu@linux.intel.com>
8+
*/
9+
10+
#define pr_fmt(fmt) "DMAR: " fmt
11+
12+
#include <linux/dmar.h>
13+
#include <linux/iommu.h>
14+
#include <linux/memory.h>
15+
#include <linux/spinlock.h>
16+
17+
#include "iommu.h"
18+
#include "pasid.h"
19+
20+
/* Check if an existing cache tag can be reused for a new association. */
21+
static bool cache_tage_match(struct cache_tag *tag, u16 domain_id,
22+
struct intel_iommu *iommu, struct device *dev,
23+
ioasid_t pasid, enum cache_tag_type type)
24+
{
25+
if (tag->type != type)
26+
return false;
27+
28+
if (tag->domain_id != domain_id || tag->pasid != pasid)
29+
return false;
30+
31+
if (type == CACHE_TAG_IOTLB || type == CACHE_TAG_NESTING_IOTLB)
32+
return tag->iommu == iommu;
33+
34+
if (type == CACHE_TAG_DEVTLB || type == CACHE_TAG_NESTING_DEVTLB)
35+
return tag->dev == dev;
36+
37+
return false;
38+
}
39+
40+
/* Assign a cache tag with specified type to domain. */
41+
static int cache_tag_assign(struct dmar_domain *domain, u16 did,
42+
struct device *dev, ioasid_t pasid,
43+
enum cache_tag_type type)
44+
{
45+
struct device_domain_info *info = dev_iommu_priv_get(dev);
46+
struct intel_iommu *iommu = info->iommu;
47+
struct cache_tag *tag, *temp;
48+
unsigned long flags;
49+
50+
tag = kzalloc(sizeof(*tag), GFP_KERNEL);
51+
if (!tag)
52+
return -ENOMEM;
53+
54+
tag->type = type;
55+
tag->iommu = iommu;
56+
tag->domain_id = did;
57+
tag->pasid = pasid;
58+
tag->users = 1;
59+
60+
if (type == CACHE_TAG_DEVTLB || type == CACHE_TAG_NESTING_DEVTLB)
61+
tag->dev = dev;
62+
else
63+
tag->dev = iommu->iommu.dev;
64+
65+
spin_lock_irqsave(&domain->cache_lock, flags);
66+
list_for_each_entry(temp, &domain->cache_tags, node) {
67+
if (cache_tage_match(temp, did, iommu, dev, pasid, type)) {
68+
temp->users++;
69+
spin_unlock_irqrestore(&domain->cache_lock, flags);
70+
kfree(tag);
71+
return 0;
72+
}
73+
}
74+
list_add_tail(&tag->node, &domain->cache_tags);
75+
spin_unlock_irqrestore(&domain->cache_lock, flags);
76+
77+
return 0;
78+
}
79+
80+
/* Unassign a cache tag with specified type from domain. */
81+
static void cache_tag_unassign(struct dmar_domain *domain, u16 did,
82+
struct device *dev, ioasid_t pasid,
83+
enum cache_tag_type type)
84+
{
85+
struct device_domain_info *info = dev_iommu_priv_get(dev);
86+
struct intel_iommu *iommu = info->iommu;
87+
struct cache_tag *tag;
88+
unsigned long flags;
89+
90+
spin_lock_irqsave(&domain->cache_lock, flags);
91+
list_for_each_entry(tag, &domain->cache_tags, node) {
92+
if (cache_tage_match(tag, did, iommu, dev, pasid, type)) {
93+
if (--tag->users == 0) {
94+
list_del(&tag->node);
95+
kfree(tag);
96+
}
97+
break;
98+
}
99+
}
100+
spin_unlock_irqrestore(&domain->cache_lock, flags);
101+
}
102+
103+
static int __cache_tag_assign_domain(struct dmar_domain *domain, u16 did,
104+
struct device *dev, ioasid_t pasid)
105+
{
106+
struct device_domain_info *info = dev_iommu_priv_get(dev);
107+
int ret;
108+
109+
ret = cache_tag_assign(domain, did, dev, pasid, CACHE_TAG_IOTLB);
110+
if (ret || !info->ats_enabled)
111+
return ret;
112+
113+
ret = cache_tag_assign(domain, did, dev, pasid, CACHE_TAG_DEVTLB);
114+
if (ret)
115+
cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_IOTLB);
116+
117+
return ret;
118+
}
119+
120+
static void __cache_tag_unassign_domain(struct dmar_domain *domain, u16 did,
121+
struct device *dev, ioasid_t pasid)
122+
{
123+
struct device_domain_info *info = dev_iommu_priv_get(dev);
124+
125+
cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_IOTLB);
126+
127+
if (info->ats_enabled)
128+
cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_DEVTLB);
129+
}
130+
131+
static int __cache_tag_assign_parent_domain(struct dmar_domain *domain, u16 did,
132+
struct device *dev, ioasid_t pasid)
133+
{
134+
struct device_domain_info *info = dev_iommu_priv_get(dev);
135+
int ret;
136+
137+
ret = cache_tag_assign(domain, did, dev, pasid, CACHE_TAG_NESTING_IOTLB);
138+
if (ret || !info->ats_enabled)
139+
return ret;
140+
141+
ret = cache_tag_assign(domain, did, dev, pasid, CACHE_TAG_NESTING_DEVTLB);
142+
if (ret)
143+
cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_NESTING_IOTLB);
144+
145+
return ret;
146+
}
147+
148+
static void __cache_tag_unassign_parent_domain(struct dmar_domain *domain, u16 did,
149+
struct device *dev, ioasid_t pasid)
150+
{
151+
struct device_domain_info *info = dev_iommu_priv_get(dev);
152+
153+
cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_NESTING_IOTLB);
154+
155+
if (info->ats_enabled)
156+
cache_tag_unassign(domain, did, dev, pasid, CACHE_TAG_NESTING_DEVTLB);
157+
}
158+
159+
static u16 domain_get_id_for_dev(struct dmar_domain *domain, struct device *dev)
160+
{
161+
struct device_domain_info *info = dev_iommu_priv_get(dev);
162+
struct intel_iommu *iommu = info->iommu;
163+
164+
/*
165+
* The driver assigns different domain IDs for all domains except
166+
* the SVA type.
167+
*/
168+
if (domain->domain.type == IOMMU_DOMAIN_SVA)
169+
return FLPT_DEFAULT_DID;
170+
171+
return domain_id_iommu(domain, iommu);
172+
}
173+
174+
/*
175+
* Assign cache tags to a domain when it's associated with a device's
176+
* PASID using a specific domain ID.
177+
*
178+
* On success (return value of 0), cache tags are created and added to the
179+
* domain's cache tag list. On failure (negative return value), an error
180+
* code is returned indicating the reason for the failure.
181+
*/
182+
int cache_tag_assign_domain(struct dmar_domain *domain,
183+
struct device *dev, ioasid_t pasid)
184+
{
185+
u16 did = domain_get_id_for_dev(domain, dev);
186+
int ret;
187+
188+
ret = __cache_tag_assign_domain(domain, did, dev, pasid);
189+
if (ret || domain->domain.type != IOMMU_DOMAIN_NESTED)
190+
return ret;
191+
192+
ret = __cache_tag_assign_parent_domain(domain->s2_domain, did, dev, pasid);
193+
if (ret)
194+
__cache_tag_unassign_domain(domain, did, dev, pasid);
195+
196+
return ret;
197+
}
198+
199+
/*
200+
* Remove the cache tags associated with a device's PASID when the domain is
201+
* detached from the device.
202+
*
203+
* The cache tags must be previously assigned to the domain by calling the
204+
* assign interface.
205+
*/
206+
void cache_tag_unassign_domain(struct dmar_domain *domain,
207+
struct device *dev, ioasid_t pasid)
208+
{
209+
u16 did = domain_get_id_for_dev(domain, dev);
210+
211+
__cache_tag_unassign_domain(domain, did, dev, pasid);
212+
if (domain->domain.type == IOMMU_DOMAIN_NESTED)
213+
__cache_tag_unassign_parent_domain(domain->s2_domain, did, dev, pasid);
214+
}

drivers/iommu/intel/iommu.c

+26-2
Original file line numberDiff line numberDiff line change
@@ -1746,7 +1746,9 @@ static struct dmar_domain *alloc_domain(unsigned int type)
17461746
domain->has_iotlb_device = false;
17471747
INIT_LIST_HEAD(&domain->devices);
17481748
INIT_LIST_HEAD(&domain->dev_pasids);
1749+
INIT_LIST_HEAD(&domain->cache_tags);
17491750
spin_lock_init(&domain->lock);
1751+
spin_lock_init(&domain->cache_lock);
17501752
xa_init(&domain->iommu_array);
17511753

17521754
return domain;
@@ -1758,6 +1760,9 @@ int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)
17581760
unsigned long ndomains;
17591761
int num, ret = -ENOSPC;
17601762

1763+
if (domain->domain.type == IOMMU_DOMAIN_SVA)
1764+
return 0;
1765+
17611766
info = kzalloc(sizeof(*info), GFP_KERNEL);
17621767
if (!info)
17631768
return -ENOMEM;
@@ -1805,6 +1810,9 @@ void domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)
18051810
{
18061811
struct iommu_domain_info *info;
18071812

1813+
if (domain->domain.type == IOMMU_DOMAIN_SVA)
1814+
return;
1815+
18081816
spin_lock(&iommu->lock);
18091817
info = xa_load(&domain->iommu_array, iommu->seq_id);
18101818
if (--info->refcnt == 0) {
@@ -2323,6 +2331,13 @@ static int dmar_domain_attach_device(struct dmar_domain *domain,
23232331
ret = domain_attach_iommu(domain, iommu);
23242332
if (ret)
23252333
return ret;
2334+
2335+
ret = cache_tag_assign_domain(domain, dev, IOMMU_NO_PASID);
2336+
if (ret) {
2337+
domain_detach_iommu(domain, iommu);
2338+
return ret;
2339+
}
2340+
23262341
info->domain = domain;
23272342
spin_lock_irqsave(&domain->lock, flags);
23282343
list_add(&info->link, &domain->devices);
@@ -3811,6 +3826,7 @@ void device_block_translation(struct device *dev)
38113826
list_del(&info->link);
38123827
spin_unlock_irqrestore(&info->domain->lock, flags);
38133828

3829+
cache_tag_unassign_domain(info->domain, dev, IOMMU_NO_PASID);
38143830
domain_detach_iommu(info->domain, iommu);
38153831
info->domain = NULL;
38163832
}
@@ -4598,6 +4614,7 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
45984614
domain = iommu_get_domain_for_dev_pasid(dev, pasid, 0);
45994615
if (WARN_ON_ONCE(!domain))
46004616
goto out_tear_down;
4617+
dmar_domain = to_dmar_domain(domain);
46014618

46024619
/*
46034620
* The SVA implementation needs to handle its own stuffs like the mm
@@ -4606,10 +4623,10 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
46064623
*/
46074624
if (domain->type == IOMMU_DOMAIN_SVA) {
46084625
intel_svm_remove_dev_pasid(dev, pasid);
4626+
cache_tag_unassign_domain(dmar_domain, dev, pasid);
46094627
goto out_tear_down;
46104628
}
46114629

4612-
dmar_domain = to_dmar_domain(domain);
46134630
spin_lock_irqsave(&dmar_domain->lock, flags);
46144631
list_for_each_entry(curr, &dmar_domain->dev_pasids, link_domain) {
46154632
if (curr->dev == dev && curr->pasid == pasid) {
@@ -4621,6 +4638,7 @@ static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
46214638
WARN_ON_ONCE(!dev_pasid);
46224639
spin_unlock_irqrestore(&dmar_domain->lock, flags);
46234640

4641+
cache_tag_unassign_domain(dmar_domain, dev, pasid);
46244642
domain_detach_iommu(dmar_domain, iommu);
46254643
intel_iommu_debugfs_remove_dev_pasid(dev_pasid);
46264644
kfree(dev_pasid);
@@ -4660,6 +4678,10 @@ static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
46604678
if (ret)
46614679
goto out_free;
46624680

4681+
ret = cache_tag_assign_domain(dmar_domain, dev, pasid);
4682+
if (ret)
4683+
goto out_detach_iommu;
4684+
46634685
if (domain_type_is_si(dmar_domain))
46644686
ret = intel_pasid_setup_pass_through(iommu, dev, pasid);
46654687
else if (dmar_domain->use_first_level)
@@ -4669,7 +4691,7 @@ static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
46694691
ret = intel_pasid_setup_second_level(iommu, dmar_domain,
46704692
dev, pasid);
46714693
if (ret)
4672-
goto out_detach_iommu;
4694+
goto out_unassign_tag;
46734695

46744696
dev_pasid->dev = dev;
46754697
dev_pasid->pasid = pasid;
@@ -4681,6 +4703,8 @@ static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
46814703
intel_iommu_debugfs_create_dev_pasid(dev_pasid);
46824704

46834705
return 0;
4706+
out_unassign_tag:
4707+
cache_tag_unassign_domain(dmar_domain, dev, pasid);
46844708
out_detach_iommu:
46854709
domain_detach_iommu(dmar_domain, iommu);
46864710
out_free:

drivers/iommu/intel/iommu.h

+31
Original file line numberDiff line numberDiff line change
@@ -606,6 +606,9 @@ struct dmar_domain {
606606
struct list_head devices; /* all devices' list */
607607
struct list_head dev_pasids; /* all attached pasids */
608608

609+
spinlock_t cache_lock; /* Protect the cache tag list */
610+
struct list_head cache_tags; /* Cache tag list */
611+
609612
int iommu_superpage;/* Level of superpages supported:
610613
0 == 4KiB (no superpages), 1 == 2MiB,
611614
2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
@@ -1091,6 +1094,34 @@ struct iommu_domain *intel_nested_domain_alloc(struct iommu_domain *parent,
10911094
const struct iommu_user_data *user_data);
10921095
struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid);
10931096

1097+
enum cache_tag_type {
1098+
CACHE_TAG_IOTLB,
1099+
CACHE_TAG_DEVTLB,
1100+
CACHE_TAG_NESTING_IOTLB,
1101+
CACHE_TAG_NESTING_DEVTLB,
1102+
};
1103+
1104+
struct cache_tag {
1105+
struct list_head node;
1106+
enum cache_tag_type type;
1107+
struct intel_iommu *iommu;
1108+
/*
1109+
* The @dev field represents the location of the cache. For IOTLB, it
1110+
* resides on the IOMMU hardware. @dev stores the device pointer to
1111+
* the IOMMU hardware. For DevTLB, it locates in the PCIe endpoint.
1112+
* @dev stores the device pointer to that endpoint.
1113+
*/
1114+
struct device *dev;
1115+
u16 domain_id;
1116+
ioasid_t pasid;
1117+
unsigned int users;
1118+
};
1119+
1120+
int cache_tag_assign_domain(struct dmar_domain *domain,
1121+
struct device *dev, ioasid_t pasid);
1122+
void cache_tag_unassign_domain(struct dmar_domain *domain,
1123+
struct device *dev, ioasid_t pasid);
1124+
10941125
#ifdef CONFIG_INTEL_IOMMU_SVM
10951126
void intel_svm_check(struct intel_iommu *iommu);
10961127
int intel_svm_enable_prq(struct intel_iommu *iommu);

0 commit comments

Comments
 (0)