1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 * Leo Duran <leo.duran@amd.com>
6 */
7
8 #define pr_fmt(fmt) "AMD-Vi: " fmt
9 #define dev_fmt(fmt) pr_fmt(fmt)
10
11 #include <linux/ratelimit.h>
12 #include <linux/pci.h>
13 #include <linux/acpi.h>
14 #include <linux/pci-ats.h>
15 #include <linux/bitmap.h>
16 #include <linux/slab.h>
17 #include <linux/debugfs.h>
18 #include <linux/scatterlist.h>
19 #include <linux/dma-map-ops.h>
20 #include <linux/dma-direct.h>
21 #include <linux/iommu-helper.h>
22 #include <linux/delay.h>
23 #include <linux/amd-iommu.h>
24 #include <linux/notifier.h>
25 #include <linux/export.h>
26 #include <linux/irq.h>
27 #include <linux/msi.h>
28 #include <linux/irqdomain.h>
29 #include <linux/percpu.h>
30 #include <linux/io-pgtable.h>
31 #include <linux/cc_platform.h>
32 #include <asm/irq_remapping.h>
33 #include <asm/io_apic.h>
34 #include <asm/apic.h>
35 #include <asm/hw_irq.h>
36 #include <asm/proto.h>
37 #include <asm/iommu.h>
38 #include <asm/gart.h>
39 #include <asm/dma.h>
40 #include <uapi/linux/iommufd.h>
41
42 #include "amd_iommu.h"
43 #include "../dma-iommu.h"
44 #include "../irq_remapping.h"
45 #include "../iommu-pages.h"
46
47 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
48
49 /* Reserved IOVA ranges */
50 #define MSI_RANGE_START (0xfee00000)
51 #define MSI_RANGE_END (0xfeefffff)
52 #define HT_RANGE_START (0xfd00000000ULL)
53 #define HT_RANGE_END (0xffffffffffULL)
54
55 static DEFINE_SPINLOCK(pd_bitmap_lock);
56
57 LIST_HEAD(ioapic_map);
58 LIST_HEAD(hpet_map);
59 LIST_HEAD(acpihid_map);
60
61 const struct iommu_ops amd_iommu_ops;
62 static const struct iommu_dirty_ops amd_dirty_ops;
63
64 int amd_iommu_max_glx_val = -1;
65
66 /*
67 * general struct to manage commands send to an IOMMU
68 */
69 struct iommu_cmd {
70 u32 data[4];
71 };
72
73 struct kmem_cache *amd_iommu_irq_cache;
74
75 static void detach_device(struct device *dev);
76
77 static void set_dte_entry(struct amd_iommu *iommu,
78 struct iommu_dev_data *dev_data);
79
80 /****************************************************************************
81 *
82 * Helper functions
83 *
84 ****************************************************************************/
85
pdom_is_v2_pgtbl_mode(struct protection_domain * pdom)86 static inline bool pdom_is_v2_pgtbl_mode(struct protection_domain *pdom)
87 {
88 return (pdom && (pdom->pd_mode == PD_MODE_V2));
89 }
90
pdom_is_in_pt_mode(struct protection_domain * pdom)91 static inline bool pdom_is_in_pt_mode(struct protection_domain *pdom)
92 {
93 return (pdom->domain.type == IOMMU_DOMAIN_IDENTITY);
94 }
95
96 /*
97 * We cannot support PASID w/ existing v1 page table in the same domain
98 * since it will be nested. However, existing domain w/ v2 page table
99 * or passthrough mode can be used for PASID.
100 */
pdom_is_sva_capable(struct protection_domain * pdom)101 static inline bool pdom_is_sva_capable(struct protection_domain *pdom)
102 {
103 return pdom_is_v2_pgtbl_mode(pdom) || pdom_is_in_pt_mode(pdom);
104 }
105
get_acpihid_device_id(struct device * dev,struct acpihid_map_entry ** entry)106 static inline int get_acpihid_device_id(struct device *dev,
107 struct acpihid_map_entry **entry)
108 {
109 struct acpi_device *adev = ACPI_COMPANION(dev);
110 struct acpihid_map_entry *p;
111
112 if (!adev)
113 return -ENODEV;
114
115 list_for_each_entry(p, &acpihid_map, list) {
116 if (acpi_dev_hid_uid_match(adev, p->hid,
117 p->uid[0] ? p->uid : NULL)) {
118 if (entry)
119 *entry = p;
120 return p->devid;
121 }
122 }
123 return -EINVAL;
124 }
125
get_device_sbdf_id(struct device * dev)126 static inline int get_device_sbdf_id(struct device *dev)
127 {
128 int sbdf;
129
130 if (dev_is_pci(dev))
131 sbdf = get_pci_sbdf_id(to_pci_dev(dev));
132 else
133 sbdf = get_acpihid_device_id(dev, NULL);
134
135 return sbdf;
136 }
137
get_dev_table(struct amd_iommu * iommu)138 struct dev_table_entry *get_dev_table(struct amd_iommu *iommu)
139 {
140 struct dev_table_entry *dev_table;
141 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
142
143 BUG_ON(pci_seg == NULL);
144 dev_table = pci_seg->dev_table;
145 BUG_ON(dev_table == NULL);
146
147 return dev_table;
148 }
149
get_device_segment(struct device * dev)150 static inline u16 get_device_segment(struct device *dev)
151 {
152 u16 seg;
153
154 if (dev_is_pci(dev)) {
155 struct pci_dev *pdev = to_pci_dev(dev);
156
157 seg = pci_domain_nr(pdev->bus);
158 } else {
159 u32 devid = get_acpihid_device_id(dev, NULL);
160
161 seg = PCI_SBDF_TO_SEGID(devid);
162 }
163
164 return seg;
165 }
166
167 /* Writes the specific IOMMU for a device into the PCI segment rlookup table */
amd_iommu_set_rlookup_table(struct amd_iommu * iommu,u16 devid)168 void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid)
169 {
170 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
171
172 pci_seg->rlookup_table[devid] = iommu;
173 }
174
__rlookup_amd_iommu(u16 seg,u16 devid)175 static struct amd_iommu *__rlookup_amd_iommu(u16 seg, u16 devid)
176 {
177 struct amd_iommu_pci_seg *pci_seg;
178
179 for_each_pci_segment(pci_seg) {
180 if (pci_seg->id == seg)
181 return pci_seg->rlookup_table[devid];
182 }
183 return NULL;
184 }
185
rlookup_amd_iommu(struct device * dev)186 static struct amd_iommu *rlookup_amd_iommu(struct device *dev)
187 {
188 u16 seg = get_device_segment(dev);
189 int devid = get_device_sbdf_id(dev);
190
191 if (devid < 0)
192 return NULL;
193 return __rlookup_amd_iommu(seg, PCI_SBDF_TO_DEVID(devid));
194 }
195
alloc_dev_data(struct amd_iommu * iommu,u16 devid)196 static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid)
197 {
198 struct iommu_dev_data *dev_data;
199 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
200
201 dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
202 if (!dev_data)
203 return NULL;
204
205 spin_lock_init(&dev_data->lock);
206 dev_data->devid = devid;
207 ratelimit_default_init(&dev_data->rs);
208
209 llist_add(&dev_data->dev_data_list, &pci_seg->dev_data_list);
210 return dev_data;
211 }
212
search_dev_data(struct amd_iommu * iommu,u16 devid)213 static struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid)
214 {
215 struct iommu_dev_data *dev_data;
216 struct llist_node *node;
217 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
218
219 if (llist_empty(&pci_seg->dev_data_list))
220 return NULL;
221
222 node = pci_seg->dev_data_list.first;
223 llist_for_each_entry(dev_data, node, dev_data_list) {
224 if (dev_data->devid == devid)
225 return dev_data;
226 }
227
228 return NULL;
229 }
230
clone_alias(struct pci_dev * pdev,u16 alias,void * data)231 static int clone_alias(struct pci_dev *pdev, u16 alias, void *data)
232 {
233 struct amd_iommu *iommu;
234 struct dev_table_entry *dev_table;
235 u16 devid = pci_dev_id(pdev);
236
237 if (devid == alias)
238 return 0;
239
240 iommu = rlookup_amd_iommu(&pdev->dev);
241 if (!iommu)
242 return 0;
243
244 amd_iommu_set_rlookup_table(iommu, alias);
245 dev_table = get_dev_table(iommu);
246 memcpy(dev_table[alias].data,
247 dev_table[devid].data,
248 sizeof(dev_table[alias].data));
249
250 return 0;
251 }
252
clone_aliases(struct amd_iommu * iommu,struct device * dev)253 static void clone_aliases(struct amd_iommu *iommu, struct device *dev)
254 {
255 struct pci_dev *pdev;
256
257 if (!dev_is_pci(dev))
258 return;
259 pdev = to_pci_dev(dev);
260
261 /*
262 * The IVRS alias stored in the alias table may not be
263 * part of the PCI DMA aliases if it's bus differs
264 * from the original device.
265 */
266 clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], NULL);
267
268 pci_for_each_dma_alias(pdev, clone_alias, NULL);
269 }
270
setup_aliases(struct amd_iommu * iommu,struct device * dev)271 static void setup_aliases(struct amd_iommu *iommu, struct device *dev)
272 {
273 struct pci_dev *pdev = to_pci_dev(dev);
274 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
275 u16 ivrs_alias;
276
277 /* For ACPI HID devices, there are no aliases */
278 if (!dev_is_pci(dev))
279 return;
280
281 /*
282 * Add the IVRS alias to the pci aliases if it is on the same
283 * bus. The IVRS table may know about a quirk that we don't.
284 */
285 ivrs_alias = pci_seg->alias_table[pci_dev_id(pdev)];
286 if (ivrs_alias != pci_dev_id(pdev) &&
287 PCI_BUS_NUM(ivrs_alias) == pdev->bus->number)
288 pci_add_dma_alias(pdev, ivrs_alias & 0xff, 1);
289
290 clone_aliases(iommu, dev);
291 }
292
find_dev_data(struct amd_iommu * iommu,u16 devid)293 static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid)
294 {
295 struct iommu_dev_data *dev_data;
296
297 dev_data = search_dev_data(iommu, devid);
298
299 if (dev_data == NULL) {
300 dev_data = alloc_dev_data(iommu, devid);
301 if (!dev_data)
302 return NULL;
303
304 if (translation_pre_enabled(iommu))
305 dev_data->defer_attach = true;
306 }
307
308 return dev_data;
309 }
310
311 /*
312 * Find or create an IOMMU group for a acpihid device.
313 */
acpihid_device_group(struct device * dev)314 static struct iommu_group *acpihid_device_group(struct device *dev)
315 {
316 struct acpihid_map_entry *p, *entry = NULL;
317 int devid;
318
319 devid = get_acpihid_device_id(dev, &entry);
320 if (devid < 0)
321 return ERR_PTR(devid);
322
323 list_for_each_entry(p, &acpihid_map, list) {
324 if ((devid == p->devid) && p->group)
325 entry->group = p->group;
326 }
327
328 if (!entry->group)
329 entry->group = generic_device_group(dev);
330 else
331 iommu_group_ref_get(entry->group);
332
333 return entry->group;
334 }
335
pdev_pasid_supported(struct iommu_dev_data * dev_data)336 static inline bool pdev_pasid_supported(struct iommu_dev_data *dev_data)
337 {
338 return (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP);
339 }
340
pdev_get_caps(struct pci_dev * pdev)341 static u32 pdev_get_caps(struct pci_dev *pdev)
342 {
343 int features;
344 u32 flags = 0;
345
346 if (pci_ats_supported(pdev))
347 flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
348
349 if (pci_pri_supported(pdev))
350 flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;
351
352 features = pci_pasid_features(pdev);
353 if (features >= 0) {
354 flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
355
356 if (features & PCI_PASID_CAP_EXEC)
357 flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
358
359 if (features & PCI_PASID_CAP_PRIV)
360 flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
361 }
362
363 return flags;
364 }
365
pdev_enable_cap_ats(struct pci_dev * pdev)366 static inline int pdev_enable_cap_ats(struct pci_dev *pdev)
367 {
368 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
369 int ret = -EINVAL;
370
371 if (dev_data->ats_enabled)
372 return 0;
373
374 if (amd_iommu_iotlb_sup &&
375 (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP)) {
376 ret = pci_enable_ats(pdev, PAGE_SHIFT);
377 if (!ret) {
378 dev_data->ats_enabled = 1;
379 dev_data->ats_qdep = pci_ats_queue_depth(pdev);
380 }
381 }
382
383 return ret;
384 }
385
pdev_disable_cap_ats(struct pci_dev * pdev)386 static inline void pdev_disable_cap_ats(struct pci_dev *pdev)
387 {
388 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
389
390 if (dev_data->ats_enabled) {
391 pci_disable_ats(pdev);
392 dev_data->ats_enabled = 0;
393 }
394 }
395
pdev_enable_cap_pri(struct pci_dev * pdev)396 static inline int pdev_enable_cap_pri(struct pci_dev *pdev)
397 {
398 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
399 int ret = -EINVAL;
400
401 if (dev_data->pri_enabled)
402 return 0;
403
404 if (!dev_data->ats_enabled)
405 return 0;
406
407 if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) {
408 /*
409 * First reset the PRI state of the device.
410 * FIXME: Hardcode number of outstanding requests for now
411 */
412 if (!pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32)) {
413 dev_data->pri_enabled = 1;
414 dev_data->pri_tlp = pci_prg_resp_pasid_required(pdev);
415
416 ret = 0;
417 }
418 }
419
420 return ret;
421 }
422
pdev_disable_cap_pri(struct pci_dev * pdev)423 static inline void pdev_disable_cap_pri(struct pci_dev *pdev)
424 {
425 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
426
427 if (dev_data->pri_enabled) {
428 pci_disable_pri(pdev);
429 dev_data->pri_enabled = 0;
430 }
431 }
432
pdev_enable_cap_pasid(struct pci_dev * pdev)433 static inline int pdev_enable_cap_pasid(struct pci_dev *pdev)
434 {
435 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
436 int ret = -EINVAL;
437
438 if (dev_data->pasid_enabled)
439 return 0;
440
441 if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) {
442 /* Only allow access to user-accessible pages */
443 ret = pci_enable_pasid(pdev, 0);
444 if (!ret)
445 dev_data->pasid_enabled = 1;
446 }
447
448 return ret;
449 }
450
pdev_disable_cap_pasid(struct pci_dev * pdev)451 static inline void pdev_disable_cap_pasid(struct pci_dev *pdev)
452 {
453 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
454
455 if (dev_data->pasid_enabled) {
456 pci_disable_pasid(pdev);
457 dev_data->pasid_enabled = 0;
458 }
459 }
460
pdev_enable_caps(struct pci_dev * pdev)461 static void pdev_enable_caps(struct pci_dev *pdev)
462 {
463 pdev_enable_cap_ats(pdev);
464 pdev_enable_cap_pasid(pdev);
465 pdev_enable_cap_pri(pdev);
466 }
467
pdev_disable_caps(struct pci_dev * pdev)468 static void pdev_disable_caps(struct pci_dev *pdev)
469 {
470 pdev_disable_cap_ats(pdev);
471 pdev_disable_cap_pasid(pdev);
472 pdev_disable_cap_pri(pdev);
473 }
474
475 /*
476 * This function checks if the driver got a valid device from the caller to
477 * avoid dereferencing invalid pointers.
478 */
check_device(struct device * dev)479 static bool check_device(struct device *dev)
480 {
481 struct amd_iommu_pci_seg *pci_seg;
482 struct amd_iommu *iommu;
483 int devid, sbdf;
484
485 if (!dev)
486 return false;
487
488 sbdf = get_device_sbdf_id(dev);
489 if (sbdf < 0)
490 return false;
491 devid = PCI_SBDF_TO_DEVID(sbdf);
492
493 iommu = rlookup_amd_iommu(dev);
494 if (!iommu)
495 return false;
496
497 /* Out of our scope? */
498 pci_seg = iommu->pci_seg;
499 if (devid > pci_seg->last_bdf)
500 return false;
501
502 return true;
503 }
504
iommu_init_device(struct amd_iommu * iommu,struct device * dev)505 static int iommu_init_device(struct amd_iommu *iommu, struct device *dev)
506 {
507 struct iommu_dev_data *dev_data;
508 int devid, sbdf;
509
510 if (dev_iommu_priv_get(dev))
511 return 0;
512
513 sbdf = get_device_sbdf_id(dev);
514 if (sbdf < 0)
515 return sbdf;
516
517 devid = PCI_SBDF_TO_DEVID(sbdf);
518 dev_data = find_dev_data(iommu, devid);
519 if (!dev_data)
520 return -ENOMEM;
521
522 dev_data->dev = dev;
523 setup_aliases(iommu, dev);
524
525 /*
526 * By default we use passthrough mode for IOMMUv2 capable device.
527 * But if amd_iommu=force_isolation is set (e.g. to debug DMA to
528 * invalid address), we ignore the capability for the device so
529 * it'll be forced to go into translation mode.
530 */
531 if ((iommu_default_passthrough() || !amd_iommu_force_isolation) &&
532 dev_is_pci(dev) && amd_iommu_gt_ppr_supported()) {
533 dev_data->flags = pdev_get_caps(to_pci_dev(dev));
534 }
535
536 dev_iommu_priv_set(dev, dev_data);
537
538 return 0;
539 }
540
iommu_ignore_device(struct amd_iommu * iommu,struct device * dev)541 static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev)
542 {
543 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
544 struct dev_table_entry *dev_table = get_dev_table(iommu);
545 int devid, sbdf;
546
547 sbdf = get_device_sbdf_id(dev);
548 if (sbdf < 0)
549 return;
550
551 devid = PCI_SBDF_TO_DEVID(sbdf);
552 pci_seg->rlookup_table[devid] = NULL;
553 memset(&dev_table[devid], 0, sizeof(struct dev_table_entry));
554
555 setup_aliases(iommu, dev);
556 }
557
amd_iommu_uninit_device(struct device * dev)558 static void amd_iommu_uninit_device(struct device *dev)
559 {
560 struct iommu_dev_data *dev_data;
561
562 dev_data = dev_iommu_priv_get(dev);
563 if (!dev_data)
564 return;
565
566 if (dev_data->domain)
567 detach_device(dev);
568
569 /*
570 * We keep dev_data around for unplugged devices and reuse it when the
571 * device is re-plugged - not doing so would introduce a ton of races.
572 */
573 }
574
575 /****************************************************************************
576 *
577 * Interrupt handling functions
578 *
579 ****************************************************************************/
580
dump_dte_entry(struct amd_iommu * iommu,u16 devid)581 static void dump_dte_entry(struct amd_iommu *iommu, u16 devid)
582 {
583 int i;
584 struct dev_table_entry *dev_table = get_dev_table(iommu);
585
586 for (i = 0; i < 4; ++i)
587 pr_err("DTE[%d]: %016llx\n", i, dev_table[devid].data[i]);
588 }
589
dump_command(unsigned long phys_addr)590 static void dump_command(unsigned long phys_addr)
591 {
592 struct iommu_cmd *cmd = iommu_phys_to_virt(phys_addr);
593 int i;
594
595 for (i = 0; i < 4; ++i)
596 pr_err("CMD[%d]: %08x\n", i, cmd->data[i]);
597 }
598
amd_iommu_report_rmp_hw_error(struct amd_iommu * iommu,volatile u32 * event)599 static void amd_iommu_report_rmp_hw_error(struct amd_iommu *iommu, volatile u32 *event)
600 {
601 struct iommu_dev_data *dev_data = NULL;
602 int devid, vmg_tag, flags;
603 struct pci_dev *pdev;
604 u64 spa;
605
606 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
607 vmg_tag = (event[1]) & 0xFFFF;
608 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
609 spa = ((u64)event[3] << 32) | (event[2] & 0xFFFFFFF8);
610
611 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),
612 devid & 0xff);
613 if (pdev)
614 dev_data = dev_iommu_priv_get(&pdev->dev);
615
616 if (dev_data) {
617 if (__ratelimit(&dev_data->rs)) {
618 pci_err(pdev, "Event logged [RMP_HW_ERROR vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n",
619 vmg_tag, spa, flags);
620 }
621 } else {
622 pr_err_ratelimited("Event logged [RMP_HW_ERROR device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n",
623 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
624 vmg_tag, spa, flags);
625 }
626
627 if (pdev)
628 pci_dev_put(pdev);
629 }
630
amd_iommu_report_rmp_fault(struct amd_iommu * iommu,volatile u32 * event)631 static void amd_iommu_report_rmp_fault(struct amd_iommu *iommu, volatile u32 *event)
632 {
633 struct iommu_dev_data *dev_data = NULL;
634 int devid, flags_rmp, vmg_tag, flags;
635 struct pci_dev *pdev;
636 u64 gpa;
637
638 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
639 flags_rmp = (event[0] >> EVENT_FLAGS_SHIFT) & 0xFF;
640 vmg_tag = (event[1]) & 0xFFFF;
641 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
642 gpa = ((u64)event[3] << 32) | event[2];
643
644 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),
645 devid & 0xff);
646 if (pdev)
647 dev_data = dev_iommu_priv_get(&pdev->dev);
648
649 if (dev_data) {
650 if (__ratelimit(&dev_data->rs)) {
651 pci_err(pdev, "Event logged [RMP_PAGE_FAULT vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n",
652 vmg_tag, gpa, flags_rmp, flags);
653 }
654 } else {
655 pr_err_ratelimited("Event logged [RMP_PAGE_FAULT device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n",
656 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
657 vmg_tag, gpa, flags_rmp, flags);
658 }
659
660 if (pdev)
661 pci_dev_put(pdev);
662 }
663
664 #define IS_IOMMU_MEM_TRANSACTION(flags) \
665 (((flags) & EVENT_FLAG_I) == 0)
666
667 #define IS_WRITE_REQUEST(flags) \
668 ((flags) & EVENT_FLAG_RW)
669
amd_iommu_report_page_fault(struct amd_iommu * iommu,u16 devid,u16 domain_id,u64 address,int flags)670 static void amd_iommu_report_page_fault(struct amd_iommu *iommu,
671 u16 devid, u16 domain_id,
672 u64 address, int flags)
673 {
674 struct iommu_dev_data *dev_data = NULL;
675 struct pci_dev *pdev;
676
677 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),
678 devid & 0xff);
679 if (pdev)
680 dev_data = dev_iommu_priv_get(&pdev->dev);
681
682 if (dev_data) {
683 /*
684 * If this is a DMA fault (for which the I(nterrupt)
685 * bit will be unset), allow report_iommu_fault() to
686 * prevent logging it.
687 */
688 if (IS_IOMMU_MEM_TRANSACTION(flags)) {
689 /* Device not attached to domain properly */
690 if (dev_data->domain == NULL) {
691 pr_err_ratelimited("Event logged [Device not attached to domain properly]\n");
692 pr_err_ratelimited(" device=%04x:%02x:%02x.%x domain=0x%04x\n",
693 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid),
694 PCI_FUNC(devid), domain_id);
695 goto out;
696 }
697
698 if (!report_iommu_fault(&dev_data->domain->domain,
699 &pdev->dev, address,
700 IS_WRITE_REQUEST(flags) ?
701 IOMMU_FAULT_WRITE :
702 IOMMU_FAULT_READ))
703 goto out;
704 }
705
706 if (__ratelimit(&dev_data->rs)) {
707 pci_err(pdev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%llx flags=0x%04x]\n",
708 domain_id, address, flags);
709 }
710 } else {
711 pr_err_ratelimited("Event logged [IO_PAGE_FAULT device=%04x:%02x:%02x.%x domain=0x%04x address=0x%llx flags=0x%04x]\n",
712 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
713 domain_id, address, flags);
714 }
715
716 out:
717 if (pdev)
718 pci_dev_put(pdev);
719 }
720
iommu_print_event(struct amd_iommu * iommu,void * __evt)721 static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
722 {
723 struct device *dev = iommu->iommu.dev;
724 int type, devid, flags, tag;
725 volatile u32 *event = __evt;
726 int count = 0;
727 u64 address;
728 u32 pasid;
729
730 retry:
731 type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
732 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
733 pasid = (event[0] & EVENT_DOMID_MASK_HI) |
734 (event[1] & EVENT_DOMID_MASK_LO);
735 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
736 address = (u64)(((u64)event[3]) << 32) | event[2];
737
738 if (type == 0) {
739 /* Did we hit the erratum? */
740 if (++count == LOOP_TIMEOUT) {
741 pr_err("No event written to event log\n");
742 return;
743 }
744 udelay(1);
745 goto retry;
746 }
747
748 if (type == EVENT_TYPE_IO_FAULT) {
749 amd_iommu_report_page_fault(iommu, devid, pasid, address, flags);
750 return;
751 }
752
753 switch (type) {
754 case EVENT_TYPE_ILL_DEV:
755 dev_err(dev, "Event logged [ILLEGAL_DEV_TABLE_ENTRY device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n",
756 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
757 pasid, address, flags);
758 dump_dte_entry(iommu, devid);
759 break;
760 case EVENT_TYPE_DEV_TAB_ERR:
761 dev_err(dev, "Event logged [DEV_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x "
762 "address=0x%llx flags=0x%04x]\n",
763 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
764 address, flags);
765 break;
766 case EVENT_TYPE_PAGE_TAB_ERR:
767 dev_err(dev, "Event logged [PAGE_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x pasid=0x%04x address=0x%llx flags=0x%04x]\n",
768 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
769 pasid, address, flags);
770 break;
771 case EVENT_TYPE_ILL_CMD:
772 dev_err(dev, "Event logged [ILLEGAL_COMMAND_ERROR address=0x%llx]\n", address);
773 dump_command(address);
774 break;
775 case EVENT_TYPE_CMD_HARD_ERR:
776 dev_err(dev, "Event logged [COMMAND_HARDWARE_ERROR address=0x%llx flags=0x%04x]\n",
777 address, flags);
778 break;
779 case EVENT_TYPE_IOTLB_INV_TO:
780 dev_err(dev, "Event logged [IOTLB_INV_TIMEOUT device=%04x:%02x:%02x.%x address=0x%llx]\n",
781 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
782 address);
783 break;
784 case EVENT_TYPE_INV_DEV_REQ:
785 dev_err(dev, "Event logged [INVALID_DEVICE_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n",
786 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
787 pasid, address, flags);
788 break;
789 case EVENT_TYPE_RMP_FAULT:
790 amd_iommu_report_rmp_fault(iommu, event);
791 break;
792 case EVENT_TYPE_RMP_HW_ERR:
793 amd_iommu_report_rmp_hw_error(iommu, event);
794 break;
795 case EVENT_TYPE_INV_PPR_REQ:
796 pasid = PPR_PASID(*((u64 *)__evt));
797 tag = event[1] & 0x03FF;
798 dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x tag=0x%03x]\n",
799 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
800 pasid, address, flags, tag);
801 break;
802 default:
803 dev_err(dev, "Event logged [UNKNOWN event[0]=0x%08x event[1]=0x%08x event[2]=0x%08x event[3]=0x%08x\n",
804 event[0], event[1], event[2], event[3]);
805 }
806
807 /*
808 * To detect the hardware errata 732 we need to clear the
809 * entry back to zero. This issue does not exist on SNP
810 * enabled system. Also this buffer is not writeable on
811 * SNP enabled system.
812 */
813 if (!amd_iommu_snp_en)
814 memset(__evt, 0, 4 * sizeof(u32));
815 }
816
iommu_poll_events(struct amd_iommu * iommu)817 static void iommu_poll_events(struct amd_iommu *iommu)
818 {
819 u32 head, tail;
820
821 head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
822 tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
823
824 while (head != tail) {
825 iommu_print_event(iommu, iommu->evt_buf + head);
826
827 /* Update head pointer of hardware ring-buffer */
828 head = (head + EVENT_ENTRY_SIZE) % EVT_BUFFER_SIZE;
829 writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
830 }
831
832 }
833
834 #ifdef CONFIG_IRQ_REMAP
835 static int (*iommu_ga_log_notifier)(u32);
836
amd_iommu_register_ga_log_notifier(int (* notifier)(u32))837 int amd_iommu_register_ga_log_notifier(int (*notifier)(u32))
838 {
839 iommu_ga_log_notifier = notifier;
840
841 return 0;
842 }
843 EXPORT_SYMBOL(amd_iommu_register_ga_log_notifier);
844
iommu_poll_ga_log(struct amd_iommu * iommu)845 static void iommu_poll_ga_log(struct amd_iommu *iommu)
846 {
847 u32 head, tail;
848
849 if (iommu->ga_log == NULL)
850 return;
851
852 head = readl(iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
853 tail = readl(iommu->mmio_base + MMIO_GA_TAIL_OFFSET);
854
855 while (head != tail) {
856 volatile u64 *raw;
857 u64 log_entry;
858
859 raw = (u64 *)(iommu->ga_log + head);
860
861 /* Avoid memcpy function-call overhead */
862 log_entry = *raw;
863
864 /* Update head pointer of hardware ring-buffer */
865 head = (head + GA_ENTRY_SIZE) % GA_LOG_SIZE;
866 writel(head, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
867
868 /* Handle GA entry */
869 switch (GA_REQ_TYPE(log_entry)) {
870 case GA_GUEST_NR:
871 if (!iommu_ga_log_notifier)
872 break;
873
874 pr_debug("%s: devid=%#x, ga_tag=%#x\n",
875 __func__, GA_DEVID(log_entry),
876 GA_TAG(log_entry));
877
878 if (iommu_ga_log_notifier(GA_TAG(log_entry)) != 0)
879 pr_err("GA log notifier failed.\n");
880 break;
881 default:
882 break;
883 }
884 }
885 }
886
887 static void
amd_iommu_set_pci_msi_domain(struct device * dev,struct amd_iommu * iommu)888 amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu)
889 {
890 if (!irq_remapping_enabled || !dev_is_pci(dev) ||
891 !pci_dev_has_default_msi_parent_domain(to_pci_dev(dev)))
892 return;
893
894 dev_set_msi_domain(dev, iommu->ir_domain);
895 }
896
897 #else /* CONFIG_IRQ_REMAP */
898 static inline void
amd_iommu_set_pci_msi_domain(struct device * dev,struct amd_iommu * iommu)899 amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu) { }
900 #endif /* !CONFIG_IRQ_REMAP */
901
amd_iommu_handle_irq(void * data,const char * evt_type,u32 int_mask,u32 overflow_mask,void (* int_handler)(struct amd_iommu *),void (* overflow_handler)(struct amd_iommu *))902 static void amd_iommu_handle_irq(void *data, const char *evt_type,
903 u32 int_mask, u32 overflow_mask,
904 void (*int_handler)(struct amd_iommu *),
905 void (*overflow_handler)(struct amd_iommu *))
906 {
907 struct amd_iommu *iommu = (struct amd_iommu *) data;
908 u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
909 u32 mask = int_mask | overflow_mask;
910
911 while (status & mask) {
912 /* Enable interrupt sources again */
913 writel(mask, iommu->mmio_base + MMIO_STATUS_OFFSET);
914
915 if (int_handler) {
916 pr_devel("Processing IOMMU (ivhd%d) %s Log\n",
917 iommu->index, evt_type);
918 int_handler(iommu);
919 }
920
921 if ((status & overflow_mask) && overflow_handler)
922 overflow_handler(iommu);
923
924 /*
925 * Hardware bug: ERBT1312
926 * When re-enabling interrupt (by writing 1
927 * to clear the bit), the hardware might also try to set
928 * the interrupt bit in the event status register.
929 * In this scenario, the bit will be set, and disable
930 * subsequent interrupts.
931 *
932 * Workaround: The IOMMU driver should read back the
933 * status register and check if the interrupt bits are cleared.
934 * If not, driver will need to go through the interrupt handler
935 * again and re-clear the bits
936 */
937 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
938 }
939 }
940
amd_iommu_int_thread_evtlog(int irq,void * data)941 irqreturn_t amd_iommu_int_thread_evtlog(int irq, void *data)
942 {
943 amd_iommu_handle_irq(data, "Evt", MMIO_STATUS_EVT_INT_MASK,
944 MMIO_STATUS_EVT_OVERFLOW_MASK,
945 iommu_poll_events, amd_iommu_restart_event_logging);
946
947 return IRQ_HANDLED;
948 }
949
amd_iommu_int_thread_pprlog(int irq,void * data)950 irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data)
951 {
952 amd_iommu_handle_irq(data, "PPR", MMIO_STATUS_PPR_INT_MASK,
953 MMIO_STATUS_PPR_OVERFLOW_MASK,
954 amd_iommu_poll_ppr_log, amd_iommu_restart_ppr_log);
955
956 return IRQ_HANDLED;
957 }
958
amd_iommu_int_thread_galog(int irq,void * data)959 irqreturn_t amd_iommu_int_thread_galog(int irq, void *data)
960 {
961 #ifdef CONFIG_IRQ_REMAP
962 amd_iommu_handle_irq(data, "GA", MMIO_STATUS_GALOG_INT_MASK,
963 MMIO_STATUS_GALOG_OVERFLOW_MASK,
964 iommu_poll_ga_log, amd_iommu_restart_ga_log);
965 #endif
966
967 return IRQ_HANDLED;
968 }
969
amd_iommu_int_thread(int irq,void * data)970 irqreturn_t amd_iommu_int_thread(int irq, void *data)
971 {
972 amd_iommu_int_thread_evtlog(irq, data);
973 amd_iommu_int_thread_pprlog(irq, data);
974 amd_iommu_int_thread_galog(irq, data);
975
976 return IRQ_HANDLED;
977 }
978
amd_iommu_int_handler(int irq,void * data)979 irqreturn_t amd_iommu_int_handler(int irq, void *data)
980 {
981 return IRQ_WAKE_THREAD;
982 }
983
984 /****************************************************************************
985 *
986 * IOMMU command queuing functions
987 *
988 ****************************************************************************/
989
wait_on_sem(struct amd_iommu * iommu,u64 data)990 static int wait_on_sem(struct amd_iommu *iommu, u64 data)
991 {
992 int i = 0;
993
994 while (*iommu->cmd_sem != data && i < LOOP_TIMEOUT) {
995 udelay(1);
996 i += 1;
997 }
998
999 if (i == LOOP_TIMEOUT) {
1000 pr_alert("Completion-Wait loop timed out\n");
1001 return -EIO;
1002 }
1003
1004 return 0;
1005 }
1006
copy_cmd_to_buffer(struct amd_iommu * iommu,struct iommu_cmd * cmd)1007 static void copy_cmd_to_buffer(struct amd_iommu *iommu,
1008 struct iommu_cmd *cmd)
1009 {
1010 u8 *target;
1011 u32 tail;
1012
1013 /* Copy command to buffer */
1014 tail = iommu->cmd_buf_tail;
1015 target = iommu->cmd_buf + tail;
1016 memcpy(target, cmd, sizeof(*cmd));
1017
1018 tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
1019 iommu->cmd_buf_tail = tail;
1020
1021 /* Tell the IOMMU about it */
1022 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
1023 }
1024
build_completion_wait(struct iommu_cmd * cmd,struct amd_iommu * iommu,u64 data)1025 static void build_completion_wait(struct iommu_cmd *cmd,
1026 struct amd_iommu *iommu,
1027 u64 data)
1028 {
1029 u64 paddr = iommu_virt_to_phys((void *)iommu->cmd_sem);
1030
1031 memset(cmd, 0, sizeof(*cmd));
1032 cmd->data[0] = lower_32_bits(paddr) | CMD_COMPL_WAIT_STORE_MASK;
1033 cmd->data[1] = upper_32_bits(paddr);
1034 cmd->data[2] = lower_32_bits(data);
1035 cmd->data[3] = upper_32_bits(data);
1036 CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
1037 }
1038
build_inv_dte(struct iommu_cmd * cmd,u16 devid)1039 static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
1040 {
1041 memset(cmd, 0, sizeof(*cmd));
1042 cmd->data[0] = devid;
1043 CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
1044 }
1045
1046 /*
1047 * Builds an invalidation address which is suitable for one page or multiple
1048 * pages. Sets the size bit (S) as needed is more than one page is flushed.
1049 */
build_inv_address(u64 address,size_t size)1050 static inline u64 build_inv_address(u64 address, size_t size)
1051 {
1052 u64 pages, end, msb_diff;
1053
1054 pages = iommu_num_pages(address, size, PAGE_SIZE);
1055
1056 if (pages == 1)
1057 return address & PAGE_MASK;
1058
1059 end = address + size - 1;
1060
1061 /*
1062 * msb_diff would hold the index of the most significant bit that
1063 * flipped between the start and end.
1064 */
1065 msb_diff = fls64(end ^ address) - 1;
1066
1067 /*
1068 * Bits 63:52 are sign extended. If for some reason bit 51 is different
1069 * between the start and the end, invalidate everything.
1070 */
1071 if (unlikely(msb_diff > 51)) {
1072 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
1073 } else {
1074 /*
1075 * The msb-bit must be clear on the address. Just set all the
1076 * lower bits.
1077 */
1078 address |= (1ull << msb_diff) - 1;
1079 }
1080
1081 /* Clear bits 11:0 */
1082 address &= PAGE_MASK;
1083
1084 /* Set the size bit - we flush more than one 4kb page */
1085 return address | CMD_INV_IOMMU_PAGES_SIZE_MASK;
1086 }
1087
build_inv_iommu_pages(struct iommu_cmd * cmd,u64 address,size_t size,u16 domid,ioasid_t pasid,bool gn)1088 static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
1089 size_t size, u16 domid,
1090 ioasid_t pasid, bool gn)
1091 {
1092 u64 inv_address = build_inv_address(address, size);
1093
1094 memset(cmd, 0, sizeof(*cmd));
1095
1096 cmd->data[1] |= domid;
1097 cmd->data[2] = lower_32_bits(inv_address);
1098 cmd->data[3] = upper_32_bits(inv_address);
1099 /* PDE bit - we want to flush everything, not only the PTEs */
1100 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
1101 if (gn) {
1102 cmd->data[0] |= pasid;
1103 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
1104 }
1105 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
1106 }
1107
build_inv_iotlb_pages(struct iommu_cmd * cmd,u16 devid,int qdep,u64 address,size_t size,ioasid_t pasid,bool gn)1108 static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
1109 u64 address, size_t size,
1110 ioasid_t pasid, bool gn)
1111 {
1112 u64 inv_address = build_inv_address(address, size);
1113
1114 memset(cmd, 0, sizeof(*cmd));
1115
1116 cmd->data[0] = devid;
1117 cmd->data[0] |= (qdep & 0xff) << 24;
1118 cmd->data[1] = devid;
1119 cmd->data[2] = lower_32_bits(inv_address);
1120 cmd->data[3] = upper_32_bits(inv_address);
1121 if (gn) {
1122 cmd->data[0] |= ((pasid >> 8) & 0xff) << 16;
1123 cmd->data[1] |= (pasid & 0xff) << 16;
1124 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
1125 }
1126
1127 CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
1128 }
1129
build_complete_ppr(struct iommu_cmd * cmd,u16 devid,u32 pasid,int status,int tag,u8 gn)1130 static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, u32 pasid,
1131 int status, int tag, u8 gn)
1132 {
1133 memset(cmd, 0, sizeof(*cmd));
1134
1135 cmd->data[0] = devid;
1136 if (gn) {
1137 cmd->data[1] = pasid;
1138 cmd->data[2] = CMD_INV_IOMMU_PAGES_GN_MASK;
1139 }
1140 cmd->data[3] = tag & 0x1ff;
1141 cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;
1142
1143 CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);
1144 }
1145
build_inv_all(struct iommu_cmd * cmd)1146 static void build_inv_all(struct iommu_cmd *cmd)
1147 {
1148 memset(cmd, 0, sizeof(*cmd));
1149 CMD_SET_TYPE(cmd, CMD_INV_ALL);
1150 }
1151
build_inv_irt(struct iommu_cmd * cmd,u16 devid)1152 static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)
1153 {
1154 memset(cmd, 0, sizeof(*cmd));
1155 cmd->data[0] = devid;
1156 CMD_SET_TYPE(cmd, CMD_INV_IRT);
1157 }
1158
1159 /*
1160 * Writes the command to the IOMMUs command buffer and informs the
1161 * hardware about the new command.
1162 */
__iommu_queue_command_sync(struct amd_iommu * iommu,struct iommu_cmd * cmd,bool sync)1163 static int __iommu_queue_command_sync(struct amd_iommu *iommu,
1164 struct iommu_cmd *cmd,
1165 bool sync)
1166 {
1167 unsigned int count = 0;
1168 u32 left, next_tail;
1169
1170 next_tail = (iommu->cmd_buf_tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
1171 again:
1172 left = (iommu->cmd_buf_head - next_tail) % CMD_BUFFER_SIZE;
1173
1174 if (left <= 0x20) {
1175 /* Skip udelay() the first time around */
1176 if (count++) {
1177 if (count == LOOP_TIMEOUT) {
1178 pr_err("Command buffer timeout\n");
1179 return -EIO;
1180 }
1181
1182 udelay(1);
1183 }
1184
1185 /* Update head and recheck remaining space */
1186 iommu->cmd_buf_head = readl(iommu->mmio_base +
1187 MMIO_CMD_HEAD_OFFSET);
1188
1189 goto again;
1190 }
1191
1192 copy_cmd_to_buffer(iommu, cmd);
1193
1194 /* Do we need to make sure all commands are processed? */
1195 iommu->need_sync = sync;
1196
1197 return 0;
1198 }
1199
iommu_queue_command_sync(struct amd_iommu * iommu,struct iommu_cmd * cmd,bool sync)1200 static int iommu_queue_command_sync(struct amd_iommu *iommu,
1201 struct iommu_cmd *cmd,
1202 bool sync)
1203 {
1204 unsigned long flags;
1205 int ret;
1206
1207 raw_spin_lock_irqsave(&iommu->lock, flags);
1208 ret = __iommu_queue_command_sync(iommu, cmd, sync);
1209 raw_spin_unlock_irqrestore(&iommu->lock, flags);
1210
1211 return ret;
1212 }
1213
iommu_queue_command(struct amd_iommu * iommu,struct iommu_cmd * cmd)1214 static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
1215 {
1216 return iommu_queue_command_sync(iommu, cmd, true);
1217 }
1218
1219 /*
1220 * This function queues a completion wait command into the command
1221 * buffer of an IOMMU
1222 */
iommu_completion_wait(struct amd_iommu * iommu)1223 static int iommu_completion_wait(struct amd_iommu *iommu)
1224 {
1225 struct iommu_cmd cmd;
1226 unsigned long flags;
1227 int ret;
1228 u64 data;
1229
1230 if (!iommu->need_sync)
1231 return 0;
1232
1233 data = atomic64_add_return(1, &iommu->cmd_sem_val);
1234 build_completion_wait(&cmd, iommu, data);
1235
1236 raw_spin_lock_irqsave(&iommu->lock, flags);
1237
1238 ret = __iommu_queue_command_sync(iommu, &cmd, false);
1239 if (ret)
1240 goto out_unlock;
1241
1242 ret = wait_on_sem(iommu, data);
1243
1244 out_unlock:
1245 raw_spin_unlock_irqrestore(&iommu->lock, flags);
1246
1247 return ret;
1248 }
1249
domain_flush_complete(struct protection_domain * domain)1250 static void domain_flush_complete(struct protection_domain *domain)
1251 {
1252 int i;
1253
1254 for (i = 0; i < amd_iommu_get_num_iommus(); ++i) {
1255 if (domain && !domain->dev_iommu[i])
1256 continue;
1257
1258 /*
1259 * Devices of this domain are behind this IOMMU
1260 * We need to wait for completion of all commands.
1261 */
1262 iommu_completion_wait(amd_iommus[i]);
1263 }
1264 }
1265
iommu_flush_dte(struct amd_iommu * iommu,u16 devid)1266 static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
1267 {
1268 struct iommu_cmd cmd;
1269
1270 build_inv_dte(&cmd, devid);
1271
1272 return iommu_queue_command(iommu, &cmd);
1273 }
1274
amd_iommu_flush_dte_all(struct amd_iommu * iommu)1275 static void amd_iommu_flush_dte_all(struct amd_iommu *iommu)
1276 {
1277 u32 devid;
1278 u16 last_bdf = iommu->pci_seg->last_bdf;
1279
1280 for (devid = 0; devid <= last_bdf; ++devid)
1281 iommu_flush_dte(iommu, devid);
1282
1283 iommu_completion_wait(iommu);
1284 }
1285
1286 /*
1287 * This function uses heavy locking and may disable irqs for some time. But
1288 * this is no issue because it is only called during resume.
1289 */
amd_iommu_flush_tlb_all(struct amd_iommu * iommu)1290 static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu)
1291 {
1292 u32 dom_id;
1293 u16 last_bdf = iommu->pci_seg->last_bdf;
1294
1295 for (dom_id = 0; dom_id <= last_bdf; ++dom_id) {
1296 struct iommu_cmd cmd;
1297 build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
1298 dom_id, IOMMU_NO_PASID, false);
1299 iommu_queue_command(iommu, &cmd);
1300 }
1301
1302 iommu_completion_wait(iommu);
1303 }
1304
amd_iommu_flush_tlb_domid(struct amd_iommu * iommu,u32 dom_id)1305 static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id)
1306 {
1307 struct iommu_cmd cmd;
1308
1309 build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
1310 dom_id, IOMMU_NO_PASID, false);
1311 iommu_queue_command(iommu, &cmd);
1312
1313 iommu_completion_wait(iommu);
1314 }
1315
amd_iommu_flush_all(struct amd_iommu * iommu)1316 static void amd_iommu_flush_all(struct amd_iommu *iommu)
1317 {
1318 struct iommu_cmd cmd;
1319
1320 build_inv_all(&cmd);
1321
1322 iommu_queue_command(iommu, &cmd);
1323 iommu_completion_wait(iommu);
1324 }
1325
iommu_flush_irt(struct amd_iommu * iommu,u16 devid)1326 static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)
1327 {
1328 struct iommu_cmd cmd;
1329
1330 build_inv_irt(&cmd, devid);
1331
1332 iommu_queue_command(iommu, &cmd);
1333 }
1334
amd_iommu_flush_irt_all(struct amd_iommu * iommu)1335 static void amd_iommu_flush_irt_all(struct amd_iommu *iommu)
1336 {
1337 u32 devid;
1338 u16 last_bdf = iommu->pci_seg->last_bdf;
1339
1340 if (iommu->irtcachedis_enabled)
1341 return;
1342
1343 for (devid = 0; devid <= last_bdf; devid++)
1344 iommu_flush_irt(iommu, devid);
1345
1346 iommu_completion_wait(iommu);
1347 }
1348
amd_iommu_flush_all_caches(struct amd_iommu * iommu)1349 void amd_iommu_flush_all_caches(struct amd_iommu *iommu)
1350 {
1351 if (check_feature(FEATURE_IA)) {
1352 amd_iommu_flush_all(iommu);
1353 } else {
1354 amd_iommu_flush_dte_all(iommu);
1355 amd_iommu_flush_irt_all(iommu);
1356 amd_iommu_flush_tlb_all(iommu);
1357 }
1358 }
1359
1360 /*
1361 * Command send function for flushing on-device TLB
1362 */
device_flush_iotlb(struct iommu_dev_data * dev_data,u64 address,size_t size,ioasid_t pasid,bool gn)1363 static int device_flush_iotlb(struct iommu_dev_data *dev_data, u64 address,
1364 size_t size, ioasid_t pasid, bool gn)
1365 {
1366 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
1367 struct iommu_cmd cmd;
1368 int qdep = dev_data->ats_qdep;
1369
1370 build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address,
1371 size, pasid, gn);
1372
1373 return iommu_queue_command(iommu, &cmd);
1374 }
1375
device_flush_dte_alias(struct pci_dev * pdev,u16 alias,void * data)1376 static int device_flush_dte_alias(struct pci_dev *pdev, u16 alias, void *data)
1377 {
1378 struct amd_iommu *iommu = data;
1379
1380 return iommu_flush_dte(iommu, alias);
1381 }
1382
1383 /*
1384 * Command send function for invalidating a device table entry
1385 */
device_flush_dte(struct iommu_dev_data * dev_data)1386 static int device_flush_dte(struct iommu_dev_data *dev_data)
1387 {
1388 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
1389 struct pci_dev *pdev = NULL;
1390 struct amd_iommu_pci_seg *pci_seg;
1391 u16 alias;
1392 int ret;
1393
1394 if (dev_is_pci(dev_data->dev))
1395 pdev = to_pci_dev(dev_data->dev);
1396
1397 if (pdev)
1398 ret = pci_for_each_dma_alias(pdev,
1399 device_flush_dte_alias, iommu);
1400 else
1401 ret = iommu_flush_dte(iommu, dev_data->devid);
1402 if (ret)
1403 return ret;
1404
1405 pci_seg = iommu->pci_seg;
1406 alias = pci_seg->alias_table[dev_data->devid];
1407 if (alias != dev_data->devid) {
1408 ret = iommu_flush_dte(iommu, alias);
1409 if (ret)
1410 return ret;
1411 }
1412
1413 if (dev_data->ats_enabled) {
1414 /* Invalidate the entire contents of an IOTLB */
1415 ret = device_flush_iotlb(dev_data, 0, ~0UL,
1416 IOMMU_NO_PASID, false);
1417 }
1418
1419 return ret;
1420 }
1421
domain_flush_pages_v2(struct protection_domain * pdom,u64 address,size_t size)1422 static int domain_flush_pages_v2(struct protection_domain *pdom,
1423 u64 address, size_t size)
1424 {
1425 struct iommu_dev_data *dev_data;
1426 struct iommu_cmd cmd;
1427 int ret = 0;
1428
1429 list_for_each_entry(dev_data, &pdom->dev_list, list) {
1430 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);
1431 u16 domid = dev_data->gcr3_info.domid;
1432
1433 build_inv_iommu_pages(&cmd, address, size,
1434 domid, IOMMU_NO_PASID, true);
1435
1436 ret |= iommu_queue_command(iommu, &cmd);
1437 }
1438
1439 return ret;
1440 }
1441
domain_flush_pages_v1(struct protection_domain * pdom,u64 address,size_t size)1442 static int domain_flush_pages_v1(struct protection_domain *pdom,
1443 u64 address, size_t size)
1444 {
1445 struct iommu_cmd cmd;
1446 int ret = 0, i;
1447
1448 build_inv_iommu_pages(&cmd, address, size,
1449 pdom->id, IOMMU_NO_PASID, false);
1450
1451 for (i = 0; i < amd_iommu_get_num_iommus(); ++i) {
1452 if (!pdom->dev_iommu[i])
1453 continue;
1454
1455 /*
1456 * Devices of this domain are behind this IOMMU
1457 * We need a TLB flush
1458 */
1459 ret |= iommu_queue_command(amd_iommus[i], &cmd);
1460 }
1461
1462 return ret;
1463 }
1464
1465 /*
1466 * TLB invalidation function which is called from the mapping functions.
1467 * It flushes range of PTEs of the domain.
1468 */
__domain_flush_pages(struct protection_domain * domain,u64 address,size_t size)1469 static void __domain_flush_pages(struct protection_domain *domain,
1470 u64 address, size_t size)
1471 {
1472 struct iommu_dev_data *dev_data;
1473 int ret = 0;
1474 ioasid_t pasid = IOMMU_NO_PASID;
1475 bool gn = false;
1476
1477 if (pdom_is_v2_pgtbl_mode(domain)) {
1478 gn = true;
1479 ret = domain_flush_pages_v2(domain, address, size);
1480 } else {
1481 ret = domain_flush_pages_v1(domain, address, size);
1482 }
1483
1484 list_for_each_entry(dev_data, &domain->dev_list, list) {
1485
1486 if (!dev_data->ats_enabled)
1487 continue;
1488
1489 ret |= device_flush_iotlb(dev_data, address, size, pasid, gn);
1490 }
1491
1492 WARN_ON(ret);
1493 }
1494
amd_iommu_domain_flush_pages(struct protection_domain * domain,u64 address,size_t size)1495 void amd_iommu_domain_flush_pages(struct protection_domain *domain,
1496 u64 address, size_t size)
1497 {
1498 if (likely(!amd_iommu_np_cache)) {
1499 __domain_flush_pages(domain, address, size);
1500
1501 /* Wait until IOMMU TLB and all device IOTLB flushes are complete */
1502 domain_flush_complete(domain);
1503
1504 return;
1505 }
1506
1507 /*
1508 * When NpCache is on, we infer that we run in a VM and use a vIOMMU.
1509 * In such setups it is best to avoid flushes of ranges which are not
1510 * naturally aligned, since it would lead to flushes of unmodified
1511 * PTEs. Such flushes would require the hypervisor to do more work than
1512 * necessary. Therefore, perform repeated flushes of aligned ranges
1513 * until you cover the range. Each iteration flushes the smaller
1514 * between the natural alignment of the address that we flush and the
1515 * greatest naturally aligned region that fits in the range.
1516 */
1517 while (size != 0) {
1518 int addr_alignment = __ffs(address);
1519 int size_alignment = __fls(size);
1520 int min_alignment;
1521 size_t flush_size;
1522
1523 /*
1524 * size is always non-zero, but address might be zero, causing
1525 * addr_alignment to be negative. As the casting of the
1526 * argument in __ffs(address) to long might trim the high bits
1527 * of the address on x86-32, cast to long when doing the check.
1528 */
1529 if (likely((unsigned long)address != 0))
1530 min_alignment = min(addr_alignment, size_alignment);
1531 else
1532 min_alignment = size_alignment;
1533
1534 flush_size = 1ul << min_alignment;
1535
1536 __domain_flush_pages(domain, address, flush_size);
1537 address += flush_size;
1538 size -= flush_size;
1539 }
1540
1541 /* Wait until IOMMU TLB and all device IOTLB flushes are complete */
1542 domain_flush_complete(domain);
1543 }
1544
1545 /* Flush the whole IO/TLB for a given protection domain - including PDE */
amd_iommu_domain_flush_all(struct protection_domain * domain)1546 static void amd_iommu_domain_flush_all(struct protection_domain *domain)
1547 {
1548 amd_iommu_domain_flush_pages(domain, 0,
1549 CMD_INV_IOMMU_ALL_PAGES_ADDRESS);
1550 }
1551
amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data * dev_data,ioasid_t pasid,u64 address,size_t size)1552 void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data,
1553 ioasid_t pasid, u64 address, size_t size)
1554 {
1555 struct iommu_cmd cmd;
1556 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);
1557
1558 build_inv_iommu_pages(&cmd, address, size,
1559 dev_data->gcr3_info.domid, pasid, true);
1560 iommu_queue_command(iommu, &cmd);
1561
1562 if (dev_data->ats_enabled)
1563 device_flush_iotlb(dev_data, address, size, pasid, true);
1564
1565 iommu_completion_wait(iommu);
1566 }
1567
dev_flush_pasid_all(struct iommu_dev_data * dev_data,ioasid_t pasid)1568 static void dev_flush_pasid_all(struct iommu_dev_data *dev_data,
1569 ioasid_t pasid)
1570 {
1571 amd_iommu_dev_flush_pasid_pages(dev_data, pasid, 0,
1572 CMD_INV_IOMMU_ALL_PAGES_ADDRESS);
1573 }
1574
1575 /* Flush the not present cache if it exists */
domain_flush_np_cache(struct protection_domain * domain,dma_addr_t iova,size_t size)1576 static void domain_flush_np_cache(struct protection_domain *domain,
1577 dma_addr_t iova, size_t size)
1578 {
1579 if (unlikely(amd_iommu_np_cache)) {
1580 unsigned long flags;
1581
1582 spin_lock_irqsave(&domain->lock, flags);
1583 amd_iommu_domain_flush_pages(domain, iova, size);
1584 spin_unlock_irqrestore(&domain->lock, flags);
1585 }
1586 }
1587
1588
1589 /*
1590 * This function flushes the DTEs for all devices in domain
1591 */
amd_iommu_update_and_flush_device_table(struct protection_domain * domain)1592 void amd_iommu_update_and_flush_device_table(struct protection_domain *domain)
1593 {
1594 struct iommu_dev_data *dev_data;
1595
1596 list_for_each_entry(dev_data, &domain->dev_list, list) {
1597 struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev);
1598
1599 set_dte_entry(iommu, dev_data);
1600 clone_aliases(iommu, dev_data->dev);
1601 }
1602
1603 list_for_each_entry(dev_data, &domain->dev_list, list)
1604 device_flush_dte(dev_data);
1605
1606 domain_flush_complete(domain);
1607 }
1608
amd_iommu_domain_update(struct protection_domain * domain)1609 void amd_iommu_domain_update(struct protection_domain *domain)
1610 {
1611 /* Update device table */
1612 amd_iommu_update_and_flush_device_table(domain);
1613
1614 /* Flush domain TLB(s) and wait for completion */
1615 amd_iommu_domain_flush_all(domain);
1616 }
1617
amd_iommu_complete_ppr(struct device * dev,u32 pasid,int status,int tag)1618 int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag)
1619 {
1620 struct iommu_dev_data *dev_data;
1621 struct amd_iommu *iommu;
1622 struct iommu_cmd cmd;
1623
1624 dev_data = dev_iommu_priv_get(dev);
1625 iommu = get_amd_iommu_from_dev(dev);
1626
1627 build_complete_ppr(&cmd, dev_data->devid, pasid, status,
1628 tag, dev_data->pri_tlp);
1629
1630 return iommu_queue_command(iommu, &cmd);
1631 }
1632
1633 /****************************************************************************
1634 *
1635 * The next functions belong to the domain allocation. A domain is
1636 * allocated for every IOMMU as the default domain. If device isolation
1637 * is enabled, every device get its own domain. The most important thing
1638 * about domains is the page table mapping the DMA address space they
1639 * contain.
1640 *
1641 ****************************************************************************/
1642
domain_id_alloc(void)1643 static u16 domain_id_alloc(void)
1644 {
1645 unsigned long flags;
1646 int id;
1647
1648 spin_lock_irqsave(&pd_bitmap_lock, flags);
1649 id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
1650 BUG_ON(id == 0);
1651 if (id > 0 && id < MAX_DOMAIN_ID)
1652 __set_bit(id, amd_iommu_pd_alloc_bitmap);
1653 else
1654 id = 0;
1655 spin_unlock_irqrestore(&pd_bitmap_lock, flags);
1656
1657 return id;
1658 }
1659
domain_id_free(int id)1660 static void domain_id_free(int id)
1661 {
1662 unsigned long flags;
1663
1664 spin_lock_irqsave(&pd_bitmap_lock, flags);
1665 if (id > 0 && id < MAX_DOMAIN_ID)
1666 __clear_bit(id, amd_iommu_pd_alloc_bitmap);
1667 spin_unlock_irqrestore(&pd_bitmap_lock, flags);
1668 }
1669
free_gcr3_tbl_level1(u64 * tbl)1670 static void free_gcr3_tbl_level1(u64 *tbl)
1671 {
1672 u64 *ptr;
1673 int i;
1674
1675 for (i = 0; i < 512; ++i) {
1676 if (!(tbl[i] & GCR3_VALID))
1677 continue;
1678
1679 ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK);
1680
1681 iommu_free_page(ptr);
1682 }
1683 }
1684
free_gcr3_tbl_level2(u64 * tbl)1685 static void free_gcr3_tbl_level2(u64 *tbl)
1686 {
1687 u64 *ptr;
1688 int i;
1689
1690 for (i = 0; i < 512; ++i) {
1691 if (!(tbl[i] & GCR3_VALID))
1692 continue;
1693
1694 ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK);
1695
1696 free_gcr3_tbl_level1(ptr);
1697 }
1698 }
1699
free_gcr3_table(struct gcr3_tbl_info * gcr3_info)1700 static void free_gcr3_table(struct gcr3_tbl_info *gcr3_info)
1701 {
1702 if (gcr3_info->glx == 2)
1703 free_gcr3_tbl_level2(gcr3_info->gcr3_tbl);
1704 else if (gcr3_info->glx == 1)
1705 free_gcr3_tbl_level1(gcr3_info->gcr3_tbl);
1706 else
1707 WARN_ON_ONCE(gcr3_info->glx != 0);
1708
1709 gcr3_info->glx = 0;
1710
1711 /* Free per device domain ID */
1712 domain_id_free(gcr3_info->domid);
1713
1714 iommu_free_page(gcr3_info->gcr3_tbl);
1715 gcr3_info->gcr3_tbl = NULL;
1716 }
1717
1718 /*
1719 * Number of GCR3 table levels required. Level must be 4-Kbyte
1720 * page and can contain up to 512 entries.
1721 */
get_gcr3_levels(int pasids)1722 static int get_gcr3_levels(int pasids)
1723 {
1724 int levels;
1725
1726 if (pasids == -1)
1727 return amd_iommu_max_glx_val;
1728
1729 levels = get_count_order(pasids);
1730
1731 return levels ? (DIV_ROUND_UP(levels, 9) - 1) : levels;
1732 }
1733
setup_gcr3_table(struct gcr3_tbl_info * gcr3_info,struct amd_iommu * iommu,int pasids)1734 static int setup_gcr3_table(struct gcr3_tbl_info *gcr3_info,
1735 struct amd_iommu *iommu, int pasids)
1736 {
1737 int levels = get_gcr3_levels(pasids);
1738 int nid = iommu ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;
1739
1740 if (levels > amd_iommu_max_glx_val)
1741 return -EINVAL;
1742
1743 if (gcr3_info->gcr3_tbl)
1744 return -EBUSY;
1745
1746 /* Allocate per device domain ID */
1747 gcr3_info->domid = domain_id_alloc();
1748
1749 gcr3_info->gcr3_tbl = iommu_alloc_page_node(nid, GFP_ATOMIC);
1750 if (gcr3_info->gcr3_tbl == NULL) {
1751 domain_id_free(gcr3_info->domid);
1752 return -ENOMEM;
1753 }
1754
1755 gcr3_info->glx = levels;
1756
1757 return 0;
1758 }
1759
__get_gcr3_pte(struct gcr3_tbl_info * gcr3_info,ioasid_t pasid,bool alloc)1760 static u64 *__get_gcr3_pte(struct gcr3_tbl_info *gcr3_info,
1761 ioasid_t pasid, bool alloc)
1762 {
1763 int index;
1764 u64 *pte;
1765 u64 *root = gcr3_info->gcr3_tbl;
1766 int level = gcr3_info->glx;
1767
1768 while (true) {
1769
1770 index = (pasid >> (9 * level)) & 0x1ff;
1771 pte = &root[index];
1772
1773 if (level == 0)
1774 break;
1775
1776 if (!(*pte & GCR3_VALID)) {
1777 if (!alloc)
1778 return NULL;
1779
1780 root = (void *)get_zeroed_page(GFP_ATOMIC);
1781 if (root == NULL)
1782 return NULL;
1783
1784 *pte = iommu_virt_to_phys(root) | GCR3_VALID;
1785 }
1786
1787 root = iommu_phys_to_virt(*pte & PAGE_MASK);
1788
1789 level -= 1;
1790 }
1791
1792 return pte;
1793 }
1794
update_gcr3(struct iommu_dev_data * dev_data,ioasid_t pasid,unsigned long gcr3,bool set)1795 static int update_gcr3(struct iommu_dev_data *dev_data,
1796 ioasid_t pasid, unsigned long gcr3, bool set)
1797 {
1798 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
1799 u64 *pte;
1800
1801 pte = __get_gcr3_pte(gcr3_info, pasid, true);
1802 if (pte == NULL)
1803 return -ENOMEM;
1804
1805 if (set)
1806 *pte = (gcr3 & PAGE_MASK) | GCR3_VALID;
1807 else
1808 *pte = 0;
1809
1810 dev_flush_pasid_all(dev_data, pasid);
1811 return 0;
1812 }
1813
amd_iommu_set_gcr3(struct iommu_dev_data * dev_data,ioasid_t pasid,unsigned long gcr3)1814 int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid,
1815 unsigned long gcr3)
1816 {
1817 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
1818 int ret;
1819
1820 iommu_group_mutex_assert(dev_data->dev);
1821
1822 ret = update_gcr3(dev_data, pasid, gcr3, true);
1823 if (ret)
1824 return ret;
1825
1826 gcr3_info->pasid_cnt++;
1827 return ret;
1828 }
1829
amd_iommu_clear_gcr3(struct iommu_dev_data * dev_data,ioasid_t pasid)1830 int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid)
1831 {
1832 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
1833 int ret;
1834
1835 iommu_group_mutex_assert(dev_data->dev);
1836
1837 ret = update_gcr3(dev_data, pasid, 0, false);
1838 if (ret)
1839 return ret;
1840
1841 gcr3_info->pasid_cnt--;
1842 return ret;
1843 }
1844
set_dte_entry(struct amd_iommu * iommu,struct iommu_dev_data * dev_data)1845 static void set_dte_entry(struct amd_iommu *iommu,
1846 struct iommu_dev_data *dev_data)
1847 {
1848 u64 pte_root = 0;
1849 u64 flags = 0;
1850 u32 old_domid;
1851 u16 devid = dev_data->devid;
1852 u16 domid;
1853 struct protection_domain *domain = dev_data->domain;
1854 struct dev_table_entry *dev_table = get_dev_table(iommu);
1855 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
1856
1857 if (gcr3_info && gcr3_info->gcr3_tbl)
1858 domid = dev_data->gcr3_info.domid;
1859 else
1860 domid = domain->id;
1861
1862 if (domain->iop.mode != PAGE_MODE_NONE)
1863 pte_root = iommu_virt_to_phys(domain->iop.root);
1864
1865 pte_root |= (domain->iop.mode & DEV_ENTRY_MODE_MASK)
1866 << DEV_ENTRY_MODE_SHIFT;
1867
1868 pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V;
1869
1870 /*
1871 * When SNP is enabled, Only set TV bit when IOMMU
1872 * page translation is in use.
1873 */
1874 if (!amd_iommu_snp_en || (domid != 0))
1875 pte_root |= DTE_FLAG_TV;
1876
1877 flags = dev_table[devid].data[1];
1878
1879 if (dev_data->ats_enabled)
1880 flags |= DTE_FLAG_IOTLB;
1881
1882 if (dev_data->ppr)
1883 pte_root |= 1ULL << DEV_ENTRY_PPR;
1884
1885 if (domain->dirty_tracking)
1886 pte_root |= DTE_FLAG_HAD;
1887
1888 if (gcr3_info && gcr3_info->gcr3_tbl) {
1889 u64 gcr3 = iommu_virt_to_phys(gcr3_info->gcr3_tbl);
1890 u64 glx = gcr3_info->glx;
1891 u64 tmp;
1892
1893 pte_root |= DTE_FLAG_GV;
1894 pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;
1895
1896 /* First mask out possible old values for GCR3 table */
1897 tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
1898 flags &= ~tmp;
1899
1900 tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
1901 flags &= ~tmp;
1902
1903 /* Encode GCR3 table into DTE */
1904 tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
1905 pte_root |= tmp;
1906
1907 tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
1908 flags |= tmp;
1909
1910 tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
1911 flags |= tmp;
1912
1913 if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL) {
1914 dev_table[devid].data[2] |=
1915 ((u64)GUEST_PGTABLE_5_LEVEL << DTE_GPT_LEVEL_SHIFT);
1916 }
1917
1918 /* GIOV is supported with V2 page table mode only */
1919 if (pdom_is_v2_pgtbl_mode(domain))
1920 pte_root |= DTE_FLAG_GIOV;
1921 }
1922
1923 flags &= ~DEV_DOMID_MASK;
1924 flags |= domid;
1925
1926 old_domid = dev_table[devid].data[1] & DEV_DOMID_MASK;
1927 dev_table[devid].data[1] = flags;
1928 dev_table[devid].data[0] = pte_root;
1929
1930 /*
1931 * A kdump kernel might be replacing a domain ID that was copied from
1932 * the previous kernel--if so, it needs to flush the translation cache
1933 * entries for the old domain ID that is being overwritten
1934 */
1935 if (old_domid) {
1936 amd_iommu_flush_tlb_domid(iommu, old_domid);
1937 }
1938 }
1939
clear_dte_entry(struct amd_iommu * iommu,u16 devid)1940 static void clear_dte_entry(struct amd_iommu *iommu, u16 devid)
1941 {
1942 struct dev_table_entry *dev_table = get_dev_table(iommu);
1943
1944 /* remove entry from the device table seen by the hardware */
1945 dev_table[devid].data[0] = DTE_FLAG_V;
1946
1947 if (!amd_iommu_snp_en)
1948 dev_table[devid].data[0] |= DTE_FLAG_TV;
1949
1950 dev_table[devid].data[1] &= DTE_FLAG_MASK;
1951
1952 amd_iommu_apply_erratum_63(iommu, devid);
1953 }
1954
1955 /* Update and flush DTE for the given device */
dev_update_dte(struct iommu_dev_data * dev_data,bool set)1956 static void dev_update_dte(struct iommu_dev_data *dev_data, bool set)
1957 {
1958 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);
1959
1960 if (set)
1961 set_dte_entry(iommu, dev_data);
1962 else
1963 clear_dte_entry(iommu, dev_data->devid);
1964
1965 clone_aliases(iommu, dev_data->dev);
1966 device_flush_dte(dev_data);
1967 iommu_completion_wait(iommu);
1968 }
1969
1970 /*
1971 * If domain is SVA capable then initialize GCR3 table. Also if domain is
1972 * in v2 page table mode then update GCR3[0].
1973 */
init_gcr3_table(struct iommu_dev_data * dev_data,struct protection_domain * pdom)1974 static int init_gcr3_table(struct iommu_dev_data *dev_data,
1975 struct protection_domain *pdom)
1976 {
1977 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
1978 int max_pasids = dev_data->max_pasids;
1979 int ret = 0;
1980
1981 /*
1982 * If domain is in pt mode then setup GCR3 table only if device
1983 * is PASID capable
1984 */
1985 if (pdom_is_in_pt_mode(pdom) && !pdev_pasid_supported(dev_data))
1986 return ret;
1987
1988 /*
1989 * By default, setup GCR3 table to support MAX PASIDs
1990 * supported by the device/IOMMU.
1991 */
1992 ret = setup_gcr3_table(&dev_data->gcr3_info, iommu,
1993 max_pasids > 0 ? max_pasids : 1);
1994 if (ret)
1995 return ret;
1996
1997 /* Setup GCR3[0] only if domain is setup with v2 page table mode */
1998 if (!pdom_is_v2_pgtbl_mode(pdom))
1999 return ret;
2000
2001 ret = update_gcr3(dev_data, 0, iommu_virt_to_phys(pdom->iop.pgd), true);
2002 if (ret)
2003 free_gcr3_table(&dev_data->gcr3_info);
2004
2005 return ret;
2006 }
2007
destroy_gcr3_table(struct iommu_dev_data * dev_data,struct protection_domain * pdom)2008 static void destroy_gcr3_table(struct iommu_dev_data *dev_data,
2009 struct protection_domain *pdom)
2010 {
2011 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
2012
2013 if (pdom_is_v2_pgtbl_mode(pdom))
2014 update_gcr3(dev_data, 0, 0, false);
2015
2016 if (gcr3_info->gcr3_tbl == NULL)
2017 return;
2018
2019 free_gcr3_table(gcr3_info);
2020 }
2021
do_attach(struct iommu_dev_data * dev_data,struct protection_domain * domain)2022 static int do_attach(struct iommu_dev_data *dev_data,
2023 struct protection_domain *domain)
2024 {
2025 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
2026 struct io_pgtable_cfg *cfg = &domain->iop.pgtbl.cfg;
2027 int ret = 0;
2028
2029 /* Update data structures */
2030 dev_data->domain = domain;
2031 list_add(&dev_data->list, &domain->dev_list);
2032
2033 /* Update NUMA Node ID */
2034 if (cfg->amd.nid == NUMA_NO_NODE)
2035 cfg->amd.nid = dev_to_node(dev_data->dev);
2036
2037 /* Do reference counting */
2038 domain->dev_iommu[iommu->index] += 1;
2039 domain->dev_cnt += 1;
2040
2041 /* Setup GCR3 table */
2042 if (pdom_is_sva_capable(domain)) {
2043 ret = init_gcr3_table(dev_data, domain);
2044 if (ret)
2045 return ret;
2046 }
2047
2048 return ret;
2049 }
2050
do_detach(struct iommu_dev_data * dev_data)2051 static void do_detach(struct iommu_dev_data *dev_data)
2052 {
2053 struct protection_domain *domain = dev_data->domain;
2054 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
2055
2056 /* Clear DTE and flush the entry */
2057 dev_update_dte(dev_data, false);
2058
2059 /* Flush IOTLB and wait for the flushes to finish */
2060 amd_iommu_domain_flush_all(domain);
2061
2062 /* Clear GCR3 table */
2063 if (pdom_is_sva_capable(domain))
2064 destroy_gcr3_table(dev_data, domain);
2065
2066 /* Update data structures */
2067 dev_data->domain = NULL;
2068 list_del(&dev_data->list);
2069
2070 /* decrease reference counters - needs to happen after the flushes */
2071 domain->dev_iommu[iommu->index] -= 1;
2072 domain->dev_cnt -= 1;
2073 }
2074
2075 /*
2076 * If a device is not yet associated with a domain, this function makes the
2077 * device visible in the domain
2078 */
attach_device(struct device * dev,struct protection_domain * domain)2079 static int attach_device(struct device *dev,
2080 struct protection_domain *domain)
2081 {
2082 struct iommu_dev_data *dev_data;
2083 unsigned long flags;
2084 int ret = 0;
2085
2086 spin_lock_irqsave(&domain->lock, flags);
2087
2088 dev_data = dev_iommu_priv_get(dev);
2089
2090 spin_lock(&dev_data->lock);
2091
2092 if (dev_data->domain != NULL) {
2093 ret = -EBUSY;
2094 goto out;
2095 }
2096
2097 ret = do_attach(dev_data, domain);
2098
2099 out:
2100 spin_unlock(&dev_data->lock);
2101
2102 spin_unlock_irqrestore(&domain->lock, flags);
2103
2104 return ret;
2105 }
2106
2107 /*
2108 * Removes a device from a protection domain (with devtable_lock held)
2109 */
detach_device(struct device * dev)2110 static void detach_device(struct device *dev)
2111 {
2112 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
2113 struct protection_domain *domain = dev_data->domain;
2114 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
2115 unsigned long flags;
2116 bool ppr = dev_data->ppr;
2117
2118 spin_lock_irqsave(&domain->lock, flags);
2119
2120 spin_lock(&dev_data->lock);
2121
2122 /*
2123 * First check if the device is still attached. It might already
2124 * be detached from its domain because the generic
2125 * iommu_detach_group code detached it and we try again here in
2126 * our alias handling.
2127 */
2128 if (WARN_ON(!dev_data->domain))
2129 goto out;
2130
2131 if (ppr) {
2132 iopf_queue_flush_dev(dev);
2133
2134 /* Updated here so that it gets reflected in DTE */
2135 dev_data->ppr = false;
2136 }
2137
2138 do_detach(dev_data);
2139
2140 out:
2141 spin_unlock(&dev_data->lock);
2142
2143 spin_unlock_irqrestore(&domain->lock, flags);
2144
2145 /* Remove IOPF handler */
2146 if (ppr)
2147 amd_iommu_iopf_remove_device(iommu, dev_data);
2148
2149 if (dev_is_pci(dev))
2150 pdev_disable_caps(to_pci_dev(dev));
2151
2152 }
2153
amd_iommu_probe_device(struct device * dev)2154 static struct iommu_device *amd_iommu_probe_device(struct device *dev)
2155 {
2156 struct iommu_device *iommu_dev;
2157 struct amd_iommu *iommu;
2158 struct iommu_dev_data *dev_data;
2159 int ret;
2160
2161 if (!check_device(dev))
2162 return ERR_PTR(-ENODEV);
2163
2164 iommu = rlookup_amd_iommu(dev);
2165 if (!iommu)
2166 return ERR_PTR(-ENODEV);
2167
2168 /* Not registered yet? */
2169 if (!iommu->iommu.ops)
2170 return ERR_PTR(-ENODEV);
2171
2172 if (dev_iommu_priv_get(dev))
2173 return &iommu->iommu;
2174
2175 ret = iommu_init_device(iommu, dev);
2176 if (ret) {
2177 dev_err(dev, "Failed to initialize - trying to proceed anyway\n");
2178 iommu_dev = ERR_PTR(ret);
2179 iommu_ignore_device(iommu, dev);
2180 goto out_err;
2181 }
2182
2183 amd_iommu_set_pci_msi_domain(dev, iommu);
2184 iommu_dev = &iommu->iommu;
2185
2186 /*
2187 * If IOMMU and device supports PASID then it will contain max
2188 * supported PASIDs, else it will be zero.
2189 */
2190 dev_data = dev_iommu_priv_get(dev);
2191 if (amd_iommu_pasid_supported() && dev_is_pci(dev) &&
2192 pdev_pasid_supported(dev_data)) {
2193 dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids,
2194 pci_max_pasids(to_pci_dev(dev)));
2195 }
2196
2197 out_err:
2198 iommu_completion_wait(iommu);
2199
2200 if (dev_is_pci(dev))
2201 pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT);
2202
2203 return iommu_dev;
2204 }
2205
amd_iommu_release_device(struct device * dev)2206 static void amd_iommu_release_device(struct device *dev)
2207 {
2208 struct amd_iommu *iommu;
2209
2210 if (!check_device(dev))
2211 return;
2212
2213 iommu = rlookup_amd_iommu(dev);
2214 if (!iommu)
2215 return;
2216
2217 amd_iommu_uninit_device(dev);
2218 iommu_completion_wait(iommu);
2219 }
2220
amd_iommu_device_group(struct device * dev)2221 static struct iommu_group *amd_iommu_device_group(struct device *dev)
2222 {
2223 if (dev_is_pci(dev))
2224 return pci_device_group(dev);
2225
2226 return acpihid_device_group(dev);
2227 }
2228
2229 /*****************************************************************************
2230 *
2231 * The following functions belong to the exported interface of AMD IOMMU
2232 *
2233 * This interface allows access to lower level functions of the IOMMU
2234 * like protection domain handling and assignement of devices to domains
2235 * which is not possible with the dma_ops interface.
2236 *
2237 *****************************************************************************/
2238
cleanup_domain(struct protection_domain * domain)2239 static void cleanup_domain(struct protection_domain *domain)
2240 {
2241 struct iommu_dev_data *entry;
2242
2243 lockdep_assert_held(&domain->lock);
2244
2245 if (!domain->dev_cnt)
2246 return;
2247
2248 while (!list_empty(&domain->dev_list)) {
2249 entry = list_first_entry(&domain->dev_list,
2250 struct iommu_dev_data, list);
2251 BUG_ON(!entry->domain);
2252 do_detach(entry);
2253 }
2254 WARN_ON(domain->dev_cnt != 0);
2255 }
2256
protection_domain_free(struct protection_domain * domain)2257 void protection_domain_free(struct protection_domain *domain)
2258 {
2259 WARN_ON(!list_empty(&domain->dev_list));
2260 if (domain->domain.type & __IOMMU_DOMAIN_PAGING)
2261 free_io_pgtable_ops(&domain->iop.pgtbl.ops);
2262 domain_id_free(domain->id);
2263 kfree(domain);
2264 }
2265
protection_domain_alloc(unsigned int type,int nid)2266 struct protection_domain *protection_domain_alloc(unsigned int type, int nid)
2267 {
2268 struct io_pgtable_ops *pgtbl_ops;
2269 struct protection_domain *domain;
2270 int pgtable;
2271
2272 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
2273 if (!domain)
2274 return NULL;
2275
2276 domain->id = domain_id_alloc();
2277 if (!domain->id)
2278 goto err_free;
2279
2280 spin_lock_init(&domain->lock);
2281 INIT_LIST_HEAD(&domain->dev_list);
2282 INIT_LIST_HEAD(&domain->dev_data_list);
2283 domain->iop.pgtbl.cfg.amd.nid = nid;
2284
2285 switch (type) {
2286 /* No need to allocate io pgtable ops in passthrough mode */
2287 case IOMMU_DOMAIN_IDENTITY:
2288 case IOMMU_DOMAIN_SVA:
2289 return domain;
2290 case IOMMU_DOMAIN_DMA:
2291 pgtable = amd_iommu_pgtable;
2292 break;
2293 /*
2294 * Force IOMMU v1 page table when allocating
2295 * domain for pass-through devices.
2296 */
2297 case IOMMU_DOMAIN_UNMANAGED:
2298 pgtable = AMD_IOMMU_V1;
2299 break;
2300 default:
2301 goto err_id;
2302 }
2303
2304 switch (pgtable) {
2305 case AMD_IOMMU_V1:
2306 domain->pd_mode = PD_MODE_V1;
2307 break;
2308 case AMD_IOMMU_V2:
2309 domain->pd_mode = PD_MODE_V2;
2310 break;
2311 default:
2312 goto err_id;
2313 }
2314
2315 pgtbl_ops =
2316 alloc_io_pgtable_ops(pgtable, &domain->iop.pgtbl.cfg, domain);
2317 if (!pgtbl_ops)
2318 goto err_id;
2319
2320 return domain;
2321 err_id:
2322 domain_id_free(domain->id);
2323 err_free:
2324 kfree(domain);
2325 return NULL;
2326 }
2327
dma_max_address(void)2328 static inline u64 dma_max_address(void)
2329 {
2330 if (amd_iommu_pgtable == AMD_IOMMU_V1)
2331 return ~0ULL;
2332
2333 /* V2 with 4/5 level page table */
2334 return ((1ULL << PM_LEVEL_SHIFT(amd_iommu_gpt_level)) - 1);
2335 }
2336
amd_iommu_hd_support(struct amd_iommu * iommu)2337 static bool amd_iommu_hd_support(struct amd_iommu *iommu)
2338 {
2339 return iommu && (iommu->features & FEATURE_HDSUP);
2340 }
2341
do_iommu_domain_alloc(unsigned int type,struct device * dev,u32 flags)2342 static struct iommu_domain *do_iommu_domain_alloc(unsigned int type,
2343 struct device *dev, u32 flags)
2344 {
2345 bool dirty_tracking = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
2346 struct protection_domain *domain;
2347 struct amd_iommu *iommu = NULL;
2348
2349 if (dev)
2350 iommu = get_amd_iommu_from_dev(dev);
2351
2352 /*
2353 * Since DTE[Mode]=0 is prohibited on SNP-enabled system,
2354 * default to use IOMMU_DOMAIN_DMA[_FQ].
2355 */
2356 if (amd_iommu_snp_en && (type == IOMMU_DOMAIN_IDENTITY))
2357 return ERR_PTR(-EINVAL);
2358
2359 if (dirty_tracking && !amd_iommu_hd_support(iommu))
2360 return ERR_PTR(-EOPNOTSUPP);
2361
2362 domain = protection_domain_alloc(type,
2363 dev ? dev_to_node(dev) : NUMA_NO_NODE);
2364 if (!domain)
2365 return ERR_PTR(-ENOMEM);
2366
2367 domain->domain.geometry.aperture_start = 0;
2368 domain->domain.geometry.aperture_end = dma_max_address();
2369 domain->domain.geometry.force_aperture = true;
2370 domain->domain.pgsize_bitmap = domain->iop.pgtbl.cfg.pgsize_bitmap;
2371
2372 if (iommu) {
2373 domain->domain.type = type;
2374 domain->domain.ops = iommu->iommu.ops->default_domain_ops;
2375
2376 if (dirty_tracking)
2377 domain->domain.dirty_ops = &amd_dirty_ops;
2378 }
2379
2380 return &domain->domain;
2381 }
2382
amd_iommu_domain_alloc(unsigned int type)2383 static struct iommu_domain *amd_iommu_domain_alloc(unsigned int type)
2384 {
2385 struct iommu_domain *domain;
2386
2387 domain = do_iommu_domain_alloc(type, NULL, 0);
2388 if (IS_ERR(domain))
2389 return NULL;
2390
2391 return domain;
2392 }
2393
2394 static struct iommu_domain *
amd_iommu_domain_alloc_user(struct device * dev,u32 flags,struct iommu_domain * parent,const struct iommu_user_data * user_data)2395 amd_iommu_domain_alloc_user(struct device *dev, u32 flags,
2396 struct iommu_domain *parent,
2397 const struct iommu_user_data *user_data)
2398
2399 {
2400 unsigned int type = IOMMU_DOMAIN_UNMANAGED;
2401
2402 if ((flags & ~IOMMU_HWPT_ALLOC_DIRTY_TRACKING) || parent || user_data)
2403 return ERR_PTR(-EOPNOTSUPP);
2404
2405 return do_iommu_domain_alloc(type, dev, flags);
2406 }
2407
amd_iommu_domain_free(struct iommu_domain * dom)2408 void amd_iommu_domain_free(struct iommu_domain *dom)
2409 {
2410 struct protection_domain *domain;
2411 unsigned long flags;
2412
2413 domain = to_pdomain(dom);
2414
2415 spin_lock_irqsave(&domain->lock, flags);
2416
2417 cleanup_domain(domain);
2418
2419 spin_unlock_irqrestore(&domain->lock, flags);
2420
2421 protection_domain_free(domain);
2422 }
2423
blocked_domain_attach_device(struct iommu_domain * domain,struct device * dev)2424 static int blocked_domain_attach_device(struct iommu_domain *domain,
2425 struct device *dev)
2426 {
2427 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
2428
2429 if (dev_data->domain)
2430 detach_device(dev);
2431
2432 /* Clear DTE and flush the entry */
2433 spin_lock(&dev_data->lock);
2434 dev_update_dte(dev_data, false);
2435 spin_unlock(&dev_data->lock);
2436
2437 return 0;
2438 }
2439
2440 static struct iommu_domain blocked_domain = {
2441 .type = IOMMU_DOMAIN_BLOCKED,
2442 .ops = &(const struct iommu_domain_ops) {
2443 .attach_dev = blocked_domain_attach_device,
2444 }
2445 };
2446
amd_iommu_attach_device(struct iommu_domain * dom,struct device * dev)2447 static int amd_iommu_attach_device(struct iommu_domain *dom,
2448 struct device *dev)
2449 {
2450 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
2451 struct protection_domain *domain = to_pdomain(dom);
2452 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);
2453 struct pci_dev *pdev;
2454 int ret;
2455
2456 /*
2457 * Skip attach device to domain if new domain is same as
2458 * devices current domain
2459 */
2460 if (dev_data->domain == domain)
2461 return 0;
2462
2463 dev_data->defer_attach = false;
2464
2465 /*
2466 * Restrict to devices with compatible IOMMU hardware support
2467 * when enforcement of dirty tracking is enabled.
2468 */
2469 if (dom->dirty_ops && !amd_iommu_hd_support(iommu))
2470 return -EINVAL;
2471
2472 if (dev_data->domain)
2473 detach_device(dev);
2474
2475 ret = attach_device(dev, domain);
2476
2477 #ifdef CONFIG_IRQ_REMAP
2478 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) {
2479 if (dom->type == IOMMU_DOMAIN_UNMANAGED)
2480 dev_data->use_vapic = 1;
2481 else
2482 dev_data->use_vapic = 0;
2483 }
2484 #endif
2485
2486 pdev = dev_is_pci(dev_data->dev) ? to_pci_dev(dev_data->dev) : NULL;
2487 if (pdev && pdom_is_sva_capable(domain)) {
2488 pdev_enable_caps(pdev);
2489
2490 /*
2491 * Device can continue to function even if IOPF
2492 * enablement failed. Hence in error path just
2493 * disable device PRI support.
2494 */
2495 if (amd_iommu_iopf_add_device(iommu, dev_data))
2496 pdev_disable_cap_pri(pdev);
2497 } else if (pdev) {
2498 pdev_enable_cap_ats(pdev);
2499 }
2500
2501 /* Update device table */
2502 dev_update_dte(dev_data, true);
2503
2504 return ret;
2505 }
2506
amd_iommu_iotlb_sync_map(struct iommu_domain * dom,unsigned long iova,size_t size)2507 static int amd_iommu_iotlb_sync_map(struct iommu_domain *dom,
2508 unsigned long iova, size_t size)
2509 {
2510 struct protection_domain *domain = to_pdomain(dom);
2511 struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops;
2512
2513 if (ops->map_pages)
2514 domain_flush_np_cache(domain, iova, size);
2515 return 0;
2516 }
2517
amd_iommu_map_pages(struct iommu_domain * dom,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int iommu_prot,gfp_t gfp,size_t * mapped)2518 static int amd_iommu_map_pages(struct iommu_domain *dom, unsigned long iova,
2519 phys_addr_t paddr, size_t pgsize, size_t pgcount,
2520 int iommu_prot, gfp_t gfp, size_t *mapped)
2521 {
2522 struct protection_domain *domain = to_pdomain(dom);
2523 struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops;
2524 int prot = 0;
2525 int ret = -EINVAL;
2526
2527 if ((domain->pd_mode == PD_MODE_V1) &&
2528 (domain->iop.mode == PAGE_MODE_NONE))
2529 return -EINVAL;
2530
2531 if (iommu_prot & IOMMU_READ)
2532 prot |= IOMMU_PROT_IR;
2533 if (iommu_prot & IOMMU_WRITE)
2534 prot |= IOMMU_PROT_IW;
2535
2536 if (ops->map_pages) {
2537 ret = ops->map_pages(ops, iova, paddr, pgsize,
2538 pgcount, prot, gfp, mapped);
2539 }
2540
2541 return ret;
2542 }
2543
amd_iommu_iotlb_gather_add_page(struct iommu_domain * domain,struct iommu_iotlb_gather * gather,unsigned long iova,size_t size)2544 static void amd_iommu_iotlb_gather_add_page(struct iommu_domain *domain,
2545 struct iommu_iotlb_gather *gather,
2546 unsigned long iova, size_t size)
2547 {
2548 /*
2549 * AMD's IOMMU can flush as many pages as necessary in a single flush.
2550 * Unless we run in a virtual machine, which can be inferred according
2551 * to whether "non-present cache" is on, it is probably best to prefer
2552 * (potentially) too extensive TLB flushing (i.e., more misses) over
2553 * mutliple TLB flushes (i.e., more flushes). For virtual machines the
2554 * hypervisor needs to synchronize the host IOMMU PTEs with those of
2555 * the guest, and the trade-off is different: unnecessary TLB flushes
2556 * should be avoided.
2557 */
2558 if (amd_iommu_np_cache &&
2559 iommu_iotlb_gather_is_disjoint(gather, iova, size))
2560 iommu_iotlb_sync(domain, gather);
2561
2562 iommu_iotlb_gather_add_range(gather, iova, size);
2563 }
2564
amd_iommu_unmap_pages(struct iommu_domain * dom,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * gather)2565 static size_t amd_iommu_unmap_pages(struct iommu_domain *dom, unsigned long iova,
2566 size_t pgsize, size_t pgcount,
2567 struct iommu_iotlb_gather *gather)
2568 {
2569 struct protection_domain *domain = to_pdomain(dom);
2570 struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops;
2571 size_t r;
2572
2573 if ((domain->pd_mode == PD_MODE_V1) &&
2574 (domain->iop.mode == PAGE_MODE_NONE))
2575 return 0;
2576
2577 r = (ops->unmap_pages) ? ops->unmap_pages(ops, iova, pgsize, pgcount, NULL) : 0;
2578
2579 if (r)
2580 amd_iommu_iotlb_gather_add_page(dom, gather, iova, r);
2581
2582 return r;
2583 }
2584
amd_iommu_iova_to_phys(struct iommu_domain * dom,dma_addr_t iova)2585 static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
2586 dma_addr_t iova)
2587 {
2588 struct protection_domain *domain = to_pdomain(dom);
2589 struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops;
2590
2591 return ops->iova_to_phys(ops, iova);
2592 }
2593
amd_iommu_capable(struct device * dev,enum iommu_cap cap)2594 static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap)
2595 {
2596 switch (cap) {
2597 case IOMMU_CAP_CACHE_COHERENCY:
2598 return true;
2599 case IOMMU_CAP_NOEXEC:
2600 return false;
2601 case IOMMU_CAP_PRE_BOOT_PROTECTION:
2602 return amdr_ivrs_remap_support;
2603 case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
2604 return true;
2605 case IOMMU_CAP_DEFERRED_FLUSH:
2606 return true;
2607 case IOMMU_CAP_DIRTY_TRACKING: {
2608 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);
2609
2610 return amd_iommu_hd_support(iommu);
2611 }
2612 default:
2613 break;
2614 }
2615
2616 return false;
2617 }
2618
amd_iommu_set_dirty_tracking(struct iommu_domain * domain,bool enable)2619 static int amd_iommu_set_dirty_tracking(struct iommu_domain *domain,
2620 bool enable)
2621 {
2622 struct protection_domain *pdomain = to_pdomain(domain);
2623 struct dev_table_entry *dev_table;
2624 struct iommu_dev_data *dev_data;
2625 bool domain_flush = false;
2626 struct amd_iommu *iommu;
2627 unsigned long flags;
2628 u64 pte_root;
2629
2630 spin_lock_irqsave(&pdomain->lock, flags);
2631 if (!(pdomain->dirty_tracking ^ enable)) {
2632 spin_unlock_irqrestore(&pdomain->lock, flags);
2633 return 0;
2634 }
2635
2636 list_for_each_entry(dev_data, &pdomain->dev_list, list) {
2637 iommu = get_amd_iommu_from_dev_data(dev_data);
2638
2639 dev_table = get_dev_table(iommu);
2640 pte_root = dev_table[dev_data->devid].data[0];
2641
2642 pte_root = (enable ? pte_root | DTE_FLAG_HAD :
2643 pte_root & ~DTE_FLAG_HAD);
2644
2645 /* Flush device DTE */
2646 dev_table[dev_data->devid].data[0] = pte_root;
2647 device_flush_dte(dev_data);
2648 domain_flush = true;
2649 }
2650
2651 /* Flush IOTLB to mark IOPTE dirty on the next translation(s) */
2652 if (domain_flush)
2653 amd_iommu_domain_flush_all(pdomain);
2654
2655 pdomain->dirty_tracking = enable;
2656 spin_unlock_irqrestore(&pdomain->lock, flags);
2657
2658 return 0;
2659 }
2660
amd_iommu_read_and_clear_dirty(struct iommu_domain * domain,unsigned long iova,size_t size,unsigned long flags,struct iommu_dirty_bitmap * dirty)2661 static int amd_iommu_read_and_clear_dirty(struct iommu_domain *domain,
2662 unsigned long iova, size_t size,
2663 unsigned long flags,
2664 struct iommu_dirty_bitmap *dirty)
2665 {
2666 struct protection_domain *pdomain = to_pdomain(domain);
2667 struct io_pgtable_ops *ops = &pdomain->iop.pgtbl.ops;
2668 unsigned long lflags;
2669
2670 if (!ops || !ops->read_and_clear_dirty)
2671 return -EOPNOTSUPP;
2672
2673 spin_lock_irqsave(&pdomain->lock, lflags);
2674 if (!pdomain->dirty_tracking && dirty->bitmap) {
2675 spin_unlock_irqrestore(&pdomain->lock, lflags);
2676 return -EINVAL;
2677 }
2678 spin_unlock_irqrestore(&pdomain->lock, lflags);
2679
2680 return ops->read_and_clear_dirty(ops, iova, size, flags, dirty);
2681 }
2682
amd_iommu_get_resv_regions(struct device * dev,struct list_head * head)2683 static void amd_iommu_get_resv_regions(struct device *dev,
2684 struct list_head *head)
2685 {
2686 struct iommu_resv_region *region;
2687 struct unity_map_entry *entry;
2688 struct amd_iommu *iommu;
2689 struct amd_iommu_pci_seg *pci_seg;
2690 int devid, sbdf;
2691
2692 sbdf = get_device_sbdf_id(dev);
2693 if (sbdf < 0)
2694 return;
2695
2696 devid = PCI_SBDF_TO_DEVID(sbdf);
2697 iommu = get_amd_iommu_from_dev(dev);
2698 pci_seg = iommu->pci_seg;
2699
2700 list_for_each_entry(entry, &pci_seg->unity_map, list) {
2701 int type, prot = 0;
2702 size_t length;
2703
2704 if (devid < entry->devid_start || devid > entry->devid_end)
2705 continue;
2706
2707 type = IOMMU_RESV_DIRECT;
2708 length = entry->address_end - entry->address_start;
2709 if (entry->prot & IOMMU_PROT_IR)
2710 prot |= IOMMU_READ;
2711 if (entry->prot & IOMMU_PROT_IW)
2712 prot |= IOMMU_WRITE;
2713 if (entry->prot & IOMMU_UNITY_MAP_FLAG_EXCL_RANGE)
2714 /* Exclusion range */
2715 type = IOMMU_RESV_RESERVED;
2716
2717 region = iommu_alloc_resv_region(entry->address_start,
2718 length, prot, type,
2719 GFP_KERNEL);
2720 if (!region) {
2721 dev_err(dev, "Out of memory allocating dm-regions\n");
2722 return;
2723 }
2724 list_add_tail(®ion->list, head);
2725 }
2726
2727 region = iommu_alloc_resv_region(MSI_RANGE_START,
2728 MSI_RANGE_END - MSI_RANGE_START + 1,
2729 0, IOMMU_RESV_MSI, GFP_KERNEL);
2730 if (!region)
2731 return;
2732 list_add_tail(®ion->list, head);
2733
2734 region = iommu_alloc_resv_region(HT_RANGE_START,
2735 HT_RANGE_END - HT_RANGE_START + 1,
2736 0, IOMMU_RESV_RESERVED, GFP_KERNEL);
2737 if (!region)
2738 return;
2739 list_add_tail(®ion->list, head);
2740 }
2741
amd_iommu_is_attach_deferred(struct device * dev)2742 static bool amd_iommu_is_attach_deferred(struct device *dev)
2743 {
2744 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
2745
2746 return dev_data->defer_attach;
2747 }
2748
amd_iommu_flush_iotlb_all(struct iommu_domain * domain)2749 static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain)
2750 {
2751 struct protection_domain *dom = to_pdomain(domain);
2752 unsigned long flags;
2753
2754 spin_lock_irqsave(&dom->lock, flags);
2755 amd_iommu_domain_flush_all(dom);
2756 spin_unlock_irqrestore(&dom->lock, flags);
2757 }
2758
amd_iommu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)2759 static void amd_iommu_iotlb_sync(struct iommu_domain *domain,
2760 struct iommu_iotlb_gather *gather)
2761 {
2762 struct protection_domain *dom = to_pdomain(domain);
2763 unsigned long flags;
2764
2765 spin_lock_irqsave(&dom->lock, flags);
2766 amd_iommu_domain_flush_pages(dom, gather->start,
2767 gather->end - gather->start + 1);
2768 spin_unlock_irqrestore(&dom->lock, flags);
2769 }
2770
amd_iommu_def_domain_type(struct device * dev)2771 static int amd_iommu_def_domain_type(struct device *dev)
2772 {
2773 struct iommu_dev_data *dev_data;
2774
2775 dev_data = dev_iommu_priv_get(dev);
2776 if (!dev_data)
2777 return 0;
2778
2779 /* Always use DMA domain for untrusted device */
2780 if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted)
2781 return IOMMU_DOMAIN_DMA;
2782
2783 /*
2784 * Do not identity map IOMMUv2 capable devices when:
2785 * - memory encryption is active, because some of those devices
2786 * (AMD GPUs) don't have the encryption bit in their DMA-mask
2787 * and require remapping.
2788 * - SNP is enabled, because it prohibits DTE[Mode]=0.
2789 */
2790 if (pdev_pasid_supported(dev_data) &&
2791 !cc_platform_has(CC_ATTR_MEM_ENCRYPT) &&
2792 !amd_iommu_snp_en) {
2793 return IOMMU_DOMAIN_IDENTITY;
2794 }
2795
2796 return 0;
2797 }
2798
amd_iommu_enforce_cache_coherency(struct iommu_domain * domain)2799 static bool amd_iommu_enforce_cache_coherency(struct iommu_domain *domain)
2800 {
2801 /* IOMMU_PTE_FC is always set */
2802 return true;
2803 }
2804
2805 static const struct iommu_dirty_ops amd_dirty_ops = {
2806 .set_dirty_tracking = amd_iommu_set_dirty_tracking,
2807 .read_and_clear_dirty = amd_iommu_read_and_clear_dirty,
2808 };
2809
amd_iommu_dev_enable_feature(struct device * dev,enum iommu_dev_features feat)2810 static int amd_iommu_dev_enable_feature(struct device *dev,
2811 enum iommu_dev_features feat)
2812 {
2813 int ret = 0;
2814
2815 switch (feat) {
2816 case IOMMU_DEV_FEAT_IOPF:
2817 case IOMMU_DEV_FEAT_SVA:
2818 break;
2819 default:
2820 ret = -EINVAL;
2821 break;
2822 }
2823 return ret;
2824 }
2825
amd_iommu_dev_disable_feature(struct device * dev,enum iommu_dev_features feat)2826 static int amd_iommu_dev_disable_feature(struct device *dev,
2827 enum iommu_dev_features feat)
2828 {
2829 int ret = 0;
2830
2831 switch (feat) {
2832 case IOMMU_DEV_FEAT_IOPF:
2833 case IOMMU_DEV_FEAT_SVA:
2834 break;
2835 default:
2836 ret = -EINVAL;
2837 break;
2838 }
2839 return ret;
2840 }
2841
2842 const struct iommu_ops amd_iommu_ops = {
2843 .capable = amd_iommu_capable,
2844 .blocked_domain = &blocked_domain,
2845 .domain_alloc = amd_iommu_domain_alloc,
2846 .domain_alloc_user = amd_iommu_domain_alloc_user,
2847 .domain_alloc_sva = amd_iommu_domain_alloc_sva,
2848 .probe_device = amd_iommu_probe_device,
2849 .release_device = amd_iommu_release_device,
2850 .device_group = amd_iommu_device_group,
2851 .get_resv_regions = amd_iommu_get_resv_regions,
2852 .is_attach_deferred = amd_iommu_is_attach_deferred,
2853 .def_domain_type = amd_iommu_def_domain_type,
2854 .dev_enable_feat = amd_iommu_dev_enable_feature,
2855 .dev_disable_feat = amd_iommu_dev_disable_feature,
2856 .remove_dev_pasid = amd_iommu_remove_dev_pasid,
2857 .page_response = amd_iommu_page_response,
2858 .default_domain_ops = &(const struct iommu_domain_ops) {
2859 .attach_dev = amd_iommu_attach_device,
2860 .map_pages = amd_iommu_map_pages,
2861 .unmap_pages = amd_iommu_unmap_pages,
2862 .iotlb_sync_map = amd_iommu_iotlb_sync_map,
2863 .iova_to_phys = amd_iommu_iova_to_phys,
2864 .flush_iotlb_all = amd_iommu_flush_iotlb_all,
2865 .iotlb_sync = amd_iommu_iotlb_sync,
2866 .free = amd_iommu_domain_free,
2867 .enforce_cache_coherency = amd_iommu_enforce_cache_coherency,
2868 }
2869 };
2870
2871 #ifdef CONFIG_IRQ_REMAP
2872
2873 /*****************************************************************************
2874 *
2875 * Interrupt Remapping Implementation
2876 *
2877 *****************************************************************************/
2878
2879 static struct irq_chip amd_ir_chip;
2880 static DEFINE_SPINLOCK(iommu_table_lock);
2881
iommu_flush_irt_and_complete(struct amd_iommu * iommu,u16 devid)2882 static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid)
2883 {
2884 int ret;
2885 u64 data;
2886 unsigned long flags;
2887 struct iommu_cmd cmd, cmd2;
2888
2889 if (iommu->irtcachedis_enabled)
2890 return;
2891
2892 build_inv_irt(&cmd, devid);
2893 data = atomic64_add_return(1, &iommu->cmd_sem_val);
2894 build_completion_wait(&cmd2, iommu, data);
2895
2896 raw_spin_lock_irqsave(&iommu->lock, flags);
2897 ret = __iommu_queue_command_sync(iommu, &cmd, true);
2898 if (ret)
2899 goto out;
2900 ret = __iommu_queue_command_sync(iommu, &cmd2, false);
2901 if (ret)
2902 goto out;
2903 wait_on_sem(iommu, data);
2904 out:
2905 raw_spin_unlock_irqrestore(&iommu->lock, flags);
2906 }
2907
set_dte_irq_entry(struct amd_iommu * iommu,u16 devid,struct irq_remap_table * table)2908 static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid,
2909 struct irq_remap_table *table)
2910 {
2911 u64 dte;
2912 struct dev_table_entry *dev_table = get_dev_table(iommu);
2913
2914 dte = dev_table[devid].data[2];
2915 dte &= ~DTE_IRQ_PHYS_ADDR_MASK;
2916 dte |= iommu_virt_to_phys(table->table);
2917 dte |= DTE_IRQ_REMAP_INTCTL;
2918 dte |= DTE_INTTABLEN;
2919 dte |= DTE_IRQ_REMAP_ENABLE;
2920
2921 dev_table[devid].data[2] = dte;
2922 }
2923
get_irq_table(struct amd_iommu * iommu,u16 devid)2924 static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid)
2925 {
2926 struct irq_remap_table *table;
2927 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
2928
2929 if (WARN_ONCE(!pci_seg->rlookup_table[devid],
2930 "%s: no iommu for devid %x:%x\n",
2931 __func__, pci_seg->id, devid))
2932 return NULL;
2933
2934 table = pci_seg->irq_lookup_table[devid];
2935 if (WARN_ONCE(!table, "%s: no table for devid %x:%x\n",
2936 __func__, pci_seg->id, devid))
2937 return NULL;
2938
2939 return table;
2940 }
2941
__alloc_irq_table(void)2942 static struct irq_remap_table *__alloc_irq_table(void)
2943 {
2944 struct irq_remap_table *table;
2945
2946 table = kzalloc(sizeof(*table), GFP_KERNEL);
2947 if (!table)
2948 return NULL;
2949
2950 table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_KERNEL);
2951 if (!table->table) {
2952 kfree(table);
2953 return NULL;
2954 }
2955 raw_spin_lock_init(&table->lock);
2956
2957 if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))
2958 memset(table->table, 0,
2959 MAX_IRQS_PER_TABLE * sizeof(u32));
2960 else
2961 memset(table->table, 0,
2962 (MAX_IRQS_PER_TABLE * (sizeof(u64) * 2)));
2963 return table;
2964 }
2965
set_remap_table_entry(struct amd_iommu * iommu,u16 devid,struct irq_remap_table * table)2966 static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid,
2967 struct irq_remap_table *table)
2968 {
2969 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
2970
2971 pci_seg->irq_lookup_table[devid] = table;
2972 set_dte_irq_entry(iommu, devid, table);
2973 iommu_flush_dte(iommu, devid);
2974 }
2975
set_remap_table_entry_alias(struct pci_dev * pdev,u16 alias,void * data)2976 static int set_remap_table_entry_alias(struct pci_dev *pdev, u16 alias,
2977 void *data)
2978 {
2979 struct irq_remap_table *table = data;
2980 struct amd_iommu_pci_seg *pci_seg;
2981 struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev);
2982
2983 if (!iommu)
2984 return -EINVAL;
2985
2986 pci_seg = iommu->pci_seg;
2987 pci_seg->irq_lookup_table[alias] = table;
2988 set_dte_irq_entry(iommu, alias, table);
2989 iommu_flush_dte(pci_seg->rlookup_table[alias], alias);
2990
2991 return 0;
2992 }
2993
alloc_irq_table(struct amd_iommu * iommu,u16 devid,struct pci_dev * pdev)2994 static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu,
2995 u16 devid, struct pci_dev *pdev)
2996 {
2997 struct irq_remap_table *table = NULL;
2998 struct irq_remap_table *new_table = NULL;
2999 struct amd_iommu_pci_seg *pci_seg;
3000 unsigned long flags;
3001 u16 alias;
3002
3003 spin_lock_irqsave(&iommu_table_lock, flags);
3004
3005 pci_seg = iommu->pci_seg;
3006 table = pci_seg->irq_lookup_table[devid];
3007 if (table)
3008 goto out_unlock;
3009
3010 alias = pci_seg->alias_table[devid];
3011 table = pci_seg->irq_lookup_table[alias];
3012 if (table) {
3013 set_remap_table_entry(iommu, devid, table);
3014 goto out_wait;
3015 }
3016 spin_unlock_irqrestore(&iommu_table_lock, flags);
3017
3018 /* Nothing there yet, allocate new irq remapping table */
3019 new_table = __alloc_irq_table();
3020 if (!new_table)
3021 return NULL;
3022
3023 spin_lock_irqsave(&iommu_table_lock, flags);
3024
3025 table = pci_seg->irq_lookup_table[devid];
3026 if (table)
3027 goto out_unlock;
3028
3029 table = pci_seg->irq_lookup_table[alias];
3030 if (table) {
3031 set_remap_table_entry(iommu, devid, table);
3032 goto out_wait;
3033 }
3034
3035 table = new_table;
3036 new_table = NULL;
3037
3038 if (pdev)
3039 pci_for_each_dma_alias(pdev, set_remap_table_entry_alias,
3040 table);
3041 else
3042 set_remap_table_entry(iommu, devid, table);
3043
3044 if (devid != alias)
3045 set_remap_table_entry(iommu, alias, table);
3046
3047 out_wait:
3048 iommu_completion_wait(iommu);
3049
3050 out_unlock:
3051 spin_unlock_irqrestore(&iommu_table_lock, flags);
3052
3053 if (new_table) {
3054 kmem_cache_free(amd_iommu_irq_cache, new_table->table);
3055 kfree(new_table);
3056 }
3057 return table;
3058 }
3059
alloc_irq_index(struct amd_iommu * iommu,u16 devid,int count,bool align,struct pci_dev * pdev)3060 static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count,
3061 bool align, struct pci_dev *pdev)
3062 {
3063 struct irq_remap_table *table;
3064 int index, c, alignment = 1;
3065 unsigned long flags;
3066
3067 table = alloc_irq_table(iommu, devid, pdev);
3068 if (!table)
3069 return -ENODEV;
3070
3071 if (align)
3072 alignment = roundup_pow_of_two(count);
3073
3074 raw_spin_lock_irqsave(&table->lock, flags);
3075
3076 /* Scan table for free entries */
3077 for (index = ALIGN(table->min_index, alignment), c = 0;
3078 index < MAX_IRQS_PER_TABLE;) {
3079 if (!iommu->irte_ops->is_allocated(table, index)) {
3080 c += 1;
3081 } else {
3082 c = 0;
3083 index = ALIGN(index + 1, alignment);
3084 continue;
3085 }
3086
3087 if (c == count) {
3088 for (; c != 0; --c)
3089 iommu->irte_ops->set_allocated(table, index - c + 1);
3090
3091 index -= count - 1;
3092 goto out;
3093 }
3094
3095 index++;
3096 }
3097
3098 index = -ENOSPC;
3099
3100 out:
3101 raw_spin_unlock_irqrestore(&table->lock, flags);
3102
3103 return index;
3104 }
3105
__modify_irte_ga(struct amd_iommu * iommu,u16 devid,int index,struct irte_ga * irte)3106 static int __modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,
3107 struct irte_ga *irte)
3108 {
3109 struct irq_remap_table *table;
3110 struct irte_ga *entry;
3111 unsigned long flags;
3112 u128 old;
3113
3114 table = get_irq_table(iommu, devid);
3115 if (!table)
3116 return -ENOMEM;
3117
3118 raw_spin_lock_irqsave(&table->lock, flags);
3119
3120 entry = (struct irte_ga *)table->table;
3121 entry = &entry[index];
3122
3123 /*
3124 * We use cmpxchg16 to atomically update the 128-bit IRTE,
3125 * and it cannot be updated by the hardware or other processors
3126 * behind us, so the return value of cmpxchg16 should be the
3127 * same as the old value.
3128 */
3129 old = entry->irte;
3130 WARN_ON(!try_cmpxchg128(&entry->irte, &old, irte->irte));
3131
3132 raw_spin_unlock_irqrestore(&table->lock, flags);
3133
3134 return 0;
3135 }
3136
modify_irte_ga(struct amd_iommu * iommu,u16 devid,int index,struct irte_ga * irte)3137 static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,
3138 struct irte_ga *irte)
3139 {
3140 bool ret;
3141
3142 ret = __modify_irte_ga(iommu, devid, index, irte);
3143 if (ret)
3144 return ret;
3145
3146 iommu_flush_irt_and_complete(iommu, devid);
3147
3148 return 0;
3149 }
3150
modify_irte(struct amd_iommu * iommu,u16 devid,int index,union irte * irte)3151 static int modify_irte(struct amd_iommu *iommu,
3152 u16 devid, int index, union irte *irte)
3153 {
3154 struct irq_remap_table *table;
3155 unsigned long flags;
3156
3157 table = get_irq_table(iommu, devid);
3158 if (!table)
3159 return -ENOMEM;
3160
3161 raw_spin_lock_irqsave(&table->lock, flags);
3162 table->table[index] = irte->val;
3163 raw_spin_unlock_irqrestore(&table->lock, flags);
3164
3165 iommu_flush_irt_and_complete(iommu, devid);
3166
3167 return 0;
3168 }
3169
free_irte(struct amd_iommu * iommu,u16 devid,int index)3170 static void free_irte(struct amd_iommu *iommu, u16 devid, int index)
3171 {
3172 struct irq_remap_table *table;
3173 unsigned long flags;
3174
3175 table = get_irq_table(iommu, devid);
3176 if (!table)
3177 return;
3178
3179 raw_spin_lock_irqsave(&table->lock, flags);
3180 iommu->irte_ops->clear_allocated(table, index);
3181 raw_spin_unlock_irqrestore(&table->lock, flags);
3182
3183 iommu_flush_irt_and_complete(iommu, devid);
3184 }
3185
irte_prepare(void * entry,u32 delivery_mode,bool dest_mode,u8 vector,u32 dest_apicid,int devid)3186 static void irte_prepare(void *entry,
3187 u32 delivery_mode, bool dest_mode,
3188 u8 vector, u32 dest_apicid, int devid)
3189 {
3190 union irte *irte = (union irte *) entry;
3191
3192 irte->val = 0;
3193 irte->fields.vector = vector;
3194 irte->fields.int_type = delivery_mode;
3195 irte->fields.destination = dest_apicid;
3196 irte->fields.dm = dest_mode;
3197 irte->fields.valid = 1;
3198 }
3199
irte_ga_prepare(void * entry,u32 delivery_mode,bool dest_mode,u8 vector,u32 dest_apicid,int devid)3200 static void irte_ga_prepare(void *entry,
3201 u32 delivery_mode, bool dest_mode,
3202 u8 vector, u32 dest_apicid, int devid)
3203 {
3204 struct irte_ga *irte = (struct irte_ga *) entry;
3205
3206 irte->lo.val = 0;
3207 irte->hi.val = 0;
3208 irte->lo.fields_remap.int_type = delivery_mode;
3209 irte->lo.fields_remap.dm = dest_mode;
3210 irte->hi.fields.vector = vector;
3211 irte->lo.fields_remap.destination = APICID_TO_IRTE_DEST_LO(dest_apicid);
3212 irte->hi.fields.destination = APICID_TO_IRTE_DEST_HI(dest_apicid);
3213 irte->lo.fields_remap.valid = 1;
3214 }
3215
irte_activate(struct amd_iommu * iommu,void * entry,u16 devid,u16 index)3216 static void irte_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
3217 {
3218 union irte *irte = (union irte *) entry;
3219
3220 irte->fields.valid = 1;
3221 modify_irte(iommu, devid, index, irte);
3222 }
3223
irte_ga_activate(struct amd_iommu * iommu,void * entry,u16 devid,u16 index)3224 static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
3225 {
3226 struct irte_ga *irte = (struct irte_ga *) entry;
3227
3228 irte->lo.fields_remap.valid = 1;
3229 modify_irte_ga(iommu, devid, index, irte);
3230 }
3231
irte_deactivate(struct amd_iommu * iommu,void * entry,u16 devid,u16 index)3232 static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
3233 {
3234 union irte *irte = (union irte *) entry;
3235
3236 irte->fields.valid = 0;
3237 modify_irte(iommu, devid, index, irte);
3238 }
3239
irte_ga_deactivate(struct amd_iommu * iommu,void * entry,u16 devid,u16 index)3240 static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
3241 {
3242 struct irte_ga *irte = (struct irte_ga *) entry;
3243
3244 irte->lo.fields_remap.valid = 0;
3245 modify_irte_ga(iommu, devid, index, irte);
3246 }
3247
irte_set_affinity(struct amd_iommu * iommu,void * entry,u16 devid,u16 index,u8 vector,u32 dest_apicid)3248 static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,
3249 u8 vector, u32 dest_apicid)
3250 {
3251 union irte *irte = (union irte *) entry;
3252
3253 irte->fields.vector = vector;
3254 irte->fields.destination = dest_apicid;
3255 modify_irte(iommu, devid, index, irte);
3256 }
3257
irte_ga_set_affinity(struct amd_iommu * iommu,void * entry,u16 devid,u16 index,u8 vector,u32 dest_apicid)3258 static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,
3259 u8 vector, u32 dest_apicid)
3260 {
3261 struct irte_ga *irte = (struct irte_ga *) entry;
3262
3263 if (!irte->lo.fields_remap.guest_mode) {
3264 irte->hi.fields.vector = vector;
3265 irte->lo.fields_remap.destination =
3266 APICID_TO_IRTE_DEST_LO(dest_apicid);
3267 irte->hi.fields.destination =
3268 APICID_TO_IRTE_DEST_HI(dest_apicid);
3269 modify_irte_ga(iommu, devid, index, irte);
3270 }
3271 }
3272
3273 #define IRTE_ALLOCATED (~1U)
irte_set_allocated(struct irq_remap_table * table,int index)3274 static void irte_set_allocated(struct irq_remap_table *table, int index)
3275 {
3276 table->table[index] = IRTE_ALLOCATED;
3277 }
3278
irte_ga_set_allocated(struct irq_remap_table * table,int index)3279 static void irte_ga_set_allocated(struct irq_remap_table *table, int index)
3280 {
3281 struct irte_ga *ptr = (struct irte_ga *)table->table;
3282 struct irte_ga *irte = &ptr[index];
3283
3284 memset(&irte->lo.val, 0, sizeof(u64));
3285 memset(&irte->hi.val, 0, sizeof(u64));
3286 irte->hi.fields.vector = 0xff;
3287 }
3288
irte_is_allocated(struct irq_remap_table * table,int index)3289 static bool irte_is_allocated(struct irq_remap_table *table, int index)
3290 {
3291 union irte *ptr = (union irte *)table->table;
3292 union irte *irte = &ptr[index];
3293
3294 return irte->val != 0;
3295 }
3296
irte_ga_is_allocated(struct irq_remap_table * table,int index)3297 static bool irte_ga_is_allocated(struct irq_remap_table *table, int index)
3298 {
3299 struct irte_ga *ptr = (struct irte_ga *)table->table;
3300 struct irte_ga *irte = &ptr[index];
3301
3302 return irte->hi.fields.vector != 0;
3303 }
3304
irte_clear_allocated(struct irq_remap_table * table,int index)3305 static void irte_clear_allocated(struct irq_remap_table *table, int index)
3306 {
3307 table->table[index] = 0;
3308 }
3309
irte_ga_clear_allocated(struct irq_remap_table * table,int index)3310 static void irte_ga_clear_allocated(struct irq_remap_table *table, int index)
3311 {
3312 struct irte_ga *ptr = (struct irte_ga *)table->table;
3313 struct irte_ga *irte = &ptr[index];
3314
3315 memset(&irte->lo.val, 0, sizeof(u64));
3316 memset(&irte->hi.val, 0, sizeof(u64));
3317 }
3318
get_devid(struct irq_alloc_info * info)3319 static int get_devid(struct irq_alloc_info *info)
3320 {
3321 switch (info->type) {
3322 case X86_IRQ_ALLOC_TYPE_IOAPIC:
3323 return get_ioapic_devid(info->devid);
3324 case X86_IRQ_ALLOC_TYPE_HPET:
3325 return get_hpet_devid(info->devid);
3326 case X86_IRQ_ALLOC_TYPE_PCI_MSI:
3327 case X86_IRQ_ALLOC_TYPE_PCI_MSIX:
3328 return get_device_sbdf_id(msi_desc_to_dev(info->desc));
3329 default:
3330 WARN_ON_ONCE(1);
3331 return -1;
3332 }
3333 }
3334
3335 struct irq_remap_ops amd_iommu_irq_ops = {
3336 .prepare = amd_iommu_prepare,
3337 .enable = amd_iommu_enable,
3338 .disable = amd_iommu_disable,
3339 .reenable = amd_iommu_reenable,
3340 .enable_faulting = amd_iommu_enable_faulting,
3341 };
3342
fill_msi_msg(struct msi_msg * msg,u32 index)3343 static void fill_msi_msg(struct msi_msg *msg, u32 index)
3344 {
3345 msg->data = index;
3346 msg->address_lo = 0;
3347 msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;
3348 msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;
3349 }
3350
irq_remapping_prepare_irte(struct amd_ir_data * data,struct irq_cfg * irq_cfg,struct irq_alloc_info * info,int devid,int index,int sub_handle)3351 static void irq_remapping_prepare_irte(struct amd_ir_data *data,
3352 struct irq_cfg *irq_cfg,
3353 struct irq_alloc_info *info,
3354 int devid, int index, int sub_handle)
3355 {
3356 struct irq_2_irte *irte_info = &data->irq_2_irte;
3357 struct amd_iommu *iommu = data->iommu;
3358
3359 if (!iommu)
3360 return;
3361
3362 data->irq_2_irte.devid = devid;
3363 data->irq_2_irte.index = index + sub_handle;
3364 iommu->irte_ops->prepare(data->entry, APIC_DELIVERY_MODE_FIXED,
3365 apic->dest_mode_logical, irq_cfg->vector,
3366 irq_cfg->dest_apicid, devid);
3367
3368 switch (info->type) {
3369 case X86_IRQ_ALLOC_TYPE_IOAPIC:
3370 case X86_IRQ_ALLOC_TYPE_HPET:
3371 case X86_IRQ_ALLOC_TYPE_PCI_MSI:
3372 case X86_IRQ_ALLOC_TYPE_PCI_MSIX:
3373 fill_msi_msg(&data->msi_entry, irte_info->index);
3374 break;
3375
3376 default:
3377 BUG_ON(1);
3378 break;
3379 }
3380 }
3381
3382 struct amd_irte_ops irte_32_ops = {
3383 .prepare = irte_prepare,
3384 .activate = irte_activate,
3385 .deactivate = irte_deactivate,
3386 .set_affinity = irte_set_affinity,
3387 .set_allocated = irte_set_allocated,
3388 .is_allocated = irte_is_allocated,
3389 .clear_allocated = irte_clear_allocated,
3390 };
3391
3392 struct amd_irte_ops irte_128_ops = {
3393 .prepare = irte_ga_prepare,
3394 .activate = irte_ga_activate,
3395 .deactivate = irte_ga_deactivate,
3396 .set_affinity = irte_ga_set_affinity,
3397 .set_allocated = irte_ga_set_allocated,
3398 .is_allocated = irte_ga_is_allocated,
3399 .clear_allocated = irte_ga_clear_allocated,
3400 };
3401
irq_remapping_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)3402 static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
3403 unsigned int nr_irqs, void *arg)
3404 {
3405 struct irq_alloc_info *info = arg;
3406 struct irq_data *irq_data;
3407 struct amd_ir_data *data = NULL;
3408 struct amd_iommu *iommu;
3409 struct irq_cfg *cfg;
3410 int i, ret, devid, seg, sbdf;
3411 int index;
3412
3413 if (!info)
3414 return -EINVAL;
3415 if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_PCI_MSI)
3416 return -EINVAL;
3417
3418 sbdf = get_devid(info);
3419 if (sbdf < 0)
3420 return -EINVAL;
3421
3422 seg = PCI_SBDF_TO_SEGID(sbdf);
3423 devid = PCI_SBDF_TO_DEVID(sbdf);
3424 iommu = __rlookup_amd_iommu(seg, devid);
3425 if (!iommu)
3426 return -EINVAL;
3427
3428 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
3429 if (ret < 0)
3430 return ret;
3431
3432 if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) {
3433 struct irq_remap_table *table;
3434
3435 table = alloc_irq_table(iommu, devid, NULL);
3436 if (table) {
3437 if (!table->min_index) {
3438 /*
3439 * Keep the first 32 indexes free for IOAPIC
3440 * interrupts.
3441 */
3442 table->min_index = 32;
3443 for (i = 0; i < 32; ++i)
3444 iommu->irte_ops->set_allocated(table, i);
3445 }
3446 WARN_ON(table->min_index != 32);
3447 index = info->ioapic.pin;
3448 } else {
3449 index = -ENOMEM;
3450 }
3451 } else if (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI ||
3452 info->type == X86_IRQ_ALLOC_TYPE_PCI_MSIX) {
3453 bool align = (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI);
3454
3455 index = alloc_irq_index(iommu, devid, nr_irqs, align,
3456 msi_desc_to_pci_dev(info->desc));
3457 } else {
3458 index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL);
3459 }
3460
3461 if (index < 0) {
3462 pr_warn("Failed to allocate IRTE\n");
3463 ret = index;
3464 goto out_free_parent;
3465 }
3466
3467 for (i = 0; i < nr_irqs; i++) {
3468 irq_data = irq_domain_get_irq_data(domain, virq + i);
3469 cfg = irq_data ? irqd_cfg(irq_data) : NULL;
3470 if (!cfg) {
3471 ret = -EINVAL;
3472 goto out_free_data;
3473 }
3474
3475 ret = -ENOMEM;
3476 data = kzalloc(sizeof(*data), GFP_KERNEL);
3477 if (!data)
3478 goto out_free_data;
3479
3480 if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))
3481 data->entry = kzalloc(sizeof(union irte), GFP_KERNEL);
3482 else
3483 data->entry = kzalloc(sizeof(struct irte_ga),
3484 GFP_KERNEL);
3485 if (!data->entry) {
3486 kfree(data);
3487 goto out_free_data;
3488 }
3489
3490 data->iommu = iommu;
3491 irq_data->hwirq = (devid << 16) + i;
3492 irq_data->chip_data = data;
3493 irq_data->chip = &amd_ir_chip;
3494 irq_remapping_prepare_irte(data, cfg, info, devid, index, i);
3495 irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT);
3496 }
3497
3498 return 0;
3499
3500 out_free_data:
3501 for (i--; i >= 0; i--) {
3502 irq_data = irq_domain_get_irq_data(domain, virq + i);
3503 if (irq_data)
3504 kfree(irq_data->chip_data);
3505 }
3506 for (i = 0; i < nr_irqs; i++)
3507 free_irte(iommu, devid, index + i);
3508 out_free_parent:
3509 irq_domain_free_irqs_common(domain, virq, nr_irqs);
3510 return ret;
3511 }
3512
irq_remapping_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)3513 static void irq_remapping_free(struct irq_domain *domain, unsigned int virq,
3514 unsigned int nr_irqs)
3515 {
3516 struct irq_2_irte *irte_info;
3517 struct irq_data *irq_data;
3518 struct amd_ir_data *data;
3519 int i;
3520
3521 for (i = 0; i < nr_irqs; i++) {
3522 irq_data = irq_domain_get_irq_data(domain, virq + i);
3523 if (irq_data && irq_data->chip_data) {
3524 data = irq_data->chip_data;
3525 irte_info = &data->irq_2_irte;
3526 free_irte(data->iommu, irte_info->devid, irte_info->index);
3527 kfree(data->entry);
3528 kfree(data);
3529 }
3530 }
3531 irq_domain_free_irqs_common(domain, virq, nr_irqs);
3532 }
3533
3534 static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,
3535 struct amd_ir_data *ir_data,
3536 struct irq_2_irte *irte_info,
3537 struct irq_cfg *cfg);
3538
irq_remapping_activate(struct irq_domain * domain,struct irq_data * irq_data,bool reserve)3539 static int irq_remapping_activate(struct irq_domain *domain,
3540 struct irq_data *irq_data, bool reserve)
3541 {
3542 struct amd_ir_data *data = irq_data->chip_data;
3543 struct irq_2_irte *irte_info = &data->irq_2_irte;
3544 struct amd_iommu *iommu = data->iommu;
3545 struct irq_cfg *cfg = irqd_cfg(irq_data);
3546
3547 if (!iommu)
3548 return 0;
3549
3550 iommu->irte_ops->activate(iommu, data->entry, irte_info->devid,
3551 irte_info->index);
3552 amd_ir_update_irte(irq_data, iommu, data, irte_info, cfg);
3553 return 0;
3554 }
3555
irq_remapping_deactivate(struct irq_domain * domain,struct irq_data * irq_data)3556 static void irq_remapping_deactivate(struct irq_domain *domain,
3557 struct irq_data *irq_data)
3558 {
3559 struct amd_ir_data *data = irq_data->chip_data;
3560 struct irq_2_irte *irte_info = &data->irq_2_irte;
3561 struct amd_iommu *iommu = data->iommu;
3562
3563 if (iommu)
3564 iommu->irte_ops->deactivate(iommu, data->entry, irte_info->devid,
3565 irte_info->index);
3566 }
3567
irq_remapping_select(struct irq_domain * d,struct irq_fwspec * fwspec,enum irq_domain_bus_token bus_token)3568 static int irq_remapping_select(struct irq_domain *d, struct irq_fwspec *fwspec,
3569 enum irq_domain_bus_token bus_token)
3570 {
3571 struct amd_iommu *iommu;
3572 int devid = -1;
3573
3574 if (!amd_iommu_irq_remap)
3575 return 0;
3576
3577 if (x86_fwspec_is_ioapic(fwspec))
3578 devid = get_ioapic_devid(fwspec->param[0]);
3579 else if (x86_fwspec_is_hpet(fwspec))
3580 devid = get_hpet_devid(fwspec->param[0]);
3581
3582 if (devid < 0)
3583 return 0;
3584 iommu = __rlookup_amd_iommu((devid >> 16), (devid & 0xffff));
3585
3586 return iommu && iommu->ir_domain == d;
3587 }
3588
3589 static const struct irq_domain_ops amd_ir_domain_ops = {
3590 .select = irq_remapping_select,
3591 .alloc = irq_remapping_alloc,
3592 .free = irq_remapping_free,
3593 .activate = irq_remapping_activate,
3594 .deactivate = irq_remapping_deactivate,
3595 };
3596
amd_iommu_activate_guest_mode(void * data)3597 int amd_iommu_activate_guest_mode(void *data)
3598 {
3599 struct amd_ir_data *ir_data = (struct amd_ir_data *)data;
3600 struct irte_ga *entry = (struct irte_ga *) ir_data->entry;
3601 u64 valid;
3602
3603 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) || !entry)
3604 return 0;
3605
3606 valid = entry->lo.fields_vapic.valid;
3607
3608 entry->lo.val = 0;
3609 entry->hi.val = 0;
3610
3611 entry->lo.fields_vapic.valid = valid;
3612 entry->lo.fields_vapic.guest_mode = 1;
3613 entry->lo.fields_vapic.ga_log_intr = 1;
3614 entry->hi.fields.ga_root_ptr = ir_data->ga_root_ptr;
3615 entry->hi.fields.vector = ir_data->ga_vector;
3616 entry->lo.fields_vapic.ga_tag = ir_data->ga_tag;
3617
3618 return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,
3619 ir_data->irq_2_irte.index, entry);
3620 }
3621 EXPORT_SYMBOL(amd_iommu_activate_guest_mode);
3622
amd_iommu_deactivate_guest_mode(void * data)3623 int amd_iommu_deactivate_guest_mode(void *data)
3624 {
3625 struct amd_ir_data *ir_data = (struct amd_ir_data *)data;
3626 struct irte_ga *entry = (struct irte_ga *) ir_data->entry;
3627 struct irq_cfg *cfg = ir_data->cfg;
3628 u64 valid;
3629
3630 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) ||
3631 !entry || !entry->lo.fields_vapic.guest_mode)
3632 return 0;
3633
3634 valid = entry->lo.fields_remap.valid;
3635
3636 entry->lo.val = 0;
3637 entry->hi.val = 0;
3638
3639 entry->lo.fields_remap.valid = valid;
3640 entry->lo.fields_remap.dm = apic->dest_mode_logical;
3641 entry->lo.fields_remap.int_type = APIC_DELIVERY_MODE_FIXED;
3642 entry->hi.fields.vector = cfg->vector;
3643 entry->lo.fields_remap.destination =
3644 APICID_TO_IRTE_DEST_LO(cfg->dest_apicid);
3645 entry->hi.fields.destination =
3646 APICID_TO_IRTE_DEST_HI(cfg->dest_apicid);
3647
3648 return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,
3649 ir_data->irq_2_irte.index, entry);
3650 }
3651 EXPORT_SYMBOL(amd_iommu_deactivate_guest_mode);
3652
amd_ir_set_vcpu_affinity(struct irq_data * data,void * vcpu_info)3653 static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info)
3654 {
3655 int ret;
3656 struct amd_iommu_pi_data *pi_data = vcpu_info;
3657 struct vcpu_data *vcpu_pi_info = pi_data->vcpu_data;
3658 struct amd_ir_data *ir_data = data->chip_data;
3659 struct irq_2_irte *irte_info = &ir_data->irq_2_irte;
3660 struct iommu_dev_data *dev_data;
3661
3662 if (ir_data->iommu == NULL)
3663 return -EINVAL;
3664
3665 dev_data = search_dev_data(ir_data->iommu, irte_info->devid);
3666
3667 /* Note:
3668 * This device has never been set up for guest mode.
3669 * we should not modify the IRTE
3670 */
3671 if (!dev_data || !dev_data->use_vapic)
3672 return 0;
3673
3674 ir_data->cfg = irqd_cfg(data);
3675 pi_data->ir_data = ir_data;
3676
3677 /* Note:
3678 * SVM tries to set up for VAPIC mode, but we are in
3679 * legacy mode. So, we force legacy mode instead.
3680 */
3681 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) {
3682 pr_debug("%s: Fall back to using intr legacy remap\n",
3683 __func__);
3684 pi_data->is_guest_mode = false;
3685 }
3686
3687 pi_data->prev_ga_tag = ir_data->cached_ga_tag;
3688 if (pi_data->is_guest_mode) {
3689 ir_data->ga_root_ptr = (pi_data->base >> 12);
3690 ir_data->ga_vector = vcpu_pi_info->vector;
3691 ir_data->ga_tag = pi_data->ga_tag;
3692 ret = amd_iommu_activate_guest_mode(ir_data);
3693 if (!ret)
3694 ir_data->cached_ga_tag = pi_data->ga_tag;
3695 } else {
3696 ret = amd_iommu_deactivate_guest_mode(ir_data);
3697
3698 /*
3699 * This communicates the ga_tag back to the caller
3700 * so that it can do all the necessary clean up.
3701 */
3702 if (!ret)
3703 ir_data->cached_ga_tag = 0;
3704 }
3705
3706 return ret;
3707 }
3708
3709
amd_ir_update_irte(struct irq_data * irqd,struct amd_iommu * iommu,struct amd_ir_data * ir_data,struct irq_2_irte * irte_info,struct irq_cfg * cfg)3710 static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,
3711 struct amd_ir_data *ir_data,
3712 struct irq_2_irte *irte_info,
3713 struct irq_cfg *cfg)
3714 {
3715
3716 /*
3717 * Atomically updates the IRTE with the new destination, vector
3718 * and flushes the interrupt entry cache.
3719 */
3720 iommu->irte_ops->set_affinity(iommu, ir_data->entry, irte_info->devid,
3721 irte_info->index, cfg->vector,
3722 cfg->dest_apicid);
3723 }
3724
amd_ir_set_affinity(struct irq_data * data,const struct cpumask * mask,bool force)3725 static int amd_ir_set_affinity(struct irq_data *data,
3726 const struct cpumask *mask, bool force)
3727 {
3728 struct amd_ir_data *ir_data = data->chip_data;
3729 struct irq_2_irte *irte_info = &ir_data->irq_2_irte;
3730 struct irq_cfg *cfg = irqd_cfg(data);
3731 struct irq_data *parent = data->parent_data;
3732 struct amd_iommu *iommu = ir_data->iommu;
3733 int ret;
3734
3735 if (!iommu)
3736 return -ENODEV;
3737
3738 ret = parent->chip->irq_set_affinity(parent, mask, force);
3739 if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
3740 return ret;
3741
3742 amd_ir_update_irte(data, iommu, ir_data, irte_info, cfg);
3743 /*
3744 * After this point, all the interrupts will start arriving
3745 * at the new destination. So, time to cleanup the previous
3746 * vector allocation.
3747 */
3748 vector_schedule_cleanup(cfg);
3749
3750 return IRQ_SET_MASK_OK_DONE;
3751 }
3752
ir_compose_msi_msg(struct irq_data * irq_data,struct msi_msg * msg)3753 static void ir_compose_msi_msg(struct irq_data *irq_data, struct msi_msg *msg)
3754 {
3755 struct amd_ir_data *ir_data = irq_data->chip_data;
3756
3757 *msg = ir_data->msi_entry;
3758 }
3759
3760 static struct irq_chip amd_ir_chip = {
3761 .name = "AMD-IR",
3762 .irq_ack = apic_ack_irq,
3763 .irq_set_affinity = amd_ir_set_affinity,
3764 .irq_set_vcpu_affinity = amd_ir_set_vcpu_affinity,
3765 .irq_compose_msi_msg = ir_compose_msi_msg,
3766 };
3767
3768 static const struct msi_parent_ops amdvi_msi_parent_ops = {
3769 .supported_flags = X86_VECTOR_MSI_FLAGS_SUPPORTED | MSI_FLAG_MULTI_PCI_MSI,
3770 .prefix = "IR-",
3771 .init_dev_msi_info = msi_parent_init_dev_msi_info,
3772 };
3773
amd_iommu_create_irq_domain(struct amd_iommu * iommu)3774 int amd_iommu_create_irq_domain(struct amd_iommu *iommu)
3775 {
3776 struct fwnode_handle *fn;
3777
3778 fn = irq_domain_alloc_named_id_fwnode("AMD-IR", iommu->index);
3779 if (!fn)
3780 return -ENOMEM;
3781 iommu->ir_domain = irq_domain_create_hierarchy(arch_get_ir_parent_domain(), 0, 0,
3782 fn, &amd_ir_domain_ops, iommu);
3783 if (!iommu->ir_domain) {
3784 irq_domain_free_fwnode(fn);
3785 return -ENOMEM;
3786 }
3787
3788 irq_domain_update_bus_token(iommu->ir_domain, DOMAIN_BUS_AMDVI);
3789 iommu->ir_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT |
3790 IRQ_DOMAIN_FLAG_ISOLATED_MSI;
3791 iommu->ir_domain->msi_parent_ops = &amdvi_msi_parent_ops;
3792
3793 return 0;
3794 }
3795
amd_iommu_update_ga(int cpu,bool is_run,void * data)3796 int amd_iommu_update_ga(int cpu, bool is_run, void *data)
3797 {
3798 struct amd_ir_data *ir_data = (struct amd_ir_data *)data;
3799 struct irte_ga *entry = (struct irte_ga *) ir_data->entry;
3800
3801 if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) ||
3802 !entry || !entry->lo.fields_vapic.guest_mode)
3803 return 0;
3804
3805 if (!ir_data->iommu)
3806 return -ENODEV;
3807
3808 if (cpu >= 0) {
3809 entry->lo.fields_vapic.destination =
3810 APICID_TO_IRTE_DEST_LO(cpu);
3811 entry->hi.fields.destination =
3812 APICID_TO_IRTE_DEST_HI(cpu);
3813 }
3814 entry->lo.fields_vapic.is_run = is_run;
3815
3816 return __modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,
3817 ir_data->irq_2_irte.index, entry);
3818 }
3819 EXPORT_SYMBOL(amd_iommu_update_ga);
3820 #endif
3821