1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2023 Advanced Micro Devices, Inc. 4 */ 5 6 #define pr_fmt(fmt) "AMD-Vi: " fmt 7 #define dev_fmt(fmt) pr_fmt(fmt) 8 9 #include <linux/amd-iommu.h> 10 #include <linux/delay.h> 11 #include <linux/mmu_notifier.h> 12 13 #include <asm/iommu.h> 14 15 #include "amd_iommu.h" 16 #include "amd_iommu_types.h" 17 18 #include "../iommu-pages.h" 19 20 int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu) 21 { 22 iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, 23 amd_iommu_pprlog_size); 24 return iommu->ppr_log ? 0 : -ENOMEM; 25 } 26 27 void amd_iommu_enable_ppr_log(struct amd_iommu *iommu) 28 { 29 u64 entry; 30 31 if (iommu->ppr_log == NULL) 32 return; 33 34 iommu_feature_enable(iommu, CONTROL_PPR_EN); 35 36 entry = iommu_virt_to_phys(iommu->ppr_log); 37 entry |= (amd_iommu_pprlog_size == PPRLOG_SIZE_DEF) ? 38 PPRLOG_LEN_MASK_DEF : PPRLOG_LEN_MASK_MAX; 39 40 memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET, 41 &entry, sizeof(entry)); 42 43 /* set head and tail to zero manually */ 44 writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); 45 writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); 46 47 iommu_feature_enable(iommu, CONTROL_PPRINT_EN); 48 iommu_feature_enable(iommu, CONTROL_PPRLOG_EN); 49 } 50 51 void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu) 52 { 53 iommu_free_pages(iommu->ppr_log); 54 } 55 56 /* 57 * This function restarts ppr logging in case the IOMMU experienced 58 * PPR log overflow. 59 */ 60 void amd_iommu_restart_ppr_log(struct amd_iommu *iommu) 61 { 62 amd_iommu_restart_log(iommu, "PPR", CONTROL_PPRINT_EN, 63 CONTROL_PPRLOG_EN, MMIO_STATUS_PPR_RUN_MASK, 64 MMIO_STATUS_PPR_OVERFLOW_MASK); 65 } 66 67 static inline u32 ppr_flag_to_fault_perm(u16 flag) 68 { 69 int perm = 0; 70 71 if (flag & PPR_FLAG_READ) 72 perm |= IOMMU_FAULT_PERM_READ; 73 if (flag & PPR_FLAG_WRITE) 74 perm |= IOMMU_FAULT_PERM_WRITE; 75 if (flag & PPR_FLAG_EXEC) 76 perm |= IOMMU_FAULT_PERM_EXEC; 77 if (!(flag & PPR_FLAG_US)) 78 perm |= IOMMU_FAULT_PERM_PRIV; 79 80 return perm; 81 } 82 83 static bool ppr_is_valid(struct amd_iommu *iommu, u64 *raw) 84 { 85 struct device *dev = iommu->iommu.dev; 86 u16 devid = PPR_DEVID(raw[0]); 87 88 if (!(PPR_FLAGS(raw[0]) & PPR_FLAG_GN)) { 89 dev_dbg(dev, "PPR logged [Request ignored due to GN=0 (device=%04x:%02x:%02x.%x " 90 "pasid=0x%05llx address=0x%llx flags=0x%04llx tag=0x%03llx]\n", 91 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 92 PPR_PASID(raw[0]), raw[1], PPR_FLAGS(raw[0]), PPR_TAG(raw[0])); 93 return false; 94 } 95 96 if (PPR_FLAGS(raw[0]) & PPR_FLAG_RVSD) { 97 dev_dbg(dev, "PPR logged [Invalid request format (device=%04x:%02x:%02x.%x " 98 "pasid=0x%05llx address=0x%llx flags=0x%04llx tag=0x%03llx]\n", 99 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 100 PPR_PASID(raw[0]), raw[1], PPR_FLAGS(raw[0]), PPR_TAG(raw[0])); 101 return false; 102 } 103 104 return true; 105 } 106 107 static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw) 108 { 109 struct iommu_dev_data *dev_data; 110 struct iopf_fault event; 111 struct pci_dev *pdev; 112 u16 devid = PPR_DEVID(raw[0]); 113 114 if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) { 115 pr_info_ratelimited("Unknown PPR request received\n"); 116 return; 117 } 118 119 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, 120 PCI_BUS_NUM(devid), devid & 0xff); 121 if (!pdev) 122 return; 123 124 if (!ppr_is_valid(iommu, raw)) 125 goto out; 126 127 memset(&event, 0, sizeof(struct iopf_fault)); 128 129 event.fault.type = IOMMU_FAULT_PAGE_REQ; 130 event.fault.prm.perm = ppr_flag_to_fault_perm(PPR_FLAGS(raw[0])); 131 event.fault.prm.addr = (u64)(raw[1] & PAGE_MASK); 132 event.fault.prm.pasid = PPR_PASID(raw[0]); 133 event.fault.prm.grpid = PPR_TAG(raw[0]) & 0x1FF; 134 135 /* 136 * PASID zero is used for requests from the I/O device without 137 * a PASID 138 */ 139 dev_data = dev_iommu_priv_get(&pdev->dev); 140 if (event.fault.prm.pasid == 0 || 141 event.fault.prm.pasid >= dev_data->max_pasids) { 142 pr_info_ratelimited("Invalid PASID : 0x%x, device : 0x%x\n", 143 event.fault.prm.pasid, pdev->dev.id); 144 goto out; 145 } 146 147 event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; 148 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 149 if (PPR_TAG(raw[0]) & 0x200) 150 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 151 152 /* Submit event */ 153 iommu_report_device_fault(&pdev->dev, &event); 154 155 return; 156 157 out: 158 /* Nobody cared, abort */ 159 amd_iommu_complete_ppr(&pdev->dev, PPR_PASID(raw[0]), 160 IOMMU_PAGE_RESP_FAILURE, 161 PPR_TAG(raw[0]) & 0x1FF); 162 } 163 164 void amd_iommu_poll_ppr_log(struct amd_iommu *iommu) 165 { 166 u32 head, tail; 167 168 if (iommu->ppr_log == NULL) 169 return; 170 171 head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); 172 tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); 173 174 while (head != tail) { 175 volatile u64 *raw; 176 u64 entry[2]; 177 int i; 178 179 raw = (u64 *)(iommu->ppr_log + head); 180 181 /* 182 * Hardware bug: Interrupt may arrive before the entry is 183 * written to memory. If this happens we need to wait for the 184 * entry to arrive. 185 */ 186 for (i = 0; i < LOOP_TIMEOUT; ++i) { 187 if (PPR_REQ_TYPE(raw[0]) != 0) 188 break; 189 udelay(1); 190 } 191 192 /* Avoid memcpy function-call overhead */ 193 entry[0] = raw[0]; 194 entry[1] = raw[1]; 195 196 /* 197 * To detect the hardware errata 733 we need to clear the 198 * entry back to zero. This issue does not exist on SNP 199 * enabled system. Also this buffer is not writeable on 200 * SNP enabled system. 201 */ 202 if (!amd_iommu_snp_en) 203 raw[0] = raw[1] = 0UL; 204 205 /* Update head pointer of hardware ring-buffer */ 206 head = (head + PPRLOG_ENTRY_SIZE) % amd_iommu_pprlog_size; 207 writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); 208 209 /* Handle PPR entry */ 210 iommu_call_iopf_notifier(iommu, entry); 211 } 212 } 213 214 /************************************************************** 215 * 216 * IOPF handling stuff 217 */ 218 219 /* Setup per-IOMMU IOPF queue if not exist. */ 220 int amd_iommu_iopf_init(struct amd_iommu *iommu) 221 { 222 int ret = 0; 223 224 if (iommu->iopf_queue) 225 return ret; 226 227 snprintf(iommu->iopfq_name, sizeof(iommu->iopfq_name), "amdvi-%#x", 228 PCI_SEG_DEVID_TO_SBDF(iommu->pci_seg->id, iommu->devid)); 229 230 iommu->iopf_queue = iopf_queue_alloc(iommu->iopfq_name); 231 if (!iommu->iopf_queue) 232 ret = -ENOMEM; 233 234 return ret; 235 } 236 237 /* Destroy per-IOMMU IOPF queue if no longer needed. */ 238 void amd_iommu_iopf_uninit(struct amd_iommu *iommu) 239 { 240 iopf_queue_free(iommu->iopf_queue); 241 iommu->iopf_queue = NULL; 242 } 243 244 void amd_iommu_page_response(struct device *dev, struct iopf_fault *evt, 245 struct iommu_page_response *resp) 246 { 247 amd_iommu_complete_ppr(dev, resp->pasid, resp->code, resp->grpid); 248 } 249 250 int amd_iommu_iopf_add_device(struct amd_iommu *iommu, 251 struct iommu_dev_data *dev_data) 252 { 253 int ret = 0; 254 255 if (!dev_data->pri_enabled) 256 return ret; 257 258 if (!iommu->iopf_queue) 259 return -EINVAL; 260 261 ret = iopf_queue_add_device(iommu->iopf_queue, dev_data->dev); 262 if (ret) 263 return ret; 264 265 dev_data->ppr = true; 266 return 0; 267 } 268 269 /* Its assumed that caller has verified that device was added to iopf queue */ 270 void amd_iommu_iopf_remove_device(struct amd_iommu *iommu, 271 struct iommu_dev_data *dev_data) 272 { 273 iopf_queue_remove_device(iommu->iopf_queue, dev_data->dev); 274 dev_data->ppr = false; 275 } 276