1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <stdint.h> 3 #include <unistd.h> 4 5 #include <linux/bits.h> 6 #include <linux/errno.h> 7 #include <linux/idxd.h> 8 #include <linux/io.h> 9 #include <linux/pci_ids.h> 10 #include <linux/sizes.h> 11 12 #include <libvfio.h> 13 14 #include "registers.h" 15 16 /* Vectors 1+ are available for work queue completion interrupts. */ 17 #define MSIX_VECTOR 1 18 19 struct dsa_state { 20 /* Descriptors for copy and batch operations. */ 21 struct dsa_hw_desc batch[32]; 22 struct dsa_hw_desc copy[1024]; 23 24 /* Completion records for copy and batch operations. */ 25 struct dsa_completion_record copy_completion; 26 struct dsa_completion_record batch_completion; 27 28 /* Cached device registers (and derived data) for easy access */ 29 union gen_cap_reg gen_cap; 30 union wq_cap_reg wq_cap; 31 union group_cap_reg group_cap; 32 union engine_cap_reg engine_cap; 33 union offsets_reg table_offsets; 34 void *wqcfg_table; 35 void *grpcfg_table; 36 u64 max_batches; 37 u64 max_copies_per_batch; 38 39 /* The number of ongoing memcpy operations. */ 40 u64 memcpy_count; 41 42 /* Buffers used by dsa_send_msi() to generate an interrupt */ 43 u64 send_msi_src; 44 u64 send_msi_dst; 45 }; 46 47 static inline struct dsa_state *to_dsa_state(struct vfio_pci_device *device) 48 { 49 return device->driver.region.vaddr; 50 } 51 52 static bool dsa_int_handle_request_required(struct vfio_pci_device *device) 53 { 54 void *bar0 = device->bars[0].vaddr; 55 union gen_cap_reg gen_cap; 56 u32 cmd_cap; 57 58 gen_cap.bits = readq(bar0 + IDXD_GENCAP_OFFSET); 59 if (!gen_cap.cmd_cap) 60 return false; 61 62 cmd_cap = readl(bar0 + IDXD_CMDCAP_OFFSET); 63 return (cmd_cap >> IDXD_CMD_REQUEST_INT_HANDLE) & 1; 64 } 65 66 static int dsa_probe(struct vfio_pci_device *device) 67 { 68 const u16 vendor_id = vfio_pci_config_readw(device, PCI_VENDOR_ID); 69 const u16 device_id = vfio_pci_config_readw(device, PCI_DEVICE_ID); 70 71 if (vendor_id != PCI_VENDOR_ID_INTEL) 72 return -EINVAL; 73 74 switch (device_id) { 75 case PCI_DEVICE_ID_INTEL_DSA_SPR0: 76 case PCI_DEVICE_ID_INTEL_DSA_DMR: 77 case PCI_DEVICE_ID_INTEL_DSA_GNRD: 78 break; 79 default: 80 return -EINVAL; 81 } 82 83 if (dsa_int_handle_request_required(device)) { 84 dev_err(device, "Device requires requesting interrupt handles\n"); 85 return -EINVAL; 86 } 87 88 return 0; 89 } 90 91 static void dsa_check_sw_err(struct vfio_pci_device *device) 92 { 93 void *reg = device->bars[0].vaddr + IDXD_SWERR_OFFSET; 94 union sw_err_reg err = {}; 95 int i; 96 97 for (i = 0; i < ARRAY_SIZE(err.bits); i++) { 98 err.bits[i] = readq(reg + offsetof(union sw_err_reg, bits[i])); 99 100 /* No errors */ 101 if (i == 0 && !err.valid) 102 return; 103 } 104 105 dev_err(device, "SWERR: 0x%016lx 0x%016lx 0x%016lx 0x%016lx\n", 106 err.bits[0], err.bits[1], err.bits[2], err.bits[3]); 107 108 dev_err(device, " valid: 0x%x\n", err.valid); 109 dev_err(device, " overflow: 0x%x\n", err.overflow); 110 dev_err(device, " desc_valid: 0x%x\n", err.desc_valid); 111 dev_err(device, " wq_idx_valid: 0x%x\n", err.wq_idx_valid); 112 dev_err(device, " batch: 0x%x\n", err.batch); 113 dev_err(device, " fault_rw: 0x%x\n", err.fault_rw); 114 dev_err(device, " priv: 0x%x\n", err.priv); 115 dev_err(device, " error: 0x%x\n", err.error); 116 dev_err(device, " wq_idx: 0x%x\n", err.wq_idx); 117 dev_err(device, " operation: 0x%x\n", err.operation); 118 dev_err(device, " pasid: 0x%x\n", err.pasid); 119 dev_err(device, " batch_idx: 0x%x\n", err.batch_idx); 120 dev_err(device, " invalid_flags: 0x%x\n", err.invalid_flags); 121 dev_err(device, " fault_addr: 0x%lx\n", err.fault_addr); 122 123 VFIO_FAIL("Software Error Detected!\n"); 124 } 125 126 static void dsa_command(struct vfio_pci_device *device, u32 cmd) 127 { 128 union idxd_command_reg cmd_reg = { .cmd = cmd }; 129 u32 sleep_ms = 1, attempts = 5000 / sleep_ms; 130 void *bar0 = device->bars[0].vaddr; 131 u32 status; 132 u8 err; 133 134 writel(cmd_reg.bits, bar0 + IDXD_CMD_OFFSET); 135 136 for (;;) { 137 dsa_check_sw_err(device); 138 139 status = readl(bar0 + IDXD_CMDSTS_OFFSET); 140 if (!(status & IDXD_CMDSTS_ACTIVE)) 141 break; 142 143 VFIO_ASSERT_GT(--attempts, 0); 144 usleep(sleep_ms * 1000); 145 } 146 147 err = status & IDXD_CMDSTS_ERR_MASK; 148 VFIO_ASSERT_EQ(err, 0, "Error issuing command 0x%x: 0x%x\n", cmd, err); 149 } 150 151 static void dsa_wq_init(struct vfio_pci_device *device) 152 { 153 struct dsa_state *dsa = to_dsa_state(device); 154 union wq_cap_reg wq_cap = dsa->wq_cap; 155 union wqcfg wqcfg; 156 u64 wqcfg_size; 157 int i; 158 159 VFIO_ASSERT_GT((u32)wq_cap.num_wqs, 0); 160 161 wqcfg = (union wqcfg) { 162 .wq_size = wq_cap.total_wq_size, 163 .mode = 1, 164 .priority = 1, 165 /* 166 * Disable Address Translation Service (if enabled) so that VFIO 167 * selftests using this driver can generate I/O page faults. 168 */ 169 .wq_ats_disable = wq_cap.wq_ats_support, 170 .max_xfer_shift = dsa->gen_cap.max_xfer_shift, 171 .max_batch_shift = dsa->gen_cap.max_batch_shift, 172 .op_config[0] = BIT(DSA_OPCODE_MEMMOVE) | BIT(DSA_OPCODE_BATCH), 173 }; 174 175 wqcfg_size = 1UL << (wq_cap.wqcfg_size + IDXD_WQCFG_MIN); 176 177 for (i = 0; i < wqcfg_size / sizeof(wqcfg.bits[0]); i++) 178 writel(wqcfg.bits[i], dsa->wqcfg_table + offsetof(union wqcfg, bits[i])); 179 } 180 181 static void dsa_group_init(struct vfio_pci_device *device) 182 { 183 struct dsa_state *dsa = to_dsa_state(device); 184 union group_cap_reg group_cap = dsa->group_cap; 185 union engine_cap_reg engine_cap = dsa->engine_cap; 186 187 VFIO_ASSERT_GT((u32)group_cap.num_groups, 0); 188 VFIO_ASSERT_GT((u32)engine_cap.num_engines, 0); 189 190 /* Assign work queue 0 and engine 0 to group 0 */ 191 writeq(1, dsa->grpcfg_table + offsetof(struct grpcfg, wqs[0])); 192 writeq(1, dsa->grpcfg_table + offsetof(struct grpcfg, engines)); 193 } 194 195 static void dsa_register_cache_init(struct vfio_pci_device *device) 196 { 197 struct dsa_state *dsa = to_dsa_state(device); 198 void *bar0 = device->bars[0].vaddr; 199 200 dsa->gen_cap.bits = readq(bar0 + IDXD_GENCAP_OFFSET); 201 dsa->wq_cap.bits = readq(bar0 + IDXD_WQCAP_OFFSET); 202 dsa->group_cap.bits = readq(bar0 + IDXD_GRPCAP_OFFSET); 203 dsa->engine_cap.bits = readq(bar0 + IDXD_ENGCAP_OFFSET); 204 205 dsa->table_offsets.bits[0] = readq(bar0 + IDXD_TABLE_OFFSET); 206 dsa->table_offsets.bits[1] = readq(bar0 + IDXD_TABLE_OFFSET + 8); 207 208 dsa->wqcfg_table = bar0 + dsa->table_offsets.wqcfg * IDXD_TABLE_MULT; 209 dsa->grpcfg_table = bar0 + dsa->table_offsets.grpcfg * IDXD_TABLE_MULT; 210 211 dsa->max_batches = 1U << (dsa->wq_cap.total_wq_size + IDXD_WQCFG_MIN); 212 dsa->max_batches = min(dsa->max_batches, ARRAY_SIZE(dsa->batch)); 213 214 dsa->max_copies_per_batch = 1UL << dsa->gen_cap.max_batch_shift; 215 dsa->max_copies_per_batch = min(dsa->max_copies_per_batch, ARRAY_SIZE(dsa->copy)); 216 } 217 218 static void dsa_init(struct vfio_pci_device *device) 219 { 220 struct dsa_state *dsa = to_dsa_state(device); 221 222 VFIO_ASSERT_GE(device->driver.region.size, sizeof(*dsa)); 223 224 vfio_pci_config_writew(device, PCI_COMMAND, 225 PCI_COMMAND_MEMORY | 226 PCI_COMMAND_MASTER | 227 PCI_COMMAND_INTX_DISABLE); 228 229 dsa_command(device, IDXD_CMD_RESET_DEVICE); 230 231 dsa_register_cache_init(device); 232 dsa_wq_init(device); 233 dsa_group_init(device); 234 235 dsa_command(device, IDXD_CMD_ENABLE_DEVICE); 236 dsa_command(device, IDXD_CMD_ENABLE_WQ); 237 238 vfio_pci_msix_enable(device, MSIX_VECTOR, 1); 239 240 device->driver.max_memcpy_count = 241 dsa->max_batches * dsa->max_copies_per_batch; 242 device->driver.max_memcpy_size = 1UL << dsa->gen_cap.max_xfer_shift; 243 device->driver.msi = MSIX_VECTOR; 244 } 245 246 static void dsa_remove(struct vfio_pci_device *device) 247 { 248 dsa_command(device, IDXD_CMD_RESET_DEVICE); 249 vfio_pci_msix_disable(device); 250 } 251 252 static int dsa_completion_wait(struct vfio_pci_device *device, 253 struct dsa_completion_record *completion) 254 { 255 u8 status; 256 257 for (;;) { 258 dsa_check_sw_err(device); 259 260 status = READ_ONCE(completion->status); 261 if (status) 262 break; 263 264 usleep(1000); 265 } 266 267 if (status == DSA_COMP_SUCCESS) 268 return 0; 269 270 dev_err(device, "Error detected during memcpy operation: 0x%x\n", status); 271 return -1; 272 } 273 274 static void dsa_copy_desc_init(struct vfio_pci_device *device, 275 struct dsa_hw_desc *desc, 276 iova_t src, iova_t dst, u64 size, 277 bool interrupt) 278 { 279 struct dsa_state *dsa = to_dsa_state(device); 280 u16 flags; 281 282 flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; 283 284 if (interrupt) 285 flags |= IDXD_OP_FLAG_RCI; 286 287 *desc = (struct dsa_hw_desc) { 288 .opcode = DSA_OPCODE_MEMMOVE, 289 .flags = flags, 290 .priv = 1, 291 .src_addr = src, 292 .dst_addr = dst, 293 .xfer_size = size, 294 .completion_addr = to_iova(device, &dsa->copy_completion), 295 .int_handle = interrupt ? MSIX_VECTOR : 0, 296 }; 297 } 298 299 static void dsa_batch_desc_init(struct vfio_pci_device *device, 300 struct dsa_hw_desc *desc, 301 u64 count) 302 { 303 struct dsa_state *dsa = to_dsa_state(device); 304 305 *desc = (struct dsa_hw_desc) { 306 .opcode = DSA_OPCODE_BATCH, 307 .flags = IDXD_OP_FLAG_CRAV, 308 .priv = 1, 309 .completion_addr = to_iova(device, &dsa->batch_completion), 310 .desc_list_addr = to_iova(device, &dsa->copy[0]), 311 .desc_count = count, 312 }; 313 } 314 315 static void dsa_desc_write(struct vfio_pci_device *device, struct dsa_hw_desc *desc) 316 { 317 /* Write the contents (not address) of the 64-byte descriptor to the device. */ 318 iosubmit_cmds512(device->bars[2].vaddr, desc, 1); 319 } 320 321 static void dsa_memcpy_one(struct vfio_pci_device *device, 322 iova_t src, iova_t dst, u64 size, bool interrupt) 323 { 324 struct dsa_state *dsa = to_dsa_state(device); 325 326 memset(&dsa->copy_completion, 0, sizeof(dsa->copy_completion)); 327 328 dsa_copy_desc_init(device, &dsa->copy[0], src, dst, size, interrupt); 329 dsa_desc_write(device, &dsa->copy[0]); 330 } 331 332 static void dsa_memcpy_batch(struct vfio_pci_device *device, 333 iova_t src, iova_t dst, u64 size, u64 count) 334 { 335 struct dsa_state *dsa = to_dsa_state(device); 336 int i; 337 338 memset(&dsa->batch_completion, 0, sizeof(dsa->batch_completion)); 339 340 for (i = 0; i < ARRAY_SIZE(dsa->copy); i++) { 341 struct dsa_hw_desc *copy_desc = &dsa->copy[i]; 342 343 dsa_copy_desc_init(device, copy_desc, src, dst, size, false); 344 345 /* Don't request completions for individual copies. */ 346 copy_desc->flags &= ~IDXD_OP_FLAG_RCR; 347 } 348 349 for (i = 0; i < ARRAY_SIZE(dsa->batch) && count; i++) { 350 struct dsa_hw_desc *batch_desc = &dsa->batch[i]; 351 int nr_copies; 352 353 nr_copies = min(count, dsa->max_copies_per_batch); 354 count -= nr_copies; 355 356 /* 357 * Batches must have at least 2 copies, so handle the case where 358 * there is exactly 1 copy left by doing one less copy in this 359 * batch and then 2 in the next. 360 */ 361 if (count == 1) { 362 nr_copies--; 363 count++; 364 } 365 366 dsa_batch_desc_init(device, batch_desc, nr_copies); 367 368 /* Request a completion for the last batch. */ 369 if (!count) 370 batch_desc->flags |= IDXD_OP_FLAG_RCR; 371 372 dsa_desc_write(device, batch_desc); 373 } 374 375 VFIO_ASSERT_EQ(count, 0, "Failed to start %lu copies.\n", count); 376 } 377 378 static void dsa_memcpy_start(struct vfio_pci_device *device, 379 iova_t src, iova_t dst, u64 size, u64 count) 380 { 381 struct dsa_state *dsa = to_dsa_state(device); 382 383 /* DSA devices require at least 2 copies per batch. */ 384 if (count == 1) 385 dsa_memcpy_one(device, src, dst, size, false); 386 else 387 dsa_memcpy_batch(device, src, dst, size, count); 388 389 dsa->memcpy_count = count; 390 } 391 392 static int dsa_memcpy_wait(struct vfio_pci_device *device) 393 { 394 struct dsa_state *dsa = to_dsa_state(device); 395 int r; 396 397 if (dsa->memcpy_count == 1) 398 r = dsa_completion_wait(device, &dsa->copy_completion); 399 else 400 r = dsa_completion_wait(device, &dsa->batch_completion); 401 402 dsa->memcpy_count = 0; 403 404 return r; 405 } 406 407 static void dsa_send_msi(struct vfio_pci_device *device) 408 { 409 struct dsa_state *dsa = to_dsa_state(device); 410 411 dsa_memcpy_one(device, 412 to_iova(device, &dsa->send_msi_src), 413 to_iova(device, &dsa->send_msi_dst), 414 sizeof(dsa->send_msi_src), true); 415 416 VFIO_ASSERT_EQ(dsa_completion_wait(device, &dsa->copy_completion), 0); 417 } 418 419 const struct vfio_pci_driver_ops dsa_ops = { 420 .name = "dsa", 421 .probe = dsa_probe, 422 .init = dsa_init, 423 .remove = dsa_remove, 424 .memcpy_start = dsa_memcpy_start, 425 .memcpy_wait = dsa_memcpy_wait, 426 .send_msi = dsa_send_msi, 427 }; 428