1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Intel I/OAT DMA Linux driver 4 * Copyright(c) 2004 - 2015 Intel Corporation. 5 */ 6 7 #include <linux/init.h> 8 #include <linux/module.h> 9 #include <linux/slab.h> 10 #include <linux/pci.h> 11 #include <linux/interrupt.h> 12 #include <linux/dmaengine.h> 13 #include <linux/delay.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/workqueue.h> 16 #include <linux/prefetch.h> 17 #include <linux/dca.h> 18 #include <linux/sizes.h> 19 #include "dma.h" 20 #include "registers.h" 21 #include "hw.h" 22 23 #include "../dmaengine.h" 24 25 MODULE_VERSION(IOAT_DMA_VERSION); 26 MODULE_DESCRIPTION("Intel I/OAT DMA Linux driver"); 27 MODULE_LICENSE("Dual BSD/GPL"); 28 MODULE_AUTHOR("Intel Corporation"); 29 30 static const struct pci_device_id ioat_pci_tbl[] = { 31 /* I/OAT v3 platforms */ 32 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) }, 33 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) }, 34 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) }, 35 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) }, 36 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) }, 37 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) }, 38 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) }, 39 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) }, 40 41 /* I/OAT v3.2 platforms */ 42 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF0) }, 43 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF1) }, 44 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF2) }, 45 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF3) }, 46 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF4) }, 47 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF5) }, 48 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF6) }, 49 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF7) }, 50 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF8) }, 51 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF9) }, 52 53 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB0) }, 54 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB1) }, 55 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB2) }, 56 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB3) }, 57 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB4) }, 58 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB5) }, 59 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB6) }, 60 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB7) }, 61 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB8) }, 62 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB9) }, 63 64 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB0) }, 65 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB1) }, 66 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB2) }, 67 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB3) }, 68 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB4) }, 69 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB5) }, 70 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB6) }, 71 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB7) }, 72 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB8) }, 73 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB9) }, 74 75 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW0) }, 76 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW1) }, 77 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW2) }, 78 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW3) }, 79 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW4) }, 80 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW5) }, 81 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW6) }, 82 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW7) }, 83 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW8) }, 84 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW9) }, 85 86 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX0) }, 87 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX1) }, 88 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX2) }, 89 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX3) }, 90 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX4) }, 91 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX5) }, 92 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX6) }, 93 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX7) }, 94 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX8) }, 95 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX9) }, 96 97 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SKX) }, 98 99 /* I/OAT v3.3 platforms */ 100 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0) }, 101 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1) }, 102 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD2) }, 103 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD3) }, 104 105 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE0) }, 106 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE1) }, 107 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE2) }, 108 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE3) }, 109 110 /* I/OAT v3.4 platforms */ 111 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_ICX) }, 112 113 { 0, } 114 }; 115 MODULE_DEVICE_TABLE(pci, ioat_pci_tbl); 116 117 static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id); 118 static void ioat_remove(struct pci_dev *pdev); 119 static void 120 ioat_init_channel(struct ioatdma_device *ioat_dma, 121 struct ioatdma_chan *ioat_chan, int idx); 122 static void ioat_intr_quirk(struct ioatdma_device *ioat_dma); 123 static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma); 124 static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma); 125 126 static int ioat_dca_enabled = 1; 127 module_param(ioat_dca_enabled, int, 0644); 128 MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)"); 129 int ioat_pending_level = 7; 130 module_param(ioat_pending_level, int, 0644); 131 MODULE_PARM_DESC(ioat_pending_level, 132 "high-water mark for pushing ioat descriptors (default: 7)"); 133 static char ioat_interrupt_style[32] = "msix"; 134 module_param_string(ioat_interrupt_style, ioat_interrupt_style, 135 sizeof(ioat_interrupt_style), 0644); 136 MODULE_PARM_DESC(ioat_interrupt_style, 137 "set ioat interrupt style: msix (default), msi, intx"); 138 139 struct kmem_cache *ioat_cache; 140 struct kmem_cache *ioat_sed_cache; 141 142 static bool is_jf_ioat(struct pci_dev *pdev) 143 { 144 switch (pdev->device) { 145 case PCI_DEVICE_ID_INTEL_IOAT_JSF0: 146 case PCI_DEVICE_ID_INTEL_IOAT_JSF1: 147 case PCI_DEVICE_ID_INTEL_IOAT_JSF2: 148 case PCI_DEVICE_ID_INTEL_IOAT_JSF3: 149 case PCI_DEVICE_ID_INTEL_IOAT_JSF4: 150 case PCI_DEVICE_ID_INTEL_IOAT_JSF5: 151 case PCI_DEVICE_ID_INTEL_IOAT_JSF6: 152 case PCI_DEVICE_ID_INTEL_IOAT_JSF7: 153 case PCI_DEVICE_ID_INTEL_IOAT_JSF8: 154 case PCI_DEVICE_ID_INTEL_IOAT_JSF9: 155 return true; 156 default: 157 return false; 158 } 159 } 160 161 static bool is_snb_ioat(struct pci_dev *pdev) 162 { 163 switch (pdev->device) { 164 case PCI_DEVICE_ID_INTEL_IOAT_SNB0: 165 case PCI_DEVICE_ID_INTEL_IOAT_SNB1: 166 case PCI_DEVICE_ID_INTEL_IOAT_SNB2: 167 case PCI_DEVICE_ID_INTEL_IOAT_SNB3: 168 case PCI_DEVICE_ID_INTEL_IOAT_SNB4: 169 case PCI_DEVICE_ID_INTEL_IOAT_SNB5: 170 case PCI_DEVICE_ID_INTEL_IOAT_SNB6: 171 case PCI_DEVICE_ID_INTEL_IOAT_SNB7: 172 case PCI_DEVICE_ID_INTEL_IOAT_SNB8: 173 case PCI_DEVICE_ID_INTEL_IOAT_SNB9: 174 return true; 175 default: 176 return false; 177 } 178 } 179 180 static bool is_ivb_ioat(struct pci_dev *pdev) 181 { 182 switch (pdev->device) { 183 case PCI_DEVICE_ID_INTEL_IOAT_IVB0: 184 case PCI_DEVICE_ID_INTEL_IOAT_IVB1: 185 case PCI_DEVICE_ID_INTEL_IOAT_IVB2: 186 case PCI_DEVICE_ID_INTEL_IOAT_IVB3: 187 case PCI_DEVICE_ID_INTEL_IOAT_IVB4: 188 case PCI_DEVICE_ID_INTEL_IOAT_IVB5: 189 case PCI_DEVICE_ID_INTEL_IOAT_IVB6: 190 case PCI_DEVICE_ID_INTEL_IOAT_IVB7: 191 case PCI_DEVICE_ID_INTEL_IOAT_IVB8: 192 case PCI_DEVICE_ID_INTEL_IOAT_IVB9: 193 return true; 194 default: 195 return false; 196 } 197 198 } 199 200 static bool is_hsw_ioat(struct pci_dev *pdev) 201 { 202 switch (pdev->device) { 203 case PCI_DEVICE_ID_INTEL_IOAT_HSW0: 204 case PCI_DEVICE_ID_INTEL_IOAT_HSW1: 205 case PCI_DEVICE_ID_INTEL_IOAT_HSW2: 206 case PCI_DEVICE_ID_INTEL_IOAT_HSW3: 207 case PCI_DEVICE_ID_INTEL_IOAT_HSW4: 208 case PCI_DEVICE_ID_INTEL_IOAT_HSW5: 209 case PCI_DEVICE_ID_INTEL_IOAT_HSW6: 210 case PCI_DEVICE_ID_INTEL_IOAT_HSW7: 211 case PCI_DEVICE_ID_INTEL_IOAT_HSW8: 212 case PCI_DEVICE_ID_INTEL_IOAT_HSW9: 213 return true; 214 default: 215 return false; 216 } 217 218 } 219 220 static bool is_bdx_ioat(struct pci_dev *pdev) 221 { 222 switch (pdev->device) { 223 case PCI_DEVICE_ID_INTEL_IOAT_BDX0: 224 case PCI_DEVICE_ID_INTEL_IOAT_BDX1: 225 case PCI_DEVICE_ID_INTEL_IOAT_BDX2: 226 case PCI_DEVICE_ID_INTEL_IOAT_BDX3: 227 case PCI_DEVICE_ID_INTEL_IOAT_BDX4: 228 case PCI_DEVICE_ID_INTEL_IOAT_BDX5: 229 case PCI_DEVICE_ID_INTEL_IOAT_BDX6: 230 case PCI_DEVICE_ID_INTEL_IOAT_BDX7: 231 case PCI_DEVICE_ID_INTEL_IOAT_BDX8: 232 case PCI_DEVICE_ID_INTEL_IOAT_BDX9: 233 return true; 234 default: 235 return false; 236 } 237 } 238 239 static inline bool is_skx_ioat(struct pci_dev *pdev) 240 { 241 return (pdev->device == PCI_DEVICE_ID_INTEL_IOAT_SKX) ? true : false; 242 } 243 244 static bool is_xeon_cb32(struct pci_dev *pdev) 245 { 246 return is_jf_ioat(pdev) || is_snb_ioat(pdev) || is_ivb_ioat(pdev) || 247 is_hsw_ioat(pdev) || is_bdx_ioat(pdev) || is_skx_ioat(pdev); 248 } 249 250 bool is_bwd_ioat(struct pci_dev *pdev) 251 { 252 switch (pdev->device) { 253 case PCI_DEVICE_ID_INTEL_IOAT_BWD0: 254 case PCI_DEVICE_ID_INTEL_IOAT_BWD1: 255 case PCI_DEVICE_ID_INTEL_IOAT_BWD2: 256 case PCI_DEVICE_ID_INTEL_IOAT_BWD3: 257 /* even though not Atom, BDX-DE has same DMA silicon */ 258 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0: 259 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1: 260 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2: 261 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3: 262 return true; 263 default: 264 return false; 265 } 266 } 267 268 static bool is_bwd_noraid(struct pci_dev *pdev) 269 { 270 switch (pdev->device) { 271 case PCI_DEVICE_ID_INTEL_IOAT_BWD2: 272 case PCI_DEVICE_ID_INTEL_IOAT_BWD3: 273 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0: 274 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1: 275 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2: 276 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3: 277 return true; 278 default: 279 return false; 280 } 281 282 } 283 284 /* 285 * Perform a IOAT transaction to verify the HW works. 286 */ 287 #define IOAT_TEST_SIZE 2000 288 289 static void ioat_dma_test_callback(void *dma_async_param) 290 { 291 struct completion *cmp = dma_async_param; 292 293 complete(cmp); 294 } 295 296 /** 297 * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works. 298 * @ioat_dma: dma device to be tested 299 */ 300 static int ioat_dma_self_test(struct ioatdma_device *ioat_dma) 301 { 302 int i; 303 u8 *src; 304 u8 *dest; 305 struct dma_device *dma = &ioat_dma->dma_dev; 306 struct device *dev = &ioat_dma->pdev->dev; 307 struct dma_chan *dma_chan; 308 struct dma_async_tx_descriptor *tx; 309 dma_addr_t dma_dest, dma_src; 310 dma_cookie_t cookie; 311 int err = 0; 312 struct completion cmp; 313 unsigned long tmo; 314 unsigned long flags; 315 316 src = kzalloc(IOAT_TEST_SIZE, GFP_KERNEL); 317 if (!src) 318 return -ENOMEM; 319 dest = kzalloc(IOAT_TEST_SIZE, GFP_KERNEL); 320 if (!dest) { 321 kfree(src); 322 return -ENOMEM; 323 } 324 325 /* Fill in src buffer */ 326 for (i = 0; i < IOAT_TEST_SIZE; i++) 327 src[i] = (u8)i; 328 329 /* Start copy, using first DMA channel */ 330 dma_chan = container_of(dma->channels.next, struct dma_chan, 331 device_node); 332 if (dma->device_alloc_chan_resources(dma_chan) < 1) { 333 dev_err(dev, "selftest cannot allocate chan resource\n"); 334 err = -ENODEV; 335 goto out; 336 } 337 338 dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE); 339 if (dma_mapping_error(dev, dma_src)) { 340 dev_err(dev, "mapping src buffer failed\n"); 341 err = -ENOMEM; 342 goto free_resources; 343 } 344 dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); 345 if (dma_mapping_error(dev, dma_dest)) { 346 dev_err(dev, "mapping dest buffer failed\n"); 347 err = -ENOMEM; 348 goto unmap_src; 349 } 350 flags = DMA_PREP_INTERRUPT; 351 tx = ioat_dma->dma_dev.device_prep_dma_memcpy(dma_chan, dma_dest, 352 dma_src, IOAT_TEST_SIZE, 353 flags); 354 if (!tx) { 355 dev_err(dev, "Self-test prep failed, disabling\n"); 356 err = -ENODEV; 357 goto unmap_dma; 358 } 359 360 async_tx_ack(tx); 361 init_completion(&cmp); 362 tx->callback = ioat_dma_test_callback; 363 tx->callback_param = &cmp; 364 cookie = tx->tx_submit(tx); 365 if (cookie < 0) { 366 dev_err(dev, "Self-test setup failed, disabling\n"); 367 err = -ENODEV; 368 goto unmap_dma; 369 } 370 dma->device_issue_pending(dma_chan); 371 372 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); 373 374 if (tmo == 0 || 375 dma->device_tx_status(dma_chan, cookie, NULL) 376 != DMA_COMPLETE) { 377 dev_err(dev, "Self-test copy timed out, disabling\n"); 378 err = -ENODEV; 379 goto unmap_dma; 380 } 381 if (memcmp(src, dest, IOAT_TEST_SIZE)) { 382 dev_err(dev, "Self-test copy failed compare, disabling\n"); 383 err = -ENODEV; 384 goto unmap_dma; 385 } 386 387 unmap_dma: 388 dma_unmap_single(dev, dma_dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); 389 unmap_src: 390 dma_unmap_single(dev, dma_src, IOAT_TEST_SIZE, DMA_TO_DEVICE); 391 free_resources: 392 dma->device_free_chan_resources(dma_chan); 393 out: 394 kfree(src); 395 kfree(dest); 396 return err; 397 } 398 399 /** 400 * ioat_dma_setup_interrupts - setup interrupt handler 401 * @ioat_dma: ioat dma device 402 */ 403 int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma) 404 { 405 struct ioatdma_chan *ioat_chan; 406 struct pci_dev *pdev = ioat_dma->pdev; 407 struct device *dev = &pdev->dev; 408 struct msix_entry *msix; 409 int i, j, msixcnt; 410 int err = -EINVAL; 411 u8 intrctrl = 0; 412 413 if (!strcmp(ioat_interrupt_style, "msix")) 414 goto msix; 415 if (!strcmp(ioat_interrupt_style, "msi")) 416 goto msi; 417 if (!strcmp(ioat_interrupt_style, "intx")) 418 goto intx; 419 dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style); 420 goto err_no_irq; 421 422 msix: 423 /* The number of MSI-X vectors should equal the number of channels */ 424 msixcnt = ioat_dma->chancnt; 425 for (i = 0; i < msixcnt; i++) 426 ioat_dma->msix_entries[i].entry = i; 427 428 err = pci_enable_msix_exact(pdev, ioat_dma->msix_entries, msixcnt); 429 if (err) 430 goto msi; 431 432 for (i = 0; i < msixcnt; i++) { 433 msix = &ioat_dma->msix_entries[i]; 434 ioat_chan = ioat_chan_by_index(ioat_dma, i); 435 err = devm_request_irq(dev, msix->vector, 436 ioat_dma_do_interrupt_msix, 0, 437 "ioat-msix", ioat_chan); 438 if (err) { 439 for (j = 0; j < i; j++) { 440 msix = &ioat_dma->msix_entries[j]; 441 ioat_chan = ioat_chan_by_index(ioat_dma, j); 442 devm_free_irq(dev, msix->vector, ioat_chan); 443 } 444 goto msi; 445 } 446 } 447 intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL; 448 ioat_dma->irq_mode = IOAT_MSIX; 449 goto done; 450 451 msi: 452 err = pci_enable_msi(pdev); 453 if (err) 454 goto intx; 455 456 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0, 457 "ioat-msi", ioat_dma); 458 if (err) { 459 pci_disable_msi(pdev); 460 goto intx; 461 } 462 ioat_dma->irq_mode = IOAT_MSI; 463 goto done; 464 465 intx: 466 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 467 IRQF_SHARED, "ioat-intx", ioat_dma); 468 if (err) 469 goto err_no_irq; 470 471 ioat_dma->irq_mode = IOAT_INTX; 472 done: 473 if (is_bwd_ioat(pdev)) 474 ioat_intr_quirk(ioat_dma); 475 intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN; 476 writeb(intrctrl, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET); 477 return 0; 478 479 err_no_irq: 480 /* Disable all interrupt generation */ 481 writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET); 482 ioat_dma->irq_mode = IOAT_NOIRQ; 483 dev_err(dev, "no usable interrupts\n"); 484 return err; 485 } 486 487 static void ioat_disable_interrupts(struct ioatdma_device *ioat_dma) 488 { 489 /* Disable all interrupt generation */ 490 writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET); 491 } 492 493 static int ioat_probe(struct ioatdma_device *ioat_dma) 494 { 495 int err = -ENODEV; 496 struct dma_device *dma = &ioat_dma->dma_dev; 497 struct pci_dev *pdev = ioat_dma->pdev; 498 struct device *dev = &pdev->dev; 499 500 ioat_dma->completion_pool = dma_pool_create("completion_pool", dev, 501 sizeof(u64), 502 SMP_CACHE_BYTES, 503 SMP_CACHE_BYTES); 504 505 if (!ioat_dma->completion_pool) { 506 err = -ENOMEM; 507 goto err_out; 508 } 509 510 ioat_enumerate_channels(ioat_dma); 511 512 dma_cap_set(DMA_MEMCPY, dma->cap_mask); 513 dma->dev = &pdev->dev; 514 515 if (!ioat_dma->chancnt) { 516 dev_err(dev, "channel enumeration error\n"); 517 goto err_setup_interrupts; 518 } 519 520 err = ioat_dma_setup_interrupts(ioat_dma); 521 if (err) 522 goto err_setup_interrupts; 523 524 err = ioat3_dma_self_test(ioat_dma); 525 if (err) 526 goto err_self_test; 527 528 return 0; 529 530 err_self_test: 531 ioat_disable_interrupts(ioat_dma); 532 err_setup_interrupts: 533 dma_pool_destroy(ioat_dma->completion_pool); 534 err_out: 535 return err; 536 } 537 538 static void ioat_dma_remove(struct ioatdma_device *ioat_dma) 539 { 540 struct dma_device *dma = &ioat_dma->dma_dev; 541 542 ioat_disable_interrupts(ioat_dma); 543 544 ioat_kobject_del(ioat_dma); 545 546 dma_async_device_unregister(dma); 547 } 548 549 /** 550 * ioat_enumerate_channels - find and initialize the device's channels 551 * @ioat_dma: the ioat dma device to be enumerated 552 */ 553 static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma) 554 { 555 struct ioatdma_chan *ioat_chan; 556 struct device *dev = &ioat_dma->pdev->dev; 557 struct dma_device *dma = &ioat_dma->dma_dev; 558 u8 xfercap_log; 559 int chancnt; 560 int i; 561 562 INIT_LIST_HEAD(&dma->channels); 563 chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET); 564 chancnt &= 0x1f; /* bits [4:0] valid */ 565 if (chancnt > ARRAY_SIZE(ioat_dma->idx)) { 566 dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n", 567 chancnt, ARRAY_SIZE(ioat_dma->idx)); 568 chancnt = ARRAY_SIZE(ioat_dma->idx); 569 } 570 xfercap_log = readb(ioat_dma->reg_base + IOAT_XFERCAP_OFFSET); 571 xfercap_log &= 0x1f; /* bits [4:0] valid */ 572 if (xfercap_log == 0) 573 return; 574 dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log); 575 576 for (i = 0; i < chancnt; i++) { 577 ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL); 578 if (!ioat_chan) 579 break; 580 581 ioat_init_channel(ioat_dma, ioat_chan, i); 582 ioat_chan->xfercap_log = xfercap_log; 583 spin_lock_init(&ioat_chan->prep_lock); 584 if (ioat_reset_hw(ioat_chan)) { 585 i = 0; 586 break; 587 } 588 } 589 ioat_dma->chancnt = i; 590 } 591 592 /** 593 * ioat_free_chan_resources - release all the descriptors 594 * @c: the channel to be cleaned 595 */ 596 static void ioat_free_chan_resources(struct dma_chan *c) 597 { 598 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 599 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; 600 struct ioat_ring_ent *desc; 601 const int total_descs = 1 << ioat_chan->alloc_order; 602 int descs; 603 int i; 604 605 /* Before freeing channel resources first check 606 * if they have been previously allocated for this channel. 607 */ 608 if (!ioat_chan->ring) 609 return; 610 611 ioat_stop(ioat_chan); 612 613 if (!test_bit(IOAT_CHAN_DOWN, &ioat_chan->state)) { 614 ioat_reset_hw(ioat_chan); 615 616 /* Put LTR to idle */ 617 if (ioat_dma->version >= IOAT_VER_3_4) 618 writeb(IOAT_CHAN_LTR_SWSEL_IDLE, 619 ioat_chan->reg_base + 620 IOAT_CHAN_LTR_SWSEL_OFFSET); 621 } 622 623 spin_lock_bh(&ioat_chan->cleanup_lock); 624 spin_lock_bh(&ioat_chan->prep_lock); 625 descs = ioat_ring_space(ioat_chan); 626 dev_dbg(to_dev(ioat_chan), "freeing %d idle descriptors\n", descs); 627 for (i = 0; i < descs; i++) { 628 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head + i); 629 ioat_free_ring_ent(desc, c); 630 } 631 632 if (descs < total_descs) 633 dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n", 634 total_descs - descs); 635 636 for (i = 0; i < total_descs - descs; i++) { 637 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail + i); 638 dump_desc_dbg(ioat_chan, desc); 639 ioat_free_ring_ent(desc, c); 640 } 641 642 for (i = 0; i < ioat_chan->desc_chunks; i++) { 643 dma_free_coherent(to_dev(ioat_chan), IOAT_CHUNK_SIZE, 644 ioat_chan->descs[i].virt, 645 ioat_chan->descs[i].hw); 646 ioat_chan->descs[i].virt = NULL; 647 ioat_chan->descs[i].hw = 0; 648 } 649 ioat_chan->desc_chunks = 0; 650 651 kfree(ioat_chan->ring); 652 ioat_chan->ring = NULL; 653 ioat_chan->alloc_order = 0; 654 dma_pool_free(ioat_dma->completion_pool, ioat_chan->completion, 655 ioat_chan->completion_dma); 656 spin_unlock_bh(&ioat_chan->prep_lock); 657 spin_unlock_bh(&ioat_chan->cleanup_lock); 658 659 ioat_chan->last_completion = 0; 660 ioat_chan->completion_dma = 0; 661 ioat_chan->dmacount = 0; 662 } 663 664 /* ioat_alloc_chan_resources - allocate/initialize ioat descriptor ring 665 * @chan: channel to be initialized 666 */ 667 static int ioat_alloc_chan_resources(struct dma_chan *c) 668 { 669 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); 670 struct ioat_ring_ent **ring; 671 u64 status; 672 int order; 673 int i = 0; 674 u32 chanerr; 675 676 /* have we already been set up? */ 677 if (ioat_chan->ring) 678 return 1 << ioat_chan->alloc_order; 679 680 /* Setup register to interrupt and write completion status on error */ 681 writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); 682 683 /* allocate a completion writeback area */ 684 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ 685 ioat_chan->completion = 686 dma_pool_zalloc(ioat_chan->ioat_dma->completion_pool, 687 GFP_NOWAIT, &ioat_chan->completion_dma); 688 if (!ioat_chan->completion) 689 return -ENOMEM; 690 691 writel(((u64)ioat_chan->completion_dma) & 0x00000000FFFFFFFF, 692 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); 693 writel(((u64)ioat_chan->completion_dma) >> 32, 694 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); 695 696 order = IOAT_MAX_ORDER; 697 ring = ioat_alloc_ring(c, order, GFP_NOWAIT); 698 if (!ring) 699 return -ENOMEM; 700 701 spin_lock_bh(&ioat_chan->cleanup_lock); 702 spin_lock_bh(&ioat_chan->prep_lock); 703 ioat_chan->ring = ring; 704 ioat_chan->head = 0; 705 ioat_chan->issued = 0; 706 ioat_chan->tail = 0; 707 ioat_chan->alloc_order = order; 708 set_bit(IOAT_RUN, &ioat_chan->state); 709 spin_unlock_bh(&ioat_chan->prep_lock); 710 spin_unlock_bh(&ioat_chan->cleanup_lock); 711 712 /* Setting up LTR values for 3.4 or later */ 713 if (ioat_chan->ioat_dma->version >= IOAT_VER_3_4) { 714 u32 lat_val; 715 716 lat_val = IOAT_CHAN_LTR_ACTIVE_SNVAL | 717 IOAT_CHAN_LTR_ACTIVE_SNLATSCALE | 718 IOAT_CHAN_LTR_ACTIVE_SNREQMNT; 719 writel(lat_val, ioat_chan->reg_base + 720 IOAT_CHAN_LTR_ACTIVE_OFFSET); 721 722 lat_val = IOAT_CHAN_LTR_IDLE_SNVAL | 723 IOAT_CHAN_LTR_IDLE_SNLATSCALE | 724 IOAT_CHAN_LTR_IDLE_SNREQMNT; 725 writel(lat_val, ioat_chan->reg_base + 726 IOAT_CHAN_LTR_IDLE_OFFSET); 727 728 /* Select to active */ 729 writeb(IOAT_CHAN_LTR_SWSEL_ACTIVE, 730 ioat_chan->reg_base + 731 IOAT_CHAN_LTR_SWSEL_OFFSET); 732 } 733 734 ioat_start_null_desc(ioat_chan); 735 736 /* check that we got off the ground */ 737 do { 738 udelay(1); 739 status = ioat_chansts(ioat_chan); 740 } while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status)); 741 742 if (is_ioat_active(status) || is_ioat_idle(status)) 743 return 1 << ioat_chan->alloc_order; 744 745 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); 746 747 dev_WARN(to_dev(ioat_chan), 748 "failed to start channel chanerr: %#x\n", chanerr); 749 ioat_free_chan_resources(c); 750 return -EFAULT; 751 } 752 753 /* common channel initialization */ 754 static void 755 ioat_init_channel(struct ioatdma_device *ioat_dma, 756 struct ioatdma_chan *ioat_chan, int idx) 757 { 758 struct dma_device *dma = &ioat_dma->dma_dev; 759 760 ioat_chan->ioat_dma = ioat_dma; 761 ioat_chan->reg_base = ioat_dma->reg_base + (0x80 * (idx + 1)); 762 spin_lock_init(&ioat_chan->cleanup_lock); 763 ioat_chan->dma_chan.device = dma; 764 dma_cookie_init(&ioat_chan->dma_chan); 765 list_add_tail(&ioat_chan->dma_chan.device_node, &dma->channels); 766 ioat_dma->idx[idx] = ioat_chan; 767 timer_setup(&ioat_chan->timer, ioat_timer_event, 0); 768 tasklet_setup(&ioat_chan->cleanup_task, ioat_cleanup_event); 769 } 770 771 #define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */ 772 static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma) 773 { 774 int i, src_idx; 775 struct page *dest; 776 struct page *xor_srcs[IOAT_NUM_SRC_TEST]; 777 struct page *xor_val_srcs[IOAT_NUM_SRC_TEST + 1]; 778 dma_addr_t dma_srcs[IOAT_NUM_SRC_TEST + 1]; 779 dma_addr_t dest_dma; 780 struct dma_async_tx_descriptor *tx; 781 struct dma_chan *dma_chan; 782 dma_cookie_t cookie; 783 u8 cmp_byte = 0; 784 u32 cmp_word; 785 u32 xor_val_result; 786 int err = 0; 787 struct completion cmp; 788 unsigned long tmo; 789 struct device *dev = &ioat_dma->pdev->dev; 790 struct dma_device *dma = &ioat_dma->dma_dev; 791 u8 op = 0; 792 793 dev_dbg(dev, "%s\n", __func__); 794 795 if (!dma_has_cap(DMA_XOR, dma->cap_mask)) 796 return 0; 797 798 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) { 799 xor_srcs[src_idx] = alloc_page(GFP_KERNEL); 800 if (!xor_srcs[src_idx]) { 801 while (src_idx--) 802 __free_page(xor_srcs[src_idx]); 803 return -ENOMEM; 804 } 805 } 806 807 dest = alloc_page(GFP_KERNEL); 808 if (!dest) { 809 while (src_idx--) 810 __free_page(xor_srcs[src_idx]); 811 return -ENOMEM; 812 } 813 814 /* Fill in src buffers */ 815 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) { 816 u8 *ptr = page_address(xor_srcs[src_idx]); 817 818 for (i = 0; i < PAGE_SIZE; i++) 819 ptr[i] = (1 << src_idx); 820 } 821 822 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) 823 cmp_byte ^= (u8) (1 << src_idx); 824 825 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | 826 (cmp_byte << 8) | cmp_byte; 827 828 memset(page_address(dest), 0, PAGE_SIZE); 829 830 dma_chan = container_of(dma->channels.next, struct dma_chan, 831 device_node); 832 if (dma->device_alloc_chan_resources(dma_chan) < 1) { 833 err = -ENODEV; 834 goto out; 835 } 836 837 /* test xor */ 838 op = IOAT_OP_XOR; 839 840 dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE); 841 if (dma_mapping_error(dev, dest_dma)) { 842 err = -ENOMEM; 843 goto free_resources; 844 } 845 846 for (i = 0; i < IOAT_NUM_SRC_TEST; i++) { 847 dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE, 848 DMA_TO_DEVICE); 849 if (dma_mapping_error(dev, dma_srcs[i])) { 850 err = -ENOMEM; 851 goto dma_unmap; 852 } 853 } 854 tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs, 855 IOAT_NUM_SRC_TEST, PAGE_SIZE, 856 DMA_PREP_INTERRUPT); 857 858 if (!tx) { 859 dev_err(dev, "Self-test xor prep failed\n"); 860 err = -ENODEV; 861 goto dma_unmap; 862 } 863 864 async_tx_ack(tx); 865 init_completion(&cmp); 866 tx->callback = ioat_dma_test_callback; 867 tx->callback_param = &cmp; 868 cookie = tx->tx_submit(tx); 869 if (cookie < 0) { 870 dev_err(dev, "Self-test xor setup failed\n"); 871 err = -ENODEV; 872 goto dma_unmap; 873 } 874 dma->device_issue_pending(dma_chan); 875 876 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); 877 878 if (tmo == 0 || 879 dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { 880 dev_err(dev, "Self-test xor timed out\n"); 881 err = -ENODEV; 882 goto dma_unmap; 883 } 884 885 for (i = 0; i < IOAT_NUM_SRC_TEST; i++) 886 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE); 887 888 dma_sync_single_for_cpu(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); 889 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { 890 u32 *ptr = page_address(dest); 891 892 if (ptr[i] != cmp_word) { 893 dev_err(dev, "Self-test xor failed compare\n"); 894 err = -ENODEV; 895 goto free_resources; 896 } 897 } 898 dma_sync_single_for_device(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); 899 900 dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); 901 902 /* skip validate if the capability is not present */ 903 if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask)) 904 goto free_resources; 905 906 op = IOAT_OP_XOR_VAL; 907 908 /* validate the sources with the destination page */ 909 for (i = 0; i < IOAT_NUM_SRC_TEST; i++) 910 xor_val_srcs[i] = xor_srcs[i]; 911 xor_val_srcs[i] = dest; 912 913 xor_val_result = 1; 914 915 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) { 916 dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE, 917 DMA_TO_DEVICE); 918 if (dma_mapping_error(dev, dma_srcs[i])) { 919 err = -ENOMEM; 920 goto dma_unmap; 921 } 922 } 923 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, 924 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, 925 &xor_val_result, DMA_PREP_INTERRUPT); 926 if (!tx) { 927 dev_err(dev, "Self-test zero prep failed\n"); 928 err = -ENODEV; 929 goto dma_unmap; 930 } 931 932 async_tx_ack(tx); 933 init_completion(&cmp); 934 tx->callback = ioat_dma_test_callback; 935 tx->callback_param = &cmp; 936 cookie = tx->tx_submit(tx); 937 if (cookie < 0) { 938 dev_err(dev, "Self-test zero setup failed\n"); 939 err = -ENODEV; 940 goto dma_unmap; 941 } 942 dma->device_issue_pending(dma_chan); 943 944 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); 945 946 if (tmo == 0 || 947 dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { 948 dev_err(dev, "Self-test validate timed out\n"); 949 err = -ENODEV; 950 goto dma_unmap; 951 } 952 953 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) 954 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE); 955 956 if (xor_val_result != 0) { 957 dev_err(dev, "Self-test validate failed compare\n"); 958 err = -ENODEV; 959 goto free_resources; 960 } 961 962 memset(page_address(dest), 0, PAGE_SIZE); 963 964 /* test for non-zero parity sum */ 965 op = IOAT_OP_XOR_VAL; 966 967 xor_val_result = 0; 968 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) { 969 dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE, 970 DMA_TO_DEVICE); 971 if (dma_mapping_error(dev, dma_srcs[i])) { 972 err = -ENOMEM; 973 goto dma_unmap; 974 } 975 } 976 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, 977 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, 978 &xor_val_result, DMA_PREP_INTERRUPT); 979 if (!tx) { 980 dev_err(dev, "Self-test 2nd zero prep failed\n"); 981 err = -ENODEV; 982 goto dma_unmap; 983 } 984 985 async_tx_ack(tx); 986 init_completion(&cmp); 987 tx->callback = ioat_dma_test_callback; 988 tx->callback_param = &cmp; 989 cookie = tx->tx_submit(tx); 990 if (cookie < 0) { 991 dev_err(dev, "Self-test 2nd zero setup failed\n"); 992 err = -ENODEV; 993 goto dma_unmap; 994 } 995 dma->device_issue_pending(dma_chan); 996 997 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); 998 999 if (tmo == 0 || 1000 dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { 1001 dev_err(dev, "Self-test 2nd validate timed out\n"); 1002 err = -ENODEV; 1003 goto dma_unmap; 1004 } 1005 1006 if (xor_val_result != SUM_CHECK_P_RESULT) { 1007 dev_err(dev, "Self-test validate failed compare\n"); 1008 err = -ENODEV; 1009 goto dma_unmap; 1010 } 1011 1012 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) 1013 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE); 1014 1015 goto free_resources; 1016 dma_unmap: 1017 if (op == IOAT_OP_XOR) { 1018 while (--i >= 0) 1019 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, 1020 DMA_TO_DEVICE); 1021 dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); 1022 } else if (op == IOAT_OP_XOR_VAL) { 1023 while (--i >= 0) 1024 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, 1025 DMA_TO_DEVICE); 1026 } 1027 free_resources: 1028 dma->device_free_chan_resources(dma_chan); 1029 out: 1030 src_idx = IOAT_NUM_SRC_TEST; 1031 while (src_idx--) 1032 __free_page(xor_srcs[src_idx]); 1033 __free_page(dest); 1034 return err; 1035 } 1036 1037 static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma) 1038 { 1039 int rc; 1040 1041 rc = ioat_dma_self_test(ioat_dma); 1042 if (rc) 1043 return rc; 1044 1045 rc = ioat_xor_val_self_test(ioat_dma); 1046 1047 return rc; 1048 } 1049 1050 static void ioat_intr_quirk(struct ioatdma_device *ioat_dma) 1051 { 1052 struct dma_device *dma; 1053 struct dma_chan *c; 1054 struct ioatdma_chan *ioat_chan; 1055 u32 errmask; 1056 1057 dma = &ioat_dma->dma_dev; 1058 1059 /* 1060 * if we have descriptor write back error status, we mask the 1061 * error interrupts 1062 */ 1063 if (ioat_dma->cap & IOAT_CAP_DWBES) { 1064 list_for_each_entry(c, &dma->channels, device_node) { 1065 ioat_chan = to_ioat_chan(c); 1066 errmask = readl(ioat_chan->reg_base + 1067 IOAT_CHANERR_MASK_OFFSET); 1068 errmask |= IOAT_CHANERR_XOR_P_OR_CRC_ERR | 1069 IOAT_CHANERR_XOR_Q_ERR; 1070 writel(errmask, ioat_chan->reg_base + 1071 IOAT_CHANERR_MASK_OFFSET); 1072 } 1073 } 1074 } 1075 1076 static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca) 1077 { 1078 struct pci_dev *pdev = ioat_dma->pdev; 1079 int dca_en = system_has_dca_enabled(pdev); 1080 struct dma_device *dma; 1081 struct dma_chan *c; 1082 struct ioatdma_chan *ioat_chan; 1083 int err; 1084 u16 val16; 1085 1086 dma = &ioat_dma->dma_dev; 1087 dma->device_prep_dma_memcpy = ioat_dma_prep_memcpy_lock; 1088 dma->device_issue_pending = ioat_issue_pending; 1089 dma->device_alloc_chan_resources = ioat_alloc_chan_resources; 1090 dma->device_free_chan_resources = ioat_free_chan_resources; 1091 1092 dma_cap_set(DMA_INTERRUPT, dma->cap_mask); 1093 dma->device_prep_dma_interrupt = ioat_prep_interrupt_lock; 1094 1095 ioat_dma->cap = readl(ioat_dma->reg_base + IOAT_DMA_CAP_OFFSET); 1096 1097 if (is_xeon_cb32(pdev) || is_bwd_noraid(pdev)) 1098 ioat_dma->cap &= 1099 ~(IOAT_CAP_XOR | IOAT_CAP_PQ | IOAT_CAP_RAID16SS); 1100 1101 /* dca is incompatible with raid operations */ 1102 if (dca_en && (ioat_dma->cap & (IOAT_CAP_XOR|IOAT_CAP_PQ))) 1103 ioat_dma->cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ); 1104 1105 if (ioat_dma->cap & IOAT_CAP_XOR) { 1106 dma->max_xor = 8; 1107 1108 dma_cap_set(DMA_XOR, dma->cap_mask); 1109 dma->device_prep_dma_xor = ioat_prep_xor; 1110 1111 dma_cap_set(DMA_XOR_VAL, dma->cap_mask); 1112 dma->device_prep_dma_xor_val = ioat_prep_xor_val; 1113 } 1114 1115 if (ioat_dma->cap & IOAT_CAP_PQ) { 1116 1117 dma->device_prep_dma_pq = ioat_prep_pq; 1118 dma->device_prep_dma_pq_val = ioat_prep_pq_val; 1119 dma_cap_set(DMA_PQ, dma->cap_mask); 1120 dma_cap_set(DMA_PQ_VAL, dma->cap_mask); 1121 1122 if (ioat_dma->cap & IOAT_CAP_RAID16SS) 1123 dma_set_maxpq(dma, 16, 0); 1124 else 1125 dma_set_maxpq(dma, 8, 0); 1126 1127 if (!(ioat_dma->cap & IOAT_CAP_XOR)) { 1128 dma->device_prep_dma_xor = ioat_prep_pqxor; 1129 dma->device_prep_dma_xor_val = ioat_prep_pqxor_val; 1130 dma_cap_set(DMA_XOR, dma->cap_mask); 1131 dma_cap_set(DMA_XOR_VAL, dma->cap_mask); 1132 1133 if (ioat_dma->cap & IOAT_CAP_RAID16SS) 1134 dma->max_xor = 16; 1135 else 1136 dma->max_xor = 8; 1137 } 1138 } 1139 1140 dma->device_tx_status = ioat_tx_status; 1141 1142 /* starting with CB3.3 super extended descriptors are supported */ 1143 if (ioat_dma->cap & IOAT_CAP_RAID16SS) { 1144 char pool_name[14]; 1145 int i; 1146 1147 for (i = 0; i < MAX_SED_POOLS; i++) { 1148 snprintf(pool_name, 14, "ioat_hw%d_sed", i); 1149 1150 /* allocate SED DMA pool */ 1151 ioat_dma->sed_hw_pool[i] = dmam_pool_create(pool_name, 1152 &pdev->dev, 1153 SED_SIZE * (i + 1), 64, 0); 1154 if (!ioat_dma->sed_hw_pool[i]) 1155 return -ENOMEM; 1156 1157 } 1158 } 1159 1160 if (!(ioat_dma->cap & (IOAT_CAP_XOR | IOAT_CAP_PQ))) 1161 dma_cap_set(DMA_PRIVATE, dma->cap_mask); 1162 1163 err = ioat_probe(ioat_dma); 1164 if (err) 1165 return err; 1166 1167 list_for_each_entry(c, &dma->channels, device_node) { 1168 ioat_chan = to_ioat_chan(c); 1169 writel(IOAT_DMA_DCA_ANY_CPU, 1170 ioat_chan->reg_base + IOAT_DCACTRL_OFFSET); 1171 } 1172 1173 err = dma_async_device_register(&ioat_dma->dma_dev); 1174 if (err) 1175 goto err_disable_interrupts; 1176 1177 ioat_kobject_add(ioat_dma, &ioat_ktype); 1178 1179 if (dca) 1180 ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base); 1181 1182 /* disable relaxed ordering */ 1183 err = pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &val16); 1184 if (err) { 1185 err = pcibios_err_to_errno(err); 1186 goto err_disable_interrupts; 1187 } 1188 1189 /* clear relaxed ordering enable */ 1190 val16 &= ~PCI_EXP_DEVCTL_RELAX_EN; 1191 err = pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, val16); 1192 if (err) { 1193 err = pcibios_err_to_errno(err); 1194 goto err_disable_interrupts; 1195 } 1196 1197 if (ioat_dma->cap & IOAT_CAP_DPS) 1198 writeb(ioat_pending_level + 1, 1199 ioat_dma->reg_base + IOAT_PREFETCH_LIMIT_OFFSET); 1200 1201 return 0; 1202 1203 err_disable_interrupts: 1204 ioat_disable_interrupts(ioat_dma); 1205 dma_pool_destroy(ioat_dma->completion_pool); 1206 return err; 1207 } 1208 1209 static void ioat_shutdown(struct pci_dev *pdev) 1210 { 1211 struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev); 1212 struct ioatdma_chan *ioat_chan; 1213 int i; 1214 1215 if (!ioat_dma) 1216 return; 1217 1218 for (i = 0; i < IOAT_MAX_CHANS; i++) { 1219 ioat_chan = ioat_dma->idx[i]; 1220 if (!ioat_chan) 1221 continue; 1222 1223 spin_lock_bh(&ioat_chan->prep_lock); 1224 set_bit(IOAT_CHAN_DOWN, &ioat_chan->state); 1225 spin_unlock_bh(&ioat_chan->prep_lock); 1226 /* 1227 * Synchronization rule for del_timer_sync(): 1228 * - The caller must not hold locks which would prevent 1229 * completion of the timer's handler. 1230 * So prep_lock cannot be held before calling it. 1231 */ 1232 del_timer_sync(&ioat_chan->timer); 1233 1234 /* this should quiesce then reset */ 1235 ioat_reset_hw(ioat_chan); 1236 } 1237 1238 ioat_disable_interrupts(ioat_dma); 1239 } 1240 1241 static void ioat_resume(struct ioatdma_device *ioat_dma) 1242 { 1243 struct ioatdma_chan *ioat_chan; 1244 u32 chanerr; 1245 int i; 1246 1247 for (i = 0; i < IOAT_MAX_CHANS; i++) { 1248 ioat_chan = ioat_dma->idx[i]; 1249 if (!ioat_chan) 1250 continue; 1251 1252 spin_lock_bh(&ioat_chan->prep_lock); 1253 clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state); 1254 spin_unlock_bh(&ioat_chan->prep_lock); 1255 1256 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); 1257 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET); 1258 1259 /* no need to reset as shutdown already did that */ 1260 } 1261 } 1262 1263 #define DRV_NAME "ioatdma" 1264 1265 static pci_ers_result_t ioat_pcie_error_detected(struct pci_dev *pdev, 1266 pci_channel_state_t error) 1267 { 1268 dev_dbg(&pdev->dev, "%s: PCIe AER error %d\n", DRV_NAME, error); 1269 1270 /* quiesce and block I/O */ 1271 ioat_shutdown(pdev); 1272 1273 return PCI_ERS_RESULT_NEED_RESET; 1274 } 1275 1276 static pci_ers_result_t ioat_pcie_error_slot_reset(struct pci_dev *pdev) 1277 { 1278 pci_ers_result_t result = PCI_ERS_RESULT_RECOVERED; 1279 1280 dev_dbg(&pdev->dev, "%s post reset handling\n", DRV_NAME); 1281 1282 if (pci_enable_device_mem(pdev) < 0) { 1283 dev_err(&pdev->dev, 1284 "Failed to enable PCIe device after reset.\n"); 1285 result = PCI_ERS_RESULT_DISCONNECT; 1286 } else { 1287 pci_set_master(pdev); 1288 pci_restore_state(pdev); 1289 pci_save_state(pdev); 1290 pci_wake_from_d3(pdev, false); 1291 } 1292 1293 return result; 1294 } 1295 1296 static void ioat_pcie_error_resume(struct pci_dev *pdev) 1297 { 1298 struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev); 1299 1300 dev_dbg(&pdev->dev, "%s: AER handling resuming\n", DRV_NAME); 1301 1302 /* initialize and bring everything back */ 1303 ioat_resume(ioat_dma); 1304 } 1305 1306 static const struct pci_error_handlers ioat_err_handler = { 1307 .error_detected = ioat_pcie_error_detected, 1308 .slot_reset = ioat_pcie_error_slot_reset, 1309 .resume = ioat_pcie_error_resume, 1310 }; 1311 1312 static struct pci_driver ioat_pci_driver = { 1313 .name = DRV_NAME, 1314 .id_table = ioat_pci_tbl, 1315 .probe = ioat_pci_probe, 1316 .remove = ioat_remove, 1317 .shutdown = ioat_shutdown, 1318 .err_handler = &ioat_err_handler, 1319 }; 1320 1321 static void release_ioatdma(struct dma_device *device) 1322 { 1323 struct ioatdma_device *d = to_ioatdma_device(device); 1324 int i; 1325 1326 for (i = 0; i < IOAT_MAX_CHANS; i++) 1327 kfree(d->idx[i]); 1328 1329 dma_pool_destroy(d->completion_pool); 1330 kfree(d); 1331 } 1332 1333 static struct ioatdma_device * 1334 alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase) 1335 { 1336 struct ioatdma_device *d = kzalloc(sizeof(*d), GFP_KERNEL); 1337 1338 if (!d) 1339 return NULL; 1340 d->pdev = pdev; 1341 d->reg_base = iobase; 1342 d->dma_dev.device_release = release_ioatdma; 1343 return d; 1344 } 1345 1346 static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1347 { 1348 void __iomem * const *iomap; 1349 struct device *dev = &pdev->dev; 1350 struct ioatdma_device *device; 1351 unsigned int i; 1352 u8 version; 1353 int err; 1354 1355 err = pcim_enable_device(pdev); 1356 if (err) 1357 return err; 1358 1359 err = pcim_iomap_regions(pdev, 1 << IOAT_MMIO_BAR, DRV_NAME); 1360 if (err) 1361 return err; 1362 iomap = pcim_iomap_table(pdev); 1363 if (!iomap) 1364 return -ENOMEM; 1365 1366 version = readb(iomap[IOAT_MMIO_BAR] + IOAT_VER_OFFSET); 1367 if (version < IOAT_VER_3_0) 1368 return -ENODEV; 1369 1370 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1371 if (err) 1372 return err; 1373 1374 device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]); 1375 if (!device) 1376 return -ENOMEM; 1377 pci_set_master(pdev); 1378 pci_set_drvdata(pdev, device); 1379 1380 device->version = version; 1381 if (device->version >= IOAT_VER_3_4) 1382 ioat_dca_enabled = 0; 1383 1384 if (is_skx_ioat(pdev)) 1385 device->version = IOAT_VER_3_2; 1386 1387 err = ioat3_dma_probe(device, ioat_dca_enabled); 1388 if (err) { 1389 for (i = 0; i < IOAT_MAX_CHANS; i++) 1390 kfree(device->idx[i]); 1391 kfree(device); 1392 dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n"); 1393 return -ENODEV; 1394 } 1395 1396 return 0; 1397 } 1398 1399 static void ioat_remove(struct pci_dev *pdev) 1400 { 1401 struct ioatdma_device *device = pci_get_drvdata(pdev); 1402 1403 if (!device) 1404 return; 1405 1406 ioat_shutdown(pdev); 1407 1408 dev_err(&pdev->dev, "Removing dma and dca services\n"); 1409 if (device->dca) { 1410 unregister_dca_provider(device->dca, &pdev->dev); 1411 free_dca_provider(device->dca); 1412 device->dca = NULL; 1413 } 1414 1415 ioat_dma_remove(device); 1416 } 1417 1418 static int __init ioat_init_module(void) 1419 { 1420 int err = -ENOMEM; 1421 1422 pr_info("%s: Intel(R) QuickData Technology Driver %s\n", 1423 DRV_NAME, IOAT_DMA_VERSION); 1424 1425 ioat_cache = kmem_cache_create("ioat", sizeof(struct ioat_ring_ent), 1426 0, SLAB_HWCACHE_ALIGN, NULL); 1427 if (!ioat_cache) 1428 return -ENOMEM; 1429 1430 ioat_sed_cache = KMEM_CACHE(ioat_sed_ent, 0); 1431 if (!ioat_sed_cache) 1432 goto err_ioat_cache; 1433 1434 err = pci_register_driver(&ioat_pci_driver); 1435 if (err) 1436 goto err_ioat3_cache; 1437 1438 return 0; 1439 1440 err_ioat3_cache: 1441 kmem_cache_destroy(ioat_sed_cache); 1442 1443 err_ioat_cache: 1444 kmem_cache_destroy(ioat_cache); 1445 1446 return err; 1447 } 1448 module_init(ioat_init_module); 1449 1450 static void __exit ioat_exit_module(void) 1451 { 1452 pci_unregister_driver(&ioat_pci_driver); 1453 kmem_cache_destroy(ioat_sed_cache); 1454 kmem_cache_destroy(ioat_cache); 1455 } 1456 module_exit(ioat_exit_module); 1457