init.c (1a50d1467077c0d6f504bdbb66e6fa0dcd5b3b01) init.c (6e2fb806e08d46cbeb96c1000ef531a92d3b2e9a)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Intel I/OAT DMA Linux driver
4 * Copyright(c) 2004 - 2015 Intel Corporation.
5 */
6
7#include <linux/init.h>
8#include <linux/module.h>

--- 9 unchanged lines hidden (view full) ---

18#include <linux/sizes.h>
19#include "dma.h"
20#include "registers.h"
21#include "hw.h"
22
23#include "../dmaengine.h"
24
25MODULE_VERSION(IOAT_DMA_VERSION);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Intel I/OAT DMA Linux driver
4 * Copyright(c) 2004 - 2015 Intel Corporation.
5 */
6
7#include <linux/init.h>
8#include <linux/module.h>

--- 9 unchanged lines hidden (view full) ---

18#include <linux/sizes.h>
19#include "dma.h"
20#include "registers.h"
21#include "hw.h"
22
23#include "../dmaengine.h"
24
25MODULE_VERSION(IOAT_DMA_VERSION);
26MODULE_DESCRIPTION("Intel I/OAT DMA Linux driver");
26MODULE_LICENSE("Dual BSD/GPL");
27MODULE_AUTHOR("Intel Corporation");
28
29static const struct pci_device_id ioat_pci_tbl[] = {
30 /* I/OAT v3 platforms */
31 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) },
32 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) },
33 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) },

--- 495 unchanged lines hidden (view full) ---

529err_self_test:
530 ioat_disable_interrupts(ioat_dma);
531err_setup_interrupts:
532 dma_pool_destroy(ioat_dma->completion_pool);
533err_out:
534 return err;
535}
536
27MODULE_LICENSE("Dual BSD/GPL");
28MODULE_AUTHOR("Intel Corporation");
29
30static const struct pci_device_id ioat_pci_tbl[] = {
31 /* I/OAT v3 platforms */
32 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) },
33 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) },
34 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) },

--- 495 unchanged lines hidden (view full) ---

530err_self_test:
531 ioat_disable_interrupts(ioat_dma);
532err_setup_interrupts:
533 dma_pool_destroy(ioat_dma->completion_pool);
534err_out:
535 return err;
536}
537
538static int ioat_register(struct ioatdma_device *ioat_dma)
539{
540 int err = dma_async_device_register(&ioat_dma->dma_dev);
541
542 if (err) {
543 ioat_disable_interrupts(ioat_dma);
544 dma_pool_destroy(ioat_dma->completion_pool);
545 }
546
547 return err;
548}
549
537static void ioat_dma_remove(struct ioatdma_device *ioat_dma)
538{
539 struct dma_device *dma = &ioat_dma->dma_dev;
540
541 ioat_disable_interrupts(ioat_dma);
542
543 ioat_kobject_del(ioat_dma);
544

--- 619 unchanged lines hidden (view full) ---

1164 return err;
1165
1166 list_for_each_entry(c, &dma->channels, device_node) {
1167 ioat_chan = to_ioat_chan(c);
1168 writel(IOAT_DMA_DCA_ANY_CPU,
1169 ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
1170 }
1171
550static void ioat_dma_remove(struct ioatdma_device *ioat_dma)
551{
552 struct dma_device *dma = &ioat_dma->dma_dev;
553
554 ioat_disable_interrupts(ioat_dma);
555
556 ioat_kobject_del(ioat_dma);
557

--- 619 unchanged lines hidden (view full) ---

1177 return err;
1178
1179 list_for_each_entry(c, &dma->channels, device_node) {
1180 ioat_chan = to_ioat_chan(c);
1181 writel(IOAT_DMA_DCA_ANY_CPU,
1182 ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
1183 }
1184
1172 err = dma_async_device_register(&ioat_dma->dma_dev);
1185 err = ioat_register(ioat_dma);
1173 if (err)
1186 if (err)
1174 goto err_disable_interrupts;
1187 return err;
1175
1176 ioat_kobject_add(ioat_dma, &ioat_ktype);
1177
1178 if (dca)
1179 ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base);
1180
1181 /* disable relaxed ordering */
1182 err = pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &val16);
1188
1189 ioat_kobject_add(ioat_dma, &ioat_ktype);
1190
1191 if (dca)
1192 ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base);
1193
1194 /* disable relaxed ordering */
1195 err = pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &val16);
1183 if (err) {
1184 err = pcibios_err_to_errno(err);
1185 goto err_disable_interrupts;
1186 }
1196 if (err)
1197 return pcibios_err_to_errno(err);
1187
1188 /* clear relaxed ordering enable */
1189 val16 &= ~PCI_EXP_DEVCTL_RELAX_EN;
1190 err = pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, val16);
1198
1199 /* clear relaxed ordering enable */
1200 val16 &= ~PCI_EXP_DEVCTL_RELAX_EN;
1201 err = pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, val16);
1191 if (err) {
1192 err = pcibios_err_to_errno(err);
1193 goto err_disable_interrupts;
1194 }
1202 if (err)
1203 return pcibios_err_to_errno(err);
1195
1196 if (ioat_dma->cap & IOAT_CAP_DPS)
1197 writeb(ioat_pending_level + 1,
1198 ioat_dma->reg_base + IOAT_PREFETCH_LIMIT_OFFSET);
1199
1200 return 0;
1204
1205 if (ioat_dma->cap & IOAT_CAP_DPS)
1206 writeb(ioat_pending_level + 1,
1207 ioat_dma->reg_base + IOAT_PREFETCH_LIMIT_OFFSET);
1208
1209 return 0;
1201
1202err_disable_interrupts:
1203 ioat_disable_interrupts(ioat_dma);
1204 dma_pool_destroy(ioat_dma->completion_pool);
1205 return err;
1206}
1207
1208static void ioat_shutdown(struct pci_dev *pdev)
1209{
1210 struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev);
1211 struct ioatdma_chan *ioat_chan;
1212 int i;
1213

--- 128 unchanged lines hidden (view full) ---

1342 return d;
1343}
1344
1345static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1346{
1347 void __iomem * const *iomap;
1348 struct device *dev = &pdev->dev;
1349 struct ioatdma_device *device;
1210}
1211
1212static void ioat_shutdown(struct pci_dev *pdev)
1213{
1214 struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev);
1215 struct ioatdma_chan *ioat_chan;
1216 int i;
1217

--- 128 unchanged lines hidden (view full) ---

1346 return d;
1347}
1348
1349static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1350{
1351 void __iomem * const *iomap;
1352 struct device *dev = &pdev->dev;
1353 struct ioatdma_device *device;
1350 unsigned int i;
1351 u8 version;
1352 int err;
1353
1354 err = pcim_enable_device(pdev);
1355 if (err)
1356 return err;
1357
1358 err = pcim_iomap_regions(pdev, 1 << IOAT_MMIO_BAR, DRV_NAME);
1359 if (err)
1360 return err;
1361 iomap = pcim_iomap_table(pdev);
1362 if (!iomap)
1363 return -ENOMEM;
1364
1354 int err;
1355
1356 err = pcim_enable_device(pdev);
1357 if (err)
1358 return err;
1359
1360 err = pcim_iomap_regions(pdev, 1 << IOAT_MMIO_BAR, DRV_NAME);
1361 if (err)
1362 return err;
1363 iomap = pcim_iomap_table(pdev);
1364 if (!iomap)
1365 return -ENOMEM;
1366
1365 version = readb(iomap[IOAT_MMIO_BAR] + IOAT_VER_OFFSET);
1366 if (version < IOAT_VER_3_0)
1367 return -ENODEV;
1368
1369 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1370 if (err)
1371 return err;
1372
1373 device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]);
1374 if (!device)
1375 return -ENOMEM;
1376 pci_set_master(pdev);
1377 pci_set_drvdata(pdev, device);
1378
1367 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1368 if (err)
1369 return err;
1370
1371 device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]);
1372 if (!device)
1373 return -ENOMEM;
1374 pci_set_master(pdev);
1375 pci_set_drvdata(pdev, device);
1376
1379 device->version = version;
1377 device->version = readb(device->reg_base + IOAT_VER_OFFSET);
1380 if (device->version >= IOAT_VER_3_4)
1381 ioat_dca_enabled = 0;
1378 if (device->version >= IOAT_VER_3_4)
1379 ioat_dca_enabled = 0;
1380 if (device->version >= IOAT_VER_3_0) {
1381 if (is_skx_ioat(pdev))
1382 device->version = IOAT_VER_3_2;
1383 err = ioat3_dma_probe(device, ioat_dca_enabled);
1384 } else
1385 return -ENODEV;
1382
1386
1383 if (is_skx_ioat(pdev))
1384 device->version = IOAT_VER_3_2;
1385
1386 err = ioat3_dma_probe(device, ioat_dca_enabled);
1387 if (err) {
1387 if (err) {
1388 for (i = 0; i < IOAT_MAX_CHANS; i++)
1389 kfree(device->idx[i]);
1390 kfree(device);
1391 dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n");
1392 return -ENODEV;
1393 }
1394
1395 return 0;
1396}
1397
1398static void ioat_remove(struct pci_dev *pdev)

--- 45 unchanged lines hidden (view full) ---

1444
1445 return err;
1446}
1447module_init(ioat_init_module);
1448
1449static void __exit ioat_exit_module(void)
1450{
1451 pci_unregister_driver(&ioat_pci_driver);
1388 dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n");
1389 return -ENODEV;
1390 }
1391
1392 return 0;
1393}
1394
1395static void ioat_remove(struct pci_dev *pdev)

--- 45 unchanged lines hidden (view full) ---

1441
1442 return err;
1443}
1444module_init(ioat_init_module);
1445
1446static void __exit ioat_exit_module(void)
1447{
1448 pci_unregister_driver(&ioat_pci_driver);
1452 kmem_cache_destroy(ioat_sed_cache);
1453 kmem_cache_destroy(ioat_cache);
1454}
1455module_exit(ioat_exit_module);
1449 kmem_cache_destroy(ioat_cache);
1450}
1451module_exit(ioat_exit_module);