init.c (3372de5813e4da8305002ff6ffbfc0c7012cb319) init.c (ef97bd0f59741ca1a555b69b8708f6601e35c3ed)
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2015 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *

--- 99 unchanged lines hidden (view full) ---

108};
109MODULE_DEVICE_TABLE(pci, ioat_pci_tbl);
110
111static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id);
112static void ioat_remove(struct pci_dev *pdev);
113static void
114ioat_init_channel(struct ioatdma_device *ioat_dma,
115 struct ioatdma_chan *ioat_chan, int idx);
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2015 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *

--- 99 unchanged lines hidden (view full) ---

108};
109MODULE_DEVICE_TABLE(pci, ioat_pci_tbl);
110
111static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id);
112static void ioat_remove(struct pci_dev *pdev);
113static void
114ioat_init_channel(struct ioatdma_device *ioat_dma,
115 struct ioatdma_chan *ioat_chan, int idx);
116static void ioat_intr_quirk(struct ioatdma_device *ioat_dma);
117static int ioat_enumerate_channels(struct ioatdma_device *ioat_dma);
118static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma);
116
117static int ioat_dca_enabled = 1;
118module_param(ioat_dca_enabled, int, 0644);
119MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
120int ioat_pending_level = 4;
121module_param(ioat_pending_level, int, 0644);
122MODULE_PARM_DESC(ioat_pending_level,
123 "high-water mark for pushing ioat descriptors (default: 4)");

--- 314 unchanged lines hidden (view full) ---

438intx:
439 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
440 IRQF_SHARED, "ioat-intx", ioat_dma);
441 if (err)
442 goto err_no_irq;
443
444 ioat_dma->irq_mode = IOAT_INTX;
445done:
119
120static int ioat_dca_enabled = 1;
121module_param(ioat_dca_enabled, int, 0644);
122MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
123int ioat_pending_level = 4;
124module_param(ioat_pending_level, int, 0644);
125MODULE_PARM_DESC(ioat_pending_level,
126 "high-water mark for pushing ioat descriptors (default: 4)");

--- 314 unchanged lines hidden (view full) ---

441intx:
442 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
443 IRQF_SHARED, "ioat-intx", ioat_dma);
444 if (err)
445 goto err_no_irq;
446
447 ioat_dma->irq_mode = IOAT_INTX;
448done:
446 if (ioat_dma->intr_quirk)
447 ioat_dma->intr_quirk(ioat_dma);
449 if (is_bwd_ioat(pdev))
450 ioat_intr_quirk(ioat_dma);
448 intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
449 writeb(intrctrl, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
450 return 0;
451
452err_no_irq:
453 /* Disable all interrupt generation */
454 writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
455 ioat_dma->irq_mode = IOAT_NOIRQ;

--- 28 unchanged lines hidden (view full) ---

484 SMP_CACHE_BYTES,
485 SMP_CACHE_BYTES);
486
487 if (!ioat_dma->completion_pool) {
488 err = -ENOMEM;
489 goto err_completion_pool;
490 }
491
451 intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
452 writeb(intrctrl, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
453 return 0;
454
455err_no_irq:
456 /* Disable all interrupt generation */
457 writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
458 ioat_dma->irq_mode = IOAT_NOIRQ;

--- 28 unchanged lines hidden (view full) ---

487 SMP_CACHE_BYTES,
488 SMP_CACHE_BYTES);
489
490 if (!ioat_dma->completion_pool) {
491 err = -ENOMEM;
492 goto err_completion_pool;
493 }
494
492 ioat_dma->enumerate_channels(ioat_dma);
495 ioat_enumerate_channels(ioat_dma);
493
494 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
495 dma->dev = &pdev->dev;
496
497 if (!dma->chancnt) {
498 dev_err(dev, "channel enumeration error\n");
499 goto err_setup_interrupts;
500 }
501
502 err = ioat_dma_setup_interrupts(ioat_dma);
503 if (err)
504 goto err_setup_interrupts;
505
496
497 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
498 dma->dev = &pdev->dev;
499
500 if (!dma->chancnt) {
501 dev_err(dev, "channel enumeration error\n");
502 goto err_setup_interrupts;
503 }
504
505 err = ioat_dma_setup_interrupts(ioat_dma);
506 if (err)
507 goto err_setup_interrupts;
508
506 err = ioat_dma->self_test(ioat_dma);
509 err = ioat3_dma_self_test(ioat_dma);
507 if (err)
508 goto err_self_test;
509
510 return 0;
511
512err_self_test:
513 ioat_disable_interrupts(ioat_dma);
514err_setup_interrupts:

--- 62 unchanged lines hidden (view full) ---

577 for (i = 0; i < dma->chancnt; i++) {
578 ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL);
579 if (!ioat_chan)
580 break;
581
582 ioat_init_channel(ioat_dma, ioat_chan, i);
583 ioat_chan->xfercap_log = xfercap_log;
584 spin_lock_init(&ioat_chan->prep_lock);
510 if (err)
511 goto err_self_test;
512
513 return 0;
514
515err_self_test:
516 ioat_disable_interrupts(ioat_dma);
517err_setup_interrupts:

--- 62 unchanged lines hidden (view full) ---

580 for (i = 0; i < dma->chancnt; i++) {
581 ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL);
582 if (!ioat_chan)
583 break;
584
585 ioat_init_channel(ioat_dma, ioat_chan, i);
586 ioat_chan->xfercap_log = xfercap_log;
587 spin_lock_init(&ioat_chan->prep_lock);
585 if (ioat_dma->reset_hw(ioat_chan)) {
588 if (ioat_reset_hw(ioat_chan)) {
586 i = 0;
587 break;
588 }
589 }
590 dma->chancnt = i;
591 return i;
592}
593

--- 12 unchanged lines hidden (view full) ---

606
607 /* Before freeing channel resources first check
608 * if they have been previously allocated for this channel.
609 */
610 if (!ioat_chan->ring)
611 return;
612
613 ioat_stop(ioat_chan);
589 i = 0;
590 break;
591 }
592 }
593 dma->chancnt = i;
594 return i;
595}
596

--- 12 unchanged lines hidden (view full) ---

609
610 /* Before freeing channel resources first check
611 * if they have been previously allocated for this channel.
612 */
613 if (!ioat_chan->ring)
614 return;
615
616 ioat_stop(ioat_chan);
614 ioat_dma->reset_hw(ioat_chan);
617 ioat_reset_hw(ioat_chan);
615
616 spin_lock_bh(&ioat_chan->cleanup_lock);
617 spin_lock_bh(&ioat_chan->prep_lock);
618 descs = ioat_ring_space(ioat_chan);
619 dev_dbg(to_dev(ioat_chan), "freeing %d idle descriptors\n", descs);
620 for (i = 0; i < descs; i++) {
621 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head + i);
622 ioat_free_ring_ent(desc, c);

--- 102 unchanged lines hidden (view full) ---

725 ioat_chan->ioat_dma = ioat_dma;
726 ioat_chan->reg_base = ioat_dma->reg_base + (0x80 * (idx + 1));
727 spin_lock_init(&ioat_chan->cleanup_lock);
728 ioat_chan->dma_chan.device = dma;
729 dma_cookie_init(&ioat_chan->dma_chan);
730 list_add_tail(&ioat_chan->dma_chan.device_node, &dma->channels);
731 ioat_dma->idx[idx] = ioat_chan;
732 init_timer(&ioat_chan->timer);
618
619 spin_lock_bh(&ioat_chan->cleanup_lock);
620 spin_lock_bh(&ioat_chan->prep_lock);
621 descs = ioat_ring_space(ioat_chan);
622 dev_dbg(to_dev(ioat_chan), "freeing %d idle descriptors\n", descs);
623 for (i = 0; i < descs; i++) {
624 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head + i);
625 ioat_free_ring_ent(desc, c);

--- 102 unchanged lines hidden (view full) ---

728 ioat_chan->ioat_dma = ioat_dma;
729 ioat_chan->reg_base = ioat_dma->reg_base + (0x80 * (idx + 1));
730 spin_lock_init(&ioat_chan->cleanup_lock);
731 ioat_chan->dma_chan.device = dma;
732 dma_cookie_init(&ioat_chan->dma_chan);
733 list_add_tail(&ioat_chan->dma_chan.device_node, &dma->channels);
734 ioat_dma->idx[idx] = ioat_chan;
735 init_timer(&ioat_chan->timer);
733 ioat_chan->timer.function = ioat_dma->timer_fn;
736 ioat_chan->timer.function = ioat_timer_event;
734 ioat_chan->timer.data = data;
737 ioat_chan->timer.data = data;
735 tasklet_init(&ioat_chan->cleanup_task, ioat_dma->cleanup_fn, data);
738 tasklet_init(&ioat_chan->cleanup_task, ioat_cleanup_event, data);
736}
737
738#define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */
739static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma)
740{
741 int i, src_idx;
742 struct page *dest;
743 struct page *xor_srcs[IOAT_NUM_SRC_TEST];

--- 304 unchanged lines hidden (view full) ---

1048 struct pci_dev *pdev = ioat_dma->pdev;
1049 int dca_en = system_has_dca_enabled(pdev);
1050 struct dma_device *dma;
1051 struct dma_chan *c;
1052 struct ioatdma_chan *ioat_chan;
1053 bool is_raid_device = false;
1054 int err;
1055
739}
740
741#define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */
742static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma)
743{
744 int i, src_idx;
745 struct page *dest;
746 struct page *xor_srcs[IOAT_NUM_SRC_TEST];

--- 304 unchanged lines hidden (view full) ---

1051 struct pci_dev *pdev = ioat_dma->pdev;
1052 int dca_en = system_has_dca_enabled(pdev);
1053 struct dma_device *dma;
1054 struct dma_chan *c;
1055 struct ioatdma_chan *ioat_chan;
1056 bool is_raid_device = false;
1057 int err;
1058
1056 ioat_dma->enumerate_channels = ioat_enumerate_channels;
1057 ioat_dma->reset_hw = ioat_reset_hw;
1058 ioat_dma->self_test = ioat3_dma_self_test;
1059 ioat_dma->intr_quirk = ioat_intr_quirk;
1060 dma = &ioat_dma->dma_dev;
1061 dma->device_prep_dma_memcpy = ioat_dma_prep_memcpy_lock;
1062 dma->device_issue_pending = ioat_issue_pending;
1063 dma->device_alloc_chan_resources = ioat_alloc_chan_resources;
1064 dma->device_free_chan_resources = ioat_free_chan_resources;
1065
1066 dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
1067 dma->device_prep_dma_interrupt = ioat_prep_interrupt_lock;

--- 41 unchanged lines hidden (view full) ---

1109 if (ioat_dma->cap & IOAT_CAP_RAID16SS)
1110 dma->max_xor = 16;
1111 else
1112 dma->max_xor = 8;
1113 }
1114 }
1115
1116 dma->device_tx_status = ioat_tx_status;
1059 dma = &ioat_dma->dma_dev;
1060 dma->device_prep_dma_memcpy = ioat_dma_prep_memcpy_lock;
1061 dma->device_issue_pending = ioat_issue_pending;
1062 dma->device_alloc_chan_resources = ioat_alloc_chan_resources;
1063 dma->device_free_chan_resources = ioat_free_chan_resources;
1064
1065 dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
1066 dma->device_prep_dma_interrupt = ioat_prep_interrupt_lock;

--- 41 unchanged lines hidden (view full) ---

1108 if (ioat_dma->cap & IOAT_CAP_RAID16SS)
1109 dma->max_xor = 16;
1110 else
1111 dma->max_xor = 8;
1112 }
1113 }
1114
1115 dma->device_tx_status = ioat_tx_status;
1117 ioat_dma->cleanup_fn = ioat_cleanup_event;
1118 ioat_dma->timer_fn = ioat_timer_event;
1119
1120 /* starting with CB3.3 super extended descriptors are supported */
1121 if (ioat_dma->cap & IOAT_CAP_RAID16SS) {
1122 char pool_name[14];
1123 int i;
1124
1125 for (i = 0; i < MAX_SED_POOLS; i++) {
1126 snprintf(pool_name, 14, "ioat_hw%d_sed", i);

--- 162 unchanged lines hidden --- 		1116
1117 /* starting with CB3.3 super extended descriptors are supported */
1118 if (ioat_dma->cap & IOAT_CAP_RAID16SS) {
1119 char pool_name[14];
1120 int i;
1121
1122 for (i = 0; i < MAX_SED_POOLS; i++) {
1123 snprintf(pool_name, 14, "ioat_hw%d_sed", i);

--- 162 unchanged lines hidden ---