14fa9c49fSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2584ec227SDan Williams /*
3584ec227SDan Williams * Intel I/OAT DMA Linux driver
4584ec227SDan Williams * Copyright(c) 2007 - 2009 Intel Corporation.
5584ec227SDan Williams */
6584ec227SDan Williams
7584ec227SDan Williams #include <linux/kernel.h>
8584ec227SDan Williams #include <linux/pci.h>
9584ec227SDan Williams #include <linux/smp.h>
10584ec227SDan Williams #include <linux/interrupt.h>
11584ec227SDan Williams #include <linux/dca.h>
12584ec227SDan Williams
13584ec227SDan Williams /* either a kernel change is needed, or we need something like this in kernel */
14584ec227SDan Williams #ifndef CONFIG_SMP
15584ec227SDan Williams #include <asm/smp.h>
16584ec227SDan Williams #undef cpu_physical_id
17584ec227SDan Williams #define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24)
18584ec227SDan Williams #endif
19584ec227SDan Williams
20584ec227SDan Williams #include "dma.h"
21584ec227SDan Williams #include "registers.h"
22584ec227SDan Williams
23584ec227SDan Williams /*
24584ec227SDan Williams * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
25584ec227SDan Williams * contain the bit number of the APIC ID to map into the DCA tag. If the valid
26584ec227SDan Williams * bit is not set, then the value must be 0 or 1 and defines the bit in the tag.
27584ec227SDan Williams */
28584ec227SDan Williams #define DCA_TAG_MAP_VALID 0x80
29584ec227SDan Williams
30584ec227SDan Williams #define DCA3_TAG_MAP_BIT_TO_INV 0x80
31584ec227SDan Williams #define DCA3_TAG_MAP_BIT_TO_SEL 0x40
32584ec227SDan Williams #define DCA3_TAG_MAP_LITERAL_VAL 0x1
33584ec227SDan Williams
34584ec227SDan Williams #define DCA_TAG_MAP_MASK 0xDF
35584ec227SDan Williams
36584ec227SDan Williams /* expected tag map bytes for I/OAT ver.2 */
37584ec227SDan Williams #define DCA2_TAG_MAP_BYTE0 0x80
38584ec227SDan Williams #define DCA2_TAG_MAP_BYTE1 0x0
39584ec227SDan Williams #define DCA2_TAG_MAP_BYTE2 0x81
40584ec227SDan Williams #define DCA2_TAG_MAP_BYTE3 0x82
41584ec227SDan Williams #define DCA2_TAG_MAP_BYTE4 0x82
42584ec227SDan Williams
43584ec227SDan Williams /*
44584ec227SDan Williams * "Legacy" DCA systems do not implement the DCA register set in the
45584ec227SDan Williams * I/OAT device. Software needs direct support for their tag mappings.
46584ec227SDan Williams */
47584ec227SDan Williams
48584ec227SDan Williams #define APICID_BIT(x) (DCA_TAG_MAP_VALID | (x))
49584ec227SDan Williams #define IOAT_TAG_MAP_LEN 8
50584ec227SDan Williams
51584ec227SDan Williams /* pack PCI B/D/F into a u16 */
dcaid_from_pcidev(struct pci_dev * pci)52584ec227SDan Williams static inline u16 dcaid_from_pcidev(struct pci_dev *pci)
53584ec227SDan Williams {
54*c65029b1SJialin Zhang return pci_dev_id(pci);
55584ec227SDan Williams }
56584ec227SDan Williams
dca_enabled_in_bios(struct pci_dev * pdev)57584ec227SDan Williams static int dca_enabled_in_bios(struct pci_dev *pdev)
58584ec227SDan Williams {
59584ec227SDan Williams /* CPUID level 9 returns DCA configuration */
60584ec227SDan Williams /* Bit 0 indicates DCA enabled by the BIOS */
61584ec227SDan Williams unsigned long cpuid_level_9;
62584ec227SDan Williams int res;
63584ec227SDan Williams
64584ec227SDan Williams cpuid_level_9 = cpuid_eax(9);
65584ec227SDan Williams res = test_bit(0, &cpuid_level_9);
66584ec227SDan Williams if (!res)
67e22dde99SDan Williams dev_dbg(&pdev->dev, "DCA is disabled in BIOS\n");
68584ec227SDan Williams
69584ec227SDan Williams return res;
70584ec227SDan Williams }
71584ec227SDan Williams
system_has_dca_enabled(struct pci_dev * pdev)72228c4f5cSDan Williams int system_has_dca_enabled(struct pci_dev *pdev)
73584ec227SDan Williams {
74584ec227SDan Williams if (boot_cpu_has(X86_FEATURE_DCA))
75584ec227SDan Williams return dca_enabled_in_bios(pdev);
76584ec227SDan Williams
77e22dde99SDan Williams dev_dbg(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n");
78584ec227SDan Williams return 0;
79584ec227SDan Williams }
80584ec227SDan Williams
81584ec227SDan Williams struct ioat_dca_slot {
82584ec227SDan Williams struct pci_dev *pdev; /* requester device */
83584ec227SDan Williams u16 rid; /* requester id, as used by IOAT */
84584ec227SDan Williams };
85584ec227SDan Williams
86584ec227SDan Williams #define IOAT_DCA_MAX_REQ 6
87584ec227SDan Williams #define IOAT3_DCA_MAX_REQ 2
88584ec227SDan Williams
89584ec227SDan Williams struct ioat_dca_priv {
90584ec227SDan Williams void __iomem *iobase;
91584ec227SDan Williams void __iomem *dca_base;
92584ec227SDan Williams int max_requesters;
93584ec227SDan Williams int requester_count;
94584ec227SDan Williams u8 tag_map[IOAT_TAG_MAP_LEN];
9535e03246SGustavo A. R. Silva struct ioat_dca_slot req_slots[];
96584ec227SDan Williams };
97584ec227SDan Williams
ioat_dca_dev_managed(struct dca_provider * dca,struct device * dev)98584ec227SDan Williams static int ioat_dca_dev_managed(struct dca_provider *dca,
99584ec227SDan Williams struct device *dev)
100584ec227SDan Williams {
101584ec227SDan Williams struct ioat_dca_priv *ioatdca = dca_priv(dca);
102584ec227SDan Williams struct pci_dev *pdev;
103584ec227SDan Williams int i;
104584ec227SDan Williams
105584ec227SDan Williams pdev = to_pci_dev(dev);
106584ec227SDan Williams for (i = 0; i < ioatdca->max_requesters; i++) {
107584ec227SDan Williams if (ioatdca->req_slots[i].pdev == pdev)
108584ec227SDan Williams return 1;
109584ec227SDan Williams }
110584ec227SDan Williams return 0;
111584ec227SDan Williams }
112584ec227SDan Williams
ioat_dca_add_requester(struct dca_provider * dca,struct device * dev)1133372de58SDave Jiang static int ioat_dca_add_requester(struct dca_provider *dca, struct device *dev)
114584ec227SDan Williams {
115584ec227SDan Williams struct ioat_dca_priv *ioatdca = dca_priv(dca);
116584ec227SDan Williams struct pci_dev *pdev;
117584ec227SDan Williams int i;
118584ec227SDan Williams u16 id;
119584ec227SDan Williams u16 global_req_table;
120584ec227SDan Williams
121584ec227SDan Williams /* This implementation only supports PCI-Express */
1221fde2548SYijing Wang if (!dev_is_pci(dev))
123584ec227SDan Williams return -ENODEV;
124584ec227SDan Williams pdev = to_pci_dev(dev);
125584ec227SDan Williams id = dcaid_from_pcidev(pdev);
126584ec227SDan Williams
127584ec227SDan Williams if (ioatdca->requester_count == ioatdca->max_requesters)
128584ec227SDan Williams return -ENODEV;
129584ec227SDan Williams
130584ec227SDan Williams for (i = 0; i < ioatdca->max_requesters; i++) {
131584ec227SDan Williams if (ioatdca->req_slots[i].pdev == NULL) {
132584ec227SDan Williams /* found an empty slot */
133584ec227SDan Williams ioatdca->requester_count++;
134584ec227SDan Williams ioatdca->req_slots[i].pdev = pdev;
135584ec227SDan Williams ioatdca->req_slots[i].rid = id;
136584ec227SDan Williams global_req_table =
137584ec227SDan Williams readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
138584ec227SDan Williams writel(id | IOAT_DCA_GREQID_VALID,
139584ec227SDan Williams ioatdca->iobase + global_req_table + (i * 4));
140584ec227SDan Williams return i;
141584ec227SDan Williams }
142584ec227SDan Williams }
143584ec227SDan Williams /* Error, ioatdma->requester_count is out of whack */
144584ec227SDan Williams return -EFAULT;
145584ec227SDan Williams }
146584ec227SDan Williams
ioat_dca_remove_requester(struct dca_provider * dca,struct device * dev)1473372de58SDave Jiang static int ioat_dca_remove_requester(struct dca_provider *dca,
148584ec227SDan Williams struct device *dev)
149584ec227SDan Williams {
150584ec227SDan Williams struct ioat_dca_priv *ioatdca = dca_priv(dca);
151584ec227SDan Williams struct pci_dev *pdev;
152584ec227SDan Williams int i;
153584ec227SDan Williams u16 global_req_table;
154584ec227SDan Williams
155584ec227SDan Williams /* This implementation only supports PCI-Express */
1561fde2548SYijing Wang if (!dev_is_pci(dev))
157584ec227SDan Williams return -ENODEV;
158584ec227SDan Williams pdev = to_pci_dev(dev);
159584ec227SDan Williams
160584ec227SDan Williams for (i = 0; i < ioatdca->max_requesters; i++) {
161584ec227SDan Williams if (ioatdca->req_slots[i].pdev == pdev) {
162584ec227SDan Williams global_req_table =
163584ec227SDan Williams readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
164584ec227SDan Williams writel(0, ioatdca->iobase + global_req_table + (i * 4));
165584ec227SDan Williams ioatdca->req_slots[i].pdev = NULL;
166584ec227SDan Williams ioatdca->req_slots[i].rid = 0;
167584ec227SDan Williams ioatdca->requester_count--;
168584ec227SDan Williams return i;
169584ec227SDan Williams }
170584ec227SDan Williams }
171584ec227SDan Williams return -ENODEV;
172584ec227SDan Williams }
173584ec227SDan Williams
ioat_dca_get_tag(struct dca_provider * dca,struct device * dev,int cpu)1743372de58SDave Jiang static u8 ioat_dca_get_tag(struct dca_provider *dca,
175584ec227SDan Williams struct device *dev,
176584ec227SDan Williams int cpu)
177584ec227SDan Williams {
178584ec227SDan Williams u8 tag;
179584ec227SDan Williams
180584ec227SDan Williams struct ioat_dca_priv *ioatdca = dca_priv(dca);
181584ec227SDan Williams int i, apic_id, bit, value;
182584ec227SDan Williams u8 entry;
183584ec227SDan Williams
184584ec227SDan Williams tag = 0;
185584ec227SDan Williams apic_id = cpu_physical_id(cpu);
186584ec227SDan Williams
187584ec227SDan Williams for (i = 0; i < IOAT_TAG_MAP_LEN; i++) {
188584ec227SDan Williams entry = ioatdca->tag_map[i];
189584ec227SDan Williams if (entry & DCA3_TAG_MAP_BIT_TO_SEL) {
190584ec227SDan Williams bit = entry &
191584ec227SDan Williams ~(DCA3_TAG_MAP_BIT_TO_SEL | DCA3_TAG_MAP_BIT_TO_INV);
192584ec227SDan Williams value = (apic_id & (1 << bit)) ? 1 : 0;
193584ec227SDan Williams } else if (entry & DCA3_TAG_MAP_BIT_TO_INV) {
194584ec227SDan Williams bit = entry & ~DCA3_TAG_MAP_BIT_TO_INV;
195584ec227SDan Williams value = (apic_id & (1 << bit)) ? 0 : 1;
196584ec227SDan Williams } else {
197584ec227SDan Williams value = (entry & DCA3_TAG_MAP_LITERAL_VAL) ? 1 : 0;
198584ec227SDan Williams }
199584ec227SDan Williams tag |= (value << i);
200584ec227SDan Williams }
201584ec227SDan Williams
202584ec227SDan Williams return tag;
203584ec227SDan Williams }
204584ec227SDan Williams
2052bb129ebSJulia Lawall static const struct dca_ops ioat_dca_ops = {
2063372de58SDave Jiang .add_requester = ioat_dca_add_requester,
2073372de58SDave Jiang .remove_requester = ioat_dca_remove_requester,
2083372de58SDave Jiang .get_tag = ioat_dca_get_tag,
209584ec227SDan Williams .dev_managed = ioat_dca_dev_managed,
210584ec227SDan Williams };
211584ec227SDan Williams
ioat_dca_count_dca_slots(void * iobase,u16 dca_offset)2123372de58SDave Jiang static int ioat_dca_count_dca_slots(void *iobase, u16 dca_offset)
213584ec227SDan Williams {
214584ec227SDan Williams int slots = 0;
215584ec227SDan Williams u32 req;
216584ec227SDan Williams u16 global_req_table;
217584ec227SDan Williams
218584ec227SDan Williams global_req_table = readw(iobase + dca_offset + IOAT3_DCA_GREQID_OFFSET);
219584ec227SDan Williams if (global_req_table == 0)
220584ec227SDan Williams return 0;
221584ec227SDan Williams
222584ec227SDan Williams do {
223584ec227SDan Williams req = readl(iobase + global_req_table + (slots * sizeof(u32)));
224584ec227SDan Williams slots++;
225584ec227SDan Williams } while ((req & IOAT_DCA_GREQID_LASTID) == 0);
226584ec227SDan Williams
227584ec227SDan Williams return slots;
228584ec227SDan Williams }
229584ec227SDan Williams
dca3_tag_map_invalid(u8 * tag_map)23007bd34dbSAlexander Duyck static inline int dca3_tag_map_invalid(u8 *tag_map)
23107bd34dbSAlexander Duyck {
23207bd34dbSAlexander Duyck /*
23307bd34dbSAlexander Duyck * If the tag map is not programmed by the BIOS the default is:
23407bd34dbSAlexander Duyck * 0x80 0x80 0x80 0x80 0x80 0x00 0x00 0x00
23507bd34dbSAlexander Duyck *
23607bd34dbSAlexander Duyck * This an invalid map and will result in only 2 possible tags
23707bd34dbSAlexander Duyck * 0x1F and 0x00. 0x00 is an invalid DCA tag so we know that
23807bd34dbSAlexander Duyck * this entire definition is invalid.
23907bd34dbSAlexander Duyck */
24007bd34dbSAlexander Duyck return ((tag_map[0] == DCA_TAG_MAP_VALID) &&
24107bd34dbSAlexander Duyck (tag_map[1] == DCA_TAG_MAP_VALID) &&
24207bd34dbSAlexander Duyck (tag_map[2] == DCA_TAG_MAP_VALID) &&
24307bd34dbSAlexander Duyck (tag_map[3] == DCA_TAG_MAP_VALID) &&
24407bd34dbSAlexander Duyck (tag_map[4] == DCA_TAG_MAP_VALID));
24507bd34dbSAlexander Duyck }
24607bd34dbSAlexander Duyck
ioat_dca_init(struct pci_dev * pdev,void __iomem * iobase)2473372de58SDave Jiang struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
248584ec227SDan Williams {
249584ec227SDan Williams struct dca_provider *dca;
250584ec227SDan Williams struct ioat_dca_priv *ioatdca;
251584ec227SDan Williams int slots;
252584ec227SDan Williams int i;
253584ec227SDan Williams int err;
254584ec227SDan Williams u16 dca_offset;
255584ec227SDan Williams u16 csi_fsb_control;
256584ec227SDan Williams u16 pcie_control;
257584ec227SDan Williams u8 bit;
258584ec227SDan Williams
259584ec227SDan Williams union {
260584ec227SDan Williams u64 full;
261584ec227SDan Williams struct {
262584ec227SDan Williams u32 low;
263584ec227SDan Williams u32 high;
264584ec227SDan Williams };
265584ec227SDan Williams } tag_map;
266584ec227SDan Williams
267584ec227SDan Williams if (!system_has_dca_enabled(pdev))
268584ec227SDan Williams return NULL;
269584ec227SDan Williams
270584ec227SDan Williams dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET);
271584ec227SDan Williams if (dca_offset == 0)
272584ec227SDan Williams return NULL;
273584ec227SDan Williams
2743372de58SDave Jiang slots = ioat_dca_count_dca_slots(iobase, dca_offset);
275584ec227SDan Williams if (slots == 0)
276584ec227SDan Williams return NULL;
277584ec227SDan Williams
2783372de58SDave Jiang dca = alloc_dca_provider(&ioat_dca_ops,
27925af5afeSGustavo A. R. Silva struct_size(ioatdca, req_slots, slots));
280584ec227SDan Williams if (!dca)
281584ec227SDan Williams return NULL;
282584ec227SDan Williams
283584ec227SDan Williams ioatdca = dca_priv(dca);
284584ec227SDan Williams ioatdca->iobase = iobase;
285584ec227SDan Williams ioatdca->dca_base = iobase + dca_offset;
286584ec227SDan Williams ioatdca->max_requesters = slots;
287584ec227SDan Williams
288584ec227SDan Williams /* some bios might not know to turn these on */
289584ec227SDan Williams csi_fsb_control = readw(ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
290584ec227SDan Williams if ((csi_fsb_control & IOAT3_CSI_CONTROL_PREFETCH) == 0) {
291584ec227SDan Williams csi_fsb_control |= IOAT3_CSI_CONTROL_PREFETCH;
292584ec227SDan Williams writew(csi_fsb_control,
293584ec227SDan Williams ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
294584ec227SDan Williams }
295584ec227SDan Williams pcie_control = readw(ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
296584ec227SDan Williams if ((pcie_control & IOAT3_PCI_CONTROL_MEMWR) == 0) {
297584ec227SDan Williams pcie_control |= IOAT3_PCI_CONTROL_MEMWR;
298584ec227SDan Williams writew(pcie_control,
299584ec227SDan Williams ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
300584ec227SDan Williams }
301584ec227SDan Williams
302584ec227SDan Williams
303584ec227SDan Williams /* TODO version, compatibility and configuration checks */
304584ec227SDan Williams
305584ec227SDan Williams /* copy out the APIC to DCA tag map */
306584ec227SDan Williams tag_map.low =
307584ec227SDan Williams readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_LOW);
308584ec227SDan Williams tag_map.high =
309584ec227SDan Williams readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_HIGH);
310584ec227SDan Williams for (i = 0; i < 8; i++) {
311584ec227SDan Williams bit = tag_map.full >> (8 * i);
312584ec227SDan Williams ioatdca->tag_map[i] = bit & DCA_TAG_MAP_MASK;
313584ec227SDan Williams }
314584ec227SDan Williams
31507bd34dbSAlexander Duyck if (dca3_tag_map_invalid(ioatdca->tag_map)) {
316036e9ef8SPrarit Bhargava add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
317036e9ef8SPrarit Bhargava pr_warn_once("%s %s: APICID_TAG_MAP set incorrectly by BIOS, disabling DCA\n",
318f3c78f85SAlexander Duyck dev_driver_string(&pdev->dev),
319f3c78f85SAlexander Duyck dev_name(&pdev->dev));
32007bd34dbSAlexander Duyck free_dca_provider(dca);
32107bd34dbSAlexander Duyck return NULL;
32207bd34dbSAlexander Duyck }
32307bd34dbSAlexander Duyck
324584ec227SDan Williams err = register_dca_provider(dca, &pdev->dev);
325584ec227SDan Williams if (err) {
326584ec227SDan Williams free_dca_provider(dca);
327584ec227SDan Williams return NULL;
328584ec227SDan Williams }
329584ec227SDan Williams
330584ec227SDan Williams return dca;
331584ec227SDan Williams }
332