1 /*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
9 * Copyright (C) 2016 T-Platforms. All Rights Reserved.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of version 2 of the GNU General Public License as
13 * published by the Free Software Foundation.
14 *
15 * BSD LICENSE
16 *
17 * Copyright(c) 2012 Intel Corporation. All rights reserved.
18 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
19 * Copyright (C) 2016 T-Platforms. All Rights Reserved.
20 *
21 * Redistribution and use in source and binary forms, with or without
22 * modification, are permitted provided that the following conditions
23 * are met:
24 *
25 * * Redistributions of source code must retain the above copyright
26 * notice, this list of conditions and the following disclaimer.
27 * * Redistributions in binary form must reproduce the above copy
28 * notice, this list of conditions and the following disclaimer in
29 * the documentation and/or other materials provided with the
30 * distribution.
31 * * Neither the name of Intel Corporation nor the names of its
32 * contributors may be used to endorse or promote products derived
33 * from this software without specific prior written permission.
34 *
35 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46 *
47 * Intel PCIe NTB Linux driver
48 */
49
50 #include <linux/debugfs.h>
51 #include <linux/delay.h>
52 #include <linux/init.h>
53 #include <linux/interrupt.h>
54 #include <linux/module.h>
55 #include <linux/pci.h>
56 #include <linux/random.h>
57 #include <linux/slab.h>
58 #include <linux/ntb.h>
59
60 #include "ntb_hw_intel.h"
61 #include "ntb_hw_gen1.h"
62 #include "ntb_hw_gen3.h"
63 #include "ntb_hw_gen4.h"
64
65 #define NTB_NAME "ntb_hw_intel"
66 #define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver"
67 #define NTB_VER "2.0"
68
69 MODULE_DESCRIPTION(NTB_DESC);
70 MODULE_VERSION(NTB_VER);
71 MODULE_LICENSE("Dual BSD/GPL");
72 MODULE_AUTHOR("Intel Corporation");
73
74 #define bar0_off(base, bar) ((base) + ((bar) << 2))
75 #define bar2_off(base, bar) bar0_off(base, (bar) - 2)
76
77 static const struct intel_ntb_reg xeon_reg;
78 static const struct intel_ntb_alt_reg xeon_pri_reg;
79 static const struct intel_ntb_alt_reg xeon_sec_reg;
80 static const struct intel_ntb_alt_reg xeon_b2b_reg;
81 static const struct intel_ntb_xlat_reg xeon_pri_xlat;
82 static const struct intel_ntb_xlat_reg xeon_sec_xlat;
83 static const struct ntb_dev_ops intel_ntb_ops;
84
85 static const struct file_operations intel_ntb_debugfs_info;
86 static struct dentry *debugfs_dir;
87
88 static int b2b_mw_idx = -1;
89 module_param(b2b_mw_idx, int, 0644);
90 MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb. A "
91 "value of zero or positive starts from first mw idx, and a "
92 "negative value starts from last mw idx. Both sides MUST "
93 "set the same value here!");
94
95 static unsigned int b2b_mw_share;
96 module_param(b2b_mw_share, uint, 0644);
97 MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
98 "ntb so that the peer ntb only occupies the first half of "
99 "the mw, so the second half can still be used as a mw. Both "
100 "sides MUST set the same value here!");
101
102 module_param_named(xeon_b2b_usd_bar2_addr64,
103 xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
104 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
105 "XEON B2B USD BAR 2 64-bit address");
106
107 module_param_named(xeon_b2b_usd_bar4_addr64,
108 xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
109 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64,
110 "XEON B2B USD BAR 4 64-bit address");
111
112 module_param_named(xeon_b2b_usd_bar4_addr32,
113 xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
114 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32,
115 "XEON B2B USD split-BAR 4 32-bit address");
116
117 module_param_named(xeon_b2b_usd_bar5_addr32,
118 xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
119 MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32,
120 "XEON B2B USD split-BAR 5 32-bit address");
121
122 module_param_named(xeon_b2b_dsd_bar2_addr64,
123 xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
124 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
125 "XEON B2B DSD BAR 2 64-bit address");
126
127 module_param_named(xeon_b2b_dsd_bar4_addr64,
128 xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
129 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64,
130 "XEON B2B DSD BAR 4 64-bit address");
131
132 module_param_named(xeon_b2b_dsd_bar4_addr32,
133 xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
134 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32,
135 "XEON B2B DSD split-BAR 4 32-bit address");
136
137 module_param_named(xeon_b2b_dsd_bar5_addr32,
138 xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
139 MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32,
140 "XEON B2B DSD split-BAR 5 32-bit address");
141
142
143 static int xeon_init_isr(struct intel_ntb_dev *ndev);
144
ndev_reset_unsafe_flags(struct intel_ntb_dev * ndev)145 static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
146 {
147 ndev->unsafe_flags = 0;
148 ndev->unsafe_flags_ignore = 0;
149
150 /* Only B2B has a workaround to avoid SDOORBELL */
151 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
152 if (!ntb_topo_is_b2b(ndev->ntb.topo))
153 ndev->unsafe_flags |= NTB_UNSAFE_DB;
154
155 /* No low level workaround to avoid SB01BASE */
156 if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
157 ndev->unsafe_flags |= NTB_UNSAFE_DB;
158 ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
159 }
160 }
161
ndev_is_unsafe(struct intel_ntb_dev * ndev,unsigned long flag)162 static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
163 unsigned long flag)
164 {
165 return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
166 }
167
ndev_ignore_unsafe(struct intel_ntb_dev * ndev,unsigned long flag)168 static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
169 unsigned long flag)
170 {
171 flag &= ndev->unsafe_flags;
172 ndev->unsafe_flags_ignore |= flag;
173
174 return !!flag;
175 }
176
ndev_mw_to_bar(struct intel_ntb_dev * ndev,int idx)177 int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
178 {
179 if (idx < 0 || idx >= ndev->mw_count)
180 return -EINVAL;
181 return ndev->reg->mw_bar[idx];
182 }
183
ndev_db_addr(struct intel_ntb_dev * ndev,phys_addr_t * db_addr,resource_size_t * db_size,phys_addr_t reg_addr,unsigned long reg)184 void ndev_db_addr(struct intel_ntb_dev *ndev,
185 phys_addr_t *db_addr, resource_size_t *db_size,
186 phys_addr_t reg_addr, unsigned long reg)
187 {
188 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
189 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
190
191 if (db_addr) {
192 *db_addr = reg_addr + reg;
193 dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx\n", *db_addr);
194 }
195
196 if (db_size) {
197 *db_size = ndev->reg->db_size;
198 dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size);
199 }
200 }
201
ndev_db_read(struct intel_ntb_dev * ndev,void __iomem * mmio)202 u64 ndev_db_read(struct intel_ntb_dev *ndev,
203 void __iomem *mmio)
204 {
205 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
206 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
207
208 return ndev->reg->db_ioread(mmio);
209 }
210
ndev_db_write(struct intel_ntb_dev * ndev,u64 db_bits,void __iomem * mmio)211 int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
212 void __iomem *mmio)
213 {
214 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
215 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
216
217 if (db_bits & ~ndev->db_valid_mask)
218 return -EINVAL;
219
220 ndev->reg->db_iowrite(db_bits, mmio);
221
222 return 0;
223 }
224
ndev_db_set_mask(struct intel_ntb_dev * ndev,u64 db_bits,void __iomem * mmio)225 static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
226 void __iomem *mmio)
227 {
228 unsigned long irqflags;
229
230 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
231 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
232
233 if (db_bits & ~ndev->db_valid_mask)
234 return -EINVAL;
235
236 spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
237 {
238 ndev->db_mask |= db_bits;
239 ndev->reg->db_iowrite(ndev->db_mask, mmio);
240 }
241 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
242
243 return 0;
244 }
245
ndev_db_clear_mask(struct intel_ntb_dev * ndev,u64 db_bits,void __iomem * mmio)246 static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
247 void __iomem *mmio)
248 {
249 unsigned long irqflags;
250
251 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
252 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
253
254 if (db_bits & ~ndev->db_valid_mask)
255 return -EINVAL;
256
257 spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
258 {
259 ndev->db_mask &= ~db_bits;
260 ndev->reg->db_iowrite(ndev->db_mask, mmio);
261 }
262 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
263
264 return 0;
265 }
266
ndev_vec_mask(struct intel_ntb_dev * ndev,int db_vector)267 static inline u64 ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
268 {
269 u64 shift, mask;
270
271 shift = ndev->db_vec_shift;
272 mask = BIT_ULL(shift) - 1;
273
274 return mask << (shift * db_vector);
275 }
276
ndev_spad_addr(struct intel_ntb_dev * ndev,int idx,phys_addr_t * spad_addr,phys_addr_t reg_addr,unsigned long reg)277 static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
278 phys_addr_t *spad_addr, phys_addr_t reg_addr,
279 unsigned long reg)
280 {
281 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
282 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
283
284 if (idx < 0 || idx >= ndev->spad_count)
285 return -EINVAL;
286
287 if (spad_addr) {
288 *spad_addr = reg_addr + reg + (idx << 2);
289 dev_dbg(&ndev->ntb.pdev->dev, "Peer spad addr %llx\n",
290 *spad_addr);
291 }
292
293 return 0;
294 }
295
ndev_spad_read(struct intel_ntb_dev * ndev,int idx,void __iomem * mmio)296 static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
297 void __iomem *mmio)
298 {
299 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
300 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
301
302 if (idx < 0 || idx >= ndev->spad_count)
303 return 0;
304
305 return ioread32(mmio + (idx << 2));
306 }
307
ndev_spad_write(struct intel_ntb_dev * ndev,int idx,u32 val,void __iomem * mmio)308 static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
309 void __iomem *mmio)
310 {
311 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
312 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
313
314 if (idx < 0 || idx >= ndev->spad_count)
315 return -EINVAL;
316
317 iowrite32(val, mmio + (idx << 2));
318
319 return 0;
320 }
321
ndev_interrupt(struct intel_ntb_dev * ndev,int vec)322 static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
323 {
324 u64 vec_mask;
325
326 vec_mask = ndev_vec_mask(ndev, vec);
327
328 if ((ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) && (vec == 31))
329 vec_mask |= ndev->db_link_mask;
330
331 dev_dbg(&ndev->ntb.pdev->dev, "vec %d vec_mask %llx\n", vec, vec_mask);
332
333 ndev->last_ts = jiffies;
334
335 if (vec_mask & ndev->db_link_mask) {
336 if (ndev->reg->poll_link(ndev))
337 ntb_link_event(&ndev->ntb);
338 }
339
340 if (vec_mask & ndev->db_valid_mask)
341 ntb_db_event(&ndev->ntb, vec);
342
343 return IRQ_HANDLED;
344 }
345
ndev_vec_isr(int irq,void * dev)346 static irqreturn_t ndev_vec_isr(int irq, void *dev)
347 {
348 struct intel_ntb_vec *nvec = dev;
349
350 dev_dbg(&nvec->ndev->ntb.pdev->dev, "irq: %d nvec->num: %d\n",
351 irq, nvec->num);
352
353 return ndev_interrupt(nvec->ndev, nvec->num);
354 }
355
ndev_irq_isr(int irq,void * dev)356 static irqreturn_t ndev_irq_isr(int irq, void *dev)
357 {
358 struct intel_ntb_dev *ndev = dev;
359
360 return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
361 }
362
ndev_init_isr(struct intel_ntb_dev * ndev,int msix_min,int msix_max,int msix_shift,int total_shift)363 int ndev_init_isr(struct intel_ntb_dev *ndev,
364 int msix_min, int msix_max,
365 int msix_shift, int total_shift)
366 {
367 struct pci_dev *pdev;
368 int rc, i, msix_count, node;
369
370 pdev = ndev->ntb.pdev;
371
372 node = dev_to_node(&pdev->dev);
373
374 /* Mask all doorbell interrupts */
375 ndev->db_mask = ndev->db_valid_mask;
376 ndev->reg->db_iowrite(ndev->db_mask,
377 ndev->self_mmio +
378 ndev->self_reg->db_mask);
379
380 /* Try to set up msix irq */
381
382 ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec),
383 GFP_KERNEL, node);
384 if (!ndev->vec)
385 goto err_msix_vec_alloc;
386
387 ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix),
388 GFP_KERNEL, node);
389 if (!ndev->msix)
390 goto err_msix_alloc;
391
392 for (i = 0; i < msix_max; ++i)
393 ndev->msix[i].entry = i;
394
395 msix_count = pci_enable_msix_range(pdev, ndev->msix,
396 msix_min, msix_max);
397 if (msix_count < 0)
398 goto err_msix_enable;
399
400 for (i = 0; i < msix_count; ++i) {
401 ndev->vec[i].ndev = ndev;
402 ndev->vec[i].num = i;
403 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
404 "ndev_vec_isr", &ndev->vec[i]);
405 if (rc)
406 goto err_msix_request;
407 }
408
409 dev_dbg(&pdev->dev, "Using %d msix interrupts\n", msix_count);
410 ndev->db_vec_count = msix_count;
411 ndev->db_vec_shift = msix_shift;
412 return 0;
413
414 err_msix_request:
415 while (i-- > 0)
416 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
417 pci_disable_msix(pdev);
418 err_msix_enable:
419 kfree(ndev->msix);
420 err_msix_alloc:
421 kfree(ndev->vec);
422 err_msix_vec_alloc:
423 ndev->msix = NULL;
424 ndev->vec = NULL;
425
426 /* Try to set up msi irq */
427
428 rc = pci_enable_msi(pdev);
429 if (rc)
430 goto err_msi_enable;
431
432 rc = request_irq(pdev->irq, ndev_irq_isr, 0,
433 "ndev_irq_isr", ndev);
434 if (rc)
435 goto err_msi_request;
436
437 dev_dbg(&pdev->dev, "Using msi interrupts\n");
438 ndev->db_vec_count = 1;
439 ndev->db_vec_shift = total_shift;
440 return 0;
441
442 err_msi_request:
443 pci_disable_msi(pdev);
444 err_msi_enable:
445
446 /* Try to set up intx irq */
447
448 pci_intx(pdev, 1);
449
450 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
451 "ndev_irq_isr", ndev);
452 if (rc)
453 goto err_intx_request;
454
455 dev_dbg(&pdev->dev, "Using intx interrupts\n");
456 ndev->db_vec_count = 1;
457 ndev->db_vec_shift = total_shift;
458 return 0;
459
460 err_intx_request:
461 return rc;
462 }
463
ndev_deinit_isr(struct intel_ntb_dev * ndev)464 static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
465 {
466 struct pci_dev *pdev;
467 int i;
468
469 pdev = ndev->ntb.pdev;
470
471 /* Mask all doorbell interrupts */
472 ndev->db_mask = ndev->db_valid_mask;
473 ndev->reg->db_iowrite(ndev->db_mask,
474 ndev->self_mmio +
475 ndev->self_reg->db_mask);
476
477 if (ndev->msix) {
478 i = ndev->db_vec_count;
479 while (i--)
480 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
481 pci_disable_msix(pdev);
482 kfree(ndev->msix);
483 kfree(ndev->vec);
484 } else {
485 free_irq(pdev->irq, ndev);
486 if (pci_dev_msi_enabled(pdev))
487 pci_disable_msi(pdev);
488 }
489 }
490
ndev_ntb_debugfs_read(struct file * filp,char __user * ubuf,size_t count,loff_t * offp)491 static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf,
492 size_t count, loff_t *offp)
493 {
494 struct intel_ntb_dev *ndev;
495 struct pci_dev *pdev;
496 void __iomem *mmio;
497 char *buf;
498 size_t buf_size;
499 ssize_t ret, off;
500 union { u64 v64; u32 v32; u16 v16; u8 v8; } u;
501
502 ndev = filp->private_data;
503 pdev = ndev->ntb.pdev;
504 mmio = ndev->self_mmio;
505
506 buf_size = min(count, 0x800ul);
507
508 buf = kmalloc(buf_size, GFP_KERNEL);
509 if (!buf)
510 return -ENOMEM;
511
512 off = 0;
513
514 off += scnprintf(buf + off, buf_size - off,
515 "NTB Device Information:\n");
516
517 off += scnprintf(buf + off, buf_size - off,
518 "Connection Topology -\t%s\n",
519 ntb_topo_string(ndev->ntb.topo));
520
521 if (ndev->b2b_idx != UINT_MAX) {
522 off += scnprintf(buf + off, buf_size - off,
523 "B2B MW Idx -\t\t%u\n", ndev->b2b_idx);
524 off += scnprintf(buf + off, buf_size - off,
525 "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
526 }
527
528 off += scnprintf(buf + off, buf_size - off,
529 "BAR4 Split -\t\t%s\n",
530 ndev->bar4_split ? "yes" : "no");
531
532 off += scnprintf(buf + off, buf_size - off,
533 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
534 off += scnprintf(buf + off, buf_size - off,
535 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
536
537 if (!ndev->reg->link_is_up(ndev)) {
538 off += scnprintf(buf + off, buf_size - off,
539 "Link Status -\t\tDown\n");
540 } else {
541 off += scnprintf(buf + off, buf_size - off,
542 "Link Status -\t\tUp\n");
543 off += scnprintf(buf + off, buf_size - off,
544 "Link Speed -\t\tPCI-E Gen %u\n",
545 NTB_LNK_STA_SPEED(ndev->lnk_sta));
546 off += scnprintf(buf + off, buf_size - off,
547 "Link Width -\t\tx%u\n",
548 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
549 }
550
551 off += scnprintf(buf + off, buf_size - off,
552 "Memory Window Count -\t%u\n", ndev->mw_count);
553 off += scnprintf(buf + off, buf_size - off,
554 "Scratchpad Count -\t%u\n", ndev->spad_count);
555 off += scnprintf(buf + off, buf_size - off,
556 "Doorbell Count -\t%u\n", ndev->db_count);
557 off += scnprintf(buf + off, buf_size - off,
558 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
559 off += scnprintf(buf + off, buf_size - off,
560 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
561
562 off += scnprintf(buf + off, buf_size - off,
563 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
564 off += scnprintf(buf + off, buf_size - off,
565 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
566 off += scnprintf(buf + off, buf_size - off,
567 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
568
569 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
570 off += scnprintf(buf + off, buf_size - off,
571 "Doorbell Mask -\t\t%#llx\n", u.v64);
572
573 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
574 off += scnprintf(buf + off, buf_size - off,
575 "Doorbell Bell -\t\t%#llx\n", u.v64);
576
577 off += scnprintf(buf + off, buf_size - off,
578 "\nNTB Window Size:\n");
579
580 pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &u.v8);
581 off += scnprintf(buf + off, buf_size - off,
582 "PBAR23SZ %hhu\n", u.v8);
583 if (!ndev->bar4_split) {
584 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &u.v8);
585 off += scnprintf(buf + off, buf_size - off,
586 "PBAR45SZ %hhu\n", u.v8);
587 } else {
588 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &u.v8);
589 off += scnprintf(buf + off, buf_size - off,
590 "PBAR4SZ %hhu\n", u.v8);
591 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &u.v8);
592 off += scnprintf(buf + off, buf_size - off,
593 "PBAR5SZ %hhu\n", u.v8);
594 }
595
596 pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &u.v8);
597 off += scnprintf(buf + off, buf_size - off,
598 "SBAR23SZ %hhu\n", u.v8);
599 if (!ndev->bar4_split) {
600 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &u.v8);
601 off += scnprintf(buf + off, buf_size - off,
602 "SBAR45SZ %hhu\n", u.v8);
603 } else {
604 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &u.v8);
605 off += scnprintf(buf + off, buf_size - off,
606 "SBAR4SZ %hhu\n", u.v8);
607 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &u.v8);
608 off += scnprintf(buf + off, buf_size - off,
609 "SBAR5SZ %hhu\n", u.v8);
610 }
611
612 off += scnprintf(buf + off, buf_size - off,
613 "\nNTB Incoming XLAT:\n");
614
615 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
616 off += scnprintf(buf + off, buf_size - off,
617 "XLAT23 -\t\t%#018llx\n", u.v64);
618
619 if (ndev->bar4_split) {
620 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
621 off += scnprintf(buf + off, buf_size - off,
622 "XLAT4 -\t\t\t%#06x\n", u.v32);
623
624 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5));
625 off += scnprintf(buf + off, buf_size - off,
626 "XLAT5 -\t\t\t%#06x\n", u.v32);
627 } else {
628 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
629 off += scnprintf(buf + off, buf_size - off,
630 "XLAT45 -\t\t%#018llx\n", u.v64);
631 }
632
633 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
634 off += scnprintf(buf + off, buf_size - off,
635 "LMT23 -\t\t\t%#018llx\n", u.v64);
636
637 if (ndev->bar4_split) {
638 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
639 off += scnprintf(buf + off, buf_size - off,
640 "LMT4 -\t\t\t%#06x\n", u.v32);
641 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5));
642 off += scnprintf(buf + off, buf_size - off,
643 "LMT5 -\t\t\t%#06x\n", u.v32);
644 } else {
645 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
646 off += scnprintf(buf + off, buf_size - off,
647 "LMT45 -\t\t\t%#018llx\n", u.v64);
648 }
649
650 if (pdev_is_gen1(pdev)) {
651 if (ntb_topo_is_b2b(ndev->ntb.topo)) {
652 off += scnprintf(buf + off, buf_size - off,
653 "\nNTB Outgoing B2B XLAT:\n");
654
655 u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
656 off += scnprintf(buf + off, buf_size - off,
657 "B2B XLAT23 -\t\t%#018llx\n", u.v64);
658
659 if (ndev->bar4_split) {
660 u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
661 off += scnprintf(buf + off, buf_size - off,
662 "B2B XLAT4 -\t\t%#06x\n",
663 u.v32);
664 u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
665 off += scnprintf(buf + off, buf_size - off,
666 "B2B XLAT5 -\t\t%#06x\n",
667 u.v32);
668 } else {
669 u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
670 off += scnprintf(buf + off, buf_size - off,
671 "B2B XLAT45 -\t\t%#018llx\n",
672 u.v64);
673 }
674
675 u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
676 off += scnprintf(buf + off, buf_size - off,
677 "B2B LMT23 -\t\t%#018llx\n", u.v64);
678
679 if (ndev->bar4_split) {
680 u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET);
681 off += scnprintf(buf + off, buf_size - off,
682 "B2B LMT4 -\t\t%#06x\n",
683 u.v32);
684 u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET);
685 off += scnprintf(buf + off, buf_size - off,
686 "B2B LMT5 -\t\t%#06x\n",
687 u.v32);
688 } else {
689 u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
690 off += scnprintf(buf + off, buf_size - off,
691 "B2B LMT45 -\t\t%#018llx\n",
692 u.v64);
693 }
694
695 off += scnprintf(buf + off, buf_size - off,
696 "\nNTB Secondary BAR:\n");
697
698 u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
699 off += scnprintf(buf + off, buf_size - off,
700 "SBAR01 -\t\t%#018llx\n", u.v64);
701
702 u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
703 off += scnprintf(buf + off, buf_size - off,
704 "SBAR23 -\t\t%#018llx\n", u.v64);
705
706 if (ndev->bar4_split) {
707 u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
708 off += scnprintf(buf + off, buf_size - off,
709 "SBAR4 -\t\t\t%#06x\n", u.v32);
710 u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
711 off += scnprintf(buf + off, buf_size - off,
712 "SBAR5 -\t\t\t%#06x\n", u.v32);
713 } else {
714 u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
715 off += scnprintf(buf + off, buf_size - off,
716 "SBAR45 -\t\t%#018llx\n",
717 u.v64);
718 }
719 }
720
721 off += scnprintf(buf + off, buf_size - off,
722 "\nXEON NTB Statistics:\n");
723
724 u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
725 off += scnprintf(buf + off, buf_size - off,
726 "Upstream Memory Miss -\t%u\n", u.v16);
727
728 off += scnprintf(buf + off, buf_size - off,
729 "\nXEON NTB Hardware Errors:\n");
730
731 if (!pci_read_config_word(pdev,
732 XEON_DEVSTS_OFFSET, &u.v16))
733 off += scnprintf(buf + off, buf_size - off,
734 "DEVSTS -\t\t%#06x\n", u.v16);
735
736 if (!pci_read_config_word(pdev,
737 XEON_LINK_STATUS_OFFSET, &u.v16))
738 off += scnprintf(buf + off, buf_size - off,
739 "LNKSTS -\t\t%#06x\n", u.v16);
740
741 if (!pci_read_config_dword(pdev,
742 XEON_UNCERRSTS_OFFSET, &u.v32))
743 off += scnprintf(buf + off, buf_size - off,
744 "UNCERRSTS -\t\t%#06x\n", u.v32);
745
746 if (!pci_read_config_dword(pdev,
747 XEON_CORERRSTS_OFFSET, &u.v32))
748 off += scnprintf(buf + off, buf_size - off,
749 "CORERRSTS -\t\t%#06x\n", u.v32);
750 }
751
752 ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
753 kfree(buf);
754 return ret;
755 }
756
ndev_debugfs_read(struct file * filp,char __user * ubuf,size_t count,loff_t * offp)757 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
758 size_t count, loff_t *offp)
759 {
760 struct intel_ntb_dev *ndev = filp->private_data;
761
762 if (pdev_is_gen1(ndev->ntb.pdev))
763 return ndev_ntb_debugfs_read(filp, ubuf, count, offp);
764 else if (pdev_is_gen3(ndev->ntb.pdev))
765 return ndev_ntb3_debugfs_read(filp, ubuf, count, offp);
766 else if (pdev_is_gen4(ndev->ntb.pdev) || pdev_is_gen5(ndev->ntb.pdev))
767 return ndev_ntb4_debugfs_read(filp, ubuf, count, offp);
768
769 return -ENXIO;
770 }
771
ndev_init_debugfs(struct intel_ntb_dev * ndev)772 static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
773 {
774 if (!debugfs_dir) {
775 ndev->debugfs_dir = NULL;
776 ndev->debugfs_info = NULL;
777 } else {
778 ndev->debugfs_dir =
779 debugfs_create_dir(pci_name(ndev->ntb.pdev),
780 debugfs_dir);
781 if (IS_ERR(ndev->debugfs_dir))
782 ndev->debugfs_info = NULL;
783 else
784 ndev->debugfs_info =
785 debugfs_create_file("info", S_IRUSR,
786 ndev->debugfs_dir, ndev,
787 &intel_ntb_debugfs_info);
788 }
789 }
790
ndev_deinit_debugfs(struct intel_ntb_dev * ndev)791 static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
792 {
793 debugfs_remove_recursive(ndev->debugfs_dir);
794 }
795
intel_ntb_mw_count(struct ntb_dev * ntb,int pidx)796 int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx)
797 {
798 if (pidx != NTB_DEF_PEER_IDX)
799 return -EINVAL;
800
801 return ntb_ndev(ntb)->mw_count;
802 }
803
intel_ntb_mw_get_align(struct ntb_dev * ntb,int pidx,int idx,resource_size_t * addr_align,resource_size_t * size_align,resource_size_t * size_max)804 int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
805 resource_size_t *addr_align,
806 resource_size_t *size_align,
807 resource_size_t *size_max)
808 {
809 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
810 resource_size_t bar_size, mw_size;
811 int bar;
812
813 if (pidx != NTB_DEF_PEER_IDX)
814 return -EINVAL;
815
816 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
817 idx += 1;
818
819 bar = ndev_mw_to_bar(ndev, idx);
820 if (bar < 0)
821 return bar;
822
823 bar_size = pci_resource_len(ndev->ntb.pdev, bar);
824
825 if (idx == ndev->b2b_idx)
826 mw_size = bar_size - ndev->b2b_off;
827 else
828 mw_size = bar_size;
829
830 if (addr_align)
831 *addr_align = pci_resource_len(ndev->ntb.pdev, bar);
832
833 if (size_align)
834 *size_align = 1;
835
836 if (size_max)
837 *size_max = mw_size;
838
839 return 0;
840 }
841
intel_ntb_mw_set_trans(struct ntb_dev * ntb,int pidx,int idx,dma_addr_t addr,resource_size_t size)842 static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
843 dma_addr_t addr, resource_size_t size)
844 {
845 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
846 unsigned long base_reg, xlat_reg, limit_reg;
847 resource_size_t bar_size, mw_size;
848 void __iomem *mmio;
849 u64 base, limit, reg_val;
850 int bar;
851
852 if (pidx != NTB_DEF_PEER_IDX)
853 return -EINVAL;
854
855 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
856 idx += 1;
857
858 bar = ndev_mw_to_bar(ndev, idx);
859 if (bar < 0)
860 return bar;
861
862 bar_size = pci_resource_len(ndev->ntb.pdev, bar);
863
864 if (idx == ndev->b2b_idx)
865 mw_size = bar_size - ndev->b2b_off;
866 else
867 mw_size = bar_size;
868
869 /* hardware requires that addr is aligned to bar size */
870 if (addr & (bar_size - 1))
871 return -EINVAL;
872
873 /* make sure the range fits in the usable mw size */
874 if (size > mw_size)
875 return -EINVAL;
876
877 mmio = ndev->self_mmio;
878 base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
879 xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
880 limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
881
882 if (bar < 4 || !ndev->bar4_split) {
883 base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
884
885 /* Set the limit if supported, if size is not mw_size */
886 if (limit_reg && size != mw_size)
887 limit = base + size;
888 else
889 limit = 0;
890
891 /* set and verify setting the translation address */
892 iowrite64(addr, mmio + xlat_reg);
893 reg_val = ioread64(mmio + xlat_reg);
894 if (reg_val != addr) {
895 iowrite64(0, mmio + xlat_reg);
896 return -EIO;
897 }
898
899 /* set and verify setting the limit */
900 iowrite64(limit, mmio + limit_reg);
901 reg_val = ioread64(mmio + limit_reg);
902 if (reg_val != limit) {
903 iowrite64(base, mmio + limit_reg);
904 iowrite64(0, mmio + xlat_reg);
905 return -EIO;
906 }
907 } else {
908 /* split bar addr range must all be 32 bit */
909 if (addr & (~0ull << 32))
910 return -EINVAL;
911 if ((addr + size) & (~0ull << 32))
912 return -EINVAL;
913
914 base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
915
916 /* Set the limit if supported, if size is not mw_size */
917 if (limit_reg && size != mw_size)
918 limit = base + size;
919 else
920 limit = 0;
921
922 /* set and verify setting the translation address */
923 iowrite32(addr, mmio + xlat_reg);
924 reg_val = ioread32(mmio + xlat_reg);
925 if (reg_val != addr) {
926 iowrite32(0, mmio + xlat_reg);
927 return -EIO;
928 }
929
930 /* set and verify setting the limit */
931 iowrite32(limit, mmio + limit_reg);
932 reg_val = ioread32(mmio + limit_reg);
933 if (reg_val != limit) {
934 iowrite32(base, mmio + limit_reg);
935 iowrite32(0, mmio + xlat_reg);
936 return -EIO;
937 }
938 }
939
940 return 0;
941 }
942
intel_ntb_link_is_up(struct ntb_dev * ntb,enum ntb_speed * speed,enum ntb_width * width)943 u64 intel_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed,
944 enum ntb_width *width)
945 {
946 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
947
948 if (ndev->reg->link_is_up(ndev)) {
949 if (speed)
950 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
951 if (width)
952 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
953 return 1;
954 } else {
955 /* TODO MAYBE: is it possible to observe the link speed and
956 * width while link is training? */
957 if (speed)
958 *speed = NTB_SPEED_NONE;
959 if (width)
960 *width = NTB_WIDTH_NONE;
961 return 0;
962 }
963 }
964
intel_ntb_link_enable(struct ntb_dev * ntb,enum ntb_speed max_speed,enum ntb_width max_width)965 static int intel_ntb_link_enable(struct ntb_dev *ntb,
966 enum ntb_speed max_speed,
967 enum ntb_width max_width)
968 {
969 struct intel_ntb_dev *ndev;
970 u32 ntb_ctl;
971
972 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
973
974 if (ndev->ntb.topo == NTB_TOPO_SEC)
975 return -EINVAL;
976
977 dev_dbg(&ntb->pdev->dev,
978 "Enabling link with max_speed %d max_width %d\n",
979 max_speed, max_width);
980 if (max_speed != NTB_SPEED_AUTO)
981 dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed);
982 if (max_width != NTB_WIDTH_AUTO)
983 dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width);
984
985 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
986 ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
987 ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
988 ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
989 if (ndev->bar4_split)
990 ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
991 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
992
993 return 0;
994 }
995
intel_ntb_link_disable(struct ntb_dev * ntb)996 int intel_ntb_link_disable(struct ntb_dev *ntb)
997 {
998 struct intel_ntb_dev *ndev;
999 u32 ntb_cntl;
1000
1001 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1002
1003 if (ndev->ntb.topo == NTB_TOPO_SEC)
1004 return -EINVAL;
1005
1006 dev_dbg(&ntb->pdev->dev, "Disabling link\n");
1007
1008 /* Bring NTB link down */
1009 ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1010 ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
1011 ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
1012 if (ndev->bar4_split)
1013 ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
1014 ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
1015 iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
1016
1017 return 0;
1018 }
1019
intel_ntb_peer_mw_count(struct ntb_dev * ntb)1020 int intel_ntb_peer_mw_count(struct ntb_dev *ntb)
1021 {
1022 /* Numbers of inbound and outbound memory windows match */
1023 return ntb_ndev(ntb)->mw_count;
1024 }
1025
intel_ntb_peer_mw_get_addr(struct ntb_dev * ntb,int idx,phys_addr_t * base,resource_size_t * size)1026 int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
1027 phys_addr_t *base, resource_size_t *size)
1028 {
1029 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1030 int bar;
1031
1032 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1033 idx += 1;
1034
1035 bar = ndev_mw_to_bar(ndev, idx);
1036 if (bar < 0)
1037 return bar;
1038
1039 if (base)
1040 *base = pci_resource_start(ndev->ntb.pdev, bar) +
1041 (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1042
1043 if (size)
1044 *size = pci_resource_len(ndev->ntb.pdev, bar) -
1045 (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1046
1047 return 0;
1048 }
1049
intel_ntb_db_is_unsafe(struct ntb_dev * ntb)1050 static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
1051 {
1052 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
1053 }
1054
intel_ntb_db_valid_mask(struct ntb_dev * ntb)1055 u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
1056 {
1057 return ntb_ndev(ntb)->db_valid_mask;
1058 }
1059
intel_ntb_db_vector_count(struct ntb_dev * ntb)1060 int intel_ntb_db_vector_count(struct ntb_dev *ntb)
1061 {
1062 struct intel_ntb_dev *ndev;
1063
1064 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1065
1066 return ndev->db_vec_count;
1067 }
1068
intel_ntb_db_vector_mask(struct ntb_dev * ntb,int db_vector)1069 u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
1070 {
1071 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1072
1073 if (db_vector < 0 || db_vector > ndev->db_vec_count)
1074 return 0;
1075
1076 return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
1077 }
1078
intel_ntb_db_read(struct ntb_dev * ntb)1079 static u64 intel_ntb_db_read(struct ntb_dev *ntb)
1080 {
1081 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1082
1083 return ndev_db_read(ndev,
1084 ndev->self_mmio +
1085 ndev->self_reg->db_bell);
1086 }
1087
intel_ntb_db_clear(struct ntb_dev * ntb,u64 db_bits)1088 static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
1089 {
1090 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1091
1092 return ndev_db_write(ndev, db_bits,
1093 ndev->self_mmio +
1094 ndev->self_reg->db_bell);
1095 }
1096
intel_ntb_db_set_mask(struct ntb_dev * ntb,u64 db_bits)1097 int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1098 {
1099 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1100
1101 return ndev_db_set_mask(ndev, db_bits,
1102 ndev->self_mmio +
1103 ndev->self_reg->db_mask);
1104 }
1105
intel_ntb_db_clear_mask(struct ntb_dev * ntb,u64 db_bits)1106 int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1107 {
1108 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1109
1110 return ndev_db_clear_mask(ndev, db_bits,
1111 ndev->self_mmio +
1112 ndev->self_reg->db_mask);
1113 }
1114
intel_ntb_peer_db_addr(struct ntb_dev * ntb,phys_addr_t * db_addr,resource_size_t * db_size,u64 * db_data,int db_bit)1115 static int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
1116 resource_size_t *db_size, u64 *db_data, int db_bit)
1117 {
1118 u64 db_bits;
1119 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1120
1121 if (unlikely(db_bit >= BITS_PER_LONG_LONG))
1122 return -EINVAL;
1123
1124 db_bits = BIT_ULL(db_bit);
1125
1126 if (unlikely(db_bits & ~ntb_ndev(ntb)->db_valid_mask))
1127 return -EINVAL;
1128
1129 ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
1130 ndev->peer_reg->db_bell);
1131
1132 if (db_data)
1133 *db_data = db_bits;
1134
1135
1136 return 0;
1137 }
1138
intel_ntb_peer_db_set(struct ntb_dev * ntb,u64 db_bits)1139 static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1140 {
1141 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1142
1143 return ndev_db_write(ndev, db_bits,
1144 ndev->peer_mmio +
1145 ndev->peer_reg->db_bell);
1146 }
1147
intel_ntb_spad_is_unsafe(struct ntb_dev * ntb)1148 int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
1149 {
1150 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
1151 }
1152
intel_ntb_spad_count(struct ntb_dev * ntb)1153 int intel_ntb_spad_count(struct ntb_dev *ntb)
1154 {
1155 struct intel_ntb_dev *ndev;
1156
1157 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1158
1159 return ndev->spad_count;
1160 }
1161
intel_ntb_spad_read(struct ntb_dev * ntb,int idx)1162 u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
1163 {
1164 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1165
1166 return ndev_spad_read(ndev, idx,
1167 ndev->self_mmio +
1168 ndev->self_reg->spad);
1169 }
1170
intel_ntb_spad_write(struct ntb_dev * ntb,int idx,u32 val)1171 int intel_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
1172 {
1173 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1174
1175 return ndev_spad_write(ndev, idx, val,
1176 ndev->self_mmio +
1177 ndev->self_reg->spad);
1178 }
1179
intel_ntb_peer_spad_addr(struct ntb_dev * ntb,int pidx,int sidx,phys_addr_t * spad_addr)1180 int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
1181 phys_addr_t *spad_addr)
1182 {
1183 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1184
1185 return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr,
1186 ndev->peer_reg->spad);
1187 }
1188
intel_ntb_peer_spad_read(struct ntb_dev * ntb,int pidx,int sidx)1189 u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
1190 {
1191 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1192
1193 return ndev_spad_read(ndev, sidx,
1194 ndev->peer_mmio +
1195 ndev->peer_reg->spad);
1196 }
1197
intel_ntb_peer_spad_write(struct ntb_dev * ntb,int pidx,int sidx,u32 val)1198 int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx,
1199 u32 val)
1200 {
1201 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1202
1203 return ndev_spad_write(ndev, sidx, val,
1204 ndev->peer_mmio +
1205 ndev->peer_reg->spad);
1206 }
1207
xeon_db_ioread(const void __iomem * mmio)1208 static u64 xeon_db_ioread(const void __iomem *mmio)
1209 {
1210 return (u64)ioread16(mmio);
1211 }
1212
xeon_db_iowrite(u64 bits,void __iomem * mmio)1213 static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
1214 {
1215 iowrite16((u16)bits, mmio);
1216 }
1217
xeon_poll_link(struct intel_ntb_dev * ndev)1218 static int xeon_poll_link(struct intel_ntb_dev *ndev)
1219 {
1220 u16 reg_val;
1221 int rc;
1222
1223 ndev->reg->db_iowrite(ndev->db_link_mask,
1224 ndev->self_mmio +
1225 ndev->self_reg->db_bell);
1226
1227 rc = pci_read_config_word(ndev->ntb.pdev,
1228 XEON_LINK_STATUS_OFFSET, ®_val);
1229 if (rc)
1230 return 0;
1231
1232 if (reg_val == ndev->lnk_sta)
1233 return 0;
1234
1235 ndev->lnk_sta = reg_val;
1236
1237 return 1;
1238 }
1239
xeon_link_is_up(struct intel_ntb_dev * ndev)1240 int xeon_link_is_up(struct intel_ntb_dev *ndev)
1241 {
1242 if (ndev->ntb.topo == NTB_TOPO_SEC)
1243 return 1;
1244
1245 return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
1246 }
1247
xeon_ppd_topo(struct intel_ntb_dev * ndev,u8 ppd)1248 enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
1249 {
1250 switch (ppd & XEON_PPD_TOPO_MASK) {
1251 case XEON_PPD_TOPO_B2B_USD:
1252 return NTB_TOPO_B2B_USD;
1253
1254 case XEON_PPD_TOPO_B2B_DSD:
1255 return NTB_TOPO_B2B_DSD;
1256
1257 case XEON_PPD_TOPO_PRI_USD:
1258 case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1259 return NTB_TOPO_PRI;
1260
1261 case XEON_PPD_TOPO_SEC_USD:
1262 case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1263 return NTB_TOPO_SEC;
1264 }
1265
1266 return NTB_TOPO_NONE;
1267 }
1268
xeon_ppd_bar4_split(struct intel_ntb_dev * ndev,u8 ppd)1269 static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
1270 {
1271 if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
1272 dev_dbg(&ndev->ntb.pdev->dev, "PPD %d split bar\n", ppd);
1273 return 1;
1274 }
1275 return 0;
1276 }
1277
xeon_init_isr(struct intel_ntb_dev * ndev)1278 static int xeon_init_isr(struct intel_ntb_dev *ndev)
1279 {
1280 return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
1281 XEON_DB_MSIX_VECTOR_COUNT,
1282 XEON_DB_MSIX_VECTOR_SHIFT,
1283 XEON_DB_TOTAL_SHIFT);
1284 }
1285
xeon_deinit_isr(struct intel_ntb_dev * ndev)1286 static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
1287 {
1288 ndev_deinit_isr(ndev);
1289 }
1290
xeon_setup_b2b_mw(struct intel_ntb_dev * ndev,const struct intel_b2b_addr * addr,const struct intel_b2b_addr * peer_addr)1291 static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
1292 const struct intel_b2b_addr *addr,
1293 const struct intel_b2b_addr *peer_addr)
1294 {
1295 struct pci_dev *pdev;
1296 void __iomem *mmio;
1297 resource_size_t bar_size;
1298 phys_addr_t bar_addr;
1299 int b2b_bar;
1300 u8 bar_sz;
1301
1302 pdev = ndev->ntb.pdev;
1303 mmio = ndev->self_mmio;
1304
1305 if (ndev->b2b_idx == UINT_MAX) {
1306 dev_dbg(&pdev->dev, "not using b2b mw\n");
1307 b2b_bar = 0;
1308 ndev->b2b_off = 0;
1309 } else {
1310 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
1311 if (b2b_bar < 0)
1312 return -EIO;
1313
1314 dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar);
1315
1316 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
1317
1318 dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size);
1319
1320 if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
1321 dev_dbg(&pdev->dev, "b2b using first half of bar\n");
1322 ndev->b2b_off = bar_size >> 1;
1323 } else if (XEON_B2B_MIN_SIZE <= bar_size) {
1324 dev_dbg(&pdev->dev, "b2b using whole bar\n");
1325 ndev->b2b_off = 0;
1326 --ndev->mw_count;
1327 } else {
1328 dev_dbg(&pdev->dev, "b2b bar size is too small\n");
1329 return -EIO;
1330 }
1331 }
1332
1333 /* Reset the secondary bar sizes to match the primary bar sizes,
1334 * except disable or halve the size of the b2b secondary bar.
1335 *
1336 * Note: code for each specific bar size register, because the register
1337 * offsets are not in a consistent order (bar5sz comes after ppd, odd).
1338 */
1339 pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
1340 dev_dbg(&pdev->dev, "PBAR23SZ %#x\n", bar_sz);
1341 if (b2b_bar == 2) {
1342 if (ndev->b2b_off)
1343 bar_sz -= 1;
1344 else
1345 bar_sz = 0;
1346 }
1347 pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
1348 pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
1349 dev_dbg(&pdev->dev, "SBAR23SZ %#x\n", bar_sz);
1350
1351 if (!ndev->bar4_split) {
1352 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
1353 dev_dbg(&pdev->dev, "PBAR45SZ %#x\n", bar_sz);
1354 if (b2b_bar == 4) {
1355 if (ndev->b2b_off)
1356 bar_sz -= 1;
1357 else
1358 bar_sz = 0;
1359 }
1360 pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
1361 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
1362 dev_dbg(&pdev->dev, "SBAR45SZ %#x\n", bar_sz);
1363 } else {
1364 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
1365 dev_dbg(&pdev->dev, "PBAR4SZ %#x\n", bar_sz);
1366 if (b2b_bar == 4) {
1367 if (ndev->b2b_off)
1368 bar_sz -= 1;
1369 else
1370 bar_sz = 0;
1371 }
1372 pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
1373 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
1374 dev_dbg(&pdev->dev, "SBAR4SZ %#x\n", bar_sz);
1375
1376 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
1377 dev_dbg(&pdev->dev, "PBAR5SZ %#x\n", bar_sz);
1378 if (b2b_bar == 5) {
1379 if (ndev->b2b_off)
1380 bar_sz -= 1;
1381 else
1382 bar_sz = 0;
1383 }
1384 pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
1385 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
1386 dev_dbg(&pdev->dev, "SBAR5SZ %#x\n", bar_sz);
1387 }
1388
1389 /* SBAR01 hit by first part of the b2b bar */
1390 if (b2b_bar == 0)
1391 bar_addr = addr->bar0_addr;
1392 else if (b2b_bar == 2)
1393 bar_addr = addr->bar2_addr64;
1394 else if (b2b_bar == 4 && !ndev->bar4_split)
1395 bar_addr = addr->bar4_addr64;
1396 else if (b2b_bar == 4)
1397 bar_addr = addr->bar4_addr32;
1398 else if (b2b_bar == 5)
1399 bar_addr = addr->bar5_addr32;
1400 else
1401 return -EIO;
1402
1403 dev_dbg(&pdev->dev, "SBAR01 %#018llx\n", bar_addr);
1404 iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
1405
1406 /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
1407 * The b2b bar is either disabled above, or configured half-size, and
1408 * it starts at the PBAR xlat + offset.
1409 */
1410
1411 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1412 iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
1413 bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
1414 dev_dbg(&pdev->dev, "SBAR23 %#018llx\n", bar_addr);
1415
1416 if (!ndev->bar4_split) {
1417 bar_addr = addr->bar4_addr64 +
1418 (b2b_bar == 4 ? ndev->b2b_off : 0);
1419 iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
1420 bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
1421 dev_dbg(&pdev->dev, "SBAR45 %#018llx\n", bar_addr);
1422 } else {
1423 bar_addr = addr->bar4_addr32 +
1424 (b2b_bar == 4 ? ndev->b2b_off : 0);
1425 iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
1426 bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
1427 dev_dbg(&pdev->dev, "SBAR4 %#010llx\n", bar_addr);
1428
1429 bar_addr = addr->bar5_addr32 +
1430 (b2b_bar == 5 ? ndev->b2b_off : 0);
1431 iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
1432 bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
1433 dev_dbg(&pdev->dev, "SBAR5 %#010llx\n", bar_addr);
1434 }
1435
1436 /* setup incoming bar limits == base addrs (zero length windows) */
1437
1438 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1439 iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
1440 bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
1441 dev_dbg(&pdev->dev, "SBAR23LMT %#018llx\n", bar_addr);
1442
1443 if (!ndev->bar4_split) {
1444 bar_addr = addr->bar4_addr64 +
1445 (b2b_bar == 4 ? ndev->b2b_off : 0);
1446 iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
1447 bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
1448 dev_dbg(&pdev->dev, "SBAR45LMT %#018llx\n", bar_addr);
1449 } else {
1450 bar_addr = addr->bar4_addr32 +
1451 (b2b_bar == 4 ? ndev->b2b_off : 0);
1452 iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
1453 bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
1454 dev_dbg(&pdev->dev, "SBAR4LMT %#010llx\n", bar_addr);
1455
1456 bar_addr = addr->bar5_addr32 +
1457 (b2b_bar == 5 ? ndev->b2b_off : 0);
1458 iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
1459 bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
1460 dev_dbg(&pdev->dev, "SBAR5LMT %#05llx\n", bar_addr);
1461 }
1462
1463 /* zero incoming translation addrs */
1464 iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
1465
1466 if (!ndev->bar4_split) {
1467 iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
1468 } else {
1469 iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
1470 iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
1471 }
1472
1473 /* zero outgoing translation limits (whole bar size windows) */
1474 iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
1475 if (!ndev->bar4_split) {
1476 iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
1477 } else {
1478 iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
1479 iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
1480 }
1481
1482 /* set outgoing translation offsets */
1483 bar_addr = peer_addr->bar2_addr64;
1484 iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
1485 bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
1486 dev_dbg(&pdev->dev, "PBAR23XLAT %#018llx\n", bar_addr);
1487
1488 if (!ndev->bar4_split) {
1489 bar_addr = peer_addr->bar4_addr64;
1490 iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
1491 bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
1492 dev_dbg(&pdev->dev, "PBAR45XLAT %#018llx\n", bar_addr);
1493 } else {
1494 bar_addr = peer_addr->bar4_addr32;
1495 iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
1496 bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
1497 dev_dbg(&pdev->dev, "PBAR4XLAT %#010llx\n", bar_addr);
1498
1499 bar_addr = peer_addr->bar5_addr32;
1500 iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
1501 bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
1502 dev_dbg(&pdev->dev, "PBAR5XLAT %#010llx\n", bar_addr);
1503 }
1504
1505 /* set the translation offset for b2b registers */
1506 if (b2b_bar == 0)
1507 bar_addr = peer_addr->bar0_addr;
1508 else if (b2b_bar == 2)
1509 bar_addr = peer_addr->bar2_addr64;
1510 else if (b2b_bar == 4 && !ndev->bar4_split)
1511 bar_addr = peer_addr->bar4_addr64;
1512 else if (b2b_bar == 4)
1513 bar_addr = peer_addr->bar4_addr32;
1514 else if (b2b_bar == 5)
1515 bar_addr = peer_addr->bar5_addr32;
1516 else
1517 return -EIO;
1518
1519 /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
1520 dev_dbg(&pdev->dev, "B2BXLAT %#018llx\n", bar_addr);
1521 iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
1522 iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
1523
1524 if (b2b_bar) {
1525 /* map peer ntb mmio config space registers */
1526 ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
1527 XEON_B2B_MIN_SIZE);
1528 if (!ndev->peer_mmio)
1529 return -EIO;
1530
1531 ndev->peer_addr = pci_resource_start(pdev, b2b_bar);
1532 }
1533
1534 return 0;
1535 }
1536
xeon_init_ntb(struct intel_ntb_dev * ndev)1537 static int xeon_init_ntb(struct intel_ntb_dev *ndev)
1538 {
1539 struct device *dev = &ndev->ntb.pdev->dev;
1540 int rc;
1541 u32 ntb_ctl;
1542
1543 if (ndev->bar4_split)
1544 ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
1545 else
1546 ndev->mw_count = XEON_MW_COUNT;
1547
1548 ndev->spad_count = XEON_SPAD_COUNT;
1549 ndev->db_count = XEON_DB_COUNT;
1550 ndev->db_link_mask = XEON_DB_LINK_BIT;
1551
1552 switch (ndev->ntb.topo) {
1553 case NTB_TOPO_PRI:
1554 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1555 dev_err(dev, "NTB Primary config disabled\n");
1556 return -EINVAL;
1557 }
1558
1559 /* enable link to allow secondary side device to appear */
1560 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1561 ntb_ctl &= ~NTB_CTL_DISABLE;
1562 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1563
1564 /* use half the spads for the peer */
1565 ndev->spad_count >>= 1;
1566 ndev->self_reg = &xeon_pri_reg;
1567 ndev->peer_reg = &xeon_sec_reg;
1568 ndev->xlat_reg = &xeon_sec_xlat;
1569 break;
1570
1571 case NTB_TOPO_SEC:
1572 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1573 dev_err(dev, "NTB Secondary config disabled\n");
1574 return -EINVAL;
1575 }
1576 /* use half the spads for the peer */
1577 ndev->spad_count >>= 1;
1578 ndev->self_reg = &xeon_sec_reg;
1579 ndev->peer_reg = &xeon_pri_reg;
1580 ndev->xlat_reg = &xeon_pri_xlat;
1581 break;
1582
1583 case NTB_TOPO_B2B_USD:
1584 case NTB_TOPO_B2B_DSD:
1585 ndev->self_reg = &xeon_pri_reg;
1586 ndev->peer_reg = &xeon_b2b_reg;
1587 ndev->xlat_reg = &xeon_sec_xlat;
1588
1589 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1590 ndev->peer_reg = &xeon_pri_reg;
1591
1592 if (b2b_mw_idx < 0)
1593 ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
1594 else
1595 ndev->b2b_idx = b2b_mw_idx;
1596
1597 if (ndev->b2b_idx >= ndev->mw_count) {
1598 dev_dbg(dev,
1599 "b2b_mw_idx %d invalid for mw_count %u\n",
1600 b2b_mw_idx, ndev->mw_count);
1601 return -EINVAL;
1602 }
1603
1604 dev_dbg(dev, "setting up b2b mw idx %d means %d\n",
1605 b2b_mw_idx, ndev->b2b_idx);
1606
1607 } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
1608 dev_warn(dev, "Reduce doorbell count by 1\n");
1609 ndev->db_count -= 1;
1610 }
1611
1612 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
1613 rc = xeon_setup_b2b_mw(ndev,
1614 &xeon_b2b_dsd_addr,
1615 &xeon_b2b_usd_addr);
1616 } else {
1617 rc = xeon_setup_b2b_mw(ndev,
1618 &xeon_b2b_usd_addr,
1619 &xeon_b2b_dsd_addr);
1620 }
1621 if (rc)
1622 return rc;
1623
1624 /* Enable Bus Master and Memory Space on the secondary side */
1625 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1626 ndev->self_mmio + XEON_SPCICMD_OFFSET);
1627
1628 break;
1629
1630 default:
1631 return -EINVAL;
1632 }
1633
1634 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1635
1636 ndev->reg->db_iowrite(ndev->db_valid_mask,
1637 ndev->self_mmio +
1638 ndev->self_reg->db_mask);
1639
1640 return 0;
1641 }
1642
xeon_init_dev(struct intel_ntb_dev * ndev)1643 static int xeon_init_dev(struct intel_ntb_dev *ndev)
1644 {
1645 struct pci_dev *pdev;
1646 u8 ppd;
1647 int rc, mem;
1648
1649 pdev = ndev->ntb.pdev;
1650
1651 switch (pdev->device) {
1652 /* There is a Xeon hardware errata related to writes to SDOORBELL or
1653 * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
1654 * which may hang the system. To workaround this use the second memory
1655 * window to access the interrupt and scratch pad registers on the
1656 * remote system.
1657 */
1658 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1659 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1660 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1661 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1662 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1663 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1664 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1665 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1666 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1667 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1668 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1669 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1670 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1671 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1672 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1673 ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
1674 break;
1675 }
1676
1677 switch (pdev->device) {
1678 /* There is a hardware errata related to accessing any register in
1679 * SB01BASE in the presence of bidirectional traffic crossing the NTB.
1680 */
1681 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1682 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1683 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1684 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1685 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1686 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1687 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1688 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1689 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1690 ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
1691 break;
1692 }
1693
1694 switch (pdev->device) {
1695 /* HW Errata on bit 14 of b2bdoorbell register. Writes will not be
1696 * mirrored to the remote system. Shrink the number of bits by one,
1697 * since bit 14 is the last bit.
1698 */
1699 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1700 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1701 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1702 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1703 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1704 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1705 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1706 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1707 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1708 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1709 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1710 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1711 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1712 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1713 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1714 ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
1715 break;
1716 }
1717
1718 ndev->reg = &xeon_reg;
1719
1720 rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
1721 if (rc)
1722 return -EIO;
1723
1724 ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
1725 dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd,
1726 ntb_topo_string(ndev->ntb.topo));
1727 if (ndev->ntb.topo == NTB_TOPO_NONE)
1728 return -EINVAL;
1729
1730 if (ndev->ntb.topo != NTB_TOPO_SEC) {
1731 ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
1732 dev_dbg(&pdev->dev, "ppd %#x bar4_split %d\n",
1733 ppd, ndev->bar4_split);
1734 } else {
1735 /* This is a way for transparent BAR to figure out if we are
1736 * doing split BAR or not. There is no way for the hw on the
1737 * transparent side to know and set the PPD.
1738 */
1739 mem = pci_select_bars(pdev, IORESOURCE_MEM);
1740 ndev->bar4_split = hweight32(mem) ==
1741 HSX_SPLIT_BAR_MW_COUNT + 1;
1742 dev_dbg(&pdev->dev, "mem %#x bar4_split %d\n",
1743 mem, ndev->bar4_split);
1744 }
1745
1746 rc = xeon_init_ntb(ndev);
1747 if (rc)
1748 return rc;
1749
1750 return xeon_init_isr(ndev);
1751 }
1752
xeon_deinit_dev(struct intel_ntb_dev * ndev)1753 static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
1754 {
1755 xeon_deinit_isr(ndev);
1756 }
1757
intel_ntb_init_pci(struct intel_ntb_dev * ndev,struct pci_dev * pdev)1758 static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
1759 {
1760 int rc;
1761
1762 pci_set_drvdata(pdev, ndev);
1763
1764 rc = pci_enable_device(pdev);
1765 if (rc)
1766 goto err_pci_enable;
1767
1768 rc = pci_request_regions(pdev, NTB_NAME);
1769 if (rc)
1770 goto err_pci_regions;
1771
1772 pci_set_master(pdev);
1773
1774 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1775 if (rc) {
1776 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1777 if (rc)
1778 goto err_dma_mask;
1779 dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1780 }
1781
1782 ndev->self_mmio = pci_iomap(pdev, 0, 0);
1783 if (!ndev->self_mmio) {
1784 rc = -EIO;
1785 goto err_mmio;
1786 }
1787 ndev->peer_mmio = ndev->self_mmio;
1788 ndev->peer_addr = pci_resource_start(pdev, 0);
1789
1790 return 0;
1791
1792 err_mmio:
1793 err_dma_mask:
1794 pci_release_regions(pdev);
1795 err_pci_regions:
1796 pci_disable_device(pdev);
1797 err_pci_enable:
1798 pci_set_drvdata(pdev, NULL);
1799 return rc;
1800 }
1801
intel_ntb_deinit_pci(struct intel_ntb_dev * ndev)1802 static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
1803 {
1804 struct pci_dev *pdev = ndev->ntb.pdev;
1805
1806 if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
1807 pci_iounmap(pdev, ndev->peer_mmio);
1808 pci_iounmap(pdev, ndev->self_mmio);
1809
1810 pci_release_regions(pdev);
1811 pci_disable_device(pdev);
1812 pci_set_drvdata(pdev, NULL);
1813 }
1814
ndev_init_struct(struct intel_ntb_dev * ndev,struct pci_dev * pdev)1815 static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
1816 struct pci_dev *pdev)
1817 {
1818 ndev->ntb.pdev = pdev;
1819 ndev->ntb.topo = NTB_TOPO_NONE;
1820 ndev->ntb.ops = &intel_ntb_ops;
1821
1822 ndev->b2b_off = 0;
1823 ndev->b2b_idx = UINT_MAX;
1824
1825 ndev->bar4_split = 0;
1826
1827 ndev->mw_count = 0;
1828 ndev->spad_count = 0;
1829 ndev->db_count = 0;
1830 ndev->db_vec_count = 0;
1831 ndev->db_vec_shift = 0;
1832
1833 ndev->ntb_ctl = 0;
1834 ndev->lnk_sta = 0;
1835
1836 ndev->db_valid_mask = 0;
1837 ndev->db_link_mask = 0;
1838 ndev->db_mask = 0;
1839
1840 spin_lock_init(&ndev->db_mask_lock);
1841 }
1842
intel_ntb_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)1843 static int intel_ntb_pci_probe(struct pci_dev *pdev,
1844 const struct pci_device_id *id)
1845 {
1846 struct intel_ntb_dev *ndev;
1847 int rc, node;
1848
1849 node = dev_to_node(&pdev->dev);
1850 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1851 if (!ndev) {
1852 rc = -ENOMEM;
1853 goto err_ndev;
1854 }
1855
1856 ndev_init_struct(ndev, pdev);
1857
1858 if (pdev_is_gen1(pdev)) {
1859 rc = intel_ntb_init_pci(ndev, pdev);
1860 if (rc)
1861 goto err_init_pci;
1862
1863 rc = xeon_init_dev(ndev);
1864 if (rc)
1865 goto err_init_dev;
1866 } else if (pdev_is_gen3(pdev)) {
1867 ndev->ntb.ops = &intel_ntb3_ops;
1868 rc = intel_ntb_init_pci(ndev, pdev);
1869 if (rc)
1870 goto err_init_pci;
1871
1872 rc = gen3_init_dev(ndev);
1873 if (rc)
1874 goto err_init_dev;
1875 } else if (pdev_is_gen4(pdev) || pdev_is_gen5(pdev)) {
1876 ndev->ntb.ops = &intel_ntb4_ops;
1877 rc = intel_ntb_init_pci(ndev, pdev);
1878 if (rc)
1879 goto err_init_pci;
1880
1881 rc = gen4_init_dev(ndev);
1882 if (rc)
1883 goto err_init_dev;
1884 } else {
1885 rc = -EINVAL;
1886 goto err_init_pci;
1887 }
1888
1889 ndev_reset_unsafe_flags(ndev);
1890
1891 ndev->reg->poll_link(ndev);
1892
1893 ndev_init_debugfs(ndev);
1894
1895 rc = ntb_register_device(&ndev->ntb);
1896 if (rc)
1897 goto err_register;
1898
1899 dev_info(&pdev->dev, "NTB device registered.\n");
1900
1901 return 0;
1902
1903 err_register:
1904 ndev_deinit_debugfs(ndev);
1905 if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev) ||
1906 pdev_is_gen4(pdev) || pdev_is_gen5(pdev))
1907 xeon_deinit_dev(ndev);
1908 err_init_dev:
1909 intel_ntb_deinit_pci(ndev);
1910 err_init_pci:
1911 kfree(ndev);
1912 err_ndev:
1913 return rc;
1914 }
1915
intel_ntb_pci_remove(struct pci_dev * pdev)1916 static void intel_ntb_pci_remove(struct pci_dev *pdev)
1917 {
1918 struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
1919
1920 ntb_unregister_device(&ndev->ntb);
1921 ndev_deinit_debugfs(ndev);
1922 if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev) ||
1923 pdev_is_gen4(pdev) || pdev_is_gen5(pdev))
1924 xeon_deinit_dev(ndev);
1925 intel_ntb_deinit_pci(ndev);
1926 kfree(ndev);
1927 }
1928
1929 static const struct intel_ntb_reg xeon_reg = {
1930 .poll_link = xeon_poll_link,
1931 .link_is_up = xeon_link_is_up,
1932 .db_ioread = xeon_db_ioread,
1933 .db_iowrite = xeon_db_iowrite,
1934 .db_size = sizeof(u32),
1935 .ntb_ctl = XEON_NTBCNTL_OFFSET,
1936 .mw_bar = {2, 4, 5},
1937 };
1938
1939 static const struct intel_ntb_alt_reg xeon_pri_reg = {
1940 .db_bell = XEON_PDOORBELL_OFFSET,
1941 .db_mask = XEON_PDBMSK_OFFSET,
1942 .spad = XEON_SPAD_OFFSET,
1943 };
1944
1945 static const struct intel_ntb_alt_reg xeon_sec_reg = {
1946 .db_bell = XEON_SDOORBELL_OFFSET,
1947 .db_mask = XEON_SDBMSK_OFFSET,
1948 /* second half of the scratchpads */
1949 .spad = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
1950 };
1951
1952 static const struct intel_ntb_alt_reg xeon_b2b_reg = {
1953 .db_bell = XEON_B2B_DOORBELL_OFFSET,
1954 .spad = XEON_B2B_SPAD_OFFSET,
1955 };
1956
1957 static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
1958 /* Note: no primary .bar0_base visible to the secondary side.
1959 *
1960 * The secondary side cannot get the base address stored in primary
1961 * bars. The base address is necessary to set the limit register to
1962 * any value other than zero, or unlimited.
1963 *
1964 * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
1965 * window by setting the limit equal to base, nor can it limit the size
1966 * of the memory window by setting the limit to base + size.
1967 */
1968 .bar2_limit = XEON_PBAR23LMT_OFFSET,
1969 .bar2_xlat = XEON_PBAR23XLAT_OFFSET,
1970 };
1971
1972 static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
1973 .bar0_base = XEON_SBAR0BASE_OFFSET,
1974 .bar2_limit = XEON_SBAR23LMT_OFFSET,
1975 .bar2_xlat = XEON_SBAR23XLAT_OFFSET,
1976 };
1977
1978 struct intel_b2b_addr xeon_b2b_usd_addr = {
1979 .bar2_addr64 = XEON_B2B_BAR2_ADDR64,
1980 .bar4_addr64 = XEON_B2B_BAR4_ADDR64,
1981 .bar4_addr32 = XEON_B2B_BAR4_ADDR32,
1982 .bar5_addr32 = XEON_B2B_BAR5_ADDR32,
1983 };
1984
1985 struct intel_b2b_addr xeon_b2b_dsd_addr = {
1986 .bar2_addr64 = XEON_B2B_BAR2_ADDR64,
1987 .bar4_addr64 = XEON_B2B_BAR4_ADDR64,
1988 .bar4_addr32 = XEON_B2B_BAR4_ADDR32,
1989 .bar5_addr32 = XEON_B2B_BAR5_ADDR32,
1990 };
1991
1992 /* operations for primary side of local ntb */
1993 static const struct ntb_dev_ops intel_ntb_ops = {
1994 .mw_count = intel_ntb_mw_count,
1995 .mw_get_align = intel_ntb_mw_get_align,
1996 .mw_set_trans = intel_ntb_mw_set_trans,
1997 .peer_mw_count = intel_ntb_peer_mw_count,
1998 .peer_mw_get_addr = intel_ntb_peer_mw_get_addr,
1999 .link_is_up = intel_ntb_link_is_up,
2000 .link_enable = intel_ntb_link_enable,
2001 .link_disable = intel_ntb_link_disable,
2002 .db_is_unsafe = intel_ntb_db_is_unsafe,
2003 .db_valid_mask = intel_ntb_db_valid_mask,
2004 .db_vector_count = intel_ntb_db_vector_count,
2005 .db_vector_mask = intel_ntb_db_vector_mask,
2006 .db_read = intel_ntb_db_read,
2007 .db_clear = intel_ntb_db_clear,
2008 .db_set_mask = intel_ntb_db_set_mask,
2009 .db_clear_mask = intel_ntb_db_clear_mask,
2010 .peer_db_addr = intel_ntb_peer_db_addr,
2011 .peer_db_set = intel_ntb_peer_db_set,
2012 .spad_is_unsafe = intel_ntb_spad_is_unsafe,
2013 .spad_count = intel_ntb_spad_count,
2014 .spad_read = intel_ntb_spad_read,
2015 .spad_write = intel_ntb_spad_write,
2016 .peer_spad_addr = intel_ntb_peer_spad_addr,
2017 .peer_spad_read = intel_ntb_peer_spad_read,
2018 .peer_spad_write = intel_ntb_peer_spad_write,
2019 };
2020
2021 static const struct file_operations intel_ntb_debugfs_info = {
2022 .owner = THIS_MODULE,
2023 .open = simple_open,
2024 .read = ndev_debugfs_read,
2025 };
2026
2027 static const struct pci_device_id intel_ntb_pci_tbl[] = {
2028 /* GEN1 */
2029 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
2030 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
2031 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
2032 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
2033 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)},
2034 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
2035 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
2036 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
2037 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
2038 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)},
2039 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
2040 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
2041 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
2042 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
2043 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)},
2044
2045 /* GEN3 */
2046 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)},
2047
2048 /* GEN4 */
2049 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_ICX)},
2050 /* GEN5 PCIe */
2051 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_GNR)},
2052 {0}
2053 };
2054 MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
2055
2056 static struct pci_driver intel_ntb_pci_driver = {
2057 .name = KBUILD_MODNAME,
2058 .id_table = intel_ntb_pci_tbl,
2059 .probe = intel_ntb_pci_probe,
2060 .remove = intel_ntb_pci_remove,
2061 };
2062
intel_ntb_pci_driver_init(void)2063 static int __init intel_ntb_pci_driver_init(void)
2064 {
2065 int ret;
2066 pr_info("%s %s\n", NTB_DESC, NTB_VER);
2067
2068 if (debugfs_initialized())
2069 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
2070
2071 ret = pci_register_driver(&intel_ntb_pci_driver);
2072 if (ret)
2073 debugfs_remove_recursive(debugfs_dir);
2074
2075 return ret;
2076 }
2077 module_init(intel_ntb_pci_driver_init);
2078
intel_ntb_pci_driver_exit(void)2079 static void __exit intel_ntb_pci_driver_exit(void)
2080 {
2081 pci_unregister_driver(&intel_ntb_pci_driver);
2082
2083 debugfs_remove_recursive(debugfs_dir);
2084 }
2085 module_exit(intel_ntb_pci_driver_exit);
2086