1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Endpoint Function Driver to implement Non-Transparent Bridge functionality
4 * Between PCI RC and EP
5 *
6 * Copyright (C) 2020 Texas Instruments
7 * Copyright (C) 2022 NXP
8 *
9 * Based on pci-epf-ntb.c
10 * Author: Frank Li <Frank.Li@nxp.com>
11 * Author: Kishon Vijay Abraham I <kishon@ti.com>
12 */
13
14 /*
15 * +------------+ +---------------------------------------+
16 * | | | |
17 * +------------+ | +--------------+
18 * | NTB | | | NTB |
19 * | NetDev | | | NetDev |
20 * +------------+ | +--------------+
21 * | NTB | | | NTB |
22 * | Transfer | | | Transfer |
23 * +------------+ | +--------------+
24 * | | | | |
25 * | PCI NTB | | | |
26 * | EPF | | | |
27 * | Driver | | | PCI Virtual |
28 * | | +---------------+ | NTB Driver |
29 * | | | PCI EP NTB |<------>| |
30 * | | | FN Driver | | |
31 * +------------+ +---------------+ +--------------+
32 * | | | | | |
33 * | PCI Bus | <-----> | PCI EP Bus | | Virtual PCI |
34 * | | PCI | | | Bus |
35 * +------------+ +---------------+--------+--------------+
36 * PCIe Root Port PCI EP
37 */
38
39 #include <linux/delay.h>
40 #include <linux/io.h>
41 #include <linux/module.h>
42 #include <linux/slab.h>
43
44 #include <linux/pci-epc.h>
45 #include <linux/pci-epf.h>
46 #include <linux/ntb.h>
47
48 static struct workqueue_struct *kpcintb_workqueue;
49
50 #define COMMAND_CONFIGURE_DOORBELL 1
51 #define COMMAND_TEARDOWN_DOORBELL 2
52 #define COMMAND_CONFIGURE_MW 3
53 #define COMMAND_TEARDOWN_MW 4
54 #define COMMAND_LINK_UP 5
55 #define COMMAND_LINK_DOWN 6
56
57 #define COMMAND_STATUS_OK 1
58 #define COMMAND_STATUS_ERROR 2
59
60 #define LINK_STATUS_UP BIT(0)
61
62 #define SPAD_COUNT 64
63 #define DB_COUNT 4
64 #define NTB_MW_OFFSET 2
65 #define DB_COUNT_MASK GENMASK(15, 0)
66 #define MSIX_ENABLE BIT(16)
67 #define MAX_DB_COUNT 32
68 #define MAX_MW 4
69
70 enum epf_ntb_bar {
71 BAR_CONFIG,
72 BAR_DB,
73 BAR_MW0,
74 BAR_MW1,
75 BAR_MW2,
76 };
77
78 /*
79 * +--------------------------------------------------+ Base
80 * | |
81 * | |
82 * | |
83 * | Common Control Register |
84 * | |
85 * | |
86 * | |
87 * +-----------------------+--------------------------+ Base+spad_offset
88 * | | |
89 * | Peer Spad Space | Spad Space |
90 * | | |
91 * | | |
92 * +-----------------------+--------------------------+ Base+spad_offset
93 * | | | +spad_count * 4
94 * | | |
95 * | Spad Space | Peer Spad Space |
96 * | | |
97 * +-----------------------+--------------------------+
98 * Virtual PCI PCIe Endpoint
99 * NTB Driver NTB Driver
100 */
101 struct epf_ntb_ctrl {
102 u32 command;
103 u32 argument;
104 u16 command_status;
105 u16 link_status;
106 u32 topology;
107 u64 addr;
108 u64 size;
109 u32 num_mws;
110 u32 reserved;
111 u32 spad_offset;
112 u32 spad_count;
113 u32 db_entry_size;
114 u32 db_data[MAX_DB_COUNT];
115 u32 db_offset[MAX_DB_COUNT];
116 } __packed;
117
118 struct epf_ntb {
119 struct ntb_dev ntb;
120 struct pci_epf *epf;
121 struct config_group group;
122
123 u32 num_mws;
124 u32 db_count;
125 u32 spad_count;
126 u64 mws_size[MAX_MW];
127 u64 db;
128 u32 vbus_number;
129 u16 vntb_pid;
130 u16 vntb_vid;
131
132 bool linkup;
133 u32 spad_size;
134
135 enum pci_barno epf_ntb_bar[6];
136
137 struct epf_ntb_ctrl *reg;
138
139 u32 *epf_db;
140
141 phys_addr_t vpci_mw_phy[MAX_MW];
142 void __iomem *vpci_mw_addr[MAX_MW];
143
144 struct delayed_work cmd_handler;
145 };
146
147 #define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group)
148 #define ntb_ndev(__ntb) container_of(__ntb, struct epf_ntb, ntb)
149
150 static struct pci_epf_header epf_ntb_header = {
151 .vendorid = PCI_ANY_ID,
152 .deviceid = PCI_ANY_ID,
153 .baseclass_code = PCI_BASE_CLASS_MEMORY,
154 .interrupt_pin = PCI_INTERRUPT_INTA,
155 };
156
157 /**
158 * epf_ntb_link_up() - Raise link_up interrupt to Virtual Host (VHOST)
159 * @ntb: NTB device that facilitates communication between HOST and VHOST
160 * @link_up: true or false indicating Link is UP or Down
161 *
162 * Once NTB function in HOST invoke ntb_link_enable(),
163 * this NTB function driver will trigger a link event to VHOST.
164 *
165 * Returns: Zero for success, or an error code in case of failure
166 */
epf_ntb_link_up(struct epf_ntb * ntb,bool link_up)167 static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up)
168 {
169 if (link_up)
170 ntb->reg->link_status |= LINK_STATUS_UP;
171 else
172 ntb->reg->link_status &= ~LINK_STATUS_UP;
173
174 ntb_link_event(&ntb->ntb);
175 return 0;
176 }
177
178 /**
179 * epf_ntb_configure_mw() - Configure the Outbound Address Space for VHOST
180 * to access the memory window of HOST
181 * @ntb: NTB device that facilitates communication between HOST and VHOST
182 * @mw: Index of the memory window (either 0, 1, 2 or 3)
183 *
184 * EP Outbound Window
185 * +--------+ +-----------+
186 * | | | |
187 * | | | |
188 * | | | |
189 * | | | |
190 * | | +-----------+
191 * | Virtual| | Memory Win|
192 * | NTB | -----------> | |
193 * | Driver | | |
194 * | | +-----------+
195 * | | | |
196 * | | | |
197 * +--------+ +-----------+
198 * VHOST PCI EP
199 *
200 * Returns: Zero for success, or an error code in case of failure
201 */
epf_ntb_configure_mw(struct epf_ntb * ntb,u32 mw)202 static int epf_ntb_configure_mw(struct epf_ntb *ntb, u32 mw)
203 {
204 phys_addr_t phys_addr;
205 u8 func_no, vfunc_no;
206 u64 addr, size;
207 int ret = 0;
208
209 phys_addr = ntb->vpci_mw_phy[mw];
210 addr = ntb->reg->addr;
211 size = ntb->reg->size;
212
213 func_no = ntb->epf->func_no;
214 vfunc_no = ntb->epf->vfunc_no;
215
216 ret = pci_epc_map_addr(ntb->epf->epc, func_no, vfunc_no, phys_addr, addr, size);
217 if (ret)
218 dev_err(&ntb->epf->epc->dev,
219 "Failed to map memory window %d address\n", mw);
220 return ret;
221 }
222
223 /**
224 * epf_ntb_teardown_mw() - Teardown the configured OB ATU
225 * @ntb: NTB device that facilitates communication between HOST and VHOST
226 * @mw: Index of the memory window (either 0, 1, 2 or 3)
227 *
228 * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using
229 * pci_epc_unmap_addr()
230 */
epf_ntb_teardown_mw(struct epf_ntb * ntb,u32 mw)231 static void epf_ntb_teardown_mw(struct epf_ntb *ntb, u32 mw)
232 {
233 pci_epc_unmap_addr(ntb->epf->epc,
234 ntb->epf->func_no,
235 ntb->epf->vfunc_no,
236 ntb->vpci_mw_phy[mw]);
237 }
238
239 /**
240 * epf_ntb_cmd_handler() - Handle commands provided by the NTB HOST
241 * @work: work_struct for the epf_ntb_epc
242 *
243 * Workqueue function that gets invoked for the two epf_ntb_epc
244 * periodically (once every 5ms) to see if it has received any commands
245 * from NTB HOST. The HOST can send commands to configure doorbell or
246 * configure memory window or to update link status.
247 */
epf_ntb_cmd_handler(struct work_struct * work)248 static void epf_ntb_cmd_handler(struct work_struct *work)
249 {
250 struct epf_ntb_ctrl *ctrl;
251 u32 command, argument;
252 struct epf_ntb *ntb;
253 struct device *dev;
254 int ret;
255 int i;
256
257 ntb = container_of(work, struct epf_ntb, cmd_handler.work);
258
259 for (i = 1; i < ntb->db_count; i++) {
260 if (ntb->epf_db[i]) {
261 ntb->db |= 1 << (i - 1);
262 ntb_db_event(&ntb->ntb, i);
263 ntb->epf_db[i] = 0;
264 }
265 }
266
267 ctrl = ntb->reg;
268 command = ctrl->command;
269 if (!command)
270 goto reset_handler;
271 argument = ctrl->argument;
272
273 ctrl->command = 0;
274 ctrl->argument = 0;
275
276 ctrl = ntb->reg;
277 dev = &ntb->epf->dev;
278
279 switch (command) {
280 case COMMAND_CONFIGURE_DOORBELL:
281 ctrl->command_status = COMMAND_STATUS_OK;
282 break;
283 case COMMAND_TEARDOWN_DOORBELL:
284 ctrl->command_status = COMMAND_STATUS_OK;
285 break;
286 case COMMAND_CONFIGURE_MW:
287 ret = epf_ntb_configure_mw(ntb, argument);
288 if (ret < 0)
289 ctrl->command_status = COMMAND_STATUS_ERROR;
290 else
291 ctrl->command_status = COMMAND_STATUS_OK;
292 break;
293 case COMMAND_TEARDOWN_MW:
294 epf_ntb_teardown_mw(ntb, argument);
295 ctrl->command_status = COMMAND_STATUS_OK;
296 break;
297 case COMMAND_LINK_UP:
298 ntb->linkup = true;
299 ret = epf_ntb_link_up(ntb, true);
300 if (ret < 0)
301 ctrl->command_status = COMMAND_STATUS_ERROR;
302 else
303 ctrl->command_status = COMMAND_STATUS_OK;
304 goto reset_handler;
305 case COMMAND_LINK_DOWN:
306 ntb->linkup = false;
307 ret = epf_ntb_link_up(ntb, false);
308 if (ret < 0)
309 ctrl->command_status = COMMAND_STATUS_ERROR;
310 else
311 ctrl->command_status = COMMAND_STATUS_OK;
312 break;
313 default:
314 dev_err(dev, "UNKNOWN command: %d\n", command);
315 break;
316 }
317
318 reset_handler:
319 queue_delayed_work(kpcintb_workqueue, &ntb->cmd_handler,
320 msecs_to_jiffies(5));
321 }
322
323 /**
324 * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR
325 * @ntb: EPC associated with one of the HOST which holds peer's outbound
326 * address.
327 *
328 * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and
329 * self scratchpad region (removes inbound ATU configuration). While BAR0 is
330 * the default self scratchpad BAR, an NTB could have other BARs for self
331 * scratchpad (because of reserved BARs). This function can get the exact BAR
332 * used for self scratchpad from epf_ntb_bar[BAR_CONFIG].
333 *
334 * Please note the self scratchpad region and config region is combined to
335 * a single region and mapped using the same BAR. Also note VHOST's peer
336 * scratchpad is HOST's self scratchpad.
337 *
338 * Returns: void
339 */
epf_ntb_config_sspad_bar_clear(struct epf_ntb * ntb)340 static void epf_ntb_config_sspad_bar_clear(struct epf_ntb *ntb)
341 {
342 struct pci_epf_bar *epf_bar;
343 enum pci_barno barno;
344
345 barno = ntb->epf_ntb_bar[BAR_CONFIG];
346 epf_bar = &ntb->epf->bar[barno];
347
348 pci_epc_clear_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);
349 }
350
351 /**
352 * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR
353 * @ntb: NTB device that facilitates communication between HOST and VHOST
354 *
355 * Map BAR0 of EP CONTROLLER which contains the VHOST's config and
356 * self scratchpad region.
357 *
358 * Please note the self scratchpad region and config region is combined to
359 * a single region and mapped using the same BAR.
360 *
361 * Returns: Zero for success, or an error code in case of failure
362 */
epf_ntb_config_sspad_bar_set(struct epf_ntb * ntb)363 static int epf_ntb_config_sspad_bar_set(struct epf_ntb *ntb)
364 {
365 struct pci_epf_bar *epf_bar;
366 enum pci_barno barno;
367 u8 func_no, vfunc_no;
368 struct device *dev;
369 int ret;
370
371 dev = &ntb->epf->dev;
372 func_no = ntb->epf->func_no;
373 vfunc_no = ntb->epf->vfunc_no;
374 barno = ntb->epf_ntb_bar[BAR_CONFIG];
375 epf_bar = &ntb->epf->bar[barno];
376
377 ret = pci_epc_set_bar(ntb->epf->epc, func_no, vfunc_no, epf_bar);
378 if (ret) {
379 dev_err(dev, "inft: Config/Status/SPAD BAR set failed\n");
380 return ret;
381 }
382 return 0;
383 }
384
385 /**
386 * epf_ntb_config_spad_bar_free() - Free the physical memory associated with
387 * config + scratchpad region
388 * @ntb: NTB device that facilitates communication between HOST and VHOST
389 */
epf_ntb_config_spad_bar_free(struct epf_ntb * ntb)390 static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)
391 {
392 enum pci_barno barno;
393
394 barno = ntb->epf_ntb_bar[BAR_CONFIG];
395 pci_epf_free_space(ntb->epf, ntb->reg, barno, 0);
396 }
397
398 /**
399 * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad
400 * region
401 * @ntb: NTB device that facilitates communication between HOST and VHOST
402 *
403 * Allocate the Local Memory mentioned in the above diagram. The size of
404 * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION
405 * is obtained from "spad-count" configfs entry.
406 *
407 * Returns: Zero for success, or an error code in case of failure
408 */
epf_ntb_config_spad_bar_alloc(struct epf_ntb * ntb)409 static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb)
410 {
411 size_t align;
412 enum pci_barno barno;
413 struct epf_ntb_ctrl *ctrl;
414 u32 spad_size, ctrl_size;
415 u64 size;
416 struct pci_epf *epf = ntb->epf;
417 struct device *dev = &epf->dev;
418 u32 spad_count;
419 void *base;
420 int i;
421 const struct pci_epc_features *epc_features = pci_epc_get_features(epf->epc,
422 epf->func_no,
423 epf->vfunc_no);
424 barno = ntb->epf_ntb_bar[BAR_CONFIG];
425 size = epc_features->bar[barno].fixed_size;
426 align = epc_features->align;
427
428 if ((!IS_ALIGNED(size, align)))
429 return -EINVAL;
430
431 spad_count = ntb->spad_count;
432
433 ctrl_size = sizeof(struct epf_ntb_ctrl);
434 spad_size = 2 * spad_count * sizeof(u32);
435
436 if (!align) {
437 ctrl_size = roundup_pow_of_two(ctrl_size);
438 spad_size = roundup_pow_of_two(spad_size);
439 } else {
440 ctrl_size = ALIGN(ctrl_size, align);
441 spad_size = ALIGN(spad_size, align);
442 }
443
444 if (!size)
445 size = ctrl_size + spad_size;
446 else if (size < ctrl_size + spad_size)
447 return -EINVAL;
448
449 base = pci_epf_alloc_space(epf, size, barno, epc_features, 0);
450 if (!base) {
451 dev_err(dev, "Config/Status/SPAD alloc region fail\n");
452 return -ENOMEM;
453 }
454
455 ntb->reg = base;
456
457 ctrl = ntb->reg;
458 ctrl->spad_offset = ctrl_size;
459
460 ctrl->spad_count = spad_count;
461 ctrl->num_mws = ntb->num_mws;
462 ntb->spad_size = spad_size;
463
464 ctrl->db_entry_size = sizeof(u32);
465
466 for (i = 0; i < ntb->db_count; i++) {
467 ntb->reg->db_data[i] = 1 + i;
468 ntb->reg->db_offset[i] = 0;
469 }
470
471 return 0;
472 }
473
474 /**
475 * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capability
476 * @ntb: NTB device that facilitates communication between HOST and VHOST
477 *
478 * Configure MSI/MSI-X capability for each interface with number of
479 * interrupts equal to "db_count" configfs entry.
480 *
481 * Returns: Zero for success, or an error code in case of failure
482 */
epf_ntb_configure_interrupt(struct epf_ntb * ntb)483 static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)
484 {
485 const struct pci_epc_features *epc_features;
486 struct device *dev;
487 u32 db_count;
488 int ret;
489
490 dev = &ntb->epf->dev;
491
492 epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);
493
494 if (!(epc_features->msix_capable || epc_features->msi_capable)) {
495 dev_err(dev, "MSI or MSI-X is required for doorbell\n");
496 return -EINVAL;
497 }
498
499 db_count = ntb->db_count;
500 if (db_count > MAX_DB_COUNT) {
501 dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT);
502 return -EINVAL;
503 }
504
505 ntb->db_count = db_count;
506
507 if (epc_features->msi_capable) {
508 ret = pci_epc_set_msi(ntb->epf->epc,
509 ntb->epf->func_no,
510 ntb->epf->vfunc_no,
511 16);
512 if (ret) {
513 dev_err(dev, "MSI configuration failed\n");
514 return ret;
515 }
516 }
517
518 return 0;
519 }
520
521 /**
522 * epf_ntb_db_bar_init() - Configure Doorbell window BARs
523 * @ntb: NTB device that facilitates communication between HOST and VHOST
524 *
525 * Returns: Zero for success, or an error code in case of failure
526 */
epf_ntb_db_bar_init(struct epf_ntb * ntb)527 static int epf_ntb_db_bar_init(struct epf_ntb *ntb)
528 {
529 const struct pci_epc_features *epc_features;
530 struct device *dev = &ntb->epf->dev;
531 int ret;
532 struct pci_epf_bar *epf_bar;
533 void __iomem *mw_addr;
534 enum pci_barno barno;
535 size_t size = sizeof(u32) * ntb->db_count;
536
537 epc_features = pci_epc_get_features(ntb->epf->epc,
538 ntb->epf->func_no,
539 ntb->epf->vfunc_no);
540 barno = ntb->epf_ntb_bar[BAR_DB];
541
542 mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, epc_features, 0);
543 if (!mw_addr) {
544 dev_err(dev, "Failed to allocate OB address\n");
545 return -ENOMEM;
546 }
547
548 ntb->epf_db = mw_addr;
549
550 epf_bar = &ntb->epf->bar[barno];
551
552 ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);
553 if (ret) {
554 dev_err(dev, "Doorbell BAR set failed\n");
555 goto err_alloc_peer_mem;
556 }
557 return ret;
558
559 err_alloc_peer_mem:
560 pci_epf_free_space(ntb->epf, mw_addr, barno, 0);
561 return -1;
562 }
563
564 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws);
565
566 /**
567 * epf_ntb_db_bar_clear() - Clear doorbell BAR and free memory
568 * allocated in peer's outbound address space
569 * @ntb: NTB device that facilitates communication between HOST and VHOST
570 */
epf_ntb_db_bar_clear(struct epf_ntb * ntb)571 static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)
572 {
573 enum pci_barno barno;
574
575 barno = ntb->epf_ntb_bar[BAR_DB];
576 pci_epf_free_space(ntb->epf, ntb->epf_db, barno, 0);
577 pci_epc_clear_bar(ntb->epf->epc,
578 ntb->epf->func_no,
579 ntb->epf->vfunc_no,
580 &ntb->epf->bar[barno]);
581 }
582
583 /**
584 * epf_ntb_mw_bar_init() - Configure Memory window BARs
585 * @ntb: NTB device that facilitates communication between HOST and VHOST
586 *
587 * Returns: Zero for success, or an error code in case of failure
588 */
epf_ntb_mw_bar_init(struct epf_ntb * ntb)589 static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
590 {
591 int ret = 0;
592 int i;
593 u64 size;
594 enum pci_barno barno;
595 struct device *dev = &ntb->epf->dev;
596
597 for (i = 0; i < ntb->num_mws; i++) {
598 size = ntb->mws_size[i];
599 barno = ntb->epf_ntb_bar[BAR_MW0 + i];
600
601 ntb->epf->bar[barno].barno = barno;
602 ntb->epf->bar[barno].size = size;
603 ntb->epf->bar[barno].addr = NULL;
604 ntb->epf->bar[barno].phys_addr = 0;
605 ntb->epf->bar[barno].flags |= upper_32_bits(size) ?
606 PCI_BASE_ADDRESS_MEM_TYPE_64 :
607 PCI_BASE_ADDRESS_MEM_TYPE_32;
608
609 ret = pci_epc_set_bar(ntb->epf->epc,
610 ntb->epf->func_no,
611 ntb->epf->vfunc_no,
612 &ntb->epf->bar[barno]);
613 if (ret) {
614 dev_err(dev, "MW set failed\n");
615 goto err_alloc_mem;
616 }
617
618 /* Allocate EPC outbound memory windows to vpci vntb device */
619 ntb->vpci_mw_addr[i] = pci_epc_mem_alloc_addr(ntb->epf->epc,
620 &ntb->vpci_mw_phy[i],
621 size);
622 if (!ntb->vpci_mw_addr[i]) {
623 ret = -ENOMEM;
624 dev_err(dev, "Failed to allocate source address\n");
625 goto err_set_bar;
626 }
627 }
628
629 return ret;
630
631 err_set_bar:
632 pci_epc_clear_bar(ntb->epf->epc,
633 ntb->epf->func_no,
634 ntb->epf->vfunc_no,
635 &ntb->epf->bar[barno]);
636 err_alloc_mem:
637 epf_ntb_mw_bar_clear(ntb, i);
638 return ret;
639 }
640
641 /**
642 * epf_ntb_mw_bar_clear() - Clear Memory window BARs
643 * @ntb: NTB device that facilitates communication between HOST and VHOST
644 * @num_mws: the number of Memory window BARs that to be cleared
645 */
epf_ntb_mw_bar_clear(struct epf_ntb * ntb,int num_mws)646 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws)
647 {
648 enum pci_barno barno;
649 int i;
650
651 for (i = 0; i < num_mws; i++) {
652 barno = ntb->epf_ntb_bar[BAR_MW0 + i];
653 pci_epc_clear_bar(ntb->epf->epc,
654 ntb->epf->func_no,
655 ntb->epf->vfunc_no,
656 &ntb->epf->bar[barno]);
657
658 pci_epc_mem_free_addr(ntb->epf->epc,
659 ntb->vpci_mw_phy[i],
660 ntb->vpci_mw_addr[i],
661 ntb->mws_size[i]);
662 }
663 }
664
665 /**
666 * epf_ntb_epc_destroy() - Cleanup NTB EPC interface
667 * @ntb: NTB device that facilitates communication between HOST and VHOST
668 *
669 * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces
670 */
epf_ntb_epc_destroy(struct epf_ntb * ntb)671 static void epf_ntb_epc_destroy(struct epf_ntb *ntb)
672 {
673 pci_epc_remove_epf(ntb->epf->epc, ntb->epf, 0);
674 pci_epc_put(ntb->epf->epc);
675 }
676
677 /**
678 * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB
679 * constructs (scratchpad region, doorbell, memorywindow)
680 * @ntb: NTB device that facilitates communication between HOST and VHOST
681 *
682 * Returns: Zero for success, or an error code in case of failure
683 */
epf_ntb_init_epc_bar(struct epf_ntb * ntb)684 static int epf_ntb_init_epc_bar(struct epf_ntb *ntb)
685 {
686 const struct pci_epc_features *epc_features;
687 enum pci_barno barno;
688 enum epf_ntb_bar bar;
689 struct device *dev;
690 u32 num_mws;
691 int i;
692
693 barno = BAR_0;
694 num_mws = ntb->num_mws;
695 dev = &ntb->epf->dev;
696 epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);
697
698 /* These are required BARs which are mandatory for NTB functionality */
699 for (bar = BAR_CONFIG; bar <= BAR_MW0; bar++, barno++) {
700 barno = pci_epc_get_next_free_bar(epc_features, barno);
701 if (barno < 0) {
702 dev_err(dev, "Fail to get NTB function BAR\n");
703 return barno;
704 }
705 ntb->epf_ntb_bar[bar] = barno;
706 }
707
708 /* These are optional BARs which don't impact NTB functionality */
709 for (bar = BAR_MW1, i = 1; i < num_mws; bar++, barno++, i++) {
710 barno = pci_epc_get_next_free_bar(epc_features, barno);
711 if (barno < 0) {
712 ntb->num_mws = i;
713 dev_dbg(dev, "BAR not available for > MW%d\n", i + 1);
714 }
715 ntb->epf_ntb_bar[bar] = barno;
716 }
717
718 return 0;
719 }
720
721 /**
722 * epf_ntb_epc_init() - Initialize NTB interface
723 * @ntb: NTB device that facilitates communication between HOST and VHOST
724 *
725 * Wrapper to initialize a particular EPC interface and start the workqueue
726 * to check for commands from HOST. This function will write to the
727 * EP controller HW for configuring it.
728 *
729 * Returns: Zero for success, or an error code in case of failure
730 */
epf_ntb_epc_init(struct epf_ntb * ntb)731 static int epf_ntb_epc_init(struct epf_ntb *ntb)
732 {
733 u8 func_no, vfunc_no;
734 struct pci_epc *epc;
735 struct pci_epf *epf;
736 struct device *dev;
737 int ret;
738
739 epf = ntb->epf;
740 dev = &epf->dev;
741 epc = epf->epc;
742 func_no = ntb->epf->func_no;
743 vfunc_no = ntb->epf->vfunc_no;
744
745 ret = epf_ntb_config_sspad_bar_set(ntb);
746 if (ret) {
747 dev_err(dev, "Config/self SPAD BAR init failed");
748 return ret;
749 }
750
751 ret = epf_ntb_configure_interrupt(ntb);
752 if (ret) {
753 dev_err(dev, "Interrupt configuration failed\n");
754 goto err_config_interrupt;
755 }
756
757 ret = epf_ntb_db_bar_init(ntb);
758 if (ret) {
759 dev_err(dev, "DB BAR init failed\n");
760 goto err_db_bar_init;
761 }
762
763 ret = epf_ntb_mw_bar_init(ntb);
764 if (ret) {
765 dev_err(dev, "MW BAR init failed\n");
766 goto err_mw_bar_init;
767 }
768
769 if (vfunc_no <= 1) {
770 ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header);
771 if (ret) {
772 dev_err(dev, "Configuration header write failed\n");
773 goto err_write_header;
774 }
775 }
776
777 INIT_DELAYED_WORK(&ntb->cmd_handler, epf_ntb_cmd_handler);
778 queue_work(kpcintb_workqueue, &ntb->cmd_handler.work);
779
780 return 0;
781
782 err_write_header:
783 epf_ntb_mw_bar_clear(ntb, ntb->num_mws);
784 err_mw_bar_init:
785 epf_ntb_db_bar_clear(ntb);
786 err_db_bar_init:
787 err_config_interrupt:
788 epf_ntb_config_sspad_bar_clear(ntb);
789
790 return ret;
791 }
792
793
794 /**
795 * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces
796 * @ntb: NTB device that facilitates communication between HOST and VHOST
797 *
798 * Wrapper to cleanup all NTB interfaces.
799 */
epf_ntb_epc_cleanup(struct epf_ntb * ntb)800 static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)
801 {
802 epf_ntb_mw_bar_clear(ntb, ntb->num_mws);
803 epf_ntb_db_bar_clear(ntb);
804 epf_ntb_config_sspad_bar_clear(ntb);
805 }
806
807 #define EPF_NTB_R(_name) \
808 static ssize_t epf_ntb_##_name##_show(struct config_item *item, \
809 char *page) \
810 { \
811 struct config_group *group = to_config_group(item); \
812 struct epf_ntb *ntb = to_epf_ntb(group); \
813 \
814 return sprintf(page, "%d\n", ntb->_name); \
815 }
816
817 #define EPF_NTB_W(_name) \
818 static ssize_t epf_ntb_##_name##_store(struct config_item *item, \
819 const char *page, size_t len) \
820 { \
821 struct config_group *group = to_config_group(item); \
822 struct epf_ntb *ntb = to_epf_ntb(group); \
823 u32 val; \
824 int ret; \
825 \
826 ret = kstrtou32(page, 0, &val); \
827 if (ret) \
828 return ret; \
829 \
830 ntb->_name = val; \
831 \
832 return len; \
833 }
834
835 #define EPF_NTB_MW_R(_name) \
836 static ssize_t epf_ntb_##_name##_show(struct config_item *item, \
837 char *page) \
838 { \
839 struct config_group *group = to_config_group(item); \
840 struct epf_ntb *ntb = to_epf_ntb(group); \
841 struct device *dev = &ntb->epf->dev; \
842 int win_no; \
843 \
844 if (sscanf(#_name, "mw%d", &win_no) != 1) \
845 return -EINVAL; \
846 \
847 if (win_no <= 0 || win_no > ntb->num_mws) { \
848 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
849 return -EINVAL; \
850 } \
851 \
852 return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]); \
853 }
854
855 #define EPF_NTB_MW_W(_name) \
856 static ssize_t epf_ntb_##_name##_store(struct config_item *item, \
857 const char *page, size_t len) \
858 { \
859 struct config_group *group = to_config_group(item); \
860 struct epf_ntb *ntb = to_epf_ntb(group); \
861 struct device *dev = &ntb->epf->dev; \
862 int win_no; \
863 u64 val; \
864 int ret; \
865 \
866 ret = kstrtou64(page, 0, &val); \
867 if (ret) \
868 return ret; \
869 \
870 if (sscanf(#_name, "mw%d", &win_no) != 1) \
871 return -EINVAL; \
872 \
873 if (win_no <= 0 || win_no > ntb->num_mws) { \
874 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
875 return -EINVAL; \
876 } \
877 \
878 ntb->mws_size[win_no - 1] = val; \
879 \
880 return len; \
881 }
882
epf_ntb_num_mws_store(struct config_item * item,const char * page,size_t len)883 static ssize_t epf_ntb_num_mws_store(struct config_item *item,
884 const char *page, size_t len)
885 {
886 struct config_group *group = to_config_group(item);
887 struct epf_ntb *ntb = to_epf_ntb(group);
888 u32 val;
889 int ret;
890
891 ret = kstrtou32(page, 0, &val);
892 if (ret)
893 return ret;
894
895 if (val > MAX_MW)
896 return -EINVAL;
897
898 ntb->num_mws = val;
899
900 return len;
901 }
902
903 EPF_NTB_R(spad_count)
904 EPF_NTB_W(spad_count)
905 EPF_NTB_R(db_count)
906 EPF_NTB_W(db_count)
907 EPF_NTB_R(num_mws)
908 EPF_NTB_R(vbus_number)
909 EPF_NTB_W(vbus_number)
910 EPF_NTB_R(vntb_pid)
911 EPF_NTB_W(vntb_pid)
912 EPF_NTB_R(vntb_vid)
913 EPF_NTB_W(vntb_vid)
914 EPF_NTB_MW_R(mw1)
915 EPF_NTB_MW_W(mw1)
916 EPF_NTB_MW_R(mw2)
917 EPF_NTB_MW_W(mw2)
918 EPF_NTB_MW_R(mw3)
919 EPF_NTB_MW_W(mw3)
920 EPF_NTB_MW_R(mw4)
921 EPF_NTB_MW_W(mw4)
922
923 CONFIGFS_ATTR(epf_ntb_, spad_count);
924 CONFIGFS_ATTR(epf_ntb_, db_count);
925 CONFIGFS_ATTR(epf_ntb_, num_mws);
926 CONFIGFS_ATTR(epf_ntb_, mw1);
927 CONFIGFS_ATTR(epf_ntb_, mw2);
928 CONFIGFS_ATTR(epf_ntb_, mw3);
929 CONFIGFS_ATTR(epf_ntb_, mw4);
930 CONFIGFS_ATTR(epf_ntb_, vbus_number);
931 CONFIGFS_ATTR(epf_ntb_, vntb_pid);
932 CONFIGFS_ATTR(epf_ntb_, vntb_vid);
933
934 static struct configfs_attribute *epf_ntb_attrs[] = {
935 &epf_ntb_attr_spad_count,
936 &epf_ntb_attr_db_count,
937 &epf_ntb_attr_num_mws,
938 &epf_ntb_attr_mw1,
939 &epf_ntb_attr_mw2,
940 &epf_ntb_attr_mw3,
941 &epf_ntb_attr_mw4,
942 &epf_ntb_attr_vbus_number,
943 &epf_ntb_attr_vntb_pid,
944 &epf_ntb_attr_vntb_vid,
945 NULL,
946 };
947
948 static const struct config_item_type ntb_group_type = {
949 .ct_attrs = epf_ntb_attrs,
950 .ct_owner = THIS_MODULE,
951 };
952
953 /**
954 * epf_ntb_add_cfs() - Add configfs directory specific to NTB
955 * @epf: NTB endpoint function device
956 * @group: A pointer to the config_group structure referencing a group of
957 * config_items of a specific type that belong to a specific sub-system.
958 *
959 * Add configfs directory specific to NTB. This directory will hold
960 * NTB specific properties like db_count, spad_count, num_mws etc.,
961 *
962 * Returns: Pointer to config_group
963 */
epf_ntb_add_cfs(struct pci_epf * epf,struct config_group * group)964 static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf,
965 struct config_group *group)
966 {
967 struct epf_ntb *ntb = epf_get_drvdata(epf);
968 struct config_group *ntb_group = &ntb->group;
969 struct device *dev = &epf->dev;
970
971 config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type);
972
973 return ntb_group;
974 }
975
976 /*==== virtual PCI bus driver, which only load virtual NTB PCI driver ====*/
977
978 static u32 pci_space[] = {
979 0xffffffff, /* Device ID, Vendor ID */
980 0, /* Status, Command */
981 0xffffffff, /* Base Class, Subclass, Prog Intf, Revision ID */
982 0x40, /* BIST, Header Type, Latency Timer, Cache Line Size */
983 0, /* BAR 0 */
984 0, /* BAR 1 */
985 0, /* BAR 2 */
986 0, /* BAR 3 */
987 0, /* BAR 4 */
988 0, /* BAR 5 */
989 0, /* Cardbus CIS Pointer */
990 0, /* Subsystem ID, Subsystem Vendor ID */
991 0, /* ROM Base Address */
992 0, /* Reserved, Capabilities Pointer */
993 0, /* Reserved */
994 0, /* Max_Lat, Min_Gnt, Interrupt Pin, Interrupt Line */
995 };
996
pci_read(struct pci_bus * bus,unsigned int devfn,int where,int size,u32 * val)997 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val)
998 {
999 if (devfn == 0) {
1000 memcpy(val, ((u8 *)pci_space) + where, size);
1001 return PCIBIOS_SUCCESSFUL;
1002 }
1003 return PCIBIOS_DEVICE_NOT_FOUND;
1004 }
1005
pci_write(struct pci_bus * bus,unsigned int devfn,int where,int size,u32 val)1006 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val)
1007 {
1008 return 0;
1009 }
1010
1011 static struct pci_ops vpci_ops = {
1012 .read = pci_read,
1013 .write = pci_write,
1014 };
1015
vpci_scan_bus(void * sysdata)1016 static int vpci_scan_bus(void *sysdata)
1017 {
1018 struct pci_bus *vpci_bus;
1019 struct epf_ntb *ndev = sysdata;
1020
1021 vpci_bus = pci_scan_bus(ndev->vbus_number, &vpci_ops, sysdata);
1022 if (!vpci_bus) {
1023 pr_err("create pci bus failed\n");
1024 return -EINVAL;
1025 }
1026
1027 pci_bus_add_devices(vpci_bus);
1028
1029 return 0;
1030 }
1031
1032 /*==================== Virtual PCIe NTB driver ==========================*/
1033
vntb_epf_mw_count(struct ntb_dev * ntb,int pidx)1034 static int vntb_epf_mw_count(struct ntb_dev *ntb, int pidx)
1035 {
1036 struct epf_ntb *ndev = ntb_ndev(ntb);
1037
1038 return ndev->num_mws;
1039 }
1040
vntb_epf_spad_count(struct ntb_dev * ntb)1041 static int vntb_epf_spad_count(struct ntb_dev *ntb)
1042 {
1043 return ntb_ndev(ntb)->spad_count;
1044 }
1045
vntb_epf_peer_mw_count(struct ntb_dev * ntb)1046 static int vntb_epf_peer_mw_count(struct ntb_dev *ntb)
1047 {
1048 return ntb_ndev(ntb)->num_mws;
1049 }
1050
vntb_epf_db_valid_mask(struct ntb_dev * ntb)1051 static u64 vntb_epf_db_valid_mask(struct ntb_dev *ntb)
1052 {
1053 return BIT_ULL(ntb_ndev(ntb)->db_count) - 1;
1054 }
1055
vntb_epf_db_set_mask(struct ntb_dev * ntb,u64 db_bits)1056 static int vntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1057 {
1058 return 0;
1059 }
1060
vntb_epf_mw_set_trans(struct ntb_dev * ndev,int pidx,int idx,dma_addr_t addr,resource_size_t size)1061 static int vntb_epf_mw_set_trans(struct ntb_dev *ndev, int pidx, int idx,
1062 dma_addr_t addr, resource_size_t size)
1063 {
1064 struct epf_ntb *ntb = ntb_ndev(ndev);
1065 struct pci_epf_bar *epf_bar;
1066 enum pci_barno barno;
1067 int ret;
1068 struct device *dev;
1069
1070 dev = &ntb->ntb.dev;
1071 barno = ntb->epf_ntb_bar[BAR_MW0 + idx];
1072 epf_bar = &ntb->epf->bar[barno];
1073 epf_bar->phys_addr = addr;
1074 epf_bar->barno = barno;
1075 epf_bar->size = size;
1076
1077 ret = pci_epc_set_bar(ntb->epf->epc, 0, 0, epf_bar);
1078 if (ret) {
1079 dev_err(dev, "failure set mw trans\n");
1080 return ret;
1081 }
1082 return 0;
1083 }
1084
vntb_epf_mw_clear_trans(struct ntb_dev * ntb,int pidx,int idx)1085 static int vntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx)
1086 {
1087 return 0;
1088 }
1089
vntb_epf_peer_mw_get_addr(struct ntb_dev * ndev,int idx,phys_addr_t * base,resource_size_t * size)1090 static int vntb_epf_peer_mw_get_addr(struct ntb_dev *ndev, int idx,
1091 phys_addr_t *base, resource_size_t *size)
1092 {
1093
1094 struct epf_ntb *ntb = ntb_ndev(ndev);
1095
1096 if (base)
1097 *base = ntb->vpci_mw_phy[idx];
1098
1099 if (size)
1100 *size = ntb->mws_size[idx];
1101
1102 return 0;
1103 }
1104
vntb_epf_link_enable(struct ntb_dev * ntb,enum ntb_speed max_speed,enum ntb_width max_width)1105 static int vntb_epf_link_enable(struct ntb_dev *ntb,
1106 enum ntb_speed max_speed,
1107 enum ntb_width max_width)
1108 {
1109 return 0;
1110 }
1111
vntb_epf_spad_read(struct ntb_dev * ndev,int idx)1112 static u32 vntb_epf_spad_read(struct ntb_dev *ndev, int idx)
1113 {
1114 struct epf_ntb *ntb = ntb_ndev(ndev);
1115 int off = ntb->reg->spad_offset, ct = ntb->reg->spad_count * sizeof(u32);
1116 u32 val;
1117 void __iomem *base = (void __iomem *)ntb->reg;
1118
1119 val = readl(base + off + ct + idx * sizeof(u32));
1120 return val;
1121 }
1122
vntb_epf_spad_write(struct ntb_dev * ndev,int idx,u32 val)1123 static int vntb_epf_spad_write(struct ntb_dev *ndev, int idx, u32 val)
1124 {
1125 struct epf_ntb *ntb = ntb_ndev(ndev);
1126 struct epf_ntb_ctrl *ctrl = ntb->reg;
1127 int off = ctrl->spad_offset, ct = ctrl->spad_count * sizeof(u32);
1128 void __iomem *base = (void __iomem *)ntb->reg;
1129
1130 writel(val, base + off + ct + idx * sizeof(u32));
1131 return 0;
1132 }
1133
vntb_epf_peer_spad_read(struct ntb_dev * ndev,int pidx,int idx)1134 static u32 vntb_epf_peer_spad_read(struct ntb_dev *ndev, int pidx, int idx)
1135 {
1136 struct epf_ntb *ntb = ntb_ndev(ndev);
1137 struct epf_ntb_ctrl *ctrl = ntb->reg;
1138 int off = ctrl->spad_offset;
1139 void __iomem *base = (void __iomem *)ntb->reg;
1140 u32 val;
1141
1142 val = readl(base + off + idx * sizeof(u32));
1143 return val;
1144 }
1145
vntb_epf_peer_spad_write(struct ntb_dev * ndev,int pidx,int idx,u32 val)1146 static int vntb_epf_peer_spad_write(struct ntb_dev *ndev, int pidx, int idx, u32 val)
1147 {
1148 struct epf_ntb *ntb = ntb_ndev(ndev);
1149 struct epf_ntb_ctrl *ctrl = ntb->reg;
1150 int off = ctrl->spad_offset;
1151 void __iomem *base = (void __iomem *)ntb->reg;
1152
1153 writel(val, base + off + idx * sizeof(u32));
1154 return 0;
1155 }
1156
vntb_epf_peer_db_set(struct ntb_dev * ndev,u64 db_bits)1157 static int vntb_epf_peer_db_set(struct ntb_dev *ndev, u64 db_bits)
1158 {
1159 u32 interrupt_num = ffs(db_bits) + 1;
1160 struct epf_ntb *ntb = ntb_ndev(ndev);
1161 u8 func_no, vfunc_no;
1162 int ret;
1163
1164 func_no = ntb->epf->func_no;
1165 vfunc_no = ntb->epf->vfunc_no;
1166
1167 ret = pci_epc_raise_irq(ntb->epf->epc, func_no, vfunc_no,
1168 PCI_IRQ_MSI, interrupt_num + 1);
1169 if (ret)
1170 dev_err(&ntb->ntb.dev, "Failed to raise IRQ\n");
1171
1172 return ret;
1173 }
1174
vntb_epf_db_read(struct ntb_dev * ndev)1175 static u64 vntb_epf_db_read(struct ntb_dev *ndev)
1176 {
1177 struct epf_ntb *ntb = ntb_ndev(ndev);
1178
1179 return ntb->db;
1180 }
1181
vntb_epf_mw_get_align(struct ntb_dev * ndev,int pidx,int idx,resource_size_t * addr_align,resource_size_t * size_align,resource_size_t * size_max)1182 static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx,
1183 resource_size_t *addr_align,
1184 resource_size_t *size_align,
1185 resource_size_t *size_max)
1186 {
1187 struct epf_ntb *ntb = ntb_ndev(ndev);
1188
1189 if (addr_align)
1190 *addr_align = SZ_4K;
1191
1192 if (size_align)
1193 *size_align = 1;
1194
1195 if (size_max)
1196 *size_max = ntb->mws_size[idx];
1197
1198 return 0;
1199 }
1200
vntb_epf_link_is_up(struct ntb_dev * ndev,enum ntb_speed * speed,enum ntb_width * width)1201 static u64 vntb_epf_link_is_up(struct ntb_dev *ndev,
1202 enum ntb_speed *speed,
1203 enum ntb_width *width)
1204 {
1205 struct epf_ntb *ntb = ntb_ndev(ndev);
1206
1207 return ntb->reg->link_status;
1208 }
1209
vntb_epf_db_clear_mask(struct ntb_dev * ndev,u64 db_bits)1210 static int vntb_epf_db_clear_mask(struct ntb_dev *ndev, u64 db_bits)
1211 {
1212 return 0;
1213 }
1214
vntb_epf_db_clear(struct ntb_dev * ndev,u64 db_bits)1215 static int vntb_epf_db_clear(struct ntb_dev *ndev, u64 db_bits)
1216 {
1217 struct epf_ntb *ntb = ntb_ndev(ndev);
1218
1219 ntb->db &= ~db_bits;
1220 return 0;
1221 }
1222
vntb_epf_link_disable(struct ntb_dev * ntb)1223 static int vntb_epf_link_disable(struct ntb_dev *ntb)
1224 {
1225 return 0;
1226 }
1227
1228 static const struct ntb_dev_ops vntb_epf_ops = {
1229 .mw_count = vntb_epf_mw_count,
1230 .spad_count = vntb_epf_spad_count,
1231 .peer_mw_count = vntb_epf_peer_mw_count,
1232 .db_valid_mask = vntb_epf_db_valid_mask,
1233 .db_set_mask = vntb_epf_db_set_mask,
1234 .mw_set_trans = vntb_epf_mw_set_trans,
1235 .mw_clear_trans = vntb_epf_mw_clear_trans,
1236 .peer_mw_get_addr = vntb_epf_peer_mw_get_addr,
1237 .link_enable = vntb_epf_link_enable,
1238 .spad_read = vntb_epf_spad_read,
1239 .spad_write = vntb_epf_spad_write,
1240 .peer_spad_read = vntb_epf_peer_spad_read,
1241 .peer_spad_write = vntb_epf_peer_spad_write,
1242 .peer_db_set = vntb_epf_peer_db_set,
1243 .db_read = vntb_epf_db_read,
1244 .mw_get_align = vntb_epf_mw_get_align,
1245 .link_is_up = vntb_epf_link_is_up,
1246 .db_clear_mask = vntb_epf_db_clear_mask,
1247 .db_clear = vntb_epf_db_clear,
1248 .link_disable = vntb_epf_link_disable,
1249 };
1250
pci_vntb_probe(struct pci_dev * pdev,const struct pci_device_id * id)1251 static int pci_vntb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1252 {
1253 int ret;
1254 struct epf_ntb *ndev = (struct epf_ntb *)pdev->sysdata;
1255 struct device *dev = &pdev->dev;
1256
1257 ndev->ntb.pdev = pdev;
1258 ndev->ntb.topo = NTB_TOPO_NONE;
1259 ndev->ntb.ops = &vntb_epf_ops;
1260
1261 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
1262 if (ret) {
1263 dev_err(dev, "Cannot set DMA mask\n");
1264 return ret;
1265 }
1266
1267 ret = ntb_register_device(&ndev->ntb);
1268 if (ret) {
1269 dev_err(dev, "Failed to register NTB device\n");
1270 return ret;
1271 }
1272
1273 dev_dbg(dev, "PCI Virtual NTB driver loaded\n");
1274 return 0;
1275 }
1276
1277 static struct pci_device_id pci_vntb_table[] = {
1278 {
1279 PCI_DEVICE(0xffff, 0xffff),
1280 },
1281 {},
1282 };
1283
1284 static struct pci_driver vntb_pci_driver = {
1285 .name = "pci-vntb",
1286 .id_table = pci_vntb_table,
1287 .probe = pci_vntb_probe,
1288 };
1289
1290 /* ============ PCIe EPF Driver Bind ====================*/
1291
1292 /**
1293 * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality
1294 * @epf: NTB endpoint function device
1295 *
1296 * Initialize both the endpoint controllers associated with NTB function device.
1297 * Invoked when a primary interface or secondary interface is bound to EPC
1298 * device. This function will succeed only when EPC is bound to both the
1299 * interfaces.
1300 *
1301 * Returns: Zero for success, or an error code in case of failure
1302 */
epf_ntb_bind(struct pci_epf * epf)1303 static int epf_ntb_bind(struct pci_epf *epf)
1304 {
1305 struct epf_ntb *ntb = epf_get_drvdata(epf);
1306 struct device *dev = &epf->dev;
1307 int ret;
1308
1309 if (!epf->epc) {
1310 dev_dbg(dev, "PRIMARY EPC interface not yet bound\n");
1311 return 0;
1312 }
1313
1314 ret = epf_ntb_init_epc_bar(ntb);
1315 if (ret) {
1316 dev_err(dev, "Failed to create NTB EPC\n");
1317 goto err_bar_init;
1318 }
1319
1320 ret = epf_ntb_config_spad_bar_alloc(ntb);
1321 if (ret) {
1322 dev_err(dev, "Failed to allocate BAR memory\n");
1323 goto err_bar_alloc;
1324 }
1325
1326 ret = epf_ntb_epc_init(ntb);
1327 if (ret) {
1328 dev_err(dev, "Failed to initialize EPC\n");
1329 goto err_bar_alloc;
1330 }
1331
1332 epf_set_drvdata(epf, ntb);
1333
1334 pci_space[0] = (ntb->vntb_pid << 16) | ntb->vntb_vid;
1335 pci_vntb_table[0].vendor = ntb->vntb_vid;
1336 pci_vntb_table[0].device = ntb->vntb_pid;
1337
1338 ret = pci_register_driver(&vntb_pci_driver);
1339 if (ret) {
1340 dev_err(dev, "failure register vntb pci driver\n");
1341 goto err_epc_cleanup;
1342 }
1343
1344 ret = vpci_scan_bus(ntb);
1345 if (ret)
1346 goto err_unregister;
1347
1348 return 0;
1349
1350 err_unregister:
1351 pci_unregister_driver(&vntb_pci_driver);
1352 err_epc_cleanup:
1353 epf_ntb_epc_cleanup(ntb);
1354 err_bar_alloc:
1355 epf_ntb_config_spad_bar_free(ntb);
1356
1357 err_bar_init:
1358 epf_ntb_epc_destroy(ntb);
1359
1360 return ret;
1361 }
1362
1363 /**
1364 * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind()
1365 * @epf: NTB endpoint function device
1366 *
1367 * Cleanup the initialization from epf_ntb_bind()
1368 */
epf_ntb_unbind(struct pci_epf * epf)1369 static void epf_ntb_unbind(struct pci_epf *epf)
1370 {
1371 struct epf_ntb *ntb = epf_get_drvdata(epf);
1372
1373 epf_ntb_epc_cleanup(ntb);
1374 epf_ntb_config_spad_bar_free(ntb);
1375 epf_ntb_epc_destroy(ntb);
1376
1377 pci_unregister_driver(&vntb_pci_driver);
1378 }
1379
1380 // EPF driver probe
1381 static const struct pci_epf_ops epf_ntb_ops = {
1382 .bind = epf_ntb_bind,
1383 .unbind = epf_ntb_unbind,
1384 .add_cfs = epf_ntb_add_cfs,
1385 };
1386
1387 /**
1388 * epf_ntb_probe() - Probe NTB function driver
1389 * @epf: NTB endpoint function device
1390 * @id: NTB endpoint function device ID
1391 *
1392 * Probe NTB function driver when endpoint function bus detects a NTB
1393 * endpoint function.
1394 *
1395 * Returns: Zero for success, or an error code in case of failure
1396 */
epf_ntb_probe(struct pci_epf * epf,const struct pci_epf_device_id * id)1397 static int epf_ntb_probe(struct pci_epf *epf,
1398 const struct pci_epf_device_id *id)
1399 {
1400 struct epf_ntb *ntb;
1401 struct device *dev;
1402
1403 dev = &epf->dev;
1404
1405 ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL);
1406 if (!ntb)
1407 return -ENOMEM;
1408
1409 epf->header = &epf_ntb_header;
1410 ntb->epf = epf;
1411 ntb->vbus_number = 0xff;
1412 epf_set_drvdata(epf, ntb);
1413
1414 dev_info(dev, "pci-ep epf driver loaded\n");
1415 return 0;
1416 }
1417
1418 static const struct pci_epf_device_id epf_ntb_ids[] = {
1419 {
1420 .name = "pci_epf_vntb",
1421 },
1422 {},
1423 };
1424
1425 static struct pci_epf_driver epf_ntb_driver = {
1426 .driver.name = "pci_epf_vntb",
1427 .probe = epf_ntb_probe,
1428 .id_table = epf_ntb_ids,
1429 .ops = &epf_ntb_ops,
1430 .owner = THIS_MODULE,
1431 };
1432
epf_ntb_init(void)1433 static int __init epf_ntb_init(void)
1434 {
1435 int ret;
1436
1437 kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM |
1438 WQ_HIGHPRI, 0);
1439 ret = pci_epf_register_driver(&epf_ntb_driver);
1440 if (ret) {
1441 destroy_workqueue(kpcintb_workqueue);
1442 pr_err("Failed to register pci epf ntb driver --> %d\n", ret);
1443 return ret;
1444 }
1445
1446 return 0;
1447 }
1448 module_init(epf_ntb_init);
1449
epf_ntb_exit(void)1450 static void __exit epf_ntb_exit(void)
1451 {
1452 pci_epf_unregister_driver(&epf_ntb_driver);
1453 destroy_workqueue(kpcintb_workqueue);
1454 }
1455 module_exit(epf_ntb_exit);
1456
1457 MODULE_DESCRIPTION("PCI EPF NTB DRIVER");
1458 MODULE_AUTHOR("Frank Li <Frank.li@nxp.com>");
1459 MODULE_LICENSE("GPL v2");
1460