1 /*- 2 * Copyright (C) 2013 Intel Corporation 3 * Copyright (C) 2015 EMC Corporation 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include <sys/param.h> 32 #include <sys/kernel.h> 33 #include <sys/systm.h> 34 #include <sys/bitset.h> 35 #include <sys/bus.h> 36 #include <sys/ktr.h> 37 #include <sys/limits.h> 38 #include <sys/lock.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 #include <sys/mutex.h> 42 #include <sys/queue.h> 43 #include <sys/socket.h> 44 #include <sys/sockio.h> 45 #include <sys/sysctl.h> 46 #include <sys/taskqueue.h> 47 48 #include <net/if.h> 49 #include <net/if_media.h> 50 #include <net/if_types.h> 51 #include <net/if_var.h> 52 #include <net/bpf.h> 53 #include <net/ethernet.h> 54 55 #include <vm/vm.h> 56 #include <vm/pmap.h> 57 58 #include <machine/bus.h> 59 #include <machine/cpufunc.h> 60 #include <machine/pmap.h> 61 62 #include "../ntb_hw/ntb_hw.h" 63 64 /* 65 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 66 * allows you to connect two systems using a PCI-e link. 67 * 68 * This module contains a protocol for sending and receiving messages, and 69 * exposes that protocol through a simulated ethernet device called ntb. 70 * 71 * NOTE: Much of the code in this module is shared with Linux. Any patches may 72 * be picked up and redistributed in Linux with a dual GPL/BSD license. 73 */ 74 75 #define QP_SETSIZE 64 76 BITSET_DEFINE(_qpset, QP_SETSIZE); 77 #define test_bit(pos, addr) BIT_ISSET(QP_SETSIZE, (pos), (addr)) 78 #define set_bit(pos, addr) BIT_SET(QP_SETSIZE, (pos), (addr)) 79 #define clear_bit(pos, addr) BIT_CLR(QP_SETSIZE, (pos), (addr)) 80 #define ffs_bit(addr) BIT_FFS(QP_SETSIZE, (addr)) 81 82 #define KTR_NTB KTR_SPARE3 83 84 #define NTB_TRANSPORT_VERSION 4 85 #define NTB_RX_MAX_PKTS 64 86 #define NTB_RXQ_SIZE 300 87 88 enum ntb_link_event { 89 NTB_LINK_DOWN = 0, 90 NTB_LINK_UP, 91 }; 92 93 static SYSCTL_NODE(_hw, OID_AUTO, if_ntb, CTLFLAG_RW, 0, "if_ntb"); 94 95 static unsigned g_if_ntb_debug_level; 96 SYSCTL_UINT(_hw_if_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 97 &g_if_ntb_debug_level, 0, "if_ntb log level -- higher is more verbose"); 98 #define ntb_printf(lvl, ...) do { \ 99 if ((lvl) <= g_if_ntb_debug_level) { \ 100 if_printf(nt->ifp, __VA_ARGS__); \ 101 } \ 102 } while (0) 103 104 static unsigned transport_mtu = 0x10000 + ETHER_HDR_LEN + ETHER_CRC_LEN; 105 106 static uint64_t max_mw_size; 107 SYSCTL_UQUAD(_hw_if_ntb, OID_AUTO, max_mw_size, CTLFLAG_RDTUN, &max_mw_size, 0, 108 "If enabled (non-zero), limit the size of large memory windows. " 109 "Both sides of the NTB MUST set the same value here."); 110 111 static unsigned max_num_clients; 112 SYSCTL_UINT(_hw_if_ntb, OID_AUTO, max_num_clients, CTLFLAG_RDTUN, 113 &max_num_clients, 0, "Maximum number of NTB transport clients. " 114 "0 (default) - use all available NTB memory windows; " 115 "positive integer N - Limit to N memory windows."); 116 117 STAILQ_HEAD(ntb_queue_list, ntb_queue_entry); 118 119 typedef unsigned ntb_q_idx_t; 120 121 struct ntb_queue_entry { 122 /* ntb_queue list reference */ 123 STAILQ_ENTRY(ntb_queue_entry) entry; 124 125 /* info on data to be transferred */ 126 void *cb_data; 127 void *buf; 128 unsigned len; 129 unsigned flags; 130 131 struct ntb_transport_qp *qp; 132 struct ntb_payload_header *x_hdr; 133 ntb_q_idx_t index; 134 }; 135 136 struct ntb_rx_info { 137 ntb_q_idx_t entry; 138 }; 139 140 struct ntb_transport_qp { 141 struct ntb_transport_ctx *transport; 142 struct ntb_softc *ntb; 143 144 void *cb_data; 145 146 bool client_ready; 147 volatile bool link_is_up; 148 uint8_t qp_num; /* Only 64 QPs are allowed. 0-63 */ 149 150 struct ntb_rx_info *rx_info; 151 struct ntb_rx_info *remote_rx_info; 152 153 void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, 154 void *data, int len); 155 struct ntb_queue_list tx_free_q; 156 struct mtx ntb_tx_free_q_lock; 157 caddr_t tx_mw; 158 bus_addr_t tx_mw_phys; 159 ntb_q_idx_t tx_index; 160 ntb_q_idx_t tx_max_entry; 161 uint64_t tx_max_frame; 162 163 void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, 164 void *data, int len); 165 struct ntb_queue_list rx_post_q; 166 struct ntb_queue_list rx_pend_q; 167 struct ntb_queue_list rx_free_q; 168 /* ntb_rx_q_lock: synchronize access to rx_XXXX_q */ 169 struct mtx ntb_rx_q_lock; 170 struct task rx_completion_task; 171 struct task rxc_db_work; 172 caddr_t rx_buff; 173 ntb_q_idx_t rx_index; 174 ntb_q_idx_t rx_max_entry; 175 uint64_t rx_max_frame; 176 177 void (*event_handler)(void *data, enum ntb_link_event status); 178 struct callout link_work; 179 struct callout queue_full; 180 struct callout rx_full; 181 182 uint64_t last_rx_no_buf; 183 184 /* Stats */ 185 uint64_t rx_bytes; 186 uint64_t rx_pkts; 187 uint64_t rx_ring_empty; 188 uint64_t rx_err_no_buf; 189 uint64_t rx_err_oflow; 190 uint64_t rx_err_ver; 191 uint64_t tx_bytes; 192 uint64_t tx_pkts; 193 uint64_t tx_ring_full; 194 uint64_t tx_err_no_buf; 195 }; 196 197 struct ntb_queue_handlers { 198 void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, 199 void *data, int len); 200 void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, 201 void *data, int len); 202 void (*event_handler)(void *data, enum ntb_link_event status); 203 }; 204 205 struct ntb_transport_mw { 206 vm_paddr_t phys_addr; 207 size_t phys_size; 208 size_t xlat_align; 209 size_t xlat_align_size; 210 /* Tx buff is off vbase / phys_addr */ 211 caddr_t vbase; 212 size_t xlat_size; 213 size_t buff_size; 214 /* Rx buff is off virt_addr / dma_addr */ 215 caddr_t virt_addr; 216 bus_addr_t dma_addr; 217 }; 218 219 struct ntb_transport_ctx { 220 struct ntb_softc *ntb; 221 struct ifnet *ifp; 222 struct ntb_transport_mw mw_vec[NTB_MAX_NUM_MW]; 223 struct ntb_transport_qp *qp_vec; 224 struct _qpset qp_bitmap; 225 struct _qpset qp_bitmap_free; 226 unsigned mw_count; 227 unsigned qp_count; 228 volatile bool link_is_up; 229 struct callout link_work; 230 struct task link_cleanup; 231 uint64_t bufsize; 232 u_char eaddr[ETHER_ADDR_LEN]; 233 struct mtx tx_lock; 234 struct mtx rx_lock; 235 236 /* The hardcoded single queuepair in ntb_setup_interface() */ 237 struct ntb_transport_qp *qp; 238 }; 239 240 static struct ntb_transport_ctx net_softc; 241 242 enum { 243 IF_NTB_DESC_DONE_FLAG = 1 << 0, 244 IF_NTB_LINK_DOWN_FLAG = 1 << 1, 245 }; 246 247 struct ntb_payload_header { 248 uint64_t ver; 249 uint64_t len; 250 uint64_t flags; 251 }; 252 253 enum { 254 /* 255 * The order of this enum is part of the if_ntb remote protocol. Do 256 * not reorder without bumping protocol version (and it's probably best 257 * to keep the protocol in lock-step with the Linux NTB driver. 258 */ 259 IF_NTB_VERSION = 0, 260 IF_NTB_QP_LINKS, 261 IF_NTB_NUM_QPS, 262 IF_NTB_NUM_MWS, 263 /* 264 * N.B.: transport_link_work assumes MW1 enums = MW0 + 2. 265 */ 266 IF_NTB_MW0_SZ_HIGH, 267 IF_NTB_MW0_SZ_LOW, 268 IF_NTB_MW1_SZ_HIGH, 269 IF_NTB_MW1_SZ_LOW, 270 IF_NTB_MAX_SPAD, 271 }; 272 273 #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) 274 #define NTB_QP_DEF_NUM_ENTRIES 100 275 #define NTB_LINK_DOWN_TIMEOUT 10 276 277 static int ntb_handle_module_events(struct module *m, int what, void *arg); 278 static int ntb_setup_interface(void); 279 static int ntb_teardown_interface(void); 280 static void ntb_net_init(void *arg); 281 static int ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data); 282 static void ntb_start(struct ifnet *ifp); 283 static void ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, 284 void *data, int len); 285 static void ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, 286 void *data, int len); 287 static void ntb_net_event_handler(void *data, enum ntb_link_event status); 288 static int ntb_transport_probe(struct ntb_softc *ntb); 289 static void ntb_transport_free(struct ntb_transport_ctx *); 290 static void ntb_transport_init_queue(struct ntb_transport_ctx *nt, 291 unsigned int qp_num); 292 static void ntb_transport_free_queue(struct ntb_transport_qp *qp); 293 static struct ntb_transport_qp *ntb_transport_create_queue(void *data, 294 struct ntb_softc *pdev, const struct ntb_queue_handlers *handlers); 295 static void ntb_transport_link_up(struct ntb_transport_qp *qp); 296 static int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, 297 void *data, unsigned int len); 298 static int ntb_process_tx(struct ntb_transport_qp *qp, 299 struct ntb_queue_entry *entry); 300 static void ntb_memcpy_tx(struct ntb_transport_qp *qp, 301 struct ntb_queue_entry *entry, void *offset); 302 static void ntb_qp_full(void *arg); 303 static void ntb_transport_rxc_db(void *arg, int pending); 304 static int ntb_process_rxc(struct ntb_transport_qp *qp); 305 static void ntb_memcpy_rx(struct ntb_transport_qp *qp, 306 struct ntb_queue_entry *entry, void *offset); 307 static inline void ntb_rx_copy_callback(struct ntb_transport_qp *qp, 308 void *data); 309 static void ntb_complete_rxc(void *arg, int pending); 310 static void ntb_transport_doorbell_callback(void *data, uint32_t vector); 311 static void ntb_transport_event_callback(void *data); 312 static void ntb_transport_link_work(void *arg); 313 static int ntb_set_mw(struct ntb_transport_ctx *, int num_mw, size_t size); 314 static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw); 315 static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, 316 unsigned int qp_num); 317 static void ntb_qp_link_work(void *arg); 318 static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt); 319 static void ntb_transport_link_cleanup_work(void *, int); 320 static void ntb_qp_link_down(struct ntb_transport_qp *qp); 321 static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp); 322 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp); 323 static void ntb_transport_link_down(struct ntb_transport_qp *qp); 324 static void ntb_send_link_down(struct ntb_transport_qp *qp); 325 static void ntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry, 326 struct ntb_queue_list *list); 327 static struct ntb_queue_entry *ntb_list_rm(struct mtx *lock, 328 struct ntb_queue_list *list); 329 static struct ntb_queue_entry *ntb_list_mv(struct mtx *lock, 330 struct ntb_queue_list *from, struct ntb_queue_list *to); 331 static void create_random_local_eui48(u_char *eaddr); 332 static unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp); 333 334 static const struct ntb_ctx_ops ntb_transport_ops = { 335 .link_event = ntb_transport_event_callback, 336 .db_event = ntb_transport_doorbell_callback, 337 }; 338 339 MALLOC_DEFINE(M_NTB_IF, "if_ntb", "ntb network driver"); 340 341 /* Module setup and teardown */ 342 static int 343 ntb_handle_module_events(struct module *m, int what, void *arg) 344 { 345 int err = 0; 346 347 switch (what) { 348 case MOD_LOAD: 349 err = ntb_setup_interface(); 350 break; 351 case MOD_UNLOAD: 352 err = ntb_teardown_interface(); 353 break; 354 default: 355 err = EOPNOTSUPP; 356 break; 357 } 358 return (err); 359 } 360 361 static moduledata_t if_ntb_mod = { 362 "if_ntb", 363 ntb_handle_module_events, 364 NULL 365 }; 366 367 DECLARE_MODULE(if_ntb, if_ntb_mod, SI_SUB_KLD, SI_ORDER_ANY); 368 MODULE_DEPEND(if_ntb, ntb_hw, 1, 1, 1); 369 370 static int 371 ntb_setup_interface(void) 372 { 373 struct ifnet *ifp; 374 struct ntb_queue_handlers handlers = { ntb_net_rx_handler, 375 ntb_net_tx_handler, ntb_net_event_handler }; 376 int rc; 377 378 net_softc.ntb = devclass_get_softc(devclass_find("ntb_hw"), 0); 379 if (net_softc.ntb == NULL) { 380 printf("ntb: Cannot find devclass\n"); 381 return (ENXIO); 382 } 383 384 ifp = net_softc.ifp = if_alloc(IFT_ETHER); 385 if (ifp == NULL) { 386 ntb_transport_free(&net_softc); 387 printf("ntb: Cannot allocate ifnet structure\n"); 388 return (ENOMEM); 389 } 390 if_initname(ifp, "ntb", 0); 391 392 rc = ntb_transport_probe(net_softc.ntb); 393 if (rc != 0) { 394 printf("ntb: Cannot init transport: %d\n", rc); 395 if_free(net_softc.ifp); 396 return (rc); 397 } 398 399 net_softc.qp = ntb_transport_create_queue(ifp, net_softc.ntb, 400 &handlers); 401 ifp->if_init = ntb_net_init; 402 ifp->if_softc = &net_softc; 403 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX; 404 ifp->if_ioctl = ntb_ioctl; 405 ifp->if_start = ntb_start; 406 IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN); 407 ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN; 408 IFQ_SET_READY(&ifp->if_snd); 409 create_random_local_eui48(net_softc.eaddr); 410 ether_ifattach(ifp, net_softc.eaddr); 411 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_JUMBO_MTU; 412 ifp->if_capenable = ifp->if_capabilities; 413 414 ntb_transport_link_up(net_softc.qp); 415 net_softc.bufsize = ntb_transport_max_size(net_softc.qp) + 416 sizeof(struct ether_header); 417 return (0); 418 } 419 420 static int 421 ntb_teardown_interface(void) 422 { 423 424 if (net_softc.qp != NULL) { 425 ntb_transport_link_down(net_softc.qp); 426 427 ntb_transport_free_queue(net_softc.qp); 428 ntb_transport_free(&net_softc); 429 } 430 431 if (net_softc.ifp != NULL) { 432 ether_ifdetach(net_softc.ifp); 433 if_free(net_softc.ifp); 434 net_softc.ifp = NULL; 435 } 436 437 return (0); 438 } 439 440 /* Network device interface */ 441 442 static void 443 ntb_net_init(void *arg) 444 { 445 struct ntb_transport_ctx *ntb_softc = arg; 446 struct ifnet *ifp = ntb_softc->ifp; 447 448 ifp->if_drv_flags |= IFF_DRV_RUNNING; 449 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 450 ifp->if_flags |= IFF_UP; 451 if_link_state_change(ifp, LINK_STATE_UP); 452 } 453 454 static int 455 ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 456 { 457 struct ntb_transport_ctx *nt = ifp->if_softc; 458 struct ifreq *ifr = (struct ifreq *)data; 459 int error = 0; 460 461 switch (command) { 462 case SIOCSIFMTU: 463 { 464 if (ifr->ifr_mtu > ntb_transport_max_size(nt->qp) - 465 ETHER_HDR_LEN - ETHER_CRC_LEN) { 466 error = EINVAL; 467 break; 468 } 469 470 ifp->if_mtu = ifr->ifr_mtu; 471 break; 472 } 473 default: 474 error = ether_ioctl(ifp, command, data); 475 break; 476 } 477 478 return (error); 479 } 480 481 482 static void 483 ntb_start(struct ifnet *ifp) 484 { 485 struct mbuf *m_head; 486 struct ntb_transport_ctx *nt = ifp->if_softc; 487 int rc; 488 489 mtx_lock(&nt->tx_lock); 490 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 491 CTR0(KTR_NTB, "TX: ntb_start"); 492 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 493 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 494 CTR1(KTR_NTB, "TX: start mbuf %p", m_head); 495 rc = ntb_transport_tx_enqueue(nt->qp, m_head, m_head, 496 m_length(m_head, NULL)); 497 if (rc != 0) { 498 CTR1(KTR_NTB, 499 "TX: could not tx mbuf %p. Returning to snd q", 500 m_head); 501 if (rc == EAGAIN) { 502 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 503 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 504 callout_reset(&nt->qp->queue_full, hz / 1000, 505 ntb_qp_full, ifp); 506 } 507 break; 508 } 509 510 } 511 mtx_unlock(&nt->tx_lock); 512 } 513 514 /* Network Device Callbacks */ 515 static void 516 ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, 517 int len) 518 { 519 520 m_freem(data); 521 CTR1(KTR_NTB, "TX: tx_handler freeing mbuf %p", data); 522 } 523 524 static void 525 ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, 526 int len) 527 { 528 struct mbuf *m = data; 529 struct ifnet *ifp = qp_data; 530 531 CTR0(KTR_NTB, "RX: rx handler"); 532 (*ifp->if_input)(ifp, m); 533 } 534 535 static void 536 ntb_net_event_handler(void *data, enum ntb_link_event status) 537 { 538 struct ifnet *ifp; 539 540 ifp = data; 541 (void)ifp; 542 543 /* XXX The Linux driver munges with the carrier status here. */ 544 545 switch (status) { 546 case NTB_LINK_DOWN: 547 break; 548 case NTB_LINK_UP: 549 break; 550 default: 551 panic("Bogus ntb_link_event %u\n", status); 552 } 553 } 554 555 /* Transport Init and teardown */ 556 557 static int 558 ntb_transport_probe(struct ntb_softc *ntb) 559 { 560 struct ntb_transport_ctx *nt = &net_softc; 561 struct ntb_transport_mw *mw; 562 uint64_t qp_bitmap; 563 int rc; 564 unsigned i; 565 566 nt->mw_count = ntb_mw_count(ntb); 567 for (i = 0; i < nt->mw_count; i++) { 568 mw = &nt->mw_vec[i]; 569 570 rc = ntb_mw_get_range(ntb, i, &mw->phys_addr, &mw->vbase, 571 &mw->phys_size, &mw->xlat_align, &mw->xlat_align_size); 572 if (rc != 0) 573 goto err; 574 575 mw->buff_size = 0; 576 mw->xlat_size = 0; 577 mw->virt_addr = NULL; 578 mw->dma_addr = 0; 579 } 580 581 qp_bitmap = ntb_db_valid_mask(ntb); 582 nt->qp_count = flsll(qp_bitmap); 583 KASSERT(nt->qp_count != 0, ("bogus db bitmap")); 584 nt->qp_count -= 1; 585 586 if (max_num_clients != 0 && max_num_clients < nt->qp_count) 587 nt->qp_count = max_num_clients; 588 else if (nt->mw_count < nt->qp_count) 589 nt->qp_count = nt->mw_count; 590 KASSERT(nt->qp_count <= QP_SETSIZE, ("invalid qp_count")); 591 592 mtx_init(&nt->tx_lock, "ntb transport tx", NULL, MTX_DEF); 593 mtx_init(&nt->rx_lock, "ntb transport rx", NULL, MTX_DEF); 594 595 nt->qp_vec = malloc(nt->qp_count * sizeof(*nt->qp_vec), M_NTB_IF, 596 M_WAITOK | M_ZERO); 597 598 for (i = 0; i < nt->qp_count; i++) { 599 set_bit(i, &nt->qp_bitmap); 600 set_bit(i, &nt->qp_bitmap_free); 601 ntb_transport_init_queue(nt, i); 602 } 603 604 callout_init(&nt->link_work, 0); 605 TASK_INIT(&nt->link_cleanup, 0, ntb_transport_link_cleanup_work, nt); 606 607 rc = ntb_set_ctx(ntb, nt, &ntb_transport_ops); 608 if (rc != 0) 609 goto err; 610 611 nt->link_is_up = false; 612 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 613 ntb_link_event(ntb); 614 615 callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt); 616 return (0); 617 618 err: 619 free(nt->qp_vec, M_NTB_IF); 620 nt->qp_vec = NULL; 621 return (rc); 622 } 623 624 static void 625 ntb_transport_free(struct ntb_transport_ctx *nt) 626 { 627 struct ntb_softc *ntb = nt->ntb; 628 struct _qpset qp_bitmap_alloc; 629 uint8_t i; 630 631 ntb_transport_link_cleanup(nt); 632 taskqueue_drain(taskqueue_swi, &nt->link_cleanup); 633 callout_drain(&nt->link_work); 634 635 BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &qp_bitmap_alloc); 636 BIT_NAND(QP_SETSIZE, &qp_bitmap_alloc, &nt->qp_bitmap_free); 637 638 /* Verify that all the QPs are freed */ 639 for (i = 0; i < nt->qp_count; i++) 640 if (test_bit(i, &qp_bitmap_alloc)) 641 ntb_transport_free_queue(&nt->qp_vec[i]); 642 643 ntb_link_disable(ntb); 644 ntb_clear_ctx(ntb); 645 646 for (i = 0; i < nt->mw_count; i++) 647 ntb_free_mw(nt, i); 648 649 free(nt->qp_vec, M_NTB_IF); 650 } 651 652 static void 653 ntb_transport_init_queue(struct ntb_transport_ctx *nt, unsigned int qp_num) 654 { 655 struct ntb_transport_mw *mw; 656 struct ntb_transport_qp *qp; 657 vm_paddr_t mw_base; 658 uint64_t mw_size, qp_offset; 659 size_t tx_size; 660 unsigned num_qps_mw, mw_num, mw_count; 661 662 mw_count = nt->mw_count; 663 mw_num = QP_TO_MW(nt, qp_num); 664 mw = &nt->mw_vec[mw_num]; 665 666 qp = &nt->qp_vec[qp_num]; 667 qp->qp_num = qp_num; 668 qp->transport = nt; 669 qp->ntb = nt->ntb; 670 qp->client_ready = false; 671 qp->event_handler = NULL; 672 ntb_qp_link_down_reset(qp); 673 674 if (nt->qp_count % mw_count && mw_num + 1 < nt->qp_count / mw_count) 675 num_qps_mw = nt->qp_count / mw_count + 1; 676 else 677 num_qps_mw = nt->qp_count / mw_count; 678 679 mw_base = mw->phys_addr; 680 mw_size = mw->phys_size; 681 682 tx_size = mw_size / num_qps_mw; 683 qp_offset = tx_size * (qp_num / mw_count); 684 685 qp->tx_mw = mw->vbase + qp_offset; 686 KASSERT(qp->tx_mw != NULL, ("uh oh?")); 687 688 /* XXX Assumes that a vm_paddr_t is equivalent to bus_addr_t */ 689 qp->tx_mw_phys = mw_base + qp_offset; 690 KASSERT(qp->tx_mw_phys != 0, ("uh oh?")); 691 692 tx_size -= sizeof(struct ntb_rx_info); 693 qp->rx_info = (void *)(qp->tx_mw + tx_size); 694 695 /* Due to house-keeping, there must be at least 2 buffs */ 696 qp->tx_max_frame = qmin(tx_size / 2, 697 transport_mtu + sizeof(struct ntb_payload_header)); 698 qp->tx_max_entry = tx_size / qp->tx_max_frame; 699 700 callout_init(&qp->link_work, 0); 701 callout_init(&qp->queue_full, 1); 702 callout_init(&qp->rx_full, 1); 703 704 mtx_init(&qp->ntb_rx_q_lock, "ntb rx q", NULL, MTX_SPIN); 705 mtx_init(&qp->ntb_tx_free_q_lock, "ntb tx free q", NULL, MTX_SPIN); 706 TASK_INIT(&qp->rx_completion_task, 0, ntb_complete_rxc, qp); 707 TASK_INIT(&qp->rxc_db_work, 0, ntb_transport_rxc_db, qp); 708 709 STAILQ_INIT(&qp->rx_post_q); 710 STAILQ_INIT(&qp->rx_pend_q); 711 STAILQ_INIT(&qp->rx_free_q); 712 STAILQ_INIT(&qp->tx_free_q); 713 714 callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp); 715 } 716 717 static void 718 ntb_transport_free_queue(struct ntb_transport_qp *qp) 719 { 720 struct ntb_queue_entry *entry; 721 722 if (qp == NULL) 723 return; 724 725 callout_drain(&qp->link_work); 726 727 ntb_db_set_mask(qp->ntb, 1ull << qp->qp_num); 728 taskqueue_drain(taskqueue_swi, &qp->rxc_db_work); 729 taskqueue_drain(taskqueue_swi, &qp->rx_completion_task); 730 731 qp->cb_data = NULL; 732 qp->rx_handler = NULL; 733 qp->tx_handler = NULL; 734 qp->event_handler = NULL; 735 736 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q))) 737 free(entry, M_NTB_IF); 738 739 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q))) 740 free(entry, M_NTB_IF); 741 742 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q))) 743 free(entry, M_NTB_IF); 744 745 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 746 free(entry, M_NTB_IF); 747 748 set_bit(qp->qp_num, &qp->transport->qp_bitmap_free); 749 } 750 751 /** 752 * ntb_transport_create_queue - Create a new NTB transport layer queue 753 * @rx_handler: receive callback function 754 * @tx_handler: transmit callback function 755 * @event_handler: event callback function 756 * 757 * Create a new NTB transport layer queue and provide the queue with a callback 758 * routine for both transmit and receive. The receive callback routine will be 759 * used to pass up data when the transport has received it on the queue. The 760 * transmit callback routine will be called when the transport has completed the 761 * transmission of the data on the queue and the data is ready to be freed. 762 * 763 * RETURNS: pointer to newly created ntb_queue, NULL on error. 764 */ 765 static struct ntb_transport_qp * 766 ntb_transport_create_queue(void *data, struct ntb_softc *ntb, 767 const struct ntb_queue_handlers *handlers) 768 { 769 struct ntb_queue_entry *entry; 770 struct ntb_transport_qp *qp; 771 struct ntb_transport_ctx *nt; 772 unsigned int free_queue; 773 int i; 774 775 nt = ntb_get_ctx(ntb, NULL); 776 KASSERT(nt != NULL, ("bogus")); 777 778 free_queue = ffs_bit(&nt->qp_bitmap); 779 if (free_queue == 0) 780 return (NULL); 781 782 /* decrement free_queue to make it zero based */ 783 free_queue--; 784 785 qp = &nt->qp_vec[free_queue]; 786 clear_bit(qp->qp_num, &nt->qp_bitmap_free); 787 qp->cb_data = data; 788 qp->rx_handler = handlers->rx_handler; 789 qp->tx_handler = handlers->tx_handler; 790 qp->event_handler = handlers->event_handler; 791 792 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 793 entry = malloc(sizeof(*entry), M_NTB_IF, M_WAITOK | M_ZERO); 794 entry->cb_data = nt->ifp; 795 entry->buf = NULL; 796 entry->len = transport_mtu; 797 ntb_list_add(&qp->ntb_rx_q_lock, entry, &qp->rx_pend_q); 798 } 799 800 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 801 entry = malloc(sizeof(*entry), M_NTB_IF, M_WAITOK | M_ZERO); 802 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 803 } 804 805 ntb_db_clear(ntb, 1ull << qp->qp_num); 806 ntb_db_clear_mask(ntb, 1ull << qp->qp_num); 807 return (qp); 808 } 809 810 /** 811 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue 812 * @qp: NTB transport layer queue to be enabled 813 * 814 * Notify NTB transport layer of client readiness to use queue 815 */ 816 static void 817 ntb_transport_link_up(struct ntb_transport_qp *qp) 818 { 819 struct ntb_transport_ctx *nt; 820 821 if (qp == NULL) 822 return; 823 824 qp->client_ready = true; 825 826 nt = qp->transport; 827 ntb_printf(2, "qp client ready\n"); 828 829 if (qp->transport->link_is_up) 830 callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp); 831 } 832 833 834 835 /* Transport Tx */ 836 837 /** 838 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry 839 * @qp: NTB transport layer queue the entry is to be enqueued on 840 * @cb: per buffer pointer for callback function to use 841 * @data: pointer to data buffer that will be sent 842 * @len: length of the data buffer 843 * 844 * Enqueue a new transmit buffer onto the transport queue from which a NTB 845 * payload will be transmitted. This assumes that a lock is being held to 846 * serialize access to the qp. 847 * 848 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 849 */ 850 static int 851 ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 852 unsigned int len) 853 { 854 struct ntb_queue_entry *entry; 855 int rc; 856 857 if (qp == NULL || !qp->link_is_up || len == 0) { 858 CTR0(KTR_NTB, "TX: link not up"); 859 return (EINVAL); 860 } 861 862 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 863 if (entry == NULL) { 864 CTR0(KTR_NTB, "TX: could not get entry from tx_free_q"); 865 qp->tx_err_no_buf++; 866 return (EBUSY); 867 } 868 CTR1(KTR_NTB, "TX: got entry %p from tx_free_q", entry); 869 870 entry->cb_data = cb; 871 entry->buf = data; 872 entry->len = len; 873 entry->flags = 0; 874 875 rc = ntb_process_tx(qp, entry); 876 if (rc != 0) { 877 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 878 CTR1(KTR_NTB, 879 "TX: process_tx failed. Returning entry %p to tx_free_q", 880 entry); 881 } 882 return (rc); 883 } 884 885 static int 886 ntb_process_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry) 887 { 888 void *offset; 889 890 offset = (char *)qp->tx_mw + qp->tx_max_frame * qp->tx_index; 891 CTR3(KTR_NTB, 892 "TX: process_tx: tx_pkts=%u, tx_index=%u, remote entry=%u", 893 qp->tx_pkts, qp->tx_index, qp->remote_rx_info->entry); 894 if (qp->tx_index == qp->remote_rx_info->entry) { 895 CTR0(KTR_NTB, "TX: ring full"); 896 qp->tx_ring_full++; 897 return (EAGAIN); 898 } 899 900 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { 901 if (qp->tx_handler != NULL) 902 qp->tx_handler(qp, qp->cb_data, entry->buf, 903 EIO); 904 905 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 906 CTR1(KTR_NTB, 907 "TX: frame too big. returning entry %p to tx_free_q", 908 entry); 909 return (0); 910 } 911 CTR2(KTR_NTB, "TX: copying entry %p to offset %p", entry, offset); 912 ntb_memcpy_tx(qp, entry, offset); 913 914 qp->tx_index++; 915 qp->tx_index %= qp->tx_max_entry; 916 917 qp->tx_pkts++; 918 919 return (0); 920 } 921 922 static void 923 ntb_memcpy_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry, 924 void *offset) 925 { 926 struct ntb_payload_header *hdr; 927 928 /* This piece is from Linux' ntb_async_tx() */ 929 hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame - 930 sizeof(struct ntb_payload_header)); 931 entry->x_hdr = hdr; 932 hdr->len = entry->len; /* TODO: replace with bus_space_write */ 933 hdr->ver = qp->tx_pkts; /* TODO: replace with bus_space_write */ 934 935 /* This piece is ntb_memcpy_tx() */ 936 CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset); 937 if (entry->buf != NULL) { 938 m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset); 939 940 /* 941 * Ensure that the data is fully copied before setting the 942 * flags 943 */ 944 wmb(); 945 } 946 947 /* The rest is ntb_tx_copy_callback() */ 948 /* TODO: replace with bus_space_write */ 949 hdr->flags = entry->flags | IF_NTB_DESC_DONE_FLAG; 950 951 ntb_peer_db_set(qp->ntb, 1ull << qp->qp_num); 952 953 /* 954 * The entry length can only be zero if the packet is intended to be a 955 * "link down" or similar. Since no payload is being sent in these 956 * cases, there is nothing to add to the completion queue. 957 */ 958 if (entry->len > 0) { 959 qp->tx_bytes += entry->len; 960 961 if (qp->tx_handler) 962 qp->tx_handler(qp, qp->cb_data, entry->cb_data, 963 entry->len); 964 } 965 966 CTR2(KTR_NTB, 967 "TX: entry %p sent. hdr->ver = %d, Returning to tx_free_q", entry, 968 hdr->ver); 969 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 970 } 971 972 static void 973 ntb_qp_full(void *arg) 974 { 975 976 CTR0(KTR_NTB, "TX: qp_full callout"); 977 ntb_start(arg); 978 } 979 980 /* Transport Rx */ 981 static void 982 ntb_transport_rxc_db(void *arg, int pending __unused) 983 { 984 struct ntb_transport_qp *qp = arg; 985 ntb_q_idx_t i; 986 int rc; 987 988 /* 989 * Limit the number of packets processed in a single interrupt to 990 * provide fairness to others 991 */ 992 CTR0(KTR_NTB, "RX: transport_rx"); 993 mtx_lock(&qp->transport->rx_lock); 994 for (i = 0; i < qp->rx_max_entry; i++) { 995 rc = ntb_process_rxc(qp); 996 if (rc != 0) { 997 CTR0(KTR_NTB, "RX: process_rxc failed"); 998 break; 999 } 1000 } 1001 mtx_unlock(&qp->transport->rx_lock); 1002 1003 if (i == qp->rx_max_entry) 1004 taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work); 1005 else if ((ntb_db_read(qp->ntb) & (1ull << qp->qp_num)) != 0) { 1006 /* If db is set, clear it and read it back to commit clear. */ 1007 ntb_db_clear(qp->ntb, 1ull << qp->qp_num); 1008 (void)ntb_db_read(qp->ntb); 1009 1010 /* 1011 * An interrupt may have arrived between finishing 1012 * ntb_process_rxc and clearing the doorbell bit: there might 1013 * be some more work to do. 1014 */ 1015 taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work); 1016 } 1017 } 1018 1019 static int 1020 ntb_process_rxc(struct ntb_transport_qp *qp) 1021 { 1022 struct ntb_payload_header *hdr; 1023 struct ntb_queue_entry *entry; 1024 void *offset; 1025 1026 offset = (void *) 1027 ((char *)qp->rx_buff + qp->rx_max_frame * qp->rx_index); 1028 hdr = (void *) 1029 ((char *)offset + qp->rx_max_frame - 1030 sizeof(struct ntb_payload_header)); 1031 1032 CTR1(KTR_NTB, "RX: process_rxc rx_index = %u", qp->rx_index); 1033 if ((hdr->flags & IF_NTB_DESC_DONE_FLAG) == 0) { 1034 CTR0(KTR_NTB, "RX: hdr not done"); 1035 qp->rx_ring_empty++; 1036 return (EAGAIN); 1037 } 1038 1039 if ((hdr->flags & IF_NTB_LINK_DOWN_FLAG) != 0) { 1040 CTR0(KTR_NTB, "RX: link down"); 1041 ntb_qp_link_down(qp); 1042 hdr->flags = 0; 1043 return (EAGAIN); 1044 } 1045 1046 if (hdr->ver != (uint32_t)qp->rx_pkts) { 1047 CTR2(KTR_NTB,"RX: ver != rx_pkts (%x != %lx). " 1048 "Returning entry %p to rx_pend_q", hdr->ver, qp->rx_pkts); 1049 qp->rx_err_ver++; 1050 return (EIO); 1051 } 1052 1053 entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q); 1054 if (entry == NULL) { 1055 qp->rx_err_no_buf++; 1056 CTR0(KTR_NTB, "RX: No entries in rx_pend_q"); 1057 return (EAGAIN); 1058 } 1059 callout_stop(&qp->rx_full); 1060 CTR1(KTR_NTB, "RX: rx entry %p from rx_pend_q", entry); 1061 1062 entry->x_hdr = hdr; 1063 entry->index = qp->rx_index; 1064 1065 if (hdr->len > entry->len) { 1066 CTR2(KTR_NTB, "RX: len too long. Wanted %ju got %ju", 1067 (uintmax_t)hdr->len, (uintmax_t)entry->len); 1068 qp->rx_err_oflow++; 1069 1070 entry->len = -EIO; 1071 entry->flags |= IF_NTB_DESC_DONE_FLAG; 1072 1073 taskqueue_enqueue(taskqueue_swi, &qp->rx_completion_task); 1074 } else { 1075 qp->rx_bytes += hdr->len; 1076 qp->rx_pkts++; 1077 1078 CTR1(KTR_NTB, "RX: received %ld rx_pkts", qp->rx_pkts); 1079 1080 entry->len = hdr->len; 1081 1082 ntb_memcpy_rx(qp, entry, offset); 1083 } 1084 1085 qp->rx_index++; 1086 qp->rx_index %= qp->rx_max_entry; 1087 return (0); 1088 } 1089 1090 static void 1091 ntb_memcpy_rx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry, 1092 void *offset) 1093 { 1094 struct ifnet *ifp = entry->cb_data; 1095 unsigned int len = entry->len; 1096 struct mbuf *m; 1097 1098 CTR2(KTR_NTB, "RX: copying %d bytes from offset %p", len, offset); 1099 m = m_devget(offset, len, 0, ifp, NULL); 1100 m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID; 1101 1102 entry->buf = (void *)m; 1103 1104 /* Ensure that the data is globally visible before clearing the flag */ 1105 wmb(); 1106 1107 CTR2(KTR_NTB, "RX: copied entry %p to mbuf %p.", entry, m); 1108 ntb_rx_copy_callback(qp, entry); 1109 } 1110 1111 static inline void 1112 ntb_rx_copy_callback(struct ntb_transport_qp *qp, void *data) 1113 { 1114 struct ntb_queue_entry *entry; 1115 1116 entry = data; 1117 entry->flags |= IF_NTB_DESC_DONE_FLAG; 1118 taskqueue_enqueue(taskqueue_swi, &qp->rx_completion_task); 1119 } 1120 1121 static void 1122 ntb_complete_rxc(void *arg, int pending) 1123 { 1124 struct ntb_transport_qp *qp = arg; 1125 struct ntb_queue_entry *entry; 1126 struct mbuf *m; 1127 unsigned len; 1128 1129 CTR0(KTR_NTB, "RX: rx_completion_task"); 1130 1131 mtx_lock_spin(&qp->ntb_rx_q_lock); 1132 1133 while (!STAILQ_EMPTY(&qp->rx_post_q)) { 1134 entry = STAILQ_FIRST(&qp->rx_post_q); 1135 if ((entry->flags & IF_NTB_DESC_DONE_FLAG) == 0) 1136 break; 1137 1138 entry->x_hdr->flags = 0; 1139 /* XXX bus_space_write */ 1140 qp->rx_info->entry = entry->index; 1141 1142 len = entry->len; 1143 m = entry->buf; 1144 1145 STAILQ_REMOVE_HEAD(&qp->rx_post_q, entry); 1146 STAILQ_INSERT_TAIL(&qp->rx_free_q, entry, entry); 1147 1148 mtx_unlock_spin(&qp->ntb_rx_q_lock); 1149 1150 CTR2(KTR_NTB, "RX: completing entry %p, mbuf %p", entry, m); 1151 if (qp->rx_handler != NULL && qp->client_ready) 1152 qp->rx_handler(qp, qp->cb_data, m, len); 1153 1154 mtx_lock_spin(&qp->ntb_rx_q_lock); 1155 } 1156 1157 mtx_unlock_spin(&qp->ntb_rx_q_lock); 1158 } 1159 1160 static void 1161 ntb_transport_doorbell_callback(void *data, uint32_t vector) 1162 { 1163 struct ntb_transport_ctx *nt = data; 1164 struct ntb_transport_qp *qp; 1165 struct _qpset db_bits; 1166 uint64_t vec_mask; 1167 unsigned qp_num; 1168 1169 BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &db_bits); 1170 BIT_NAND(QP_SETSIZE, &db_bits, &nt->qp_bitmap_free); 1171 1172 vec_mask = ntb_db_vector_mask(nt->ntb, vector); 1173 while (vec_mask != 0) { 1174 qp_num = ffsll(vec_mask) - 1; 1175 1176 if (test_bit(qp_num, &db_bits)) { 1177 qp = &nt->qp_vec[qp_num]; 1178 taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work); 1179 } 1180 1181 vec_mask &= ~(1ull << qp_num); 1182 } 1183 } 1184 1185 /* Link Event handler */ 1186 static void 1187 ntb_transport_event_callback(void *data) 1188 { 1189 struct ntb_transport_ctx *nt = data; 1190 1191 if (ntb_link_is_up(nt->ntb, NULL, NULL)) { 1192 ntb_printf(1, "HW link up\n"); 1193 callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt); 1194 } else { 1195 ntb_printf(1, "HW link down\n"); 1196 taskqueue_enqueue(taskqueue_swi, &nt->link_cleanup); 1197 } 1198 } 1199 1200 /* Link bring up */ 1201 static void 1202 ntb_transport_link_work(void *arg) 1203 { 1204 struct ntb_transport_ctx *nt = arg; 1205 struct ntb_softc *ntb = nt->ntb; 1206 struct ntb_transport_qp *qp; 1207 uint64_t val64, size; 1208 uint32_t val; 1209 unsigned i; 1210 int rc; 1211 1212 /* send the local info, in the opposite order of the way we read it */ 1213 for (i = 0; i < nt->mw_count; i++) { 1214 size = nt->mw_vec[i].phys_size; 1215 1216 if (max_mw_size != 0 && size > max_mw_size) 1217 size = max_mw_size; 1218 1219 ntb_peer_spad_write(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2), 1220 size >> 32); 1221 ntb_peer_spad_write(ntb, IF_NTB_MW0_SZ_LOW + (i * 2), size); 1222 } 1223 1224 ntb_peer_spad_write(ntb, IF_NTB_NUM_MWS, nt->mw_count); 1225 1226 ntb_peer_spad_write(ntb, IF_NTB_NUM_QPS, nt->qp_count); 1227 1228 ntb_peer_spad_write(ntb, IF_NTB_VERSION, NTB_TRANSPORT_VERSION); 1229 1230 /* Query the remote side for its info */ 1231 val = 0; 1232 ntb_spad_read(ntb, IF_NTB_VERSION, &val); 1233 if (val != NTB_TRANSPORT_VERSION) 1234 goto out; 1235 1236 ntb_spad_read(ntb, IF_NTB_NUM_QPS, &val); 1237 if (val != nt->qp_count) 1238 goto out; 1239 1240 ntb_spad_read(ntb, IF_NTB_NUM_MWS, &val); 1241 if (val != nt->mw_count) 1242 goto out; 1243 1244 for (i = 0; i < nt->mw_count; i++) { 1245 ntb_spad_read(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2), &val); 1246 val64 = (uint64_t)val << 32; 1247 1248 ntb_spad_read(ntb, IF_NTB_MW0_SZ_LOW + (i * 2), &val); 1249 val64 |= val; 1250 1251 rc = ntb_set_mw(nt, i, val64); 1252 if (rc != 0) 1253 goto free_mws; 1254 } 1255 1256 nt->link_is_up = true; 1257 ntb_printf(1, "transport link up\n"); 1258 1259 for (i = 0; i < nt->qp_count; i++) { 1260 qp = &nt->qp_vec[i]; 1261 1262 ntb_transport_setup_qp_mw(nt, i); 1263 1264 if (qp->client_ready) 1265 callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp); 1266 } 1267 1268 return; 1269 1270 free_mws: 1271 for (i = 0; i < nt->mw_count; i++) 1272 ntb_free_mw(nt, i); 1273 out: 1274 if (ntb_link_is_up(ntb, NULL, NULL)) 1275 callout_reset(&nt->link_work, 1276 NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_transport_link_work, nt); 1277 } 1278 1279 static int 1280 ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, size_t size) 1281 { 1282 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 1283 size_t xlat_size, buff_size; 1284 int rc; 1285 1286 if (size == 0) 1287 return (EINVAL); 1288 1289 xlat_size = roundup(size, mw->xlat_align_size); 1290 buff_size = roundup(size, mw->xlat_align); 1291 1292 /* No need to re-setup */ 1293 if (mw->xlat_size == xlat_size) 1294 return (0); 1295 1296 if (mw->buff_size != 0) 1297 ntb_free_mw(nt, num_mw); 1298 1299 /* Alloc memory for receiving data. Must be aligned */ 1300 mw->xlat_size = xlat_size; 1301 mw->buff_size = buff_size; 1302 1303 mw->virt_addr = contigmalloc(mw->buff_size, M_NTB_IF, M_ZERO, 0, 1304 BUS_SPACE_MAXADDR, mw->xlat_align, 0); 1305 if (mw->virt_addr == NULL) { 1306 mw->xlat_size = 0; 1307 mw->buff_size = 0; 1308 printf("ntb: Unable to allocate MW buffer of size %zu\n", 1309 mw->xlat_size); 1310 return (ENOMEM); 1311 } 1312 /* TODO: replace with bus_space_* functions */ 1313 mw->dma_addr = vtophys(mw->virt_addr); 1314 1315 /* 1316 * Ensure that the allocation from contigmalloc is aligned as 1317 * requested. XXX: This may not be needed -- brought in for parity 1318 * with the Linux driver. 1319 */ 1320 if (mw->dma_addr % mw->xlat_align != 0) { 1321 ntb_printf(0, 1322 "DMA memory 0x%jx not aligned to BAR size 0x%zx\n", 1323 (uintmax_t)mw->dma_addr, size); 1324 ntb_free_mw(nt, num_mw); 1325 return (ENOMEM); 1326 } 1327 1328 /* Notify HW the memory location of the receive buffer */ 1329 rc = ntb_mw_set_trans(nt->ntb, num_mw, mw->dma_addr, mw->xlat_size); 1330 if (rc) { 1331 ntb_printf(0, "Unable to set mw%d translation\n", num_mw); 1332 ntb_free_mw(nt, num_mw); 1333 return (rc); 1334 } 1335 1336 return (0); 1337 } 1338 1339 static void 1340 ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) 1341 { 1342 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw]; 1343 1344 if (mw->virt_addr == NULL) 1345 return; 1346 1347 ntb_mw_clear_trans(nt->ntb, num_mw); 1348 contigfree(mw->virt_addr, mw->xlat_size, M_NTB_IF); 1349 mw->xlat_size = 0; 1350 mw->buff_size = 0; 1351 mw->virt_addr = NULL; 1352 } 1353 1354 static int 1355 ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, unsigned int qp_num) 1356 { 1357 struct ntb_transport_qp *qp = &nt->qp_vec[qp_num]; 1358 struct ntb_transport_mw *mw; 1359 void *offset; 1360 ntb_q_idx_t i; 1361 size_t rx_size; 1362 unsigned num_qps_mw, mw_num, mw_count; 1363 1364 mw_count = nt->mw_count; 1365 mw_num = QP_TO_MW(nt, qp_num); 1366 mw = &nt->mw_vec[mw_num]; 1367 1368 if (mw->virt_addr == NULL) 1369 return (ENOMEM); 1370 1371 if (nt->qp_count % mw_count && mw_num + 1 < nt->qp_count / mw_count) 1372 num_qps_mw = nt->qp_count / mw_count + 1; 1373 else 1374 num_qps_mw = nt->qp_count / mw_count; 1375 1376 rx_size = mw->xlat_size / num_qps_mw; 1377 qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count); 1378 rx_size -= sizeof(struct ntb_rx_info); 1379 1380 qp->remote_rx_info = (void*)(qp->rx_buff + rx_size); 1381 1382 /* Due to house-keeping, there must be at least 2 buffs */ 1383 qp->rx_max_frame = qmin(rx_size / 2, 1384 transport_mtu + sizeof(struct ntb_payload_header)); 1385 qp->rx_max_entry = rx_size / qp->rx_max_frame; 1386 qp->rx_index = 0; 1387 1388 qp->remote_rx_info->entry = qp->rx_max_entry - 1; 1389 1390 /* Set up the hdr offsets with 0s */ 1391 for (i = 0; i < qp->rx_max_entry; i++) { 1392 offset = (void *)(qp->rx_buff + qp->rx_max_frame * (i + 1) - 1393 sizeof(struct ntb_payload_header)); 1394 memset(offset, 0, sizeof(struct ntb_payload_header)); 1395 } 1396 1397 qp->rx_pkts = 0; 1398 qp->tx_pkts = 0; 1399 qp->tx_index = 0; 1400 1401 return (0); 1402 } 1403 1404 static void 1405 ntb_qp_link_work(void *arg) 1406 { 1407 struct ntb_transport_qp *qp = arg; 1408 struct ntb_softc *ntb = qp->ntb; 1409 struct ntb_transport_ctx *nt = qp->transport; 1410 uint32_t val, dummy; 1411 1412 ntb_spad_read(ntb, IF_NTB_QP_LINKS, &val); 1413 1414 ntb_peer_spad_write(ntb, IF_NTB_QP_LINKS, val | (1ull << qp->qp_num)); 1415 1416 /* query remote spad for qp ready bits */ 1417 ntb_peer_spad_read(ntb, IF_NTB_QP_LINKS, &dummy); 1418 1419 /* See if the remote side is up */ 1420 if ((val & (1ull << qp->qp_num)) != 0) { 1421 ntb_printf(2, "qp link up\n"); 1422 qp->link_is_up = true; 1423 1424 if (qp->event_handler != NULL) 1425 qp->event_handler(qp->cb_data, NTB_LINK_UP); 1426 1427 taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work); 1428 } else if (nt->link_is_up) 1429 callout_reset(&qp->link_work, 1430 NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp); 1431 } 1432 1433 /* Link down event*/ 1434 static void 1435 ntb_transport_link_cleanup(struct ntb_transport_ctx *nt) 1436 { 1437 struct ntb_transport_qp *qp; 1438 struct _qpset qp_bitmap_alloc; 1439 unsigned i; 1440 1441 BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &qp_bitmap_alloc); 1442 BIT_NAND(QP_SETSIZE, &qp_bitmap_alloc, &nt->qp_bitmap_free); 1443 1444 /* Pass along the info to any clients */ 1445 for (i = 0; i < nt->qp_count; i++) 1446 if (test_bit(i, &qp_bitmap_alloc)) { 1447 qp = &nt->qp_vec[i]; 1448 ntb_qp_link_cleanup(qp); 1449 callout_drain(&qp->link_work); 1450 } 1451 1452 if (!nt->link_is_up) 1453 callout_drain(&nt->link_work); 1454 1455 /* 1456 * The scratchpad registers keep the values if the remote side 1457 * goes down, blast them now to give them a sane value the next 1458 * time they are accessed 1459 */ 1460 for (i = 0; i < IF_NTB_MAX_SPAD; i++) 1461 ntb_spad_write(nt->ntb, i, 0); 1462 } 1463 1464 static void 1465 ntb_transport_link_cleanup_work(void *arg, int pending __unused) 1466 { 1467 1468 ntb_transport_link_cleanup(arg); 1469 } 1470 1471 static void 1472 ntb_qp_link_down(struct ntb_transport_qp *qp) 1473 { 1474 1475 ntb_qp_link_cleanup(qp); 1476 } 1477 1478 static void 1479 ntb_qp_link_down_reset(struct ntb_transport_qp *qp) 1480 { 1481 1482 qp->link_is_up = false; 1483 1484 qp->tx_index = qp->rx_index = 0; 1485 qp->tx_bytes = qp->rx_bytes = 0; 1486 qp->tx_pkts = qp->rx_pkts = 0; 1487 1488 qp->rx_ring_empty = 0; 1489 qp->tx_ring_full = 0; 1490 1491 qp->rx_err_no_buf = qp->tx_err_no_buf = 0; 1492 qp->rx_err_oflow = qp->rx_err_ver = 0; 1493 } 1494 1495 static void 1496 ntb_qp_link_cleanup(struct ntb_transport_qp *qp) 1497 { 1498 struct ntb_transport_ctx *nt = qp->transport; 1499 1500 callout_drain(&qp->link_work); 1501 ntb_qp_link_down_reset(qp); 1502 1503 if (qp->event_handler != NULL) 1504 qp->event_handler(qp->cb_data, NTB_LINK_DOWN); 1505 1506 if (nt->link_is_up) 1507 callout_reset(&qp->link_work, 1508 NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp); 1509 } 1510 1511 /* Link commanded down */ 1512 /** 1513 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data 1514 * @qp: NTB transport layer queue to be disabled 1515 * 1516 * Notify NTB transport layer of client's desire to no longer receive data on 1517 * transport queue specified. It is the client's responsibility to ensure all 1518 * entries on queue are purged or otherwise handled appropriately. 1519 */ 1520 static void 1521 ntb_transport_link_down(struct ntb_transport_qp *qp) 1522 { 1523 uint32_t val; 1524 1525 if (qp == NULL) 1526 return; 1527 1528 qp->client_ready = false; 1529 1530 ntb_spad_read(qp->ntb, IF_NTB_QP_LINKS, &val); 1531 1532 ntb_peer_spad_write(qp->ntb, IF_NTB_QP_LINKS, 1533 val & ~(1 << qp->qp_num)); 1534 1535 if (qp->link_is_up) 1536 ntb_send_link_down(qp); 1537 else 1538 callout_drain(&qp->link_work); 1539 } 1540 1541 static void 1542 ntb_send_link_down(struct ntb_transport_qp *qp) 1543 { 1544 struct ntb_queue_entry *entry; 1545 int i, rc; 1546 1547 if (!qp->link_is_up) 1548 return; 1549 1550 for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { 1551 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1552 if (entry != NULL) 1553 break; 1554 pause("NTB Wait for link down", hz / 10); 1555 } 1556 1557 if (entry == NULL) 1558 return; 1559 1560 entry->cb_data = NULL; 1561 entry->buf = NULL; 1562 entry->len = 0; 1563 entry->flags = IF_NTB_LINK_DOWN_FLAG; 1564 1565 mtx_lock(&qp->transport->tx_lock); 1566 rc = ntb_process_tx(qp, entry); 1567 if (rc != 0) 1568 printf("ntb: Failed to send link down\n"); 1569 mtx_unlock(&qp->transport->tx_lock); 1570 1571 ntb_qp_link_down_reset(qp); 1572 } 1573 1574 1575 /* List Management */ 1576 1577 static void 1578 ntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry, 1579 struct ntb_queue_list *list) 1580 { 1581 1582 mtx_lock_spin(lock); 1583 STAILQ_INSERT_TAIL(list, entry, entry); 1584 mtx_unlock_spin(lock); 1585 } 1586 1587 static struct ntb_queue_entry * 1588 ntb_list_rm(struct mtx *lock, struct ntb_queue_list *list) 1589 { 1590 struct ntb_queue_entry *entry; 1591 1592 mtx_lock_spin(lock); 1593 if (STAILQ_EMPTY(list)) { 1594 entry = NULL; 1595 goto out; 1596 } 1597 entry = STAILQ_FIRST(list); 1598 STAILQ_REMOVE_HEAD(list, entry); 1599 out: 1600 mtx_unlock_spin(lock); 1601 1602 return (entry); 1603 } 1604 1605 static struct ntb_queue_entry * 1606 ntb_list_mv(struct mtx *lock, struct ntb_queue_list *from, 1607 struct ntb_queue_list *to) 1608 { 1609 struct ntb_queue_entry *entry; 1610 1611 mtx_lock_spin(lock); 1612 if (STAILQ_EMPTY(from)) { 1613 entry = NULL; 1614 goto out; 1615 } 1616 entry = STAILQ_FIRST(from); 1617 STAILQ_REMOVE_HEAD(from, entry); 1618 STAILQ_INSERT_TAIL(to, entry, entry); 1619 1620 out: 1621 mtx_unlock_spin(lock); 1622 return (entry); 1623 } 1624 1625 /* Helper functions */ 1626 /* TODO: This too should really be part of the kernel */ 1627 #define EUI48_MULTICAST 1 << 0 1628 #define EUI48_LOCALLY_ADMINISTERED 1 << 1 1629 static void 1630 create_random_local_eui48(u_char *eaddr) 1631 { 1632 static uint8_t counter = 0; 1633 uint32_t seed = ticks; 1634 1635 eaddr[0] = EUI48_LOCALLY_ADMINISTERED; 1636 memcpy(&eaddr[1], &seed, sizeof(uint32_t)); 1637 eaddr[5] = counter++; 1638 } 1639 1640 /** 1641 * ntb_transport_max_size - Query the max payload size of a qp 1642 * @qp: NTB transport layer queue to be queried 1643 * 1644 * Query the maximum payload size permissible on the given qp 1645 * 1646 * RETURNS: the max payload size of a qp 1647 */ 1648 static unsigned int 1649 ntb_transport_max_size(struct ntb_transport_qp *qp) 1650 { 1651 1652 if (qp == NULL) 1653 return (0); 1654 1655 return (qp->tx_max_frame - sizeof(struct ntb_payload_header)); 1656 } 1657