1 /*- 2 * Copyright (C) 2013 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/kernel.h> 32 #include <sys/systm.h> 33 #include <sys/bus.h> 34 #include <sys/ktr.h> 35 #include <sys/lock.h> 36 #include <sys/malloc.h> 37 #include <sys/module.h> 38 #include <sys/mutex.h> 39 #include <sys/queue.h> 40 #include <sys/socket.h> 41 #include <sys/sockio.h> 42 #include <sys/taskqueue.h> 43 #include <net/if.h> 44 #include <net/if_media.h> 45 #include <net/if_types.h> 46 #include <net/if_var.h> 47 #include <net/bpf.h> 48 #include <net/ethernet.h> 49 #include <vm/vm.h> 50 #include <vm/pmap.h> 51 #include <machine/bus.h> 52 #include <machine/cpufunc.h> 53 #include <machine/pmap.h> 54 55 #include "../ntb_hw/ntb_hw.h" 56 57 /* 58 * The Non-Transparent Bridge (NTB) is a device on some Intel processors that 59 * allows you to connect two systems using a PCI-e link. 60 * 61 * This module contains a protocol for sending and receiving messages, and 62 * exposes that protocol through a simulated ethernet device called ntb. 63 * 64 * NOTE: Much of the code in this module is shared with Linux. Any patches may 65 * be picked up and redistributed in Linux with a dual GPL/BSD license. 66 */ 67 68 /* TODO: These functions should really be part of the kernel */ 69 #define test_bit(pos, bitmap_addr) (*(bitmap_addr) & 1UL << (pos)) 70 #define set_bit(pos, bitmap_addr) *(bitmap_addr) |= 1UL << (pos) 71 #define clear_bit(pos, bitmap_addr) *(bitmap_addr) &= ~(1UL << (pos)) 72 73 #define KTR_NTB KTR_SPARE3 74 75 #define NTB_TRANSPORT_VERSION 3 76 #define NTB_RX_MAX_PKTS 64 77 #define NTB_RXQ_SIZE 300 78 79 static unsigned int transport_mtu = 0x4000 + ETHER_HDR_LEN + ETHER_CRC_LEN; 80 static unsigned int max_num_clients = 1; 81 82 STAILQ_HEAD(ntb_queue_list, ntb_queue_entry); 83 84 struct ntb_queue_entry { 85 /* ntb_queue list reference */ 86 STAILQ_ENTRY(ntb_queue_entry) entry; 87 88 /* info on data to be transfered */ 89 void *cb_data; 90 void *buf; 91 uint64_t len; 92 uint64_t flags; 93 }; 94 95 struct ntb_rx_info { 96 unsigned int entry; 97 }; 98 99 struct ntb_transport_qp { 100 struct ntb_netdev *transport; 101 struct ntb_softc *ntb; 102 103 void *cb_data; 104 105 bool client_ready; 106 bool qp_link; 107 uint8_t qp_num; /* Only 64 QPs are allowed. 0-63 */ 108 109 struct ntb_rx_info *rx_info; 110 struct ntb_rx_info *remote_rx_info; 111 112 void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data, 113 void *data, int len); 114 struct ntb_queue_list tx_free_q; 115 struct mtx ntb_tx_free_q_lock; 116 void *tx_mw; 117 uint64_t tx_index; 118 uint64_t tx_max_entry; 119 uint64_t tx_max_frame; 120 121 void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data, 122 void *data, int len); 123 struct ntb_queue_list rx_pend_q; 124 struct ntb_queue_list rx_free_q; 125 struct mtx ntb_rx_pend_q_lock; 126 struct mtx ntb_rx_free_q_lock; 127 struct task rx_completion_task; 128 void *rx_buff; 129 uint64_t rx_index; 130 uint64_t rx_max_entry; 131 uint64_t rx_max_frame; 132 133 void (*event_handler) (void *data, int status); 134 struct callout link_work; 135 struct callout queue_full; 136 struct callout rx_full; 137 138 uint64_t last_rx_no_buf; 139 140 /* Stats */ 141 uint64_t rx_bytes; 142 uint64_t rx_pkts; 143 uint64_t rx_ring_empty; 144 uint64_t rx_err_no_buf; 145 uint64_t rx_err_oflow; 146 uint64_t rx_err_ver; 147 uint64_t tx_bytes; 148 uint64_t tx_pkts; 149 uint64_t tx_ring_full; 150 }; 151 152 struct ntb_queue_handlers { 153 void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data, 154 void *data, int len); 155 void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data, 156 void *data, int len); 157 void (*event_handler) (void *data, int status); 158 }; 159 160 161 struct ntb_transport_mw { 162 size_t size; 163 void *virt_addr; 164 vm_paddr_t dma_addr; 165 }; 166 167 struct ntb_netdev { 168 struct ntb_softc *ntb; 169 struct ifnet *ifp; 170 struct ntb_transport_mw mw[NTB_NUM_MW]; 171 struct ntb_transport_qp *qps; 172 uint64_t max_qps; 173 uint64_t qp_bitmap; 174 bool transport_link; 175 struct callout link_work; 176 struct ntb_transport_qp *qp; 177 uint64_t bufsize; 178 u_char eaddr[ETHER_ADDR_LEN]; 179 struct mtx tx_lock; 180 struct mtx rx_lock; 181 }; 182 183 static struct ntb_netdev net_softc; 184 185 enum { 186 IF_NTB_DESC_DONE_FLAG = 1 << 0, 187 IF_NTB_LINK_DOWN_FLAG = 1 << 1, 188 }; 189 190 struct ntb_payload_header { 191 uint64_t ver; 192 uint64_t len; 193 uint64_t flags; 194 }; 195 196 enum { 197 /* 198 * The order of this enum is part of the if_ntb remote protocol. Do 199 * not reorder without bumping protocol version (and it's probably best 200 * to keep the protocol in lock-step with the Linux NTB driver. 201 */ 202 IF_NTB_VERSION = 0, 203 IF_NTB_QP_LINKS, 204 IF_NTB_NUM_QPS, 205 IF_NTB_NUM_MWS, 206 /* 207 * N.B.: transport_link_work assumes MW1 enums = MW0 + 2. 208 */ 209 IF_NTB_MW0_SZ_HIGH, 210 IF_NTB_MW0_SZ_LOW, 211 IF_NTB_MW1_SZ_HIGH, 212 IF_NTB_MW1_SZ_LOW, 213 IF_NTB_MAX_SPAD, 214 }; 215 216 #define QP_TO_MW(qp) ((qp) % NTB_NUM_MW) 217 #define NTB_QP_DEF_NUM_ENTRIES 100 218 #define NTB_LINK_DOWN_TIMEOUT 10 219 220 static int ntb_handle_module_events(struct module *m, int what, void *arg); 221 static int ntb_setup_interface(void); 222 static int ntb_teardown_interface(void); 223 static void ntb_net_init(void *arg); 224 static int ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data); 225 static void ntb_start(struct ifnet *ifp); 226 static void ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, 227 void *data, int len); 228 static void ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, 229 void *data, int len); 230 static void ntb_net_event_handler(void *data, int status); 231 static int ntb_transport_init(struct ntb_softc *ntb); 232 static void ntb_transport_free(void *transport); 233 static void ntb_transport_init_queue(struct ntb_netdev *nt, 234 unsigned int qp_num); 235 static void ntb_transport_free_queue(struct ntb_transport_qp *qp); 236 static struct ntb_transport_qp * ntb_transport_create_queue(void *data, 237 struct ntb_softc *pdev, const struct ntb_queue_handlers *handlers); 238 static void ntb_transport_link_up(struct ntb_transport_qp *qp); 239 static int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, 240 void *data, unsigned int len); 241 static int ntb_process_tx(struct ntb_transport_qp *qp, 242 struct ntb_queue_entry *entry); 243 static void ntb_tx_copy_task(struct ntb_transport_qp *qp, 244 struct ntb_queue_entry *entry, void *offset); 245 static void ntb_qp_full(void *arg); 246 static void ntb_transport_rxc_db(void *data, int db_num); 247 static void ntb_rx_pendq_full(void *arg); 248 static void ntb_transport_rx(struct ntb_transport_qp *qp); 249 static int ntb_process_rxc(struct ntb_transport_qp *qp); 250 static void ntb_rx_copy_task(struct ntb_transport_qp *qp, 251 struct ntb_queue_entry *entry, void *offset); 252 static void ntb_rx_completion_task(void *arg, int pending); 253 static void ntb_transport_event_callback(void *data, enum ntb_hw_event event); 254 static void ntb_transport_link_work(void *arg); 255 static int ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size); 256 static void ntb_free_mw(struct ntb_netdev *nt, int num_mw); 257 static void ntb_transport_setup_qp_mw(struct ntb_netdev *nt, 258 unsigned int qp_num); 259 static void ntb_qp_link_work(void *arg); 260 static void ntb_transport_link_cleanup(struct ntb_netdev *nt); 261 static void ntb_qp_link_down(struct ntb_transport_qp *qp); 262 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp); 263 static void ntb_transport_link_down(struct ntb_transport_qp *qp); 264 static void ntb_send_link_down(struct ntb_transport_qp *qp); 265 static void ntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry, 266 struct ntb_queue_list *list); 267 static struct ntb_queue_entry *ntb_list_rm(struct mtx *lock, 268 struct ntb_queue_list *list); 269 static void create_random_local_eui48(u_char *eaddr); 270 static unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp); 271 272 MALLOC_DEFINE(M_NTB_IF, "if_ntb", "ntb network driver"); 273 274 /* Module setup and teardown */ 275 static int 276 ntb_handle_module_events(struct module *m, int what, void *arg) 277 { 278 int err = 0; 279 280 switch (what) { 281 case MOD_LOAD: 282 err = ntb_setup_interface(); 283 break; 284 case MOD_UNLOAD: 285 err = ntb_teardown_interface(); 286 break; 287 default: 288 err = EOPNOTSUPP; 289 break; 290 } 291 return (err); 292 } 293 294 static moduledata_t if_ntb_mod = { 295 "if_ntb", 296 ntb_handle_module_events, 297 NULL 298 }; 299 300 DECLARE_MODULE(if_ntb, if_ntb_mod, SI_SUB_KLD, SI_ORDER_ANY); 301 MODULE_DEPEND(if_ntb, ntb_hw, 1, 1, 1); 302 303 static int 304 ntb_setup_interface() 305 { 306 struct ifnet *ifp; 307 struct ntb_queue_handlers handlers = { ntb_net_rx_handler, 308 ntb_net_tx_handler, ntb_net_event_handler }; 309 310 net_softc.ntb = devclass_get_softc(devclass_find("ntb_hw"), 0); 311 if (net_softc.ntb == NULL) { 312 printf("ntb: Cannot find devclass\n"); 313 return (ENXIO); 314 } 315 316 ntb_transport_init(net_softc.ntb); 317 318 ifp = net_softc.ifp = if_alloc(IFT_ETHER); 319 if (ifp == NULL) { 320 printf("ntb: cannot allocate ifnet structure\n"); 321 return (ENOMEM); 322 } 323 324 net_softc.qp = ntb_transport_create_queue(ifp, net_softc.ntb, 325 &handlers); 326 if_initname(ifp, "ntb", 0); 327 ifp->if_init = ntb_net_init; 328 ifp->if_softc = &net_softc; 329 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX; 330 ifp->if_ioctl = ntb_ioctl; 331 ifp->if_start = ntb_start; 332 IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN); 333 ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN; 334 IFQ_SET_READY(&ifp->if_snd); 335 create_random_local_eui48(net_softc.eaddr); 336 ether_ifattach(ifp, net_softc.eaddr); 337 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_JUMBO_MTU; 338 ifp->if_capenable = ifp->if_capabilities; 339 340 ntb_transport_link_up(net_softc.qp); 341 net_softc.bufsize = ntb_transport_max_size(net_softc.qp) + 342 sizeof(struct ether_header); 343 return (0); 344 } 345 346 static int 347 ntb_teardown_interface() 348 { 349 350 if (net_softc.qp != NULL) 351 ntb_transport_link_down(net_softc.qp); 352 353 if (net_softc.ifp != NULL) { 354 ether_ifdetach(net_softc.ifp); 355 if_free(net_softc.ifp); 356 } 357 358 if (net_softc.qp != NULL) { 359 ntb_transport_free_queue(net_softc.qp); 360 ntb_transport_free(&net_softc); 361 } 362 363 return (0); 364 } 365 366 /* Network device interface */ 367 368 static void 369 ntb_net_init(void *arg) 370 { 371 struct ntb_netdev *ntb_softc = arg; 372 struct ifnet *ifp = ntb_softc->ifp; 373 374 ifp->if_drv_flags |= IFF_DRV_RUNNING; 375 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 376 ifp->if_flags |= IFF_UP; 377 if_link_state_change(ifp, LINK_STATE_UP); 378 } 379 380 static int 381 ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 382 { 383 struct ntb_netdev *nt = ifp->if_softc; 384 struct ifreq *ifr = (struct ifreq *)data; 385 int error = 0; 386 387 switch (command) { 388 case SIOCSIFMTU: 389 { 390 if (ifr->ifr_mtu > ntb_transport_max_size(nt->qp) - 391 ETHER_HDR_LEN - ETHER_CRC_LEN) { 392 error = EINVAL; 393 break; 394 } 395 396 ifp->if_mtu = ifr->ifr_mtu; 397 break; 398 } 399 default: 400 error = ether_ioctl(ifp, command, data); 401 break; 402 } 403 404 return (error); 405 } 406 407 408 static void 409 ntb_start(struct ifnet *ifp) 410 { 411 struct mbuf *m_head; 412 struct ntb_netdev *nt = ifp->if_softc; 413 int rc; 414 415 mtx_lock(&nt->tx_lock); 416 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 417 CTR0(KTR_NTB, "TX: ntb_start"); 418 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 419 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 420 CTR1(KTR_NTB, "TX: start mbuf %p", m_head); 421 rc = ntb_transport_tx_enqueue(nt->qp, m_head, m_head, 422 m_length(m_head, NULL)); 423 if (rc != 0) { 424 CTR1(KTR_NTB, 425 "TX: could not tx mbuf %p. Returning to snd q", 426 m_head); 427 if (rc == EAGAIN) { 428 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 429 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 430 callout_reset(&nt->qp->queue_full, hz / 1000, 431 ntb_qp_full, ifp); 432 } 433 break; 434 } 435 436 } 437 mtx_unlock(&nt->tx_lock); 438 } 439 440 /* Network Device Callbacks */ 441 static void 442 ntb_net_tx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, 443 int len) 444 { 445 446 m_freem(data); 447 CTR1(KTR_NTB, "TX: tx_handler freeing mbuf %p", data); 448 } 449 450 static void 451 ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, 452 int len) 453 { 454 struct mbuf *m = data; 455 struct ifnet *ifp = qp_data; 456 457 CTR0(KTR_NTB, "RX: rx handler"); 458 (*ifp->if_input)(ifp, m); 459 } 460 461 static void 462 ntb_net_event_handler(void *data, int status) 463 { 464 465 } 466 467 /* Transport Init and teardown */ 468 469 static int 470 ntb_transport_init(struct ntb_softc *ntb) 471 { 472 struct ntb_netdev *nt = &net_softc; 473 int rc, i; 474 475 nt->max_qps = max_num_clients; 476 ntb_register_transport(ntb, nt); 477 mtx_init(&nt->tx_lock, "ntb transport tx", NULL, MTX_DEF); 478 mtx_init(&nt->rx_lock, "ntb transport rx", NULL, MTX_DEF); 479 480 nt->qps = malloc(nt->max_qps * sizeof(struct ntb_transport_qp), 481 M_NTB_IF, M_WAITOK|M_ZERO); 482 483 nt->qp_bitmap = ((uint64_t) 1 << nt->max_qps) - 1; 484 485 for (i = 0; i < nt->max_qps; i++) 486 ntb_transport_init_queue(nt, i); 487 488 callout_init(&nt->link_work, 0); 489 490 rc = ntb_register_event_callback(ntb, 491 ntb_transport_event_callback); 492 if (rc != 0) 493 goto err; 494 495 if (ntb_query_link_status(ntb)) { 496 if (bootverbose) 497 device_printf(ntb_get_device(ntb), "link up\n"); 498 callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt); 499 } 500 501 return (0); 502 503 err: 504 free(nt->qps, M_NTB_IF); 505 ntb_unregister_transport(ntb); 506 return (rc); 507 } 508 509 static void 510 ntb_transport_free(void *transport) 511 { 512 struct ntb_netdev *nt = transport; 513 struct ntb_softc *ntb = nt->ntb; 514 int i; 515 516 nt->transport_link = NTB_LINK_DOWN; 517 518 callout_drain(&nt->link_work); 519 520 /* verify that all the qps are freed */ 521 for (i = 0; i < nt->max_qps; i++) 522 if (!test_bit(i, &nt->qp_bitmap)) 523 ntb_transport_free_queue(&nt->qps[i]); 524 525 526 ntb_unregister_event_callback(ntb); 527 528 for (i = 0; i < NTB_NUM_MW; i++) 529 ntb_free_mw(nt, i); 530 531 free(nt->qps, M_NTB_IF); 532 ntb_unregister_transport(ntb); 533 } 534 535 static void 536 ntb_transport_init_queue(struct ntb_netdev *nt, unsigned int qp_num) 537 { 538 struct ntb_transport_qp *qp; 539 unsigned int num_qps_mw, tx_size; 540 uint8_t mw_num = QP_TO_MW(qp_num); 541 542 qp = &nt->qps[qp_num]; 543 qp->qp_num = qp_num; 544 qp->transport = nt; 545 qp->ntb = nt->ntb; 546 qp->qp_link = NTB_LINK_DOWN; 547 qp->client_ready = NTB_LINK_DOWN; 548 qp->event_handler = NULL; 549 550 if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW) 551 num_qps_mw = nt->max_qps / NTB_NUM_MW + 1; 552 else 553 num_qps_mw = nt->max_qps / NTB_NUM_MW; 554 555 tx_size = (unsigned int) ntb_get_mw_size(qp->ntb, mw_num) / num_qps_mw; 556 qp->rx_info = (struct ntb_rx_info *) 557 ((char *)ntb_get_mw_vbase(qp->ntb, mw_num) + 558 (qp_num / NTB_NUM_MW * tx_size)); 559 tx_size -= sizeof(struct ntb_rx_info); 560 561 qp->tx_mw = qp->rx_info + 1; 562 /* Due to house-keeping, there must be at least 2 buffs */ 563 qp->tx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header), 564 tx_size / 2); 565 qp->tx_max_entry = tx_size / qp->tx_max_frame; 566 567 callout_init(&qp->link_work, 0); 568 callout_init(&qp->queue_full, 1); 569 callout_init(&qp->rx_full, 1); 570 571 mtx_init(&qp->ntb_rx_pend_q_lock, "ntb rx pend q", NULL, MTX_SPIN); 572 mtx_init(&qp->ntb_rx_free_q_lock, "ntb rx free q", NULL, MTX_SPIN); 573 mtx_init(&qp->ntb_tx_free_q_lock, "ntb tx free q", NULL, MTX_SPIN); 574 TASK_INIT(&qp->rx_completion_task, 0, ntb_rx_completion_task, qp); 575 576 STAILQ_INIT(&qp->rx_pend_q); 577 STAILQ_INIT(&qp->rx_free_q); 578 STAILQ_INIT(&qp->tx_free_q); 579 } 580 581 static void 582 ntb_transport_free_queue(struct ntb_transport_qp *qp) 583 { 584 struct ntb_queue_entry *entry; 585 586 if (qp == NULL) 587 return; 588 589 callout_drain(&qp->link_work); 590 591 ntb_unregister_db_callback(qp->ntb, qp->qp_num); 592 593 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) 594 free(entry, M_NTB_IF); 595 596 while ((entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q))) 597 free(entry, M_NTB_IF); 598 599 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 600 free(entry, M_NTB_IF); 601 602 set_bit(qp->qp_num, &qp->transport->qp_bitmap); 603 } 604 605 /** 606 * ntb_transport_create_queue - Create a new NTB transport layer queue 607 * @rx_handler: receive callback function 608 * @tx_handler: transmit callback function 609 * @event_handler: event callback function 610 * 611 * Create a new NTB transport layer queue and provide the queue with a callback 612 * routine for both transmit and receive. The receive callback routine will be 613 * used to pass up data when the transport has received it on the queue. The 614 * transmit callback routine will be called when the transport has completed the 615 * transmission of the data on the queue and the data is ready to be freed. 616 * 617 * RETURNS: pointer to newly created ntb_queue, NULL on error. 618 */ 619 static struct ntb_transport_qp * 620 ntb_transport_create_queue(void *data, struct ntb_softc *pdev, 621 const struct ntb_queue_handlers *handlers) 622 { 623 struct ntb_queue_entry *entry; 624 struct ntb_transport_qp *qp; 625 struct ntb_netdev *nt; 626 unsigned int free_queue; 627 int rc, i; 628 629 nt = ntb_find_transport(pdev); 630 if (nt == NULL) 631 goto err; 632 633 free_queue = ffs(nt->qp_bitmap); 634 if (free_queue == 0) 635 goto err; 636 637 /* decrement free_queue to make it zero based */ 638 free_queue--; 639 640 clear_bit(free_queue, &nt->qp_bitmap); 641 642 qp = &nt->qps[free_queue]; 643 qp->cb_data = data; 644 qp->rx_handler = handlers->rx_handler; 645 qp->tx_handler = handlers->tx_handler; 646 qp->event_handler = handlers->event_handler; 647 648 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 649 entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF, 650 M_WAITOK|M_ZERO); 651 entry->cb_data = nt->ifp; 652 entry->buf = NULL; 653 entry->len = transport_mtu; 654 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 655 } 656 657 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 658 entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF, 659 M_WAITOK|M_ZERO); 660 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 661 } 662 663 rc = ntb_register_db_callback(qp->ntb, free_queue, qp, 664 ntb_transport_rxc_db); 665 if (rc != 0) 666 goto err1; 667 668 return (qp); 669 670 err1: 671 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 672 free(entry, M_NTB_IF); 673 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) 674 free(entry, M_NTB_IF); 675 set_bit(free_queue, &nt->qp_bitmap); 676 err: 677 return (NULL); 678 } 679 680 /** 681 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue 682 * @qp: NTB transport layer queue to be enabled 683 * 684 * Notify NTB transport layer of client readiness to use queue 685 */ 686 static void 687 ntb_transport_link_up(struct ntb_transport_qp *qp) 688 { 689 690 if (qp == NULL) 691 return; 692 693 qp->client_ready = NTB_LINK_UP; 694 if (bootverbose) 695 device_printf(ntb_get_device(qp->ntb), "qp client ready\n"); 696 697 if (qp->transport->transport_link == NTB_LINK_UP) 698 callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp); 699 } 700 701 702 703 /* Transport Tx */ 704 705 /** 706 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry 707 * @qp: NTB transport layer queue the entry is to be enqueued on 708 * @cb: per buffer pointer for callback function to use 709 * @data: pointer to data buffer that will be sent 710 * @len: length of the data buffer 711 * 712 * Enqueue a new transmit buffer onto the transport queue from which a NTB 713 * payload will be transmitted. This assumes that a lock is behing held to 714 * serialize access to the qp. 715 * 716 * RETURNS: An appropriate ERRNO error value on error, or zero for success. 717 */ 718 static int 719 ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 720 unsigned int len) 721 { 722 struct ntb_queue_entry *entry; 723 int rc; 724 725 if (qp == NULL || qp->qp_link != NTB_LINK_UP || len == 0) { 726 CTR0(KTR_NTB, "TX: link not up"); 727 return (EINVAL); 728 } 729 730 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 731 if (entry == NULL) { 732 CTR0(KTR_NTB, "TX: could not get entry from tx_free_q"); 733 return (ENOMEM); 734 } 735 CTR1(KTR_NTB, "TX: got entry %p from tx_free_q", entry); 736 737 entry->cb_data = cb; 738 entry->buf = data; 739 entry->len = len; 740 entry->flags = 0; 741 742 rc = ntb_process_tx(qp, entry); 743 if (rc != 0) { 744 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 745 CTR1(KTR_NTB, 746 "TX: process_tx failed. Returning entry %p to tx_free_q", 747 entry); 748 } 749 return (rc); 750 } 751 752 static int 753 ntb_process_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry) 754 { 755 void *offset; 756 757 offset = (char *)qp->tx_mw + qp->tx_max_frame * qp->tx_index; 758 CTR3(KTR_NTB, 759 "TX: process_tx: tx_pkts=%u, tx_index=%u, remote entry=%u", 760 qp->tx_pkts, qp->tx_index, qp->remote_rx_info->entry); 761 if (qp->tx_index == qp->remote_rx_info->entry) { 762 CTR0(KTR_NTB, "TX: ring full"); 763 qp->tx_ring_full++; 764 return (EAGAIN); 765 } 766 767 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { 768 if (qp->tx_handler != NULL) 769 qp->tx_handler(qp, qp->cb_data, entry->buf, 770 EIO); 771 772 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 773 CTR1(KTR_NTB, 774 "TX: frame too big. returning entry %p to tx_free_q", 775 entry); 776 return (0); 777 } 778 CTR2(KTR_NTB, "TX: copying entry %p to offset %p", entry, offset); 779 ntb_tx_copy_task(qp, entry, offset); 780 781 qp->tx_index++; 782 qp->tx_index %= qp->tx_max_entry; 783 784 qp->tx_pkts++; 785 786 return (0); 787 } 788 789 static void 790 ntb_tx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry, 791 void *offset) 792 { 793 struct ntb_payload_header *hdr; 794 795 CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset); 796 if (entry->buf != NULL) 797 m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset); 798 799 hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame - 800 sizeof(struct ntb_payload_header)); 801 hdr->len = entry->len; /* TODO: replace with bus_space_write */ 802 hdr->ver = qp->tx_pkts; /* TODO: replace with bus_space_write */ 803 wmb(); 804 /* TODO: replace with bus_space_write */ 805 hdr->flags = entry->flags | IF_NTB_DESC_DONE_FLAG; 806 807 ntb_ring_sdb(qp->ntb, qp->qp_num); 808 809 /* 810 * The entry length can only be zero if the packet is intended to be a 811 * "link down" or similar. Since no payload is being sent in these 812 * cases, there is nothing to add to the completion queue. 813 */ 814 if (entry->len > 0) { 815 qp->tx_bytes += entry->len; 816 817 if (qp->tx_handler) 818 qp->tx_handler(qp, qp->cb_data, entry->cb_data, 819 entry->len); 820 } 821 822 CTR2(KTR_NTB, 823 "TX: entry %p sent. hdr->ver = %d, Returning to tx_free_q", entry, 824 hdr->ver); 825 ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); 826 } 827 828 static void 829 ntb_qp_full(void *arg) 830 { 831 832 CTR0(KTR_NTB, "TX: qp_full callout"); 833 ntb_start(arg); 834 } 835 836 /* Transport Rx */ 837 static void 838 ntb_transport_rxc_db(void *data, int db_num) 839 { 840 struct ntb_transport_qp *qp = data; 841 842 ntb_transport_rx(qp); 843 } 844 845 static void 846 ntb_rx_pendq_full(void *arg) 847 { 848 849 CTR0(KTR_NTB, "RX: ntb_rx_pendq_full callout"); 850 ntb_transport_rx(arg); 851 } 852 853 static void 854 ntb_transport_rx(struct ntb_transport_qp *qp) 855 { 856 uint64_t i; 857 int rc; 858 859 /* 860 * Limit the number of packets processed in a single interrupt to 861 * provide fairness to others 862 */ 863 mtx_lock(&qp->transport->rx_lock); 864 CTR0(KTR_NTB, "RX: transport_rx"); 865 for (i = 0; i < qp->rx_max_entry; i++) { 866 rc = ntb_process_rxc(qp); 867 if (rc != 0) { 868 CTR0(KTR_NTB, "RX: process_rxc failed"); 869 break; 870 } 871 } 872 mtx_unlock(&qp->transport->rx_lock); 873 } 874 875 static int 876 ntb_process_rxc(struct ntb_transport_qp *qp) 877 { 878 struct ntb_payload_header *hdr; 879 struct ntb_queue_entry *entry; 880 void *offset; 881 882 offset = (void *) 883 ((char *)qp->rx_buff + qp->rx_max_frame * qp->rx_index); 884 hdr = (void *) 885 ((char *)offset + qp->rx_max_frame - 886 sizeof(struct ntb_payload_header)); 887 888 CTR1(KTR_NTB, "RX: process_rxc rx_index = %u", qp->rx_index); 889 entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); 890 if (entry == NULL) { 891 qp->rx_err_no_buf++; 892 CTR0(KTR_NTB, "RX: No entries in rx_pend_q"); 893 return (ENOMEM); 894 } 895 callout_stop(&qp->rx_full); 896 CTR1(KTR_NTB, "RX: rx entry %p from rx_pend_q", entry); 897 898 if ((hdr->flags & IF_NTB_DESC_DONE_FLAG) == 0) { 899 CTR1(KTR_NTB, 900 "RX: hdr not done. Returning entry %p to rx_pend_q", entry); 901 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 902 qp->rx_ring_empty++; 903 return (EAGAIN); 904 } 905 906 if (hdr->ver != (uint32_t) qp->rx_pkts) { 907 CTR3(KTR_NTB,"RX: ver != rx_pkts (%x != %lx). " 908 "Returning entry %p to rx_pend_q", hdr->ver, qp->rx_pkts, 909 entry); 910 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 911 qp->rx_err_ver++; 912 return (EIO); 913 } 914 915 if ((hdr->flags & IF_NTB_LINK_DOWN_FLAG) != 0) { 916 ntb_qp_link_down(qp); 917 CTR1(KTR_NTB, 918 "RX: link down. adding entry %p back to rx_pend_q", entry); 919 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 920 goto out; 921 } 922 923 if (hdr->len <= entry->len) { 924 entry->len = hdr->len; 925 ntb_rx_copy_task(qp, entry, offset); 926 } else { 927 CTR1(KTR_NTB, 928 "RX: len too long. Returning entry %p to rx_pend_q", entry); 929 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 930 931 qp->rx_err_oflow++; 932 } 933 934 qp->rx_bytes += hdr->len; 935 qp->rx_pkts++; 936 CTR1(KTR_NTB, "RX: received %ld rx_pkts", qp->rx_pkts); 937 938 939 out: 940 /* Ensure that the data is globally visible before clearing the flag */ 941 wmb(); 942 hdr->flags = 0; 943 /* TODO: replace with bus_space_write */ 944 qp->rx_info->entry = qp->rx_index; 945 946 qp->rx_index++; 947 qp->rx_index %= qp->rx_max_entry; 948 949 return (0); 950 } 951 952 static void 953 ntb_rx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry, 954 void *offset) 955 { 956 struct ifnet *ifp = entry->cb_data; 957 unsigned int len = entry->len; 958 struct mbuf *m; 959 960 CTR2(KTR_NTB, "RX: copying %d bytes from offset %p", len, offset); 961 m = m_devget(offset, len, 0, ifp, NULL); 962 m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID; 963 964 entry->buf = (void *)m; 965 966 CTR2(KTR_NTB, 967 "RX: copied entry %p to mbuf %p. Adding entry to rx_free_q", entry, 968 m); 969 ntb_list_add(&qp->ntb_rx_free_q_lock, entry, &qp->rx_free_q); 970 971 taskqueue_enqueue(taskqueue_swi, &qp->rx_completion_task); 972 } 973 974 static void 975 ntb_rx_completion_task(void *arg, int pending) 976 { 977 struct ntb_transport_qp *qp = arg; 978 struct mbuf *m; 979 struct ntb_queue_entry *entry; 980 981 CTR0(KTR_NTB, "RX: rx_completion_task"); 982 983 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) { 984 m = entry->buf; 985 CTR2(KTR_NTB, "RX: completing entry %p, mbuf %p", entry, m); 986 if (qp->rx_handler && qp->client_ready == NTB_LINK_UP) 987 qp->rx_handler(qp, qp->cb_data, m, entry->len); 988 989 entry->buf = NULL; 990 entry->len = qp->transport->bufsize; 991 992 CTR1(KTR_NTB,"RX: entry %p removed from rx_free_q " 993 "and added to rx_pend_q", entry); 994 ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); 995 if (qp->rx_err_no_buf > qp->last_rx_no_buf) { 996 qp->last_rx_no_buf = qp->rx_err_no_buf; 997 CTR0(KTR_NTB, "RX: could spawn rx task"); 998 callout_reset(&qp->rx_full, hz / 1000, ntb_rx_pendq_full, 999 qp); 1000 } 1001 } 1002 } 1003 1004 /* Link Event handler */ 1005 static void 1006 ntb_transport_event_callback(void *data, enum ntb_hw_event event) 1007 { 1008 struct ntb_netdev *nt = data; 1009 1010 switch (event) { 1011 case NTB_EVENT_HW_LINK_UP: 1012 if (bootverbose) 1013 device_printf(ntb_get_device(nt->ntb), "HW link up\n"); 1014 callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt); 1015 break; 1016 case NTB_EVENT_HW_LINK_DOWN: 1017 if (bootverbose) 1018 device_printf(ntb_get_device(nt->ntb), "HW link down\n"); 1019 ntb_transport_link_cleanup(nt); 1020 break; 1021 default: 1022 panic("ntb: Unknown NTB event"); 1023 } 1024 } 1025 1026 /* Link bring up */ 1027 static void 1028 ntb_transport_link_work(void *arg) 1029 { 1030 struct ntb_netdev *nt = arg; 1031 struct ntb_softc *ntb = nt->ntb; 1032 struct ntb_transport_qp *qp; 1033 uint64_t val64; 1034 uint32_t val; 1035 int rc, i; 1036 1037 /* send the local info, in the opposite order of the way we read it */ 1038 for (i = 0; i < NTB_NUM_MW; i++) { 1039 rc = ntb_write_remote_spad(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2), 1040 ntb_get_mw_size(ntb, i) >> 32); 1041 if (rc != 0) 1042 goto out; 1043 1044 rc = ntb_write_remote_spad(ntb, IF_NTB_MW0_SZ_LOW + (i * 2), 1045 (uint32_t)ntb_get_mw_size(ntb, i)); 1046 if (rc != 0) 1047 goto out; 1048 } 1049 1050 rc = ntb_write_remote_spad(ntb, IF_NTB_NUM_MWS, NTB_NUM_MW); 1051 if (rc != 0) 1052 goto out; 1053 1054 rc = ntb_write_remote_spad(ntb, IF_NTB_NUM_QPS, nt->max_qps); 1055 if (rc != 0) 1056 goto out; 1057 1058 rc = ntb_write_remote_spad(ntb, IF_NTB_VERSION, NTB_TRANSPORT_VERSION); 1059 if (rc != 0) 1060 goto out; 1061 1062 /* Query the remote side for its info */ 1063 rc = ntb_read_local_spad(ntb, IF_NTB_VERSION, &val); 1064 if (rc != 0) 1065 goto out; 1066 1067 if (val != NTB_TRANSPORT_VERSION) 1068 goto out; 1069 1070 rc = ntb_read_local_spad(ntb, IF_NTB_NUM_QPS, &val); 1071 if (rc != 0) 1072 goto out; 1073 1074 if (val != nt->max_qps) 1075 goto out; 1076 1077 rc = ntb_read_local_spad(ntb, IF_NTB_NUM_MWS, &val); 1078 if (rc != 0) 1079 goto out; 1080 1081 if (val != NTB_NUM_MW) 1082 goto out; 1083 1084 for (i = 0; i < NTB_NUM_MW; i++) { 1085 rc = ntb_read_local_spad(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2), 1086 &val); 1087 if (rc != 0) 1088 goto free_mws; 1089 1090 val64 = (uint64_t)val << 32; 1091 1092 rc = ntb_read_local_spad(ntb, IF_NTB_MW0_SZ_LOW + (i * 2), 1093 &val); 1094 if (rc != 0) 1095 goto free_mws; 1096 1097 val64 |= val; 1098 1099 rc = ntb_set_mw(nt, i, val64); 1100 if (rc != 0) 1101 goto free_mws; 1102 } 1103 1104 nt->transport_link = NTB_LINK_UP; 1105 if (bootverbose) 1106 device_printf(ntb_get_device(ntb), "transport link up\n"); 1107 1108 for (i = 0; i < nt->max_qps; i++) { 1109 qp = &nt->qps[i]; 1110 1111 ntb_transport_setup_qp_mw(nt, i); 1112 1113 if (qp->client_ready == NTB_LINK_UP) 1114 callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp); 1115 } 1116 1117 return; 1118 1119 free_mws: 1120 for (i = 0; i < NTB_NUM_MW; i++) 1121 ntb_free_mw(nt, i); 1122 out: 1123 if (ntb_query_link_status(ntb)) 1124 callout_reset(&nt->link_work, 1125 NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_transport_link_work, nt); 1126 } 1127 1128 static int 1129 ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size) 1130 { 1131 struct ntb_transport_mw *mw = &nt->mw[num_mw]; 1132 1133 /* No need to re-setup */ 1134 if (mw->size == size) 1135 return (0); 1136 1137 if (mw->size != 0) 1138 ntb_free_mw(nt, num_mw); 1139 1140 /* Alloc memory for receiving data. Must be 4k aligned */ 1141 mw->size = size; 1142 1143 mw->virt_addr = contigmalloc(mw->size, M_NTB_IF, M_ZERO, 0, 1144 BUS_SPACE_MAXADDR, mw->size, 0); 1145 if (mw->virt_addr == NULL) { 1146 mw->size = 0; 1147 printf("ntb: Unable to allocate MW buffer of size %d\n", 1148 (int)mw->size); 1149 return (ENOMEM); 1150 } 1151 /* TODO: replace with bus_space_* functions */ 1152 mw->dma_addr = vtophys(mw->virt_addr); 1153 1154 /* Notify HW the memory location of the receive buffer */ 1155 ntb_set_mw_addr(nt->ntb, num_mw, mw->dma_addr); 1156 1157 return (0); 1158 } 1159 1160 static void 1161 ntb_free_mw(struct ntb_netdev *nt, int num_mw) 1162 { 1163 struct ntb_transport_mw *mw = &nt->mw[num_mw]; 1164 1165 if (mw->virt_addr == NULL) 1166 return; 1167 1168 contigfree(mw->virt_addr, mw->size, M_NTB_IF); 1169 mw->virt_addr = NULL; 1170 } 1171 1172 static void 1173 ntb_transport_setup_qp_mw(struct ntb_netdev *nt, unsigned int qp_num) 1174 { 1175 struct ntb_transport_qp *qp = &nt->qps[qp_num]; 1176 void *offset; 1177 unsigned int rx_size, num_qps_mw; 1178 uint8_t mw_num = QP_TO_MW(qp_num); 1179 unsigned int i; 1180 1181 if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW) 1182 num_qps_mw = nt->max_qps / NTB_NUM_MW + 1; 1183 else 1184 num_qps_mw = nt->max_qps / NTB_NUM_MW; 1185 1186 rx_size = (unsigned int) nt->mw[mw_num].size / num_qps_mw; 1187 qp->remote_rx_info = (void *)((uint8_t *)nt->mw[mw_num].virt_addr + 1188 (qp_num / NTB_NUM_MW * rx_size)); 1189 rx_size -= sizeof(struct ntb_rx_info); 1190 1191 qp->rx_buff = qp->remote_rx_info + 1; 1192 /* Due to house-keeping, there must be at least 2 buffs */ 1193 qp->rx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header), 1194 rx_size / 2); 1195 qp->rx_max_entry = rx_size / qp->rx_max_frame; 1196 qp->rx_index = 0; 1197 1198 qp->remote_rx_info->entry = qp->rx_max_entry - 1; 1199 1200 /* setup the hdr offsets with 0's */ 1201 for (i = 0; i < qp->rx_max_entry; i++) { 1202 offset = (void *)((uint8_t *)qp->rx_buff + 1203 qp->rx_max_frame * (i + 1) - 1204 sizeof(struct ntb_payload_header)); 1205 memset(offset, 0, sizeof(struct ntb_payload_header)); 1206 } 1207 1208 qp->rx_pkts = 0; 1209 qp->tx_pkts = 0; 1210 qp->tx_index = 0; 1211 } 1212 1213 static void 1214 ntb_qp_link_work(void *arg) 1215 { 1216 struct ntb_transport_qp *qp = arg; 1217 struct ntb_softc *ntb = qp->ntb; 1218 struct ntb_netdev *nt = qp->transport; 1219 int rc, val; 1220 1221 1222 rc = ntb_read_remote_spad(ntb, IF_NTB_QP_LINKS, &val); 1223 if (rc != 0) 1224 return; 1225 1226 rc = ntb_write_remote_spad(ntb, IF_NTB_QP_LINKS, val | 1 << qp->qp_num); 1227 1228 /* query remote spad for qp ready bits */ 1229 rc = ntb_read_local_spad(ntb, IF_NTB_QP_LINKS, &val); 1230 1231 /* See if the remote side is up */ 1232 if ((1 << qp->qp_num & val) != 0) { 1233 qp->qp_link = NTB_LINK_UP; 1234 if (qp->event_handler != NULL) 1235 qp->event_handler(qp->cb_data, NTB_LINK_UP); 1236 if (bootverbose) 1237 device_printf(ntb_get_device(ntb), "qp link up\n"); 1238 } else if (nt->transport_link == NTB_LINK_UP) { 1239 callout_reset(&qp->link_work, 1240 NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp); 1241 } 1242 } 1243 1244 /* Link down event*/ 1245 static void 1246 ntb_transport_link_cleanup(struct ntb_netdev *nt) 1247 { 1248 int i; 1249 1250 if (nt->transport_link == NTB_LINK_DOWN) 1251 callout_drain(&nt->link_work); 1252 else 1253 nt->transport_link = NTB_LINK_DOWN; 1254 1255 /* Pass along the info to any clients */ 1256 for (i = 0; i < nt->max_qps; i++) 1257 if (!test_bit(i, &nt->qp_bitmap)) 1258 ntb_qp_link_down(&nt->qps[i]); 1259 1260 /* 1261 * The scratchpad registers keep the values if the remote side 1262 * goes down, blast them now to give them a sane value the next 1263 * time they are accessed 1264 */ 1265 for (i = 0; i < IF_NTB_MAX_SPAD; i++) 1266 ntb_write_local_spad(nt->ntb, i, 0); 1267 } 1268 1269 1270 static void 1271 ntb_qp_link_down(struct ntb_transport_qp *qp) 1272 { 1273 1274 ntb_qp_link_cleanup(qp); 1275 } 1276 1277 static void 1278 ntb_qp_link_cleanup(struct ntb_transport_qp *qp) 1279 { 1280 struct ntb_netdev *nt = qp->transport; 1281 1282 if (qp->qp_link == NTB_LINK_DOWN) { 1283 callout_drain(&qp->link_work); 1284 return; 1285 } 1286 1287 if (qp->event_handler != NULL) 1288 qp->event_handler(qp->cb_data, NTB_LINK_DOWN); 1289 1290 qp->qp_link = NTB_LINK_DOWN; 1291 1292 if (nt->transport_link == NTB_LINK_UP) 1293 callout_reset(&qp->link_work, 1294 NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp); 1295 } 1296 1297 /* Link commanded down */ 1298 /** 1299 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data 1300 * @qp: NTB transport layer queue to be disabled 1301 * 1302 * Notify NTB transport layer of client's desire to no longer receive data on 1303 * transport queue specified. It is the client's responsibility to ensure all 1304 * entries on queue are purged or otherwise handled appropraitely. 1305 */ 1306 static void 1307 ntb_transport_link_down(struct ntb_transport_qp *qp) 1308 { 1309 int rc, val; 1310 1311 if (qp == NULL) 1312 return; 1313 1314 qp->client_ready = NTB_LINK_DOWN; 1315 1316 rc = ntb_read_remote_spad(qp->ntb, IF_NTB_QP_LINKS, &val); 1317 if (rc != 0) 1318 return; 1319 1320 rc = ntb_write_remote_spad(qp->ntb, IF_NTB_QP_LINKS, 1321 val & ~(1 << qp->qp_num)); 1322 1323 if (qp->qp_link == NTB_LINK_UP) 1324 ntb_send_link_down(qp); 1325 else 1326 callout_drain(&qp->link_work); 1327 1328 } 1329 1330 static void 1331 ntb_send_link_down(struct ntb_transport_qp *qp) 1332 { 1333 struct ntb_queue_entry *entry; 1334 int i, rc; 1335 1336 if (qp->qp_link == NTB_LINK_DOWN) 1337 return; 1338 1339 qp->qp_link = NTB_LINK_DOWN; 1340 1341 for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { 1342 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1343 if (entry != NULL) 1344 break; 1345 pause("NTB Wait for link down", hz / 10); 1346 } 1347 1348 if (entry == NULL) 1349 return; 1350 1351 entry->cb_data = NULL; 1352 entry->buf = NULL; 1353 entry->len = 0; 1354 entry->flags = IF_NTB_LINK_DOWN_FLAG; 1355 1356 mtx_lock(&qp->transport->tx_lock); 1357 rc = ntb_process_tx(qp, entry); 1358 if (rc != 0) 1359 printf("ntb: Failed to send link down\n"); 1360 mtx_unlock(&qp->transport->tx_lock); 1361 } 1362 1363 1364 /* List Management */ 1365 1366 static void 1367 ntb_list_add(struct mtx *lock, struct ntb_queue_entry *entry, 1368 struct ntb_queue_list *list) 1369 { 1370 1371 mtx_lock_spin(lock); 1372 STAILQ_INSERT_TAIL(list, entry, entry); 1373 mtx_unlock_spin(lock); 1374 } 1375 1376 static struct ntb_queue_entry * 1377 ntb_list_rm(struct mtx *lock, struct ntb_queue_list *list) 1378 { 1379 struct ntb_queue_entry *entry; 1380 1381 mtx_lock_spin(lock); 1382 if (STAILQ_EMPTY(list)) { 1383 entry = NULL; 1384 goto out; 1385 } 1386 entry = STAILQ_FIRST(list); 1387 STAILQ_REMOVE_HEAD(list, entry); 1388 out: 1389 mtx_unlock_spin(lock); 1390 1391 return (entry); 1392 } 1393 1394 /* Helper functions */ 1395 /* TODO: This too should really be part of the kernel */ 1396 #define EUI48_MULTICAST 1 << 0 1397 #define EUI48_LOCALLY_ADMINISTERED 1 << 1 1398 static void 1399 create_random_local_eui48(u_char *eaddr) 1400 { 1401 static uint8_t counter = 0; 1402 uint32_t seed = ticks; 1403 1404 eaddr[0] = EUI48_LOCALLY_ADMINISTERED; 1405 memcpy(&eaddr[1], &seed, sizeof(uint32_t)); 1406 eaddr[5] = counter++; 1407 } 1408 1409 /** 1410 * ntb_transport_max_size - Query the max payload size of a qp 1411 * @qp: NTB transport layer queue to be queried 1412 * 1413 * Query the maximum payload size permissible on the given qp 1414 * 1415 * RETURNS: the max payload size of a qp 1416 */ 1417 static unsigned int 1418 ntb_transport_max_size(struct ntb_transport_qp *qp) 1419 { 1420 1421 if (qp == NULL) 1422 return (0); 1423 1424 return (qp->tx_max_frame - sizeof(struct ntb_payload_header)); 1425 } 1426