1 /*-
2  * Copyright (c) 2016-2017 Alexander Motin <mav@FreeBSD.org>
30  * The Non-Transparent Bridge (NTB) is a device that allows you to connect
31  * two or more systems using a PCI-e links, providing remote memory access.
75     "ntb_transport log level -- higher is more verbose");
86     "If enabled (non-zero), limit the size of large memory windows. "
91     &enable_xeon_watchdog, 0, "If non-zero, write a register every second to "
125 	uint8_t			qp_num;	/* Only 64 QPs are allowed.  0-63 */
233 	 * to keep the protocol in lock-step with the Linux NTB driver.
248 	 * Some NTB-using hardware have a watchdog to work around NTB hangs; if
265 #define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)
328 	ntb_spad_write(nt->dev, NTBT_WATCHDOG_SPAD, 0);  in xeon_link_watchdog_hb()
329 	callout_reset(&nt->link_watchdog, 1 * hz, xeon_link_watchdog_hb, nt);  in xeon_link_watchdog_hb()
344 	struct ntb_transport_child **cpp = &nt->child;  in ntb_transport_attach()
353 	nt->dev = dev;  in ntb_transport_attach()
354 	nt->mw_count = ntb_mw_count(dev);  in ntb_transport_attach()
358 	KASSERT(db_bitmap == ((uint64_t)1 << db_count) - 1,  in ntb_transport_attach()
361 	if (nt->mw_count == 0) {  in ntb_transport_attach()
365 	nt->compact = (spad_count < 4 + 2 * nt->mw_count);  in ntb_transport_attach()
368 	TUNABLE_INT_FETCH(buf, &nt->compact);  in ntb_transport_attach()
369 	if (nt->compact) {  in ntb_transport_attach()
374 		if (spad_count < 2 + nt->mw_count) {  in ntb_transport_attach()
375 			nt->mw_count = spad_count - 2;  in ntb_transport_attach()
377 			    "memory windows.\n", nt->mw_count);  in ntb_transport_attach()
384 		if (spad_count < 4 + 2 * nt->mw_count) {  in ntb_transport_attach()
385 			nt->mw_count = (spad_count - 4) / 2;  in ntb_transport_attach()
387 			    "memory windows.\n", nt->mw_count);  in ntb_transport_attach()
395 	nt->mw_vec = malloc(nt->mw_count * sizeof(*nt->mw_vec), M_NTB_T,  in ntb_transport_attach()
397 	for (i = 0; i < nt->mw_count; i++) {  in ntb_transport_attach()
398 		mw = &nt->mw_vec[i];  in ntb_transport_attach()
400 		rc = ntb_mw_get_range(dev, i, &mw->phys_addr, &mw->vbase,  in ntb_transport_attach()
401 		    &mw->phys_size, &mw->xlat_align, &mw->xlat_align_size,  in ntb_transport_attach()
402 		    &mw->addr_limit);  in ntb_transport_attach()
406 		mw->tx_size = mw->phys_size;  in ntb_transport_attach()
407 		if (max_mw_size != 0 && mw->tx_size > max_mw_size) {  in ntb_transport_attach()
409 			    "%ju to %ju\n", i, (uintmax_t)mw->tx_size,  in ntb_transport_attach()
411 			mw->tx_size = max_mw_size;  in ntb_transport_attach()
413 		if (nt->compact && mw->tx_size > UINT32_MAX) {  in ntb_transport_attach()
415 			    "(%ju)\n", i, (uintmax_t)mw->tx_size);  in ntb_transport_attach()
420 		mw->rx_size = 0;  in ntb_transport_attach()
421 		mw->buff_size = 0;  in ntb_transport_attach()
422 		mw->virt_addr = NULL;  in ntb_transport_attach()
423 		mw->dma_addr = 0;  in ntb_transport_attach()
435 		ntb_set_mw(nt, i, mw->tx_size);  in ntb_transport_attach()
439 	qpo = imin(db_count, nt->mw_count);  in ntb_transport_attach()
452 		qp = (np && np[0] != 0) ? strtol(np, NULL, 10) : qpo - qpu;  in ntb_transport_attach()
456 		if (qp > qpt - qpu) {  in ntb_transport_attach()
462 		nc->consumer = i;  in ntb_transport_attach()
463 		nc->qpoff = qpu;  in ntb_transport_attach()
464 		nc->qpcnt = qp;  in ntb_transport_attach()
465 		nc->dev = device_add_child(dev, name, DEVICE_UNIT_ANY);  in ntb_transport_attach()
466 		if (nc->dev == NULL) {  in ntb_transport_attach()
470 		device_set_ivars(nc->dev, nc);  in ntb_transport_attach()
472 		cpp = &nc->next;  in ntb_transport_attach()
478 				printf("-%d", qpu + qp - 1);  in ntb_transport_attach()
485 	nt->qp_count = qpu;  in ntb_transport_attach()
487 	nt->qp_vec = malloc(nt->qp_count * sizeof(*nt->qp_vec), M_NTB_T,  in ntb_transport_attach()
490 	for (i = 0; i < nt->qp_count; i++)  in ntb_transport_attach()
493 	callout_init(&nt->link_work, 1);  in ntb_transport_attach()
494 	callout_init(&nt->link_watchdog, 1);  in ntb_transport_attach()
495 	TASK_INIT(&nt->link_cleanup, 0, ntb_transport_link_cleanup_work, nt);  in ntb_transport_attach()
496 	nt->link_is_up = false;  in ntb_transport_attach()
504 	for (i = 0; i < nt->mw_count; i++) {  in ntb_transport_attach()
505 		mw = &nt->mw_vec[i];  in ntb_transport_attach()
506 		rc = ntb_mw_set_trans(nt->dev, i, mw->dma_addr, mw->buff_size);  in ntb_transport_attach()
512 		callout_reset(&nt->link_watchdog, 0, xeon_link_watchdog_hb, nt);  in ntb_transport_attach()
518 	free(nt->qp_vec, M_NTB_T);  in ntb_transport_attach()
519 	free(nt->mw_vec, M_NTB_T);  in ntb_transport_attach()
527 	struct ntb_transport_child **cpp = &nt->child;  in ntb_transport_detach()
532 		*cpp = (*cpp)->next;  in ntb_transport_detach()
533 		error = device_delete_child(dev, nc->dev);  in ntb_transport_detach()
538 	KASSERT(nt->qp_bitmap == 0,  in ntb_transport_detach()
539 	    ("Some queues not freed on detach (%jx)", nt->qp_bitmap));  in ntb_transport_detach()
542 	taskqueue_drain(taskqueue_swi, &nt->link_cleanup);  in ntb_transport_detach()
543 	callout_drain(&nt->link_work);  in ntb_transport_detach()
544 	callout_drain(&nt->link_watchdog);  in ntb_transport_detach()
549 	for (i = 0; i < nt->mw_count; i++)  in ntb_transport_detach()
552 	free(nt->qp_vec, M_NTB_T);  in ntb_transport_detach()
553 	free(nt->mw_vec, M_NTB_T);  in ntb_transport_detach()
564 	if (nc->qpcnt > 0) {  in ntb_transport_print_child()
565 		printf(" queue %d", nc->qpoff);  in ntb_transport_print_child()
566 		if (nc->qpcnt > 1)  in ntb_transport_print_child()
567 			printf("-%d", nc->qpoff + nc->qpcnt - 1);  in ntb_transport_print_child()
569 	retval += printf(" at consumer %d", nc->consumer);  in ntb_transport_print_child()
581 	sbuf_printf(sb, "consumer=%d", nc->consumer);  in ntb_transport_child_location()
590 	return (nc->qpcnt);  in ntb_transport_queue_count()
603 	mw_count = nt->mw_count;  in ntb_transport_init_queue()
605 	mw = &nt->mw_vec[mw_num];  in ntb_transport_init_queue()
607 	qp = &nt->qp_vec[qp_num];  in ntb_transport_init_queue()
608 	qp->qp_num = qp_num;  in ntb_transport_init_queue()
609 	qp->transport = nt;  in ntb_transport_init_queue()
610 	qp->dev = nt->dev;  in ntb_transport_init_queue()
611 	qp->client_ready = false;  in ntb_transport_init_queue()
612 	qp->event_handler = NULL;  in ntb_transport_init_queue()
615 	if (mw_num < nt->qp_count % mw_count)  in ntb_transport_init_queue()
616 		num_qps_mw = nt->qp_count / mw_count + 1;  in ntb_transport_init_queue()
618 		num_qps_mw = nt->qp_count / mw_count;  in ntb_transport_init_queue()
620 	mw_base = mw->phys_addr;  in ntb_transport_init_queue()
622 	tx_size = mw->tx_size / num_qps_mw;  in ntb_transport_init_queue()
625 	qp->tx_mw = mw->vbase + qp_offset;  in ntb_transport_init_queue()
626 	KASSERT(qp->tx_mw != NULL, ("uh oh?"));  in ntb_transport_init_queue()
629 	qp->tx_mw_phys = mw_base + qp_offset;  in ntb_transport_init_queue()
630 	KASSERT(qp->tx_mw_phys != 0, ("uh oh?"));  in ntb_transport_init_queue()
632 	tx_size -= sizeof(struct ntb_rx_info);  in ntb_transport_init_queue()
633 	qp->rx_info = (void *)(qp->tx_mw + tx_size);  in ntb_transport_init_queue()
635 	/* Due to house-keeping, there must be at least 2 buffs */  in ntb_transport_init_queue()
636 	qp->tx_max_frame = qmin(transport_mtu, tx_size / 2);  in ntb_transport_init_queue()
637 	qp->tx_max_entry = tx_size / qp->tx_max_frame;  in ntb_transport_init_queue()
639 	callout_init(&qp->link_work, 1);  in ntb_transport_init_queue()
640 	callout_init(&qp->rx_full, 1);  in ntb_transport_init_queue()
642 	mtx_init(&qp->ntb_rx_q_lock, "ntb rx q", NULL, MTX_SPIN);  in ntb_transport_init_queue()
643 	mtx_init(&qp->ntb_tx_free_q_lock, "ntb tx free q", NULL, MTX_SPIN);  in ntb_transport_init_queue()
644 	mtx_init(&qp->tx_lock, "ntb transport tx", NULL, MTX_DEF);  in ntb_transport_init_queue()
645 	TASK_INIT(&qp->rxc_db_work, 0, ntb_transport_rxc_db, qp);  in ntb_transport_init_queue()
646 	qp->rxc_tq = taskqueue_create("ntbt_rx", M_WAITOK,  in ntb_transport_init_queue()
647 	    taskqueue_thread_enqueue, &qp->rxc_tq);  in ntb_transport_init_queue()
648 	taskqueue_start_threads(&qp->rxc_tq, 1, PI_NET, "%s rx%d",  in ntb_transport_init_queue()
649 	    device_get_nameunit(nt->dev), qp_num);  in ntb_transport_init_queue()
651 	STAILQ_INIT(&qp->rx_post_q);  in ntb_transport_init_queue()
652 	STAILQ_INIT(&qp->rx_pend_q);  in ntb_transport_init_queue()
653 	STAILQ_INIT(&qp->tx_free_q);  in ntb_transport_init_queue()
659 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_transport_free_queue()
662 	callout_drain(&qp->link_work);  in ntb_transport_free_queue()
664 	ntb_db_set_mask(qp->dev, 1ull << qp->qp_num);  in ntb_transport_free_queue()
665 	taskqueue_drain_all(qp->rxc_tq);  in ntb_transport_free_queue()
666 	taskqueue_free(qp->rxc_tq);  in ntb_transport_free_queue()
668 	qp->cb_data = NULL;  in ntb_transport_free_queue()
669 	qp->rx_handler = NULL;  in ntb_transport_free_queue()
670 	qp->tx_handler = NULL;  in ntb_transport_free_queue()
671 	qp->event_handler = NULL;  in ntb_transport_free_queue()
673 	while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q)))  in ntb_transport_free_queue()
676 	while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q)))  in ntb_transport_free_queue()
679 	while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))  in ntb_transport_free_queue()
682 	nt->qp_bitmap &= ~(1 << qp->qp_num);  in ntb_transport_free_queue()
686  * ntb_transport_create_queue - Create a new NTB transport layer queue
709 	if (q < 0 || q >= nc->qpcnt)  in ntb_transport_create_queue()
712 	qp = &nt->qp_vec[nc->qpoff + q];  in ntb_transport_create_queue()
713 	nt->qp_bitmap |= (1 << qp->qp_num);  in ntb_transport_create_queue()
714 	qp->cb_data = data;  in ntb_transport_create_queue()
715 	qp->rx_handler = handlers->rx_handler;  in ntb_transport_create_queue()
716 	qp->tx_handler = handlers->tx_handler;  in ntb_transport_create_queue()
717 	qp->event_handler = handlers->event_handler;  in ntb_transport_create_queue()
721 		entry->cb_data = data;  in ntb_transport_create_queue()
722 		entry->buf = NULL;  in ntb_transport_create_queue()
723 		entry->len = transport_mtu;  in ntb_transport_create_queue()
724 		entry->qp = qp;  in ntb_transport_create_queue()
725 		ntb_list_add(&qp->ntb_rx_q_lock, entry, &qp->rx_pend_q);  in ntb_transport_create_queue()
730 		entry->qp = qp;  in ntb_transport_create_queue()
731 		ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);  in ntb_transport_create_queue()
734 	ntb_db_clear(dev, 1ull << qp->qp_num);  in ntb_transport_create_queue()
739  * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
747 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_transport_link_up()
749 	qp->client_ready = true;  in ntb_transport_link_up()
751 	ntb_printf(2, "qp %d client ready\n", qp->qp_num);  in ntb_transport_link_up()
753 	if (nt->link_is_up)  in ntb_transport_link_up()
754 		callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp);  in ntb_transport_link_up()
760  * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry
779 	if (!qp->link_is_up || len == 0) {  in ntb_transport_tx_enqueue()
784 	entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);  in ntb_transport_tx_enqueue()
787 		qp->tx_err_no_buf++;  in ntb_transport_tx_enqueue()
792 	entry->cb_data = cb;  in ntb_transport_tx_enqueue()
793 	entry->buf = data;  in ntb_transport_tx_enqueue()
794 	entry->len = len;  in ntb_transport_tx_enqueue()
795 	entry->flags = 0;  in ntb_transport_tx_enqueue()
797 	mtx_lock(&qp->tx_lock);  in ntb_transport_tx_enqueue()
799 	mtx_unlock(&qp->tx_lock);  in ntb_transport_tx_enqueue()
801 		ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);  in ntb_transport_tx_enqueue()
813 	struct ntb_transport_qp *qp = entry->qp;  in ntb_tx_copy_callback()
814 	struct ntb_payload_header *hdr = entry->x_hdr;  in ntb_tx_copy_callback()
816 	iowrite32(entry->flags | NTBT_DESC_DONE_FLAG, &hdr->flags);  in ntb_tx_copy_callback()
819 	ntb_peer_db_set(qp->dev, 1ull << qp->qp_num);  in ntb_tx_copy_callback()
826 	if (entry->len > 0) {  in ntb_tx_copy_callback()
827 		qp->tx_bytes += entry->len;  in ntb_tx_copy_callback()
829 		if (qp->tx_handler)  in ntb_tx_copy_callback()
830 			qp->tx_handler(qp, qp->cb_data, entry->buf,  in ntb_tx_copy_callback()
831 			    entry->len);  in ntb_tx_copy_callback()
833 			m_freem(entry->buf);  in ntb_tx_copy_callback()
834 		entry->buf = NULL;  in ntb_tx_copy_callback()
838 	    "TX: entry %p sent. hdr->ver = %u, hdr->flags = 0x%x, Returning "  in ntb_tx_copy_callback()
839 	    "to tx_free_q", entry, hdr->ver, hdr->flags);  in ntb_tx_copy_callback()
840 	ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);  in ntb_tx_copy_callback()
847 	CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset);  in ntb_memcpy_tx()
848 	if (entry->buf != NULL) {  in ntb_memcpy_tx()
849 		m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset);  in ntb_memcpy_tx()
867 	offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index;  in ntb_async_tx()
868 	hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame -  in ntb_async_tx()
870 	entry->x_hdr = hdr;  in ntb_async_tx()
872 	iowrite32(entry->len, &hdr->len);  in ntb_async_tx()
873 	iowrite32(qp->tx_pkts, &hdr->ver);  in ntb_async_tx()
884 	    qp->tx_pkts, qp->tx_index, qp->remote_rx_info->entry);  in ntb_process_tx()
885 	if (qp->tx_index == qp->remote_rx_info->entry) {  in ntb_process_tx()
887 		qp->tx_ring_full++;  in ntb_process_tx()
891 	if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) {  in ntb_process_tx()
892 		if (qp->tx_handler != NULL)  in ntb_process_tx()
893 			qp->tx_handler(qp, qp->cb_data, entry->buf,  in ntb_process_tx()
896 			m_freem(entry->buf);  in ntb_process_tx()
898 		entry->buf = NULL;  in ntb_process_tx()
899 		ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);  in ntb_process_tx()
905 	CTR2(KTR_NTB, "TX: copying entry %p to index %u", entry, qp->tx_index);  in ntb_process_tx()
908 	qp->tx_index++;  in ntb_process_tx()
909 	qp->tx_index %= qp->tx_max_entry;  in ntb_process_tx()
911 	qp->tx_pkts++;  in ntb_process_tx()
921 	uint64_t qp_mask = 1ull << qp->qp_num;  in ntb_transport_rxc_db()
930 	if ((ntb_db_read(qp->dev) & qp_mask) != 0) {  in ntb_transport_rxc_db()
932 		ntb_db_clear(qp->dev, qp_mask);  in ntb_transport_rxc_db()
935 	if (qp->link_is_up)  in ntb_transport_rxc_db()
936 		ntb_db_clear_mask(qp->dev, qp_mask);  in ntb_transport_rxc_db()
946 	offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index;  in ntb_process_rxc()
947 	hdr = (void *)(offset + qp->rx_max_frame -  in ntb_process_rxc()
950 	CTR1(KTR_NTB, "RX: process_rxc rx_index = %u", qp->rx_index);  in ntb_process_rxc()
951 	if ((hdr->flags & NTBT_DESC_DONE_FLAG) == 0) {  in ntb_process_rxc()
953 		qp->rx_ring_empty++;  in ntb_process_rxc()
957 	if ((hdr->flags & NTBT_LINK_DOWN_FLAG) != 0) {  in ntb_process_rxc()
960 		hdr->flags = 0;  in ntb_process_rxc()
964 	if (hdr->ver != (uint32_t)qp->rx_pkts) {  in ntb_process_rxc()
966 		    "Returning entry to rx_pend_q", hdr->ver, qp->rx_pkts);  in ntb_process_rxc()
967 		qp->rx_err_ver++;  in ntb_process_rxc()
971 	entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q);  in ntb_process_rxc()
973 		qp->rx_err_no_buf++;  in ntb_process_rxc()
977 	callout_stop(&qp->rx_full);  in ntb_process_rxc()
980 	entry->x_hdr = hdr;  in ntb_process_rxc()
981 	entry->index = qp->rx_index;  in ntb_process_rxc()
983 	if (hdr->len > entry->len) {  in ntb_process_rxc()
985 		    (uintmax_t)hdr->len, (uintmax_t)entry->len);  in ntb_process_rxc()
986 		qp->rx_err_oflow++;  in ntb_process_rxc()
988 		entry->len = -EIO;  in ntb_process_rxc()
989 		entry->flags |= NTBT_DESC_DONE_FLAG;  in ntb_process_rxc()
993 		qp->rx_bytes += hdr->len;  in ntb_process_rxc()
994 		qp->rx_pkts++;  in ntb_process_rxc()
996 		CTR1(KTR_NTB, "RX: received %ld rx_pkts", qp->rx_pkts);  in ntb_process_rxc()
998 		entry->len = hdr->len;  in ntb_process_rxc()
1003 	qp->rx_index++;  in ntb_process_rxc()
1004 	qp->rx_index %= qp->rx_max_entry;  in ntb_process_rxc()
1012 	struct ifnet *ifp = entry->cb_data;  in ntb_memcpy_rx()
1013 	unsigned int len = entry->len;  in ntb_memcpy_rx()
1017 	entry->buf = (void *)m_devget(offset, len, 0, ifp, NULL);  in ntb_memcpy_rx()
1018 	if (entry->buf == NULL)  in ntb_memcpy_rx()
1019 		entry->len = -ENOMEM;  in ntb_memcpy_rx()
1024 	CTR2(KTR_NTB, "RX: copied entry %p to mbuf %p.", entry, entry->buf);  in ntb_memcpy_rx()
1034 	entry->flags |= NTBT_DESC_DONE_FLAG;  in ntb_rx_copy_callback()
1047 	mtx_lock_spin(&qp->ntb_rx_q_lock);  in ntb_complete_rxc()
1049 	while (!STAILQ_EMPTY(&qp->rx_post_q)) {  in ntb_complete_rxc()
1050 		entry = STAILQ_FIRST(&qp->rx_post_q);  in ntb_complete_rxc()
1051 		if ((entry->flags & NTBT_DESC_DONE_FLAG) == 0)  in ntb_complete_rxc()
1054 		entry->x_hdr->flags = 0;  in ntb_complete_rxc()
1055 		iowrite32(entry->index, &qp->rx_info->entry);  in ntb_complete_rxc()
1057 		STAILQ_REMOVE_HEAD(&qp->rx_post_q, entry);  in ntb_complete_rxc()
1059 		len = entry->len;  in ntb_complete_rxc()
1060 		m = entry->buf;  in ntb_complete_rxc()
1063 		 * Re-initialize queue_entry for reuse; rx_handler takes  in ntb_complete_rxc()
1066 		entry->buf = NULL;  in ntb_complete_rxc()
1067 		entry->len = transport_mtu;  in ntb_complete_rxc()
1068 		entry->cb_data = qp->cb_data;  in ntb_complete_rxc()
1070 		STAILQ_INSERT_TAIL(&qp->rx_pend_q, entry, entry);  in ntb_complete_rxc()
1072 		mtx_unlock_spin(&qp->ntb_rx_q_lock);  in ntb_complete_rxc()
1075 		if (qp->rx_handler != NULL && qp->client_ready)  in ntb_complete_rxc()
1076 			qp->rx_handler(qp, qp->cb_data, m, len);  in ntb_complete_rxc()
1080 		mtx_lock_spin(&qp->ntb_rx_q_lock);  in ntb_complete_rxc()
1083 	mtx_unlock_spin(&qp->ntb_rx_q_lock);  in ntb_complete_rxc()
1094 	vec_mask = ntb_db_vector_mask(nt->dev, vector);  in ntb_transport_doorbell_callback()
1095 	vec_mask &= nt->qp_bitmap;  in ntb_transport_doorbell_callback()
1096 	if ((vec_mask & (vec_mask - 1)) != 0)  in ntb_transport_doorbell_callback()
1097 		vec_mask &= ntb_db_read(nt->dev);  in ntb_transport_doorbell_callback()
1099 		ntb_db_set_mask(nt->dev, vec_mask);  in ntb_transport_doorbell_callback()
1100 		ntb_db_clear(nt->dev, vec_mask);  in ntb_transport_doorbell_callback()
1103 		qp_num = ffsll(vec_mask) - 1;  in ntb_transport_doorbell_callback()
1105 		qp = &nt->qp_vec[qp_num];  in ntb_transport_doorbell_callback()
1106 		if (qp->link_is_up)  in ntb_transport_doorbell_callback()
1107 			taskqueue_enqueue(qp->rxc_tq, &qp->rxc_db_work);  in ntb_transport_doorbell_callback()
1119 	if (ntb_link_is_up(nt->dev, &nt->link_speed, &nt->link_width)) {  in ntb_transport_event_callback()
1121 		callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt);  in ntb_transport_event_callback()
1124 		taskqueue_enqueue(taskqueue_swi, &nt->link_cleanup);  in ntb_transport_event_callback()
1134 	device_t dev = nt->dev;  in ntb_transport_link_work()
1142 	if (nt->compact) {  in ntb_transport_link_work()
1143 		for (i = 0; i < nt->mw_count; i++) {  in ntb_transport_link_work()
1144 			size = nt->mw_vec[i].tx_size;  in ntb_transport_link_work()
1150 		    (nt->qp_count << 24) | (nt->mw_count << 16) |  in ntb_transport_link_work()
1153 		for (i = 0; i < nt->mw_count; i++) {  in ntb_transport_link_work()
1154 			size = nt->mw_vec[i].tx_size;  in ntb_transport_link_work()
1159 		ntb_peer_spad_write(dev, NTBT_NUM_MWS, nt->mw_count);  in ntb_transport_link_work()
1160 		ntb_peer_spad_write(dev, NTBT_NUM_QPS, nt->qp_count);  in ntb_transport_link_work()
1167 	if (nt->compact) {  in ntb_transport_link_work()
1169 		if (val != ((nt->qp_count << 24) | (nt->mw_count << 16) |  in ntb_transport_link_work()
1178 		if (val != nt->qp_count)  in ntb_transport_link_work()
1182 		if (val != nt->mw_count)  in ntb_transport_link_work()
1186 	for (i = 0; i < nt->mw_count; i++) {  in ntb_transport_link_work()
1187 		if (nt->compact) {  in ntb_transport_link_work()
1198 		mw = &nt->mw_vec[i];  in ntb_transport_link_work()
1199 		mw->rx_size = val64;  in ntb_transport_link_work()
1200 		val64 = roundup(val64, mw->xlat_align_size);  in ntb_transport_link_work()
1201 		if (mw->buff_size != val64) {  in ntb_transport_link_work()
1210 			rc = ntb_mw_set_trans(nt->dev, i, mw->dma_addr,  in ntb_transport_link_work()
1211 			    mw->buff_size);  in ntb_transport_link_work()
1220 	nt->link_is_up = true;  in ntb_transport_link_work()
1223 	for (i = 0; i < nt->qp_count; i++) {  in ntb_transport_link_work()
1224 		qp = &nt->qp_vec[i];  in ntb_transport_link_work()
1228 		if (qp->client_ready)  in ntb_transport_link_work()
1229 			callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp);  in ntb_transport_link_work()
1235 	for (i = 0; i < nt->mw_count; i++)  in ntb_transport_link_work()
1238 	if (ntb_link_is_up(dev, &nt->link_speed, &nt->link_width))  in ntb_transport_link_work()
1239 		callout_reset(&nt->link_work,  in ntb_transport_link_work()
1240 		    NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_transport_link_work, nt);  in ntb_transport_link_work()
1253 	if (!(cba->error = error))  in ntb_load_cb()
1254 		cba->addr = segs[0].ds_addr;  in ntb_load_cb()
1260 	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];  in ntb_set_mw()
1267 	buff_size = roundup(size, mw->xlat_align_size);  in ntb_set_mw()
1269 	/* No need to re-setup */  in ntb_set_mw()
1270 	if (mw->buff_size == buff_size)  in ntb_set_mw()
1273 	if (mw->buff_size != 0)  in ntb_set_mw()
1277 	mw->buff_size = buff_size;  in ntb_set_mw()
1279 	if (bus_dma_tag_create(bus_get_dma_tag(nt->dev), mw->xlat_align, 0,  in ntb_set_mw()
1280 	    mw->addr_limit, BUS_SPACE_MAXADDR,  in ntb_set_mw()
1281 	    NULL, NULL, mw->buff_size, 1, mw->buff_size,  in ntb_set_mw()
1282 	    0, NULL, NULL, &mw->dma_tag)) {  in ntb_set_mw()
1284 		    mw->buff_size);  in ntb_set_mw()
1285 		mw->buff_size = 0;  in ntb_set_mw()
1288 	if (bus_dmamem_alloc(mw->dma_tag, (void **)&mw->virt_addr,  in ntb_set_mw()
1289 	    BUS_DMA_WAITOK | BUS_DMA_ZERO, &mw->dma_map)) {  in ntb_set_mw()
1290 		bus_dma_tag_destroy(mw->dma_tag);  in ntb_set_mw()
1292 		    mw->buff_size);  in ntb_set_mw()
1293 		mw->buff_size = 0;  in ntb_set_mw()
1296 	if (bus_dmamap_load(mw->dma_tag, mw->dma_map, mw->virt_addr,  in ntb_set_mw()
1297 	    mw->buff_size, ntb_load_cb, &cba, BUS_DMA_NOWAIT) || cba.error) {  in ntb_set_mw()
1298 		bus_dmamem_free(mw->dma_tag, mw->virt_addr, mw->dma_map);  in ntb_set_mw()
1299 		bus_dma_tag_destroy(mw->dma_tag);  in ntb_set_mw()
1301 		    mw->buff_size);  in ntb_set_mw()
1302 		mw->buff_size = 0;  in ntb_set_mw()
1305 	mw->dma_addr = cba.addr;  in ntb_set_mw()
1313 	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];  in ntb_free_mw()
1315 	if (mw->virt_addr == NULL)  in ntb_free_mw()
1318 	ntb_mw_clear_trans(nt->dev, num_mw);  in ntb_free_mw()
1319 	bus_dmamap_unload(mw->dma_tag, mw->dma_map);  in ntb_free_mw()
1320 	bus_dmamem_free(mw->dma_tag, mw->virt_addr, mw->dma_map);  in ntb_free_mw()
1321 	bus_dma_tag_destroy(mw->dma_tag);  in ntb_free_mw()
1322 	mw->buff_size = 0;  in ntb_free_mw()
1323 	mw->virt_addr = NULL;  in ntb_free_mw()
1329 	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];  in ntb_transport_setup_qp_mw()
1336 	mw_count = nt->mw_count;  in ntb_transport_setup_qp_mw()
1338 	mw = &nt->mw_vec[mw_num];  in ntb_transport_setup_qp_mw()
1340 	if (mw->virt_addr == NULL)  in ntb_transport_setup_qp_mw()
1343 	if (mw_num < nt->qp_count % mw_count)  in ntb_transport_setup_qp_mw()
1344 		num_qps_mw = nt->qp_count / mw_count + 1;  in ntb_transport_setup_qp_mw()
1346 		num_qps_mw = nt->qp_count / mw_count;  in ntb_transport_setup_qp_mw()
1348 	rx_size = mw->rx_size / num_qps_mw;  in ntb_transport_setup_qp_mw()
1349 	qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count);  in ntb_transport_setup_qp_mw()
1350 	rx_size -= sizeof(struct ntb_rx_info);  in ntb_transport_setup_qp_mw()
1352 	qp->remote_rx_info = (void*)(qp->rx_buff + rx_size);  in ntb_transport_setup_qp_mw()
1354 	/* Due to house-keeping, there must be at least 2 buffs */  in ntb_transport_setup_qp_mw()
1355 	qp->rx_max_frame = qmin(transport_mtu, rx_size / 2);  in ntb_transport_setup_qp_mw()
1356 	qp->rx_max_entry = rx_size / qp->rx_max_frame;  in ntb_transport_setup_qp_mw()
1357 	qp->rx_index = 0;  in ntb_transport_setup_qp_mw()
1359 	qp->remote_rx_info->entry = qp->rx_max_entry - 1;  in ntb_transport_setup_qp_mw()
1362 	for (i = 0; i < qp->rx_max_entry; i++) {  in ntb_transport_setup_qp_mw()
1363 		offset = (void *)(qp->rx_buff + qp->rx_max_frame * (i + 1) -  in ntb_transport_setup_qp_mw()
1368 	qp->rx_pkts = 0;  in ntb_transport_setup_qp_mw()
1369 	qp->tx_pkts = 0;  in ntb_transport_setup_qp_mw()
1370 	qp->tx_index = 0;  in ntb_transport_setup_qp_mw()
1379 	device_t dev = qp->dev;  in ntb_qp_link_work()
1380 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_qp_link_work()
1384 	/* Report queues that are up on our side */  in ntb_qp_link_work()
1385 	for (i = 0, val = 0; i < nt->qp_count; i++) {  in ntb_qp_link_work()
1386 		if (nt->qp_vec[i].client_ready)  in ntb_qp_link_work()
1393 	if ((val & (1ull << qp->qp_num)) != 0) {  in ntb_qp_link_work()
1394 		ntb_printf(2, "qp %d link up\n", qp->qp_num);  in ntb_qp_link_work()
1395 		qp->link_is_up = true;  in ntb_qp_link_work()
1397 		if (qp->event_handler != NULL)  in ntb_qp_link_work()
1398 			qp->event_handler(qp->cb_data, NTB_LINK_UP);  in ntb_qp_link_work()
1400 		ntb_db_clear_mask(dev, 1ull << qp->qp_num);  in ntb_qp_link_work()
1401 	} else if (nt->link_is_up)  in ntb_qp_link_work()
1402 		callout_reset(&qp->link_work,  in ntb_qp_link_work()
1403 		    NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp);  in ntb_qp_link_work()
1413 	callout_drain(&nt->link_work);  in ntb_transport_link_cleanup()
1414 	nt->link_is_up = 0;  in ntb_transport_link_cleanup()
1417 	for (i = 0; i < nt->qp_count; i++) {  in ntb_transport_link_cleanup()
1418 		if ((nt->qp_bitmap & (1 << i)) != 0) {  in ntb_transport_link_cleanup()
1419 			qp = &nt->qp_vec[i];  in ntb_transport_link_cleanup()
1421 			callout_drain(&qp->link_work);  in ntb_transport_link_cleanup()
1430 	ntb_spad_clear(nt->dev);  in ntb_transport_link_cleanup()
1451 	qp->link_is_up = false;  in ntb_qp_link_down_reset()
1452 	ntb_db_set_mask(qp->dev, 1ull << qp->qp_num);  in ntb_qp_link_down_reset()
1454 	qp->tx_index = qp->rx_index = 0;  in ntb_qp_link_down_reset()
1455 	qp->tx_bytes = qp->rx_bytes = 0;  in ntb_qp_link_down_reset()
1456 	qp->tx_pkts = qp->rx_pkts = 0;  in ntb_qp_link_down_reset()
1458 	qp->rx_ring_empty = 0;  in ntb_qp_link_down_reset()
1459 	qp->tx_ring_full = 0;  in ntb_qp_link_down_reset()
1461 	qp->rx_err_no_buf = qp->tx_err_no_buf = 0;  in ntb_qp_link_down_reset()
1462 	qp->rx_err_oflow = qp->rx_err_ver = 0;  in ntb_qp_link_down_reset()
1469 	callout_drain(&qp->link_work);  in ntb_qp_link_cleanup()
1472 	if (qp->event_handler != NULL)  in ntb_qp_link_cleanup()
1473 		qp->event_handler(qp->cb_data, NTB_LINK_DOWN);  in ntb_qp_link_cleanup()
1478  * ntb_transport_link_down - Notify NTB transport to no longer enqueue data
1488 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_transport_link_down()
1492 	qp->client_ready = false;  in ntb_transport_link_down()
1493 	for (i = 0, val = 0; i < nt->qp_count; i++) {  in ntb_transport_link_down()
1494 		if (nt->qp_vec[i].client_ready)  in ntb_transport_link_down()
1497 	ntb_peer_spad_write(qp->dev, NTBT_QP_LINKS, val);  in ntb_transport_link_down()
1499 	if (qp->link_is_up)  in ntb_transport_link_down()
1502 		callout_drain(&qp->link_work);  in ntb_transport_link_down()
1506  * ntb_transport_link_query - Query transport link state
1517 	return (qp->link_is_up);  in ntb_transport_link_query()
1521  * ntb_transport_link_speed - Query transport link speed
1531 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_transport_link_speed()
1532 	uint64_t rate;  in ntb_transport_link_speed()  local
1534 	if (!nt->link_is_up)  in ntb_transport_link_speed()
1536 	switch (nt->link_speed) {  in ntb_transport_link_speed()
1538 		rate = 2500000000 * 8 / 10;  in ntb_transport_link_speed()
1541 		rate = 5000000000 * 8 / 10;  in ntb_transport_link_speed()
1544 		rate = 8000000000 * 128 / 130;  in ntb_transport_link_speed()
1547 		rate = 16000000000 * 128 / 130;  in ntb_transport_link_speed()
1552 	if (nt->link_width <= 0)  in ntb_transport_link_speed()
1554 	return (rate * nt->link_width);  in ntb_transport_link_speed()
1563 	if (!qp->link_is_up)  in ntb_send_link_down()
1567 		entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);  in ntb_send_link_down()
1570 		pause("NTB Wait for link down", hz / 10);  in ntb_send_link_down()
1576 	entry->cb_data = NULL;  in ntb_send_link_down()
1577 	entry->buf = NULL;  in ntb_send_link_down()
1578 	entry->len = 0;  in ntb_send_link_down()
1579 	entry->flags = NTBT_LINK_DOWN_FLAG;  in ntb_send_link_down()
1581 	mtx_lock(&qp->tx_lock);  in ntb_send_link_down()
1583 	mtx_unlock(&qp->tx_lock);  in ntb_send_link_down()
1641  * ntb_transport_qp_num - Query the qp number
1651 	return (qp->qp_num);  in ntb_transport_qp_num()
1655  * ntb_transport_max_size - Query the max payload size of a qp
1666 	return (qp->tx_max_frame - sizeof(struct ntb_payload_header));  in ntb_transport_max_size()
1672 	unsigned int head = qp->tx_index;  in ntb_transport_tx_free_entry()
1673 	unsigned int tail = qp->remote_rx_info->entry;  in ntb_transport_tx_free_entry()
1675 	return (tail >= head ? tail - head : qp->tx_max_entry + tail - head);  in ntb_transport_tx_free_entry()