108 	struct ntb_transport_qp		*qp;  member
130 	void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
140 	void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
265 #define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)  argument
274 static int ntb_process_tx(struct ntb_transport_qp *qp,
277 static int ntb_process_rxc(struct ntb_transport_qp *qp);
278 static void ntb_memcpy_rx(struct ntb_transport_qp *qp,
280 static inline void ntb_rx_copy_callback(struct ntb_transport_qp *qp,
282 static void ntb_complete_rxc(struct ntb_transport_qp *qp);
293 static void ntb_qp_link_down(struct ntb_transport_qp *qp);
294 static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp);
295 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp);
296 static void ntb_send_link_down(struct ntb_transport_qp *qp);
348 	int rc, i, db_count, spad_count, qp, qpu, qpo, qpt;  in ntb_transport_attach()  local
452 		qp = (np && np[0] != 0) ? strtol(np, NULL, 10) : qpo - qpu;  in ntb_transport_attach()
453 		if (qp <= 0)  in ntb_transport_attach()
454 			qp = 1;  in ntb_transport_attach()
456 		if (qp > qpt - qpu) {  in ntb_transport_attach()
464 		nc->qpcnt = qp;  in ntb_transport_attach()
477 			if (qp > 1)  in ntb_transport_attach()
478 				printf("-%d", qpu + qp - 1);  in ntb_transport_attach()
482 		qpu += qp;  in ntb_transport_attach()
597 	struct ntb_transport_qp *qp;  in ntb_transport_init_queue()  local
607 	qp = &nt->qp_vec[qp_num];  in ntb_transport_init_queue()
608 	qp->qp_num = qp_num;  in ntb_transport_init_queue()
609 	qp->transport = nt;  in ntb_transport_init_queue()
610 	qp->dev = nt->dev;  in ntb_transport_init_queue()
611 	qp->client_ready = false;  in ntb_transport_init_queue()
612 	qp->event_handler = NULL;  in ntb_transport_init_queue()
613 	ntb_qp_link_down_reset(qp);  in ntb_transport_init_queue()
625 	qp->tx_mw = mw->vbase + qp_offset;  in ntb_transport_init_queue()
626 	KASSERT(qp->tx_mw != NULL, ("uh oh?"));  in ntb_transport_init_queue()
629 	qp->tx_mw_phys = mw_base + qp_offset;  in ntb_transport_init_queue()
630 	KASSERT(qp->tx_mw_phys != 0, ("uh oh?"));  in ntb_transport_init_queue()
633 	qp->rx_info = (void *)(qp->tx_mw + tx_size);  in ntb_transport_init_queue()
636 	qp->tx_max_frame = qmin(transport_mtu, tx_size / 2);  in ntb_transport_init_queue()
637 	qp->tx_max_entry = tx_size / qp->tx_max_frame;  in ntb_transport_init_queue()
639 	callout_init(&qp->link_work, 1);  in ntb_transport_init_queue()
640 	callout_init(&qp->rx_full, 1);  in ntb_transport_init_queue()
642 	mtx_init(&qp->ntb_rx_q_lock, "ntb rx q", NULL, MTX_SPIN);  in ntb_transport_init_queue()
643 	mtx_init(&qp->ntb_tx_free_q_lock, "ntb tx free q", NULL, MTX_SPIN);  in ntb_transport_init_queue()
644 	mtx_init(&qp->tx_lock, "ntb transport tx", NULL, MTX_DEF);  in ntb_transport_init_queue()
645 	TASK_INIT(&qp->rxc_db_work, 0, ntb_transport_rxc_db, qp);  in ntb_transport_init_queue()
646 	qp->rxc_tq = taskqueue_create("ntbt_rx", M_WAITOK,  in ntb_transport_init_queue()
647 	    taskqueue_thread_enqueue, &qp->rxc_tq);  in ntb_transport_init_queue()
648 	taskqueue_start_threads(&qp->rxc_tq, 1, PI_NET, "%s rx%d",  in ntb_transport_init_queue()
651 	STAILQ_INIT(&qp->rx_post_q);  in ntb_transport_init_queue()
652 	STAILQ_INIT(&qp->rx_pend_q);  in ntb_transport_init_queue()
653 	STAILQ_INIT(&qp->tx_free_q);  in ntb_transport_init_queue()
657 ntb_transport_free_queue(struct ntb_transport_qp *qp)  in ntb_transport_free_queue()  argument
659 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_transport_free_queue()
662 	callout_drain(&qp->link_work);  in ntb_transport_free_queue()
664 	ntb_db_set_mask(qp->dev, 1ull << qp->qp_num);  in ntb_transport_free_queue()
665 	taskqueue_drain_all(qp->rxc_tq);  in ntb_transport_free_queue()
666 	taskqueue_free(qp->rxc_tq);  in ntb_transport_free_queue()
668 	qp->cb_data = NULL;  in ntb_transport_free_queue()
669 	qp->rx_handler = NULL;  in ntb_transport_free_queue()
670 	qp->tx_handler = NULL;  in ntb_transport_free_queue()
671 	qp->event_handler = NULL;  in ntb_transport_free_queue()
673 	while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q)))  in ntb_transport_free_queue()
676 	while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q)))  in ntb_transport_free_queue()
679 	while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))  in ntb_transport_free_queue()
682 	nt->qp_bitmap &= ~(1 << qp->qp_num);  in ntb_transport_free_queue()
706 	struct ntb_transport_qp *qp;  in ntb_transport_create_queue()  local
712 	qp = &nt->qp_vec[nc->qpoff + q];  in ntb_transport_create_queue()
713 	nt->qp_bitmap |= (1 << qp->qp_num);  in ntb_transport_create_queue()
714 	qp->cb_data = data;  in ntb_transport_create_queue()
715 	qp->rx_handler = handlers->rx_handler;  in ntb_transport_create_queue()
716 	qp->tx_handler = handlers->tx_handler;  in ntb_transport_create_queue()
717 	qp->event_handler = handlers->event_handler;  in ntb_transport_create_queue()
724 		entry->qp = qp;  in ntb_transport_create_queue()
725 		ntb_list_add(&qp->ntb_rx_q_lock, entry, &qp->rx_pend_q);  in ntb_transport_create_queue()
730 		entry->qp = qp;  in ntb_transport_create_queue()
731 		ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);  in ntb_transport_create_queue()
734 	ntb_db_clear(dev, 1ull << qp->qp_num);  in ntb_transport_create_queue()
735 	return (qp);  in ntb_transport_create_queue()
745 ntb_transport_link_up(struct ntb_transport_qp *qp)  in ntb_transport_link_up()  argument
747 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_transport_link_up()
749 	qp->client_ready = true;  in ntb_transport_link_up()
751 	ntb_printf(2, "qp %d client ready\n", qp->qp_num);  in ntb_transport_link_up()
754 		callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp);  in ntb_transport_link_up()
773 ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,  in ntb_transport_tx_enqueue()  argument
779 	if (!qp->link_is_up || len == 0) {  in ntb_transport_tx_enqueue()
784 	entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);  in ntb_transport_tx_enqueue()
787 		qp->tx_err_no_buf++;  in ntb_transport_tx_enqueue()
797 	mtx_lock(&qp->tx_lock);  in ntb_transport_tx_enqueue()
798 	rc = ntb_process_tx(qp, entry);  in ntb_transport_tx_enqueue()
799 	mtx_unlock(&qp->tx_lock);  in ntb_transport_tx_enqueue()
801 		ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);  in ntb_transport_tx_enqueue()
813 	struct ntb_transport_qp *qp = entry->qp;  in ntb_tx_copy_callback()  local
819 	ntb_peer_db_set(qp->dev, 1ull << qp->qp_num);  in ntb_tx_copy_callback()
827 		qp->tx_bytes += entry->len;  in ntb_tx_copy_callback()
829 		if (qp->tx_handler)  in ntb_tx_copy_callback()
830 			qp->tx_handler(qp, qp->cb_data, entry->buf,  in ntb_tx_copy_callback()
840 	ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);  in ntb_tx_copy_callback()
862 ntb_async_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry)  in ntb_async_tx()  argument
867 	offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index;  in ntb_async_tx()
868 	hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame -  in ntb_async_tx()
873 	iowrite32(qp->tx_pkts, &hdr->ver);  in ntb_async_tx()
879 ntb_process_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry)  in ntb_process_tx()  argument
884 	    qp->tx_pkts, qp->tx_index, qp->remote_rx_info->entry);  in ntb_process_tx()
885 	if (qp->tx_index == qp->remote_rx_info->entry) {  in ntb_process_tx()
887 		qp->tx_ring_full++;  in ntb_process_tx()
891 	if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) {  in ntb_process_tx()
892 		if (qp->tx_handler != NULL)  in ntb_process_tx()
893 			qp->tx_handler(qp, qp->cb_data, entry->buf,  in ntb_process_tx()
899 		ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);  in ntb_process_tx()
905 	CTR2(KTR_NTB, "TX: copying entry %p to index %u", entry, qp->tx_index);  in ntb_process_tx()
906 	ntb_async_tx(qp, entry);  in ntb_process_tx()
908 	qp->tx_index++;  in ntb_process_tx()
909 	qp->tx_index %= qp->tx_max_entry;  in ntb_process_tx()
911 	qp->tx_pkts++;  in ntb_process_tx()
920 	struct ntb_transport_qp *qp = arg;  in ntb_transport_rxc_db()  local
921 	uint64_t qp_mask = 1ull << qp->qp_num;  in ntb_transport_rxc_db()
926 	while ((rc = ntb_process_rxc(qp)) == 0)  in ntb_transport_rxc_db()
930 	if ((ntb_db_read(qp->dev) & qp_mask) != 0) {  in ntb_transport_rxc_db()
932 		ntb_db_clear(qp->dev, qp_mask);  in ntb_transport_rxc_db()
935 	if (qp->link_is_up)  in ntb_transport_rxc_db()
936 		ntb_db_clear_mask(qp->dev, qp_mask);  in ntb_transport_rxc_db()
940 ntb_process_rxc(struct ntb_transport_qp *qp)  in ntb_process_rxc()  argument
946 	offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index;  in ntb_process_rxc()
947 	hdr = (void *)(offset + qp->rx_max_frame -  in ntb_process_rxc()
950 	CTR1(KTR_NTB, "RX: process_rxc rx_index = %u", qp->rx_index);  in ntb_process_rxc()
953 		qp->rx_ring_empty++;  in ntb_process_rxc()
959 		ntb_qp_link_down(qp);  in ntb_process_rxc()
964 	if (hdr->ver != (uint32_t)qp->rx_pkts) {  in ntb_process_rxc()
966 		    "Returning entry to rx_pend_q", hdr->ver, qp->rx_pkts);  in ntb_process_rxc()
967 		qp->rx_err_ver++;  in ntb_process_rxc()
971 	entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q);  in ntb_process_rxc()
973 		qp->rx_err_no_buf++;  in ntb_process_rxc()
977 	callout_stop(&qp->rx_full);  in ntb_process_rxc()
981 	entry->index = qp->rx_index;  in ntb_process_rxc()
986 		qp->rx_err_oflow++;  in ntb_process_rxc()
991 		ntb_complete_rxc(qp);  in ntb_process_rxc()
993 		qp->rx_bytes += hdr->len;  in ntb_process_rxc()
994 		qp->rx_pkts++;  in ntb_process_rxc()
996 		CTR1(KTR_NTB, "RX: received %ld rx_pkts", qp->rx_pkts);  in ntb_process_rxc()
1000 		ntb_memcpy_rx(qp, entry, offset);  in ntb_process_rxc()
1003 	qp->rx_index++;  in ntb_process_rxc()
1004 	qp->rx_index %= qp->rx_max_entry;  in ntb_process_rxc()
1009 ntb_memcpy_rx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,  in ntb_memcpy_rx()  argument
1025 	ntb_rx_copy_callback(qp, entry);  in ntb_memcpy_rx()
1029 ntb_rx_copy_callback(struct ntb_transport_qp *qp, void *data)  in ntb_rx_copy_callback()  argument
1035 	ntb_complete_rxc(qp);  in ntb_rx_copy_callback()
1039 ntb_complete_rxc(struct ntb_transport_qp *qp)  in ntb_complete_rxc()  argument
1047 	mtx_lock_spin(&qp->ntb_rx_q_lock);  in ntb_complete_rxc()
1049 	while (!STAILQ_EMPTY(&qp->rx_post_q)) {  in ntb_complete_rxc()
1050 		entry = STAILQ_FIRST(&qp->rx_post_q);  in ntb_complete_rxc()
1055 		iowrite32(entry->index, &qp->rx_info->entry);  in ntb_complete_rxc()
1057 		STAILQ_REMOVE_HEAD(&qp->rx_post_q, entry);  in ntb_complete_rxc()
1068 		entry->cb_data = qp->cb_data;  in ntb_complete_rxc()
1070 		STAILQ_INSERT_TAIL(&qp->rx_pend_q, entry, entry);  in ntb_complete_rxc()
1072 		mtx_unlock_spin(&qp->ntb_rx_q_lock);  in ntb_complete_rxc()
1075 		if (qp->rx_handler != NULL && qp->client_ready)  in ntb_complete_rxc()
1076 			qp->rx_handler(qp, qp->cb_data, m, len);  in ntb_complete_rxc()
1080 		mtx_lock_spin(&qp->ntb_rx_q_lock);  in ntb_complete_rxc()
1083 	mtx_unlock_spin(&qp->ntb_rx_q_lock);  in ntb_complete_rxc()
1090 	struct ntb_transport_qp *qp;  in ntb_transport_doorbell_callback()  local
1105 		qp = &nt->qp_vec[qp_num];  in ntb_transport_doorbell_callback()
1106 		if (qp->link_is_up)  in ntb_transport_doorbell_callback()
1107 			taskqueue_enqueue(qp->rxc_tq, &qp->rxc_db_work);  in ntb_transport_doorbell_callback()
1135 	struct ntb_transport_qp *qp;  in ntb_transport_link_work()  local
1224 		qp = &nt->qp_vec[i];  in ntb_transport_link_work()
1228 		if (qp->client_ready)  in ntb_transport_link_work()
1229 			callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp);  in ntb_transport_link_work()
1329 	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];  in ntb_transport_setup_qp_mw()  local
1349 	qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count);  in ntb_transport_setup_qp_mw()
1352 	qp->remote_rx_info = (void*)(qp->rx_buff + rx_size);  in ntb_transport_setup_qp_mw()
1355 	qp->rx_max_frame = qmin(transport_mtu, rx_size / 2);  in ntb_transport_setup_qp_mw()
1356 	qp->rx_max_entry = rx_size / qp->rx_max_frame;  in ntb_transport_setup_qp_mw()
1357 	qp->rx_index = 0;  in ntb_transport_setup_qp_mw()
1359 	qp->remote_rx_info->entry = qp->rx_max_entry - 1;  in ntb_transport_setup_qp_mw()
1362 	for (i = 0; i < qp->rx_max_entry; i++) {  in ntb_transport_setup_qp_mw()
1363 		offset = (void *)(qp->rx_buff + qp->rx_max_frame * (i + 1) -  in ntb_transport_setup_qp_mw()
1368 	qp->rx_pkts = 0;  in ntb_transport_setup_qp_mw()
1369 	qp->tx_pkts = 0;  in ntb_transport_setup_qp_mw()
1370 	qp->tx_index = 0;  in ntb_transport_setup_qp_mw()
1378 	struct ntb_transport_qp *qp = arg;  in ntb_qp_link_work()  local
1379 	device_t dev = qp->dev;  in ntb_qp_link_work()
1380 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_qp_link_work()
1393 	if ((val & (1ull << qp->qp_num)) != 0) {  in ntb_qp_link_work()
1394 		ntb_printf(2, "qp %d link up\n", qp->qp_num);  in ntb_qp_link_work()
1395 		qp->link_is_up = true;  in ntb_qp_link_work()
1397 		if (qp->event_handler != NULL)  in ntb_qp_link_work()
1398 			qp->event_handler(qp->cb_data, NTB_LINK_UP);  in ntb_qp_link_work()
1400 		ntb_db_clear_mask(dev, 1ull << qp->qp_num);  in ntb_qp_link_work()
1402 		callout_reset(&qp->link_work,  in ntb_qp_link_work()
1403 		    NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_qp_link_work, qp);  in ntb_qp_link_work()
1410 	struct ntb_transport_qp *qp;  in ntb_transport_link_cleanup()  local
1419 			qp = &nt->qp_vec[i];  in ntb_transport_link_cleanup()
1420 			ntb_qp_link_cleanup(qp);  in ntb_transport_link_cleanup()
1421 			callout_drain(&qp->link_work);  in ntb_transport_link_cleanup()
1441 ntb_qp_link_down(struct ntb_transport_qp *qp)  in ntb_qp_link_down()  argument
1444 	ntb_qp_link_cleanup(qp);  in ntb_qp_link_down()
1448 ntb_qp_link_down_reset(struct ntb_transport_qp *qp)  in ntb_qp_link_down_reset()  argument
1451 	qp->link_is_up = false;  in ntb_qp_link_down_reset()
1452 	ntb_db_set_mask(qp->dev, 1ull << qp->qp_num);  in ntb_qp_link_down_reset()
1454 	qp->tx_index = qp->rx_index = 0;  in ntb_qp_link_down_reset()
1455 	qp->tx_bytes = qp->rx_bytes = 0;  in ntb_qp_link_down_reset()
1456 	qp->tx_pkts = qp->rx_pkts = 0;  in ntb_qp_link_down_reset()
1458 	qp->rx_ring_empty = 0;  in ntb_qp_link_down_reset()
1459 	qp->tx_ring_full = 0;  in ntb_qp_link_down_reset()
1461 	qp->rx_err_no_buf = qp->tx_err_no_buf = 0;  in ntb_qp_link_down_reset()
1462 	qp->rx_err_oflow = qp->rx_err_ver = 0;  in ntb_qp_link_down_reset()
1466 ntb_qp_link_cleanup(struct ntb_transport_qp *qp)  in ntb_qp_link_cleanup()  argument
1469 	callout_drain(&qp->link_work);  in ntb_qp_link_cleanup()
1470 	ntb_qp_link_down_reset(qp);  in ntb_qp_link_cleanup()
1472 	if (qp->event_handler != NULL)  in ntb_qp_link_cleanup()
1473 		qp->event_handler(qp->cb_data, NTB_LINK_DOWN);  in ntb_qp_link_cleanup()
1486 ntb_transport_link_down(struct ntb_transport_qp *qp)  in ntb_transport_link_down()  argument
1488 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_transport_link_down()
1492 	qp->client_ready = false;  in ntb_transport_link_down()
1497 	ntb_peer_spad_write(qp->dev, NTBT_QP_LINKS, val);  in ntb_transport_link_down()
1499 	if (qp->link_is_up)  in ntb_transport_link_down()
1500 		ntb_send_link_down(qp);  in ntb_transport_link_down()
1502 		callout_drain(&qp->link_work);  in ntb_transport_link_down()
1514 ntb_transport_link_query(struct ntb_transport_qp *qp)  in ntb_transport_link_query()  argument
1517 	return (qp->link_is_up);  in ntb_transport_link_query()
1529 ntb_transport_link_speed(struct ntb_transport_qp *qp)  in ntb_transport_link_speed()  argument
1531 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_transport_link_speed()
1558 ntb_send_link_down(struct ntb_transport_qp *qp)  in ntb_send_link_down()  argument
1563 	if (!qp->link_is_up)  in ntb_send_link_down()
1567 		entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);  in ntb_send_link_down()
1581 	mtx_lock(&qp->tx_lock);  in ntb_send_link_down()
1582 	rc = ntb_process_tx(qp, entry);  in ntb_send_link_down()
1583 	mtx_unlock(&qp->tx_lock);  in ntb_send_link_down()
1587 	ntb_qp_link_down_reset(qp);  in ntb_send_link_down()
1648 unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp)  in ntb_transport_qp_num()  argument
1651 	return (qp->qp_num);  in ntb_transport_qp_num()
1663 ntb_transport_max_size(struct ntb_transport_qp *qp)  in ntb_transport_max_size()  argument
1666 	return (qp->tx_max_frame - sizeof(struct ntb_payload_header));  in ntb_transport_max_size()
1670 ntb_transport_tx_free_entry(struct ntb_transport_qp *qp)  in ntb_transport_tx_free_entry()  argument
1672 	unsigned int head = qp->tx_index;  in ntb_transport_tx_free_entry()
1673 	unsigned int tail = qp->remote_rx_info->entry;  in ntb_transport_tx_free_entry()
1675 	return (tail >= head ? tail - head : qp->tx_max_entry + tail - head);  in ntb_transport_tx_free_entry()