182 dpaa2_swp_init_portal(struct dpaa2_swp **swp, struct dpaa2_swp_desc *desc,  in dpaa2_swp_init_portal()  argument
188 	if (!swp || !desc)  in dpaa2_swp_init_portal()
304 	*swp = p;  in dpaa2_swp_init_portal()
310 dpaa2_swp_free_portal(struct dpaa2_swp *swp)  in dpaa2_swp_free_portal()  argument
314 	KASSERT(swp != NULL, ("%s: swp is NULL", __func__));  in dpaa2_swp_free_portal()
316 	DPAA2_SWP_LOCK(swp, &flags);  in dpaa2_swp_free_portal()
317 	swp->flags |= DPAA2_SWP_DESTROYED;  in dpaa2_swp_free_portal()
318 	DPAA2_SWP_UNLOCK(swp);  in dpaa2_swp_free_portal()
323 	mtx_destroy(&swp->lock);  in dpaa2_swp_free_portal()
324 	free(swp, M_DPAA2_SWP);  in dpaa2_swp_free_portal()
350 dpaa2_swp_write_reg(struct dpaa2_swp *swp, uint32_t o, uint32_t v)  in dpaa2_swp_write_reg()  argument
352 	bus_write_4(swp->cinh_map, o, v);  in dpaa2_swp_write_reg()
356 dpaa2_swp_read_reg(struct dpaa2_swp *swp, uint32_t o)  in dpaa2_swp_read_reg()  argument
358 	return (bus_read_4(swp->cinh_map, o));  in dpaa2_swp_read_reg()
394 dpaa2_swp_set_intr_trigger(struct dpaa2_swp *swp, uint32_t mask)  in dpaa2_swp_set_intr_trigger()  argument
396 	if (swp != NULL)  in dpaa2_swp_set_intr_trigger()
397 		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_IER, mask);  in dpaa2_swp_set_intr_trigger()
404 dpaa2_swp_get_intr_trigger(struct dpaa2_swp *swp)  in dpaa2_swp_get_intr_trigger()  argument
406 	if (swp != NULL)  in dpaa2_swp_get_intr_trigger()
407 		return dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_IER);  in dpaa2_swp_get_intr_trigger()
415 dpaa2_swp_read_intr_status(struct dpaa2_swp *swp)  in dpaa2_swp_read_intr_status()  argument
417 	if (swp != NULL)  in dpaa2_swp_read_intr_status()
418 		return dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_ISR);  in dpaa2_swp_read_intr_status()
426 dpaa2_swp_clear_intr_status(struct dpaa2_swp *swp, uint32_t mask)  in dpaa2_swp_clear_intr_status()  argument
428 	if (swp != NULL)  in dpaa2_swp_clear_intr_status()
429 		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_ISR, mask);  in dpaa2_swp_clear_intr_status()
435  * swp:		the software portal object
440 dpaa2_swp_set_push_dequeue(struct dpaa2_swp *swp, uint8_t chan_idx, bool en)  in dpaa2_swp_set_push_dequeue()  argument
444 	if (swp != NULL) {  in dpaa2_swp_set_push_dequeue()
446 			device_printf(swp->desc->dpio_dev, "channel index "  in dpaa2_swp_set_push_dequeue()
452 			swp->sdq |= 1 << chan_idx;  in dpaa2_swp_set_push_dequeue()
454 			swp->sdq &= ~(1 << chan_idx);  in dpaa2_swp_set_push_dequeue()
459 		dqsrc = (swp->sdq >> DPAA2_SDQCR_SRC_SHIFT) &  in dpaa2_swp_set_push_dequeue()
461 		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_SDQCR, dqsrc != 0  in dpaa2_swp_set_push_dequeue()
462 		    ? swp->sdq : 0);  in dpaa2_swp_set_push_dequeue()
469  * swp:		The software portal object.
474 int dpaa2_swp_set_irq_coalescing(struct dpaa2_swp *swp, uint32_t threshold,  in dpaa2_swp_set_irq_coalescing()  argument
479 	if (swp == NULL)  in dpaa2_swp_set_irq_coalescing()
486 	itp = (holdoff * 1000u) / swp->desc->swp_cycles_ratio;  in dpaa2_swp_set_irq_coalescing()
489 	if (threshold >= swp->dqrr.ring_size)  in dpaa2_swp_set_irq_coalescing()
490 		threshold = swp->dqrr.ring_size - 1;  in dpaa2_swp_set_irq_coalescing()
492 	swp->dqrr.irq_threshold = threshold;  in dpaa2_swp_set_irq_coalescing()
493 	swp->dqrr.irq_itp = itp;  in dpaa2_swp_set_irq_coalescing()
495 	dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_DQRR_ITR, threshold);  in dpaa2_swp_set_irq_coalescing()
496 	dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_ITPR, itp);  in dpaa2_swp_set_irq_coalescing()
510 dpaa2_swp_conf_wq_channel(struct dpaa2_swp *swp, uint16_t chan_id,  in dpaa2_swp_conf_wq_channel()  argument
532 	if (swp == NULL)  in dpaa2_swp_conf_wq_channel()
540 	error = dpaa2_swp_exec_mgmt_command(swp, (struct dpaa2_swp_cmd *) &cmd,  in dpaa2_swp_conf_wq_channel()
546 		device_printf(swp->desc->dpio_dev, "WQ channel configuration "  in dpaa2_swp_conf_wq_channel()
559 dpaa2_swp_query_bp(struct dpaa2_swp *swp, uint16_t bpid,  in dpaa2_swp_query_bp()  argument
581 	if (swp == NULL || conf == NULL)  in dpaa2_swp_query_bp()
586 	error = dpaa2_swp_exec_mgmt_command(swp, (struct dpaa2_swp_cmd *) &cmd,  in dpaa2_swp_query_bp()
592 		device_printf(swp->desc->dpio_dev, "BP query error: bpid=%d, "  in dpaa2_swp_query_bp()
605 dpaa2_swp_release_bufs(struct dpaa2_swp *swp, uint16_t bpid, bus_addr_t *buf,  in dpaa2_swp_release_bufs()  argument
618 	if (swp == NULL || buf == NULL || buf_num == 0u ||  in dpaa2_swp_release_bufs()
627 	error = dpaa2_swp_exec_br_command(swp, (struct dpaa2_swp_cmd *) &cmd,  in dpaa2_swp_release_bufs()
630 		device_printf(swp->desc->dpio_dev, "buffers release command "  in dpaa2_swp_release_bufs()
639 dpaa2_swp_dqrr_next_locked(struct dpaa2_swp *swp, struct dpaa2_dq *dq,  in dpaa2_swp_dqrr_next_locked()  argument
642 	struct resource_map *map = swp->cinh_map;  in dpaa2_swp_dqrr_next_locked()
645 	uint32_t offset = swp->cfg.mem_backed  in dpaa2_swp_dqrr_next_locked()
646 	    ? DPAA2_SWP_CENA_DQRR_MEM(swp->dqrr.next_idx)  in dpaa2_swp_dqrr_next_locked()
647 	    : DPAA2_SWP_CENA_DQRR(swp->dqrr.next_idx);  in dpaa2_swp_dqrr_next_locked()
649 	if (swp == NULL || dq == NULL)  in dpaa2_swp_dqrr_next_locked()
656 	if (swp->dqrr.reset_bug) {  in dpaa2_swp_dqrr_next_locked()
665 		pi = dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_DQPI) & DQRR_PI_MASK;  in dpaa2_swp_dqrr_next_locked()
668 		if (pi == swp->dqrr.next_idx)  in dpaa2_swp_dqrr_next_locked()
681 		if (swp->dqrr.next_idx == (swp->dqrr.ring_size - 1))  in dpaa2_swp_dqrr_next_locked()
682 			swp->dqrr.reset_bug = 0;  in dpaa2_swp_dqrr_next_locked()
686 	if ((verb & DPAA2_SWP_VALID_BIT) != swp->dqrr.valid_bit)  in dpaa2_swp_dqrr_next_locked()
695 		*idx = swp->dqrr.next_idx;  in dpaa2_swp_dqrr_next_locked()
701 	swp->dqrr.next_idx++;  in dpaa2_swp_dqrr_next_locked()
702 	swp->dqrr.next_idx &= swp->dqrr.ring_size - 1; /* wrap around */  in dpaa2_swp_dqrr_next_locked()
703 	if (swp->dqrr.next_idx == 0u)  in dpaa2_swp_dqrr_next_locked()
704 		swp->dqrr.valid_bit ^= DPAA2_SWP_VALID_BIT;  in dpaa2_swp_dqrr_next_locked()
710 dpaa2_swp_pull(struct dpaa2_swp *swp, uint16_t chan_id, struct dpaa2_buf *buf,  in dpaa2_swp_pull()  argument
748 	DPAA2_SWP_LOCK(swp, &flags);  in dpaa2_swp_pull()
751 		DPAA2_SWP_UNLOCK(swp);  in dpaa2_swp_pull()
755 	error = dpaa2_swp_exec_vdc_command_locked(swp,  in dpaa2_swp_pull()
758 		DPAA2_SWP_UNLOCK(swp);  in dpaa2_swp_pull()
776 	DPAA2_SWP_UNLOCK(swp);  in dpaa2_swp_pull()
785  * swp:		Software portal used to send this command to.
790 dpaa2_swp_enq(struct dpaa2_swp *swp, struct dpaa2_eq_desc *ed,  in dpaa2_swp_enq()  argument
794 	int rc = dpaa2_swp_enq_mult(swp, ed, fd, &flags, 1);  in dpaa2_swp_enq()
802  * swp:		Software portal used to send this command to.
811 dpaa2_swp_enq_mult(struct dpaa2_swp *swp, struct dpaa2_eq_desc *ed,  in dpaa2_swp_enq_mult()  argument
824 	if (swp == NULL || ed == NULL || fd == NULL || flags == NULL ||  in dpaa2_swp_enq_mult()
828 	DPAA2_SWP_LOCK(swp, &swp_flags);  in dpaa2_swp_enq_mult()
831 		DPAA2_SWP_UNLOCK(swp);  in dpaa2_swp_enq_mult()
835 	map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;  in dpaa2_swp_enq_mult()
836 	ci_offset = swp->cfg.mem_backed  in dpaa2_swp_enq_mult()
840 	half_mask = swp->eqcr.pi_ci_mask >> 1;  in dpaa2_swp_enq_mult()
841 	full_mask = swp->eqcr.pi_ci_mask;  in dpaa2_swp_enq_mult()
843 	if (swp->eqcr.available == 0) {  in dpaa2_swp_enq_mult()
844 		val = dpaa2_swp_read_reg(swp, ci_offset);  in dpaa2_swp_enq_mult()
845 		eqcr_ci = swp->eqcr.ci;  in dpaa2_swp_enq_mult()
846 		swp->eqcr.ci = val & full_mask;  in dpaa2_swp_enq_mult()
848 		swp->eqcr.available = dpaa2_swp_cyc_diff(swp->eqcr.pi_ring_size,  in dpaa2_swp_enq_mult()
849 		    eqcr_ci, swp->eqcr.ci);  in dpaa2_swp_enq_mult()
851 		if (swp->eqcr.available == 0) {  in dpaa2_swp_enq_mult()
852 			DPAA2_SWP_UNLOCK(swp);  in dpaa2_swp_enq_mult()
857 	eqcr_pi = swp->eqcr.pi;  in dpaa2_swp_enq_mult()
858 	num_enq = swp->eqcr.available < frames_n  in dpaa2_swp_enq_mult()
859 	    ? swp->eqcr.available : frames_n;  in dpaa2_swp_enq_mult()
860 	swp->eqcr.available -= num_enq;  in dpaa2_swp_enq_mult()
862 	KASSERT(num_enq >= 0 && num_enq <= swp->eqcr.pi_ring_size,  in dpaa2_swp_enq_mult()
864 	KASSERT(swp->eqcr.available >= 0 &&  in dpaa2_swp_enq_mult()
865 	    swp->eqcr.available <= swp->eqcr.pi_ring_size,  in dpaa2_swp_enq_mult()
866 	    ("%s: unexpected eqcr.available=%d", __func__, swp->eqcr.available));  in dpaa2_swp_enq_mult()
897 	eqcr_pi = swp->eqcr.pi;  in dpaa2_swp_enq_mult()
901 		    ed_pdat8[0] | swp->eqcr.pi_vb);  in dpaa2_swp_enq_mult()
912 			swp->eqcr.pi_vb ^= DPAA2_SWP_VALID_BIT;  in dpaa2_swp_enq_mult()
914 	swp->eqcr.pi = eqcr_pi & full_mask;  in dpaa2_swp_enq_mult()
916 	DPAA2_SWP_UNLOCK(swp);  in dpaa2_swp_enq_mult()
933 dpaa2_swp_exec_br_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,  in dpaa2_swp_exec_br_command()  argument
946 	if (!swp || !cmd)  in dpaa2_swp_exec_br_command()
949 	DPAA2_SWP_LOCK(swp, &flags);  in dpaa2_swp_exec_br_command()
952 		DPAA2_SWP_UNLOCK(swp);  in dpaa2_swp_exec_br_command()
956 	rar = dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_RAR);  in dpaa2_swp_exec_br_command()
958 		DPAA2_SWP_UNLOCK(swp);  in dpaa2_swp_exec_br_command()
962 	map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;  in dpaa2_swp_exec_br_command()
963 	offset = swp->cfg.mem_backed  in dpaa2_swp_exec_br_command()
976 	if (swp->cfg.mem_backed) {  in dpaa2_swp_exec_br_command()
979 		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_RCR_AM_RT +  in dpaa2_swp_exec_br_command()
986 	DPAA2_SWP_UNLOCK(swp);  in dpaa2_swp_exec_br_command()
1003 dpaa2_swp_exec_vdc_command_locked(struct dpaa2_swp *swp,  in dpaa2_swp_exec_vdc_command_locked()  argument
1015 	map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;  in dpaa2_swp_exec_vdc_command_locked()
1016 	offset = swp->cfg.mem_backed  in dpaa2_swp_exec_vdc_command_locked()
1028 	if (swp->cfg.mem_backed) {  in dpaa2_swp_exec_vdc_command_locked()
1029 		bus_write_1(map, offset, c->verb | swp->vdq.valid_bit);  in dpaa2_swp_exec_vdc_command_locked()
1030 		swp->vdq.valid_bit ^= DPAA2_SWP_VALID_BIT;  in dpaa2_swp_exec_vdc_command_locked()
1032 		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_VDQCR_RT,  in dpaa2_swp_exec_vdc_command_locked()
1036 		bus_write_1(map, offset, c->verb | swp->vdq.valid_bit);  in dpaa2_swp_exec_vdc_command_locked()
1037 		swp->vdq.valid_bit ^= DPAA2_SWP_VALID_BIT;  in dpaa2_swp_exec_vdc_command_locked()
1047 dpaa2_swp_exec_mgmt_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,  in dpaa2_swp_exec_mgmt_command()  argument
1059 	if (swp == NULL || cmd == NULL || rsp == NULL)  in dpaa2_swp_exec_mgmt_command()
1062 	DPAA2_SWP_LOCK(swp, &flags);  in dpaa2_swp_exec_mgmt_command()
1065 		DPAA2_SWP_UNLOCK(swp);  in dpaa2_swp_exec_mgmt_command()
1073 	dpaa2_swp_send_mgmt_command(swp, cmd, cmdid);  in dpaa2_swp_exec_mgmt_command()
1074 	error = dpaa2_swp_wait_for_mgmt_response(swp, rsp);  in dpaa2_swp_exec_mgmt_command()
1076 		DPAA2_SWP_UNLOCK(swp);  in dpaa2_swp_exec_mgmt_command()
1079 	DPAA2_SWP_UNLOCK(swp);  in dpaa2_swp_exec_mgmt_command()
1092 dpaa2_swp_send_mgmt_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,  in dpaa2_swp_send_mgmt_command()  argument
1100 	map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;  in dpaa2_swp_send_mgmt_command()
1101 	offset = swp->cfg.mem_backed ? DPAA2_SWP_CENA_CR_MEM : DPAA2_SWP_CENA_CR;  in dpaa2_swp_send_mgmt_command()
1111 	if (swp->cfg.mem_backed) {  in dpaa2_swp_send_mgmt_command()
1112 		bus_write_1(map, offset, cmdid | swp->mr.valid_bit);  in dpaa2_swp_send_mgmt_command()
1114 		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_CR_RT,  in dpaa2_swp_send_mgmt_command()
1118 		bus_write_1(map, offset, cmdid | swp->mc.valid_bit);  in dpaa2_swp_send_mgmt_command()
1125 dpaa2_swp_wait_for_mgmt_response(struct dpaa2_swp *swp, struct dpaa2_swp_rsp *rsp)  in dpaa2_swp_wait_for_mgmt_response()  argument
1127 	struct resource_map *map = swp->cfg.mem_backed  in dpaa2_swp_wait_for_mgmt_response()
1128 	    ? swp->cena_map : swp->cinh_map;  in dpaa2_swp_wait_for_mgmt_response()
1130 	const uint32_t offset = swp->cfg.mem_backed  in dpaa2_swp_wait_for_mgmt_response()
1132 	    : DPAA2_SWP_CENA_RR(swp->mc.valid_bit);  in dpaa2_swp_wait_for_mgmt_response()
1138 		if (swp->cfg.mem_backed) {  in dpaa2_swp_wait_for_mgmt_response()
1140 			if (swp->mr.valid_bit != (verb & DPAA2_SWP_VALID_BIT))  in dpaa2_swp_wait_for_mgmt_response()
1144 			swp->mr.valid_bit ^= DPAA2_SWP_VALID_BIT;  in dpaa2_swp_wait_for_mgmt_response()
1150 			swp->mc.valid_bit ^= DPAA2_SWP_VALID_BIT;  in dpaa2_swp_wait_for_mgmt_response()