1 /* SPDX-License-Identifier: BSD-3-Clause */
39  *  iavf_adminq_init_regs - Initialize AdminQ registers
47 	hw->aq.asq.tail = IAVF_VF_ATQT1;  in iavf_adminq_init_regs()
48 	hw->aq.asq.head = IAVF_VF_ATQH1;  in iavf_adminq_init_regs()
49 	hw->aq.asq.len  = IAVF_VF_ATQLEN1;  in iavf_adminq_init_regs()
50 	hw->aq.asq.bal  = IAVF_VF_ATQBAL1;  in iavf_adminq_init_regs()
51 	hw->aq.asq.bah  = IAVF_VF_ATQBAH1;  in iavf_adminq_init_regs()
52 	hw->aq.arq.tail = IAVF_VF_ARQT1;  in iavf_adminq_init_regs()
53 	hw->aq.arq.head = IAVF_VF_ARQH1;  in iavf_adminq_init_regs()
54 	hw->aq.arq.len  = IAVF_VF_ARQLEN1;  in iavf_adminq_init_regs()
55 	hw->aq.arq.bal  = IAVF_VF_ARQBAL1;  in iavf_adminq_init_regs()
56 	hw->aq.arq.bah  = IAVF_VF_ARQBAH1;  in iavf_adminq_init_regs()
60  *  iavf_alloc_adminq_asq_ring - Allocate Admin Queue send rings
67 	ret_code = iavf_allocate_dma_mem(hw, &hw->aq.asq.desc_buf,  in iavf_alloc_adminq_asq_ring()
69 					 (hw->aq.num_asq_entries *  in iavf_alloc_adminq_asq_ring()
75 	ret_code = iavf_allocate_virt_mem(hw, &hw->aq.asq.cmd_buf,  in iavf_alloc_adminq_asq_ring()
76 					  (hw->aq.num_asq_entries *  in iavf_alloc_adminq_asq_ring()
79 		iavf_free_dma_mem(hw, &hw->aq.asq.desc_buf);  in iavf_alloc_adminq_asq_ring()
87  *  iavf_alloc_adminq_arq_ring - Allocate Admin Queue receive rings
94 	ret_code = iavf_allocate_dma_mem(hw, &hw->aq.arq.desc_buf,  in iavf_alloc_adminq_arq_ring()
96 					 (hw->aq.num_arq_entries *  in iavf_alloc_adminq_arq_ring()
104  *  iavf_free_adminq_asq - Free Admin Queue send rings
108  *  and de-allocated
112 	iavf_free_virt_mem(hw, &hw->aq.asq.cmd_buf);  in iavf_free_adminq_asq()
113 	iavf_free_dma_mem(hw, &hw->aq.asq.desc_buf);  in iavf_free_adminq_asq()
117  *  iavf_free_adminq_arq - Free Admin Queue receive rings
121  *  and de-allocated
125 	iavf_free_dma_mem(hw, &hw->aq.arq.desc_buf);  in iavf_free_adminq_arq()
129  *  iavf_alloc_arq_bufs - Allocate pre-posted buffers for the receive queue
144 	ret_code = iavf_allocate_virt_mem(hw, &hw->aq.arq.dma_head,  in iavf_alloc_arq_bufs()
145 		(hw->aq.num_arq_entries * sizeof(struct iavf_dma_mem)));  in iavf_alloc_arq_bufs()
148 	hw->aq.arq.r.arq_bi = (struct iavf_dma_mem *)hw->aq.arq.dma_head.va;  in iavf_alloc_arq_bufs()
151 	for (i = 0; i < hw->aq.num_arq_entries; i++) {  in iavf_alloc_arq_bufs()
152 		bi = &hw->aq.arq.r.arq_bi[i];  in iavf_alloc_arq_bufs()
155 						 hw->aq.arq_buf_size,  in iavf_alloc_arq_bufs()
161 		desc = IAVF_ADMINQ_DESC(hw->aq.arq, i);  in iavf_alloc_arq_bufs()
163 		desc->flags = CPU_TO_LE16(IAVF_AQ_FLAG_BUF);  in iavf_alloc_arq_bufs()
164 		if (hw->aq.arq_buf_size > IAVF_AQ_LARGE_BUF)  in iavf_alloc_arq_bufs()
165 			desc->flags |= CPU_TO_LE16(IAVF_AQ_FLAG_LB);  in iavf_alloc_arq_bufs()
166 		desc->opcode = 0;  in iavf_alloc_arq_bufs()
170 		desc->datalen = CPU_TO_LE16((u16)bi->size);  in iavf_alloc_arq_bufs()
171 		desc->retval = 0;  in iavf_alloc_arq_bufs()
172 		desc->cookie_high = 0;  in iavf_alloc_arq_bufs()
173 		desc->cookie_low = 0;  in iavf_alloc_arq_bufs()
174 		desc->params.external.addr_high =  in iavf_alloc_arq_bufs()
175 			CPU_TO_LE32(IAVF_HI_DWORD(bi->pa));  in iavf_alloc_arq_bufs()
176 		desc->params.external.addr_low =  in iavf_alloc_arq_bufs()
177 			CPU_TO_LE32(IAVF_LO_DWORD(bi->pa));  in iavf_alloc_arq_bufs()
178 		desc->params.external.param0 = 0;  in iavf_alloc_arq_bufs()
179 		desc->params.external.param1 = 0;  in iavf_alloc_arq_bufs()
187 	i--;  in iavf_alloc_arq_bufs()
188 	for (; i >= 0; i--)  in iavf_alloc_arq_bufs()
189 		iavf_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]);  in iavf_alloc_arq_bufs()
190 	iavf_free_virt_mem(hw, &hw->aq.arq.dma_head);  in iavf_alloc_arq_bufs()
196  *  iavf_alloc_asq_bufs - Allocate empty buffer structs for the send queue
206 	ret_code = iavf_allocate_virt_mem(hw, &hw->aq.asq.dma_head,  in iavf_alloc_asq_bufs()
207 		(hw->aq.num_asq_entries * sizeof(struct iavf_dma_mem)));  in iavf_alloc_asq_bufs()
210 	hw->aq.asq.r.asq_bi = (struct iavf_dma_mem *)hw->aq.asq.dma_head.va;  in iavf_alloc_asq_bufs()
213 	for (i = 0; i < hw->aq.num_asq_entries; i++) {  in iavf_alloc_asq_bufs()
214 		bi = &hw->aq.asq.r.asq_bi[i];  in iavf_alloc_asq_bufs()
217 						 hw->aq.asq_buf_size,  in iavf_alloc_asq_bufs()
227 	i--;  in iavf_alloc_asq_bufs()
228 	for (; i >= 0; i--)  in iavf_alloc_asq_bufs()
229 		iavf_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]);  in iavf_alloc_asq_bufs()
230 	iavf_free_virt_mem(hw, &hw->aq.asq.dma_head);  in iavf_alloc_asq_bufs()
236  *  iavf_free_arq_bufs - Free receive queue buffer info elements
244 	for (i = 0; i < hw->aq.num_arq_entries; i++)  in iavf_free_arq_bufs()
245 		iavf_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]);  in iavf_free_arq_bufs()
248 	iavf_free_dma_mem(hw, &hw->aq.arq.desc_buf);  in iavf_free_arq_bufs()
251 	iavf_free_virt_mem(hw, &hw->aq.arq.dma_head);  in iavf_free_arq_bufs()
255  *  iavf_free_asq_bufs - Free send queue buffer info elements
262 	/* only unmap if the address is non-NULL */  in iavf_free_asq_bufs()
263 	for (i = 0; i < hw->aq.num_asq_entries; i++)  in iavf_free_asq_bufs()
264 		if (hw->aq.asq.r.asq_bi[i].pa)  in iavf_free_asq_bufs()
265 			iavf_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]);  in iavf_free_asq_bufs()
268 	iavf_free_virt_mem(hw, &hw->aq.asq.cmd_buf);  in iavf_free_asq_bufs()
271 	iavf_free_dma_mem(hw, &hw->aq.asq.desc_buf);  in iavf_free_asq_bufs()
274 	iavf_free_virt_mem(hw, &hw->aq.asq.dma_head);  in iavf_free_asq_bufs()
278  *  iavf_config_asq_regs - configure ASQ registers
289 	wr32(hw, hw->aq.asq.head, 0);  in iavf_config_asq_regs()
290 	wr32(hw, hw->aq.asq.tail, 0);  in iavf_config_asq_regs()
293 	wr32(hw, hw->aq.asq.len, (hw->aq.num_asq_entries |  in iavf_config_asq_regs()
295 	wr32(hw, hw->aq.asq.bal, IAVF_LO_DWORD(hw->aq.asq.desc_buf.pa));  in iavf_config_asq_regs()
296 	wr32(hw, hw->aq.asq.bah, IAVF_HI_DWORD(hw->aq.asq.desc_buf.pa));  in iavf_config_asq_regs()
299 	reg = rd32(hw, hw->aq.asq.bal);  in iavf_config_asq_regs()
300 	if (reg != IAVF_LO_DWORD(hw->aq.asq.desc_buf.pa))  in iavf_config_asq_regs()
307  *  iavf_config_arq_regs - ARQ register configuration
318 	wr32(hw, hw->aq.arq.head, 0);  in iavf_config_arq_regs()
319 	wr32(hw, hw->aq.arq.tail, 0);  in iavf_config_arq_regs()
322 	wr32(hw, hw->aq.arq.len, (hw->aq.num_arq_entries |  in iavf_config_arq_regs()
324 	wr32(hw, hw->aq.arq.bal, IAVF_LO_DWORD(hw->aq.arq.desc_buf.pa));  in iavf_config_arq_regs()
325 	wr32(hw, hw->aq.arq.bah, IAVF_HI_DWORD(hw->aq.arq.desc_buf.pa));  in iavf_config_arq_regs()
327 	/* Update tail in the HW to post pre-allocated buffers */  in iavf_config_arq_regs()
328 	wr32(hw, hw->aq.arq.tail, hw->aq.num_arq_entries - 1);  in iavf_config_arq_regs()
331 	reg = rd32(hw, hw->aq.arq.bal);  in iavf_config_arq_regs()
332 	if (reg != IAVF_LO_DWORD(hw->aq.arq.desc_buf.pa))  in iavf_config_arq_regs()
339  *  iavf_init_asq - main initialization routine for ASQ
344  *  in the hw->aq structure:
345  *     - hw->aq.num_asq_entries
346  *     - hw->aq.arq_buf_size
355 	if (hw->aq.asq.count > 0) {  in iavf_init_asq()
356 		/* queue already initialized */  in iavf_init_asq()
362 	if ((hw->aq.num_asq_entries == 0) ||  in iavf_init_asq()
363 	    (hw->aq.asq_buf_size == 0)) {  in iavf_init_asq()
368 	hw->aq.asq.next_to_use = 0;  in iavf_init_asq()
369 	hw->aq.asq.next_to_clean = 0;  in iavf_init_asq()
387 	hw->aq.asq.count = hw->aq.num_asq_entries;  in iavf_init_asq()
402  *  iavf_init_arq - initialize ARQ
407  *  in the hw->aq structure:
408  *     - hw->aq.num_asq_entries
409  *     - hw->aq.arq_buf_size
418 	if (hw->aq.arq.count > 0) {  in iavf_init_arq()
419 		/* queue already initialized */  in iavf_init_arq()
425 	if ((hw->aq.num_arq_entries == 0) ||  in iavf_init_arq()
426 	    (hw->aq.arq_buf_size == 0)) {  in iavf_init_arq()
431 	hw->aq.arq.next_to_use = 0;  in iavf_init_arq()
432 	hw->aq.arq.next_to_clean = 0;  in iavf_init_arq()
450 	hw->aq.arq.count = hw->aq.num_arq_entries;  in iavf_init_arq()
461  *  iavf_shutdown_asq - shutdown the ASQ
470 	iavf_acquire_spinlock(&hw->aq.asq_spinlock);  in iavf_shutdown_asq()
472 	if (hw->aq.asq.count == 0) {  in iavf_shutdown_asq()
477 	/* Stop firmware AdminQ processing */  in iavf_shutdown_asq()
478 	wr32(hw, hw->aq.asq.head, 0);  in iavf_shutdown_asq()
479 	wr32(hw, hw->aq.asq.tail, 0);  in iavf_shutdown_asq()
480 	wr32(hw, hw->aq.asq.len, 0);  in iavf_shutdown_asq()
481 	wr32(hw, hw->aq.asq.bal, 0);  in iavf_shutdown_asq()
482 	wr32(hw, hw->aq.asq.bah, 0);  in iavf_shutdown_asq()
484 	hw->aq.asq.count = 0; /* to indicate uninitialized queue */  in iavf_shutdown_asq()
490 	iavf_release_spinlock(&hw->aq.asq_spinlock);  in iavf_shutdown_asq()
495  *  iavf_shutdown_arq - shutdown ARQ
504 	iavf_acquire_spinlock(&hw->aq.arq_spinlock);  in iavf_shutdown_arq()
506 	if (hw->aq.arq.count == 0) {  in iavf_shutdown_arq()
511 	/* Stop firmware AdminQ processing */  in iavf_shutdown_arq()
512 	wr32(hw, hw->aq.arq.head, 0);  in iavf_shutdown_arq()
513 	wr32(hw, hw->aq.arq.tail, 0);  in iavf_shutdown_arq()
514 	wr32(hw, hw->aq.arq.len, 0);  in iavf_shutdown_arq()
515 	wr32(hw, hw->aq.arq.bal, 0);  in iavf_shutdown_arq()
516 	wr32(hw, hw->aq.arq.bah, 0);  in iavf_shutdown_arq()
518 	hw->aq.arq.count = 0; /* to indicate uninitialized queue */  in iavf_shutdown_arq()
524 	iavf_release_spinlock(&hw->aq.arq_spinlock);  in iavf_shutdown_arq()
529  *  iavf_init_adminq - main initialization routine for Admin Queue
533  *  in the hw->aq structure:
534  *     - hw->aq.num_asq_entries
535  *     - hw->aq.num_arq_entries
536  *     - hw->aq.arq_buf_size
537  *     - hw->aq.asq_buf_size
544 	if ((hw->aq.num_arq_entries == 0) ||  in iavf_init_adminq()
545 	    (hw->aq.num_asq_entries == 0) ||  in iavf_init_adminq()
546 	    (hw->aq.arq_buf_size == 0) ||  in iavf_init_adminq()
547 	    (hw->aq.asq_buf_size == 0)) {  in iavf_init_adminq()
551 	iavf_init_spinlock(&hw->aq.asq_spinlock);  in iavf_init_adminq()
552 	iavf_init_spinlock(&hw->aq.arq_spinlock);  in iavf_init_adminq()
558 	hw->aq.asq_cmd_timeout = IAVF_ASQ_CMD_TIMEOUT;  in iavf_init_adminq()
576 	iavf_destroy_spinlock(&hw->aq.asq_spinlock);  in iavf_init_adminq()
577 	iavf_destroy_spinlock(&hw->aq.arq_spinlock);  in iavf_init_adminq()
584  *  iavf_shutdown_adminq - shutdown routine for the Admin Queue
596 	iavf_destroy_spinlock(&hw->aq.asq_spinlock);  in iavf_shutdown_adminq()
597 	iavf_destroy_spinlock(&hw->aq.arq_spinlock);  in iavf_shutdown_adminq()
603  *  iavf_clean_asq - cleans Admin send queue
610 	struct iavf_adminq_ring *asq = &(hw->aq.asq);  in iavf_clean_asq()
612 	u16 ntc = asq->next_to_clean;  in iavf_clean_asq()
618 	while (rd32(hw, hw->aq.asq.head) != ntc) {  in iavf_clean_asq()
620 			   "ntc %d head %d.\n", ntc, rd32(hw, hw->aq.asq.head));  in iavf_clean_asq()
622 		if (details->callback) {  in iavf_clean_asq()
624 					(IAVF_ADMINQ_CALLBACK)details->callback;  in iavf_clean_asq()
632 		if (ntc == asq->count)  in iavf_clean_asq()
638 	asq->next_to_clean = ntc;  in iavf_clean_asq()
644  *  iavf_asq_done - check if FW has processed the Admin Send Queue
647  *  Returns true if the firmware has processed all descriptors on the
655 	return rd32(hw, hw->aq.asq.head) == hw->aq.asq.next_to_use;  in iavf_asq_done()
660  *  iavf_asq_send_command - send command to Admin Queue
684 	iavf_acquire_spinlock(&hw->aq.asq_spinlock);  in iavf_asq_send_command()
686 	hw->aq.asq_last_status = IAVF_AQ_RC_OK;  in iavf_asq_send_command()
688 	if (hw->aq.asq.count == 0) {  in iavf_asq_send_command()
690 			   "AQTX: Admin queue not initialized.\n");  in iavf_asq_send_command()
695 	val = rd32(hw, hw->aq.asq.head);  in iavf_asq_send_command()
696 	if (val >= hw->aq.num_asq_entries) {  in iavf_asq_send_command()
703 	details = IAVF_ADMINQ_DETAILS(hw->aq.asq, hw->aq.asq.next_to_use);  in iavf_asq_send_command()
714 		if (details->cookie) {  in iavf_asq_send_command()
715 			desc->cookie_high =  in iavf_asq_send_command()
716 				CPU_TO_LE32(IAVF_HI_DWORD(details->cookie));  in iavf_asq_send_command()
717 			desc->cookie_low =  in iavf_asq_send_command()
718 				CPU_TO_LE32(IAVF_LO_DWORD(details->cookie));  in iavf_asq_send_command()
727 	desc->flags &= ~CPU_TO_LE16(details->flags_dis);  in iavf_asq_send_command()
728 	desc->flags |= CPU_TO_LE16(details->flags_ena);  in iavf_asq_send_command()
730 	if (buff_size > hw->aq.asq_buf_size) {  in iavf_asq_send_command()
739 	if (details->postpone && !details->async) {  in iavf_asq_send_command()
763 	desc_on_ring = IAVF_ADMINQ_DESC(hw->aq.asq, hw->aq.asq.next_to_use);  in iavf_asq_send_command()
771 		dma_buff = &(hw->aq.asq.r.asq_bi[hw->aq.asq.next_to_use]);  in iavf_asq_send_command()
773 		iavf_memcpy(dma_buff->va, buff, buff_size,  in iavf_asq_send_command()
775 		desc_on_ring->datalen = CPU_TO_LE16(buff_size);  in iavf_asq_send_command()
780 		desc_on_ring->params.external.addr_high =  in iavf_asq_send_command()
781 				CPU_TO_LE32(IAVF_HI_DWORD(dma_buff->pa));  in iavf_asq_send_command()
782 		desc_on_ring->params.external.addr_low =  in iavf_asq_send_command()
783 				CPU_TO_LE32(IAVF_LO_DWORD(dma_buff->pa));  in iavf_asq_send_command()
790 	(hw->aq.asq.next_to_use)++;  in iavf_asq_send_command()
791 	if (hw->aq.asq.next_to_use == hw->aq.asq.count)  in iavf_asq_send_command()
792 		hw->aq.asq.next_to_use = 0;  in iavf_asq_send_command()
793 	if (!details->postpone)  in iavf_asq_send_command()
794 		wr32(hw, hw->aq.asq.tail, hw->aq.asq.next_to_use);  in iavf_asq_send_command()
799 	if (!details->async && !details->postpone) {  in iavf_asq_send_command()
810 		} while (total_delay < hw->aq.asq_cmd_timeout);  in iavf_asq_send_command()
818 			iavf_memcpy(buff, dma_buff->va, buff_size,  in iavf_asq_send_command()
820 		retval = LE16_TO_CPU(desc->retval);  in iavf_asq_send_command()
837 		hw->aq.asq_last_status = (enum iavf_admin_queue_err)retval;  in iavf_asq_send_command()
845 	if (details->wb_desc)  in iavf_asq_send_command()
846 		iavf_memcpy(details->wb_desc, desc_on_ring,  in iavf_asq_send_command()
851 	    (!details->async && !details->postpone)) {  in iavf_asq_send_command()
852 		if (rd32(hw, hw->aq.asq.len) & IAVF_VF_ATQLEN1_ATQCRIT_MASK) {  in iavf_asq_send_command()
864 	iavf_release_spinlock(&hw->aq.asq_spinlock);  in iavf_asq_send_command()
869  *  iavf_fill_default_direct_cmd_desc - AQ descriptor helper function
881 	desc->opcode = CPU_TO_LE16(opcode);  in iavf_fill_default_direct_cmd_desc()
882 	desc->flags = CPU_TO_LE16(IAVF_AQ_FLAG_SI);  in iavf_fill_default_direct_cmd_desc()
900 	u16 ntc = hw->aq.arq.next_to_clean;  in iavf_clean_arq_element()
908 	/* pre-clean the event info */  in iavf_clean_arq_element()
909 	iavf_memset(&e->desc, 0, sizeof(e->desc), IAVF_NONDMA_MEM);  in iavf_clean_arq_element()
912 	iavf_acquire_spinlock(&hw->aq.arq_spinlock);  in iavf_clean_arq_element()
914 	if (hw->aq.arq.count == 0) {  in iavf_clean_arq_element()
916 			   "AQRX: Admin queue not initialized.\n");  in iavf_clean_arq_element()
922 	ntu = rd32(hw, hw->aq.arq.head) & IAVF_VF_ARQH1_ARQH_MASK;  in iavf_clean_arq_element()
924 		/* nothing to do - shouldn't need to update ring's values */  in iavf_clean_arq_element()
930 	desc = IAVF_ADMINQ_DESC(hw->aq.arq, ntc);  in iavf_clean_arq_element()
933 	hw->aq.arq_last_status =  in iavf_clean_arq_element()
934 		(enum iavf_admin_queue_err)LE16_TO_CPU(desc->retval);  in iavf_clean_arq_element()
935 	flags = LE16_TO_CPU(desc->flags);  in iavf_clean_arq_element()
941 			   hw->aq.arq_last_status);  in iavf_clean_arq_element()
944 	iavf_memcpy(&e->desc, desc, sizeof(struct iavf_aq_desc),  in iavf_clean_arq_element()
946 	datalen = LE16_TO_CPU(desc->datalen);  in iavf_clean_arq_element()
947 	e->msg_len = min(datalen, e->buf_len);  in iavf_clean_arq_element()
948 	if (e->msg_buf != NULL && (e->msg_len != 0))  in iavf_clean_arq_element()
949 		iavf_memcpy(e->msg_buf,  in iavf_clean_arq_element()
950 			    hw->aq.arq.r.arq_bi[desc_idx].va,  in iavf_clean_arq_element()
951 			    e->msg_len, IAVF_DMA_TO_NONDMA);  in iavf_clean_arq_element()
954 	iavf_debug_aq(hw, IAVF_DEBUG_AQ_COMMAND, (void *)desc, e->msg_buf,  in iavf_clean_arq_element()
955 		      hw->aq.arq_buf_size);  in iavf_clean_arq_element()
961 	bi = &hw->aq.arq.r.arq_bi[ntc];  in iavf_clean_arq_element()
964 	desc->flags = CPU_TO_LE16(IAVF_AQ_FLAG_BUF);  in iavf_clean_arq_element()
965 	if (hw->aq.arq_buf_size > IAVF_AQ_LARGE_BUF)  in iavf_clean_arq_element()
966 		desc->flags |= CPU_TO_LE16(IAVF_AQ_FLAG_LB);  in iavf_clean_arq_element()
967 	desc->datalen = CPU_TO_LE16((u16)bi->size);  in iavf_clean_arq_element()
968 	desc->params.external.addr_high = CPU_TO_LE32(IAVF_HI_DWORD(bi->pa));  in iavf_clean_arq_element()
969 	desc->params.external.addr_low = CPU_TO_LE32(IAVF_LO_DWORD(bi->pa));  in iavf_clean_arq_element()
972 	wr32(hw, hw->aq.arq.tail, ntc);  in iavf_clean_arq_element()
975 	if (ntc == hw->aq.num_arq_entries)  in iavf_clean_arq_element()
977 	hw->aq.arq.next_to_clean = ntc;  in iavf_clean_arq_element()
978 	hw->aq.arq.next_to_use = ntu;  in iavf_clean_arq_element()
983 		*pending = (ntc > ntu ? hw->aq.arq.count : 0) + (ntu - ntc);  in iavf_clean_arq_element()
985 	iavf_release_spinlock(&hw->aq.arq_spinlock);  in iavf_clean_arq_element()