93 static int ena_com_admin_init_sq(struct ena_com_admin_queue *admin_queue)  in ena_com_admin_init_sq()  argument
95 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_admin_init_sq()
96 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_init_sq()
97 	u16 size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_init_sq()
99 	sq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size, &sq->dma_addr, GFP_KERNEL);  in ena_com_admin_init_sq()
115 static int ena_com_admin_init_cq(struct ena_com_admin_queue *admin_queue)  in ena_com_admin_init_cq()  argument
117 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_admin_init_cq()
118 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_init_cq()
119 	u16 size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_init_cq()
121 	cq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size, &cq->dma_addr, GFP_KERNEL);  in ena_com_admin_init_cq()
183 static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *admin_queue,  in get_comp_ctxt()  argument
186 	if (unlikely(command_id >= admin_queue->q_depth)) {  in get_comp_ctxt()
187 		netdev_err(admin_queue->ena_dev->net_device,  in get_comp_ctxt()
189 			   command_id, admin_queue->q_depth);  in get_comp_ctxt()
193 	if (unlikely(!admin_queue->comp_ctx)) {  in get_comp_ctxt()
194 		netdev_err(admin_queue->ena_dev->net_device, "Completion context is NULL\n");  in get_comp_ctxt()
198 	if (unlikely(admin_queue->comp_ctx[command_id].occupied && capture)) {  in get_comp_ctxt()
199 		netdev_err(admin_queue->ena_dev->net_device, "Completion context is occupied\n");  in get_comp_ctxt()
204 		atomic_inc(&admin_queue->outstanding_cmds);  in get_comp_ctxt()
205 		admin_queue->comp_ctx[command_id].occupied = true;  in get_comp_ctxt()
208 	return &admin_queue->comp_ctx[command_id];  in get_comp_ctxt()
211 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in __ena_com_submit_admin_cmd()  argument
222 	queue_size_mask = admin_queue->q_depth - 1;  in __ena_com_submit_admin_cmd()
224 	tail_masked = admin_queue->sq.tail & queue_size_mask;  in __ena_com_submit_admin_cmd()
227 	cnt = (u16)atomic_read(&admin_queue->outstanding_cmds);  in __ena_com_submit_admin_cmd()
228 	if (cnt >= admin_queue->q_depth) {  in __ena_com_submit_admin_cmd()
229 		netdev_dbg(admin_queue->ena_dev->net_device, "Admin queue is full.\n");  in __ena_com_submit_admin_cmd()
230 		admin_queue->stats.out_of_space++;  in __ena_com_submit_admin_cmd()
234 	cmd_id = admin_queue->curr_cmd_id;  in __ena_com_submit_admin_cmd()
236 	cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &  in __ena_com_submit_admin_cmd()
242 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);  in __ena_com_submit_admin_cmd()
253 	memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);  in __ena_com_submit_admin_cmd()
255 	admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &  in __ena_com_submit_admin_cmd()
258 	admin_queue->sq.tail++;  in __ena_com_submit_admin_cmd()
259 	admin_queue->stats.submitted_cmd++;  in __ena_com_submit_admin_cmd()
261 	if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))  in __ena_com_submit_admin_cmd()
262 		admin_queue->sq.phase = !admin_queue->sq.phase;  in __ena_com_submit_admin_cmd()
264 	writel(admin_queue->sq.tail, admin_queue->sq.db_addr);  in __ena_com_submit_admin_cmd()
269 static int ena_com_init_comp_ctxt(struct ena_com_admin_queue *admin_queue)  in ena_com_init_comp_ctxt()  argument
271 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_init_comp_ctxt()
272 	size_t size = admin_queue->q_depth * sizeof(struct ena_comp_ctx);  in ena_com_init_comp_ctxt()
276 	admin_queue->comp_ctx = devm_kzalloc(admin_queue->q_dmadev, size, GFP_KERNEL);  in ena_com_init_comp_ctxt()
277 	if (unlikely(!admin_queue->comp_ctx)) {  in ena_com_init_comp_ctxt()
282 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_init_comp_ctxt()
283 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_init_comp_ctxt()
291 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in ena_com_submit_admin_cmd()  argument
300 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
301 	if (unlikely(!admin_queue->running_state)) {  in ena_com_submit_admin_cmd()
302 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
305 	comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,  in ena_com_submit_admin_cmd()
310 		admin_queue->running_state = false;  in ena_com_submit_admin_cmd()
311 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
429 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,  in ena_com_handle_single_admin_completion()  argument
438 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);  in ena_com_handle_single_admin_completion()
440 		netdev_err(admin_queue->ena_dev->net_device,  in ena_com_handle_single_admin_completion()
442 		admin_queue->running_state = false;  in ena_com_handle_single_admin_completion()
452 	if (!admin_queue->polling)  in ena_com_handle_single_admin_completion()
456 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)  in ena_com_handle_admin_completion()  argument
463 	head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);  in ena_com_handle_admin_completion()
464 	phase = admin_queue->cq.phase;  in ena_com_handle_admin_completion()
466 	cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
475 		ena_com_handle_single_admin_completion(admin_queue, cqe);  in ena_com_handle_admin_completion()
479 		if (unlikely(head_masked == admin_queue->q_depth)) {  in ena_com_handle_admin_completion()
484 		cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
487 	admin_queue->cq.head += comp_num;  in ena_com_handle_admin_completion()
488 	admin_queue->cq.phase = phase;  in ena_com_handle_admin_completion()
489 	admin_queue->sq.head += comp_num;  in ena_com_handle_admin_completion()
490 	admin_queue->stats.completed_cmd += comp_num;  in ena_com_handle_admin_completion()
493 static int ena_com_comp_status_to_errno(struct ena_com_admin_queue *admin_queue,  in ena_com_comp_status_to_errno()  argument
497 		netdev_err(admin_queue->ena_dev->net_device, "Admin command failed[%u]\n",  in ena_com_comp_status_to_errno()
528 						     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_polling()  argument
535 	timeout = jiffies + usecs_to_jiffies(admin_queue->completion_timeout);  in ena_com_wait_and_process_admin_cq_polling()
538 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
539 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_polling()
540 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
546 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_wait_and_process_admin_cq_polling()
549 			spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
550 			admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_polling()
551 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_polling()
552 			spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
559 						 admin_queue->ena_dev->ena_min_poll_delay_us);  in ena_com_wait_and_process_admin_cq_polling()
563 		netdev_err(admin_queue->ena_dev->net_device, "Command was aborted\n");  in ena_com_wait_and_process_admin_cq_polling()
564 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
565 		admin_queue->stats.aborted_cmd++;  in ena_com_wait_and_process_admin_cq_polling()
566 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
573 	ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);  in ena_com_wait_and_process_admin_cq_polling()
575 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_polling()
587 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_llq()  local
594 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_llq()
608 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_llq()
751 							struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_interrupts()  argument
757 				    usecs_to_jiffies(admin_queue->completion_timeout));  in ena_com_wait_and_process_admin_cq_interrupts()
765 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
766 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_interrupts()
767 		admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_interrupts()
768 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
771 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_wait_and_process_admin_cq_interrupts()
775 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_wait_and_process_admin_cq_interrupts()
779 		admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_interrupts()
784 	ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);  in ena_com_wait_and_process_admin_cq_interrupts()
786 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_interrupts()
858 					     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq()  argument
860 	if (admin_queue->polling)  in ena_com_wait_and_process_admin_cq()
862 								 admin_queue);  in ena_com_wait_and_process_admin_cq()
865 							    admin_queue);  in ena_com_wait_and_process_admin_cq()
871 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_sq()  local
891 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_sq()
981 	struct ena_com_admin_queue *admin_queue;  in ena_com_get_feature_ex()  local
991 	admin_queue = &ena_dev->admin_queue;  in ena_com_get_feature_ex()
1013 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_get_feature_ex()
1176 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_sq()  local
1218 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_sq()
1297 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,  in ena_com_execute_admin_command()  argument
1306 	comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,  in ena_com_execute_admin_command()
1311 			netdev_dbg(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1314 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1320 	ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);  in ena_com_execute_admin_command()
1322 		if (admin_queue->running_state)  in ena_com_execute_admin_command()
1323 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1326 			netdev_dbg(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1335 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_cq()  local
1360 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_cq()
1403 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_abort_admin_commands()  local
1407 	if (!admin_queue->comp_ctx)  in ena_com_abort_admin_commands()
1410 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_abort_admin_commands()
1411 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_abort_admin_commands()
1423 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_wait_for_abort_completion()  local
1427 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1428 	while (atomic_read(&admin_queue->outstanding_cmds) != 0) {  in ena_com_wait_for_abort_completion()
1429 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1431 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1433 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1439 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_cq()  local
1449 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_cq()
1463 	return ena_dev->admin_queue.running_state;  in ena_com_get_admin_running_state()
1468 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_admin_running_state()  local
1471 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1472 	ena_dev->admin_queue.running_state = state;  in ena_com_set_admin_running_state()
1473 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1490 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_aenq_config()  local
1510 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_aenq_config()
1517 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_aenq_config()
1602 				      struct ena_com_admin_queue *admin_queue)  in ena_com_free_ena_admin_queue_comp_ctx()  argument
1605 	if (!admin_queue->comp_ctx)  in ena_com_free_ena_admin_queue_comp_ctx()
1608 	devm_kfree(ena_dev->dmadev, admin_queue->comp_ctx);  in ena_com_free_ena_admin_queue_comp_ctx()
1610 	admin_queue->comp_ctx = NULL;  in ena_com_free_ena_admin_queue_comp_ctx()
1615 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_destroy()  local
1616 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_destroy()
1617 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_destroy()
1621 	ena_com_free_ena_admin_queue_comp_ctx(ena_dev, admin_queue);  in ena_com_admin_destroy()
1623 	size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1628 	size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1647 	ena_dev->admin_queue.polling = polling;  in ena_com_set_admin_polling_mode()
1744 	ret = ena_com_execute_admin_command(&ena_dev->admin_queue,  in ena_com_phc_config()
1969 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_init()  local
1985 	admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;  in ena_com_admin_init()
1987 	admin_queue->q_dmadev = ena_dev->dmadev;  in ena_com_admin_init()
1988 	admin_queue->polling = false;  in ena_com_admin_init()
1989 	admin_queue->curr_cmd_id = 0;  in ena_com_admin_init()
1991 	atomic_set(&admin_queue->outstanding_cmds, 0);  in ena_com_admin_init()
1993 	spin_lock_init(&admin_queue->q_lock);  in ena_com_admin_init()
1995 	ret = ena_com_init_comp_ctxt(admin_queue);  in ena_com_admin_init()
1999 	ret = ena_com_admin_init_sq(admin_queue);  in ena_com_admin_init()
2003 	ret = ena_com_admin_init_cq(admin_queue);  in ena_com_admin_init()
2007 	admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +  in ena_com_admin_init()
2010 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);  in ena_com_admin_init()
2011 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);  in ena_com_admin_init()
2016 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);  in ena_com_admin_init()
2017 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);  in ena_com_admin_init()
2023 	aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;  in ena_com_admin_init()
2029 	acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;  in ena_com_admin_init()
2040 	admin_queue->ena_dev = ena_dev;  in ena_com_admin_init()
2041 	admin_queue->running_state = true;  in ena_com_admin_init()
2142 	struct ena_com_admin_queue *admin_queue;  in ena_get_dev_stats()  local
2145 	admin_queue = &ena_dev->admin_queue;  in ena_get_dev_stats()
2151 	ret = ena_com_execute_admin_command(admin_queue,  in ena_get_dev_stats()
2272 	ena_com_handle_admin_completion(&ena_dev->admin_queue);  in ena_com_admin_q_comp_intr_handler()
2404 		ena_dev->admin_queue.completion_timeout = timeout * 100000;  in ena_com_dev_reset()
2406 		ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US;  in ena_com_dev_reset()
2499 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_dev_mtu()  local
2510 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_dev_mtu()
2517 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_dev_mtu()
2531 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_function()  local
2575 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_function()
2702 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_ctrl()  local
2734 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_ctrl()
2863 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_indirect_table_set()  local
2902 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_indirect_table_set()
3074 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_host_attributes()  local
3085 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_host_attributes()
3108 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_host_attributes()