Lines Matching +full:cmd +full:- +full:timeout +full:- +full:ms
1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
4 * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates.
39 /* Timeout in micro-sec */
94 /* Abort - canceled by the driver */
118 if (unlikely((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr)) { in ena_com_mem_addr_set()
123 ena_addr->mem_addr_low = lower_32_bits(addr); in ena_com_mem_addr_set()
124 ena_addr->mem_addr_high = (u16)upper_32_bits(addr); in ena_com_mem_addr_set()
131 struct ena_com_dev *ena_dev = admin_queue->ena_dev; in ena_com_admin_init_sq()
132 struct ena_com_admin_sq *sq = &admin_queue->sq; in ena_com_admin_init_sq()
133 u16 size = ADMIN_SQ_SIZE(admin_queue->q_depth); in ena_com_admin_init_sq()
135 ENA_MEM_ALLOC_COHERENT(admin_queue->q_dmadev, size, sq->entries, sq->dma_addr, in ena_com_admin_init_sq()
136 sq->mem_handle); in ena_com_admin_init_sq()
138 if (unlikely(!sq->entries)) { in ena_com_admin_init_sq()
143 sq->head = 0; in ena_com_admin_init_sq()
144 sq->tail = 0; in ena_com_admin_init_sq()
145 sq->phase = 1; in ena_com_admin_init_sq()
147 sq->db_addr = NULL; in ena_com_admin_init_sq()
154 struct ena_com_dev *ena_dev = admin_queue->ena_dev; in ena_com_admin_init_cq()
155 struct ena_com_admin_cq *cq = &admin_queue->cq; in ena_com_admin_init_cq()
156 u16 size = ADMIN_CQ_SIZE(admin_queue->q_depth); in ena_com_admin_init_cq()
158 ENA_MEM_ALLOC_COHERENT(admin_queue->q_dmadev, size, cq->entries, cq->dma_addr, in ena_com_admin_init_cq()
159 cq->mem_handle); in ena_com_admin_init_cq()
161 if (unlikely(!cq->entries)) { in ena_com_admin_init_cq()
166 cq->head = 0; in ena_com_admin_init_cq()
167 cq->phase = 1; in ena_com_admin_init_cq()
175 struct ena_com_aenq *aenq = &ena_dev->aenq; in ena_com_admin_init_aenq()
179 ena_dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH; in ena_com_admin_init_aenq()
181 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, size, in ena_com_admin_init_aenq()
182 aenq->entries, in ena_com_admin_init_aenq()
183 aenq->dma_addr, in ena_com_admin_init_aenq()
184 aenq->mem_handle); in ena_com_admin_init_aenq()
186 if (unlikely(!aenq->entries)) { in ena_com_admin_init_aenq()
191 aenq->head = aenq->q_depth; in ena_com_admin_init_aenq()
192 aenq->phase = 1; in ena_com_admin_init_aenq()
194 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(aenq->dma_addr); in ena_com_admin_init_aenq()
195 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(aenq->dma_addr); in ena_com_admin_init_aenq()
197 ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_AENQ_BASE_LO_OFF); in ena_com_admin_init_aenq()
198 ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_AENQ_BASE_HI_OFF); in ena_com_admin_init_aenq()
201 aenq_caps |= ena_dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK; in ena_com_admin_init_aenq()
205 ENA_REG_WRITE32(ena_dev->bus, aenq_caps, ena_dev->reg_bar + ENA_REGS_AENQ_CAPS_OFF); in ena_com_admin_init_aenq()
212 aenq->aenq_handlers = aenq_handlers; in ena_com_admin_init_aenq()
220 comp_ctx->user_cqe = NULL; in comp_ctxt_release()
221 comp_ctx->occupied = false; in comp_ctxt_release()
222 ATOMIC32_DEC(&queue->outstanding_cmds); in comp_ctxt_release()
228 if (unlikely(command_id >= admin_queue->q_depth)) { in get_comp_ctxt()
229 ena_trc_err(admin_queue->ena_dev, in get_comp_ctxt()
231 command_id, admin_queue->q_depth); in get_comp_ctxt()
235 if (unlikely(!admin_queue->comp_ctx)) { in get_comp_ctxt()
236 ena_trc_err(admin_queue->ena_dev, in get_comp_ctxt()
241 if (unlikely(admin_queue->comp_ctx[command_id].occupied && capture)) { in get_comp_ctxt()
242 ena_trc_err(admin_queue->ena_dev, in get_comp_ctxt()
248 ATOMIC32_INC(&admin_queue->outstanding_cmds); in get_comp_ctxt()
249 admin_queue->comp_ctx[command_id].occupied = true; in get_comp_ctxt()
252 return &admin_queue->comp_ctx[command_id]; in get_comp_ctxt()
256 struct ena_admin_aq_entry *cmd, in __ena_com_submit_admin_cmd() argument
266 queue_size_mask = admin_queue->q_depth - 1; in __ena_com_submit_admin_cmd()
268 tail_masked = admin_queue->sq.tail & queue_size_mask; in __ena_com_submit_admin_cmd()
271 cnt = (u16)ATOMIC32_READ(&admin_queue->outstanding_cmds); in __ena_com_submit_admin_cmd()
272 if (unlikely(cnt >= admin_queue->q_depth)) { in __ena_com_submit_admin_cmd()
273 ena_trc_dbg(admin_queue->ena_dev, "Admin queue is full.\n"); in __ena_com_submit_admin_cmd()
274 admin_queue->stats.out_of_space++; in __ena_com_submit_admin_cmd()
278 cmd_id = admin_queue->curr_cmd_id; in __ena_com_submit_admin_cmd()
280 cmd->aq_common_descriptor.flags |= admin_queue->sq.phase & in __ena_com_submit_admin_cmd()
283 cmd->aq_common_descriptor.command_id |= cmd_id & in __ena_com_submit_admin_cmd()
290 comp_ctx->status = ENA_CMD_SUBMITTED; in __ena_com_submit_admin_cmd()
291 comp_ctx->comp_size = (u32)comp_size_in_bytes; in __ena_com_submit_admin_cmd()
292 comp_ctx->user_cqe = comp; in __ena_com_submit_admin_cmd()
293 comp_ctx->cmd_opcode = cmd->aq_common_descriptor.opcode; in __ena_com_submit_admin_cmd()
295 ENA_WAIT_EVENT_CLEAR(comp_ctx->wait_event); in __ena_com_submit_admin_cmd()
297 memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes); in __ena_com_submit_admin_cmd()
299 admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) & in __ena_com_submit_admin_cmd()
302 admin_queue->sq.tail++; in __ena_com_submit_admin_cmd()
303 admin_queue->stats.submitted_cmd++; in __ena_com_submit_admin_cmd()
305 if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0)) in __ena_com_submit_admin_cmd()
306 admin_queue->sq.phase = !admin_queue->sq.phase; in __ena_com_submit_admin_cmd()
308 ENA_DB_SYNC(&admin_queue->sq.mem_handle); in __ena_com_submit_admin_cmd()
309 ENA_REG_WRITE32(admin_queue->bus, admin_queue->sq.tail, in __ena_com_submit_admin_cmd()
310 admin_queue->sq.db_addr); in __ena_com_submit_admin_cmd()
317 struct ena_com_dev *ena_dev = admin_queue->ena_dev; in ena_com_init_comp_ctxt()
318 size_t size = admin_queue->q_depth * sizeof(struct ena_comp_ctx); in ena_com_init_comp_ctxt()
322 admin_queue->comp_ctx = ENA_MEM_ALLOC(admin_queue->q_dmadev, size); in ena_com_init_comp_ctxt()
323 if (unlikely(!admin_queue->comp_ctx)) { in ena_com_init_comp_ctxt()
328 for (i = 0; i < admin_queue->q_depth; i++) { in ena_com_init_comp_ctxt()
331 ENA_WAIT_EVENT_INIT(comp_ctx->wait_event); in ena_com_init_comp_ctxt()
338 struct ena_admin_aq_entry *cmd, in ena_com_submit_admin_cmd() argument
346 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); in ena_com_submit_admin_cmd()
347 if (unlikely(!admin_queue->running_state)) { in ena_com_submit_admin_cmd()
348 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); in ena_com_submit_admin_cmd()
351 comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd, in ena_com_submit_admin_cmd()
356 admin_queue->running_state = false; in ena_com_submit_admin_cmd()
357 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); in ena_com_submit_admin_cmd()
369 memset(&io_sq->desc_addr, 0x0, sizeof(io_sq->desc_addr)); in ena_com_init_io_sq()
371 io_sq->dma_addr_bits = (u8)ena_dev->dma_addr_bits; in ena_com_init_io_sq()
372 io_sq->desc_entry_size = in ena_com_init_io_sq()
373 (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ? in ena_com_init_io_sq()
377 size = io_sq->desc_entry_size * io_sq->q_depth; in ena_com_init_io_sq()
378 io_sq->bus = ena_dev->bus; in ena_com_init_io_sq()
380 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) { in ena_com_init_io_sq()
381 ENA_MEM_ALLOC_COHERENT_NODE(ena_dev->dmadev, in ena_com_init_io_sq()
383 io_sq->desc_addr.virt_addr, in ena_com_init_io_sq()
384 io_sq->desc_addr.phys_addr, in ena_com_init_io_sq()
385 io_sq->desc_addr.mem_handle, in ena_com_init_io_sq()
386 ctx->numa_node, in ena_com_init_io_sq()
388 if (!io_sq->desc_addr.virt_addr) { in ena_com_init_io_sq()
389 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, in ena_com_init_io_sq()
391 io_sq->desc_addr.virt_addr, in ena_com_init_io_sq()
392 io_sq->desc_addr.phys_addr, in ena_com_init_io_sq()
393 io_sq->desc_addr.mem_handle); in ena_com_init_io_sq()
396 if (unlikely(!io_sq->desc_addr.virt_addr)) { in ena_com_init_io_sq()
402 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { in ena_com_init_io_sq()
404 io_sq->bounce_buf_ctrl.buffer_size = in ena_com_init_io_sq()
405 ena_dev->llq_info.desc_list_entry_size; in ena_com_init_io_sq()
406 io_sq->bounce_buf_ctrl.buffers_num = in ena_com_init_io_sq()
408 io_sq->bounce_buf_ctrl.next_to_use = 0; in ena_com_init_io_sq()
410 size = (size_t)io_sq->bounce_buf_ctrl.buffer_size * in ena_com_init_io_sq()
411 io_sq->bounce_buf_ctrl.buffers_num; in ena_com_init_io_sq()
413 ENA_MEM_ALLOC_NODE(ena_dev->dmadev, in ena_com_init_io_sq()
415 io_sq->bounce_buf_ctrl.base_buffer, in ena_com_init_io_sq()
416 ctx->numa_node, in ena_com_init_io_sq()
418 if (!io_sq->bounce_buf_ctrl.base_buffer) in ena_com_init_io_sq()
419 io_sq->bounce_buf_ctrl.base_buffer = ENA_MEM_ALLOC(ena_dev->dmadev, size); in ena_com_init_io_sq()
421 if (unlikely(!io_sq->bounce_buf_ctrl.base_buffer)) { in ena_com_init_io_sq()
426 memcpy(&io_sq->llq_info, &ena_dev->llq_info, in ena_com_init_io_sq()
427 sizeof(io_sq->llq_info)); in ena_com_init_io_sq()
430 io_sq->llq_buf_ctrl.curr_bounce_buf = in ena_com_init_io_sq()
431 ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl); in ena_com_init_io_sq()
432 memset(io_sq->llq_buf_ctrl.curr_bounce_buf, in ena_com_init_io_sq()
433 0x0, io_sq->llq_info.desc_list_entry_size); in ena_com_init_io_sq()
434 io_sq->llq_buf_ctrl.descs_left_in_line = in ena_com_init_io_sq()
435 io_sq->llq_info.descs_num_before_header; in ena_com_init_io_sq()
436 io_sq->disable_meta_caching = in ena_com_init_io_sq()
437 io_sq->llq_info.disable_meta_caching; in ena_com_init_io_sq()
439 if (io_sq->llq_info.max_entries_in_tx_burst > 0) in ena_com_init_io_sq()
440 io_sq->entries_in_tx_burst_left = in ena_com_init_io_sq()
441 io_sq->llq_info.max_entries_in_tx_burst; in ena_com_init_io_sq()
444 io_sq->tail = 0; in ena_com_init_io_sq()
445 io_sq->next_to_comp = 0; in ena_com_init_io_sq()
446 io_sq->phase = 1; in ena_com_init_io_sq()
458 memset(&io_cq->cdesc_addr, 0x0, sizeof(io_cq->cdesc_addr)); in ena_com_init_io_cq()
461 io_cq->cdesc_entry_size_in_bytes = in ena_com_init_io_cq()
462 (io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ? in ena_com_init_io_cq()
466 size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth; in ena_com_init_io_cq()
467 io_cq->bus = ena_dev->bus; in ena_com_init_io_cq()
469 ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(ena_dev->dmadev, in ena_com_init_io_cq()
471 io_cq->cdesc_addr.virt_addr, in ena_com_init_io_cq()
472 io_cq->cdesc_addr.phys_addr, in ena_com_init_io_cq()
473 io_cq->cdesc_addr.mem_handle, in ena_com_init_io_cq()
474 ctx->numa_node, in ena_com_init_io_cq()
477 if (!io_cq->cdesc_addr.virt_addr) { in ena_com_init_io_cq()
478 ENA_MEM_ALLOC_COHERENT_ALIGNED(ena_dev->dmadev, in ena_com_init_io_cq()
480 io_cq->cdesc_addr.virt_addr, in ena_com_init_io_cq()
481 io_cq->cdesc_addr.phys_addr, in ena_com_init_io_cq()
482 io_cq->cdesc_addr.mem_handle, in ena_com_init_io_cq()
486 if (unlikely(!io_cq->cdesc_addr.virt_addr)) { in ena_com_init_io_cq()
491 io_cq->phase = 1; in ena_com_init_io_cq()
492 io_cq->head = 0; in ena_com_init_io_cq()
503 cmd_id = cqe->acq_common_descriptor.command & in ena_com_handle_single_admin_completion()
508 ena_trc_err(admin_queue->ena_dev, in ena_com_handle_single_admin_completion()
510 admin_queue->running_state = false; in ena_com_handle_single_admin_completion()
514 if (!comp_ctx->occupied) in ena_com_handle_single_admin_completion()
517 comp_ctx->status = ENA_CMD_COMPLETED; in ena_com_handle_single_admin_completion()
518 comp_ctx->comp_status = cqe->acq_common_descriptor.status; in ena_com_handle_single_admin_completion()
520 if (comp_ctx->user_cqe) in ena_com_handle_single_admin_completion()
521 memcpy(comp_ctx->user_cqe, (void *)cqe, comp_ctx->comp_size); in ena_com_handle_single_admin_completion()
523 if (!admin_queue->polling) in ena_com_handle_single_admin_completion()
524 ENA_WAIT_EVENT_SIGNAL(comp_ctx->wait_event); in ena_com_handle_single_admin_completion()
534 head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1); in ena_com_handle_admin_completion()
535 phase = admin_queue->cq.phase; in ena_com_handle_admin_completion()
537 cqe = &admin_queue->cq.entries[head_masked]; in ena_com_handle_admin_completion()
540 while ((READ_ONCE8(cqe->acq_common_descriptor.flags) & in ena_com_handle_admin_completion()
550 if (unlikely(head_masked == admin_queue->q_depth)) { in ena_com_handle_admin_completion()
555 cqe = &admin_queue->cq.entries[head_masked]; in ena_com_handle_admin_completion()
558 admin_queue->cq.head += comp_num; in ena_com_handle_admin_completion()
559 admin_queue->cq.phase = phase; in ena_com_handle_admin_completion()
560 admin_queue->sq.head += comp_num; in ena_com_handle_admin_completion()
561 admin_queue->stats.completed_cmd += comp_num; in ena_com_handle_admin_completion()
568 ena_trc_err(admin_queue->ena_dev, in ena_com_comp_status_to_errno()
602 ena_time_t timeout; in ena_com_wait_and_process_admin_cq_polling() local
606 timeout = ENA_GET_SYSTEM_TIMEOUT(admin_queue->completion_timeout); in ena_com_wait_and_process_admin_cq_polling()
609 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); in ena_com_wait_and_process_admin_cq_polling()
611 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); in ena_com_wait_and_process_admin_cq_polling()
613 if (comp_ctx->status != ENA_CMD_SUBMITTED) in ena_com_wait_and_process_admin_cq_polling()
616 if (unlikely(ENA_TIME_EXPIRE(timeout))) { in ena_com_wait_and_process_admin_cq_polling()
617 ena_trc_err(admin_queue->ena_dev, in ena_com_wait_and_process_admin_cq_polling()
618 "Wait for completion (polling) timeout\n"); in ena_com_wait_and_process_admin_cq_polling()
620 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); in ena_com_wait_and_process_admin_cq_polling()
621 admin_queue->stats.no_completion++; in ena_com_wait_and_process_admin_cq_polling()
622 admin_queue->running_state = false; in ena_com_wait_and_process_admin_cq_polling()
623 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); in ena_com_wait_and_process_admin_cq_polling()
630 admin_queue->ena_dev->ena_min_poll_delay_us); in ena_com_wait_and_process_admin_cq_polling()
633 if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) { in ena_com_wait_and_process_admin_cq_polling()
634 ena_trc_err(admin_queue->ena_dev, "Command was aborted\n"); in ena_com_wait_and_process_admin_cq_polling()
635 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); in ena_com_wait_and_process_admin_cq_polling()
636 admin_queue->stats.aborted_cmd++; in ena_com_wait_and_process_admin_cq_polling()
637 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); in ena_com_wait_and_process_admin_cq_polling()
642 ENA_WARN(comp_ctx->status != ENA_CMD_COMPLETED, in ena_com_wait_and_process_admin_cq_polling()
643 admin_queue->ena_dev, "Invalid comp status %d\n", in ena_com_wait_and_process_admin_cq_polling()
644 comp_ctx->status); in ena_com_wait_and_process_admin_cq_polling()
646 ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status); in ena_com_wait_and_process_admin_cq_polling()
661 struct ena_admin_set_feat_cmd cmd; in ena_com_set_llq() local
663 struct ena_com_llq_info *llq_info = &ena_dev->llq_info; in ena_com_set_llq()
666 memset(&cmd, 0x0, sizeof(cmd)); in ena_com_set_llq()
667 admin_queue = &ena_dev->admin_queue; in ena_com_set_llq()
669 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; in ena_com_set_llq()
670 cmd.feat_common.feature_id = ENA_ADMIN_LLQ; in ena_com_set_llq()
672 cmd.u.llq.header_location_ctrl_enabled = llq_info->header_location_ctrl; in ena_com_set_llq()
673 cmd.u.llq.entry_size_ctrl_enabled = llq_info->desc_list_entry_size_ctrl; in ena_com_set_llq()
674 cmd.u.llq.desc_num_before_header_enabled = llq_info->descs_num_before_header; in ena_com_set_llq()
675 cmd.u.llq.descriptors_stride_ctrl_enabled = llq_info->desc_stride_ctrl; in ena_com_set_llq()
677 cmd.u.llq.accel_mode.u.set.enabled_flags = in ena_com_set_llq()
682 (struct ena_admin_aq_entry *)&cmd, in ena_com_set_llq()
683 sizeof(cmd), in ena_com_set_llq()
697 struct ena_com_llq_info *llq_info = &ena_dev->llq_info; in ena_com_config_llq_info()
704 supported_feat = llq_features->header_location_ctrl_supported; in ena_com_config_llq_info()
706 if (likely(supported_feat & llq_default_cfg->llq_header_location)) { in ena_com_config_llq_info()
707 llq_info->header_location_ctrl = in ena_com_config_llq_info()
708 llq_default_cfg->llq_header_location; in ena_com_config_llq_info()
715 if (likely(llq_info->header_location_ctrl == ENA_ADMIN_INLINE_HEADER)) { in ena_com_config_llq_info()
716 supported_feat = llq_features->descriptors_stride_ctrl_supported; in ena_com_config_llq_info()
717 if (likely(supported_feat & llq_default_cfg->llq_stride_ctrl)) { in ena_com_config_llq_info()
718 llq_info->desc_stride_ctrl = llq_default_cfg->llq_stride_ctrl; in ena_com_config_llq_info()
721 llq_info->desc_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; in ena_com_config_llq_info()
723 llq_info->desc_stride_ctrl = ENA_ADMIN_SINGLE_DESC_PER_ENTRY; in ena_com_config_llq_info()
731 llq_default_cfg->llq_stride_ctrl, in ena_com_config_llq_info()
733 llq_info->desc_stride_ctrl); in ena_com_config_llq_info()
736 llq_info->desc_stride_ctrl = 0; in ena_com_config_llq_info()
739 supported_feat = llq_features->entry_size_ctrl_supported; in ena_com_config_llq_info()
740 if (likely(supported_feat & llq_default_cfg->llq_ring_entry_size)) { in ena_com_config_llq_info()
741 llq_info->desc_list_entry_size_ctrl = llq_default_cfg->llq_ring_entry_size; in ena_com_config_llq_info()
742 llq_info->desc_list_entry_size = llq_default_cfg->llq_ring_entry_size_value; in ena_com_config_llq_info()
745 llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_128B; in ena_com_config_llq_info()
746 llq_info->desc_list_entry_size = 128; in ena_com_config_llq_info()
748 llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_192B; in ena_com_config_llq_info()
749 llq_info->desc_list_entry_size = 192; in ena_com_config_llq_info()
751 llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_256B; in ena_com_config_llq_info()
752 llq_info->desc_list_entry_size = 256; in ena_com_config_llq_info()
760 llq_default_cfg->llq_ring_entry_size, in ena_com_config_llq_info()
762 llq_info->desc_list_entry_size); in ena_com_config_llq_info()
764 if (unlikely(llq_info->desc_list_entry_size & 0x7)) { in ena_com_config_llq_info()
769 llq_info->desc_list_entry_size); in ena_com_config_llq_info()
773 if (llq_info->desc_stride_ctrl == ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY) in ena_com_config_llq_info()
774 llq_info->descs_per_entry = llq_info->desc_list_entry_size / in ena_com_config_llq_info()
777 llq_info->descs_per_entry = 1; in ena_com_config_llq_info()
779 supported_feat = llq_features->desc_num_before_header_supported; in ena_com_config_llq_info()
780 if (likely(supported_feat & llq_default_cfg->llq_num_decs_before_header)) { in ena_com_config_llq_info()
781 llq_info->descs_num_before_header = llq_default_cfg->llq_num_decs_before_header; in ena_com_config_llq_info()
784 llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2; in ena_com_config_llq_info()
786 llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_1; in ena_com_config_llq_info()
788 llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_4; in ena_com_config_llq_info()
790 llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8; in ena_com_config_llq_info()
798 llq_default_cfg->llq_num_decs_before_header, in ena_com_config_llq_info()
800 llq_info->descs_num_before_header); in ena_com_config_llq_info()
803 llq_accel_mode_get = llq_features->accel_mode.u.get; in ena_com_config_llq_info()
805 llq_info->disable_meta_caching = in ena_com_config_llq_info()
810 llq_info->max_entries_in_tx_burst = in ena_com_config_llq_info()
812 llq_default_cfg->llq_ring_entry_size_value; in ena_com_config_llq_info()
827 ENA_WAIT_EVENT_WAIT(comp_ctx->wait_event, in ena_com_wait_and_process_admin_cq_interrupts()
828 admin_queue->completion_timeout); in ena_com_wait_and_process_admin_cq_interrupts()
832 * 1) No completion (timeout reached) in ena_com_wait_and_process_admin_cq_interrupts()
833 * 2) There is completion but the device didn't get any msi-x interrupt. in ena_com_wait_and_process_admin_cq_interrupts()
835 if (unlikely(comp_ctx->status == ENA_CMD_SUBMITTED)) { in ena_com_wait_and_process_admin_cq_interrupts()
836 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); in ena_com_wait_and_process_admin_cq_interrupts()
838 admin_queue->stats.no_completion++; in ena_com_wait_and_process_admin_cq_interrupts()
839 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); in ena_com_wait_and_process_admin_cq_interrupts()
841 if (comp_ctx->status == ENA_CMD_COMPLETED) { in ena_com_wait_and_process_admin_cq_interrupts()
842 admin_queue->is_missing_admin_interrupt = true; in ena_com_wait_and_process_admin_cq_interrupts()
843 ena_trc_err(admin_queue->ena_dev, in ena_com_wait_and_process_admin_cq_interrupts()
844 …"The ena device sent a completion but the driver didn't receive a MSI-X interrupt (cmd %d), autopo… in ena_com_wait_and_process_admin_cq_interrupts()
845 comp_ctx->cmd_opcode, admin_queue->auto_polling ? "ON" : "OFF"); in ena_com_wait_and_process_admin_cq_interrupts()
847 if (admin_queue->auto_polling) in ena_com_wait_and_process_admin_cq_interrupts()
848 admin_queue->polling = true; in ena_com_wait_and_process_admin_cq_interrupts()
850 ena_trc_err(admin_queue->ena_dev, in ena_com_wait_and_process_admin_cq_interrupts()
851 "The ena device didn't send a completion for the admin cmd %d status %d\n", in ena_com_wait_and_process_admin_cq_interrupts()
852 comp_ctx->cmd_opcode, comp_ctx->status); in ena_com_wait_and_process_admin_cq_interrupts()
858 if (!admin_queue->polling) { in ena_com_wait_and_process_admin_cq_interrupts()
859 admin_queue->running_state = false; in ena_com_wait_and_process_admin_cq_interrupts()
863 } else if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) { in ena_com_wait_and_process_admin_cq_interrupts()
864 ena_trc_err(admin_queue->ena_dev, "Command was aborted\n"); in ena_com_wait_and_process_admin_cq_interrupts()
865 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); in ena_com_wait_and_process_admin_cq_interrupts()
866 admin_queue->stats.aborted_cmd++; in ena_com_wait_and_process_admin_cq_interrupts()
867 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); in ena_com_wait_and_process_admin_cq_interrupts()
872 ENA_WARN(comp_ctx->status != ENA_CMD_COMPLETED, in ena_com_wait_and_process_admin_cq_interrupts()
873 admin_queue->ena_dev, "Invalid comp status %d\n", in ena_com_wait_and_process_admin_cq_interrupts()
874 comp_ctx->status); in ena_com_wait_and_process_admin_cq_interrupts()
876 ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status); in ena_com_wait_and_process_admin_cq_interrupts()
884 * On timeout the function will return ENA_MMIO_READ_TIMEOUT
888 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; in ena_com_reg_bar_read32()
890 mmio_read->read_resp; in ena_com_reg_bar_read32()
893 u32 timeout = mmio_read->reg_read_to; in ena_com_reg_bar_read32() local
897 if (timeout == 0) in ena_com_reg_bar_read32()
898 timeout = ENA_REG_READ_TIMEOUT; in ena_com_reg_bar_read32()
901 if (!mmio_read->readless_supported) in ena_com_reg_bar_read32()
902 return ENA_REG_READ32(ena_dev->bus, ena_dev->reg_bar + offset); in ena_com_reg_bar_read32()
904 ENA_SPINLOCK_LOCK(mmio_read->lock, flags); in ena_com_reg_bar_read32()
905 mmio_read->seq_num++; in ena_com_reg_bar_read32()
907 read_resp->req_id = mmio_read->seq_num + 0xDEAD; in ena_com_reg_bar_read32()
910 mmio_read_reg |= mmio_read->seq_num & in ena_com_reg_bar_read32()
913 ENA_REG_WRITE32(ena_dev->bus, mmio_read_reg, in ena_com_reg_bar_read32()
914 ena_dev->reg_bar + ENA_REGS_MMIO_REG_READ_OFF); in ena_com_reg_bar_read32()
916 for (i = 0; i < timeout; i++) { in ena_com_reg_bar_read32()
917 if (READ_ONCE16(read_resp->req_id) == mmio_read->seq_num) in ena_com_reg_bar_read32()
923 if (unlikely(i == timeout)) { in ena_com_reg_bar_read32()
924 …ena_trc_err(ena_dev, "Reading reg failed for timeout. expected: req id[%u] offset[%u] actual: req … in ena_com_reg_bar_read32()
925 mmio_read->seq_num, in ena_com_reg_bar_read32()
927 read_resp->req_id, in ena_com_reg_bar_read32()
928 read_resp->reg_off); in ena_com_reg_bar_read32()
933 if (unlikely(read_resp->reg_off != offset)) { in ena_com_reg_bar_read32()
937 ret = read_resp->reg_val; in ena_com_reg_bar_read32()
940 ENA_SPINLOCK_UNLOCK(mmio_read->lock, flags); in ena_com_reg_bar_read32()
946 * Polling mode - wait until the completion is available.
947 * Async mode - wait on wait queue until the completion is ready
948 * (or the timeout expired).
955 if (admin_queue->polling) in ena_com_wait_and_process_admin_cq()
966 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; in ena_com_destroy_io_sq()
974 if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) in ena_com_destroy_io_sq()
983 destroy_cmd.sq.sq_idx = io_sq->idx; in ena_com_destroy_io_sq()
1004 if (io_cq->cdesc_addr.virt_addr) { in ena_com_io_queue_free()
1005 size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth; in ena_com_io_queue_free()
1007 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, in ena_com_io_queue_free()
1009 io_cq->cdesc_addr.virt_addr, in ena_com_io_queue_free()
1010 io_cq->cdesc_addr.phys_addr, in ena_com_io_queue_free()
1011 io_cq->cdesc_addr.mem_handle); in ena_com_io_queue_free()
1013 io_cq->cdesc_addr.virt_addr = NULL; in ena_com_io_queue_free()
1016 if (io_sq->desc_addr.virt_addr) { in ena_com_io_queue_free()
1017 size = io_sq->desc_entry_size * io_sq->q_depth; in ena_com_io_queue_free()
1019 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, in ena_com_io_queue_free()
1021 io_sq->desc_addr.virt_addr, in ena_com_io_queue_free()
1022 io_sq->desc_addr.phys_addr, in ena_com_io_queue_free()
1023 io_sq->desc_addr.mem_handle); in ena_com_io_queue_free()
1025 io_sq->desc_addr.virt_addr = NULL; in ena_com_io_queue_free()
1028 if (io_sq->bounce_buf_ctrl.base_buffer) { in ena_com_io_queue_free()
1029 ENA_MEM_FREE(ena_dev->dmadev, in ena_com_io_queue_free()
1030 io_sq->bounce_buf_ctrl.base_buffer, in ena_com_io_queue_free()
1031 (io_sq->llq_info.desc_list_entry_size * ENA_COM_BOUNCE_BUFFER_CNTRL_CNT)); in ena_com_io_queue_free()
1032 io_sq->bounce_buf_ctrl.base_buffer = NULL; in ena_com_io_queue_free()
1036 static int wait_for_reset_state(struct ena_com_dev *ena_dev, u32 timeout, in wait_for_reset_state() argument
1042 /* Convert timeout from resolution of 100ms to us resolution. */ in wait_for_reset_state()
1043 timeout_stamp = ENA_GET_SYSTEM_TIMEOUT(100 * 1000 * timeout); in wait_for_reset_state()
1049 ena_trc_err(ena_dev, "Reg read timeout occurred\n"); in wait_for_reset_state()
1060 ena_delay_exponential_backoff_us(exp++, ena_dev->ena_min_poll_delay_us); in wait_for_reset_state()
1071 !(ena_dev->supported_features & feature_mask)) in ena_com_check_supported_feature_id()
1094 admin_queue = &ena_dev->admin_queue; in ena_com_get_feature_ex()
1146 return ena_dev->rss.hash_func; in ena_com_get_current_hash_function()
1152 (ena_dev->rss).hash_key; in ena_com_hash_key_fill_default_key()
1154 ENA_RSS_FILL_KEY(&hash_key->key, sizeof(hash_key->key)); in ena_com_hash_key_fill_default_key()
1158 hash_key->key_parts = ENA_ADMIN_RSS_KEY_PARTS; in ena_com_hash_key_fill_default_key()
1163 struct ena_rss *rss = &ena_dev->rss; in ena_com_hash_key_allocate()
1168 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, in ena_com_hash_key_allocate()
1169 sizeof(*rss->hash_key), in ena_com_hash_key_allocate()
1170 rss->hash_key, in ena_com_hash_key_allocate()
1171 rss->hash_key_dma_addr, in ena_com_hash_key_allocate()
1172 rss->hash_key_mem_handle); in ena_com_hash_key_allocate()
1174 if (unlikely(!rss->hash_key)) in ena_com_hash_key_allocate()
1182 struct ena_rss *rss = &ena_dev->rss; in ena_com_hash_key_destroy()
1184 if (rss->hash_key) in ena_com_hash_key_destroy()
1185 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, in ena_com_hash_key_destroy()
1186 sizeof(*rss->hash_key), in ena_com_hash_key_destroy()
1187 rss->hash_key, in ena_com_hash_key_destroy()
1188 rss->hash_key_dma_addr, in ena_com_hash_key_destroy()
1189 rss->hash_key_mem_handle); in ena_com_hash_key_destroy()
1190 rss->hash_key = NULL; in ena_com_hash_key_destroy()
1195 struct ena_rss *rss = &ena_dev->rss; in ena_com_hash_ctrl_init()
1197 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, in ena_com_hash_ctrl_init()
1198 sizeof(*rss->hash_ctrl), in ena_com_hash_ctrl_init()
1199 rss->hash_ctrl, in ena_com_hash_ctrl_init()
1200 rss->hash_ctrl_dma_addr, in ena_com_hash_ctrl_init()
1201 rss->hash_ctrl_mem_handle); in ena_com_hash_ctrl_init()
1203 if (unlikely(!rss->hash_ctrl)) in ena_com_hash_ctrl_init()
1211 struct ena_rss *rss = &ena_dev->rss; in ena_com_hash_ctrl_destroy()
1213 if (rss->hash_ctrl) in ena_com_hash_ctrl_destroy()
1214 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, in ena_com_hash_ctrl_destroy()
1215 sizeof(*rss->hash_ctrl), in ena_com_hash_ctrl_destroy()
1216 rss->hash_ctrl, in ena_com_hash_ctrl_destroy()
1217 rss->hash_ctrl_dma_addr, in ena_com_hash_ctrl_destroy()
1218 rss->hash_ctrl_mem_handle); in ena_com_hash_ctrl_destroy()
1219 rss->hash_ctrl = NULL; in ena_com_hash_ctrl_destroy()
1225 struct ena_rss *rss = &ena_dev->rss; in ena_com_indirect_table_allocate()
1247 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, in ena_com_indirect_table_allocate()
1249 rss->rss_ind_tbl, in ena_com_indirect_table_allocate()
1250 rss->rss_ind_tbl_dma_addr, in ena_com_indirect_table_allocate()
1251 rss->rss_ind_tbl_mem_handle); in ena_com_indirect_table_allocate()
1252 if (unlikely(!rss->rss_ind_tbl)) in ena_com_indirect_table_allocate()
1256 rss->host_rss_ind_tbl = in ena_com_indirect_table_allocate()
1257 ENA_MEM_ALLOC(ena_dev->dmadev, tbl_size); in ena_com_indirect_table_allocate()
1258 if (unlikely(!rss->host_rss_ind_tbl)) in ena_com_indirect_table_allocate()
1261 rss->tbl_log_size = log_size; in ena_com_indirect_table_allocate()
1269 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, in ena_com_indirect_table_allocate()
1271 rss->rss_ind_tbl, in ena_com_indirect_table_allocate()
1272 rss->rss_ind_tbl_dma_addr, in ena_com_indirect_table_allocate()
1273 rss->rss_ind_tbl_mem_handle); in ena_com_indirect_table_allocate()
1274 rss->rss_ind_tbl = NULL; in ena_com_indirect_table_allocate()
1276 rss->tbl_log_size = 0; in ena_com_indirect_table_allocate()
1282 struct ena_rss *rss = &ena_dev->rss; in ena_com_indirect_table_destroy()
1283 size_t tbl_size = (1ULL << rss->tbl_log_size) * in ena_com_indirect_table_destroy()
1286 if (rss->rss_ind_tbl) in ena_com_indirect_table_destroy()
1287 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, in ena_com_indirect_table_destroy()
1289 rss->rss_ind_tbl, in ena_com_indirect_table_destroy()
1290 rss->rss_ind_tbl_dma_addr, in ena_com_indirect_table_destroy()
1291 rss->rss_ind_tbl_mem_handle); in ena_com_indirect_table_destroy()
1292 rss->rss_ind_tbl = NULL; in ena_com_indirect_table_destroy()
1294 if (rss->host_rss_ind_tbl) in ena_com_indirect_table_destroy()
1295 ENA_MEM_FREE(ena_dev->dmadev, in ena_com_indirect_table_destroy()
1296 rss->host_rss_ind_tbl, in ena_com_indirect_table_destroy()
1297 ((1ULL << rss->tbl_log_size) * sizeof(u16))); in ena_com_indirect_table_destroy()
1298 rss->host_rss_ind_tbl = NULL; in ena_com_indirect_table_destroy()
1304 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; in ena_com_create_io_sq()
1314 if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) in ena_com_create_io_sq()
1323 create_cmd.sq_caps_2 |= io_sq->mem_queue_type & in ena_com_create_io_sq()
1334 create_cmd.sq_depth = io_sq->q_depth; in ena_com_create_io_sq()
1336 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) { in ena_com_create_io_sq()
1339 io_sq->desc_addr.phys_addr); in ena_com_create_io_sq()
1356 io_sq->idx = cmd_completion.sq_idx; in ena_com_create_io_sq()
1358 io_sq->db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + in ena_com_create_io_sq()
1361 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { in ena_com_create_io_sq()
1362 io_sq->desc_addr.pbuf_dev_addr = in ena_com_create_io_sq()
1363 (u8 __iomem *)((uintptr_t)ena_dev->mem_bar + in ena_com_create_io_sq()
1367 ena_trc_dbg(ena_dev, "Created sq[%u], depth[%u]\n", io_sq->idx, io_sq->q_depth); in ena_com_create_io_sq()
1374 struct ena_rss *rss = &ena_dev->rss; in ena_com_ind_tbl_convert_to_device()
1379 for (i = 0; i < 1 << rss->tbl_log_size; i++) { in ena_com_ind_tbl_convert_to_device()
1380 qid = rss->host_rss_ind_tbl[i]; in ena_com_ind_tbl_convert_to_device()
1384 io_sq = &ena_dev->io_sq_queues[qid]; in ena_com_ind_tbl_convert_to_device()
1386 if (io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX) in ena_com_ind_tbl_convert_to_device()
1389 rss->rss_ind_tbl[i].cq_idx = io_sq->idx; in ena_com_ind_tbl_convert_to_device()
1398 u16 prev_intr_delay_resolution = ena_dev->intr_delay_resolution; in ena_com_update_intr_delay_resolution()
1406 ena_dev->intr_moder_rx_interval = in ena_com_update_intr_delay_resolution()
1407 ena_dev->intr_moder_rx_interval * in ena_com_update_intr_delay_resolution()
1412 ena_dev->intr_moder_tx_interval = in ena_com_update_intr_delay_resolution()
1413 ena_dev->intr_moder_tx_interval * in ena_com_update_intr_delay_resolution()
1417 ena_dev->intr_delay_resolution = intr_delay_resolution; in ena_com_update_intr_delay_resolution()
1425 struct ena_admin_aq_entry *cmd, in ena_com_execute_admin_command() argument
1433 comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size, in ena_com_execute_admin_command()
1438 ena_trc_dbg(admin_queue->ena_dev, in ena_com_execute_admin_command()
1442 ena_trc_err(admin_queue->ena_dev, in ena_com_execute_admin_command()
1451 if (admin_queue->running_state) in ena_com_execute_admin_command()
1452 ena_trc_err(admin_queue->ena_dev, in ena_com_execute_admin_command()
1455 ena_trc_dbg(admin_queue->ena_dev, in ena_com_execute_admin_command()
1464 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; in ena_com_create_io_cq()
1473 create_cmd.cq_caps_2 |= (io_cq->cdesc_entry_size_in_bytes / 4) & in ena_com_create_io_cq()
1478 create_cmd.msix_vector = io_cq->msix_vector; in ena_com_create_io_cq()
1479 create_cmd.cq_depth = io_cq->q_depth; in ena_com_create_io_cq()
1483 io_cq->cdesc_addr.phys_addr); in ena_com_create_io_cq()
1499 io_cq->idx = cmd_completion.cq_idx; in ena_com_create_io_cq()
1501 io_cq->unmask_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + in ena_com_create_io_cq()
1505 io_cq->numa_node_cfg_reg = in ena_com_create_io_cq()
1506 (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + in ena_com_create_io_cq()
1509 ena_trc_dbg(ena_dev, "Created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth); in ena_com_create_io_cq()
1524 *io_sq = &ena_dev->io_sq_queues[qid]; in ena_com_get_io_handlers()
1525 *io_cq = &ena_dev->io_cq_queues[qid]; in ena_com_get_io_handlers()
1532 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; in ena_com_abort_admin_commands()
1536 if (!admin_queue->comp_ctx) in ena_com_abort_admin_commands()
1539 for (i = 0; i < admin_queue->q_depth; i++) { in ena_com_abort_admin_commands()
1544 comp_ctx->status = ENA_CMD_ABORTED; in ena_com_abort_admin_commands()
1546 ENA_WAIT_EVENT_SIGNAL(comp_ctx->wait_event); in ena_com_abort_admin_commands()
1552 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; in ena_com_wait_for_abort_completion()
1556 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); in ena_com_wait_for_abort_completion()
1557 while (ATOMIC32_READ(&admin_queue->outstanding_cmds) != 0) { in ena_com_wait_for_abort_completion()
1558 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); in ena_com_wait_for_abort_completion()
1559 ena_delay_exponential_backoff_us(exp++, ena_dev->ena_min_poll_delay_us); in ena_com_wait_for_abort_completion()
1560 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); in ena_com_wait_for_abort_completion()
1562 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); in ena_com_wait_for_abort_completion()
1568 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; in ena_com_destroy_io_cq()
1575 destroy_cmd.cq_idx = io_cq->idx; in ena_com_destroy_io_cq()
1592 return ena_dev->admin_queue.running_state; in ena_com_get_admin_running_state()
1597 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; in ena_com_set_admin_running_state()
1600 ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); in ena_com_set_admin_running_state()
1601 ena_dev->admin_queue.running_state = state; in ena_com_set_admin_running_state()
1602 ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); in ena_com_set_admin_running_state()
1607 u16 depth = ena_dev->aenq.q_depth; in ena_com_admin_aenq_enable()
1609 ENA_WARN(ena_dev->aenq.head != depth, ena_dev, "Invalid AENQ state\n"); in ena_com_admin_aenq_enable()
1614 ENA_REG_WRITE32(ena_dev->bus, depth, ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF); in ena_com_admin_aenq_enable()
1620 struct ena_admin_set_feat_cmd cmd; in ena_com_set_aenq_config() local
1638 memset(&cmd, 0x0, sizeof(cmd)); in ena_com_set_aenq_config()
1639 admin_queue = &ena_dev->admin_queue; in ena_com_set_aenq_config()
1641 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; in ena_com_set_aenq_config()
1642 cmd.aq_common_descriptor.flags = 0; in ena_com_set_aenq_config()
1643 cmd.feat_common.feature_id = ENA_ADMIN_AENQ_CONFIG; in ena_com_set_aenq_config()
1644 cmd.u.aenq.enabled_groups = groups_flag; in ena_com_set_aenq_config()
1647 (struct ena_admin_aq_entry *)&cmd, in ena_com_set_aenq_config()
1648 sizeof(cmd), in ena_com_set_aenq_config()
1664 ena_trc_err(ena_dev, "Reg read timeout occurred\n"); in ena_com_get_dma_width()
1678 ena_dev->dma_addr_bits = width; in ena_com_get_dma_width()
1698 ena_trc_err(ena_dev, "Reg read timeout occurred\n"); in ena_com_validate_version()
1724 return -1; in ena_com_validate_version()
1735 if (!admin_queue->comp_ctx) in ena_com_free_ena_admin_queue_comp_ctx()
1739 ENA_MEM_FREE(ena_dev->dmadev, in ena_com_free_ena_admin_queue_comp_ctx()
1740 admin_queue->comp_ctx, in ena_com_free_ena_admin_queue_comp_ctx()
1741 (admin_queue->q_depth * sizeof(struct ena_comp_ctx))); in ena_com_free_ena_admin_queue_comp_ctx()
1743 admin_queue->comp_ctx = NULL; in ena_com_free_ena_admin_queue_comp_ctx()
1748 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; in ena_com_admin_destroy()
1749 struct ena_com_admin_cq *cq = &admin_queue->cq; in ena_com_admin_destroy()
1750 struct ena_com_admin_sq *sq = &admin_queue->sq; in ena_com_admin_destroy()
1751 struct ena_com_aenq *aenq = &ena_dev->aenq; in ena_com_admin_destroy()
1756 size = ADMIN_SQ_SIZE(admin_queue->q_depth); in ena_com_admin_destroy()
1757 if (sq->entries) in ena_com_admin_destroy()
1758 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, sq->entries, in ena_com_admin_destroy()
1759 sq->dma_addr, sq->mem_handle); in ena_com_admin_destroy()
1760 sq->entries = NULL; in ena_com_admin_destroy()
1762 size = ADMIN_CQ_SIZE(admin_queue->q_depth); in ena_com_admin_destroy()
1763 if (cq->entries) in ena_com_admin_destroy()
1764 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, cq->entries, in ena_com_admin_destroy()
1765 cq->dma_addr, cq->mem_handle); in ena_com_admin_destroy()
1766 cq->entries = NULL; in ena_com_admin_destroy()
1768 size = ADMIN_AENQ_SIZE(aenq->q_depth); in ena_com_admin_destroy()
1769 if (ena_dev->aenq.entries) in ena_com_admin_destroy()
1770 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, aenq->entries, in ena_com_admin_destroy()
1771 aenq->dma_addr, aenq->mem_handle); in ena_com_admin_destroy()
1772 aenq->entries = NULL; in ena_com_admin_destroy()
1773 ENA_SPINLOCK_DESTROY(admin_queue->q_lock); in ena_com_admin_destroy()
1783 ENA_REG_WRITE32(ena_dev->bus, mask_value, in ena_com_set_admin_polling_mode()
1784 ena_dev->reg_bar + ENA_REGS_INTR_MASK_OFF); in ena_com_set_admin_polling_mode()
1785 ena_dev->admin_queue.polling = polling; in ena_com_set_admin_polling_mode()
1790 return ena_dev->admin_queue.polling; in ena_com_get_admin_polling_mode()
1796 ena_dev->admin_queue.auto_polling = polling; in ena_com_set_admin_auto_polling_mode()
1806 struct ena_com_phc_info *phc = &ena_dev->phc; in ena_com_phc_init()
1811 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, in ena_com_phc_init()
1812 sizeof(*phc->virt_addr), in ena_com_phc_init()
1813 phc->virt_addr, in ena_com_phc_init()
1814 phc->phys_addr, in ena_com_phc_init()
1815 phc->mem_handle); in ena_com_phc_init()
1816 if (unlikely(!phc->virt_addr)) in ena_com_phc_init()
1819 ENA_SPINLOCK_INIT(phc->lock); in ena_com_phc_init()
1821 phc->virt_addr->req_id = 0; in ena_com_phc_init()
1822 phc->virt_addr->timestamp = 0; in ena_com_phc_init()
1829 struct ena_com_phc_info *phc = &ena_dev->phc; in ena_com_phc_config()
1854 phc->doorbell_offset = get_feat_resp.u.phc.doorbell_offset; in ena_com_phc_config()
1856 /* Update PHC expire timeout according to device or default driver value */ in ena_com_phc_config()
1857 phc->expire_timeout_usec = (get_feat_resp.u.phc.expire_timeout_usec) ? in ena_com_phc_config()
1861 /* Update PHC block timeout according to device or default driver value */ in ena_com_phc_config()
1862 phc->block_timeout_usec = (get_feat_resp.u.phc.block_timeout_usec) ? in ena_com_phc_config()
1866 /* Sanity check - expire timeout must not exceed block timeout */ in ena_com_phc_config()
1867 if (phc->expire_timeout_usec > phc->block_timeout_usec) in ena_com_phc_config()
1868 phc->expire_timeout_usec = phc->block_timeout_usec; in ena_com_phc_config()
1874 set_feat_cmd.u.phc.output_length = sizeof(*phc->virt_addr); in ena_com_phc_config()
1875 ret = ena_com_mem_addr_set(ena_dev, &set_feat_cmd.u.phc.output_address, phc->phys_addr); in ena_com_phc_config()
1882 ret = ena_com_execute_admin_command(&ena_dev->admin_queue, in ena_com_phc_config()
1893 phc->active = true; in ena_com_phc_config()
1901 struct ena_com_phc_info *phc = &ena_dev->phc; in ena_com_phc_destroy()
1905 if (!phc->virt_addr) in ena_com_phc_destroy()
1908 ENA_SPINLOCK_LOCK(phc->lock, flags); in ena_com_phc_destroy()
1909 phc->active = false; in ena_com_phc_destroy()
1910 ENA_SPINLOCK_UNLOCK(phc->lock, flags); in ena_com_phc_destroy()
1912 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, in ena_com_phc_destroy()
1913 sizeof(*phc->virt_addr), in ena_com_phc_destroy()
1914 phc->virt_addr, in ena_com_phc_destroy()
1915 phc->phys_addr, in ena_com_phc_destroy()
1916 phc->mem_handle); in ena_com_phc_destroy()
1917 phc->virt_addr = NULL; in ena_com_phc_destroy()
1919 ENA_SPINLOCK_DESTROY(phc->lock); in ena_com_phc_destroy()
1924 volatile struct ena_admin_phc_resp *read_resp = ena_dev->phc.virt_addr; in ena_com_phc_get_timestamp()
1926 struct ena_com_phc_info *phc = &ena_dev->phc; in ena_com_phc_get_timestamp()
1932 if (!phc->active) { in ena_com_phc_get_timestamp()
1937 ENA_SPINLOCK_LOCK(phc->lock, flags); in ena_com_phc_get_timestamp()
1940 if (unlikely(ENA_TIME_COMPARE_HIGH_RES(phc->system_time, zero_system_time))) { in ena_com_phc_get_timestamp()
1942 block_time = ENA_GET_SYSTEM_TIMEOUT_HIGH_RES(phc->system_time, in ena_com_phc_get_timestamp()
1943 phc->block_timeout_usec); in ena_com_phc_get_timestamp()
1946 phc->stats.phc_skp++; in ena_com_phc_get_timestamp()
1952 if ((READ_ONCE16(read_resp->req_id) != phc->req_id) || in ena_com_phc_get_timestamp()
1953 (read_resp->error_flags & ENA_PHC_ERROR_FLAGS)) { in ena_com_phc_get_timestamp()
1957 phc->stats.phc_err++; in ena_com_phc_get_timestamp()
1960 phc->stats.phc_exp++; in ena_com_phc_get_timestamp()
1965 phc->system_time = ENA_GET_SYSTEM_TIME_HIGH_RES(); in ena_com_phc_get_timestamp()
1966 block_time = ENA_GET_SYSTEM_TIMEOUT_HIGH_RES(phc->system_time, phc->block_timeout_usec); in ena_com_phc_get_timestamp()
1967 expire_time = ENA_GET_SYSTEM_TIMEOUT_HIGH_RES(phc->system_time, phc->expire_timeout_usec); in ena_com_phc_get_timestamp()
1970 phc->req_id++; in ena_com_phc_get_timestamp()
1975 read_resp->req_id = phc->req_id + ENA_PHC_REQ_ID_OFFSET; in ena_com_phc_get_timestamp()
1978 ENA_REG_WRITE32(ena_dev->bus, phc->req_id, ena_dev->reg_bar + phc->doorbell_offset); in ena_com_phc_get_timestamp()
1987 phc->error_bound = ENA_PHC_MAX_ERROR_BOUND; in ena_com_phc_get_timestamp()
1993 if (READ_ONCE16(read_resp->req_id) != phc->req_id) { in ena_com_phc_get_timestamp()
2002 if (unlikely(read_resp->error_flags & ENA_PHC_ERROR_FLAGS)) { in ena_com_phc_get_timestamp()
2007 phc->error_bound = ENA_PHC_MAX_ERROR_BOUND; in ena_com_phc_get_timestamp()
2013 *timestamp = read_resp->timestamp; in ena_com_phc_get_timestamp()
2016 phc->error_bound = read_resp->error_bound; in ena_com_phc_get_timestamp()
2019 phc->stats.phc_cnt++; in ena_com_phc_get_timestamp()
2022 phc->system_time = zero_system_time; in ena_com_phc_get_timestamp()
2027 ENA_SPINLOCK_UNLOCK(phc->lock, flags); in ena_com_phc_get_timestamp()
2034 struct ena_com_phc_info *phc = &ena_dev->phc; in ena_com_phc_get_error_bound()
2035 u32 local_error_bound = phc->error_bound; in ena_com_phc_get_error_bound()
2037 if (!phc->active) { in ena_com_phc_get_error_bound()
2052 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; in ena_com_mmio_reg_read_request_init()
2054 ENA_SPINLOCK_INIT(mmio_read->lock); in ena_com_mmio_reg_read_request_init()
2055 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, in ena_com_mmio_reg_read_request_init()
2056 sizeof(*mmio_read->read_resp), in ena_com_mmio_reg_read_request_init()
2057 mmio_read->read_resp, in ena_com_mmio_reg_read_request_init()
2058 mmio_read->read_resp_dma_addr, in ena_com_mmio_reg_read_request_init()
2059 mmio_read->read_resp_mem_handle); in ena_com_mmio_reg_read_request_init()
2060 if (unlikely(!mmio_read->read_resp)) in ena_com_mmio_reg_read_request_init()
2065 mmio_read->read_resp->req_id = 0x0; in ena_com_mmio_reg_read_request_init()
2066 mmio_read->seq_num = 0x0; in ena_com_mmio_reg_read_request_init()
2067 mmio_read->readless_supported = true; in ena_com_mmio_reg_read_request_init()
2072 ENA_SPINLOCK_DESTROY(mmio_read->lock); in ena_com_mmio_reg_read_request_init()
2078 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; in ena_com_set_mmio_read_mode()
2080 mmio_read->readless_supported = readless_supported; in ena_com_set_mmio_read_mode()
2085 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; in ena_com_mmio_reg_read_request_destroy()
2087 ENA_REG_WRITE32(ena_dev->bus, 0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF); in ena_com_mmio_reg_read_request_destroy()
2088 ENA_REG_WRITE32(ena_dev->bus, 0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF); in ena_com_mmio_reg_read_request_destroy()
2090 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, in ena_com_mmio_reg_read_request_destroy()
2091 sizeof(*mmio_read->read_resp), in ena_com_mmio_reg_read_request_destroy()
2092 mmio_read->read_resp, in ena_com_mmio_reg_read_request_destroy()
2093 mmio_read->read_resp_dma_addr, in ena_com_mmio_reg_read_request_destroy()
2094 mmio_read->read_resp_mem_handle); in ena_com_mmio_reg_read_request_destroy()
2096 mmio_read->read_resp = NULL; in ena_com_mmio_reg_read_request_destroy()
2097 ENA_SPINLOCK_DESTROY(mmio_read->lock); in ena_com_mmio_reg_read_request_destroy()
2102 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; in ena_com_mmio_reg_read_request_write_dev_addr()
2105 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(mmio_read->read_resp_dma_addr); in ena_com_mmio_reg_read_request_write_dev_addr()
2106 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(mmio_read->read_resp_dma_addr); in ena_com_mmio_reg_read_request_write_dev_addr()
2108 ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF); in ena_com_mmio_reg_read_request_write_dev_addr()
2109 ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF); in ena_com_mmio_reg_read_request_write_dev_addr()
2115 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; in ena_com_admin_init()
2122 ena_trc_err(ena_dev, "Reg read timeout occurred\n"); in ena_com_admin_init()
2131 admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH; in ena_com_admin_init()
2133 admin_queue->bus = ena_dev->bus; in ena_com_admin_init()
2134 admin_queue->q_dmadev = ena_dev->dmadev; in ena_com_admin_init()
2135 admin_queue->polling = false; in ena_com_admin_init()
2136 admin_queue->curr_cmd_id = 0; in ena_com_admin_init()
2138 ATOMIC32_SET(&admin_queue->outstanding_cmds, 0); in ena_com_admin_init()
2140 ENA_SPINLOCK_INIT(admin_queue->q_lock); in ena_com_admin_init()
2154 admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + in ena_com_admin_init()
2157 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr); in ena_com_admin_init()
2158 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr); in ena_com_admin_init()
2160 ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_AQ_BASE_LO_OFF); in ena_com_admin_init()
2161 ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_AQ_BASE_HI_OFF); in ena_com_admin_init()
2163 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr); in ena_com_admin_init()
2164 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr); in ena_com_admin_init()
2166 ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_LO_OFF); in ena_com_admin_init()
2167 ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_HI_OFF); in ena_com_admin_init()
2170 aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK; in ena_com_admin_init()
2176 acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK; in ena_com_admin_init()
2181 ENA_REG_WRITE32(ena_dev->bus, aq_caps, ena_dev->reg_bar + ENA_REGS_AQ_CAPS_OFF); in ena_com_admin_init()
2182 ENA_REG_WRITE32(ena_dev->bus, acq_caps, ena_dev->reg_bar + ENA_REGS_ACQ_CAPS_OFF); in ena_com_admin_init()
2187 admin_queue->ena_dev = ena_dev; in ena_com_admin_init()
2188 admin_queue->running_state = true; in ena_com_admin_init()
2189 admin_queue->is_missing_admin_interrupt = false; in ena_com_admin_init()
2205 if (unlikely(ctx->qid >= ENA_TOTAL_NUM_QUEUES)) { in ena_com_create_io_queue()
2207 ctx->qid, ENA_TOTAL_NUM_QUEUES); in ena_com_create_io_queue()
2211 io_sq = &ena_dev->io_sq_queues[ctx->qid]; in ena_com_create_io_queue()
2212 io_cq = &ena_dev->io_cq_queues[ctx->qid]; in ena_com_create_io_queue()
2218 io_cq->q_depth = ctx->queue_size; in ena_com_create_io_queue()
2219 io_cq->direction = ctx->direction; in ena_com_create_io_queue()
2220 io_cq->qid = ctx->qid; in ena_com_create_io_queue()
2222 io_cq->msix_vector = ctx->msix_vector; in ena_com_create_io_queue()
2224 io_sq->q_depth = ctx->queue_size; in ena_com_create_io_queue()
2225 io_sq->direction = ctx->direction; in ena_com_create_io_queue()
2226 io_sq->qid = ctx->qid; in ena_com_create_io_queue()
2228 io_sq->mem_queue_type = ctx->mem_queue_type; in ena_com_create_io_queue()
2230 if (ctx->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) in ena_com_create_io_queue()
2232 io_sq->tx_max_header_size = in ena_com_create_io_queue()
2233 ENA_MIN32(ena_dev->tx_max_header_size, SZ_256); in ena_com_create_io_queue()
2246 ret = ena_com_create_io_sq(ena_dev, io_sq, io_cq->idx); in ena_com_create_io_queue()
2270 io_sq = &ena_dev->io_sq_queues[qid]; in ena_com_destroy_io_queue()
2271 io_cq = &ena_dev->io_cq_queues[qid]; in ena_com_destroy_io_queue()
2289 struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp; in ena_get_dev_stats()
2290 struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd; in ena_get_dev_stats()
2294 admin_queue = &ena_dev->admin_queue; in ena_get_dev_stats()
2296 get_cmd->aq_common_descriptor.opcode = ENA_ADMIN_GET_STATS; in ena_get_dev_stats()
2297 get_cmd->aq_common_descriptor.flags = 0; in ena_get_dev_stats()
2298 get_cmd->type = type; in ena_get_dev_stats()
2318 customer_metrics = &ena_dev->customer_metrics; in ena_com_set_supported_customer_metrics()
2320 customer_metrics->supported_metrics = ENA_ADMIN_CUSTOMER_METRICS_MIN_SUPPORT_MASK; in ena_com_set_supported_customer_metrics()
2328 customer_metrics->supported_metrics = in ena_com_set_supported_customer_metrics()
2345 memcpy(&get_feat_ctx->dev_attr, &get_resp.u.dev_attr, in ena_com_get_dev_attr_feat()
2348 ena_dev->supported_features = get_resp.u.dev_attr.supported_features; in ena_com_get_dev_attr_feat()
2349 ena_dev->capabilities = get_resp.u.dev_attr.capabilities; in ena_com_get_dev_attr_feat()
2351 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { in ena_com_get_dev_attr_feat()
2361 memcpy(&get_feat_ctx->max_queue_ext, &get_resp.u.max_queue_ext, in ena_com_get_dev_attr_feat()
2363 ena_dev->tx_max_header_size = in ena_com_get_dev_attr_feat()
2368 memcpy(&get_feat_ctx->max_queues, &get_resp.u.max_queue, in ena_com_get_dev_attr_feat()
2370 ena_dev->tx_max_header_size = in ena_com_get_dev_attr_feat()
2382 memcpy(&get_feat_ctx->aenq, &get_resp.u.aenq, in ena_com_get_dev_attr_feat()
2390 memcpy(&get_feat_ctx->offload, &get_resp.u.offload, in ena_com_get_dev_attr_feat()
2399 memcpy(&get_feat_ctx->hw_hints, &get_resp.u.hw_hints, in ena_com_get_dev_attr_feat()
2402 memset(&get_feat_ctx->hw_hints, 0x0, sizeof(get_feat_ctx->hw_hints)); in ena_com_get_dev_attr_feat()
2409 memcpy(&get_feat_ctx->llq, &get_resp.u.llq, in ena_com_get_dev_attr_feat()
2412 memset(&get_feat_ctx->llq, 0x0, sizeof(get_feat_ctx->llq)); in ena_com_get_dev_attr_feat()
2423 ena_com_handle_admin_completion(&ena_dev->admin_queue); in ena_com_admin_q_comp_intr_handler()
2432 struct ena_aenq_handlers *aenq_handlers = ena_dev->aenq.aenq_handlers; in ena_com_get_specific_aenq_cb()
2434 if ((group < ENA_MAX_HANDLERS) && aenq_handlers->handlers[group]) in ena_com_get_specific_aenq_cb()
2435 return aenq_handlers->handlers[group]; in ena_com_get_specific_aenq_cb()
2437 return aenq_handlers->unimplemented_handler; in ena_com_get_specific_aenq_cb()
2448 struct ena_com_aenq *aenq = &ena_dev->aenq; in ena_com_aenq_intr_handler()
2454 masked_head = aenq->head & (aenq->q_depth - 1); in ena_com_aenq_intr_handler()
2455 phase = aenq->phase; in ena_com_aenq_intr_handler()
2456 aenq_e = &aenq->entries[masked_head]; /* Get first entry */ in ena_com_aenq_intr_handler()
2457 aenq_common = &aenq_e->aenq_common_desc; in ena_com_aenq_intr_handler()
2460 while ((READ_ONCE8(aenq_common->flags) & in ena_com_aenq_intr_handler()
2467 timestamp = (u64)aenq_common->timestamp_low | in ena_com_aenq_intr_handler()
2468 ((u64)aenq_common->timestamp_high << 32); in ena_com_aenq_intr_handler()
2471 aenq_common->group, in ena_com_aenq_intr_handler()
2472 aenq_common->syndrome, in ena_com_aenq_intr_handler()
2477 aenq_common->group); in ena_com_aenq_intr_handler()
2484 if (unlikely(masked_head == aenq->q_depth)) { in ena_com_aenq_intr_handler()
2488 aenq_e = &aenq->entries[masked_head]; in ena_com_aenq_intr_handler()
2489 aenq_common = &aenq_e->aenq_common_desc; in ena_com_aenq_intr_handler()
2492 aenq->head += processed; in ena_com_aenq_intr_handler()
2493 aenq->phase = phase; in ena_com_aenq_intr_handler()
2501 ENA_REG_WRITE32_RELAXED(ena_dev->bus, (u32)aenq->head, in ena_com_aenq_intr_handler()
2502 ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF); in ena_com_aenq_intr_handler()
2509 struct ena_com_aenq *aenq = &ena_dev->aenq; in ena_com_aenq_has_keep_alive()
2511 u8 phase = aenq->phase; in ena_com_aenq_has_keep_alive()
2514 masked_head = aenq->head & (aenq->q_depth - 1); in ena_com_aenq_has_keep_alive()
2515 aenq_e = &aenq->entries[masked_head]; /* Get first entry */ in ena_com_aenq_has_keep_alive()
2516 aenq_common = &aenq_e->aenq_common_desc; in ena_com_aenq_has_keep_alive()
2519 while ((READ_ONCE8(aenq_common->flags) & in ena_com_aenq_has_keep_alive()
2526 if (aenq_common->group == ENA_ADMIN_KEEP_ALIVE) in ena_com_aenq_has_keep_alive()
2532 if (unlikely(masked_head == aenq->q_depth)) { in ena_com_aenq_has_keep_alive()
2537 aenq_e = &aenq->entries[masked_head]; in ena_com_aenq_has_keep_alive()
2538 aenq_common = &aenq_e->aenq_common_desc; in ena_com_aenq_has_keep_alive()
2558 ena_dev->stats_func = ENA_EXTENDED_STAT_GET_FUNCT(func_queue); in ena_com_extended_stats_set_func_queue()
2559 ena_dev->stats_queue = ENA_EXTENDED_STAT_GET_QUEUE(func_queue); in ena_com_extended_stats_set_func_queue()
2570 u32 stat, timeout, cap, reset_val; in ena_com_dev_reset() local
2578 ena_trc_err(ena_dev, "Reg read32 timeout occurred\n"); in ena_com_dev_reset()
2587 timeout = (cap & ENA_REGS_CAPS_RESET_TIMEOUT_MASK) >> in ena_com_dev_reset()
2589 if (timeout == 0) { in ena_com_dev_reset()
2590 ena_trc_err(ena_dev, "Invalid timeout value\n"); in ena_com_dev_reset()
2598 * bits 24-27 as MSB, bits 28-31 as LSB in ena_com_dev_reset()
2618 ENA_REG_WRITE32(ena_dev->bus, reset_val, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF); in ena_com_dev_reset()
2623 rc = wait_for_reset_state(ena_dev, timeout, in ena_com_dev_reset()
2631 ENA_REG_WRITE32(ena_dev->bus, 0, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF); in ena_com_dev_reset()
2632 rc = wait_for_reset_state(ena_dev, timeout, 0); in ena_com_dev_reset()
2638 timeout = (cap & ENA_REGS_CAPS_ADMIN_CMD_TO_MASK) >> in ena_com_dev_reset()
2640 if (timeout) in ena_com_dev_reset()
2641 /* the resolution of timeout reg is 100ms */ in ena_com_dev_reset()
2642 ena_dev->admin_queue.completion_timeout = timeout * 100000; in ena_com_dev_reset()
2644 ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US; in ena_com_dev_reset()
2715 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, len, in ena_com_get_dev_extended_stats()
2723 &get_cmd->u.control_buffer.address, in ena_com_get_dev_extended_stats()
2729 get_cmd->u.control_buffer.length = len; in ena_com_get_dev_extended_stats()
2731 get_cmd->device_id = ena_dev->stats_func; in ena_com_get_dev_extended_stats()
2732 get_cmd->queue_idx = ena_dev->stats_queue; in ena_com_get_dev_extended_stats()
2742 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, len, virt_addr, phys_addr, in ena_com_get_dev_extended_stats()
2755 if (unlikely(len > ena_dev->customer_metrics.buffer_len)) { in ena_com_get_customer_metrics()
2765 if (!ena_dev->customer_metrics.supported_metrics) { in ena_com_get_customer_metrics()
2773 &get_cmd->u.control_buffer.address, in ena_com_get_customer_metrics()
2774 ena_dev->customer_metrics.buffer_dma_addr); in ena_com_get_customer_metrics()
2780 get_cmd->u.control_buffer.length = ena_dev->customer_metrics.buffer_len; in ena_com_get_customer_metrics()
2781 get_cmd->requested_metrics = ena_dev->customer_metrics.supported_metrics; in ena_com_get_customer_metrics()
2784 memcpy(buffer, ena_dev->customer_metrics.buffer_virt_addr, len); in ena_com_get_customer_metrics()
2794 struct ena_admin_set_feat_cmd cmd; in ena_com_set_dev_mtu() local
2803 memset(&cmd, 0x0, sizeof(cmd)); in ena_com_set_dev_mtu()
2804 admin_queue = &ena_dev->admin_queue; in ena_com_set_dev_mtu()
2806 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; in ena_com_set_dev_mtu()
2807 cmd.aq_common_descriptor.flags = 0; in ena_com_set_dev_mtu()
2808 cmd.feat_common.feature_id = ENA_ADMIN_MTU; in ena_com_set_dev_mtu()
2809 cmd.u.mtu.mtu = mtu; in ena_com_set_dev_mtu()
2812 (struct ena_admin_aq_entry *)&cmd, in ena_com_set_dev_mtu()
2813 sizeof(cmd), in ena_com_set_dev_mtu()
2843 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; in ena_com_set_hash_function()
2844 struct ena_rss *rss = &ena_dev->rss; in ena_com_set_hash_function()
2845 struct ena_admin_set_feat_cmd cmd; in ena_com_set_hash_function() local
2863 if (!(get_resp.u.flow_hash_func.supported_func & BIT(rss->hash_func))) { in ena_com_set_hash_function()
2865 rss->hash_func); in ena_com_set_hash_function()
2869 memset(&cmd, 0x0, sizeof(cmd)); in ena_com_set_hash_function()
2871 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; in ena_com_set_hash_function()
2872 cmd.aq_common_descriptor.flags = in ena_com_set_hash_function()
2874 cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_FUNCTION; in ena_com_set_hash_function()
2875 cmd.u.flow_hash_func.init_val = rss->hash_init_val; in ena_com_set_hash_function()
2876 cmd.u.flow_hash_func.selected_func = 1 << rss->hash_func; in ena_com_set_hash_function()
2879 &cmd.control_buffer.address, in ena_com_set_hash_function()
2880 rss->hash_key_dma_addr); in ena_com_set_hash_function()
2886 cmd.control_buffer.length = sizeof(*rss->hash_key); in ena_com_set_hash_function()
2889 (struct ena_admin_aq_entry *)&cmd, in ena_com_set_hash_function()
2890 sizeof(cmd), in ena_com_set_hash_function()
2895 rss->hash_func, ret); in ena_com_set_hash_function()
2909 struct ena_rss *rss = &ena_dev->rss; in ena_com_fill_hash_function()
2912 hash_key = rss->hash_key; in ena_com_fill_hash_function()
2920 rss->hash_key_dma_addr, in ena_com_fill_hash_function()
2921 sizeof(*rss->hash_key), 0); in ena_com_fill_hash_function()
2931 if (key_len != sizeof(hash_key->key)) { in ena_com_fill_hash_function()
2933 key_len, sizeof(hash_key->key)); in ena_com_fill_hash_function()
2936 memcpy(hash_key->key, key, key_len); in ena_com_fill_hash_function()
2937 hash_key->key_parts = key_len / sizeof(hash_key->key[0]); in ena_com_fill_hash_function()
2940 rss->hash_init_val = init_val; in ena_com_fill_hash_function()
2941 old_func = rss->hash_func; in ena_com_fill_hash_function()
2942 rss->hash_func = func; in ena_com_fill_hash_function()
2947 rss->hash_func = old_func; in ena_com_fill_hash_function()
2955 struct ena_rss *rss = &ena_dev->rss; in ena_com_get_hash_function()
2964 rss->hash_key_dma_addr, in ena_com_get_hash_function()
2965 sizeof(*rss->hash_key), 0); in ena_com_get_hash_function()
2970 rss->hash_func = ENA_FFS(get_resp.u.flow_hash_func.selected_func); in ena_com_get_hash_function()
2971 if (rss->hash_func) in ena_com_get_hash_function()
2972 rss->hash_func--; in ena_com_get_hash_function()
2974 *func = rss->hash_func; in ena_com_get_hash_function()
2982 ena_dev->rss.hash_key; in ena_com_get_hash_key()
2985 memcpy(key, hash_key->key, in ena_com_get_hash_key()
2986 (size_t)(hash_key->key_parts) * sizeof(hash_key->key[0])); in ena_com_get_hash_key()
2995 struct ena_rss *rss = &ena_dev->rss; in ena_com_get_hash_ctrl()
3001 rss->hash_ctrl_dma_addr, in ena_com_get_hash_ctrl()
3002 sizeof(*rss->hash_ctrl), 0); in ena_com_get_hash_ctrl()
3007 *fields = rss->hash_ctrl->selected_fields[proto].fields; in ena_com_get_hash_ctrl()
3014 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; in ena_com_set_hash_ctrl()
3015 struct ena_rss *rss = &ena_dev->rss; in ena_com_set_hash_ctrl()
3016 struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl; in ena_com_set_hash_ctrl()
3017 struct ena_admin_set_feat_cmd cmd; in ena_com_set_hash_ctrl() local
3028 memset(&cmd, 0x0, sizeof(cmd)); in ena_com_set_hash_ctrl()
3030 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; in ena_com_set_hash_ctrl()
3031 cmd.aq_common_descriptor.flags = in ena_com_set_hash_ctrl()
3033 cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_INPUT; in ena_com_set_hash_ctrl()
3034 cmd.u.flow_hash_input.enabled_input_sort = in ena_com_set_hash_ctrl()
3039 &cmd.control_buffer.address, in ena_com_set_hash_ctrl()
3040 rss->hash_ctrl_dma_addr); in ena_com_set_hash_ctrl()
3045 cmd.control_buffer.length = sizeof(*hash_ctrl); in ena_com_set_hash_ctrl()
3048 (struct ena_admin_aq_entry *)&cmd, in ena_com_set_hash_ctrl()
3049 sizeof(cmd), in ena_com_set_hash_ctrl()
3060 struct ena_rss *rss = &ena_dev->rss; in ena_com_set_default_hash_ctrl()
3062 rss->hash_ctrl; in ena_com_set_default_hash_ctrl()
3071 hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP4].fields = in ena_com_set_default_hash_ctrl()
3075 hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP4].fields = in ena_com_set_default_hash_ctrl()
3079 hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP6].fields = in ena_com_set_default_hash_ctrl()
3083 hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP6].fields = in ena_com_set_default_hash_ctrl()
3087 hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4].fields = in ena_com_set_default_hash_ctrl()
3090 hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP6].fields = in ena_com_set_default_hash_ctrl()
3093 hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields = in ena_com_set_default_hash_ctrl()
3096 hash_ctrl->selected_fields[ENA_ADMIN_RSS_NOT_IP].fields = in ena_com_set_default_hash_ctrl()
3100 available_fields = hash_ctrl->selected_fields[i].fields & in ena_com_set_default_hash_ctrl()
3101 hash_ctrl->supported_fields[i].fields; in ena_com_set_default_hash_ctrl()
3102 if (available_fields != hash_ctrl->selected_fields[i].fields) { in ena_com_set_default_hash_ctrl()
3104 i, hash_ctrl->supported_fields[i].fields, in ena_com_set_default_hash_ctrl()
3105 hash_ctrl->selected_fields[i].fields); in ena_com_set_default_hash_ctrl()
3123 struct ena_rss *rss = &ena_dev->rss; in ena_com_fill_hash_ctrl()
3124 struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl; in ena_com_fill_hash_ctrl()
3139 supported_fields = hash_ctrl->supported_fields[proto].fields; in ena_com_fill_hash_ctrl()
3145 hash_ctrl->selected_fields[proto].fields = hash_fields; in ena_com_fill_hash_ctrl()
3159 struct ena_rss *rss = &ena_dev->rss; in ena_com_indirect_table_fill_entry()
3161 if (unlikely(entry_idx >= (1 << rss->tbl_log_size))) in ena_com_indirect_table_fill_entry()
3167 rss->host_rss_ind_tbl[entry_idx] = entry_value; in ena_com_indirect_table_fill_entry()
3174 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; in ena_com_indirect_table_set()
3175 struct ena_rss *rss = &ena_dev->rss; in ena_com_indirect_table_set()
3176 struct ena_admin_set_feat_cmd cmd; in ena_com_indirect_table_set() local
3193 memset(&cmd, 0x0, sizeof(cmd)); in ena_com_indirect_table_set()
3195 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; in ena_com_indirect_table_set()
3196 cmd.aq_common_descriptor.flags = in ena_com_indirect_table_set()
3198 cmd.feat_common.feature_id = ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG; in ena_com_indirect_table_set()
3199 cmd.u.ind_table.size = rss->tbl_log_size; in ena_com_indirect_table_set()
3200 cmd.u.ind_table.inline_index = 0xFFFFFFFF; in ena_com_indirect_table_set()
3203 &cmd.control_buffer.address, in ena_com_indirect_table_set()
3204 rss->rss_ind_tbl_dma_addr); in ena_com_indirect_table_set()
3210 cmd.control_buffer.length = (1ULL << rss->tbl_log_size) * in ena_com_indirect_table_set()
3214 (struct ena_admin_aq_entry *)&cmd, in ena_com_indirect_table_set()
3215 sizeof(cmd), in ena_com_indirect_table_set()
3227 struct ena_rss *rss = &ena_dev->rss; in ena_com_indirect_table_get()
3232 tbl_size = (1ULL << rss->tbl_log_size) * in ena_com_indirect_table_get()
3237 rss->rss_ind_tbl_dma_addr, in ena_com_indirect_table_get()
3245 for (i = 0; i < (1 << rss->tbl_log_size); i++) in ena_com_indirect_table_get()
3246 ind_tbl[i] = rss->host_rss_ind_tbl[i]; in ena_com_indirect_table_get()
3255 memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss)); in ena_com_rss_init()
3292 memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss)); in ena_com_rss_destroy()
3297 struct ena_host_attribute *host_attr = &ena_dev->host_attr; in ena_com_allocate_host_info()
3299 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, in ena_com_allocate_host_info()
3301 host_attr->host_info, in ena_com_allocate_host_info()
3302 host_attr->host_info_dma_addr, in ena_com_allocate_host_info()
3303 host_attr->host_info_dma_handle); in ena_com_allocate_host_info()
3304 if (unlikely(!host_attr->host_info)) in ena_com_allocate_host_info()
3307 host_attr->host_info->ena_spec_version = ((ENA_COMMON_SPEC_VERSION_MAJOR << in ena_com_allocate_host_info()
3317 struct ena_host_attribute *host_attr = &ena_dev->host_attr; in ena_com_allocate_debug_area()
3319 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, in ena_com_allocate_debug_area()
3321 host_attr->debug_area_virt_addr, in ena_com_allocate_debug_area()
3322 host_attr->debug_area_dma_addr, in ena_com_allocate_debug_area()
3323 host_attr->debug_area_dma_handle); in ena_com_allocate_debug_area()
3324 if (unlikely(!host_attr->debug_area_virt_addr)) { in ena_com_allocate_debug_area()
3325 host_attr->debug_area_size = 0; in ena_com_allocate_debug_area()
3329 host_attr->debug_area_size = debug_area_size; in ena_com_allocate_debug_area()
3336 struct ena_customer_metrics *customer_metrics = &ena_dev->customer_metrics; in ena_com_allocate_customer_metrics_buffer()
3338 customer_metrics->buffer_len = ENA_CUSTOMER_METRICS_BUFFER_SIZE; in ena_com_allocate_customer_metrics_buffer()
3339 ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, in ena_com_allocate_customer_metrics_buffer()
3340 customer_metrics->buffer_len, in ena_com_allocate_customer_metrics_buffer()
3341 customer_metrics->buffer_virt_addr, in ena_com_allocate_customer_metrics_buffer()
3342 customer_metrics->buffer_dma_addr, in ena_com_allocate_customer_metrics_buffer()
3343 customer_metrics->buffer_dma_handle); in ena_com_allocate_customer_metrics_buffer()
3344 if (unlikely(!customer_metrics->buffer_virt_addr)) in ena_com_allocate_customer_metrics_buffer()
3352 struct ena_host_attribute *host_attr = &ena_dev->host_attr; in ena_com_delete_host_info()
3354 if (host_attr->host_info) { in ena_com_delete_host_info()
3355 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, in ena_com_delete_host_info()
3357 host_attr->host_info, in ena_com_delete_host_info()
3358 host_attr->host_info_dma_addr, in ena_com_delete_host_info()
3359 host_attr->host_info_dma_handle); in ena_com_delete_host_info()
3360 host_attr->host_info = NULL; in ena_com_delete_host_info()
3366 struct ena_host_attribute *host_attr = &ena_dev->host_attr; in ena_com_delete_debug_area()
3368 if (host_attr->debug_area_virt_addr) { in ena_com_delete_debug_area()
3369 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, in ena_com_delete_debug_area()
3370 host_attr->debug_area_size, in ena_com_delete_debug_area()
3371 host_attr->debug_area_virt_addr, in ena_com_delete_debug_area()
3372 host_attr->debug_area_dma_addr, in ena_com_delete_debug_area()
3373 host_attr->debug_area_dma_handle); in ena_com_delete_debug_area()
3374 host_attr->debug_area_virt_addr = NULL; in ena_com_delete_debug_area()
3380 struct ena_customer_metrics *customer_metrics = &ena_dev->customer_metrics; in ena_com_delete_customer_metrics_buffer()
3382 if (customer_metrics->buffer_virt_addr) { in ena_com_delete_customer_metrics_buffer()
3383 ENA_MEM_FREE_COHERENT(ena_dev->dmadev, in ena_com_delete_customer_metrics_buffer()
3384 customer_metrics->buffer_len, in ena_com_delete_customer_metrics_buffer()
3385 customer_metrics->buffer_virt_addr, in ena_com_delete_customer_metrics_buffer()
3386 customer_metrics->buffer_dma_addr, in ena_com_delete_customer_metrics_buffer()
3387 customer_metrics->buffer_dma_handle); in ena_com_delete_customer_metrics_buffer()
3388 customer_metrics->buffer_virt_addr = NULL; in ena_com_delete_customer_metrics_buffer()
3394 struct ena_host_attribute *host_attr = &ena_dev->host_attr; in ena_com_set_host_attributes()
3396 struct ena_admin_set_feat_cmd cmd; in ena_com_set_host_attributes() local
3405 memset(&cmd, 0x0, sizeof(cmd)); in ena_com_set_host_attributes()
3406 admin_queue = &ena_dev->admin_queue; in ena_com_set_host_attributes()
3408 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; in ena_com_set_host_attributes()
3409 cmd.feat_common.feature_id = ENA_ADMIN_HOST_ATTR_CONFIG; in ena_com_set_host_attributes()
3412 &cmd.u.host_attr.debug_ba, in ena_com_set_host_attributes()
3413 host_attr->debug_area_dma_addr); in ena_com_set_host_attributes()
3420 &cmd.u.host_attr.os_info_ba, in ena_com_set_host_attributes()
3421 host_attr->host_info_dma_addr); in ena_com_set_host_attributes()
3427 cmd.u.host_attr.debug_area_size = host_attr->debug_area_size; in ena_com_set_host_attributes()
3430 (struct ena_admin_aq_entry *)&cmd, in ena_com_set_host_attributes()
3431 sizeof(cmd), in ena_com_set_host_attributes()
3468 ena_dev->intr_delay_resolution, in ena_com_update_nonadaptive_moderation_interval_tx()
3469 &ena_dev->intr_moder_tx_interval); in ena_com_update_nonadaptive_moderation_interval_tx()
3477 ena_dev->intr_delay_resolution, in ena_com_update_nonadaptive_moderation_interval_rx()
3478 &ena_dev->intr_moder_rx_interval); in ena_com_update_nonadaptive_moderation_interval_rx()
3497 "Failed to get interrupt moderation admin cmd. rc: %d\n", rc); in ena_com_init_interrupt_moderation()
3509 /* Disable adaptive moderation by default - can be enabled later */ in ena_com_init_interrupt_moderation()
3517 return ena_dev->intr_moder_tx_interval; in ena_com_get_nonadaptive_moderation_interval_tx()
3522 return ena_dev->intr_moder_rx_interval; in ena_com_get_nonadaptive_moderation_interval_rx()
3529 struct ena_com_llq_info *llq_info = &ena_dev->llq_info; in ena_com_config_dev_mode()
3532 if (!llq_features->max_llq_num) { in ena_com_config_dev_mode()
3533 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; in ena_com_config_dev_mode()
3541 ena_dev->tx_max_header_size = llq_info->desc_list_entry_size - in ena_com_config_dev_mode()
3542 (llq_info->descs_num_before_header * sizeof(struct ena_eth_io_tx_desc)); in ena_com_config_dev_mode()
3544 if (unlikely(ena_dev->tx_max_header_size == 0)) { in ena_com_config_dev_mode()
3549 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV; in ena_com_config_dev_mode()