Lines Matching +full:axi +full:- +full:spi +full:- +full:engine +full:- +full:1
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2022 HabanaLabs, Ltd.
26 * 1. Host is protected by:
27 * - Range registers
28 * - MMU
31 * - Range registers (protect the first 512MB)
34 * - Range registers
35 * - Protection bits
39 * QMAN DMA channels 0,1 (PCI DMAN):
40 * - DMA is not secured.
41 * - PQ and CQ are secured.
42 * - CP is secured: The driver needs to parse CB but WREG should be allowed
52 * - Clear SRAM on context switch (happens on context switch when device is
54 * - MMU page tables area clear (happens on init)
56 * QMAN DMA 2-7, TPC, MME, NIC:
58 * CQ, CP and the engine are not secured
62 #define GAUDI_BOOT_FIT_FILE "habanalabs/gaudi/gaudi-boot-fit.itb"
63 #define GAUDI_LINUX_FW_FILE "habanalabs/gaudi/gaudi-fit.itb"
73 #define GAUDI_RESET_WAIT_MSEC 1 /* 1ms */
77 #define GAUDI_PLDM_RESET_WAIT_MSEC 1000 /* 1s */
79 #define GAUDI_PLDM_TEST_QUEUE_WAIT_USEC 1000000 /* 1s */
104 #define HBM_SCRUBBING_TIMEOUT_US 1000000 /* 1s */
134 [1] = GAUDI_QUEUE_ID_DMA_0_1,
209 "PQ AXI HBW error",
210 "CQ AXI HBW error",
211 "CP AXI HBW error",
214 "CP AXI LBW error",
218 "FENCE 1 inc over max value and clipped",
222 "FENCE 1 dec under min value and clipped",
231 "MSG AXI LBW returned with error"
352 { .id = 1, .name = "SYNC_OBJ_DMA_UP_FEEDBACK" },
408 [SP_NEXT_TPC] = mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0,
411 [SP_NEXT_MME] = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0,
414 [SP_DMA_QUEUES_OFFSET] = mmDMA1_QM_GLBL_CFG0 - mmDMA0_QM_GLBL_CFG0,
423 mmDMA0_QM_CP_FENCE0_CNT_0 - mmDMA0_QM_GLBL_CFG0,
425 mmDMA0_QM_CP_FENCE0_RDATA_0 - mmDMA0_QM_GLBL_CFG0,
426 [SP_CP_STS_OFFSET] = mmDMA0_QM_CP_STS_0 - mmDMA0_QM_GLBL_CFG0,
427 [SP_NUM_CORES] = 1,
463 * i.e. SYNC_MGR_W_S is actually 0, SYNC_MGR_E_S is 1, etc.
520 struct asic_fixed_properties *prop = &hdev->asic_prop; in set_default_power_values()
522 if (hdev->card_type == cpucp_card_type_pmc) { in set_default_power_values()
523 prop->max_power_default = MAX_POWER_DEFAULT_PMC; in set_default_power_values()
525 if (prop->fw_security_enabled) in set_default_power_values()
526 prop->dc_power_default = DC_POWER_DEFAULT_PMC_SEC; in set_default_power_values()
528 prop->dc_power_default = DC_POWER_DEFAULT_PMC; in set_default_power_values()
530 prop->max_power_default = MAX_POWER_DEFAULT_PCI; in set_default_power_values()
531 prop->dc_power_default = DC_POWER_DEFAULT_PCI; in set_default_power_values()
537 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi_set_fixed_properties()
541 prop->max_queues = GAUDI_QUEUE_ID_SIZE; in gaudi_set_fixed_properties()
542 prop->hw_queues_props = kcalloc(prop->max_queues, in gaudi_set_fixed_properties()
546 if (!prop->hw_queues_props) in gaudi_set_fixed_properties()
547 return -ENOMEM; in gaudi_set_fixed_properties()
549 for (i = 0 ; i < prop->max_queues ; i++) { in gaudi_set_fixed_properties()
551 prop->hw_queues_props[i].type = QUEUE_TYPE_EXT; in gaudi_set_fixed_properties()
552 prop->hw_queues_props[i].driver_only = 0; in gaudi_set_fixed_properties()
553 prop->hw_queues_props[i].supports_sync_stream = 1; in gaudi_set_fixed_properties()
554 prop->hw_queues_props[i].cb_alloc_flags = in gaudi_set_fixed_properties()
558 prop->hw_queues_props[i].type = QUEUE_TYPE_CPU; in gaudi_set_fixed_properties()
559 prop->hw_queues_props[i].driver_only = 1; in gaudi_set_fixed_properties()
560 prop->hw_queues_props[i].supports_sync_stream = 0; in gaudi_set_fixed_properties()
561 prop->hw_queues_props[i].cb_alloc_flags = in gaudi_set_fixed_properties()
564 prop->hw_queues_props[i].type = QUEUE_TYPE_INT; in gaudi_set_fixed_properties()
565 prop->hw_queues_props[i].driver_only = 0; in gaudi_set_fixed_properties()
566 prop->hw_queues_props[i].supports_sync_stream = 0; in gaudi_set_fixed_properties()
567 prop->hw_queues_props[i].cb_alloc_flags = in gaudi_set_fixed_properties()
571 prop->hw_queues_props[i].collective_mode = in gaudi_set_fixed_properties()
575 prop->cache_line_size = DEVICE_CACHE_LINE_SIZE; in gaudi_set_fixed_properties()
576 prop->cfg_base_address = CFG_BASE; in gaudi_set_fixed_properties()
577 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE; in gaudi_set_fixed_properties()
578 prop->host_base_address = HOST_PHYS_BASE; in gaudi_set_fixed_properties()
579 prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE; in gaudi_set_fixed_properties()
580 prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES; in gaudi_set_fixed_properties()
581 prop->completion_mode = HL_COMPLETION_MODE_JOB; in gaudi_set_fixed_properties()
582 prop->collective_first_sob = 0; in gaudi_set_fixed_properties()
583 prop->collective_first_mon = 0; in gaudi_set_fixed_properties()
586 prop->sync_stream_first_sob = in gaudi_set_fixed_properties()
590 /* 1 monitor per internal queue stream are reserved for collective in gaudi_set_fixed_properties()
593 prop->sync_stream_first_mon = in gaudi_set_fixed_properties()
597 prop->dram_base_address = DRAM_PHYS_BASE; in gaudi_set_fixed_properties()
598 prop->dram_size = GAUDI_HBM_SIZE_32GB; in gaudi_set_fixed_properties()
599 prop->dram_end_address = prop->dram_base_address + prop->dram_size; in gaudi_set_fixed_properties()
600 prop->dram_user_base_address = DRAM_BASE_ADDR_USER; in gaudi_set_fixed_properties()
602 prop->sram_base_address = SRAM_BASE_ADDR; in gaudi_set_fixed_properties()
603 prop->sram_size = SRAM_SIZE; in gaudi_set_fixed_properties()
604 prop->sram_end_address = prop->sram_base_address + prop->sram_size; in gaudi_set_fixed_properties()
605 prop->sram_user_base_address = in gaudi_set_fixed_properties()
606 prop->sram_base_address + SRAM_USER_BASE_OFFSET; in gaudi_set_fixed_properties()
608 prop->mmu_cache_mng_addr = MMU_CACHE_MNG_ADDR; in gaudi_set_fixed_properties()
609 prop->mmu_cache_mng_size = MMU_CACHE_MNG_SIZE; in gaudi_set_fixed_properties()
611 prop->mmu_pgt_addr = MMU_PAGE_TABLES_ADDR; in gaudi_set_fixed_properties()
612 if (hdev->pldm) in gaudi_set_fixed_properties()
613 prop->mmu_pgt_size = 0x800000; /* 8MB */ in gaudi_set_fixed_properties()
615 prop->mmu_pgt_size = MMU_PAGE_TABLES_SIZE; in gaudi_set_fixed_properties()
616 prop->mmu_pte_size = HL_PTE_SIZE; in gaudi_set_fixed_properties()
617 prop->dram_page_size = PAGE_SIZE_2MB; in gaudi_set_fixed_properties()
618 prop->device_mem_alloc_default_page_size = prop->dram_page_size; in gaudi_set_fixed_properties()
619 prop->dram_supports_virtual_memory = false; in gaudi_set_fixed_properties()
621 prop->pmmu.hop_shifts[MMU_HOP0] = MMU_V1_1_HOP0_SHIFT; in gaudi_set_fixed_properties()
622 prop->pmmu.hop_shifts[MMU_HOP1] = MMU_V1_1_HOP1_SHIFT; in gaudi_set_fixed_properties()
623 prop->pmmu.hop_shifts[MMU_HOP2] = MMU_V1_1_HOP2_SHIFT; in gaudi_set_fixed_properties()
624 prop->pmmu.hop_shifts[MMU_HOP3] = MMU_V1_1_HOP3_SHIFT; in gaudi_set_fixed_properties()
625 prop->pmmu.hop_shifts[MMU_HOP4] = MMU_V1_1_HOP4_SHIFT; in gaudi_set_fixed_properties()
626 prop->pmmu.hop_masks[MMU_HOP0] = MMU_V1_1_HOP0_MASK; in gaudi_set_fixed_properties()
627 prop->pmmu.hop_masks[MMU_HOP1] = MMU_V1_1_HOP1_MASK; in gaudi_set_fixed_properties()
628 prop->pmmu.hop_masks[MMU_HOP2] = MMU_V1_1_HOP2_MASK; in gaudi_set_fixed_properties()
629 prop->pmmu.hop_masks[MMU_HOP3] = MMU_V1_1_HOP3_MASK; in gaudi_set_fixed_properties()
630 prop->pmmu.hop_masks[MMU_HOP4] = MMU_V1_1_HOP4_MASK; in gaudi_set_fixed_properties()
631 prop->pmmu.start_addr = VA_HOST_SPACE_START; in gaudi_set_fixed_properties()
632 prop->pmmu.end_addr = in gaudi_set_fixed_properties()
633 (VA_HOST_SPACE_START + VA_HOST_SPACE_SIZE / 2) - 1; in gaudi_set_fixed_properties()
634 prop->pmmu.page_size = PAGE_SIZE_4KB; in gaudi_set_fixed_properties()
635 prop->pmmu.num_hops = MMU_ARCH_5_HOPS; in gaudi_set_fixed_properties()
636 prop->pmmu.last_mask = LAST_MASK; in gaudi_set_fixed_properties()
637 /* TODO: will be duplicated until implementing per-MMU props */ in gaudi_set_fixed_properties()
638 prop->pmmu.hop_table_size = HOP_TABLE_SIZE_512_PTE; in gaudi_set_fixed_properties()
639 prop->pmmu.hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE; in gaudi_set_fixed_properties()
642 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); in gaudi_set_fixed_properties()
643 prop->pmmu_huge.page_size = PAGE_SIZE_2MB; in gaudi_set_fixed_properties()
646 memcpy(&prop->dmmu, &prop->pmmu, sizeof(prop->pmmu)); in gaudi_set_fixed_properties()
647 prop->dmmu.start_addr = (VA_HOST_SPACE_START + VA_HOST_SPACE_SIZE / 2); in gaudi_set_fixed_properties()
648 prop->dmmu.end_addr = VA_HOST_SPACE_END; in gaudi_set_fixed_properties()
649 prop->dmmu.page_size = PAGE_SIZE_2MB; in gaudi_set_fixed_properties()
650 prop->dmmu.pgt_size = prop->mmu_pgt_size; in gaudi_set_fixed_properties()
652 prop->cfg_size = CFG_SIZE; in gaudi_set_fixed_properties()
653 prop->max_asid = MAX_ASID; in gaudi_set_fixed_properties()
654 prop->num_of_events = GAUDI_EVENT_SIZE; in gaudi_set_fixed_properties()
655 prop->max_num_of_engines = GAUDI_ENGINE_ID_SIZE; in gaudi_set_fixed_properties()
656 prop->tpc_enabled_mask = TPC_ENABLED_MASK; in gaudi_set_fixed_properties()
660 prop->cb_pool_cb_cnt = GAUDI_CB_POOL_CB_CNT; in gaudi_set_fixed_properties()
661 prop->cb_pool_cb_size = GAUDI_CB_POOL_CB_SIZE; in gaudi_set_fixed_properties()
663 prop->pcie_dbi_base_address = mmPCIE_DBI_BASE; in gaudi_set_fixed_properties()
664 prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; in gaudi_set_fixed_properties()
666 strscpy_pad(prop->cpucp_info.card_name, GAUDI_DEFAULT_CARD_NAME, in gaudi_set_fixed_properties()
669 prop->max_pending_cs = GAUDI_MAX_PENDING_CS; in gaudi_set_fixed_properties()
671 prop->first_available_user_sob[HL_GAUDI_WS_DCORE] = in gaudi_set_fixed_properties()
672 prop->sync_stream_first_sob + in gaudi_set_fixed_properties()
674 prop->first_available_user_mon[HL_GAUDI_WS_DCORE] = in gaudi_set_fixed_properties()
675 prop->sync_stream_first_mon + in gaudi_set_fixed_properties()
678 prop->first_available_user_interrupt = USHRT_MAX; in gaudi_set_fixed_properties()
679 prop->tpc_interrupt_id = USHRT_MAX; in gaudi_set_fixed_properties()
682 prop->eq_interrupt_id = 0; in gaudi_set_fixed_properties()
685 prop->first_available_cq[i] = USHRT_MAX; in gaudi_set_fixed_properties()
687 prop->fw_cpu_boot_dev_sts0_valid = false; in gaudi_set_fixed_properties()
688 prop->fw_cpu_boot_dev_sts1_valid = false; in gaudi_set_fixed_properties()
689 prop->hard_reset_done_by_fw = false; in gaudi_set_fixed_properties()
690 prop->gic_interrupts_enable = true; in gaudi_set_fixed_properties()
692 prop->server_type = HL_SERVER_TYPE_UNKNOWN; in gaudi_set_fixed_properties()
694 prop->clk_pll_index = HL_GAUDI_MME_PLL; in gaudi_set_fixed_properties()
695 prop->max_freq_value = GAUDI_MAX_CLK_FREQ; in gaudi_set_fixed_properties()
697 prop->use_get_power_for_reset_history = true; in gaudi_set_fixed_properties()
699 prop->configurable_stop_on_err = true; in gaudi_set_fixed_properties()
701 prop->set_max_power_on_device_init = true; in gaudi_set_fixed_properties()
703 prop->dma_mask = 48; in gaudi_set_fixed_properties()
705 prop->hbw_flush_reg = mmPCIE_WRAP_RR_ELBI_RD_SEC_REG_CTRL; in gaudi_set_fixed_properties()
720 hdev->rmmio = hdev->pcie_bar[CFG_BAR_ID] + in gaudi_pci_bars_map()
721 (CFG_BASE - SPI_FLASH_BASE_ADDR); in gaudi_pci_bars_map()
728 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_set_hbm_bar_base()
733 if ((gaudi) && (gaudi->hbm_bar_cur_addr == addr)) in gaudi_set_hbm_bar_base()
736 if (hdev->asic_prop.iatu_done_by_fw) in gaudi_set_hbm_bar_base()
739 /* Inbound Region 2 - Bar 4 - Point to HBM */ in gaudi_set_hbm_bar_base()
748 old_addr = gaudi->hbm_bar_cur_addr; in gaudi_set_hbm_bar_base()
749 gaudi->hbm_bar_cur_addr = addr; in gaudi_set_hbm_bar_base()
761 if (hdev->asic_prop.iatu_done_by_fw) in gaudi_init_iatu()
764 /* Inbound Region 0 - Bar 0 - Point to SRAM + CFG */ in gaudi_init_iatu()
772 /* Inbound Region 1 - Bar 2 - Point to SPI FLASH */ in gaudi_init_iatu()
776 rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region); in gaudi_init_iatu()
780 /* Inbound Region 2 - Bar 4 - Point to HBM */ in gaudi_init_iatu()
788 /* Outbound Region 0 - Point to Host */ in gaudi_init_iatu()
804 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi_early_init()
805 struct pci_dev *pdev = hdev->pdev; in gaudi_early_init()
812 dev_err(hdev->dev, "Failed setting fixed properties\n"); in gaudi_early_init()
820 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", in gaudi_early_init()
822 rc = -ENODEV; in gaudi_early_init()
829 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", in gaudi_early_init()
831 rc = -ENODEV; in gaudi_early_init()
835 prop->dram_pci_bar_size = pci_resource_len(pdev, HBM_BAR_ID); in gaudi_early_init()
836 hdev->dram_pci_bar_start = pci_resource_start(pdev, HBM_BAR_ID); in gaudi_early_init()
839 if (hdev->asic_prop.fw_security_enabled) { in gaudi_early_init()
840 hdev->asic_prop.iatu_done_by_fw = true; in gaudi_early_init()
843 * GIC-security-bit can ONLY be set by CPUCP, so in this stage in gaudi_early_init()
846 hdev->asic_prop.gic_interrupts_enable = false; in gaudi_early_init()
858 hdev->asic_prop.iatu_done_by_fw = true; in gaudi_early_init()
866 * version to determine whether we run with a security-enabled firmware in gaudi_early_init()
870 if (hdev->reset_on_preboot_fail) in gaudi_early_init()
872 hdev->asic_funcs->hw_fini(hdev, true, false); in gaudi_early_init()
877 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n"); in gaudi_early_init()
878 rc = hdev->asic_funcs->hw_fini(hdev, true, false); in gaudi_early_init()
880 dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc); in gaudi_early_init()
890 kfree(hdev->asic_prop.hw_queues_props); in gaudi_early_init()
896 kfree(hdev->asic_prop.hw_queues_props); in gaudi_early_fini()
903 * gaudi_fetch_psoc_frequency - Fetch PSOC frequency values
911 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi_fetch_psoc_frequency()
915 if ((hdev->fw_components & FW_TYPE_LINUX) && in gaudi_fetch_psoc_frequency()
916 (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PLL_INFO_EN)) { in gaudi_fetch_psoc_frequency()
917 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_fetch_psoc_frequency()
919 if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi_fetch_psoc_frequency()
941 freq = PLL_REF_CLK / (div_fctr + 1); in gaudi_fetch_psoc_frequency()
944 pll_clk = PLL_REF_CLK * (nf + 1) / in gaudi_fetch_psoc_frequency()
945 ((nr + 1) * (od + 1)); in gaudi_fetch_psoc_frequency()
949 freq = pll_clk / (div_fctr + 1); in gaudi_fetch_psoc_frequency()
951 dev_warn(hdev->dev, "Received invalid div select value: %#x", div_sel); in gaudi_fetch_psoc_frequency()
956 prop->psoc_timestamp_frequency = freq; in gaudi_fetch_psoc_frequency()
957 prop->psoc_pci_pll_nr = nr; in gaudi_fetch_psoc_frequency()
958 prop->psoc_pci_pll_nf = nf; in gaudi_fetch_psoc_frequency()
959 prop->psoc_pci_pll_od = od; in gaudi_fetch_psoc_frequency()
960 prop->psoc_pci_pll_div_factor = div_fctr; in gaudi_fetch_psoc_frequency()
968 struct asic_fixed_properties *prop = &hdev->asic_prop; in _gaudi_init_tpc_mem()
979 return -EFAULT; in _gaudi_init_tpc_mem()
981 init_tpc_mem_pkt = cb->kernel_address; in _gaudi_init_tpc_mem()
985 init_tpc_mem_pkt->tsize = cpu_to_le32(tpc_kernel_size); in _gaudi_init_tpc_mem()
988 ctl |= FIELD_PREP(GAUDI_PKT_LIN_DMA_CTL_LIN_MASK, 1); in _gaudi_init_tpc_mem()
989 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); in _gaudi_init_tpc_mem()
990 ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); in _gaudi_init_tpc_mem()
992 init_tpc_mem_pkt->ctl = cpu_to_le32(ctl); in _gaudi_init_tpc_mem()
994 init_tpc_mem_pkt->src_addr = cpu_to_le64(tpc_kernel_src_addr); in _gaudi_init_tpc_mem()
998 round_up(prop->sram_user_base_address, SZ_8K)); in _gaudi_init_tpc_mem()
999 init_tpc_mem_pkt->dst_addr |= cpu_to_le64(dst_addr); in _gaudi_init_tpc_mem()
1003 dev_err(hdev->dev, "Failed to allocate a new job\n"); in _gaudi_init_tpc_mem()
1004 rc = -ENOMEM; in _gaudi_init_tpc_mem()
1008 job->id = 0; in _gaudi_init_tpc_mem()
1009 job->user_cb = cb; in _gaudi_init_tpc_mem()
1010 atomic_inc(&job->user_cb->cs_cnt); in _gaudi_init_tpc_mem()
1011 job->user_cb_size = cb_size; in _gaudi_init_tpc_mem()
1012 job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0; in _gaudi_init_tpc_mem()
1013 job->patched_cb = job->user_cb; in _gaudi_init_tpc_mem()
1014 job->job_cb_size = job->user_cb_size + sizeof(struct packet_msg_prot); in _gaudi_init_tpc_mem()
1030 hl_userptr_delete_list(hdev, &job->userptr_list); in _gaudi_init_tpc_mem()
1033 atomic_dec(&cb->cs_cnt); in _gaudi_init_tpc_mem()
1037 hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle); in _gaudi_init_tpc_mem()
1043 * gaudi_init_tpc_mem() - Initialize TPC memories.
1059 rc = request_firmware(&fw, GAUDI_TPC_FW_FILE, hdev->dev); in gaudi_init_tpc_mem()
1060 if (rc == -EINTR && count-- > 0) { in gaudi_init_tpc_mem()
1066 dev_err(hdev->dev, "Failed to load firmware file %s\n", in gaudi_init_tpc_mem()
1071 fw_size = fw->size; in gaudi_init_tpc_mem()
1074 dev_err(hdev->dev, in gaudi_init_tpc_mem()
1077 rc = -ENOMEM; in gaudi_init_tpc_mem()
1081 memcpy(cpu_addr, fw->data, fw_size); in gaudi_init_tpc_mem()
1085 hl_asic_dma_free_coherent(hdev, fw->size, cpu_addr, dma_handle); in gaudi_init_tpc_mem()
1094 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_collective_map_sobs()
1095 struct gaudi_collective_properties *prop = &gaudi->collective_props; in gaudi_collective_map_sobs()
1101 stream * HL_RSVD_SOBS + prop->curr_sob_group_idx[stream]; in gaudi_collective_map_sobs()
1102 sob_id = prop->hw_sob_group[sob_group_id].base_sob_id; in gaudi_collective_map_sobs()
1106 q = &hdev->kernel_queues[queue_id + (4 * i)]; in gaudi_collective_map_sobs()
1107 q->sync_stream_prop.collective_sob_id = sob_id + i; in gaudi_collective_map_sobs()
1111 * engine need to participate in the reduction process in gaudi_collective_map_sobs()
1114 q = &hdev->kernel_queues[queue_id]; in gaudi_collective_map_sobs()
1115 q->sync_stream_prop.collective_sob_id = in gaudi_collective_map_sobs()
1119 q = &hdev->kernel_queues[queue_id]; in gaudi_collective_map_sobs()
1120 q->sync_stream_prop.collective_sob_id = in gaudi_collective_map_sobs()
1128 struct hl_device *hdev = hw_sob_group->hdev; in gaudi_sob_group_hw_reset()
1133 (hw_sob_group->base_sob_id * 4) + (i * 4)), 0); in gaudi_sob_group_hw_reset()
1135 kref_init(&hw_sob_group->kref); in gaudi_sob_group_hw_reset()
1142 struct hl_device *hdev = hw_sob_group->hdev; in gaudi_sob_group_reset_error()
1144 dev_crit(hdev->dev, in gaudi_sob_group_reset_error()
1146 hw_sob_group->base_sob_id); in gaudi_sob_group_reset_error()
1154 prop = &gaudi->collective_props; in gaudi_collective_mstr_sob_mask_set()
1156 memset(prop->mstr_sob_mask, 0, sizeof(prop->mstr_sob_mask)); in gaudi_collective_mstr_sob_mask_set()
1159 if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i)) in gaudi_collective_mstr_sob_mask_set()
1160 prop->mstr_sob_mask[i / HL_MAX_SOBS_PER_MONITOR] |= in gaudi_collective_mstr_sob_mask_set()
1162 /* Set collective engine bit */ in gaudi_collective_mstr_sob_mask_set()
1163 prop->mstr_sob_mask[i / HL_MAX_SOBS_PER_MONITOR] |= in gaudi_collective_mstr_sob_mask_set()
1173 gaudi = hdev->asic_specific; in gaudi_collective_init()
1174 prop = &gaudi->collective_props; in gaudi_collective_init()
1175 sob_id = hdev->asic_prop.collective_first_sob; in gaudi_collective_init()
1183 prop->hw_sob_group[i].hdev = hdev; in gaudi_collective_init()
1184 prop->hw_sob_group[i].base_sob_id = sob_id; in gaudi_collective_init()
1186 gaudi_sob_group_hw_reset(&prop->hw_sob_group[i].kref); in gaudi_collective_init()
1190 prop->next_sob_group_val[i] = 1; in gaudi_collective_init()
1191 prop->curr_sob_group_idx[i] = 0; in gaudi_collective_init()
1202 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_reset_sob_group()
1203 struct gaudi_collective_properties *cprop = &gaudi->collective_props; in gaudi_reset_sob_group()
1205 kref_put(&cprop->hw_sob_group[sob_group].kref, in gaudi_reset_sob_group()
1218 gaudi = hdev->asic_specific; in gaudi_collective_master_init_job()
1219 cprop = &gaudi->collective_props; in gaudi_collective_master_init_job()
1220 queue_id = job->hw_queue_id; in gaudi_collective_master_init_job()
1221 prop = &hdev->kernel_queues[queue_id].sync_stream_prop; in gaudi_collective_master_init_job()
1224 cprop->hw_sob_group[sob_group_offset].base_sob_id; in gaudi_collective_master_init_job()
1225 master_monitor = prop->collective_mstr_mon_id[0]; in gaudi_collective_master_init_job()
1227 cprop->hw_sob_group[sob_group_offset].queue_id = queue_id; in gaudi_collective_master_init_job()
1229 dev_dbg(hdev->dev, in gaudi_collective_master_init_job()
1231 master_sob_base, cprop->mstr_sob_mask[0], in gaudi_collective_master_init_job()
1232 cprop->next_sob_group_val[stream], in gaudi_collective_master_init_job()
1235 wait_prop.data = (void *) job->patched_cb; in gaudi_collective_master_init_job()
1237 wait_prop.sob_mask = cprop->mstr_sob_mask[0]; in gaudi_collective_master_init_job()
1238 wait_prop.sob_val = cprop->next_sob_group_val[stream]; in gaudi_collective_master_init_job()
1245 master_monitor = prop->collective_mstr_mon_id[1]; in gaudi_collective_master_init_job()
1247 dev_dbg(hdev->dev, in gaudi_collective_master_init_job()
1249 master_sob_base, cprop->mstr_sob_mask[1], in gaudi_collective_master_init_job()
1250 cprop->next_sob_group_val[stream], in gaudi_collective_master_init_job()
1254 wait_prop.sob_mask = cprop->mstr_sob_mask[1]; in gaudi_collective_master_init_job()
1267 queue_id = job->hw_queue_id; in gaudi_collective_slave_init_job()
1268 prop = &hdev->kernel_queues[queue_id].sync_stream_prop; in gaudi_collective_slave_init_job()
1270 if (job->cs->encaps_signals) { in gaudi_collective_slave_init_job()
1275 hl_hw_queue_encaps_sig_set_sob_info(hdev, job->cs, job, in gaudi_collective_slave_init_job()
1278 dev_dbg(hdev->dev, "collective wait: Sequence %llu found, sob_id: %u, wait for sob_val: %u\n", in gaudi_collective_slave_init_job()
1279 job->cs->sequence, in gaudi_collective_slave_init_job()
1280 cs_cmpl->hw_sob->sob_id, in gaudi_collective_slave_init_job()
1281 cs_cmpl->sob_val); in gaudi_collective_slave_init_job()
1285 wait_prop.data = (void *) job->user_cb; in gaudi_collective_slave_init_job()
1286 wait_prop.sob_base = cs_cmpl->hw_sob->sob_id; in gaudi_collective_slave_init_job()
1288 wait_prop.sob_val = cs_cmpl->sob_val; in gaudi_collective_slave_init_job()
1289 wait_prop.mon_id = prop->collective_slave_mon_id; in gaudi_collective_slave_init_job()
1293 dev_dbg(hdev->dev, in gaudi_collective_slave_init_job()
1295 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val, in gaudi_collective_slave_init_job()
1296 prop->collective_slave_mon_id, queue_id); in gaudi_collective_slave_init_job()
1300 dev_dbg(hdev->dev, in gaudi_collective_slave_init_job()
1301 "generate signal CB, sob_id: %d, sob val: 1, q_idx: %d\n", in gaudi_collective_slave_init_job()
1302 prop->collective_sob_id, queue_id); in gaudi_collective_slave_init_job()
1304 cb_size += gaudi_gen_signal_cb(hdev, job->user_cb, in gaudi_collective_slave_init_job()
1305 prop->collective_sob_id, cb_size, false); in gaudi_collective_slave_init_job()
1311 container_of(cs->signal_fence, struct hl_cs_compl, base_fence); in gaudi_collective_wait_init_cs()
1313 container_of(cs->fence, struct hl_cs_compl, base_fence); in gaudi_collective_wait_init_cs()
1314 struct hl_cs_encaps_sig_handle *handle = cs->encaps_sig_hdl; in gaudi_collective_wait_init_cs()
1322 ctx = cs->ctx; in gaudi_collective_wait_init_cs()
1323 hdev = ctx->hdev; in gaudi_collective_wait_init_cs()
1324 gaudi = hdev->asic_specific; in gaudi_collective_wait_init_cs()
1325 cprop = &gaudi->collective_props; in gaudi_collective_wait_init_cs()
1327 if (cs->encaps_signals) { in gaudi_collective_wait_init_cs()
1328 cs_cmpl->hw_sob = handle->hw_sob; in gaudi_collective_wait_init_cs()
1335 cs_cmpl->sob_val = 0; in gaudi_collective_wait_init_cs()
1338 cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob; in gaudi_collective_wait_init_cs()
1339 cs_cmpl->sob_val = signal_cs_cmpl->sob_val; in gaudi_collective_wait_init_cs()
1354 spin_lock(&signal_cs_cmpl->lock); in gaudi_collective_wait_init_cs()
1356 if (completion_done(&cs->signal_fence->completion)) { in gaudi_collective_wait_init_cs()
1357 spin_unlock(&signal_cs_cmpl->lock); in gaudi_collective_wait_init_cs()
1358 return -EINVAL; in gaudi_collective_wait_init_cs()
1361 kref_get(&cs_cmpl->hw_sob->kref); in gaudi_collective_wait_init_cs()
1363 spin_unlock(&signal_cs_cmpl->lock); in gaudi_collective_wait_init_cs()
1365 /* Calculate the stream from collective master queue (1st job) */ in gaudi_collective_wait_init_cs()
1366 job = list_first_entry(&cs->job_list, struct hl_cs_job, cs_node); in gaudi_collective_wait_init_cs()
1367 stream = job->hw_queue_id % 4; in gaudi_collective_wait_init_cs()
1369 stream * HL_RSVD_SOBS + cprop->curr_sob_group_idx[stream]; in gaudi_collective_wait_init_cs()
1371 list_for_each_entry(job, &cs->job_list, cs_node) { in gaudi_collective_wait_init_cs()
1372 queue_id = job->hw_queue_id; in gaudi_collective_wait_init_cs()
1374 if (hdev->kernel_queues[queue_id].collective_mode == in gaudi_collective_wait_init_cs()
1382 cs_cmpl->sob_group = sob_group_offset; in gaudi_collective_wait_init_cs()
1385 kref_get(&cprop->hw_sob_group[sob_group_offset].kref); in gaudi_collective_wait_init_cs()
1386 cprop->next_sob_group_val[stream]++; in gaudi_collective_wait_init_cs()
1388 if (cprop->next_sob_group_val[stream] == HL_MAX_SOB_VAL) { in gaudi_collective_wait_init_cs()
1394 kref_put(&cprop->hw_sob_group[sob_group_offset].kref, in gaudi_collective_wait_init_cs()
1396 cprop->next_sob_group_val[stream] = 1; in gaudi_collective_wait_init_cs()
1398 cprop->curr_sob_group_idx[stream] = in gaudi_collective_wait_init_cs()
1399 (cprop->curr_sob_group_idx[stream] + 1) & in gaudi_collective_wait_init_cs()
1400 (HL_RSVD_SOBS - 1); in gaudi_collective_wait_init_cs()
1404 dev_dbg(hdev->dev, "switched to SOB group %d, stream: %d\n", in gaudi_collective_wait_init_cs()
1405 cprop->curr_sob_group_idx[stream], stream); in gaudi_collective_wait_init_cs()
1409 hl_fence_put(cs->signal_fence); in gaudi_collective_wait_init_cs()
1410 cs->signal_fence = NULL; in gaudi_collective_wait_init_cs()
1423 return cacheline_end - user_cb_size + additional_commands; in gaudi_get_patched_cb_extra_size()
1440 cntr = &hdev->aggregated_cs_counters; in gaudi_collective_wait_create_job()
1444 * 4 msg short packets for monitor 1 configuration in gaudi_collective_wait_create_job()
1445 * 1 fence packet in gaudi_collective_wait_create_job()
1447 * 1 fence packet in gaudi_collective_wait_create_job()
1457 * 1 fence packet in gaudi_collective_wait_create_job()
1458 * 1 additional msg short packet for sob signal in gaudi_collective_wait_create_job()
1465 hw_queue_prop = &hdev->asic_prop.hw_queues_props[queue_id]; in gaudi_collective_wait_create_job()
1466 job = hl_cs_allocate_job(hdev, hw_queue_prop->type, true); in gaudi_collective_wait_create_job()
1468 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt); in gaudi_collective_wait_create_job()
1469 atomic64_inc(&cntr->out_of_mem_drop_cnt); in gaudi_collective_wait_create_job()
1470 dev_err(hdev->dev, "Failed to allocate a new job\n"); in gaudi_collective_wait_create_job()
1471 return -ENOMEM; in gaudi_collective_wait_create_job()
1477 atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt); in gaudi_collective_wait_create_job()
1478 atomic64_inc(&cntr->out_of_mem_drop_cnt); in gaudi_collective_wait_create_job()
1480 return -EFAULT; in gaudi_collective_wait_create_job()
1483 job->id = 0; in gaudi_collective_wait_create_job()
1484 job->cs = cs; in gaudi_collective_wait_create_job()
1485 job->user_cb = cb; in gaudi_collective_wait_create_job()
1486 atomic_inc(&job->user_cb->cs_cnt); in gaudi_collective_wait_create_job()
1487 job->user_cb_size = cb_size; in gaudi_collective_wait_create_job()
1488 job->hw_queue_id = queue_id; in gaudi_collective_wait_create_job()
1494 if (cs->encaps_signals) in gaudi_collective_wait_create_job()
1495 job->encaps_sig_wait_offset = encaps_signal_offset; in gaudi_collective_wait_create_job()
1499 * We call hl_cb_destroy() out of two reasons - we don't need in gaudi_collective_wait_create_job()
1504 job->patched_cb = job->user_cb; in gaudi_collective_wait_create_job()
1506 job->patched_cb = NULL; in gaudi_collective_wait_create_job()
1508 job->job_cb_size = job->user_cb_size; in gaudi_collective_wait_create_job()
1509 hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle); in gaudi_collective_wait_create_job()
1512 if (hw_queue_prop->type == QUEUE_TYPE_EXT) in gaudi_collective_wait_create_job()
1515 cs->jobs_in_queue_cnt[job->hw_queue_id]++; in gaudi_collective_wait_create_job()
1517 list_add_tail(&job->cs_node, &cs->job_list); in gaudi_collective_wait_create_job()
1529 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_collective_wait_create_jobs()
1537 hw_queue_prop = &hdev->asic_prop.hw_queues_props[wait_queue_id]; in gaudi_collective_wait_create_jobs()
1538 if (!(hw_queue_prop->collective_mode == HL_COLLECTIVE_MASTER)) { in gaudi_collective_wait_create_jobs()
1539 dev_err(hdev->dev, in gaudi_collective_wait_create_jobs()
1542 return -EINVAL; in gaudi_collective_wait_create_jobs()
1545 /* Verify engine id is supported */ in gaudi_collective_wait_create_jobs()
1548 dev_err(hdev->dev, in gaudi_collective_wait_create_jobs()
1549 "Collective wait does not support engine %u\n", in gaudi_collective_wait_create_jobs()
1551 return -EINVAL; in gaudi_collective_wait_create_jobs()
1561 num_jobs = NUMBER_OF_SOBS_IN_GRP + 1; in gaudi_collective_wait_create_jobs()
1567 * First monitor for NICs 0-7, second monitor for NICs 8-9 and the in gaudi_collective_wait_create_jobs()
1568 * reduction engine (DMA5/TPC7). in gaudi_collective_wait_create_jobs()
1571 * all wait for the user to signal sob 'cs_cmpl->sob_val'. in gaudi_collective_wait_create_jobs()
1581 if (gaudi->hw_cap_initialized & in gaudi_collective_wait_create_jobs()
1611 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_late_init()
1614 rc = gaudi->cpucp_info_get(hdev); in gaudi_late_init()
1616 dev_err(hdev->dev, "Failed to get cpucp info\n"); in gaudi_late_init()
1620 if ((hdev->card_type == cpucp_card_type_pci) && in gaudi_late_init()
1621 (hdev->nic_ports_mask & 0x3)) { in gaudi_late_init()
1622 dev_info(hdev->dev, in gaudi_late_init()
1624 hdev->nic_ports_mask &= ~0x3; in gaudi_late_init()
1638 gaudi->hw_cap_initialized &= ~(HW_CAP_NIC0 | HW_CAP_NIC1); in gaudi_late_init()
1646 rc = hdev->asic_funcs->scrub_device_mem(hdev); in gaudi_late_init()
1652 dev_err(hdev->dev, "Failed to fetch psoc frequency\n"); in gaudi_late_init()
1658 dev_err(hdev->dev, "Failed to clear MMU page tables range\n"); in gaudi_late_init()
1664 dev_err(hdev->dev, "Failed to initialize TPC memories\n"); in gaudi_late_init()
1670 dev_err(hdev->dev, "Failed to init collective\n"); in gaudi_late_init()
1675 * initialize the ASID one time during device initialization with the fixed value of 1 in gaudi_late_init()
1677 gaudi_mmu_prepare(hdev, 1); in gaudi_late_init()
1701 * The device CPU works with 40-bits addresses, while bit 39 must be set in gaudi_alloc_cpu_accessible_dma_mem()
1702 * to '1' when accessing the host. in gaudi_alloc_cpu_accessible_dma_mem()
1714 rc = -ENOMEM; in gaudi_alloc_cpu_accessible_dma_mem()
1718 end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1; in gaudi_alloc_cpu_accessible_dma_mem()
1725 dev_err(hdev->dev, in gaudi_alloc_cpu_accessible_dma_mem()
1727 rc = -EFAULT; in gaudi_alloc_cpu_accessible_dma_mem()
1731 hdev->cpu_accessible_dma_mem = virt_addr_arr[i]; in gaudi_alloc_cpu_accessible_dma_mem()
1732 hdev->cpu_accessible_dma_address = dma_addr_arr[i]; in gaudi_alloc_cpu_accessible_dma_mem()
1733 hdev->cpu_pci_msb_addr = in gaudi_alloc_cpu_accessible_dma_mem()
1734 GAUDI_CPU_PCI_MSB_ADDR(hdev->cpu_accessible_dma_address); in gaudi_alloc_cpu_accessible_dma_mem()
1736 if (!hdev->asic_prop.fw_security_enabled) in gaudi_alloc_cpu_accessible_dma_mem()
1737 GAUDI_PCI_TO_CPU_ADDR(hdev->cpu_accessible_dma_address); in gaudi_alloc_cpu_accessible_dma_mem()
1749 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_free_internal_qmans_pq_mem()
1754 q = &gaudi->internal_qmans[i]; in gaudi_free_internal_qmans_pq_mem()
1755 if (!q->pq_kernel_addr) in gaudi_free_internal_qmans_pq_mem()
1757 hl_asic_dma_free_coherent(hdev, q->pq_size, q->pq_kernel_addr, q->pq_dma_addr); in gaudi_free_internal_qmans_pq_mem()
1763 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_alloc_internal_qmans_pq_mem()
1771 q = &gaudi->internal_qmans[i]; in gaudi_alloc_internal_qmans_pq_mem()
1775 q->pq_size = HBM_DMA_QMAN_SIZE_IN_BYTES; in gaudi_alloc_internal_qmans_pq_mem()
1778 q->pq_size = MME_QMAN_SIZE_IN_BYTES; in gaudi_alloc_internal_qmans_pq_mem()
1781 q->pq_size = TPC_QMAN_SIZE_IN_BYTES; in gaudi_alloc_internal_qmans_pq_mem()
1784 q->pq_size = NIC_QMAN_SIZE_IN_BYTES; in gaudi_alloc_internal_qmans_pq_mem()
1787 dev_err(hdev->dev, "Bad internal queue index %d", i); in gaudi_alloc_internal_qmans_pq_mem()
1788 rc = -EINVAL; in gaudi_alloc_internal_qmans_pq_mem()
1792 q->pq_kernel_addr = hl_asic_dma_alloc_coherent(hdev, q->pq_size, &q->pq_dma_addr, in gaudi_alloc_internal_qmans_pq_mem()
1794 if (!q->pq_kernel_addr) { in gaudi_alloc_internal_qmans_pq_mem()
1795 rc = -ENOMEM; in gaudi_alloc_internal_qmans_pq_mem()
1809 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi_set_pci_memory_regions()
1813 region = &hdev->pci_mem_region[PCI_REGION_CFG]; in gaudi_set_pci_memory_regions()
1814 region->region_base = CFG_BASE; in gaudi_set_pci_memory_regions()
1815 region->region_size = CFG_SIZE; in gaudi_set_pci_memory_regions()
1816 region->offset_in_bar = CFG_BASE - SPI_FLASH_BASE_ADDR; in gaudi_set_pci_memory_regions()
1817 region->bar_size = CFG_BAR_SIZE; in gaudi_set_pci_memory_regions()
1818 region->bar_id = CFG_BAR_ID; in gaudi_set_pci_memory_regions()
1819 region->used = 1; in gaudi_set_pci_memory_regions()
1822 region = &hdev->pci_mem_region[PCI_REGION_SRAM]; in gaudi_set_pci_memory_regions()
1823 region->region_base = SRAM_BASE_ADDR; in gaudi_set_pci_memory_regions()
1824 region->region_size = SRAM_SIZE; in gaudi_set_pci_memory_regions()
1825 region->offset_in_bar = 0; in gaudi_set_pci_memory_regions()
1826 region->bar_size = SRAM_BAR_SIZE; in gaudi_set_pci_memory_regions()
1827 region->bar_id = SRAM_BAR_ID; in gaudi_set_pci_memory_regions()
1828 region->used = 1; in gaudi_set_pci_memory_regions()
1831 region = &hdev->pci_mem_region[PCI_REGION_DRAM]; in gaudi_set_pci_memory_regions()
1832 region->region_base = DRAM_PHYS_BASE; in gaudi_set_pci_memory_regions()
1833 region->region_size = hdev->asic_prop.dram_size; in gaudi_set_pci_memory_regions()
1834 region->offset_in_bar = 0; in gaudi_set_pci_memory_regions()
1835 region->bar_size = prop->dram_pci_bar_size; in gaudi_set_pci_memory_regions()
1836 region->bar_id = HBM_BAR_ID; in gaudi_set_pci_memory_regions()
1837 region->used = 1; in gaudi_set_pci_memory_regions()
1840 region = &hdev->pci_mem_region[PCI_REGION_SP_SRAM]; in gaudi_set_pci_memory_regions()
1841 region->region_base = PSOC_SCRATCHPAD_ADDR; in gaudi_set_pci_memory_regions()
1842 region->region_size = PSOC_SCRATCHPAD_SIZE; in gaudi_set_pci_memory_regions()
1843 region->offset_in_bar = PSOC_SCRATCHPAD_ADDR - SPI_FLASH_BASE_ADDR; in gaudi_set_pci_memory_regions()
1844 region->bar_size = CFG_BAR_SIZE; in gaudi_set_pci_memory_regions()
1845 region->bar_id = CFG_BAR_ID; in gaudi_set_pci_memory_regions()
1846 region->used = 1; in gaudi_set_pci_memory_regions()
1858 return -ENOMEM; in gaudi_sw_init()
1863 dev_err(hdev->dev, in gaudi_sw_init()
1866 rc = -EINVAL; in gaudi_sw_init()
1870 gaudi->events[event_id++] = in gaudi_sw_init()
1875 gaudi->cpucp_info_get = gaudi_cpucp_info_get; in gaudi_sw_init()
1877 hdev->asic_specific = gaudi; in gaudi_sw_init()
1880 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), in gaudi_sw_init()
1881 &hdev->pdev->dev, GAUDI_DMA_POOL_BLK_SIZE, 8, 0); in gaudi_sw_init()
1882 if (!hdev->dma_pool) { in gaudi_sw_init()
1883 dev_err(hdev->dev, "failed to create DMA pool\n"); in gaudi_sw_init()
1884 rc = -ENOMEM; in gaudi_sw_init()
1892 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); in gaudi_sw_init()
1893 if (!hdev->cpu_accessible_dma_pool) { in gaudi_sw_init()
1894 dev_err(hdev->dev, in gaudi_sw_init()
1896 rc = -ENOMEM; in gaudi_sw_init()
1900 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, in gaudi_sw_init()
1901 (uintptr_t) hdev->cpu_accessible_dma_mem, in gaudi_sw_init()
1902 HL_CPU_ACCESSIBLE_MEM_SIZE, -1); in gaudi_sw_init()
1904 dev_err(hdev->dev, in gaudi_sw_init()
1906 rc = -EFAULT; in gaudi_sw_init()
1914 spin_lock_init(&gaudi->hw_queues_lock); in gaudi_sw_init()
1916 hdev->supports_sync_stream = true; in gaudi_sw_init()
1917 hdev->supports_coresight = true; in gaudi_sw_init()
1918 hdev->supports_staged_submission = true; in gaudi_sw_init()
1919 hdev->supports_wait_for_multi_cs = true; in gaudi_sw_init()
1921 hdev->asic_funcs->set_pci_memory_regions(hdev); in gaudi_sw_init()
1922 hdev->stream_master_qid_arr = in gaudi_sw_init()
1923 hdev->asic_funcs->get_stream_master_qid_arr(); in gaudi_sw_init()
1924 hdev->stream_master_qid_arr_size = GAUDI_STREAM_MASTER_ARR_SIZE; in gaudi_sw_init()
1929 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in gaudi_sw_init()
1931 if (!hdev->asic_prop.fw_security_enabled) in gaudi_sw_init()
1932 GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address, in gaudi_sw_init()
1933 hdev->cpu_pci_msb_addr); in gaudi_sw_init()
1934 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, in gaudi_sw_init()
1935 hdev->cpu_accessible_dma_address); in gaudi_sw_init()
1937 dma_pool_destroy(hdev->dma_pool); in gaudi_sw_init()
1945 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_sw_fini()
1949 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in gaudi_sw_fini()
1951 if (!hdev->asic_prop.fw_security_enabled) in gaudi_sw_fini()
1952 GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address, in gaudi_sw_fini()
1953 hdev->cpu_pci_msb_addr); in gaudi_sw_fini()
1955 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, in gaudi_sw_fini()
1956 hdev->cpu_accessible_dma_address); in gaudi_sw_fini()
1958 dma_pool_destroy(hdev->dma_pool); in gaudi_sw_fini()
1970 if (hdev->disabled) in gaudi_irq_handler_single()
1973 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in gaudi_irq_handler_single()
1974 hl_irq_handler_cq(irq, &hdev->completion_queue[i]); in gaudi_irq_handler_single()
1976 hl_irq_handler_eq(irq, &hdev->event_queue); in gaudi_irq_handler_single()
1991 dev_crit(hdev->dev, "CPU EQ must use IRQ %d\n", in gaudi_pci_irq_vector()
1995 (nr + NIC_NUMBER_OF_ENGINES + 1); in gaudi_pci_irq_vector()
1997 return pci_irq_vector(hdev->pdev, msi_vec); in gaudi_pci_irq_vector()
2004 dev_dbg(hdev->dev, "Working in single MSI IRQ mode\n"); in gaudi_enable_msi_single()
2010 dev_err(hdev->dev, in gaudi_enable_msi_single()
2018 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_enable_msi()
2021 if (gaudi->hw_cap_initialized & HW_CAP_MSI) in gaudi_enable_msi()
2024 rc = pci_alloc_irq_vectors(hdev->pdev, 1, 1, PCI_IRQ_MSI); in gaudi_enable_msi()
2026 dev_err(hdev->dev, "MSI: Failed to enable support %d\n", rc); in gaudi_enable_msi()
2034 gaudi->hw_cap_initialized |= HW_CAP_MSI; in gaudi_enable_msi()
2039 pci_free_irq_vectors(hdev->pdev); in gaudi_enable_msi()
2045 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_sync_irqs()
2047 if (!(gaudi->hw_cap_initialized & HW_CAP_MSI)) in gaudi_sync_irqs()
2056 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_disable_msi()
2058 if (!(gaudi->hw_cap_initialized & HW_CAP_MSI)) in gaudi_disable_msi()
2063 pci_free_irq_vectors(hdev->pdev); in gaudi_disable_msi()
2065 gaudi->hw_cap_initialized &= ~HW_CAP_MSI; in gaudi_disable_msi()
2070 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_init_scrambler_sram()
2072 if (hdev->asic_prop.fw_security_enabled) in gaudi_init_scrambler_sram()
2075 if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 & in gaudi_init_scrambler_sram()
2079 if (gaudi->hw_cap_initialized & HW_CAP_SRAM_SCRAMBLER) in gaudi_init_scrambler_sram()
2083 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2085 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2087 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2089 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2091 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2093 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2095 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2097 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2100 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2102 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2104 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2106 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2108 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2110 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2112 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2114 1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2117 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2119 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2121 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2123 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2125 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2127 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2129 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2131 1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT); in gaudi_init_scrambler_sram()
2133 gaudi->hw_cap_initialized |= HW_CAP_SRAM_SCRAMBLER; in gaudi_init_scrambler_sram()
2138 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_init_scrambler_hbm()
2140 if (hdev->asic_prop.fw_security_enabled) in gaudi_init_scrambler_hbm()
2143 if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 & in gaudi_init_scrambler_hbm()
2147 if (gaudi->hw_cap_initialized & HW_CAP_HBM_SCRAMBLER) in gaudi_init_scrambler_hbm()
2151 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2153 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2155 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2157 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2159 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2161 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2163 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2165 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2168 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2170 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2172 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2174 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2176 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2178 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2180 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2182 1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2185 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2187 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2189 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2191 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2193 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2195 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2197 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2199 1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT); in gaudi_init_scrambler_hbm()
2201 gaudi->hw_cap_initialized |= HW_CAP_HBM_SCRAMBLER; in gaudi_init_scrambler_hbm()
2206 if (hdev->asic_prop.fw_security_enabled) in gaudi_init_e2e()
2209 if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 & in gaudi_init_e2e()
2220 WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_WR_SIZE, 1); in gaudi_init_e2e()
2223 WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_WR_SIZE, 1); in gaudi_init_e2e()
2224 WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_RD_SIZE, 1); in gaudi_init_e2e()
2225 WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_WR_SIZE, 1); in gaudi_init_e2e()
2238 WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_WR_SIZE, 1); in gaudi_init_e2e()
2239 WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_RD_SIZE, 1); in gaudi_init_e2e()
2240 WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_WR_SIZE, 1); in gaudi_init_e2e()
2245 WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_WR_SIZE, 1); in gaudi_init_e2e()
2260 WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_WR_SIZE, 1); in gaudi_init_e2e()
2263 WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_WR_SIZE, 1); in gaudi_init_e2e()
2264 WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_RD_SIZE, 1); in gaudi_init_e2e()
2265 WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_WR_SIZE, 1); in gaudi_init_e2e()
2278 WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_WR_SIZE, 1); in gaudi_init_e2e()
2279 WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_RD_SIZE, 1); in gaudi_init_e2e()
2280 WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_WR_SIZE, 1); in gaudi_init_e2e()
2285 WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_WR_SIZE, 1); in gaudi_init_e2e()
2334 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2336 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2339 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2341 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2344 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2346 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2349 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2351 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2354 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2356 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2359 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2361 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2364 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2366 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2369 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2371 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2374 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2376 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2379 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2381 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2384 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2386 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2389 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2391 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2394 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2396 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2399 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2401 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2404 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2406 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2409 1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2411 1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2414 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2416 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2419 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2421 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2424 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2426 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2429 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2431 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2434 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2436 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2439 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2441 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2444 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2446 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2449 1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT); in gaudi_init_e2e()
2451 1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT); in gaudi_init_e2e()
2458 if (hdev->asic_prop.fw_security_enabled) in gaudi_init_hbm_cred()
2461 if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 & in gaudi_init_hbm_cred()
2491 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | in gaudi_init_hbm_cred()
2492 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); in gaudi_init_hbm_cred()
2494 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | in gaudi_init_hbm_cred()
2495 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); in gaudi_init_hbm_cred()
2497 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | in gaudi_init_hbm_cred()
2498 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); in gaudi_init_hbm_cred()
2500 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | in gaudi_init_hbm_cred()
2501 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); in gaudi_init_hbm_cred()
2504 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | in gaudi_init_hbm_cred()
2505 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); in gaudi_init_hbm_cred()
2507 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | in gaudi_init_hbm_cred()
2508 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); in gaudi_init_hbm_cred()
2510 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | in gaudi_init_hbm_cred()
2511 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); in gaudi_init_hbm_cred()
2513 (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) | in gaudi_init_hbm_cred()
2514 (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT)); in gaudi_init_hbm_cred()
2535 /* Make sure 1st 128 bytes in SRAM are 0 for Tensor DMA */ in gaudi_init_golden_registers()
2537 writeq(0, hdev->pcie_bar[SRAM_BAR_ID] + i); in gaudi_init_golden_registers()
2549 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi_init_pci_dma_qman()
2602 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? in gaudi_init_pci_dma_qman()
2604 le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); in gaudi_init_pci_dma_qman()
2608 if (hdev->stop_on_err) in gaudi_init_pci_dma_qman()
2639 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi_init_dma_core()
2640 u32 dma_err_cfg = 1 << DMA0_CORE_ERR_CFG_ERR_MSG_EN_SHIFT; in gaudi_init_dma_core()
2648 /* WA for H/W bug H3-2116 */ in gaudi_init_dma_core()
2652 if (hdev->stop_on_err) in gaudi_init_dma_core()
2653 dma_err_cfg |= 1 << DMA0_CORE_ERR_CFG_STOP_ON_ERR_SHIFT; in gaudi_init_dma_core()
2657 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? in gaudi_init_dma_core()
2659 le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl); in gaudi_init_dma_core()
2669 1 << DMA0_CORE_PROT_ERR_VAL_SHIFT); in gaudi_init_dma_core()
2672 1 << DMA0_CORE_SECURE_PROPS_MMBP_SHIFT); in gaudi_init_dma_core()
2673 WREG32(mmDMA0_CORE_CFG_0 + dma_offset, 1 << DMA0_CORE_CFG_0_EN_SHIFT); in gaudi_init_dma_core()
2686 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_init_pci_dma_qmans()
2690 if (gaudi->hw_cap_initialized & HW_CAP_PCI_DMA) in gaudi_init_pci_dma_qmans()
2696 * For queues after the CPU Q need to add 1 to get the correct in gaudi_init_pci_dma_qmans()
2700 if (dma_id > 1) { in gaudi_init_pci_dma_qmans()
2701 cpu_skip = 1; in gaudi_init_pci_dma_qmans()
2710 q = &hdev->kernel_queues[q_idx]; in gaudi_init_pci_dma_qmans()
2711 q->cq_id = cq_id++; in gaudi_init_pci_dma_qmans()
2712 q->msi_vec = nic_skip + cpu_skip + msi_vec++; in gaudi_init_pci_dma_qmans()
2714 q->bus_address); in gaudi_init_pci_dma_qmans()
2722 gaudi->hw_cap_initialized |= HW_CAP_PCI_DMA; in gaudi_init_pci_dma_qmans()
2729 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi_init_hbm_dma_qman()
2773 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? in gaudi_init_hbm_dma_qman()
2775 le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); in gaudi_init_hbm_dma_qman()
2786 if (hdev->stop_on_err) in gaudi_init_hbm_dma_qman()
2832 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_init_hbm_dma_qmans()
2837 if (gaudi->hw_cap_initialized & HW_CAP_HBM_DMA) in gaudi_init_hbm_dma_qmans()
2848 internal_q_index = dma_id * QMAN_STREAMS + j + 1; in gaudi_init_hbm_dma_qmans()
2850 q = &gaudi->internal_qmans[internal_q_index]; in gaudi_init_hbm_dma_qmans()
2851 qman_base_addr = (u64) q->pq_dma_addr; in gaudi_init_hbm_dma_qmans()
2864 gaudi->hw_cap_initialized |= HW_CAP_HBM_DMA; in gaudi_init_hbm_dma_qmans()
2871 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi_init_mme_qman()
2906 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? in gaudi_init_mme_qman()
2908 le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl); in gaudi_init_mme_qman()
2919 (mmMME1_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0) / 2; in gaudi_init_mme_qman()
2922 if (hdev->stop_on_err) in gaudi_init_mme_qman()
2956 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_init_mme_qmans()
2962 if (gaudi->hw_cap_initialized & HW_CAP_MME) in gaudi_init_mme_qmans()
2970 mme_offset = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0; in gaudi_init_mme_qmans()
2974 q = &gaudi->internal_qmans[internal_q_index]; in gaudi_init_mme_qmans()
2975 qman_base_addr = (u64) q->pq_dma_addr; in gaudi_init_mme_qmans()
2983 mme_offset = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0; in gaudi_init_mme_qmans()
2990 gaudi->hw_cap_initialized |= HW_CAP_MME; in gaudi_init_mme_qmans()
2997 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi_init_tpc_qman()
3023 (mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0); in gaudi_init_tpc_qman()
3042 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? in gaudi_init_tpc_qman()
3044 le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl); in gaudi_init_tpc_qman()
3055 if (hdev->stop_on_err) in gaudi_init_tpc_qman()
3101 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_init_tpc_qmans()
3105 u32 tpc_delta = mmTPC1_CFG_SM_BASE_ADDRESS_HIGH - in gaudi_init_tpc_qmans()
3109 if (gaudi->hw_cap_initialized & HW_CAP_TPC_MASK) in gaudi_init_tpc_qmans()
3119 q = &gaudi->internal_qmans[internal_q_index]; in gaudi_init_tpc_qmans()
3120 qman_base_addr = (u64) q->pq_dma_addr; in gaudi_init_tpc_qmans()
3137 tpc_offset += mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0; in gaudi_init_tpc_qmans()
3139 gaudi->hw_cap_initialized |= in gaudi_init_tpc_qmans()
3140 FIELD_PREP(HW_CAP_TPC_MASK, 1 << tpc_id); in gaudi_init_tpc_qmans()
3148 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi_init_nic_qman()
3199 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? in gaudi_init_nic_qman()
3201 le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl); in gaudi_init_nic_qman()
3205 if (hdev->stop_on_err) in gaudi_init_nic_qman()
3234 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_init_nic_qmans()
3239 mmNIC0_QM1_GLBL_CFG0 - mmNIC0_QM0_GLBL_CFG0; in gaudi_init_nic_qmans()
3241 mmNIC1_QM0_GLBL_CFG0 - mmNIC0_QM0_GLBL_CFG0; in gaudi_init_nic_qmans()
3244 if (!hdev->nic_ports_mask) in gaudi_init_nic_qmans()
3247 if (gaudi->hw_cap_initialized & HW_CAP_NIC_MASK) in gaudi_init_nic_qmans()
3250 dev_dbg(hdev->dev, "Initializing NIC QMANs\n"); in gaudi_init_nic_qmans()
3253 if (!(hdev->nic_ports_mask & (1 << nic_id))) { in gaudi_init_nic_qmans()
3255 if (nic_id & 1) { in gaudi_init_nic_qmans()
3256 nic_offset -= (nic_delta_between_qmans * 2); in gaudi_init_nic_qmans()
3265 q = &gaudi->internal_qmans[internal_q_index]; in gaudi_init_nic_qmans()
3266 qman_base_addr = (u64) q->pq_dma_addr; in gaudi_init_nic_qmans()
3275 if (nic_id & 1) { in gaudi_init_nic_qmans()
3276 nic_offset -= (nic_delta_between_qmans * 2); in gaudi_init_nic_qmans()
3280 gaudi->hw_cap_initialized |= 1 << (HW_CAP_NIC_SHIFT + nic_id); in gaudi_init_nic_qmans()
3286 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_disable_pci_dma_qmans()
3288 if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA)) in gaudi_disable_pci_dma_qmans()
3298 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_disable_hbm_dma_qmans()
3300 if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA)) in gaudi_disable_hbm_dma_qmans()
3312 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_disable_mme_qmans()
3314 if (!(gaudi->hw_cap_initialized & HW_CAP_MME)) in gaudi_disable_mme_qmans()
3323 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_disable_tpc_qmans()
3327 if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi_disable_tpc_qmans()
3332 tpc_offset += mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0; in gaudi_disable_tpc_qmans()
3338 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_disable_nic_qmans()
3341 mmNIC0_QM1_GLBL_CFG0 - mmNIC0_QM0_GLBL_CFG0; in gaudi_disable_nic_qmans()
3343 mmNIC1_QM0_GLBL_CFG0 - mmNIC0_QM0_GLBL_CFG0; in gaudi_disable_nic_qmans()
3347 nic_mask = 1 << (HW_CAP_NIC_SHIFT + nic_id); in gaudi_disable_nic_qmans()
3349 if (gaudi->hw_cap_initialized & nic_mask) in gaudi_disable_nic_qmans()
3353 if (nic_id & 1) { in gaudi_disable_nic_qmans()
3354 nic_offset -= (nic_delta_between_qmans * 2); in gaudi_disable_nic_qmans()
3362 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_stop_pci_dma_qmans()
3364 if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA)) in gaudi_stop_pci_dma_qmans()
3375 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_stop_hbm_dma_qmans()
3377 if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA)) in gaudi_stop_hbm_dma_qmans()
3391 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_stop_mme_qmans()
3393 if (!(gaudi->hw_cap_initialized & HW_CAP_MME)) in gaudi_stop_mme_qmans()
3403 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_stop_tpc_qmans()
3405 if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi_stop_tpc_qmans()
3420 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_stop_nic_qmans()
3424 if (gaudi->hw_cap_initialized & HW_CAP_NIC0) in gaudi_stop_nic_qmans()
3430 if (gaudi->hw_cap_initialized & HW_CAP_NIC1) in gaudi_stop_nic_qmans()
3436 if (gaudi->hw_cap_initialized & HW_CAP_NIC2) in gaudi_stop_nic_qmans()
3442 if (gaudi->hw_cap_initialized & HW_CAP_NIC3) in gaudi_stop_nic_qmans()
3448 if (gaudi->hw_cap_initialized & HW_CAP_NIC4) in gaudi_stop_nic_qmans()
3454 if (gaudi->hw_cap_initialized & HW_CAP_NIC5) in gaudi_stop_nic_qmans()
3460 if (gaudi->hw_cap_initialized & HW_CAP_NIC6) in gaudi_stop_nic_qmans()
3466 if (gaudi->hw_cap_initialized & HW_CAP_NIC7) in gaudi_stop_nic_qmans()
3472 if (gaudi->hw_cap_initialized & HW_CAP_NIC8) in gaudi_stop_nic_qmans()
3478 if (gaudi->hw_cap_initialized & HW_CAP_NIC9) in gaudi_stop_nic_qmans()
3487 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_pci_dma_stall()
3489 if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA)) in gaudi_pci_dma_stall()
3492 WREG32(mmDMA0_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); in gaudi_pci_dma_stall()
3493 WREG32(mmDMA1_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); in gaudi_pci_dma_stall()
3494 WREG32(mmDMA5_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); in gaudi_pci_dma_stall()
3499 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_hbm_dma_stall()
3501 if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA)) in gaudi_hbm_dma_stall()
3504 WREG32(mmDMA2_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); in gaudi_hbm_dma_stall()
3505 WREG32(mmDMA3_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); in gaudi_hbm_dma_stall()
3506 WREG32(mmDMA4_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); in gaudi_hbm_dma_stall()
3507 WREG32(mmDMA6_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); in gaudi_hbm_dma_stall()
3508 WREG32(mmDMA7_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT); in gaudi_hbm_dma_stall()
3513 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_mme_stall()
3515 if (!(gaudi->hw_cap_initialized & HW_CAP_MME)) in gaudi_mme_stall()
3518 /* WA for H3-1800 bug: do ACC and SBAB writes twice */ in gaudi_mme_stall()
3519 WREG32(mmMME0_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); in gaudi_mme_stall()
3520 WREG32(mmMME0_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); in gaudi_mme_stall()
3521 WREG32(mmMME0_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); in gaudi_mme_stall()
3522 WREG32(mmMME0_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); in gaudi_mme_stall()
3523 WREG32(mmMME1_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); in gaudi_mme_stall()
3524 WREG32(mmMME1_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); in gaudi_mme_stall()
3525 WREG32(mmMME1_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); in gaudi_mme_stall()
3526 WREG32(mmMME1_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); in gaudi_mme_stall()
3527 WREG32(mmMME2_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); in gaudi_mme_stall()
3528 WREG32(mmMME2_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); in gaudi_mme_stall()
3529 WREG32(mmMME2_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); in gaudi_mme_stall()
3530 WREG32(mmMME2_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); in gaudi_mme_stall()
3531 WREG32(mmMME3_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); in gaudi_mme_stall()
3532 WREG32(mmMME3_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT); in gaudi_mme_stall()
3533 WREG32(mmMME3_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); in gaudi_mme_stall()
3534 WREG32(mmMME3_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT); in gaudi_mme_stall()
3539 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_tpc_stall()
3541 if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi_tpc_stall()
3544 WREG32(mmTPC0_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); in gaudi_tpc_stall()
3545 WREG32(mmTPC1_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); in gaudi_tpc_stall()
3546 WREG32(mmTPC2_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); in gaudi_tpc_stall()
3547 WREG32(mmTPC3_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); in gaudi_tpc_stall()
3548 WREG32(mmTPC4_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); in gaudi_tpc_stall()
3549 WREG32(mmTPC5_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); in gaudi_tpc_stall()
3550 WREG32(mmTPC6_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); in gaudi_tpc_stall()
3551 WREG32(mmTPC7_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT); in gaudi_tpc_stall()
3559 if (hdev->asic_prop.fw_security_enabled) in gaudi_disable_clock_gating()
3566 qman_offset += (mmDMA1_QM_CGM_CFG - mmDMA0_QM_CGM_CFG); in gaudi_disable_clock_gating()
3578 qman_offset += (mmTPC1_QM_CGM_CFG - mmTPC0_QM_CGM_CFG); in gaudi_disable_clock_gating()
3585 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0); in gaudi_enable_timestamp()
3587 /* Zero the lower/upper parts of the 64-bit counter */ in gaudi_enable_timestamp()
3588 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0xC, 0); in gaudi_enable_timestamp()
3589 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0x8, 0); in gaudi_enable_timestamp()
3592 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 1); in gaudi_enable_timestamp()
3598 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0); in gaudi_disable_timestamp()
3605 if (hdev->pldm) in gaudi_halt_engines()
3642 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi_mmu_init()
3643 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_mmu_init()
3647 if (gaudi->hw_cap_initialized & HW_CAP_MMU) in gaudi_mmu_init()
3650 for (i = 0 ; i < prop->max_asid ; i++) { in gaudi_mmu_init()
3651 hop0_addr = prop->mmu_pgt_addr + in gaudi_mmu_init()
3652 (i * prop->dmmu.hop_table_size); in gaudi_mmu_init()
3656 dev_err(hdev->dev, in gaudi_mmu_init()
3663 WREG32(mmSTLB_CACHE_INV_BASE_39_8, prop->mmu_cache_mng_addr >> 8); in gaudi_mmu_init()
3664 WREG32(mmSTLB_CACHE_INV_BASE_49_40, prop->mmu_cache_mng_addr >> 40); in gaudi_mmu_init()
3667 WREG32(mmSTLB_MEM_CACHE_INVALIDATION, 1); in gaudi_mmu_init()
3673 WREG32(mmMMU_UP_MMU_ENABLE, 1); in gaudi_mmu_init()
3679 * The H/W expects the first PI after init to be 1. After wraparound in gaudi_mmu_init()
3682 gaudi->mmu_cache_inv_pi = 1; in gaudi_mmu_init()
3684 gaudi->hw_cap_initialized |= HW_CAP_MMU; in gaudi_mmu_init()
3693 dst = hdev->pcie_bar[HBM_BAR_ID] + LINUX_FW_OFFSET; in gaudi_load_firmware_to_device()
3702 dst = hdev->pcie_bar[SRAM_BAR_ID] + BOOT_FIT_SRAM_OFFSET; in gaudi_load_boot_fit_to_device()
3712 dynamic_loader = &hdev->fw_loader.dynamic_loader; in gaudi_init_dynamic_firmware_loader()
3717 * hard-coded) in later stages of the protocol those values will be in gaudi_init_dynamic_firmware_loader()
3719 * will always be up-to-date in gaudi_init_dynamic_firmware_loader()
3721 dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; in gaudi_init_dynamic_firmware_loader()
3722 dyn_regs->kmd_msg_to_cpu = in gaudi_init_dynamic_firmware_loader()
3724 dyn_regs->cpu_cmd_status_to_host = in gaudi_init_dynamic_firmware_loader()
3727 dynamic_loader->wait_for_bl_timeout = GAUDI_WAIT_FOR_BL_TIMEOUT_USEC; in gaudi_init_dynamic_firmware_loader()
3734 static_loader = &hdev->fw_loader.static_loader; in gaudi_init_static_firmware_loader()
3736 static_loader->preboot_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; in gaudi_init_static_firmware_loader()
3737 static_loader->boot_fit_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; in gaudi_init_static_firmware_loader()
3738 static_loader->kmd_msg_to_cpu_reg = mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU; in gaudi_init_static_firmware_loader()
3739 static_loader->cpu_cmd_status_to_host_reg = mmCPU_CMD_STATUS_TO_HOST; in gaudi_init_static_firmware_loader()
3740 static_loader->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; in gaudi_init_static_firmware_loader()
3741 static_loader->cpu_boot_dev_status0_reg = mmCPU_BOOT_DEV_STS0; in gaudi_init_static_firmware_loader()
3742 static_loader->cpu_boot_dev_status1_reg = mmCPU_BOOT_DEV_STS1; in gaudi_init_static_firmware_loader()
3743 static_loader->boot_err0_reg = mmCPU_BOOT_ERR0; in gaudi_init_static_firmware_loader()
3744 static_loader->boot_err1_reg = mmCPU_BOOT_ERR1; in gaudi_init_static_firmware_loader()
3745 static_loader->preboot_version_offset_reg = mmPREBOOT_VER_OFFSET; in gaudi_init_static_firmware_loader()
3746 static_loader->boot_fit_version_offset_reg = mmUBOOT_VER_OFFSET; in gaudi_init_static_firmware_loader()
3747 static_loader->sram_offset_mask = ~(lower_32_bits(SRAM_BASE_ADDR)); in gaudi_init_static_firmware_loader()
3748 static_loader->cpu_reset_wait_msec = hdev->pldm ? in gaudi_init_static_firmware_loader()
3755 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load; in gaudi_init_firmware_preload_params()
3757 pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; in gaudi_init_firmware_preload_params()
3758 pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0; in gaudi_init_firmware_preload_params()
3759 pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1; in gaudi_init_firmware_preload_params()
3760 pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0; in gaudi_init_firmware_preload_params()
3761 pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1; in gaudi_init_firmware_preload_params()
3762 pre_fw_load->wait_for_preboot_timeout = GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC; in gaudi_init_firmware_preload_params()
3767 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi_init_firmware_loader()
3768 struct fw_load_mgr *fw_loader = &hdev->fw_loader; in gaudi_init_firmware_loader()
3771 fw_loader->fw_comp_loaded = FW_TYPE_NONE; in gaudi_init_firmware_loader()
3772 fw_loader->boot_fit_img.image_name = GAUDI_BOOT_FIT_FILE; in gaudi_init_firmware_loader()
3773 fw_loader->linux_img.image_name = GAUDI_LINUX_FW_FILE; in gaudi_init_firmware_loader()
3774 fw_loader->cpu_timeout = GAUDI_CPU_TIMEOUT_USEC; in gaudi_init_firmware_loader()
3775 fw_loader->boot_fit_timeout = GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC; in gaudi_init_firmware_loader()
3776 fw_loader->skip_bmc = !hdev->bmc_enable; in gaudi_init_firmware_loader()
3777 fw_loader->sram_bar_id = SRAM_BAR_ID; in gaudi_init_firmware_loader()
3778 fw_loader->dram_bar_id = HBM_BAR_ID; in gaudi_init_firmware_loader()
3780 if (prop->dynamic_fw_load) in gaudi_init_firmware_loader()
3788 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_init_cpu()
3791 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) in gaudi_init_cpu()
3794 if (gaudi->hw_cap_initialized & HW_CAP_CPU) in gaudi_init_cpu()
3801 if (!hdev->asic_prop.fw_security_enabled) in gaudi_init_cpu()
3802 WREG32(mmCPU_IF_CPU_MSB_ADDR, hdev->cpu_pci_msb_addr); in gaudi_init_cpu()
3809 gaudi->hw_cap_initialized |= HW_CAP_CPU; in gaudi_init_cpu()
3817 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi_init_cpu_queues()
3818 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi_init_cpu_queues()
3819 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_init_cpu_queues()
3823 &hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ]; in gaudi_init_cpu_queues()
3826 if (!hdev->cpu_queues_enable) in gaudi_init_cpu_queues()
3829 if (gaudi->hw_cap_initialized & HW_CAP_CPU_Q) in gaudi_init_cpu_queues()
3832 eq = &hdev->event_queue; in gaudi_init_cpu_queues()
3834 WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); in gaudi_init_cpu_queues()
3835 WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); in gaudi_init_cpu_queues()
3837 WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); in gaudi_init_cpu_queues()
3838 WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); in gaudi_init_cpu_queues()
3841 lower_32_bits(hdev->cpu_accessible_dma_address)); in gaudi_init_cpu_queues()
3843 upper_32_bits(hdev->cpu_accessible_dma_address)); in gaudi_init_cpu_queues()
3856 irq_handler_offset = prop->gic_interrupts_enable ? in gaudi_init_cpu_queues()
3858 le32_to_cpu(dyn_regs->gic_host_pi_upd_irq); in gaudi_init_cpu_queues()
3872 dev_err(hdev->dev, in gaudi_init_cpu_queues()
3873 "Failed to communicate with Device CPU (CPU-CP timeout)\n"); in gaudi_init_cpu_queues()
3874 return -EIO; in gaudi_init_cpu_queues()
3878 if (prop->fw_cpu_boot_dev_sts0_valid) in gaudi_init_cpu_queues()
3879 prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); in gaudi_init_cpu_queues()
3880 if (prop->fw_cpu_boot_dev_sts1_valid) in gaudi_init_cpu_queues()
3881 prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); in gaudi_init_cpu_queues()
3883 gaudi->hw_cap_initialized |= HW_CAP_CPU_Q; in gaudi_init_cpu_queues()
3892 if (!hdev->asic_prop.fw_security_enabled) { in gaudi_pre_hw_init()
3917 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_hw_init()
3926 if (hdev->asic_prop.iatu_done_by_fw) in gaudi_hw_init()
3927 gaudi->hbm_bar_cur_addr = DRAM_PHYS_BASE; in gaudi_hw_init()
3930 * Before pushing u-boot/linux to device, need to set the hbm bar to in gaudi_hw_init()
3934 dev_err(hdev->dev, in gaudi_hw_init()
3936 return -EIO; in gaudi_hw_init()
3941 dev_err(hdev->dev, "failed to initialize CPU\n"); in gaudi_hw_init()
3984 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", in gaudi_hw_init()
4006 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi_hw_fini()
4008 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_hw_fini()
4012 dev_err(hdev->dev, "GAUDI doesn't support soft-reset\n"); in gaudi_hw_fini()
4016 if (hdev->pldm) { in gaudi_hw_fini()
4025 dev_dbg(hdev->dev, in gaudi_hw_fini()
4032 driver_performs_reset = !!(!hdev->asic_prop.fw_security_enabled && in gaudi_hw_fini()
4033 !hdev->asic_prop.hard_reset_done_by_fw); in gaudi_hw_fini()
4046 if (hdev->fw_loader.fw_comp_loaded & FW_TYPE_LINUX) { in gaudi_hw_fini()
4047 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? in gaudi_hw_fini()
4049 le32_to_cpu(dyn_regs->gic_host_halt_irq); in gaudi_hw_fini()
4054 /* This is a hail-mary attempt to revive the card in the small chance that the in gaudi_hw_fini()
4065 if (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT) { in gaudi_hw_fini()
4066 if (hdev->asic_prop.hard_reset_done_by_fw) in gaudi_hw_fini()
4072 if (hdev->asic_prop.hard_reset_done_by_fw) in gaudi_hw_fini()
4108 /* Tell ASIC not to re-initialize PCIe */ in gaudi_hw_fini()
4111 /* Restart BTL/BLR upon hard-reset */ in gaudi_hw_fini()
4112 WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START, 1); in gaudi_hw_fini()
4115 1 << PSOC_GLOBAL_CONF_SW_ALL_RST_IND_SHIFT); in gaudi_hw_fini()
4117 dev_dbg(hdev->dev, in gaudi_hw_fini()
4121 dev_dbg(hdev->dev, in gaudi_hw_fini()
4135 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", status); in gaudi_hw_fini()
4136 return -ETIMEDOUT; in gaudi_hw_fini()
4140 gaudi->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q | HW_CAP_HBM | in gaudi_hw_fini()
4146 memset(gaudi->events_stat, 0, sizeof(gaudi->events_stat)); in gaudi_hw_fini()
4148 hdev->device_cpu_is_halted = false; in gaudi_hw_fini()
4171 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, in gaudi_mmap()
4172 (dma_addr - HOST_PHYS_BASE), size); in gaudi_mmap()
4174 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); in gaudi_mmap()
4182 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi_ring_doorbell()
4184 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_ring_doorbell()
4206 q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4213 q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4220 q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4227 q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4234 q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4241 q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4246 if (gaudi->hw_cap_initialized & HW_CAP_CPU_Q) in gaudi_ring_doorbell()
4413 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC0)) in gaudi_ring_doorbell()
4416 q_off = ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4421 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC1)) in gaudi_ring_doorbell()
4424 q_off = ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4429 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC2)) in gaudi_ring_doorbell()
4432 q_off = ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4437 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC3)) in gaudi_ring_doorbell()
4440 q_off = ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4445 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC4)) in gaudi_ring_doorbell()
4448 q_off = ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4453 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC5)) in gaudi_ring_doorbell()
4456 q_off = ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4461 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC6)) in gaudi_ring_doorbell()
4464 q_off = ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4469 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC7)) in gaudi_ring_doorbell()
4472 q_off = ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4477 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC8)) in gaudi_ring_doorbell()
4480 q_off = ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4485 if (!(gaudi->hw_cap_initialized & HW_CAP_NIC9)) in gaudi_ring_doorbell()
4488 q_off = ((hw_queue_id - 1) & 0x3) * 4; in gaudi_ring_doorbell()
4498 dev_err(hdev->dev, "h/w queue %d is invalid. Can't set pi\n", in gaudi_ring_doorbell()
4512 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? in gaudi_ring_doorbell()
4514 le32_to_cpu(dyn_regs->gic_host_pi_upd_irq); in gaudi_ring_doorbell()
4528 pqe[1] = pbd[1]; in gaudi_pqe_write()
4534 void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size, in gaudi_dma_alloc_coherent()
4548 dma_addr_t fixed_dma_handle = dma_handle - HOST_PHYS_BASE; in gaudi_dma_free_coherent()
4550 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle); in gaudi_dma_free_coherent()
4555 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi_scrub_device_dram()
4556 u64 cur_addr = prop->dram_user_base_address; in gaudi_scrub_device_dram()
4560 while (cur_addr < prop->dram_end_address) { in gaudi_scrub_device_dram()
4565 min((u64)SZ_2G, prop->dram_end_address - cur_addr); in gaudi_scrub_device_dram()
4567 dev_dbg(hdev->dev, in gaudi_scrub_device_dram()
4568 "Doing HBM scrubbing for 0x%09llx - 0x%09llx\n", in gaudi_scrub_device_dram()
4582 ((1 << DMA0_CORE_COMMIT_LIN_SHIFT) | in gaudi_scrub_device_dram()
4583 (1 << DMA0_CORE_COMMIT_MEM_SET_SHIFT))); in gaudi_scrub_device_dram()
4587 if (cur_addr == prop->dram_end_address) in gaudi_scrub_device_dram()
4603 dev_err(hdev->dev, in gaudi_scrub_device_dram()
4606 return -EIO; in gaudi_scrub_device_dram()
4616 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi_scrub_device_mem()
4618 u64 addr, size, val = hdev->memory_scrub_val; in gaudi_scrub_device_mem()
4622 if (!hdev->memory_scrub) in gaudi_scrub_device_mem()
4626 while (!hdev->asic_funcs->is_device_idle(hdev, NULL, 0, NULL)) { in gaudi_scrub_device_mem()
4628 dev_err(hdev->dev, "waiting for idle timeout\n"); in gaudi_scrub_device_mem()
4629 return -ETIMEDOUT; in gaudi_scrub_device_mem()
4631 usleep_range((1000 >> 2) + 1, 1000); in gaudi_scrub_device_mem()
4635 addr = prop->sram_user_base_address; in gaudi_scrub_device_mem()
4636 size = hdev->pldm ? 0x10000 : prop->sram_size - SRAM_USER_BASE_OFFSET; in gaudi_scrub_device_mem()
4638 dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx val: 0x%llx\n", in gaudi_scrub_device_mem()
4642 dev_err(hdev->dev, "Failed to clear SRAM (%d)\n", rc); in gaudi_scrub_device_mem()
4649 dev_err(hdev->dev, "Failed to clear HBM (%d)\n", rc); in gaudi_scrub_device_mem()
4660 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_get_int_queue_base()
4665 dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id); in gaudi_get_int_queue_base()
4669 q = &gaudi->internal_qmans[queue_id]; in gaudi_get_int_queue_base()
4670 *dma_handle = q->pq_dma_addr; in gaudi_get_int_queue_base()
4671 *queue_len = q->pq_size / QMAN_PQ_ENTRY_SIZE; in gaudi_get_int_queue_base()
4673 return q->pq_kernel_addr; in gaudi_get_int_queue_base()
4679 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_send_cpu_message()
4681 if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) { in gaudi_send_cpu_message()
4703 if (hdev->pldm) in gaudi_test_queue()
4712 dev_err(hdev->dev, in gaudi_test_queue()
4715 return -ENOMEM; in gaudi_test_queue()
4723 dev_err(hdev->dev, in gaudi_test_queue()
4726 rc = -ENOMEM; in gaudi_test_queue()
4731 tmp |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1); in gaudi_test_queue()
4732 tmp |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); in gaudi_test_queue()
4734 fence_pkt->ctl = cpu_to_le32(tmp); in gaudi_test_queue()
4735 fence_pkt->value = cpu_to_le32(fence_val); in gaudi_test_queue()
4736 fence_pkt->addr = cpu_to_le64(fence_dma_addr); in gaudi_test_queue()
4742 dev_err(hdev->dev, in gaudi_test_queue()
4753 if (rc == -ETIMEDOUT) { in gaudi_test_queue()
4754 dev_err(hdev->dev, in gaudi_test_queue()
4757 rc = -EIO; in gaudi_test_queue()
4769 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_test_cpu_queue()
4775 if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi_test_cpu_queue()
4785 for (i = 0 ; i < hdev->asic_prop.max_queues ; i++) { in gaudi_test_queues()
4786 if (hdev->asic_prop.hw_queues_props[i].type == QUEUE_TYPE_EXT) { in gaudi_test_queues()
4789 ret_val = -EINVAL; in gaudi_test_queues()
4795 ret_val = -EINVAL; in gaudi_test_queues()
4808 kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); in gaudi_dma_pool_zalloc()
4821 dma_addr_t fixed_dma_addr = dma_addr - HOST_PHYS_BASE; in gaudi_dma_pool_free()
4823 dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr); in gaudi_dma_pool_free()
4854 while ((count + 1) < sgt->nents) { in gaudi_get_dma_desc_list_size()
4886 if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), in gaudi_pin_memory_before_cs()
4887 parser->job_userptr_list, &userptr)) in gaudi_pin_memory_before_cs()
4892 return -ENOMEM; in gaudi_pin_memory_before_cs()
4894 rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), in gaudi_pin_memory_before_cs()
4899 list_add_tail(&userptr->job_node, parser->job_userptr_list); in gaudi_pin_memory_before_cs()
4901 rc = hl_dma_map_sgtable(hdev, userptr->sgt, dir); in gaudi_pin_memory_before_cs()
4903 dev_err(hdev->dev, "failed to map sgt with DMA region\n"); in gaudi_pin_memory_before_cs()
4907 userptr->dma_mapped = true; in gaudi_pin_memory_before_cs()
4908 userptr->dir = dir; in gaudi_pin_memory_before_cs()
4911 parser->patched_cb_size += in gaudi_pin_memory_before_cs()
4912 gaudi_get_dma_desc_list_size(hdev, userptr->sgt); in gaudi_pin_memory_before_cs()
4917 list_del(&userptr->job_node); in gaudi_pin_memory_before_cs()
4934 user_memset = (le32_to_cpu(user_dma_pkt->ctl) & in gaudi_validate_dma_pkt_host()
4942 dev_dbg(hdev->dev, "DMA direction is HOST --> DEVICE\n"); in gaudi_validate_dma_pkt_host()
4944 addr = le64_to_cpu(user_dma_pkt->src_addr); in gaudi_validate_dma_pkt_host()
4946 dev_dbg(hdev->dev, "DMA direction is DEVICE --> HOST\n"); in gaudi_validate_dma_pkt_host()
4948 addr = (le64_to_cpu(user_dma_pkt->dst_addr) & in gaudi_validate_dma_pkt_host()
4954 parser->patched_cb_size += sizeof(*user_dma_pkt); in gaudi_validate_dma_pkt_host()
4967 u64 dst_addr = (le64_to_cpu(user_dma_pkt->dst_addr) & in gaudi_validate_dma_pkt_no_mmu()
4971 dev_dbg(hdev->dev, "DMA packet details:\n"); in gaudi_validate_dma_pkt_no_mmu()
4972 dev_dbg(hdev->dev, "source == 0x%llx\n", in gaudi_validate_dma_pkt_no_mmu()
4973 le64_to_cpu(user_dma_pkt->src_addr)); in gaudi_validate_dma_pkt_no_mmu()
4974 dev_dbg(hdev->dev, "destination == 0x%llx\n", dst_addr); in gaudi_validate_dma_pkt_no_mmu()
4975 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); in gaudi_validate_dma_pkt_no_mmu()
4982 if (!le32_to_cpu(user_dma_pkt->tsize)) { in gaudi_validate_dma_pkt_no_mmu()
4983 parser->patched_cb_size += sizeof(*user_dma_pkt); in gaudi_validate_dma_pkt_no_mmu()
4987 if (parser->hw_queue_id <= GAUDI_QUEUE_ID_DMA_0_3) in gaudi_validate_dma_pkt_no_mmu()
5000 cfg = le32_to_cpu(user_pkt->cfg); in gaudi_validate_load_and_exe_pkt()
5003 dev_err(hdev->dev, in gaudi_validate_load_and_exe_pkt()
5005 return -EPERM; in gaudi_validate_load_and_exe_pkt()
5008 parser->patched_cb_size += sizeof(struct packet_load_and_exe); in gaudi_validate_load_and_exe_pkt()
5019 parser->patched_cb_size = 0; in gaudi_validate_cb()
5022 while (cb_parsed_length < parser->user_cb_size) { in gaudi_validate_cb()
5027 user_pkt = parser->user_cb->kernel_address + cb_parsed_length; in gaudi_validate_cb()
5030 (le64_to_cpu(user_pkt->header) & in gaudi_validate_cb()
5035 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); in gaudi_validate_cb()
5036 rc = -EINVAL; in gaudi_validate_cb()
5042 if (cb_parsed_length > parser->user_cb_size) { in gaudi_validate_cb()
5043 dev_err(hdev->dev, in gaudi_validate_cb()
5045 rc = -EINVAL; in gaudi_validate_cb()
5051 dev_err(hdev->dev, in gaudi_validate_cb()
5053 rc = -EPERM; in gaudi_validate_cb()
5057 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); in gaudi_validate_cb()
5058 rc = -EPERM; in gaudi_validate_cb()
5062 dev_err(hdev->dev, "User not allowed to use STOP\n"); in gaudi_validate_cb()
5063 rc = -EPERM; in gaudi_validate_cb()
5067 dev_err(hdev->dev, in gaudi_validate_cb()
5069 rc = -EPERM; in gaudi_validate_cb()
5078 parser->contains_dma_pkt = true; in gaudi_validate_cb()
5080 parser->patched_cb_size += pkt_size; in gaudi_validate_cb()
5093 parser->patched_cb_size += pkt_size; in gaudi_validate_cb()
5097 dev_err(hdev->dev, "Invalid packet header 0x%x\n", in gaudi_validate_cb()
5099 rc = -EINVAL; in gaudi_validate_cb()
5109 * 1. Optional NOP padding for cacheline alignment in gaudi_validate_cb()
5113 if (parser->completion) in gaudi_validate_cb()
5114 parser->patched_cb_size += gaudi_get_patched_cb_extra_size( in gaudi_validate_cb()
5115 parser->patched_cb_size); in gaudi_validate_cb()
5138 ctl = le32_to_cpu(user_dma_pkt->ctl); in gaudi_patch_dma_packet()
5140 if (parser->hw_queue_id <= GAUDI_QUEUE_ID_DMA_0_3) in gaudi_patch_dma_packet()
5147 addr = le64_to_cpu(user_dma_pkt->src_addr); in gaudi_patch_dma_packet()
5148 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in gaudi_patch_dma_packet()
5153 addr = le64_to_cpu(user_dma_pkt->dst_addr); in gaudi_patch_dma_packet()
5154 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in gaudi_patch_dma_packet()
5160 le32_to_cpu(user_dma_pkt->tsize), in gaudi_patch_dma_packet()
5161 parser->job_userptr_list, &userptr))) { in gaudi_patch_dma_packet()
5162 dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n", in gaudi_patch_dma_packet()
5163 addr, user_dma_pkt->tsize); in gaudi_patch_dma_packet()
5164 return -EFAULT; in gaudi_patch_dma_packet()
5175 sgt = userptr->sgt; in gaudi_patch_dma_packet()
5185 while ((count + 1) < sgt->nents) { in gaudi_patch_dma_packet()
5203 ctl = le32_to_cpu(user_dma_pkt->ctl); in gaudi_patch_dma_packet()
5207 new_dma_pkt->ctl = cpu_to_le32(ctl); in gaudi_patch_dma_packet()
5208 new_dma_pkt->tsize = cpu_to_le32(len); in gaudi_patch_dma_packet()
5211 new_dma_pkt->src_addr = cpu_to_le64(dma_addr); in gaudi_patch_dma_packet()
5212 new_dma_pkt->dst_addr = cpu_to_le64(device_memory_addr); in gaudi_patch_dma_packet()
5214 new_dma_pkt->src_addr = cpu_to_le64(device_memory_addr); in gaudi_patch_dma_packet()
5215 new_dma_pkt->dst_addr = cpu_to_le64(dma_addr); in gaudi_patch_dma_packet()
5225 dev_err(hdev->dev, in gaudi_patch_dma_packet()
5227 return -EFAULT; in gaudi_patch_dma_packet()
5230 /* Fix the last dma packet - wrcomp must be as user set it */ in gaudi_patch_dma_packet()
5231 new_dma_pkt--; in gaudi_patch_dma_packet()
5232 new_dma_pkt->ctl |= cpu_to_le32(user_wrcomp_en_mask); in gaudi_patch_dma_packet()
5247 while (cb_parsed_length < parser->user_cb_size) { in gaudi_patch_cb()
5253 user_pkt = parser->user_cb->kernel_address + cb_parsed_length; in gaudi_patch_cb()
5254 kernel_pkt = parser->patched_cb->kernel_address + in gaudi_patch_cb()
5258 (le64_to_cpu(user_pkt->header) & in gaudi_patch_cb()
5263 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); in gaudi_patch_cb()
5264 rc = -EINVAL; in gaudi_patch_cb()
5270 if (cb_parsed_length > parser->user_cb_size) { in gaudi_patch_cb()
5271 dev_err(hdev->dev, in gaudi_patch_cb()
5273 rc = -EINVAL; in gaudi_patch_cb()
5287 dev_err(hdev->dev, in gaudi_patch_cb()
5289 rc = -EPERM; in gaudi_patch_cb()
5293 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); in gaudi_patch_cb()
5294 rc = -EPERM; in gaudi_patch_cb()
5298 dev_err(hdev->dev, "User not allowed to use STOP\n"); in gaudi_patch_cb()
5299 rc = -EPERM; in gaudi_patch_cb()
5316 dev_err(hdev->dev, "Invalid packet header 0x%x\n", in gaudi_patch_cb()
5318 rc = -EINVAL; in gaudi_patch_cb()
5339 * 1. Optional NOP padding for cacheline alignment in gaudi_parse_cb_mmu()
5343 if (parser->completion) in gaudi_parse_cb_mmu()
5344 parser->patched_cb_size = parser->user_cb_size + in gaudi_parse_cb_mmu()
5345 gaudi_get_patched_cb_extra_size(parser->user_cb_size); in gaudi_parse_cb_mmu()
5347 parser->patched_cb_size = parser->user_cb_size; in gaudi_parse_cb_mmu()
5349 rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, in gaudi_parse_cb_mmu()
5350 parser->patched_cb_size, false, false, in gaudi_parse_cb_mmu()
5354 dev_err(hdev->dev, in gaudi_parse_cb_mmu()
5360 parser->patched_cb = hl_cb_get(&hdev->kernel_mem_mgr, handle); in gaudi_parse_cb_mmu()
5362 if (!parser->patched_cb) { in gaudi_parse_cb_mmu()
5363 dev_crit(hdev->dev, "DMA CB handle invalid 0x%llx\n", handle); in gaudi_parse_cb_mmu()
5364 rc = -EFAULT; in gaudi_parse_cb_mmu()
5370 * "parser->user_cb_size <= parser->user_cb->size" was done in get_cb_from_cs_chunk() in gaudi_parse_cb_mmu()
5374 * 1. validate_queue_index() assigns true to is_kernel_allocated_cb for any submission to in gaudi_parse_cb_mmu()
5378 memcpy(parser->patched_cb->kernel_address, in gaudi_parse_cb_mmu()
5379 parser->user_cb->kernel_address, in gaudi_parse_cb_mmu()
5380 parser->user_cb_size); in gaudi_parse_cb_mmu()
5382 patched_cb_size = parser->patched_cb_size; in gaudi_parse_cb_mmu()
5385 user_cb = parser->user_cb; in gaudi_parse_cb_mmu()
5386 parser->user_cb = parser->patched_cb; in gaudi_parse_cb_mmu()
5388 parser->user_cb = user_cb; in gaudi_parse_cb_mmu()
5391 hl_cb_put(parser->patched_cb); in gaudi_parse_cb_mmu()
5395 if (patched_cb_size != parser->patched_cb_size) { in gaudi_parse_cb_mmu()
5396 dev_err(hdev->dev, "user CB size mismatch\n"); in gaudi_parse_cb_mmu()
5397 hl_cb_put(parser->patched_cb); in gaudi_parse_cb_mmu()
5398 rc = -EINVAL; in gaudi_parse_cb_mmu()
5404 * Always call cb destroy here because we still have 1 reference in gaudi_parse_cb_mmu()
5409 hl_cb_destroy(&hdev->kernel_mem_mgr, handle); in gaudi_parse_cb_mmu()
5425 rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, in gaudi_parse_cb_no_mmu()
5426 parser->patched_cb_size, false, false, in gaudi_parse_cb_no_mmu()
5429 dev_err(hdev->dev, in gaudi_parse_cb_no_mmu()
5434 parser->patched_cb = hl_cb_get(&hdev->kernel_mem_mgr, handle); in gaudi_parse_cb_no_mmu()
5436 if (!parser->patched_cb) { in gaudi_parse_cb_no_mmu()
5437 dev_crit(hdev->dev, "DMA CB handle invalid 0x%llx\n", handle); in gaudi_parse_cb_no_mmu()
5438 rc = -EFAULT; in gaudi_parse_cb_no_mmu()
5445 hl_cb_put(parser->patched_cb); in gaudi_parse_cb_no_mmu()
5449 * Always call cb destroy here because we still have 1 reference in gaudi_parse_cb_no_mmu()
5454 hl_cb_destroy(&hdev->kernel_mem_mgr, handle); in gaudi_parse_cb_no_mmu()
5458 hl_userptr_delete_list(hdev, parser->job_userptr_list); in gaudi_parse_cb_no_mmu()
5465 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; in gaudi_parse_cb_no_ext_queue()
5466 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_parse_cb_no_ext_queue()
5469 if ((parser->hw_queue_id >= GAUDI_QUEUE_ID_NIC_0_0) && in gaudi_parse_cb_no_ext_queue()
5470 (parser->hw_queue_id <= GAUDI_QUEUE_ID_NIC_9_3)) { in gaudi_parse_cb_no_ext_queue()
5471 nic_queue_offset = parser->hw_queue_id - GAUDI_QUEUE_ID_NIC_0_0; in gaudi_parse_cb_no_ext_queue()
5472 nic_mask_q_id = 1 << (HW_CAP_NIC_SHIFT + (nic_queue_offset >> 2)); in gaudi_parse_cb_no_ext_queue()
5474 if (!(gaudi->hw_cap_initialized & nic_mask_q_id)) { in gaudi_parse_cb_no_ext_queue()
5475 dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id); in gaudi_parse_cb_no_ext_queue()
5476 return -EINVAL; in gaudi_parse_cb_no_ext_queue()
5481 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi_parse_cb_no_ext_queue()
5482 parser->user_cb_size, in gaudi_parse_cb_no_ext_queue()
5483 asic_prop->sram_user_base_address, in gaudi_parse_cb_no_ext_queue()
5484 asic_prop->sram_end_address)) in gaudi_parse_cb_no_ext_queue()
5487 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi_parse_cb_no_ext_queue()
5488 parser->user_cb_size, in gaudi_parse_cb_no_ext_queue()
5489 asic_prop->dram_user_base_address, in gaudi_parse_cb_no_ext_queue()
5490 asic_prop->dram_end_address)) in gaudi_parse_cb_no_ext_queue()
5494 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi_parse_cb_no_ext_queue()
5495 parser->user_cb_size, in gaudi_parse_cb_no_ext_queue()
5496 asic_prop->pmmu.start_addr, in gaudi_parse_cb_no_ext_queue()
5497 asic_prop->pmmu.end_addr)) in gaudi_parse_cb_no_ext_queue()
5500 dev_err(hdev->dev, in gaudi_parse_cb_no_ext_queue()
5502 parser->user_cb, parser->user_cb_size); in gaudi_parse_cb_no_ext_queue()
5504 return -EFAULT; in gaudi_parse_cb_no_ext_queue()
5509 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_cs_parser()
5511 if (parser->queue_type == QUEUE_TYPE_INT) in gaudi_cs_parser()
5514 if (gaudi->hw_cap_initialized & HW_CAP_MMU) in gaudi_cs_parser()
5530 cq_pkt = kernel_address + len - (sizeof(struct packet_msg_prot) * 2); in gaudi_add_end_of_cb_packets()
5533 cq_padding->ctl = cpu_to_le32(FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_NOP)); in gaudi_add_end_of_cb_packets()
5538 tmp |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); in gaudi_add_end_of_cb_packets()
5541 tmp |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1); in gaudi_add_end_of_cb_packets()
5543 cq_pkt->ctl = cpu_to_le32(tmp); in gaudi_add_end_of_cb_packets()
5544 cq_pkt->value = cpu_to_le32(cq_val); in gaudi_add_end_of_cb_packets()
5545 cq_pkt->addr = cpu_to_le64(cq_addr); in gaudi_add_end_of_cb_packets()
5550 tmp |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); in gaudi_add_end_of_cb_packets()
5551 cq_pkt->ctl = cpu_to_le32(tmp); in gaudi_add_end_of_cb_packets()
5552 cq_pkt->value = cpu_to_le32(1); in gaudi_add_end_of_cb_packets()
5553 msi_addr = hdev->pdev ? mmPCIE_CORE_MSI_REQ : mmPCIE_MSI_INTR_0 + msi_vec * 4; in gaudi_add_end_of_cb_packets()
5554 cq_pkt->addr = cpu_to_le64(CFG_BASE + msi_addr); in gaudi_add_end_of_cb_packets()
5573 return -EFAULT; in gaudi_memset_device_memory()
5575 lin_dma_pkt = cb->kernel_address; in gaudi_memset_device_memory()
5580 ctl |= FIELD_PREP(GAUDI_PKT_LIN_DMA_CTL_MEMSET_MASK, 1); in gaudi_memset_device_memory()
5581 ctl |= FIELD_PREP(GAUDI_PKT_LIN_DMA_CTL_LIN_MASK, 1); in gaudi_memset_device_memory()
5582 ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); in gaudi_memset_device_memory()
5583 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); in gaudi_memset_device_memory()
5585 lin_dma_pkt->ctl = cpu_to_le32(ctl); in gaudi_memset_device_memory()
5586 lin_dma_pkt->src_addr = cpu_to_le64(val); in gaudi_memset_device_memory()
5587 lin_dma_pkt->dst_addr |= cpu_to_le64(addr); in gaudi_memset_device_memory()
5588 lin_dma_pkt->tsize = cpu_to_le32(size); in gaudi_memset_device_memory()
5592 dev_err(hdev->dev, "Failed to allocate a new job\n"); in gaudi_memset_device_memory()
5593 rc = -ENOMEM; in gaudi_memset_device_memory()
5599 if (err_cause && !hdev->init_done) { in gaudi_memset_device_memory()
5600 dev_dbg(hdev->dev, in gaudi_memset_device_memory()
5601 "Clearing DMA0 engine from errors (cause 0x%x)\n", in gaudi_memset_device_memory()
5606 job->id = 0; in gaudi_memset_device_memory()
5607 job->user_cb = cb; in gaudi_memset_device_memory()
5608 atomic_inc(&job->user_cb->cs_cnt); in gaudi_memset_device_memory()
5609 job->user_cb_size = cb_size; in gaudi_memset_device_memory()
5610 job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0; in gaudi_memset_device_memory()
5611 job->patched_cb = job->user_cb; in gaudi_memset_device_memory()
5612 job->job_cb_size = job->user_cb_size + sizeof(struct packet_msg_prot); in gaudi_memset_device_memory()
5619 atomic_dec(&cb->cs_cnt); in gaudi_memset_device_memory()
5624 dev_err(hdev->dev, "DMA Failed, cause 0x%x\n", err_cause); in gaudi_memset_device_memory()
5625 rc = -EIO; in gaudi_memset_device_memory()
5626 if (!hdev->init_done) { in gaudi_memset_device_memory()
5627 dev_dbg(hdev->dev, in gaudi_memset_device_memory()
5628 "Clearing DMA0 engine from errors (cause 0x%x)\n", in gaudi_memset_device_memory()
5636 hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle); in gaudi_memset_device_memory()
5653 dev_err(hdev->dev, "CB size must be smaller than %uMB", SZ_2M); in gaudi_memset_registers()
5654 return -ENOMEM; in gaudi_memset_registers()
5659 return -EFAULT; in gaudi_memset_registers()
5661 pkt = cb->kernel_address; in gaudi_memset_registers()
5665 ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1); in gaudi_memset_registers()
5666 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); in gaudi_memset_registers()
5667 ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); in gaudi_memset_registers()
5670 pkt->ctl = cpu_to_le32(ctl); in gaudi_memset_registers()
5671 pkt->value = cpu_to_le32(val); in gaudi_memset_registers()
5672 pkt->addr = cpu_to_le64(reg_base + (i * 4)); in gaudi_memset_registers()
5677 dev_err(hdev->dev, "Failed to allocate a new job\n"); in gaudi_memset_registers()
5678 rc = -ENOMEM; in gaudi_memset_registers()
5682 job->id = 0; in gaudi_memset_registers()
5683 job->user_cb = cb; in gaudi_memset_registers()
5684 atomic_inc(&job->user_cb->cs_cnt); in gaudi_memset_registers()
5685 job->user_cb_size = cb_size; in gaudi_memset_registers()
5686 job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0; in gaudi_memset_registers()
5687 job->patched_cb = job->user_cb; in gaudi_memset_registers()
5688 job->job_cb_size = cb_size; in gaudi_memset_registers()
5695 atomic_dec(&cb->cs_cnt); in gaudi_memset_registers()
5699 hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle); in gaudi_memset_registers()
5714 dev_err(hdev->dev, "failed resetting SM registers"); in gaudi_restore_sm_registers()
5715 return -ENOMEM; in gaudi_restore_sm_registers()
5722 dev_err(hdev->dev, "failed resetting SM registers"); in gaudi_restore_sm_registers()
5723 return -ENOMEM; in gaudi_restore_sm_registers()
5730 dev_err(hdev->dev, "failed resetting SM registers"); in gaudi_restore_sm_registers()
5731 return -ENOMEM; in gaudi_restore_sm_registers()
5738 dev_err(hdev->dev, "failed resetting SM registers"); in gaudi_restore_sm_registers()
5739 return -ENOMEM; in gaudi_restore_sm_registers()
5746 dev_err(hdev->dev, "failed resetting SM registers"); in gaudi_restore_sm_registers()
5747 return -ENOMEM; in gaudi_restore_sm_registers()
5754 dev_err(hdev->dev, "failed resetting SM registers"); in gaudi_restore_sm_registers()
5755 return -ENOMEM; in gaudi_restore_sm_registers()
5760 num_regs = NUM_OF_SOB_IN_BLOCK - GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT; in gaudi_restore_sm_registers()
5763 dev_err(hdev->dev, "failed resetting SM registers"); in gaudi_restore_sm_registers()
5764 return -ENOMEM; in gaudi_restore_sm_registers()
5769 num_regs = NUM_OF_MONITORS_IN_BLOCK - GAUDI_FIRST_AVAILABLE_W_S_MONITOR; in gaudi_restore_sm_registers()
5772 dev_err(hdev->dev, "failed resetting SM registers"); in gaudi_restore_sm_registers()
5773 return -ENOMEM; in gaudi_restore_sm_registers()
5781 u32 sob_delta = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_1 - in gaudi_restore_dma_registers()
5797 /* For DMAs 2-7, need to restore WR_AWUSER_31_11 as it can be in gaudi_restore_dma_registers()
5800 if (i > 1) in gaudi_restore_dma_registers()
5817 qman_offset = i * (mmMME2_QM_BASE - mmMME0_QM_BASE); in gaudi_restore_qm_registers()
5827 qman_offset = (i >> 1) * NIC_MACRO_QMAN_OFFSET + in gaudi_restore_qm_registers()
5854 u32 size = hdev->asic_prop.mmu_pgt_size + in gaudi_mmu_clear_pgt_range()
5855 hdev->asic_prop.mmu_cache_mng_size; in gaudi_mmu_clear_pgt_range()
5856 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_mmu_clear_pgt_range()
5857 u64 addr = hdev->asic_prop.mmu_pgt_addr; in gaudi_mmu_clear_pgt_range()
5859 if (!(gaudi->hw_cap_initialized & HW_CAP_MMU)) in gaudi_mmu_clear_pgt_range()
5885 (1 << DMA0_CORE_COMMIT_LIN_SHIFT)); in gaudi_dma_core_transfer()
5896 dev_err(hdev->dev, in gaudi_dma_core_transfer()
5897 "DMA %d timed-out during reading of 0x%llx\n", in gaudi_dma_core_transfer()
5899 return -EIO; in gaudi_dma_core_transfer()
5905 dev_err(hdev->dev, "DMA Failed, cause 0x%x\n", err_cause); in gaudi_dma_core_transfer()
5906 dev_dbg(hdev->dev, in gaudi_dma_core_transfer()
5907 "Clearing DMA0 engine from errors (cause 0x%x)\n", in gaudi_dma_core_transfer()
5911 return -EIO; in gaudi_dma_core_transfer()
5931 return -ENOMEM; in gaudi_debugfs_read_dma()
5933 hdev->asic_funcs->hw_queues_lock(hdev); in gaudi_debugfs_read_dma()
5955 dev_err_ratelimited(hdev->dev, in gaudi_debugfs_read_dma()
5957 rc = -EAGAIN; in gaudi_debugfs_read_dma()
5975 dev_dbg(hdev->dev, in gaudi_debugfs_read_dma()
5976 "Clearing DMA0 engine from errors (cause 0x%x)\n", in gaudi_debugfs_read_dma()
6002 size_left -= SZ_2M; in gaudi_debugfs_read_dma()
6015 hdev->asic_funcs->hw_queues_unlock(hdev); in gaudi_debugfs_read_dma()
6024 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_read_pte()
6026 if (hdev->reset_info.hard_reset_pending) in gaudi_read_pte()
6029 return readq(hdev->pcie_bar[HBM_BAR_ID] + in gaudi_read_pte()
6030 (addr - gaudi->hbm_bar_cur_addr)); in gaudi_read_pte()
6035 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_write_pte()
6037 if (hdev->reset_info.hard_reset_pending) in gaudi_write_pte()
6040 writeq(val, hdev->pcie_bar[HBM_BAR_ID] + in gaudi_write_pte()
6041 (addr - gaudi->hbm_bar_cur_addr)); in gaudi_write_pte()
6053 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_mmu_prepare()
6055 if (!(gaudi->hw_cap_initialized & HW_CAP_MMU)) in gaudi_mmu_prepare()
6059 dev_crit(hdev->dev, "asid %u is too big\n", asid); in gaudi_mmu_prepare()
6208 if (gaudi->hw_cap_initialized & HW_CAP_NIC0) { in gaudi_mmu_prepare()
6221 if (gaudi->hw_cap_initialized & HW_CAP_NIC1) { in gaudi_mmu_prepare()
6234 if (gaudi->hw_cap_initialized & HW_CAP_NIC2) { in gaudi_mmu_prepare()
6247 if (gaudi->hw_cap_initialized & HW_CAP_NIC3) { in gaudi_mmu_prepare()
6260 if (gaudi->hw_cap_initialized & HW_CAP_NIC4) { in gaudi_mmu_prepare()
6273 if (gaudi->hw_cap_initialized & HW_CAP_NIC5) { in gaudi_mmu_prepare()
6286 if (gaudi->hw_cap_initialized & HW_CAP_NIC6) { in gaudi_mmu_prepare()
6299 if (gaudi->hw_cap_initialized & HW_CAP_NIC7) { in gaudi_mmu_prepare()
6312 if (gaudi->hw_cap_initialized & HW_CAP_NIC8) { in gaudi_mmu_prepare()
6325 if (gaudi->hw_cap_initialized & HW_CAP_NIC9) { in gaudi_mmu_prepare()
6352 if (hdev->pldm) in gaudi_send_job_on_qman0()
6359 dev_err(hdev->dev, in gaudi_send_job_on_qman0()
6361 return -ENOMEM; in gaudi_send_job_on_qman0()
6364 cb = job->patched_cb; in gaudi_send_job_on_qman0()
6366 fence_pkt = cb->kernel_address + in gaudi_send_job_on_qman0()
6367 job->job_cb_size - sizeof(struct packet_msg_prot); in gaudi_send_job_on_qman0()
6370 tmp |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1); in gaudi_send_job_on_qman0()
6371 tmp |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); in gaudi_send_job_on_qman0()
6373 fence_pkt->ctl = cpu_to_le32(tmp); in gaudi_send_job_on_qman0()
6374 fence_pkt->value = cpu_to_le32(GAUDI_QMAN0_FENCE_VAL); in gaudi_send_job_on_qman0()
6375 fence_pkt->addr = cpu_to_le64(fence_dma_addr); in gaudi_send_job_on_qman0()
6383 job->job_cb_size, cb->bus_address); in gaudi_send_job_on_qman0()
6385 dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc); in gaudi_send_job_on_qman0()
6395 if (rc == -ETIMEDOUT) { in gaudi_send_job_on_qman0()
6396 dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp); in gaudi_send_job_on_qman0()
6436 dma_id[1] = 2; in gaudi_get_razwi_initiator_dma_name()
6440 dma_id[0] = 1; in gaudi_get_razwi_initiator_dma_name()
6441 dma_id[1] = 3; in gaudi_get_razwi_initiator_dma_name()
6446 dma_id[1] = 6; in gaudi_get_razwi_initiator_dma_name()
6451 dma_id[1] = 7; in gaudi_get_razwi_initiator_dma_name()
6465 if ((err_cause[0] & mask) && !(err_cause[1] & mask)) { in gaudi_get_razwi_initiator_dma_name()
6468 } else if (!(err_cause[0] & mask) && (err_cause[1] & mask)) { in gaudi_get_razwi_initiator_dma_name()
6478 if ((err_cause[0] & mask) && !(err_cause[1] & mask)) { in gaudi_get_razwi_initiator_dma_name()
6481 } else if (!(err_cause[0] & mask) && (err_cause[1] & mask)) { in gaudi_get_razwi_initiator_dma_name()
6491 if ((err_cause[0] & mask) && !(err_cause[1] & mask)) { in gaudi_get_razwi_initiator_dma_name()
6494 } else if (!(err_cause[0] & mask) && (err_cause[1] & mask)) { in gaudi_get_razwi_initiator_dma_name()
6504 if ((err_cause[0] & mask) && !(err_cause[1] & mask)) { in gaudi_get_razwi_initiator_dma_name()
6507 } else if (!(err_cause[0] & mask) && (err_cause[1] & mask)) { in gaudi_get_razwi_initiator_dma_name()
6563 /* PCI, CPU or PSOC does not have engine id*/ in gaudi_get_razwi_initiator_name()
6627 dev_err(hdev->dev, in gaudi_get_razwi_initiator_name()
6643 dev_err_ratelimited(hdev->dev, in gaudi_print_and_get_razwi_info()
6651 dev_err_ratelimited(hdev->dev, in gaudi_print_and_get_razwi_info()
6661 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_print_and_get_mmu_error_info()
6664 if (!(gaudi->hw_cap_initialized & HW_CAP_MMU)) in gaudi_print_and_get_mmu_error_info()
6673 dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n", *addr); in gaudi_print_and_get_mmu_error_info()
6685 dev_err_ratelimited(hdev->dev, "MMU access error on va 0x%llx\n", *addr); in gaudi_print_and_get_mmu_error_info()
6692 * +-------------------+------------------------------------------------------+
6695 * +-------------------+------------------------------------------------------+
6696 * | 0xF30 - 0xF3F |ECC single error indication (1 bit per memory wrapper)|
6701 * +-------------------+------------------------------------------------------+
6702 * | 0xF40 - 0xF4F |ECC double error indication (1 bit per memory wrapper)|
6707 * +-------------------+------------------------------------------------------+
6716 num_mem_regs = params->num_memories / 32 + in gaudi_extract_ecc_info()
6717 ((params->num_memories % 32) ? 1 : 0); in gaudi_extract_ecc_info()
6719 if (params->block_address >= CFG_BASE) in gaudi_extract_ecc_info()
6720 params->block_address -= CFG_BASE; in gaudi_extract_ecc_info()
6722 if (params->derr) in gaudi_extract_ecc_info()
6723 err_addr = params->block_address + GAUDI_ECC_DERR0_OFFSET; in gaudi_extract_ecc_info()
6725 err_addr = params->block_address + GAUDI_ECC_SERR0_OFFSET; in gaudi_extract_ecc_info()
6742 dev_err(hdev->dev, "ECC error information cannot be found\n"); in gaudi_extract_ecc_info()
6743 return -EINVAL; in gaudi_extract_ecc_info()
6746 WREG32(params->block_address + GAUDI_ECC_MEM_SEL_OFFSET, in gaudi_extract_ecc_info()
6750 RREG32(params->block_address + GAUDI_ECC_ADDRESS_OFFSET); in gaudi_extract_ecc_info()
6752 RREG32(params->block_address + GAUDI_ECC_SYNDROME_OFFSET); in gaudi_extract_ecc_info()
6755 reg = RREG32(params->block_address + GAUDI_ECC_MEM_INFO_CLR_OFFSET); in gaudi_extract_ecc_info()
6756 if (params->derr) in gaudi_extract_ecc_info()
6757 reg |= FIELD_PREP(GAUDI_ECC_MEM_INFO_CLR_DERR_MASK, 1); in gaudi_extract_ecc_info()
6759 reg |= FIELD_PREP(GAUDI_ECC_MEM_INFO_CLR_SERR_MASK, 1); in gaudi_extract_ecc_info()
6761 WREG32(params->block_address + GAUDI_ECC_MEM_INFO_CLR_OFFSET, reg); in gaudi_extract_ecc_info()
6767 * gaudi_queue_idx_dec - decrement queue index (pi/ci) and handle wrap
6776 u32 mask = q_len - 1; in gaudi_queue_idx_dec()
6779 * modular decrement is equivalent to adding (queue_size -1) in gaudi_queue_idx_dec()
6781 * range [0, queue_len - 1] in gaudi_queue_idx_dec()
6783 return (idx + q_len - 1) & mask; in gaudi_queue_idx_dec()
6787 * gaudi_handle_sw_config_stream_data - print SW config stream data
6800 cq_ptr_lo_off = mmTPC0_QM_CQ_PTR_LO_1 - mmTPC0_QM_CQ_PTR_LO_0; in gaudi_handle_sw_config_stream_data()
6802 cq_ptr_lo = qman_base + (mmTPC0_QM_CQ_PTR_LO_0 - mmTPC0_QM_BASE) + in gaudi_handle_sw_config_stream_data()
6805 (mmTPC0_QM_CQ_PTR_HI_0 - mmTPC0_QM_CQ_PTR_LO_0); in gaudi_handle_sw_config_stream_data()
6807 (mmTPC0_QM_CQ_TSIZE_0 - mmTPC0_QM_CQ_PTR_LO_0); in gaudi_handle_sw_config_stream_data()
6811 dev_info(hdev->dev, "stop on err: stream: %u, addr: %#llx, size: %u\n", in gaudi_handle_sw_config_stream_data()
6815 hdev->captured_err_info.undef_opcode.cq_addr = cq_ptr; in gaudi_handle_sw_config_stream_data()
6816 hdev->captured_err_info.undef_opcode.cq_size = size; in gaudi_handle_sw_config_stream_data()
6817 hdev->captured_err_info.undef_opcode.stream_id = stream; in gaudi_handle_sw_config_stream_data()
6822 * gaudi_handle_last_pqes_on_err - print last PQEs on error
6841 q = &hdev->kernel_queues[qid_base + stream]; in gaudi_handle_last_pqes_on_err()
6843 qm_ci_stream_off = mmTPC0_QM_PQ_CI_1 - mmTPC0_QM_PQ_CI_0; in gaudi_handle_last_pqes_on_err()
6844 pq_ci = qman_base + (mmTPC0_QM_PQ_CI_0 - mmTPC0_QM_BASE) + in gaudi_handle_last_pqes_on_err()
6847 queue_len = (q->queue_type == QUEUE_TYPE_INT) ? in gaudi_handle_last_pqes_on_err()
6848 q->int_queue_len : HL_QUEUE_LENGTH; in gaudi_handle_last_pqes_on_err()
6850 hdev->asic_funcs->hw_queues_lock(hdev); in gaudi_handle_last_pqes_on_err()
6857 /* we should start printing form ci -1 */ in gaudi_handle_last_pqes_on_err()
6865 bd = q->kernel_address; in gaudi_handle_last_pqes_on_err()
6868 len = le32_to_cpu(bd->len); in gaudi_handle_last_pqes_on_err()
6869 /* len 0 means uninitialized entry- break */ in gaudi_handle_last_pqes_on_err()
6873 addr[i] = le64_to_cpu(bd->ptr); in gaudi_handle_last_pqes_on_err()
6875 dev_info(hdev->dev, "stop on err PQE(stream %u): ci: %u, addr: %#llx, size: %u\n", in gaudi_handle_last_pqes_on_err()
6883 struct undefined_opcode_info *undef_opcode = &hdev->captured_err_info.undef_opcode; in gaudi_handle_last_pqes_on_err()
6884 u32 arr_idx = undef_opcode->cb_addr_streams_len; in gaudi_handle_last_pqes_on_err()
6887 undef_opcode->timestamp = ktime_get(); in gaudi_handle_last_pqes_on_err()
6888 undef_opcode->engine_id = gaudi_queue_id_to_engine_id[qid_base]; in gaudi_handle_last_pqes_on_err()
6891 memcpy(undef_opcode->cb_addr_streams[arr_idx], addr, sizeof(addr)); in gaudi_handle_last_pqes_on_err()
6892 undef_opcode->cb_addr_streams_len++; in gaudi_handle_last_pqes_on_err()
6895 hdev->asic_funcs->hw_queues_unlock(hdev); in gaudi_handle_last_pqes_on_err()
6899 * handle_qman_data_on_err - extract QMAN data on error
6922 /* handle Lower-CP */ in handle_qman_data_on_err()
6940 glbl_sts_addr = qman_base + (mmTPC0_QM_GLBL_STS1_0 - mmTPC0_QM_BASE); in gaudi_handle_qman_err_generic()
6941 arb_err_addr = qman_base + (mmTPC0_QM_ARB_ERR_CAUSE - mmTPC0_QM_BASE); in gaudi_handle_qman_err_generic()
6944 for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) { in gaudi_handle_qman_err_generic()
6958 dev_err_ratelimited(hdev->dev, in gaudi_handle_qman_err_generic()
6967 hdev->captured_err_info.undef_opcode.write_enable) { in gaudi_handle_qman_err_generic()
6968 memset(&hdev->captured_err_info.undef_opcode, 0, in gaudi_handle_qman_err_generic()
6969 sizeof(hdev->captured_err_info.undef_opcode)); in gaudi_handle_qman_err_generic()
6971 hdev->captured_err_info.undef_opcode.write_enable = false; in gaudi_handle_qman_err_generic()
6975 /* Write 1 clear errors */ in gaudi_handle_qman_err_generic()
6976 if (!hdev->stop_on_err) in gaudi_handle_qman_err_generic()
6989 dev_err_ratelimited(hdev->dev, in gaudi_handle_qman_err_generic()
7000 u32 index = event_type - GAUDI_EVENT_DMA_IF_SEI_0; in gaudi_print_sm_sei_info()
7005 switch (sei_data->sei_cause) { in gaudi_print_sm_sei_info()
7007 dev_err_ratelimited(hdev->dev, in gaudi_print_sm_sei_info()
7010 le32_to_cpu(sei_data->sei_log)); in gaudi_print_sm_sei_info()
7013 dev_err_ratelimited(hdev->dev, in gaudi_print_sm_sei_info()
7014 "%s SEI Error: Unaligned 4B LBW access, monitor agent address low - %#x", in gaudi_print_sm_sei_info()
7016 le32_to_cpu(sei_data->sei_log)); in gaudi_print_sm_sei_info()
7019 dev_err_ratelimited(hdev->dev, in gaudi_print_sm_sei_info()
7020 "%s SEI Error: AXI ID %u response error", in gaudi_print_sm_sei_info()
7022 le32_to_cpu(sei_data->sei_log)); in gaudi_print_sm_sei_info()
7025 dev_err_ratelimited(hdev->dev, "Unknown SM SEI cause %u", in gaudi_print_sm_sei_info()
7026 le32_to_cpu(sei_data->sei_log)); in gaudi_print_sm_sei_info()
7040 if (hdev->asic_prop.fw_security_enabled) { in gaudi_handle_ecc_event()
7051 index = event_type - GAUDI_EVENT_TPC0_SERR; in gaudi_handle_ecc_event()
7058 index = event_type - GAUDI_EVENT_TPC0_DERR; in gaudi_handle_ecc_event()
7069 index = (event_type - GAUDI_EVENT_MME0_ACC_SERR) / 4; in gaudi_handle_ecc_event()
7079 index = (event_type - GAUDI_EVENT_MME0_ACC_DERR) / 4; in gaudi_handle_ecc_event()
7089 index = (event_type - GAUDI_EVENT_MME0_SBAB_SERR) / 4; in gaudi_handle_ecc_event()
7100 index = (event_type - GAUDI_EVENT_MME0_SBAB_DERR) / 4; in gaudi_handle_ecc_event()
7113 ecc_address = le64_to_cpu(ecc_data->ecc_address); in gaudi_handle_ecc_event()
7114 ecc_syndrom = le64_to_cpu(ecc_data->ecc_syndrom); in gaudi_handle_ecc_event()
7115 memory_wrapper_idx = ecc_data->memory_wrapper_idx; in gaudi_handle_ecc_event()
7123 dev_err(hdev->dev, in gaudi_handle_ecc_event()
7137 index = event_type - GAUDI_EVENT_TPC0_QM; in gaudi_handle_qman_err()
7154 index = event_type - GAUDI_EVENT_DMA0_QM; in gaudi_handle_qman_err()
7157 if (index > 1) in gaudi_handle_qman_err()
7229 * Init engine id by default as not valid and only if razwi initiated from engine with in gaudi_print_irq_info()
7230 * engine id it will get valid value. in gaudi_print_irq_info()
7233 engine_id[1] = HL_RAZWI_NA_ENG_ID; in gaudi_print_irq_info()
7236 dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n", in gaudi_print_irq_info()
7240 gaudi_print_and_get_razwi_info(hdev, &engine_id[0], &engine_id[1], &is_read, in gaudi_print_irq_info()
7250 if (engine_id[1] != HL_RAZWI_NA_ENG_ID) in gaudi_print_irq_info()
7253 num_of_razwi_eng = 1; in gaudi_print_irq_info()
7265 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ]; in gaudi_print_out_of_sync_info()
7267 dev_err(hdev->dev, "Out of sync with FW, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d\n", in gaudi_print_out_of_sync_info()
7268 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci)); in gaudi_print_out_of_sync_info()
7274 dev_err(hdev->dev, in gaudi_print_fw_alive_info()
7276 (fw_alive->severity == FW_ALIVE_SEVERITY_MINOR) ? "Minor" : "Critical", in gaudi_print_fw_alive_info()
7277 le32_to_cpu(fw_alive->process_id), in gaudi_print_fw_alive_info()
7278 le32_to_cpu(fw_alive->thread_id), in gaudi_print_fw_alive_info()
7279 le64_to_cpu(fw_alive->uptime_seconds)); in gaudi_print_fw_alive_info()
7287 u16 nic_id = event_type - GAUDI_EVENT_NIC_SEI_0; in gaudi_print_nic_axi_irq_info()
7289 switch (eq_nic_sei->axi_error_cause) { in gaudi_print_nic_axi_irq_info()
7312 dev_err(hdev->dev, "unknown NIC AXI cause %d\n", in gaudi_print_nic_axi_irq_info()
7313 eq_nic_sei->axi_error_cause); in gaudi_print_nic_axi_irq_info()
7319 eq_nic_sei->id); in gaudi_print_nic_axi_irq_info()
7320 dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n", in gaudi_print_nic_axi_irq_info()
7326 /* GAUDI doesn't support any reset except hard-reset */ in gaudi_compute_reset_late_init()
7327 return -EPERM; in gaudi_compute_reset_late_init()
7336 if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 & in gaudi_hbm_read_interrupts()
7339 dev_err(hdev->dev, "No FW ECC data"); in gaudi_hbm_read_interrupts()
7344 le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); in gaudi_hbm_read_interrupts()
7346 le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); in gaudi_hbm_read_interrupts()
7348 le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); in gaudi_hbm_read_interrupts()
7350 le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); in gaudi_hbm_read_interrupts()
7352 le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); in gaudi_hbm_read_interrupts()
7354 le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); in gaudi_hbm_read_interrupts()
7356 le32_to_cpu(hbm_ecc_data->hbm_ecc_info)); in gaudi_hbm_read_interrupts()
7358 dev_err(hdev->dev, in gaudi_hbm_read_interrupts()
7361 dev_err(hdev->dev, in gaudi_hbm_read_interrupts()
7362 …"HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%u, SEC_CNT=%d, DEC_CNT=%d\… in gaudi_hbm_read_interrupts()
7363 device, ch, hbm_ecc_data->first_addr, type, in gaudi_hbm_read_interrupts()
7364 hbm_ecc_data->sec_cont_cnt, hbm_ecc_data->sec_cnt, in gaudi_hbm_read_interrupts()
7365 hbm_ecc_data->dec_cnt); in gaudi_hbm_read_interrupts()
7369 if (hdev->asic_prop.fw_security_enabled) { in gaudi_hbm_read_interrupts()
7370 dev_info(hdev->dev, "Cannot access MC regs for ECC data while security is enabled\n"); in gaudi_hbm_read_interrupts()
7379 rc = -EIO; in gaudi_hbm_read_interrupts()
7380 dev_err(hdev->dev, in gaudi_hbm_read_interrupts()
7382 device, ch * 2, val & 0x1, (val >> 1) & 0x1, in gaudi_hbm_read_interrupts()
7387 dev_err(hdev->dev, in gaudi_hbm_read_interrupts()
7388 …"HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%d, SEC_CNT=%d, DEC_CNT=%d\… in gaudi_hbm_read_interrupts()
7399 rc = -EIO; in gaudi_hbm_read_interrupts()
7400 dev_err(hdev->dev, in gaudi_hbm_read_interrupts()
7402 device, ch * 2 + 1, val & 0x1, (val >> 1) & 0x1, in gaudi_hbm_read_interrupts()
7407 dev_err(hdev->dev, in gaudi_hbm_read_interrupts()
7408 …"HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%d, SEC_CNT=%d, DEC_CNT=%d\… in gaudi_hbm_read_interrupts()
7409 device, ch * 2 + 1, in gaudi_hbm_read_interrupts()
7428 rc = -EIO; in gaudi_hbm_read_interrupts()
7429 dev_err(hdev->dev, in gaudi_hbm_read_interrupts()
7436 rc = -EIO; in gaudi_hbm_read_interrupts()
7437 dev_err(hdev->dev, in gaudi_hbm_read_interrupts()
7453 return 1; in gaudi_hbm_event_to_dev()
7479 dev_err_ratelimited(hdev->dev, in gaudi_tpc_read_interrupts()
7483 /* If this is QM error, we need to soft-reset */ in gaudi_tpc_read_interrupts()
7496 return (tpc_dec_event_type - GAUDI_EVENT_TPC0_DEC) >> 1; in tpc_dec_event_to_tpc_id()
7501 return (tpc_dec_event_type - GAUDI_EVENT_TPC0_KRN_ERR) / 6; in tpc_krn_event_to_tpc_id()
7508 mutex_lock(&hdev->clk_throttling.lock); in gaudi_print_clk_change_info()
7512 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER; in gaudi_print_clk_change_info()
7513 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER; in gaudi_print_clk_change_info()
7514 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get(); in gaudi_print_clk_change_info()
7515 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time; in gaudi_print_clk_change_info()
7516 dev_info_ratelimited(hdev->dev, in gaudi_print_clk_change_info()
7521 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER; in gaudi_print_clk_change_info()
7522 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get(); in gaudi_print_clk_change_info()
7523 dev_info_ratelimited(hdev->dev, in gaudi_print_clk_change_info()
7528 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL; in gaudi_print_clk_change_info()
7529 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL; in gaudi_print_clk_change_info()
7530 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get(); in gaudi_print_clk_change_info()
7531 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time; in gaudi_print_clk_change_info()
7533 dev_info_ratelimited(hdev->dev, in gaudi_print_clk_change_info()
7538 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL; in gaudi_print_clk_change_info()
7539 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get(); in gaudi_print_clk_change_info()
7541 dev_info_ratelimited(hdev->dev, in gaudi_print_clk_change_info()
7546 dev_err(hdev->dev, "Received invalid clock change event %d\n", in gaudi_print_clk_change_info()
7551 mutex_unlock(&hdev->clk_throttling.lock); in gaudi_print_clk_change_info()
7556 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_handle_eqe()
7558 u64 data = le64_to_cpu(eq_entry->data[0]), event_mask = 0; in gaudi_handle_eqe()
7559 u32 ctl = le32_to_cpu(eq_entry->hdr.ctl); in gaudi_handle_eqe()
7568 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", in gaudi_handle_eqe()
7569 event_type, GAUDI_EVENT_SIZE - 1); in gaudi_handle_eqe()
7573 gaudi->events_stat[event_type]++; in gaudi_handle_eqe()
7574 gaudi->events_stat_aggregate[event_type]++; in gaudi_handle_eqe()
7601 gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); in gaudi_handle_eqe()
7622 &eq_entry->hbm_ecc_data); in gaudi_handle_eqe()
7634 &eq_entry->hbm_ecc_data); in gaudi_handle_eqe()
7659 dev_err(hdev->dev, "reset required due to %s\n", in gaudi_handle_eqe()
7684 dev_err(hdev->dev, "reset required due to %s\n", in gaudi_handle_eqe()
7718 gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); in gaudi_handle_eqe()
7797 &eq_entry->sm_sei_data); in gaudi_handle_eqe()
7801 dev_err(hdev->dev, in gaudi_handle_eqe()
7815 cause = le64_to_cpu(eq_entry->data[0]) & 0xFF; in gaudi_handle_eqe()
7816 dev_err(hdev->dev, in gaudi_handle_eqe()
7829 gaudi_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err); in gaudi_handle_eqe()
7835 gaudi_print_fw_alive_info(hdev, &eq_entry->fw_alive); in gaudi_handle_eqe()
7843 dev_err(hdev->dev, "Received invalid H/W interrupt %d\n", in gaudi_handle_eqe()
7856 if (hdev->asic_prop.fw_security_enabled && !reset_direct) { in gaudi_handle_eqe()
7862 } else if (hdev->hard_reset_on_fw_events) { in gaudi_handle_eqe()
7885 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_get_events_stat()
7888 *size = (u32) sizeof(gaudi->events_stat_aggregate); in gaudi_get_events_stat()
7889 return gaudi->events_stat_aggregate; in gaudi_get_events_stat()
7892 *size = (u32) sizeof(gaudi->events_stat); in gaudi_get_events_stat()
7893 return gaudi->events_stat; in gaudi_get_events_stat()
7898 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_mmu_invalidate_cache()
7902 if (!(gaudi->hw_cap_initialized & HW_CAP_MMU) || in gaudi_mmu_invalidate_cache()
7903 hdev->reset_info.hard_reset_pending) in gaudi_mmu_invalidate_cache()
7906 if (hdev->pldm) in gaudi_mmu_invalidate_cache()
7913 WREG32(mmSTLB_CACHE_INV, gaudi->mmu_cache_inv_pi++); in gaudi_mmu_invalidate_cache()
7936 return hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags); in gaudi_mmu_invalidate_cache_range()
7944 if (hdev->pldm) in gaudi_mmu_update_asid_hop0_addr()
7963 dev_err(hdev->dev, in gaudi_mmu_update_asid_hop0_addr()
7973 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_send_heartbeat()
7975 if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi_send_heartbeat()
7983 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_cpucp_info_get()
7984 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi_cpucp_info_get()
7987 if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi_cpucp_info_get()
7996 if (!strlen(prop->cpucp_info.card_name)) in gaudi_cpucp_info_get()
7997 strscpy_pad(prop->cpucp_info.card_name, GAUDI_DEFAULT_CARD_NAME, in gaudi_cpucp_info_get()
8000 hdev->card_type = le32_to_cpu(hdev->asic_prop.cpucp_info.card_type); in gaudi_cpucp_info_get()
8010 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_is_device_idle()
8011 const char *fmt = "%-5d%-9s%#-14x%#-12x%#x\n"; in gaudi_is_device_idle()
8012 const char *mme_slave_fmt = "%-5d%-9s%-14s%-12s%#x\n"; in gaudi_is_device_idle()
8013 const char *nic_fmt = "%-5d%-9s%#-14x%#x\n"; in gaudi_is_device_idle()
8023 "--- ------- ------------ ---------- -------------\n"); in gaudi_is_device_idle()
8047 "--- ------- ------------ ---------- ----------\n"); in gaudi_is_device_idle()
8069 "--- ------- ------------ ---------- -----------\n"); in gaudi_is_device_idle()
8076 /* MME 1 & 3 are slaves, no need to check their QMANs */ in gaudi_is_device_idle()
8095 is_eng_idle ? "Y" : "N", "-", in gaudi_is_device_idle()
8096 "-", mme_arch_sts); in gaudi_is_device_idle()
8103 "--- ------- ------------ ----------\n"); in gaudi_is_device_idle()
8108 if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + port)) { in gaudi_is_device_idle()
8122 port = 2 * i + 1; in gaudi_is_device_idle()
8123 if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + port)) { in gaudi_is_device_idle()
8145 __acquires(&gaudi->hw_queues_lock) in gaudi_hw_queues_lock()
8147 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_hw_queues_lock()
8149 spin_lock(&gaudi->hw_queues_lock); in gaudi_hw_queues_lock()
8153 __releases(&gaudi->hw_queues_lock) in gaudi_hw_queues_unlock()
8155 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_hw_queues_unlock()
8157 spin_unlock(&gaudi->hw_queues_lock); in gaudi_hw_queues_unlock()
8162 return hdev->pdev->device; in gaudi_get_pci_id()
8168 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_get_eeprom_data()
8170 if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi_get_eeprom_data()
8178 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_get_monitor_dump()
8180 if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi_get_monitor_dump()
8196 offset = tpc_id * (mmTPC1_CFG_STATUS - mmTPC0_CFG_STATUS); in gaudi_run_tpc_kernel()
8198 if (hdev->pldm) in gaudi_run_tpc_kernel()
8223 (1 << TPC0_CFG_TPC_CMD_ICACHE_INVALIDATE_SHIFT | in gaudi_run_tpc_kernel()
8224 1 << TPC0_CFG_TPC_CMD_ICACHE_PREFETCH_64KB_SHIFT)); in gaudi_run_tpc_kernel()
8225 /* wait a bit for the engine to start executing */ in gaudi_run_tpc_kernel()
8228 /* wait until engine has finished executing */ in gaudi_run_tpc_kernel()
8239 dev_err(hdev->dev, in gaudi_run_tpc_kernel()
8242 return -EIO; in gaudi_run_tpc_kernel()
8246 1 << TPC0_CFG_TPC_EXECUTE_V_SHIFT); in gaudi_run_tpc_kernel()
8248 /* wait a bit for the engine to start executing */ in gaudi_run_tpc_kernel()
8251 /* wait until engine has finished executing */ in gaudi_run_tpc_kernel()
8262 dev_err(hdev->dev, in gaudi_run_tpc_kernel()
8265 return -EIO; in gaudi_run_tpc_kernel()
8277 dev_err(hdev->dev, in gaudi_run_tpc_kernel()
8280 return -EIO; in gaudi_run_tpc_kernel()
8289 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_internal_cb_pool_init()
8292 if (!(gaudi->hw_cap_initialized & HW_CAP_MMU)) in gaudi_internal_cb_pool_init()
8295 hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev, in gaudi_internal_cb_pool_init()
8297 &hdev->internal_cb_pool_dma_addr, in gaudi_internal_cb_pool_init()
8300 if (!hdev->internal_cb_pool_virt_addr) in gaudi_internal_cb_pool_init()
8301 return -ENOMEM; in gaudi_internal_cb_pool_init()
8307 hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1); in gaudi_internal_cb_pool_init()
8308 if (!hdev->internal_cb_pool) { in gaudi_internal_cb_pool_init()
8309 dev_err(hdev->dev, in gaudi_internal_cb_pool_init()
8311 rc = -ENOMEM; in gaudi_internal_cb_pool_init()
8315 rc = gen_pool_add(hdev->internal_cb_pool, in gaudi_internal_cb_pool_init()
8316 (uintptr_t) hdev->internal_cb_pool_virt_addr, in gaudi_internal_cb_pool_init()
8317 HOST_SPACE_INTERNAL_CB_SZ, -1); in gaudi_internal_cb_pool_init()
8319 dev_err(hdev->dev, in gaudi_internal_cb_pool_init()
8321 rc = -EFAULT; in gaudi_internal_cb_pool_init()
8325 hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, in gaudi_internal_cb_pool_init()
8329 if (!hdev->internal_cb_va_base) { in gaudi_internal_cb_pool_init()
8330 rc = -ENOMEM; in gaudi_internal_cb_pool_init()
8334 mutex_lock(&hdev->mmu_lock); in gaudi_internal_cb_pool_init()
8336 rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, in gaudi_internal_cb_pool_init()
8337 hdev->internal_cb_pool_dma_addr, in gaudi_internal_cb_pool_init()
8346 mutex_unlock(&hdev->mmu_lock); in gaudi_internal_cb_pool_init()
8351 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, in gaudi_internal_cb_pool_init()
8354 mutex_unlock(&hdev->mmu_lock); in gaudi_internal_cb_pool_init()
8355 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, in gaudi_internal_cb_pool_init()
8358 gen_pool_destroy(hdev->internal_cb_pool); in gaudi_internal_cb_pool_init()
8360 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, in gaudi_internal_cb_pool_init()
8361 hdev->internal_cb_pool_dma_addr); in gaudi_internal_cb_pool_init()
8369 struct gaudi_device *gaudi = hdev->asic_specific; in gaudi_internal_cb_pool_fini()
8371 if (!(gaudi->hw_cap_initialized & HW_CAP_MMU)) in gaudi_internal_cb_pool_fini()
8374 mutex_lock(&hdev->mmu_lock); in gaudi_internal_cb_pool_fini()
8375 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, in gaudi_internal_cb_pool_fini()
8377 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, in gaudi_internal_cb_pool_fini()
8380 mutex_unlock(&hdev->mmu_lock); in gaudi_internal_cb_pool_fini()
8382 gen_pool_destroy(hdev->internal_cb_pool); in gaudi_internal_cb_pool_fini()
8384 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, in gaudi_internal_cb_pool_fini()
8385 hdev->internal_cb_pool_dma_addr); in gaudi_internal_cb_pool_fini()
8392 if (ctx->asid == HL_KERNEL_ASID_ID) in gaudi_ctx_init()
8395 rc = gaudi_internal_cb_pool_init(ctx->hdev, ctx); in gaudi_ctx_init()
8399 rc = gaudi_restore_user_registers(ctx->hdev); in gaudi_ctx_init()
8401 gaudi_internal_cb_pool_fini(ctx->hdev, ctx); in gaudi_ctx_init()
8408 if (ctx->asid == HL_KERNEL_ASID_ID) in gaudi_ctx_fini()
8411 gaudi_internal_cb_pool_fini(ctx->hdev, ctx); in gaudi_ctx_fini()
8449 pkt = cb->kernel_address + size; in gaudi_gen_signal_cb()
8452 /* Inc by 1, Mode ADD */ in gaudi_gen_signal_cb()
8453 value = FIELD_PREP(GAUDI_PKT_SHORT_VAL_SOB_SYNC_VAL_MASK, 1); in gaudi_gen_signal_cb()
8454 value |= FIELD_PREP(GAUDI_PKT_SHORT_VAL_SOB_MOD_MASK, 1); in gaudi_gen_signal_cb()
8461 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); in gaudi_gen_signal_cb()
8462 ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); in gaudi_gen_signal_cb()
8464 pkt->value = cpu_to_le32(value); in gaudi_gen_signal_cb()
8465 pkt->ctl = cpu_to_le32(ctl); in gaudi_gen_signal_cb()
8481 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); in gaudi_add_mon_msg_short()
8484 pkt->value = cpu_to_le32(value); in gaudi_add_mon_msg_short()
8485 pkt->ctl = cpu_to_le32(ctl); in gaudi_add_mon_msg_short()
8500 dev_err(hdev->dev, in gaudi_add_arm_monitor_pkt()
8513 (mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0 + mon_id * 4) - in gaudi_add_arm_monitor_pkt()
8530 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); in gaudi_add_arm_monitor_pkt()
8531 ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); in gaudi_add_arm_monitor_pkt()
8533 pkt->value = cpu_to_le32(value); in gaudi_add_arm_monitor_pkt()
8534 pkt->ctl = cpu_to_le32(ctl); in gaudi_add_arm_monitor_pkt()
8545 cfg = FIELD_PREP(GAUDI_PKT_FENCE_CFG_DEC_VAL_MASK, 1); in gaudi_add_fence_pkt()
8546 cfg |= FIELD_PREP(GAUDI_PKT_FENCE_CFG_TARGET_VAL_MASK, 1); in gaudi_add_fence_pkt()
8551 ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1); in gaudi_add_fence_pkt()
8552 ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1); in gaudi_add_fence_pkt()
8554 pkt->cfg = cpu_to_le32(cfg); in gaudi_add_fence_pkt()
8555 pkt->ctl = cpu_to_le32(ctl); in gaudi_add_fence_pkt()
8623 nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_0) >> 2; in gaudi_get_fence_addr()
8625 (nic_index >> 1) * NIC_MACRO_QMAN_OFFSET + in gaudi_get_fence_addr()
8638 nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_1) >> 2; in gaudi_get_fence_addr()
8640 (nic_index >> 1) * NIC_MACRO_QMAN_OFFSET + in gaudi_get_fence_addr()
8653 nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_2) >> 2; in gaudi_get_fence_addr()
8655 (nic_index >> 1) * NIC_MACRO_QMAN_OFFSET + in gaudi_get_fence_addr()
8668 nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_3) >> 2; in gaudi_get_fence_addr()
8670 (nic_index >> 1) * NIC_MACRO_QMAN_OFFSET + in gaudi_get_fence_addr()
8674 return -EINVAL; in gaudi_get_fence_addr()
8696 (mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_id * 4) - in gaudi_add_mon_pkts()
8704 (mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_id * 4) - in gaudi_add_mon_pkts()
8715 (mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_id * 4) - in gaudi_add_mon_pkts()
8718 size += gaudi_add_mon_msg_short(buf + size, 1, msg_addr_offset); in gaudi_add_mon_pkts()
8726 struct hl_cb *cb = (struct hl_cb *) prop->data; in gaudi_gen_wait_cb()
8727 void *buf = cb->kernel_address; in gaudi_gen_wait_cb()
8729 u32 size = prop->size; in gaudi_gen_wait_cb()
8731 if (gaudi_get_fence_addr(hdev, prop->q_idx, &fence_addr)) { in gaudi_gen_wait_cb()
8732 dev_crit(hdev->dev, "wrong queue id %d for wait packet\n", in gaudi_gen_wait_cb()
8733 prop->q_idx); in gaudi_gen_wait_cb()
8737 size += gaudi_add_mon_pkts(buf + size, prop->mon_id, fence_addr); in gaudi_gen_wait_cb()
8738 size += gaudi_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, in gaudi_gen_wait_cb()
8739 prop->sob_mask, prop->sob_val, prop->mon_id); in gaudi_gen_wait_cb()
8749 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, in gaudi_reset_sob()
8750 hw_sob->sob_id); in gaudi_reset_sob()
8753 hw_sob->sob_id * 4, 0); in gaudi_reset_sob()
8755 kref_init(&hw_sob->kref); in gaudi_reset_sob()
8768 return -EPERM; in gaudi_get_hw_block_id()
8775 return -EPERM; in gaudi_block_mmap()
8781 &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi_enable_events_from_fw()
8782 u32 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ? in gaudi_enable_events_from_fw()
8784 le32_to_cpu(dyn_regs->gic_host_ints_irq); in gaudi_enable_events_from_fw()
8792 return -EINVAL; in gaudi_ack_mmu_page_fault_or_access_error()
8808 default: return -EINVAL; in gaudi_map_pll_idx_to_fw_idx()
8824 reg_value -= lower_32_bits(CFG_BASE); in gaudi_add_sync_to_engine_map_entry()
8829 return -ENOMEM; in gaudi_add_sync_to_engine_map_entry()
8830 entry->engine_type = engine_type; in gaudi_add_sync_to_engine_map_entry()
8831 entry->engine_id = engine_id; in gaudi_add_sync_to_engine_map_entry()
8832 entry->sync_id = reg_value; in gaudi_add_sync_to_engine_map_entry()
8833 hash_add(map->tb, &entry->node, reg_value); in gaudi_add_sync_to_engine_map_entry()
8841 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; in gaudi_gen_sync_to_engine_map()
8846 for (i = 0; i < sds->props[SP_NUM_OF_TPC_ENGINES]; ++i) { in gaudi_gen_sync_to_engine_map()
8848 reg_value = RREG32(sds->props[SP_TPC0_CFG_SO] + in gaudi_gen_sync_to_engine_map()
8849 sds->props[SP_NEXT_TPC] * i); in gaudi_gen_sync_to_engine_map()
8858 for (i = 0; i < sds->props[SP_NUM_OF_MME_ENGINES]; ++i) { in gaudi_gen_sync_to_engine_map()
8859 for (j = 0; j < sds->props[SP_SUB_MME_ENG_NUM]; ++j) { in gaudi_gen_sync_to_engine_map()
8861 reg_value = RREG32(sds->props[SP_MME_CFG_SO] + in gaudi_gen_sync_to_engine_map()
8862 sds->props[SP_NEXT_MME] * i + in gaudi_gen_sync_to_engine_map()
8867 i * sds->props[SP_SUB_MME_ENG_NUM] + j); in gaudi_gen_sync_to_engine_map()
8874 for (i = 0; i < sds->props[SP_NUM_OF_DMA_ENGINES]; ++i) { in gaudi_gen_sync_to_engine_map()
8875 reg_value = RREG32(sds->props[SP_DMA_CFG_SO] + in gaudi_gen_sync_to_engine_map()
8876 sds->props[SP_DMA_QUEUES_OFFSET] * i); in gaudi_gen_sync_to_engine_map()
8895 mon->status); in gaudi_monitor_valid()
8908 mon->arm_data); in gaudi_fill_sobs_from_mon()
8910 mon->arm_data); in gaudi_fill_sobs_from_mon()
8912 for (i = 0, offset = 0; mask && offset < MONITOR_SOB_STRING_SIZE - in gaudi_fill_sobs_from_mon()
8913 max_write; mask >>= 1, i++) { in gaudi_fill_sobs_from_mon()
8914 if (!(mask & 1)) { in gaudi_fill_sobs_from_mon()
8944 mon->id, name, in gaudi_print_single_monitor()
8946 mon->arm_data), in gaudi_print_single_monitor()
8951 mon->arm_data)), in gaudi_print_single_monitor()
8953 mon->arm_data), in gaudi_print_single_monitor()
8954 mon->wr_data, in gaudi_print_single_monitor()
8955 (((u64)mon->wr_addr_high) << 32) | mon->wr_addr_low, in gaudi_print_single_monitor()
8960 mon->status)), in gaudi_print_single_monitor()
8970 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; in gaudi_print_fences_single_engine()
8971 int rc = -ENOMEM, i; in gaudi_print_fences_single_engine()
8974 statuses = kcalloc(sds->props[SP_ENGINE_NUM_OF_QUEUES], in gaudi_print_fences_single_engine()
8979 fences = kcalloc(sds->props[SP_ENGINE_NUM_OF_FENCES] * in gaudi_print_fences_single_engine()
8980 sds->props[SP_ENGINE_NUM_OF_QUEUES], in gaudi_print_fences_single_engine()
8985 for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_FENCES]; ++i) in gaudi_print_fences_single_engine()
8988 for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_FENCES] * in gaudi_print_fences_single_engine()
8989 sds->props[SP_ENGINE_NUM_OF_QUEUES]; ++i) in gaudi_print_fences_single_engine()
8993 for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_QUEUES]; ++i) { in gaudi_print_fences_single_engine()
9006 (i + fence_id * sds->props[SP_ENGINE_NUM_OF_QUEUES]); in gaudi_print_fences_single_engine()
9007 fence_rdata = fence_cnt - sds->props[SP_FENCE0_CNT_OFFSET] + in gaudi_print_fences_single_engine()
9008 sds->props[SP_FENCE0_RDATA_OFFSET]; in gaudi_print_fences_single_engine()
9044 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; in gaudi_state_dump_init()
9048 hash_add(sds->so_id_to_str_tb, in gaudi_state_dump_init()
9053 hash_add(sds->monitor_id_to_str_tb, in gaudi_state_dump_init()
9057 sds->props = gaudi_state_dump_specs_props; in gaudi_state_dump_init()
9059 sds->sync_namager_names = gaudi_sync_manager_names; in gaudi_state_dump_init()
9061 sds->funcs = gaudi_state_dump_funcs; in gaudi_state_dump_init()
9088 cpucp_info = &hdev->asic_prop.cpucp_info; in infineon_ver_show()
9090 return sprintf(buf, "%#04x\n", le32_to_cpu(cpucp_info->infineon_version)); in infineon_ver_show()
9104 dev_vrm_attr_grp->attrs = gaudi_vrm_dev_attrs; in gaudi_add_device_attr()
9210 * gaudi_set_asic_funcs - set GAUDI function pointers
9217 hdev->asic_funcs = &gaudi_funcs; in gaudi_set_asic_funcs()