1 // SPDX-License-Identifier: GPL-2.0
4  * Copyright 2020-2022 HabanaLabs, Ltd.
32 #define GAUDI2_RESET_WAIT_MSEC			1		/* 1ms */
34 #define GAUDI2_PLDM_RESET_WAIT_MSEC		1000		/* 1s */
40 #define GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC	1000000		/* 1s */
45  * since the code already has built-in support for binning of up to MAX_FAULTY_TPCS TPCs
49 #define MAX_CLUSTER_BINNING_FAULTY_TPCS		1
50 #define MAX_FAULTY_XBARS			1
51 #define MAX_FAULTY_EDMAS			1
52 #define MAX_FAULTY_DECODERS			1
120 #define MMU_RANGE_INV_ASID_EN_SHIFT		1
126 #define GAUDI2_PMMU_SPI_SEI_ENABLE_MASK		GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 2, 0)
127 #define GAUDI2_HMMU_SPI_SEI_ENABLE_MASK		GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 1, 0)
131 #define GAUDI2_VDEC_MSIX_ENTRIES		(GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \
132 							GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 1)
134 #define ENGINE_ID_DCORE_OFFSET (GAUDI2_DCORE1_ENGINE_ID_EDMA_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)
164 /* HW scrambles only bits 0-25 */
283 		{RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF1_RTR_CTRL_BASE,
285 		{RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF0_RTR_CTRL_BASE,
291 		{RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE,
293 		{RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE,
299 		{RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
303 		{RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
307 		{RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
311 		{RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
315 		{RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
319 		{RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
323 		{RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
327 		{RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
438 	[1] = HBM_ID1,
739 	"AXI SPLIT SEI Status"
788 	"PQ AXI HBW error",
789 	"CQ AXI HBW error",
790 	"CP AXI HBW error",
793 	"CP AXI LBW error",
797 	"FENCE 1 inc over max value and clipped",
801 	"FENCE 1 dec under min value and clipped",
810 	"CQ AXI HBW error",
811 	"CP AXI HBW error",
814 	"CP AXI LBW error",
818 	"FENCE 1 inc over max value and clipped",
822 	"FENCE 1 dec under min value and clipped",
839 	"MSG AXI LBW returned with error"
904 	"wap sei (wbc axi err)",
937 	"TRANSPOSE ENGINE DESC FIFO OVERFLOW",
948 	"TRANSPOSE ENGINE DESC FIFO OVERFLOW",
961 	{"armed monitor write got BRESP (SLVERR or DECERR)", "AXI id"},
978 	"AXI drain HBW",
979 	"AXI drain LBW",
1950 	"gaudi2 vdec 1_0", "gaudi2 vdec 1_0 abnormal",
1951 	"gaudi2 vdec 1_1", "gaudi2 vdec 1_1 abnormal",
1995 static const u32 gaudi2_tpc_initiator_hbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = {
2003 static const u32 gaudi2_tpc_initiator_lbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = {
2114  * and read global errors. Most HW blocks are addressable and those who aren't (N/A)-
2137 	{HBM_MC_SPI_THR_ENG_MASK, "temperature-based throttling engaged"},
2138 	{HBM_MC_SPI_THR_DIS_ENG_MASK, "temperature-based throttling disengaged"},
2161 	{"page fault", 1},		/* INTERRUPT_CLR[1] */
2162 	{"page access", 1},		/* INTERRUPT_CLR[1] */
2165 	{"mmu rei0", -1},		/* no clear register bit */
2166 	{"mmu rei1", -1},		/* no clear register bit */
2167 	{"stlb rei0", -1},		/* no clear register bit */
2168 	{"stlb rei1", -1},		/* no clear register bit */
2237 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_iterate_tpcs()
2242 	ctx->rc = 0;  in gaudi2_iterate_tpcs()
2248 			if (!(prop->tpc_enabled_mask & BIT(tpc_seq)))  in gaudi2_iterate_tpcs()
2253 			ctx->fn(hdev, dcore, inst, offset, ctx);  in gaudi2_iterate_tpcs()
2254 			if (ctx->rc) {  in gaudi2_iterate_tpcs()
2255 				dev_err(hdev->dev, "TPC iterator failed for DCORE%d TPC%d\n",  in gaudi2_iterate_tpcs()
2262 	if (!(prop->tpc_enabled_mask & BIT(TPC_ID_DCORE0_TPC6)))  in gaudi2_iterate_tpcs()
2266 	offset = DCORE_TPC_OFFSET * (NUM_DCORE0_TPC - 1);  in gaudi2_iterate_tpcs()
2267 	ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx);  in gaudi2_iterate_tpcs()
2268 	if (ctx->rc)  in gaudi2_iterate_tpcs()
2269 		dev_err(hdev->dev, "TPC iterator failed for DCORE0 TPC6\n");  in gaudi2_iterate_tpcs()
2282 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in set_number_of_functional_hbms()
2283 	u8 faulty_hbms = hweight64(hdev->dram_binning);  in set_number_of_functional_hbms()
2287 		dev_dbg(hdev->dev, "All HBM are in use (no binning)\n");  in set_number_of_functional_hbms()
2288 		prop->num_functional_hbms = GAUDI2_HBM_NUM;  in set_number_of_functional_hbms()
2299 		dev_err(hdev->dev,  in set_number_of_functional_hbms()
2301 			MAX_FAULTY_HBMS, hdev->dram_binning);  in set_number_of_functional_hbms()
2302 		return -EINVAL;  in set_number_of_functional_hbms()
2307 	 * GAUDI2_HBM_NUM - 1.  in set_number_of_functional_hbms()
2309 	prop->num_functional_hbms = GAUDI2_HBM_NUM - faulty_hbms;  in set_number_of_functional_hbms()
2329 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_set_dram_properties()
2336 		return -EINVAL;  in gaudi2_set_dram_properties()
2343 	basic_hbm_page_size = prop->num_functional_hbms * SZ_8M;  in gaudi2_set_dram_properties()
2344 	prop->dram_page_size = GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR * basic_hbm_page_size;  in gaudi2_set_dram_properties()
2345 	prop->device_mem_alloc_default_page_size = prop->dram_page_size;  in gaudi2_set_dram_properties()
2346 	prop->dram_size = prop->num_functional_hbms * SZ_16G;  in gaudi2_set_dram_properties()
2347 	prop->dram_base_address = DRAM_PHYS_BASE;  in gaudi2_set_dram_properties()
2348 	prop->dram_end_address = prop->dram_base_address + prop->dram_size;  in gaudi2_set_dram_properties()
2349 	prop->dram_supports_virtual_memory = true;  in gaudi2_set_dram_properties()
2351 	prop->dram_user_base_address = DRAM_PHYS_BASE + prop->dram_page_size;  in gaudi2_set_dram_properties()
2352 	prop->dram_hints_align_mask = ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK;  in gaudi2_set_dram_properties()
2353 	prop->hints_dram_reserved_va_range.start_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_START;  in gaudi2_set_dram_properties()
2354 	prop->hints_dram_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_END;  in gaudi2_set_dram_properties()
2362 	 * 1. partition the virtual address space to DRAM-page (whole) pages.  in gaudi2_set_dram_properties()
2364 	 * 2. limit the amount of virtual address space we got from 1 above to  in gaudi2_set_dram_properties()
2373 	prop->dmmu.start_addr = prop->dram_base_address +  in gaudi2_set_dram_properties()
2374 			(prop->dram_page_size *  in gaudi2_set_dram_properties()
2375 				DIV_ROUND_UP_SECTOR_T(prop->dram_size, prop->dram_page_size));  in gaudi2_set_dram_properties()
2376 	prop->dmmu.end_addr = prop->dmmu.start_addr + prop->dram_page_size *  in gaudi2_set_dram_properties()
2377 			div_u64((VA_HBM_SPACE_END - prop->dmmu.start_addr), prop->dmmu.page_size);  in gaudi2_set_dram_properties()
2382 	hbm_drv_base_offset = roundup(CPU_FW_IMAGE_SIZE, prop->num_functional_hbms * SZ_8M);  in gaudi2_set_dram_properties()
2391 	prop->mmu_pgt_addr = DRAM_PHYS_BASE + hbm_drv_base_offset +  in gaudi2_set_dram_properties()
2392 				((prop->dram_page_size - hbm_drv_base_offset) -  in gaudi2_set_dram_properties()
2396 	edma_pq_base_addr = prop->mmu_pgt_addr + HMMU_PAGE_TABLES_SIZE;  in gaudi2_set_dram_properties()
2400 			prop->hw_queues_props[i].q_dram_bd_address = edma_pq_base_addr +  in gaudi2_set_dram_properties()
2411 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_set_fixed_properties()
2416 	prop->max_queues = GAUDI2_QUEUE_ID_SIZE;  in gaudi2_set_fixed_properties()
2417 	prop->hw_queues_props = kcalloc(prop->max_queues, sizeof(struct hw_queue_properties),  in gaudi2_set_fixed_properties()
2420 	if (!prop->hw_queues_props)  in gaudi2_set_fixed_properties()
2421 		return -ENOMEM;  in gaudi2_set_fixed_properties()
2423 	q_props = prop->hw_queues_props;  in gaudi2_set_fixed_properties()
2432 			q_props[i].supports_sync_stream = 1;  in gaudi2_set_fixed_properties()
2439 			q_props[i].dram_bd = 1;  in gaudi2_set_fixed_properties()
2443 	q_props[GAUDI2_QUEUE_ID_CPU_PQ].driver_only = 1;  in gaudi2_set_fixed_properties()
2446 	prop->cache_line_size = DEVICE_CACHE_LINE_SIZE;  in gaudi2_set_fixed_properties()
2447 	prop->cfg_base_address = CFG_BASE;  in gaudi2_set_fixed_properties()
2448 	prop->device_dma_offset_for_host_access = HOST_PHYS_BASE_0;  in gaudi2_set_fixed_properties()
2449 	prop->host_base_address = HOST_PHYS_BASE_0;  in gaudi2_set_fixed_properties()
2450 	prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE_0;  in gaudi2_set_fixed_properties()
2451 	prop->max_pending_cs = GAUDI2_MAX_PENDING_CS;  in gaudi2_set_fixed_properties()
2452 	prop->completion_queues_count = GAUDI2_RESERVED_CQ_NUMBER;  in gaudi2_set_fixed_properties()
2453 	prop->user_dec_intr_count = NUMBER_OF_DEC;  in gaudi2_set_fixed_properties()
2454 	prop->user_interrupt_count = GAUDI2_IRQ_NUM_USER_LAST - GAUDI2_IRQ_NUM_USER_FIRST + 1;  in gaudi2_set_fixed_properties()
2455 	prop->completion_mode = HL_COMPLETION_MODE_CS;  in gaudi2_set_fixed_properties()
2456 	prop->sync_stream_first_sob = GAUDI2_RESERVED_SOB_NUMBER;  in gaudi2_set_fixed_properties()
2457 	prop->sync_stream_first_mon = GAUDI2_RESERVED_MON_NUMBER;  in gaudi2_set_fixed_properties()
2459 	prop->sram_base_address = SRAM_BASE_ADDR;  in gaudi2_set_fixed_properties()
2460 	prop->sram_size = SRAM_SIZE;  in gaudi2_set_fixed_properties()
2461 	prop->sram_end_address = prop->sram_base_address + prop->sram_size;  in gaudi2_set_fixed_properties()
2462 	prop->sram_user_base_address = prop->sram_base_address + SRAM_USER_BASE_OFFSET;  in gaudi2_set_fixed_properties()
2464 	prop->hints_range_reservation = true;  in gaudi2_set_fixed_properties()
2466 	prop->rotator_enabled_mask = BIT(NUM_OF_ROT) - 1;  in gaudi2_set_fixed_properties()
2468 	prop->max_asid = 2;  in gaudi2_set_fixed_properties()
2470 	prop->dmmu.pgt_size = HMMU_PAGE_TABLES_SIZE;  in gaudi2_set_fixed_properties()
2471 	prop->mmu_pte_size = HL_PTE_SIZE;  in gaudi2_set_fixed_properties()
2473 	prop->dmmu.hop_shifts[MMU_HOP0] = DHOP0_SHIFT;  in gaudi2_set_fixed_properties()
2474 	prop->dmmu.hop_shifts[MMU_HOP1] = DHOP1_SHIFT;  in gaudi2_set_fixed_properties()
2475 	prop->dmmu.hop_shifts[MMU_HOP2] = DHOP2_SHIFT;  in gaudi2_set_fixed_properties()
2476 	prop->dmmu.hop_shifts[MMU_HOP3] = DHOP3_SHIFT;  in gaudi2_set_fixed_properties()
2477 	prop->dmmu.hop_masks[MMU_HOP0] = DHOP0_MASK;  in gaudi2_set_fixed_properties()
2478 	prop->dmmu.hop_masks[MMU_HOP1] = DHOP1_MASK;  in gaudi2_set_fixed_properties()
2479 	prop->dmmu.hop_masks[MMU_HOP2] = DHOP2_MASK;  in gaudi2_set_fixed_properties()
2480 	prop->dmmu.hop_masks[MMU_HOP3] = DHOP3_MASK;  in gaudi2_set_fixed_properties()
2481 	prop->dmmu.page_size = PAGE_SIZE_1GB;  in gaudi2_set_fixed_properties()
2482 	prop->dmmu.num_hops = MMU_ARCH_4_HOPS;  in gaudi2_set_fixed_properties()
2483 	prop->dmmu.last_mask = LAST_MASK;  in gaudi2_set_fixed_properties()
2484 	prop->dmmu.host_resident = 0;  in gaudi2_set_fixed_properties()
2485 	prop->dmmu.hop_table_size = HOP_TABLE_SIZE_512_PTE;  in gaudi2_set_fixed_properties()
2486 	prop->dmmu.hop0_tables_total_size = HOP_TABLE_SIZE_512_PTE * prop->max_asid;  in gaudi2_set_fixed_properties()
2492 	rc = hdev->asic_funcs->set_dram_properties(hdev);  in gaudi2_set_fixed_properties()
2496 	prop->mmu_pgt_size = PMMU_PAGE_TABLES_SIZE;  in gaudi2_set_fixed_properties()
2498 	prop->pmmu.pgt_size = prop->mmu_pgt_size;  in gaudi2_set_fixed_properties()
2499 	hdev->pmmu_huge_range = true;  in gaudi2_set_fixed_properties()
2500 	prop->pmmu.host_resident = 1;  in gaudi2_set_fixed_properties()
2501 	prop->pmmu.num_hops = MMU_ARCH_6_HOPS;  in gaudi2_set_fixed_properties()
2502 	prop->pmmu.last_mask = LAST_MASK;  in gaudi2_set_fixed_properties()
2503 	prop->pmmu.hop_table_size = HOP_TABLE_SIZE_512_PTE;  in gaudi2_set_fixed_properties()
2504 	prop->pmmu.hop0_tables_total_size = HOP_TABLE_SIZE_512_PTE * prop->max_asid;  in gaudi2_set_fixed_properties()
2506 	prop->hints_host_reserved_va_range.start_addr = RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START;  in gaudi2_set_fixed_properties()
2507 	prop->hints_host_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HOST_END;  in gaudi2_set_fixed_properties()
2508 	prop->hints_host_hpage_reserved_va_range.start_addr =  in gaudi2_set_fixed_properties()
2510 	prop->hints_host_hpage_reserved_va_range.end_addr =  in gaudi2_set_fixed_properties()
2514 		prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_64K;  in gaudi2_set_fixed_properties()
2515 		prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_64K;  in gaudi2_set_fixed_properties()
2516 		prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_64K;  in gaudi2_set_fixed_properties()
2517 		prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_64K;  in gaudi2_set_fixed_properties()
2518 		prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_64K;  in gaudi2_set_fixed_properties()
2519 		prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_64K;  in gaudi2_set_fixed_properties()
2520 		prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_64K;  in gaudi2_set_fixed_properties()
2521 		prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_64K;  in gaudi2_set_fixed_properties()
2522 		prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_64K;  in gaudi2_set_fixed_properties()
2523 		prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_64K;  in gaudi2_set_fixed_properties()
2524 		prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_64K;  in gaudi2_set_fixed_properties()
2525 		prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_64K;  in gaudi2_set_fixed_properties()
2526 		prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START;  in gaudi2_set_fixed_properties()
2527 		prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END;  in gaudi2_set_fixed_properties()
2528 		prop->pmmu.page_size = PAGE_SIZE_64KB;  in gaudi2_set_fixed_properties()
2531 		memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));  in gaudi2_set_fixed_properties()
2532 		prop->pmmu_huge.page_size = PAGE_SIZE_16MB;  in gaudi2_set_fixed_properties()
2533 		prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START;  in gaudi2_set_fixed_properties()
2534 		prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END;  in gaudi2_set_fixed_properties()
2536 		prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_4K;  in gaudi2_set_fixed_properties()
2537 		prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_4K;  in gaudi2_set_fixed_properties()
2538 		prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_4K;  in gaudi2_set_fixed_properties()
2539 		prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_4K;  in gaudi2_set_fixed_properties()
2540 		prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_4K;  in gaudi2_set_fixed_properties()
2541 		prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_4K;  in gaudi2_set_fixed_properties()
2542 		prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_4K;  in gaudi2_set_fixed_properties()
2543 		prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_4K;  in gaudi2_set_fixed_properties()
2544 		prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_4K;  in gaudi2_set_fixed_properties()
2545 		prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_4K;  in gaudi2_set_fixed_properties()
2546 		prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_4K;  in gaudi2_set_fixed_properties()
2547 		prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_4K;  in gaudi2_set_fixed_properties()
2548 		prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START;  in gaudi2_set_fixed_properties()
2549 		prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END;  in gaudi2_set_fixed_properties()
2550 		prop->pmmu.page_size = PAGE_SIZE_4KB;  in gaudi2_set_fixed_properties()
2553 		memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));  in gaudi2_set_fixed_properties()
2554 		prop->pmmu_huge.page_size = PAGE_SIZE_2MB;  in gaudi2_set_fixed_properties()
2555 		prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START;  in gaudi2_set_fixed_properties()
2556 		prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END;  in gaudi2_set_fixed_properties()
2559 	prop->max_num_of_engines = GAUDI2_ENGINE_ID_SIZE;  in gaudi2_set_fixed_properties()
2560 	prop->num_engine_cores = CPU_ID_MAX;  in gaudi2_set_fixed_properties()
2561 	prop->cfg_size = CFG_SIZE;  in gaudi2_set_fixed_properties()
2562 	prop->num_of_events = GAUDI2_EVENT_SIZE;  in gaudi2_set_fixed_properties()
2564 	prop->supports_engine_modes = true;  in gaudi2_set_fixed_properties()
2566 	prop->dc_power_default = DC_POWER_DEFAULT;  in gaudi2_set_fixed_properties()
2568 	prop->cb_pool_cb_cnt = GAUDI2_CB_POOL_CB_CNT;  in gaudi2_set_fixed_properties()
2569 	prop->cb_pool_cb_size = GAUDI2_CB_POOL_CB_SIZE;  in gaudi2_set_fixed_properties()
2570 	prop->pcie_dbi_base_address = CFG_BASE + mmPCIE_DBI_BASE;  in gaudi2_set_fixed_properties()
2571 	prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;  in gaudi2_set_fixed_properties()
2573 	strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN);  in gaudi2_set_fixed_properties()
2575 	prop->mme_master_slave_mode = 1;  in gaudi2_set_fixed_properties()
2577 	prop->first_available_user_sob[0] = GAUDI2_RESERVED_SOB_NUMBER +  in gaudi2_set_fixed_properties()
2580 	prop->first_available_user_mon[0] = GAUDI2_RESERVED_MON_NUMBER +  in gaudi2_set_fixed_properties()
2583 	prop->first_available_user_interrupt = GAUDI2_IRQ_NUM_USER_FIRST;  in gaudi2_set_fixed_properties()
2584 	prop->tpc_interrupt_id = GAUDI2_IRQ_NUM_TPC_ASSERT;  in gaudi2_set_fixed_properties()
2585 	prop->eq_interrupt_id = GAUDI2_IRQ_NUM_EVENT_QUEUE;  in gaudi2_set_fixed_properties()
2587 	prop->first_available_cq[0] = GAUDI2_RESERVED_CQ_NUMBER;  in gaudi2_set_fixed_properties()
2589 	prop->fw_cpu_boot_dev_sts0_valid = false;  in gaudi2_set_fixed_properties()
2590 	prop->fw_cpu_boot_dev_sts1_valid = false;  in gaudi2_set_fixed_properties()
2591 	prop->hard_reset_done_by_fw = false;  in gaudi2_set_fixed_properties()
2592 	prop->gic_interrupts_enable = true;  in gaudi2_set_fixed_properties()
2594 	prop->server_type = HL_SERVER_TYPE_UNKNOWN;  in gaudi2_set_fixed_properties()
2596 	prop->max_dec = NUMBER_OF_DEC;  in gaudi2_set_fixed_properties()
2598 	prop->clk_pll_index = HL_GAUDI2_MME_PLL;  in gaudi2_set_fixed_properties()
2600 	prop->dma_mask = 64;  in gaudi2_set_fixed_properties()
2602 	prop->hbw_flush_reg = mmPCIE_WRAP_SPECIAL_GLBL_SPARE_0;  in gaudi2_set_fixed_properties()
2604 	prop->supports_advanced_cpucp_rc = true;  in gaudi2_set_fixed_properties()
2609 	kfree(prop->hw_queues_props);  in gaudi2_set_fixed_properties()
2623 	hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + (CFG_BASE - STM_FLASH_BASE_ADDR);  in gaudi2_pci_bars_map()
2630 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_set_hbm_bar_base()
2635 	if ((gaudi2) && (gaudi2->dram_bar_cur_addr == addr))  in gaudi2_set_hbm_bar_base()
2638 	if (hdev->asic_prop.iatu_done_by_fw)  in gaudi2_set_hbm_bar_base()
2641 	/* Inbound Region 2 - Bar 4 - Point to DRAM */  in gaudi2_set_hbm_bar_base()
2650 		old_addr = gaudi2->dram_bar_cur_addr;  in gaudi2_set_hbm_bar_base()
2651 		gaudi2->dram_bar_cur_addr = addr;  in gaudi2_set_hbm_bar_base()
2664 	if (hdev->asic_prop.iatu_done_by_fw)  in gaudi2_init_iatu()
2667 	/* Temporary inbound Region 0 - Bar 0 - Point to CFG  in gaudi2_init_iatu()
2674 	inbound_region.addr = STM_FLASH_BASE_ADDR - STM_FLASH_ALIGNED_OFF;  in gaudi2_init_iatu()
2683 	hdev->pcie_bar_phys[SRAM_CFG_BAR_ID] = (u64)bar_addr_high << 32 | bar_addr_low;  in gaudi2_init_iatu()
2685 	/* Inbound Region 0 - Bar 0 - Point to CFG */  in gaudi2_init_iatu()
2695 	/* Inbound Region 1 - Bar 0 - Point to BAR0_RESERVED + SRAM */  in gaudi2_init_iatu()
2701 	rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region);  in gaudi2_init_iatu()
2705 	/* Inbound Region 2 - Bar 4 - Point to DRAM */  in gaudi2_init_iatu()
2713 	/* Outbound Region 0 - Point to Host */  in gaudi2_init_iatu()
2728 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_tpc_binning_init_prop()
2734 	if (hweight64(hdev->tpc_binning) > MAX_CLUSTER_BINNING_FAULTY_TPCS) {  in gaudi2_tpc_binning_init_prop()
2735 		dev_err(hdev->dev, "TPC binning is supported for max of %d faulty TPCs, provided mask 0x%llx\n",  in gaudi2_tpc_binning_init_prop()
2737 					hdev->tpc_binning);  in gaudi2_tpc_binning_init_prop()
2738 		return -EINVAL;  in gaudi2_tpc_binning_init_prop()
2741 	prop->tpc_binning_mask = hdev->tpc_binning;  in gaudi2_tpc_binning_init_prop()
2742 	prop->tpc_enabled_mask = GAUDI2_TPC_FULL_MASK;  in gaudi2_tpc_binning_init_prop()
2749 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_set_tpc_binning_masks()
2750 	struct hw_queue_properties *q_props = prop->hw_queues_props;  in gaudi2_set_tpc_binning_masks()
2759 	tpc_binning_mask = prop->tpc_binning_mask;  in gaudi2_set_tpc_binning_masks()
2779 		 * Coverity complains about possible out-of-bound access in  in gaudi2_set_tpc_binning_masks()
2783 			dev_err(hdev->dev,  in gaudi2_set_tpc_binning_masks()
2786 			return -EINVAL;  in gaudi2_set_tpc_binning_masks()
2791 		clear_bit(subst_seq, (unsigned long *)&prop->tpc_enabled_mask);  in gaudi2_set_tpc_binning_masks()
2794 		q_props[qid_base].binned = 1;  in gaudi2_set_tpc_binning_masks()
2795 		q_props[qid_base + 1].binned = 1;  in gaudi2_set_tpc_binning_masks()
2796 		q_props[qid_base + 2].binned = 1;  in gaudi2_set_tpc_binning_masks()
2797 		q_props[qid_base + 3].binned = 1;  in gaudi2_set_tpc_binning_masks()
2807 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_set_dec_binning_masks()
2810 	num_faulty = hweight32(hdev->decoder_binning);  in gaudi2_set_dec_binning_masks()
2817 …dev_err(hdev->dev, "decoder binning is supported for max of single faulty decoder, provided mask 0…  in gaudi2_set_dec_binning_masks()
2818 						hdev->decoder_binning);  in gaudi2_set_dec_binning_masks()
2819 		return -EINVAL;  in gaudi2_set_dec_binning_masks()
2822 	prop->decoder_binning_mask = (hdev->decoder_binning & GAUDI2_DECODER_FULL_MASK);  in gaudi2_set_dec_binning_masks()
2824 	if (prop->decoder_binning_mask)  in gaudi2_set_dec_binning_masks()
2825 		prop->decoder_enabled_mask = (GAUDI2_DECODER_FULL_MASK & ~BIT(DEC_ID_PCIE_VDEC1));  in gaudi2_set_dec_binning_masks()
2827 		prop->decoder_enabled_mask = GAUDI2_DECODER_FULL_MASK;  in gaudi2_set_dec_binning_masks()
2834 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_set_dram_binning_masks()
2837 	if (!hdev->dram_binning) {  in gaudi2_set_dram_binning_masks()
2838 		prop->dram_binning_mask = 0;  in gaudi2_set_dram_binning_masks()
2839 		prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK;  in gaudi2_set_dram_binning_masks()
2844 	prop->faulty_dram_cluster_map |= hdev->dram_binning;  in gaudi2_set_dram_binning_masks()
2845 	prop->dram_binning_mask = hdev->dram_binning;  in gaudi2_set_dram_binning_masks()
2846 	prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK & ~BIT(HBM_ID5);  in gaudi2_set_dram_binning_masks()
2851 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_set_edma_binning_masks()
2855 	num_faulty = hweight32(hdev->edma_binning);  in gaudi2_set_edma_binning_masks()
2862 		dev_err(hdev->dev,  in gaudi2_set_edma_binning_masks()
2864 			hdev->edma_binning);  in gaudi2_set_edma_binning_masks()
2865 		return -EINVAL;  in gaudi2_set_edma_binning_masks()
2868 	if (!hdev->edma_binning) {  in gaudi2_set_edma_binning_masks()
2869 		prop->edma_binning_mask = 0;  in gaudi2_set_edma_binning_masks()
2870 		prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK;  in gaudi2_set_edma_binning_masks()
2874 	seq = __ffs((unsigned long)hdev->edma_binning);  in gaudi2_set_edma_binning_masks()
2877 	prop->faulty_dram_cluster_map |= BIT(edma_to_hbm_cluster[seq]);  in gaudi2_set_edma_binning_masks()
2878 	prop->edma_binning_mask = hdev->edma_binning;  in gaudi2_set_edma_binning_masks()
2879 	prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK & ~BIT(EDMA_ID_DCORE3_INSTANCE1);  in gaudi2_set_edma_binning_masks()
2882 	q_props = prop->hw_queues_props;  in gaudi2_set_edma_binning_masks()
2883 	q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0].binned = 1;  in gaudi2_set_edma_binning_masks()
2884 	q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1].binned = 1;  in gaudi2_set_edma_binning_masks()
2885 	q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2].binned = 1;  in gaudi2_set_edma_binning_masks()
2886 	q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3].binned = 1;  in gaudi2_set_edma_binning_masks()
2893 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_set_xbar_edge_enable_mask()
2898 		prop->xbar_edge_enabled_mask = GAUDI2_XBAR_EDGE_FULL_MASK;  in gaudi2_set_xbar_edge_enable_mask()
2913 		dev_err(hdev->dev, "we cannot have more than %d faulty XBAR EDGE\n",  in gaudi2_set_xbar_edge_enable_mask()
2915 		return -EINVAL;  in gaudi2_set_xbar_edge_enable_mask()
2921 	prop->faulty_dram_cluster_map |= BIT(xbar_edge_to_hbm_cluster[seq]);  in gaudi2_set_xbar_edge_enable_mask()
2922 	prop->xbar_edge_enabled_mask = (~xbar_edge_iso_mask) & GAUDI2_XBAR_EDGE_FULL_MASK;  in gaudi2_set_xbar_edge_enable_mask()
2934 	 * If more than single cluster is faulty- the chip is unusable  in gaudi2_set_cluster_binning_masks_common()
2936 	hdev->asic_prop.faulty_dram_cluster_map = 0;  in gaudi2_set_cluster_binning_masks_common()
2950 	hdev->asic_prop.hmmu_hif_enabled_mask = GAUDI2_HIF_HMMU_FULL_MASK;  in gaudi2_set_cluster_binning_masks_common()
2957 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_set_cluster_binning_masks()
2960 	rc = gaudi2_set_cluster_binning_masks_common(hdev, prop->cpucp_info.xbar_binning_mask);  in gaudi2_set_cluster_binning_masks()
2965 	if (prop->faulty_dram_cluster_map) {  in gaudi2_set_cluster_binning_masks()
2966 		u8 cluster_seq = __ffs((unsigned long)prop->faulty_dram_cluster_map);  in gaudi2_set_cluster_binning_masks()
2968 		prop->hmmu_hif_enabled_mask = cluster_hmmu_hif_enabled_mask[cluster_seq];  in gaudi2_set_cluster_binning_masks()
2995 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_cpucp_info_get()
2996 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_cpucp_info_get()
3001 	if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))  in gaudi2_cpucp_info_get()
3007 	if (hdev->reset_info.in_compute_reset)  in gaudi2_cpucp_info_get()
3015 	dram_size = le64_to_cpu(prop->cpucp_info.dram_size);  in gaudi2_cpucp_info_get()
3019 		if ((dram_size != ((GAUDI2_HBM_NUM - 1) * SZ_16G)) &&  in gaudi2_cpucp_info_get()
3021 			dev_err(hdev->dev,  in gaudi2_cpucp_info_get()
3023 				dram_size, prop->dram_size);  in gaudi2_cpucp_info_get()
3024 			dram_size = prop->dram_size;  in gaudi2_cpucp_info_get()
3027 		prop->dram_size = dram_size;  in gaudi2_cpucp_info_get()
3028 		prop->dram_end_address = prop->dram_base_address + dram_size;  in gaudi2_cpucp_info_get()
3031 	if (!strlen(prop->cpucp_info.card_name))  in gaudi2_cpucp_info_get()
3032 		strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME,  in gaudi2_cpucp_info_get()
3036 	hdev->dram_binning = prop->cpucp_info.dram_binning_mask;  in gaudi2_cpucp_info_get()
3037 	hdev->edma_binning = prop->cpucp_info.edma_binning_mask;  in gaudi2_cpucp_info_get()
3038 	hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask);  in gaudi2_cpucp_info_get()
3039 	hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask));  in gaudi2_cpucp_info_get()
3041 	dev_dbg(hdev->dev, "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x\n",  in gaudi2_cpucp_info_get()
3042 			hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning,  in gaudi2_cpucp_info_get()
3043 			hdev->decoder_binning);  in gaudi2_cpucp_info_get()
3049 	rc = hdev->asic_funcs->set_dram_properties(hdev);  in gaudi2_cpucp_info_get()
3053 	rc = hdev->asic_funcs->set_binning_masks(hdev);  in gaudi2_cpucp_info_get()
3061 	prop->max_power_default = (u64) max_power;  in gaudi2_cpucp_info_get()
3068 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_fetch_psoc_frequency()
3072 	if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))  in gaudi2_fetch_psoc_frequency()
3079 	hdev->asic_prop.psoc_timestamp_frequency = pll_freq_arr[3];  in gaudi2_fetch_psoc_frequency()
3086 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_mmu_clear_pgt_range()
3087 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_mmu_clear_pgt_range()
3090 	if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK))  in gaudi2_mmu_clear_pgt_range()
3093 	if (prop->dmmu.host_resident)  in gaudi2_mmu_clear_pgt_range()
3096 	rc = gaudi2_memset_device_memory(hdev, prop->mmu_pgt_addr, prop->dmmu.pgt_size, 0);  in gaudi2_mmu_clear_pgt_range()
3098 		dev_err(hdev->dev, "Failed to clear mmu pgt");  in gaudi2_mmu_clear_pgt_range()
3105 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_early_init()
3106 	struct pci_dev *pdev = hdev->pdev;  in gaudi2_early_init()
3118 		dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n",  in gaudi2_early_init()
3120 		rc = -ENODEV;  in gaudi2_early_init()
3126 		dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n",  in gaudi2_early_init()
3128 		rc = -ENODEV;  in gaudi2_early_init()
3132 	prop->dram_pci_bar_size = pci_resource_len(pdev, DRAM_BAR_ID);  in gaudi2_early_init()
3133 	hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID);  in gaudi2_early_init()
3138 	if (hdev->pldm)  in gaudi2_early_init()
3139 		hdev->asic_prop.iatu_done_by_fw = false;  in gaudi2_early_init()
3141 		hdev->asic_prop.iatu_done_by_fw = true;  in gaudi2_early_init()
3148 	 * version to determine whether we run with a security-enabled firmware  in gaudi2_early_init()
3152 		if (hdev->reset_on_preboot_fail)  in gaudi2_early_init()
3154 			hdev->asic_funcs->hw_fini(hdev, true, false);  in gaudi2_early_init()
3159 		dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n");  in gaudi2_early_init()
3160 		rc = hdev->asic_funcs->hw_fini(hdev, true, false);  in gaudi2_early_init()
3162 			dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc);  in gaudi2_early_init()
3172 	kfree(hdev->asic_prop.hw_queues_props);  in gaudi2_early_init()
3178 	kfree(hdev->asic_prop.hw_queues_props);  in gaudi2_early_fini()
3206 	struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;  in gaudi2_init_arcs()
3207 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_init_arcs()
3227 				!(hdev->nic_ports_mask & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)))  in gaudi2_init_arcs()
3230 		if (gaudi2_is_arc_tpc_owned(arc_id) && !(gaudi2->tpc_hw_cap_initialized &  in gaudi2_init_arcs()
3231 							BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)))  in gaudi2_init_arcs()
3238 	hdev->asic_prop.engine_core_interrupt_reg_addr =  in gaudi2_init_arcs()
3239 		CFG_BASE + le32_to_cpu(dyn_regs->eng_arc_irq_ctrl);  in gaudi2_init_arcs()
3268 		reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 1);  in gaudi2_scrub_arc_dccm()
3310 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_late_init()
3314 					gaudi2->virt_msix_db_dma_addr);  in gaudi2_late_init()
3320 		dev_err(hdev->dev, "Failed to fetch psoc frequency\n");  in gaudi2_late_init()
3326 		dev_err(hdev->dev, "Failed to clear MMU page tables range\n");  in gaudi2_late_init()
3334 		dev_err(hdev->dev, "Failed to scrub arcs DCCM\n");  in gaudi2_late_init()
3355 	struct user_mapped_block *blocks = gaudi2->mapped_blocks;  in gaudi2_user_mapped_dec_init()
3371 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_user_mapped_blocks_init()
3372 	struct user_mapped_block *blocks = gaudi2->mapped_blocks;  in gaudi2_user_mapped_blocks_init()
3389 	blocks[NUM_ARC_CPUS + 1].address = mmARC_FARM_ARC1_ACP_ENG_BASE;  in gaudi2_user_mapped_blocks_init()
3390 	blocks[NUM_ARC_CPUS + 1].size = HL_BLOCK_SIZE;  in gaudi2_user_mapped_blocks_init()
3429 	for (i = 1; i < NUM_OF_DCORES; ++i) {  in gaudi2_user_mapped_blocks_init()
3430 		blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].size = SM_OBJS_BLOCK_SIZE;  in gaudi2_user_mapped_blocks_init()
3431 		blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].size = HL_BLOCK_SIZE;  in gaudi2_user_mapped_blocks_init()
3433 		blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].address =  in gaudi2_user_mapped_blocks_init()
3436 		blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].address =  in gaudi2_user_mapped_blocks_init()
3447 	/* The device ARC works with 32-bits addresses, and because there is a single HW register  in gaudi2_alloc_cpu_accessible_dma_mem()
3456 			rc = -ENOMEM;  in gaudi2_alloc_cpu_accessible_dma_mem()
3460 		end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1;  in gaudi2_alloc_cpu_accessible_dma_mem()
3466 		dev_err(hdev->dev,  in gaudi2_alloc_cpu_accessible_dma_mem()
3468 		rc = -EFAULT;  in gaudi2_alloc_cpu_accessible_dma_mem()
3472 	hdev->cpu_accessible_dma_mem = virt_addr_arr[i];  in gaudi2_alloc_cpu_accessible_dma_mem()
3473 	hdev->cpu_accessible_dma_address = dma_addr_arr[i];  in gaudi2_alloc_cpu_accessible_dma_mem()
3485 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_set_pci_memory_regions()
3489 	region = &hdev->pci_mem_region[PCI_REGION_CFG];  in gaudi2_set_pci_memory_regions()
3490 	region->region_base = CFG_BASE;  in gaudi2_set_pci_memory_regions()
3491 	region->region_size = CFG_SIZE;  in gaudi2_set_pci_memory_regions()
3492 	region->offset_in_bar = CFG_BASE - STM_FLASH_BASE_ADDR;  in gaudi2_set_pci_memory_regions()
3493 	region->bar_size = CFG_BAR_SIZE;  in gaudi2_set_pci_memory_regions()
3494 	region->bar_id = SRAM_CFG_BAR_ID;  in gaudi2_set_pci_memory_regions()
3495 	region->used = 1;  in gaudi2_set_pci_memory_regions()
3498 	region = &hdev->pci_mem_region[PCI_REGION_SRAM];  in gaudi2_set_pci_memory_regions()
3499 	region->region_base = SRAM_BASE_ADDR;  in gaudi2_set_pci_memory_regions()
3500 	region->region_size = SRAM_SIZE;  in gaudi2_set_pci_memory_regions()
3501 	region->offset_in_bar = CFG_REGION_SIZE + BAR0_RSRVD_SIZE;  in gaudi2_set_pci_memory_regions()
3502 	region->bar_size = CFG_BAR_SIZE;  in gaudi2_set_pci_memory_regions()
3503 	region->bar_id = SRAM_CFG_BAR_ID;  in gaudi2_set_pci_memory_regions()
3504 	region->used = 1;  in gaudi2_set_pci_memory_regions()
3507 	region = &hdev->pci_mem_region[PCI_REGION_DRAM];  in gaudi2_set_pci_memory_regions()
3508 	region->region_base = DRAM_PHYS_BASE;  in gaudi2_set_pci_memory_regions()
3509 	region->region_size = hdev->asic_prop.dram_size;  in gaudi2_set_pci_memory_regions()
3510 	region->offset_in_bar = 0;  in gaudi2_set_pci_memory_regions()
3511 	region->bar_size = prop->dram_pci_bar_size;  in gaudi2_set_pci_memory_regions()
3512 	region->bar_id = DRAM_BAR_ID;  in gaudi2_set_pci_memory_regions()
3513 	region->used = 1;  in gaudi2_set_pci_memory_regions()
3518 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_user_interrupt_setup()
3522 	HL_USR_INTR_STRUCT_INIT(hdev->tpc_interrupt, hdev, 0, HL_USR_INTERRUPT_TPC);  in gaudi2_user_interrupt_setup()
3525 	HL_USR_INTR_STRUCT_INIT(hdev->unexpected_error_interrupt, hdev, 0,  in gaudi2_user_interrupt_setup()
3529 	HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev,  in gaudi2_user_interrupt_setup()
3533 	HL_USR_INTR_STRUCT_INIT(hdev->common_decoder_interrupt, hdev,  in gaudi2_user_interrupt_setup()
3547 		HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i,  in gaudi2_user_interrupt_setup()
3550 	for (i = GAUDI2_IRQ_NUM_USER_FIRST, k = 0 ; k < prop->user_interrupt_count; i++, j++, k++)  in gaudi2_user_interrupt_setup()
3551 		HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, HL_USR_INTERRUPT_CQ);  in gaudi2_user_interrupt_setup()
3558 	return rand ? rand : 1;  in gaudi2_get_non_zero_random_int()
3563 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_special_blocks_free()
3565 			&prop->skip_special_blocks_cfg;  in gaudi2_special_blocks_free()
3567 	kfree(prop->special_blocks);  in gaudi2_special_blocks_free()
3568 	kfree(skip_special_blocks_cfg->block_types);  in gaudi2_special_blocks_free()
3569 	kfree(skip_special_blocks_cfg->block_ranges);  in gaudi2_special_blocks_free()
3586 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_special_blocks_config()
3590 	prop->glbl_err_max_cause_num = GAUDI2_GLBL_ERR_MAX_CAUSE_NUM;  in gaudi2_special_blocks_config()
3591 	prop->num_of_special_blocks = ARRAY_SIZE(gaudi2_special_blocks);  in gaudi2_special_blocks_config()
3592 	prop->special_blocks = kmalloc_array(prop->num_of_special_blocks,  in gaudi2_special_blocks_config()
3593 			sizeof(*prop->special_blocks), GFP_KERNEL);  in gaudi2_special_blocks_config()
3594 	if (!prop->special_blocks)  in gaudi2_special_blocks_config()
3595 		return -ENOMEM;  in gaudi2_special_blocks_config()
3597 	for (i = 0 ; i < prop->num_of_special_blocks ; i++)  in gaudi2_special_blocks_config()
3598 		memcpy(&prop->special_blocks[i], &gaudi2_special_blocks[i],  in gaudi2_special_blocks_config()
3599 				sizeof(*prop->special_blocks));  in gaudi2_special_blocks_config()
3602 	memset(&prop->skip_special_blocks_cfg, 0, sizeof(prop->skip_special_blocks_cfg));  in gaudi2_special_blocks_config()
3603 	prop->skip_special_blocks_cfg.skip_block_hook = gaudi2_special_block_skip;  in gaudi2_special_blocks_config()
3606 		prop->skip_special_blocks_cfg.block_types =  in gaudi2_special_blocks_config()
3609 		if (!prop->skip_special_blocks_cfg.block_types) {  in gaudi2_special_blocks_config()
3610 			rc = -ENOMEM;  in gaudi2_special_blocks_config()
3614 		memcpy(prop->skip_special_blocks_cfg.block_types, gaudi2_iterator_skip_block_types,  in gaudi2_special_blocks_config()
3617 		prop->skip_special_blocks_cfg.block_types_len =  in gaudi2_special_blocks_config()
3622 		prop->skip_special_blocks_cfg.block_ranges =  in gaudi2_special_blocks_config()
3625 		if (!prop->skip_special_blocks_cfg.block_ranges) {  in gaudi2_special_blocks_config()
3626 			rc = -ENOMEM;  in gaudi2_special_blocks_config()
3631 			memcpy(&prop->skip_special_blocks_cfg.block_ranges[i],  in gaudi2_special_blocks_config()
3635 		prop->skip_special_blocks_cfg.block_ranges_len =  in gaudi2_special_blocks_config()
3642 	kfree(prop->skip_special_blocks_cfg.block_types);  in gaudi2_special_blocks_config()
3644 	kfree(prop->special_blocks);  in gaudi2_special_blocks_config()
3656 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_test_queues_msgs_free()
3657 	struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info;  in gaudi2_test_queues_msgs_free()
3661 		/* bail-out if this is an allocation failure point */  in gaudi2_test_queues_msgs_free()
3672 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_test_queues_msgs_alloc()
3673 	struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info;  in gaudi2_test_queues_msgs_alloc()
3676 	/* allocate a message-short buf for each Q we intend to test */  in gaudi2_test_queues_msgs_alloc()
3682 			dev_err(hdev->dev,  in gaudi2_test_queues_msgs_alloc()
3684 			rc = -ENOMEM;  in gaudi2_test_queues_msgs_alloc()
3698 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_sw_init()
3705 		return -ENOMEM;  in gaudi2_sw_init()
3711 		if (gaudi2->num_of_valid_hw_events == GAUDI2_EVENT_SIZE) {  in gaudi2_sw_init()
3712 			dev_err(hdev->dev, "H/W events array exceeds the limit of %u events\n",  in gaudi2_sw_init()
3714 			rc = -EINVAL;  in gaudi2_sw_init()
3718 		gaudi2->hw_events[gaudi2->num_of_valid_hw_events++] = gaudi2_irq_map_table[i].fc_id;  in gaudi2_sw_init()
3722 		gaudi2->lfsr_rand_seeds[i] = gaudi2_get_non_zero_random_int();  in gaudi2_sw_init()
3724 	gaudi2->cpucp_info_get = gaudi2_cpucp_info_get;  in gaudi2_sw_init()
3726 	hdev->asic_specific = gaudi2;  in gaudi2_sw_init()
3729 	 * Use DEVICE_CACHE_LINE_SIZE for alignment since the NIC memory-mapped  in gaudi2_sw_init()
3732 	hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), &hdev->pdev->dev,  in gaudi2_sw_init()
3734 	if (!hdev->dma_pool) {  in gaudi2_sw_init()
3735 		dev_err(hdev->dev, "failed to create DMA pool\n");  in gaudi2_sw_init()
3736 		rc = -ENOMEM;  in gaudi2_sw_init()
3744 	hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);  in gaudi2_sw_init()
3745 	if (!hdev->cpu_accessible_dma_pool) {  in gaudi2_sw_init()
3746 		dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n");  in gaudi2_sw_init()
3747 		rc = -ENOMEM;  in gaudi2_sw_init()
3751 	rc = gen_pool_add(hdev->cpu_accessible_dma_pool, (uintptr_t) hdev->cpu_accessible_dma_mem,  in gaudi2_sw_init()
3752 				HL_CPU_ACCESSIBLE_MEM_SIZE, -1);  in gaudi2_sw_init()
3754 		dev_err(hdev->dev, "Failed to add memory to CPU accessible DMA pool\n");  in gaudi2_sw_init()
3755 		rc = -EFAULT;  in gaudi2_sw_init()
3759 	gaudi2->virt_msix_db_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, prop->pmmu.page_size,  in gaudi2_sw_init()
3760 								&gaudi2->virt_msix_db_dma_addr);  in gaudi2_sw_init()
3761 	if (!gaudi2->virt_msix_db_cpu_addr) {  in gaudi2_sw_init()
3762 		dev_err(hdev->dev, "Failed to allocate DMA memory for virtual MSI-X doorbell\n");  in gaudi2_sw_init()
3763 		rc = -ENOMEM;  in gaudi2_sw_init()
3767 	spin_lock_init(&gaudi2->hw_queues_lock);  in gaudi2_sw_init()
3769 	gaudi2->scratchpad_bus_address = prop->mmu_pgt_addr + HMMU_PAGE_TABLES_SIZE + EDMA_PQS_SIZE;  in gaudi2_sw_init()
3776 	hdev->supports_coresight = true;  in gaudi2_sw_init()
3777 	hdev->supports_sync_stream = true;  in gaudi2_sw_init()
3778 	hdev->supports_cb_mapping = true;  in gaudi2_sw_init()
3779 	hdev->supports_wait_for_multi_cs = false;  in gaudi2_sw_init()
3781 	prop->supports_compute_reset = true;  in gaudi2_sw_init()
3784 	if (hl_fw_version_cmp(hdev, 1, 11, 0) < 0)  in gaudi2_sw_init()
3785 		hdev->event_queue.check_eqe_index = false;  in gaudi2_sw_init()
3787 		hdev->event_queue.check_eqe_index = true;  in gaudi2_sw_init()
3789 	hdev->asic_funcs->set_pci_memory_regions(hdev);  in gaudi2_sw_init()
3799 	hdev->heartbeat_debug_info.cpu_queue_id = GAUDI2_QUEUE_ID_CPU_PQ;  in gaudi2_sw_init()
3806 	hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr);  in gaudi2_sw_init()
3808 	gen_pool_destroy(hdev->cpu_accessible_dma_pool);  in gaudi2_sw_init()
3810 	hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem,  in gaudi2_sw_init()
3811 					hdev->cpu_accessible_dma_address);  in gaudi2_sw_init()
3813 	dma_pool_destroy(hdev->dma_pool);  in gaudi2_sw_init()
3821 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_sw_fini()
3822 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_sw_fini()
3828 	hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr);  in gaudi2_sw_fini()
3830 	gen_pool_destroy(hdev->cpu_accessible_dma_pool);  in gaudi2_sw_fini()
3832 	hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem,  in gaudi2_sw_fini()
3833 						hdev->cpu_accessible_dma_address);  in gaudi2_sw_fini()
3835 	dma_pool_destroy(hdev->dma_pool);  in gaudi2_sw_fini()
3865  * gaudi2_clear_qm_fence_counters_common - clear QM's fence counters
3882 	size = mmPDMA0_QM_CP_BARRIER_CFG - mmPDMA0_QM_CP_FENCE0_CNT_0;  in gaudi2_clear_qm_fence_counters_common()
3905 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_stop_dma_qmans()
3908 	if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK))  in gaudi2_stop_dma_qmans()
3916 	if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))  in gaudi2_stop_dma_qmans()
3924 			if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq)))  in gaudi2_stop_dma_qmans()
3938 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_stop_mme_qmans()
3941 	offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE;  in gaudi2_stop_mme_qmans()
3944 		if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)))  in gaudi2_stop_mme_qmans()
3953 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_stop_tpc_qmans()
3957 	if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))  in gaudi2_stop_tpc_qmans()
3961 		if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i)))  in gaudi2_stop_tpc_qmans()
3971 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_stop_rot_qmans()
3975 	if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK))  in gaudi2_stop_rot_qmans()
3979 		if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i)))  in gaudi2_stop_rot_qmans()
3989 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_stop_nic_qmans()
3993 	if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK))  in gaudi2_stop_nic_qmans()
3999 		if (!(hdev->nic_ports_mask & BIT(i)))  in gaudi2_stop_nic_qmans()
4017 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_dma_stall()
4020 	if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK))  in gaudi2_dma_stall()
4027 	if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))  in gaudi2_dma_stall()
4035 			if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq)))  in gaudi2_dma_stall()
4049 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_mme_stall()
4052 	offset = mmDCORE1_MME_CTRL_LO_QM_STALL - mmDCORE0_MME_CTRL_LO_QM_STALL;  in gaudi2_mme_stall()
4055 		if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))  in gaudi2_mme_stall()
4056 			WREG32(mmDCORE0_MME_CTRL_LO_QM_STALL + (i * offset), 1);  in gaudi2_mme_stall()
4061 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_tpc_stall()
4065 	if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))  in gaudi2_tpc_stall()
4069 		if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i)))  in gaudi2_tpc_stall()
4073 		WREG32(reg_base + TPC_CFG_STALL_OFFSET, 1);  in gaudi2_tpc_stall()
4079 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_rotator_stall()
4083 	if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK))  in gaudi2_rotator_stall()
4091 		if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i)))  in gaudi2_rotator_stall()
4105 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_disable_dma_qmans()
4108 	if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK))  in gaudi2_disable_dma_qmans()
4115 	if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))  in gaudi2_disable_dma_qmans()
4123 			if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq)))  in gaudi2_disable_dma_qmans()
4137 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_disable_mme_qmans()
4140 	offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE;  in gaudi2_disable_mme_qmans()
4143 		if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))  in gaudi2_disable_mme_qmans()
4149 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_disable_tpc_qmans()
4153 	if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))  in gaudi2_disable_tpc_qmans()
4157 		if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i)))  in gaudi2_disable_tpc_qmans()
4167 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_disable_rot_qmans()
4171 	if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK))  in gaudi2_disable_rot_qmans()
4175 		if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i)))  in gaudi2_disable_rot_qmans()
4185 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_disable_nic_qmans()
4189 	if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK))  in gaudi2_disable_nic_qmans()
4195 		if (!(hdev->nic_ports_mask & BIT(i)))  in gaudi2_disable_nic_qmans()
4208 	/* Zero the lower/upper parts of the 64-bit counter */  in gaudi2_enable_timestamp()
4213 	WREG32(mmPSOC_TIMESTAMP_BASE, 1);  in gaudi2_enable_timestamp()
4230 		return gaudi2_vdec_irq_name[irq_number - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM];  in gaudi2_irq_name()
4250 		irq = pci_irq_vector(hdev->pdev, i);  in gaudi2_dec_disable_msix()
4251 		relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM;  in gaudi2_dec_disable_msix()
4253 		dec = hdev->dec + relative_idx / 2;  in gaudi2_dec_disable_msix()
4261 				(void *) &hdev->user_interrupt[dec->core_id]));  in gaudi2_dec_disable_msix()
4274 		irq = pci_irq_vector(hdev->pdev, i);  in gaudi2_dec_enable_msix()
4275 		relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM;  in gaudi2_dec_enable_msix()
4284 		dec = hdev->dec + relative_idx / 2;  in gaudi2_dec_enable_msix()
4290 					(void *) &hdev->user_interrupt[dec->core_id]);  in gaudi2_dec_enable_msix()
4294 			dev_err(hdev->dev, "Failed to request IRQ %d", irq);  in gaudi2_dec_enable_msix()
4308 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_enable_msix()
4309 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_enable_msix()
4313 	if (gaudi2->hw_cap_initialized & HW_CAP_MSIX)  in gaudi2_enable_msix()
4318 	rc = pci_alloc_irq_vectors(hdev->pdev, GAUDI2_MSIX_ENTRIES, GAUDI2_MSIX_ENTRIES,  in gaudi2_enable_msix()
4321 		dev_err(hdev->dev, "MSI-X: Failed to enable support -- %d/%d\n",  in gaudi2_enable_msix()
4326 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);  in gaudi2_enable_msix()
4327 	cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION];  in gaudi2_enable_msix()
4330 		dev_err(hdev->dev, "Failed to request IRQ %d", irq);  in gaudi2_enable_msix()
4334 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);  in gaudi2_enable_msix()
4336 			&hdev->event_queue);  in gaudi2_enable_msix()
4338 		dev_err(hdev->dev, "Failed to request IRQ %d", irq);  in gaudi2_enable_msix()
4344 		dev_err(hdev->dev, "Failed to enable decoder IRQ");  in gaudi2_enable_msix()
4348 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);  in gaudi2_enable_msix()
4351 					&hdev->tpc_interrupt);  in gaudi2_enable_msix()
4353 		dev_err(hdev->dev, "Failed to request IRQ %d", irq);  in gaudi2_enable_msix()
4357 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);  in gaudi2_enable_msix()
4360 					&hdev->unexpected_error_interrupt);  in gaudi2_enable_msix()
4362 		dev_err(hdev->dev, "Failed to request IRQ %d", irq);  in gaudi2_enable_msix()
4366 	for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0;  in gaudi2_enable_msix()
4367 			user_irq_init_cnt < prop->user_interrupt_count;  in gaudi2_enable_msix()
4370 		irq = pci_irq_vector(hdev->pdev, i);  in gaudi2_enable_msix()
4373 				&hdev->user_interrupt[j]);  in gaudi2_enable_msix()
4375 			dev_err(hdev->dev, "Failed to request IRQ %d", irq);  in gaudi2_enable_msix()
4380 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR);  in gaudi2_enable_msix()
4385 		dev_err(hdev->dev, "Failed to request IRQ %d", irq);  in gaudi2_enable_msix()
4389 	gaudi2->hw_cap_initialized |= HW_CAP_MSIX;  in gaudi2_enable_msix()
4394 	for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count;  in gaudi2_enable_msix()
4397 		irq = pci_irq_vector(hdev->pdev, i);  in gaudi2_enable_msix()
4399 		free_irq(irq, &hdev->user_interrupt[j]);  in gaudi2_enable_msix()
4401 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);  in gaudi2_enable_msix()
4402 	free_irq(irq, &hdev->unexpected_error_interrupt);  in gaudi2_enable_msix()
4404 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);  in gaudi2_enable_msix()
4405 	free_irq(irq, &hdev->tpc_interrupt);  in gaudi2_enable_msix()
4407 	gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_DEC_LAST + 1);  in gaudi2_enable_msix()
4409 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);  in gaudi2_enable_msix()
4413 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);  in gaudi2_enable_msix()
4417 	pci_free_irq_vectors(hdev->pdev);  in gaudi2_enable_msix()
4424 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_sync_irqs()
4428 	if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX))  in gaudi2_sync_irqs()
4432 	synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION));  in gaudi2_sync_irqs()
4435 		irq = pci_irq_vector(hdev->pdev, i);  in gaudi2_sync_irqs()
4439 	synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT));  in gaudi2_sync_irqs()
4440 	synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR));  in gaudi2_sync_irqs()
4442 	for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count;  in gaudi2_sync_irqs()
4444 		irq = pci_irq_vector(hdev->pdev, i);  in gaudi2_sync_irqs()
4448 	synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE));  in gaudi2_sync_irqs()
4449 	synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR));  in gaudi2_sync_irqs()
4454 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_disable_msix()
4455 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_disable_msix()
4459 	if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX))  in gaudi2_disable_msix()
4464 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE);  in gaudi2_disable_msix()
4465 	free_irq(irq, &hdev->event_queue);  in gaudi2_disable_msix()
4467 	gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1);  in gaudi2_disable_msix()
4469 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT);  in gaudi2_disable_msix()
4470 	free_irq(irq, &hdev->tpc_interrupt);  in gaudi2_disable_msix()
4472 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR);  in gaudi2_disable_msix()
4473 	free_irq(irq, &hdev->unexpected_error_interrupt);  in gaudi2_disable_msix()
4475 	for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, k = 0;  in gaudi2_disable_msix()
4476 			k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) {  in gaudi2_disable_msix()
4478 		irq = pci_irq_vector(hdev->pdev, i);  in gaudi2_disable_msix()
4480 		free_irq(irq, &hdev->user_interrupt[j]);  in gaudi2_disable_msix()
4483 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION);  in gaudi2_disable_msix()
4484 	cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION];  in gaudi2_disable_msix()
4487 	irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR);  in gaudi2_disable_msix()
4490 	pci_free_irq_vectors(hdev->pdev);  in gaudi2_disable_msix()
4492 	gaudi2->hw_cap_initialized &= ~HW_CAP_MSIX;  in gaudi2_disable_msix()
4502 	if (hdev->pldm)  in gaudi2_stop_dcore_dec()
4509 		if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))  in gaudi2_stop_dcore_dec()
4529 			dev_err(hdev->dev,  in gaudi2_stop_dcore_dec()
4542 	if (hdev->pldm)  in gaudi2_stop_pcie_dec()
4549 		if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))  in gaudi2_stop_pcie_dec()
4569 			dev_err(hdev->dev,  in gaudi2_stop_pcie_dec()
4577 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_stop_dec()
4580 	if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == 0)  in gaudi2_stop_dec()
4595 		reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 1);  in gaudi2_set_arc_running_mode()
4597 		reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_HALT_REQ_MASK, 1);  in gaudi2_set_arc_running_mode()
4617 	if (hdev->pldm)  in gaudi2_verify_arc_running_mode()
4641 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_reset_arcs()
4654 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_nic_qmans_manual_flush()
4658 	if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK))  in gaudi2_nic_qmans_manual_flush()
4664 		if (!(hdev->nic_ports_mask & BIT(i)))  in gaudi2_nic_qmans_manual_flush()
4686 				dev_err(hdev->dev, "failed to %s arc: %d\n",  in gaudi2_set_engine_cores()
4689 				return -1;  in gaudi2_set_engine_cores()
4699 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_set_tpc_engine_mode()
4702 	if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK))  in gaudi2_set_tpc_engine_mode()
4706 	if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + tpc_id)))  in gaudi2_set_tpc_engine_mode()
4712 			(engine_command == HL_ENGINE_STALL) ? 1 : 0);  in gaudi2_set_tpc_engine_mode()
4726 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_set_mme_engine_mode()
4730 	if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + mme_id)))  in gaudi2_set_mme_engine_mode()
4736 			(engine_command == HL_ENGINE_STALL) ? 1 : 0);  in gaudi2_set_mme_engine_mode()
4744 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_set_edma_engine_mode()
4747 	if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK))  in gaudi2_set_edma_engine_mode()
4751 	if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + edma_id)))  in gaudi2_set_edma_engine_mode()
4757 			(engine_command == HL_ENGINE_STALL) ? 1 : 0);  in gaudi2_set_edma_engine_mode()
4804 			dev_err(hdev->dev, "Invalid engine ID %u\n", engine_ids[i]);  in gaudi2_set_engine_modes()
4805 			return -EINVAL;  in gaudi2_set_engine_modes()
4825 		dev_err(hdev->dev, "failed to execute command id %u\n", engine_command);  in gaudi2_set_engines()
4826 		return -EINVAL;  in gaudi2_set_engines()
4834 	if (hdev->pldm)  in gaudi2_halt_engines()
4884 	struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load;  in gaudi2_init_firmware_preload_params()
4886 	pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS;  in gaudi2_init_firmware_preload_params()
4887 	pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0;  in gaudi2_init_firmware_preload_params()
4888 	pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1;  in gaudi2_init_firmware_preload_params()
4889 	pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0;  in gaudi2_init_firmware_preload_params()
4890 	pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1;  in gaudi2_init_firmware_preload_params()
4891 	pre_fw_load->wait_for_preboot_timeout = GAUDI2_PREBOOT_REQ_TIMEOUT_USEC;  in gaudi2_init_firmware_preload_params()
4892 	pre_fw_load->wait_for_preboot_extended_timeout =  in gaudi2_init_firmware_preload_params()
4898 	struct fw_load_mgr *fw_loader = &hdev->fw_loader;  in gaudi2_init_firmware_loader()
4903 	fw_loader->fw_comp_loaded = FW_TYPE_NONE;  in gaudi2_init_firmware_loader()
4904 	fw_loader->boot_fit_img.image_name = GAUDI2_BOOT_FIT_FILE;  in gaudi2_init_firmware_loader()
4905 	fw_loader->linux_img.image_name = GAUDI2_LINUX_FW_FILE;  in gaudi2_init_firmware_loader()
4906 	fw_loader->boot_fit_timeout = GAUDI2_BOOT_FIT_REQ_TIMEOUT_USEC;  in gaudi2_init_firmware_loader()
4907 	fw_loader->skip_bmc = false;  in gaudi2_init_firmware_loader()
4908 	fw_loader->sram_bar_id = SRAM_CFG_BAR_ID;  in gaudi2_init_firmware_loader()
4909 	fw_loader->dram_bar_id = DRAM_BAR_ID;  in gaudi2_init_firmware_loader()
4910 	fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC;  in gaudi2_init_firmware_loader()
4914 	 * hard-coded). in later stages of the protocol those values will be  in gaudi2_init_firmware_loader()
4916 	 * will always be up-to-date  in gaudi2_init_firmware_loader()
4918 	dynamic_loader = &hdev->fw_loader.dynamic_loader;  in gaudi2_init_firmware_loader()
4919 	dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs;  in gaudi2_init_firmware_loader()
4920 	dyn_regs->kmd_msg_to_cpu = cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU);  in gaudi2_init_firmware_loader()
4921 	dyn_regs->cpu_cmd_status_to_host = cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST);  in gaudi2_init_firmware_loader()
4922 	dynamic_loader->wait_for_bl_timeout = GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC;  in gaudi2_init_firmware_loader()
4927 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_init_cpu()
4930 	if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))  in gaudi2_init_cpu()
4933 	if (gaudi2->hw_cap_initialized & HW_CAP_CPU)  in gaudi2_init_cpu()
4940 	gaudi2->hw_cap_initialized |= HW_CAP_CPU;  in gaudi2_init_cpu()
4947 	struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];  in gaudi2_init_cpu_queues()
4948 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_init_cpu_queues()
4949 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_init_cpu_queues()
4955 	if (!hdev->cpu_queues_enable)  in gaudi2_init_cpu_queues()
4958 	if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)  in gaudi2_init_cpu_queues()
4961 	eq = &hdev->event_queue;  in gaudi2_init_cpu_queues()
4963 	dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;  in gaudi2_init_cpu_queues()
4965 	WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address));  in gaudi2_init_cpu_queues()
4966 	WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address));  in gaudi2_init_cpu_queues()
4968 	WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address));  in gaudi2_init_cpu_queues()
4969 	WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address));  in gaudi2_init_cpu_queues()
4971 	WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, lower_32_bits(hdev->cpu_accessible_dma_address));  in gaudi2_init_cpu_queues()
4972 	WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, upper_32_bits(hdev->cpu_accessible_dma_address));  in gaudi2_init_cpu_queues()
4987 	WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq),  in gaudi2_init_cpu_queues()
4999 		dev_err(hdev->dev, "Failed to communicate with device CPU (timeout)\n");  in gaudi2_init_cpu_queues()
5000 		return -EIO;  in gaudi2_init_cpu_queues()
5004 	if (prop->fw_cpu_boot_dev_sts0_valid)  in gaudi2_init_cpu_queues()
5005 		prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0);  in gaudi2_init_cpu_queues()
5007 	if (prop->fw_cpu_boot_dev_sts1_valid)  in gaudi2_init_cpu_queues()
5008 		prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1);  in gaudi2_init_cpu_queues()
5010 	gaudi2->hw_cap_initialized |= HW_CAP_CPU_Q;  in gaudi2_init_cpu_queues()
5021 		q = &hdev->kernel_queues[queue_id_base + pq_id];  in gaudi2_init_qman_pq()
5024 		if (q->dram_bd) {  in gaudi2_init_qman_pq()
5026 					lower_32_bits(q->pq_dram_address));  in gaudi2_init_qman_pq()
5028 					upper_32_bits(q->pq_dram_address));  in gaudi2_init_qman_pq()
5031 					lower_32_bits(q->bus_address));  in gaudi2_init_qman_pq()
5033 					upper_32_bits(q->bus_address));  in gaudi2_init_qman_pq()
5066 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_init_qman_pqc()
5077 				lower_32_bits(gaudi2->scratchpad_bus_address));  in gaudi2_init_qman_pqc()
5079 				upper_32_bits(gaudi2->scratchpad_bus_address));  in gaudi2_init_qman_pqc()
5090 	WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT);  in gaudi2_init_qman_pqc()
5095 	struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;  in gaudi2_get_dyn_sp_reg()
5108 		sp_reg_addr = le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl);  in gaudi2_get_dyn_sp_reg()
5117 		sp_reg_addr = le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl);  in gaudi2_get_dyn_sp_reg()
5126 		sp_reg_addr = le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl);  in gaudi2_get_dyn_sp_reg()
5129 		sp_reg_addr = le32_to_cpu(dyn_regs->gic_rot_qm_irq_ctrl);  in gaudi2_get_dyn_sp_reg()
5132 		sp_reg_addr = le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl);  in gaudi2_get_dyn_sp_reg()
5135 		dev_err(hdev->dev, "Unexpected h/w queue %d\n", queue_id_base);  in gaudi2_get_dyn_sp_reg()
5184 		hdev->kernel_queues[queue_id_base + pq_id].cq_id = GAUDI2_RESERVED_CQ_CS_COMPLETION;  in gaudi2_init_qman()
5199 	prot = 1 << ARC_FARM_KDMA_PROT_ERR_VAL_SHIFT;  in gaudi2_init_dma_core()
5201 		prot |= 1 << ARC_FARM_KDMA_PROT_VAL_SHIFT;  in gaudi2_init_dma_core()
5205 	dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;  in gaudi2_init_dma_core()
5206 	irq_handler_offset = le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl);  in gaudi2_init_dma_core()
5219 	WREG32(reg_base + DMA_CORE_CFG_0_OFFSET, 1 << ARC_FARM_KDMA_CFG_0_EN_SHIFT);  in gaudi2_init_dma_core()
5224 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_init_kdma()
5227 	if ((gaudi2->hw_cap_initialized & HW_CAP_KDMA) == HW_CAP_KDMA)  in gaudi2_init_kdma()
5234 	gaudi2->hw_cap_initialized |= HW_CAP_KDMA;  in gaudi2_init_kdma()
5239 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_init_pdma()
5242 	if ((gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK) == HW_CAP_PDMA_MASK)  in gaudi2_init_pdma()
5257 	gaudi2->hw_cap_initialized |= HW_CAP_PDMA_MASK;  in gaudi2_init_pdma()
5276 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_init_edma()
5277 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_init_edma()
5280 	if ((gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK) == HW_CAP_EDMA_MASK)  in gaudi2_init_edma()
5287 			if (!(prop->edma_enabled_mask & BIT(seq)))  in gaudi2_init_edma()
5292 			gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_EDMA_SHIFT + seq);  in gaudi2_init_edma()
5298  * gaudi2_arm_monitors_for_virt_msix_db() - Arm monitors for writing to the virtual MSI-X doorbell.
5305  * write directly to the HBW host memory of the virtual MSI-X doorbell.
5310  * completion, by decrementing the sync object value and re-arming the monitor.
5316 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_arm_monitors_for_virt_msix_db()
5325 	 * 1. Write interrupt ID to the virtual MSI-X doorbell (master monitor)  in gaudi2_arm_monitors_for_virt_msix_db()
5326 	 * 2. Decrement SOB value by 1.  in gaudi2_arm_monitors_for_virt_msix_db()
5327 	 * 3. Re-arm the master monitor.  in gaudi2_arm_monitors_for_virt_msix_db()
5332 	/* 2nd monitor: Decrement SOB value by 1 */  in gaudi2_arm_monitors_for_virt_msix_db()
5339 	payload = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 0x7FFF) | /* "-1" */  in gaudi2_arm_monitors_for_virt_msix_db()
5340 			FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_SIGN_MASK, 1) |  in gaudi2_arm_monitors_for_virt_msix_db()
5341 			FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1);  in gaudi2_arm_monitors_for_virt_msix_db()
5344 	/* 3rd monitor: Re-arm the master monitor */  in gaudi2_arm_monitors_for_virt_msix_db()
5357 			FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, 1);  in gaudi2_arm_monitors_for_virt_msix_db()
5362 	/* 1st monitor (master): Write interrupt ID to the virtual MSI-X doorbell */  in gaudi2_arm_monitors_for_virt_msix_db()
5368 	addr = gaudi2->virt_msix_db_dma_addr;  in gaudi2_arm_monitors_for_virt_msix_db()
5381 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_prepare_sm_for_virt_msix_db()
5385 		if (!(prop->decoder_enabled_mask & BIT(decoder_id)))  in gaudi2_prepare_sm_for_virt_msix_db()
5395 		interrupt_id += 1;  in gaudi2_prepare_sm_for_virt_msix_db()
5402 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_init_sm()
5408 	reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1);  in gaudi2_init_sm()
5409 	reg_val |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_LBW_EN_MASK, 1);  in gaudi2_init_sm()
5415 	reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1);  in gaudi2_init_sm()
5418 	/* Init CQ0 DB - configure the monitor to trigger MSI-X interrupt */  in gaudi2_init_sm()
5419 	WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(gaudi2->virt_msix_db_dma_addr));  in gaudi2_init_sm()
5420 	WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(gaudi2->virt_msix_db_dma_addr));  in gaudi2_init_sm()
5425 			hdev->completion_queue[i].bus_address;  in gaudi2_init_sm()
5439 	/* Initialize sync objects and monitors which are used for the virtual MSI-X doorbell */  in gaudi2_init_sm()
5445 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_init_mme_acc()
5450 	reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_POS_INF_MASK, 1);  in gaudi2_init_mme_acc()
5451 	reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NEG_INF_MASK, 1);  in gaudi2_init_mme_acc()
5452 	reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NAN_MASK, 1);  in gaudi2_init_mme_acc()
5453 	reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_POS_INF_MASK, 1);  in gaudi2_init_mme_acc()
5454 	reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_NEG_INF_MASK, 1);  in gaudi2_init_mme_acc()
5461 		WREG32(reg_base + MME_ACC_AP_LFSR_SEED_WDATA_OFFSET, gaudi2->lfsr_rand_seeds[i]);  in gaudi2_init_mme_acc()
5474 	case 1:  in gaudi2_init_dcore_mme()
5484 		dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id);  in gaudi2_init_dcore_mme()
5499 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_init_mme()
5502 	if ((gaudi2->hw_cap_initialized & HW_CAP_MME_MASK) == HW_CAP_MME_MASK)  in gaudi2_init_mme()
5508 		gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_MME_SHIFT + i);  in gaudi2_init_mme()
5529 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_init_tpc_config()
5530 	struct gaudi2_tpc_init_cfg_data *cfg_data = ctx->data;  in gaudi2_init_tpc_config()
5534 	queue_id_base = cfg_data->dcore_tpc_qid_base[dcore] + (inst * NUM_OF_PQ_PER_QMAN);  in gaudi2_init_tpc_config()
5536 	if (dcore == 0 && inst == (NUM_DCORE0_TPC - 1))  in gaudi2_init_tpc_config()
5545 	gaudi2->tpc_hw_cap_initialized |= BIT_ULL(HW_CAP_TPC_SHIFT + seq);  in gaudi2_init_tpc_config()
5550 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_init_tpc()
5554 	if (!hdev->asic_prop.tpc_enabled_mask)  in gaudi2_init_tpc()
5557 	if ((gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK) == HW_CAP_TPC_MASK)  in gaudi2_init_tpc()
5561 	init_cfg_data.dcore_tpc_qid_base[1] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0;  in gaudi2_init_tpc()
5571 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_init_rotator()
5580 		gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_ROT_SHIFT + i);  in gaudi2_init_rotator()
5603 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_init_dec()
5607 	if (!hdev->asic_prop.decoder_enabled_mask)  in gaudi2_init_dec()
5610 	if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == HW_CAP_DEC_MASK)  in gaudi2_init_dec()
5617 			if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))  in gaudi2_init_dec()
5627 			gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit);  in gaudi2_init_dec()
5632 		if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit)))  in gaudi2_init_dec()
5640 		gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit);  in gaudi2_init_dec()
5650 	if (hdev->pldm || !hdev->pdev)  in gaudi2_mmu_update_asid_hop0_addr()
5669 		dev_err(hdev->dev, "Timeout during MMU hop0 config of asid %d\n", asid);  in gaudi2_mmu_update_asid_hop0_addr()
5696 	timeout_usec = (hdev->pldm) ? GAUDI2_PLDM_MMU_TIMEOUT_USEC :  in gaudi2_mmu_invalidate_cache_status_poll()
5700 	if (inv_params->flags & MMU_OP_CLEAR_MEMCACHE) {  in gaudi2_mmu_invalidate_cache_status_poll()
5719 	if (inv_params->flags & MMU_OP_SKIP_LOW_CACHE_INV)  in gaudi2_mmu_invalidate_cache_status_poll()
5722 	start_offset = inv_params->range_invalidation ?  in gaudi2_mmu_invalidate_cache_status_poll()
5738 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_is_hmmu_enabled()
5743 	if (gaudi2->hw_cap_initialized & hw_cap)  in gaudi2_is_hmmu_enabled()
5763 	if (inv_params->range_invalidation) {  in gaudi2_mmu_invalidate_cache_trigger()
5770 		u64 start = inv_params->start_va - 1;  in gaudi2_mmu_invalidate_cache_trigger()
5781 				inv_params->end_va >> MMU_RANGE_INV_VA_LSB_SHIFT);  in gaudi2_mmu_invalidate_cache_trigger()
5784 				inv_params->end_va >> MMU_RANGE_INV_VA_MSB_SHIFT);  in gaudi2_mmu_invalidate_cache_trigger()
5790 						inv_params->inv_start_val, inv_params->flags);  in gaudi2_mmu_invalidate_cache_trigger()
5846 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_mmu_invalidate_cache()
5850 	if (hdev->reset_info.hard_reset_pending)  in gaudi2_mmu_invalidate_cache()
5854 	invld_params.inv_start_val = 1;  in gaudi2_mmu_invalidate_cache()
5856 	if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) {  in gaudi2_mmu_invalidate_cache()
5873 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_mmu_invalidate_cache_range()
5878 	if (hdev->reset_info.hard_reset_pending)  in gaudi2_mmu_invalidate_cache_range()
5881 	inv_start_val = (1 << MMU_RANGE_INV_EN_SHIFT |  in gaudi2_mmu_invalidate_cache_range()
5882 			1 << MMU_RANGE_INV_ASID_EN_SHIFT |  in gaudi2_mmu_invalidate_cache_range()
5887 	if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) {  in gaudi2_mmu_invalidate_cache_range()
5899 			invld_params.inv_start_val = 1;  in gaudi2_mmu_invalidate_cache_range()
5924 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_mmu_update_hop0_addr()
5926 	u32 asid, max_asid = prop->max_asid;  in gaudi2_mmu_update_hop0_addr()
5930 	if (hdev->pldm)  in gaudi2_mmu_update_hop0_addr()
5935 			hop0_addr = hdev->mmu_priv.hr.mmu_asid_hop0[asid].phys_addr;  in gaudi2_mmu_update_hop0_addr()
5937 			hop0_addr = prop->mmu_pgt_addr + (asid * prop->dmmu.hop_table_size);  in gaudi2_mmu_update_hop0_addr()
5941 			dev_err(hdev->dev, "failed to set hop0 addr for asid %d\n", asid);  in gaudi2_mmu_update_hop0_addr()
5955 	if (hdev->pldm || !hdev->pdev)  in gaudi2_mmu_init_common()
5960 	WREG32(stlb_base + STLB_INV_ALL_START_OFFSET, 1);  in gaudi2_mmu_init_common()
5971 		dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU SRAM init\n");  in gaudi2_mmu_init_common()
5988 		dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU invalidate all\n");  in gaudi2_mmu_init_common()
5990 	WREG32(mmu_base + MMU_ENABLE_OFFSET, 1);  in gaudi2_mmu_init_common()
5997 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_pci_mmu_init()
5998 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_pci_mmu_init()
6002 	if (gaudi2->hw_cap_initialized & HW_CAP_PMMU)  in gaudi2_pci_mmu_init()
6029 				1),  in gaudi2_pci_mmu_init()
6037 	rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base, prop->pmmu.host_resident);  in gaudi2_pci_mmu_init()
6041 	gaudi2->hw_cap_initialized |= HW_CAP_PMMU;  in gaudi2_pci_mmu_init()
6049 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_dcore_hmmu_init()
6050 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_dcore_hmmu_init()
6062 	if ((gaudi2->hw_cap_initialized & hw_cap) || !(prop->hmmu_hif_enabled_mask & BIT(dmmu_seq)))  in gaudi2_dcore_hmmu_init()
6084 	RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 1,  in gaudi2_dcore_hmmu_init()
6089 	rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base, prop->dmmu.host_resident);  in gaudi2_dcore_hmmu_init()
6093 	gaudi2->hw_cap_initialized |= hw_cap;  in gaudi2_dcore_hmmu_init()
6129 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_hw_init()
6146 	if (hdev->asic_prop.iatu_done_by_fw)  in gaudi2_hw_init()
6147 		gaudi2->dram_bar_cur_addr = DRAM_PHYS_BASE;  in gaudi2_hw_init()
6150 	 * Before pushing u-boot/linux to device, need to set the hbm bar to  in gaudi2_hw_init()
6154 		dev_err(hdev->dev, "failed to map HBM bar to DRAM base address\n");  in gaudi2_hw_init()
6155 		return -EIO;  in gaudi2_hw_init()
6160 		dev_err(hdev->dev, "failed to initialize CPU\n");  in gaudi2_hw_init()
6169 		dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", rc);  in gaudi2_hw_init()
6173 	rc = gaudi2->cpucp_info_get(hdev);  in gaudi2_hw_init()
6175 		dev_err(hdev->dev, "Failed to get cpucp info\n");  in gaudi2_hw_init()
6218  * gaudi2_send_hard_reset_cmd - common function to handle reset
6227 	struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;  in gaudi2_send_hard_reset_cmd()
6229 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_send_hard_reset_cmd()
6232 	preboot_only = (hdev->fw_loader.fw_comp_loaded == FW_TYPE_PREBOOT_CPU);  in gaudi2_send_hard_reset_cmd()
6233 	heartbeat_reset = (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT);  in gaudi2_send_hard_reset_cmd()
6248 	if (gaudi2 && (gaudi2->hw_cap_initialized & HW_CAP_CPU) &&  in gaudi2_send_hard_reset_cmd()
6254 	 * 1. FW reset: FW initiate the reset sequence  in gaudi2_send_hard_reset_cmd()
6261 		WREG32(le32_to_cpu(dyn_regs->gic_host_halt_irq),  in gaudi2_send_hard_reset_cmd()
6271 	 * For the case in which we are working with Linux/Bootfit this is a hail-mary  in gaudi2_send_hard_reset_cmd()
6285 		if (hdev->asic_prop.hard_reset_done_by_fw)  in gaudi2_send_hard_reset_cmd()
6293  * gaudi2_execute_hard_reset - execute hard reset by driver/FW
6301 	if (hdev->asic_prop.hard_reset_done_by_fw) {  in gaudi2_execute_hard_reset()
6314 	WREG32(mmPSOC_RESET_CONF_SW_ALL_RST, 1);  in gaudi2_execute_hard_reset()
6318  * gaudi2_execute_soft_reset - execute soft reset by driver/FW
6336 	gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 1,  in gaudi2_execute_soft_reset()
6339 	gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 2,  in gaudi2_execute_soft_reset()
6343 	WREG32(mmPSOC_RESET_CONF_SOFT_RST, 1);  in gaudi2_execute_soft_reset()
6365 		dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val);  in gaudi2_poll_btm_indication()
6370 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_hw_fini()
6375 	if (hdev->pldm) {  in gaudi2_hw_fini()
6390 		driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw;  in gaudi2_hw_fini()
6399 		driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU &&  in gaudi2_hw_fini()
6400 							!hdev->asic_prop.fw_security_enabled);  in gaudi2_hw_fini()
6411 		 * 1. FW update is being triggered  in gaudi2_hw_fini()
6412 		 *        - setting the dirty bit  in gaudi2_hw_fini()
6415 		 *        - dirty bit cleared  in gaudi2_hw_fini()
6416 		 *        - BTM indication cleared  in gaudi2_hw_fini()
6417 		 *        - preboot ready indication cleared  in gaudi2_hw_fini()
6419 		 *        - BTM indication will be set  in gaudi2_hw_fini()
6420 		 *        - BIST test performed and another reset triggered  in gaudi2_hw_fini()
6431 		if (hdev->fw_components & FW_TYPE_PREBOOT_CPU)  in gaudi2_hw_fini()
6440 	gaudi2->dec_hw_cap_initialized &= ~(HW_CAP_DEC_MASK);  in gaudi2_hw_fini()
6441 	gaudi2->tpc_hw_cap_initialized &= ~(HW_CAP_TPC_MASK);  in gaudi2_hw_fini()
6444 	 * Clear NIC capability mask in order for driver to re-configure  in gaudi2_hw_fini()
6445 	 * NIC QMANs. NIC ports will not be re-configured during soft  in gaudi2_hw_fini()
6448 	gaudi2->nic_hw_cap_initialized &= ~(HW_CAP_NIC_MASK);  in gaudi2_hw_fini()
6451 		gaudi2->hw_cap_initialized &=  in gaudi2_hw_fini()
6458 		memset(gaudi2->events_stat, 0, sizeof(gaudi2->events_stat));  in gaudi2_hw_fini()
6460 		gaudi2->hw_cap_initialized &=  in gaudi2_hw_fini()
6488 	rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size);  in gaudi2_mmap()
6490 		dev_err(hdev->dev, "dma_mmap_coherent error %d", rc);  in gaudi2_mmap()
6494 	rc = remap_pfn_range(vma, vma->vm_start,  in gaudi2_mmap()
6496 				size, vma->vm_page_prot);  in gaudi2_mmap()
6498 		dev_err(hdev->dev, "remap_pfn_range error %d", rc);  in gaudi2_mmap()
6507 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_is_queue_enabled()
6521 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0) >> 2);  in gaudi2_is_queue_enabled()
6525 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0) >> 2);  in gaudi2_is_queue_enabled()
6529 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0) >> 2);  in gaudi2_is_queue_enabled()
6533 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0) >> 2);  in gaudi2_is_queue_enabled()
6541 		hw_test_cap_bit = HW_CAP_MME_SHIFT + 1;  in gaudi2_is_queue_enabled()
6554 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_TPC_0_0) >> 2);  in gaudi2_is_queue_enabled()
6558 			return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(0));  in gaudi2_is_queue_enabled()
6563 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_TPC_0_0) >> 2);  in gaudi2_is_queue_enabled()
6568 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_TPC_0_0) >> 2);  in gaudi2_is_queue_enabled()
6573 			((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_TPC_0_0) >> 2);  in gaudi2_is_queue_enabled()
6581 		hw_test_cap_bit = HW_CAP_ROT_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_ROT_0_0) >> 2);  in gaudi2_is_queue_enabled()
6585 		hw_nic_cap_bit = HW_CAP_NIC_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_NIC_0_0) >> 2);  in gaudi2_is_queue_enabled()
6589 			return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(0));  in gaudi2_is_queue_enabled()
6593 		return !!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q);  in gaudi2_is_queue_enabled()
6600 		return  !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(hw_tpc_cap_bit));  in gaudi2_is_queue_enabled()
6603 		return  !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(hw_nic_cap_bit));  in gaudi2_is_queue_enabled()
6608 	return !!(gaudi2->hw_cap_initialized & hw_cap_mask);  in gaudi2_is_queue_enabled()
6613 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_is_arc_enabled()
6618 		return !!(gaudi2->active_hw_arc & BIT_ULL(arc_id));  in gaudi2_is_arc_enabled()
6621 		return !!(gaudi2->active_tpc_arc & BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0));  in gaudi2_is_arc_enabled()
6624 		return !!(gaudi2->active_nic_arc & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0));  in gaudi2_is_arc_enabled()
6633 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_clr_arc_id_cap()
6638 		gaudi2->active_hw_arc &= ~(BIT_ULL(arc_id));  in gaudi2_clr_arc_id_cap()
6642 		gaudi2->active_tpc_arc &= ~(BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0));  in gaudi2_clr_arc_id_cap()
6646 		gaudi2->active_nic_arc &= ~(BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0));  in gaudi2_clr_arc_id_cap()
6656 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_set_arc_id_cap()
6661 		gaudi2->active_hw_arc |= BIT_ULL(arc_id);  in gaudi2_set_arc_id_cap()
6665 		gaudi2->active_tpc_arc |= BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0);  in gaudi2_set_arc_id_cap()
6669 		gaudi2->active_nic_arc |= BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0);  in gaudi2_set_arc_id_cap()
6679 	struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;  in gaudi2_ring_doorbell()
6684 		 * QMAN has 4 successive PQ_PI registers, 1 for each of the QMAN PQs.  in gaudi2_ring_doorbell()
6703 		WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq),  in gaudi2_ring_doorbell()
6714 	pqe[1] = pbd[1];  in gaudi2_pqe_write()
6720 	return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags);  in gaudi2_dma_alloc_coherent()
6726 	dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle);  in gaudi2_dma_free_coherent()
6732 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_send_cpu_message()
6734 	if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) {  in gaudi2_send_cpu_message()
6752 	return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);  in gaudi2_dma_pool_zalloc()
6757 	dma_pool_free(hdev->dma_pool, vaddr, dma_addr);  in gaudi2_dma_pool_free()
6773 	struct asic_fixed_properties *asic_prop = &hdev->asic_prop;  in gaudi2_validate_cb_address()
6774 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_validate_cb_address()
6776 	if (!gaudi2_is_queue_enabled(hdev, parser->hw_queue_id)) {  in gaudi2_validate_cb_address()
6777 		dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id);  in gaudi2_validate_cb_address()
6778 		return -EINVAL;  in gaudi2_validate_cb_address()
6783 	if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,  in gaudi2_validate_cb_address()
6784 					parser->user_cb_size,  in gaudi2_validate_cb_address()
6785 					asic_prop->sram_user_base_address,  in gaudi2_validate_cb_address()
6786 					asic_prop->sram_end_address))  in gaudi2_validate_cb_address()
6789 	if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,  in gaudi2_validate_cb_address()
6790 					parser->user_cb_size,  in gaudi2_validate_cb_address()
6791 					asic_prop->dram_user_base_address,  in gaudi2_validate_cb_address()
6792 					asic_prop->dram_end_address))  in gaudi2_validate_cb_address()
6795 	if ((gaudi2->hw_cap_initialized & HW_CAP_DMMU_MASK) &&  in gaudi2_validate_cb_address()
6796 		hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,  in gaudi2_validate_cb_address()
6797 						parser->user_cb_size,  in gaudi2_validate_cb_address()
6798 						asic_prop->dmmu.start_addr,  in gaudi2_validate_cb_address()
6799 						asic_prop->dmmu.end_addr))  in gaudi2_validate_cb_address()
6802 	if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) {  in gaudi2_validate_cb_address()
6803 		if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,  in gaudi2_validate_cb_address()
6804 					parser->user_cb_size,  in gaudi2_validate_cb_address()
6805 					asic_prop->pmmu.start_addr,  in gaudi2_validate_cb_address()
6806 					asic_prop->pmmu.end_addr) ||  in gaudi2_validate_cb_address()
6808 					(u64) (uintptr_t) parser->user_cb,  in gaudi2_validate_cb_address()
6809 					parser->user_cb_size,  in gaudi2_validate_cb_address()
6810 					asic_prop->pmmu_huge.start_addr,  in gaudi2_validate_cb_address()
6811 					asic_prop->pmmu_huge.end_addr))  in gaudi2_validate_cb_address()
6814 	} else if (gaudi2_host_phys_addr_valid((u64) (uintptr_t) parser->user_cb)) {  in gaudi2_validate_cb_address()
6815 		if (!hdev->pdev)  in gaudi2_validate_cb_address()
6818 		if (!device_iommu_mapped(&hdev->pdev->dev))  in gaudi2_validate_cb_address()
6822 	dev_err(hdev->dev, "CB address %p + 0x%x for internal QMAN is not valid\n",  in gaudi2_validate_cb_address()
6823 		parser->user_cb, parser->user_cb_size);  in gaudi2_validate_cb_address()
6825 	return -EFAULT;  in gaudi2_validate_cb_address()
6830 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_cs_parser()
6832 	if (!parser->is_kernel_allocated_cb)  in gaudi2_cs_parser()
6835 	if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) {  in gaudi2_cs_parser()
6836 		dev_err(hdev->dev, "PMMU not initialized - Unsupported mode in Gaudi2\n");  in gaudi2_cs_parser()
6837 		return -EINVAL;  in gaudi2_cs_parser()
6845 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_send_heartbeat()
6847 	if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))  in gaudi2_send_heartbeat()
6890 	mask = ~(1 << (sob_id & 0x7));  in gaudi2_arm_cq_monitor()
6891 	mode = 1; /* comparison mode is "equal to" */  in gaudi2_arm_cq_monitor()
6913 				GAUDI2_RESERVED_CQ_KDMA_COMPLETION, 1, 1);  in gaudi2_send_job_to_kdma()
6918 	comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) |  in gaudi2_send_job_to_kdma()
6919 			FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1);  in gaudi2_send_job_to_kdma()
6930 	commit_mask = FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_LIN_MASK, 1) |  in gaudi2_send_job_to_kdma()
6931 				FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_WR_COMP_EN_MASK, 1);  in gaudi2_send_job_to_kdma()
6934 		commit_mask |= FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_MEM_SET_MASK, 1);  in gaudi2_send_job_to_kdma()
6939 	cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_KDMA_COMPLETION];  in gaudi2_send_job_to_kdma()
6940 	cq_base = cq->kernel_address;  in gaudi2_send_job_to_kdma()
6941 	polling_addr = (u32 *)&cq_base[cq->ci];  in gaudi2_send_job_to_kdma()
6943 	if (hdev->pldm)  in gaudi2_send_job_to_kdma()
6944 		/* for each 1MB 20 second of timeout */  in gaudi2_send_job_to_kdma()
6945 		timeout = ((size / SZ_1M) + 1) * USEC_PER_SEC * 20;  in gaudi2_send_job_to_kdma()
6954 			(status == 1),  in gaudi2_send_job_to_kdma()
6962 		dev_err(hdev->dev, "Timeout while waiting for KDMA to be idle\n");  in gaudi2_send_job_to_kdma()
6963 		WREG32(mmARC_FARM_KDMA_CFG_1, 1 << ARC_FARM_KDMA_CFG_1_HALT_SHIFT);  in gaudi2_send_job_to_kdma()
6967 	cq->ci = hl_cq_inc_ptr(cq->ci);  in gaudi2_send_job_to_kdma()
6989 		WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT);  in gaudi2_qman_set_test_mode()
6995 	return hdev->asic_prop.first_available_user_sob[0] +  in gaudi2_test_queue_hw_queue_id_to_sob_id()
6996 				hw_queue_id - GAUDI2_QUEUE_ID_PDMA_0_0;  in gaudi2_test_queue_hw_queue_id_to_sob_id()
7012 	u32 tmp, sob_base = 1;  in gaudi2_test_queue_send_msg_short()
7013 	struct packet_msg_short *msg_short_pkt = msg_info->kern_addr;  in gaudi2_test_queue_send_msg_short()
7018 		(1 << GAUDI2_PKT_CTL_EB_SHIFT) |  in gaudi2_test_queue_send_msg_short()
7019 		(1 << GAUDI2_PKT_CTL_MB_SHIFT) |  in gaudi2_test_queue_send_msg_short()
7023 	msg_short_pkt->value = cpu_to_le32(sob_val);  in gaudi2_test_queue_send_msg_short()
7024 	msg_short_pkt->ctl = cpu_to_le32(tmp);  in gaudi2_test_queue_send_msg_short()
7026 	rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, msg_info->dma_addr);  in gaudi2_test_queue_send_msg_short()
7028 		dev_err(hdev->dev,  in gaudi2_test_queue_send_msg_short()
7041 	if (hdev->pldm)  in gaudi2_test_queue_wait_completion()
7054 	if (rc == -ETIMEDOUT) {  in gaudi2_test_queue_wait_completion()
7055 		dev_err(hdev->dev, "H/W queue %d test failed (SOB_OBJ_0 == 0x%x)\n",  in gaudi2_test_queue_wait_completion()
7057 		rc = -EIO;  in gaudi2_test_queue_wait_completion()
7065 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_test_cpu_queue()
7071 	if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))  in gaudi2_test_cpu_queue()
7079 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_test_queues()
7089 		msg_info = &gaudi2->queues_test_info[i - GAUDI2_QUEUE_ID_PDMA_0_0];  in gaudi2_test_queues()
7108 			/* chip is not usable, no need for cleanups, just bail-out with error */  in gaudi2_test_queues()
7121 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_compute_reset_late_init()
7129 		dev_err(hdev->dev, "Failed to scrub arcs DCCM\n");  in gaudi2_compute_reset_late_init()
7136 	irq_arr_size = gaudi2->num_of_valid_hw_events * sizeof(gaudi2->hw_events[0]);  in gaudi2_compute_reset_late_init()
7137 	return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size);  in gaudi2_compute_reset_late_init()
7144 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_get_edma_idle_status()
7146 	const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#-15x%#x\n";  in gaudi2_get_edma_idle_status()
7154 			"----  ----  -------  ------------  -------------  -------------\n");  in gaudi2_get_edma_idle_status()
7160 			if (!(prop->edma_enabled_mask & BIT(seq)))  in gaudi2_get_edma_idle_status()
7195 	const char *pdma_fmt = "%-6d%-9s%#-14x%#-15x%#x\n";  in gaudi2_get_pdma_idle_status()
7203 					"----  -------  ------------  -------------  -------------\n");  in gaudi2_get_pdma_idle_status()
7234 	const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n";  in gaudi2_get_nic_idle_status()
7241 	if (e && hdev->nic_ports_mask)  in gaudi2_get_nic_idle_status()
7244 					"---  -------  ------------  ----------\n");  in gaudi2_get_nic_idle_status()
7247 		if (!(i & 1))  in gaudi2_get_nic_idle_status()
7252 		if (!(hdev->nic_ports_mask & BIT(i)))  in gaudi2_get_nic_idle_status()
7281 	const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n";  in gaudi2_get_mme_idle_status()
7289 					"---  ----  -------  ------------  ---------------\n");  in gaudi2_get_mme_idle_status()
7322 	struct gaudi2_tpc_idle_data *idle_data = ctx->data;  in gaudi2_is_tpc_engine_idle()
7327 	if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1)))  in gaudi2_is_tpc_engine_idle()
7340 	*(idle_data->is_idle) &= is_eng_idle;  in gaudi2_is_tpc_engine_idle()
7342 	if (idle_data->mask && !is_eng_idle)  in gaudi2_is_tpc_engine_idle()
7343 		set_bit(engine_idx, idle_data->mask);  in gaudi2_is_tpc_engine_idle()
7345 	if (idle_data->e)  in gaudi2_is_tpc_engine_idle()
7346 		hl_engine_data_sprintf(idle_data->e,  in gaudi2_is_tpc_engine_idle()
7347 					idle_data->tpc_fmt, dcore, inst,  in gaudi2_is_tpc_engine_idle()
7355 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_get_tpc_idle_status()
7360 		.tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n",  in gaudi2_get_tpc_idle_status()
7370 	if (e && prop->tpc_enabled_mask)  in gaudi2_get_tpc_idle_status()
7373 			"----  ---  -------  ------------  ----------  ------\n");  in gaudi2_get_tpc_idle_status()
7383 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_get_decoder_idle_status()
7385 	const char *pcie_dec_fmt = "%-10d%-9s%#x\n";  in gaudi2_get_decoder_idle_status()
7386 	const char *dec_fmt = "%-6d%-5d%-9s%#x\n";  in gaudi2_get_decoder_idle_status()
7393 	if (e && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK)))  in gaudi2_get_decoder_idle_status()
7396 			"----  ---  -------  ---------------\n");  in gaudi2_get_decoder_idle_status()
7400 			dec_enabled_bit = 1 << (i * NUM_OF_DEC_PER_DCORE + j);  in gaudi2_get_decoder_idle_status()
7401 			if (!(prop->decoder_enabled_mask & dec_enabled_bit))  in gaudi2_get_decoder_idle_status()
7421 	if (e && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK))  in gaudi2_get_decoder_idle_status()
7424 			"--------  -------  ---------------\n");  in gaudi2_get_decoder_idle_status()
7429 		if (!(prop->decoder_enabled_mask & BIT(dec_enabled_bit)))  in gaudi2_get_decoder_idle_status()
7452 	const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-14x%#x\n";  in gaudi2_get_rotator_idle_status()
7462 			"----  ---  -------  ------------  ------------  ----------\n");  in gaudi2_get_rotator_idle_status()
7504 	__acquires(&gaudi2->hw_queues_lock)  in gaudi2_hw_queues_lock()
7506 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_hw_queues_lock()
7508 	spin_lock(&gaudi2->hw_queues_lock);  in gaudi2_hw_queues_lock()
7512 	__releases(&gaudi2->hw_queues_lock)  in gaudi2_hw_queues_unlock()
7514 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_hw_queues_unlock()
7516 	spin_unlock(&gaudi2->hw_queues_lock);  in gaudi2_hw_queues_unlock()
7521 	return hdev->pdev->device;  in gaudi2_get_pci_id()
7526 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_get_eeprom_data()
7528 	if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))  in gaudi2_get_eeprom_data()
7541 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_get_events_stat()
7544 		*size = (u32) sizeof(gaudi2->events_stat_aggregate);  in gaudi2_get_events_stat()
7545 		return gaudi2->events_stat_aggregate;  in gaudi2_get_events_stat()
7548 	*size = (u32) sizeof(gaudi2->events_stat);  in gaudi2_get_events_stat()
7549 	return gaudi2->events_stat;  in gaudi2_get_events_stat()
7555 	u32 offset = (mmDCORE0_VDEC1_BRDG_CTRL_BASE - mmDCORE0_VDEC0_BRDG_CTRL_BASE) *  in gaudi2_mmu_vdec_dcore_prepare()
7578 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_mmu_dcore_prepare()
7585 	if (prop->edma_enabled_mask & BIT(edma_seq_base)) {  in gaudi2_mmu_dcore_prepare()
7592 	if (prop->edma_enabled_mask & BIT(edma_seq_base + 1)) {  in gaudi2_mmu_dcore_prepare()
7602 	 * Sync Mngrs on dcores 1 - 3 are exposed to user, so must use user ASID  in gaudi2_mmu_dcore_prepare()
7638 		if (prop->decoder_enabled_mask & BIT(dcore_id * NUM_OF_DEC_PER_DCORE + vdec_id))  in gaudi2_mmu_dcore_prepare()
7646 	u32 offset = (mmPCIE_VDEC1_BRDG_CTRL_BASE - mmPCIE_VDEC0_BRDG_CTRL_BASE) * shared_vdec_id;  in gudi2_mmu_vdec_shared_prepare()
7667 	u32 offset = (mmARC_FARM_ARC1_DUP_ENG_BASE - mmARC_FARM_ARC0_DUP_ENG_BASE) * arc_farm_id;  in gudi2_mmu_arc_farm_arc_dup_eng_prepare()
7721 	if (hdev->fw_components & FW_TYPE_BOOT_CPU)  in gaudi2_arc_mmu_prepare_all()
7739 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_mmu_shared_prepare()
7767 	if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 0))  in gaudi2_mmu_shared_prepare()
7770 	if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 1))  in gaudi2_mmu_shared_prepare()
7771 		gudi2_mmu_vdec_shared_prepare(hdev, 1, rw_asid, 0);  in gaudi2_mmu_shared_prepare()
7787 	struct gaudi2_tpc_mmu_data *mmu_data = ctx->data;  in gaudi2_tpc_mmu_prepare()
7790 	WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_ASID + offset, mmu_data->rw_asid);  in gaudi2_tpc_mmu_prepare()
7792 	WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_ASID + offset, mmu_data->rw_asid);  in gaudi2_tpc_mmu_prepare()
7798 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_mmu_prepare()
7807 		dev_crit(hdev->dev, "asid %u is too big\n", asid);  in gaudi2_mmu_prepare()
7808 		return -EINVAL;  in gaudi2_mmu_prepare()
7811 	if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK))  in gaudi2_mmu_prepare()
7835 	/* return in case of NIC status event - these events are received periodically and not as  in is_info_event()
7857 		dev_err_ratelimited(hdev->dev, "%s: %pV\n",  in gaudi2_print_event()
7861 		dev_err(hdev->dev, "%s: %pV\n",  in gaudi2_print_event()
7876 	if (hl_fw_version_cmp(hdev, 1, 12, 0) >= 0)  in gaudi2_handle_ecc_event()
7879 	ecc_address = le64_to_cpu(ecc_data->ecc_address);  in gaudi2_handle_ecc_event()
7880 	ecc_syndrome = le64_to_cpu(ecc_data->ecc_syndrom);  in gaudi2_handle_ecc_event()
7881 	memory_wrapper_idx = ecc_data->memory_wrapper_idx;  in gaudi2_handle_ecc_event()
7884 		block_id = le16_to_cpu(ecc_data->block_id);  in gaudi2_handle_ecc_event()
7885 		gaudi2_print_event(hdev, event_type, !ecc_data->is_critical,  in gaudi2_handle_ecc_event()
7888 			ecc_data->is_critical);  in gaudi2_handle_ecc_event()
7890 		gaudi2_print_event(hdev, event_type, !ecc_data->is_critical,  in gaudi2_handle_ecc_event()
7892 			ecc_address, ecc_syndrome, memory_wrapper_idx, ecc_data->is_critical);  in gaudi2_handle_ecc_event()
7895 	return !!ecc_data->is_critical;  in gaudi2_handle_ecc_event()
7900 	struct undefined_opcode_info *undef_opcode = &hdev->captured_err_info.undef_opcode;  in handle_lower_qman_data_on_err()
7906 	is_arc_cq = FIELD_GET(PDMA0_QM_CP_STS_CUR_CQ_MASK, cp_sts); /* 0 - legacy CQ, 1 - ARC_CQ */  in handle_lower_qman_data_on_err()
7924 	dev_info(hdev->dev,  in handle_lower_qman_data_on_err()
7928 	if (undef_opcode->write_enable) {  in handle_lower_qman_data_on_err()
7930 		undef_opcode->timestamp = ktime_get();  in handle_lower_qman_data_on_err()
7931 		undef_opcode->cq_addr = cq_ptr;  in handle_lower_qman_data_on_err()
7932 		undef_opcode->cq_size = cq_size;  in handle_lower_qman_data_on_err()
7933 		undef_opcode->engine_id = engine_id;  in handle_lower_qman_data_on_err()
7934 		undef_opcode->stream_id = QMAN_STREAMS;  in handle_lower_qman_data_on_err()
7935 		undef_opcode->write_enable = 0;  in handle_lower_qman_data_on_err()
7946 	glbl_sts_addr = qman_base + (mmDCORE0_TPC0_QM_GLBL_ERR_STS_0 - mmDCORE0_TPC0_QM_BASE);  in gaudi2_handle_qman_err_generic()
7947 	arb_err_addr = qman_base + (mmDCORE0_TPC0_QM_ARB_ERR_CAUSE - mmDCORE0_TPC0_QM_BASE);  in gaudi2_handle_qman_err_generic()
7950 	for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) {  in gaudi2_handle_qman_err_generic()
8021 	hl_handle_razwi(hdev, (u64)razwi_hi << 32 | razwi_lo, &eng_id, 1,  in gaudi2_razwi_rr_hbw_shared_printf_info()
8024 	dev_err_ratelimited(hdev->dev,  in gaudi2_razwi_rr_hbw_shared_printf_info()
8025 		"%s-RAZWI SHARED RR HBW %s error, address %#llx, Initiator coordinates 0x%x\n",  in gaudi2_razwi_rr_hbw_shared_printf_info()
8048 	hl_handle_razwi(hdev, razwi_addr, &eng_id, 1, rd_wr_flag | HL_RAZWI_LBW, event_mask);  in gaudi2_razwi_rr_lbw_shared_printf_info()
8049 	dev_err_ratelimited(hdev->dev,  in gaudi2_razwi_rr_lbw_shared_printf_info()
8050 …"%s-RAZWI SHARED RR LBW %s error, mstr_if 0x%llx, captured address 0x%llX Initiator coordinates 0x…  in gaudi2_razwi_rr_lbw_shared_printf_info()
8064 				(GAUDI2_DCORE0_ENGINE_ID_TPC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0));  in gaudi2_razwi_calc_engine_id()
8067 		return ((GAUDI2_DCORE0_ENGINE_ID_MME - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) +  in gaudi2_razwi_calc_engine_id()
8089 				(GAUDI2_DCORE0_ENGINE_ID_DEC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0));  in gaudi2_razwi_calc_engine_id()
8120 		if (hdev->tpc_binning) {  in gaudi2_ack_module_razwi_event_handler()
8121 			binned_idx = __ffs(hdev->tpc_binning);  in gaudi2_ack_module_razwi_event_handler()
8187 		if (hdev->decoder_binning) {  in gaudi2_ack_module_razwi_event_handler()
8188 			binned_idx = __ffs(hdev->decoder_binning);  in gaudi2_ack_module_razwi_event_handler()
8218 				(((s32)lbw_rtr_id - hbw_rtr_id) * DCORE_RTR_OFFSET);  in gaudi2_ack_module_razwi_event_handler()
8263 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_check_if_razwi_happened()
8267 	for (mod_idx = 0 ; mod_idx < (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1) ; mod_idx++) {  in gaudi2_check_if_razwi_happened()
8268 		if (prop->tpc_enabled_mask & BIT(mod_idx))  in gaudi2_check_if_razwi_happened()
8280 		if (prop->edma_enabled_mask & BIT(mod_idx))  in gaudi2_check_if_razwi_happened()
8289 		if (hdev->nic_ports_mask & BIT(mod_idx))  in gaudi2_check_if_razwi_happened()
8290 			gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_NIC, mod_idx >> 1, 0,  in gaudi2_check_if_razwi_happened()
8295 		if (prop->decoder_enabled_mask & BIT(mod_idx))  in gaudi2_check_if_razwi_happened()
8319 						PSOC_RAZWI_ENG_STR_SIZE - str_size, "%s",  in gaudi2_psoc_razwi_get_engines()
8323 						PSOC_RAZWI_ENG_STR_SIZE - str_size, " or %s",  in gaudi2_psoc_razwi_get_engines()
8360 				dev_err(hdev->dev,  in gaudi2_handle_psoc_razwi_happened()
8374 				dev_err(hdev->dev,  in gaudi2_handle_psoc_razwi_happened()
8386 				dev_err(hdev->dev,  in gaudi2_handle_psoc_razwi_happened()
8398 				dev_err(hdev->dev,  in gaudi2_handle_psoc_razwi_happened()
8406 		/* In common case the loop will break, when there is only one engine id, or  in gaudi2_handle_psoc_razwi_happened()
8424 	if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) {  in gaudi2_ack_psoc_razwi_event_handler()
8432 	dev_err_ratelimited(hdev->dev,  in gaudi2_ack_psoc_razwi_event_handler()
8443 		dev_err_ratelimited(hdev->dev,  in gaudi2_ack_psoc_razwi_event_handler()
8448 	if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX))  in gaudi2_ack_psoc_razwi_event_handler()
8484 		index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP;  in gaudi2_handle_qm_sei_err()
8498 		index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) /  in gaudi2_handle_qm_sei_err()
8499 				(GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE -  in gaudi2_handle_qm_sei_err()
8506 		index = event_type - GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP;  in gaudi2_handle_qm_sei_err()
8512 		index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE;  in gaudi2_handle_qm_sei_err()
8545 		index = event_type - GAUDI2_EVENT_TPC0_QM;  in gaudi2_handle_qman_err()
8550 		index = event_type - GAUDI2_EVENT_TPC6_QM;  in gaudi2_handle_qman_err()
8555 		index = event_type - GAUDI2_EVENT_TPC12_QM;  in gaudi2_handle_qman_err()
8560 		index = event_type - GAUDI2_EVENT_TPC18_QM;  in gaudi2_handle_qman_err()
8590 		index = 1;  in gaudi2_handle_qman_err()
8647 	/* Handle EDMA QM SEI here because there is no AXI error response event for EDMA */  in gaudi2_handle_qman_err()
8712 	u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data);  in gaudi2_handle_rot_err()
8734 	u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data);  in gaudi2_tpc_ack_interrupts()
8766 				(dec_index - NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES);  in gaudi2_handle_dec_err()
8783 	/* Write 1 clear errors */  in gaudi2_handle_dec_err()
8852 	/* check if RAZWI happened on WAP0/1 */  in gaudi2_handle_mme_wap_err()
8868 	/* If an AXI read or write error is received, an error is reported and  in gaudi2_handle_kdma_core_event()
8870 	 * register of the KDMA engine, the reported error is always HBW even if  in gaudi2_handle_kdma_core_event()
9013 		dev_err_ratelimited(hdev->dev, "PMMU page fault on va 0x%llx\n", addr);  in gaudi2_handle_page_error()
9017 		dev_err_ratelimited(hdev->dev, "HMMU page fault on va range 0x%llx - 0x%llx\n",  in gaudi2_handle_page_error()
9044 	dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n",  in gaudi2_handle_access_error()
9064 			else if (i == 1)  in gaudi2_handle_mmu_spi_sei_generic()
9125 		dev_err_ratelimited(hdev->dev, "SM%u err. err cause: CQ_INTR. queue index: %u\n",  in gaudi2_handle_sm_err()
9150 		dcore = 1;  in get_hmmu_base()
9156 		index_in_dcore = 1;  in get_hmmu_base()
9160 		dcore = 1;  in get_hmmu_base()
9161 		index_in_dcore = 1;  in get_hmmu_base()
9190 		dcore = 1;  in get_hmmu_base()
9200 		dcore = 1;  in get_hmmu_base()
9216 		index_in_dcore = 1;  in get_hmmu_base()
9221 		index_in_dcore = 1;  in get_hmmu_base()
9269 	dev_err_ratelimited(hdev->dev,  in gaudi2_hbm_sei_handle_read_err()
9275 	addr = le32_to_cpu(rd_err_data->dbg_rd_err_addr.rd_addr_val);  in gaudi2_hbm_sei_handle_read_err()
9276 	dev_err_ratelimited(hdev->dev,  in gaudi2_hbm_sei_handle_read_err()
9286 		if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) &  in gaudi2_hbm_sei_handle_read_err()
9288 			dev_err_ratelimited(hdev->dev, "Beat%d ECC SERR: DM: %#x, Syndrome: %#x\n",  in gaudi2_hbm_sei_handle_read_err()
9290 						le32_to_cpu(rd_err_data->dbg_rd_err_dm),  in gaudi2_hbm_sei_handle_read_err()
9291 						le32_to_cpu(rd_err_data->dbg_rd_err_syndrome));  in gaudi2_hbm_sei_handle_read_err()
9293 		if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) &  in gaudi2_hbm_sei_handle_read_err()
9295 			dev_err_ratelimited(hdev->dev, "Beat%d ECC DERR: DM: %#x, Syndrome: %#x\n",  in gaudi2_hbm_sei_handle_read_err()
9297 						le32_to_cpu(rd_err_data->dbg_rd_err_dm),  in gaudi2_hbm_sei_handle_read_err()
9298 						le32_to_cpu(rd_err_data->dbg_rd_err_syndrome));  in gaudi2_hbm_sei_handle_read_err()
9303 		if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) &  in gaudi2_hbm_sei_handle_read_err()
9305 			dev_err_ratelimited(hdev->dev,  in gaudi2_hbm_sei_handle_read_err()
9308 					le32_to_cpu(rd_err_data->dbg_rd_err_dm),  in gaudi2_hbm_sei_handle_read_err()
9309 					(le32_to_cpu(rd_err_data->dbg_rd_err_misc) &  in gaudi2_hbm_sei_handle_read_err()
9315 		dev_err_ratelimited(hdev->dev, "Beat%d DQ data:\n", beat);  in gaudi2_hbm_sei_handle_read_err()
9316 		dev_err_ratelimited(hdev->dev, "\t0x%08x\n",  in gaudi2_hbm_sei_handle_read_err()
9317 					le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2]));  in gaudi2_hbm_sei_handle_read_err()
9318 		dev_err_ratelimited(hdev->dev, "\t0x%08x\n",  in gaudi2_hbm_sei_handle_read_err()
9319 					le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2 + 1]));  in gaudi2_hbm_sei_handle_read_err()
9328 	struct hbm_sei_wr_cmd_address *wr_cmd_addr = wr_par_err_data->dbg_last_wr_cmds;  in gaudi2_hbm_sei_print_wr_par_info()
9329 	u32 i, curr_addr, derr = wr_par_err_data->dbg_derr;  in gaudi2_hbm_sei_print_wr_par_info()
9331 	dev_err_ratelimited(hdev->dev, "WRITE PARITY ERROR count: %d\n", err_cnt);  in gaudi2_hbm_sei_print_wr_par_info()
9333 	dev_err_ratelimited(hdev->dev, "CK-0 DERR: 0x%02x, CK-1 DERR: 0x%02x\n",  in gaudi2_hbm_sei_print_wr_par_info()
9336 	/* JIRA H6-3286 - the following prints may not be valid */  in gaudi2_hbm_sei_print_wr_par_info()
9337 	dev_err_ratelimited(hdev->dev, "Last latched write commands addresses:\n");  in gaudi2_hbm_sei_print_wr_par_info()
9340 		dev_err_ratelimited(hdev->dev,  in gaudi2_hbm_sei_print_wr_par_info()
9353 	__le32 *col_cmd = ca_par_err_data->dbg_col;  in gaudi2_hbm_sei_print_ca_par_info()
9354 	__le16 *row_cmd = ca_par_err_data->dbg_row;  in gaudi2_hbm_sei_print_ca_par_info()
9357 	dev_err_ratelimited(hdev->dev, "CA ERROR count: %d\n", err_cnt);  in gaudi2_hbm_sei_print_ca_par_info()
9359 	dev_err_ratelimited(hdev->dev, "Last latched C&R bus commands:\n");  in gaudi2_hbm_sei_print_ca_par_info()
9361 		dev_err_ratelimited(hdev->dev, "cmd%u: ROW(0x%04x) COL(0x%05x)\n", i,  in gaudi2_hbm_sei_print_ca_par_info()
9373 	hbm_id = (event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 4;  in gaudi2_handle_hbm_mc_sei_err()
9374 	mc_id = ((event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 2) % 2;  in gaudi2_handle_hbm_mc_sei_err()
9376 	cause_idx = sei_data->hdr.sei_cause;  in gaudi2_handle_hbm_mc_sei_err()
9377 	if (cause_idx > GAUDI2_NUM_OF_HBM_SEI_CAUSE - 1) {  in gaudi2_handle_hbm_mc_sei_err()
9384 	gaudi2_print_event(hdev, event_type, !sei_data->hdr.is_critical,  in gaudi2_handle_hbm_mc_sei_err()
9385 		"System %s Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s",  in gaudi2_handle_hbm_mc_sei_err()
9386 		sei_data->hdr.is_critical ? "Critical" : "Non-critical",  in gaudi2_handle_hbm_mc_sei_err()
9387 		hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel,  in gaudi2_handle_hbm_mc_sei_err()
9390 	/* Print error-specific info */  in gaudi2_handle_hbm_mc_sei_err()
9397 		gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_even_info,  in gaudi2_handle_hbm_mc_sei_err()
9398 						le32_to_cpu(sei_data->hdr.cnt));  in gaudi2_handle_hbm_mc_sei_err()
9403 		gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_odd_info,  in gaudi2_handle_hbm_mc_sei_err()
9404 						le32_to_cpu(sei_data->hdr.cnt));  in gaudi2_handle_hbm_mc_sei_err()
9409 		gaudi2_hbm_sei_print_wr_par_info(hdev, &sei_data->wr_parity_info,  in gaudi2_handle_hbm_mc_sei_err()
9410 						le32_to_cpu(sei_data->hdr.cnt));  in gaudi2_handle_hbm_mc_sei_err()
9419 								&sei_data->read_err_info,  in gaudi2_handle_hbm_mc_sei_err()
9420 								le32_to_cpu(sei_data->hdr.cnt));  in gaudi2_handle_hbm_mc_sei_err()
9427 	require_hard_reset |= !!sei_data->hdr.is_critical;  in gaudi2_handle_hbm_mc_sei_err()
9438 		return 1;  in gaudi2_handle_hbm_cattrip()
9450 			dev_dbg(hdev->dev, "HBM spi event: notification cause(%s)\n",  in gaudi2_handle_hbm_mc_spi()
9462 	mutex_lock(&hdev->clk_throttling.lock);  in gaudi2_print_clk_change_info()
9466 		hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER;  in gaudi2_print_clk_change_info()
9467 		hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER;  in gaudi2_print_clk_change_info()
9468 		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get();  in gaudi2_print_clk_change_info()
9469 		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time;  in gaudi2_print_clk_change_info()
9470 		dev_dbg_ratelimited(hdev->dev, "Clock throttling due to power consumption\n");  in gaudi2_print_clk_change_info()
9474 		hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER;  in gaudi2_print_clk_change_info()
9475 		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get();  in gaudi2_print_clk_change_info()
9476 		dev_dbg_ratelimited(hdev->dev, "Power envelop is safe, back to optimal clock\n");  in gaudi2_print_clk_change_info()
9480 		hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL;  in gaudi2_print_clk_change_info()
9481 		hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL;  in gaudi2_print_clk_change_info()
9482 		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get();  in gaudi2_print_clk_change_info()
9483 		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time;  in gaudi2_print_clk_change_info()
9485 		dev_info_ratelimited(hdev->dev, "Clock throttling due to overheating\n");  in gaudi2_print_clk_change_info()
9489 		hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL;  in gaudi2_print_clk_change_info()
9490 		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get();  in gaudi2_print_clk_change_info()
9492 		dev_info_ratelimited(hdev->dev, "Thermal envelop is safe, back to optimal clock\n");  in gaudi2_print_clk_change_info()
9496 		dev_err(hdev->dev, "Received invalid clock change event %d\n", event_type);  in gaudi2_print_clk_change_info()
9500 	mutex_unlock(&hdev->clk_throttling.lock);  in gaudi2_print_clk_change_info()
9506 	struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];  in gaudi2_print_out_of_sync_info()
9510 		le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci),  in gaudi2_print_out_of_sync_info()
9511 		q->pi, atomic_read(&q->ci));  in gaudi2_print_out_of_sync_info()
9532 			"pcie msi-x gen denied due to vector num check failure, vec(0x%X)",  in gaudi2_handle_pcie_p2p_msix()
9547 	cause = le64_to_cpu(drain_data->intr_cause.intr_cause_data);  in gaudi2_handle_pcie_drain()
9550 		dev_err_ratelimited(hdev->dev, "PCIE AXI drain LBW completed\n");  in gaudi2_handle_pcie_drain()
9554 	if (cause & BIT_ULL(1)) {  in gaudi2_handle_pcie_drain()
9555 		dev_err_ratelimited(hdev->dev, "PCIE AXI drain HBW completed\n");  in gaudi2_handle_pcie_drain()
9569 			dev_err_ratelimited(hdev->dev, "PSOC %s completed\n",  in gaudi2_handle_psoc_drain()
9583 	struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ];  in gaudi2_print_cpu_pkt_failure_info()
9587 		le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci));  in gaudi2_print_cpu_pkt_failure_info()
9597 	intr_type = le32_to_cpu(data->intr_type);  in hl_arc_event_handle()
9598 	engine_id = le32_to_cpu(data->engine_id);  in hl_arc_event_handle()
9599 	payload = le64_to_cpu(data->payload);  in hl_arc_event_handle()
9607 				engine_id, intr_type, q->queue_index);  in hl_arc_event_handle()
9608 		return 1;  in hl_arc_event_handle()
9622 		index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP;  in event_id_to_engine_id()
9626 		index = event_type - GAUDI2_EVENT_TPC0_QM;  in event_id_to_engine_id()
9638 		index = 1;  in event_id_to_engine_id()
9668 		index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE;  in event_id_to_engine_id()
9672 		index = (event_type - GAUDI2_EVENT_DEC0_SPI) >> 1;  in event_id_to_engine_id()
9676 		index = event_type - GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE;  in event_id_to_engine_id()
9679 		index = event_type - GAUDI2_EVENT_NIC0_QM0;  in event_id_to_engine_id()
9682 		index = event_type - GAUDI2_EVENT_NIC0_BMON_SPMU;  in event_id_to_engine_id()
9685 		index = (event_type - GAUDI2_EVENT_TPC0_BMON_SPMU) >> 1;  in event_id_to_engine_id()
9738 			return GAUDI2_DCORE1_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE1_TPC0;  in event_id_to_engine_id()
9740 			return GAUDI2_DCORE2_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE2_TPC0;  in event_id_to_engine_id()
9742 			return GAUDI2_DCORE3_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE3_TPC0;  in event_id_to_engine_id()
9782 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_handle_eqe()
9788 	ctl = le32_to_cpu(eq_entry->hdr.ctl);  in gaudi2_handle_eqe()
9792 		dev_err(hdev->dev, "Event type %u exceeds maximum of %u",  in gaudi2_handle_eqe()
9793 				event_type, GAUDI2_EVENT_SIZE - 1);  in gaudi2_handle_eqe()
9797 	gaudi2->events_stat[event_type]++;  in gaudi2_handle_eqe()
9798 	gaudi2->events_stat_aggregate[event_type]++;  in gaudi2_handle_eqe()
9806 		reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);  in gaudi2_handle_eqe()
9807 		is_critical = eq_entry->ecc_data.is_critical;  in gaudi2_handle_eqe()
9839 		index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE;  in gaudi2_handle_eqe()
9841 					&eq_entry->razwi_with_intr_cause, &event_mask);  in gaudi2_handle_eqe()
9847 		index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP;  in gaudi2_handle_eqe()
9849 						&eq_entry->razwi_with_intr_cause, &event_mask);  in gaudi2_handle_eqe()
9855 		index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE;  in gaudi2_handle_eqe()
9885 		index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) /  in gaudi2_handle_eqe()
9886 			(GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR);  in gaudi2_handle_eqe()
9888 					&eq_entry->razwi_with_intr_cause, &event_mask);  in gaudi2_handle_eqe()
9902 		index = (event_type - GAUDI2_EVENT_DEC0_SPI) /  in gaudi2_handle_eqe()
9903 				(GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI);  in gaudi2_handle_eqe()
9912 		index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) /  in gaudi2_handle_eqe()
9913 				(GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE -  in gaudi2_handle_eqe()
9924 		index = (event_type - GAUDI2_EVENT_MME0_QMAN_SW_ERROR) /  in gaudi2_handle_eqe()
9925 				(GAUDI2_EVENT_MME1_QMAN_SW_ERROR -  in gaudi2_handle_eqe()
9935 		index = (event_type - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID) /  in gaudi2_handle_eqe()
9936 				(GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID -  in gaudi2_handle_eqe()
9945 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));  in gaudi2_handle_eqe()
9951 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));  in gaudi2_handle_eqe()
9957 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));  in gaudi2_handle_eqe()
9963 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data), &event_mask);  in gaudi2_handle_eqe()
9979 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));  in gaudi2_handle_eqe()
9986 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));  in gaudi2_handle_eqe()
9998 		if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) {  in gaudi2_handle_eqe()
10001 			is_critical = eq_entry->sei_data.hdr.is_critical;  in gaudi2_handle_eqe()
10008 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));  in gaudi2_handle_eqe()
10014 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));  in gaudi2_handle_eqe()
10019 		error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data);  in gaudi2_handle_eqe()
10022 		if (hl_fw_version_cmp(hdev, 1, 13, 0) >= 0)  in gaudi2_handle_eqe()
10028 				le64_to_cpu(eq_entry->intr_cause.intr_cause_data));  in gaudi2_handle_eqe()
10137 		gaudi2_print_out_of_sync_info(hdev, event_type, &eq_entry->pkt_sync_err);  in gaudi2_handle_eqe()
10146 		/* Do nothing- FW will handle it */  in gaudi2_handle_eqe()
10155 		index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE;  in gaudi2_handle_eqe()
10166 		dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n",  in gaudi2_handle_eqe()
10167 						le64_to_cpu(eq_entry->data[0]));  in gaudi2_handle_eqe()
10172 		dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n",  in gaudi2_handle_eqe()
10173 						le64_to_cpu(eq_entry->data[0]));  in gaudi2_handle_eqe()
10179 		gaudi2_print_cpu_pkt_failure_info(hdev, event_type, &eq_entry->pkt_sync_err);  in gaudi2_handle_eqe()
10186 		error_count = hl_arc_event_handle(hdev, event_type, &eq_entry->arc_data);  in gaudi2_handle_eqe()
10204 		dev_info_ratelimited(hdev->dev, "%s event received\n",  in gaudi2_handle_eqe()
10214 			dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n",  in gaudi2_handle_eqe()
10238 		if (hdev->hard_reset_on_fw_events ||  in gaudi2_handle_eqe()
10239 				(hdev->asic_prop.fw_security_enabled && is_critical))  in gaudi2_handle_eqe()
10253 	if (hdev->asic_prop.fw_security_enabled && is_critical) {  in gaudi2_handle_eqe()
10275 	ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_MEMSET_MASK, 1);  in gaudi2_memset_memory_chunk_using_edma_qm()
10276 	ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_WRCOMP_MASK, 1);  in gaudi2_memset_memory_chunk_using_edma_qm()
10277 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 1);  in gaudi2_memset_memory_chunk_using_edma_qm()
10279 	lin_dma_pkt->ctl = cpu_to_le32(ctl);  in gaudi2_memset_memory_chunk_using_edma_qm()
10280 	lin_dma_pkt->src_addr = cpu_to_le64(val);  in gaudi2_memset_memory_chunk_using_edma_qm()
10281 	lin_dma_pkt->dst_addr = cpu_to_le64(addr);  in gaudi2_memset_memory_chunk_using_edma_qm()
10282 	lin_dma_pkt->tsize = cpu_to_le32(size);  in gaudi2_memset_memory_chunk_using_edma_qm()
10287 		rc = hdev->asic_funcs->access_dev_mem(hdev, PCI_REGION_DRAM,  in gaudi2_memset_memory_chunk_using_edma_qm()
10291 			dev_err(hdev->dev, "Failed to copy lin_dma packet to HBM (%#llx)\n",  in gaudi2_memset_memory_chunk_using_edma_qm()
10299 		dev_err(hdev->dev, "Failed to send lin_dma packet to H/W queue %d\n",  in gaudi2_memset_memory_chunk_using_edma_qm()
10314 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_memset_device_memory()
10318 	if (prop->edma_enabled_mask == 0) {  in gaudi2_memset_device_memory()
10319 		dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n");  in gaudi2_memset_device_memory()
10320 		return -EIO;  in gaudi2_memset_device_memory()
10323 	sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4;  in gaudi2_memset_device_memory()
10326 	comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) |  in gaudi2_memset_device_memory()
10327 		FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1);  in gaudi2_memset_device_memory()
10328 	mmubp = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_MASK, 1) |  in gaudi2_memset_device_memory()
10329 		FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_MASK, 1);  in gaudi2_memset_device_memory()
10342 	if ((addr >= prop->dram_user_base_address) &&  in gaudi2_memset_device_memory()
10343 				(addr < prop->dram_user_base_address + cb_len))  in gaudi2_memset_device_memory()
10344 		cur_addr += (prop->dram_user_base_address + cb_len) - addr;  in gaudi2_memset_device_memory()
10348 		return -ENOMEM;  in gaudi2_memset_device_memory()
10351 	 * set mmu bypass for the scrubbing - all ddmas are configured the same so save  in gaudi2_memset_device_memory()
10362 			if (!(prop->edma_enabled_mask & BIT(edma_bit)))  in gaudi2_memset_device_memory()
10385 				if (!(prop->edma_enabled_mask & BIT(edma_bit)))  in gaudi2_memset_device_memory()
10388 				chunk_size = min_t(u64, SZ_2G, end_addr - cur_addr);  in gaudi2_memset_device_memory()
10392 					prop->dram_user_base_address + (dma_num * pkt_size),  in gaudi2_memset_device_memory()
10409 		dev_err(hdev->dev, "DMA Timeout during HBM scrubbing(sob: 0x%x, dma_num: 0x%x)\n",  in gaudi2_memset_device_memory()
10419 			if (!(prop->edma_enabled_mask & BIT(edma_bit)))  in gaudi2_memset_device_memory()
10435 		rc = hdev->asic_funcs->access_dev_mem(hdev, PCI_REGION_DRAM,  in gaudi2_memset_device_memory()
10436 			prop->dram_user_base_address + i,  in gaudi2_memset_device_memory()
10448 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_scrub_device_dram()
10449 	u64 size = prop->dram_end_address - prop->dram_user_base_address;  in gaudi2_scrub_device_dram()
10451 	rc = gaudi2_memset_device_memory(hdev, prop->dram_user_base_address, size, val);  in gaudi2_scrub_device_dram()
10454 		dev_err(hdev->dev, "Failed to scrub dram, address: 0x%llx size: %llu\n",  in gaudi2_scrub_device_dram()
10455 				prop->dram_user_base_address, size);  in gaudi2_scrub_device_dram()
10462 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_scrub_device_mem()
10463 	u64 val = hdev->memory_scrub_val;  in gaudi2_scrub_device_mem()
10466 	if (!hdev->memory_scrub)  in gaudi2_scrub_device_mem()
10470 	addr = prop->sram_user_base_address;  in gaudi2_scrub_device_mem()
10471 	size = hdev->pldm ? 0x10000 : (prop->sram_size - SRAM_USER_BASE_OFFSET);  in gaudi2_scrub_device_mem()
10472 	dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx, val: 0x%llx\n",  in gaudi2_scrub_device_mem()
10476 		dev_err(hdev->dev, "scrubbing SRAM failed (%d)\n", rc);  in gaudi2_scrub_device_mem()
10483 		dev_err(hdev->dev, "scrubbing DRAM failed (%d)\n", rc);  in gaudi2_scrub_device_mem()
10496 	offset = hdev->asic_prop.first_available_cq[0] * 4;  in gaudi2_restore_user_sm_registers()
10503 	size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 -  in gaudi2_restore_user_sm_registers()
10520 	size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0;  in gaudi2_restore_user_sm_registers()
10522 	for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {  in gaudi2_restore_user_sm_registers()
10538 	offset = hdev->asic_prop.first_available_user_mon[0] * 4;  in gaudi2_restore_user_sm_registers()
10540 	val = 1 << DCORE0_SYNC_MNGR_OBJS_MON_STATUS_PROT_SHIFT;  in gaudi2_restore_user_sm_registers()
10541 	size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - (mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset);  in gaudi2_restore_user_sm_registers()
10551 	size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0;  in gaudi2_restore_user_sm_registers()
10553 	for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {  in gaudi2_restore_user_sm_registers()
10560 	offset = hdev->asic_prop.first_available_user_sob[0] * 4;  in gaudi2_restore_user_sm_registers()
10563 	size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 -  in gaudi2_restore_user_sm_registers()
10570 	size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0;  in gaudi2_restore_user_sm_registers()
10572 	for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) {  in gaudi2_restore_user_sm_registers()
10631 	u64 block_base = cfg_ctx->base + block_idx * cfg_ctx->block_off;  in gaudi2_init_block_instances()
10635 	for (i = 0 ; i < cfg_ctx->instances ; i++) {  in gaudi2_init_block_instances()
10636 		seq = block_idx * cfg_ctx->instances + i;  in gaudi2_init_block_instances()
10639 		if (!(cfg_ctx->enabled_mask & BIT_ULL(seq)))  in gaudi2_init_block_instances()
10642 		cfg_ctx->instance_cfg_fn(hdev, block_base + i * cfg_ctx->instance_off,  in gaudi2_init_block_instances()
10643 					cfg_ctx->data);  in gaudi2_init_block_instances()
10652 	cfg_ctx->enabled_mask = mask;  in gaudi2_init_blocks_with_mask()
10654 	for (i = 0 ; i < cfg_ctx->blocks ; i++)  in gaudi2_init_blocks_with_mask()
10675 		dev_err(hdev->dev, "No ctx available\n");  in gaudi2_debugfs_read_dma()
10676 		return -EINVAL;  in gaudi2_debugfs_read_dma()
10683 		dev_err(hdev->dev, "Failed to allocate memory for KDMA read\n");  in gaudi2_debugfs_read_dma()
10684 		rc = -ENOMEM;  in gaudi2_debugfs_read_dma()
10692 		dev_err(hdev->dev, "Failed to reserve vmem on asic\n");  in gaudi2_debugfs_read_dma()
10693 		rc = -ENOMEM;  in gaudi2_debugfs_read_dma()
10698 	mutex_lock(&hdev->mmu_lock);  in gaudi2_debugfs_read_dma()
10702 		dev_err(hdev->dev, "Failed to create mapping on asic mmu\n");  in gaudi2_debugfs_read_dma()
10708 				      ctx->asid, reserved_va_base, SZ_2M);  in gaudi2_debugfs_read_dma()
10714 	mutex_unlock(&hdev->mmu_lock);  in gaudi2_debugfs_read_dma()
10717 	gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid);  in gaudi2_debugfs_read_dma()
10738 		size_left -= SZ_2M;  in gaudi2_debugfs_read_dma()
10743 	mutex_lock(&hdev->mmu_lock);  in gaudi2_debugfs_read_dma()
10750 				      ctx->asid, reserved_va_base, SZ_2M);  in gaudi2_debugfs_read_dma()
10753 	mutex_unlock(&hdev->mmu_lock);  in gaudi2_debugfs_read_dma()
10765 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_internal_cb_pool_init()
10768 	if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU))  in gaudi2_internal_cb_pool_init()
10771 	hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev,  in gaudi2_internal_cb_pool_init()
10773 								&hdev->internal_cb_pool_dma_addr,  in gaudi2_internal_cb_pool_init()
10776 	if (!hdev->internal_cb_pool_virt_addr)  in gaudi2_internal_cb_pool_init()
10777 		return -ENOMEM;  in gaudi2_internal_cb_pool_init()
10782 	hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1);  in gaudi2_internal_cb_pool_init()
10783 	if (!hdev->internal_cb_pool) {  in gaudi2_internal_cb_pool_init()
10784 		dev_err(hdev->dev, "Failed to create internal CB pool\n");  in gaudi2_internal_cb_pool_init()
10785 		rc = -ENOMEM;  in gaudi2_internal_cb_pool_init()
10789 	rc = gen_pool_add(hdev->internal_cb_pool, (uintptr_t) hdev->internal_cb_pool_virt_addr,  in gaudi2_internal_cb_pool_init()
10790 				HOST_SPACE_INTERNAL_CB_SZ, -1);  in gaudi2_internal_cb_pool_init()
10792 		dev_err(hdev->dev, "Failed to add memory to internal CB pool\n");  in gaudi2_internal_cb_pool_init()
10793 		rc = -EFAULT;  in gaudi2_internal_cb_pool_init()
10797 	hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST,  in gaudi2_internal_cb_pool_init()
10800 	if (!hdev->internal_cb_va_base) {  in gaudi2_internal_cb_pool_init()
10801 		rc = -ENOMEM;  in gaudi2_internal_cb_pool_init()
10805 	mutex_lock(&hdev->mmu_lock);  in gaudi2_internal_cb_pool_init()
10807 	rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr,  in gaudi2_internal_cb_pool_init()
10816 	mutex_unlock(&hdev->mmu_lock);  in gaudi2_internal_cb_pool_init()
10821 	hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);  in gaudi2_internal_cb_pool_init()
10823 	mutex_unlock(&hdev->mmu_lock);  in gaudi2_internal_cb_pool_init()
10824 	hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);  in gaudi2_internal_cb_pool_init()
10826 	gen_pool_destroy(hdev->internal_cb_pool);  in gaudi2_internal_cb_pool_init()
10828 	hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr,  in gaudi2_internal_cb_pool_init()
10829 					hdev->internal_cb_pool_dma_addr);  in gaudi2_internal_cb_pool_init()
10836 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_internal_cb_pool_fini()
10838 	if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU))  in gaudi2_internal_cb_pool_fini()
10841 	mutex_lock(&hdev->mmu_lock);  in gaudi2_internal_cb_pool_fini()
10842 	hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);  in gaudi2_internal_cb_pool_fini()
10843 	hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);  in gaudi2_internal_cb_pool_fini()
10845 	mutex_unlock(&hdev->mmu_lock);  in gaudi2_internal_cb_pool_fini()
10847 	gen_pool_destroy(hdev->internal_cb_pool);  in gaudi2_internal_cb_pool_fini()
10849 	hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr,  in gaudi2_internal_cb_pool_fini()
10850 					hdev->internal_cb_pool_dma_addr);  in gaudi2_internal_cb_pool_fini()
10861 	struct hl_device *hdev = ctx->hdev;  in gaudi2_map_virtual_msix_doorbell_memory()
10862 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_map_virtual_msix_doorbell_memory()
10863 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_map_virtual_msix_doorbell_memory()
10867 				gaudi2->virt_msix_db_dma_addr, prop->pmmu.page_size, true);  in gaudi2_map_virtual_msix_doorbell_memory()
10869 		dev_err(hdev->dev, "Failed to map VA %#llx for virtual MSI-X doorbell memory\n",  in gaudi2_map_virtual_msix_doorbell_memory()
10877 	struct hl_device *hdev = ctx->hdev;  in gaudi2_unmap_virtual_msix_doorbell_memory()
10878 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_unmap_virtual_msix_doorbell_memory()
10882 				prop->pmmu.page_size, true);  in gaudi2_unmap_virtual_msix_doorbell_memory()
10884 		dev_err(hdev->dev, "Failed to unmap VA %#llx of virtual MSI-X doorbell memory\n",  in gaudi2_unmap_virtual_msix_doorbell_memory()
10892 	if (ctx->asid == HL_KERNEL_ASID_ID)  in gaudi2_ctx_init()
10895 	rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid);  in gaudi2_ctx_init()
10902 	if (ctx->hdev->reset_upon_device_release)  in gaudi2_ctx_init()
10903 		gaudi2_restore_nic_qm_registers(ctx->hdev);  in gaudi2_ctx_init()
10905 		gaudi2_restore_user_registers(ctx->hdev);  in gaudi2_ctx_init()
10907 	rc = gaudi2_internal_cb_pool_init(ctx->hdev, ctx);  in gaudi2_ctx_init()
10913 		gaudi2_internal_cb_pool_fini(ctx->hdev, ctx);  in gaudi2_ctx_init()
10920 	if (ctx->asid == HL_KERNEL_ASID_ID)  in gaudi2_ctx_fini()
10923 	gaudi2_internal_cb_pool_fini(ctx->hdev, ctx);  in gaudi2_ctx_fini()
10930 	struct hl_device *hdev = cs->ctx->hdev;  in gaudi2_pre_schedule_cs()
10931 	int index = cs->sequence & (hdev->asic_prop.max_pending_cs - 1);  in gaudi2_pre_schedule_cs()
10942 	 * generates MSI-X interrupt.  in gaudi2_pre_schedule_cs()
10946 	mon_payload = (1 << CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT) |  in gaudi2_pre_schedule_cs()
10947 				(1 << CQ_ENTRY_READY_SHIFT) | index;  in gaudi2_pre_schedule_cs()
10950 				cs->jobs_cnt);  in gaudi2_pre_schedule_cs()
10966 	pkt = (struct packet_msg_short *) (uintptr_t) (cb->kernel_address + size);  in gaudi2_gen_signal_cb()
10969 	/* Inc by 1, Mode ADD */  in gaudi2_gen_signal_cb()
10970 	value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_SYNC_VAL_MASK, 1);  in gaudi2_gen_signal_cb()
10971 	value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_MOD_MASK, 1);  in gaudi2_gen_signal_cb()
10974 	ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 1); /* SOB base */  in gaudi2_gen_signal_cb()
10977 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1);  in gaudi2_gen_signal_cb()
10979 	pkt->value = cpu_to_le32(value);  in gaudi2_gen_signal_cb()
10980 	pkt->ctl = cpu_to_le32(ctl);  in gaudi2_gen_signal_cb()
10997 	pkt->value = cpu_to_le32(value);  in gaudi2_add_mon_msg_short()
10998 	pkt->ctl = cpu_to_le32(ctl);  in gaudi2_add_mon_msg_short()
11010 		dev_err(hdev->dev, "sob_base %u (mask %#x) is not valid\n", sob_base, sob_mask);  in gaudi2_add_arm_monitor_pkt()
11025 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1);  in gaudi2_add_arm_monitor_pkt()
11027 	pkt->value = cpu_to_le32(value);  in gaudi2_add_arm_monitor_pkt()
11028 	pkt->ctl = cpu_to_le32(ctl);  in gaudi2_add_arm_monitor_pkt()
11039 	cfg = FIELD_PREP(GAUDI2_PKT_FENCE_CFG_DEC_VAL_MASK, 1);  in gaudi2_add_fence_pkt()
11040 	cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_TARGET_VAL_MASK, 1);  in gaudi2_add_fence_pkt()
11045 	ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1);  in gaudi2_add_fence_pkt()
11047 	pkt->cfg = cpu_to_le32(cfg);  in gaudi2_add_fence_pkt()
11048 	pkt->ctl = cpu_to_le32(ctl);  in gaudi2_add_fence_pkt()
11055 	struct hl_cb *cb = prop->data;  in gaudi2_gen_wait_cb()
11056 	void *buf = (void *) (uintptr_t) (cb->kernel_address);  in gaudi2_gen_wait_cb()
11059 	u32 stream_index, size = prop->size;  in gaudi2_gen_wait_cb()
11062 	stream_index = prop->q_idx % 4;  in gaudi2_gen_wait_cb()
11063 	fence_addr = CFG_BASE + gaudi2_qm_blocks_bases[prop->q_idx] +  in gaudi2_gen_wait_cb()
11073 	msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + prop->mon_id * 4) -  in gaudi2_gen_wait_cb()
11079 	msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + prop->mon_id * 4) -  in gaudi2_gen_wait_cb()
11088 	msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + prop->mon_id * 4) -  in gaudi2_gen_wait_cb()
11091 	size += gaudi2_add_mon_msg_short(buf + size, 1, msg_addr_offset);  in gaudi2_gen_wait_cb()
11094 	msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + prop->mon_id * 4) - monitor_base;  in gaudi2_gen_wait_cb()
11096 	size += gaudi2_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, prop->sob_mask,  in gaudi2_gen_wait_cb()
11097 						prop->sob_val, msg_addr_offset);  in gaudi2_gen_wait_cb()
11109 	dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id);  in gaudi2_reset_sob()
11111 	WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, 0);  in gaudi2_reset_sob()
11113 	kref_init(&hw_sob->kref);  in gaudi2_reset_sob()
11136 	return -EINVAL;  in gaudi2_collective_wait_create_jobs()
11140  * hl_mmu_scramble - converts a dram (non power of 2) page-size aligned address
11141  *                   to DMMU page-size address (64MB) before mapping it in
11153  * PA1 0x3000000  VA1 0x9C000000  SVA1= (VA1/48M)*64M 0xD0000000  <- PA1/48M 0x1
11154  * PA2 0x9000000  VA2 0x9F000000  SVA2= (VA2/48M)*64M 0xD4000000  <- PA2/48M 0x3
11159 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_mmu_scramble_addr()
11167 		divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE;  in gaudi2_mmu_scramble_addr()
11179 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_mmu_descramble_addr()
11187 		divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE;  in gaudi2_mmu_descramble_addr()
11203 		dev_err(hdev->dev, "Unexpected core number %d for DEC\n", core_id);  in gaudi2_get_dec_base_addr()
11224 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_get_hw_block_id()
11228 		if (block_addr == CFG_BASE + gaudi2->mapped_blocks[i].address) {  in gaudi2_get_hw_block_id()
11231 				*block_size = gaudi2->mapped_blocks[i].size;  in gaudi2_get_hw_block_id()
11236 	dev_err(hdev->dev, "Invalid block address %#llx", block_addr);  in gaudi2_get_hw_block_id()
11238 	return -EINVAL;  in gaudi2_get_hw_block_id()
11244 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_block_mmap()
11250 		dev_err(hdev->dev, "Invalid block id %u", block_id);  in gaudi2_block_mmap()
11251 		return -EINVAL;  in gaudi2_block_mmap()
11255 	if (block_size != gaudi2->mapped_blocks[block_id].size) {  in gaudi2_block_mmap()
11256 		dev_err(hdev->dev, "Invalid block size %u", block_size);  in gaudi2_block_mmap()
11257 		return -EINVAL;  in gaudi2_block_mmap()
11260 	offset_in_bar = CFG_BASE + gaudi2->mapped_blocks[block_id].address - STM_FLASH_BASE_ADDR;  in gaudi2_block_mmap()
11262 	address = pci_resource_start(hdev->pdev, SRAM_CFG_BAR_ID) + offset_in_bar;  in gaudi2_block_mmap()
11267 	rc = remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT,  in gaudi2_block_mmap()
11268 			block_size, vma->vm_page_prot);  in gaudi2_block_mmap()
11270 		dev_err(hdev->dev, "remap_pfn_range error %d", rc);  in gaudi2_block_mmap()
11277 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_enable_events_from_fw()
11279 	struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;  in gaudi2_enable_events_from_fw()
11280 	u32 irq_handler_offset = le32_to_cpu(dyn_regs->gic_host_ints_irq);  in gaudi2_enable_events_from_fw()
11282 	if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)  in gaudi2_enable_events_from_fw()
11342 		return -EINVAL;  in gaudi2_get_mmu_base()
11351 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_ack_mmu_error()
11354 	if (!(gaudi2->hw_cap_initialized & mmu_id))  in gaudi2_ack_mmu_error()
11404 	default: return -EINVAL;  in gaudi2_map_pll_idx_to_fw_idx()
11447 	hdev->state_dump_specs.props = gaudi2_state_dump_specs_props;  in gaudi2_state_dump_init()
11448 	hdev->state_dump_specs.funcs = gaudi2_state_dump_funcs;  in gaudi2_state_dump_init()
11471 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in gaudi2_mmu_get_real_page_size()
11475 		if (page_size % mmu_prop->page_size)  in gaudi2_mmu_get_real_page_size()
11478 		*real_page_size = mmu_prop->page_size;  in gaudi2_mmu_get_real_page_size()
11482 	if ((page_size % prop->dram_page_size) || (prop->dram_page_size > mmu_prop->page_size))  in gaudi2_mmu_get_real_page_size()
11493 	*real_page_size = prop->dram_page_size;  in gaudi2_mmu_get_real_page_size()
11498 	dev_err(hdev->dev, "page size of 0x%X is not 0x%X aligned, can't map\n",  in gaudi2_mmu_get_real_page_size()
11499 							page_size, mmu_prop->page_size >> 10);  in gaudi2_mmu_get_real_page_size()
11500 	return -EFAULT;  in gaudi2_mmu_get_real_page_size()
11505 	return -EOPNOTSUPP;  in gaudi2_get_monitor_dump()
11510 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_send_device_activity()
11512 	if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q))  in gaudi2_send_device_activity()
11520 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_read_pte()
11523 	if (hdev->reset_info.hard_reset_pending)  in gaudi2_read_pte()
11526 	val = readq(hdev->pcie_bar[DRAM_BAR_ID] + (addr - gaudi2->dram_bar_cur_addr));  in gaudi2_read_pte()
11533 	struct gaudi2_device *gaudi2 = hdev->asic_specific;  in gaudi2_write_pte()
11535 	if (hdev->reset_info.hard_reset_pending)  in gaudi2_write_pte()
11538 	writeq(val, hdev->pcie_bar[DRAM_BAR_ID] + (addr - gaudi2->dram_bar_cur_addr));  in gaudi2_write_pte()
11643 	hdev->asic_funcs = &gaudi2_funcs;  in gaudi2_set_asic_funcs()