Lines Matching refs:hdev
355 static int goya_mmu_clear_pgt_range(struct hl_device *hdev);
356 static int goya_mmu_set_dram_default_page(struct hl_device *hdev);
357 static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev);
358 static void goya_mmu_prepare(struct hl_device *hdev, u32 asid);
360 int goya_set_fixed_properties(struct hl_device *hdev) in goya_set_fixed_properties() argument
362 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_set_fixed_properties()
411 if (hdev->pldm) in goya_set_fixed_properties()
508 static int goya_pci_bars_map(struct hl_device *hdev) in goya_pci_bars_map() argument
514 rc = hl_pci_bars_map(hdev, name, is_wc); in goya_pci_bars_map()
518 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_pci_bars_map()
524 static u64 goya_set_ddr_bar_base(struct hl_device *hdev, u64 addr) in goya_set_ddr_bar_base() argument
526 struct goya_device *goya = hdev->asic_specific; in goya_set_ddr_bar_base()
538 rc = hl_pci_set_inbound_region(hdev, 1, &pci_region); in goya_set_ddr_bar_base()
558 static int goya_init_iatu(struct hl_device *hdev) in goya_init_iatu() argument
564 if (hdev->asic_prop.iatu_done_by_fw) in goya_init_iatu()
571 rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region); in goya_init_iatu()
579 rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region); in goya_init_iatu()
586 rc = hl_pci_set_outbound_region(hdev, &outbound_region); in goya_init_iatu()
592 static enum hl_device_hw_state goya_get_hw_state(struct hl_device *hdev) in goya_get_hw_state() argument
608 static int goya_early_init(struct hl_device *hdev) in goya_early_init() argument
610 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_early_init()
611 struct pci_dev *pdev = hdev->pdev; in goya_early_init()
616 rc = goya_set_fixed_properties(hdev); in goya_early_init()
618 dev_err(hdev->dev, "Failed to get fixed properties\n"); in goya_early_init()
626 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", in goya_early_init()
635 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", in goya_early_init()
642 hdev->dram_pci_bar_start = pci_resource_start(pdev, DDR_BAR_ID); in goya_early_init()
645 if (hdev->asic_prop.fw_security_enabled) { in goya_early_init()
646 hdev->asic_prop.iatu_done_by_fw = true; in goya_early_init()
650 rc = hl_pci_elbi_read(hdev, CFG_BASE + mmCPU_BOOT_DEV_STS0, in goya_early_init()
658 hdev->asic_prop.iatu_done_by_fw = true; in goya_early_init()
661 rc = hl_pci_init(hdev); in goya_early_init()
668 rc = hl_fw_read_preboot_status(hdev); in goya_early_init()
670 if (hdev->reset_on_preboot_fail) in goya_early_init()
672 hdev->asic_funcs->hw_fini(hdev, true, false); in goya_early_init()
676 if (goya_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) { in goya_early_init()
677 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n"); in goya_early_init()
678 rc = hdev->asic_funcs->hw_fini(hdev, true, false); in goya_early_init()
680 dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc); in goya_early_init()
685 if (!hdev->pldm) { in goya_early_init()
688 dev_warn(hdev->dev, in goya_early_init()
695 hl_pci_fini(hdev); in goya_early_init()
697 kfree(hdev->asic_prop.hw_queues_props); in goya_early_init()
709 static int goya_early_fini(struct hl_device *hdev) in goya_early_fini() argument
711 kfree(hdev->asic_prop.hw_queues_props); in goya_early_fini()
712 hl_pci_fini(hdev); in goya_early_fini()
717 static void goya_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid) in goya_mmu_prepare_reg() argument
724 static void goya_qman0_set_security(struct hl_device *hdev, bool secure) in goya_qman0_set_security() argument
726 struct goya_device *goya = hdev->asic_specific; in goya_qman0_set_security()
745 static void goya_fetch_psoc_frequency(struct hl_device *hdev) in goya_fetch_psoc_frequency() argument
747 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_fetch_psoc_frequency()
752 if (hdev->asic_prop.fw_security_enabled) { in goya_fetch_psoc_frequency()
753 struct goya_device *goya = hdev->asic_specific; in goya_fetch_psoc_frequency()
758 rc = hl_fw_cpucp_pll_info_get(hdev, HL_GOYA_PCI_PLL, in goya_fetch_psoc_frequency()
787 dev_warn(hdev->dev, in goya_fetch_psoc_frequency()
814 int goya_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq) in goya_set_frequency() argument
816 struct goya_device *goya = hdev->asic_specific; in goya_set_frequency()
822 dev_dbg(hdev->dev, "Changing device frequency to %s\n", in goya_set_frequency()
825 goya_set_pll_profile(hdev, freq); in goya_set_frequency()
837 struct hl_device *hdev = goya_work->hdev; in goya_set_freq_to_low_job() local
839 mutex_lock(&hdev->fpriv_list_lock); in goya_set_freq_to_low_job()
841 if (!hdev->is_compute_ctx_active) in goya_set_freq_to_low_job()
842 goya_set_frequency(hdev, PLL_LOW); in goya_set_freq_to_low_job()
844 mutex_unlock(&hdev->fpriv_list_lock); in goya_set_freq_to_low_job()
850 int goya_late_init(struct hl_device *hdev) in goya_late_init() argument
852 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_late_init()
853 struct goya_device *goya = hdev->asic_specific; in goya_late_init()
856 goya_fetch_psoc_frequency(hdev); in goya_late_init()
858 rc = goya_mmu_clear_pgt_range(hdev); in goya_late_init()
860 dev_err(hdev->dev, in goya_late_init()
865 rc = goya_mmu_set_dram_default_page(hdev); in goya_late_init()
867 dev_err(hdev->dev, "Failed to set DRAM default page %d\n", rc); in goya_late_init()
871 rc = goya_mmu_add_mappings_for_device_cpu(hdev); in goya_late_init()
875 rc = goya_init_cpu_queues(hdev); in goya_late_init()
879 rc = goya_test_cpu_queue(hdev); in goya_late_init()
883 rc = goya_cpucp_info_get(hdev); in goya_late_init()
885 dev_err(hdev->dev, "Failed to get cpucp info %d\n", rc); in goya_late_init()
895 rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS, 0x0); in goya_late_init()
904 goya_set_pll_profile(hdev, PLL_LOW); in goya_late_init()
919 void goya_late_fini(struct hl_device *hdev) in goya_late_fini() argument
921 struct goya_device *goya = hdev->asic_specific; in goya_late_fini()
925 hl_hwmon_release_resources(hdev); in goya_late_fini()
928 static void goya_set_pci_memory_regions(struct hl_device *hdev) in goya_set_pci_memory_regions() argument
930 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_set_pci_memory_regions()
934 region = &hdev->pci_mem_region[PCI_REGION_CFG]; in goya_set_pci_memory_regions()
943 region = &hdev->pci_mem_region[PCI_REGION_SRAM]; in goya_set_pci_memory_regions()
952 region = &hdev->pci_mem_region[PCI_REGION_DRAM]; in goya_set_pci_memory_regions()
954 region->region_size = hdev->asic_prop.dram_size; in goya_set_pci_memory_regions()
967 static int goya_sw_init(struct hl_device *hdev) in goya_sw_init() argument
984 hdev->asic_specific = goya; in goya_sw_init()
987 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), in goya_sw_init()
988 &hdev->pdev->dev, GOYA_DMA_POOL_BLK_SIZE, 8, 0); in goya_sw_init()
989 if (!hdev->dma_pool) { in goya_sw_init()
990 dev_err(hdev->dev, "failed to create DMA pool\n"); in goya_sw_init()
995 hdev->cpu_accessible_dma_mem = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, in goya_sw_init()
996 &hdev->cpu_accessible_dma_address, in goya_sw_init()
999 if (!hdev->cpu_accessible_dma_mem) { in goya_sw_init()
1004 dev_dbg(hdev->dev, "cpu accessible memory at bus address %pad\n", in goya_sw_init()
1005 &hdev->cpu_accessible_dma_address); in goya_sw_init()
1007 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); in goya_sw_init()
1008 if (!hdev->cpu_accessible_dma_pool) { in goya_sw_init()
1009 dev_err(hdev->dev, in goya_sw_init()
1015 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, in goya_sw_init()
1016 (uintptr_t) hdev->cpu_accessible_dma_mem, in goya_sw_init()
1019 dev_err(hdev->dev, in goya_sw_init()
1026 hdev->supports_coresight = true; in goya_sw_init()
1027 hdev->asic_prop.supports_compute_reset = true; in goya_sw_init()
1028 hdev->asic_prop.allow_inference_soft_reset = true; in goya_sw_init()
1029 hdev->supports_wait_for_multi_cs = false; in goya_sw_init()
1030 hdev->supports_ctx_switch = true; in goya_sw_init()
1032 hdev->asic_funcs->set_pci_memory_regions(hdev); in goya_sw_init()
1040 goya->goya_work->hdev = hdev; in goya_sw_init()
1046 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in goya_sw_init()
1048 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, in goya_sw_init()
1049 hdev->cpu_accessible_dma_address); in goya_sw_init()
1051 dma_pool_destroy(hdev->dma_pool); in goya_sw_init()
1064 static int goya_sw_fini(struct hl_device *hdev) in goya_sw_fini() argument
1066 struct goya_device *goya = hdev->asic_specific; in goya_sw_fini()
1068 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in goya_sw_fini()
1070 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, in goya_sw_fini()
1071 hdev->cpu_accessible_dma_address); in goya_sw_fini()
1073 dma_pool_destroy(hdev->dma_pool); in goya_sw_fini()
1081 static void goya_init_dma_qman(struct hl_device *hdev, int dma_id, in goya_init_dma_qman() argument
1084 struct goya_device *goya = hdev->asic_specific; in goya_init_dma_qman()
1126 if (hdev->stop_on_err) in goya_init_dma_qman()
1133 static void goya_init_dma_ch(struct hl_device *hdev, int dma_id) in goya_init_dma_ch() argument
1167 void goya_init_dma_qmans(struct hl_device *hdev) in goya_init_dma_qmans() argument
1169 struct goya_device *goya = hdev->asic_specific; in goya_init_dma_qmans()
1176 q = &hdev->kernel_queues[0]; in goya_init_dma_qmans()
1180 goya_init_dma_qman(hdev, i, q->bus_address); in goya_init_dma_qmans()
1181 goya_init_dma_ch(hdev, i); in goya_init_dma_qmans()
1193 static void goya_disable_external_queues(struct hl_device *hdev) in goya_disable_external_queues() argument
1195 struct goya_device *goya = hdev->asic_specific; in goya_disable_external_queues()
1207 static int goya_stop_queue(struct hl_device *hdev, u32 cfg_reg, in goya_stop_queue() argument
1220 hdev, in goya_stop_queue()
1233 hdev, in goya_stop_queue()
1241 dev_err(hdev->dev, in goya_stop_queue()
1257 static int goya_stop_external_queues(struct hl_device *hdev) in goya_stop_external_queues() argument
1261 struct goya_device *goya = hdev->asic_specific; in goya_stop_external_queues()
1266 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
1272 dev_err(hdev->dev, "failed to stop DMA QMAN 0\n"); in goya_stop_external_queues()
1276 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
1282 dev_err(hdev->dev, "failed to stop DMA QMAN 1\n"); in goya_stop_external_queues()
1286 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
1292 dev_err(hdev->dev, "failed to stop DMA QMAN 2\n"); in goya_stop_external_queues()
1296 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
1302 dev_err(hdev->dev, "failed to stop DMA QMAN 3\n"); in goya_stop_external_queues()
1306 rc = goya_stop_queue(hdev, in goya_stop_external_queues()
1312 dev_err(hdev->dev, "failed to stop DMA QMAN 4\n"); in goya_stop_external_queues()
1327 int goya_init_cpu_queues(struct hl_device *hdev) in goya_init_cpu_queues() argument
1329 struct goya_device *goya = hdev->asic_specific; in goya_init_cpu_queues()
1330 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_init_cpu_queues()
1333 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ]; in goya_init_cpu_queues()
1336 if (!hdev->cpu_queues_enable) in goya_init_cpu_queues()
1342 eq = &hdev->event_queue; in goya_init_cpu_queues()
1370 hdev, in goya_init_cpu_queues()
1378 dev_err(hdev->dev, in goya_init_cpu_queues()
1394 static void goya_set_pll_refclk(struct hl_device *hdev) in goya_set_pll_refclk() argument
1432 static void goya_disable_clk_rlx(struct hl_device *hdev) in goya_disable_clk_rlx() argument
1438 static void _goya_tpc_mbist_workaround(struct hl_device *hdev, u8 tpc_id) in _goya_tpc_mbist_workaround() argument
1456 dev_warn(hdev->dev, "TPC%d MBIST ACTIVE is not cleared\n", in _goya_tpc_mbist_workaround()
1476 hdev, in _goya_tpc_mbist_workaround()
1484 dev_err(hdev->dev, in _goya_tpc_mbist_workaround()
1503 static void goya_tpc_mbist_workaround(struct hl_device *hdev) in goya_tpc_mbist_workaround() argument
1505 struct goya_device *goya = hdev->asic_specific; in goya_tpc_mbist_workaround()
1508 if (hdev->pldm) in goya_tpc_mbist_workaround()
1517 _goya_tpc_mbist_workaround(hdev, i); in goya_tpc_mbist_workaround()
1530 static void goya_init_golden_registers(struct hl_device *hdev) in goya_init_golden_registers() argument
1532 struct goya_device *goya = hdev->asic_specific; in goya_init_golden_registers()
1825 static void goya_init_mme_qman(struct hl_device *hdev) in goya_init_mme_qman() argument
1842 qman_base_addr = hdev->asic_prop.sram_base_address + in goya_init_mme_qman()
1875 static void goya_init_mme_cmdq(struct hl_device *hdev) in goya_init_mme_cmdq() argument
1911 void goya_init_mme_qmans(struct hl_device *hdev) in goya_init_mme_qmans() argument
1913 struct goya_device *goya = hdev->asic_specific; in goya_init_mme_qmans()
1925 goya_init_mme_qman(hdev); in goya_init_mme_qmans()
1926 goya_init_mme_cmdq(hdev); in goya_init_mme_qmans()
1931 static void goya_init_tpc_qman(struct hl_device *hdev, u32 base_off, int tpc_id) in goya_init_tpc_qman() argument
1949 qman_base_addr = hdev->asic_prop.sram_base_address + base_off; in goya_init_tpc_qman()
1981 static void goya_init_tpc_cmdq(struct hl_device *hdev, int tpc_id) in goya_init_tpc_cmdq() argument
2018 void goya_init_tpc_qmans(struct hl_device *hdev) in goya_init_tpc_qmans() argument
2020 struct goya_device *goya = hdev->asic_specific; in goya_init_tpc_qmans()
2039 goya_init_tpc_qman(hdev, TPC0_QMAN_BASE_OFFSET, 0); in goya_init_tpc_qmans()
2040 goya_init_tpc_qman(hdev, TPC1_QMAN_BASE_OFFSET, 1); in goya_init_tpc_qmans()
2041 goya_init_tpc_qman(hdev, TPC2_QMAN_BASE_OFFSET, 2); in goya_init_tpc_qmans()
2042 goya_init_tpc_qman(hdev, TPC3_QMAN_BASE_OFFSET, 3); in goya_init_tpc_qmans()
2043 goya_init_tpc_qman(hdev, TPC4_QMAN_BASE_OFFSET, 4); in goya_init_tpc_qmans()
2044 goya_init_tpc_qman(hdev, TPC5_QMAN_BASE_OFFSET, 5); in goya_init_tpc_qmans()
2045 goya_init_tpc_qman(hdev, TPC6_QMAN_BASE_OFFSET, 6); in goya_init_tpc_qmans()
2046 goya_init_tpc_qman(hdev, TPC7_QMAN_BASE_OFFSET, 7); in goya_init_tpc_qmans()
2049 goya_init_tpc_cmdq(hdev, i); in goya_init_tpc_qmans()
2060 static void goya_disable_internal_queues(struct hl_device *hdev) in goya_disable_internal_queues() argument
2062 struct goya_device *goya = hdev->asic_specific; in goya_disable_internal_queues()
2107 static int goya_stop_internal_queues(struct hl_device *hdev) in goya_stop_internal_queues() argument
2109 struct goya_device *goya = hdev->asic_specific; in goya_stop_internal_queues()
2121 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2127 dev_err(hdev->dev, "failed to stop MME QMAN\n"); in goya_stop_internal_queues()
2131 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2137 dev_err(hdev->dev, "failed to stop MME CMDQ\n"); in goya_stop_internal_queues()
2145 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2151 dev_err(hdev->dev, "failed to stop TPC 0 QMAN\n"); in goya_stop_internal_queues()
2155 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2161 dev_err(hdev->dev, "failed to stop TPC 0 CMDQ\n"); in goya_stop_internal_queues()
2165 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2171 dev_err(hdev->dev, "failed to stop TPC 1 QMAN\n"); in goya_stop_internal_queues()
2175 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2181 dev_err(hdev->dev, "failed to stop TPC 1 CMDQ\n"); in goya_stop_internal_queues()
2185 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2191 dev_err(hdev->dev, "failed to stop TPC 2 QMAN\n"); in goya_stop_internal_queues()
2195 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2201 dev_err(hdev->dev, "failed to stop TPC 2 CMDQ\n"); in goya_stop_internal_queues()
2205 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2211 dev_err(hdev->dev, "failed to stop TPC 3 QMAN\n"); in goya_stop_internal_queues()
2215 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2221 dev_err(hdev->dev, "failed to stop TPC 3 CMDQ\n"); in goya_stop_internal_queues()
2225 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2231 dev_err(hdev->dev, "failed to stop TPC 4 QMAN\n"); in goya_stop_internal_queues()
2235 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2241 dev_err(hdev->dev, "failed to stop TPC 4 CMDQ\n"); in goya_stop_internal_queues()
2245 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2251 dev_err(hdev->dev, "failed to stop TPC 5 QMAN\n"); in goya_stop_internal_queues()
2255 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2261 dev_err(hdev->dev, "failed to stop TPC 5 CMDQ\n"); in goya_stop_internal_queues()
2265 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2271 dev_err(hdev->dev, "failed to stop TPC 6 QMAN\n"); in goya_stop_internal_queues()
2275 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2281 dev_err(hdev->dev, "failed to stop TPC 6 CMDQ\n"); in goya_stop_internal_queues()
2285 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2291 dev_err(hdev->dev, "failed to stop TPC 7 QMAN\n"); in goya_stop_internal_queues()
2295 rc = goya_stop_queue(hdev, in goya_stop_internal_queues()
2301 dev_err(hdev->dev, "failed to stop TPC 7 CMDQ\n"); in goya_stop_internal_queues()
2308 static void goya_dma_stall(struct hl_device *hdev) in goya_dma_stall() argument
2310 struct goya_device *goya = hdev->asic_specific; in goya_dma_stall()
2322 static void goya_tpc_stall(struct hl_device *hdev) in goya_tpc_stall() argument
2324 struct goya_device *goya = hdev->asic_specific; in goya_tpc_stall()
2339 static void goya_mme_stall(struct hl_device *hdev) in goya_mme_stall() argument
2341 struct goya_device *goya = hdev->asic_specific; in goya_mme_stall()
2349 static int goya_enable_msix(struct hl_device *hdev) in goya_enable_msix() argument
2351 struct goya_device *goya = hdev->asic_specific; in goya_enable_msix()
2352 int cq_cnt = hdev->asic_prop.completion_queues_count; in goya_enable_msix()
2358 rc = pci_alloc_irq_vectors(hdev->pdev, GOYA_MSIX_ENTRIES, in goya_enable_msix()
2361 dev_err(hdev->dev, in goya_enable_msix()
2368 irq = pci_irq_vector(hdev->pdev, i); in goya_enable_msix()
2370 &hdev->completion_queue[i]); in goya_enable_msix()
2372 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in goya_enable_msix()
2377 irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX); in goya_enable_msix()
2381 &hdev->event_queue); in goya_enable_msix()
2383 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in goya_enable_msix()
2392 free_irq(pci_irq_vector(hdev->pdev, i), in goya_enable_msix()
2393 &hdev->completion_queue[i]); in goya_enable_msix()
2395 pci_free_irq_vectors(hdev->pdev); in goya_enable_msix()
2399 static void goya_sync_irqs(struct hl_device *hdev) in goya_sync_irqs() argument
2401 struct goya_device *goya = hdev->asic_specific; in goya_sync_irqs()
2408 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in goya_sync_irqs()
2409 synchronize_irq(pci_irq_vector(hdev->pdev, i)); in goya_sync_irqs()
2411 synchronize_irq(pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX)); in goya_sync_irqs()
2414 static void goya_disable_msix(struct hl_device *hdev) in goya_disable_msix() argument
2416 struct goya_device *goya = hdev->asic_specific; in goya_disable_msix()
2422 goya_sync_irqs(hdev); in goya_disable_msix()
2424 irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX); in goya_disable_msix()
2425 free_irq(irq, &hdev->event_queue); in goya_disable_msix()
2427 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) { in goya_disable_msix()
2428 irq = pci_irq_vector(hdev->pdev, i); in goya_disable_msix()
2429 free_irq(irq, &hdev->completion_queue[i]); in goya_disable_msix()
2432 pci_free_irq_vectors(hdev->pdev); in goya_disable_msix()
2437 static void goya_enable_timestamp(struct hl_device *hdev) in goya_enable_timestamp() argument
2450 static void goya_disable_timestamp(struct hl_device *hdev) in goya_disable_timestamp() argument
2456 static void goya_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset) in goya_halt_engines() argument
2460 if (hdev->pldm) in goya_halt_engines()
2465 goya_stop_external_queues(hdev); in goya_halt_engines()
2466 goya_stop_internal_queues(hdev); in goya_halt_engines()
2470 goya_dma_stall(hdev); in goya_halt_engines()
2471 goya_tpc_stall(hdev); in goya_halt_engines()
2472 goya_mme_stall(hdev); in goya_halt_engines()
2476 goya_disable_external_queues(hdev); in goya_halt_engines()
2477 goya_disable_internal_queues(hdev); in goya_halt_engines()
2479 goya_disable_timestamp(hdev); in goya_halt_engines()
2482 goya_disable_msix(hdev); in goya_halt_engines()
2483 goya_mmu_remove_device_cpu_mappings(hdev); in goya_halt_engines()
2485 goya_sync_irqs(hdev); in goya_halt_engines()
2497 static int goya_load_firmware_to_device(struct hl_device *hdev) in goya_load_firmware_to_device() argument
2501 dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET; in goya_load_firmware_to_device()
2503 return hl_fw_load_fw_to_device(hdev, GOYA_LINUX_FW_FILE, dst, 0, 0); in goya_load_firmware_to_device()
2514 static int goya_load_boot_fit_to_device(struct hl_device *hdev) in goya_load_boot_fit_to_device() argument
2518 dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + BOOT_FIT_SRAM_OFFSET; in goya_load_boot_fit_to_device()
2520 return hl_fw_load_fw_to_device(hdev, GOYA_BOOT_FIT_FILE, dst, 0, 0); in goya_load_boot_fit_to_device()
2523 static void goya_init_dynamic_firmware_loader(struct hl_device *hdev) in goya_init_dynamic_firmware_loader() argument
2528 dynamic_loader = &hdev->fw_loader.dynamic_loader; in goya_init_dynamic_firmware_loader()
2546 static void goya_init_static_firmware_loader(struct hl_device *hdev) in goya_init_static_firmware_loader() argument
2550 static_loader = &hdev->fw_loader.static_loader; in goya_init_static_firmware_loader()
2566 static void goya_init_firmware_preload_params(struct hl_device *hdev) in goya_init_firmware_preload_params() argument
2568 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load; in goya_init_firmware_preload_params()
2578 static void goya_init_firmware_loader(struct hl_device *hdev) in goya_init_firmware_loader() argument
2580 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_init_firmware_loader()
2581 struct fw_load_mgr *fw_loader = &hdev->fw_loader; in goya_init_firmware_loader()
2594 goya_init_dynamic_firmware_loader(hdev); in goya_init_firmware_loader()
2596 goya_init_static_firmware_loader(hdev); in goya_init_firmware_loader()
2599 static int goya_init_cpu(struct hl_device *hdev) in goya_init_cpu() argument
2601 struct goya_device *goya = hdev->asic_specific; in goya_init_cpu()
2604 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) in goya_init_cpu()
2614 if (goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) { in goya_init_cpu()
2615 dev_err(hdev->dev, in goya_init_cpu()
2620 rc = hl_fw_init_cpu(hdev); in goya_init_cpu()
2630 static int goya_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid, in goya_mmu_update_asid_hop0_addr() argument
2636 if (hdev->pldm) in goya_mmu_update_asid_hop0_addr()
2646 hdev, in goya_mmu_update_asid_hop0_addr()
2654 dev_err(hdev->dev, in goya_mmu_update_asid_hop0_addr()
2662 int goya_mmu_init(struct hl_device *hdev) in goya_mmu_init() argument
2664 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_init()
2665 struct goya_device *goya = hdev->asic_specific; in goya_mmu_init()
2672 hdev->dram_default_page_mapping = true; in goya_mmu_init()
2678 rc = goya_mmu_update_asid_hop0_addr(hdev, i, hop0_addr); in goya_mmu_init()
2680 dev_err(hdev->dev, in goya_mmu_init()
2697 hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR | MMU_OP_PHYS_PACK); in goya_mmu_init()
2716 static int goya_hw_init(struct hl_device *hdev) in goya_hw_init() argument
2718 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_hw_init()
2732 rc = goya_init_cpu(hdev); in goya_hw_init()
2734 dev_err(hdev->dev, "failed to initialize CPU\n"); in goya_hw_init()
2738 goya_tpc_mbist_workaround(hdev); in goya_hw_init()
2740 goya_init_golden_registers(hdev); in goya_hw_init()
2746 if (goya_set_ddr_bar_base(hdev, (MMU_PAGE_TABLES_ADDR & in goya_hw_init()
2748 dev_err(hdev->dev, in goya_hw_init()
2753 rc = goya_mmu_init(hdev); in goya_hw_init()
2757 goya_init_security(hdev); in goya_hw_init()
2759 goya_init_dma_qmans(hdev); in goya_hw_init()
2761 goya_init_mme_qmans(hdev); in goya_hw_init()
2763 goya_init_tpc_qmans(hdev); in goya_hw_init()
2765 goya_enable_timestamp(hdev); in goya_hw_init()
2768 rc = goya_enable_msix(hdev); in goya_hw_init()
2778 goya_disable_internal_queues(hdev); in goya_hw_init()
2779 goya_disable_external_queues(hdev); in goya_hw_init()
2784 static int goya_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset) in goya_hw_fini() argument
2786 struct goya_device *goya = hdev->asic_specific; in goya_hw_fini()
2789 if (hdev->pldm) { in goya_hw_fini()
2807 goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE); in goya_hw_fini()
2808 goya_disable_clk_rlx(hdev); in goya_hw_fini()
2809 goya_set_pll_refclk(hdev); in goya_hw_fini()
2812 dev_dbg(hdev->dev, in goya_hw_fini()
2817 dev_dbg(hdev->dev, in goya_hw_fini()
2831 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", status); in goya_hw_fini()
2862 int goya_suspend(struct hl_device *hdev) in goya_suspend() argument
2864 return hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0); in goya_suspend()
2867 int goya_resume(struct hl_device *hdev) in goya_resume() argument
2869 return goya_init_iatu(hdev); in goya_resume()
2872 static int goya_mmap(struct hl_device *hdev, struct vm_area_struct *vma, in goya_mmap() argument
2880 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, in goya_mmap()
2883 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); in goya_mmap()
2888 void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) in goya_ring_doorbell() argument
2955 dev_err(hdev->dev, "H/W queue %d is invalid. Can't set pi\n", in goya_ring_doorbell()
2973 void goya_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd) in goya_pqe_write() argument
2979 static void *goya_dma_alloc_coherent(struct hl_device *hdev, size_t size, in goya_dma_alloc_coherent() argument
2982 void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size, in goya_dma_alloc_coherent()
2992 static void goya_dma_free_coherent(struct hl_device *hdev, size_t size, in goya_dma_free_coherent() argument
2998 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle); in goya_dma_free_coherent()
3001 int goya_scrub_device_mem(struct hl_device *hdev) in goya_scrub_device_mem() argument
3006 void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id, in goya_get_int_queue_base() argument
3012 *dma_handle = hdev->asic_prop.sram_base_address; in goya_get_int_queue_base()
3014 base = (__force void *) hdev->pcie_bar[SRAM_CFG_BAR_ID]; in goya_get_int_queue_base()
3054 dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id); in goya_get_int_queue_base()
3064 static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job) in goya_send_job_on_qman0() argument
3073 if (hdev->pldm) in goya_send_job_on_qman0()
3078 if (!hdev->asic_funcs->is_device_idle(hdev, NULL, 0, NULL)) { in goya_send_job_on_qman0()
3079 dev_err_ratelimited(hdev->dev, in goya_send_job_on_qman0()
3084 fence_ptr = hl_asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, &fence_dma_addr); in goya_send_job_on_qman0()
3086 dev_err(hdev->dev, in goya_send_job_on_qman0()
3091 goya_qman0_set_security(hdev, true); in goya_send_job_on_qman0()
3105 rc = hl_hw_queue_send_cb_no_cmpl(hdev, GOYA_QUEUE_ID_DMA_0, in goya_send_job_on_qman0()
3108 dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc); in goya_send_job_on_qman0()
3112 rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, in goya_send_job_on_qman0()
3116 hl_hw_queue_inc_ci_kernel(hdev, GOYA_QUEUE_ID_DMA_0); in goya_send_job_on_qman0()
3119 dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp); in goya_send_job_on_qman0()
3124 hl_asic_dma_pool_free(hdev, (void *) fence_ptr, fence_dma_addr); in goya_send_job_on_qman0()
3126 goya_qman0_set_security(hdev, false); in goya_send_job_on_qman0()
3131 int goya_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len, in goya_send_cpu_message() argument
3134 struct goya_device *goya = hdev->asic_specific; in goya_send_cpu_message()
3145 return hl_fw_send_cpu_message(hdev, GOYA_QUEUE_ID_CPU_PQ, msg, len, in goya_send_cpu_message()
3149 int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id) in goya_test_queue() argument
3160 fence_ptr = hl_asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, &fence_dma_addr); in goya_test_queue()
3162 dev_err(hdev->dev, in goya_test_queue()
3170 fence_pkt = hl_asic_dma_pool_zalloc(hdev, sizeof(struct packet_msg_prot), GFP_KERNEL, in goya_test_queue()
3173 dev_err(hdev->dev, in goya_test_queue()
3187 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, in goya_test_queue()
3191 dev_err(hdev->dev, in goya_test_queue()
3197 rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val), in goya_test_queue()
3200 hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id); in goya_test_queue()
3203 dev_err(hdev->dev, in goya_test_queue()
3210 hl_asic_dma_pool_free(hdev, (void *) fence_pkt, pkt_dma_addr); in goya_test_queue()
3212 hl_asic_dma_pool_free(hdev, (void *) fence_ptr, fence_dma_addr); in goya_test_queue()
3216 int goya_test_cpu_queue(struct hl_device *hdev) in goya_test_cpu_queue() argument
3218 struct goya_device *goya = hdev->asic_specific; in goya_test_cpu_queue()
3227 return hl_fw_test_cpu_queue(hdev); in goya_test_cpu_queue()
3230 int goya_test_queues(struct hl_device *hdev) in goya_test_queues() argument
3235 rc = goya_test_queue(hdev, i); in goya_test_queues()
3243 static void *goya_dma_pool_zalloc(struct hl_device *hdev, size_t size, in goya_dma_pool_zalloc() argument
3251 kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); in goya_dma_pool_zalloc()
3260 static void goya_dma_pool_free(struct hl_device *hdev, void *vaddr, in goya_dma_pool_free() argument
3266 dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr); in goya_dma_pool_free()
3269 void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, in goya_cpu_accessible_dma_pool_alloc() argument
3274 vaddr = hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle); in goya_cpu_accessible_dma_pool_alloc()
3275 *dma_handle = (*dma_handle) - hdev->cpu_accessible_dma_address + in goya_cpu_accessible_dma_pool_alloc()
3281 void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, in goya_cpu_accessible_dma_pool_free() argument
3284 hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr); in goya_cpu_accessible_dma_pool_free()
3287 u32 goya_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt) in goya_get_dma_desc_list_size() argument
3327 static int goya_pin_memory_before_cs(struct hl_device *hdev, in goya_pin_memory_before_cs() argument
3335 if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), in goya_pin_memory_before_cs()
3343 rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), in goya_pin_memory_before_cs()
3350 rc = hl_dma_map_sgtable(hdev, userptr->sgt, dir); in goya_pin_memory_before_cs()
3352 dev_err(hdev->dev, "failed to map sgt with DMA region\n"); in goya_pin_memory_before_cs()
3361 goya_get_dma_desc_list_size(hdev, userptr->sgt); in goya_pin_memory_before_cs()
3367 hl_unpin_host_memory(hdev, userptr); in goya_pin_memory_before_cs()
3373 static int goya_validate_dma_pkt_host(struct hl_device *hdev, in goya_validate_dma_pkt_host() argument
3396 dev_dbg(hdev->dev, "DMA direction is HOST --> DRAM\n"); in goya_validate_dma_pkt_host()
3406 dev_dbg(hdev->dev, "DMA direction is DRAM --> HOST\n"); in goya_validate_dma_pkt_host()
3414 dev_dbg(hdev->dev, "DMA direction is HOST --> SRAM\n"); in goya_validate_dma_pkt_host()
3423 dev_dbg(hdev->dev, "DMA direction is SRAM --> HOST\n"); in goya_validate_dma_pkt_host()
3429 dev_err(hdev->dev, "DMA direction %d is unsupported/undefined\n", user_dir); in goya_validate_dma_pkt_host()
3436 hdev->asic_prop.sram_user_base_address, in goya_validate_dma_pkt_host()
3437 hdev->asic_prop.sram_end_address)) { in goya_validate_dma_pkt_host()
3439 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3448 hdev->asic_prop.dram_user_base_address, in goya_validate_dma_pkt_host()
3449 hdev->asic_prop.dram_end_address)) { in goya_validate_dma_pkt_host()
3451 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3464 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3469 rc = goya_pin_memory_before_cs(hdev, parser, user_dma_pkt, in goya_validate_dma_pkt_host()
3476 static int goya_validate_dma_pkt_no_host(struct hl_device *hdev, in goya_validate_dma_pkt_no_host() argument
3489 dev_dbg(hdev->dev, "DMA direction is DRAM --> SRAM\n"); in goya_validate_dma_pkt_no_host()
3493 dev_dbg(hdev->dev, "DMA direction is SRAM --> DRAM\n"); in goya_validate_dma_pkt_no_host()
3500 hdev->asic_prop.sram_user_base_address, in goya_validate_dma_pkt_no_host()
3501 hdev->asic_prop.sram_end_address)) { in goya_validate_dma_pkt_no_host()
3502 dev_err(hdev->dev, "SRAM address 0x%llx + 0x%x is invalid\n", in goya_validate_dma_pkt_no_host()
3509 hdev->asic_prop.dram_user_base_address, in goya_validate_dma_pkt_no_host()
3510 hdev->asic_prop.dram_end_address)) { in goya_validate_dma_pkt_no_host()
3511 dev_err(hdev->dev, "DRAM address 0x%llx + 0x%x is invalid\n", in goya_validate_dma_pkt_no_host()
3521 static int goya_validate_dma_pkt_no_mmu(struct hl_device *hdev, in goya_validate_dma_pkt_no_mmu() argument
3529 dev_dbg(hdev->dev, "DMA packet details:\n"); in goya_validate_dma_pkt_no_mmu()
3530 dev_dbg(hdev->dev, "source == 0x%llx\n", in goya_validate_dma_pkt_no_mmu()
3532 dev_dbg(hdev->dev, "destination == 0x%llx\n", in goya_validate_dma_pkt_no_mmu()
3534 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); in goya_validate_dma_pkt_no_mmu()
3545 dev_err(hdev->dev, in goya_validate_dma_pkt_no_mmu()
3551 rc = goya_validate_dma_pkt_no_host(hdev, parser, user_dma_pkt); in goya_validate_dma_pkt_no_mmu()
3553 rc = goya_validate_dma_pkt_host(hdev, parser, user_dma_pkt); in goya_validate_dma_pkt_no_mmu()
3558 static int goya_validate_dma_pkt_mmu(struct hl_device *hdev, in goya_validate_dma_pkt_mmu() argument
3562 dev_dbg(hdev->dev, "DMA packet details:\n"); in goya_validate_dma_pkt_mmu()
3563 dev_dbg(hdev->dev, "source == 0x%llx\n", in goya_validate_dma_pkt_mmu()
3565 dev_dbg(hdev->dev, "destination == 0x%llx\n", in goya_validate_dma_pkt_mmu()
3567 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); in goya_validate_dma_pkt_mmu()
3577 hdev->asic_prop.pmmu.start_addr, in goya_validate_dma_pkt_mmu()
3578 hdev->asic_prop.pmmu.end_addr)) { in goya_validate_dma_pkt_mmu()
3579 dev_err(hdev->dev, in goya_validate_dma_pkt_mmu()
3585 dev_err(hdev->dev, in goya_validate_dma_pkt_mmu()
3595 static int goya_validate_wreg32(struct hl_device *hdev, in goya_validate_wreg32() argument
3599 struct goya_device *goya = hdev->asic_specific; in goya_validate_wreg32()
3606 dev_dbg(hdev->dev, "WREG32 packet details:\n"); in goya_validate_wreg32()
3607 dev_dbg(hdev->dev, "reg_offset == 0x%x\n", reg_offset); in goya_validate_wreg32()
3608 dev_dbg(hdev->dev, "value == 0x%x\n", in goya_validate_wreg32()
3612 dev_err(hdev->dev, "WREG32 packet with illegal address 0x%x\n", in goya_validate_wreg32()
3631 dev_err(hdev->dev, "WREG32 packet with illegal value 0x%x\n", in goya_validate_wreg32()
3639 static int goya_validate_cb(struct hl_device *hdev, in goya_validate_cb() argument
3661 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); in goya_validate_cb()
3669 dev_err(hdev->dev, in goya_validate_cb()
3682 rc = goya_validate_wreg32(hdev, in goya_validate_cb()
3688 dev_err(hdev->dev, in goya_validate_cb()
3694 dev_err(hdev->dev, in goya_validate_cb()
3700 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); in goya_validate_cb()
3705 dev_err(hdev->dev, "User not allowed to use STOP\n"); in goya_validate_cb()
3711 rc = goya_validate_dma_pkt_mmu(hdev, parser, in goya_validate_cb()
3714 rc = goya_validate_dma_pkt_no_mmu(hdev, parser, in goya_validate_cb()
3726 dev_err(hdev->dev, "Invalid packet header 0x%x\n", in goya_validate_cb()
3746 static int goya_patch_dma_packet(struct hl_device *hdev, in goya_patch_dma_packet() argument
3793 (hl_userptr_is_pinned(hdev, addr, in goya_patch_dma_packet()
3796 dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n", in goya_patch_dma_packet()
3862 dev_err(hdev->dev, in goya_patch_dma_packet()
3876 static int goya_patch_cb(struct hl_device *hdev, in goya_patch_cb() argument
3900 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); in goya_patch_cb()
3908 dev_err(hdev->dev, in goya_patch_cb()
3916 rc = goya_patch_dma_packet(hdev, parser, in goya_patch_cb()
3926 rc = goya_validate_wreg32(hdev, parser, in goya_patch_cb()
3931 dev_err(hdev->dev, in goya_patch_cb()
3937 dev_err(hdev->dev, in goya_patch_cb()
3943 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); in goya_patch_cb()
3948 dev_err(hdev->dev, "User not allowed to use STOP\n"); in goya_patch_cb()
3961 dev_err(hdev->dev, "Invalid packet header 0x%x\n", in goya_patch_cb()
3974 static int goya_parse_cb_mmu(struct hl_device *hdev, in goya_parse_cb_mmu() argument
3990 rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, in goya_parse_cb_mmu()
3995 dev_err(hdev->dev, in goya_parse_cb_mmu()
4001 parser->patched_cb = hl_cb_get(&hdev->kernel_mem_mgr, handle); in goya_parse_cb_mmu()
4004 dev_crit(hdev->dev, "DMA CB handle invalid 0x%llx\n", handle); in goya_parse_cb_mmu()
4022 rc = goya_validate_cb(hdev, parser, true); in goya_parse_cb_mmu()
4031 dev_err(hdev->dev, "user CB size mismatch\n"); in goya_parse_cb_mmu()
4044 hl_cb_destroy(&hdev->kernel_mem_mgr, handle); in goya_parse_cb_mmu()
4049 static int goya_parse_cb_no_mmu(struct hl_device *hdev, in goya_parse_cb_no_mmu() argument
4055 rc = goya_validate_cb(hdev, parser, false); in goya_parse_cb_no_mmu()
4060 rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, in goya_parse_cb_no_mmu()
4064 dev_err(hdev->dev, in goya_parse_cb_no_mmu()
4069 parser->patched_cb = hl_cb_get(&hdev->kernel_mem_mgr, handle); in goya_parse_cb_no_mmu()
4072 dev_crit(hdev->dev, "DMA CB handle invalid 0x%llx\n", handle); in goya_parse_cb_no_mmu()
4077 rc = goya_patch_cb(hdev, parser); in goya_parse_cb_no_mmu()
4089 hl_cb_destroy(&hdev->kernel_mem_mgr, handle); in goya_parse_cb_no_mmu()
4093 hl_userptr_delete_list(hdev, parser->job_userptr_list); in goya_parse_cb_no_mmu()
4097 static int goya_parse_cb_no_ext_queue(struct hl_device *hdev, in goya_parse_cb_no_ext_queue() argument
4100 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; in goya_parse_cb_no_ext_queue()
4101 struct goya_device *goya = hdev->asic_specific; in goya_parse_cb_no_ext_queue()
4121 dev_err(hdev->dev, in goya_parse_cb_no_ext_queue()
4128 int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser) in goya_cs_parser() argument
4130 struct goya_device *goya = hdev->asic_specific; in goya_cs_parser()
4133 return goya_parse_cb_no_ext_queue(hdev, parser); in goya_cs_parser()
4136 return goya_parse_cb_mmu(hdev, parser); in goya_cs_parser()
4138 return goya_parse_cb_no_mmu(hdev, parser); in goya_cs_parser()
4141 void goya_add_end_of_cb_packets(struct hl_device *hdev, void *kernel_address, in goya_add_end_of_cb_packets() argument
4166 void goya_update_eq_ci(struct hl_device *hdev, u32 val) in goya_update_eq_ci() argument
4171 void goya_restore_phase_topology(struct hl_device *hdev) in goya_restore_phase_topology() argument
4176 static void goya_clear_sm_regs(struct hl_device *hdev) in goya_clear_sm_regs() argument
4196 static int goya_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, void *blob_addr) in goya_debugfs_read_dma() argument
4198 dev_err(hdev->dev, "Reading via DMA is unimplemented yet\n"); in goya_debugfs_read_dma()
4202 static u64 goya_read_pte(struct hl_device *hdev, u64 addr) in goya_read_pte() argument
4204 struct goya_device *goya = hdev->asic_specific; in goya_read_pte()
4206 if (hdev->reset_info.hard_reset_pending) in goya_read_pte()
4209 return readq(hdev->pcie_bar[DDR_BAR_ID] + in goya_read_pte()
4213 static void goya_write_pte(struct hl_device *hdev, u64 addr, u64 val) in goya_write_pte() argument
4215 struct goya_device *goya = hdev->asic_specific; in goya_write_pte()
4217 if (hdev->reset_info.hard_reset_pending) in goya_write_pte()
4220 writeq(val, hdev->pcie_bar[DDR_BAR_ID] + in goya_write_pte()
4422 static void goya_print_razwi_info(struct hl_device *hdev) in goya_print_razwi_info() argument
4425 dev_err_ratelimited(hdev->dev, "Illegal write to LBW\n"); in goya_print_razwi_info()
4430 dev_err_ratelimited(hdev->dev, "Illegal read from LBW\n"); in goya_print_razwi_info()
4435 dev_err_ratelimited(hdev->dev, "Illegal write to HBW\n"); in goya_print_razwi_info()
4440 dev_err_ratelimited(hdev->dev, "Illegal read from HBW\n"); in goya_print_razwi_info()
4445 static void goya_print_mmu_error_info(struct hl_device *hdev) in goya_print_mmu_error_info() argument
4447 struct goya_device *goya = hdev->asic_specific; in goya_print_mmu_error_info()
4460 dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n", in goya_print_mmu_error_info()
4467 static void goya_print_out_of_sync_info(struct hl_device *hdev, in goya_print_out_of_sync_info() argument
4470 struct hl_hw_queue *q = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ]; in goya_print_out_of_sync_info()
4472 dev_err(hdev->dev, "Out of sync with FW, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d\n", in goya_print_out_of_sync_info()
4476 static void goya_print_irq_info(struct hl_device *hdev, u16 event_type, in goya_print_irq_info() argument
4482 dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n", in goya_print_irq_info()
4486 goya_print_razwi_info(hdev); in goya_print_irq_info()
4487 goya_print_mmu_error_info(hdev); in goya_print_irq_info()
4491 static int goya_unmask_irq_arr(struct hl_device *hdev, u32 *irq_arr, in goya_unmask_irq_arr() argument
4509 dev_err(hdev->dev, "too many elements in IRQ array\n"); in goya_unmask_irq_arr()
4531 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt, in goya_unmask_irq_arr()
4535 dev_err(hdev->dev, "failed to unmask IRQ array\n"); in goya_unmask_irq_arr()
4542 static int goya_compute_reset_late_init(struct hl_device *hdev) in goya_compute_reset_late_init() argument
4548 return goya_unmask_irq_arr(hdev, goya_all_events, in goya_compute_reset_late_init()
4552 static int goya_unmask_irq(struct hl_device *hdev, u16 event_type) in goya_unmask_irq() argument
4564 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), in goya_unmask_irq()
4568 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type); in goya_unmask_irq()
4573 static void goya_print_clk_change_info(struct hl_device *hdev, u16 event_type) in goya_print_clk_change_info() argument
4577 mutex_lock(&hdev->clk_throttling.lock); in goya_print_clk_change_info()
4581 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER; in goya_print_clk_change_info()
4582 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER; in goya_print_clk_change_info()
4583 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get(); in goya_print_clk_change_info()
4584 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time; in goya_print_clk_change_info()
4585 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4590 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER; in goya_print_clk_change_info()
4591 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get(); in goya_print_clk_change_info()
4592 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4597 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL; in goya_print_clk_change_info()
4598 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL; in goya_print_clk_change_info()
4599 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get(); in goya_print_clk_change_info()
4600 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time; in goya_print_clk_change_info()
4601 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4606 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL; in goya_print_clk_change_info()
4607 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get(); in goya_print_clk_change_info()
4608 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4613 dev_err(hdev->dev, "Received invalid clock change event %d\n", in goya_print_clk_change_info()
4618 mutex_unlock(&hdev->clk_throttling.lock); in goya_print_clk_change_info()
4621 void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry) in goya_handle_eqe() argument
4626 struct goya_device *goya = hdev->asic_specific; in goya_handle_eqe()
4629 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", in goya_handle_eqe()
4660 goya_print_irq_info(hdev, event_type, false); in goya_handle_eqe()
4661 if (hdev->hard_reset_on_fw_events) in goya_handle_eqe()
4662 hl_device_reset(hdev, (HL_DRV_RESET_HARD | in goya_handle_eqe()
4667 goya_print_irq_info(hdev, event_type, false); in goya_handle_eqe()
4668 if (hdev->hard_reset_on_fw_events) in goya_handle_eqe()
4669 hl_device_reset(hdev, HL_DRV_RESET_HARD); in goya_handle_eqe()
4699 goya_print_irq_info(hdev, event_type, true); in goya_handle_eqe()
4700 goya_unmask_irq(hdev, event_type); in goya_handle_eqe()
4713 goya_print_irq_info(hdev, event_type, false); in goya_handle_eqe()
4714 goya_unmask_irq(hdev, event_type); in goya_handle_eqe()
4721 goya_print_clk_change_info(hdev, event_type); in goya_handle_eqe()
4722 goya_unmask_irq(hdev, event_type); in goya_handle_eqe()
4726 goya_print_irq_info(hdev, event_type, false); in goya_handle_eqe()
4727 goya_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err); in goya_handle_eqe()
4728 if (hdev->hard_reset_on_fw_events) in goya_handle_eqe()
4729 hl_device_reset(hdev, HL_DRV_RESET_HARD); in goya_handle_eqe()
4731 hl_fw_unmask_irq(hdev, event_type); in goya_handle_eqe()
4735 dev_err(hdev->dev, "Received invalid H/W interrupt %d\n", in goya_handle_eqe()
4741 void *goya_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size) in goya_get_events_stat() argument
4743 struct goya_device *goya = hdev->asic_specific; in goya_get_events_stat()
4754 static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, in goya_memset_device_memory() argument
4766 cb = hl_cb_kernel_create(hdev, cb_size, false); in goya_memset_device_memory()
4796 job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true); in goya_memset_device_memory()
4798 dev_err(hdev->dev, "Failed to allocate a new job\n"); in goya_memset_device_memory()
4811 hl_debugfs_add_job(hdev, job); in goya_memset_device_memory()
4813 rc = goya_send_job_on_qman0(hdev, job); in goya_memset_device_memory()
4815 hl_debugfs_remove_job(hdev, job); in goya_memset_device_memory()
4821 hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle); in goya_memset_device_memory()
4826 int goya_context_switch(struct hl_device *hdev, u32 asid) in goya_context_switch() argument
4828 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_context_switch()
4830 u32 size = hdev->pldm ? 0x10000 : prop->sram_size; in goya_context_switch()
4836 rc = goya_memset_device_memory(hdev, addr, size, val, false); in goya_context_switch()
4838 dev_err(hdev->dev, "Failed to clear SRAM in context switch\n"); in goya_context_switch()
4855 goya_clear_sm_regs(hdev); in goya_context_switch()
4860 static int goya_mmu_clear_pgt_range(struct hl_device *hdev) in goya_mmu_clear_pgt_range() argument
4862 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_clear_pgt_range()
4863 struct goya_device *goya = hdev->asic_specific; in goya_mmu_clear_pgt_range()
4871 return goya_memset_device_memory(hdev, addr, size, 0, true); in goya_mmu_clear_pgt_range()
4874 static int goya_mmu_set_dram_default_page(struct hl_device *hdev) in goya_mmu_set_dram_default_page() argument
4876 struct goya_device *goya = hdev->asic_specific; in goya_mmu_set_dram_default_page()
4877 u64 addr = hdev->asic_prop.mmu_dram_default_page_addr; in goya_mmu_set_dram_default_page()
4884 return goya_memset_device_memory(hdev, addr, size, val, true); in goya_mmu_set_dram_default_page()
4887 static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev) in goya_mmu_add_mappings_for_device_cpu() argument
4889 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_add_mappings_for_device_cpu()
4890 struct goya_device *goya = hdev->asic_specific; in goya_mmu_add_mappings_for_device_cpu()
4898 rc = hl_mmu_map_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4903 dev_err(hdev->dev, "Map failed for address 0x%llx\n", in goya_mmu_add_mappings_for_device_cpu()
4909 if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { in goya_mmu_add_mappings_for_device_cpu()
4910 rc = hl_mmu_map_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4912 hdev->cpu_accessible_dma_address, in goya_mmu_add_mappings_for_device_cpu()
4916 dev_err(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4923 rc = hl_mmu_map_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4925 hdev->cpu_accessible_dma_address + cpu_off, in goya_mmu_add_mappings_for_device_cpu()
4928 dev_err(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4936 goya_mmu_prepare_reg(hdev, mmCPU_IF_ARUSER_OVR, HL_KERNEL_ASID_ID); in goya_mmu_add_mappings_for_device_cpu()
4937 goya_mmu_prepare_reg(hdev, mmCPU_IF_AWUSER_OVR, HL_KERNEL_ASID_ID); in goya_mmu_add_mappings_for_device_cpu()
4950 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4953 dev_warn_ratelimited(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4958 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4961 dev_warn_ratelimited(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4968 void goya_mmu_remove_device_cpu_mappings(struct hl_device *hdev) in goya_mmu_remove_device_cpu_mappings() argument
4970 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_remove_device_cpu_mappings()
4971 struct goya_device *goya = hdev->asic_specific; in goya_mmu_remove_device_cpu_mappings()
4983 if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { in goya_mmu_remove_device_cpu_mappings()
4984 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
4987 dev_warn(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
4991 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
4995 dev_warn_ratelimited(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
5001 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
5004 dev_warn_ratelimited(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
5011 static void goya_mmu_prepare(struct hl_device *hdev, u32 asid) in goya_mmu_prepare() argument
5013 struct goya_device *goya = hdev->asic_specific; in goya_mmu_prepare()
5020 dev_crit(hdev->dev, "asid %u is too big\n", asid); in goya_mmu_prepare()
5026 goya_mmu_prepare_reg(hdev, goya_mmu_regs[i], asid); in goya_mmu_prepare()
5029 static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, in goya_mmu_invalidate_cache() argument
5032 struct goya_device *goya = hdev->asic_specific; in goya_mmu_invalidate_cache()
5037 hdev->reset_info.hard_reset_pending) in goya_mmu_invalidate_cache()
5044 if (hdev->pldm) in goya_mmu_invalidate_cache()
5053 hdev, in goya_mmu_invalidate_cache()
5063 static int goya_mmu_invalidate_cache_range(struct hl_device *hdev, in goya_mmu_invalidate_cache_range() argument
5070 return hl_mmu_invalidate_cache(hdev, is_hard, flags); in goya_mmu_invalidate_cache_range()
5073 int goya_send_heartbeat(struct hl_device *hdev) in goya_send_heartbeat() argument
5075 struct goya_device *goya = hdev->asic_specific; in goya_send_heartbeat()
5080 return hl_fw_send_heartbeat(hdev); in goya_send_heartbeat()
5083 int goya_cpucp_info_get(struct hl_device *hdev) in goya_cpucp_info_get() argument
5085 struct goya_device *goya = hdev->asic_specific; in goya_cpucp_info_get()
5086 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_cpucp_info_get()
5093 rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, in goya_cpucp_info_get()
5103 dev_err(hdev->dev, in goya_cpucp_info_get()
5120 static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, in goya_is_device_idle() argument
5199 static void goya_hw_queues_lock(struct hl_device *hdev) in goya_hw_queues_lock() argument
5202 struct goya_device *goya = hdev->asic_specific; in goya_hw_queues_lock()
5207 static void goya_hw_queues_unlock(struct hl_device *hdev) in goya_hw_queues_unlock() argument
5210 struct goya_device *goya = hdev->asic_specific; in goya_hw_queues_unlock()
5215 static u32 goya_get_pci_id(struct hl_device *hdev) in goya_get_pci_id() argument
5217 return hdev->pdev->device; in goya_get_pci_id()
5220 static int goya_get_eeprom_data(struct hl_device *hdev, void *data, in goya_get_eeprom_data() argument
5223 struct goya_device *goya = hdev->asic_specific; in goya_get_eeprom_data()
5228 return hl_fw_get_eeprom_data(hdev, data, max_size); in goya_get_eeprom_data()
5231 static void goya_cpu_init_scrambler_dram(struct hl_device *hdev) in goya_cpu_init_scrambler_dram() argument
5239 goya_mmu_prepare(ctx->hdev, ctx->asid); in goya_ctx_init()
5249 u32 goya_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) in goya_get_queue_id_for_cq() argument
5254 static u32 goya_get_signal_cb_size(struct hl_device *hdev) in goya_get_signal_cb_size() argument
5259 static u32 goya_get_wait_cb_size(struct hl_device *hdev) in goya_get_wait_cb_size() argument
5264 static u32 goya_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, in goya_gen_signal_cb() argument
5270 static u32 goya_gen_wait_cb(struct hl_device *hdev, in goya_gen_wait_cb() argument
5276 static void goya_reset_sob(struct hl_device *hdev, void *data) in goya_reset_sob() argument
5281 static void goya_reset_sob_group(struct hl_device *hdev, u16 sob_group) in goya_reset_sob_group() argument
5286 u64 goya_get_device_time(struct hl_device *hdev) in goya_get_device_time() argument
5298 static int goya_collective_wait_create_jobs(struct hl_device *hdev, in goya_collective_wait_create_jobs() argument
5310 static int goya_get_hw_block_id(struct hl_device *hdev, u64 block_addr, in goya_get_hw_block_id() argument
5316 static int goya_block_mmap(struct hl_device *hdev, struct vm_area_struct *vma, in goya_block_mmap() argument
5322 static void goya_enable_events_from_fw(struct hl_device *hdev) in goya_enable_events_from_fw() argument
5328 static int goya_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask) in goya_ack_mmu_page_fault_or_access_error() argument
5347 static int goya_gen_sync_to_engine_map(struct hl_device *hdev, in goya_gen_sync_to_engine_map() argument
5361 struct hl_device *hdev, in goya_print_single_monitor() argument
5370 struct hl_device *hdev, u64 base_offset, u64 status_base_offset, in goya_print_fences_single_engine() argument
5386 static void goya_state_dump_init(struct hl_device *hdev) in goya_state_dump_init() argument
5389 hdev->state_dump_specs.props = goya_state_dump_specs_props; in goya_state_dump_init()
5390 hdev->state_dump_specs.funcs = goya_state_dump_funcs; in goya_state_dump_init()
5393 static u32 goya_get_sob_addr(struct hl_device *hdev, u32 sob_id) in goya_get_sob_addr() argument
5403 static int goya_get_monitor_dump(struct hl_device *hdev, void *data) in goya_get_monitor_dump() argument
5408 static void goya_check_if_razwi_happened(struct hl_device *hdev) in goya_check_if_razwi_happened() argument
5412 static int goya_scrub_device_dram(struct hl_device *hdev, u64 val) in goya_scrub_device_dram() argument
5417 static int goya_set_dram_properties(struct hl_device *hdev) in goya_set_dram_properties() argument
5422 static int goya_set_binning_masks(struct hl_device *hdev) in goya_set_binning_masks() argument
5427 static int goya_send_device_activity(struct hl_device *hdev, bool open) in goya_send_device_activity() argument
5535 void goya_set_asic_funcs(struct hl_device *hdev) in goya_set_asic_funcs() argument
5537 hdev->asic_funcs = &goya_funcs; in goya_set_asic_funcs()