Lines Matching +full:mmu +full:- +full:500 +full:s
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2022 HabanaLabs, Ltd.
9 #include "../include/hw_ip/mmu/mmu_general.h"
10 #include "../include/hw_ip/mmu/mmu_v1_0.h"
23 * - Range registers (When MMU is enabled, DMA RR does NOT protect host)
24 * - MMU
27 * - Range registers (protect the first 512MB)
28 * - MMU (isolation between users)
31 * - Range registers
32 * - Protection bits
34 * When MMU is disabled:
44 * - checks DMA pointer
45 * - WREG, MSG_PROT are not allowed.
46 * - MSG_LONG/SHORT are allowed.
49 * and only if the QMAN's CP is secured and MSG_PROT is used
52 * When MMU is enabled:
55 * MMU is set to bypass on the Secure props register of the QMAN.
56 * The reasons we don't enable MMU for PQ, CQ and CP are:
57 * - PQ entry is in kernel address space and the driver doesn't map it.
58 * - CP writes to MSIX register and to kernel address space (completion
65 * the driver doesn't map memory in MMU.
67 * QMAN TPC/MME: PQ, CQ and CP aren't secured (no change from MMU disabled mode)
73 #define GOYA_BOOT_FIT_FILE "habanalabs/goya/goya-boot-fit.itb"
74 #define GOYA_LINUX_FW_FILE "habanalabs/goya/goya-fit.itb"
80 #define GOYA_RESET_TIMEOUT_MSEC 500 /* 500ms */
81 #define GOYA_PLDM_RESET_TIMEOUT_MSEC 20000 /* 20s */
84 #define GOYA_PLDM_RESET_WAIT_MSEC 1000 /* 1s */
88 #define GOYA_BOOT_FIT_REQ_TIMEOUT_USEC 1000000 /* 1s */
89 #define GOYA_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */
90 #define GOYA_WAIT_FOR_BL_TIMEOUT_USEC 15000000 /* 15s */
362 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_set_fixed_properties()
365 prop->max_queues = GOYA_QUEUE_ID_SIZE; in goya_set_fixed_properties()
366 prop->hw_queues_props = kcalloc(prop->max_queues, in goya_set_fixed_properties()
370 if (!prop->hw_queues_props) in goya_set_fixed_properties()
371 return -ENOMEM; in goya_set_fixed_properties()
374 prop->hw_queues_props[i].type = QUEUE_TYPE_EXT; in goya_set_fixed_properties()
375 prop->hw_queues_props[i].driver_only = 0; in goya_set_fixed_properties()
376 prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_KERNEL; in goya_set_fixed_properties()
380 prop->hw_queues_props[i].type = QUEUE_TYPE_CPU; in goya_set_fixed_properties()
381 prop->hw_queues_props[i].driver_only = 1; in goya_set_fixed_properties()
382 prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_KERNEL; in goya_set_fixed_properties()
387 prop->hw_queues_props[i].type = QUEUE_TYPE_INT; in goya_set_fixed_properties()
388 prop->hw_queues_props[i].driver_only = 0; in goya_set_fixed_properties()
389 prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_USER; in goya_set_fixed_properties()
392 prop->cfg_base_address = CFG_BASE; in goya_set_fixed_properties()
393 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE; in goya_set_fixed_properties()
394 prop->host_base_address = HOST_PHYS_BASE; in goya_set_fixed_properties()
395 prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE; in goya_set_fixed_properties()
396 prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES; in goya_set_fixed_properties()
397 prop->completion_mode = HL_COMPLETION_MODE_JOB; in goya_set_fixed_properties()
398 prop->dram_base_address = DRAM_PHYS_BASE; in goya_set_fixed_properties()
399 prop->dram_size = DRAM_PHYS_DEFAULT_SIZE; in goya_set_fixed_properties()
400 prop->dram_end_address = prop->dram_base_address + prop->dram_size; in goya_set_fixed_properties()
401 prop->dram_user_base_address = DRAM_BASE_ADDR_USER; in goya_set_fixed_properties()
403 prop->sram_base_address = SRAM_BASE_ADDR; in goya_set_fixed_properties()
404 prop->sram_size = SRAM_SIZE; in goya_set_fixed_properties()
405 prop->sram_end_address = prop->sram_base_address + prop->sram_size; in goya_set_fixed_properties()
406 prop->sram_user_base_address = prop->sram_base_address + in goya_set_fixed_properties()
409 prop->mmu_pgt_addr = MMU_PAGE_TABLES_ADDR; in goya_set_fixed_properties()
410 prop->mmu_dram_default_page_addr = MMU_DRAM_DEFAULT_PAGE_ADDR; in goya_set_fixed_properties()
411 if (hdev->pldm) in goya_set_fixed_properties()
412 prop->mmu_pgt_size = 0x800000; /* 8MB */ in goya_set_fixed_properties()
414 prop->mmu_pgt_size = MMU_PAGE_TABLES_SIZE; in goya_set_fixed_properties()
415 prop->mmu_pte_size = HL_PTE_SIZE; in goya_set_fixed_properties()
416 prop->dram_page_size = PAGE_SIZE_2MB; in goya_set_fixed_properties()
417 prop->device_mem_alloc_default_page_size = prop->dram_page_size; in goya_set_fixed_properties()
418 prop->dram_supports_virtual_memory = true; in goya_set_fixed_properties()
420 prop->dmmu.hop_shifts[MMU_HOP0] = MMU_V1_0_HOP0_SHIFT; in goya_set_fixed_properties()
421 prop->dmmu.hop_shifts[MMU_HOP1] = MMU_V1_0_HOP1_SHIFT; in goya_set_fixed_properties()
422 prop->dmmu.hop_shifts[MMU_HOP2] = MMU_V1_0_HOP2_SHIFT; in goya_set_fixed_properties()
423 prop->dmmu.hop_shifts[MMU_HOP3] = MMU_V1_0_HOP3_SHIFT; in goya_set_fixed_properties()
424 prop->dmmu.hop_shifts[MMU_HOP4] = MMU_V1_0_HOP4_SHIFT; in goya_set_fixed_properties()
425 prop->dmmu.hop_masks[MMU_HOP0] = MMU_V1_0_HOP0_MASK; in goya_set_fixed_properties()
426 prop->dmmu.hop_masks[MMU_HOP1] = MMU_V1_0_HOP1_MASK; in goya_set_fixed_properties()
427 prop->dmmu.hop_masks[MMU_HOP2] = MMU_V1_0_HOP2_MASK; in goya_set_fixed_properties()
428 prop->dmmu.hop_masks[MMU_HOP3] = MMU_V1_0_HOP3_MASK; in goya_set_fixed_properties()
429 prop->dmmu.hop_masks[MMU_HOP4] = MMU_V1_0_HOP4_MASK; in goya_set_fixed_properties()
430 prop->dmmu.start_addr = VA_DDR_SPACE_START; in goya_set_fixed_properties()
431 prop->dmmu.end_addr = VA_DDR_SPACE_END; in goya_set_fixed_properties()
432 prop->dmmu.page_size = PAGE_SIZE_2MB; in goya_set_fixed_properties()
433 prop->dmmu.num_hops = MMU_ARCH_5_HOPS; in goya_set_fixed_properties()
434 prop->dmmu.last_mask = LAST_MASK; in goya_set_fixed_properties()
435 /* TODO: will be duplicated until implementing per-MMU props */ in goya_set_fixed_properties()
436 prop->dmmu.hop_table_size = HOP_TABLE_SIZE_512_PTE; in goya_set_fixed_properties()
437 prop->dmmu.hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE; in goya_set_fixed_properties()
440 memcpy(&prop->pmmu, &prop->dmmu, sizeof(prop->dmmu)); in goya_set_fixed_properties()
441 prop->pmmu.start_addr = VA_HOST_SPACE_START; in goya_set_fixed_properties()
442 prop->pmmu.end_addr = VA_HOST_SPACE_END; in goya_set_fixed_properties()
443 prop->pmmu.page_size = PAGE_SIZE_4KB; in goya_set_fixed_properties()
444 prop->pmmu.num_hops = MMU_ARCH_5_HOPS; in goya_set_fixed_properties()
445 prop->pmmu.last_mask = LAST_MASK; in goya_set_fixed_properties()
446 /* TODO: will be duplicated until implementing per-MMU props */ in goya_set_fixed_properties()
447 prop->pmmu.hop_table_size = HOP_TABLE_SIZE_512_PTE; in goya_set_fixed_properties()
448 prop->pmmu.hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE; in goya_set_fixed_properties()
451 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); in goya_set_fixed_properties()
452 prop->pmmu_huge.page_size = PAGE_SIZE_2MB; in goya_set_fixed_properties()
454 prop->dram_size_for_default_page_mapping = VA_DDR_SPACE_END; in goya_set_fixed_properties()
455 prop->cfg_size = CFG_SIZE; in goya_set_fixed_properties()
456 prop->max_asid = MAX_ASID; in goya_set_fixed_properties()
457 prop->num_of_events = GOYA_ASYNC_EVENT_ID_SIZE; in goya_set_fixed_properties()
458 prop->high_pll = PLL_HIGH_DEFAULT; in goya_set_fixed_properties()
459 prop->cb_pool_cb_cnt = GOYA_CB_POOL_CB_CNT; in goya_set_fixed_properties()
460 prop->cb_pool_cb_size = GOYA_CB_POOL_CB_SIZE; in goya_set_fixed_properties()
461 prop->max_power_default = MAX_POWER_DEFAULT; in goya_set_fixed_properties()
462 prop->dc_power_default = DC_POWER_DEFAULT; in goya_set_fixed_properties()
463 prop->tpc_enabled_mask = TPC_ENABLED_MASK; in goya_set_fixed_properties()
464 prop->pcie_dbi_base_address = mmPCIE_DBI_BASE; in goya_set_fixed_properties()
465 prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; in goya_set_fixed_properties()
467 strscpy_pad(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME, in goya_set_fixed_properties()
470 prop->max_pending_cs = GOYA_MAX_PENDING_CS; in goya_set_fixed_properties()
472 prop->first_available_user_interrupt = USHRT_MAX; in goya_set_fixed_properties()
473 prop->tpc_interrupt_id = USHRT_MAX; in goya_set_fixed_properties()
474 prop->eq_interrupt_id = GOYA_EVENT_QUEUE_MSIX_IDX; in goya_set_fixed_properties()
477 prop->first_available_cq[i] = USHRT_MAX; in goya_set_fixed_properties()
479 prop->fw_cpu_boot_dev_sts0_valid = false; in goya_set_fixed_properties()
480 prop->fw_cpu_boot_dev_sts1_valid = false; in goya_set_fixed_properties()
481 prop->hard_reset_done_by_fw = false; in goya_set_fixed_properties()
482 prop->gic_interrupts_enable = true; in goya_set_fixed_properties()
484 prop->server_type = HL_SERVER_TYPE_UNKNOWN; in goya_set_fixed_properties()
486 prop->clk_pll_index = HL_GOYA_MME_PLL; in goya_set_fixed_properties()
488 prop->use_get_power_for_reset_history = true; in goya_set_fixed_properties()
490 prop->configurable_stop_on_err = true; in goya_set_fixed_properties()
492 prop->set_max_power_on_device_init = true; in goya_set_fixed_properties()
494 prop->dma_mask = 48; in goya_set_fixed_properties()
500 * goya_pci_bars_map - Map PCI BARS of Goya device
518 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + in goya_pci_bars_map()
519 (CFG_BASE - SRAM_BASE_ADDR); in goya_pci_bars_map()
526 struct goya_device *goya = hdev->asic_specific; in goya_set_ddr_bar_base()
531 if ((goya) && (goya->ddr_bar_cur_addr == addr)) in goya_set_ddr_bar_base()
534 /* Inbound Region 1 - Bar 4 - Point to DDR */ in goya_set_ddr_bar_base()
543 old_addr = goya->ddr_bar_cur_addr; in goya_set_ddr_bar_base()
544 goya->ddr_bar_cur_addr = addr; in goya_set_ddr_bar_base()
551 * goya_init_iatu - Initialize the iATU unit inside the PCI controller
564 if (hdev->asic_prop.iatu_done_by_fw) in goya_init_iatu()
567 /* Inbound Region 0 - Bar 0 - Point to SRAM and CFG */ in goya_init_iatu()
575 /* Inbound Region 1 - Bar 4 - Point to DDR */ in goya_init_iatu()
583 /* Outbound Region 0 - Point to Host */ in goya_init_iatu()
598 * goya_early_init - GOYA early initialization code
610 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_early_init()
611 struct pci_dev *pdev = hdev->pdev; in goya_early_init()
618 dev_err(hdev->dev, "Failed to get fixed properties\n"); in goya_early_init()
626 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", in goya_early_init()
628 rc = -ENODEV; in goya_early_init()
635 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", in goya_early_init()
637 rc = -ENODEV; in goya_early_init()
641 prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID); in goya_early_init()
642 hdev->dram_pci_bar_start = pci_resource_start(pdev, DDR_BAR_ID); in goya_early_init()
645 if (hdev->asic_prop.fw_security_enabled) { in goya_early_init()
646 hdev->asic_prop.iatu_done_by_fw = true; in goya_early_init()
658 hdev->asic_prop.iatu_done_by_fw = true; in goya_early_init()
666 * version to determine whether we run with a security-enabled firmware in goya_early_init()
670 if (hdev->reset_on_preboot_fail) in goya_early_init()
672 hdev->asic_funcs->hw_fini(hdev, true, false); in goya_early_init()
677 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n"); in goya_early_init()
678 rc = hdev->asic_funcs->hw_fini(hdev, true, false); in goya_early_init()
680 dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc); in goya_early_init()
685 if (!hdev->pldm) { in goya_early_init()
688 dev_warn(hdev->dev, in goya_early_init()
697 kfree(hdev->asic_prop.hw_queues_props); in goya_early_init()
702 * goya_early_fini - GOYA early finalization code
711 kfree(hdev->asic_prop.hw_queues_props); in goya_early_fini()
726 struct goya_device *goya = hdev->asic_specific; in goya_qman0_set_security()
728 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_qman0_set_security()
740 * goya_fetch_psoc_frequency - Fetch PSOC frequency values
747 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_fetch_psoc_frequency()
752 if (hdev->asic_prop.fw_security_enabled) { in goya_fetch_psoc_frequency()
753 struct goya_device *goya = hdev->asic_specific; in goya_fetch_psoc_frequency()
755 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) in goya_fetch_psoc_frequency()
787 dev_warn(hdev->dev, in goya_fetch_psoc_frequency()
794 prop->psoc_timestamp_frequency = freq; in goya_fetch_psoc_frequency()
795 prop->psoc_pci_pll_nr = nr; in goya_fetch_psoc_frequency()
796 prop->psoc_pci_pll_nf = nf; in goya_fetch_psoc_frequency()
797 prop->psoc_pci_pll_od = od; in goya_fetch_psoc_frequency()
798 prop->psoc_pci_pll_div_factor = div_fctr; in goya_fetch_psoc_frequency()
802 * goya_set_frequency - set the frequency of the device
816 struct goya_device *goya = hdev->asic_specific; in goya_set_frequency()
818 if ((goya->pm_mng_profile == PM_MANUAL) || in goya_set_frequency()
819 (goya->curr_pll_profile == freq)) in goya_set_frequency()
822 dev_dbg(hdev->dev, "Changing device frequency to %s\n", in goya_set_frequency()
827 goya->curr_pll_profile = freq; in goya_set_frequency()
837 struct hl_device *hdev = goya_work->hdev; in goya_set_freq_to_low_job()
839 mutex_lock(&hdev->fpriv_list_lock); in goya_set_freq_to_low_job()
841 if (!hdev->is_compute_ctx_active) in goya_set_freq_to_low_job()
844 mutex_unlock(&hdev->fpriv_list_lock); in goya_set_freq_to_low_job()
846 schedule_delayed_work(&goya_work->work_freq, in goya_set_freq_to_low_job()
852 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_late_init()
853 struct goya_device *goya = hdev->asic_specific; in goya_late_init()
860 dev_err(hdev->dev, in goya_late_init()
861 "Failed to clear MMU page tables range %d\n", rc); in goya_late_init()
867 dev_err(hdev->dev, "Failed to set DRAM default page %d\n", rc); in goya_late_init()
885 dev_err(hdev->dev, "Failed to get cpucp info %d\n", rc); in goya_late_init()
891 * the MMU block) accordingly. The value is the log2 of the DRAM size in goya_late_init()
893 WREG32(mmMMU_LOG2_DDR_SIZE, ilog2(prop->dram_size)); in goya_late_init()
900 goya->curr_pll_profile = PLL_LOW; in goya_late_init()
902 goya->pm_mng_profile = PM_AUTO; in goya_late_init()
906 schedule_delayed_work(&goya->goya_work->work_freq, in goya_late_init()
913 * goya_late_fini - GOYA late tear-down code
921 struct goya_device *goya = hdev->asic_specific; in goya_late_fini()
923 cancel_delayed_work_sync(&goya->goya_work->work_freq); in goya_late_fini()
930 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_set_pci_memory_regions()
934 region = &hdev->pci_mem_region[PCI_REGION_CFG]; in goya_set_pci_memory_regions()
935 region->region_base = CFG_BASE; in goya_set_pci_memory_regions()
936 region->region_size = CFG_SIZE; in goya_set_pci_memory_regions()
937 region->offset_in_bar = CFG_BASE - SRAM_BASE_ADDR; in goya_set_pci_memory_regions()
938 region->bar_size = CFG_BAR_SIZE; in goya_set_pci_memory_regions()
939 region->bar_id = SRAM_CFG_BAR_ID; in goya_set_pci_memory_regions()
940 region->used = 1; in goya_set_pci_memory_regions()
943 region = &hdev->pci_mem_region[PCI_REGION_SRAM]; in goya_set_pci_memory_regions()
944 region->region_base = SRAM_BASE_ADDR; in goya_set_pci_memory_regions()
945 region->region_size = SRAM_SIZE; in goya_set_pci_memory_regions()
946 region->offset_in_bar = 0; in goya_set_pci_memory_regions()
947 region->bar_size = CFG_BAR_SIZE; in goya_set_pci_memory_regions()
948 region->bar_id = SRAM_CFG_BAR_ID; in goya_set_pci_memory_regions()
949 region->used = 1; in goya_set_pci_memory_regions()
952 region = &hdev->pci_mem_region[PCI_REGION_DRAM]; in goya_set_pci_memory_regions()
953 region->region_base = DRAM_PHYS_BASE; in goya_set_pci_memory_regions()
954 region->region_size = hdev->asic_prop.dram_size; in goya_set_pci_memory_regions()
955 region->offset_in_bar = 0; in goya_set_pci_memory_regions()
956 region->bar_size = prop->dram_pci_bar_size; in goya_set_pci_memory_regions()
957 region->bar_id = DDR_BAR_ID; in goya_set_pci_memory_regions()
958 region->used = 1; in goya_set_pci_memory_regions()
962 * goya_sw_init - Goya software initialization code
975 return -ENOMEM; in goya_sw_init()
978 goya->ddr_bar_cur_addr = DRAM_PHYS_BASE; in goya_sw_init()
980 goya->mme_clk = GOYA_PLL_FREQ_LOW; in goya_sw_init()
981 goya->tpc_clk = GOYA_PLL_FREQ_LOW; in goya_sw_init()
982 goya->ic_clk = GOYA_PLL_FREQ_LOW; in goya_sw_init()
984 hdev->asic_specific = goya; in goya_sw_init()
987 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), in goya_sw_init()
988 &hdev->pdev->dev, GOYA_DMA_POOL_BLK_SIZE, 8, 0); in goya_sw_init()
989 if (!hdev->dma_pool) { in goya_sw_init()
990 dev_err(hdev->dev, "failed to create DMA pool\n"); in goya_sw_init()
991 rc = -ENOMEM; in goya_sw_init()
995 hdev->cpu_accessible_dma_mem = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, in goya_sw_init()
996 &hdev->cpu_accessible_dma_address, in goya_sw_init()
999 if (!hdev->cpu_accessible_dma_mem) { in goya_sw_init()
1000 rc = -ENOMEM; in goya_sw_init()
1004 dev_dbg(hdev->dev, "cpu accessible memory at bus address %pad\n", in goya_sw_init()
1005 &hdev->cpu_accessible_dma_address); in goya_sw_init()
1007 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); in goya_sw_init()
1008 if (!hdev->cpu_accessible_dma_pool) { in goya_sw_init()
1009 dev_err(hdev->dev, in goya_sw_init()
1011 rc = -ENOMEM; in goya_sw_init()
1015 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, in goya_sw_init()
1016 (uintptr_t) hdev->cpu_accessible_dma_mem, in goya_sw_init()
1017 HL_CPU_ACCESSIBLE_MEM_SIZE, -1); in goya_sw_init()
1019 dev_err(hdev->dev, in goya_sw_init()
1021 rc = -EFAULT; in goya_sw_init()
1025 spin_lock_init(&goya->hw_queues_lock); in goya_sw_init()
1026 hdev->supports_coresight = true; in goya_sw_init()
1027 hdev->asic_prop.supports_compute_reset = true; in goya_sw_init()
1028 hdev->asic_prop.allow_inference_soft_reset = true; in goya_sw_init()
1029 hdev->supports_wait_for_multi_cs = false; in goya_sw_init()
1030 hdev->supports_ctx_switch = true; in goya_sw_init()
1032 hdev->asic_funcs->set_pci_memory_regions(hdev); in goya_sw_init()
1034 goya->goya_work = kmalloc(sizeof(struct goya_work_freq), GFP_KERNEL); in goya_sw_init()
1035 if (!goya->goya_work) { in goya_sw_init()
1036 rc = -ENOMEM; in goya_sw_init()
1040 goya->goya_work->hdev = hdev; in goya_sw_init()
1041 INIT_DELAYED_WORK(&goya->goya_work->work_freq, goya_set_freq_to_low_job); in goya_sw_init()
1046 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in goya_sw_init()
1048 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, in goya_sw_init()
1049 hdev->cpu_accessible_dma_address); in goya_sw_init()
1051 dma_pool_destroy(hdev->dma_pool); in goya_sw_init()
1059 * goya_sw_fini - Goya software tear-down code
1066 struct goya_device *goya = hdev->asic_specific; in goya_sw_fini()
1068 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in goya_sw_fini()
1070 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, in goya_sw_fini()
1071 hdev->cpu_accessible_dma_address); in goya_sw_fini()
1073 dma_pool_destroy(hdev->dma_pool); in goya_sw_fini()
1075 kfree(goya->goya_work); in goya_sw_fini()
1084 struct goya_device *goya = hdev->asic_specific; in goya_init_dma_qman()
1088 u32 reg_off = dma_id * (mmDMA_QM_1_PQ_PI - mmDMA_QM_0_PQ_PI); in goya_init_dma_qman()
1121 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_init_dma_qman()
1126 if (hdev->stop_on_err) in goya_init_dma_qman()
1137 u32 reg_off = dma_id * (mmDMA_CH_1_CFG1 - mmDMA_CH_0_CFG1); in goya_init_dma_ch()
1151 (dma_id - 1) * 4; in goya_init_dma_ch()
1160 * goya_init_dma_qmans - Initialize QMAN DMA registers
1169 struct goya_device *goya = hdev->asic_specific; in goya_init_dma_qmans()
1173 if (goya->hw_cap_initialized & HW_CAP_DMA) in goya_init_dma_qmans()
1176 q = &hdev->kernel_queues[0]; in goya_init_dma_qmans()
1179 q->cq_id = q->msi_vec = i; in goya_init_dma_qmans()
1180 goya_init_dma_qman(hdev, i, q->bus_address); in goya_init_dma_qmans()
1184 goya->hw_cap_initialized |= HW_CAP_DMA; in goya_init_dma_qmans()
1188 * goya_disable_external_queues - Disable external queues
1195 struct goya_device *goya = hdev->asic_specific; in goya_disable_external_queues()
1197 if (!(goya->hw_cap_initialized & HW_CAP_DMA)) in goya_disable_external_queues()
1241 dev_err(hdev->dev, in goya_stop_queue()
1243 return -EINVAL; in goya_stop_queue()
1250 * goya_stop_external_queues - Stop external queues
1261 struct goya_device *goya = hdev->asic_specific; in goya_stop_external_queues()
1263 if (!(goya->hw_cap_initialized & HW_CAP_DMA)) in goya_stop_external_queues()
1272 dev_err(hdev->dev, "failed to stop DMA QMAN 0\n"); in goya_stop_external_queues()
1273 retval = -EIO; in goya_stop_external_queues()
1282 dev_err(hdev->dev, "failed to stop DMA QMAN 1\n"); in goya_stop_external_queues()
1283 retval = -EIO; in goya_stop_external_queues()
1292 dev_err(hdev->dev, "failed to stop DMA QMAN 2\n"); in goya_stop_external_queues()
1293 retval = -EIO; in goya_stop_external_queues()
1302 dev_err(hdev->dev, "failed to stop DMA QMAN 3\n"); in goya_stop_external_queues()
1303 retval = -EIO; in goya_stop_external_queues()
1312 dev_err(hdev->dev, "failed to stop DMA QMAN 4\n"); in goya_stop_external_queues()
1313 retval = -EIO; in goya_stop_external_queues()
1320 * goya_init_cpu_queues - Initialize PQ/CQ/EQ of CPU
1329 struct goya_device *goya = hdev->asic_specific; in goya_init_cpu_queues()
1330 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_init_cpu_queues()
1333 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ]; in goya_init_cpu_queues()
1336 if (!hdev->cpu_queues_enable) in goya_init_cpu_queues()
1339 if (goya->hw_cap_initialized & HW_CAP_CPU_Q) in goya_init_cpu_queues()
1342 eq = &hdev->event_queue; in goya_init_cpu_queues()
1344 WREG32(mmCPU_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); in goya_init_cpu_queues()
1345 WREG32(mmCPU_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); in goya_init_cpu_queues()
1347 WREG32(mmCPU_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); in goya_init_cpu_queues()
1348 WREG32(mmCPU_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); in goya_init_cpu_queues()
1378 dev_err(hdev->dev, in goya_init_cpu_queues()
1380 return -EIO; in goya_init_cpu_queues()
1384 if (prop->fw_cpu_boot_dev_sts0_valid) in goya_init_cpu_queues()
1385 prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); in goya_init_cpu_queues()
1387 if (prop->fw_cpu_boot_dev_sts1_valid) in goya_init_cpu_queues()
1388 prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); in goya_init_cpu_queues()
1390 goya->hw_cap_initialized |= HW_CAP_CPU_Q; in goya_init_cpu_queues()
1446 tpc_eml_address = (mmTPC0_EML_CFG_BASE + tpc_eml_offset - CFG_BASE); in _goya_tpc_mbist_workaround()
1456 dev_warn(hdev->dev, "TPC%d MBIST ACTIVE is not cleared\n", in _goya_tpc_mbist_workaround()
1484 dev_err(hdev->dev, in _goya_tpc_mbist_workaround()
1505 struct goya_device *goya = hdev->asic_specific; in goya_tpc_mbist_workaround()
1508 if (hdev->pldm) in goya_tpc_mbist_workaround()
1511 if (goya->hw_cap_initialized & HW_CAP_TPC_MBIST) in goya_tpc_mbist_workaround()
1519 goya->hw_cap_initialized |= HW_CAP_TPC_MBIST; in goya_tpc_mbist_workaround()
1523 * goya_init_golden_registers - Initialize golden registers
1532 struct goya_device *goya = hdev->asic_specific; in goya_init_golden_registers()
1536 if (goya->hw_cap_initialized & HW_CAP_GOLDEN) in goya_init_golden_registers()
1812 * Workaround for H2 #HW-23 bug in goya_init_golden_registers()
1822 goya->hw_cap_initialized |= HW_CAP_GOLDEN; in goya_init_golden_registers()
1842 qman_base_addr = hdev->asic_prop.sram_base_address + in goya_init_mme_qman()
1913 struct goya_device *goya = hdev->asic_specific; in goya_init_mme_qmans()
1916 if (goya->hw_cap_initialized & HW_CAP_MME) in goya_init_mme_qmans()
1928 goya->hw_cap_initialized |= HW_CAP_MME; in goya_init_mme_qmans()
1937 u32 reg_off = tpc_id * (mmTPC1_QM_PQ_PI - mmTPC0_QM_PQ_PI); in goya_init_tpc_qman()
1949 qman_base_addr = hdev->asic_prop.sram_base_address + base_off; in goya_init_tpc_qman()
1986 u32 reg_off = tpc_id * (mmTPC1_CMDQ_CQ_CFG1 - mmTPC0_CMDQ_CQ_CFG1); in goya_init_tpc_cmdq()
2020 struct goya_device *goya = hdev->asic_specific; in goya_init_tpc_qmans()
2022 u32 cfg_off = mmTPC1_CFG_SM_BASE_ADDRESS_LOW - in goya_init_tpc_qmans()
2026 if (goya->hw_cap_initialized & HW_CAP_TPC) in goya_init_tpc_qmans()
2051 goya->hw_cap_initialized |= HW_CAP_TPC; in goya_init_tpc_qmans()
2055 * goya_disable_internal_queues - Disable internal queues
2062 struct goya_device *goya = hdev->asic_specific; in goya_disable_internal_queues()
2064 if (!(goya->hw_cap_initialized & HW_CAP_MME)) in goya_disable_internal_queues()
2071 if (!(goya->hw_cap_initialized & HW_CAP_TPC)) in goya_disable_internal_queues()
2100 * goya_stop_internal_queues - Stop internal queues
2109 struct goya_device *goya = hdev->asic_specific; in goya_stop_internal_queues()
2112 if (!(goya->hw_cap_initialized & HW_CAP_MME)) in goya_stop_internal_queues()
2127 dev_err(hdev->dev, "failed to stop MME QMAN\n"); in goya_stop_internal_queues()
2128 retval = -EIO; in goya_stop_internal_queues()
2137 dev_err(hdev->dev, "failed to stop MME CMDQ\n"); in goya_stop_internal_queues()
2138 retval = -EIO; in goya_stop_internal_queues()
2142 if (!(goya->hw_cap_initialized & HW_CAP_TPC)) in goya_stop_internal_queues()
2151 dev_err(hdev->dev, "failed to stop TPC 0 QMAN\n"); in goya_stop_internal_queues()
2152 retval = -EIO; in goya_stop_internal_queues()
2161 dev_err(hdev->dev, "failed to stop TPC 0 CMDQ\n"); in goya_stop_internal_queues()
2162 retval = -EIO; in goya_stop_internal_queues()
2171 dev_err(hdev->dev, "failed to stop TPC 1 QMAN\n"); in goya_stop_internal_queues()
2172 retval = -EIO; in goya_stop_internal_queues()
2181 dev_err(hdev->dev, "failed to stop TPC 1 CMDQ\n"); in goya_stop_internal_queues()
2182 retval = -EIO; in goya_stop_internal_queues()
2191 dev_err(hdev->dev, "failed to stop TPC 2 QMAN\n"); in goya_stop_internal_queues()
2192 retval = -EIO; in goya_stop_internal_queues()
2201 dev_err(hdev->dev, "failed to stop TPC 2 CMDQ\n"); in goya_stop_internal_queues()
2202 retval = -EIO; in goya_stop_internal_queues()
2211 dev_err(hdev->dev, "failed to stop TPC 3 QMAN\n"); in goya_stop_internal_queues()
2212 retval = -EIO; in goya_stop_internal_queues()
2221 dev_err(hdev->dev, "failed to stop TPC 3 CMDQ\n"); in goya_stop_internal_queues()
2222 retval = -EIO; in goya_stop_internal_queues()
2231 dev_err(hdev->dev, "failed to stop TPC 4 QMAN\n"); in goya_stop_internal_queues()
2232 retval = -EIO; in goya_stop_internal_queues()
2241 dev_err(hdev->dev, "failed to stop TPC 4 CMDQ\n"); in goya_stop_internal_queues()
2242 retval = -EIO; in goya_stop_internal_queues()
2251 dev_err(hdev->dev, "failed to stop TPC 5 QMAN\n"); in goya_stop_internal_queues()
2252 retval = -EIO; in goya_stop_internal_queues()
2261 dev_err(hdev->dev, "failed to stop TPC 5 CMDQ\n"); in goya_stop_internal_queues()
2262 retval = -EIO; in goya_stop_internal_queues()
2271 dev_err(hdev->dev, "failed to stop TPC 6 QMAN\n"); in goya_stop_internal_queues()
2272 retval = -EIO; in goya_stop_internal_queues()
2281 dev_err(hdev->dev, "failed to stop TPC 6 CMDQ\n"); in goya_stop_internal_queues()
2282 retval = -EIO; in goya_stop_internal_queues()
2291 dev_err(hdev->dev, "failed to stop TPC 7 QMAN\n"); in goya_stop_internal_queues()
2292 retval = -EIO; in goya_stop_internal_queues()
2301 dev_err(hdev->dev, "failed to stop TPC 7 CMDQ\n"); in goya_stop_internal_queues()
2302 retval = -EIO; in goya_stop_internal_queues()
2310 struct goya_device *goya = hdev->asic_specific; in goya_dma_stall()
2312 if (!(goya->hw_cap_initialized & HW_CAP_DMA)) in goya_dma_stall()
2324 struct goya_device *goya = hdev->asic_specific; in goya_tpc_stall()
2326 if (!(goya->hw_cap_initialized & HW_CAP_TPC)) in goya_tpc_stall()
2341 struct goya_device *goya = hdev->asic_specific; in goya_mme_stall()
2343 if (!(goya->hw_cap_initialized & HW_CAP_MME)) in goya_mme_stall()
2351 struct goya_device *goya = hdev->asic_specific; in goya_enable_msix()
2352 int cq_cnt = hdev->asic_prop.completion_queues_count; in goya_enable_msix()
2355 if (goya->hw_cap_initialized & HW_CAP_MSIX) in goya_enable_msix()
2358 rc = pci_alloc_irq_vectors(hdev->pdev, GOYA_MSIX_ENTRIES, in goya_enable_msix()
2361 dev_err(hdev->dev, in goya_enable_msix()
2362 "MSI-X: Failed to enable support -- %d/%d\n", in goya_enable_msix()
2368 irq = pci_irq_vector(hdev->pdev, i); in goya_enable_msix()
2370 &hdev->completion_queue[i]); in goya_enable_msix()
2372 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in goya_enable_msix()
2377 irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX); in goya_enable_msix()
2381 &hdev->event_queue); in goya_enable_msix()
2383 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in goya_enable_msix()
2387 goya->hw_cap_initialized |= HW_CAP_MSIX; in goya_enable_msix()
2392 free_irq(pci_irq_vector(hdev->pdev, i), in goya_enable_msix()
2393 &hdev->completion_queue[i]); in goya_enable_msix()
2395 pci_free_irq_vectors(hdev->pdev); in goya_enable_msix()
2401 struct goya_device *goya = hdev->asic_specific; in goya_sync_irqs()
2404 if (!(goya->hw_cap_initialized & HW_CAP_MSIX)) in goya_sync_irqs()
2408 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) in goya_sync_irqs()
2409 synchronize_irq(pci_irq_vector(hdev->pdev, i)); in goya_sync_irqs()
2411 synchronize_irq(pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX)); in goya_sync_irqs()
2416 struct goya_device *goya = hdev->asic_specific; in goya_disable_msix()
2419 if (!(goya->hw_cap_initialized & HW_CAP_MSIX)) in goya_disable_msix()
2424 irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX); in goya_disable_msix()
2425 free_irq(irq, &hdev->event_queue); in goya_disable_msix()
2427 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) { in goya_disable_msix()
2428 irq = pci_irq_vector(hdev->pdev, i); in goya_disable_msix()
2429 free_irq(irq, &hdev->completion_queue[i]); in goya_disable_msix()
2432 pci_free_irq_vectors(hdev->pdev); in goya_disable_msix()
2434 goya->hw_cap_initialized &= ~HW_CAP_MSIX; in goya_disable_msix()
2440 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0); in goya_enable_timestamp()
2442 /* Zero the lower/upper parts of the 64-bit counter */ in goya_enable_timestamp()
2443 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0xC, 0); in goya_enable_timestamp()
2444 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0x8, 0); in goya_enable_timestamp()
2447 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 1); in goya_enable_timestamp()
2453 WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0); in goya_disable_timestamp()
2460 if (hdev->pldm) in goya_halt_engines()
2490 * goya_load_firmware_to_device() - Load LINUX FW code to device.
2495 * Return: 0 on success, non-zero for failure.
2501 dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET; in goya_load_firmware_to_device()
2507 * goya_load_boot_fit_to_device() - Load boot fit to device.
2512 * Return: 0 on success, non-zero for failure.
2518 dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + BOOT_FIT_SRAM_OFFSET; in goya_load_boot_fit_to_device()
2528 dynamic_loader = &hdev->fw_loader.dynamic_loader; in goya_init_dynamic_firmware_loader()
2533 * hard-coded) in later stages of the protocol those values will be in goya_init_dynamic_firmware_loader()
2535 * will always be up-to-date in goya_init_dynamic_firmware_loader()
2537 dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; in goya_init_dynamic_firmware_loader()
2538 dyn_regs->kmd_msg_to_cpu = in goya_init_dynamic_firmware_loader()
2540 dyn_regs->cpu_cmd_status_to_host = in goya_init_dynamic_firmware_loader()
2543 dynamic_loader->wait_for_bl_timeout = GOYA_WAIT_FOR_BL_TIMEOUT_USEC; in goya_init_dynamic_firmware_loader()
2550 static_loader = &hdev->fw_loader.static_loader; in goya_init_static_firmware_loader()
2552 static_loader->preboot_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; in goya_init_static_firmware_loader()
2553 static_loader->boot_fit_version_max_off = SRAM_SIZE - VERSION_MAX_LEN; in goya_init_static_firmware_loader()
2554 static_loader->kmd_msg_to_cpu_reg = mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU; in goya_init_static_firmware_loader()
2555 static_loader->cpu_cmd_status_to_host_reg = mmCPU_CMD_STATUS_TO_HOST; in goya_init_static_firmware_loader()
2556 static_loader->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; in goya_init_static_firmware_loader()
2557 static_loader->cpu_boot_dev_status0_reg = mmCPU_BOOT_DEV_STS0; in goya_init_static_firmware_loader()
2558 static_loader->cpu_boot_dev_status1_reg = mmCPU_BOOT_DEV_STS1; in goya_init_static_firmware_loader()
2559 static_loader->boot_err0_reg = mmCPU_BOOT_ERR0; in goya_init_static_firmware_loader()
2560 static_loader->boot_err1_reg = mmCPU_BOOT_ERR1; in goya_init_static_firmware_loader()
2561 static_loader->preboot_version_offset_reg = mmPREBOOT_VER_OFFSET; in goya_init_static_firmware_loader()
2562 static_loader->boot_fit_version_offset_reg = mmUBOOT_VER_OFFSET; in goya_init_static_firmware_loader()
2563 static_loader->sram_offset_mask = ~(lower_32_bits(SRAM_BASE_ADDR)); in goya_init_static_firmware_loader()
2568 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load; in goya_init_firmware_preload_params()
2570 pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; in goya_init_firmware_preload_params()
2571 pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0; in goya_init_firmware_preload_params()
2572 pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1; in goya_init_firmware_preload_params()
2573 pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0; in goya_init_firmware_preload_params()
2574 pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1; in goya_init_firmware_preload_params()
2575 pre_fw_load->wait_for_preboot_timeout = GOYA_BOOT_FIT_REQ_TIMEOUT_USEC; in goya_init_firmware_preload_params()
2580 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_init_firmware_loader()
2581 struct fw_load_mgr *fw_loader = &hdev->fw_loader; in goya_init_firmware_loader()
2584 fw_loader->fw_comp_loaded = FW_TYPE_NONE; in goya_init_firmware_loader()
2585 fw_loader->boot_fit_img.image_name = GOYA_BOOT_FIT_FILE; in goya_init_firmware_loader()
2586 fw_loader->linux_img.image_name = GOYA_LINUX_FW_FILE; in goya_init_firmware_loader()
2587 fw_loader->cpu_timeout = GOYA_CPU_TIMEOUT_USEC; in goya_init_firmware_loader()
2588 fw_loader->boot_fit_timeout = GOYA_BOOT_FIT_REQ_TIMEOUT_USEC; in goya_init_firmware_loader()
2589 fw_loader->skip_bmc = false; in goya_init_firmware_loader()
2590 fw_loader->sram_bar_id = SRAM_CFG_BAR_ID; in goya_init_firmware_loader()
2591 fw_loader->dram_bar_id = DDR_BAR_ID; in goya_init_firmware_loader()
2593 if (prop->dynamic_fw_load) in goya_init_firmware_loader()
2601 struct goya_device *goya = hdev->asic_specific; in goya_init_cpu()
2604 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) in goya_init_cpu()
2607 if (goya->hw_cap_initialized & HW_CAP_CPU) in goya_init_cpu()
2611 * Before pushing u-boot/linux to device, need to set the ddr bar to in goya_init_cpu()
2615 dev_err(hdev->dev, in goya_init_cpu()
2617 return -EIO; in goya_init_cpu()
2625 goya->hw_cap_initialized |= HW_CAP_CPU; in goya_init_cpu()
2636 if (hdev->pldm) in goya_mmu_update_asid_hop0_addr()
2654 dev_err(hdev->dev, in goya_mmu_update_asid_hop0_addr()
2655 "Timeout during MMU hop0 config of asid %d\n", asid); in goya_mmu_update_asid_hop0_addr()
2664 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_init()
2665 struct goya_device *goya = hdev->asic_specific; in goya_mmu_init()
2669 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_mmu_init()
2672 hdev->dram_default_page_mapping = true; in goya_mmu_init()
2674 for (i = 0 ; i < prop->max_asid ; i++) { in goya_mmu_init()
2675 hop0_addr = prop->mmu_pgt_addr + in goya_mmu_init()
2676 (i * prop->dmmu.hop_table_size); in goya_mmu_init()
2680 dev_err(hdev->dev, in goya_mmu_init()
2686 goya->hw_cap_initialized |= HW_CAP_MMU; in goya_mmu_init()
2688 /* init MMU cache manage page */ in goya_mmu_init()
2709 * goya_hw_init - Goya hardware initialization code
2718 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_hw_init()
2725 * Let's mark in the H/W that we have reached this point. We check in goya_hw_init()
2734 dev_err(hdev->dev, "failed to initialize CPU\n"); in goya_hw_init()
2744 * iATU to point to the start address of the MMU page tables in goya_hw_init()
2747 ~(prop->dram_pci_bar_size - 0x1ull))) == U64_MAX) { in goya_hw_init()
2748 dev_err(hdev->dev, in goya_hw_init()
2749 "failed to map DDR bar to MMU page tables\n"); in goya_hw_init()
2750 return -EIO; in goya_hw_init()
2767 /* MSI-X must be enabled before CPU queues are initialized */ in goya_hw_init()
2772 /* Perform read from the device to flush all MSI-X configuration */ in goya_hw_init()
2786 struct goya_device *goya = hdev->asic_specific; in goya_hw_fini()
2789 if (hdev->pldm) { in goya_hw_fini()
2812 dev_dbg(hdev->dev, in goya_hw_fini()
2817 dev_dbg(hdev->dev, in goya_hw_fini()
2831 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", status); in goya_hw_fini()
2832 return -ETIMEDOUT; in goya_hw_fini()
2836 goya->hw_cap_initialized &= ~(HW_CAP_DMA | HW_CAP_MME | in goya_hw_fini()
2843 /* Chicken bit to re-initiate boot sequencer flow */ in goya_hw_fini()
2851 goya->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q | in goya_hw_fini()
2857 memset(goya->events_stat, 0, sizeof(goya->events_stat)); in goya_hw_fini()
2880 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, in goya_mmap()
2881 (dma_addr - HOST_PHYS_BASE), size); in goya_mmap()
2883 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); in goya_mmap()
2955 dev_err(hdev->dev, "H/W queue %d is invalid. Can't set pi\n", in goya_ring_doorbell()
2982 void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size, in goya_dma_alloc_coherent()
2985 /* Shift to the device's base physical address of host memory */ in goya_dma_alloc_coherent()
2995 /* Cancel the device's base physical address of host memory */ in goya_dma_free_coherent()
2996 dma_addr_t fixed_dma_handle = dma_handle - HOST_PHYS_BASE; in goya_dma_free_coherent()
2998 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle); in goya_dma_free_coherent()
3012 *dma_handle = hdev->asic_prop.sram_base_address; in goya_get_int_queue_base()
3014 base = (__force void *) hdev->pcie_bar[SRAM_CFG_BAR_ID]; in goya_get_int_queue_base()
3054 dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id); in goya_get_int_queue_base()
3073 if (hdev->pldm) in goya_send_job_on_qman0()
3078 if (!hdev->asic_funcs->is_device_idle(hdev, NULL, 0, NULL)) { in goya_send_job_on_qman0()
3079 dev_err_ratelimited(hdev->dev, in goya_send_job_on_qman0()
3081 return -EBUSY; in goya_send_job_on_qman0()
3086 dev_err(hdev->dev, in goya_send_job_on_qman0()
3088 return -ENOMEM; in goya_send_job_on_qman0()
3093 cb = job->patched_cb; in goya_send_job_on_qman0()
3095 fence_pkt = cb->kernel_address + in goya_send_job_on_qman0()
3096 job->job_cb_size - sizeof(struct packet_msg_prot); in goya_send_job_on_qman0()
3101 fence_pkt->ctl = cpu_to_le32(tmp); in goya_send_job_on_qman0()
3102 fence_pkt->value = cpu_to_le32(GOYA_QMAN0_FENCE_VAL); in goya_send_job_on_qman0()
3103 fence_pkt->addr = cpu_to_le64(fence_dma_addr); in goya_send_job_on_qman0()
3106 job->job_cb_size, cb->bus_address); in goya_send_job_on_qman0()
3108 dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc); in goya_send_job_on_qman0()
3118 if (rc == -ETIMEDOUT) { in goya_send_job_on_qman0()
3119 dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp); in goya_send_job_on_qman0()
3134 struct goya_device *goya = hdev->asic_specific; in goya_send_cpu_message()
3136 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) { in goya_send_cpu_message()
3162 dev_err(hdev->dev, in goya_test_queue()
3165 return -ENOMEM; in goya_test_queue()
3173 dev_err(hdev->dev, in goya_test_queue()
3176 rc = -ENOMEM; in goya_test_queue()
3183 fence_pkt->ctl = cpu_to_le32(tmp); in goya_test_queue()
3184 fence_pkt->value = cpu_to_le32(fence_val); in goya_test_queue()
3185 fence_pkt->addr = cpu_to_le64(fence_dma_addr); in goya_test_queue()
3191 dev_err(hdev->dev, in goya_test_queue()
3202 if (rc == -ETIMEDOUT) { in goya_test_queue()
3203 dev_err(hdev->dev, in goya_test_queue()
3206 rc = -EIO; in goya_test_queue()
3218 struct goya_device *goya = hdev->asic_specific; in goya_test_cpu_queue()
3224 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) in goya_test_cpu_queue()
3237 ret_val = -EINVAL; in goya_test_queues()
3251 kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); in goya_dma_pool_zalloc()
3253 /* Shift to the device's base physical address of host memory */ in goya_dma_pool_zalloc()
3263 /* Cancel the device's base physical address of host memory */ in goya_dma_pool_free()
3264 dma_addr_t fixed_dma_addr = dma_addr - HOST_PHYS_BASE; in goya_dma_pool_free()
3266 dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr); in goya_dma_pool_free()
3275 *dma_handle = (*dma_handle) - hdev->cpu_accessible_dma_address + in goya_cpu_accessible_dma_pool_alloc()
3303 while ((count + 1) < sgt->nents) { in goya_get_dma_desc_list_size()
3335 if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), in goya_pin_memory_before_cs()
3336 parser->job_userptr_list, &userptr)) in goya_pin_memory_before_cs()
3341 return -ENOMEM; in goya_pin_memory_before_cs()
3343 rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize), in goya_pin_memory_before_cs()
3348 list_add_tail(&userptr->job_node, parser->job_userptr_list); in goya_pin_memory_before_cs()
3350 rc = hl_dma_map_sgtable(hdev, userptr->sgt, dir); in goya_pin_memory_before_cs()
3352 dev_err(hdev->dev, "failed to map sgt with DMA region\n"); in goya_pin_memory_before_cs()
3356 userptr->dma_mapped = true; in goya_pin_memory_before_cs()
3357 userptr->dir = dir; in goya_pin_memory_before_cs()
3360 parser->patched_cb_size += in goya_pin_memory_before_cs()
3361 goya_get_dma_desc_list_size(hdev, userptr->sgt); in goya_pin_memory_before_cs()
3366 list_del(&userptr->job_node); in goya_pin_memory_before_cs()
3386 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_validate_dma_pkt_host()
3396 dev_dbg(hdev->dev, "DMA direction is HOST --> DRAM\n"); in goya_validate_dma_pkt_host()
3399 addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_host()
3400 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_host()
3406 dev_dbg(hdev->dev, "DMA direction is DRAM --> HOST\n"); in goya_validate_dma_pkt_host()
3409 addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_host()
3410 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_host()
3414 dev_dbg(hdev->dev, "DMA direction is HOST --> SRAM\n"); in goya_validate_dma_pkt_host()
3416 addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_host()
3417 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_host()
3423 dev_dbg(hdev->dev, "DMA direction is SRAM --> HOST\n"); in goya_validate_dma_pkt_host()
3425 addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_host()
3426 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_host()
3429 dev_err(hdev->dev, "DMA direction %d is unsupported/undefined\n", user_dir); in goya_validate_dma_pkt_host()
3430 return -EFAULT; in goya_validate_dma_pkt_host()
3435 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_host()
3436 hdev->asic_prop.sram_user_base_address, in goya_validate_dma_pkt_host()
3437 hdev->asic_prop.sram_end_address)) { in goya_validate_dma_pkt_host()
3439 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3442 user_dma_pkt->tsize); in goya_validate_dma_pkt_host()
3443 return -EFAULT; in goya_validate_dma_pkt_host()
3447 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_host()
3448 hdev->asic_prop.dram_user_base_address, in goya_validate_dma_pkt_host()
3449 hdev->asic_prop.dram_end_address)) { in goya_validate_dma_pkt_host()
3451 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3454 user_dma_pkt->tsize); in goya_validate_dma_pkt_host()
3455 return -EFAULT; in goya_validate_dma_pkt_host()
3460 parser->patched_cb_size += sizeof(*user_dma_pkt); in goya_validate_dma_pkt_host()
3463 (parser->hw_queue_id > GOYA_QUEUE_ID_DMA_1)) { in goya_validate_dma_pkt_host()
3464 dev_err(hdev->dev, in goya_validate_dma_pkt_host()
3466 return -EFAULT; in goya_validate_dma_pkt_host()
3484 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_validate_dma_pkt_no_host()
3489 dev_dbg(hdev->dev, "DMA direction is DRAM --> SRAM\n"); in goya_validate_dma_pkt_no_host()
3490 dram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_no_host()
3491 sram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_no_host()
3493 dev_dbg(hdev->dev, "DMA direction is SRAM --> DRAM\n"); in goya_validate_dma_pkt_no_host()
3494 sram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_validate_dma_pkt_no_host()
3495 dram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_validate_dma_pkt_no_host()
3499 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_no_host()
3500 hdev->asic_prop.sram_user_base_address, in goya_validate_dma_pkt_no_host()
3501 hdev->asic_prop.sram_end_address)) { in goya_validate_dma_pkt_no_host()
3502 dev_err(hdev->dev, "SRAM address 0x%llx + 0x%x is invalid\n", in goya_validate_dma_pkt_no_host()
3503 sram_memory_addr, user_dma_pkt->tsize); in goya_validate_dma_pkt_no_host()
3504 return -EFAULT; in goya_validate_dma_pkt_no_host()
3508 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_no_host()
3509 hdev->asic_prop.dram_user_base_address, in goya_validate_dma_pkt_no_host()
3510 hdev->asic_prop.dram_end_address)) { in goya_validate_dma_pkt_no_host()
3511 dev_err(hdev->dev, "DRAM address 0x%llx + 0x%x is invalid\n", in goya_validate_dma_pkt_no_host()
3512 dram_memory_addr, user_dma_pkt->tsize); in goya_validate_dma_pkt_no_host()
3513 return -EFAULT; in goya_validate_dma_pkt_no_host()
3516 parser->patched_cb_size += sizeof(*user_dma_pkt); in goya_validate_dma_pkt_no_host()
3529 dev_dbg(hdev->dev, "DMA packet details:\n"); in goya_validate_dma_pkt_no_mmu()
3530 dev_dbg(hdev->dev, "source == 0x%llx\n", in goya_validate_dma_pkt_no_mmu()
3531 le64_to_cpu(user_dma_pkt->src_addr)); in goya_validate_dma_pkt_no_mmu()
3532 dev_dbg(hdev->dev, "destination == 0x%llx\n", in goya_validate_dma_pkt_no_mmu()
3533 le64_to_cpu(user_dma_pkt->dst_addr)); in goya_validate_dma_pkt_no_mmu()
3534 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); in goya_validate_dma_pkt_no_mmu()
3536 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_validate_dma_pkt_no_mmu()
3544 if (user_dma_pkt->tsize == 0) { in goya_validate_dma_pkt_no_mmu()
3545 dev_err(hdev->dev, in goya_validate_dma_pkt_no_mmu()
3547 return -EINVAL; in goya_validate_dma_pkt_no_mmu()
3562 dev_dbg(hdev->dev, "DMA packet details:\n"); in goya_validate_dma_pkt_mmu()
3563 dev_dbg(hdev->dev, "source == 0x%llx\n", in goya_validate_dma_pkt_mmu()
3564 le64_to_cpu(user_dma_pkt->src_addr)); in goya_validate_dma_pkt_mmu()
3565 dev_dbg(hdev->dev, "destination == 0x%llx\n", in goya_validate_dma_pkt_mmu()
3566 le64_to_cpu(user_dma_pkt->dst_addr)); in goya_validate_dma_pkt_mmu()
3567 dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize)); in goya_validate_dma_pkt_mmu()
3570 * WA for HW-23. in goya_validate_dma_pkt_mmu()
3574 if (parser->hw_queue_id != GOYA_QUEUE_ID_DMA_1 && in goya_validate_dma_pkt_mmu()
3575 hl_mem_area_inside_range(le64_to_cpu(user_dma_pkt->src_addr), in goya_validate_dma_pkt_mmu()
3576 le32_to_cpu(user_dma_pkt->tsize), in goya_validate_dma_pkt_mmu()
3577 hdev->asic_prop.pmmu.start_addr, in goya_validate_dma_pkt_mmu()
3578 hdev->asic_prop.pmmu.end_addr)) { in goya_validate_dma_pkt_mmu()
3579 dev_err(hdev->dev, in goya_validate_dma_pkt_mmu()
3581 return -EFAULT; in goya_validate_dma_pkt_mmu()
3584 if (user_dma_pkt->tsize == 0) { in goya_validate_dma_pkt_mmu()
3585 dev_err(hdev->dev, in goya_validate_dma_pkt_mmu()
3587 return -EINVAL; in goya_validate_dma_pkt_mmu()
3590 parser->patched_cb_size += sizeof(*user_dma_pkt); in goya_validate_dma_pkt_mmu()
3599 struct goya_device *goya = hdev->asic_specific; in goya_validate_wreg32()
3603 reg_offset = le32_to_cpu(wreg_pkt->ctl) & in goya_validate_wreg32()
3606 dev_dbg(hdev->dev, "WREG32 packet details:\n"); in goya_validate_wreg32()
3607 dev_dbg(hdev->dev, "reg_offset == 0x%x\n", reg_offset); in goya_validate_wreg32()
3608 dev_dbg(hdev->dev, "value == 0x%x\n", in goya_validate_wreg32()
3609 le32_to_cpu(wreg_pkt->value)); in goya_validate_wreg32()
3612 dev_err(hdev->dev, "WREG32 packet with illegal address 0x%x\n", in goya_validate_wreg32()
3614 return -EPERM; in goya_validate_wreg32()
3618 * With MMU, DMA channels are not secured, so it doesn't matter where in goya_validate_wreg32()
3620 * non-secured property in goya_validate_wreg32()
3622 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_validate_wreg32()
3628 if ((le32_to_cpu(wreg_pkt->value) < sob_start_addr) || in goya_validate_wreg32()
3629 (le32_to_cpu(wreg_pkt->value) > sob_end_addr)) { in goya_validate_wreg32()
3631 dev_err(hdev->dev, "WREG32 packet with illegal value 0x%x\n", in goya_validate_wreg32()
3632 wreg_pkt->value); in goya_validate_wreg32()
3633 return -EPERM; in goya_validate_wreg32()
3645 parser->patched_cb_size = 0; in goya_validate_cb()
3648 while (cb_parsed_length < parser->user_cb_size) { in goya_validate_cb()
3653 user_pkt = parser->user_cb->kernel_address + cb_parsed_length; in goya_validate_cb()
3656 (le64_to_cpu(user_pkt->header) & in goya_validate_cb()
3661 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); in goya_validate_cb()
3662 rc = -EINVAL; in goya_validate_cb()
3668 if (cb_parsed_length > parser->user_cb_size) { in goya_validate_cb()
3669 dev_err(hdev->dev, in goya_validate_cb()
3671 rc = -EINVAL; in goya_validate_cb()
3680 * not called in MMU path while this function is called in goya_validate_cb()
3684 parser->patched_cb_size += pkt_size; in goya_validate_cb()
3688 dev_err(hdev->dev, in goya_validate_cb()
3690 rc = -EPERM; in goya_validate_cb()
3694 dev_err(hdev->dev, in goya_validate_cb()
3696 rc = -EPERM; in goya_validate_cb()
3700 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); in goya_validate_cb()
3701 rc = -EPERM; in goya_validate_cb()
3705 dev_err(hdev->dev, "User not allowed to use STOP\n"); in goya_validate_cb()
3706 rc = -EPERM; in goya_validate_cb()
3722 parser->patched_cb_size += pkt_size; in goya_validate_cb()
3726 dev_err(hdev->dev, "Invalid packet header 0x%x\n", in goya_validate_cb()
3728 rc = -EINVAL; in goya_validate_cb()
3739 * 2. A packet that will generate MSI-X interrupt in goya_validate_cb()
3741 parser->patched_cb_size += sizeof(struct packet_msg_prot) * 2; in goya_validate_cb()
3765 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_patch_dma_packet()
3774 (user_dma_pkt->tsize == 0)) { in goya_patch_dma_packet()
3781 addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_patch_dma_packet()
3782 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_patch_dma_packet()
3787 addr = le64_to_cpu(user_dma_pkt->dst_addr); in goya_patch_dma_packet()
3788 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr); in goya_patch_dma_packet()
3794 le32_to_cpu(user_dma_pkt->tsize), in goya_patch_dma_packet()
3795 parser->job_userptr_list, &userptr) == false)) { in goya_patch_dma_packet()
3796 dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n", in goya_patch_dma_packet()
3797 addr, user_dma_pkt->tsize); in goya_patch_dma_packet()
3798 return -EFAULT; in goya_patch_dma_packet()
3811 sgt = userptr->sgt; in goya_patch_dma_packet()
3821 while ((count + 1) < sgt->nents) { in goya_patch_dma_packet()
3839 ctl = le32_to_cpu(user_dma_pkt->ctl); in goya_patch_dma_packet()
3844 new_dma_pkt->ctl = cpu_to_le32(ctl); in goya_patch_dma_packet()
3845 new_dma_pkt->tsize = cpu_to_le32((u32) len); in goya_patch_dma_packet()
3848 new_dma_pkt->src_addr = cpu_to_le64(dma_addr); in goya_patch_dma_packet()
3849 new_dma_pkt->dst_addr = cpu_to_le64(device_memory_addr); in goya_patch_dma_packet()
3851 new_dma_pkt->src_addr = cpu_to_le64(device_memory_addr); in goya_patch_dma_packet()
3852 new_dma_pkt->dst_addr = cpu_to_le64(dma_addr); in goya_patch_dma_packet()
3862 dev_err(hdev->dev, in goya_patch_dma_packet()
3864 return -EFAULT; in goya_patch_dma_packet()
3867 /* Fix the last dma packet - rdcomp/wrcomp must be as user set them */ in goya_patch_dma_packet()
3868 new_dma_pkt--; in goya_patch_dma_packet()
3869 new_dma_pkt->ctl |= cpu_to_le32(user_rdcomp_mask | user_wrcomp_mask); in goya_patch_dma_packet()
3884 while (cb_parsed_length < parser->user_cb_size) { in goya_patch_cb()
3890 user_pkt = parser->user_cb->kernel_address + cb_parsed_length; in goya_patch_cb()
3891 kernel_pkt = parser->patched_cb->kernel_address + in goya_patch_cb()
3895 (le64_to_cpu(user_pkt->header) & in goya_patch_cb()
3900 dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id); in goya_patch_cb()
3901 rc = -EINVAL; in goya_patch_cb()
3907 if (cb_parsed_length > parser->user_cb_size) { in goya_patch_cb()
3908 dev_err(hdev->dev, in goya_patch_cb()
3910 rc = -EINVAL; in goya_patch_cb()
3931 dev_err(hdev->dev, in goya_patch_cb()
3933 rc = -EPERM; in goya_patch_cb()
3937 dev_err(hdev->dev, in goya_patch_cb()
3939 rc = -EPERM; in goya_patch_cb()
3943 dev_err(hdev->dev, "User not allowed to use CP_DMA\n"); in goya_patch_cb()
3944 rc = -EPERM; in goya_patch_cb()
3948 dev_err(hdev->dev, "User not allowed to use STOP\n"); in goya_patch_cb()
3949 rc = -EPERM; in goya_patch_cb()
3961 dev_err(hdev->dev, "Invalid packet header 0x%x\n", in goya_patch_cb()
3963 rc = -EINVAL; in goya_patch_cb()
3985 * 2. A packet that will generate MSI-X interrupt in goya_parse_cb_mmu()
3987 parser->patched_cb_size = parser->user_cb_size + in goya_parse_cb_mmu()
3990 rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, in goya_parse_cb_mmu()
3991 parser->patched_cb_size, false, false, in goya_parse_cb_mmu()
3995 dev_err(hdev->dev, in goya_parse_cb_mmu()
4001 parser->patched_cb = hl_cb_get(&hdev->kernel_mem_mgr, handle); in goya_parse_cb_mmu()
4003 if (!parser->patched_cb) { in goya_parse_cb_mmu()
4004 dev_crit(hdev->dev, "DMA CB handle invalid 0x%llx\n", handle); in goya_parse_cb_mmu()
4005 rc = -EFAULT; in goya_parse_cb_mmu()
4010 * The check that parser->user_cb_size <= parser->user_cb->size was done in goya_parse_cb_mmu()
4013 memcpy(parser->patched_cb->kernel_address, in goya_parse_cb_mmu()
4014 parser->user_cb->kernel_address, in goya_parse_cb_mmu()
4015 parser->user_cb_size); in goya_parse_cb_mmu()
4017 patched_cb_size = parser->patched_cb_size; in goya_parse_cb_mmu()
4020 user_cb = parser->user_cb; in goya_parse_cb_mmu()
4021 parser->user_cb = parser->patched_cb; in goya_parse_cb_mmu()
4023 parser->user_cb = user_cb; in goya_parse_cb_mmu()
4026 hl_cb_put(parser->patched_cb); in goya_parse_cb_mmu()
4030 if (patched_cb_size != parser->patched_cb_size) { in goya_parse_cb_mmu()
4031 dev_err(hdev->dev, "user CB size mismatch\n"); in goya_parse_cb_mmu()
4032 hl_cb_put(parser->patched_cb); in goya_parse_cb_mmu()
4033 rc = -EINVAL; in goya_parse_cb_mmu()
4044 hl_cb_destroy(&hdev->kernel_mem_mgr, handle); in goya_parse_cb_mmu()
4060 rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx, in goya_parse_cb_no_mmu()
4061 parser->patched_cb_size, false, false, in goya_parse_cb_no_mmu()
4064 dev_err(hdev->dev, in goya_parse_cb_no_mmu()
4069 parser->patched_cb = hl_cb_get(&hdev->kernel_mem_mgr, handle); in goya_parse_cb_no_mmu()
4071 if (!parser->patched_cb) { in goya_parse_cb_no_mmu()
4072 dev_crit(hdev->dev, "DMA CB handle invalid 0x%llx\n", handle); in goya_parse_cb_no_mmu()
4073 rc = -EFAULT; in goya_parse_cb_no_mmu()
4080 hl_cb_put(parser->patched_cb); in goya_parse_cb_no_mmu()
4089 hl_cb_destroy(&hdev->kernel_mem_mgr, handle); in goya_parse_cb_no_mmu()
4093 hl_userptr_delete_list(hdev, parser->job_userptr_list); in goya_parse_cb_no_mmu()
4100 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; in goya_parse_cb_no_ext_queue()
4101 struct goya_device *goya = hdev->asic_specific; in goya_parse_cb_no_ext_queue()
4103 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_parse_cb_no_ext_queue()
4108 (u64) (uintptr_t) parser->user_cb, in goya_parse_cb_no_ext_queue()
4109 parser->user_cb_size, in goya_parse_cb_no_ext_queue()
4110 asic_prop->sram_user_base_address, in goya_parse_cb_no_ext_queue()
4111 asic_prop->sram_end_address)) in goya_parse_cb_no_ext_queue()
4115 (u64) (uintptr_t) parser->user_cb, in goya_parse_cb_no_ext_queue()
4116 parser->user_cb_size, in goya_parse_cb_no_ext_queue()
4117 asic_prop->dram_user_base_address, in goya_parse_cb_no_ext_queue()
4118 asic_prop->dram_end_address)) in goya_parse_cb_no_ext_queue()
4121 dev_err(hdev->dev, in goya_parse_cb_no_ext_queue()
4123 parser->user_cb, parser->user_cb_size); in goya_parse_cb_no_ext_queue()
4125 return -EFAULT; in goya_parse_cb_no_ext_queue()
4130 struct goya_device *goya = hdev->asic_specific; in goya_cs_parser()
4132 if (parser->queue_type == QUEUE_TYPE_INT) in goya_cs_parser()
4135 if (goya->hw_cap_initialized & HW_CAP_MMU) in goya_cs_parser()
4148 cq_pkt = kernel_address + len - (sizeof(struct packet_msg_prot) * 2); in goya_add_end_of_cb_packets()
4153 cq_pkt->ctl = cpu_to_le32(tmp); in goya_add_end_of_cb_packets()
4154 cq_pkt->value = cpu_to_le32(cq_val); in goya_add_end_of_cb_packets()
4155 cq_pkt->addr = cpu_to_le64(cq_addr); in goya_add_end_of_cb_packets()
4161 cq_pkt->ctl = cpu_to_le32(tmp); in goya_add_end_of_cb_packets()
4162 cq_pkt->value = cpu_to_le32(msix_vec & 0x7FF); in goya_add_end_of_cb_packets()
4163 cq_pkt->addr = cpu_to_le64(CFG_BASE + mmPCIE_DBI_MSIX_DOORBELL_OFF); in goya_add_end_of_cb_packets()
4181 ((mmSYNC_MNGR_SOB_OBJ_1023 - mmSYNC_MNGR_SOB_OBJ_0) + 4); in goya_clear_sm_regs()
4184 ((mmSYNC_MNGR_MON_STATUS_255 - mmSYNC_MNGR_MON_STATUS_0) + 4); in goya_clear_sm_regs()
4198 dev_err(hdev->dev, "Reading via DMA is unimplemented yet\n"); in goya_debugfs_read_dma()
4199 return -EPERM; in goya_debugfs_read_dma()
4204 struct goya_device *goya = hdev->asic_specific; in goya_read_pte()
4206 if (hdev->reset_info.hard_reset_pending) in goya_read_pte()
4209 return readq(hdev->pcie_bar[DDR_BAR_ID] + in goya_read_pte()
4210 (addr - goya->ddr_bar_cur_addr)); in goya_read_pte()
4215 struct goya_device *goya = hdev->asic_specific; in goya_write_pte()
4217 if (hdev->reset_info.hard_reset_pending) in goya_write_pte()
4220 writeq(val, hdev->pcie_bar[DDR_BAR_ID] + in goya_write_pte()
4221 (addr - goya->ddr_bar_cur_addr)); in goya_write_pte()
4349 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_ECC) / 3; in goya_get_event_desc()
4353 index = event_type - GOYA_ASYNC_EVENT_ID_SRAM0; in goya_get_event_desc()
4357 index = event_type - GOYA_ASYNC_EVENT_ID_PLL0; in goya_get_event_desc()
4368 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_DEC) / 3; in goya_get_event_desc()
4379 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR) / 10; in goya_get_event_desc()
4383 index = event_type - GOYA_ASYNC_EVENT_ID_TPC0_CMDQ; in goya_get_event_desc()
4387 index = event_type - GOYA_ASYNC_EVENT_ID_TPC0_QM; in goya_get_event_desc()
4391 index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_QM; in goya_get_event_desc()
4395 index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_CH; in goya_get_event_desc()
4406 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU) / 10; in goya_get_event_desc()
4410 index = event_type - GOYA_ASYNC_EVENT_ID_DMA_BM_CH0; in goya_get_event_desc()
4425 dev_err_ratelimited(hdev->dev, "Illegal write to LBW\n"); in goya_print_razwi_info()
4430 dev_err_ratelimited(hdev->dev, "Illegal read from LBW\n"); in goya_print_razwi_info()
4435 dev_err_ratelimited(hdev->dev, "Illegal write to HBW\n"); in goya_print_razwi_info()
4440 dev_err_ratelimited(hdev->dev, "Illegal read from HBW\n"); in goya_print_razwi_info()
4447 struct goya_device *goya = hdev->asic_specific; in goya_print_mmu_error_info()
4451 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_print_mmu_error_info()
4460 dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n", in goya_print_mmu_error_info()
4470 struct hl_hw_queue *q = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ]; in goya_print_out_of_sync_info()
4472 dev_err(hdev->dev, "Out of sync with FW, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d\n", in goya_print_out_of_sync_info()
4473 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci)); in goya_print_out_of_sync_info()
4482 dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n", in goya_print_irq_info()
4504 /* data should be aligned to 8 bytes in order to CPU-CP to copy it */ in goya_unmask_irq_arr()
4509 dev_err(hdev->dev, "too many elements in IRQ array\n"); in goya_unmask_irq_arr()
4510 return -EINVAL; in goya_unmask_irq_arr()
4515 return -ENOMEM; in goya_unmask_irq_arr()
4518 pkt->length = cpu_to_le32(irq_num_entries); in goya_unmask_irq_arr()
4523 for (irq_arr_index = 0, goya_irq_arr = (__le32 *) &pkt->irqs; in goya_unmask_irq_arr()
4528 pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY << in goya_unmask_irq_arr()
4531 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt, in goya_unmask_irq_arr()
4535 dev_err(hdev->dev, "failed to unmask IRQ array\n"); in goya_unmask_irq_arr()
4564 rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), in goya_unmask_irq()
4568 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type); in goya_unmask_irq()
4577 mutex_lock(&hdev->clk_throttling.lock); in goya_print_clk_change_info()
4581 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER; in goya_print_clk_change_info()
4582 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER; in goya_print_clk_change_info()
4583 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get(); in goya_print_clk_change_info()
4584 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time; in goya_print_clk_change_info()
4585 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4590 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER; in goya_print_clk_change_info()
4591 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get(); in goya_print_clk_change_info()
4592 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4597 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL; in goya_print_clk_change_info()
4598 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL; in goya_print_clk_change_info()
4599 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get(); in goya_print_clk_change_info()
4600 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time; in goya_print_clk_change_info()
4601 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4606 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL; in goya_print_clk_change_info()
4607 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get(); in goya_print_clk_change_info()
4608 dev_info_ratelimited(hdev->dev, in goya_print_clk_change_info()
4613 dev_err(hdev->dev, "Received invalid clock change event %d\n", in goya_print_clk_change_info()
4618 mutex_unlock(&hdev->clk_throttling.lock); in goya_print_clk_change_info()
4623 u32 ctl = le32_to_cpu(eq_entry->hdr.ctl); in goya_handle_eqe()
4626 struct goya_device *goya = hdev->asic_specific; in goya_handle_eqe()
4629 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", in goya_handle_eqe()
4630 event_type, GOYA_ASYNC_EVENT_ID_SIZE - 1); in goya_handle_eqe()
4634 goya->events_stat[event_type]++; in goya_handle_eqe()
4635 goya->events_stat_aggregate[event_type]++; in goya_handle_eqe()
4661 if (hdev->hard_reset_on_fw_events) in goya_handle_eqe()
4668 if (hdev->hard_reset_on_fw_events) in goya_handle_eqe()
4727 goya_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err); in goya_handle_eqe()
4728 if (hdev->hard_reset_on_fw_events) in goya_handle_eqe()
4735 dev_err(hdev->dev, "Received invalid H/W interrupt %d\n", in goya_handle_eqe()
4743 struct goya_device *goya = hdev->asic_specific; in goya_get_events_stat()
4746 *size = (u32) sizeof(goya->events_stat_aggregate); in goya_get_events_stat()
4747 return goya->events_stat_aggregate; in goya_get_events_stat()
4750 *size = (u32) sizeof(goya->events_stat); in goya_get_events_stat()
4751 return goya->events_stat; in goya_get_events_stat()
4768 return -ENOMEM; in goya_memset_device_memory()
4770 lin_dma_pkt = cb->kernel_address; in goya_memset_device_memory()
4782 lin_dma_pkt->ctl = cpu_to_le32(ctl); in goya_memset_device_memory()
4784 lin_dma_pkt->src_addr = cpu_to_le64(val); in goya_memset_device_memory()
4785 lin_dma_pkt->dst_addr = cpu_to_le64(addr); in goya_memset_device_memory()
4787 lin_dma_pkt->tsize = cpu_to_le32(SZ_2G); in goya_memset_device_memory()
4789 lin_dma_pkt->tsize = cpu_to_le32(size); in goya_memset_device_memory()
4791 size -= SZ_2G; in goya_memset_device_memory()
4794 } while (--lin_dma_pkts_cnt); in goya_memset_device_memory()
4798 dev_err(hdev->dev, "Failed to allocate a new job\n"); in goya_memset_device_memory()
4799 rc = -ENOMEM; in goya_memset_device_memory()
4803 job->id = 0; in goya_memset_device_memory()
4804 job->user_cb = cb; in goya_memset_device_memory()
4805 atomic_inc(&job->user_cb->cs_cnt); in goya_memset_device_memory()
4806 job->user_cb_size = cb_size; in goya_memset_device_memory()
4807 job->hw_queue_id = GOYA_QUEUE_ID_DMA_0; in goya_memset_device_memory()
4808 job->patched_cb = job->user_cb; in goya_memset_device_memory()
4809 job->job_cb_size = job->user_cb_size; in goya_memset_device_memory()
4817 atomic_dec(&cb->cs_cnt); in goya_memset_device_memory()
4821 hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle); in goya_memset_device_memory()
4828 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_context_switch()
4829 u64 addr = prop->sram_base_address, sob_addr; in goya_context_switch()
4830 u32 size = hdev->pldm ? 0x10000 : prop->sram_size; in goya_context_switch()
4833 u32 channel_off = mmDMA_CH_1_WR_COMP_ADDR_LO - in goya_context_switch()
4838 dev_err(hdev->dev, "Failed to clear SRAM in context switch\n"); in goya_context_switch()
4848 (dma_id - 1) * 4; in goya_context_switch()
4862 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_clear_pgt_range()
4863 struct goya_device *goya = hdev->asic_specific; in goya_mmu_clear_pgt_range()
4864 u64 addr = prop->mmu_pgt_addr; in goya_mmu_clear_pgt_range()
4865 u32 size = prop->mmu_pgt_size + MMU_DRAM_DEFAULT_PAGE_SIZE + in goya_mmu_clear_pgt_range()
4868 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_clear_pgt_range()
4876 struct goya_device *goya = hdev->asic_specific; in goya_mmu_set_dram_default_page()
4877 u64 addr = hdev->asic_prop.mmu_dram_default_page_addr; in goya_mmu_set_dram_default_page()
4881 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_set_dram_default_page()
4889 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_add_mappings_for_device_cpu()
4890 struct goya_device *goya = hdev->asic_specific; in goya_mmu_add_mappings_for_device_cpu()
4894 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_add_mappings_for_device_cpu()
4898 rc = hl_mmu_map_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4899 prop->dram_base_address + off, in goya_mmu_add_mappings_for_device_cpu()
4900 prop->dram_base_address + off, PAGE_SIZE_2MB, in goya_mmu_add_mappings_for_device_cpu()
4903 dev_err(hdev->dev, "Map failed for address 0x%llx\n", in goya_mmu_add_mappings_for_device_cpu()
4904 prop->dram_base_address + off); in goya_mmu_add_mappings_for_device_cpu()
4909 if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { in goya_mmu_add_mappings_for_device_cpu()
4910 rc = hl_mmu_map_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4912 hdev->cpu_accessible_dma_address, in goya_mmu_add_mappings_for_device_cpu()
4916 dev_err(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4918 off -= PAGE_SIZE_2MB; in goya_mmu_add_mappings_for_device_cpu()
4923 rc = hl_mmu_map_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4925 hdev->cpu_accessible_dma_address + cpu_off, in goya_mmu_add_mappings_for_device_cpu()
4928 dev_err(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4930 cpu_off -= PAGE_SIZE_4KB; in goya_mmu_add_mappings_for_device_cpu()
4944 goya->device_cpu_mmu_mappings_done = true; in goya_mmu_add_mappings_for_device_cpu()
4949 for (; cpu_off >= 0 ; cpu_off -= PAGE_SIZE_4KB) in goya_mmu_add_mappings_for_device_cpu()
4950 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4953 dev_warn_ratelimited(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4957 for (; off >= 0 ; off -= PAGE_SIZE_2MB) in goya_mmu_add_mappings_for_device_cpu()
4958 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_add_mappings_for_device_cpu()
4959 prop->dram_base_address + off, PAGE_SIZE_2MB, in goya_mmu_add_mappings_for_device_cpu()
4961 dev_warn_ratelimited(hdev->dev, in goya_mmu_add_mappings_for_device_cpu()
4963 prop->dram_base_address + off); in goya_mmu_add_mappings_for_device_cpu()
4970 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_mmu_remove_device_cpu_mappings()
4971 struct goya_device *goya = hdev->asic_specific; in goya_mmu_remove_device_cpu_mappings()
4974 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_remove_device_cpu_mappings()
4977 if (!goya->device_cpu_mmu_mappings_done) in goya_mmu_remove_device_cpu_mappings()
4983 if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) { in goya_mmu_remove_device_cpu_mappings()
4984 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
4987 dev_warn(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
4991 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
4995 dev_warn_ratelimited(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
5001 if (hl_mmu_unmap_page(hdev->kernel_ctx, in goya_mmu_remove_device_cpu_mappings()
5002 prop->dram_base_address + off, PAGE_SIZE_2MB, in goya_mmu_remove_device_cpu_mappings()
5004 dev_warn_ratelimited(hdev->dev, in goya_mmu_remove_device_cpu_mappings()
5006 prop->dram_base_address + off); in goya_mmu_remove_device_cpu_mappings()
5008 goya->device_cpu_mmu_mappings_done = false; in goya_mmu_remove_device_cpu_mappings()
5013 struct goya_device *goya = hdev->asic_specific; in goya_mmu_prepare()
5016 if (!(goya->hw_cap_initialized & HW_CAP_MMU)) in goya_mmu_prepare()
5020 dev_crit(hdev->dev, "asid %u is too big\n", asid); in goya_mmu_prepare()
5032 struct goya_device *goya = hdev->asic_specific; in goya_mmu_invalidate_cache()
5036 if (!(goya->hw_cap_initialized & HW_CAP_MMU) || in goya_mmu_invalidate_cache()
5037 hdev->reset_info.hard_reset_pending) in goya_mmu_invalidate_cache()
5044 if (hdev->pldm) in goya_mmu_invalidate_cache()
5075 struct goya_device *goya = hdev->asic_specific; in goya_send_heartbeat()
5077 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) in goya_send_heartbeat()
5085 struct goya_device *goya = hdev->asic_specific; in goya_cpucp_info_get()
5086 struct asic_fixed_properties *prop = &hdev->asic_prop; in goya_cpucp_info_get()
5090 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) in goya_cpucp_info_get()
5099 dram_size = le64_to_cpu(prop->cpucp_info.dram_size); in goya_cpucp_info_get()
5103 dev_err(hdev->dev, in goya_cpucp_info_get()
5109 prop->dram_size = dram_size; in goya_cpucp_info_get()
5110 prop->dram_end_address = prop->dram_base_address + dram_size; in goya_cpucp_info_get()
5113 if (!strlen(prop->cpucp_info.card_name)) in goya_cpucp_info_get()
5114 strscpy_pad(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME, in goya_cpucp_info_get()
5123 const char *fmt = "%-5d%-9s%#-14x%#-16x%#x\n"; in goya_is_device_idle()
5124 const char *dma_fmt = "%-5d%-9s%#-14x%#x\n"; in goya_is_device_idle()
5134 "--- ------- ------------ -------------\n"); in goya_is_device_idle()
5136 offset = mmDMA_QM_1_GLBL_STS0 - mmDMA_QM_0_GLBL_STS0; in goya_is_device_idle()
5155 "--- ------- ------------ -------------- ----------\n"); in goya_is_device_idle()
5157 offset = mmTPC1_QM_GLBL_STS0 - mmTPC0_QM_GLBL_STS0; in goya_is_device_idle()
5178 "--- ------- ------------ -------------- -----------\n"); in goya_is_device_idle()
5200 __acquires(&goya->hw_queues_lock) in goya_hw_queues_lock()
5202 struct goya_device *goya = hdev->asic_specific; in goya_hw_queues_lock()
5204 spin_lock(&goya->hw_queues_lock); in goya_hw_queues_lock()
5208 __releases(&goya->hw_queues_lock) in goya_hw_queues_unlock()
5210 struct goya_device *goya = hdev->asic_specific; in goya_hw_queues_unlock()
5212 spin_unlock(&goya->hw_queues_lock); in goya_hw_queues_unlock()
5217 return hdev->pdev->device; in goya_get_pci_id()
5223 struct goya_device *goya = hdev->asic_specific; in goya_get_eeprom_data()
5225 if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) in goya_get_eeprom_data()
5238 if (ctx->asid != HL_KERNEL_ASID_ID) in goya_ctx_init()
5239 goya_mmu_prepare(ctx->hdev, ctx->asid); in goya_ctx_init()
5302 return -EINVAL; in goya_collective_wait_create_jobs()
5313 return -EPERM; in goya_get_hw_block_id()
5319 return -EPERM; in goya_block_mmap()
5330 return -EINVAL; in goya_ack_mmu_page_fault_or_access_error()
5343 default: return -EINVAL; in goya_map_pll_idx_to_fw_idx()
5389 hdev->state_dump_specs.props = goya_state_dump_specs_props; in goya_state_dump_init()
5390 hdev->state_dump_specs.funcs = goya_state_dump_funcs; in goya_state_dump_init()
5405 return -EOPNOTSUPP; in goya_get_monitor_dump()
5414 return -EOPNOTSUPP; in goya_scrub_device_dram()
5530 * goya_set_asic_funcs - set Goya function pointers
5537 hdev->asic_funcs = &goya_funcs; in goya_set_asic_funcs()