301 static void ivpu_mmu_config_check(struct ivpu_device *vdev)  in ivpu_mmu_config_check()  argument
306 	if (ivpu_is_simics(vdev))  in ivpu_mmu_config_check()
313 		ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref);  in ivpu_mmu_config_check()
317 		ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF);  in ivpu_mmu_config_check()
321 		ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF);  in ivpu_mmu_config_check()
323 	if (ivpu_is_simics(vdev))  in ivpu_mmu_config_check()
325 	else if (ivpu_is_fpga(vdev))  in ivpu_mmu_config_check()
332 		ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref);  in ivpu_mmu_config_check()
335 static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev)  in ivpu_mmu_cdtab_alloc()  argument
337 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_cdtab_alloc()
341 	cdtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &cdtab->dma, GFP_KERNEL);  in ivpu_mmu_cdtab_alloc()
345 	ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size);  in ivpu_mmu_cdtab_alloc()
350 static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev)  in ivpu_mmu_strtab_alloc()  argument
352 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_strtab_alloc()
356 	strtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &strtab->dma, GFP_KERNEL);  in ivpu_mmu_strtab_alloc()
364 	ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n",  in ivpu_mmu_strtab_alloc()
370 static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev)  in ivpu_mmu_cmdq_alloc()  argument
372 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_cmdq_alloc()
375 	q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, &q->dma, GFP_KERNEL);  in ivpu_mmu_cmdq_alloc()
383 	ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n",  in ivpu_mmu_cmdq_alloc()
389 static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev)  in ivpu_mmu_evtq_alloc()  argument
391 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_evtq_alloc()
394 	q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, &q->dma, GFP_KERNEL);  in ivpu_mmu_evtq_alloc()
402 	ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n",  in ivpu_mmu_evtq_alloc()
408 static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev)  in ivpu_mmu_structs_alloc()  argument
412 	ret = ivpu_mmu_cdtab_alloc(vdev);  in ivpu_mmu_structs_alloc()
414 		ivpu_err(vdev, "Failed to allocate cdtab: %d\n", ret);  in ivpu_mmu_structs_alloc()
418 	ret = ivpu_mmu_strtab_alloc(vdev);  in ivpu_mmu_structs_alloc()
420 		ivpu_err(vdev, "Failed to allocate strtab: %d\n", ret);  in ivpu_mmu_structs_alloc()
424 	ret = ivpu_mmu_cmdq_alloc(vdev);  in ivpu_mmu_structs_alloc()
426 		ivpu_err(vdev, "Failed to allocate cmdq: %d\n", ret);  in ivpu_mmu_structs_alloc()
430 	ret = ivpu_mmu_evtq_alloc(vdev);  in ivpu_mmu_structs_alloc()
432 		ivpu_err(vdev, "Failed to allocate evtq: %d\n", ret);  in ivpu_mmu_structs_alloc()
437 static int ivpu_mmu_reg_write_cr0(struct ivpu_device *vdev, u32 val)  in ivpu_mmu_reg_write_cr0()  argument
444 static int ivpu_mmu_reg_write_irq_ctrl(struct ivpu_device *vdev, u32 val)  in ivpu_mmu_reg_write_irq_ctrl()  argument
451 static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev)  in ivpu_mmu_irqs_setup()  argument
456 	ret = ivpu_mmu_reg_write_irq_ctrl(vdev, 0);  in ivpu_mmu_irqs_setup()
460 	return ivpu_mmu_reg_write_irq_ctrl(vdev, irq_ctrl);  in ivpu_mmu_irqs_setup()
463 static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev)  in ivpu_mmu_cmdq_wait_for_cons()  argument
465 	struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;  in ivpu_mmu_cmdq_wait_for_cons()
490 static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1)  in ivpu_mmu_cmdq_cmd_write()  argument
492 	struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;  in ivpu_mmu_cmdq_cmd_write()
497 		ivpu_err(vdev, "Failed to write MMU CMD %s\n", name);  in ivpu_mmu_cmdq_cmd_write()
505 	ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1);  in ivpu_mmu_cmdq_cmd_write()
510 static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)  in ivpu_mmu_cmdq_sync()  argument
512 	struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;  in ivpu_mmu_cmdq_sync()
518 	ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0);  in ivpu_mmu_cmdq_sync()
522 	if (!ivpu_is_force_snoop_enabled(vdev))  in ivpu_mmu_cmdq_sync()
526 	ret = ivpu_mmu_cmdq_wait_for_cons(vdev);  in ivpu_mmu_cmdq_sync()
533 		ivpu_err(vdev, "Timed out waiting for MMU consumer: %d, error: %s\n", ret,  in ivpu_mmu_cmdq_sync()
535 		ivpu_hw_diagnose_failure(vdev);  in ivpu_mmu_cmdq_sync()
541 static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev)  in ivpu_mmu_cmdq_write_cfgi_all()  argument
546 	return ivpu_mmu_cmdq_cmd_write(vdev, "CFGI_ALL", data0, data1);  in ivpu_mmu_cmdq_write_cfgi_all()
549 static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid)  in ivpu_mmu_cmdq_write_tlbi_nh_asid()  argument
554 	return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NH_ASID", val, 0);  in ivpu_mmu_cmdq_write_tlbi_nh_asid()
557 static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev)  in ivpu_mmu_cmdq_write_tlbi_nsnh_all()  argument
561 	return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NSNH_ALL", val, 0);  in ivpu_mmu_cmdq_write_tlbi_nsnh_all()
564 static int ivpu_mmu_reset(struct ivpu_device *vdev)  in ivpu_mmu_reset()  argument
566 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_reset()
571 	if (!ivpu_is_force_snoop_enabled(vdev))  in ivpu_mmu_reset()
580 	ret = ivpu_mmu_reg_write_cr0(vdev, 0);  in ivpu_mmu_reset()
600 	ret = ivpu_mmu_reg_write_cr0(vdev, val);  in ivpu_mmu_reset()
604 	ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);  in ivpu_mmu_reset()
608 	ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);  in ivpu_mmu_reset()
612 	ret = ivpu_mmu_cmdq_sync(vdev);  in ivpu_mmu_reset()
621 	ret = ivpu_mmu_reg_write_cr0(vdev, val);  in ivpu_mmu_reset()
626 	ret = ivpu_mmu_reg_write_cr0(vdev, val);  in ivpu_mmu_reset()
630 	ret = ivpu_mmu_irqs_setup(vdev);  in ivpu_mmu_reset()
635 	return ivpu_mmu_reg_write_cr0(vdev, val);  in ivpu_mmu_reset()
638 static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid)  in ivpu_mmu_strtab_link_cd()  argument
640 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_strtab_link_cd()
666 	if (!ivpu_is_force_snoop_enabled(vdev))  in ivpu_mmu_strtab_link_cd()
669 	ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]);  in ivpu_mmu_strtab_link_cd()
672 static int ivpu_mmu_strtab_init(struct ivpu_device *vdev)  in ivpu_mmu_strtab_init()  argument
674 	ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0);  in ivpu_mmu_strtab_init()
675 	ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3);  in ivpu_mmu_strtab_init()
680 int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid)  in ivpu_mmu_invalidate_tlb()  argument
682 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_invalidate_tlb()
689 	ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid);  in ivpu_mmu_invalidate_tlb()
693 	ret = ivpu_mmu_cmdq_sync(vdev);  in ivpu_mmu_invalidate_tlb()
699 static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma)  in ivpu_mmu_cd_add()  argument
701 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_cd_add()
741 	if (!ivpu_is_force_snoop_enabled(vdev))  in ivpu_mmu_cd_add()
744 	ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",  in ivpu_mmu_cd_add()
751 	ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);  in ivpu_mmu_cd_add()
755 	ret = ivpu_mmu_cmdq_sync(vdev);  in ivpu_mmu_cd_add()
761 static int ivpu_mmu_cd_add_gbl(struct ivpu_device *vdev)  in ivpu_mmu_cd_add_gbl()  argument
765 	ret = ivpu_mmu_cd_add(vdev, 0, vdev->gctx.pgtable.pgd_dma);  in ivpu_mmu_cd_add_gbl()
767 		ivpu_err(vdev, "Failed to add global CD entry: %d\n", ret);  in ivpu_mmu_cd_add_gbl()
772 static int ivpu_mmu_cd_add_user(struct ivpu_device *vdev, u32 ssid, dma_addr_t cd_dma)  in ivpu_mmu_cd_add_user()  argument
777 		ivpu_err(vdev, "Invalid SSID: %u\n", ssid);  in ivpu_mmu_cd_add_user()
781 	ret = ivpu_mmu_cd_add(vdev, ssid, cd_dma);  in ivpu_mmu_cd_add_user()
783 		ivpu_err(vdev, "Failed to add CD entry SSID=%u: %d\n", ssid, ret);  in ivpu_mmu_cd_add_user()
788 int ivpu_mmu_init(struct ivpu_device *vdev)  in ivpu_mmu_init()  argument
790 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_init()
793 	ivpu_dbg(vdev, MMU, "Init..\n");  in ivpu_mmu_init()
795 	ivpu_mmu_config_check(vdev);  in ivpu_mmu_init()
797 	ret = drmm_mutex_init(&vdev->drm, &mmu->lock);  in ivpu_mmu_init()
801 	ret = ivpu_mmu_structs_alloc(vdev);  in ivpu_mmu_init()
805 	ret = ivpu_mmu_strtab_init(vdev);  in ivpu_mmu_init()
807 		ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret);  in ivpu_mmu_init()
811 	ret = ivpu_mmu_cd_add_gbl(vdev);  in ivpu_mmu_init()
813 		ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret);  in ivpu_mmu_init()
817 	ret = ivpu_mmu_enable(vdev);  in ivpu_mmu_init()
819 		ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);  in ivpu_mmu_init()
823 	ivpu_dbg(vdev, MMU, "Init done\n");  in ivpu_mmu_init()
828 int ivpu_mmu_enable(struct ivpu_device *vdev)  in ivpu_mmu_enable()  argument
830 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_enable()
837 	ret = ivpu_mmu_reset(vdev);  in ivpu_mmu_enable()
839 		ivpu_err(vdev, "Failed to reset MMU: %d\n", ret);  in ivpu_mmu_enable()
843 	ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);  in ivpu_mmu_enable()
847 	ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);  in ivpu_mmu_enable()
851 	ret = ivpu_mmu_cmdq_sync(vdev);  in ivpu_mmu_enable()
864 void ivpu_mmu_disable(struct ivpu_device *vdev)  in ivpu_mmu_disable()  argument
866 	struct ivpu_mmu_info *mmu = vdev->mmu;  in ivpu_mmu_disable()
873 static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event)  in ivpu_mmu_dump_event()  argument
881 …ivpu_err_ratelimited(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0…  in ivpu_mmu_dump_event()
886 static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev)  in ivpu_mmu_get_event()  argument
888 	struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq;  in ivpu_mmu_get_event()
900 void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)  in ivpu_mmu_irq_evtq_handler()  argument
905 	ivpu_dbg(vdev, IRQ, "MMU event queue\n");  in ivpu_mmu_irq_evtq_handler()
907 	while ((event = ivpu_mmu_get_event(vdev)) != NULL) {  in ivpu_mmu_irq_evtq_handler()
908 		ivpu_mmu_dump_event(vdev, event);  in ivpu_mmu_irq_evtq_handler()
912 			ivpu_pm_trigger_recovery(vdev, "MMU event");  in ivpu_mmu_irq_evtq_handler()
916 		ivpu_mmu_user_context_mark_invalid(vdev, ssid);  in ivpu_mmu_irq_evtq_handler()
917 		REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, vdev->mmu->evtq.cons);  in ivpu_mmu_irq_evtq_handler()
920 	if (!kfifo_put(&vdev->hw->irq.fifo, IVPU_HW_IRQ_SRC_MMU_EVTQ))  in ivpu_mmu_irq_evtq_handler()
921 		ivpu_err_ratelimited(vdev, "IRQ FIFO full\n");  in ivpu_mmu_irq_evtq_handler()
924 void ivpu_mmu_evtq_dump(struct ivpu_device *vdev)  in ivpu_mmu_evtq_dump()  argument
928 	while ((event = ivpu_mmu_get_event(vdev)) != NULL)  in ivpu_mmu_evtq_dump()
929 		ivpu_mmu_dump_event(vdev, event);  in ivpu_mmu_evtq_dump()
932 void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev)  in ivpu_mmu_irq_gerr_handler()  argument
936 	ivpu_dbg(vdev, IRQ, "MMU error\n");  in ivpu_mmu_irq_gerr_handler()
946 		ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
949 		ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
952 		ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
955 		ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n");  in ivpu_mmu_irq_gerr_handler()
958 		ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n");  in ivpu_mmu_irq_gerr_handler()
961 		ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n");  in ivpu_mmu_irq_gerr_handler()
964 		ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n");  in ivpu_mmu_irq_gerr_handler()
969 int ivpu_mmu_set_pgtable(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable)  in ivpu_mmu_set_pgtable()  argument
971 	return ivpu_mmu_cd_add_user(vdev, ssid, pgtable->pgd_dma);  in ivpu_mmu_set_pgtable()
974 void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid)  in ivpu_mmu_clear_pgtable()  argument
976 	ivpu_mmu_cd_add_user(vdev, ssid, 0); /* 0 will clear CD entry */  in ivpu_mmu_clear_pgtable()