1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6 #include <linux/circ_buf.h> 7 #include <linux/highmem.h> 8 9 #include "ivpu_drv.h" 10 #include "ivpu_hw.h" 11 #include "ivpu_hw_reg_io.h" 12 #include "ivpu_mmu.h" 13 #include "ivpu_mmu_context.h" 14 #include "ivpu_pm.h" 15 16 #define IVPU_MMU_REG_IDR0 0x00200000u 17 #define IVPU_MMU_REG_IDR1 0x00200004u 18 #define IVPU_MMU_REG_IDR3 0x0020000cu 19 #define IVPU_MMU_REG_IDR5 0x00200014u 20 #define IVPU_MMU_REG_CR0 0x00200020u 21 #define IVPU_MMU_REG_CR0ACK 0x00200024u 22 #define IVPU_MMU_REG_CR0ACK_VAL_MASK GENMASK(31, 0) 23 #define IVPU_MMU_REG_CR1 0x00200028u 24 #define IVPU_MMU_REG_CR2 0x0020002cu 25 #define IVPU_MMU_REG_IRQ_CTRL 0x00200050u 26 #define IVPU_MMU_REG_IRQ_CTRLACK 0x00200054u 27 #define IVPU_MMU_REG_IRQ_CTRLACK_VAL_MASK GENMASK(31, 0) 28 29 #define IVPU_MMU_REG_GERROR 0x00200060u 30 #define IVPU_MMU_REG_GERROR_CMDQ_MASK BIT_MASK(0) 31 #define IVPU_MMU_REG_GERROR_EVTQ_ABT_MASK BIT_MASK(2) 32 #define IVPU_MMU_REG_GERROR_PRIQ_ABT_MASK BIT_MASK(3) 33 #define IVPU_MMU_REG_GERROR_MSI_CMDQ_ABT_MASK BIT_MASK(4) 34 #define IVPU_MMU_REG_GERROR_MSI_EVTQ_ABT_MASK BIT_MASK(5) 35 #define IVPU_MMU_REG_GERROR_MSI_PRIQ_ABT_MASK BIT_MASK(6) 36 #define IVPU_MMU_REG_GERROR_MSI_ABT_MASK BIT_MASK(7) 37 38 #define IVPU_MMU_REG_GERRORN 0x00200064u 39 40 #define IVPU_MMU_REG_STRTAB_BASE 0x00200080u 41 #define IVPU_MMU_REG_STRTAB_BASE_CFG 0x00200088u 42 #define IVPU_MMU_REG_CMDQ_BASE 0x00200090u 43 #define IVPU_MMU_REG_CMDQ_PROD 0x00200098u 44 #define IVPU_MMU_REG_CMDQ_CONS 0x0020009cu 45 #define IVPU_MMU_REG_CMDQ_CONS_VAL_MASK GENMASK(23, 0) 46 #define IVPU_MMU_REG_CMDQ_CONS_ERR_MASK GENMASK(30, 24) 47 #define IVPU_MMU_REG_EVTQ_BASE 0x002000a0u 48 #define IVPU_MMU_REG_EVTQ_PROD 0x002000a8u 49 #define IVPU_MMU_REG_EVTQ_CONS 0x002000acu 50 #define IVPU_MMU_REG_EVTQ_PROD_SEC (0x002000a8u + SZ_64K) 51 #define IVPU_MMU_REG_EVTQ_CONS_SEC (0x002000acu + SZ_64K) 52 53 #define IVPU_MMU_IDR0_REF 0x080f3e0f 54 #define IVPU_MMU_IDR0_REF_SIMICS 0x080f3e1f 55 #define IVPU_MMU_IDR1_REF 0x0e739d18 56 #define IVPU_MMU_IDR3_REF 0x0000003c 57 #define IVPU_MMU_IDR5_REF 0x00040070 58 #define IVPU_MMU_IDR5_REF_SIMICS 0x00000075 59 #define IVPU_MMU_IDR5_REF_FPGA 0x00800075 60 61 #define IVPU_MMU_CDTAB_ENT_SIZE 64 62 #define IVPU_MMU_CDTAB_ENT_COUNT_LOG2 8 /* 256 entries */ 63 #define IVPU_MMU_CDTAB_ENT_COUNT ((u32)1 << IVPU_MMU_CDTAB_ENT_COUNT_LOG2) 64 65 #define IVPU_MMU_STREAM_ID0 0 66 #define IVPU_MMU_STREAM_ID3 3 67 68 #define IVPU_MMU_STRTAB_ENT_SIZE 64 69 #define IVPU_MMU_STRTAB_ENT_COUNT 4 70 #define IVPU_MMU_STRTAB_CFG_LOG2SIZE 2 71 #define IVPU_MMU_STRTAB_CFG IVPU_MMU_STRTAB_CFG_LOG2SIZE 72 73 #define IVPU_MMU_Q_COUNT_LOG2 4 /* 16 entries */ 74 #define IVPU_MMU_Q_COUNT ((u32)1 << IVPU_MMU_Q_COUNT_LOG2) 75 #define IVPU_MMU_Q_WRAP_BIT (IVPU_MMU_Q_COUNT << 1) 76 #define IVPU_MMU_Q_WRAP_MASK (IVPU_MMU_Q_WRAP_BIT - 1) 77 #define IVPU_MMU_Q_IDX_MASK (IVPU_MMU_Q_COUNT - 1) 78 #define IVPU_MMU_Q_IDX(val) ((val) & IVPU_MMU_Q_IDX_MASK) 79 80 #define IVPU_MMU_CMDQ_CMD_SIZE 16 81 #define IVPU_MMU_CMDQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_CMDQ_CMD_SIZE) 82 83 #define IVPU_MMU_EVTQ_CMD_SIZE 32 84 #define IVPU_MMU_EVTQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_EVTQ_CMD_SIZE) 85 86 #define IVPU_MMU_CMD_OPCODE GENMASK(7, 0) 87 88 #define IVPU_MMU_CMD_SYNC_0_CS GENMASK(13, 12) 89 #define IVPU_MMU_CMD_SYNC_0_MSH GENMASK(23, 22) 90 #define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24) 91 #define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24) 92 #define IVPU_MMU_CMD_SYNC_0_MSI_DATA GENMASK(63, 32) 93 94 #define IVPU_MMU_CMD_CFGI_0_SSEC BIT(10) 95 #define IVPU_MMU_CMD_CFGI_0_SSV BIT(11) 96 #define IVPU_MMU_CMD_CFGI_0_SSID GENMASK(31, 12) 97 #define IVPU_MMU_CMD_CFGI_0_SID GENMASK(63, 32) 98 #define IVPU_MMU_CMD_CFGI_1_RANGE GENMASK(4, 0) 99 100 #define IVPU_MMU_CMD_TLBI_0_ASID GENMASK(63, 48) 101 #define IVPU_MMU_CMD_TLBI_0_VMID GENMASK(47, 32) 102 103 #define CMD_PREFETCH_CFG 0x1 104 #define CMD_CFGI_STE 0x3 105 #define CMD_CFGI_ALL 0x4 106 #define CMD_CFGI_CD 0x5 107 #define CMD_CFGI_CD_ALL 0x6 108 #define CMD_TLBI_NH_ASID 0x11 109 #define CMD_TLBI_EL2_ALL 0x20 110 #define CMD_TLBI_NSNH_ALL 0x30 111 #define CMD_SYNC 0x46 112 113 #define IVPU_MMU_EVT_F_UUT 0x01 114 #define IVPU_MMU_EVT_C_BAD_STREAMID 0x02 115 #define IVPU_MMU_EVT_F_STE_FETCH 0x03 116 #define IVPU_MMU_EVT_C_BAD_STE 0x04 117 #define IVPU_MMU_EVT_F_BAD_ATS_TREQ 0x05 118 #define IVPU_MMU_EVT_F_STREAM_DISABLED 0x06 119 #define IVPU_MMU_EVT_F_TRANSL_FORBIDDEN 0x07 120 #define IVPU_MMU_EVT_C_BAD_SUBSTREAMID 0x08 121 #define IVPU_MMU_EVT_F_CD_FETCH 0x09 122 #define IVPU_MMU_EVT_C_BAD_CD 0x0a 123 #define IVPU_MMU_EVT_F_WALK_EABT 0x0b 124 #define IVPU_MMU_EVT_F_TRANSLATION 0x10 125 #define IVPU_MMU_EVT_F_ADDR_SIZE 0x11 126 #define IVPU_MMU_EVT_F_ACCESS 0x12 127 #define IVPU_MMU_EVT_F_PERMISSION 0x13 128 #define IVPU_MMU_EVT_F_TLB_CONFLICT 0x20 129 #define IVPU_MMU_EVT_F_CFG_CONFLICT 0x21 130 #define IVPU_MMU_EVT_E_PAGE_REQUEST 0x24 131 #define IVPU_MMU_EVT_F_VMS_FETCH 0x25 132 133 #define IVPU_MMU_EVT_OP_MASK GENMASK_ULL(7, 0) 134 #define IVPU_MMU_EVT_SSID_MASK GENMASK_ULL(31, 12) 135 136 #define IVPU_MMU_Q_BASE_RWA BIT(62) 137 #define IVPU_MMU_Q_BASE_ADDR_MASK GENMASK_ULL(51, 5) 138 #define IVPU_MMU_STRTAB_BASE_RA BIT(62) 139 #define IVPU_MMU_STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6) 140 141 #define IVPU_MMU_IRQ_EVTQ_EN BIT(2) 142 #define IVPU_MMU_IRQ_GERROR_EN BIT(0) 143 144 #define IVPU_MMU_CR0_ATSCHK BIT(4) 145 #define IVPU_MMU_CR0_CMDQEN BIT(3) 146 #define IVPU_MMU_CR0_EVTQEN BIT(2) 147 #define IVPU_MMU_CR0_PRIQEN BIT(1) 148 #define IVPU_MMU_CR0_SMMUEN BIT(0) 149 150 #define IVPU_MMU_CR1_TABLE_SH GENMASK(11, 10) 151 #define IVPU_MMU_CR1_TABLE_OC GENMASK(9, 8) 152 #define IVPU_MMU_CR1_TABLE_IC GENMASK(7, 6) 153 #define IVPU_MMU_CR1_QUEUE_SH GENMASK(5, 4) 154 #define IVPU_MMU_CR1_QUEUE_OC GENMASK(3, 2) 155 #define IVPU_MMU_CR1_QUEUE_IC GENMASK(1, 0) 156 #define IVPU_MMU_CACHE_NC 0 157 #define IVPU_MMU_CACHE_WB 1 158 #define IVPU_MMU_CACHE_WT 2 159 #define IVPU_MMU_SH_NSH 0 160 #define IVPU_MMU_SH_OSH 2 161 #define IVPU_MMU_SH_ISH 3 162 163 #define IVPU_MMU_CMDQ_OP GENMASK_ULL(7, 0) 164 165 #define IVPU_MMU_CD_0_TCR_T0SZ GENMASK_ULL(5, 0) 166 #define IVPU_MMU_CD_0_TCR_TG0 GENMASK_ULL(7, 6) 167 #define IVPU_MMU_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8) 168 #define IVPU_MMU_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10) 169 #define IVPU_MMU_CD_0_TCR_SH0 GENMASK_ULL(13, 12) 170 #define IVPU_MMU_CD_0_TCR_EPD0 BIT_ULL(14) 171 #define IVPU_MMU_CD_0_TCR_EPD1 BIT_ULL(30) 172 #define IVPU_MMU_CD_0_ENDI BIT(15) 173 #define IVPU_MMU_CD_0_V BIT(31) 174 #define IVPU_MMU_CD_0_TCR_IPS GENMASK_ULL(34, 32) 175 #define IVPU_MMU_CD_0_TCR_TBI0 BIT_ULL(38) 176 #define IVPU_MMU_CD_0_AA64 BIT(41) 177 #define IVPU_MMU_CD_0_S BIT(44) 178 #define IVPU_MMU_CD_0_R BIT(45) 179 #define IVPU_MMU_CD_0_A BIT(46) 180 #define IVPU_MMU_CD_0_ASET BIT(47) 181 #define IVPU_MMU_CD_0_ASID GENMASK_ULL(63, 48) 182 183 #define IVPU_MMU_T0SZ_48BIT 16 184 #define IVPU_MMU_T0SZ_38BIT 26 185 186 #define IVPU_MMU_IPS_48BIT 5 187 #define IVPU_MMU_IPS_44BIT 4 188 #define IVPU_MMU_IPS_42BIT 3 189 #define IVPU_MMU_IPS_40BIT 2 190 #define IVPU_MMU_IPS_36BIT 1 191 #define IVPU_MMU_IPS_32BIT 0 192 193 #define IVPU_MMU_CD_1_TTB0_MASK GENMASK_ULL(51, 4) 194 195 #define IVPU_MMU_STE_0_S1CDMAX GENMASK_ULL(63, 59) 196 #define IVPU_MMU_STE_0_S1FMT GENMASK_ULL(5, 4) 197 #define IVPU_MMU_STE_0_S1FMT_LINEAR 0 198 #define IVPU_MMU_STE_DWORDS 8 199 #define IVPU_MMU_STE_0_CFG_S1_TRANS 5 200 #define IVPU_MMU_STE_0_CFG GENMASK_ULL(3, 1) 201 #define IVPU_MMU_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6) 202 #define IVPU_MMU_STE_0_V BIT(0) 203 204 #define IVPU_MMU_STE_1_STRW_NSEL1 0ul 205 #define IVPU_MMU_STE_1_CONT GENMASK_ULL(16, 13) 206 #define IVPU_MMU_STE_1_STRW GENMASK_ULL(31, 30) 207 #define IVPU_MMU_STE_1_PRIVCFG GENMASK_ULL(49, 48) 208 #define IVPU_MMU_STE_1_PRIVCFG_UNPRIV 2ul 209 #define IVPU_MMU_STE_1_INSTCFG GENMASK_ULL(51, 50) 210 #define IVPU_MMU_STE_1_INSTCFG_DATA 2ul 211 #define IVPU_MMU_STE_1_MEV BIT(19) 212 #define IVPU_MMU_STE_1_S1STALLD BIT(27) 213 #define IVPU_MMU_STE_1_S1C_CACHE_NC 0ul 214 #define IVPU_MMU_STE_1_S1C_CACHE_WBRA 1ul 215 #define IVPU_MMU_STE_1_S1C_CACHE_WT 2ul 216 #define IVPU_MMU_STE_1_S1C_CACHE_WB 3ul 217 #define IVPU_MMU_STE_1_S1CIR GENMASK_ULL(3, 2) 218 #define IVPU_MMU_STE_1_S1COR GENMASK_ULL(5, 4) 219 #define IVPU_MMU_STE_1_S1CSH GENMASK_ULL(7, 6) 220 #define IVPU_MMU_STE_1_S1DSS GENMASK_ULL(1, 0) 221 #define IVPU_MMU_STE_1_S1DSS_TERMINATE 0x0 222 223 #define IVPU_MMU_REG_TIMEOUT_US (10 * USEC_PER_MSEC) 224 #define IVPU_MMU_QUEUE_TIMEOUT_US (100 * USEC_PER_MSEC) 225 226 #define IVPU_MMU_GERROR_ERR_MASK ((REG_FLD(IVPU_MMU_REG_GERROR, CMDQ)) | \ 227 (REG_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT)) | \ 228 (REG_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT)) | \ 229 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT)) | \ 230 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT)) | \ 231 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT)) | \ 232 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_ABT))) 233 234 #define IVPU_MMU_CERROR_NONE 0x0 235 #define IVPU_MMU_CERROR_ILL 0x1 236 #define IVPU_MMU_CERROR_ABT 0x2 237 #define IVPU_MMU_CERROR_ATC_INV_SYNC 0x3 238 239 static const char *ivpu_mmu_event_to_str(u32 cmd) 240 { 241 switch (cmd) { 242 case IVPU_MMU_EVT_F_UUT: 243 return "Unsupported Upstream Transaction"; 244 case IVPU_MMU_EVT_C_BAD_STREAMID: 245 return "Transaction StreamID out of range"; 246 case IVPU_MMU_EVT_F_STE_FETCH: 247 return "Fetch of STE caused external abort"; 248 case IVPU_MMU_EVT_C_BAD_STE: 249 return "Used STE invalid"; 250 case IVPU_MMU_EVT_F_BAD_ATS_TREQ: 251 return "Address Request disallowed for a StreamID"; 252 case IVPU_MMU_EVT_F_STREAM_DISABLED: 253 return "Transaction marks non-substream disabled"; 254 case IVPU_MMU_EVT_F_TRANSL_FORBIDDEN: 255 return "MMU bypass is disallowed for this StreamID"; 256 case IVPU_MMU_EVT_C_BAD_SUBSTREAMID: 257 return "Invalid StreamID"; 258 case IVPU_MMU_EVT_F_CD_FETCH: 259 return "Fetch of CD caused external abort"; 260 case IVPU_MMU_EVT_C_BAD_CD: 261 return "Fetched CD invalid"; 262 case IVPU_MMU_EVT_F_WALK_EABT: 263 return " An external abort occurred fetching a TLB"; 264 case IVPU_MMU_EVT_F_TRANSLATION: 265 return "Translation fault"; 266 case IVPU_MMU_EVT_F_ADDR_SIZE: 267 return " Output address caused address size fault"; 268 case IVPU_MMU_EVT_F_ACCESS: 269 return "Access flag fault"; 270 case IVPU_MMU_EVT_F_PERMISSION: 271 return "Permission fault occurred on page access"; 272 case IVPU_MMU_EVT_F_TLB_CONFLICT: 273 return "A TLB conflict"; 274 case IVPU_MMU_EVT_F_CFG_CONFLICT: 275 return "A configuration cache conflict"; 276 case IVPU_MMU_EVT_E_PAGE_REQUEST: 277 return "Page request hint from a client device"; 278 case IVPU_MMU_EVT_F_VMS_FETCH: 279 return "Fetch of VMS caused external abort"; 280 default: 281 return "Unknown CMDQ command"; 282 } 283 } 284 285 static const char *ivpu_mmu_cmdq_err_to_str(u32 err) 286 { 287 switch (err) { 288 case IVPU_MMU_CERROR_NONE: 289 return "No CMDQ Error"; 290 case IVPU_MMU_CERROR_ILL: 291 return "Illegal command"; 292 case IVPU_MMU_CERROR_ABT: 293 return "External abort on CMDQ read"; 294 case IVPU_MMU_CERROR_ATC_INV_SYNC: 295 return "Sync failed to complete ATS invalidation"; 296 default: 297 return "Unknown CMDQ Error"; 298 } 299 } 300 301 static void ivpu_mmu_config_check(struct ivpu_device *vdev) 302 { 303 u32 val_ref; 304 u32 val; 305 306 if (ivpu_is_simics(vdev)) 307 val_ref = IVPU_MMU_IDR0_REF_SIMICS; 308 else 309 val_ref = IVPU_MMU_IDR0_REF; 310 311 val = REGV_RD32(IVPU_MMU_REG_IDR0); 312 if (val != val_ref) 313 ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref); 314 315 val = REGV_RD32(IVPU_MMU_REG_IDR1); 316 if (val != IVPU_MMU_IDR1_REF) 317 ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF); 318 319 val = REGV_RD32(IVPU_MMU_REG_IDR3); 320 if (val != IVPU_MMU_IDR3_REF) 321 ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF); 322 323 if (ivpu_is_simics(vdev)) 324 val_ref = IVPU_MMU_IDR5_REF_SIMICS; 325 else if (ivpu_is_fpga(vdev)) 326 val_ref = IVPU_MMU_IDR5_REF_FPGA; 327 else 328 val_ref = IVPU_MMU_IDR5_REF; 329 330 val = REGV_RD32(IVPU_MMU_REG_IDR5); 331 if (val != val_ref) 332 ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref); 333 } 334 335 static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev) 336 { 337 struct ivpu_mmu_info *mmu = vdev->mmu; 338 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; 339 size_t size = IVPU_MMU_CDTAB_ENT_COUNT * IVPU_MMU_CDTAB_ENT_SIZE; 340 341 cdtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &cdtab->dma, GFP_KERNEL); 342 if (!cdtab->base) 343 return -ENOMEM; 344 345 ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size); 346 347 return 0; 348 } 349 350 static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev) 351 { 352 struct ivpu_mmu_info *mmu = vdev->mmu; 353 struct ivpu_mmu_strtab *strtab = &mmu->strtab; 354 size_t size = IVPU_MMU_STRTAB_ENT_COUNT * IVPU_MMU_STRTAB_ENT_SIZE; 355 356 strtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &strtab->dma, GFP_KERNEL); 357 if (!strtab->base) 358 return -ENOMEM; 359 360 strtab->base_cfg = IVPU_MMU_STRTAB_CFG; 361 strtab->dma_q = IVPU_MMU_STRTAB_BASE_RA; 362 strtab->dma_q |= strtab->dma & IVPU_MMU_STRTAB_BASE_ADDR_MASK; 363 364 ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n", 365 &strtab->dma, &strtab->dma_q, size); 366 367 return 0; 368 } 369 370 static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev) 371 { 372 struct ivpu_mmu_info *mmu = vdev->mmu; 373 struct ivpu_mmu_queue *q = &mmu->cmdq; 374 375 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, &q->dma, GFP_KERNEL); 376 if (!q->base) 377 return -ENOMEM; 378 379 q->dma_q = IVPU_MMU_Q_BASE_RWA; 380 q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK; 381 q->dma_q |= IVPU_MMU_Q_COUNT_LOG2; 382 383 ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n", 384 &q->dma, &q->dma_q, IVPU_MMU_CMDQ_SIZE); 385 386 return 0; 387 } 388 389 static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev) 390 { 391 struct ivpu_mmu_info *mmu = vdev->mmu; 392 struct ivpu_mmu_queue *q = &mmu->evtq; 393 394 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, &q->dma, GFP_KERNEL); 395 if (!q->base) 396 return -ENOMEM; 397 398 q->dma_q = IVPU_MMU_Q_BASE_RWA; 399 q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK; 400 q->dma_q |= IVPU_MMU_Q_COUNT_LOG2; 401 402 ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n", 403 &q->dma, &q->dma_q, IVPU_MMU_EVTQ_SIZE); 404 405 return 0; 406 } 407 408 static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev) 409 { 410 int ret; 411 412 ret = ivpu_mmu_cdtab_alloc(vdev); 413 if (ret) { 414 ivpu_err(vdev, "Failed to allocate cdtab: %d\n", ret); 415 return ret; 416 } 417 418 ret = ivpu_mmu_strtab_alloc(vdev); 419 if (ret) { 420 ivpu_err(vdev, "Failed to allocate strtab: %d\n", ret); 421 return ret; 422 } 423 424 ret = ivpu_mmu_cmdq_alloc(vdev); 425 if (ret) { 426 ivpu_err(vdev, "Failed to allocate cmdq: %d\n", ret); 427 return ret; 428 } 429 430 ret = ivpu_mmu_evtq_alloc(vdev); 431 if (ret) 432 ivpu_err(vdev, "Failed to allocate evtq: %d\n", ret); 433 434 return ret; 435 } 436 437 static int ivpu_mmu_reg_write_cr0(struct ivpu_device *vdev, u32 val) 438 { 439 REGV_WR32(IVPU_MMU_REG_CR0, val); 440 441 return REGV_POLL_FLD(IVPU_MMU_REG_CR0ACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US); 442 } 443 444 static int ivpu_mmu_reg_write_irq_ctrl(struct ivpu_device *vdev, u32 val) 445 { 446 REGV_WR32(IVPU_MMU_REG_IRQ_CTRL, val); 447 448 return REGV_POLL_FLD(IVPU_MMU_REG_IRQ_CTRLACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US); 449 } 450 451 static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev) 452 { 453 u32 irq_ctrl = IVPU_MMU_IRQ_EVTQ_EN | IVPU_MMU_IRQ_GERROR_EN; 454 int ret; 455 456 ret = ivpu_mmu_reg_write_irq_ctrl(vdev, 0); 457 if (ret) 458 return ret; 459 460 return ivpu_mmu_reg_write_irq_ctrl(vdev, irq_ctrl); 461 } 462 463 static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev) 464 { 465 struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq; 466 int ret; 467 468 ret = REGV_POLL_FLD(IVPU_MMU_REG_CMDQ_CONS, VAL, cmdq->prod, 469 IVPU_MMU_QUEUE_TIMEOUT_US); 470 if (ret) 471 return ret; 472 473 cmdq->cons = cmdq->prod; 474 475 return 0; 476 } 477 478 static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1) 479 { 480 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq; 481 u64 *queue_buffer = q->base; 482 int idx = IVPU_MMU_Q_IDX(q->prod) * (IVPU_MMU_CMDQ_CMD_SIZE / sizeof(*queue_buffer)); 483 484 if (!CIRC_SPACE(IVPU_MMU_Q_IDX(q->prod), IVPU_MMU_Q_IDX(q->cons), IVPU_MMU_Q_COUNT)) { 485 ivpu_err(vdev, "Failed to write MMU CMD %s\n", name); 486 return -EBUSY; 487 } 488 489 queue_buffer[idx] = data0; 490 queue_buffer[idx + 1] = data1; 491 q->prod = (q->prod + 1) & IVPU_MMU_Q_WRAP_MASK; 492 493 ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1); 494 495 return 0; 496 } 497 498 static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev) 499 { 500 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq; 501 u64 val; 502 int ret; 503 504 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC); 505 506 ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0); 507 if (ret) 508 return ret; 509 510 clflush_cache_range(q->base, IVPU_MMU_CMDQ_SIZE); 511 REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, q->prod); 512 513 ret = ivpu_mmu_cmdq_wait_for_cons(vdev); 514 if (ret) { 515 u32 err; 516 517 val = REGV_RD32(IVPU_MMU_REG_CMDQ_CONS); 518 err = REG_GET_FLD(IVPU_MMU_REG_CMDQ_CONS, ERR, val); 519 520 ivpu_err(vdev, "Timed out waiting for MMU consumer: %d, error: %s\n", ret, 521 ivpu_mmu_cmdq_err_to_str(err)); 522 ivpu_hw_diagnose_failure(vdev); 523 } 524 525 return ret; 526 } 527 528 static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev) 529 { 530 u64 data0 = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_CFGI_ALL); 531 u64 data1 = FIELD_PREP(IVPU_MMU_CMD_CFGI_1_RANGE, 0x1f); 532 533 return ivpu_mmu_cmdq_cmd_write(vdev, "CFGI_ALL", data0, data1); 534 } 535 536 static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid) 537 { 538 u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NH_ASID) | 539 FIELD_PREP(IVPU_MMU_CMD_TLBI_0_ASID, ssid); 540 541 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NH_ASID", val, 0); 542 } 543 544 static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev) 545 { 546 u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NSNH_ALL); 547 548 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NSNH_ALL", val, 0); 549 } 550 551 static int ivpu_mmu_reset(struct ivpu_device *vdev) 552 { 553 struct ivpu_mmu_info *mmu = vdev->mmu; 554 u32 val; 555 int ret; 556 557 memset(mmu->cmdq.base, 0, IVPU_MMU_CMDQ_SIZE); 558 clflush_cache_range(mmu->cmdq.base, IVPU_MMU_CMDQ_SIZE); 559 mmu->cmdq.prod = 0; 560 mmu->cmdq.cons = 0; 561 562 memset(mmu->evtq.base, 0, IVPU_MMU_EVTQ_SIZE); 563 mmu->evtq.prod = 0; 564 mmu->evtq.cons = 0; 565 566 ret = ivpu_mmu_reg_write_cr0(vdev, 0); 567 if (ret) 568 return ret; 569 570 val = FIELD_PREP(IVPU_MMU_CR1_TABLE_SH, IVPU_MMU_SH_ISH) | 571 FIELD_PREP(IVPU_MMU_CR1_TABLE_OC, IVPU_MMU_CACHE_WB) | 572 FIELD_PREP(IVPU_MMU_CR1_TABLE_IC, IVPU_MMU_CACHE_WB) | 573 FIELD_PREP(IVPU_MMU_CR1_QUEUE_SH, IVPU_MMU_SH_ISH) | 574 FIELD_PREP(IVPU_MMU_CR1_QUEUE_OC, IVPU_MMU_CACHE_WB) | 575 FIELD_PREP(IVPU_MMU_CR1_QUEUE_IC, IVPU_MMU_CACHE_WB); 576 REGV_WR32(IVPU_MMU_REG_CR1, val); 577 578 REGV_WR64(IVPU_MMU_REG_STRTAB_BASE, mmu->strtab.dma_q); 579 REGV_WR32(IVPU_MMU_REG_STRTAB_BASE_CFG, mmu->strtab.base_cfg); 580 581 REGV_WR64(IVPU_MMU_REG_CMDQ_BASE, mmu->cmdq.dma_q); 582 REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, 0); 583 REGV_WR32(IVPU_MMU_REG_CMDQ_CONS, 0); 584 585 val = IVPU_MMU_CR0_CMDQEN; 586 ret = ivpu_mmu_reg_write_cr0(vdev, val); 587 if (ret) 588 return ret; 589 590 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); 591 if (ret) 592 return ret; 593 594 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev); 595 if (ret) 596 return ret; 597 598 ret = ivpu_mmu_cmdq_sync(vdev); 599 if (ret) 600 return ret; 601 602 REGV_WR64(IVPU_MMU_REG_EVTQ_BASE, mmu->evtq.dma_q); 603 REGV_WR32(IVPU_MMU_REG_EVTQ_PROD_SEC, 0); 604 REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, 0); 605 606 val |= IVPU_MMU_CR0_EVTQEN; 607 ret = ivpu_mmu_reg_write_cr0(vdev, val); 608 if (ret) 609 return ret; 610 611 val |= IVPU_MMU_CR0_ATSCHK; 612 ret = ivpu_mmu_reg_write_cr0(vdev, val); 613 if (ret) 614 return ret; 615 616 ret = ivpu_mmu_irqs_setup(vdev); 617 if (ret) 618 return ret; 619 620 val |= IVPU_MMU_CR0_SMMUEN; 621 return ivpu_mmu_reg_write_cr0(vdev, val); 622 } 623 624 static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid) 625 { 626 struct ivpu_mmu_info *mmu = vdev->mmu; 627 struct ivpu_mmu_strtab *strtab = &mmu->strtab; 628 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; 629 u64 *entry = strtab->base + (sid * IVPU_MMU_STRTAB_ENT_SIZE); 630 u64 str[2]; 631 632 str[0] = FIELD_PREP(IVPU_MMU_STE_0_CFG, IVPU_MMU_STE_0_CFG_S1_TRANS) | 633 FIELD_PREP(IVPU_MMU_STE_0_S1CDMAX, IVPU_MMU_CDTAB_ENT_COUNT_LOG2) | 634 FIELD_PREP(IVPU_MMU_STE_0_S1FMT, IVPU_MMU_STE_0_S1FMT_LINEAR) | 635 IVPU_MMU_STE_0_V | 636 (cdtab->dma & IVPU_MMU_STE_0_S1CTXPTR_MASK); 637 638 str[1] = FIELD_PREP(IVPU_MMU_STE_1_S1DSS, IVPU_MMU_STE_1_S1DSS_TERMINATE) | 639 FIELD_PREP(IVPU_MMU_STE_1_S1CIR, IVPU_MMU_STE_1_S1C_CACHE_NC) | 640 FIELD_PREP(IVPU_MMU_STE_1_S1COR, IVPU_MMU_STE_1_S1C_CACHE_NC) | 641 FIELD_PREP(IVPU_MMU_STE_1_S1CSH, IVPU_MMU_SH_NSH) | 642 FIELD_PREP(IVPU_MMU_STE_1_PRIVCFG, IVPU_MMU_STE_1_PRIVCFG_UNPRIV) | 643 FIELD_PREP(IVPU_MMU_STE_1_INSTCFG, IVPU_MMU_STE_1_INSTCFG_DATA) | 644 FIELD_PREP(IVPU_MMU_STE_1_STRW, IVPU_MMU_STE_1_STRW_NSEL1) | 645 FIELD_PREP(IVPU_MMU_STE_1_CONT, IVPU_MMU_STRTAB_CFG_LOG2SIZE) | 646 IVPU_MMU_STE_1_MEV | 647 IVPU_MMU_STE_1_S1STALLD; 648 649 WRITE_ONCE(entry[1], str[1]); 650 WRITE_ONCE(entry[0], str[0]); 651 652 clflush_cache_range(entry, IVPU_MMU_STRTAB_ENT_SIZE); 653 654 ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]); 655 } 656 657 static int ivpu_mmu_strtab_init(struct ivpu_device *vdev) 658 { 659 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0); 660 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3); 661 662 return 0; 663 } 664 665 int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid) 666 { 667 struct ivpu_mmu_info *mmu = vdev->mmu; 668 int ret = 0; 669 670 mutex_lock(&mmu->lock); 671 if (!mmu->on) 672 goto unlock; 673 674 ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid); 675 if (ret) 676 goto unlock; 677 678 ret = ivpu_mmu_cmdq_sync(vdev); 679 unlock: 680 mutex_unlock(&mmu->lock); 681 return ret; 682 } 683 684 static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma) 685 { 686 struct ivpu_mmu_info *mmu = vdev->mmu; 687 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; 688 u64 *entry; 689 u64 cd[4]; 690 int ret = 0; 691 692 if (ssid > IVPU_MMU_CDTAB_ENT_COUNT) 693 return -EINVAL; 694 695 entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE); 696 697 if (cd_dma != 0) { 698 cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, IVPU_MMU_T0SZ_48BIT) | 699 FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) | 700 FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) | 701 FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) | 702 FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) | 703 FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, IVPU_MMU_IPS_48BIT) | 704 FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) | 705 IVPU_MMU_CD_0_TCR_EPD1 | 706 IVPU_MMU_CD_0_AA64 | 707 IVPU_MMU_CD_0_R | 708 IVPU_MMU_CD_0_ASET | 709 IVPU_MMU_CD_0_V; 710 cd[1] = cd_dma & IVPU_MMU_CD_1_TTB0_MASK; 711 cd[2] = 0; 712 cd[3] = 0x0000000000007444; 713 714 /* For global context generate memory fault on VPU */ 715 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID) 716 cd[0] |= IVPU_MMU_CD_0_A; 717 } else { 718 memset(cd, 0, sizeof(cd)); 719 } 720 721 WRITE_ONCE(entry[1], cd[1]); 722 WRITE_ONCE(entry[2], cd[2]); 723 WRITE_ONCE(entry[3], cd[3]); 724 WRITE_ONCE(entry[0], cd[0]); 725 726 clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE); 727 728 ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n", 729 cd_dma ? "write" : "clear", ssid, &cd_dma, cd[0], cd[1], cd[2], cd[3]); 730 731 mutex_lock(&mmu->lock); 732 if (!mmu->on) 733 goto unlock; 734 735 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); 736 if (ret) 737 goto unlock; 738 739 ret = ivpu_mmu_cmdq_sync(vdev); 740 unlock: 741 mutex_unlock(&mmu->lock); 742 return ret; 743 } 744 745 static int ivpu_mmu_cd_add_gbl(struct ivpu_device *vdev) 746 { 747 int ret; 748 749 ret = ivpu_mmu_cd_add(vdev, 0, vdev->gctx.pgtable.pgd_dma); 750 if (ret) 751 ivpu_err(vdev, "Failed to add global CD entry: %d\n", ret); 752 753 return ret; 754 } 755 756 static int ivpu_mmu_cd_add_user(struct ivpu_device *vdev, u32 ssid, dma_addr_t cd_dma) 757 { 758 int ret; 759 760 if (ssid == 0) { 761 ivpu_err(vdev, "Invalid SSID: %u\n", ssid); 762 return -EINVAL; 763 } 764 765 ret = ivpu_mmu_cd_add(vdev, ssid, cd_dma); 766 if (ret) 767 ivpu_err(vdev, "Failed to add CD entry SSID=%u: %d\n", ssid, ret); 768 769 return ret; 770 } 771 772 int ivpu_mmu_init(struct ivpu_device *vdev) 773 { 774 struct ivpu_mmu_info *mmu = vdev->mmu; 775 int ret; 776 777 ivpu_dbg(vdev, MMU, "Init..\n"); 778 779 ivpu_mmu_config_check(vdev); 780 781 ret = drmm_mutex_init(&vdev->drm, &mmu->lock); 782 if (ret) 783 return ret; 784 785 ret = ivpu_mmu_structs_alloc(vdev); 786 if (ret) 787 return ret; 788 789 ret = ivpu_mmu_strtab_init(vdev); 790 if (ret) { 791 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret); 792 return ret; 793 } 794 795 ret = ivpu_mmu_cd_add_gbl(vdev); 796 if (ret) { 797 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret); 798 return ret; 799 } 800 801 ret = ivpu_mmu_enable(vdev); 802 if (ret) { 803 ivpu_err(vdev, "Failed to resume MMU: %d\n", ret); 804 return ret; 805 } 806 807 ivpu_dbg(vdev, MMU, "Init done\n"); 808 809 return 0; 810 } 811 812 int ivpu_mmu_enable(struct ivpu_device *vdev) 813 { 814 struct ivpu_mmu_info *mmu = vdev->mmu; 815 int ret; 816 817 mutex_lock(&mmu->lock); 818 819 mmu->on = true; 820 821 ret = ivpu_mmu_reset(vdev); 822 if (ret) { 823 ivpu_err(vdev, "Failed to reset MMU: %d\n", ret); 824 goto err; 825 } 826 827 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); 828 if (ret) 829 goto err; 830 831 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev); 832 if (ret) 833 goto err; 834 835 ret = ivpu_mmu_cmdq_sync(vdev); 836 if (ret) 837 goto err; 838 839 mutex_unlock(&mmu->lock); 840 841 return 0; 842 err: 843 mmu->on = false; 844 mutex_unlock(&mmu->lock); 845 return ret; 846 } 847 848 void ivpu_mmu_disable(struct ivpu_device *vdev) 849 { 850 struct ivpu_mmu_info *mmu = vdev->mmu; 851 852 mutex_lock(&mmu->lock); 853 mmu->on = false; 854 mutex_unlock(&mmu->lock); 855 } 856 857 static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event) 858 { 859 u32 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]); 860 u32 op = FIELD_GET(IVPU_MMU_EVT_OP_MASK, event[0]); 861 u64 fetch_addr = ((u64)event[7]) << 32 | event[6]; 862 u64 in_addr = ((u64)event[5]) << 32 | event[4]; 863 u32 sid = event[1]; 864 865 ivpu_err(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch addr: 0x%llx\n", 866 op, ivpu_mmu_event_to_str(op), ssid, sid, event[2], event[3], in_addr, fetch_addr); 867 } 868 869 static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev) 870 { 871 struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq; 872 u32 idx = IVPU_MMU_Q_IDX(evtq->cons); 873 u32 *evt = evtq->base + (idx * IVPU_MMU_EVTQ_CMD_SIZE); 874 875 evtq->prod = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC); 876 if (!CIRC_CNT(IVPU_MMU_Q_IDX(evtq->prod), IVPU_MMU_Q_IDX(evtq->cons), IVPU_MMU_Q_COUNT)) 877 return NULL; 878 879 evtq->cons = (evtq->cons + 1) & IVPU_MMU_Q_WRAP_MASK; 880 REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, evtq->cons); 881 882 return evt; 883 } 884 885 void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev) 886 { 887 u32 *event; 888 u32 ssid; 889 890 ivpu_dbg(vdev, IRQ, "MMU event queue\n"); 891 892 while ((event = ivpu_mmu_get_event(vdev)) != NULL) { 893 ivpu_mmu_dump_event(vdev, event); 894 895 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]); 896 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID) { 897 ivpu_pm_trigger_recovery(vdev, "MMU event"); 898 return; 899 } 900 901 ivpu_mmu_user_context_mark_invalid(vdev, ssid); 902 } 903 } 904 905 void ivpu_mmu_evtq_dump(struct ivpu_device *vdev) 906 { 907 u32 *event; 908 909 while ((event = ivpu_mmu_get_event(vdev)) != NULL) 910 ivpu_mmu_dump_event(vdev, event); 911 } 912 913 void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev) 914 { 915 u32 gerror_val, gerrorn_val, active; 916 917 ivpu_dbg(vdev, IRQ, "MMU error\n"); 918 919 gerror_val = REGV_RD32(IVPU_MMU_REG_GERROR); 920 gerrorn_val = REGV_RD32(IVPU_MMU_REG_GERRORN); 921 922 active = gerror_val ^ gerrorn_val; 923 if (!(active & IVPU_MMU_GERROR_ERR_MASK)) 924 return; 925 926 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_ABT, active)) 927 ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n"); 928 929 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT, active)) 930 ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n"); 931 932 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT, active)) 933 ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n"); 934 935 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT, active)) 936 ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n"); 937 938 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT, active)) 939 ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n"); 940 941 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT, active)) 942 ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n"); 943 944 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, CMDQ, active)) 945 ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n"); 946 947 REGV_WR32(IVPU_MMU_REG_GERRORN, gerror_val); 948 } 949 950 int ivpu_mmu_set_pgtable(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable) 951 { 952 return ivpu_mmu_cd_add_user(vdev, ssid, pgtable->pgd_dma); 953 } 954 955 void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid) 956 { 957 ivpu_mmu_cd_add_user(vdev, ssid, 0); /* 0 will clear CD entry */ 958 } 959