1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6 #include <linux/circ_buf.h> 7 #include <linux/highmem.h> 8 9 #include "ivpu_drv.h" 10 #include "ivpu_hw_mtl_reg.h" 11 #include "ivpu_hw_reg_io.h" 12 #include "ivpu_mmu.h" 13 #include "ivpu_mmu_context.h" 14 15 #define IVPU_MMU_IDR0_REF 0x080f3e0f 16 #define IVPU_MMU_IDR0_REF_SIMICS 0x080f3e1f 17 #define IVPU_MMU_IDR1_REF 0x0e739d18 18 #define IVPU_MMU_IDR3_REF 0x0000003c 19 #define IVPU_MMU_IDR5_REF 0x00040070 20 #define IVPU_MMU_IDR5_REF_SIMICS 0x00000075 21 #define IVPU_MMU_IDR5_REF_FPGA 0x00800075 22 23 #define IVPU_MMU_CDTAB_ENT_SIZE 64 24 #define IVPU_MMU_CDTAB_ENT_COUNT_LOG2 8 /* 256 entries */ 25 #define IVPU_MMU_CDTAB_ENT_COUNT ((u32)1 << IVPU_MMU_CDTAB_ENT_COUNT_LOG2) 26 27 #define IVPU_MMU_STREAM_ID0 0 28 #define IVPU_MMU_STREAM_ID3 3 29 30 #define IVPU_MMU_STRTAB_ENT_SIZE 64 31 #define IVPU_MMU_STRTAB_ENT_COUNT 4 32 #define IVPU_MMU_STRTAB_CFG_LOG2SIZE 2 33 #define IVPU_MMU_STRTAB_CFG IVPU_MMU_STRTAB_CFG_LOG2SIZE 34 35 #define IVPU_MMU_Q_COUNT_LOG2 4 /* 16 entries */ 36 #define IVPU_MMU_Q_COUNT ((u32)1 << IVPU_MMU_Q_COUNT_LOG2) 37 #define IVPU_MMU_Q_WRAP_BIT (IVPU_MMU_Q_COUNT << 1) 38 #define IVPU_MMU_Q_WRAP_MASK (IVPU_MMU_Q_WRAP_BIT - 1) 39 #define IVPU_MMU_Q_IDX_MASK (IVPU_MMU_Q_COUNT - 1) 40 #define IVPU_MMU_Q_IDX(val) ((val) & IVPU_MMU_Q_IDX_MASK) 41 42 #define IVPU_MMU_CMDQ_CMD_SIZE 16 43 #define IVPU_MMU_CMDQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_CMDQ_CMD_SIZE) 44 45 #define IVPU_MMU_EVTQ_CMD_SIZE 32 46 #define IVPU_MMU_EVTQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_EVTQ_CMD_SIZE) 47 48 #define IVPU_MMU_CMD_OPCODE GENMASK(7, 0) 49 50 #define IVPU_MMU_CMD_SYNC_0_CS GENMASK(13, 12) 51 #define IVPU_MMU_CMD_SYNC_0_MSH GENMASK(23, 22) 52 #define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24) 53 #define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24) 54 #define IVPU_MMU_CMD_SYNC_0_MSI_DATA GENMASK(63, 32) 55 56 #define IVPU_MMU_CMD_CFGI_0_SSEC BIT(10) 57 #define IVPU_MMU_CMD_CFGI_0_SSV BIT(11) 58 #define IVPU_MMU_CMD_CFGI_0_SSID GENMASK(31, 12) 59 #define IVPU_MMU_CMD_CFGI_0_SID GENMASK(63, 32) 60 #define IVPU_MMU_CMD_CFGI_1_RANGE GENMASK(4, 0) 61 62 #define IVPU_MMU_CMD_TLBI_0_ASID GENMASK(63, 48) 63 #define IVPU_MMU_CMD_TLBI_0_VMID GENMASK(47, 32) 64 65 #define CMD_PREFETCH_CFG 0x1 66 #define CMD_CFGI_STE 0x3 67 #define CMD_CFGI_ALL 0x4 68 #define CMD_CFGI_CD 0x5 69 #define CMD_CFGI_CD_ALL 0x6 70 #define CMD_TLBI_NH_ASID 0x11 71 #define CMD_TLBI_EL2_ALL 0x20 72 #define CMD_TLBI_NSNH_ALL 0x30 73 #define CMD_SYNC 0x46 74 75 #define IVPU_MMU_EVT_F_UUT 0x01 76 #define IVPU_MMU_EVT_C_BAD_STREAMID 0x02 77 #define IVPU_MMU_EVT_F_STE_FETCH 0x03 78 #define IVPU_MMU_EVT_C_BAD_STE 0x04 79 #define IVPU_MMU_EVT_F_BAD_ATS_TREQ 0x05 80 #define IVPU_MMU_EVT_F_STREAM_DISABLED 0x06 81 #define IVPU_MMU_EVT_F_TRANSL_FORBIDDEN 0x07 82 #define IVPU_MMU_EVT_C_BAD_SUBSTREAMID 0x08 83 #define IVPU_MMU_EVT_F_CD_FETCH 0x09 84 #define IVPU_MMU_EVT_C_BAD_CD 0x0a 85 #define IVPU_MMU_EVT_F_WALK_EABT 0x0b 86 #define IVPU_MMU_EVT_F_TRANSLATION 0x10 87 #define IVPU_MMU_EVT_F_ADDR_SIZE 0x11 88 #define IVPU_MMU_EVT_F_ACCESS 0x12 89 #define IVPU_MMU_EVT_F_PERMISSION 0x13 90 #define IVPU_MMU_EVT_F_TLB_CONFLICT 0x20 91 #define IVPU_MMU_EVT_F_CFG_CONFLICT 0x21 92 #define IVPU_MMU_EVT_E_PAGE_REQUEST 0x24 93 #define IVPU_MMU_EVT_F_VMS_FETCH 0x25 94 95 #define IVPU_MMU_EVT_OP_MASK GENMASK_ULL(7, 0) 96 #define IVPU_MMU_EVT_SSID_MASK GENMASK_ULL(31, 12) 97 98 #define IVPU_MMU_Q_BASE_RWA BIT(62) 99 #define IVPU_MMU_Q_BASE_ADDR_MASK GENMASK_ULL(51, 5) 100 #define IVPU_MMU_STRTAB_BASE_RA BIT(62) 101 #define IVPU_MMU_STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6) 102 103 #define IVPU_MMU_IRQ_EVTQ_EN BIT(2) 104 #define IVPU_MMU_IRQ_GERROR_EN BIT(0) 105 106 #define IVPU_MMU_CR0_ATSCHK BIT(4) 107 #define IVPU_MMU_CR0_CMDQEN BIT(3) 108 #define IVPU_MMU_CR0_EVTQEN BIT(2) 109 #define IVPU_MMU_CR0_PRIQEN BIT(1) 110 #define IVPU_MMU_CR0_SMMUEN BIT(0) 111 112 #define IVPU_MMU_CR1_TABLE_SH GENMASK(11, 10) 113 #define IVPU_MMU_CR1_TABLE_OC GENMASK(9, 8) 114 #define IVPU_MMU_CR1_TABLE_IC GENMASK(7, 6) 115 #define IVPU_MMU_CR1_QUEUE_SH GENMASK(5, 4) 116 #define IVPU_MMU_CR1_QUEUE_OC GENMASK(3, 2) 117 #define IVPU_MMU_CR1_QUEUE_IC GENMASK(1, 0) 118 #define IVPU_MMU_CACHE_NC 0 119 #define IVPU_MMU_CACHE_WB 1 120 #define IVPU_MMU_CACHE_WT 2 121 #define IVPU_MMU_SH_NSH 0 122 #define IVPU_MMU_SH_OSH 2 123 #define IVPU_MMU_SH_ISH 3 124 125 #define IVPU_MMU_CMDQ_OP GENMASK_ULL(7, 0) 126 127 #define IVPU_MMU_CD_0_TCR_T0SZ GENMASK_ULL(5, 0) 128 #define IVPU_MMU_CD_0_TCR_TG0 GENMASK_ULL(7, 6) 129 #define IVPU_MMU_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8) 130 #define IVPU_MMU_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10) 131 #define IVPU_MMU_CD_0_TCR_SH0 GENMASK_ULL(13, 12) 132 #define IVPU_MMU_CD_0_TCR_EPD0 BIT_ULL(14) 133 #define IVPU_MMU_CD_0_TCR_EPD1 BIT_ULL(30) 134 #define IVPU_MMU_CD_0_ENDI BIT(15) 135 #define IVPU_MMU_CD_0_V BIT(31) 136 #define IVPU_MMU_CD_0_TCR_IPS GENMASK_ULL(34, 32) 137 #define IVPU_MMU_CD_0_TCR_TBI0 BIT_ULL(38) 138 #define IVPU_MMU_CD_0_AA64 BIT(41) 139 #define IVPU_MMU_CD_0_S BIT(44) 140 #define IVPU_MMU_CD_0_R BIT(45) 141 #define IVPU_MMU_CD_0_A BIT(46) 142 #define IVPU_MMU_CD_0_ASET BIT(47) 143 #define IVPU_MMU_CD_0_ASID GENMASK_ULL(63, 48) 144 145 #define IVPU_MMU_CD_1_TTB0_MASK GENMASK_ULL(51, 4) 146 147 #define IVPU_MMU_STE_0_S1CDMAX GENMASK_ULL(63, 59) 148 #define IVPU_MMU_STE_0_S1FMT GENMASK_ULL(5, 4) 149 #define IVPU_MMU_STE_0_S1FMT_LINEAR 0 150 #define IVPU_MMU_STE_DWORDS 8 151 #define IVPU_MMU_STE_0_CFG_S1_TRANS 5 152 #define IVPU_MMU_STE_0_CFG GENMASK_ULL(3, 1) 153 #define IVPU_MMU_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6) 154 #define IVPU_MMU_STE_0_V BIT(0) 155 156 #define IVPU_MMU_STE_1_STRW_NSEL1 0ul 157 #define IVPU_MMU_STE_1_CONT GENMASK_ULL(16, 13) 158 #define IVPU_MMU_STE_1_STRW GENMASK_ULL(31, 30) 159 #define IVPU_MMU_STE_1_PRIVCFG GENMASK_ULL(49, 48) 160 #define IVPU_MMU_STE_1_PRIVCFG_UNPRIV 2ul 161 #define IVPU_MMU_STE_1_INSTCFG GENMASK_ULL(51, 50) 162 #define IVPU_MMU_STE_1_INSTCFG_DATA 2ul 163 #define IVPU_MMU_STE_1_MEV BIT(19) 164 #define IVPU_MMU_STE_1_S1STALLD BIT(27) 165 #define IVPU_MMU_STE_1_S1C_CACHE_NC 0ul 166 #define IVPU_MMU_STE_1_S1C_CACHE_WBRA 1ul 167 #define IVPU_MMU_STE_1_S1C_CACHE_WT 2ul 168 #define IVPU_MMU_STE_1_S1C_CACHE_WB 3ul 169 #define IVPU_MMU_STE_1_S1CIR GENMASK_ULL(3, 2) 170 #define IVPU_MMU_STE_1_S1COR GENMASK_ULL(5, 4) 171 #define IVPU_MMU_STE_1_S1CSH GENMASK_ULL(7, 6) 172 #define IVPU_MMU_STE_1_S1DSS GENMASK_ULL(1, 0) 173 #define IVPU_MMU_STE_1_S1DSS_TERMINATE 0x0 174 175 #define IVPU_MMU_REG_TIMEOUT_US (10 * USEC_PER_MSEC) 176 #define IVPU_MMU_QUEUE_TIMEOUT_US (100 * USEC_PER_MSEC) 177 178 #define IVPU_MMU_GERROR_ERR_MASK ((REG_FLD(MTL_VPU_HOST_MMU_GERROR, CMDQ)) | \ 179 (REG_FLD(MTL_VPU_HOST_MMU_GERROR, EVTQ_ABT)) | \ 180 (REG_FLD(MTL_VPU_HOST_MMU_GERROR, PRIQ_ABT)) | \ 181 (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_CMDQ_ABT)) | \ 182 (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_EVTQ_ABT)) | \ 183 (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_PRIQ_ABT)) | \ 184 (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_ABT))) 185 186 static char *ivpu_mmu_event_to_str(u32 cmd) 187 { 188 switch (cmd) { 189 case IVPU_MMU_EVT_F_UUT: 190 return "Unsupported Upstream Transaction"; 191 case IVPU_MMU_EVT_C_BAD_STREAMID: 192 return "Transaction StreamID out of range"; 193 case IVPU_MMU_EVT_F_STE_FETCH: 194 return "Fetch of STE caused external abort"; 195 case IVPU_MMU_EVT_C_BAD_STE: 196 return "Used STE invalid"; 197 case IVPU_MMU_EVT_F_BAD_ATS_TREQ: 198 return "Address Request disallowed for a StreamID"; 199 case IVPU_MMU_EVT_F_STREAM_DISABLED: 200 return "Transaction marks non-substream disabled"; 201 case IVPU_MMU_EVT_F_TRANSL_FORBIDDEN: 202 return "MMU bypass is disallowed for this StreamID"; 203 case IVPU_MMU_EVT_C_BAD_SUBSTREAMID: 204 return "Invalid StreamID"; 205 case IVPU_MMU_EVT_F_CD_FETCH: 206 return "Fetch of CD caused external abort"; 207 case IVPU_MMU_EVT_C_BAD_CD: 208 return "Fetched CD invalid"; 209 case IVPU_MMU_EVT_F_WALK_EABT: 210 return " An external abort occurred fetching a TLB"; 211 case IVPU_MMU_EVT_F_TRANSLATION: 212 return "Translation fault"; 213 case IVPU_MMU_EVT_F_ADDR_SIZE: 214 return " Output address caused address size fault"; 215 case IVPU_MMU_EVT_F_ACCESS: 216 return "Access flag fault"; 217 case IVPU_MMU_EVT_F_PERMISSION: 218 return "Permission fault occurred on page access"; 219 case IVPU_MMU_EVT_F_TLB_CONFLICT: 220 return "A TLB conflict"; 221 case IVPU_MMU_EVT_F_CFG_CONFLICT: 222 return "A configuration cache conflict"; 223 case IVPU_MMU_EVT_E_PAGE_REQUEST: 224 return "Page request hint from a client device"; 225 case IVPU_MMU_EVT_F_VMS_FETCH: 226 return "Fetch of VMS caused external abort"; 227 default: 228 return "Unknown CMDQ command"; 229 } 230 } 231 232 static void ivpu_mmu_config_check(struct ivpu_device *vdev) 233 { 234 u32 val_ref; 235 u32 val; 236 237 if (ivpu_is_simics(vdev)) 238 val_ref = IVPU_MMU_IDR0_REF_SIMICS; 239 else 240 val_ref = IVPU_MMU_IDR0_REF; 241 242 val = REGV_RD32(MTL_VPU_HOST_MMU_IDR0); 243 if (val != val_ref) 244 ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref); 245 246 val = REGV_RD32(MTL_VPU_HOST_MMU_IDR1); 247 if (val != IVPU_MMU_IDR1_REF) 248 ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF); 249 250 val = REGV_RD32(MTL_VPU_HOST_MMU_IDR3); 251 if (val != IVPU_MMU_IDR3_REF) 252 ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF); 253 254 if (ivpu_is_simics(vdev)) 255 val_ref = IVPU_MMU_IDR5_REF_SIMICS; 256 else if (ivpu_is_fpga(vdev)) 257 val_ref = IVPU_MMU_IDR5_REF_FPGA; 258 else 259 val_ref = IVPU_MMU_IDR5_REF; 260 261 val = REGV_RD32(MTL_VPU_HOST_MMU_IDR5); 262 if (val != val_ref) 263 ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref); 264 } 265 266 static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev) 267 { 268 struct ivpu_mmu_info *mmu = vdev->mmu; 269 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; 270 size_t size = IVPU_MMU_CDTAB_ENT_COUNT * IVPU_MMU_CDTAB_ENT_SIZE; 271 272 cdtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &cdtab->dma, GFP_KERNEL); 273 if (!cdtab->base) 274 return -ENOMEM; 275 276 ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size); 277 278 return 0; 279 } 280 281 static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev) 282 { 283 struct ivpu_mmu_info *mmu = vdev->mmu; 284 struct ivpu_mmu_strtab *strtab = &mmu->strtab; 285 size_t size = IVPU_MMU_STRTAB_ENT_COUNT * IVPU_MMU_STRTAB_ENT_SIZE; 286 287 strtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &strtab->dma, GFP_KERNEL); 288 if (!strtab->base) 289 return -ENOMEM; 290 291 strtab->base_cfg = IVPU_MMU_STRTAB_CFG; 292 strtab->dma_q = IVPU_MMU_STRTAB_BASE_RA; 293 strtab->dma_q |= strtab->dma & IVPU_MMU_STRTAB_BASE_ADDR_MASK; 294 295 ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n", 296 &strtab->dma, &strtab->dma_q, size); 297 298 return 0; 299 } 300 301 static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev) 302 { 303 struct ivpu_mmu_info *mmu = vdev->mmu; 304 struct ivpu_mmu_queue *q = &mmu->cmdq; 305 306 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, &q->dma, GFP_KERNEL); 307 if (!q->base) 308 return -ENOMEM; 309 310 q->dma_q = IVPU_MMU_Q_BASE_RWA; 311 q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK; 312 q->dma_q |= IVPU_MMU_Q_COUNT_LOG2; 313 314 ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n", 315 &q->dma, &q->dma_q, IVPU_MMU_CMDQ_SIZE); 316 317 return 0; 318 } 319 320 static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev) 321 { 322 struct ivpu_mmu_info *mmu = vdev->mmu; 323 struct ivpu_mmu_queue *q = &mmu->evtq; 324 325 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, &q->dma, GFP_KERNEL); 326 if (!q->base) 327 return -ENOMEM; 328 329 q->dma_q = IVPU_MMU_Q_BASE_RWA; 330 q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK; 331 q->dma_q |= IVPU_MMU_Q_COUNT_LOG2; 332 333 ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n", 334 &q->dma, &q->dma_q, IVPU_MMU_EVTQ_SIZE); 335 336 return 0; 337 } 338 339 static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev) 340 { 341 int ret; 342 343 ret = ivpu_mmu_cdtab_alloc(vdev); 344 if (ret) { 345 ivpu_err(vdev, "Failed to allocate cdtab: %d\n", ret); 346 return ret; 347 } 348 349 ret = ivpu_mmu_strtab_alloc(vdev); 350 if (ret) { 351 ivpu_err(vdev, "Failed to allocate strtab: %d\n", ret); 352 return ret; 353 } 354 355 ret = ivpu_mmu_cmdq_alloc(vdev); 356 if (ret) { 357 ivpu_err(vdev, "Failed to allocate cmdq: %d\n", ret); 358 return ret; 359 } 360 361 ret = ivpu_mmu_evtq_alloc(vdev); 362 if (ret) 363 ivpu_err(vdev, "Failed to allocate evtq: %d\n", ret); 364 365 return ret; 366 } 367 368 static int ivpu_mmu_reg_write(struct ivpu_device *vdev, u32 reg, u32 val) 369 { 370 u32 reg_ack = reg + 4; /* ACK register is 4B after base register */ 371 u32 val_ack; 372 int ret; 373 374 REGV_WR32(reg, val); 375 376 ret = REGV_POLL(reg_ack, val_ack, (val == val_ack), IVPU_MMU_REG_TIMEOUT_US); 377 if (ret) 378 ivpu_err(vdev, "Failed to write register 0x%x\n", reg); 379 380 return ret; 381 } 382 383 static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev) 384 { 385 u32 irq_ctrl = IVPU_MMU_IRQ_EVTQ_EN | IVPU_MMU_IRQ_GERROR_EN; 386 int ret; 387 388 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_IRQ_CTRL, 0); 389 if (ret) 390 return ret; 391 392 return ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_IRQ_CTRL, irq_ctrl); 393 } 394 395 static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev) 396 { 397 struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq; 398 399 return REGV_POLL(MTL_VPU_HOST_MMU_CMDQ_CONS, cmdq->cons, (cmdq->prod == cmdq->cons), 400 IVPU_MMU_QUEUE_TIMEOUT_US); 401 } 402 403 static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1) 404 { 405 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq; 406 u64 *queue_buffer = q->base; 407 int idx = IVPU_MMU_Q_IDX(q->prod) * (IVPU_MMU_CMDQ_CMD_SIZE / sizeof(*queue_buffer)); 408 409 if (!CIRC_SPACE(IVPU_MMU_Q_IDX(q->prod), IVPU_MMU_Q_IDX(q->cons), IVPU_MMU_Q_COUNT)) { 410 ivpu_err(vdev, "Failed to write MMU CMD %s\n", name); 411 return -EBUSY; 412 } 413 414 queue_buffer[idx] = data0; 415 queue_buffer[idx + 1] = data1; 416 q->prod = (q->prod + 1) & IVPU_MMU_Q_WRAP_MASK; 417 418 ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1); 419 420 return 0; 421 } 422 423 static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev) 424 { 425 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq; 426 u64 val; 427 int ret; 428 429 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC) | 430 FIELD_PREP(IVPU_MMU_CMD_SYNC_0_CS, 0x2) | 431 FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSH, 0x3) | 432 FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSI_ATTR, 0xf); 433 434 ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0); 435 if (ret) 436 return ret; 437 438 clflush_cache_range(q->base, IVPU_MMU_CMDQ_SIZE); 439 REGV_WR32(MTL_VPU_HOST_MMU_CMDQ_PROD, q->prod); 440 441 ret = ivpu_mmu_cmdq_wait_for_cons(vdev); 442 if (ret) 443 ivpu_err(vdev, "Timed out waiting for consumer: %d\n", ret); 444 445 return ret; 446 } 447 448 static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev) 449 { 450 u64 data0 = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_CFGI_ALL); 451 u64 data1 = FIELD_PREP(IVPU_MMU_CMD_CFGI_1_RANGE, 0x1f); 452 453 return ivpu_mmu_cmdq_cmd_write(vdev, "CFGI_ALL", data0, data1); 454 } 455 456 static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid) 457 { 458 u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NH_ASID) | 459 FIELD_PREP(IVPU_MMU_CMD_TLBI_0_ASID, ssid); 460 461 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NH_ASID", val, 0); 462 } 463 464 static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev) 465 { 466 u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NSNH_ALL); 467 468 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NSNH_ALL", val, 0); 469 } 470 471 static int ivpu_mmu_reset(struct ivpu_device *vdev) 472 { 473 struct ivpu_mmu_info *mmu = vdev->mmu; 474 u32 val; 475 int ret; 476 477 memset(mmu->cmdq.base, 0, IVPU_MMU_CMDQ_SIZE); 478 clflush_cache_range(mmu->cmdq.base, IVPU_MMU_CMDQ_SIZE); 479 mmu->cmdq.prod = 0; 480 mmu->cmdq.cons = 0; 481 482 memset(mmu->evtq.base, 0, IVPU_MMU_EVTQ_SIZE); 483 clflush_cache_range(mmu->evtq.base, IVPU_MMU_EVTQ_SIZE); 484 mmu->evtq.prod = 0; 485 mmu->evtq.cons = 0; 486 487 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, 0); 488 if (ret) 489 return ret; 490 491 val = FIELD_PREP(IVPU_MMU_CR1_TABLE_SH, IVPU_MMU_SH_ISH) | 492 FIELD_PREP(IVPU_MMU_CR1_TABLE_OC, IVPU_MMU_CACHE_WB) | 493 FIELD_PREP(IVPU_MMU_CR1_TABLE_IC, IVPU_MMU_CACHE_WB) | 494 FIELD_PREP(IVPU_MMU_CR1_QUEUE_SH, IVPU_MMU_SH_ISH) | 495 FIELD_PREP(IVPU_MMU_CR1_QUEUE_OC, IVPU_MMU_CACHE_WB) | 496 FIELD_PREP(IVPU_MMU_CR1_QUEUE_IC, IVPU_MMU_CACHE_WB); 497 REGV_WR32(MTL_VPU_HOST_MMU_CR1, val); 498 499 REGV_WR64(MTL_VPU_HOST_MMU_STRTAB_BASE, mmu->strtab.dma_q); 500 REGV_WR32(MTL_VPU_HOST_MMU_STRTAB_BASE_CFG, mmu->strtab.base_cfg); 501 502 REGV_WR64(MTL_VPU_HOST_MMU_CMDQ_BASE, mmu->cmdq.dma_q); 503 REGV_WR32(MTL_VPU_HOST_MMU_CMDQ_PROD, 0); 504 REGV_WR32(MTL_VPU_HOST_MMU_CMDQ_CONS, 0); 505 506 val = IVPU_MMU_CR0_CMDQEN; 507 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val); 508 if (ret) 509 return ret; 510 511 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); 512 if (ret) 513 return ret; 514 515 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev); 516 if (ret) 517 return ret; 518 519 ret = ivpu_mmu_cmdq_sync(vdev); 520 if (ret) 521 return ret; 522 523 REGV_WR64(MTL_VPU_HOST_MMU_EVTQ_BASE, mmu->evtq.dma_q); 524 REGV_WR32(MTL_VPU_HOST_MMU_EVTQ_PROD_SEC, 0); 525 REGV_WR32(MTL_VPU_HOST_MMU_EVTQ_CONS_SEC, 0); 526 527 val |= IVPU_MMU_CR0_EVTQEN; 528 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val); 529 if (ret) 530 return ret; 531 532 val |= IVPU_MMU_CR0_ATSCHK; 533 ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val); 534 if (ret) 535 return ret; 536 537 ret = ivpu_mmu_irqs_setup(vdev); 538 if (ret) 539 return ret; 540 541 val |= IVPU_MMU_CR0_SMMUEN; 542 return ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val); 543 } 544 545 static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid) 546 { 547 struct ivpu_mmu_info *mmu = vdev->mmu; 548 struct ivpu_mmu_strtab *strtab = &mmu->strtab; 549 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; 550 u64 *entry = strtab->base + (sid * IVPU_MMU_STRTAB_ENT_SIZE); 551 u64 str[2]; 552 553 str[0] = FIELD_PREP(IVPU_MMU_STE_0_CFG, IVPU_MMU_STE_0_CFG_S1_TRANS) | 554 FIELD_PREP(IVPU_MMU_STE_0_S1CDMAX, IVPU_MMU_CDTAB_ENT_COUNT_LOG2) | 555 FIELD_PREP(IVPU_MMU_STE_0_S1FMT, IVPU_MMU_STE_0_S1FMT_LINEAR) | 556 IVPU_MMU_STE_0_V | 557 (cdtab->dma & IVPU_MMU_STE_0_S1CTXPTR_MASK); 558 559 str[1] = FIELD_PREP(IVPU_MMU_STE_1_S1DSS, IVPU_MMU_STE_1_S1DSS_TERMINATE) | 560 FIELD_PREP(IVPU_MMU_STE_1_S1CIR, IVPU_MMU_STE_1_S1C_CACHE_NC) | 561 FIELD_PREP(IVPU_MMU_STE_1_S1COR, IVPU_MMU_STE_1_S1C_CACHE_NC) | 562 FIELD_PREP(IVPU_MMU_STE_1_S1CSH, IVPU_MMU_SH_NSH) | 563 FIELD_PREP(IVPU_MMU_STE_1_PRIVCFG, IVPU_MMU_STE_1_PRIVCFG_UNPRIV) | 564 FIELD_PREP(IVPU_MMU_STE_1_INSTCFG, IVPU_MMU_STE_1_INSTCFG_DATA) | 565 FIELD_PREP(IVPU_MMU_STE_1_STRW, IVPU_MMU_STE_1_STRW_NSEL1) | 566 FIELD_PREP(IVPU_MMU_STE_1_CONT, IVPU_MMU_STRTAB_CFG_LOG2SIZE) | 567 IVPU_MMU_STE_1_MEV | 568 IVPU_MMU_STE_1_S1STALLD; 569 570 WRITE_ONCE(entry[1], str[1]); 571 WRITE_ONCE(entry[0], str[0]); 572 573 clflush_cache_range(entry, IVPU_MMU_STRTAB_ENT_SIZE); 574 575 ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]); 576 } 577 578 static int ivpu_mmu_strtab_init(struct ivpu_device *vdev) 579 { 580 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0); 581 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3); 582 583 return 0; 584 } 585 586 int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid) 587 { 588 struct ivpu_mmu_info *mmu = vdev->mmu; 589 int ret; 590 591 ret = mutex_lock_interruptible(&mmu->lock); 592 if (ret) 593 return ret; 594 595 if (!mmu->on) { 596 ret = 0; 597 goto unlock; 598 } 599 600 ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid); 601 if (ret) 602 goto unlock; 603 604 ret = ivpu_mmu_cmdq_sync(vdev); 605 unlock: 606 mutex_unlock(&mmu->lock); 607 return ret; 608 } 609 610 static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma) 611 { 612 struct ivpu_mmu_info *mmu = vdev->mmu; 613 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; 614 u64 *entry; 615 u64 cd[4]; 616 int ret; 617 618 if (ssid > IVPU_MMU_CDTAB_ENT_COUNT) 619 return -EINVAL; 620 621 entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE); 622 623 if (cd_dma != 0) { 624 cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, 26) | 625 FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) | 626 FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) | 627 FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) | 628 FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) | 629 FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, 3) | 630 FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) | 631 IVPU_MMU_CD_0_TCR_EPD1 | 632 IVPU_MMU_CD_0_AA64 | 633 IVPU_MMU_CD_0_R | 634 IVPU_MMU_CD_0_ASET | 635 IVPU_MMU_CD_0_V; 636 cd[1] = cd_dma & IVPU_MMU_CD_1_TTB0_MASK; 637 cd[2] = 0; 638 cd[3] = 0x0000000000007444; 639 640 /* For global context generate memory fault on VPU */ 641 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID) 642 cd[0] |= IVPU_MMU_CD_0_A; 643 } else { 644 memset(cd, 0, sizeof(cd)); 645 } 646 647 WRITE_ONCE(entry[1], cd[1]); 648 WRITE_ONCE(entry[2], cd[2]); 649 WRITE_ONCE(entry[3], cd[3]); 650 WRITE_ONCE(entry[0], cd[0]); 651 652 clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE); 653 654 ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n", 655 cd_dma ? "write" : "clear", ssid, &cd_dma, cd[0], cd[1], cd[2], cd[3]); 656 657 ret = mutex_lock_interruptible(&mmu->lock); 658 if (ret) 659 return ret; 660 661 if (!mmu->on) { 662 ret = 0; 663 goto unlock; 664 } 665 666 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); 667 if (ret) 668 goto unlock; 669 670 ret = ivpu_mmu_cmdq_sync(vdev); 671 unlock: 672 mutex_unlock(&mmu->lock); 673 return ret; 674 } 675 676 static int ivpu_mmu_cd_add_gbl(struct ivpu_device *vdev) 677 { 678 int ret; 679 680 ret = ivpu_mmu_cd_add(vdev, 0, vdev->gctx.pgtable.pgd_dma); 681 if (ret) 682 ivpu_err(vdev, "Failed to add global CD entry: %d\n", ret); 683 684 return ret; 685 } 686 687 static int ivpu_mmu_cd_add_user(struct ivpu_device *vdev, u32 ssid, dma_addr_t cd_dma) 688 { 689 int ret; 690 691 if (ssid == 0) { 692 ivpu_err(vdev, "Invalid SSID: %u\n", ssid); 693 return -EINVAL; 694 } 695 696 ret = ivpu_mmu_cd_add(vdev, ssid, cd_dma); 697 if (ret) 698 ivpu_err(vdev, "Failed to add CD entry SSID=%u: %d\n", ssid, ret); 699 700 return ret; 701 } 702 703 int ivpu_mmu_init(struct ivpu_device *vdev) 704 { 705 struct ivpu_mmu_info *mmu = vdev->mmu; 706 int ret; 707 708 ivpu_dbg(vdev, MMU, "Init..\n"); 709 710 drmm_mutex_init(&vdev->drm, &mmu->lock); 711 ivpu_mmu_config_check(vdev); 712 713 ret = ivpu_mmu_structs_alloc(vdev); 714 if (ret) 715 return ret; 716 717 ret = ivpu_mmu_strtab_init(vdev); 718 if (ret) { 719 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret); 720 return ret; 721 } 722 723 ret = ivpu_mmu_cd_add_gbl(vdev); 724 if (ret) { 725 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret); 726 return ret; 727 } 728 729 ret = ivpu_mmu_enable(vdev); 730 if (ret) { 731 ivpu_err(vdev, "Failed to resume MMU: %d\n", ret); 732 return ret; 733 } 734 735 ivpu_dbg(vdev, MMU, "Init done\n"); 736 737 return 0; 738 } 739 740 int ivpu_mmu_enable(struct ivpu_device *vdev) 741 { 742 struct ivpu_mmu_info *mmu = vdev->mmu; 743 int ret; 744 745 mutex_lock(&mmu->lock); 746 747 mmu->on = true; 748 749 ret = ivpu_mmu_reset(vdev); 750 if (ret) { 751 ivpu_err(vdev, "Failed to reset MMU: %d\n", ret); 752 goto err; 753 } 754 755 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); 756 if (ret) 757 goto err; 758 759 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev); 760 if (ret) 761 goto err; 762 763 ret = ivpu_mmu_cmdq_sync(vdev); 764 if (ret) 765 goto err; 766 767 mutex_unlock(&mmu->lock); 768 769 return 0; 770 err: 771 mmu->on = false; 772 mutex_unlock(&mmu->lock); 773 return ret; 774 } 775 776 void ivpu_mmu_disable(struct ivpu_device *vdev) 777 { 778 struct ivpu_mmu_info *mmu = vdev->mmu; 779 780 mutex_lock(&mmu->lock); 781 mmu->on = false; 782 mutex_unlock(&mmu->lock); 783 } 784 785 static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event) 786 { 787 u32 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]); 788 u32 op = FIELD_GET(IVPU_MMU_EVT_OP_MASK, event[0]); 789 u64 fetch_addr = ((u64)event[7]) << 32 | event[6]; 790 u64 in_addr = ((u64)event[5]) << 32 | event[4]; 791 u32 sid = event[1]; 792 793 ivpu_err(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch addr: 0x%llx\n", 794 op, ivpu_mmu_event_to_str(op), ssid, sid, event[2], event[3], in_addr, fetch_addr); 795 } 796 797 static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev) 798 { 799 struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq; 800 u32 idx = IVPU_MMU_Q_IDX(evtq->cons); 801 u32 *evt = evtq->base + (idx * IVPU_MMU_EVTQ_CMD_SIZE); 802 803 evtq->prod = REGV_RD32(MTL_VPU_HOST_MMU_EVTQ_PROD_SEC); 804 if (!CIRC_CNT(IVPU_MMU_Q_IDX(evtq->prod), IVPU_MMU_Q_IDX(evtq->cons), IVPU_MMU_Q_COUNT)) 805 return NULL; 806 807 clflush_cache_range(evt, IVPU_MMU_EVTQ_CMD_SIZE); 808 809 evtq->cons = (evtq->cons + 1) & IVPU_MMU_Q_WRAP_MASK; 810 REGV_WR32(MTL_VPU_HOST_MMU_EVTQ_CONS_SEC, evtq->cons); 811 812 return evt; 813 } 814 815 void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev) 816 { 817 u32 *event; 818 u32 ssid; 819 820 ivpu_dbg(vdev, IRQ, "MMU event queue\n"); 821 822 while ((event = ivpu_mmu_get_event(vdev)) != NULL) { 823 ivpu_mmu_dump_event(vdev, event); 824 825 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]); 826 if (ssid != IVPU_GLOBAL_CONTEXT_MMU_SSID) 827 ivpu_mmu_user_context_mark_invalid(vdev, ssid); 828 } 829 } 830 831 void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev) 832 { 833 u32 gerror_val, gerrorn_val, active; 834 835 ivpu_dbg(vdev, IRQ, "MMU error\n"); 836 837 gerror_val = REGV_RD32(MTL_VPU_HOST_MMU_GERROR); 838 gerrorn_val = REGV_RD32(MTL_VPU_HOST_MMU_GERRORN); 839 840 active = gerror_val ^ gerrorn_val; 841 if (!(active & IVPU_MMU_GERROR_ERR_MASK)) 842 return; 843 844 if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_ABT, active)) 845 ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n"); 846 847 if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_PRIQ_ABT, active)) 848 ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n"); 849 850 if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_EVTQ_ABT, active)) 851 ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n"); 852 853 if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_CMDQ_ABT, active)) 854 ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n"); 855 856 if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, PRIQ_ABT, active)) 857 ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n"); 858 859 if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, EVTQ_ABT, active)) 860 ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n"); 861 862 if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, CMDQ, active)) 863 ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n"); 864 865 REGV_WR32(MTL_VPU_HOST_MMU_GERRORN, gerror_val); 866 } 867 868 int ivpu_mmu_set_pgtable(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable) 869 { 870 return ivpu_mmu_cd_add_user(vdev, ssid, pgtable->pgd_dma); 871 } 872 873 void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid) 874 { 875 ivpu_mmu_cd_add_user(vdev, ssid, 0); /* 0 will clear CD entry */ 876 } 877