1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6 #include <linux/circ_buf.h> 7 #include <linux/highmem.h> 8 9 #include "ivpu_drv.h" 10 #include "ivpu_hw_reg_io.h" 11 #include "ivpu_mmu.h" 12 #include "ivpu_mmu_context.h" 13 #include "ivpu_pm.h" 14 15 #define IVPU_MMU_REG_IDR0 0x00200000u 16 #define IVPU_MMU_REG_IDR1 0x00200004u 17 #define IVPU_MMU_REG_IDR3 0x0020000cu 18 #define IVPU_MMU_REG_IDR5 0x00200014u 19 #define IVPU_MMU_REG_CR0 0x00200020u 20 #define IVPU_MMU_REG_CR0ACK 0x00200024u 21 #define IVPU_MMU_REG_CR1 0x00200028u 22 #define IVPU_MMU_REG_CR2 0x0020002cu 23 #define IVPU_MMU_REG_IRQ_CTRL 0x00200050u 24 #define IVPU_MMU_REG_IRQ_CTRLACK 0x00200054u 25 26 #define IVPU_MMU_REG_GERROR 0x00200060u 27 #define IVPU_MMU_REG_GERROR_CMDQ_MASK BIT_MASK(0) 28 #define IVPU_MMU_REG_GERROR_EVTQ_ABT_MASK BIT_MASK(2) 29 #define IVPU_MMU_REG_GERROR_PRIQ_ABT_MASK BIT_MASK(3) 30 #define IVPU_MMU_REG_GERROR_MSI_CMDQ_ABT_MASK BIT_MASK(4) 31 #define IVPU_MMU_REG_GERROR_MSI_EVTQ_ABT_MASK BIT_MASK(5) 32 #define IVPU_MMU_REG_GERROR_MSI_PRIQ_ABT_MASK BIT_MASK(6) 33 #define IVPU_MMU_REG_GERROR_MSI_ABT_MASK BIT_MASK(7) 34 35 #define IVPU_MMU_REG_GERRORN 0x00200064u 36 37 #define IVPU_MMU_REG_STRTAB_BASE 0x00200080u 38 #define IVPU_MMU_REG_STRTAB_BASE_CFG 0x00200088u 39 #define IVPU_MMU_REG_CMDQ_BASE 0x00200090u 40 #define IVPU_MMU_REG_CMDQ_PROD 0x00200098u 41 #define IVPU_MMU_REG_CMDQ_CONS 0x0020009cu 42 #define IVPU_MMU_REG_EVTQ_BASE 0x002000a0u 43 #define IVPU_MMU_REG_EVTQ_PROD 0x002000a8u 44 #define IVPU_MMU_REG_EVTQ_CONS 0x002000acu 45 #define IVPU_MMU_REG_EVTQ_PROD_SEC (0x002000a8u + SZ_64K) 46 #define IVPU_MMU_REG_EVTQ_CONS_SEC (0x002000acu + SZ_64K) 47 #define IVPU_MMU_REG_CMDQ_CONS_ERR_MASK GENMASK(30, 24) 48 49 #define IVPU_MMU_IDR0_REF 0x080f3e0f 50 #define IVPU_MMU_IDR0_REF_SIMICS 0x080f3e1f 51 #define IVPU_MMU_IDR1_REF 0x0e739d18 52 #define IVPU_MMU_IDR3_REF 0x0000003c 53 #define IVPU_MMU_IDR5_REF 0x00040070 54 #define IVPU_MMU_IDR5_REF_SIMICS 0x00000075 55 #define IVPU_MMU_IDR5_REF_FPGA 0x00800075 56 57 #define IVPU_MMU_CDTAB_ENT_SIZE 64 58 #define IVPU_MMU_CDTAB_ENT_COUNT_LOG2 8 /* 256 entries */ 59 #define IVPU_MMU_CDTAB_ENT_COUNT ((u32)1 << IVPU_MMU_CDTAB_ENT_COUNT_LOG2) 60 61 #define IVPU_MMU_STREAM_ID0 0 62 #define IVPU_MMU_STREAM_ID3 3 63 64 #define IVPU_MMU_STRTAB_ENT_SIZE 64 65 #define IVPU_MMU_STRTAB_ENT_COUNT 4 66 #define IVPU_MMU_STRTAB_CFG_LOG2SIZE 2 67 #define IVPU_MMU_STRTAB_CFG IVPU_MMU_STRTAB_CFG_LOG2SIZE 68 69 #define IVPU_MMU_Q_COUNT_LOG2 4 /* 16 entries */ 70 #define IVPU_MMU_Q_COUNT ((u32)1 << IVPU_MMU_Q_COUNT_LOG2) 71 #define IVPU_MMU_Q_WRAP_BIT (IVPU_MMU_Q_COUNT << 1) 72 #define IVPU_MMU_Q_WRAP_MASK (IVPU_MMU_Q_WRAP_BIT - 1) 73 #define IVPU_MMU_Q_IDX_MASK (IVPU_MMU_Q_COUNT - 1) 74 #define IVPU_MMU_Q_IDX(val) ((val) & IVPU_MMU_Q_IDX_MASK) 75 76 #define IVPU_MMU_CMDQ_CMD_SIZE 16 77 #define IVPU_MMU_CMDQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_CMDQ_CMD_SIZE) 78 79 #define IVPU_MMU_EVTQ_CMD_SIZE 32 80 #define IVPU_MMU_EVTQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_EVTQ_CMD_SIZE) 81 82 #define IVPU_MMU_CMD_OPCODE GENMASK(7, 0) 83 84 #define IVPU_MMU_CMD_SYNC_0_CS GENMASK(13, 12) 85 #define IVPU_MMU_CMD_SYNC_0_MSH GENMASK(23, 22) 86 #define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24) 87 #define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24) 88 #define IVPU_MMU_CMD_SYNC_0_MSI_DATA GENMASK(63, 32) 89 90 #define IVPU_MMU_CMD_CFGI_0_SSEC BIT(10) 91 #define IVPU_MMU_CMD_CFGI_0_SSV BIT(11) 92 #define IVPU_MMU_CMD_CFGI_0_SSID GENMASK(31, 12) 93 #define IVPU_MMU_CMD_CFGI_0_SID GENMASK(63, 32) 94 #define IVPU_MMU_CMD_CFGI_1_RANGE GENMASK(4, 0) 95 96 #define IVPU_MMU_CMD_TLBI_0_ASID GENMASK(63, 48) 97 #define IVPU_MMU_CMD_TLBI_0_VMID GENMASK(47, 32) 98 99 #define CMD_PREFETCH_CFG 0x1 100 #define CMD_CFGI_STE 0x3 101 #define CMD_CFGI_ALL 0x4 102 #define CMD_CFGI_CD 0x5 103 #define CMD_CFGI_CD_ALL 0x6 104 #define CMD_TLBI_NH_ASID 0x11 105 #define CMD_TLBI_EL2_ALL 0x20 106 #define CMD_TLBI_NSNH_ALL 0x30 107 #define CMD_SYNC 0x46 108 109 #define IVPU_MMU_EVT_F_UUT 0x01 110 #define IVPU_MMU_EVT_C_BAD_STREAMID 0x02 111 #define IVPU_MMU_EVT_F_STE_FETCH 0x03 112 #define IVPU_MMU_EVT_C_BAD_STE 0x04 113 #define IVPU_MMU_EVT_F_BAD_ATS_TREQ 0x05 114 #define IVPU_MMU_EVT_F_STREAM_DISABLED 0x06 115 #define IVPU_MMU_EVT_F_TRANSL_FORBIDDEN 0x07 116 #define IVPU_MMU_EVT_C_BAD_SUBSTREAMID 0x08 117 #define IVPU_MMU_EVT_F_CD_FETCH 0x09 118 #define IVPU_MMU_EVT_C_BAD_CD 0x0a 119 #define IVPU_MMU_EVT_F_WALK_EABT 0x0b 120 #define IVPU_MMU_EVT_F_TRANSLATION 0x10 121 #define IVPU_MMU_EVT_F_ADDR_SIZE 0x11 122 #define IVPU_MMU_EVT_F_ACCESS 0x12 123 #define IVPU_MMU_EVT_F_PERMISSION 0x13 124 #define IVPU_MMU_EVT_F_TLB_CONFLICT 0x20 125 #define IVPU_MMU_EVT_F_CFG_CONFLICT 0x21 126 #define IVPU_MMU_EVT_E_PAGE_REQUEST 0x24 127 #define IVPU_MMU_EVT_F_VMS_FETCH 0x25 128 129 #define IVPU_MMU_EVT_OP_MASK GENMASK_ULL(7, 0) 130 #define IVPU_MMU_EVT_SSID_MASK GENMASK_ULL(31, 12) 131 132 #define IVPU_MMU_Q_BASE_RWA BIT(62) 133 #define IVPU_MMU_Q_BASE_ADDR_MASK GENMASK_ULL(51, 5) 134 #define IVPU_MMU_STRTAB_BASE_RA BIT(62) 135 #define IVPU_MMU_STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6) 136 137 #define IVPU_MMU_IRQ_EVTQ_EN BIT(2) 138 #define IVPU_MMU_IRQ_GERROR_EN BIT(0) 139 140 #define IVPU_MMU_CR0_ATSCHK BIT(4) 141 #define IVPU_MMU_CR0_CMDQEN BIT(3) 142 #define IVPU_MMU_CR0_EVTQEN BIT(2) 143 #define IVPU_MMU_CR0_PRIQEN BIT(1) 144 #define IVPU_MMU_CR0_SMMUEN BIT(0) 145 146 #define IVPU_MMU_CR1_TABLE_SH GENMASK(11, 10) 147 #define IVPU_MMU_CR1_TABLE_OC GENMASK(9, 8) 148 #define IVPU_MMU_CR1_TABLE_IC GENMASK(7, 6) 149 #define IVPU_MMU_CR1_QUEUE_SH GENMASK(5, 4) 150 #define IVPU_MMU_CR1_QUEUE_OC GENMASK(3, 2) 151 #define IVPU_MMU_CR1_QUEUE_IC GENMASK(1, 0) 152 #define IVPU_MMU_CACHE_NC 0 153 #define IVPU_MMU_CACHE_WB 1 154 #define IVPU_MMU_CACHE_WT 2 155 #define IVPU_MMU_SH_NSH 0 156 #define IVPU_MMU_SH_OSH 2 157 #define IVPU_MMU_SH_ISH 3 158 159 #define IVPU_MMU_CMDQ_OP GENMASK_ULL(7, 0) 160 161 #define IVPU_MMU_CD_0_TCR_T0SZ GENMASK_ULL(5, 0) 162 #define IVPU_MMU_CD_0_TCR_TG0 GENMASK_ULL(7, 6) 163 #define IVPU_MMU_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8) 164 #define IVPU_MMU_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10) 165 #define IVPU_MMU_CD_0_TCR_SH0 GENMASK_ULL(13, 12) 166 #define IVPU_MMU_CD_0_TCR_EPD0 BIT_ULL(14) 167 #define IVPU_MMU_CD_0_TCR_EPD1 BIT_ULL(30) 168 #define IVPU_MMU_CD_0_ENDI BIT(15) 169 #define IVPU_MMU_CD_0_V BIT(31) 170 #define IVPU_MMU_CD_0_TCR_IPS GENMASK_ULL(34, 32) 171 #define IVPU_MMU_CD_0_TCR_TBI0 BIT_ULL(38) 172 #define IVPU_MMU_CD_0_AA64 BIT(41) 173 #define IVPU_MMU_CD_0_S BIT(44) 174 #define IVPU_MMU_CD_0_R BIT(45) 175 #define IVPU_MMU_CD_0_A BIT(46) 176 #define IVPU_MMU_CD_0_ASET BIT(47) 177 #define IVPU_MMU_CD_0_ASID GENMASK_ULL(63, 48) 178 179 #define IVPU_MMU_T0SZ_48BIT 16 180 #define IVPU_MMU_T0SZ_38BIT 26 181 182 #define IVPU_MMU_IPS_48BIT 5 183 #define IVPU_MMU_IPS_44BIT 4 184 #define IVPU_MMU_IPS_42BIT 3 185 #define IVPU_MMU_IPS_40BIT 2 186 #define IVPU_MMU_IPS_36BIT 1 187 #define IVPU_MMU_IPS_32BIT 0 188 189 #define IVPU_MMU_CD_1_TTB0_MASK GENMASK_ULL(51, 4) 190 191 #define IVPU_MMU_STE_0_S1CDMAX GENMASK_ULL(63, 59) 192 #define IVPU_MMU_STE_0_S1FMT GENMASK_ULL(5, 4) 193 #define IVPU_MMU_STE_0_S1FMT_LINEAR 0 194 #define IVPU_MMU_STE_DWORDS 8 195 #define IVPU_MMU_STE_0_CFG_S1_TRANS 5 196 #define IVPU_MMU_STE_0_CFG GENMASK_ULL(3, 1) 197 #define IVPU_MMU_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6) 198 #define IVPU_MMU_STE_0_V BIT(0) 199 200 #define IVPU_MMU_STE_1_STRW_NSEL1 0ul 201 #define IVPU_MMU_STE_1_CONT GENMASK_ULL(16, 13) 202 #define IVPU_MMU_STE_1_STRW GENMASK_ULL(31, 30) 203 #define IVPU_MMU_STE_1_PRIVCFG GENMASK_ULL(49, 48) 204 #define IVPU_MMU_STE_1_PRIVCFG_UNPRIV 2ul 205 #define IVPU_MMU_STE_1_INSTCFG GENMASK_ULL(51, 50) 206 #define IVPU_MMU_STE_1_INSTCFG_DATA 2ul 207 #define IVPU_MMU_STE_1_MEV BIT(19) 208 #define IVPU_MMU_STE_1_S1STALLD BIT(27) 209 #define IVPU_MMU_STE_1_S1C_CACHE_NC 0ul 210 #define IVPU_MMU_STE_1_S1C_CACHE_WBRA 1ul 211 #define IVPU_MMU_STE_1_S1C_CACHE_WT 2ul 212 #define IVPU_MMU_STE_1_S1C_CACHE_WB 3ul 213 #define IVPU_MMU_STE_1_S1CIR GENMASK_ULL(3, 2) 214 #define IVPU_MMU_STE_1_S1COR GENMASK_ULL(5, 4) 215 #define IVPU_MMU_STE_1_S1CSH GENMASK_ULL(7, 6) 216 #define IVPU_MMU_STE_1_S1DSS GENMASK_ULL(1, 0) 217 #define IVPU_MMU_STE_1_S1DSS_TERMINATE 0x0 218 219 #define IVPU_MMU_REG_TIMEOUT_US (10 * USEC_PER_MSEC) 220 #define IVPU_MMU_QUEUE_TIMEOUT_US (100 * USEC_PER_MSEC) 221 222 #define IVPU_MMU_GERROR_ERR_MASK ((REG_FLD(IVPU_MMU_REG_GERROR, CMDQ)) | \ 223 (REG_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT)) | \ 224 (REG_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT)) | \ 225 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT)) | \ 226 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT)) | \ 227 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT)) | \ 228 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_ABT))) 229 230 static char *ivpu_mmu_event_to_str(u32 cmd) 231 { 232 switch (cmd) { 233 case IVPU_MMU_EVT_F_UUT: 234 return "Unsupported Upstream Transaction"; 235 case IVPU_MMU_EVT_C_BAD_STREAMID: 236 return "Transaction StreamID out of range"; 237 case IVPU_MMU_EVT_F_STE_FETCH: 238 return "Fetch of STE caused external abort"; 239 case IVPU_MMU_EVT_C_BAD_STE: 240 return "Used STE invalid"; 241 case IVPU_MMU_EVT_F_BAD_ATS_TREQ: 242 return "Address Request disallowed for a StreamID"; 243 case IVPU_MMU_EVT_F_STREAM_DISABLED: 244 return "Transaction marks non-substream disabled"; 245 case IVPU_MMU_EVT_F_TRANSL_FORBIDDEN: 246 return "MMU bypass is disallowed for this StreamID"; 247 case IVPU_MMU_EVT_C_BAD_SUBSTREAMID: 248 return "Invalid StreamID"; 249 case IVPU_MMU_EVT_F_CD_FETCH: 250 return "Fetch of CD caused external abort"; 251 case IVPU_MMU_EVT_C_BAD_CD: 252 return "Fetched CD invalid"; 253 case IVPU_MMU_EVT_F_WALK_EABT: 254 return " An external abort occurred fetching a TLB"; 255 case IVPU_MMU_EVT_F_TRANSLATION: 256 return "Translation fault"; 257 case IVPU_MMU_EVT_F_ADDR_SIZE: 258 return " Output address caused address size fault"; 259 case IVPU_MMU_EVT_F_ACCESS: 260 return "Access flag fault"; 261 case IVPU_MMU_EVT_F_PERMISSION: 262 return "Permission fault occurred on page access"; 263 case IVPU_MMU_EVT_F_TLB_CONFLICT: 264 return "A TLB conflict"; 265 case IVPU_MMU_EVT_F_CFG_CONFLICT: 266 return "A configuration cache conflict"; 267 case IVPU_MMU_EVT_E_PAGE_REQUEST: 268 return "Page request hint from a client device"; 269 case IVPU_MMU_EVT_F_VMS_FETCH: 270 return "Fetch of VMS caused external abort"; 271 default: 272 return "Unknown CMDQ command"; 273 } 274 } 275 276 static void ivpu_mmu_config_check(struct ivpu_device *vdev) 277 { 278 u32 val_ref; 279 u32 val; 280 281 if (ivpu_is_simics(vdev)) 282 val_ref = IVPU_MMU_IDR0_REF_SIMICS; 283 else 284 val_ref = IVPU_MMU_IDR0_REF; 285 286 val = REGV_RD32(IVPU_MMU_REG_IDR0); 287 if (val != val_ref) 288 ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref); 289 290 val = REGV_RD32(IVPU_MMU_REG_IDR1); 291 if (val != IVPU_MMU_IDR1_REF) 292 ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF); 293 294 val = REGV_RD32(IVPU_MMU_REG_IDR3); 295 if (val != IVPU_MMU_IDR3_REF) 296 ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF); 297 298 if (ivpu_is_simics(vdev)) 299 val_ref = IVPU_MMU_IDR5_REF_SIMICS; 300 else if (ivpu_is_fpga(vdev)) 301 val_ref = IVPU_MMU_IDR5_REF_FPGA; 302 else 303 val_ref = IVPU_MMU_IDR5_REF; 304 305 val = REGV_RD32(IVPU_MMU_REG_IDR5); 306 if (val != val_ref) 307 ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref); 308 } 309 310 static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev) 311 { 312 struct ivpu_mmu_info *mmu = vdev->mmu; 313 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; 314 size_t size = IVPU_MMU_CDTAB_ENT_COUNT * IVPU_MMU_CDTAB_ENT_SIZE; 315 316 cdtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &cdtab->dma, GFP_KERNEL); 317 if (!cdtab->base) 318 return -ENOMEM; 319 320 ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size); 321 322 return 0; 323 } 324 325 static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev) 326 { 327 struct ivpu_mmu_info *mmu = vdev->mmu; 328 struct ivpu_mmu_strtab *strtab = &mmu->strtab; 329 size_t size = IVPU_MMU_STRTAB_ENT_COUNT * IVPU_MMU_STRTAB_ENT_SIZE; 330 331 strtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &strtab->dma, GFP_KERNEL); 332 if (!strtab->base) 333 return -ENOMEM; 334 335 strtab->base_cfg = IVPU_MMU_STRTAB_CFG; 336 strtab->dma_q = IVPU_MMU_STRTAB_BASE_RA; 337 strtab->dma_q |= strtab->dma & IVPU_MMU_STRTAB_BASE_ADDR_MASK; 338 339 ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n", 340 &strtab->dma, &strtab->dma_q, size); 341 342 return 0; 343 } 344 345 static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev) 346 { 347 struct ivpu_mmu_info *mmu = vdev->mmu; 348 struct ivpu_mmu_queue *q = &mmu->cmdq; 349 350 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, &q->dma, GFP_KERNEL); 351 if (!q->base) 352 return -ENOMEM; 353 354 q->dma_q = IVPU_MMU_Q_BASE_RWA; 355 q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK; 356 q->dma_q |= IVPU_MMU_Q_COUNT_LOG2; 357 358 ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n", 359 &q->dma, &q->dma_q, IVPU_MMU_CMDQ_SIZE); 360 361 return 0; 362 } 363 364 static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev) 365 { 366 struct ivpu_mmu_info *mmu = vdev->mmu; 367 struct ivpu_mmu_queue *q = &mmu->evtq; 368 369 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, &q->dma, GFP_KERNEL); 370 if (!q->base) 371 return -ENOMEM; 372 373 q->dma_q = IVPU_MMU_Q_BASE_RWA; 374 q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK; 375 q->dma_q |= IVPU_MMU_Q_COUNT_LOG2; 376 377 ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n", 378 &q->dma, &q->dma_q, IVPU_MMU_EVTQ_SIZE); 379 380 return 0; 381 } 382 383 static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev) 384 { 385 int ret; 386 387 ret = ivpu_mmu_cdtab_alloc(vdev); 388 if (ret) { 389 ivpu_err(vdev, "Failed to allocate cdtab: %d\n", ret); 390 return ret; 391 } 392 393 ret = ivpu_mmu_strtab_alloc(vdev); 394 if (ret) { 395 ivpu_err(vdev, "Failed to allocate strtab: %d\n", ret); 396 return ret; 397 } 398 399 ret = ivpu_mmu_cmdq_alloc(vdev); 400 if (ret) { 401 ivpu_err(vdev, "Failed to allocate cmdq: %d\n", ret); 402 return ret; 403 } 404 405 ret = ivpu_mmu_evtq_alloc(vdev); 406 if (ret) 407 ivpu_err(vdev, "Failed to allocate evtq: %d\n", ret); 408 409 return ret; 410 } 411 412 static int ivpu_mmu_reg_write(struct ivpu_device *vdev, u32 reg, u32 val) 413 { 414 u32 reg_ack = reg + 4; /* ACK register is 4B after base register */ 415 u32 val_ack; 416 int ret; 417 418 REGV_WR32(reg, val); 419 420 ret = REGV_POLL(reg_ack, val_ack, (val == val_ack), IVPU_MMU_REG_TIMEOUT_US); 421 if (ret) 422 ivpu_err(vdev, "Failed to write register 0x%x\n", reg); 423 424 return ret; 425 } 426 427 static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev) 428 { 429 u32 irq_ctrl = IVPU_MMU_IRQ_EVTQ_EN | IVPU_MMU_IRQ_GERROR_EN; 430 int ret; 431 432 ret = ivpu_mmu_reg_write(vdev, IVPU_MMU_REG_IRQ_CTRL, 0); 433 if (ret) 434 return ret; 435 436 return ivpu_mmu_reg_write(vdev, IVPU_MMU_REG_IRQ_CTRL, irq_ctrl); 437 } 438 439 static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev) 440 { 441 struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq; 442 443 return REGV_POLL(IVPU_MMU_REG_CMDQ_CONS, cmdq->cons, (cmdq->prod == cmdq->cons), 444 IVPU_MMU_QUEUE_TIMEOUT_US); 445 } 446 447 static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1) 448 { 449 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq; 450 u64 *queue_buffer = q->base; 451 int idx = IVPU_MMU_Q_IDX(q->prod) * (IVPU_MMU_CMDQ_CMD_SIZE / sizeof(*queue_buffer)); 452 453 if (!CIRC_SPACE(IVPU_MMU_Q_IDX(q->prod), IVPU_MMU_Q_IDX(q->cons), IVPU_MMU_Q_COUNT)) { 454 ivpu_err(vdev, "Failed to write MMU CMD %s\n", name); 455 return -EBUSY; 456 } 457 458 queue_buffer[idx] = data0; 459 queue_buffer[idx + 1] = data1; 460 q->prod = (q->prod + 1) & IVPU_MMU_Q_WRAP_MASK; 461 462 ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1); 463 464 return 0; 465 } 466 467 static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev) 468 { 469 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq; 470 u64 val; 471 int ret; 472 473 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC) | 474 FIELD_PREP(IVPU_MMU_CMD_SYNC_0_CS, 0x2) | 475 FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSH, 0x3) | 476 FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSI_ATTR, 0xf); 477 478 ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0); 479 if (ret) 480 return ret; 481 482 clflush_cache_range(q->base, IVPU_MMU_CMDQ_SIZE); 483 REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, q->prod); 484 485 ret = ivpu_mmu_cmdq_wait_for_cons(vdev); 486 if (ret) 487 ivpu_err(vdev, "Timed out waiting for consumer: %d\n", ret); 488 489 return ret; 490 } 491 492 static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev) 493 { 494 u64 data0 = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_CFGI_ALL); 495 u64 data1 = FIELD_PREP(IVPU_MMU_CMD_CFGI_1_RANGE, 0x1f); 496 497 return ivpu_mmu_cmdq_cmd_write(vdev, "CFGI_ALL", data0, data1); 498 } 499 500 static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid) 501 { 502 u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NH_ASID) | 503 FIELD_PREP(IVPU_MMU_CMD_TLBI_0_ASID, ssid); 504 505 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NH_ASID", val, 0); 506 } 507 508 static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev) 509 { 510 u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NSNH_ALL); 511 512 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NSNH_ALL", val, 0); 513 } 514 515 static int ivpu_mmu_reset(struct ivpu_device *vdev) 516 { 517 struct ivpu_mmu_info *mmu = vdev->mmu; 518 u32 val; 519 int ret; 520 521 memset(mmu->cmdq.base, 0, IVPU_MMU_CMDQ_SIZE); 522 clflush_cache_range(mmu->cmdq.base, IVPU_MMU_CMDQ_SIZE); 523 mmu->cmdq.prod = 0; 524 mmu->cmdq.cons = 0; 525 526 memset(mmu->evtq.base, 0, IVPU_MMU_EVTQ_SIZE); 527 clflush_cache_range(mmu->evtq.base, IVPU_MMU_EVTQ_SIZE); 528 mmu->evtq.prod = 0; 529 mmu->evtq.cons = 0; 530 531 ret = ivpu_mmu_reg_write(vdev, IVPU_MMU_REG_CR0, 0); 532 if (ret) 533 return ret; 534 535 val = FIELD_PREP(IVPU_MMU_CR1_TABLE_SH, IVPU_MMU_SH_ISH) | 536 FIELD_PREP(IVPU_MMU_CR1_TABLE_OC, IVPU_MMU_CACHE_WB) | 537 FIELD_PREP(IVPU_MMU_CR1_TABLE_IC, IVPU_MMU_CACHE_WB) | 538 FIELD_PREP(IVPU_MMU_CR1_QUEUE_SH, IVPU_MMU_SH_ISH) | 539 FIELD_PREP(IVPU_MMU_CR1_QUEUE_OC, IVPU_MMU_CACHE_WB) | 540 FIELD_PREP(IVPU_MMU_CR1_QUEUE_IC, IVPU_MMU_CACHE_WB); 541 REGV_WR32(IVPU_MMU_REG_CR1, val); 542 543 REGV_WR64(IVPU_MMU_REG_STRTAB_BASE, mmu->strtab.dma_q); 544 REGV_WR32(IVPU_MMU_REG_STRTAB_BASE_CFG, mmu->strtab.base_cfg); 545 546 REGV_WR64(IVPU_MMU_REG_CMDQ_BASE, mmu->cmdq.dma_q); 547 REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, 0); 548 REGV_WR32(IVPU_MMU_REG_CMDQ_CONS, 0); 549 550 val = IVPU_MMU_CR0_CMDQEN; 551 ret = ivpu_mmu_reg_write(vdev, IVPU_MMU_REG_CR0, val); 552 if (ret) 553 return ret; 554 555 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); 556 if (ret) 557 return ret; 558 559 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev); 560 if (ret) 561 return ret; 562 563 ret = ivpu_mmu_cmdq_sync(vdev); 564 if (ret) 565 return ret; 566 567 REGV_WR64(IVPU_MMU_REG_EVTQ_BASE, mmu->evtq.dma_q); 568 REGV_WR32(IVPU_MMU_REG_EVTQ_PROD_SEC, 0); 569 REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, 0); 570 571 val |= IVPU_MMU_CR0_EVTQEN; 572 ret = ivpu_mmu_reg_write(vdev, IVPU_MMU_REG_CR0, val); 573 if (ret) 574 return ret; 575 576 val |= IVPU_MMU_CR0_ATSCHK; 577 ret = ivpu_mmu_reg_write(vdev, IVPU_MMU_REG_CR0, val); 578 if (ret) 579 return ret; 580 581 ret = ivpu_mmu_irqs_setup(vdev); 582 if (ret) 583 return ret; 584 585 val |= IVPU_MMU_CR0_SMMUEN; 586 return ivpu_mmu_reg_write(vdev, IVPU_MMU_REG_CR0, val); 587 } 588 589 static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid) 590 { 591 struct ivpu_mmu_info *mmu = vdev->mmu; 592 struct ivpu_mmu_strtab *strtab = &mmu->strtab; 593 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; 594 u64 *entry = strtab->base + (sid * IVPU_MMU_STRTAB_ENT_SIZE); 595 u64 str[2]; 596 597 str[0] = FIELD_PREP(IVPU_MMU_STE_0_CFG, IVPU_MMU_STE_0_CFG_S1_TRANS) | 598 FIELD_PREP(IVPU_MMU_STE_0_S1CDMAX, IVPU_MMU_CDTAB_ENT_COUNT_LOG2) | 599 FIELD_PREP(IVPU_MMU_STE_0_S1FMT, IVPU_MMU_STE_0_S1FMT_LINEAR) | 600 IVPU_MMU_STE_0_V | 601 (cdtab->dma & IVPU_MMU_STE_0_S1CTXPTR_MASK); 602 603 str[1] = FIELD_PREP(IVPU_MMU_STE_1_S1DSS, IVPU_MMU_STE_1_S1DSS_TERMINATE) | 604 FIELD_PREP(IVPU_MMU_STE_1_S1CIR, IVPU_MMU_STE_1_S1C_CACHE_NC) | 605 FIELD_PREP(IVPU_MMU_STE_1_S1COR, IVPU_MMU_STE_1_S1C_CACHE_NC) | 606 FIELD_PREP(IVPU_MMU_STE_1_S1CSH, IVPU_MMU_SH_NSH) | 607 FIELD_PREP(IVPU_MMU_STE_1_PRIVCFG, IVPU_MMU_STE_1_PRIVCFG_UNPRIV) | 608 FIELD_PREP(IVPU_MMU_STE_1_INSTCFG, IVPU_MMU_STE_1_INSTCFG_DATA) | 609 FIELD_PREP(IVPU_MMU_STE_1_STRW, IVPU_MMU_STE_1_STRW_NSEL1) | 610 FIELD_PREP(IVPU_MMU_STE_1_CONT, IVPU_MMU_STRTAB_CFG_LOG2SIZE) | 611 IVPU_MMU_STE_1_MEV | 612 IVPU_MMU_STE_1_S1STALLD; 613 614 WRITE_ONCE(entry[1], str[1]); 615 WRITE_ONCE(entry[0], str[0]); 616 617 clflush_cache_range(entry, IVPU_MMU_STRTAB_ENT_SIZE); 618 619 ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]); 620 } 621 622 static int ivpu_mmu_strtab_init(struct ivpu_device *vdev) 623 { 624 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0); 625 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3); 626 627 return 0; 628 } 629 630 int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid) 631 { 632 struct ivpu_mmu_info *mmu = vdev->mmu; 633 int ret = 0; 634 635 mutex_lock(&mmu->lock); 636 if (!mmu->on) 637 goto unlock; 638 639 ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid); 640 if (ret) 641 goto unlock; 642 643 ret = ivpu_mmu_cmdq_sync(vdev); 644 unlock: 645 mutex_unlock(&mmu->lock); 646 return ret; 647 } 648 649 static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma) 650 { 651 struct ivpu_mmu_info *mmu = vdev->mmu; 652 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; 653 u64 *entry; 654 u64 cd[4]; 655 int ret = 0; 656 657 if (ssid > IVPU_MMU_CDTAB_ENT_COUNT) 658 return -EINVAL; 659 660 entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE); 661 662 if (cd_dma != 0) { 663 cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, IVPU_MMU_T0SZ_48BIT) | 664 FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) | 665 FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) | 666 FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) | 667 FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) | 668 FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, IVPU_MMU_IPS_48BIT) | 669 FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) | 670 IVPU_MMU_CD_0_TCR_EPD1 | 671 IVPU_MMU_CD_0_AA64 | 672 IVPU_MMU_CD_0_R | 673 IVPU_MMU_CD_0_ASET | 674 IVPU_MMU_CD_0_V; 675 cd[1] = cd_dma & IVPU_MMU_CD_1_TTB0_MASK; 676 cd[2] = 0; 677 cd[3] = 0x0000000000007444; 678 679 /* For global context generate memory fault on VPU */ 680 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID) 681 cd[0] |= IVPU_MMU_CD_0_A; 682 } else { 683 memset(cd, 0, sizeof(cd)); 684 } 685 686 WRITE_ONCE(entry[1], cd[1]); 687 WRITE_ONCE(entry[2], cd[2]); 688 WRITE_ONCE(entry[3], cd[3]); 689 WRITE_ONCE(entry[0], cd[0]); 690 691 clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE); 692 693 ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n", 694 cd_dma ? "write" : "clear", ssid, &cd_dma, cd[0], cd[1], cd[2], cd[3]); 695 696 mutex_lock(&mmu->lock); 697 if (!mmu->on) 698 goto unlock; 699 700 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); 701 if (ret) 702 goto unlock; 703 704 ret = ivpu_mmu_cmdq_sync(vdev); 705 unlock: 706 mutex_unlock(&mmu->lock); 707 return ret; 708 } 709 710 static int ivpu_mmu_cd_add_gbl(struct ivpu_device *vdev) 711 { 712 int ret; 713 714 ret = ivpu_mmu_cd_add(vdev, 0, vdev->gctx.pgtable.pgd_dma); 715 if (ret) 716 ivpu_err(vdev, "Failed to add global CD entry: %d\n", ret); 717 718 return ret; 719 } 720 721 static int ivpu_mmu_cd_add_user(struct ivpu_device *vdev, u32 ssid, dma_addr_t cd_dma) 722 { 723 int ret; 724 725 if (ssid == 0) { 726 ivpu_err(vdev, "Invalid SSID: %u\n", ssid); 727 return -EINVAL; 728 } 729 730 ret = ivpu_mmu_cd_add(vdev, ssid, cd_dma); 731 if (ret) 732 ivpu_err(vdev, "Failed to add CD entry SSID=%u: %d\n", ssid, ret); 733 734 return ret; 735 } 736 737 int ivpu_mmu_init(struct ivpu_device *vdev) 738 { 739 struct ivpu_mmu_info *mmu = vdev->mmu; 740 int ret; 741 742 ivpu_dbg(vdev, MMU, "Init..\n"); 743 744 drmm_mutex_init(&vdev->drm, &mmu->lock); 745 ivpu_mmu_config_check(vdev); 746 747 ret = ivpu_mmu_structs_alloc(vdev); 748 if (ret) 749 return ret; 750 751 ret = ivpu_mmu_strtab_init(vdev); 752 if (ret) { 753 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret); 754 return ret; 755 } 756 757 ret = ivpu_mmu_cd_add_gbl(vdev); 758 if (ret) { 759 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret); 760 return ret; 761 } 762 763 ret = ivpu_mmu_enable(vdev); 764 if (ret) { 765 ivpu_err(vdev, "Failed to resume MMU: %d\n", ret); 766 return ret; 767 } 768 769 ivpu_dbg(vdev, MMU, "Init done\n"); 770 771 return 0; 772 } 773 774 int ivpu_mmu_enable(struct ivpu_device *vdev) 775 { 776 struct ivpu_mmu_info *mmu = vdev->mmu; 777 int ret; 778 779 mutex_lock(&mmu->lock); 780 781 mmu->on = true; 782 783 ret = ivpu_mmu_reset(vdev); 784 if (ret) { 785 ivpu_err(vdev, "Failed to reset MMU: %d\n", ret); 786 goto err; 787 } 788 789 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); 790 if (ret) 791 goto err; 792 793 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev); 794 if (ret) 795 goto err; 796 797 ret = ivpu_mmu_cmdq_sync(vdev); 798 if (ret) 799 goto err; 800 801 mutex_unlock(&mmu->lock); 802 803 return 0; 804 err: 805 mmu->on = false; 806 mutex_unlock(&mmu->lock); 807 return ret; 808 } 809 810 void ivpu_mmu_disable(struct ivpu_device *vdev) 811 { 812 struct ivpu_mmu_info *mmu = vdev->mmu; 813 814 mutex_lock(&mmu->lock); 815 mmu->on = false; 816 mutex_unlock(&mmu->lock); 817 } 818 819 static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event) 820 { 821 u32 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]); 822 u32 op = FIELD_GET(IVPU_MMU_EVT_OP_MASK, event[0]); 823 u64 fetch_addr = ((u64)event[7]) << 32 | event[6]; 824 u64 in_addr = ((u64)event[5]) << 32 | event[4]; 825 u32 sid = event[1]; 826 827 ivpu_err(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch addr: 0x%llx\n", 828 op, ivpu_mmu_event_to_str(op), ssid, sid, event[2], event[3], in_addr, fetch_addr); 829 } 830 831 static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev) 832 { 833 struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq; 834 u32 idx = IVPU_MMU_Q_IDX(evtq->cons); 835 u32 *evt = evtq->base + (idx * IVPU_MMU_EVTQ_CMD_SIZE); 836 837 evtq->prod = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC); 838 if (!CIRC_CNT(IVPU_MMU_Q_IDX(evtq->prod), IVPU_MMU_Q_IDX(evtq->cons), IVPU_MMU_Q_COUNT)) 839 return NULL; 840 841 clflush_cache_range(evt, IVPU_MMU_EVTQ_CMD_SIZE); 842 843 evtq->cons = (evtq->cons + 1) & IVPU_MMU_Q_WRAP_MASK; 844 REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, evtq->cons); 845 846 return evt; 847 } 848 849 void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev) 850 { 851 bool schedule_recovery = false; 852 u32 *event; 853 u32 ssid; 854 855 ivpu_dbg(vdev, IRQ, "MMU event queue\n"); 856 857 while ((event = ivpu_mmu_get_event(vdev)) != NULL) { 858 ivpu_mmu_dump_event(vdev, event); 859 860 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]); 861 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID) 862 schedule_recovery = true; 863 else 864 ivpu_mmu_user_context_mark_invalid(vdev, ssid); 865 } 866 867 if (schedule_recovery) 868 ivpu_pm_schedule_recovery(vdev); 869 } 870 871 void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev) 872 { 873 u32 gerror_val, gerrorn_val, active; 874 875 ivpu_dbg(vdev, IRQ, "MMU error\n"); 876 877 gerror_val = REGV_RD32(IVPU_MMU_REG_GERROR); 878 gerrorn_val = REGV_RD32(IVPU_MMU_REG_GERRORN); 879 880 active = gerror_val ^ gerrorn_val; 881 if (!(active & IVPU_MMU_GERROR_ERR_MASK)) 882 return; 883 884 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_ABT, active)) 885 ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n"); 886 887 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT, active)) 888 ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n"); 889 890 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT, active)) 891 ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n"); 892 893 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT, active)) 894 ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n"); 895 896 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT, active)) 897 ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n"); 898 899 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT, active)) 900 ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n"); 901 902 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, CMDQ, active)) 903 ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n"); 904 905 REGV_WR32(IVPU_MMU_REG_GERRORN, gerror_val); 906 } 907 908 int ivpu_mmu_set_pgtable(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable) 909 { 910 return ivpu_mmu_cd_add_user(vdev, ssid, pgtable->pgd_dma); 911 } 912 913 void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid) 914 { 915 ivpu_mmu_cd_add_user(vdev, ssid, 0); /* 0 will clear CD entry */ 916 } 917