1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6 #include <linux/circ_buf.h> 7 #include <linux/highmem.h> 8 9 #include "ivpu_drv.h" 10 #include "ivpu_hw_reg_io.h" 11 #include "ivpu_mmu.h" 12 #include "ivpu_mmu_context.h" 13 #include "ivpu_pm.h" 14 15 #define IVPU_MMU_REG_IDR0 0x00200000u 16 #define IVPU_MMU_REG_IDR1 0x00200004u 17 #define IVPU_MMU_REG_IDR3 0x0020000cu 18 #define IVPU_MMU_REG_IDR5 0x00200014u 19 #define IVPU_MMU_REG_CR0 0x00200020u 20 #define IVPU_MMU_REG_CR0ACK 0x00200024u 21 #define IVPU_MMU_REG_CR0ACK_VAL_MASK GENMASK(31, 0) 22 #define IVPU_MMU_REG_CR1 0x00200028u 23 #define IVPU_MMU_REG_CR2 0x0020002cu 24 #define IVPU_MMU_REG_IRQ_CTRL 0x00200050u 25 #define IVPU_MMU_REG_IRQ_CTRLACK 0x00200054u 26 #define IVPU_MMU_REG_IRQ_CTRLACK_VAL_MASK GENMASK(31, 0) 27 28 #define IVPU_MMU_REG_GERROR 0x00200060u 29 #define IVPU_MMU_REG_GERROR_CMDQ_MASK BIT_MASK(0) 30 #define IVPU_MMU_REG_GERROR_EVTQ_ABT_MASK BIT_MASK(2) 31 #define IVPU_MMU_REG_GERROR_PRIQ_ABT_MASK BIT_MASK(3) 32 #define IVPU_MMU_REG_GERROR_MSI_CMDQ_ABT_MASK BIT_MASK(4) 33 #define IVPU_MMU_REG_GERROR_MSI_EVTQ_ABT_MASK BIT_MASK(5) 34 #define IVPU_MMU_REG_GERROR_MSI_PRIQ_ABT_MASK BIT_MASK(6) 35 #define IVPU_MMU_REG_GERROR_MSI_ABT_MASK BIT_MASK(7) 36 37 #define IVPU_MMU_REG_GERRORN 0x00200064u 38 39 #define IVPU_MMU_REG_STRTAB_BASE 0x00200080u 40 #define IVPU_MMU_REG_STRTAB_BASE_CFG 0x00200088u 41 #define IVPU_MMU_REG_CMDQ_BASE 0x00200090u 42 #define IVPU_MMU_REG_CMDQ_PROD 0x00200098u 43 #define IVPU_MMU_REG_CMDQ_CONS 0x0020009cu 44 #define IVPU_MMU_REG_CMDQ_CONS_VAL_MASK GENMASK(23, 0) 45 #define IVPU_MMU_REG_CMDQ_CONS_ERR_MASK GENMASK(30, 24) 46 #define IVPU_MMU_REG_EVTQ_BASE 0x002000a0u 47 #define IVPU_MMU_REG_EVTQ_PROD 0x002000a8u 48 #define IVPU_MMU_REG_EVTQ_CONS 0x002000acu 49 #define IVPU_MMU_REG_EVTQ_PROD_SEC (0x002000a8u + SZ_64K) 50 #define IVPU_MMU_REG_EVTQ_CONS_SEC (0x002000acu + SZ_64K) 51 52 #define IVPU_MMU_IDR0_REF 0x080f3e0f 53 #define IVPU_MMU_IDR0_REF_SIMICS 0x080f3e1f 54 #define IVPU_MMU_IDR1_REF 0x0e739d18 55 #define IVPU_MMU_IDR3_REF 0x0000003c 56 #define IVPU_MMU_IDR5_REF 0x00040070 57 #define IVPU_MMU_IDR5_REF_SIMICS 0x00000075 58 #define IVPU_MMU_IDR5_REF_FPGA 0x00800075 59 60 #define IVPU_MMU_CDTAB_ENT_SIZE 64 61 #define IVPU_MMU_CDTAB_ENT_COUNT_LOG2 8 /* 256 entries */ 62 #define IVPU_MMU_CDTAB_ENT_COUNT ((u32)1 << IVPU_MMU_CDTAB_ENT_COUNT_LOG2) 63 64 #define IVPU_MMU_STREAM_ID0 0 65 #define IVPU_MMU_STREAM_ID3 3 66 67 #define IVPU_MMU_STRTAB_ENT_SIZE 64 68 #define IVPU_MMU_STRTAB_ENT_COUNT 4 69 #define IVPU_MMU_STRTAB_CFG_LOG2SIZE 2 70 #define IVPU_MMU_STRTAB_CFG IVPU_MMU_STRTAB_CFG_LOG2SIZE 71 72 #define IVPU_MMU_Q_COUNT_LOG2 4 /* 16 entries */ 73 #define IVPU_MMU_Q_COUNT ((u32)1 << IVPU_MMU_Q_COUNT_LOG2) 74 #define IVPU_MMU_Q_WRAP_BIT (IVPU_MMU_Q_COUNT << 1) 75 #define IVPU_MMU_Q_WRAP_MASK (IVPU_MMU_Q_WRAP_BIT - 1) 76 #define IVPU_MMU_Q_IDX_MASK (IVPU_MMU_Q_COUNT - 1) 77 #define IVPU_MMU_Q_IDX(val) ((val) & IVPU_MMU_Q_IDX_MASK) 78 79 #define IVPU_MMU_CMDQ_CMD_SIZE 16 80 #define IVPU_MMU_CMDQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_CMDQ_CMD_SIZE) 81 82 #define IVPU_MMU_EVTQ_CMD_SIZE 32 83 #define IVPU_MMU_EVTQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_EVTQ_CMD_SIZE) 84 85 #define IVPU_MMU_CMD_OPCODE GENMASK(7, 0) 86 87 #define IVPU_MMU_CMD_SYNC_0_CS GENMASK(13, 12) 88 #define IVPU_MMU_CMD_SYNC_0_MSH GENMASK(23, 22) 89 #define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24) 90 #define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24) 91 #define IVPU_MMU_CMD_SYNC_0_MSI_DATA GENMASK(63, 32) 92 93 #define IVPU_MMU_CMD_CFGI_0_SSEC BIT(10) 94 #define IVPU_MMU_CMD_CFGI_0_SSV BIT(11) 95 #define IVPU_MMU_CMD_CFGI_0_SSID GENMASK(31, 12) 96 #define IVPU_MMU_CMD_CFGI_0_SID GENMASK(63, 32) 97 #define IVPU_MMU_CMD_CFGI_1_RANGE GENMASK(4, 0) 98 99 #define IVPU_MMU_CMD_TLBI_0_ASID GENMASK(63, 48) 100 #define IVPU_MMU_CMD_TLBI_0_VMID GENMASK(47, 32) 101 102 #define CMD_PREFETCH_CFG 0x1 103 #define CMD_CFGI_STE 0x3 104 #define CMD_CFGI_ALL 0x4 105 #define CMD_CFGI_CD 0x5 106 #define CMD_CFGI_CD_ALL 0x6 107 #define CMD_TLBI_NH_ASID 0x11 108 #define CMD_TLBI_EL2_ALL 0x20 109 #define CMD_TLBI_NSNH_ALL 0x30 110 #define CMD_SYNC 0x46 111 112 #define IVPU_MMU_EVT_F_UUT 0x01 113 #define IVPU_MMU_EVT_C_BAD_STREAMID 0x02 114 #define IVPU_MMU_EVT_F_STE_FETCH 0x03 115 #define IVPU_MMU_EVT_C_BAD_STE 0x04 116 #define IVPU_MMU_EVT_F_BAD_ATS_TREQ 0x05 117 #define IVPU_MMU_EVT_F_STREAM_DISABLED 0x06 118 #define IVPU_MMU_EVT_F_TRANSL_FORBIDDEN 0x07 119 #define IVPU_MMU_EVT_C_BAD_SUBSTREAMID 0x08 120 #define IVPU_MMU_EVT_F_CD_FETCH 0x09 121 #define IVPU_MMU_EVT_C_BAD_CD 0x0a 122 #define IVPU_MMU_EVT_F_WALK_EABT 0x0b 123 #define IVPU_MMU_EVT_F_TRANSLATION 0x10 124 #define IVPU_MMU_EVT_F_ADDR_SIZE 0x11 125 #define IVPU_MMU_EVT_F_ACCESS 0x12 126 #define IVPU_MMU_EVT_F_PERMISSION 0x13 127 #define IVPU_MMU_EVT_F_TLB_CONFLICT 0x20 128 #define IVPU_MMU_EVT_F_CFG_CONFLICT 0x21 129 #define IVPU_MMU_EVT_E_PAGE_REQUEST 0x24 130 #define IVPU_MMU_EVT_F_VMS_FETCH 0x25 131 132 #define IVPU_MMU_EVT_OP_MASK GENMASK_ULL(7, 0) 133 #define IVPU_MMU_EVT_SSID_MASK GENMASK_ULL(31, 12) 134 135 #define IVPU_MMU_Q_BASE_RWA BIT(62) 136 #define IVPU_MMU_Q_BASE_ADDR_MASK GENMASK_ULL(51, 5) 137 #define IVPU_MMU_STRTAB_BASE_RA BIT(62) 138 #define IVPU_MMU_STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6) 139 140 #define IVPU_MMU_IRQ_EVTQ_EN BIT(2) 141 #define IVPU_MMU_IRQ_GERROR_EN BIT(0) 142 143 #define IVPU_MMU_CR0_ATSCHK BIT(4) 144 #define IVPU_MMU_CR0_CMDQEN BIT(3) 145 #define IVPU_MMU_CR0_EVTQEN BIT(2) 146 #define IVPU_MMU_CR0_PRIQEN BIT(1) 147 #define IVPU_MMU_CR0_SMMUEN BIT(0) 148 149 #define IVPU_MMU_CR1_TABLE_SH GENMASK(11, 10) 150 #define IVPU_MMU_CR1_TABLE_OC GENMASK(9, 8) 151 #define IVPU_MMU_CR1_TABLE_IC GENMASK(7, 6) 152 #define IVPU_MMU_CR1_QUEUE_SH GENMASK(5, 4) 153 #define IVPU_MMU_CR1_QUEUE_OC GENMASK(3, 2) 154 #define IVPU_MMU_CR1_QUEUE_IC GENMASK(1, 0) 155 #define IVPU_MMU_CACHE_NC 0 156 #define IVPU_MMU_CACHE_WB 1 157 #define IVPU_MMU_CACHE_WT 2 158 #define IVPU_MMU_SH_NSH 0 159 #define IVPU_MMU_SH_OSH 2 160 #define IVPU_MMU_SH_ISH 3 161 162 #define IVPU_MMU_CMDQ_OP GENMASK_ULL(7, 0) 163 164 #define IVPU_MMU_CD_0_TCR_T0SZ GENMASK_ULL(5, 0) 165 #define IVPU_MMU_CD_0_TCR_TG0 GENMASK_ULL(7, 6) 166 #define IVPU_MMU_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8) 167 #define IVPU_MMU_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10) 168 #define IVPU_MMU_CD_0_TCR_SH0 GENMASK_ULL(13, 12) 169 #define IVPU_MMU_CD_0_TCR_EPD0 BIT_ULL(14) 170 #define IVPU_MMU_CD_0_TCR_EPD1 BIT_ULL(30) 171 #define IVPU_MMU_CD_0_ENDI BIT(15) 172 #define IVPU_MMU_CD_0_V BIT(31) 173 #define IVPU_MMU_CD_0_TCR_IPS GENMASK_ULL(34, 32) 174 #define IVPU_MMU_CD_0_TCR_TBI0 BIT_ULL(38) 175 #define IVPU_MMU_CD_0_AA64 BIT(41) 176 #define IVPU_MMU_CD_0_S BIT(44) 177 #define IVPU_MMU_CD_0_R BIT(45) 178 #define IVPU_MMU_CD_0_A BIT(46) 179 #define IVPU_MMU_CD_0_ASET BIT(47) 180 #define IVPU_MMU_CD_0_ASID GENMASK_ULL(63, 48) 181 182 #define IVPU_MMU_T0SZ_48BIT 16 183 #define IVPU_MMU_T0SZ_38BIT 26 184 185 #define IVPU_MMU_IPS_48BIT 5 186 #define IVPU_MMU_IPS_44BIT 4 187 #define IVPU_MMU_IPS_42BIT 3 188 #define IVPU_MMU_IPS_40BIT 2 189 #define IVPU_MMU_IPS_36BIT 1 190 #define IVPU_MMU_IPS_32BIT 0 191 192 #define IVPU_MMU_CD_1_TTB0_MASK GENMASK_ULL(51, 4) 193 194 #define IVPU_MMU_STE_0_S1CDMAX GENMASK_ULL(63, 59) 195 #define IVPU_MMU_STE_0_S1FMT GENMASK_ULL(5, 4) 196 #define IVPU_MMU_STE_0_S1FMT_LINEAR 0 197 #define IVPU_MMU_STE_DWORDS 8 198 #define IVPU_MMU_STE_0_CFG_S1_TRANS 5 199 #define IVPU_MMU_STE_0_CFG GENMASK_ULL(3, 1) 200 #define IVPU_MMU_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6) 201 #define IVPU_MMU_STE_0_V BIT(0) 202 203 #define IVPU_MMU_STE_1_STRW_NSEL1 0ul 204 #define IVPU_MMU_STE_1_CONT GENMASK_ULL(16, 13) 205 #define IVPU_MMU_STE_1_STRW GENMASK_ULL(31, 30) 206 #define IVPU_MMU_STE_1_PRIVCFG GENMASK_ULL(49, 48) 207 #define IVPU_MMU_STE_1_PRIVCFG_UNPRIV 2ul 208 #define IVPU_MMU_STE_1_INSTCFG GENMASK_ULL(51, 50) 209 #define IVPU_MMU_STE_1_INSTCFG_DATA 2ul 210 #define IVPU_MMU_STE_1_MEV BIT(19) 211 #define IVPU_MMU_STE_1_S1STALLD BIT(27) 212 #define IVPU_MMU_STE_1_S1C_CACHE_NC 0ul 213 #define IVPU_MMU_STE_1_S1C_CACHE_WBRA 1ul 214 #define IVPU_MMU_STE_1_S1C_CACHE_WT 2ul 215 #define IVPU_MMU_STE_1_S1C_CACHE_WB 3ul 216 #define IVPU_MMU_STE_1_S1CIR GENMASK_ULL(3, 2) 217 #define IVPU_MMU_STE_1_S1COR GENMASK_ULL(5, 4) 218 #define IVPU_MMU_STE_1_S1CSH GENMASK_ULL(7, 6) 219 #define IVPU_MMU_STE_1_S1DSS GENMASK_ULL(1, 0) 220 #define IVPU_MMU_STE_1_S1DSS_TERMINATE 0x0 221 222 #define IVPU_MMU_REG_TIMEOUT_US (10 * USEC_PER_MSEC) 223 #define IVPU_MMU_QUEUE_TIMEOUT_US (100 * USEC_PER_MSEC) 224 225 #define IVPU_MMU_GERROR_ERR_MASK ((REG_FLD(IVPU_MMU_REG_GERROR, CMDQ)) | \ 226 (REG_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT)) | \ 227 (REG_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT)) | \ 228 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT)) | \ 229 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT)) | \ 230 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT)) | \ 231 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_ABT))) 232 233 static char *ivpu_mmu_event_to_str(u32 cmd) 234 { 235 switch (cmd) { 236 case IVPU_MMU_EVT_F_UUT: 237 return "Unsupported Upstream Transaction"; 238 case IVPU_MMU_EVT_C_BAD_STREAMID: 239 return "Transaction StreamID out of range"; 240 case IVPU_MMU_EVT_F_STE_FETCH: 241 return "Fetch of STE caused external abort"; 242 case IVPU_MMU_EVT_C_BAD_STE: 243 return "Used STE invalid"; 244 case IVPU_MMU_EVT_F_BAD_ATS_TREQ: 245 return "Address Request disallowed for a StreamID"; 246 case IVPU_MMU_EVT_F_STREAM_DISABLED: 247 return "Transaction marks non-substream disabled"; 248 case IVPU_MMU_EVT_F_TRANSL_FORBIDDEN: 249 return "MMU bypass is disallowed for this StreamID"; 250 case IVPU_MMU_EVT_C_BAD_SUBSTREAMID: 251 return "Invalid StreamID"; 252 case IVPU_MMU_EVT_F_CD_FETCH: 253 return "Fetch of CD caused external abort"; 254 case IVPU_MMU_EVT_C_BAD_CD: 255 return "Fetched CD invalid"; 256 case IVPU_MMU_EVT_F_WALK_EABT: 257 return " An external abort occurred fetching a TLB"; 258 case IVPU_MMU_EVT_F_TRANSLATION: 259 return "Translation fault"; 260 case IVPU_MMU_EVT_F_ADDR_SIZE: 261 return " Output address caused address size fault"; 262 case IVPU_MMU_EVT_F_ACCESS: 263 return "Access flag fault"; 264 case IVPU_MMU_EVT_F_PERMISSION: 265 return "Permission fault occurred on page access"; 266 case IVPU_MMU_EVT_F_TLB_CONFLICT: 267 return "A TLB conflict"; 268 case IVPU_MMU_EVT_F_CFG_CONFLICT: 269 return "A configuration cache conflict"; 270 case IVPU_MMU_EVT_E_PAGE_REQUEST: 271 return "Page request hint from a client device"; 272 case IVPU_MMU_EVT_F_VMS_FETCH: 273 return "Fetch of VMS caused external abort"; 274 default: 275 return "Unknown CMDQ command"; 276 } 277 } 278 279 static void ivpu_mmu_config_check(struct ivpu_device *vdev) 280 { 281 u32 val_ref; 282 u32 val; 283 284 if (ivpu_is_simics(vdev)) 285 val_ref = IVPU_MMU_IDR0_REF_SIMICS; 286 else 287 val_ref = IVPU_MMU_IDR0_REF; 288 289 val = REGV_RD32(IVPU_MMU_REG_IDR0); 290 if (val != val_ref) 291 ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref); 292 293 val = REGV_RD32(IVPU_MMU_REG_IDR1); 294 if (val != IVPU_MMU_IDR1_REF) 295 ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF); 296 297 val = REGV_RD32(IVPU_MMU_REG_IDR3); 298 if (val != IVPU_MMU_IDR3_REF) 299 ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF); 300 301 if (ivpu_is_simics(vdev)) 302 val_ref = IVPU_MMU_IDR5_REF_SIMICS; 303 else if (ivpu_is_fpga(vdev)) 304 val_ref = IVPU_MMU_IDR5_REF_FPGA; 305 else 306 val_ref = IVPU_MMU_IDR5_REF; 307 308 val = REGV_RD32(IVPU_MMU_REG_IDR5); 309 if (val != val_ref) 310 ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref); 311 } 312 313 static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev) 314 { 315 struct ivpu_mmu_info *mmu = vdev->mmu; 316 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; 317 size_t size = IVPU_MMU_CDTAB_ENT_COUNT * IVPU_MMU_CDTAB_ENT_SIZE; 318 319 cdtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &cdtab->dma, GFP_KERNEL); 320 if (!cdtab->base) 321 return -ENOMEM; 322 323 ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size); 324 325 return 0; 326 } 327 328 static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev) 329 { 330 struct ivpu_mmu_info *mmu = vdev->mmu; 331 struct ivpu_mmu_strtab *strtab = &mmu->strtab; 332 size_t size = IVPU_MMU_STRTAB_ENT_COUNT * IVPU_MMU_STRTAB_ENT_SIZE; 333 334 strtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &strtab->dma, GFP_KERNEL); 335 if (!strtab->base) 336 return -ENOMEM; 337 338 strtab->base_cfg = IVPU_MMU_STRTAB_CFG; 339 strtab->dma_q = IVPU_MMU_STRTAB_BASE_RA; 340 strtab->dma_q |= strtab->dma & IVPU_MMU_STRTAB_BASE_ADDR_MASK; 341 342 ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n", 343 &strtab->dma, &strtab->dma_q, size); 344 345 return 0; 346 } 347 348 static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev) 349 { 350 struct ivpu_mmu_info *mmu = vdev->mmu; 351 struct ivpu_mmu_queue *q = &mmu->cmdq; 352 353 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, &q->dma, GFP_KERNEL); 354 if (!q->base) 355 return -ENOMEM; 356 357 q->dma_q = IVPU_MMU_Q_BASE_RWA; 358 q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK; 359 q->dma_q |= IVPU_MMU_Q_COUNT_LOG2; 360 361 ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n", 362 &q->dma, &q->dma_q, IVPU_MMU_CMDQ_SIZE); 363 364 return 0; 365 } 366 367 static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev) 368 { 369 struct ivpu_mmu_info *mmu = vdev->mmu; 370 struct ivpu_mmu_queue *q = &mmu->evtq; 371 372 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, &q->dma, GFP_KERNEL); 373 if (!q->base) 374 return -ENOMEM; 375 376 q->dma_q = IVPU_MMU_Q_BASE_RWA; 377 q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK; 378 q->dma_q |= IVPU_MMU_Q_COUNT_LOG2; 379 380 ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n", 381 &q->dma, &q->dma_q, IVPU_MMU_EVTQ_SIZE); 382 383 return 0; 384 } 385 386 static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev) 387 { 388 int ret; 389 390 ret = ivpu_mmu_cdtab_alloc(vdev); 391 if (ret) { 392 ivpu_err(vdev, "Failed to allocate cdtab: %d\n", ret); 393 return ret; 394 } 395 396 ret = ivpu_mmu_strtab_alloc(vdev); 397 if (ret) { 398 ivpu_err(vdev, "Failed to allocate strtab: %d\n", ret); 399 return ret; 400 } 401 402 ret = ivpu_mmu_cmdq_alloc(vdev); 403 if (ret) { 404 ivpu_err(vdev, "Failed to allocate cmdq: %d\n", ret); 405 return ret; 406 } 407 408 ret = ivpu_mmu_evtq_alloc(vdev); 409 if (ret) 410 ivpu_err(vdev, "Failed to allocate evtq: %d\n", ret); 411 412 return ret; 413 } 414 415 static int ivpu_mmu_reg_write_cr0(struct ivpu_device *vdev, u32 val) 416 { 417 REGV_WR32(IVPU_MMU_REG_CR0, val); 418 419 return REGV_POLL_FLD(IVPU_MMU_REG_CR0ACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US); 420 } 421 422 static int ivpu_mmu_reg_write_irq_ctrl(struct ivpu_device *vdev, u32 val) 423 { 424 REGV_WR32(IVPU_MMU_REG_IRQ_CTRL, val); 425 426 return REGV_POLL_FLD(IVPU_MMU_REG_IRQ_CTRLACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US); 427 } 428 429 static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev) 430 { 431 u32 irq_ctrl = IVPU_MMU_IRQ_EVTQ_EN | IVPU_MMU_IRQ_GERROR_EN; 432 int ret; 433 434 ret = ivpu_mmu_reg_write_irq_ctrl(vdev, 0); 435 if (ret) 436 return ret; 437 438 return ivpu_mmu_reg_write_irq_ctrl(vdev, irq_ctrl); 439 } 440 441 static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev) 442 { 443 struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq; 444 int ret; 445 446 ret = REGV_POLL_FLD(IVPU_MMU_REG_CMDQ_CONS, VAL, cmdq->prod, 447 IVPU_MMU_QUEUE_TIMEOUT_US); 448 if (ret) 449 return ret; 450 451 cmdq->cons = cmdq->prod; 452 453 return 0; 454 } 455 456 static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1) 457 { 458 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq; 459 u64 *queue_buffer = q->base; 460 int idx = IVPU_MMU_Q_IDX(q->prod) * (IVPU_MMU_CMDQ_CMD_SIZE / sizeof(*queue_buffer)); 461 462 if (!CIRC_SPACE(IVPU_MMU_Q_IDX(q->prod), IVPU_MMU_Q_IDX(q->cons), IVPU_MMU_Q_COUNT)) { 463 ivpu_err(vdev, "Failed to write MMU CMD %s\n", name); 464 return -EBUSY; 465 } 466 467 queue_buffer[idx] = data0; 468 queue_buffer[idx + 1] = data1; 469 q->prod = (q->prod + 1) & IVPU_MMU_Q_WRAP_MASK; 470 471 ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1); 472 473 return 0; 474 } 475 476 static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev) 477 { 478 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq; 479 u64 val; 480 int ret; 481 482 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC) | 483 FIELD_PREP(IVPU_MMU_CMD_SYNC_0_CS, 0x2) | 484 FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSH, 0x3) | 485 FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSI_ATTR, 0xf); 486 487 ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0); 488 if (ret) 489 return ret; 490 491 clflush_cache_range(q->base, IVPU_MMU_CMDQ_SIZE); 492 REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, q->prod); 493 494 ret = ivpu_mmu_cmdq_wait_for_cons(vdev); 495 if (ret) 496 ivpu_err(vdev, "Timed out waiting for consumer: %d\n", ret); 497 498 return ret; 499 } 500 501 static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev) 502 { 503 u64 data0 = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_CFGI_ALL); 504 u64 data1 = FIELD_PREP(IVPU_MMU_CMD_CFGI_1_RANGE, 0x1f); 505 506 return ivpu_mmu_cmdq_cmd_write(vdev, "CFGI_ALL", data0, data1); 507 } 508 509 static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid) 510 { 511 u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NH_ASID) | 512 FIELD_PREP(IVPU_MMU_CMD_TLBI_0_ASID, ssid); 513 514 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NH_ASID", val, 0); 515 } 516 517 static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev) 518 { 519 u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NSNH_ALL); 520 521 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NSNH_ALL", val, 0); 522 } 523 524 static int ivpu_mmu_reset(struct ivpu_device *vdev) 525 { 526 struct ivpu_mmu_info *mmu = vdev->mmu; 527 u32 val; 528 int ret; 529 530 memset(mmu->cmdq.base, 0, IVPU_MMU_CMDQ_SIZE); 531 clflush_cache_range(mmu->cmdq.base, IVPU_MMU_CMDQ_SIZE); 532 mmu->cmdq.prod = 0; 533 mmu->cmdq.cons = 0; 534 535 memset(mmu->evtq.base, 0, IVPU_MMU_EVTQ_SIZE); 536 clflush_cache_range(mmu->evtq.base, IVPU_MMU_EVTQ_SIZE); 537 mmu->evtq.prod = 0; 538 mmu->evtq.cons = 0; 539 540 ret = ivpu_mmu_reg_write_cr0(vdev, 0); 541 if (ret) 542 return ret; 543 544 val = FIELD_PREP(IVPU_MMU_CR1_TABLE_SH, IVPU_MMU_SH_ISH) | 545 FIELD_PREP(IVPU_MMU_CR1_TABLE_OC, IVPU_MMU_CACHE_WB) | 546 FIELD_PREP(IVPU_MMU_CR1_TABLE_IC, IVPU_MMU_CACHE_WB) | 547 FIELD_PREP(IVPU_MMU_CR1_QUEUE_SH, IVPU_MMU_SH_ISH) | 548 FIELD_PREP(IVPU_MMU_CR1_QUEUE_OC, IVPU_MMU_CACHE_WB) | 549 FIELD_PREP(IVPU_MMU_CR1_QUEUE_IC, IVPU_MMU_CACHE_WB); 550 REGV_WR32(IVPU_MMU_REG_CR1, val); 551 552 REGV_WR64(IVPU_MMU_REG_STRTAB_BASE, mmu->strtab.dma_q); 553 REGV_WR32(IVPU_MMU_REG_STRTAB_BASE_CFG, mmu->strtab.base_cfg); 554 555 REGV_WR64(IVPU_MMU_REG_CMDQ_BASE, mmu->cmdq.dma_q); 556 REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, 0); 557 REGV_WR32(IVPU_MMU_REG_CMDQ_CONS, 0); 558 559 val = IVPU_MMU_CR0_CMDQEN; 560 ret = ivpu_mmu_reg_write_cr0(vdev, val); 561 if (ret) 562 return ret; 563 564 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); 565 if (ret) 566 return ret; 567 568 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev); 569 if (ret) 570 return ret; 571 572 ret = ivpu_mmu_cmdq_sync(vdev); 573 if (ret) 574 return ret; 575 576 REGV_WR64(IVPU_MMU_REG_EVTQ_BASE, mmu->evtq.dma_q); 577 REGV_WR32(IVPU_MMU_REG_EVTQ_PROD_SEC, 0); 578 REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, 0); 579 580 val |= IVPU_MMU_CR0_EVTQEN; 581 ret = ivpu_mmu_reg_write_cr0(vdev, val); 582 if (ret) 583 return ret; 584 585 val |= IVPU_MMU_CR0_ATSCHK; 586 ret = ivpu_mmu_reg_write_cr0(vdev, val); 587 if (ret) 588 return ret; 589 590 ret = ivpu_mmu_irqs_setup(vdev); 591 if (ret) 592 return ret; 593 594 val |= IVPU_MMU_CR0_SMMUEN; 595 return ivpu_mmu_reg_write_cr0(vdev, val); 596 } 597 598 static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid) 599 { 600 struct ivpu_mmu_info *mmu = vdev->mmu; 601 struct ivpu_mmu_strtab *strtab = &mmu->strtab; 602 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; 603 u64 *entry = strtab->base + (sid * IVPU_MMU_STRTAB_ENT_SIZE); 604 u64 str[2]; 605 606 str[0] = FIELD_PREP(IVPU_MMU_STE_0_CFG, IVPU_MMU_STE_0_CFG_S1_TRANS) | 607 FIELD_PREP(IVPU_MMU_STE_0_S1CDMAX, IVPU_MMU_CDTAB_ENT_COUNT_LOG2) | 608 FIELD_PREP(IVPU_MMU_STE_0_S1FMT, IVPU_MMU_STE_0_S1FMT_LINEAR) | 609 IVPU_MMU_STE_0_V | 610 (cdtab->dma & IVPU_MMU_STE_0_S1CTXPTR_MASK); 611 612 str[1] = FIELD_PREP(IVPU_MMU_STE_1_S1DSS, IVPU_MMU_STE_1_S1DSS_TERMINATE) | 613 FIELD_PREP(IVPU_MMU_STE_1_S1CIR, IVPU_MMU_STE_1_S1C_CACHE_NC) | 614 FIELD_PREP(IVPU_MMU_STE_1_S1COR, IVPU_MMU_STE_1_S1C_CACHE_NC) | 615 FIELD_PREP(IVPU_MMU_STE_1_S1CSH, IVPU_MMU_SH_NSH) | 616 FIELD_PREP(IVPU_MMU_STE_1_PRIVCFG, IVPU_MMU_STE_1_PRIVCFG_UNPRIV) | 617 FIELD_PREP(IVPU_MMU_STE_1_INSTCFG, IVPU_MMU_STE_1_INSTCFG_DATA) | 618 FIELD_PREP(IVPU_MMU_STE_1_STRW, IVPU_MMU_STE_1_STRW_NSEL1) | 619 FIELD_PREP(IVPU_MMU_STE_1_CONT, IVPU_MMU_STRTAB_CFG_LOG2SIZE) | 620 IVPU_MMU_STE_1_MEV | 621 IVPU_MMU_STE_1_S1STALLD; 622 623 WRITE_ONCE(entry[1], str[1]); 624 WRITE_ONCE(entry[0], str[0]); 625 626 clflush_cache_range(entry, IVPU_MMU_STRTAB_ENT_SIZE); 627 628 ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]); 629 } 630 631 static int ivpu_mmu_strtab_init(struct ivpu_device *vdev) 632 { 633 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0); 634 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3); 635 636 return 0; 637 } 638 639 int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid) 640 { 641 struct ivpu_mmu_info *mmu = vdev->mmu; 642 int ret = 0; 643 644 mutex_lock(&mmu->lock); 645 if (!mmu->on) 646 goto unlock; 647 648 ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid); 649 if (ret) 650 goto unlock; 651 652 ret = ivpu_mmu_cmdq_sync(vdev); 653 unlock: 654 mutex_unlock(&mmu->lock); 655 return ret; 656 } 657 658 static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma) 659 { 660 struct ivpu_mmu_info *mmu = vdev->mmu; 661 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab; 662 u64 *entry; 663 u64 cd[4]; 664 int ret = 0; 665 666 if (ssid > IVPU_MMU_CDTAB_ENT_COUNT) 667 return -EINVAL; 668 669 entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE); 670 671 if (cd_dma != 0) { 672 cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, IVPU_MMU_T0SZ_48BIT) | 673 FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) | 674 FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) | 675 FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) | 676 FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) | 677 FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, IVPU_MMU_IPS_48BIT) | 678 FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) | 679 IVPU_MMU_CD_0_TCR_EPD1 | 680 IVPU_MMU_CD_0_AA64 | 681 IVPU_MMU_CD_0_R | 682 IVPU_MMU_CD_0_ASET | 683 IVPU_MMU_CD_0_V; 684 cd[1] = cd_dma & IVPU_MMU_CD_1_TTB0_MASK; 685 cd[2] = 0; 686 cd[3] = 0x0000000000007444; 687 688 /* For global context generate memory fault on VPU */ 689 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID) 690 cd[0] |= IVPU_MMU_CD_0_A; 691 } else { 692 memset(cd, 0, sizeof(cd)); 693 } 694 695 WRITE_ONCE(entry[1], cd[1]); 696 WRITE_ONCE(entry[2], cd[2]); 697 WRITE_ONCE(entry[3], cd[3]); 698 WRITE_ONCE(entry[0], cd[0]); 699 700 clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE); 701 702 ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n", 703 cd_dma ? "write" : "clear", ssid, &cd_dma, cd[0], cd[1], cd[2], cd[3]); 704 705 mutex_lock(&mmu->lock); 706 if (!mmu->on) 707 goto unlock; 708 709 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); 710 if (ret) 711 goto unlock; 712 713 ret = ivpu_mmu_cmdq_sync(vdev); 714 unlock: 715 mutex_unlock(&mmu->lock); 716 return ret; 717 } 718 719 static int ivpu_mmu_cd_add_gbl(struct ivpu_device *vdev) 720 { 721 int ret; 722 723 ret = ivpu_mmu_cd_add(vdev, 0, vdev->gctx.pgtable.pgd_dma); 724 if (ret) 725 ivpu_err(vdev, "Failed to add global CD entry: %d\n", ret); 726 727 return ret; 728 } 729 730 static int ivpu_mmu_cd_add_user(struct ivpu_device *vdev, u32 ssid, dma_addr_t cd_dma) 731 { 732 int ret; 733 734 if (ssid == 0) { 735 ivpu_err(vdev, "Invalid SSID: %u\n", ssid); 736 return -EINVAL; 737 } 738 739 ret = ivpu_mmu_cd_add(vdev, ssid, cd_dma); 740 if (ret) 741 ivpu_err(vdev, "Failed to add CD entry SSID=%u: %d\n", ssid, ret); 742 743 return ret; 744 } 745 746 int ivpu_mmu_init(struct ivpu_device *vdev) 747 { 748 struct ivpu_mmu_info *mmu = vdev->mmu; 749 int ret; 750 751 ivpu_dbg(vdev, MMU, "Init..\n"); 752 753 drmm_mutex_init(&vdev->drm, &mmu->lock); 754 ivpu_mmu_config_check(vdev); 755 756 ret = ivpu_mmu_structs_alloc(vdev); 757 if (ret) 758 return ret; 759 760 ret = ivpu_mmu_strtab_init(vdev); 761 if (ret) { 762 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret); 763 return ret; 764 } 765 766 ret = ivpu_mmu_cd_add_gbl(vdev); 767 if (ret) { 768 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret); 769 return ret; 770 } 771 772 ret = ivpu_mmu_enable(vdev); 773 if (ret) { 774 ivpu_err(vdev, "Failed to resume MMU: %d\n", ret); 775 return ret; 776 } 777 778 ivpu_dbg(vdev, MMU, "Init done\n"); 779 780 return 0; 781 } 782 783 int ivpu_mmu_enable(struct ivpu_device *vdev) 784 { 785 struct ivpu_mmu_info *mmu = vdev->mmu; 786 int ret; 787 788 mutex_lock(&mmu->lock); 789 790 mmu->on = true; 791 792 ret = ivpu_mmu_reset(vdev); 793 if (ret) { 794 ivpu_err(vdev, "Failed to reset MMU: %d\n", ret); 795 goto err; 796 } 797 798 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev); 799 if (ret) 800 goto err; 801 802 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev); 803 if (ret) 804 goto err; 805 806 ret = ivpu_mmu_cmdq_sync(vdev); 807 if (ret) 808 goto err; 809 810 mutex_unlock(&mmu->lock); 811 812 return 0; 813 err: 814 mmu->on = false; 815 mutex_unlock(&mmu->lock); 816 return ret; 817 } 818 819 void ivpu_mmu_disable(struct ivpu_device *vdev) 820 { 821 struct ivpu_mmu_info *mmu = vdev->mmu; 822 823 mutex_lock(&mmu->lock); 824 mmu->on = false; 825 mutex_unlock(&mmu->lock); 826 } 827 828 static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event) 829 { 830 u32 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]); 831 u32 op = FIELD_GET(IVPU_MMU_EVT_OP_MASK, event[0]); 832 u64 fetch_addr = ((u64)event[7]) << 32 | event[6]; 833 u64 in_addr = ((u64)event[5]) << 32 | event[4]; 834 u32 sid = event[1]; 835 836 ivpu_err(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch addr: 0x%llx\n", 837 op, ivpu_mmu_event_to_str(op), ssid, sid, event[2], event[3], in_addr, fetch_addr); 838 } 839 840 static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev) 841 { 842 struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq; 843 u32 idx = IVPU_MMU_Q_IDX(evtq->cons); 844 u32 *evt = evtq->base + (idx * IVPU_MMU_EVTQ_CMD_SIZE); 845 846 evtq->prod = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC); 847 if (!CIRC_CNT(IVPU_MMU_Q_IDX(evtq->prod), IVPU_MMU_Q_IDX(evtq->cons), IVPU_MMU_Q_COUNT)) 848 return NULL; 849 850 clflush_cache_range(evt, IVPU_MMU_EVTQ_CMD_SIZE); 851 852 evtq->cons = (evtq->cons + 1) & IVPU_MMU_Q_WRAP_MASK; 853 REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, evtq->cons); 854 855 return evt; 856 } 857 858 void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev) 859 { 860 bool schedule_recovery = false; 861 u32 *event; 862 u32 ssid; 863 864 ivpu_dbg(vdev, IRQ, "MMU event queue\n"); 865 866 while ((event = ivpu_mmu_get_event(vdev)) != NULL) { 867 ivpu_mmu_dump_event(vdev, event); 868 869 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]); 870 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID) 871 schedule_recovery = true; 872 else 873 ivpu_mmu_user_context_mark_invalid(vdev, ssid); 874 } 875 876 if (schedule_recovery) 877 ivpu_pm_schedule_recovery(vdev); 878 } 879 880 void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev) 881 { 882 u32 gerror_val, gerrorn_val, active; 883 884 ivpu_dbg(vdev, IRQ, "MMU error\n"); 885 886 gerror_val = REGV_RD32(IVPU_MMU_REG_GERROR); 887 gerrorn_val = REGV_RD32(IVPU_MMU_REG_GERRORN); 888 889 active = gerror_val ^ gerrorn_val; 890 if (!(active & IVPU_MMU_GERROR_ERR_MASK)) 891 return; 892 893 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_ABT, active)) 894 ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n"); 895 896 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT, active)) 897 ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n"); 898 899 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT, active)) 900 ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n"); 901 902 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT, active)) 903 ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n"); 904 905 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT, active)) 906 ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n"); 907 908 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT, active)) 909 ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n"); 910 911 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, CMDQ, active)) 912 ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n"); 913 914 REGV_WR32(IVPU_MMU_REG_GERRORN, gerror_val); 915 } 916 917 int ivpu_mmu_set_pgtable(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable) 918 { 919 return ivpu_mmu_cd_add_user(vdev, ssid, pgtable->pgd_dma); 920 } 921 922 void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid) 923 { 924 ivpu_mmu_cd_add_user(vdev, ssid, 0); /* 0 will clear CD entry */ 925 } 926