1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2021-2024 NVIDIA CORPORATION & AFFILIATES. */ 3 4 #define dev_fmt(fmt) "tegra241_cmdqv: " fmt 5 6 #include <linux/acpi.h> 7 #include <linux/debugfs.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/interrupt.h> 10 #include <linux/iommu.h> 11 #include <linux/iommufd.h> 12 #include <linux/iopoll.h> 13 #include <uapi/linux/iommufd.h> 14 15 #include <acpi/acpixf.h> 16 17 #include "arm-smmu-v3.h" 18 19 /* CMDQV register page base and size defines */ 20 #define TEGRA241_CMDQV_CONFIG_BASE (0) 21 #define TEGRA241_CMDQV_CONFIG_SIZE (SZ_64K) 22 #define TEGRA241_VCMDQ_PAGE0_BASE (TEGRA241_CMDQV_CONFIG_BASE + SZ_64K) 23 #define TEGRA241_VCMDQ_PAGE1_BASE (TEGRA241_VCMDQ_PAGE0_BASE + SZ_64K) 24 #define TEGRA241_VINTF_PAGE_BASE (TEGRA241_VCMDQ_PAGE1_BASE + SZ_64K) 25 26 /* CMDQV global base regs */ 27 #define TEGRA241_CMDQV_CONFIG 0x0000 28 #define CMDQV_EN BIT(0) 29 30 #define TEGRA241_CMDQV_PARAM 0x0004 31 #define CMDQV_NUM_SID_PER_VM_LOG2 GENMASK(15, 12) 32 #define CMDQV_NUM_VINTF_LOG2 GENMASK(11, 8) 33 #define CMDQV_NUM_VCMDQ_LOG2 GENMASK(7, 4) 34 #define CMDQV_VER GENMASK(3, 0) 35 36 #define TEGRA241_CMDQV_STATUS 0x0008 37 #define CMDQV_ENABLED BIT(0) 38 39 #define TEGRA241_CMDQV_VINTF_ERR_MAP 0x0014 40 #define TEGRA241_CMDQV_VINTF_INT_MASK 0x001C 41 #define TEGRA241_CMDQV_CMDQ_ERR_MAP(m) (0x0024 + 0x4*(m)) 42 43 #define TEGRA241_CMDQV_CMDQ_ALLOC(q) (0x0200 + 0x4*(q)) 44 #define CMDQV_CMDQ_ALLOC_VINTF GENMASK(20, 15) 45 #define CMDQV_CMDQ_ALLOC_LVCMDQ GENMASK(7, 1) 46 #define CMDQV_CMDQ_ALLOCATED BIT(0) 47 48 /* VINTF base regs */ 49 #define TEGRA241_VINTF(v) (0x1000 + 0x100*(v)) 50 51 #define TEGRA241_VINTF_CONFIG 0x0000 52 #define VINTF_HYP_OWN BIT(17) 53 #define VINTF_VMID GENMASK(16, 1) 54 #define VINTF_EN BIT(0) 55 56 #define TEGRA241_VINTF_STATUS 0x0004 57 #define VINTF_STATUS GENMASK(3, 1) 58 #define VINTF_ENABLED BIT(0) 59 60 #define TEGRA241_VINTF_SID_MATCH(s) (0x0040 + 0x4*(s)) 61 #define TEGRA241_VINTF_SID_REPLACE(s) (0x0080 + 0x4*(s)) 62 63 #define TEGRA241_VINTF_LVCMDQ_ERR_MAP_64(m) \ 64 (0x00C0 + 0x8*(m)) 65 #define LVCMDQ_ERR_MAP_NUM_64 2 66 67 /* VCMDQ base regs */ 68 /* -- PAGE0 -- */ 69 #define TEGRA241_VCMDQ_PAGE0(q) (TEGRA241_VCMDQ_PAGE0_BASE + 0x80*(q)) 70 71 #define TEGRA241_VCMDQ_CONS 0x00000 72 #define VCMDQ_CONS_ERR GENMASK(30, 24) 73 74 #define TEGRA241_VCMDQ_PROD 0x00004 75 76 #define TEGRA241_VCMDQ_CONFIG 0x00008 77 #define VCMDQ_EN BIT(0) 78 79 #define TEGRA241_VCMDQ_STATUS 0x0000C 80 #define VCMDQ_ENABLED BIT(0) 81 82 #define TEGRA241_VCMDQ_GERROR 0x00010 83 #define TEGRA241_VCMDQ_GERRORN 0x00014 84 85 /* -- PAGE1 -- */ 86 #define TEGRA241_VCMDQ_PAGE1(q) (TEGRA241_VCMDQ_PAGE1_BASE + 0x80*(q)) 87 #define VCMDQ_ADDR GENMASK(47, 5) 88 #define VCMDQ_LOG2SIZE GENMASK(4, 0) 89 90 #define TEGRA241_VCMDQ_BASE 0x00000 91 #define TEGRA241_VCMDQ_CONS_INDX_BASE 0x00008 92 93 /* VINTF logical-VCMDQ pages */ 94 #define TEGRA241_VINTFi_PAGE0(i) (TEGRA241_VINTF_PAGE_BASE + SZ_128K*(i)) 95 #define TEGRA241_VINTFi_PAGE1(i) (TEGRA241_VINTFi_PAGE0(i) + SZ_64K) 96 #define TEGRA241_VINTFi_LVCMDQ_PAGE0(i, q) \ 97 (TEGRA241_VINTFi_PAGE0(i) + 0x80*(q)) 98 #define TEGRA241_VINTFi_LVCMDQ_PAGE1(i, q) \ 99 (TEGRA241_VINTFi_PAGE1(i) + 0x80*(q)) 100 101 /* MMIO helpers */ 102 #define REG_CMDQV(_cmdqv, _regname) \ 103 ((_cmdqv)->base + TEGRA241_CMDQV_##_regname) 104 #define REG_VINTF(_vintf, _regname) \ 105 ((_vintf)->base + TEGRA241_VINTF_##_regname) 106 #define REG_VCMDQ_PAGE0(_vcmdq, _regname) \ 107 ((_vcmdq)->page0 + TEGRA241_VCMDQ_##_regname) 108 #define REG_VCMDQ_PAGE1(_vcmdq, _regname) \ 109 ((_vcmdq)->page1 + TEGRA241_VCMDQ_##_regname) 110 111 112 static bool disable_cmdqv; 113 module_param(disable_cmdqv, bool, 0444); 114 MODULE_PARM_DESC(disable_cmdqv, 115 "This allows to disable CMDQV HW and use default SMMU internal CMDQ."); 116 117 static bool bypass_vcmdq; 118 module_param(bypass_vcmdq, bool, 0444); 119 MODULE_PARM_DESC(bypass_vcmdq, 120 "This allows to bypass VCMDQ for debugging use or perf comparison."); 121 122 /** 123 * struct tegra241_vcmdq - Virtual Command Queue 124 * @core: Embedded iommufd_hw_queue structure 125 * @idx: Global index in the CMDQV 126 * @lidx: Local index in the VINTF 127 * @enabled: Enable status 128 * @cmdqv: Parent CMDQV pointer 129 * @vintf: Parent VINTF pointer 130 * @prev: Previous LVCMDQ to depend on 131 * @cmdq: Command Queue struct 132 * @page0: MMIO Page0 base address 133 * @page1: MMIO Page1 base address 134 */ 135 struct tegra241_vcmdq { 136 struct iommufd_hw_queue core; 137 138 u16 idx; 139 u16 lidx; 140 141 bool enabled; 142 143 struct tegra241_cmdqv *cmdqv; 144 struct tegra241_vintf *vintf; 145 struct tegra241_vcmdq *prev; 146 struct arm_smmu_cmdq cmdq; 147 148 void __iomem *page0; 149 void __iomem *page1; 150 }; 151 #define hw_queue_to_vcmdq(v) container_of(v, struct tegra241_vcmdq, core) 152 153 /** 154 * struct tegra241_vintf - Virtual Interface 155 * @vsmmu: Embedded arm_vsmmu structure 156 * @idx: Global index in the CMDQV 157 * @enabled: Enable status 158 * @hyp_own: Owned by hypervisor (in-kernel) 159 * @cmdqv: Parent CMDQV pointer 160 * @lvcmdqs: List of logical VCMDQ pointers 161 * @lvcmdq_mutex: Lock to serialize user-allocated lvcmdqs 162 * @base: MMIO base address 163 * @mmap_offset: Offset argument for mmap() syscall 164 * @sids: Stream ID mapping resources 165 */ 166 struct tegra241_vintf { 167 struct arm_vsmmu vsmmu; 168 169 u16 idx; 170 171 bool enabled; 172 bool hyp_own; 173 174 struct tegra241_cmdqv *cmdqv; 175 struct tegra241_vcmdq **lvcmdqs; 176 struct mutex lvcmdq_mutex; /* user space race */ 177 178 void __iomem *base; 179 unsigned long mmap_offset; 180 181 struct ida sids; 182 }; 183 #define viommu_to_vintf(v) container_of(v, struct tegra241_vintf, vsmmu.core) 184 185 /** 186 * struct tegra241_vintf_sid - Virtual Interface Stream ID Mapping 187 * @core: Embedded iommufd_vdevice structure, holding virtual Stream ID 188 * @vintf: Parent VINTF pointer 189 * @sid: Physical Stream ID 190 * @idx: Mapping index in the VINTF 191 */ 192 struct tegra241_vintf_sid { 193 struct iommufd_vdevice core; 194 struct tegra241_vintf *vintf; 195 u32 sid; 196 u8 idx; 197 }; 198 #define vdev_to_vsid(v) container_of(v, struct tegra241_vintf_sid, core) 199 200 /** 201 * struct tegra241_cmdqv - CMDQ-V for SMMUv3 202 * @smmu: SMMUv3 device 203 * @dev: CMDQV device 204 * @base: MMIO base address 205 * @base_phys: MMIO physical base address, for mmap 206 * @irq: IRQ number 207 * @num_vintfs: Total number of VINTFs 208 * @num_vcmdqs: Total number of VCMDQs 209 * @num_lvcmdqs_per_vintf: Number of logical VCMDQs per VINTF 210 * @num_sids_per_vintf: Total number of SID mappings per VINTF 211 * @vintf_ids: VINTF id allocator 212 * @vintfs: List of VINTFs 213 */ 214 struct tegra241_cmdqv { 215 struct arm_smmu_device smmu; 216 struct device *dev; 217 218 void __iomem *base; 219 phys_addr_t base_phys; 220 int irq; 221 222 /* CMDQV Hardware Params */ 223 u16 num_vintfs; 224 u16 num_vcmdqs; 225 u16 num_lvcmdqs_per_vintf; 226 u16 num_sids_per_vintf; 227 228 struct ida vintf_ids; 229 230 struct tegra241_vintf **vintfs; 231 }; 232 233 /* Config and Polling Helpers */ 234 235 static inline int tegra241_cmdqv_write_config(struct tegra241_cmdqv *cmdqv, 236 void __iomem *addr_config, 237 void __iomem *addr_status, 238 u32 regval, const char *header, 239 bool *out_enabled) 240 { 241 bool en = regval & BIT(0); 242 int ret; 243 244 writel(regval, addr_config); 245 ret = readl_poll_timeout(addr_status, regval, 246 en ? regval & BIT(0) : !(regval & BIT(0)), 247 1, ARM_SMMU_POLL_TIMEOUT_US); 248 if (ret) 249 dev_err(cmdqv->dev, "%sfailed to %sable, STATUS=0x%08X\n", 250 header, en ? "en" : "dis", regval); 251 if (out_enabled) 252 WRITE_ONCE(*out_enabled, regval & BIT(0)); 253 return ret; 254 } 255 256 static inline int cmdqv_write_config(struct tegra241_cmdqv *cmdqv, u32 regval) 257 { 258 return tegra241_cmdqv_write_config(cmdqv, 259 REG_CMDQV(cmdqv, CONFIG), 260 REG_CMDQV(cmdqv, STATUS), 261 regval, "CMDQV: ", NULL); 262 } 263 264 static inline int vintf_write_config(struct tegra241_vintf *vintf, u32 regval) 265 { 266 char header[16]; 267 268 snprintf(header, 16, "VINTF%u: ", vintf->idx); 269 return tegra241_cmdqv_write_config(vintf->cmdqv, 270 REG_VINTF(vintf, CONFIG), 271 REG_VINTF(vintf, STATUS), 272 regval, header, &vintf->enabled); 273 } 274 275 static inline char *lvcmdq_error_header(struct tegra241_vcmdq *vcmdq, 276 char *header, int hlen) 277 { 278 WARN_ON(hlen < 64); 279 if (WARN_ON(!vcmdq->vintf)) 280 return ""; 281 snprintf(header, hlen, "VINTF%u: VCMDQ%u/LVCMDQ%u: ", 282 vcmdq->vintf->idx, vcmdq->idx, vcmdq->lidx); 283 return header; 284 } 285 286 static inline int vcmdq_write_config(struct tegra241_vcmdq *vcmdq, u32 regval) 287 { 288 char header[64], *h = lvcmdq_error_header(vcmdq, header, 64); 289 290 return tegra241_cmdqv_write_config(vcmdq->cmdqv, 291 REG_VCMDQ_PAGE0(vcmdq, CONFIG), 292 REG_VCMDQ_PAGE0(vcmdq, STATUS), 293 regval, h, &vcmdq->enabled); 294 } 295 296 /* ISR Functions */ 297 298 static void tegra241_vintf_user_handle_error(struct tegra241_vintf *vintf) 299 { 300 struct iommufd_viommu *viommu = &vintf->vsmmu.core; 301 struct iommu_vevent_tegra241_cmdqv vevent_data; 302 int i; 303 304 for (i = 0; i < LVCMDQ_ERR_MAP_NUM_64; i++) 305 vevent_data.lvcmdq_err_map[i] = 306 readq_relaxed(REG_VINTF(vintf, LVCMDQ_ERR_MAP_64(i))); 307 308 iommufd_viommu_report_event(viommu, IOMMU_VEVENTQ_TYPE_TEGRA241_CMDQV, 309 &vevent_data, sizeof(vevent_data)); 310 } 311 312 static void tegra241_vintf0_handle_error(struct tegra241_vintf *vintf) 313 { 314 int i; 315 316 for (i = 0; i < LVCMDQ_ERR_MAP_NUM_64; i++) { 317 u64 map = readq_relaxed(REG_VINTF(vintf, LVCMDQ_ERR_MAP_64(i))); 318 319 while (map) { 320 unsigned long lidx = __ffs64(map); 321 struct tegra241_vcmdq *vcmdq = vintf->lvcmdqs[lidx]; 322 u32 gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR)); 323 324 __arm_smmu_cmdq_skip_err(&vintf->cmdqv->smmu, &vcmdq->cmdq); 325 writel(gerror, REG_VCMDQ_PAGE0(vcmdq, GERRORN)); 326 map &= ~BIT_ULL(lidx); 327 } 328 } 329 } 330 331 static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid) 332 { 333 struct tegra241_cmdqv *cmdqv = (struct tegra241_cmdqv *)devid; 334 void __iomem *reg_vintf_map = REG_CMDQV(cmdqv, VINTF_ERR_MAP); 335 char err_str[256]; 336 u64 vintf_map; 337 338 /* Use readl_relaxed() as register addresses are not 64-bit aligned */ 339 vintf_map = (u64)readl_relaxed(reg_vintf_map + 0x4) << 32 | 340 (u64)readl_relaxed(reg_vintf_map); 341 342 snprintf(err_str, sizeof(err_str), 343 "vintf_map: %016llx, vcmdq_map %08x:%08x:%08x:%08x", vintf_map, 344 readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(3))), 345 readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(2))), 346 readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(1))), 347 readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(0)))); 348 349 dev_warn(cmdqv->dev, "unexpected error reported. %s\n", err_str); 350 351 /* Handle VINTF0 and its LVCMDQs */ 352 if (vintf_map & BIT_ULL(0)) { 353 tegra241_vintf0_handle_error(cmdqv->vintfs[0]); 354 vintf_map &= ~BIT_ULL(0); 355 } 356 357 /* Handle other user VINTFs and their LVCMDQs */ 358 while (vintf_map) { 359 unsigned long idx = __ffs64(vintf_map); 360 361 tegra241_vintf_user_handle_error(cmdqv->vintfs[idx]); 362 vintf_map &= ~BIT_ULL(idx); 363 } 364 365 return IRQ_HANDLED; 366 } 367 368 /* Command Queue Function */ 369 370 static bool tegra241_guest_vcmdq_supports_cmd(struct arm_smmu_cmdq_ent *ent) 371 { 372 switch (ent->opcode) { 373 case CMDQ_OP_TLBI_NH_ASID: 374 case CMDQ_OP_TLBI_NH_VA: 375 case CMDQ_OP_ATC_INV: 376 return true; 377 default: 378 return false; 379 } 380 } 381 382 static struct arm_smmu_cmdq * 383 tegra241_cmdqv_get_cmdq(struct arm_smmu_device *smmu, 384 struct arm_smmu_cmdq_ent *ent) 385 { 386 struct tegra241_cmdqv *cmdqv = 387 container_of(smmu, struct tegra241_cmdqv, smmu); 388 struct tegra241_vintf *vintf = cmdqv->vintfs[0]; 389 struct tegra241_vcmdq *vcmdq; 390 u16 lidx; 391 392 if (READ_ONCE(bypass_vcmdq)) 393 return NULL; 394 395 /* Use SMMU CMDQ if VINTF0 is uninitialized */ 396 if (!READ_ONCE(vintf->enabled)) 397 return NULL; 398 399 /* 400 * Select a LVCMDQ to use. Here we use a temporal solution to 401 * balance out traffic on cmdq issuing: each cmdq has its own 402 * lock, if all cpus issue cmdlist using the same cmdq, only 403 * one CPU at a time can enter the process, while the others 404 * will be spinning at the same lock. 405 */ 406 lidx = raw_smp_processor_id() % cmdqv->num_lvcmdqs_per_vintf; 407 vcmdq = vintf->lvcmdqs[lidx]; 408 if (!vcmdq || !READ_ONCE(vcmdq->enabled)) 409 return NULL; 410 411 /* Unsupported CMD goes for smmu->cmdq pathway */ 412 if (!arm_smmu_cmdq_supports_cmd(&vcmdq->cmdq, ent)) 413 return NULL; 414 return &vcmdq->cmdq; 415 } 416 417 /* HW Reset Functions */ 418 419 /* 420 * When a guest-owned VCMDQ is disabled, if the guest did not enqueue a CMD_SYNC 421 * following an ATC_INV command at the end of the guest queue while this ATC_INV 422 * is timed out, the TIMEOUT will not be reported until this VCMDQ gets assigned 423 * to the next VM, which will be a false alarm potentially causing some unwanted 424 * behavior in the new VM. Thus, a guest-owned VCMDQ must flush the TIMEOUT when 425 * it gets disabled. This can be done by just issuing a CMD_SYNC to SMMU CMDQ. 426 */ 427 static void tegra241_vcmdq_hw_flush_timeout(struct tegra241_vcmdq *vcmdq) 428 { 429 struct arm_smmu_device *smmu = &vcmdq->cmdqv->smmu; 430 u64 cmd_sync[CMDQ_ENT_DWORDS] = {}; 431 432 cmd_sync[0] = FIELD_PREP(CMDQ_0_OP, CMDQ_OP_CMD_SYNC) | 433 FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_NONE); 434 435 /* 436 * It does not hurt to insert another CMD_SYNC, taking advantage of the 437 * arm_smmu_cmdq_issue_cmdlist() that waits for the CMD_SYNC completion. 438 */ 439 arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, cmd_sync, 1, true); 440 } 441 442 /* This function is for LVCMDQ, so @vcmdq must not be unmapped yet */ 443 static void tegra241_vcmdq_hw_deinit(struct tegra241_vcmdq *vcmdq) 444 { 445 char header[64], *h = lvcmdq_error_header(vcmdq, header, 64); 446 u32 gerrorn, gerror; 447 448 if (vcmdq_write_config(vcmdq, 0)) { 449 dev_err(vcmdq->cmdqv->dev, 450 "%sGERRORN=0x%X, GERROR=0x%X, CONS=0x%X\n", h, 451 readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERRORN)), 452 readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR)), 453 readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, CONS))); 454 } 455 tegra241_vcmdq_hw_flush_timeout(vcmdq); 456 457 writel_relaxed(0, REG_VCMDQ_PAGE0(vcmdq, PROD)); 458 writel_relaxed(0, REG_VCMDQ_PAGE0(vcmdq, CONS)); 459 writeq_relaxed(0, REG_VCMDQ_PAGE1(vcmdq, BASE)); 460 writeq_relaxed(0, REG_VCMDQ_PAGE1(vcmdq, CONS_INDX_BASE)); 461 462 gerrorn = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERRORN)); 463 gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR)); 464 if (gerror != gerrorn) { 465 dev_warn(vcmdq->cmdqv->dev, 466 "%suncleared error detected, resetting\n", h); 467 writel(gerror, REG_VCMDQ_PAGE0(vcmdq, GERRORN)); 468 } 469 470 dev_dbg(vcmdq->cmdqv->dev, "%sdeinited\n", h); 471 } 472 473 /* This function is for LVCMDQ, so @vcmdq must be mapped prior */ 474 static int tegra241_vcmdq_hw_init(struct tegra241_vcmdq *vcmdq) 475 { 476 char header[64], *h = lvcmdq_error_header(vcmdq, header, 64); 477 int ret; 478 479 /* Reset VCMDQ */ 480 tegra241_vcmdq_hw_deinit(vcmdq); 481 482 /* Configure and enable VCMDQ */ 483 writeq_relaxed(vcmdq->cmdq.q.q_base, REG_VCMDQ_PAGE1(vcmdq, BASE)); 484 485 ret = vcmdq_write_config(vcmdq, VCMDQ_EN); 486 if (ret) { 487 dev_err(vcmdq->cmdqv->dev, 488 "%sGERRORN=0x%X, GERROR=0x%X, CONS=0x%X\n", h, 489 readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERRORN)), 490 readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR)), 491 readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, CONS))); 492 return ret; 493 } 494 495 dev_dbg(vcmdq->cmdqv->dev, "%sinited\n", h); 496 return 0; 497 } 498 499 /* Unmap a global VCMDQ from the pre-assigned LVCMDQ */ 500 static void tegra241_vcmdq_unmap_lvcmdq(struct tegra241_vcmdq *vcmdq) 501 { 502 u32 regval = readl(REG_CMDQV(vcmdq->cmdqv, CMDQ_ALLOC(vcmdq->idx))); 503 char header[64], *h = lvcmdq_error_header(vcmdq, header, 64); 504 505 writel(regval & ~CMDQV_CMDQ_ALLOCATED, 506 REG_CMDQV(vcmdq->cmdqv, CMDQ_ALLOC(vcmdq->idx))); 507 dev_dbg(vcmdq->cmdqv->dev, "%sunmapped\n", h); 508 } 509 510 static void tegra241_vintf_hw_deinit(struct tegra241_vintf *vintf) 511 { 512 u16 lidx = vintf->cmdqv->num_lvcmdqs_per_vintf; 513 int sidx; 514 515 /* HW requires to unmap LVCMDQs in descending order */ 516 while (lidx--) { 517 if (vintf->lvcmdqs && vintf->lvcmdqs[lidx]) { 518 tegra241_vcmdq_hw_deinit(vintf->lvcmdqs[lidx]); 519 tegra241_vcmdq_unmap_lvcmdq(vintf->lvcmdqs[lidx]); 520 } 521 } 522 vintf_write_config(vintf, 0); 523 for (sidx = 0; sidx < vintf->cmdqv->num_sids_per_vintf; sidx++) { 524 writel(0, REG_VINTF(vintf, SID_MATCH(sidx))); 525 writel(0, REG_VINTF(vintf, SID_REPLACE(sidx))); 526 } 527 } 528 529 /* Map a global VCMDQ to the pre-assigned LVCMDQ */ 530 static void tegra241_vcmdq_map_lvcmdq(struct tegra241_vcmdq *vcmdq) 531 { 532 u32 regval = readl(REG_CMDQV(vcmdq->cmdqv, CMDQ_ALLOC(vcmdq->idx))); 533 char header[64], *h = lvcmdq_error_header(vcmdq, header, 64); 534 535 writel(regval | CMDQV_CMDQ_ALLOCATED, 536 REG_CMDQV(vcmdq->cmdqv, CMDQ_ALLOC(vcmdq->idx))); 537 dev_dbg(vcmdq->cmdqv->dev, "%smapped\n", h); 538 } 539 540 static int tegra241_vintf_hw_init(struct tegra241_vintf *vintf, bool hyp_own) 541 { 542 u32 regval; 543 u16 lidx; 544 int ret; 545 546 /* Reset VINTF */ 547 tegra241_vintf_hw_deinit(vintf); 548 549 /* Configure and enable VINTF */ 550 /* 551 * Note that HYP_OWN bit is wired to zero when running in guest kernel, 552 * whether enabling it here or not, as !HYP_OWN cmdq HWs only support a 553 * restricted set of supported commands. 554 */ 555 regval = FIELD_PREP(VINTF_HYP_OWN, hyp_own) | 556 FIELD_PREP(VINTF_VMID, vintf->vsmmu.vmid); 557 writel(regval, REG_VINTF(vintf, CONFIG)); 558 559 ret = vintf_write_config(vintf, regval | VINTF_EN); 560 if (ret) 561 return ret; 562 /* 563 * As being mentioned above, HYP_OWN bit is wired to zero for a guest 564 * kernel, so read it back from HW to ensure that reflects in hyp_own 565 */ 566 vintf->hyp_own = !!(VINTF_HYP_OWN & readl(REG_VINTF(vintf, CONFIG))); 567 568 /* HW requires to map LVCMDQs in ascending order */ 569 for (lidx = 0; lidx < vintf->cmdqv->num_lvcmdqs_per_vintf; lidx++) { 570 if (vintf->lvcmdqs && vintf->lvcmdqs[lidx]) { 571 tegra241_vcmdq_map_lvcmdq(vintf->lvcmdqs[lidx]); 572 ret = tegra241_vcmdq_hw_init(vintf->lvcmdqs[lidx]); 573 if (ret) { 574 tegra241_vintf_hw_deinit(vintf); 575 return ret; 576 } 577 } 578 } 579 580 return 0; 581 } 582 583 static int tegra241_cmdqv_hw_reset(struct arm_smmu_device *smmu) 584 { 585 struct tegra241_cmdqv *cmdqv = 586 container_of(smmu, struct tegra241_cmdqv, smmu); 587 u16 qidx, lidx, idx; 588 u32 regval; 589 int ret; 590 591 /* Reset CMDQV */ 592 regval = readl_relaxed(REG_CMDQV(cmdqv, CONFIG)); 593 ret = cmdqv_write_config(cmdqv, regval & ~CMDQV_EN); 594 if (ret) 595 return ret; 596 ret = cmdqv_write_config(cmdqv, regval | CMDQV_EN); 597 if (ret) 598 return ret; 599 600 /* Assign preallocated global VCMDQs to each VINTF as LVCMDQs */ 601 for (idx = 0, qidx = 0; idx < cmdqv->num_vintfs; idx++) { 602 for (lidx = 0; lidx < cmdqv->num_lvcmdqs_per_vintf; lidx++) { 603 regval = FIELD_PREP(CMDQV_CMDQ_ALLOC_VINTF, idx); 604 regval |= FIELD_PREP(CMDQV_CMDQ_ALLOC_LVCMDQ, lidx); 605 writel_relaxed(regval, 606 REG_CMDQV(cmdqv, CMDQ_ALLOC(qidx++))); 607 } 608 } 609 610 return tegra241_vintf_hw_init(cmdqv->vintfs[0], true); 611 } 612 613 /* VCMDQ Resource Helpers */ 614 615 static int tegra241_vcmdq_alloc_smmu_cmdq(struct tegra241_vcmdq *vcmdq) 616 { 617 struct arm_smmu_device *smmu = &vcmdq->cmdqv->smmu; 618 struct arm_smmu_cmdq *cmdq = &vcmdq->cmdq; 619 struct arm_smmu_queue *q = &cmdq->q; 620 char name[16]; 621 u32 regval; 622 int ret; 623 624 snprintf(name, 16, "vcmdq%u", vcmdq->idx); 625 626 /* Cap queue size to SMMU's IDR1.CMDQS and ensure natural alignment */ 627 regval = readl_relaxed(smmu->base + ARM_SMMU_IDR1); 628 q->llq.max_n_shift = 629 min_t(u32, CMDQ_MAX_SZ_SHIFT, FIELD_GET(IDR1_CMDQS, regval)); 630 631 /* Use the common helper to init the VCMDQ, and then... */ 632 ret = arm_smmu_init_one_queue(smmu, q, vcmdq->page0, 633 TEGRA241_VCMDQ_PROD, TEGRA241_VCMDQ_CONS, 634 CMDQ_ENT_DWORDS, name); 635 if (ret) 636 return ret; 637 638 /* ...override q_base to write VCMDQ_BASE registers */ 639 q->q_base = q->base_dma & VCMDQ_ADDR; 640 q->q_base |= FIELD_PREP(VCMDQ_LOG2SIZE, q->llq.max_n_shift); 641 642 if (!vcmdq->vintf->hyp_own) 643 cmdq->supports_cmd = tegra241_guest_vcmdq_supports_cmd; 644 645 return arm_smmu_cmdq_init(smmu, cmdq); 646 } 647 648 /* VINTF Logical VCMDQ Resource Helpers */ 649 650 static void tegra241_vintf_deinit_lvcmdq(struct tegra241_vintf *vintf, u16 lidx) 651 { 652 vintf->lvcmdqs[lidx] = NULL; 653 } 654 655 static int tegra241_vintf_init_lvcmdq(struct tegra241_vintf *vintf, u16 lidx, 656 struct tegra241_vcmdq *vcmdq) 657 { 658 struct tegra241_cmdqv *cmdqv = vintf->cmdqv; 659 u16 idx = vintf->idx; 660 661 vcmdq->idx = idx * cmdqv->num_lvcmdqs_per_vintf + lidx; 662 vcmdq->lidx = lidx; 663 vcmdq->cmdqv = cmdqv; 664 vcmdq->vintf = vintf; 665 vcmdq->page0 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE0(idx, lidx); 666 vcmdq->page1 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE1(idx, lidx); 667 668 vintf->lvcmdqs[lidx] = vcmdq; 669 return 0; 670 } 671 672 static void tegra241_vintf_free_lvcmdq(struct tegra241_vintf *vintf, u16 lidx) 673 { 674 struct tegra241_vcmdq *vcmdq = vintf->lvcmdqs[lidx]; 675 char header[64]; 676 677 /* Note that the lvcmdq queue memory space is managed by devres */ 678 679 tegra241_vintf_deinit_lvcmdq(vintf, lidx); 680 681 dev_dbg(vintf->cmdqv->dev, 682 "%sdeallocated\n", lvcmdq_error_header(vcmdq, header, 64)); 683 /* Guest-owned VCMDQ is free-ed with hw_queue by iommufd core */ 684 if (vcmdq->vintf->hyp_own) 685 kfree(vcmdq); 686 } 687 688 static struct tegra241_vcmdq * 689 tegra241_vintf_alloc_lvcmdq(struct tegra241_vintf *vintf, u16 lidx) 690 { 691 struct tegra241_cmdqv *cmdqv = vintf->cmdqv; 692 struct tegra241_vcmdq *vcmdq; 693 char header[64]; 694 int ret; 695 696 vcmdq = kzalloc(sizeof(*vcmdq), GFP_KERNEL); 697 if (!vcmdq) 698 return ERR_PTR(-ENOMEM); 699 700 ret = tegra241_vintf_init_lvcmdq(vintf, lidx, vcmdq); 701 if (ret) 702 goto free_vcmdq; 703 704 /* Build an arm_smmu_cmdq for each LVCMDQ */ 705 ret = tegra241_vcmdq_alloc_smmu_cmdq(vcmdq); 706 if (ret) 707 goto deinit_lvcmdq; 708 709 dev_dbg(cmdqv->dev, 710 "%sallocated\n", lvcmdq_error_header(vcmdq, header, 64)); 711 return vcmdq; 712 713 deinit_lvcmdq: 714 tegra241_vintf_deinit_lvcmdq(vintf, lidx); 715 free_vcmdq: 716 kfree(vcmdq); 717 return ERR_PTR(ret); 718 } 719 720 /* VINTF Resource Helpers */ 721 722 static void tegra241_cmdqv_deinit_vintf(struct tegra241_cmdqv *cmdqv, u16 idx) 723 { 724 kfree(cmdqv->vintfs[idx]->lvcmdqs); 725 ida_free(&cmdqv->vintf_ids, idx); 726 cmdqv->vintfs[idx] = NULL; 727 } 728 729 static int tegra241_cmdqv_init_vintf(struct tegra241_cmdqv *cmdqv, u16 max_idx, 730 struct tegra241_vintf *vintf) 731 { 732 733 u16 idx; 734 int ret; 735 736 ret = ida_alloc_max(&cmdqv->vintf_ids, max_idx, GFP_KERNEL); 737 if (ret < 0) 738 return ret; 739 idx = ret; 740 741 vintf->idx = idx; 742 vintf->cmdqv = cmdqv; 743 vintf->base = cmdqv->base + TEGRA241_VINTF(idx); 744 745 vintf->lvcmdqs = kcalloc(cmdqv->num_lvcmdqs_per_vintf, 746 sizeof(*vintf->lvcmdqs), GFP_KERNEL); 747 if (!vintf->lvcmdqs) { 748 ida_free(&cmdqv->vintf_ids, idx); 749 return -ENOMEM; 750 } 751 752 cmdqv->vintfs[idx] = vintf; 753 return ret; 754 } 755 756 /* Remove Helpers */ 757 758 static void tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv *cmdqv, u16 idx) 759 { 760 struct tegra241_vintf *vintf = cmdqv->vintfs[idx]; 761 u16 lidx; 762 763 tegra241_vintf_hw_deinit(vintf); 764 765 /* Remove LVCMDQ resources */ 766 for (lidx = 0; lidx < vintf->cmdqv->num_lvcmdqs_per_vintf; lidx++) 767 if (vintf->lvcmdqs[lidx]) 768 tegra241_vintf_free_lvcmdq(vintf, lidx); 769 770 dev_dbg(cmdqv->dev, "VINTF%u: deallocated\n", vintf->idx); 771 tegra241_cmdqv_deinit_vintf(cmdqv, idx); 772 if (!vintf->hyp_own) { 773 mutex_destroy(&vintf->lvcmdq_mutex); 774 ida_destroy(&vintf->sids); 775 /* Guest-owned VINTF is free-ed with viommu by iommufd core */ 776 } else { 777 kfree(vintf); 778 } 779 } 780 781 static void tegra241_cmdqv_remove(struct arm_smmu_device *smmu) 782 { 783 struct tegra241_cmdqv *cmdqv = 784 container_of(smmu, struct tegra241_cmdqv, smmu); 785 u16 idx; 786 787 /* Remove VINTF resources */ 788 for (idx = 0; idx < cmdqv->num_vintfs; idx++) { 789 if (cmdqv->vintfs[idx]) { 790 /* Only vintf0 should remain at this stage */ 791 WARN_ON(idx > 0); 792 tegra241_cmdqv_remove_vintf(cmdqv, idx); 793 } 794 } 795 796 /* Remove cmdqv resources */ 797 ida_destroy(&cmdqv->vintf_ids); 798 799 if (cmdqv->irq > 0) 800 free_irq(cmdqv->irq, cmdqv); 801 iounmap(cmdqv->base); 802 kfree(cmdqv->vintfs); 803 put_device(cmdqv->dev); /* smmu->impl_dev */ 804 } 805 806 static int 807 tegra241_cmdqv_init_vintf_user(struct arm_vsmmu *vsmmu, 808 const struct iommu_user_data *user_data); 809 810 static void *tegra241_cmdqv_hw_info(struct arm_smmu_device *smmu, u32 *length, 811 enum iommu_hw_info_type *type) 812 { 813 struct tegra241_cmdqv *cmdqv = 814 container_of(smmu, struct tegra241_cmdqv, smmu); 815 struct iommu_hw_info_tegra241_cmdqv *info; 816 u32 regval; 817 818 if (*type != IOMMU_HW_INFO_TYPE_TEGRA241_CMDQV) 819 return ERR_PTR(-EOPNOTSUPP); 820 821 info = kzalloc(sizeof(*info), GFP_KERNEL); 822 if (!info) 823 return ERR_PTR(-ENOMEM); 824 825 regval = readl_relaxed(REG_CMDQV(cmdqv, PARAM)); 826 info->log2vcmdqs = ilog2(cmdqv->num_lvcmdqs_per_vintf); 827 info->log2vsids = ilog2(cmdqv->num_sids_per_vintf); 828 info->version = FIELD_GET(CMDQV_VER, regval); 829 830 *length = sizeof(*info); 831 *type = IOMMU_HW_INFO_TYPE_TEGRA241_CMDQV; 832 return info; 833 } 834 835 static size_t tegra241_cmdqv_get_vintf_size(enum iommu_viommu_type viommu_type) 836 { 837 if (viommu_type != IOMMU_VIOMMU_TYPE_TEGRA241_CMDQV) 838 return 0; 839 return VIOMMU_STRUCT_SIZE(struct tegra241_vintf, vsmmu.core); 840 } 841 842 static struct arm_smmu_impl_ops tegra241_cmdqv_impl_ops = { 843 /* For in-kernel use */ 844 .get_secondary_cmdq = tegra241_cmdqv_get_cmdq, 845 .device_reset = tegra241_cmdqv_hw_reset, 846 .device_remove = tegra241_cmdqv_remove, 847 /* For user-space use */ 848 .hw_info = tegra241_cmdqv_hw_info, 849 .get_viommu_size = tegra241_cmdqv_get_vintf_size, 850 .vsmmu_init = tegra241_cmdqv_init_vintf_user, 851 }; 852 853 /* Probe Functions */ 854 855 static int tegra241_cmdqv_acpi_is_memory(struct acpi_resource *res, void *data) 856 { 857 struct resource_win win; 858 859 return !acpi_dev_resource_address_space(res, &win); 860 } 861 862 static int tegra241_cmdqv_acpi_get_irqs(struct acpi_resource *ares, void *data) 863 { 864 struct resource r; 865 int *irq = data; 866 867 if (*irq <= 0 && acpi_dev_resource_interrupt(ares, 0, &r)) 868 *irq = r.start; 869 return 1; /* No need to add resource to the list */ 870 } 871 872 static struct resource * 873 tegra241_cmdqv_find_acpi_resource(struct device *dev, int *irq) 874 { 875 struct acpi_device *adev = to_acpi_device(dev); 876 struct list_head resource_list; 877 struct resource_entry *rentry; 878 struct resource *res = NULL; 879 int ret; 880 881 INIT_LIST_HEAD(&resource_list); 882 ret = acpi_dev_get_resources(adev, &resource_list, 883 tegra241_cmdqv_acpi_is_memory, NULL); 884 if (ret < 0) { 885 dev_err(dev, "failed to get memory resource: %d\n", ret); 886 return NULL; 887 } 888 889 rentry = list_first_entry_or_null(&resource_list, 890 struct resource_entry, node); 891 if (!rentry) { 892 dev_err(dev, "failed to get memory resource entry\n"); 893 goto free_list; 894 } 895 896 /* Caller must free the res */ 897 res = kzalloc(sizeof(*res), GFP_KERNEL); 898 if (!res) 899 goto free_list; 900 901 *res = *rentry->res; 902 903 acpi_dev_free_resource_list(&resource_list); 904 905 INIT_LIST_HEAD(&resource_list); 906 907 if (irq) 908 ret = acpi_dev_get_resources(adev, &resource_list, 909 tegra241_cmdqv_acpi_get_irqs, irq); 910 if (ret < 0 || !irq || *irq <= 0) 911 dev_warn(dev, "no interrupt. errors will not be reported\n"); 912 913 free_list: 914 acpi_dev_free_resource_list(&resource_list); 915 return res; 916 } 917 918 static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu) 919 { 920 struct tegra241_cmdqv *cmdqv = 921 container_of(smmu, struct tegra241_cmdqv, smmu); 922 struct tegra241_vintf *vintf; 923 int lidx; 924 int ret; 925 926 vintf = kzalloc(sizeof(*vintf), GFP_KERNEL); 927 if (!vintf) 928 return -ENOMEM; 929 930 /* Init VINTF0 for in-kernel use */ 931 ret = tegra241_cmdqv_init_vintf(cmdqv, 0, vintf); 932 if (ret) { 933 dev_err(cmdqv->dev, "failed to init vintf0: %d\n", ret); 934 return ret; 935 } 936 937 /* Preallocate logical VCMDQs to VINTF0 */ 938 for (lidx = 0; lidx < cmdqv->num_lvcmdqs_per_vintf; lidx++) { 939 struct tegra241_vcmdq *vcmdq; 940 941 vcmdq = tegra241_vintf_alloc_lvcmdq(vintf, lidx); 942 if (IS_ERR(vcmdq)) 943 return PTR_ERR(vcmdq); 944 } 945 946 /* Now, we are ready to run all the impl ops */ 947 smmu->impl_ops = &tegra241_cmdqv_impl_ops; 948 return 0; 949 } 950 951 #ifdef CONFIG_IOMMU_DEBUGFS 952 static struct dentry *cmdqv_debugfs_dir; 953 #endif 954 955 static struct arm_smmu_device * 956 __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res, 957 int irq) 958 { 959 static const struct arm_smmu_impl_ops init_ops = { 960 .init_structures = tegra241_cmdqv_init_structures, 961 .device_remove = tegra241_cmdqv_remove, 962 }; 963 struct tegra241_cmdqv *cmdqv = NULL; 964 struct arm_smmu_device *new_smmu; 965 void __iomem *base; 966 u32 regval; 967 int ret; 968 969 static_assert(offsetof(struct tegra241_cmdqv, smmu) == 0); 970 971 base = ioremap(res->start, resource_size(res)); 972 if (!base) { 973 dev_err(smmu->dev, "failed to ioremap\n"); 974 return NULL; 975 } 976 977 regval = readl(base + TEGRA241_CMDQV_CONFIG); 978 if (disable_cmdqv) { 979 dev_info(smmu->dev, "Detected disable_cmdqv=true\n"); 980 writel(regval & ~CMDQV_EN, base + TEGRA241_CMDQV_CONFIG); 981 goto iounmap; 982 } 983 984 cmdqv = devm_krealloc(smmu->dev, smmu, sizeof(*cmdqv), GFP_KERNEL); 985 if (!cmdqv) 986 goto iounmap; 987 new_smmu = &cmdqv->smmu; 988 989 cmdqv->irq = irq; 990 cmdqv->base = base; 991 cmdqv->dev = smmu->impl_dev; 992 cmdqv->base_phys = res->start; 993 994 if (cmdqv->irq > 0) { 995 ret = request_threaded_irq(irq, NULL, tegra241_cmdqv_isr, 996 IRQF_ONESHOT, "tegra241-cmdqv", 997 cmdqv); 998 if (ret) { 999 dev_err(cmdqv->dev, "failed to request irq (%d): %d\n", 1000 cmdqv->irq, ret); 1001 goto iounmap; 1002 } 1003 } 1004 1005 regval = readl_relaxed(REG_CMDQV(cmdqv, PARAM)); 1006 cmdqv->num_vintfs = 1 << FIELD_GET(CMDQV_NUM_VINTF_LOG2, regval); 1007 cmdqv->num_vcmdqs = 1 << FIELD_GET(CMDQV_NUM_VCMDQ_LOG2, regval); 1008 cmdqv->num_lvcmdqs_per_vintf = cmdqv->num_vcmdqs / cmdqv->num_vintfs; 1009 cmdqv->num_sids_per_vintf = 1010 1 << FIELD_GET(CMDQV_NUM_SID_PER_VM_LOG2, regval); 1011 1012 cmdqv->vintfs = 1013 kcalloc(cmdqv->num_vintfs, sizeof(*cmdqv->vintfs), GFP_KERNEL); 1014 if (!cmdqv->vintfs) 1015 goto free_irq; 1016 1017 ida_init(&cmdqv->vintf_ids); 1018 1019 #ifdef CONFIG_IOMMU_DEBUGFS 1020 if (!cmdqv_debugfs_dir) { 1021 cmdqv_debugfs_dir = 1022 debugfs_create_dir("tegra241_cmdqv", iommu_debugfs_dir); 1023 debugfs_create_bool("bypass_vcmdq", 0644, cmdqv_debugfs_dir, 1024 &bypass_vcmdq); 1025 } 1026 #endif 1027 1028 /* Provide init-level ops only, until tegra241_cmdqv_init_structures */ 1029 new_smmu->impl_ops = &init_ops; 1030 1031 return new_smmu; 1032 1033 free_irq: 1034 if (cmdqv->irq > 0) 1035 free_irq(cmdqv->irq, cmdqv); 1036 iounmap: 1037 iounmap(base); 1038 return NULL; 1039 } 1040 1041 struct arm_smmu_device *tegra241_cmdqv_probe(struct arm_smmu_device *smmu) 1042 { 1043 struct arm_smmu_device *new_smmu; 1044 struct resource *res = NULL; 1045 int irq; 1046 1047 if (!smmu->dev->of_node) 1048 res = tegra241_cmdqv_find_acpi_resource(smmu->impl_dev, &irq); 1049 if (!res) 1050 goto out_fallback; 1051 1052 new_smmu = __tegra241_cmdqv_probe(smmu, res, irq); 1053 kfree(res); 1054 1055 if (new_smmu) 1056 return new_smmu; 1057 1058 out_fallback: 1059 dev_info(smmu->impl_dev, "Falling back to standard SMMU CMDQ\n"); 1060 smmu->options &= ~ARM_SMMU_OPT_TEGRA241_CMDQV; 1061 put_device(smmu->impl_dev); 1062 return ERR_PTR(-ENODEV); 1063 } 1064 1065 /* User space VINTF and VCMDQ Functions */ 1066 1067 static size_t tegra241_vintf_get_vcmdq_size(struct iommufd_viommu *viommu, 1068 enum iommu_hw_queue_type queue_type) 1069 { 1070 if (queue_type != IOMMU_HW_QUEUE_TYPE_TEGRA241_CMDQV) 1071 return 0; 1072 return HW_QUEUE_STRUCT_SIZE(struct tegra241_vcmdq, core); 1073 } 1074 1075 static int tegra241_vcmdq_hw_init_user(struct tegra241_vcmdq *vcmdq) 1076 { 1077 char header[64]; 1078 1079 /* Configure the vcmdq only; User space does the enabling */ 1080 writeq_relaxed(vcmdq->cmdq.q.q_base, REG_VCMDQ_PAGE1(vcmdq, BASE)); 1081 1082 dev_dbg(vcmdq->cmdqv->dev, "%sinited at host PA 0x%llx size 0x%lx\n", 1083 lvcmdq_error_header(vcmdq, header, 64), 1084 vcmdq->cmdq.q.q_base & VCMDQ_ADDR, 1085 1UL << (vcmdq->cmdq.q.q_base & VCMDQ_LOG2SIZE)); 1086 return 0; 1087 } 1088 1089 static void 1090 tegra241_vintf_destroy_lvcmdq_user(struct iommufd_hw_queue *hw_queue) 1091 { 1092 struct tegra241_vcmdq *vcmdq = hw_queue_to_vcmdq(hw_queue); 1093 1094 mutex_lock(&vcmdq->vintf->lvcmdq_mutex); 1095 tegra241_vcmdq_hw_deinit(vcmdq); 1096 tegra241_vcmdq_unmap_lvcmdq(vcmdq); 1097 tegra241_vintf_free_lvcmdq(vcmdq->vintf, vcmdq->lidx); 1098 if (vcmdq->prev) 1099 iommufd_hw_queue_undepend(vcmdq, vcmdq->prev, core); 1100 mutex_unlock(&vcmdq->vintf->lvcmdq_mutex); 1101 } 1102 1103 static int tegra241_vintf_alloc_lvcmdq_user(struct iommufd_hw_queue *hw_queue, 1104 u32 lidx, phys_addr_t base_addr_pa) 1105 { 1106 struct tegra241_vintf *vintf = viommu_to_vintf(hw_queue->viommu); 1107 struct tegra241_vcmdq *vcmdq = hw_queue_to_vcmdq(hw_queue); 1108 struct tegra241_cmdqv *cmdqv = vintf->cmdqv; 1109 struct arm_smmu_device *smmu = &cmdqv->smmu; 1110 struct tegra241_vcmdq *prev = NULL; 1111 u32 log2size, max_n_shift; 1112 char header[64]; 1113 int ret; 1114 1115 if (hw_queue->type != IOMMU_HW_QUEUE_TYPE_TEGRA241_CMDQV) 1116 return -EOPNOTSUPP; 1117 if (lidx >= cmdqv->num_lvcmdqs_per_vintf) 1118 return -EINVAL; 1119 1120 mutex_lock(&vintf->lvcmdq_mutex); 1121 1122 if (vintf->lvcmdqs[lidx]) { 1123 ret = -EEXIST; 1124 goto unlock; 1125 } 1126 1127 /* 1128 * HW requires to map LVCMDQs in ascending order, so reject if the 1129 * previous lvcmdqs is not allocated yet. 1130 */ 1131 if (lidx) { 1132 prev = vintf->lvcmdqs[lidx - 1]; 1133 if (!prev) { 1134 ret = -EIO; 1135 goto unlock; 1136 } 1137 } 1138 1139 /* 1140 * hw_queue->length must be a power of 2, in range of 1141 * [ 32, 2 ^ (idr[1].CMDQS + CMDQ_ENT_SZ_SHIFT) ] 1142 */ 1143 max_n_shift = FIELD_GET(IDR1_CMDQS, 1144 readl_relaxed(smmu->base + ARM_SMMU_IDR1)); 1145 if (!is_power_of_2(hw_queue->length) || hw_queue->length < 32 || 1146 hw_queue->length > (1 << (max_n_shift + CMDQ_ENT_SZ_SHIFT))) { 1147 ret = -EINVAL; 1148 goto unlock; 1149 } 1150 log2size = ilog2(hw_queue->length) - CMDQ_ENT_SZ_SHIFT; 1151 1152 /* base_addr_pa must be aligned to hw_queue->length */ 1153 if (base_addr_pa & ~VCMDQ_ADDR || 1154 base_addr_pa & (hw_queue->length - 1)) { 1155 ret = -EINVAL; 1156 goto unlock; 1157 } 1158 1159 /* 1160 * HW requires to unmap LVCMDQs in descending order, so destroy() must 1161 * follow this rule. Set a dependency on its previous LVCMDQ so iommufd 1162 * core will help enforce it. 1163 */ 1164 if (prev) { 1165 ret = iommufd_hw_queue_depend(vcmdq, prev, core); 1166 if (ret) 1167 goto unlock; 1168 } 1169 vcmdq->prev = prev; 1170 1171 ret = tegra241_vintf_init_lvcmdq(vintf, lidx, vcmdq); 1172 if (ret) 1173 goto undepend_vcmdq; 1174 1175 dev_dbg(cmdqv->dev, "%sallocated\n", 1176 lvcmdq_error_header(vcmdq, header, 64)); 1177 1178 tegra241_vcmdq_map_lvcmdq(vcmdq); 1179 1180 vcmdq->cmdq.q.q_base = base_addr_pa & VCMDQ_ADDR; 1181 vcmdq->cmdq.q.q_base |= log2size; 1182 1183 ret = tegra241_vcmdq_hw_init_user(vcmdq); 1184 if (ret) 1185 goto unmap_lvcmdq; 1186 1187 hw_queue->destroy = &tegra241_vintf_destroy_lvcmdq_user; 1188 mutex_unlock(&vintf->lvcmdq_mutex); 1189 return 0; 1190 1191 unmap_lvcmdq: 1192 tegra241_vcmdq_unmap_lvcmdq(vcmdq); 1193 tegra241_vintf_deinit_lvcmdq(vintf, lidx); 1194 undepend_vcmdq: 1195 if (vcmdq->prev) 1196 iommufd_hw_queue_undepend(vcmdq, vcmdq->prev, core); 1197 unlock: 1198 mutex_unlock(&vintf->lvcmdq_mutex); 1199 return ret; 1200 } 1201 1202 static void tegra241_cmdqv_destroy_vintf_user(struct iommufd_viommu *viommu) 1203 { 1204 struct tegra241_vintf *vintf = viommu_to_vintf(viommu); 1205 1206 if (vintf->mmap_offset) 1207 iommufd_viommu_destroy_mmap(&vintf->vsmmu.core, 1208 vintf->mmap_offset); 1209 tegra241_cmdqv_remove_vintf(vintf->cmdqv, vintf->idx); 1210 } 1211 1212 static void tegra241_vintf_destroy_vsid(struct iommufd_vdevice *vdev) 1213 { 1214 struct tegra241_vintf_sid *vsid = vdev_to_vsid(vdev); 1215 struct tegra241_vintf *vintf = vsid->vintf; 1216 1217 writel(0, REG_VINTF(vintf, SID_MATCH(vsid->idx))); 1218 writel(0, REG_VINTF(vintf, SID_REPLACE(vsid->idx))); 1219 ida_free(&vintf->sids, vsid->idx); 1220 dev_dbg(vintf->cmdqv->dev, 1221 "VINTF%u: deallocated SID_REPLACE%d for pSID=%x\n", vintf->idx, 1222 vsid->idx, vsid->sid); 1223 } 1224 1225 static int tegra241_vintf_init_vsid(struct iommufd_vdevice *vdev) 1226 { 1227 struct device *dev = iommufd_vdevice_to_device(vdev); 1228 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 1229 struct tegra241_vintf *vintf = viommu_to_vintf(vdev->viommu); 1230 struct tegra241_vintf_sid *vsid = vdev_to_vsid(vdev); 1231 struct arm_smmu_stream *stream = &master->streams[0]; 1232 u64 virt_sid = vdev->virt_id; 1233 int sidx; 1234 1235 if (virt_sid > UINT_MAX) 1236 return -EINVAL; 1237 1238 WARN_ON_ONCE(master->num_streams != 1); 1239 1240 /* Find an empty pair of SID_REPLACE and SID_MATCH */ 1241 sidx = ida_alloc_max(&vintf->sids, vintf->cmdqv->num_sids_per_vintf - 1, 1242 GFP_KERNEL); 1243 if (sidx < 0) 1244 return sidx; 1245 1246 writel(stream->id, REG_VINTF(vintf, SID_REPLACE(sidx))); 1247 writel(virt_sid << 1 | 0x1, REG_VINTF(vintf, SID_MATCH(sidx))); 1248 dev_dbg(vintf->cmdqv->dev, 1249 "VINTF%u: allocated SID_REPLACE%d for pSID=%x, vSID=%x\n", 1250 vintf->idx, sidx, stream->id, (u32)virt_sid); 1251 1252 vsid->idx = sidx; 1253 vsid->vintf = vintf; 1254 vsid->sid = stream->id; 1255 1256 vdev->destroy = &tegra241_vintf_destroy_vsid; 1257 return 0; 1258 } 1259 1260 static struct iommufd_viommu_ops tegra241_cmdqv_viommu_ops = { 1261 .destroy = tegra241_cmdqv_destroy_vintf_user, 1262 .alloc_domain_nested = arm_vsmmu_alloc_domain_nested, 1263 /* Non-accelerated commands will be still handled by the kernel */ 1264 .cache_invalidate = arm_vsmmu_cache_invalidate, 1265 .vdevice_size = VDEVICE_STRUCT_SIZE(struct tegra241_vintf_sid, core), 1266 .vdevice_init = tegra241_vintf_init_vsid, 1267 .get_hw_queue_size = tegra241_vintf_get_vcmdq_size, 1268 .hw_queue_init_phys = tegra241_vintf_alloc_lvcmdq_user, 1269 }; 1270 1271 static int 1272 tegra241_cmdqv_init_vintf_user(struct arm_vsmmu *vsmmu, 1273 const struct iommu_user_data *user_data) 1274 { 1275 struct tegra241_cmdqv *cmdqv = 1276 container_of(vsmmu->smmu, struct tegra241_cmdqv, smmu); 1277 struct tegra241_vintf *vintf = viommu_to_vintf(&vsmmu->core); 1278 struct iommu_viommu_tegra241_cmdqv data; 1279 phys_addr_t page0_base; 1280 int ret; 1281 1282 /* 1283 * Unsupported type should be rejected by tegra241_cmdqv_get_vintf_size. 1284 * Seeing one here indicates a kernel bug or some data corruption. 1285 */ 1286 if (WARN_ON(vsmmu->core.type != IOMMU_VIOMMU_TYPE_TEGRA241_CMDQV)) 1287 return -EOPNOTSUPP; 1288 1289 if (!user_data) 1290 return -EINVAL; 1291 1292 ret = iommu_copy_struct_from_user(&data, user_data, 1293 IOMMU_VIOMMU_TYPE_TEGRA241_CMDQV, 1294 out_vintf_mmap_length); 1295 if (ret) 1296 return ret; 1297 1298 ret = tegra241_cmdqv_init_vintf(cmdqv, cmdqv->num_vintfs - 1, vintf); 1299 if (ret < 0) { 1300 dev_err(cmdqv->dev, "no more available vintf\n"); 1301 return ret; 1302 } 1303 1304 /* 1305 * Initialize the user-owned VINTF without a LVCMDQ, as it cannot pre- 1306 * allocate a LVCMDQ until user space wants one, for security reasons. 1307 * It is different than the kernel-owned VINTF0, which had pre-assigned 1308 * and pre-allocated global VCMDQs that would be mapped to the LVCMDQs 1309 * by the tegra241_vintf_hw_init() call. 1310 */ 1311 ret = tegra241_vintf_hw_init(vintf, false); 1312 if (ret) 1313 goto deinit_vintf; 1314 1315 page0_base = cmdqv->base_phys + TEGRA241_VINTFi_PAGE0(vintf->idx); 1316 ret = iommufd_viommu_alloc_mmap(&vintf->vsmmu.core, page0_base, SZ_64K, 1317 &vintf->mmap_offset); 1318 if (ret) 1319 goto hw_deinit_vintf; 1320 1321 data.out_vintf_mmap_length = SZ_64K; 1322 data.out_vintf_mmap_offset = vintf->mmap_offset; 1323 ret = iommu_copy_struct_to_user(user_data, &data, 1324 IOMMU_VIOMMU_TYPE_TEGRA241_CMDQV, 1325 out_vintf_mmap_length); 1326 if (ret) 1327 goto free_mmap; 1328 1329 ida_init(&vintf->sids); 1330 mutex_init(&vintf->lvcmdq_mutex); 1331 1332 dev_dbg(cmdqv->dev, "VINTF%u: allocated with vmid (%d)\n", vintf->idx, 1333 vintf->vsmmu.vmid); 1334 1335 vsmmu->core.ops = &tegra241_cmdqv_viommu_ops; 1336 return 0; 1337 1338 free_mmap: 1339 iommufd_viommu_destroy_mmap(&vintf->vsmmu.core, vintf->mmap_offset); 1340 hw_deinit_vintf: 1341 tegra241_vintf_hw_deinit(vintf); 1342 deinit_vintf: 1343 tegra241_cmdqv_deinit_vintf(cmdqv, vintf->idx); 1344 return ret; 1345 } 1346 1347 MODULE_IMPORT_NS("IOMMUFD"); 1348