1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2019, The Linux Foundation. All rights reserved. 4 */ 5 6 #include <linux/acpi.h> 7 #include <linux/adreno-smmu-priv.h> 8 #include <linux/delay.h> 9 #include <linux/of_device.h> 10 #include <linux/firmware/qcom/qcom_scm.h> 11 #include <linux/platform_device.h> 12 #include <linux/pm_runtime.h> 13 14 #include "arm-smmu.h" 15 #include "arm-smmu-qcom.h" 16 17 #define QCOM_DUMMY_VAL -1 18 19 /* 20 * SMMU-500 TRM defines BIT(0) as CMTLB (Enable context caching in the 21 * macro TLB) and BIT(1) as CPRE (Enable context caching in the prefetch 22 * buffer). The remaining bits are implementation defined and vary across 23 * SoCs. 24 */ 25 26 #define CPRE (1 << 1) 27 #define CMTLB (1 << 0) 28 #define PREFETCH_SHIFT 8 29 #define PREFETCH_DEFAULT 0 30 #define PREFETCH_SHALLOW (1 << PREFETCH_SHIFT) 31 #define PREFETCH_MODERATE (2 << PREFETCH_SHIFT) 32 #define PREFETCH_DEEP (3 << PREFETCH_SHIFT) 33 #define GFX_ACTLR_PRR (1 << 5) 34 35 static const struct of_device_id qcom_smmu_actlr_client_of_match[] = { 36 { .compatible = "qcom,adreno", 37 .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, 38 { .compatible = "qcom,adreno-gmu", 39 .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, 40 { .compatible = "qcom,adreno-smmu", 41 .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, 42 { .compatible = "qcom,fastrpc", 43 .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, 44 { .compatible = "qcom,sc7280-mdss", 45 .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, 46 { .compatible = "qcom,sc7280-venus", 47 .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, 48 { .compatible = "qcom,sm8550-mdss", 49 .data = (const void *) (PREFETCH_DEFAULT | CMTLB) }, 50 { } 51 }; 52 53 static struct qcom_smmu *to_qcom_smmu(struct arm_smmu_device *smmu) 54 { 55 return container_of(smmu, struct qcom_smmu, smmu); 56 } 57 58 static void qcom_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, 59 int sync, int status) 60 { 61 unsigned int spin_cnt, delay; 62 u32 reg; 63 64 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL); 65 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { 66 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { 67 reg = arm_smmu_readl(smmu, page, status); 68 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE)) 69 return; 70 cpu_relax(); 71 } 72 udelay(delay); 73 } 74 75 qcom_smmu_tlb_sync_debug(smmu); 76 } 77 78 static void qcom_adreno_smmu_write_sctlr(struct arm_smmu_device *smmu, int idx, 79 u32 reg) 80 { 81 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); 82 83 /* 84 * On the GPU device we want to process subsequent transactions after a 85 * fault to keep the GPU from hanging 86 */ 87 reg |= ARM_SMMU_SCTLR_HUPCF; 88 89 if (qsmmu->stall_enabled & BIT(idx)) 90 reg |= ARM_SMMU_SCTLR_CFCFG; 91 92 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg); 93 } 94 95 static void qcom_adreno_smmu_get_fault_info(const void *cookie, 96 struct adreno_smmu_fault_info *info) 97 { 98 struct arm_smmu_domain *smmu_domain = (void *)cookie; 99 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 100 struct arm_smmu_device *smmu = smmu_domain->smmu; 101 102 info->fsr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSR); 103 info->fsynr0 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR0); 104 info->fsynr1 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR1); 105 info->far = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_FAR); 106 info->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx)); 107 info->ttbr0 = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_TTBR0); 108 info->contextidr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_CONTEXTIDR); 109 } 110 111 static void qcom_adreno_smmu_set_stall(const void *cookie, bool enabled) 112 { 113 struct arm_smmu_domain *smmu_domain = (void *)cookie; 114 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 115 struct arm_smmu_device *smmu = smmu_domain->smmu; 116 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); 117 u32 mask = BIT(cfg->cbndx); 118 bool stall_changed = !!(qsmmu->stall_enabled & mask) != enabled; 119 unsigned long flags; 120 121 if (enabled) 122 qsmmu->stall_enabled |= mask; 123 else 124 qsmmu->stall_enabled &= ~mask; 125 126 /* 127 * If the device is on and we changed the setting, update the register. 128 * The spec pseudocode says that CFCFG is resampled after a fault, and 129 * we believe that no implementations cache it in the TLB, so it should 130 * be safe to change it without a TLB invalidation. 131 */ 132 if (stall_changed && pm_runtime_get_if_active(smmu->dev) > 0) { 133 u32 reg; 134 135 spin_lock_irqsave(&smmu_domain->cb_lock, flags); 136 reg = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_SCTLR); 137 138 if (enabled) 139 reg |= ARM_SMMU_SCTLR_CFCFG; 140 else 141 reg &= ~ARM_SMMU_SCTLR_CFCFG; 142 143 arm_smmu_cb_write(smmu, cfg->cbndx, ARM_SMMU_CB_SCTLR, reg); 144 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 145 146 pm_runtime_put_autosuspend(smmu->dev); 147 } 148 } 149 150 static void qcom_adreno_smmu_set_prr_bit(const void *cookie, bool set) 151 { 152 struct arm_smmu_domain *smmu_domain = (void *)cookie; 153 struct arm_smmu_device *smmu = smmu_domain->smmu; 154 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 155 u32 reg = 0; 156 int ret; 157 158 ret = pm_runtime_resume_and_get(smmu->dev); 159 if (ret < 0) { 160 dev_err(smmu->dev, "failed to get runtime PM: %d\n", ret); 161 return; 162 } 163 164 reg = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_ACTLR); 165 reg &= ~GFX_ACTLR_PRR; 166 if (set) 167 reg |= FIELD_PREP(GFX_ACTLR_PRR, 1); 168 arm_smmu_cb_write(smmu, cfg->cbndx, ARM_SMMU_CB_ACTLR, reg); 169 pm_runtime_put_autosuspend(smmu->dev); 170 } 171 172 static void qcom_adreno_smmu_set_prr_addr(const void *cookie, phys_addr_t page_addr) 173 { 174 struct arm_smmu_domain *smmu_domain = (void *)cookie; 175 struct arm_smmu_device *smmu = smmu_domain->smmu; 176 int ret; 177 178 ret = pm_runtime_resume_and_get(smmu->dev); 179 if (ret < 0) { 180 dev_err(smmu->dev, "failed to get runtime PM: %d\n", ret); 181 return; 182 } 183 184 writel_relaxed(lower_32_bits(page_addr), 185 smmu->base + ARM_SMMU_GFX_PRR_CFG_LADDR); 186 writel_relaxed(upper_32_bits(page_addr), 187 smmu->base + ARM_SMMU_GFX_PRR_CFG_UADDR); 188 pm_runtime_put_autosuspend(smmu->dev); 189 } 190 191 #define QCOM_ADRENO_SMMU_GPU_SID 0 192 193 static bool qcom_adreno_smmu_is_gpu_device(struct device *dev) 194 { 195 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 196 int i; 197 198 /* 199 * The GPU will always use SID 0 so that is a handy way to uniquely 200 * identify it and configure it for per-instance pagetables 201 */ 202 for (i = 0; i < fwspec->num_ids; i++) { 203 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); 204 205 if (sid == QCOM_ADRENO_SMMU_GPU_SID) 206 return true; 207 } 208 209 return false; 210 } 211 212 static const struct io_pgtable_cfg *qcom_adreno_smmu_get_ttbr1_cfg( 213 const void *cookie) 214 { 215 struct arm_smmu_domain *smmu_domain = (void *)cookie; 216 struct io_pgtable *pgtable = 217 io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops); 218 return &pgtable->cfg; 219 } 220 221 /* 222 * Local implementation to configure TTBR0 with the specified pagetable config. 223 * The GPU driver will call this to enable TTBR0 when per-instance pagetables 224 * are active 225 */ 226 227 static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie, 228 const struct io_pgtable_cfg *pgtbl_cfg) 229 { 230 struct arm_smmu_domain *smmu_domain = (void *)cookie; 231 struct io_pgtable *pgtable = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops); 232 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 233 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx]; 234 235 /* The domain must have split pagetables already enabled */ 236 if (cb->tcr[0] & ARM_SMMU_TCR_EPD1) 237 return -EINVAL; 238 239 /* If the pagetable config is NULL, disable TTBR0 */ 240 if (!pgtbl_cfg) { 241 /* Do nothing if it is already disabled */ 242 if ((cb->tcr[0] & ARM_SMMU_TCR_EPD0)) 243 return -EINVAL; 244 245 /* Set TCR to the original configuration */ 246 cb->tcr[0] = arm_smmu_lpae_tcr(&pgtable->cfg); 247 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid); 248 } else { 249 u32 tcr = cb->tcr[0]; 250 251 /* Don't call this again if TTBR0 is already enabled */ 252 if (!(cb->tcr[0] & ARM_SMMU_TCR_EPD0)) 253 return -EINVAL; 254 255 tcr |= arm_smmu_lpae_tcr(pgtbl_cfg); 256 tcr &= ~(ARM_SMMU_TCR_EPD0 | ARM_SMMU_TCR_EPD1); 257 258 cb->tcr[0] = tcr; 259 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr; 260 cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid); 261 } 262 263 arm_smmu_write_context_bank(smmu_domain->smmu, cb->cfg->cbndx); 264 265 return 0; 266 } 267 268 static int qcom_adreno_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain, 269 struct arm_smmu_device *smmu, 270 struct device *dev, int start) 271 { 272 int count; 273 274 /* 275 * Assign context bank 0 to the GPU device so the GPU hardware can 276 * switch pagetables 277 */ 278 if (qcom_adreno_smmu_is_gpu_device(dev)) { 279 start = 0; 280 count = 1; 281 } else { 282 start = 1; 283 count = smmu->num_context_banks; 284 } 285 286 return __arm_smmu_alloc_bitmap(smmu->context_map, start, count); 287 } 288 289 static bool qcom_adreno_can_do_ttbr1(struct arm_smmu_device *smmu) 290 { 291 const struct device_node *np = smmu->dev->of_node; 292 293 if (of_device_is_compatible(np, "qcom,msm8996-smmu-v2")) 294 return false; 295 296 return true; 297 } 298 299 static void qcom_smmu_set_actlr_dev(struct device *dev, struct arm_smmu_device *smmu, int cbndx, 300 const struct of_device_id *client_match) 301 { 302 const struct of_device_id *match = 303 of_match_device(client_match, dev); 304 305 if (!match) { 306 dev_dbg(dev, "no ACTLR settings present\n"); 307 return; 308 } 309 310 arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR, (unsigned long)match->data); 311 } 312 313 static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain, 314 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev) 315 { 316 const struct device_node *np = smmu_domain->smmu->dev->of_node; 317 struct arm_smmu_device *smmu = smmu_domain->smmu; 318 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); 319 const struct of_device_id *client_match; 320 int cbndx = smmu_domain->cfg.cbndx; 321 struct adreno_smmu_priv *priv; 322 323 smmu_domain->cfg.flush_walk_prefer_tlbiasid = true; 324 325 client_match = qsmmu->data->client_match; 326 327 if (client_match) 328 qcom_smmu_set_actlr_dev(dev, smmu, cbndx, client_match); 329 330 /* Only enable split pagetables for the GPU device (SID 0) */ 331 if (!qcom_adreno_smmu_is_gpu_device(dev)) 332 return 0; 333 334 /* 335 * All targets that use the qcom,adreno-smmu compatible string *should* 336 * be AARCH64 stage 1 but double check because the arm-smmu code assumes 337 * that is the case when the TTBR1 quirk is enabled 338 */ 339 if (qcom_adreno_can_do_ttbr1(smmu_domain->smmu) && 340 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) && 341 (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)) 342 pgtbl_cfg->quirks |= IO_PGTABLE_QUIRK_ARM_TTBR1; 343 344 /* 345 * Initialize private interface with GPU: 346 */ 347 348 priv = dev_get_drvdata(dev); 349 priv->cookie = smmu_domain; 350 priv->get_ttbr1_cfg = qcom_adreno_smmu_get_ttbr1_cfg; 351 priv->set_ttbr0_cfg = qcom_adreno_smmu_set_ttbr0_cfg; 352 priv->get_fault_info = qcom_adreno_smmu_get_fault_info; 353 priv->set_stall = qcom_adreno_smmu_set_stall; 354 priv->set_prr_bit = NULL; 355 priv->set_prr_addr = NULL; 356 357 if (of_device_is_compatible(np, "qcom,smmu-500") && 358 of_device_is_compatible(np, "qcom,adreno-smmu")) { 359 priv->set_prr_bit = qcom_adreno_smmu_set_prr_bit; 360 priv->set_prr_addr = qcom_adreno_smmu_set_prr_addr; 361 } 362 363 return 0; 364 } 365 366 static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = { 367 { .compatible = "qcom,adreno" }, 368 { .compatible = "qcom,adreno-gmu" }, 369 { .compatible = "qcom,mdp4" }, 370 { .compatible = "qcom,mdss" }, 371 { .compatible = "qcom,qcm2290-mdss" }, 372 { .compatible = "qcom,sar2130p-mdss" }, 373 { .compatible = "qcom,sc7180-mdss" }, 374 { .compatible = "qcom,sc7180-mss-pil" }, 375 { .compatible = "qcom,sc7280-mdss" }, 376 { .compatible = "qcom,sc7280-mss-pil" }, 377 { .compatible = "qcom,sc8180x-mdss" }, 378 { .compatible = "qcom,sc8280xp-mdss" }, 379 { .compatible = "qcom,sdm670-mdss" }, 380 { .compatible = "qcom,sdm845-mdss" }, 381 { .compatible = "qcom,sdm845-mss-pil" }, 382 { .compatible = "qcom,sm6350-mdss" }, 383 { .compatible = "qcom,sm6375-mdss" }, 384 { .compatible = "qcom,sm8150-mdss" }, 385 { .compatible = "qcom,sm8250-mdss" }, 386 { .compatible = "qcom,x1e80100-mdss" }, 387 { } 388 }; 389 390 static int qcom_smmu_init_context(struct arm_smmu_domain *smmu_domain, 391 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev) 392 { 393 struct arm_smmu_device *smmu = smmu_domain->smmu; 394 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); 395 const struct of_device_id *client_match; 396 int cbndx = smmu_domain->cfg.cbndx; 397 398 smmu_domain->cfg.flush_walk_prefer_tlbiasid = true; 399 400 client_match = qsmmu->data->client_match; 401 402 if (client_match) 403 qcom_smmu_set_actlr_dev(dev, smmu, cbndx, client_match); 404 405 return 0; 406 } 407 408 static int qcom_smmu_cfg_probe(struct arm_smmu_device *smmu) 409 { 410 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); 411 unsigned int last_s2cr; 412 u32 reg; 413 u32 smr; 414 int i; 415 416 /* 417 * MSM8998 LPASS SMMU reports 13 context banks, but accessing 418 * the last context bank crashes the system. 419 */ 420 if (of_device_is_compatible(smmu->dev->of_node, "qcom,msm8998-smmu-v2") && 421 smmu->num_context_banks == 13) { 422 smmu->num_context_banks = 12; 423 } else if (of_device_is_compatible(smmu->dev->of_node, "qcom,sdm630-smmu-v2")) { 424 if (smmu->num_context_banks == 21) /* SDM630 / SDM660 A2NOC SMMU */ 425 smmu->num_context_banks = 7; 426 else if (smmu->num_context_banks == 14) /* SDM630 / SDM660 LPASS SMMU */ 427 smmu->num_context_banks = 13; 428 } 429 430 /* 431 * Some platforms support more than the Arm SMMU architected maximum of 432 * 128 stream matching groups. For unknown reasons, the additional 433 * groups don't exhibit the same behavior as the architected registers, 434 * so limit the groups to 128 until the behavior is fixed for the other 435 * groups. 436 */ 437 if (smmu->num_mapping_groups > 128) { 438 dev_notice(smmu->dev, "\tLimiting the stream matching groups to 128\n"); 439 smmu->num_mapping_groups = 128; 440 } 441 442 last_s2cr = ARM_SMMU_GR0_S2CR(smmu->num_mapping_groups - 1); 443 444 /* 445 * With some firmware versions writes to S2CR of type FAULT are 446 * ignored, and writing BYPASS will end up written as FAULT in the 447 * register. Perform a write to S2CR to detect if this is the case and 448 * if so reserve a context bank to emulate bypass streams. 449 */ 450 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, S2CR_TYPE_BYPASS) | 451 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, 0xff) | 452 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, S2CR_PRIVCFG_DEFAULT); 453 arm_smmu_gr0_write(smmu, last_s2cr, reg); 454 reg = arm_smmu_gr0_read(smmu, last_s2cr); 455 if (FIELD_GET(ARM_SMMU_S2CR_TYPE, reg) != S2CR_TYPE_BYPASS) { 456 qsmmu->bypass_quirk = true; 457 qsmmu->bypass_cbndx = smmu->num_context_banks - 1; 458 459 set_bit(qsmmu->bypass_cbndx, smmu->context_map); 460 461 arm_smmu_cb_write(smmu, qsmmu->bypass_cbndx, ARM_SMMU_CB_SCTLR, 0); 462 463 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, CBAR_TYPE_S1_TRANS_S2_BYPASS); 464 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(qsmmu->bypass_cbndx), reg); 465 } 466 467 for (i = 0; i < smmu->num_mapping_groups; i++) { 468 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); 469 470 if (FIELD_GET(ARM_SMMU_SMR_VALID, smr)) { 471 /* Ignore valid bit for SMR mask extraction. */ 472 smr &= ~ARM_SMMU_SMR_VALID; 473 smmu->smrs[i].id = FIELD_GET(ARM_SMMU_SMR_ID, smr); 474 smmu->smrs[i].mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr); 475 smmu->smrs[i].valid = true; 476 477 smmu->s2crs[i].type = S2CR_TYPE_BYPASS; 478 smmu->s2crs[i].privcfg = S2CR_PRIVCFG_DEFAULT; 479 smmu->s2crs[i].cbndx = 0xff; 480 } 481 } 482 483 return 0; 484 } 485 486 static int qcom_adreno_smmuv2_cfg_probe(struct arm_smmu_device *smmu) 487 { 488 /* Support for 16K pages is advertised on some SoCs, but it doesn't seem to work */ 489 smmu->features &= ~ARM_SMMU_FEAT_FMT_AARCH64_16K; 490 491 /* TZ protects several last context banks, hide them from Linux */ 492 if (of_device_is_compatible(smmu->dev->of_node, "qcom,sdm630-smmu-v2") && 493 smmu->num_context_banks == 5) 494 smmu->num_context_banks = 2; 495 496 return 0; 497 } 498 499 static void qcom_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) 500 { 501 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx; 502 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); 503 u32 cbndx = s2cr->cbndx; 504 u32 type = s2cr->type; 505 u32 reg; 506 507 if (qsmmu->bypass_quirk) { 508 if (type == S2CR_TYPE_BYPASS) { 509 /* 510 * Firmware with quirky S2CR handling will substitute 511 * BYPASS writes with FAULT, so point the stream to the 512 * reserved context bank and ask for translation on the 513 * stream 514 */ 515 type = S2CR_TYPE_TRANS; 516 cbndx = qsmmu->bypass_cbndx; 517 } else if (type == S2CR_TYPE_FAULT) { 518 /* 519 * Firmware with quirky S2CR handling will ignore FAULT 520 * writes, so trick it to write FAULT by asking for a 521 * BYPASS. 522 */ 523 type = S2CR_TYPE_BYPASS; 524 cbndx = 0xff; 525 } 526 } 527 528 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, type) | 529 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, cbndx) | 530 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg); 531 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg); 532 } 533 534 static int qcom_smmu_def_domain_type(struct device *dev) 535 { 536 const struct of_device_id *match = 537 of_match_device(qcom_smmu_client_of_match, dev); 538 539 return match ? IOMMU_DOMAIN_IDENTITY : 0; 540 } 541 542 static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu) 543 { 544 int ret; 545 546 arm_mmu500_reset(smmu); 547 548 /* 549 * To address performance degradation in non-real time clients, 550 * such as USB and UFS, turn off wait-for-safe on sdm845 based boards, 551 * such as MTP and db845, whose firmwares implement secure monitor 552 * call handlers to turn on/off the wait-for-safe logic. 553 */ 554 ret = qcom_scm_qsmmu500_wait_safe_toggle(0); 555 if (ret) 556 dev_warn(smmu->dev, "Failed to turn off SAFE logic\n"); 557 558 return ret; 559 } 560 561 static const struct arm_smmu_impl qcom_smmu_v2_impl = { 562 .init_context = qcom_smmu_init_context, 563 .cfg_probe = qcom_smmu_cfg_probe, 564 .def_domain_type = qcom_smmu_def_domain_type, 565 .write_s2cr = qcom_smmu_write_s2cr, 566 .tlb_sync = qcom_smmu_tlb_sync, 567 }; 568 569 static const struct arm_smmu_impl qcom_smmu_500_impl = { 570 .init_context = qcom_smmu_init_context, 571 .cfg_probe = qcom_smmu_cfg_probe, 572 .def_domain_type = qcom_smmu_def_domain_type, 573 .reset = arm_mmu500_reset, 574 .write_s2cr = qcom_smmu_write_s2cr, 575 .tlb_sync = qcom_smmu_tlb_sync, 576 #ifdef CONFIG_ARM_SMMU_QCOM_DEBUG 577 .context_fault = qcom_smmu_context_fault, 578 .context_fault_needs_threaded_irq = true, 579 #endif 580 }; 581 582 static const struct arm_smmu_impl sdm845_smmu_500_impl = { 583 .init_context = qcom_smmu_init_context, 584 .cfg_probe = qcom_smmu_cfg_probe, 585 .def_domain_type = qcom_smmu_def_domain_type, 586 .reset = qcom_sdm845_smmu500_reset, 587 .write_s2cr = qcom_smmu_write_s2cr, 588 .tlb_sync = qcom_smmu_tlb_sync, 589 #ifdef CONFIG_ARM_SMMU_QCOM_DEBUG 590 .context_fault = qcom_smmu_context_fault, 591 .context_fault_needs_threaded_irq = true, 592 #endif 593 }; 594 595 static const struct arm_smmu_impl qcom_adreno_smmu_v2_impl = { 596 .init_context = qcom_adreno_smmu_init_context, 597 .cfg_probe = qcom_adreno_smmuv2_cfg_probe, 598 .def_domain_type = qcom_smmu_def_domain_type, 599 .alloc_context_bank = qcom_adreno_smmu_alloc_context_bank, 600 .write_sctlr = qcom_adreno_smmu_write_sctlr, 601 .tlb_sync = qcom_smmu_tlb_sync, 602 .context_fault_needs_threaded_irq = true, 603 }; 604 605 static const struct arm_smmu_impl qcom_adreno_smmu_500_impl = { 606 .init_context = qcom_adreno_smmu_init_context, 607 .def_domain_type = qcom_smmu_def_domain_type, 608 .reset = arm_mmu500_reset, 609 .alloc_context_bank = qcom_adreno_smmu_alloc_context_bank, 610 .write_sctlr = qcom_adreno_smmu_write_sctlr, 611 .tlb_sync = qcom_smmu_tlb_sync, 612 .context_fault_needs_threaded_irq = true, 613 }; 614 615 static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu, 616 const struct qcom_smmu_match_data *data) 617 { 618 const struct device_node *np = smmu->dev->of_node; 619 const struct arm_smmu_impl *impl; 620 struct qcom_smmu *qsmmu; 621 622 if (!data) 623 return ERR_PTR(-EINVAL); 624 625 if (np && of_device_is_compatible(np, "qcom,adreno-smmu")) 626 impl = data->adreno_impl; 627 else 628 impl = data->impl; 629 630 if (!impl) 631 return smmu; 632 633 /* Check to make sure qcom_scm has finished probing */ 634 if (!qcom_scm_is_available()) 635 return ERR_PTR(dev_err_probe(smmu->dev, -EPROBE_DEFER, 636 "qcom_scm not ready\n")); 637 638 qsmmu = devm_krealloc(smmu->dev, smmu, sizeof(*qsmmu), GFP_KERNEL); 639 if (!qsmmu) 640 return ERR_PTR(-ENOMEM); 641 642 qsmmu->smmu.impl = impl; 643 qsmmu->data = data; 644 645 return &qsmmu->smmu; 646 } 647 648 /* Implementation Defined Register Space 0 register offsets */ 649 static const u32 qcom_smmu_impl0_reg_offset[] = { 650 [QCOM_SMMU_TBU_PWR_STATUS] = 0x2204, 651 [QCOM_SMMU_STATS_SYNC_INV_TBU_ACK] = 0x25dc, 652 [QCOM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR] = 0x2670, 653 }; 654 655 static const struct qcom_smmu_config qcom_smmu_impl0_cfg = { 656 .reg_offset = qcom_smmu_impl0_reg_offset, 657 }; 658 659 /* 660 * It is not yet possible to use MDP SMMU with the bypass quirk on the msm8996, 661 * there are not enough context banks. 662 */ 663 static const struct qcom_smmu_match_data msm8996_smmu_data = { 664 .impl = NULL, 665 .adreno_impl = &qcom_adreno_smmu_v2_impl, 666 }; 667 668 static const struct qcom_smmu_match_data qcom_smmu_v2_data = { 669 .impl = &qcom_smmu_v2_impl, 670 .adreno_impl = &qcom_adreno_smmu_v2_impl, 671 }; 672 673 static const struct qcom_smmu_match_data sdm845_smmu_500_data = { 674 .impl = &sdm845_smmu_500_impl, 675 /* 676 * No need for adreno impl here. On sdm845 the Adreno SMMU is handled 677 * by the separate sdm845-smmu-v2 device. 678 */ 679 /* Also no debug configuration. */ 680 }; 681 682 static const struct qcom_smmu_match_data qcom_smmu_500_impl0_data = { 683 .impl = &qcom_smmu_500_impl, 684 .adreno_impl = &qcom_adreno_smmu_500_impl, 685 .cfg = &qcom_smmu_impl0_cfg, 686 .client_match = qcom_smmu_actlr_client_of_match, 687 }; 688 689 /* 690 * Do not add any more qcom,SOC-smmu-500 entries to this list, unless they need 691 * special handling and can not be covered by the qcom,smmu-500 entry. 692 */ 693 static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = { 694 { .compatible = "qcom,msm8996-smmu-v2", .data = &msm8996_smmu_data }, 695 { .compatible = "qcom,msm8998-smmu-v2", .data = &qcom_smmu_v2_data }, 696 { .compatible = "qcom,qcm2290-smmu-500", .data = &qcom_smmu_500_impl0_data }, 697 { .compatible = "qcom,qdu1000-smmu-500", .data = &qcom_smmu_500_impl0_data }, 698 { .compatible = "qcom,sc7180-smmu-500", .data = &qcom_smmu_500_impl0_data }, 699 { .compatible = "qcom,sc7180-smmu-v2", .data = &qcom_smmu_v2_data }, 700 { .compatible = "qcom,sc7280-smmu-500", .data = &qcom_smmu_500_impl0_data }, 701 { .compatible = "qcom,sc8180x-smmu-500", .data = &qcom_smmu_500_impl0_data }, 702 { .compatible = "qcom,sc8280xp-smmu-500", .data = &qcom_smmu_500_impl0_data }, 703 { .compatible = "qcom,sdm630-smmu-v2", .data = &qcom_smmu_v2_data }, 704 { .compatible = "qcom,sdm670-smmu-v2", .data = &qcom_smmu_v2_data }, 705 { .compatible = "qcom,sdm845-smmu-v2", .data = &qcom_smmu_v2_data }, 706 { .compatible = "qcom,sdm845-smmu-500", .data = &sdm845_smmu_500_data }, 707 { .compatible = "qcom,sm6115-smmu-500", .data = &qcom_smmu_500_impl0_data}, 708 { .compatible = "qcom,sm6125-smmu-500", .data = &qcom_smmu_500_impl0_data }, 709 { .compatible = "qcom,sm6350-smmu-v2", .data = &qcom_smmu_v2_data }, 710 { .compatible = "qcom,sm6350-smmu-500", .data = &qcom_smmu_500_impl0_data }, 711 { .compatible = "qcom,sm6375-smmu-v2", .data = &qcom_smmu_v2_data }, 712 { .compatible = "qcom,sm6375-smmu-500", .data = &qcom_smmu_500_impl0_data }, 713 { .compatible = "qcom,sm7150-smmu-v2", .data = &qcom_smmu_v2_data }, 714 { .compatible = "qcom,sm8150-smmu-500", .data = &qcom_smmu_500_impl0_data }, 715 { .compatible = "qcom,sm8250-smmu-500", .data = &qcom_smmu_500_impl0_data }, 716 { .compatible = "qcom,sm8350-smmu-500", .data = &qcom_smmu_500_impl0_data }, 717 { .compatible = "qcom,sm8450-smmu-500", .data = &qcom_smmu_500_impl0_data }, 718 { .compatible = "qcom,smmu-500", .data = &qcom_smmu_500_impl0_data }, 719 { } 720 }; 721 722 #ifdef CONFIG_ACPI 723 static struct acpi_platform_list qcom_acpi_platlist[] = { 724 { "LENOVO", "CB-01 ", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" }, 725 { "QCOM ", "QCOMEDK2", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" }, 726 { } 727 }; 728 #endif 729 730 static int qcom_smmu_tbu_probe(struct platform_device *pdev) 731 { 732 struct device *dev = &pdev->dev; 733 int ret; 734 735 if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM_DEBUG)) { 736 ret = qcom_tbu_probe(pdev); 737 if (ret) 738 return ret; 739 } 740 741 if (dev->pm_domain) { 742 pm_runtime_set_active(dev); 743 pm_runtime_enable(dev); 744 } 745 746 return 0; 747 } 748 749 static const struct of_device_id qcom_smmu_tbu_of_match[] = { 750 { .compatible = "qcom,sc7280-tbu" }, 751 { .compatible = "qcom,sdm845-tbu" }, 752 { } 753 }; 754 755 static struct platform_driver qcom_smmu_tbu_driver = { 756 .driver = { 757 .name = "qcom_tbu", 758 .of_match_table = qcom_smmu_tbu_of_match, 759 }, 760 .probe = qcom_smmu_tbu_probe, 761 }; 762 763 struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) 764 { 765 const struct device_node *np = smmu->dev->of_node; 766 const struct of_device_id *match; 767 static u8 tbu_registered; 768 769 if (!tbu_registered++) 770 platform_driver_register(&qcom_smmu_tbu_driver); 771 772 #ifdef CONFIG_ACPI 773 if (np == NULL) { 774 /* Match platform for ACPI boot */ 775 if (acpi_match_platform_list(qcom_acpi_platlist) >= 0) 776 return qcom_smmu_create(smmu, &qcom_smmu_500_impl0_data); 777 } 778 #endif 779 780 match = of_match_node(qcom_smmu_impl_of_match, np); 781 if (match) 782 return qcom_smmu_create(smmu, match->data); 783 784 /* 785 * If you hit this WARN_ON() you are missing an entry in the 786 * qcom_smmu_impl_of_match[] table, and GPU per-process page- 787 * tables will be broken. 788 */ 789 WARN(of_device_is_compatible(np, "qcom,adreno-smmu"), 790 "Missing qcom_smmu_impl_of_match entry for: %s", 791 dev_name(smmu->dev)); 792 793 return smmu; 794 } 795