1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2019, The Linux Foundation. All rights reserved. 4 */ 5 6 #include <linux/acpi.h> 7 #include <linux/adreno-smmu-priv.h> 8 #include <linux/delay.h> 9 #include <linux/of_device.h> 10 #include <linux/firmware/qcom/qcom_scm.h> 11 #include <linux/platform_device.h> 12 #include <linux/pm_runtime.h> 13 14 #include "arm-smmu.h" 15 #include "arm-smmu-qcom.h" 16 17 #define QCOM_DUMMY_VAL -1 18 19 /* 20 * SMMU-500 TRM defines BIT(0) as CMTLB (Enable context caching in the 21 * macro TLB) and BIT(1) as CPRE (Enable context caching in the prefetch 22 * buffer). The remaining bits are implementation defined and vary across 23 * SoCs. 24 */ 25 26 #define CPRE (1 << 1) 27 #define CMTLB (1 << 0) 28 #define PREFETCH_SHIFT 8 29 #define PREFETCH_DEFAULT 0 30 #define PREFETCH_SHALLOW (1 << PREFETCH_SHIFT) 31 #define PREFETCH_MODERATE (2 << PREFETCH_SHIFT) 32 #define PREFETCH_DEEP (3 << PREFETCH_SHIFT) 33 #define GFX_ACTLR_PRR (1 << 5) 34 35 static const struct of_device_id qcom_smmu_actlr_client_of_match[] = { 36 { .compatible = "qcom,adreno", 37 .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, 38 { .compatible = "qcom,adreno-gmu", 39 .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, 40 { .compatible = "qcom,adreno-smmu", 41 .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, 42 { .compatible = "qcom,fastrpc", 43 .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, 44 { .compatible = "qcom,sc7280-mdss", 45 .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, 46 { .compatible = "qcom,sc7280-venus", 47 .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, 48 { .compatible = "qcom,sm8550-mdss", 49 .data = (const void *) (PREFETCH_DEFAULT | CMTLB) }, 50 { } 51 }; 52 53 static struct qcom_smmu *to_qcom_smmu(struct arm_smmu_device *smmu) 54 { 55 return container_of(smmu, struct qcom_smmu, smmu); 56 } 57 58 static void qcom_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, 59 int sync, int status) 60 { 61 unsigned int spin_cnt, delay; 62 u32 reg; 63 64 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL); 65 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { 66 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { 67 reg = arm_smmu_readl(smmu, page, status); 68 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE)) 69 return; 70 cpu_relax(); 71 } 72 udelay(delay); 73 } 74 75 qcom_smmu_tlb_sync_debug(smmu); 76 } 77 78 static void qcom_adreno_smmu_write_sctlr(struct arm_smmu_device *smmu, int idx, 79 u32 reg) 80 { 81 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); 82 83 /* 84 * On the GPU device we want to process subsequent transactions after a 85 * fault to keep the GPU from hanging 86 */ 87 reg |= ARM_SMMU_SCTLR_HUPCF; 88 89 if (qsmmu->stall_enabled & BIT(idx)) 90 reg |= ARM_SMMU_SCTLR_CFCFG; 91 92 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg); 93 } 94 95 static void qcom_adreno_smmu_get_fault_info(const void *cookie, 96 struct adreno_smmu_fault_info *info) 97 { 98 struct arm_smmu_domain *smmu_domain = (void *)cookie; 99 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 100 struct arm_smmu_device *smmu = smmu_domain->smmu; 101 102 info->fsr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSR); 103 info->fsynr0 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR0); 104 info->fsynr1 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR1); 105 info->far = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_FAR); 106 info->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx)); 107 info->ttbr0 = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_TTBR0); 108 info->contextidr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_CONTEXTIDR); 109 } 110 111 static void qcom_adreno_smmu_set_stall(const void *cookie, bool enabled) 112 { 113 struct arm_smmu_domain *smmu_domain = (void *)cookie; 114 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 115 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu_domain->smmu); 116 117 if (enabled) 118 qsmmu->stall_enabled |= BIT(cfg->cbndx); 119 else 120 qsmmu->stall_enabled &= ~BIT(cfg->cbndx); 121 } 122 123 static void qcom_adreno_smmu_resume_translation(const void *cookie, bool terminate) 124 { 125 struct arm_smmu_domain *smmu_domain = (void *)cookie; 126 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 127 struct arm_smmu_device *smmu = smmu_domain->smmu; 128 u32 reg = 0; 129 130 if (terminate) 131 reg |= ARM_SMMU_RESUME_TERMINATE; 132 133 arm_smmu_cb_write(smmu, cfg->cbndx, ARM_SMMU_CB_RESUME, reg); 134 } 135 136 static void qcom_adreno_smmu_set_prr_bit(const void *cookie, bool set) 137 { 138 struct arm_smmu_domain *smmu_domain = (void *)cookie; 139 struct arm_smmu_device *smmu = smmu_domain->smmu; 140 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 141 u32 reg = 0; 142 int ret; 143 144 ret = pm_runtime_resume_and_get(smmu->dev); 145 if (ret < 0) { 146 dev_err(smmu->dev, "failed to get runtime PM: %d\n", ret); 147 return; 148 } 149 150 reg = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_ACTLR); 151 reg &= ~GFX_ACTLR_PRR; 152 if (set) 153 reg |= FIELD_PREP(GFX_ACTLR_PRR, 1); 154 arm_smmu_cb_write(smmu, cfg->cbndx, ARM_SMMU_CB_ACTLR, reg); 155 pm_runtime_put_autosuspend(smmu->dev); 156 } 157 158 static void qcom_adreno_smmu_set_prr_addr(const void *cookie, phys_addr_t page_addr) 159 { 160 struct arm_smmu_domain *smmu_domain = (void *)cookie; 161 struct arm_smmu_device *smmu = smmu_domain->smmu; 162 int ret; 163 164 ret = pm_runtime_resume_and_get(smmu->dev); 165 if (ret < 0) { 166 dev_err(smmu->dev, "failed to get runtime PM: %d\n", ret); 167 return; 168 } 169 170 writel_relaxed(lower_32_bits(page_addr), 171 smmu->base + ARM_SMMU_GFX_PRR_CFG_LADDR); 172 writel_relaxed(upper_32_bits(page_addr), 173 smmu->base + ARM_SMMU_GFX_PRR_CFG_UADDR); 174 pm_runtime_put_autosuspend(smmu->dev); 175 } 176 177 #define QCOM_ADRENO_SMMU_GPU_SID 0 178 179 static bool qcom_adreno_smmu_is_gpu_device(struct device *dev) 180 { 181 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 182 int i; 183 184 /* 185 * The GPU will always use SID 0 so that is a handy way to uniquely 186 * identify it and configure it for per-instance pagetables 187 */ 188 for (i = 0; i < fwspec->num_ids; i++) { 189 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); 190 191 if (sid == QCOM_ADRENO_SMMU_GPU_SID) 192 return true; 193 } 194 195 return false; 196 } 197 198 static const struct io_pgtable_cfg *qcom_adreno_smmu_get_ttbr1_cfg( 199 const void *cookie) 200 { 201 struct arm_smmu_domain *smmu_domain = (void *)cookie; 202 struct io_pgtable *pgtable = 203 io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops); 204 return &pgtable->cfg; 205 } 206 207 /* 208 * Local implementation to configure TTBR0 with the specified pagetable config. 209 * The GPU driver will call this to enable TTBR0 when per-instance pagetables 210 * are active 211 */ 212 213 static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie, 214 const struct io_pgtable_cfg *pgtbl_cfg) 215 { 216 struct arm_smmu_domain *smmu_domain = (void *)cookie; 217 struct io_pgtable *pgtable = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops); 218 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 219 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx]; 220 221 /* The domain must have split pagetables already enabled */ 222 if (cb->tcr[0] & ARM_SMMU_TCR_EPD1) 223 return -EINVAL; 224 225 /* If the pagetable config is NULL, disable TTBR0 */ 226 if (!pgtbl_cfg) { 227 /* Do nothing if it is already disabled */ 228 if ((cb->tcr[0] & ARM_SMMU_TCR_EPD0)) 229 return -EINVAL; 230 231 /* Set TCR to the original configuration */ 232 cb->tcr[0] = arm_smmu_lpae_tcr(&pgtable->cfg); 233 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid); 234 } else { 235 u32 tcr = cb->tcr[0]; 236 237 /* Don't call this again if TTBR0 is already enabled */ 238 if (!(cb->tcr[0] & ARM_SMMU_TCR_EPD0)) 239 return -EINVAL; 240 241 tcr |= arm_smmu_lpae_tcr(pgtbl_cfg); 242 tcr &= ~(ARM_SMMU_TCR_EPD0 | ARM_SMMU_TCR_EPD1); 243 244 cb->tcr[0] = tcr; 245 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr; 246 cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid); 247 } 248 249 arm_smmu_write_context_bank(smmu_domain->smmu, cb->cfg->cbndx); 250 251 return 0; 252 } 253 254 static int qcom_adreno_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain, 255 struct arm_smmu_device *smmu, 256 struct device *dev, int start) 257 { 258 int count; 259 260 /* 261 * Assign context bank 0 to the GPU device so the GPU hardware can 262 * switch pagetables 263 */ 264 if (qcom_adreno_smmu_is_gpu_device(dev)) { 265 start = 0; 266 count = 1; 267 } else { 268 start = 1; 269 count = smmu->num_context_banks; 270 } 271 272 return __arm_smmu_alloc_bitmap(smmu->context_map, start, count); 273 } 274 275 static bool qcom_adreno_can_do_ttbr1(struct arm_smmu_device *smmu) 276 { 277 const struct device_node *np = smmu->dev->of_node; 278 279 if (of_device_is_compatible(np, "qcom,msm8996-smmu-v2")) 280 return false; 281 282 return true; 283 } 284 285 static void qcom_smmu_set_actlr_dev(struct device *dev, struct arm_smmu_device *smmu, int cbndx, 286 const struct of_device_id *client_match) 287 { 288 const struct of_device_id *match = 289 of_match_device(client_match, dev); 290 291 if (!match) { 292 dev_dbg(dev, "no ACTLR settings present\n"); 293 return; 294 } 295 296 arm_smmu_cb_write(smmu, cbndx, ARM_SMMU_CB_ACTLR, (unsigned long)match->data); 297 } 298 299 static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain, 300 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev) 301 { 302 const struct device_node *np = smmu_domain->smmu->dev->of_node; 303 struct arm_smmu_device *smmu = smmu_domain->smmu; 304 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); 305 const struct of_device_id *client_match; 306 int cbndx = smmu_domain->cfg.cbndx; 307 struct adreno_smmu_priv *priv; 308 309 smmu_domain->cfg.flush_walk_prefer_tlbiasid = true; 310 311 client_match = qsmmu->data->client_match; 312 313 if (client_match) 314 qcom_smmu_set_actlr_dev(dev, smmu, cbndx, client_match); 315 316 /* Only enable split pagetables for the GPU device (SID 0) */ 317 if (!qcom_adreno_smmu_is_gpu_device(dev)) 318 return 0; 319 320 /* 321 * All targets that use the qcom,adreno-smmu compatible string *should* 322 * be AARCH64 stage 1 but double check because the arm-smmu code assumes 323 * that is the case when the TTBR1 quirk is enabled 324 */ 325 if (qcom_adreno_can_do_ttbr1(smmu_domain->smmu) && 326 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) && 327 (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)) 328 pgtbl_cfg->quirks |= IO_PGTABLE_QUIRK_ARM_TTBR1; 329 330 /* 331 * Initialize private interface with GPU: 332 */ 333 334 priv = dev_get_drvdata(dev); 335 priv->cookie = smmu_domain; 336 priv->get_ttbr1_cfg = qcom_adreno_smmu_get_ttbr1_cfg; 337 priv->set_ttbr0_cfg = qcom_adreno_smmu_set_ttbr0_cfg; 338 priv->get_fault_info = qcom_adreno_smmu_get_fault_info; 339 priv->set_stall = qcom_adreno_smmu_set_stall; 340 priv->resume_translation = qcom_adreno_smmu_resume_translation; 341 priv->set_prr_bit = NULL; 342 priv->set_prr_addr = NULL; 343 344 if (of_device_is_compatible(np, "qcom,smmu-500") && 345 of_device_is_compatible(np, "qcom,adreno-smmu")) { 346 priv->set_prr_bit = qcom_adreno_smmu_set_prr_bit; 347 priv->set_prr_addr = qcom_adreno_smmu_set_prr_addr; 348 } 349 350 return 0; 351 } 352 353 static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = { 354 { .compatible = "qcom,adreno" }, 355 { .compatible = "qcom,adreno-gmu" }, 356 { .compatible = "qcom,mdp4" }, 357 { .compatible = "qcom,mdss" }, 358 { .compatible = "qcom,qcm2290-mdss" }, 359 { .compatible = "qcom,sc7180-mdss" }, 360 { .compatible = "qcom,sc7180-mss-pil" }, 361 { .compatible = "qcom,sc7280-mdss" }, 362 { .compatible = "qcom,sc7280-mss-pil" }, 363 { .compatible = "qcom,sc8180x-mdss" }, 364 { .compatible = "qcom,sc8280xp-mdss" }, 365 { .compatible = "qcom,sdm670-mdss" }, 366 { .compatible = "qcom,sdm845-mdss" }, 367 { .compatible = "qcom,sdm845-mss-pil" }, 368 { .compatible = "qcom,sm6350-mdss" }, 369 { .compatible = "qcom,sm6375-mdss" }, 370 { .compatible = "qcom,sm8150-mdss" }, 371 { .compatible = "qcom,sm8250-mdss" }, 372 { .compatible = "qcom,x1e80100-mdss" }, 373 { } 374 }; 375 376 static int qcom_smmu_init_context(struct arm_smmu_domain *smmu_domain, 377 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev) 378 { 379 struct arm_smmu_device *smmu = smmu_domain->smmu; 380 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); 381 const struct of_device_id *client_match; 382 int cbndx = smmu_domain->cfg.cbndx; 383 384 smmu_domain->cfg.flush_walk_prefer_tlbiasid = true; 385 386 client_match = qsmmu->data->client_match; 387 388 if (client_match) 389 qcom_smmu_set_actlr_dev(dev, smmu, cbndx, client_match); 390 391 return 0; 392 } 393 394 static int qcom_smmu_cfg_probe(struct arm_smmu_device *smmu) 395 { 396 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); 397 unsigned int last_s2cr; 398 u32 reg; 399 u32 smr; 400 int i; 401 402 /* 403 * MSM8998 LPASS SMMU reports 13 context banks, but accessing 404 * the last context bank crashes the system. 405 */ 406 if (of_device_is_compatible(smmu->dev->of_node, "qcom,msm8998-smmu-v2") && 407 smmu->num_context_banks == 13) { 408 smmu->num_context_banks = 12; 409 } else if (of_device_is_compatible(smmu->dev->of_node, "qcom,sdm630-smmu-v2")) { 410 if (smmu->num_context_banks == 21) /* SDM630 / SDM660 A2NOC SMMU */ 411 smmu->num_context_banks = 7; 412 else if (smmu->num_context_banks == 14) /* SDM630 / SDM660 LPASS SMMU */ 413 smmu->num_context_banks = 13; 414 } 415 416 /* 417 * Some platforms support more than the Arm SMMU architected maximum of 418 * 128 stream matching groups. For unknown reasons, the additional 419 * groups don't exhibit the same behavior as the architected registers, 420 * so limit the groups to 128 until the behavior is fixed for the other 421 * groups. 422 */ 423 if (smmu->num_mapping_groups > 128) { 424 dev_notice(smmu->dev, "\tLimiting the stream matching groups to 128\n"); 425 smmu->num_mapping_groups = 128; 426 } 427 428 last_s2cr = ARM_SMMU_GR0_S2CR(smmu->num_mapping_groups - 1); 429 430 /* 431 * With some firmware versions writes to S2CR of type FAULT are 432 * ignored, and writing BYPASS will end up written as FAULT in the 433 * register. Perform a write to S2CR to detect if this is the case and 434 * if so reserve a context bank to emulate bypass streams. 435 */ 436 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, S2CR_TYPE_BYPASS) | 437 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, 0xff) | 438 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, S2CR_PRIVCFG_DEFAULT); 439 arm_smmu_gr0_write(smmu, last_s2cr, reg); 440 reg = arm_smmu_gr0_read(smmu, last_s2cr); 441 if (FIELD_GET(ARM_SMMU_S2CR_TYPE, reg) != S2CR_TYPE_BYPASS) { 442 qsmmu->bypass_quirk = true; 443 qsmmu->bypass_cbndx = smmu->num_context_banks - 1; 444 445 set_bit(qsmmu->bypass_cbndx, smmu->context_map); 446 447 arm_smmu_cb_write(smmu, qsmmu->bypass_cbndx, ARM_SMMU_CB_SCTLR, 0); 448 449 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, CBAR_TYPE_S1_TRANS_S2_BYPASS); 450 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(qsmmu->bypass_cbndx), reg); 451 } 452 453 for (i = 0; i < smmu->num_mapping_groups; i++) { 454 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); 455 456 if (FIELD_GET(ARM_SMMU_SMR_VALID, smr)) { 457 /* Ignore valid bit for SMR mask extraction. */ 458 smr &= ~ARM_SMMU_SMR_VALID; 459 smmu->smrs[i].id = FIELD_GET(ARM_SMMU_SMR_ID, smr); 460 smmu->smrs[i].mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr); 461 smmu->smrs[i].valid = true; 462 463 smmu->s2crs[i].type = S2CR_TYPE_BYPASS; 464 smmu->s2crs[i].privcfg = S2CR_PRIVCFG_DEFAULT; 465 smmu->s2crs[i].cbndx = 0xff; 466 } 467 } 468 469 return 0; 470 } 471 472 static int qcom_adreno_smmuv2_cfg_probe(struct arm_smmu_device *smmu) 473 { 474 /* Support for 16K pages is advertised on some SoCs, but it doesn't seem to work */ 475 smmu->features &= ~ARM_SMMU_FEAT_FMT_AARCH64_16K; 476 477 /* TZ protects several last context banks, hide them from Linux */ 478 if (of_device_is_compatible(smmu->dev->of_node, "qcom,sdm630-smmu-v2") && 479 smmu->num_context_banks == 5) 480 smmu->num_context_banks = 2; 481 482 return 0; 483 } 484 485 static void qcom_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) 486 { 487 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx; 488 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); 489 u32 cbndx = s2cr->cbndx; 490 u32 type = s2cr->type; 491 u32 reg; 492 493 if (qsmmu->bypass_quirk) { 494 if (type == S2CR_TYPE_BYPASS) { 495 /* 496 * Firmware with quirky S2CR handling will substitute 497 * BYPASS writes with FAULT, so point the stream to the 498 * reserved context bank and ask for translation on the 499 * stream 500 */ 501 type = S2CR_TYPE_TRANS; 502 cbndx = qsmmu->bypass_cbndx; 503 } else if (type == S2CR_TYPE_FAULT) { 504 /* 505 * Firmware with quirky S2CR handling will ignore FAULT 506 * writes, so trick it to write FAULT by asking for a 507 * BYPASS. 508 */ 509 type = S2CR_TYPE_BYPASS; 510 cbndx = 0xff; 511 } 512 } 513 514 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, type) | 515 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, cbndx) | 516 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg); 517 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg); 518 } 519 520 static int qcom_smmu_def_domain_type(struct device *dev) 521 { 522 const struct of_device_id *match = 523 of_match_device(qcom_smmu_client_of_match, dev); 524 525 return match ? IOMMU_DOMAIN_IDENTITY : 0; 526 } 527 528 static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu) 529 { 530 int ret; 531 532 arm_mmu500_reset(smmu); 533 534 /* 535 * To address performance degradation in non-real time clients, 536 * such as USB and UFS, turn off wait-for-safe on sdm845 based boards, 537 * such as MTP and db845, whose firmwares implement secure monitor 538 * call handlers to turn on/off the wait-for-safe logic. 539 */ 540 ret = qcom_scm_qsmmu500_wait_safe_toggle(0); 541 if (ret) 542 dev_warn(smmu->dev, "Failed to turn off SAFE logic\n"); 543 544 return ret; 545 } 546 547 static const struct arm_smmu_impl qcom_smmu_v2_impl = { 548 .init_context = qcom_smmu_init_context, 549 .cfg_probe = qcom_smmu_cfg_probe, 550 .def_domain_type = qcom_smmu_def_domain_type, 551 .write_s2cr = qcom_smmu_write_s2cr, 552 .tlb_sync = qcom_smmu_tlb_sync, 553 }; 554 555 static const struct arm_smmu_impl qcom_smmu_500_impl = { 556 .init_context = qcom_smmu_init_context, 557 .cfg_probe = qcom_smmu_cfg_probe, 558 .def_domain_type = qcom_smmu_def_domain_type, 559 .reset = arm_mmu500_reset, 560 .write_s2cr = qcom_smmu_write_s2cr, 561 .tlb_sync = qcom_smmu_tlb_sync, 562 #ifdef CONFIG_ARM_SMMU_QCOM_DEBUG 563 .context_fault = qcom_smmu_context_fault, 564 .context_fault_needs_threaded_irq = true, 565 #endif 566 }; 567 568 static const struct arm_smmu_impl sdm845_smmu_500_impl = { 569 .init_context = qcom_smmu_init_context, 570 .cfg_probe = qcom_smmu_cfg_probe, 571 .def_domain_type = qcom_smmu_def_domain_type, 572 .reset = qcom_sdm845_smmu500_reset, 573 .write_s2cr = qcom_smmu_write_s2cr, 574 .tlb_sync = qcom_smmu_tlb_sync, 575 #ifdef CONFIG_ARM_SMMU_QCOM_DEBUG 576 .context_fault = qcom_smmu_context_fault, 577 .context_fault_needs_threaded_irq = true, 578 #endif 579 }; 580 581 static const struct arm_smmu_impl qcom_adreno_smmu_v2_impl = { 582 .init_context = qcom_adreno_smmu_init_context, 583 .cfg_probe = qcom_adreno_smmuv2_cfg_probe, 584 .def_domain_type = qcom_smmu_def_domain_type, 585 .alloc_context_bank = qcom_adreno_smmu_alloc_context_bank, 586 .write_sctlr = qcom_adreno_smmu_write_sctlr, 587 .tlb_sync = qcom_smmu_tlb_sync, 588 }; 589 590 static const struct arm_smmu_impl qcom_adreno_smmu_500_impl = { 591 .init_context = qcom_adreno_smmu_init_context, 592 .def_domain_type = qcom_smmu_def_domain_type, 593 .reset = arm_mmu500_reset, 594 .alloc_context_bank = qcom_adreno_smmu_alloc_context_bank, 595 .write_sctlr = qcom_adreno_smmu_write_sctlr, 596 .tlb_sync = qcom_smmu_tlb_sync, 597 }; 598 599 static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu, 600 const struct qcom_smmu_match_data *data) 601 { 602 const struct device_node *np = smmu->dev->of_node; 603 const struct arm_smmu_impl *impl; 604 struct qcom_smmu *qsmmu; 605 606 if (!data) 607 return ERR_PTR(-EINVAL); 608 609 if (np && of_device_is_compatible(np, "qcom,adreno-smmu")) 610 impl = data->adreno_impl; 611 else 612 impl = data->impl; 613 614 if (!impl) 615 return smmu; 616 617 /* Check to make sure qcom_scm has finished probing */ 618 if (!qcom_scm_is_available()) 619 return ERR_PTR(dev_err_probe(smmu->dev, -EPROBE_DEFER, 620 "qcom_scm not ready\n")); 621 622 qsmmu = devm_krealloc(smmu->dev, smmu, sizeof(*qsmmu), GFP_KERNEL); 623 if (!qsmmu) 624 return ERR_PTR(-ENOMEM); 625 626 qsmmu->smmu.impl = impl; 627 qsmmu->data = data; 628 629 return &qsmmu->smmu; 630 } 631 632 /* Implementation Defined Register Space 0 register offsets */ 633 static const u32 qcom_smmu_impl0_reg_offset[] = { 634 [QCOM_SMMU_TBU_PWR_STATUS] = 0x2204, 635 [QCOM_SMMU_STATS_SYNC_INV_TBU_ACK] = 0x25dc, 636 [QCOM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR] = 0x2670, 637 }; 638 639 static const struct qcom_smmu_config qcom_smmu_impl0_cfg = { 640 .reg_offset = qcom_smmu_impl0_reg_offset, 641 }; 642 643 /* 644 * It is not yet possible to use MDP SMMU with the bypass quirk on the msm8996, 645 * there are not enough context banks. 646 */ 647 static const struct qcom_smmu_match_data msm8996_smmu_data = { 648 .impl = NULL, 649 .adreno_impl = &qcom_adreno_smmu_v2_impl, 650 }; 651 652 static const struct qcom_smmu_match_data qcom_smmu_v2_data = { 653 .impl = &qcom_smmu_v2_impl, 654 .adreno_impl = &qcom_adreno_smmu_v2_impl, 655 }; 656 657 static const struct qcom_smmu_match_data sdm845_smmu_500_data = { 658 .impl = &sdm845_smmu_500_impl, 659 /* 660 * No need for adreno impl here. On sdm845 the Adreno SMMU is handled 661 * by the separate sdm845-smmu-v2 device. 662 */ 663 /* Also no debug configuration. */ 664 }; 665 666 static const struct qcom_smmu_match_data qcom_smmu_500_impl0_data = { 667 .impl = &qcom_smmu_500_impl, 668 .adreno_impl = &qcom_adreno_smmu_500_impl, 669 .cfg = &qcom_smmu_impl0_cfg, 670 .client_match = qcom_smmu_actlr_client_of_match, 671 }; 672 673 /* 674 * Do not add any more qcom,SOC-smmu-500 entries to this list, unless they need 675 * special handling and can not be covered by the qcom,smmu-500 entry. 676 */ 677 static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = { 678 { .compatible = "qcom,msm8996-smmu-v2", .data = &msm8996_smmu_data }, 679 { .compatible = "qcom,msm8998-smmu-v2", .data = &qcom_smmu_v2_data }, 680 { .compatible = "qcom,qcm2290-smmu-500", .data = &qcom_smmu_500_impl0_data }, 681 { .compatible = "qcom,qdu1000-smmu-500", .data = &qcom_smmu_500_impl0_data }, 682 { .compatible = "qcom,sc7180-smmu-500", .data = &qcom_smmu_500_impl0_data }, 683 { .compatible = "qcom,sc7180-smmu-v2", .data = &qcom_smmu_v2_data }, 684 { .compatible = "qcom,sc7280-smmu-500", .data = &qcom_smmu_500_impl0_data }, 685 { .compatible = "qcom,sc8180x-smmu-500", .data = &qcom_smmu_500_impl0_data }, 686 { .compatible = "qcom,sc8280xp-smmu-500", .data = &qcom_smmu_500_impl0_data }, 687 { .compatible = "qcom,sdm630-smmu-v2", .data = &qcom_smmu_v2_data }, 688 { .compatible = "qcom,sdm670-smmu-v2", .data = &qcom_smmu_v2_data }, 689 { .compatible = "qcom,sdm845-smmu-v2", .data = &qcom_smmu_v2_data }, 690 { .compatible = "qcom,sdm845-smmu-500", .data = &sdm845_smmu_500_data }, 691 { .compatible = "qcom,sm6115-smmu-500", .data = &qcom_smmu_500_impl0_data}, 692 { .compatible = "qcom,sm6125-smmu-500", .data = &qcom_smmu_500_impl0_data }, 693 { .compatible = "qcom,sm6350-smmu-v2", .data = &qcom_smmu_v2_data }, 694 { .compatible = "qcom,sm6350-smmu-500", .data = &qcom_smmu_500_impl0_data }, 695 { .compatible = "qcom,sm6375-smmu-v2", .data = &qcom_smmu_v2_data }, 696 { .compatible = "qcom,sm6375-smmu-500", .data = &qcom_smmu_500_impl0_data }, 697 { .compatible = "qcom,sm7150-smmu-v2", .data = &qcom_smmu_v2_data }, 698 { .compatible = "qcom,sm8150-smmu-500", .data = &qcom_smmu_500_impl0_data }, 699 { .compatible = "qcom,sm8250-smmu-500", .data = &qcom_smmu_500_impl0_data }, 700 { .compatible = "qcom,sm8350-smmu-500", .data = &qcom_smmu_500_impl0_data }, 701 { .compatible = "qcom,sm8450-smmu-500", .data = &qcom_smmu_500_impl0_data }, 702 { .compatible = "qcom,smmu-500", .data = &qcom_smmu_500_impl0_data }, 703 { } 704 }; 705 706 #ifdef CONFIG_ACPI 707 static struct acpi_platform_list qcom_acpi_platlist[] = { 708 { "LENOVO", "CB-01 ", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" }, 709 { "QCOM ", "QCOMEDK2", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" }, 710 { } 711 }; 712 #endif 713 714 static int qcom_smmu_tbu_probe(struct platform_device *pdev) 715 { 716 struct device *dev = &pdev->dev; 717 int ret; 718 719 if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM_DEBUG)) { 720 ret = qcom_tbu_probe(pdev); 721 if (ret) 722 return ret; 723 } 724 725 if (dev->pm_domain) { 726 pm_runtime_set_active(dev); 727 pm_runtime_enable(dev); 728 } 729 730 return 0; 731 } 732 733 static const struct of_device_id qcom_smmu_tbu_of_match[] = { 734 { .compatible = "qcom,sc7280-tbu" }, 735 { .compatible = "qcom,sdm845-tbu" }, 736 { } 737 }; 738 739 static struct platform_driver qcom_smmu_tbu_driver = { 740 .driver = { 741 .name = "qcom_tbu", 742 .of_match_table = qcom_smmu_tbu_of_match, 743 }, 744 .probe = qcom_smmu_tbu_probe, 745 }; 746 747 struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) 748 { 749 const struct device_node *np = smmu->dev->of_node; 750 const struct of_device_id *match; 751 static u8 tbu_registered; 752 753 if (!tbu_registered++) 754 platform_driver_register(&qcom_smmu_tbu_driver); 755 756 #ifdef CONFIG_ACPI 757 if (np == NULL) { 758 /* Match platform for ACPI boot */ 759 if (acpi_match_platform_list(qcom_acpi_platlist) >= 0) 760 return qcom_smmu_create(smmu, &qcom_smmu_500_impl0_data); 761 } 762 #endif 763 764 match = of_match_node(qcom_smmu_impl_of_match, np); 765 if (match) 766 return qcom_smmu_create(smmu, match->data); 767 768 /* 769 * If you hit this WARN_ON() you are missing an entry in the 770 * qcom_smmu_impl_of_match[] table, and GPU per-process page- 771 * tables will be broken. 772 */ 773 WARN(of_device_is_compatible(np, "qcom,adreno-smmu"), 774 "Missing qcom_smmu_impl_of_match entry for: %s", 775 dev_name(smmu->dev)); 776 777 return smmu; 778 } 779