1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * IOMMU API for ARM architected SMMU implementations. 4 * 5 * Copyright (C) 2013 ARM Limited 6 * 7 * Author: Will Deacon <will.deacon@arm.com> 8 * 9 * This driver currently supports: 10 * - SMMUv1 and v2 implementations 11 * - Stream-matching and stream-indexing 12 * - v7/v8 long-descriptor format 13 * - Non-secure access to the SMMU 14 * - Context fault reporting 15 * - Extended Stream ID (16 bit) 16 */ 17 18 #define pr_fmt(fmt) "arm-smmu: " fmt 19 20 #include <linux/acpi.h> 21 #include <linux/acpi_iort.h> 22 #include <linux/bitfield.h> 23 #include <linux/delay.h> 24 #include <linux/dma-mapping.h> 25 #include <linux/err.h> 26 #include <linux/interrupt.h> 27 #include <linux/io.h> 28 #include <linux/iopoll.h> 29 #include <linux/module.h> 30 #include <linux/of.h> 31 #include <linux/of_address.h> 32 #include <linux/pci.h> 33 #include <linux/platform_device.h> 34 #include <linux/pm_runtime.h> 35 #include <linux/ratelimit.h> 36 #include <linux/slab.h> 37 #include <linux/string_choices.h> 38 39 #include <linux/fsl/mc.h> 40 41 #include "arm-smmu.h" 42 #include "../../dma-iommu.h" 43 44 /* 45 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU 46 * global register space are still, in fact, using a hypervisor to mediate it 47 * by trapping and emulating register accesses. Sadly, some deployed versions 48 * of said trapping code have bugs wherein they go horribly wrong for stores 49 * using r31 (i.e. XZR/WZR) as the source register. 50 */ 51 #define QCOM_DUMMY_VAL -1 52 53 #define MSI_IOVA_BASE 0x8000000 54 #define MSI_IOVA_LENGTH 0x100000 55 56 static int force_stage; 57 module_param(force_stage, int, S_IRUGO); 58 MODULE_PARM_DESC(force_stage, 59 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); 60 static bool disable_bypass = 61 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT); 62 module_param(disable_bypass, bool, S_IRUGO); 63 MODULE_PARM_DESC(disable_bypass, 64 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); 65 66 #define s2cr_init_val (struct arm_smmu_s2cr){ \ 67 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \ 68 } 69 70 static bool using_legacy_binding, using_generic_binding; 71 72 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu) 73 { 74 if (pm_runtime_enabled(smmu->dev)) 75 return pm_runtime_resume_and_get(smmu->dev); 76 77 return 0; 78 } 79 80 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu) 81 { 82 if (pm_runtime_enabled(smmu->dev)) { 83 pm_runtime_mark_last_busy(smmu->dev); 84 __pm_runtime_put_autosuspend(smmu->dev); 85 86 } 87 } 88 89 static void arm_smmu_rpm_use_autosuspend(struct arm_smmu_device *smmu) 90 { 91 /* 92 * Setup an autosuspend delay to avoid bouncing runpm state. 93 * Otherwise, if a driver for a suspended consumer device 94 * unmaps buffers, it will runpm resume/suspend for each one. 95 * 96 * For example, when used by a GPU device, when an application 97 * or game exits, it can trigger unmapping 100s or 1000s of 98 * buffers. With a runpm cycle for each buffer, that adds up 99 * to 5-10sec worth of reprogramming the context bank, while 100 * the system appears to be locked up to the user. 101 */ 102 pm_runtime_set_autosuspend_delay(smmu->dev, 20); 103 pm_runtime_use_autosuspend(smmu->dev); 104 } 105 106 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) 107 { 108 return container_of(dom, struct arm_smmu_domain, domain); 109 } 110 111 static struct platform_driver arm_smmu_driver; 112 static const struct iommu_ops arm_smmu_ops; 113 114 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS 115 static struct device_node *dev_get_dev_node(struct device *dev) 116 { 117 if (dev_is_pci(dev)) { 118 struct pci_bus *bus = to_pci_dev(dev)->bus; 119 120 while (!pci_is_root_bus(bus)) 121 bus = bus->parent; 122 return of_node_get(bus->bridge->parent->of_node); 123 } 124 125 return of_node_get(dev->of_node); 126 } 127 128 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data) 129 { 130 *((__be32 *)data) = cpu_to_be32(alias); 131 return 0; /* Continue walking */ 132 } 133 134 static int __find_legacy_master_phandle(struct device *dev, void *data) 135 { 136 struct of_phandle_iterator *it = *(void **)data; 137 struct device_node *np = it->node; 138 int err; 139 140 of_for_each_phandle(it, err, dev->of_node, "mmu-masters", 141 "#stream-id-cells", -1) 142 if (it->node == np) { 143 *(void **)data = dev; 144 return 1; 145 } 146 it->node = np; 147 return err == -ENOENT ? 0 : err; 148 } 149 150 static int arm_smmu_register_legacy_master(struct device *dev, 151 struct arm_smmu_device **smmu) 152 { 153 struct device *smmu_dev; 154 struct device_node *np; 155 struct of_phandle_iterator it; 156 void *data = ⁢ 157 u32 *sids; 158 __be32 pci_sid; 159 int err; 160 161 np = dev_get_dev_node(dev); 162 if (!np || !of_property_present(np, "#stream-id-cells")) { 163 of_node_put(np); 164 return -ENODEV; 165 } 166 167 it.node = np; 168 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data, 169 __find_legacy_master_phandle); 170 smmu_dev = data; 171 of_node_put(np); 172 if (err == 0) 173 return -ENODEV; 174 if (err < 0) 175 return err; 176 177 if (dev_is_pci(dev)) { 178 /* "mmu-masters" assumes Stream ID == Requester ID */ 179 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid, 180 &pci_sid); 181 it.cur = &pci_sid; 182 it.cur_count = 1; 183 } 184 185 err = iommu_fwspec_init(dev, NULL); 186 if (err) 187 return err; 188 189 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL); 190 if (!sids) 191 return -ENOMEM; 192 193 *smmu = dev_get_drvdata(smmu_dev); 194 of_phandle_iterator_args(&it, sids, it.cur_count); 195 err = iommu_fwspec_add_ids(dev, sids, it.cur_count); 196 kfree(sids); 197 return err; 198 } 199 #else 200 static int arm_smmu_register_legacy_master(struct device *dev, 201 struct arm_smmu_device **smmu) 202 { 203 return -ENODEV; 204 } 205 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */ 206 207 static void __arm_smmu_free_bitmap(unsigned long *map, int idx) 208 { 209 clear_bit(idx, map); 210 } 211 212 /* Wait for any pending TLB invalidations to complete */ 213 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, 214 int sync, int status) 215 { 216 unsigned int spin_cnt, delay; 217 u32 reg; 218 219 if (smmu->impl && unlikely(smmu->impl->tlb_sync)) 220 return smmu->impl->tlb_sync(smmu, page, sync, status); 221 222 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL); 223 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { 224 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { 225 reg = arm_smmu_readl(smmu, page, status); 226 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE)) 227 return; 228 cpu_relax(); 229 } 230 udelay(delay); 231 } 232 dev_err_ratelimited(smmu->dev, 233 "TLB sync timed out -- SMMU may be deadlocked\n"); 234 } 235 236 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) 237 { 238 unsigned long flags; 239 240 spin_lock_irqsave(&smmu->global_sync_lock, flags); 241 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC, 242 ARM_SMMU_GR0_sTLBGSTATUS); 243 spin_unlock_irqrestore(&smmu->global_sync_lock, flags); 244 } 245 246 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain) 247 { 248 struct arm_smmu_device *smmu = smmu_domain->smmu; 249 unsigned long flags; 250 251 spin_lock_irqsave(&smmu_domain->cb_lock, flags); 252 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx), 253 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS); 254 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 255 } 256 257 static void arm_smmu_tlb_inv_context_s1(void *cookie) 258 { 259 struct arm_smmu_domain *smmu_domain = cookie; 260 /* 261 * The TLBI write may be relaxed, so ensure that PTEs cleared by the 262 * current CPU are visible beforehand. 263 */ 264 wmb(); 265 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx, 266 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid); 267 arm_smmu_tlb_sync_context(smmu_domain); 268 } 269 270 static void arm_smmu_tlb_inv_context_s2(void *cookie) 271 { 272 struct arm_smmu_domain *smmu_domain = cookie; 273 struct arm_smmu_device *smmu = smmu_domain->smmu; 274 275 /* See above */ 276 wmb(); 277 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid); 278 arm_smmu_tlb_sync_global(smmu); 279 } 280 281 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size, 282 size_t granule, void *cookie, int reg) 283 { 284 struct arm_smmu_domain *smmu_domain = cookie; 285 struct arm_smmu_device *smmu = smmu_domain->smmu; 286 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 287 int idx = cfg->cbndx; 288 289 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 290 wmb(); 291 292 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) { 293 iova = (iova >> 12) << 12; 294 iova |= cfg->asid; 295 do { 296 arm_smmu_cb_write(smmu, idx, reg, iova); 297 iova += granule; 298 } while (size -= granule); 299 } else { 300 iova >>= 12; 301 iova |= (u64)cfg->asid << 48; 302 do { 303 arm_smmu_cb_writeq(smmu, idx, reg, iova); 304 iova += granule >> 12; 305 } while (size -= granule); 306 } 307 } 308 309 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size, 310 size_t granule, void *cookie, int reg) 311 { 312 struct arm_smmu_domain *smmu_domain = cookie; 313 struct arm_smmu_device *smmu = smmu_domain->smmu; 314 int idx = smmu_domain->cfg.cbndx; 315 316 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 317 wmb(); 318 319 iova >>= 12; 320 do { 321 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64) 322 arm_smmu_cb_writeq(smmu, idx, reg, iova); 323 else 324 arm_smmu_cb_write(smmu, idx, reg, iova); 325 iova += granule >> 12; 326 } while (size -= granule); 327 } 328 329 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size, 330 size_t granule, void *cookie) 331 { 332 struct arm_smmu_domain *smmu_domain = cookie; 333 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 334 335 if (cfg->flush_walk_prefer_tlbiasid) { 336 arm_smmu_tlb_inv_context_s1(cookie); 337 } else { 338 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie, 339 ARM_SMMU_CB_S1_TLBIVA); 340 arm_smmu_tlb_sync_context(cookie); 341 } 342 } 343 344 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather, 345 unsigned long iova, size_t granule, 346 void *cookie) 347 { 348 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie, 349 ARM_SMMU_CB_S1_TLBIVAL); 350 } 351 352 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size, 353 size_t granule, void *cookie) 354 { 355 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie, 356 ARM_SMMU_CB_S2_TLBIIPAS2); 357 arm_smmu_tlb_sync_context(cookie); 358 } 359 360 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather, 361 unsigned long iova, size_t granule, 362 void *cookie) 363 { 364 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie, 365 ARM_SMMU_CB_S2_TLBIIPAS2L); 366 } 367 368 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size, 369 size_t granule, void *cookie) 370 { 371 arm_smmu_tlb_inv_context_s2(cookie); 372 } 373 /* 374 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears 375 * almost negligible, but the benefit of getting the first one in as far ahead 376 * of the sync as possible is significant, hence we don't just make this a 377 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might 378 * think. 379 */ 380 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather, 381 unsigned long iova, size_t granule, 382 void *cookie) 383 { 384 struct arm_smmu_domain *smmu_domain = cookie; 385 struct arm_smmu_device *smmu = smmu_domain->smmu; 386 387 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 388 wmb(); 389 390 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid); 391 } 392 393 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = { 394 .tlb_flush_all = arm_smmu_tlb_inv_context_s1, 395 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1, 396 .tlb_add_page = arm_smmu_tlb_add_page_s1, 397 }; 398 399 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = { 400 .tlb_flush_all = arm_smmu_tlb_inv_context_s2, 401 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2, 402 .tlb_add_page = arm_smmu_tlb_add_page_s2, 403 }; 404 405 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = { 406 .tlb_flush_all = arm_smmu_tlb_inv_context_s2, 407 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1, 408 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1, 409 }; 410 411 412 void arm_smmu_read_context_fault_info(struct arm_smmu_device *smmu, int idx, 413 struct arm_smmu_context_fault_info *cfi) 414 { 415 cfi->iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); 416 cfi->fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); 417 cfi->fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); 418 cfi->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); 419 } 420 421 void arm_smmu_print_context_fault_info(struct arm_smmu_device *smmu, int idx, 422 const struct arm_smmu_context_fault_info *cfi) 423 { 424 dev_err(smmu->dev, 425 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", 426 cfi->fsr, cfi->iova, cfi->fsynr, cfi->cbfrsynra, idx); 427 428 dev_err(smmu->dev, "FSR = %08x [%s%sFormat=%u%s%s%s%s%s%s%s%s], SID=0x%x\n", 429 cfi->fsr, 430 (cfi->fsr & ARM_SMMU_CB_FSR_MULTI) ? "MULTI " : "", 431 (cfi->fsr & ARM_SMMU_CB_FSR_SS) ? "SS " : "", 432 (u32)FIELD_GET(ARM_SMMU_CB_FSR_FORMAT, cfi->fsr), 433 (cfi->fsr & ARM_SMMU_CB_FSR_UUT) ? " UUT" : "", 434 (cfi->fsr & ARM_SMMU_CB_FSR_ASF) ? " ASF" : "", 435 (cfi->fsr & ARM_SMMU_CB_FSR_TLBLKF) ? " TLBLKF" : "", 436 (cfi->fsr & ARM_SMMU_CB_FSR_TLBMCF) ? " TLBMCF" : "", 437 (cfi->fsr & ARM_SMMU_CB_FSR_EF) ? " EF" : "", 438 (cfi->fsr & ARM_SMMU_CB_FSR_PF) ? " PF" : "", 439 (cfi->fsr & ARM_SMMU_CB_FSR_AFF) ? " AFF" : "", 440 (cfi->fsr & ARM_SMMU_CB_FSR_TF) ? " TF" : "", 441 cfi->cbfrsynra); 442 443 dev_err(smmu->dev, "FSYNR0 = %08x [S1CBNDX=%u%s%s%s%s%s%s PLVL=%u]\n", 444 cfi->fsynr, 445 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_S1CBNDX, cfi->fsynr), 446 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_AFR) ? " AFR" : "", 447 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PTWF) ? " PTWF" : "", 448 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_NSATTR) ? " NSATTR" : "", 449 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_IND) ? " IND" : "", 450 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PNU) ? " PNU" : "", 451 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_WNR) ? " WNR" : "", 452 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_PLVL, cfi->fsynr)); 453 } 454 455 static irqreturn_t arm_smmu_context_fault(int irq, void *dev) 456 { 457 struct arm_smmu_context_fault_info cfi; 458 struct arm_smmu_domain *smmu_domain = dev; 459 struct arm_smmu_device *smmu = smmu_domain->smmu; 460 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 461 DEFAULT_RATELIMIT_BURST); 462 int idx = smmu_domain->cfg.cbndx; 463 int ret; 464 465 arm_smmu_read_context_fault_info(smmu, idx, &cfi); 466 467 if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT)) 468 return IRQ_NONE; 469 470 ret = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova, 471 cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); 472 473 if (ret == -ENOSYS && __ratelimit(&rs)) 474 arm_smmu_print_context_fault_info(smmu, idx, &cfi); 475 476 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr); 477 478 if (cfi.fsr & ARM_SMMU_CB_FSR_SS) { 479 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, 480 ret == -EAGAIN ? 0 : ARM_SMMU_RESUME_TERMINATE); 481 } 482 483 return IRQ_HANDLED; 484 } 485 486 static irqreturn_t arm_smmu_global_fault(int irq, void *dev) 487 { 488 u32 gfsr, gfsynr0, gfsynr1, gfsynr2; 489 struct arm_smmu_device *smmu = dev; 490 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 491 DEFAULT_RATELIMIT_BURST); 492 493 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); 494 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0); 495 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1); 496 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2); 497 498 if (!gfsr) 499 return IRQ_NONE; 500 501 if (__ratelimit(&rs)) { 502 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) && 503 (gfsr & ARM_SMMU_sGFSR_USF)) 504 dev_err(smmu->dev, 505 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n", 506 (u16)gfsynr1); 507 else 508 dev_err(smmu->dev, 509 "Unexpected global fault, this could be serious\n"); 510 dev_err(smmu->dev, 511 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n", 512 gfsr, gfsynr0, gfsynr1, gfsynr2); 513 } 514 515 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr); 516 return IRQ_HANDLED; 517 } 518 519 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, 520 struct io_pgtable_cfg *pgtbl_cfg) 521 { 522 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 523 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx]; 524 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; 525 526 cb->cfg = cfg; 527 528 /* TCR */ 529 if (stage1) { 530 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 531 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr; 532 } else { 533 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg); 534 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg); 535 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 536 cb->tcr[1] |= ARM_SMMU_TCR2_AS; 537 else 538 cb->tcr[0] |= ARM_SMMU_TCR_EAE; 539 } 540 } else { 541 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg); 542 } 543 544 /* TTBRs */ 545 if (stage1) { 546 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 547 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr; 548 cb->ttbr[1] = 0; 549 } else { 550 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, 551 cfg->asid); 552 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, 553 cfg->asid); 554 555 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) 556 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr; 557 else 558 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr; 559 } 560 } else { 561 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; 562 } 563 564 /* MAIRs (stage-1 only) */ 565 if (stage1) { 566 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 567 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr; 568 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr; 569 } else { 570 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair; 571 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32; 572 } 573 } 574 } 575 576 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) 577 { 578 u32 reg; 579 bool stage1; 580 struct arm_smmu_cb *cb = &smmu->cbs[idx]; 581 struct arm_smmu_cfg *cfg = cb->cfg; 582 583 /* Unassigned context banks only need disabling */ 584 if (!cfg) { 585 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0); 586 return; 587 } 588 589 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; 590 591 /* CBA2R */ 592 if (smmu->version > ARM_SMMU_V1) { 593 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 594 reg = ARM_SMMU_CBA2R_VA64; 595 else 596 reg = 0; 597 /* 16-bit VMIDs live in CBA2R */ 598 if (smmu->features & ARM_SMMU_FEAT_VMID16) 599 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid); 600 601 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg); 602 } 603 604 /* CBAR */ 605 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar); 606 if (smmu->version < ARM_SMMU_V2) 607 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx); 608 609 /* 610 * Use the weakest shareability/memory types, so they are 611 * overridden by the ttbcr/pte. 612 */ 613 if (stage1) { 614 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG, 615 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) | 616 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR, 617 ARM_SMMU_CBAR_S1_MEMATTR_WB); 618 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) { 619 /* 8-bit VMIDs live in CBAR */ 620 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid); 621 } 622 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg); 623 624 /* 625 * TCR 626 * We must write this before the TTBRs, since it determines the 627 * access behaviour of some fields (in particular, ASID[15:8]). 628 */ 629 if (stage1 && smmu->version > ARM_SMMU_V1) 630 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]); 631 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]); 632 633 /* TTBRs */ 634 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 635 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid); 636 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]); 637 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]); 638 } else { 639 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]); 640 if (stage1) 641 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1, 642 cb->ttbr[1]); 643 } 644 645 /* MAIRs (stage-1 only) */ 646 if (stage1) { 647 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]); 648 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]); 649 } 650 651 /* SCTLR */ 652 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE | 653 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M; 654 if (stage1) 655 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE; 656 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) 657 reg |= ARM_SMMU_SCTLR_E; 658 659 if (smmu->impl && smmu->impl->write_sctlr) 660 smmu->impl->write_sctlr(smmu, idx, reg); 661 else 662 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg); 663 } 664 665 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain, 666 struct arm_smmu_device *smmu, 667 struct device *dev, unsigned int start) 668 { 669 if (smmu->impl && smmu->impl->alloc_context_bank) 670 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start); 671 672 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks); 673 } 674 675 static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain, 676 struct arm_smmu_device *smmu, 677 struct device *dev) 678 { 679 int irq, start, ret = 0; 680 unsigned long ias, oas; 681 struct io_pgtable_ops *pgtbl_ops; 682 struct io_pgtable_cfg pgtbl_cfg; 683 enum io_pgtable_fmt fmt; 684 struct iommu_domain *domain = &smmu_domain->domain; 685 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 686 irqreturn_t (*context_fault)(int irq, void *dev); 687 688 mutex_lock(&smmu_domain->init_mutex); 689 if (smmu_domain->smmu) 690 goto out_unlock; 691 692 /* 693 * Mapping the requested stage onto what we support is surprisingly 694 * complicated, mainly because the spec allows S1+S2 SMMUs without 695 * support for nested translation. That means we end up with the 696 * following table: 697 * 698 * Requested Supported Actual 699 * S1 N S1 700 * S1 S1+S2 S1 701 * S1 S2 S2 702 * S1 S1 S1 703 * N N N 704 * N S1+S2 S2 705 * N S2 S2 706 * N S1 S1 707 * 708 * Note that you can't actually request stage-2 mappings. 709 */ 710 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) 711 smmu_domain->stage = ARM_SMMU_DOMAIN_S2; 712 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) 713 smmu_domain->stage = ARM_SMMU_DOMAIN_S1; 714 715 /* 716 * Choosing a suitable context format is even more fiddly. Until we 717 * grow some way for the caller to express a preference, and/or move 718 * the decision into the io-pgtable code where it arguably belongs, 719 * just aim for the closest thing to the rest of the system, and hope 720 * that the hardware isn't esoteric enough that we can't assume AArch64 721 * support to be a superset of AArch32 support... 722 */ 723 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L) 724 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L; 725 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) && 726 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) && 727 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) && 728 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)) 729 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S; 730 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) && 731 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K | 732 ARM_SMMU_FEAT_FMT_AARCH64_16K | 733 ARM_SMMU_FEAT_FMT_AARCH64_4K))) 734 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64; 735 736 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) { 737 ret = -EINVAL; 738 goto out_unlock; 739 } 740 741 switch (smmu_domain->stage) { 742 case ARM_SMMU_DOMAIN_S1: 743 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS; 744 start = smmu->num_s2_context_banks; 745 ias = smmu->va_size; 746 oas = smmu->ipa_size; 747 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { 748 fmt = ARM_64_LPAE_S1; 749 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) { 750 fmt = ARM_32_LPAE_S1; 751 ias = min(ias, 32UL); 752 oas = min(oas, 40UL); 753 } else { 754 fmt = ARM_V7S; 755 ias = min(ias, 32UL); 756 oas = min(oas, 32UL); 757 } 758 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops; 759 break; 760 case ARM_SMMU_DOMAIN_NESTED: 761 /* 762 * We will likely want to change this if/when KVM gets 763 * involved. 764 */ 765 case ARM_SMMU_DOMAIN_S2: 766 cfg->cbar = CBAR_TYPE_S2_TRANS; 767 start = 0; 768 ias = smmu->ipa_size; 769 oas = smmu->pa_size; 770 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { 771 fmt = ARM_64_LPAE_S2; 772 } else { 773 fmt = ARM_32_LPAE_S2; 774 ias = min(ias, 40UL); 775 oas = min(oas, 40UL); 776 } 777 if (smmu->version == ARM_SMMU_V2) 778 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2; 779 else 780 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1; 781 break; 782 default: 783 ret = -EINVAL; 784 goto out_unlock; 785 } 786 787 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start); 788 if (ret < 0) { 789 goto out_unlock; 790 } 791 792 smmu_domain->smmu = smmu; 793 794 cfg->cbndx = ret; 795 if (smmu->version < ARM_SMMU_V2) { 796 cfg->irptndx = atomic_inc_return(&smmu->irptndx); 797 cfg->irptndx %= smmu->num_context_irqs; 798 } else { 799 cfg->irptndx = cfg->cbndx; 800 } 801 802 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2) 803 cfg->vmid = cfg->cbndx + 1; 804 else 805 cfg->asid = cfg->cbndx; 806 807 pgtbl_cfg = (struct io_pgtable_cfg) { 808 .pgsize_bitmap = smmu->pgsize_bitmap, 809 .ias = ias, 810 .oas = oas, 811 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK, 812 .tlb = smmu_domain->flush_ops, 813 .iommu_dev = smmu->dev, 814 }; 815 816 if (smmu->impl && smmu->impl->init_context) { 817 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev); 818 if (ret) 819 goto out_clear_smmu; 820 } 821 822 if (smmu_domain->pgtbl_quirks) 823 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks; 824 825 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); 826 if (!pgtbl_ops) { 827 ret = -ENOMEM; 828 goto out_clear_smmu; 829 } 830 831 /* Update the domain's page sizes to reflect the page table format */ 832 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; 833 834 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) { 835 domain->geometry.aperture_start = ~0UL << ias; 836 domain->geometry.aperture_end = ~0UL; 837 } else { 838 domain->geometry.aperture_end = (1UL << ias) - 1; 839 } 840 841 domain->geometry.force_aperture = true; 842 843 /* Initialise the context bank with our page table cfg */ 844 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); 845 arm_smmu_write_context_bank(smmu, cfg->cbndx); 846 847 /* 848 * Request context fault interrupt. Do this last to avoid the 849 * handler seeing a half-initialised domain state. 850 */ 851 irq = smmu->irqs[cfg->irptndx]; 852 853 if (smmu->impl && smmu->impl->context_fault) 854 context_fault = smmu->impl->context_fault; 855 else 856 context_fault = arm_smmu_context_fault; 857 858 if (smmu->impl && smmu->impl->context_fault_needs_threaded_irq) 859 ret = devm_request_threaded_irq(smmu->dev, irq, NULL, 860 context_fault, 861 IRQF_ONESHOT | IRQF_SHARED, 862 "arm-smmu-context-fault", 863 smmu_domain); 864 else 865 ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED, 866 "arm-smmu-context-fault", smmu_domain); 867 868 if (ret < 0) { 869 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", 870 cfg->irptndx, irq); 871 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX; 872 } 873 874 mutex_unlock(&smmu_domain->init_mutex); 875 876 /* Publish page table ops for map/unmap */ 877 smmu_domain->pgtbl_ops = pgtbl_ops; 878 return 0; 879 880 out_clear_smmu: 881 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); 882 smmu_domain->smmu = NULL; 883 out_unlock: 884 mutex_unlock(&smmu_domain->init_mutex); 885 return ret; 886 } 887 888 static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain) 889 { 890 struct arm_smmu_device *smmu = smmu_domain->smmu; 891 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 892 int ret, irq; 893 894 if (!smmu) 895 return; 896 897 ret = arm_smmu_rpm_get(smmu); 898 if (ret < 0) 899 return; 900 901 /* 902 * Disable the context bank and free the page tables before freeing 903 * it. 904 */ 905 smmu->cbs[cfg->cbndx].cfg = NULL; 906 arm_smmu_write_context_bank(smmu, cfg->cbndx); 907 908 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) { 909 irq = smmu->irqs[cfg->irptndx]; 910 devm_free_irq(smmu->dev, irq, smmu_domain); 911 } 912 913 free_io_pgtable_ops(smmu_domain->pgtbl_ops); 914 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); 915 916 arm_smmu_rpm_put(smmu); 917 } 918 919 static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev) 920 { 921 struct arm_smmu_domain *smmu_domain; 922 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 923 struct arm_smmu_device *smmu = cfg->smmu; 924 925 /* 926 * Allocate the domain and initialise some of its data structures. 927 * We can't really do anything meaningful until we've added a 928 * master. 929 */ 930 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); 931 if (!smmu_domain) 932 return NULL; 933 934 mutex_init(&smmu_domain->init_mutex); 935 spin_lock_init(&smmu_domain->cb_lock); 936 smmu_domain->domain.pgsize_bitmap = smmu->pgsize_bitmap; 937 938 return &smmu_domain->domain; 939 } 940 941 static void arm_smmu_domain_free(struct iommu_domain *domain) 942 { 943 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 944 945 /* 946 * Free the domain resources. We assume that all devices have 947 * already been detached. 948 */ 949 arm_smmu_destroy_domain_context(smmu_domain); 950 kfree(smmu_domain); 951 } 952 953 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx) 954 { 955 struct arm_smmu_smr *smr = smmu->smrs + idx; 956 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) | 957 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask); 958 959 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid) 960 reg |= ARM_SMMU_SMR_VALID; 961 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg); 962 } 963 964 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) 965 { 966 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx; 967 u32 reg; 968 969 if (smmu->impl && smmu->impl->write_s2cr) { 970 smmu->impl->write_s2cr(smmu, idx); 971 return; 972 } 973 974 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) | 975 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) | 976 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg); 977 978 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs && 979 smmu->smrs[idx].valid) 980 reg |= ARM_SMMU_S2CR_EXIDVALID; 981 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg); 982 } 983 984 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) 985 { 986 arm_smmu_write_s2cr(smmu, idx); 987 if (smmu->smrs) 988 arm_smmu_write_smr(smmu, idx); 989 } 990 991 /* 992 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function 993 * should be called after sCR0 is written. 994 */ 995 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu) 996 { 997 u32 smr; 998 int i; 999 1000 if (!smmu->smrs) 1001 return; 1002 /* 1003 * If we've had to accommodate firmware memory regions, we may 1004 * have live SMRs by now; tread carefully... 1005 * 1006 * Somewhat perversely, not having a free SMR for this test implies we 1007 * can get away without it anyway, as we'll only be able to 'allocate' 1008 * these SMRs for the ID/mask values we're already trusting to be OK. 1009 */ 1010 for (i = 0; i < smmu->num_mapping_groups; i++) 1011 if (!smmu->smrs[i].valid) 1012 goto smr_ok; 1013 return; 1014 smr_ok: 1015 /* 1016 * SMR.ID bits may not be preserved if the corresponding MASK 1017 * bits are set, so check each one separately. We can reject 1018 * masters later if they try to claim IDs outside these masks. 1019 */ 1020 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask); 1021 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr); 1022 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); 1023 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr); 1024 1025 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask); 1026 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr); 1027 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); 1028 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr); 1029 } 1030 1031 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask) 1032 { 1033 struct arm_smmu_smr *smrs = smmu->smrs; 1034 int i, free_idx = -ENOSPC; 1035 1036 /* Stream indexing is blissfully easy */ 1037 if (!smrs) 1038 return id; 1039 1040 /* Validating SMRs is... less so */ 1041 for (i = 0; i < smmu->num_mapping_groups; ++i) { 1042 if (!smrs[i].valid) { 1043 /* 1044 * Note the first free entry we come across, which 1045 * we'll claim in the end if nothing else matches. 1046 */ 1047 if (free_idx < 0) 1048 free_idx = i; 1049 continue; 1050 } 1051 /* 1052 * If the new entry is _entirely_ matched by an existing entry, 1053 * then reuse that, with the guarantee that there also cannot 1054 * be any subsequent conflicting entries. In normal use we'd 1055 * expect simply identical entries for this case, but there's 1056 * no harm in accommodating the generalisation. 1057 */ 1058 if ((mask & smrs[i].mask) == mask && 1059 !((id ^ smrs[i].id) & ~smrs[i].mask)) 1060 return i; 1061 /* 1062 * If the new entry has any other overlap with an existing one, 1063 * though, then there always exists at least one stream ID 1064 * which would cause a conflict, and we can't allow that risk. 1065 */ 1066 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask))) 1067 return -EINVAL; 1068 } 1069 1070 return free_idx; 1071 } 1072 1073 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) 1074 { 1075 if (--smmu->s2crs[idx].count) 1076 return false; 1077 1078 smmu->s2crs[idx] = s2cr_init_val; 1079 if (smmu->smrs) 1080 smmu->smrs[idx].valid = false; 1081 1082 return true; 1083 } 1084 1085 static int arm_smmu_master_alloc_smes(struct device *dev) 1086 { 1087 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1088 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1089 struct arm_smmu_device *smmu = cfg->smmu; 1090 struct arm_smmu_smr *smrs = smmu->smrs; 1091 int i, idx, ret; 1092 1093 mutex_lock(&smmu->stream_map_mutex); 1094 /* Figure out a viable stream map entry allocation */ 1095 for_each_cfg_sme(cfg, fwspec, i, idx) { 1096 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); 1097 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]); 1098 1099 if (idx != INVALID_SMENDX) { 1100 ret = -EEXIST; 1101 goto out_err; 1102 } 1103 1104 ret = arm_smmu_find_sme(smmu, sid, mask); 1105 if (ret < 0) 1106 goto out_err; 1107 1108 idx = ret; 1109 if (smrs && smmu->s2crs[idx].count == 0) { 1110 smrs[idx].id = sid; 1111 smrs[idx].mask = mask; 1112 smrs[idx].valid = true; 1113 } 1114 smmu->s2crs[idx].count++; 1115 cfg->smendx[i] = (s16)idx; 1116 } 1117 1118 /* It worked! Now, poke the actual hardware */ 1119 for_each_cfg_sme(cfg, fwspec, i, idx) 1120 arm_smmu_write_sme(smmu, idx); 1121 1122 mutex_unlock(&smmu->stream_map_mutex); 1123 return 0; 1124 1125 out_err: 1126 while (i--) { 1127 arm_smmu_free_sme(smmu, cfg->smendx[i]); 1128 cfg->smendx[i] = INVALID_SMENDX; 1129 } 1130 mutex_unlock(&smmu->stream_map_mutex); 1131 return ret; 1132 } 1133 1134 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg, 1135 struct iommu_fwspec *fwspec) 1136 { 1137 struct arm_smmu_device *smmu = cfg->smmu; 1138 int i, idx; 1139 1140 mutex_lock(&smmu->stream_map_mutex); 1141 for_each_cfg_sme(cfg, fwspec, i, idx) { 1142 if (arm_smmu_free_sme(smmu, idx)) 1143 arm_smmu_write_sme(smmu, idx); 1144 cfg->smendx[i] = INVALID_SMENDX; 1145 } 1146 mutex_unlock(&smmu->stream_map_mutex); 1147 } 1148 1149 static void arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg *cfg, 1150 enum arm_smmu_s2cr_type type, 1151 u8 cbndx, struct iommu_fwspec *fwspec) 1152 { 1153 struct arm_smmu_device *smmu = cfg->smmu; 1154 struct arm_smmu_s2cr *s2cr = smmu->s2crs; 1155 int i, idx; 1156 1157 for_each_cfg_sme(cfg, fwspec, i, idx) { 1158 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx) 1159 continue; 1160 1161 s2cr[idx].type = type; 1162 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT; 1163 s2cr[idx].cbndx = cbndx; 1164 arm_smmu_write_s2cr(smmu, idx); 1165 } 1166 } 1167 1168 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) 1169 { 1170 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1171 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1172 struct arm_smmu_master_cfg *cfg; 1173 struct arm_smmu_device *smmu; 1174 int ret; 1175 1176 /* 1177 * FIXME: The arch/arm DMA API code tries to attach devices to its own 1178 * domains between of_xlate() and probe_device() - we have no way to cope 1179 * with that, so until ARM gets converted to rely on groups and default 1180 * domains, just say no (but more politely than by dereferencing NULL). 1181 * This should be at least a WARN_ON once that's sorted. 1182 */ 1183 cfg = dev_iommu_priv_get(dev); 1184 if (!cfg) 1185 return -ENODEV; 1186 1187 smmu = cfg->smmu; 1188 1189 ret = arm_smmu_rpm_get(smmu); 1190 if (ret < 0) 1191 return ret; 1192 1193 /* Ensure that the domain is finalised */ 1194 ret = arm_smmu_init_domain_context(smmu_domain, smmu, dev); 1195 if (ret < 0) 1196 goto rpm_put; 1197 1198 /* 1199 * Sanity check the domain. We don't support domains across 1200 * different SMMUs. 1201 */ 1202 if (smmu_domain->smmu != smmu) { 1203 ret = -EINVAL; 1204 goto rpm_put; 1205 } 1206 1207 /* Looks ok, so add the device to the domain */ 1208 arm_smmu_master_install_s2crs(cfg, S2CR_TYPE_TRANS, 1209 smmu_domain->cfg.cbndx, fwspec); 1210 rpm_put: 1211 arm_smmu_rpm_put(smmu); 1212 return ret; 1213 } 1214 1215 static int arm_smmu_attach_dev_type(struct device *dev, 1216 enum arm_smmu_s2cr_type type) 1217 { 1218 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1219 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1220 struct arm_smmu_device *smmu; 1221 int ret; 1222 1223 if (!cfg) 1224 return -ENODEV; 1225 smmu = cfg->smmu; 1226 1227 ret = arm_smmu_rpm_get(smmu); 1228 if (ret < 0) 1229 return ret; 1230 1231 arm_smmu_master_install_s2crs(cfg, type, 0, fwspec); 1232 arm_smmu_rpm_put(smmu); 1233 return 0; 1234 } 1235 1236 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain, 1237 struct device *dev) 1238 { 1239 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_BYPASS); 1240 } 1241 1242 static const struct iommu_domain_ops arm_smmu_identity_ops = { 1243 .attach_dev = arm_smmu_attach_dev_identity, 1244 }; 1245 1246 static struct iommu_domain arm_smmu_identity_domain = { 1247 .type = IOMMU_DOMAIN_IDENTITY, 1248 .ops = &arm_smmu_identity_ops, 1249 }; 1250 1251 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain, 1252 struct device *dev) 1253 { 1254 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_FAULT); 1255 } 1256 1257 static const struct iommu_domain_ops arm_smmu_blocked_ops = { 1258 .attach_dev = arm_smmu_attach_dev_blocked, 1259 }; 1260 1261 static struct iommu_domain arm_smmu_blocked_domain = { 1262 .type = IOMMU_DOMAIN_BLOCKED, 1263 .ops = &arm_smmu_blocked_ops, 1264 }; 1265 1266 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, 1267 phys_addr_t paddr, size_t pgsize, size_t pgcount, 1268 int prot, gfp_t gfp, size_t *mapped) 1269 { 1270 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 1271 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; 1272 int ret; 1273 1274 if (!ops) 1275 return -ENODEV; 1276 1277 arm_smmu_rpm_get(smmu); 1278 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped); 1279 arm_smmu_rpm_put(smmu); 1280 1281 return ret; 1282 } 1283 1284 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova, 1285 size_t pgsize, size_t pgcount, 1286 struct iommu_iotlb_gather *iotlb_gather) 1287 { 1288 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 1289 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; 1290 size_t ret; 1291 1292 if (!ops) 1293 return 0; 1294 1295 arm_smmu_rpm_get(smmu); 1296 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather); 1297 arm_smmu_rpm_put(smmu); 1298 1299 return ret; 1300 } 1301 1302 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain) 1303 { 1304 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1305 struct arm_smmu_device *smmu = smmu_domain->smmu; 1306 1307 if (smmu_domain->flush_ops) { 1308 arm_smmu_rpm_get(smmu); 1309 smmu_domain->flush_ops->tlb_flush_all(smmu_domain); 1310 arm_smmu_rpm_put(smmu); 1311 } 1312 } 1313 1314 static void arm_smmu_iotlb_sync(struct iommu_domain *domain, 1315 struct iommu_iotlb_gather *gather) 1316 { 1317 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1318 struct arm_smmu_device *smmu = smmu_domain->smmu; 1319 1320 if (!smmu) 1321 return; 1322 1323 arm_smmu_rpm_get(smmu); 1324 if (smmu->version == ARM_SMMU_V2 || 1325 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) 1326 arm_smmu_tlb_sync_context(smmu_domain); 1327 else 1328 arm_smmu_tlb_sync_global(smmu); 1329 arm_smmu_rpm_put(smmu); 1330 } 1331 1332 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, 1333 dma_addr_t iova) 1334 { 1335 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1336 struct arm_smmu_device *smmu = smmu_domain->smmu; 1337 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 1338 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; 1339 struct device *dev = smmu->dev; 1340 void __iomem *reg; 1341 u32 tmp; 1342 u64 phys; 1343 unsigned long va, flags; 1344 int ret, idx = cfg->cbndx; 1345 phys_addr_t addr = 0; 1346 1347 ret = arm_smmu_rpm_get(smmu); 1348 if (ret < 0) 1349 return 0; 1350 1351 spin_lock_irqsave(&smmu_domain->cb_lock, flags); 1352 va = iova & ~0xfffUL; 1353 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 1354 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va); 1355 else 1356 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va); 1357 1358 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR; 1359 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_CB_ATSR_ACTIVE), 1360 5, 50)) { 1361 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 1362 dev_err(dev, 1363 "iova to phys timed out on %pad. Falling back to software table walk.\n", 1364 &iova); 1365 arm_smmu_rpm_put(smmu); 1366 return ops->iova_to_phys(ops, iova); 1367 } 1368 1369 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR); 1370 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 1371 if (phys & ARM_SMMU_CB_PAR_F) { 1372 dev_err(dev, "translation fault!\n"); 1373 dev_err(dev, "PAR = 0x%llx\n", phys); 1374 goto out; 1375 } 1376 1377 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff); 1378 out: 1379 arm_smmu_rpm_put(smmu); 1380 1381 return addr; 1382 } 1383 1384 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, 1385 dma_addr_t iova) 1386 { 1387 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1388 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; 1389 1390 if (!ops) 1391 return 0; 1392 1393 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && 1394 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) 1395 return arm_smmu_iova_to_phys_hard(domain, iova); 1396 1397 return ops->iova_to_phys(ops, iova); 1398 } 1399 1400 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap) 1401 { 1402 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1403 1404 switch (cap) { 1405 case IOMMU_CAP_CACHE_COHERENCY: 1406 /* 1407 * It's overwhelmingly the case in practice that when the pagetable 1408 * walk interface is connected to a coherent interconnect, all the 1409 * translation interfaces are too. Furthermore if the device is 1410 * natively coherent, then its translation interface must also be. 1411 */ 1412 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK || 1413 device_get_dma_attr(dev) == DEV_DMA_COHERENT; 1414 case IOMMU_CAP_NOEXEC: 1415 case IOMMU_CAP_DEFERRED_FLUSH: 1416 return true; 1417 default: 1418 return false; 1419 } 1420 } 1421 1422 static 1423 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode) 1424 { 1425 struct device *dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode); 1426 1427 put_device(dev); 1428 return dev ? dev_get_drvdata(dev) : NULL; 1429 } 1430 1431 static struct iommu_device *arm_smmu_probe_device(struct device *dev) 1432 { 1433 struct arm_smmu_device *smmu = NULL; 1434 struct arm_smmu_master_cfg *cfg; 1435 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1436 int i, ret; 1437 1438 if (using_legacy_binding) { 1439 ret = arm_smmu_register_legacy_master(dev, &smmu); 1440 1441 /* 1442 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master() 1443 * will allocate/initialise a new one. Thus we need to update fwspec for 1444 * later use. 1445 */ 1446 fwspec = dev_iommu_fwspec_get(dev); 1447 if (ret) 1448 goto out_free; 1449 } else { 1450 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); 1451 } 1452 1453 ret = -EINVAL; 1454 for (i = 0; i < fwspec->num_ids; i++) { 1455 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); 1456 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]); 1457 1458 if (sid & ~smmu->streamid_mask) { 1459 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n", 1460 sid, smmu->streamid_mask); 1461 goto out_free; 1462 } 1463 if (mask & ~smmu->smr_mask_mask) { 1464 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n", 1465 mask, smmu->smr_mask_mask); 1466 goto out_free; 1467 } 1468 } 1469 1470 ret = -ENOMEM; 1471 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]), 1472 GFP_KERNEL); 1473 if (!cfg) 1474 goto out_free; 1475 1476 cfg->smmu = smmu; 1477 dev_iommu_priv_set(dev, cfg); 1478 while (i--) 1479 cfg->smendx[i] = INVALID_SMENDX; 1480 1481 ret = arm_smmu_rpm_get(smmu); 1482 if (ret < 0) 1483 goto out_cfg_free; 1484 1485 ret = arm_smmu_master_alloc_smes(dev); 1486 arm_smmu_rpm_put(smmu); 1487 1488 if (ret) 1489 goto out_cfg_free; 1490 1491 device_link_add(dev, smmu->dev, 1492 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); 1493 1494 return &smmu->iommu; 1495 1496 out_cfg_free: 1497 kfree(cfg); 1498 out_free: 1499 return ERR_PTR(ret); 1500 } 1501 1502 static void arm_smmu_release_device(struct device *dev) 1503 { 1504 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1505 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1506 int ret; 1507 1508 ret = arm_smmu_rpm_get(cfg->smmu); 1509 if (ret < 0) 1510 return; 1511 1512 arm_smmu_master_free_smes(cfg, fwspec); 1513 1514 arm_smmu_rpm_put(cfg->smmu); 1515 1516 kfree(cfg); 1517 } 1518 1519 static void arm_smmu_probe_finalize(struct device *dev) 1520 { 1521 struct arm_smmu_master_cfg *cfg; 1522 struct arm_smmu_device *smmu; 1523 1524 cfg = dev_iommu_priv_get(dev); 1525 smmu = cfg->smmu; 1526 1527 if (smmu->impl && smmu->impl->probe_finalize) 1528 smmu->impl->probe_finalize(smmu, dev); 1529 } 1530 1531 static struct iommu_group *arm_smmu_device_group(struct device *dev) 1532 { 1533 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1534 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1535 struct arm_smmu_device *smmu = cfg->smmu; 1536 struct iommu_group *group = NULL; 1537 int i, idx; 1538 1539 mutex_lock(&smmu->stream_map_mutex); 1540 for_each_cfg_sme(cfg, fwspec, i, idx) { 1541 if (group && smmu->s2crs[idx].group && 1542 group != smmu->s2crs[idx].group) { 1543 mutex_unlock(&smmu->stream_map_mutex); 1544 return ERR_PTR(-EINVAL); 1545 } 1546 1547 group = smmu->s2crs[idx].group; 1548 } 1549 1550 if (group) { 1551 mutex_unlock(&smmu->stream_map_mutex); 1552 return iommu_group_ref_get(group); 1553 } 1554 1555 if (dev_is_pci(dev)) 1556 group = pci_device_group(dev); 1557 else if (dev_is_fsl_mc(dev)) 1558 group = fsl_mc_device_group(dev); 1559 else 1560 group = generic_device_group(dev); 1561 1562 /* Remember group for faster lookups */ 1563 if (!IS_ERR(group)) 1564 for_each_cfg_sme(cfg, fwspec, i, idx) 1565 smmu->s2crs[idx].group = group; 1566 1567 mutex_unlock(&smmu->stream_map_mutex); 1568 return group; 1569 } 1570 1571 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain, 1572 unsigned long quirks) 1573 { 1574 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1575 int ret = 0; 1576 1577 mutex_lock(&smmu_domain->init_mutex); 1578 if (smmu_domain->smmu) 1579 ret = -EPERM; 1580 else 1581 smmu_domain->pgtbl_quirks = quirks; 1582 mutex_unlock(&smmu_domain->init_mutex); 1583 1584 return ret; 1585 } 1586 1587 static int arm_smmu_of_xlate(struct device *dev, 1588 const struct of_phandle_args *args) 1589 { 1590 u32 mask, fwid = 0; 1591 1592 if (args->args_count > 0) 1593 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]); 1594 1595 if (args->args_count > 1) 1596 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]); 1597 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask)) 1598 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask); 1599 1600 return iommu_fwspec_add_ids(dev, &fwid, 1); 1601 } 1602 1603 static void arm_smmu_get_resv_regions(struct device *dev, 1604 struct list_head *head) 1605 { 1606 struct iommu_resv_region *region; 1607 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; 1608 1609 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH, 1610 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL); 1611 if (!region) 1612 return; 1613 1614 list_add_tail(®ion->list, head); 1615 1616 iommu_dma_get_resv_regions(dev, head); 1617 } 1618 1619 static int arm_smmu_def_domain_type(struct device *dev) 1620 { 1621 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1622 const struct arm_smmu_impl *impl = cfg->smmu->impl; 1623 1624 if (using_legacy_binding) 1625 return IOMMU_DOMAIN_IDENTITY; 1626 1627 if (impl && impl->def_domain_type) 1628 return impl->def_domain_type(dev); 1629 1630 return 0; 1631 } 1632 1633 static const struct iommu_ops arm_smmu_ops = { 1634 .identity_domain = &arm_smmu_identity_domain, 1635 .blocked_domain = &arm_smmu_blocked_domain, 1636 .capable = arm_smmu_capable, 1637 .domain_alloc_paging = arm_smmu_domain_alloc_paging, 1638 .probe_device = arm_smmu_probe_device, 1639 .release_device = arm_smmu_release_device, 1640 .probe_finalize = arm_smmu_probe_finalize, 1641 .device_group = arm_smmu_device_group, 1642 .of_xlate = arm_smmu_of_xlate, 1643 .get_resv_regions = arm_smmu_get_resv_regions, 1644 .def_domain_type = arm_smmu_def_domain_type, 1645 .owner = THIS_MODULE, 1646 .default_domain_ops = &(const struct iommu_domain_ops) { 1647 .attach_dev = arm_smmu_attach_dev, 1648 .map_pages = arm_smmu_map_pages, 1649 .unmap_pages = arm_smmu_unmap_pages, 1650 .flush_iotlb_all = arm_smmu_flush_iotlb_all, 1651 .iotlb_sync = arm_smmu_iotlb_sync, 1652 .iova_to_phys = arm_smmu_iova_to_phys, 1653 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks, 1654 .free = arm_smmu_domain_free, 1655 } 1656 }; 1657 1658 static void arm_smmu_device_reset(struct arm_smmu_device *smmu) 1659 { 1660 int i; 1661 u32 reg; 1662 1663 /* clear global FSR */ 1664 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); 1665 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg); 1666 1667 /* 1668 * Reset stream mapping groups: Initial values mark all SMRn as 1669 * invalid and all S2CRn as bypass unless overridden. 1670 */ 1671 for (i = 0; i < smmu->num_mapping_groups; ++i) 1672 arm_smmu_write_sme(smmu, i); 1673 1674 /* Make sure all context banks are disabled and clear CB_FSR */ 1675 for (i = 0; i < smmu->num_context_banks; ++i) { 1676 arm_smmu_write_context_bank(smmu, i); 1677 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_CB_FSR_FAULT); 1678 } 1679 1680 /* Invalidate the TLB, just in case */ 1681 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL); 1682 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL); 1683 1684 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0); 1685 1686 /* Enable fault reporting */ 1687 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE | 1688 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE); 1689 1690 /* Disable TLB broadcasting. */ 1691 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM); 1692 1693 /* Enable client access, handling unmatched streams as appropriate */ 1694 reg &= ~ARM_SMMU_sCR0_CLIENTPD; 1695 if (disable_bypass) 1696 reg |= ARM_SMMU_sCR0_USFCFG; 1697 else 1698 reg &= ~ARM_SMMU_sCR0_USFCFG; 1699 1700 /* Disable forced broadcasting */ 1701 reg &= ~ARM_SMMU_sCR0_FB; 1702 1703 /* Don't upgrade barriers */ 1704 reg &= ~(ARM_SMMU_sCR0_BSU); 1705 1706 if (smmu->features & ARM_SMMU_FEAT_VMID16) 1707 reg |= ARM_SMMU_sCR0_VMID16EN; 1708 1709 if (smmu->features & ARM_SMMU_FEAT_EXIDS) 1710 reg |= ARM_SMMU_sCR0_EXIDENABLE; 1711 1712 if (smmu->impl && smmu->impl->reset) 1713 smmu->impl->reset(smmu); 1714 1715 /* Push the button */ 1716 arm_smmu_tlb_sync_global(smmu); 1717 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg); 1718 } 1719 1720 static int arm_smmu_id_size_to_bits(int size) 1721 { 1722 switch (size) { 1723 case 0: 1724 return 32; 1725 case 1: 1726 return 36; 1727 case 2: 1728 return 40; 1729 case 3: 1730 return 42; 1731 case 4: 1732 return 44; 1733 case 5: 1734 default: 1735 return 48; 1736 } 1737 } 1738 1739 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) 1740 { 1741 unsigned int size; 1742 u32 id; 1743 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK; 1744 int i, ret; 1745 1746 dev_notice(smmu->dev, "probing hardware configuration...\n"); 1747 dev_notice(smmu->dev, "SMMUv%d with:\n", 1748 smmu->version == ARM_SMMU_V2 ? 2 : 1); 1749 1750 /* ID0 */ 1751 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0); 1752 1753 /* Restrict available stages based on module parameter */ 1754 if (force_stage == 1) 1755 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS); 1756 else if (force_stage == 2) 1757 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS); 1758 1759 if (id & ARM_SMMU_ID0_S1TS) { 1760 smmu->features |= ARM_SMMU_FEAT_TRANS_S1; 1761 dev_notice(smmu->dev, "\tstage 1 translation\n"); 1762 } 1763 1764 if (id & ARM_SMMU_ID0_S2TS) { 1765 smmu->features |= ARM_SMMU_FEAT_TRANS_S2; 1766 dev_notice(smmu->dev, "\tstage 2 translation\n"); 1767 } 1768 1769 if (id & ARM_SMMU_ID0_NTS) { 1770 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED; 1771 dev_notice(smmu->dev, "\tnested translation\n"); 1772 } 1773 1774 if (!(smmu->features & 1775 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) { 1776 dev_err(smmu->dev, "\tno translation support!\n"); 1777 return -ENODEV; 1778 } 1779 1780 if ((id & ARM_SMMU_ID0_S1TS) && 1781 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) { 1782 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; 1783 dev_notice(smmu->dev, "\taddress translation ops\n"); 1784 } 1785 1786 /* 1787 * In order for DMA API calls to work properly, we must defer to what 1788 * the FW says about coherency, regardless of what the hardware claims. 1789 * Fortunately, this also opens up a workaround for systems where the 1790 * ID register value has ended up configured incorrectly. 1791 */ 1792 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW); 1793 if (cttw_fw || cttw_reg) 1794 dev_notice(smmu->dev, "\t%scoherent table walk\n", 1795 cttw_fw ? "" : "non-"); 1796 if (cttw_fw != cttw_reg) 1797 dev_notice(smmu->dev, 1798 "\t(IDR0.CTTW overridden by FW configuration)\n"); 1799 1800 /* Max. number of entries we have for stream matching/indexing */ 1801 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) { 1802 smmu->features |= ARM_SMMU_FEAT_EXIDS; 1803 size = 1 << 16; 1804 } else { 1805 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id); 1806 } 1807 smmu->streamid_mask = size - 1; 1808 if (id & ARM_SMMU_ID0_SMS) { 1809 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH; 1810 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id); 1811 if (size == 0) { 1812 dev_err(smmu->dev, 1813 "stream-matching supported, but no SMRs present!\n"); 1814 return -ENODEV; 1815 } 1816 1817 /* Zero-initialised to mark as invalid */ 1818 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs), 1819 GFP_KERNEL); 1820 if (!smmu->smrs) 1821 return -ENOMEM; 1822 1823 dev_notice(smmu->dev, 1824 "\tstream matching with %u register groups", size); 1825 } 1826 /* s2cr->type == 0 means translation, so initialise explicitly */ 1827 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs), 1828 GFP_KERNEL); 1829 if (!smmu->s2crs) 1830 return -ENOMEM; 1831 for (i = 0; i < size; i++) 1832 smmu->s2crs[i] = s2cr_init_val; 1833 1834 smmu->num_mapping_groups = size; 1835 mutex_init(&smmu->stream_map_mutex); 1836 spin_lock_init(&smmu->global_sync_lock); 1837 1838 if (smmu->version < ARM_SMMU_V2 || 1839 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) { 1840 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L; 1841 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S)) 1842 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S; 1843 } 1844 1845 /* ID1 */ 1846 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1); 1847 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12; 1848 1849 /* Check for size mismatch of SMMU address space from mapped region */ 1850 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1); 1851 if (smmu->numpage != 2 * size << smmu->pgshift) 1852 dev_warn(smmu->dev, 1853 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n", 1854 2 * size << smmu->pgshift, smmu->numpage); 1855 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */ 1856 smmu->numpage = size; 1857 1858 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id); 1859 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id); 1860 if (smmu->num_s2_context_banks > smmu->num_context_banks) { 1861 dev_err(smmu->dev, "impossible number of S2 context banks!\n"); 1862 return -ENODEV; 1863 } 1864 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n", 1865 smmu->num_context_banks, smmu->num_s2_context_banks); 1866 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks, 1867 sizeof(*smmu->cbs), GFP_KERNEL); 1868 if (!smmu->cbs) 1869 return -ENOMEM; 1870 1871 /* ID2 */ 1872 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2); 1873 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id)); 1874 smmu->ipa_size = size; 1875 1876 /* The output mask is also applied for bypass */ 1877 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id)); 1878 smmu->pa_size = size; 1879 1880 if (id & ARM_SMMU_ID2_VMID16) 1881 smmu->features |= ARM_SMMU_FEAT_VMID16; 1882 1883 /* 1884 * What the page table walker can address actually depends on which 1885 * descriptor format is in use, but since a) we don't know that yet, 1886 * and b) it can vary per context bank, this will have to do... 1887 */ 1888 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size))) 1889 dev_warn(smmu->dev, 1890 "failed to set DMA mask for table walker\n"); 1891 1892 if (smmu->version < ARM_SMMU_V2) { 1893 smmu->va_size = smmu->ipa_size; 1894 if (smmu->version == ARM_SMMU_V1_64K) 1895 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; 1896 } else { 1897 size = FIELD_GET(ARM_SMMU_ID2_UBS, id); 1898 smmu->va_size = arm_smmu_id_size_to_bits(size); 1899 if (id & ARM_SMMU_ID2_PTFS_4K) 1900 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K; 1901 if (id & ARM_SMMU_ID2_PTFS_16K) 1902 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K; 1903 if (id & ARM_SMMU_ID2_PTFS_64K) 1904 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; 1905 } 1906 1907 if (smmu->impl && smmu->impl->cfg_probe) { 1908 ret = smmu->impl->cfg_probe(smmu); 1909 if (ret) 1910 return ret; 1911 } 1912 1913 /* Now we've corralled the various formats, what'll it do? */ 1914 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) 1915 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M; 1916 if (smmu->features & 1917 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K)) 1918 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; 1919 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K) 1920 smmu->pgsize_bitmap |= SZ_16K | SZ_32M; 1921 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) 1922 smmu->pgsize_bitmap |= SZ_64K | SZ_512M; 1923 1924 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", 1925 smmu->pgsize_bitmap); 1926 1927 1928 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) 1929 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n", 1930 smmu->va_size, smmu->ipa_size); 1931 1932 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2) 1933 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n", 1934 smmu->ipa_size, smmu->pa_size); 1935 1936 return 0; 1937 } 1938 1939 struct arm_smmu_match_data { 1940 enum arm_smmu_arch_version version; 1941 enum arm_smmu_implementation model; 1942 }; 1943 1944 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \ 1945 static const struct arm_smmu_match_data name = { .version = ver, .model = imp } 1946 1947 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); 1948 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); 1949 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU); 1950 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500); 1951 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); 1952 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2); 1953 1954 static const struct of_device_id arm_smmu_of_match[] = { 1955 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, 1956 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 }, 1957 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, 1958 { .compatible = "arm,mmu-401", .data = &arm_mmu401 }, 1959 { .compatible = "arm,mmu-500", .data = &arm_mmu500 }, 1960 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, 1961 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 }, 1962 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 }, 1963 { }, 1964 }; 1965 MODULE_DEVICE_TABLE(of, arm_smmu_of_match); 1966 1967 #ifdef CONFIG_ACPI 1968 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu) 1969 { 1970 int ret = 0; 1971 1972 switch (model) { 1973 case ACPI_IORT_SMMU_V1: 1974 case ACPI_IORT_SMMU_CORELINK_MMU400: 1975 smmu->version = ARM_SMMU_V1; 1976 smmu->model = GENERIC_SMMU; 1977 break; 1978 case ACPI_IORT_SMMU_CORELINK_MMU401: 1979 smmu->version = ARM_SMMU_V1_64K; 1980 smmu->model = GENERIC_SMMU; 1981 break; 1982 case ACPI_IORT_SMMU_V2: 1983 smmu->version = ARM_SMMU_V2; 1984 smmu->model = GENERIC_SMMU; 1985 break; 1986 case ACPI_IORT_SMMU_CORELINK_MMU500: 1987 smmu->version = ARM_SMMU_V2; 1988 smmu->model = ARM_MMU500; 1989 break; 1990 case ACPI_IORT_SMMU_CAVIUM_THUNDERX: 1991 smmu->version = ARM_SMMU_V2; 1992 smmu->model = CAVIUM_SMMUV2; 1993 break; 1994 default: 1995 ret = -ENODEV; 1996 } 1997 1998 return ret; 1999 } 2000 2001 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu, 2002 u32 *global_irqs, u32 *pmu_irqs) 2003 { 2004 struct device *dev = smmu->dev; 2005 struct acpi_iort_node *node = 2006 *(struct acpi_iort_node **)dev_get_platdata(dev); 2007 struct acpi_iort_smmu *iort_smmu; 2008 int ret; 2009 2010 /* Retrieve SMMU1/2 specific data */ 2011 iort_smmu = (struct acpi_iort_smmu *)node->node_data; 2012 2013 ret = acpi_smmu_get_data(iort_smmu->model, smmu); 2014 if (ret < 0) 2015 return ret; 2016 2017 /* Ignore the configuration access interrupt */ 2018 *global_irqs = 1; 2019 *pmu_irqs = 0; 2020 2021 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK) 2022 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; 2023 2024 return 0; 2025 } 2026 #else 2027 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu, 2028 u32 *global_irqs, u32 *pmu_irqs) 2029 { 2030 return -ENODEV; 2031 } 2032 #endif 2033 2034 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu, 2035 u32 *global_irqs, u32 *pmu_irqs) 2036 { 2037 const struct arm_smmu_match_data *data; 2038 struct device *dev = smmu->dev; 2039 bool legacy_binding; 2040 2041 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs)) 2042 return dev_err_probe(dev, -ENODEV, 2043 "missing #global-interrupts property\n"); 2044 *pmu_irqs = 0; 2045 2046 data = of_device_get_match_data(dev); 2047 smmu->version = data->version; 2048 smmu->model = data->model; 2049 2050 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL); 2051 if (legacy_binding && !using_generic_binding) { 2052 if (!using_legacy_binding) { 2053 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n", 2054 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU"); 2055 } 2056 using_legacy_binding = true; 2057 } else if (!legacy_binding && !using_legacy_binding) { 2058 using_generic_binding = true; 2059 } else { 2060 dev_err(dev, "not probing due to mismatched DT properties\n"); 2061 return -ENODEV; 2062 } 2063 2064 if (of_dma_is_coherent(dev->of_node)) 2065 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; 2066 2067 return 0; 2068 } 2069 2070 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu) 2071 { 2072 struct list_head rmr_list; 2073 struct iommu_resv_region *e; 2074 int idx, cnt = 0; 2075 u32 reg; 2076 2077 INIT_LIST_HEAD(&rmr_list); 2078 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); 2079 2080 /* 2081 * Rather than trying to look at existing mappings that 2082 * are setup by the firmware and then invalidate the ones 2083 * that do no have matching RMR entries, just disable the 2084 * SMMU until it gets enabled again in the reset routine. 2085 */ 2086 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0); 2087 reg |= ARM_SMMU_sCR0_CLIENTPD; 2088 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg); 2089 2090 list_for_each_entry(e, &rmr_list, list) { 2091 struct iommu_iort_rmr_data *rmr; 2092 int i; 2093 2094 rmr = container_of(e, struct iommu_iort_rmr_data, rr); 2095 for (i = 0; i < rmr->num_sids; i++) { 2096 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0); 2097 if (idx < 0) 2098 continue; 2099 2100 if (smmu->s2crs[idx].count == 0) { 2101 smmu->smrs[idx].id = rmr->sids[i]; 2102 smmu->smrs[idx].mask = 0; 2103 smmu->smrs[idx].valid = true; 2104 } 2105 smmu->s2crs[idx].count++; 2106 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS; 2107 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT; 2108 2109 cnt++; 2110 } 2111 } 2112 2113 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt, 2114 str_plural(cnt)); 2115 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); 2116 } 2117 2118 static int arm_smmu_device_probe(struct platform_device *pdev) 2119 { 2120 struct resource *res; 2121 struct arm_smmu_device *smmu; 2122 struct device *dev = &pdev->dev; 2123 int num_irqs, i, err; 2124 u32 global_irqs, pmu_irqs; 2125 irqreturn_t (*global_fault)(int irq, void *dev); 2126 2127 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); 2128 if (!smmu) { 2129 dev_err(dev, "failed to allocate arm_smmu_device\n"); 2130 return -ENOMEM; 2131 } 2132 smmu->dev = dev; 2133 2134 if (dev->of_node) 2135 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs); 2136 else 2137 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs); 2138 if (err) 2139 return err; 2140 2141 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); 2142 if (IS_ERR(smmu->base)) 2143 return PTR_ERR(smmu->base); 2144 smmu->ioaddr = res->start; 2145 2146 /* 2147 * The resource size should effectively match the value of SMMU_TOP; 2148 * stash that temporarily until we know PAGESIZE to validate it with. 2149 */ 2150 smmu->numpage = resource_size(res); 2151 2152 smmu = arm_smmu_impl_init(smmu); 2153 if (IS_ERR(smmu)) 2154 return PTR_ERR(smmu); 2155 2156 num_irqs = platform_irq_count(pdev); 2157 2158 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs; 2159 if (smmu->num_context_irqs <= 0) 2160 return dev_err_probe(dev, -ENODEV, 2161 "found %d interrupts but expected at least %d\n", 2162 num_irqs, global_irqs + pmu_irqs + 1); 2163 2164 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs, 2165 sizeof(*smmu->irqs), GFP_KERNEL); 2166 if (!smmu->irqs) 2167 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n", 2168 smmu->num_context_irqs); 2169 2170 for (i = 0; i < smmu->num_context_irqs; i++) { 2171 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i); 2172 2173 if (irq < 0) 2174 return irq; 2175 smmu->irqs[i] = irq; 2176 } 2177 2178 err = devm_clk_bulk_get_all(dev, &smmu->clks); 2179 if (err < 0) { 2180 dev_err(dev, "failed to get clocks %d\n", err); 2181 return err; 2182 } 2183 smmu->num_clks = err; 2184 2185 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks); 2186 if (err) 2187 return err; 2188 2189 err = arm_smmu_device_cfg_probe(smmu); 2190 if (err) 2191 return err; 2192 2193 if (smmu->version == ARM_SMMU_V2) { 2194 if (smmu->num_context_banks > smmu->num_context_irqs) { 2195 dev_err(dev, 2196 "found only %d context irq(s) but %d required\n", 2197 smmu->num_context_irqs, smmu->num_context_banks); 2198 return -ENODEV; 2199 } 2200 2201 /* Ignore superfluous interrupts */ 2202 smmu->num_context_irqs = smmu->num_context_banks; 2203 } 2204 2205 if (smmu->impl && smmu->impl->global_fault) 2206 global_fault = smmu->impl->global_fault; 2207 else 2208 global_fault = arm_smmu_global_fault; 2209 2210 for (i = 0; i < global_irqs; i++) { 2211 int irq = platform_get_irq(pdev, i); 2212 2213 if (irq < 0) 2214 return irq; 2215 2216 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED, 2217 "arm-smmu global fault", smmu); 2218 if (err) 2219 return dev_err_probe(dev, err, 2220 "failed to request global IRQ %d (%u)\n", 2221 i, irq); 2222 } 2223 2224 platform_set_drvdata(pdev, smmu); 2225 2226 /* Check for RMRs and install bypass SMRs if any */ 2227 arm_smmu_rmr_install_bypass_smr(smmu); 2228 2229 arm_smmu_device_reset(smmu); 2230 arm_smmu_test_smr_masks(smmu); 2231 2232 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL, 2233 "smmu.%pa", &smmu->ioaddr); 2234 if (err) 2235 return dev_err_probe(dev, err, "Failed to register iommu in sysfs\n"); 2236 2237 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, 2238 using_legacy_binding ? NULL : dev); 2239 if (err) { 2240 iommu_device_sysfs_remove(&smmu->iommu); 2241 return dev_err_probe(dev, err, "Failed to register iommu\n"); 2242 } 2243 2244 /* 2245 * We want to avoid touching dev->power.lock in fastpaths unless 2246 * it's really going to do something useful - pm_runtime_enabled() 2247 * can serve as an ideal proxy for that decision. So, conditionally 2248 * enable pm_runtime. 2249 */ 2250 if (dev->pm_domain) { 2251 pm_runtime_set_active(dev); 2252 pm_runtime_enable(dev); 2253 arm_smmu_rpm_use_autosuspend(smmu); 2254 } 2255 2256 return 0; 2257 } 2258 2259 static void arm_smmu_device_shutdown(struct platform_device *pdev) 2260 { 2261 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); 2262 2263 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) 2264 dev_notice(&pdev->dev, "disabling translation\n"); 2265 2266 arm_smmu_rpm_get(smmu); 2267 /* Turn the thing off */ 2268 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD); 2269 arm_smmu_rpm_put(smmu); 2270 2271 if (pm_runtime_enabled(smmu->dev)) 2272 pm_runtime_force_suspend(smmu->dev); 2273 else 2274 clk_bulk_disable(smmu->num_clks, smmu->clks); 2275 2276 clk_bulk_unprepare(smmu->num_clks, smmu->clks); 2277 } 2278 2279 static void arm_smmu_device_remove(struct platform_device *pdev) 2280 { 2281 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); 2282 2283 iommu_device_unregister(&smmu->iommu); 2284 iommu_device_sysfs_remove(&smmu->iommu); 2285 2286 arm_smmu_device_shutdown(pdev); 2287 } 2288 2289 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev) 2290 { 2291 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2292 int ret; 2293 2294 ret = clk_bulk_enable(smmu->num_clks, smmu->clks); 2295 if (ret) 2296 return ret; 2297 2298 arm_smmu_device_reset(smmu); 2299 2300 return 0; 2301 } 2302 2303 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev) 2304 { 2305 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2306 2307 clk_bulk_disable(smmu->num_clks, smmu->clks); 2308 2309 return 0; 2310 } 2311 2312 static int __maybe_unused arm_smmu_pm_resume(struct device *dev) 2313 { 2314 int ret; 2315 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2316 2317 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks); 2318 if (ret) 2319 return ret; 2320 2321 if (pm_runtime_suspended(dev)) 2322 return 0; 2323 2324 ret = arm_smmu_runtime_resume(dev); 2325 if (ret) 2326 clk_bulk_unprepare(smmu->num_clks, smmu->clks); 2327 2328 return ret; 2329 } 2330 2331 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev) 2332 { 2333 int ret = 0; 2334 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2335 2336 if (pm_runtime_suspended(dev)) 2337 goto clk_unprepare; 2338 2339 ret = arm_smmu_runtime_suspend(dev); 2340 if (ret) 2341 return ret; 2342 2343 clk_unprepare: 2344 clk_bulk_unprepare(smmu->num_clks, smmu->clks); 2345 return ret; 2346 } 2347 2348 static const struct dev_pm_ops arm_smmu_pm_ops = { 2349 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume) 2350 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend, 2351 arm_smmu_runtime_resume, NULL) 2352 }; 2353 2354 static struct platform_driver arm_smmu_driver = { 2355 .driver = { 2356 .name = "arm-smmu", 2357 .of_match_table = arm_smmu_of_match, 2358 .pm = &arm_smmu_pm_ops, 2359 .suppress_bind_attrs = true, 2360 }, 2361 .probe = arm_smmu_device_probe, 2362 .remove = arm_smmu_device_remove, 2363 .shutdown = arm_smmu_device_shutdown, 2364 }; 2365 module_platform_driver(arm_smmu_driver); 2366 2367 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); 2368 MODULE_AUTHOR("Will Deacon <will@kernel.org>"); 2369 MODULE_ALIAS("platform:arm-smmu"); 2370 MODULE_LICENSE("GPL v2"); 2371