1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Tegra host1x driver 4 * 5 * Copyright (c) 2010-2013, NVIDIA Corporation. 6 */ 7 8 #include <linux/clk.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/io.h> 12 #include <linux/list.h> 13 #include <linux/module.h> 14 #include <linux/of.h> 15 #include <linux/of_platform.h> 16 #include <linux/platform_device.h> 17 #include <linux/pm_runtime.h> 18 #include <linux/slab.h> 19 20 #include <soc/tegra/common.h> 21 22 #define CREATE_TRACE_POINTS 23 #include <trace/events/host1x.h> 24 #undef CREATE_TRACE_POINTS 25 26 #if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU) 27 #include <asm/dma-iommu.h> 28 #endif 29 30 #include "bus.h" 31 #include "channel.h" 32 #include "context.h" 33 #include "debug.h" 34 #include "dev.h" 35 #include "intr.h" 36 37 #include "hw/host1x01.h" 38 #include "hw/host1x02.h" 39 #include "hw/host1x04.h" 40 #include "hw/host1x05.h" 41 #include "hw/host1x06.h" 42 #include "hw/host1x07.h" 43 #include "hw/host1x08.h" 44 #include "hw/host1x10.h" 45 46 void host1x_common_writel(struct host1x *host1x, u32 v, u32 r) 47 { 48 writel(v, host1x->common_regs + r); 49 } 50 51 void host1x_hypervisor_writel(struct host1x *host1x, u32 v, u32 r) 52 { 53 writel(v, host1x->hv_regs + r); 54 } 55 56 u32 host1x_hypervisor_readl(struct host1x *host1x, u32 r) 57 { 58 return readl(host1x->hv_regs + r); 59 } 60 61 void host1x_sync_writel(struct host1x *host1x, u32 v, u32 r) 62 { 63 void __iomem *sync_regs = host1x->regs + host1x->info->sync_offset; 64 65 writel(v, sync_regs + r); 66 } 67 68 u32 host1x_sync_readl(struct host1x *host1x, u32 r) 69 { 70 void __iomem *sync_regs = host1x->regs + host1x->info->sync_offset; 71 72 return readl(sync_regs + r); 73 } 74 75 #ifdef CONFIG_64BIT 76 u64 host1x_sync_readq(struct host1x *host1x, u32 r) 77 { 78 void __iomem *sync_regs = host1x->regs + host1x->info->sync_offset; 79 80 return readq(sync_regs + r); 81 } 82 #endif 83 84 void host1x_ch_writel(struct host1x_channel *ch, u32 v, u32 r) 85 { 86 writel(v, ch->regs + r); 87 } 88 89 u32 host1x_ch_readl(struct host1x_channel *ch, u32 r) 90 { 91 return readl(ch->regs + r); 92 } 93 94 static const struct host1x_info host1x01_info = { 95 .nb_channels = 8, 96 .nb_pts = 32, 97 .nb_mlocks = 16, 98 .nb_bases = 8, 99 .init = host1x01_init, 100 .sync_offset = 0x3000, 101 .dma_mask = DMA_BIT_MASK(32), 102 .has_wide_gather = false, 103 .has_hypervisor = false, 104 .num_sid_entries = 0, 105 .sid_table = NULL, 106 .reserve_vblank_syncpts = true, 107 }; 108 109 static const struct host1x_info host1x02_info = { 110 .nb_channels = 9, 111 .nb_pts = 32, 112 .nb_mlocks = 16, 113 .nb_bases = 12, 114 .init = host1x02_init, 115 .sync_offset = 0x3000, 116 .dma_mask = DMA_BIT_MASK(32), 117 .has_wide_gather = false, 118 .has_hypervisor = false, 119 .num_sid_entries = 0, 120 .sid_table = NULL, 121 .reserve_vblank_syncpts = true, 122 }; 123 124 static const struct host1x_info host1x04_info = { 125 .nb_channels = 12, 126 .nb_pts = 192, 127 .nb_mlocks = 16, 128 .nb_bases = 64, 129 .init = host1x04_init, 130 .sync_offset = 0x2100, 131 .dma_mask = DMA_BIT_MASK(34), 132 .has_wide_gather = false, 133 .has_hypervisor = false, 134 .num_sid_entries = 0, 135 .sid_table = NULL, 136 .reserve_vblank_syncpts = false, 137 }; 138 139 static const struct host1x_info host1x05_info = { 140 .nb_channels = 14, 141 .nb_pts = 192, 142 .nb_mlocks = 16, 143 .nb_bases = 64, 144 .init = host1x05_init, 145 .sync_offset = 0x2100, 146 .dma_mask = DMA_BIT_MASK(34), 147 .has_wide_gather = false, 148 .has_hypervisor = false, 149 .num_sid_entries = 0, 150 .sid_table = NULL, 151 .reserve_vblank_syncpts = false, 152 }; 153 154 static const struct host1x_sid_entry tegra186_sid_table[] = { 155 { /* SE1 */ .base = 0x1ac8, .offset = 0x90, .limit = 0x90 }, 156 { /* SE2 */ .base = 0x1ad0, .offset = 0x90, .limit = 0x90 }, 157 { /* SE3 */ .base = 0x1ad8, .offset = 0x90, .limit = 0x90 }, 158 { /* SE4 */ .base = 0x1ae0, .offset = 0x90, .limit = 0x90 }, 159 { /* ISP */ .base = 0x1ae8, .offset = 0x50, .limit = 0x50 }, 160 { /* VIC */ .base = 0x1af0, .offset = 0x30, .limit = 0x34 }, 161 { /* NVENC */ .base = 0x1af8, .offset = 0x30, .limit = 0x34 }, 162 { /* NVDEC */ .base = 0x1b00, .offset = 0x30, .limit = 0x34 }, 163 { /* NVJPG */ .base = 0x1b08, .offset = 0x30, .limit = 0x34 }, 164 { /* TSEC */ .base = 0x1b10, .offset = 0x30, .limit = 0x34 }, 165 { /* TSECB */ .base = 0x1b18, .offset = 0x30, .limit = 0x34 }, 166 { /* VI 0 */ .base = 0x1b80, .offset = 0x10000, .limit = 0x10000 }, 167 { /* VI 1 */ .base = 0x1b88, .offset = 0x20000, .limit = 0x20000 }, 168 { /* VI 2 */ .base = 0x1b90, .offset = 0x30000, .limit = 0x30000 }, 169 { /* VI 3 */ .base = 0x1b98, .offset = 0x40000, .limit = 0x40000 }, 170 { /* VI 4 */ .base = 0x1ba0, .offset = 0x50000, .limit = 0x50000 }, 171 { /* VI 5 */ .base = 0x1ba8, .offset = 0x60000, .limit = 0x60000 }, 172 { /* VI 6 */ .base = 0x1bb0, .offset = 0x70000, .limit = 0x70000 }, 173 { /* VI 7 */ .base = 0x1bb8, .offset = 0x80000, .limit = 0x80000 }, 174 { /* VI 8 */ .base = 0x1bc0, .offset = 0x90000, .limit = 0x90000 }, 175 { /* VI 9 */ .base = 0x1bc8, .offset = 0xa0000, .limit = 0xa0000 }, 176 { /* VI 10 */ .base = 0x1bd0, .offset = 0xb0000, .limit = 0xb0000 }, 177 { /* VI 11 */ .base = 0x1bd8, .offset = 0xc0000, .limit = 0xc0000 }, 178 }; 179 180 static const struct host1x_info host1x06_info = { 181 .nb_channels = 63, 182 .nb_pts = 576, 183 .nb_mlocks = 24, 184 .nb_bases = 16, 185 .init = host1x06_init, 186 .sync_offset = 0x0, 187 .dma_mask = DMA_BIT_MASK(40), 188 .has_wide_gather = true, 189 .has_hypervisor = true, 190 .num_sid_entries = ARRAY_SIZE(tegra186_sid_table), 191 .sid_table = tegra186_sid_table, 192 .reserve_vblank_syncpts = false, 193 .skip_reset_assert = true, 194 }; 195 196 static const struct host1x_sid_entry tegra194_sid_table[] = { 197 { /* SE1 */ .base = 0x1ac8, .offset = 0x90, .limit = 0x90 }, 198 { /* SE2 */ .base = 0x1ad0, .offset = 0x90, .limit = 0x90 }, 199 { /* SE3 */ .base = 0x1ad8, .offset = 0x90, .limit = 0x90 }, 200 { /* SE4 */ .base = 0x1ae0, .offset = 0x90, .limit = 0x90 }, 201 { /* ISP */ .base = 0x1ae8, .offset = 0x800, .limit = 0x800 }, 202 { /* VIC */ .base = 0x1af0, .offset = 0x30, .limit = 0x34 }, 203 { /* NVENC */ .base = 0x1af8, .offset = 0x30, .limit = 0x34 }, 204 { /* NVDEC */ .base = 0x1b00, .offset = 0x30, .limit = 0x34 }, 205 { /* NVJPG */ .base = 0x1b08, .offset = 0x30, .limit = 0x34 }, 206 { /* TSEC */ .base = 0x1b10, .offset = 0x30, .limit = 0x34 }, 207 { /* TSECB */ .base = 0x1b18, .offset = 0x30, .limit = 0x34 }, 208 { /* VI */ .base = 0x1b80, .offset = 0x800, .limit = 0x800 }, 209 { /* VI_THI */ .base = 0x1b88, .offset = 0x30, .limit = 0x34 }, 210 { /* ISP_THI */ .base = 0x1b90, .offset = 0x30, .limit = 0x34 }, 211 { /* PVA0_CLUSTER */ .base = 0x1b98, .offset = 0x0, .limit = 0x0 }, 212 { /* PVA0_CLUSTER */ .base = 0x1ba0, .offset = 0x0, .limit = 0x0 }, 213 { /* NVDLA0 */ .base = 0x1ba8, .offset = 0x30, .limit = 0x34 }, 214 { /* NVDLA1 */ .base = 0x1bb0, .offset = 0x30, .limit = 0x34 }, 215 { /* NVENC1 */ .base = 0x1bb8, .offset = 0x30, .limit = 0x34 }, 216 { /* NVDEC1 */ .base = 0x1bc0, .offset = 0x30, .limit = 0x34 }, 217 }; 218 219 static const struct host1x_info host1x07_info = { 220 .nb_channels = 63, 221 .nb_pts = 704, 222 .nb_mlocks = 32, 223 .nb_bases = 0, 224 .init = host1x07_init, 225 .sync_offset = 0x0, 226 .dma_mask = DMA_BIT_MASK(40), 227 .has_wide_gather = true, 228 .has_hypervisor = true, 229 .num_sid_entries = ARRAY_SIZE(tegra194_sid_table), 230 .sid_table = tegra194_sid_table, 231 .reserve_vblank_syncpts = false, 232 }; 233 234 /* 235 * Tegra234 has two stream ID protection tables, one for setting stream IDs 236 * through the channel path via SETSTREAMID, and one for setting them via 237 * MMIO. We program each engine's data stream ID in the channel path table 238 * and firmware stream ID in the MMIO path table. 239 */ 240 static const struct host1x_sid_entry tegra234_sid_table[] = { 241 { /* SE1 MMIO */ .base = 0x1650, .offset = 0x90, .limit = 0x90 }, 242 { /* SE1 ch */ .base = 0x1730, .offset = 0x90, .limit = 0x90 }, 243 { /* SE2 MMIO */ .base = 0x1658, .offset = 0x90, .limit = 0x90 }, 244 { /* SE2 ch */ .base = 0x1738, .offset = 0x90, .limit = 0x90 }, 245 { /* SE4 MMIO */ .base = 0x1660, .offset = 0x90, .limit = 0x90 }, 246 { /* SE4 ch */ .base = 0x1740, .offset = 0x90, .limit = 0x90 }, 247 { /* ISP MMIO */ .base = 0x1680, .offset = 0x800, .limit = 0x800 }, 248 { /* VIC MMIO */ .base = 0x1688, .offset = 0x34, .limit = 0x34 }, 249 { /* VIC ch */ .base = 0x17b8, .offset = 0x30, .limit = 0x30 }, 250 { /* NVENC MMIO */ .base = 0x1690, .offset = 0x34, .limit = 0x34 }, 251 { /* NVENC ch */ .base = 0x17c0, .offset = 0x30, .limit = 0x30 }, 252 { /* NVDEC MMIO */ .base = 0x1698, .offset = 0x34, .limit = 0x34 }, 253 { /* NVDEC ch */ .base = 0x17c8, .offset = 0x30, .limit = 0x30 }, 254 { /* NVJPG MMIO */ .base = 0x16a0, .offset = 0x34, .limit = 0x34 }, 255 { /* NVJPG ch */ .base = 0x17d0, .offset = 0x30, .limit = 0x30 }, 256 { /* TSEC MMIO */ .base = 0x16a8, .offset = 0x30, .limit = 0x34 }, 257 { /* NVJPG1 MMIO */ .base = 0x16b0, .offset = 0x34, .limit = 0x34 }, 258 { /* NVJPG1 ch */ .base = 0x17a8, .offset = 0x30, .limit = 0x30 }, 259 { /* VI MMIO */ .base = 0x16b8, .offset = 0x800, .limit = 0x800 }, 260 { /* VI_THI MMIO */ .base = 0x16c0, .offset = 0x30, .limit = 0x34 }, 261 { /* ISP_THI MMIO */ .base = 0x16c8, .offset = 0x30, .limit = 0x34 }, 262 { /* NVDLA MMIO */ .base = 0x16d8, .offset = 0x30, .limit = 0x34 }, 263 { /* NVDLA ch */ .base = 0x17e0, .offset = 0x30, .limit = 0x34 }, 264 { /* NVDLA1 MMIO */ .base = 0x16e0, .offset = 0x30, .limit = 0x34 }, 265 { /* NVDLA1 ch */ .base = 0x17e8, .offset = 0x30, .limit = 0x34 }, 266 { /* OFA MMIO */ .base = 0x16e8, .offset = 0x34, .limit = 0x34 }, 267 { /* OFA ch */ .base = 0x1768, .offset = 0x30, .limit = 0x30 }, 268 { /* VI2 MMIO */ .base = 0x16f0, .offset = 0x800, .limit = 0x800 }, 269 { /* VI2_THI MMIO */ .base = 0x16f8, .offset = 0x30, .limit = 0x34 }, 270 }; 271 272 static const struct host1x_info host1x08_info = { 273 .nb_channels = 63, 274 .nb_pts = 1024, 275 .nb_mlocks = 24, 276 .nb_bases = 0, 277 .init = host1x08_init, 278 .sync_offset = 0x0, 279 .dma_mask = DMA_BIT_MASK(40), 280 .has_wide_gather = true, 281 .has_hypervisor = true, 282 .has_common = true, 283 .num_sid_entries = ARRAY_SIZE(tegra234_sid_table), 284 .sid_table = tegra234_sid_table, 285 .streamid_vm_table = { 0x1004, 128 }, 286 .classid_vm_table = { 0x1404, 25 }, 287 .mmio_vm_table = { 0x1504, 25 }, 288 .reserve_vblank_syncpts = false, 289 }; 290 291 static const struct host1x_sid_entry tegra264_sid_table[] = { 292 { /* SE1 MMIO */ .base = 0x1650, .offset = 0x90, .limit = 0x90 }, 293 { /* SE2 MMIO */ .base = 0x1658, .offset = 0x90, .limit = 0x90 }, 294 { /* SE4 MMIO */ .base = 0x1660, .offset = 0x90, .limit = 0x90 }, 295 { /* SE1 ch */ .base = 0x1738, .offset = 0x90, .limit = 0x90 }, 296 { /* SE2 ch */ .base = 0x1740, .offset = 0x90, .limit = 0x90 }, 297 { /* SE4 ch */ .base = 0x1748, .offset = 0x90, .limit = 0x90 }, 298 { /* VIC ch */ .base = 0x1790, .offset = 0x30, .limit = 0x30 }, 299 { /* VIC MMIO */ .base = 0x1688, .offset = 0x34, .limit = 0x34 }, 300 { /* TSEC MMIO */ .base = 0x1690, .offset = 0x30, .limit = 0x34 }, 301 { /* VI MMIO */ .base = 0x1698, .offset = 0x800, .limit = 0x800 }, 302 { /* VI_THI MMIO */ .base = 0x16a0, .offset = 0x30, .limit = 0x34 }, 303 { /* ISP MMIO */ .base = 0x1680, .offset = 0x800, .limit = 0x800 }, 304 { /* ISP_THI MMIO */ .base = 0x16a8, .offset = 0x30, .limit = 0x34 }, 305 { /* VI2 MMIO */ .base = 0x16b8, .offset = 0x800, .limit = 0x800 }, 306 { /* VI2_THI MMIO */ .base = 0x16c0, .offset = 0x30, .limit = 0x34 }, 307 { /* ISP1 MMIO */ .base = 0x16c8, .offset = 0x800, .limit = 0x800 }, 308 { /* ISP1_THI MMIO */ .base = 0x16d0, .offset = 0x30, .limit = 0x34 }, 309 }; 310 311 static const struct host1x_info host1x10_info = { 312 .nb_channels = 63, 313 .nb_pts = 1024, 314 .nb_mlocks = 24, 315 .nb_bases = 0, 316 .init = host1x10_init, 317 .sync_offset = 0x0, 318 .dma_mask = DMA_BIT_MASK(40), 319 .has_wide_gather = true, 320 .has_hypervisor = true, 321 .has_common = true, 322 .num_sid_entries = ARRAY_SIZE(tegra264_sid_table), 323 .sid_table = tegra264_sid_table, 324 .streamid_vm_table = { 0x1004, 128 }, 325 .classid_vm_table = { 0x1404, 25 }, 326 .mmio_vm_table = { 0x1504, 25 }, 327 .reserve_vblank_syncpts = false, 328 }; 329 330 static const struct of_device_id host1x_of_match[] = { 331 { .compatible = "nvidia,tegra264-host1x", .data = &host1x10_info, }, 332 { .compatible = "nvidia,tegra234-host1x", .data = &host1x08_info, }, 333 { .compatible = "nvidia,tegra194-host1x", .data = &host1x07_info, }, 334 { .compatible = "nvidia,tegra186-host1x", .data = &host1x06_info, }, 335 { .compatible = "nvidia,tegra210-host1x", .data = &host1x05_info, }, 336 { .compatible = "nvidia,tegra124-host1x", .data = &host1x04_info, }, 337 { .compatible = "nvidia,tegra114-host1x", .data = &host1x02_info, }, 338 { .compatible = "nvidia,tegra30-host1x", .data = &host1x01_info, }, 339 { .compatible = "nvidia,tegra20-host1x", .data = &host1x01_info, }, 340 { }, 341 }; 342 MODULE_DEVICE_TABLE(of, host1x_of_match); 343 344 static void host1x_setup_virtualization_tables(struct host1x *host) 345 { 346 const struct host1x_info *info = host->info; 347 unsigned int i; 348 349 if (!info->has_hypervisor) 350 return; 351 352 for (i = 0; i < info->num_sid_entries; i++) { 353 const struct host1x_sid_entry *entry = &info->sid_table[i]; 354 355 host1x_hypervisor_writel(host, entry->offset, entry->base); 356 host1x_hypervisor_writel(host, entry->limit, entry->base + 4); 357 } 358 359 for (i = 0; i < info->streamid_vm_table.count; i++) { 360 /* Allow access to all stream IDs to all VMs. */ 361 host1x_hypervisor_writel(host, 0xff, info->streamid_vm_table.base + 4 * i); 362 } 363 364 for (i = 0; i < info->classid_vm_table.count; i++) { 365 /* Allow access to all classes to all VMs. */ 366 host1x_hypervisor_writel(host, 0xff, info->classid_vm_table.base + 4 * i); 367 } 368 369 for (i = 0; i < info->mmio_vm_table.count; i++) { 370 /* Use VM1 (that's us) as originator VMID for engine MMIO accesses. */ 371 host1x_hypervisor_writel(host, 0x1, info->mmio_vm_table.base + 4 * i); 372 } 373 } 374 375 static bool host1x_wants_iommu(struct host1x *host1x) 376 { 377 /* Our IOMMU usage policy doesn't currently play well with GART */ 378 if (of_machine_is_compatible("nvidia,tegra20")) 379 return false; 380 381 /* 382 * If we support addressing a maximum of 32 bits of physical memory 383 * and if the host1x firewall is enabled, there's no need to enable 384 * IOMMU support. This can happen for example on Tegra20, Tegra30 385 * and Tegra114. 386 * 387 * Tegra124 and later can address up to 34 bits of physical memory and 388 * many platforms come equipped with more than 2 GiB of system memory, 389 * which requires crossing the 4 GiB boundary. But there's a catch: on 390 * SoCs before Tegra186 (i.e. Tegra124 and Tegra210), the host1x can 391 * only address up to 32 bits of memory in GATHER opcodes, which means 392 * that command buffers need to either be in the first 2 GiB of system 393 * memory (which could quickly lead to memory exhaustion), or command 394 * buffers need to be treated differently from other buffers (which is 395 * not possible with the current ABI). 396 * 397 * A third option is to use the IOMMU in these cases to make sure all 398 * buffers will be mapped into a 32-bit IOVA space that host1x can 399 * address. This allows all of the system memory to be used and works 400 * within the limitations of the host1x on these SoCs. 401 * 402 * In summary, default to enable IOMMU on Tegra124 and later. For any 403 * of the earlier SoCs, only use the IOMMU for additional safety when 404 * the host1x firewall is disabled. 405 */ 406 if (host1x->info->dma_mask <= DMA_BIT_MASK(32)) { 407 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL)) 408 return false; 409 } 410 411 return true; 412 } 413 414 /* 415 * Returns ERR_PTR on failure, NULL if the translation is IDENTITY, otherwise a 416 * valid paging domain. 417 */ 418 static struct iommu_domain *host1x_iommu_attach(struct host1x *host) 419 { 420 struct iommu_domain *domain = iommu_get_domain_for_dev(host->dev); 421 int err; 422 423 #if IS_ENABLED(CONFIG_ARM_DMA_USE_IOMMU) 424 if (host->dev->archdata.mapping) { 425 struct dma_iommu_mapping *mapping = 426 to_dma_iommu_mapping(host->dev); 427 arm_iommu_detach_device(host->dev); 428 arm_iommu_release_mapping(mapping); 429 430 domain = iommu_get_domain_for_dev(host->dev); 431 } 432 #endif 433 434 /* 435 * We may not always want to enable IOMMU support (for example if the 436 * host1x firewall is already enabled and we don't support addressing 437 * more than 32 bits of physical memory), so check for that first. 438 * 439 * Similarly, if host1x is already attached to an IOMMU (via the DMA 440 * API), don't try to attach again. 441 */ 442 if (domain && domain->type == IOMMU_DOMAIN_IDENTITY) 443 domain = NULL; 444 if (!host1x_wants_iommu(host) || domain) 445 return domain; 446 447 host->group = iommu_group_get(host->dev); 448 if (host->group) { 449 struct iommu_domain_geometry *geometry; 450 dma_addr_t start, end; 451 unsigned long order; 452 453 err = iova_cache_get(); 454 if (err < 0) 455 goto put_group; 456 457 host->domain = iommu_paging_domain_alloc(host->dev); 458 if (IS_ERR(host->domain)) { 459 err = PTR_ERR(host->domain); 460 host->domain = NULL; 461 goto put_cache; 462 } 463 464 err = iommu_attach_group(host->domain, host->group); 465 if (err) { 466 if (err == -ENODEV) 467 err = 0; 468 469 goto free_domain; 470 } 471 472 geometry = &host->domain->geometry; 473 start = geometry->aperture_start & host->info->dma_mask; 474 end = geometry->aperture_end & host->info->dma_mask; 475 476 order = __ffs(host->domain->pgsize_bitmap); 477 init_iova_domain(&host->iova, 1UL << order, start >> order); 478 host->iova_end = end; 479 480 domain = host->domain; 481 } 482 483 return domain; 484 485 free_domain: 486 iommu_domain_free(host->domain); 487 host->domain = NULL; 488 put_cache: 489 iova_cache_put(); 490 put_group: 491 iommu_group_put(host->group); 492 host->group = NULL; 493 494 return ERR_PTR(err); 495 } 496 497 static int host1x_iommu_init(struct host1x *host) 498 { 499 u64 mask = host->info->dma_mask; 500 struct iommu_domain *domain; 501 int err; 502 503 domain = host1x_iommu_attach(host); 504 if (IS_ERR(domain)) { 505 err = PTR_ERR(domain); 506 dev_err(host->dev, "failed to attach to IOMMU: %d\n", err); 507 return err; 508 } 509 510 /* 511 * If we're not behind an IOMMU make sure we don't get push buffers 512 * that are allocated outside of the range addressable by the GATHER 513 * opcode. 514 * 515 * Newer generations of Tegra (Tegra186 and later) support a wide 516 * variant of the GATHER opcode that allows addressing more bits. 517 */ 518 if (!domain && !host->info->has_wide_gather) 519 mask = DMA_BIT_MASK(32); 520 521 err = dma_coerce_mask_and_coherent(host->dev, mask); 522 if (err < 0) { 523 dev_err(host->dev, "failed to set DMA mask: %d\n", err); 524 return err; 525 } 526 527 return 0; 528 } 529 530 static void host1x_iommu_exit(struct host1x *host) 531 { 532 if (host->domain) { 533 put_iova_domain(&host->iova); 534 iommu_detach_group(host->domain, host->group); 535 536 iommu_domain_free(host->domain); 537 host->domain = NULL; 538 539 iova_cache_put(); 540 541 iommu_group_put(host->group); 542 host->group = NULL; 543 } 544 } 545 546 static int host1x_get_resets(struct host1x *host) 547 { 548 int err; 549 550 host->resets[0].id = "mc"; 551 host->resets[1].id = "host1x"; 552 host->nresets = ARRAY_SIZE(host->resets); 553 554 err = devm_reset_control_bulk_get_optional_exclusive_released( 555 host->dev, host->nresets, host->resets); 556 if (err) { 557 dev_err(host->dev, "failed to get reset: %d\n", err); 558 return err; 559 } 560 561 return 0; 562 } 563 564 static int host1x_probe(struct platform_device *pdev) 565 { 566 struct host1x *host; 567 int err, i; 568 569 host = devm_kzalloc(&pdev->dev, sizeof(*host), GFP_KERNEL); 570 if (!host) 571 return -ENOMEM; 572 573 host->info = of_device_get_match_data(&pdev->dev); 574 575 if (host->info->has_hypervisor) { 576 host->regs = devm_platform_ioremap_resource_byname(pdev, "vm"); 577 if (IS_ERR(host->regs)) 578 return PTR_ERR(host->regs); 579 580 host->hv_regs = devm_platform_ioremap_resource_byname(pdev, "hypervisor"); 581 if (IS_ERR(host->hv_regs)) 582 return PTR_ERR(host->hv_regs); 583 584 if (host->info->has_common) { 585 host->common_regs = devm_platform_ioremap_resource_byname(pdev, "common"); 586 if (IS_ERR(host->common_regs)) 587 return PTR_ERR(host->common_regs); 588 } 589 } else { 590 host->regs = devm_platform_ioremap_resource(pdev, 0); 591 if (IS_ERR(host->regs)) 592 return PTR_ERR(host->regs); 593 } 594 595 for (i = 0; i < ARRAY_SIZE(host->syncpt_irqs); i++) { 596 char irq_name[] = "syncptX"; 597 598 sprintf(irq_name, "syncpt%d", i); 599 600 err = platform_get_irq_byname_optional(pdev, irq_name); 601 if (err == -ENXIO) 602 break; 603 if (err < 0) 604 return err; 605 606 host->syncpt_irqs[i] = err; 607 } 608 609 host->num_syncpt_irqs = i; 610 611 /* Device tree without irq names */ 612 if (i == 0) { 613 host->syncpt_irqs[0] = platform_get_irq(pdev, 0); 614 if (host->syncpt_irqs[0] < 0) 615 return host->syncpt_irqs[0]; 616 617 host->num_syncpt_irqs = 1; 618 } 619 620 mutex_init(&host->devices_lock); 621 INIT_LIST_HEAD(&host->devices); 622 INIT_LIST_HEAD(&host->list); 623 host->dev = &pdev->dev; 624 625 /* set common host1x device data */ 626 platform_set_drvdata(pdev, host); 627 628 host->dev->dma_parms = &host->dma_parms; 629 dma_set_max_seg_size(host->dev, UINT_MAX); 630 631 if (host->info->init) { 632 err = host->info->init(host); 633 if (err) 634 return err; 635 } 636 637 host->clk = devm_clk_get(&pdev->dev, NULL); 638 if (IS_ERR(host->clk)) 639 return dev_err_probe(&pdev->dev, PTR_ERR(host->clk), "failed to get clock\n"); 640 641 err = host1x_get_resets(host); 642 if (err) 643 return err; 644 645 host1x_bo_cache_init(&host->cache); 646 647 err = host1x_iommu_init(host); 648 if (err < 0) { 649 dev_err(&pdev->dev, "failed to setup IOMMU: %d\n", err); 650 goto destroy_cache; 651 } 652 653 err = host1x_channel_list_init(&host->channel_list, 654 host->info->nb_channels); 655 if (err) { 656 dev_err(&pdev->dev, "failed to initialize channel list\n"); 657 goto iommu_exit; 658 } 659 660 err = host1x_memory_context_list_init(host); 661 if (err) { 662 dev_err(&pdev->dev, "failed to initialize context list\n"); 663 goto free_channels; 664 } 665 666 err = host1x_syncpt_init(host); 667 if (err) { 668 dev_err(&pdev->dev, "failed to initialize syncpts\n"); 669 goto free_contexts; 670 } 671 672 mutex_init(&host->intr_mutex); 673 674 pm_runtime_enable(&pdev->dev); 675 676 err = devm_tegra_core_dev_init_opp_table_common(&pdev->dev); 677 if (err) 678 goto pm_disable; 679 680 /* the driver's code isn't ready yet for the dynamic RPM */ 681 err = pm_runtime_resume_and_get(&pdev->dev); 682 if (err) 683 goto pm_disable; 684 685 err = host1x_intr_init(host); 686 if (err) { 687 dev_err(&pdev->dev, "failed to initialize interrupts\n"); 688 goto pm_put; 689 } 690 691 host1x_debug_init(host); 692 693 err = host1x_register(host); 694 if (err < 0) 695 goto deinit_debugfs; 696 697 err = devm_of_platform_populate(&pdev->dev); 698 if (err < 0) 699 goto unregister; 700 701 return 0; 702 703 unregister: 704 host1x_unregister(host); 705 deinit_debugfs: 706 host1x_debug_deinit(host); 707 host1x_intr_deinit(host); 708 pm_put: 709 pm_runtime_put_sync_suspend(&pdev->dev); 710 pm_disable: 711 pm_runtime_disable(&pdev->dev); 712 host1x_syncpt_deinit(host); 713 free_contexts: 714 host1x_memory_context_list_free(&host->context_list); 715 free_channels: 716 host1x_channel_list_free(&host->channel_list); 717 iommu_exit: 718 host1x_iommu_exit(host); 719 destroy_cache: 720 host1x_bo_cache_destroy(&host->cache); 721 722 return err; 723 } 724 725 static void host1x_remove(struct platform_device *pdev) 726 { 727 struct host1x *host = platform_get_drvdata(pdev); 728 729 host1x_unregister(host); 730 host1x_debug_deinit(host); 731 732 pm_runtime_force_suspend(&pdev->dev); 733 734 host1x_intr_deinit(host); 735 host1x_syncpt_deinit(host); 736 host1x_memory_context_list_free(&host->context_list); 737 host1x_channel_list_free(&host->channel_list); 738 host1x_iommu_exit(host); 739 host1x_bo_cache_destroy(&host->cache); 740 } 741 742 static int __maybe_unused host1x_runtime_suspend(struct device *dev) 743 { 744 struct host1x *host = dev_get_drvdata(dev); 745 int err; 746 747 host1x_channel_stop_all(host); 748 host1x_intr_stop(host); 749 host1x_syncpt_save(host); 750 751 if (!host->info->skip_reset_assert) { 752 err = reset_control_bulk_assert(host->nresets, host->resets); 753 if (err) { 754 dev_err(dev, "failed to assert reset: %d\n", err); 755 goto resume_host1x; 756 } 757 758 usleep_range(1000, 2000); 759 } 760 761 clk_disable_unprepare(host->clk); 762 reset_control_bulk_release(host->nresets, host->resets); 763 764 return 0; 765 766 resume_host1x: 767 host1x_setup_virtualization_tables(host); 768 host1x_syncpt_restore(host); 769 host1x_intr_start(host); 770 771 return err; 772 } 773 774 static int __maybe_unused host1x_runtime_resume(struct device *dev) 775 { 776 struct host1x *host = dev_get_drvdata(dev); 777 int err; 778 779 err = reset_control_bulk_acquire(host->nresets, host->resets); 780 if (err) { 781 dev_err(dev, "failed to acquire reset: %d\n", err); 782 return err; 783 } 784 785 err = clk_prepare_enable(host->clk); 786 if (err) { 787 dev_err(dev, "failed to enable clock: %d\n", err); 788 goto release_reset; 789 } 790 791 err = reset_control_bulk_deassert(host->nresets, host->resets); 792 if (err < 0) { 793 dev_err(dev, "failed to deassert reset: %d\n", err); 794 goto disable_clk; 795 } 796 797 host1x_setup_virtualization_tables(host); 798 host1x_syncpt_restore(host); 799 host1x_intr_start(host); 800 801 return 0; 802 803 disable_clk: 804 clk_disable_unprepare(host->clk); 805 release_reset: 806 reset_control_bulk_release(host->nresets, host->resets); 807 808 return err; 809 } 810 811 static const struct dev_pm_ops host1x_pm_ops = { 812 SET_RUNTIME_PM_OPS(host1x_runtime_suspend, host1x_runtime_resume, 813 NULL) 814 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) 815 }; 816 817 static struct platform_driver tegra_host1x_driver = { 818 .driver = { 819 .name = "tegra-host1x", 820 .of_match_table = host1x_of_match, 821 .pm = &host1x_pm_ops, 822 }, 823 .probe = host1x_probe, 824 .remove = host1x_remove, 825 }; 826 827 static struct platform_driver * const drivers[] = { 828 &tegra_host1x_driver, 829 &tegra_mipi_driver, 830 }; 831 832 static int __init tegra_host1x_init(void) 833 { 834 int err; 835 836 err = bus_register(&host1x_bus_type); 837 if (err < 0) 838 return err; 839 840 err = platform_register_drivers(drivers, ARRAY_SIZE(drivers)); 841 if (err < 0) 842 bus_unregister(&host1x_bus_type); 843 844 return err; 845 } 846 module_init(tegra_host1x_init); 847 848 static void __exit tegra_host1x_exit(void) 849 { 850 platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); 851 bus_unregister(&host1x_bus_type); 852 } 853 module_exit(tegra_host1x_exit); 854 855 /** 856 * host1x_get_dma_mask() - query the supported DMA mask for host1x 857 * @host1x: host1x instance 858 * 859 * Note that this returns the supported DMA mask for host1x, which can be 860 * different from the applicable DMA mask under certain circumstances. 861 */ 862 u64 host1x_get_dma_mask(struct host1x *host1x) 863 { 864 return host1x->info->dma_mask; 865 } 866 EXPORT_SYMBOL(host1x_get_dma_mask); 867 868 MODULE_SOFTDEP("post: tegra-drm"); 869 MODULE_AUTHOR("Thierry Reding <thierry.reding@avionic-design.de>"); 870 MODULE_AUTHOR("Terje Bergstrom <tbergstrom@nvidia.com>"); 871 MODULE_DESCRIPTION("Host1x driver for Tegra products"); 872 MODULE_LICENSE("GPL"); 873