1 // SPDX-License-Identifier: GPL-2.0+ 2 /* Copyright (C) 2014-2018 Broadcom */ 3 4 /** 5 * DOC: Broadcom V3D Graphics Driver 6 * 7 * This driver supports the Broadcom V3D 3.3 and 4.1 OpenGL ES GPUs. 8 * For V3D 2.x support, see the VC4 driver. 9 * 10 * The V3D GPU includes a tiled render (composed of a bin and render 11 * pipelines), the TFU (texture formatting unit), and the CSD (compute 12 * shader dispatch). 13 */ 14 15 #include <linux/clk.h> 16 #include <linux/device.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/io.h> 19 #include <linux/module.h> 20 #include <linux/of.h> 21 #include <linux/of_platform.h> 22 #include <linux/platform_device.h> 23 #include <linux/sched/clock.h> 24 #include <linux/reset.h> 25 26 #include <drm/drm_drv.h> 27 #include <drm/drm_managed.h> 28 #include <drm/drm_print.h> 29 #include <uapi/drm/v3d_drm.h> 30 31 #include "v3d_drv.h" 32 #include "v3d_regs.h" 33 34 #define DRIVER_NAME "v3d" 35 #define DRIVER_DESC "Broadcom V3D graphics" 36 #define DRIVER_MAJOR 1 37 #define DRIVER_MINOR 0 38 #define DRIVER_PATCHLEVEL 0 39 40 /* Only expose the `super_pages` modparam if THP is enabled. */ 41 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 42 bool super_pages = true; 43 module_param_named(super_pages, super_pages, bool, 0400); 44 MODULE_PARM_DESC(super_pages, "Enable/Disable Super Pages support."); 45 #endif 46 47 static int v3d_get_param_ioctl(struct drm_device *dev, void *data, 48 struct drm_file *file_priv) 49 { 50 struct v3d_file_priv *v3d_priv = file_priv->driver_priv; 51 struct v3d_dev *v3d = to_v3d_dev(dev); 52 struct drm_v3d_get_param *args = data; 53 static const u32 reg_map[] = { 54 [DRM_V3D_PARAM_V3D_UIFCFG] = V3D_HUB_UIFCFG, 55 [DRM_V3D_PARAM_V3D_HUB_IDENT1] = V3D_HUB_IDENT1, 56 [DRM_V3D_PARAM_V3D_HUB_IDENT2] = V3D_HUB_IDENT2, 57 [DRM_V3D_PARAM_V3D_HUB_IDENT3] = V3D_HUB_IDENT3, 58 [DRM_V3D_PARAM_V3D_CORE0_IDENT0] = V3D_CTL_IDENT0, 59 [DRM_V3D_PARAM_V3D_CORE0_IDENT1] = V3D_CTL_IDENT1, 60 [DRM_V3D_PARAM_V3D_CORE0_IDENT2] = V3D_CTL_IDENT2, 61 }; 62 int ret; 63 64 if (args->pad != 0) 65 return -EINVAL; 66 67 /* Note that DRM_V3D_PARAM_V3D_CORE0_IDENT0 is 0, so we need 68 * to explicitly allow it in the "the register in our 69 * parameter map" check. 70 */ 71 if (args->param < ARRAY_SIZE(reg_map) && 72 (reg_map[args->param] || 73 args->param == DRM_V3D_PARAM_V3D_CORE0_IDENT0)) { 74 u32 offset = reg_map[args->param]; 75 76 if (args->value != 0) 77 return -EINVAL; 78 79 ret = v3d_pm_runtime_get(v3d); 80 if (ret) 81 return ret; 82 83 if (args->param >= DRM_V3D_PARAM_V3D_CORE0_IDENT0 && 84 args->param <= DRM_V3D_PARAM_V3D_CORE0_IDENT2) { 85 args->value = V3D_CORE_READ(0, offset); 86 } else { 87 args->value = V3D_READ(offset); 88 } 89 90 v3d_pm_runtime_put(v3d); 91 92 return 0; 93 } 94 95 switch (args->param) { 96 case DRM_V3D_PARAM_SUPPORTS_TFU: 97 args->value = 1; 98 return 0; 99 case DRM_V3D_PARAM_SUPPORTS_CSD: 100 args->value = v3d_has_csd(v3d); 101 return 0; 102 case DRM_V3D_PARAM_SUPPORTS_CACHE_FLUSH: 103 args->value = 1; 104 return 0; 105 case DRM_V3D_PARAM_SUPPORTS_PERFMON: 106 args->value = (v3d->ver >= V3D_GEN_41); 107 return 0; 108 case DRM_V3D_PARAM_SUPPORTS_MULTISYNC_EXT: 109 args->value = 1; 110 return 0; 111 case DRM_V3D_PARAM_SUPPORTS_CPU_QUEUE: 112 args->value = 1; 113 return 0; 114 case DRM_V3D_PARAM_MAX_PERF_COUNTERS: 115 args->value = v3d->perfmon_info.max_counters; 116 return 0; 117 case DRM_V3D_PARAM_SUPPORTS_SUPER_PAGES: 118 args->value = !!drm_gem_get_huge_mnt(dev); 119 return 0; 120 case DRM_V3D_PARAM_GLOBAL_RESET_COUNTER: 121 args->value = atomic_read(&v3d->reset_counter); 122 return 0; 123 case DRM_V3D_PARAM_CONTEXT_RESET_COUNTER: 124 args->value = 0; 125 for (enum v3d_queue q = 0; q < V3D_MAX_QUEUES; q++) 126 args->value += atomic_read(&v3d_priv->stats[q]->reset_counter); 127 return 0; 128 default: 129 drm_dbg(dev, "Unknown parameter %d\n", args->param); 130 return -EINVAL; 131 } 132 } 133 134 static int 135 v3d_open(struct drm_device *dev, struct drm_file *file) 136 { 137 struct v3d_dev *v3d = to_v3d_dev(dev); 138 struct v3d_file_priv *v3d_priv; 139 struct drm_gpu_scheduler *sched; 140 int i, ret; 141 142 v3d_priv = kzalloc_obj(*v3d_priv); 143 if (!v3d_priv) 144 return -ENOMEM; 145 146 v3d_priv->v3d = v3d; 147 148 for (i = 0; i < V3D_MAX_QUEUES; i++) { 149 v3d_priv->stats[i] = v3d_stats_alloc(); 150 if (!v3d_priv->stats[i]) { 151 ret = -ENOMEM; 152 goto err_stats; 153 } 154 155 sched = &v3d->queue[i].sched; 156 ret = drm_sched_entity_init(&v3d_priv->sched_entity[i], 157 DRM_SCHED_PRIORITY_NORMAL, &sched, 158 1, NULL); 159 if (ret) 160 goto err_sched; 161 } 162 163 v3d_perfmon_open_file(v3d_priv); 164 file->driver_priv = v3d_priv; 165 166 return 0; 167 168 err_sched: 169 v3d_stats_put(v3d_priv->stats[i]); 170 err_stats: 171 for (i--; i >= 0; i--) { 172 drm_sched_entity_destroy(&v3d_priv->sched_entity[i]); 173 v3d_stats_put(v3d_priv->stats[i]); 174 } 175 kfree(v3d_priv); 176 return ret; 177 } 178 179 static void 180 v3d_postclose(struct drm_device *dev, struct drm_file *file) 181 { 182 struct v3d_file_priv *v3d_priv = file->driver_priv; 183 enum v3d_queue q; 184 185 for (q = 0; q < V3D_MAX_QUEUES; q++) { 186 drm_sched_entity_destroy(&v3d_priv->sched_entity[q]); 187 v3d_stats_put(v3d_priv->stats[q]); 188 } 189 190 v3d_perfmon_close_file(v3d_priv); 191 kfree(v3d_priv); 192 } 193 194 void v3d_get_stats(const struct v3d_stats *stats, u64 timestamp, 195 u64 *active_runtime, u64 *jobs_completed) 196 { 197 unsigned int seq; 198 199 do { 200 seq = raw_read_seqcount_begin(&stats->lock); 201 *active_runtime = stats->enabled_ns; 202 if (stats->start_ns) 203 *active_runtime += timestamp - stats->start_ns; 204 *jobs_completed = stats->jobs_completed; 205 } while (read_seqcount_retry(&stats->lock, seq)); 206 } 207 208 static void v3d_show_fdinfo(struct drm_printer *p, struct drm_file *file) 209 { 210 struct v3d_file_priv *file_priv = file->driver_priv; 211 u64 timestamp = local_clock(); 212 enum v3d_queue queue; 213 214 for (queue = 0; queue < V3D_MAX_QUEUES; queue++) { 215 struct v3d_stats *stats = file_priv->stats[queue]; 216 u64 active_runtime, jobs_completed; 217 218 v3d_get_stats(stats, timestamp, &active_runtime, &jobs_completed); 219 220 /* Note that, in case of a GPU reset, the time spent during an 221 * attempt of executing the job is not computed in the runtime. 222 */ 223 drm_printf(p, "drm-engine-%s: \t%llu ns\n", 224 v3d_queue_to_string(queue), active_runtime); 225 226 /* Note that we only count jobs that completed. Therefore, jobs 227 * that were resubmitted due to a GPU reset are not computed. 228 */ 229 drm_printf(p, "v3d-jobs-%s: \t%llu jobs\n", 230 v3d_queue_to_string(queue), jobs_completed); 231 } 232 233 drm_show_memory_stats(p, file); 234 } 235 236 static const struct file_operations v3d_drm_fops = { 237 .owner = THIS_MODULE, 238 DRM_GEM_FOPS, 239 .show_fdinfo = drm_show_fdinfo, 240 }; 241 242 /* DRM_AUTH is required on SUBMIT_CL for now, while we don't have GMP 243 * protection between clients. Note that render nodes would be 244 * able to submit CLs that could access BOs from clients authenticated 245 * with the master node. The TFU doesn't use the GMP, so it would 246 * need to stay DRM_AUTH until we do buffer size/offset validation. 247 */ 248 static const struct drm_ioctl_desc v3d_drm_ioctls[] = { 249 DRM_IOCTL_DEF_DRV(V3D_SUBMIT_CL, v3d_submit_cl_ioctl, DRM_RENDER_ALLOW | DRM_AUTH), 250 DRM_IOCTL_DEF_DRV(V3D_WAIT_BO, v3d_wait_bo_ioctl, DRM_RENDER_ALLOW), 251 DRM_IOCTL_DEF_DRV(V3D_CREATE_BO, v3d_create_bo_ioctl, DRM_RENDER_ALLOW), 252 DRM_IOCTL_DEF_DRV(V3D_MMAP_BO, v3d_mmap_bo_ioctl, DRM_RENDER_ALLOW), 253 DRM_IOCTL_DEF_DRV(V3D_GET_PARAM, v3d_get_param_ioctl, DRM_RENDER_ALLOW), 254 DRM_IOCTL_DEF_DRV(V3D_GET_BO_OFFSET, v3d_get_bo_offset_ioctl, DRM_RENDER_ALLOW), 255 DRM_IOCTL_DEF_DRV(V3D_SUBMIT_TFU, v3d_submit_tfu_ioctl, DRM_RENDER_ALLOW | DRM_AUTH), 256 DRM_IOCTL_DEF_DRV(V3D_SUBMIT_CSD, v3d_submit_csd_ioctl, DRM_RENDER_ALLOW | DRM_AUTH), 257 DRM_IOCTL_DEF_DRV(V3D_PERFMON_CREATE, v3d_perfmon_create_ioctl, DRM_RENDER_ALLOW), 258 DRM_IOCTL_DEF_DRV(V3D_PERFMON_DESTROY, v3d_perfmon_destroy_ioctl, DRM_RENDER_ALLOW), 259 DRM_IOCTL_DEF_DRV(V3D_PERFMON_GET_VALUES, v3d_perfmon_get_values_ioctl, DRM_RENDER_ALLOW), 260 DRM_IOCTL_DEF_DRV(V3D_SUBMIT_CPU, v3d_submit_cpu_ioctl, DRM_RENDER_ALLOW | DRM_AUTH), 261 DRM_IOCTL_DEF_DRV(V3D_PERFMON_GET_COUNTER, v3d_perfmon_get_counter_ioctl, DRM_RENDER_ALLOW), 262 DRM_IOCTL_DEF_DRV(V3D_PERFMON_SET_GLOBAL, v3d_perfmon_set_global_ioctl, DRM_RENDER_ALLOW), 263 }; 264 265 static const struct drm_driver v3d_drm_driver = { 266 .driver_features = (DRIVER_GEM | 267 DRIVER_RENDER | 268 DRIVER_SYNCOBJ), 269 270 .open = v3d_open, 271 .postclose = v3d_postclose, 272 273 #if defined(CONFIG_DEBUG_FS) 274 .debugfs_init = v3d_debugfs_init, 275 #endif 276 277 .gem_create_object = v3d_create_object, 278 .gem_prime_import_sg_table = v3d_prime_import_sg_table, 279 280 .ioctls = v3d_drm_ioctls, 281 .num_ioctls = ARRAY_SIZE(v3d_drm_ioctls), 282 .fops = &v3d_drm_fops, 283 .show_fdinfo = v3d_show_fdinfo, 284 285 .name = DRIVER_NAME, 286 .desc = DRIVER_DESC, 287 .major = DRIVER_MAJOR, 288 .minor = DRIVER_MINOR, 289 .patchlevel = DRIVER_PATCHLEVEL, 290 }; 291 292 static const struct of_device_id v3d_of_match[] = { 293 { .compatible = "brcm,2711-v3d", .data = (void *)V3D_GEN_42 }, 294 { .compatible = "brcm,2712-v3d", .data = (void *)V3D_GEN_71 }, 295 { .compatible = "brcm,7268-v3d", .data = (void *)V3D_GEN_33 }, 296 { .compatible = "brcm,7278-v3d", .data = (void *)V3D_GEN_41 }, 297 {}, 298 }; 299 MODULE_DEVICE_TABLE(of, v3d_of_match); 300 301 static int 302 map_regs(struct v3d_dev *v3d, void __iomem **regs, const char *name) 303 { 304 *regs = devm_platform_ioremap_resource_byname(v3d_to_pdev(v3d), name); 305 return PTR_ERR_OR_ZERO(*regs); 306 } 307 308 static int v3d_platform_drm_probe(struct platform_device *pdev) 309 { 310 struct device *dev = &pdev->dev; 311 struct drm_device *drm; 312 struct v3d_dev *v3d; 313 enum v3d_gen gen; 314 int ret; 315 u32 mmu_debug; 316 u32 ident1, ident3; 317 u64 mask; 318 319 v3d = devm_drm_dev_alloc(dev, &v3d_drm_driver, struct v3d_dev, drm); 320 if (IS_ERR(v3d)) 321 return PTR_ERR(v3d); 322 323 drm = &v3d->drm; 324 325 platform_set_drvdata(pdev, drm); 326 327 gen = (uintptr_t)of_device_get_match_data(dev); 328 v3d->ver = gen; 329 330 ret = map_regs(v3d, &v3d->hub_regs, "hub"); 331 if (ret) 332 return ret; 333 334 ret = map_regs(v3d, &v3d->core_regs[0], "core0"); 335 if (ret) 336 return ret; 337 338 if (v3d->ver >= V3D_GEN_71) { 339 ret = map_regs(v3d, &v3d->sms_regs, "sms"); 340 if (ret) 341 return ret; 342 } 343 344 if (v3d->ver < V3D_GEN_41) { 345 ret = map_regs(v3d, &v3d->gca_regs, "gca"); 346 if (ret) 347 return ret; 348 } 349 350 v3d->reset = devm_reset_control_get_optional_exclusive(dev, NULL); 351 if (IS_ERR(v3d->reset)) 352 return dev_err_probe(dev, PTR_ERR(v3d->reset), 353 "Failed to get reset control\n"); 354 355 if (!v3d->reset) { 356 ret = map_regs(v3d, &v3d->bridge_regs, "bridge"); 357 if (ret) { 358 dev_err(dev, "Failed to get bridge registers\n"); 359 return ret; 360 } 361 } 362 363 v3d->clk = devm_clk_get_optional(dev, NULL); 364 if (IS_ERR(v3d->clk)) 365 return dev_err_probe(dev, PTR_ERR(v3d->clk), "Failed to get V3D clock\n"); 366 367 ret = v3d_irq_init(v3d); 368 if (ret) 369 return ret; 370 371 v3d_perfmon_init(v3d); 372 373 v3d->mmu_scratch = dma_alloc_wc(dev, 4096, &v3d->mmu_scratch_paddr, 374 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO); 375 if (!v3d->mmu_scratch) { 376 dev_err(dev, "Failed to allocate MMU scratch page\n"); 377 return -ENOMEM; 378 } 379 380 ret = v3d_gem_init(drm); 381 if (ret) 382 goto dma_free; 383 384 ret = devm_pm_runtime_enable(dev); 385 if (ret) 386 goto gem_destroy; 387 388 ret = pm_runtime_resume_and_get(dev); 389 if (ret) 390 goto gem_destroy; 391 392 /* If PM is disabled, we need to call v3d_power_resume() manually. */ 393 if (!IS_ENABLED(CONFIG_PM)) { 394 ret = v3d_power_resume(dev); 395 if (ret) 396 goto gem_destroy; 397 } 398 399 mmu_debug = V3D_READ(V3D_MMU_DEBUG_INFO); 400 mask = DMA_BIT_MASK(30 + V3D_GET_FIELD(mmu_debug, V3D_MMU_PA_WIDTH)); 401 ret = dma_set_mask_and_coherent(dev, mask); 402 if (ret) 403 goto runtime_pm_put; 404 405 dma_set_max_seg_size(&pdev->dev, UINT_MAX); 406 407 v3d->va_width = 30 + V3D_GET_FIELD(mmu_debug, V3D_MMU_VA_WIDTH); 408 409 ident1 = V3D_READ(V3D_HUB_IDENT1); 410 v3d->ver = (V3D_GET_FIELD(ident1, V3D_HUB_IDENT1_TVER) * 10 + 411 V3D_GET_FIELD(ident1, V3D_HUB_IDENT1_REV)); 412 /* Make sure that the V3D tech version retrieved from the HW is equal 413 * to the one advertised by the device tree. 414 */ 415 WARN_ON(v3d->ver != gen); 416 417 v3d->cores = V3D_GET_FIELD(ident1, V3D_HUB_IDENT1_NCORES); 418 WARN_ON(v3d->cores > 1); /* multicore not yet implemented */ 419 420 ident3 = V3D_READ(V3D_HUB_IDENT3); 421 v3d->rev = V3D_GET_FIELD(ident3, V3D_HUB_IDENT3_IPREV); 422 423 pm_runtime_set_autosuspend_delay(dev, 100); 424 pm_runtime_use_autosuspend(dev); 425 426 ret = drm_dev_register(drm, 0); 427 if (ret) 428 goto runtime_pm_put; 429 430 ret = v3d_sysfs_init(dev); 431 if (ret) 432 goto drm_unregister; 433 434 pm_runtime_mark_last_busy(dev); 435 pm_runtime_put_autosuspend(dev); 436 437 return 0; 438 439 drm_unregister: 440 drm_dev_unregister(drm); 441 runtime_pm_put: 442 pm_runtime_put_sync_suspend(dev); 443 gem_destroy: 444 v3d_gem_destroy(drm); 445 dma_free: 446 dma_free_wc(dev, 4096, v3d->mmu_scratch, v3d->mmu_scratch_paddr); 447 return ret; 448 } 449 450 static void v3d_platform_drm_remove(struct platform_device *pdev) 451 { 452 struct drm_device *drm = platform_get_drvdata(pdev); 453 struct v3d_dev *v3d = to_v3d_dev(drm); 454 struct device *dev = &pdev->dev; 455 456 v3d_sysfs_destroy(dev); 457 458 drm_dev_unregister(drm); 459 460 pm_runtime_suspend(dev); 461 462 /* If PM is disabled, we need to call v3d_power_suspend() manually. */ 463 if (!IS_ENABLED(CONFIG_PM)) 464 v3d_power_suspend(dev); 465 466 v3d_gem_destroy(drm); 467 468 dma_free_wc(dev, 4096, v3d->mmu_scratch, v3d->mmu_scratch_paddr); 469 } 470 471 static DEFINE_RUNTIME_DEV_PM_OPS(v3d_pm_ops, v3d_power_suspend, 472 v3d_power_resume, NULL); 473 474 static struct platform_driver v3d_platform_driver = { 475 .probe = v3d_platform_drm_probe, 476 .remove = v3d_platform_drm_remove, 477 .driver = { 478 .name = "v3d", 479 .of_match_table = v3d_of_match, 480 .pm = pm_ptr(&v3d_pm_ops), 481 }, 482 }; 483 484 module_platform_driver(v3d_platform_driver); 485 486 MODULE_ALIAS("platform:v3d-drm"); 487 MODULE_DESCRIPTION("Broadcom V3D DRM Driver"); 488 MODULE_AUTHOR("Eric Anholt <eric@anholt.net>"); 489 MODULE_LICENSE("GPL v2"); 490