1 /* 2 * Copyright (C) 2015 Etnaviv Project 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 as published by 6 * the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program. If not, see <http://www.gnu.org/licenses/>. 15 */ 16 17 #include <linux/component.h> 18 #include <linux/fence.h> 19 #include <linux/moduleparam.h> 20 #include <linux/of_device.h> 21 #include "etnaviv_dump.h" 22 #include "etnaviv_gpu.h" 23 #include "etnaviv_gem.h" 24 #include "etnaviv_mmu.h" 25 #include "etnaviv_iommu.h" 26 #include "etnaviv_iommu_v2.h" 27 #include "common.xml.h" 28 #include "state.xml.h" 29 #include "state_hi.xml.h" 30 #include "cmdstream.xml.h" 31 32 static const struct platform_device_id gpu_ids[] = { 33 { .name = "etnaviv-gpu,2d" }, 34 { }, 35 }; 36 37 static bool etnaviv_dump_core = true; 38 module_param_named(dump_core, etnaviv_dump_core, bool, 0600); 39 40 /* 41 * Driver functions: 42 */ 43 44 int etnaviv_gpu_get_param(struct etnaviv_gpu *gpu, u32 param, u64 *value) 45 { 46 switch (param) { 47 case ETNAVIV_PARAM_GPU_MODEL: 48 *value = gpu->identity.model; 49 break; 50 51 case ETNAVIV_PARAM_GPU_REVISION: 52 *value = gpu->identity.revision; 53 break; 54 55 case ETNAVIV_PARAM_GPU_FEATURES_0: 56 *value = gpu->identity.features; 57 break; 58 59 case ETNAVIV_PARAM_GPU_FEATURES_1: 60 *value = gpu->identity.minor_features0; 61 break; 62 63 case ETNAVIV_PARAM_GPU_FEATURES_2: 64 *value = gpu->identity.minor_features1; 65 break; 66 67 case ETNAVIV_PARAM_GPU_FEATURES_3: 68 *value = gpu->identity.minor_features2; 69 break; 70 71 case ETNAVIV_PARAM_GPU_FEATURES_4: 72 *value = gpu->identity.minor_features3; 73 break; 74 75 case ETNAVIV_PARAM_GPU_FEATURES_5: 76 *value = gpu->identity.minor_features4; 77 break; 78 79 case ETNAVIV_PARAM_GPU_FEATURES_6: 80 *value = gpu->identity.minor_features5; 81 break; 82 83 case ETNAVIV_PARAM_GPU_STREAM_COUNT: 84 *value = gpu->identity.stream_count; 85 break; 86 87 case ETNAVIV_PARAM_GPU_REGISTER_MAX: 88 *value = gpu->identity.register_max; 89 break; 90 91 case ETNAVIV_PARAM_GPU_THREAD_COUNT: 92 *value = gpu->identity.thread_count; 93 break; 94 95 case ETNAVIV_PARAM_GPU_VERTEX_CACHE_SIZE: 96 *value = gpu->identity.vertex_cache_size; 97 break; 98 99 case ETNAVIV_PARAM_GPU_SHADER_CORE_COUNT: 100 *value = gpu->identity.shader_core_count; 101 break; 102 103 case ETNAVIV_PARAM_GPU_PIXEL_PIPES: 104 *value = gpu->identity.pixel_pipes; 105 break; 106 107 case ETNAVIV_PARAM_GPU_VERTEX_OUTPUT_BUFFER_SIZE: 108 *value = gpu->identity.vertex_output_buffer_size; 109 break; 110 111 case ETNAVIV_PARAM_GPU_BUFFER_SIZE: 112 *value = gpu->identity.buffer_size; 113 break; 114 115 case ETNAVIV_PARAM_GPU_INSTRUCTION_COUNT: 116 *value = gpu->identity.instruction_count; 117 break; 118 119 case ETNAVIV_PARAM_GPU_NUM_CONSTANTS: 120 *value = gpu->identity.num_constants; 121 break; 122 123 case ETNAVIV_PARAM_GPU_NUM_VARYINGS: 124 *value = gpu->identity.varyings_count; 125 break; 126 127 default: 128 DBG("%s: invalid param: %u", dev_name(gpu->dev), param); 129 return -EINVAL; 130 } 131 132 return 0; 133 } 134 135 136 #define etnaviv_is_model_rev(gpu, mod, rev) \ 137 ((gpu)->identity.model == chipModel_##mod && \ 138 (gpu)->identity.revision == rev) 139 #define etnaviv_field(val, field) \ 140 (((val) & field##__MASK) >> field##__SHIFT) 141 142 static void etnaviv_hw_specs(struct etnaviv_gpu *gpu) 143 { 144 if (gpu->identity.minor_features0 & 145 chipMinorFeatures0_MORE_MINOR_FEATURES) { 146 u32 specs[4]; 147 unsigned int streams; 148 149 specs[0] = gpu_read(gpu, VIVS_HI_CHIP_SPECS); 150 specs[1] = gpu_read(gpu, VIVS_HI_CHIP_SPECS_2); 151 specs[2] = gpu_read(gpu, VIVS_HI_CHIP_SPECS_3); 152 specs[3] = gpu_read(gpu, VIVS_HI_CHIP_SPECS_4); 153 154 gpu->identity.stream_count = etnaviv_field(specs[0], 155 VIVS_HI_CHIP_SPECS_STREAM_COUNT); 156 gpu->identity.register_max = etnaviv_field(specs[0], 157 VIVS_HI_CHIP_SPECS_REGISTER_MAX); 158 gpu->identity.thread_count = etnaviv_field(specs[0], 159 VIVS_HI_CHIP_SPECS_THREAD_COUNT); 160 gpu->identity.vertex_cache_size = etnaviv_field(specs[0], 161 VIVS_HI_CHIP_SPECS_VERTEX_CACHE_SIZE); 162 gpu->identity.shader_core_count = etnaviv_field(specs[0], 163 VIVS_HI_CHIP_SPECS_SHADER_CORE_COUNT); 164 gpu->identity.pixel_pipes = etnaviv_field(specs[0], 165 VIVS_HI_CHIP_SPECS_PIXEL_PIPES); 166 gpu->identity.vertex_output_buffer_size = 167 etnaviv_field(specs[0], 168 VIVS_HI_CHIP_SPECS_VERTEX_OUTPUT_BUFFER_SIZE); 169 170 gpu->identity.buffer_size = etnaviv_field(specs[1], 171 VIVS_HI_CHIP_SPECS_2_BUFFER_SIZE); 172 gpu->identity.instruction_count = etnaviv_field(specs[1], 173 VIVS_HI_CHIP_SPECS_2_INSTRUCTION_COUNT); 174 gpu->identity.num_constants = etnaviv_field(specs[1], 175 VIVS_HI_CHIP_SPECS_2_NUM_CONSTANTS); 176 177 gpu->identity.varyings_count = etnaviv_field(specs[2], 178 VIVS_HI_CHIP_SPECS_3_VARYINGS_COUNT); 179 180 /* This overrides the value from older register if non-zero */ 181 streams = etnaviv_field(specs[3], 182 VIVS_HI_CHIP_SPECS_4_STREAM_COUNT); 183 if (streams) 184 gpu->identity.stream_count = streams; 185 } 186 187 /* Fill in the stream count if not specified */ 188 if (gpu->identity.stream_count == 0) { 189 if (gpu->identity.model >= 0x1000) 190 gpu->identity.stream_count = 4; 191 else 192 gpu->identity.stream_count = 1; 193 } 194 195 /* Convert the register max value */ 196 if (gpu->identity.register_max) 197 gpu->identity.register_max = 1 << gpu->identity.register_max; 198 else if (gpu->identity.model == chipModel_GC400) 199 gpu->identity.register_max = 32; 200 else 201 gpu->identity.register_max = 64; 202 203 /* Convert thread count */ 204 if (gpu->identity.thread_count) 205 gpu->identity.thread_count = 1 << gpu->identity.thread_count; 206 else if (gpu->identity.model == chipModel_GC400) 207 gpu->identity.thread_count = 64; 208 else if (gpu->identity.model == chipModel_GC500 || 209 gpu->identity.model == chipModel_GC530) 210 gpu->identity.thread_count = 128; 211 else 212 gpu->identity.thread_count = 256; 213 214 if (gpu->identity.vertex_cache_size == 0) 215 gpu->identity.vertex_cache_size = 8; 216 217 if (gpu->identity.shader_core_count == 0) { 218 if (gpu->identity.model >= 0x1000) 219 gpu->identity.shader_core_count = 2; 220 else 221 gpu->identity.shader_core_count = 1; 222 } 223 224 if (gpu->identity.pixel_pipes == 0) 225 gpu->identity.pixel_pipes = 1; 226 227 /* Convert virtex buffer size */ 228 if (gpu->identity.vertex_output_buffer_size) { 229 gpu->identity.vertex_output_buffer_size = 230 1 << gpu->identity.vertex_output_buffer_size; 231 } else if (gpu->identity.model == chipModel_GC400) { 232 if (gpu->identity.revision < 0x4000) 233 gpu->identity.vertex_output_buffer_size = 512; 234 else if (gpu->identity.revision < 0x4200) 235 gpu->identity.vertex_output_buffer_size = 256; 236 else 237 gpu->identity.vertex_output_buffer_size = 128; 238 } else { 239 gpu->identity.vertex_output_buffer_size = 512; 240 } 241 242 switch (gpu->identity.instruction_count) { 243 case 0: 244 if (etnaviv_is_model_rev(gpu, GC2000, 0x5108) || 245 gpu->identity.model == chipModel_GC880) 246 gpu->identity.instruction_count = 512; 247 else 248 gpu->identity.instruction_count = 256; 249 break; 250 251 case 1: 252 gpu->identity.instruction_count = 1024; 253 break; 254 255 case 2: 256 gpu->identity.instruction_count = 2048; 257 break; 258 259 default: 260 gpu->identity.instruction_count = 256; 261 break; 262 } 263 264 if (gpu->identity.num_constants == 0) 265 gpu->identity.num_constants = 168; 266 267 if (gpu->identity.varyings_count == 0) { 268 if (gpu->identity.minor_features1 & chipMinorFeatures1_HALTI0) 269 gpu->identity.varyings_count = 12; 270 else 271 gpu->identity.varyings_count = 8; 272 } 273 274 /* 275 * For some cores, two varyings are consumed for position, so the 276 * maximum varying count needs to be reduced by one. 277 */ 278 if (etnaviv_is_model_rev(gpu, GC5000, 0x5434) || 279 etnaviv_is_model_rev(gpu, GC4000, 0x5222) || 280 etnaviv_is_model_rev(gpu, GC4000, 0x5245) || 281 etnaviv_is_model_rev(gpu, GC4000, 0x5208) || 282 etnaviv_is_model_rev(gpu, GC3000, 0x5435) || 283 etnaviv_is_model_rev(gpu, GC2200, 0x5244) || 284 etnaviv_is_model_rev(gpu, GC2100, 0x5108) || 285 etnaviv_is_model_rev(gpu, GC2000, 0x5108) || 286 etnaviv_is_model_rev(gpu, GC1500, 0x5246) || 287 etnaviv_is_model_rev(gpu, GC880, 0x5107) || 288 etnaviv_is_model_rev(gpu, GC880, 0x5106)) 289 gpu->identity.varyings_count -= 1; 290 } 291 292 static void etnaviv_hw_identify(struct etnaviv_gpu *gpu) 293 { 294 u32 chipIdentity; 295 296 chipIdentity = gpu_read(gpu, VIVS_HI_CHIP_IDENTITY); 297 298 /* Special case for older graphic cores. */ 299 if (etnaviv_field(chipIdentity, VIVS_HI_CHIP_IDENTITY_FAMILY) == 0x01) { 300 gpu->identity.model = chipModel_GC500; 301 gpu->identity.revision = etnaviv_field(chipIdentity, 302 VIVS_HI_CHIP_IDENTITY_REVISION); 303 } else { 304 305 gpu->identity.model = gpu_read(gpu, VIVS_HI_CHIP_MODEL); 306 gpu->identity.revision = gpu_read(gpu, VIVS_HI_CHIP_REV); 307 308 /* 309 * !!!! HACK ALERT !!!! 310 * Because people change device IDs without letting software 311 * know about it - here is the hack to make it all look the 312 * same. Only for GC400 family. 313 */ 314 if ((gpu->identity.model & 0xff00) == 0x0400 && 315 gpu->identity.model != chipModel_GC420) { 316 gpu->identity.model = gpu->identity.model & 0x0400; 317 } 318 319 /* Another special case */ 320 if (etnaviv_is_model_rev(gpu, GC300, 0x2201)) { 321 u32 chipDate = gpu_read(gpu, VIVS_HI_CHIP_DATE); 322 u32 chipTime = gpu_read(gpu, VIVS_HI_CHIP_TIME); 323 324 if (chipDate == 0x20080814 && chipTime == 0x12051100) { 325 /* 326 * This IP has an ECO; put the correct 327 * revision in it. 328 */ 329 gpu->identity.revision = 0x1051; 330 } 331 } 332 } 333 334 dev_info(gpu->dev, "model: GC%x, revision: %x\n", 335 gpu->identity.model, gpu->identity.revision); 336 337 gpu->identity.features = gpu_read(gpu, VIVS_HI_CHIP_FEATURE); 338 339 /* Disable fast clear on GC700. */ 340 if (gpu->identity.model == chipModel_GC700) 341 gpu->identity.features &= ~chipFeatures_FAST_CLEAR; 342 343 if ((gpu->identity.model == chipModel_GC500 && 344 gpu->identity.revision < 2) || 345 (gpu->identity.model == chipModel_GC300 && 346 gpu->identity.revision < 0x2000)) { 347 348 /* 349 * GC500 rev 1.x and GC300 rev < 2.0 doesn't have these 350 * registers. 351 */ 352 gpu->identity.minor_features0 = 0; 353 gpu->identity.minor_features1 = 0; 354 gpu->identity.minor_features2 = 0; 355 gpu->identity.minor_features3 = 0; 356 gpu->identity.minor_features4 = 0; 357 gpu->identity.minor_features5 = 0; 358 } else 359 gpu->identity.minor_features0 = 360 gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_0); 361 362 if (gpu->identity.minor_features0 & 363 chipMinorFeatures0_MORE_MINOR_FEATURES) { 364 gpu->identity.minor_features1 = 365 gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_1); 366 gpu->identity.minor_features2 = 367 gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_2); 368 gpu->identity.minor_features3 = 369 gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_3); 370 gpu->identity.minor_features4 = 371 gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_4); 372 gpu->identity.minor_features5 = 373 gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_5); 374 } 375 376 /* GC600 idle register reports zero bits where modules aren't present */ 377 if (gpu->identity.model == chipModel_GC600) { 378 gpu->idle_mask = VIVS_HI_IDLE_STATE_TX | 379 VIVS_HI_IDLE_STATE_RA | 380 VIVS_HI_IDLE_STATE_SE | 381 VIVS_HI_IDLE_STATE_PA | 382 VIVS_HI_IDLE_STATE_SH | 383 VIVS_HI_IDLE_STATE_PE | 384 VIVS_HI_IDLE_STATE_DE | 385 VIVS_HI_IDLE_STATE_FE; 386 } else { 387 gpu->idle_mask = ~VIVS_HI_IDLE_STATE_AXI_LP; 388 } 389 390 etnaviv_hw_specs(gpu); 391 } 392 393 static void etnaviv_gpu_load_clock(struct etnaviv_gpu *gpu, u32 clock) 394 { 395 gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, clock | 396 VIVS_HI_CLOCK_CONTROL_FSCALE_CMD_LOAD); 397 gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, clock); 398 } 399 400 static int etnaviv_hw_reset(struct etnaviv_gpu *gpu) 401 { 402 u32 control, idle; 403 unsigned long timeout; 404 bool failed = true; 405 406 /* TODO 407 * 408 * - clock gating 409 * - puls eater 410 * - what about VG? 411 */ 412 413 /* We hope that the GPU resets in under one second */ 414 timeout = jiffies + msecs_to_jiffies(1000); 415 416 while (time_is_after_jiffies(timeout)) { 417 control = VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS | 418 VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(0x40); 419 420 /* enable clock */ 421 etnaviv_gpu_load_clock(gpu, control); 422 423 /* Wait for stable clock. Vivante's code waited for 1ms */ 424 usleep_range(1000, 10000); 425 426 /* isolate the GPU. */ 427 control |= VIVS_HI_CLOCK_CONTROL_ISOLATE_GPU; 428 gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); 429 430 /* set soft reset. */ 431 control |= VIVS_HI_CLOCK_CONTROL_SOFT_RESET; 432 gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); 433 434 /* wait for reset. */ 435 msleep(1); 436 437 /* reset soft reset bit. */ 438 control &= ~VIVS_HI_CLOCK_CONTROL_SOFT_RESET; 439 gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); 440 441 /* reset GPU isolation. */ 442 control &= ~VIVS_HI_CLOCK_CONTROL_ISOLATE_GPU; 443 gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); 444 445 /* read idle register. */ 446 idle = gpu_read(gpu, VIVS_HI_IDLE_STATE); 447 448 /* try reseting again if FE it not idle */ 449 if ((idle & VIVS_HI_IDLE_STATE_FE) == 0) { 450 dev_dbg(gpu->dev, "FE is not idle\n"); 451 continue; 452 } 453 454 /* read reset register. */ 455 control = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL); 456 457 /* is the GPU idle? */ 458 if (((control & VIVS_HI_CLOCK_CONTROL_IDLE_3D) == 0) || 459 ((control & VIVS_HI_CLOCK_CONTROL_IDLE_2D) == 0)) { 460 dev_dbg(gpu->dev, "GPU is not idle\n"); 461 continue; 462 } 463 464 failed = false; 465 break; 466 } 467 468 if (failed) { 469 idle = gpu_read(gpu, VIVS_HI_IDLE_STATE); 470 control = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL); 471 472 dev_err(gpu->dev, "GPU failed to reset: FE %sidle, 3D %sidle, 2D %sidle\n", 473 idle & VIVS_HI_IDLE_STATE_FE ? "" : "not ", 474 control & VIVS_HI_CLOCK_CONTROL_IDLE_3D ? "" : "not ", 475 control & VIVS_HI_CLOCK_CONTROL_IDLE_2D ? "" : "not "); 476 477 return -EBUSY; 478 } 479 480 /* We rely on the GPU running, so program the clock */ 481 control = VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS | 482 VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(0x40); 483 484 /* enable clock */ 485 etnaviv_gpu_load_clock(gpu, control); 486 487 return 0; 488 } 489 490 static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu) 491 { 492 u16 prefetch; 493 494 if ((etnaviv_is_model_rev(gpu, GC320, 0x5007) || 495 etnaviv_is_model_rev(gpu, GC320, 0x5220)) && 496 gpu_read(gpu, VIVS_HI_CHIP_TIME) != 0x2062400) { 497 u32 mc_memory_debug; 498 499 mc_memory_debug = gpu_read(gpu, VIVS_MC_DEBUG_MEMORY) & ~0xff; 500 501 if (gpu->identity.revision == 0x5007) 502 mc_memory_debug |= 0x0c; 503 else 504 mc_memory_debug |= 0x08; 505 506 gpu_write(gpu, VIVS_MC_DEBUG_MEMORY, mc_memory_debug); 507 } 508 509 /* 510 * Update GPU AXI cache atttribute to "cacheable, no allocate". 511 * This is necessary to prevent the iMX6 SoC locking up. 512 */ 513 gpu_write(gpu, VIVS_HI_AXI_CONFIG, 514 VIVS_HI_AXI_CONFIG_AWCACHE(2) | 515 VIVS_HI_AXI_CONFIG_ARCACHE(2)); 516 517 /* GC2000 rev 5108 needs a special bus config */ 518 if (etnaviv_is_model_rev(gpu, GC2000, 0x5108)) { 519 u32 bus_config = gpu_read(gpu, VIVS_MC_BUS_CONFIG); 520 bus_config &= ~(VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG__MASK | 521 VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG__MASK); 522 bus_config |= VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG(1) | 523 VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG(0); 524 gpu_write(gpu, VIVS_MC_BUS_CONFIG, bus_config); 525 } 526 527 /* set base addresses */ 528 gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_RA, gpu->memory_base); 529 gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_FE, gpu->memory_base); 530 gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_TX, gpu->memory_base); 531 gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_PEZ, gpu->memory_base); 532 gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_PE, gpu->memory_base); 533 534 /* setup the MMU page table pointers */ 535 etnaviv_iommu_domain_restore(gpu, gpu->mmu->domain); 536 537 /* Start command processor */ 538 prefetch = etnaviv_buffer_init(gpu); 539 540 gpu_write(gpu, VIVS_HI_INTR_ENBL, ~0U); 541 gpu_write(gpu, VIVS_FE_COMMAND_ADDRESS, 542 gpu->buffer->paddr - gpu->memory_base); 543 gpu_write(gpu, VIVS_FE_COMMAND_CONTROL, 544 VIVS_FE_COMMAND_CONTROL_ENABLE | 545 VIVS_FE_COMMAND_CONTROL_PREFETCH(prefetch)); 546 } 547 548 int etnaviv_gpu_init(struct etnaviv_gpu *gpu) 549 { 550 int ret, i; 551 struct iommu_domain *iommu; 552 enum etnaviv_iommu_version version; 553 bool mmuv2; 554 555 ret = pm_runtime_get_sync(gpu->dev); 556 if (ret < 0) 557 return ret; 558 559 etnaviv_hw_identify(gpu); 560 561 if (gpu->identity.model == 0) { 562 dev_err(gpu->dev, "Unknown GPU model\n"); 563 ret = -ENXIO; 564 goto fail; 565 } 566 567 /* Exclude VG cores with FE2.0 */ 568 if (gpu->identity.features & chipFeatures_PIPE_VG && 569 gpu->identity.features & chipFeatures_FE20) { 570 dev_info(gpu->dev, "Ignoring GPU with VG and FE2.0\n"); 571 ret = -ENXIO; 572 goto fail; 573 } 574 575 /* 576 * Set the GPU linear window to be at the end of the DMA window, where 577 * the CMA area is likely to reside. This ensures that we are able to 578 * map the command buffers while having the linear window overlap as 579 * much RAM as possible, so we can optimize mappings for other buffers. 580 * 581 * For 3D cores only do this if MC2.0 is present, as with MC1.0 it leads 582 * to different views of the memory on the individual engines. 583 */ 584 if (!(gpu->identity.features & chipFeatures_PIPE_3D) || 585 (gpu->identity.minor_features0 & chipMinorFeatures0_MC20)) { 586 u32 dma_mask = (u32)dma_get_required_mask(gpu->dev); 587 if (dma_mask < PHYS_OFFSET + SZ_2G) 588 gpu->memory_base = PHYS_OFFSET; 589 else 590 gpu->memory_base = dma_mask - SZ_2G + 1; 591 } 592 593 ret = etnaviv_hw_reset(gpu); 594 if (ret) 595 goto fail; 596 597 /* Setup IOMMU.. eventually we will (I think) do this once per context 598 * and have separate page tables per context. For now, to keep things 599 * simple and to get something working, just use a single address space: 600 */ 601 mmuv2 = gpu->identity.minor_features1 & chipMinorFeatures1_MMU_VERSION; 602 dev_dbg(gpu->dev, "mmuv2: %d\n", mmuv2); 603 604 if (!mmuv2) { 605 iommu = etnaviv_iommu_domain_alloc(gpu); 606 version = ETNAVIV_IOMMU_V1; 607 } else { 608 iommu = etnaviv_iommu_v2_domain_alloc(gpu); 609 version = ETNAVIV_IOMMU_V2; 610 } 611 612 if (!iommu) { 613 ret = -ENOMEM; 614 goto fail; 615 } 616 617 gpu->mmu = etnaviv_iommu_new(gpu, iommu, version); 618 if (!gpu->mmu) { 619 iommu_domain_free(iommu); 620 ret = -ENOMEM; 621 goto fail; 622 } 623 624 /* Create buffer: */ 625 gpu->buffer = etnaviv_gpu_cmdbuf_new(gpu, PAGE_SIZE, 0); 626 if (!gpu->buffer) { 627 ret = -ENOMEM; 628 dev_err(gpu->dev, "could not create command buffer\n"); 629 goto destroy_iommu; 630 } 631 if (gpu->buffer->paddr - gpu->memory_base > 0x80000000) { 632 ret = -EINVAL; 633 dev_err(gpu->dev, 634 "command buffer outside valid memory window\n"); 635 goto free_buffer; 636 } 637 638 /* Setup event management */ 639 spin_lock_init(&gpu->event_spinlock); 640 init_completion(&gpu->event_free); 641 for (i = 0; i < ARRAY_SIZE(gpu->event); i++) { 642 gpu->event[i].used = false; 643 complete(&gpu->event_free); 644 } 645 646 /* Now program the hardware */ 647 mutex_lock(&gpu->lock); 648 etnaviv_gpu_hw_init(gpu); 649 gpu->exec_state = -1; 650 mutex_unlock(&gpu->lock); 651 652 pm_runtime_mark_last_busy(gpu->dev); 653 pm_runtime_put_autosuspend(gpu->dev); 654 655 return 0; 656 657 free_buffer: 658 etnaviv_gpu_cmdbuf_free(gpu->buffer); 659 gpu->buffer = NULL; 660 destroy_iommu: 661 etnaviv_iommu_destroy(gpu->mmu); 662 gpu->mmu = NULL; 663 fail: 664 pm_runtime_mark_last_busy(gpu->dev); 665 pm_runtime_put_autosuspend(gpu->dev); 666 667 return ret; 668 } 669 670 #ifdef CONFIG_DEBUG_FS 671 struct dma_debug { 672 u32 address[2]; 673 u32 state[2]; 674 }; 675 676 static void verify_dma(struct etnaviv_gpu *gpu, struct dma_debug *debug) 677 { 678 u32 i; 679 680 debug->address[0] = gpu_read(gpu, VIVS_FE_DMA_ADDRESS); 681 debug->state[0] = gpu_read(gpu, VIVS_FE_DMA_DEBUG_STATE); 682 683 for (i = 0; i < 500; i++) { 684 debug->address[1] = gpu_read(gpu, VIVS_FE_DMA_ADDRESS); 685 debug->state[1] = gpu_read(gpu, VIVS_FE_DMA_DEBUG_STATE); 686 687 if (debug->address[0] != debug->address[1]) 688 break; 689 690 if (debug->state[0] != debug->state[1]) 691 break; 692 } 693 } 694 695 int etnaviv_gpu_debugfs(struct etnaviv_gpu *gpu, struct seq_file *m) 696 { 697 struct dma_debug debug; 698 u32 dma_lo, dma_hi, axi, idle; 699 int ret; 700 701 seq_printf(m, "%s Status:\n", dev_name(gpu->dev)); 702 703 ret = pm_runtime_get_sync(gpu->dev); 704 if (ret < 0) 705 return ret; 706 707 dma_lo = gpu_read(gpu, VIVS_FE_DMA_LOW); 708 dma_hi = gpu_read(gpu, VIVS_FE_DMA_HIGH); 709 axi = gpu_read(gpu, VIVS_HI_AXI_STATUS); 710 idle = gpu_read(gpu, VIVS_HI_IDLE_STATE); 711 712 verify_dma(gpu, &debug); 713 714 seq_puts(m, "\tfeatures\n"); 715 seq_printf(m, "\t minor_features0: 0x%08x\n", 716 gpu->identity.minor_features0); 717 seq_printf(m, "\t minor_features1: 0x%08x\n", 718 gpu->identity.minor_features1); 719 seq_printf(m, "\t minor_features2: 0x%08x\n", 720 gpu->identity.minor_features2); 721 seq_printf(m, "\t minor_features3: 0x%08x\n", 722 gpu->identity.minor_features3); 723 seq_printf(m, "\t minor_features4: 0x%08x\n", 724 gpu->identity.minor_features4); 725 seq_printf(m, "\t minor_features5: 0x%08x\n", 726 gpu->identity.minor_features5); 727 728 seq_puts(m, "\tspecs\n"); 729 seq_printf(m, "\t stream_count: %d\n", 730 gpu->identity.stream_count); 731 seq_printf(m, "\t register_max: %d\n", 732 gpu->identity.register_max); 733 seq_printf(m, "\t thread_count: %d\n", 734 gpu->identity.thread_count); 735 seq_printf(m, "\t vertex_cache_size: %d\n", 736 gpu->identity.vertex_cache_size); 737 seq_printf(m, "\t shader_core_count: %d\n", 738 gpu->identity.shader_core_count); 739 seq_printf(m, "\t pixel_pipes: %d\n", 740 gpu->identity.pixel_pipes); 741 seq_printf(m, "\t vertex_output_buffer_size: %d\n", 742 gpu->identity.vertex_output_buffer_size); 743 seq_printf(m, "\t buffer_size: %d\n", 744 gpu->identity.buffer_size); 745 seq_printf(m, "\t instruction_count: %d\n", 746 gpu->identity.instruction_count); 747 seq_printf(m, "\t num_constants: %d\n", 748 gpu->identity.num_constants); 749 seq_printf(m, "\t varyings_count: %d\n", 750 gpu->identity.varyings_count); 751 752 seq_printf(m, "\taxi: 0x%08x\n", axi); 753 seq_printf(m, "\tidle: 0x%08x\n", idle); 754 idle |= ~gpu->idle_mask & ~VIVS_HI_IDLE_STATE_AXI_LP; 755 if ((idle & VIVS_HI_IDLE_STATE_FE) == 0) 756 seq_puts(m, "\t FE is not idle\n"); 757 if ((idle & VIVS_HI_IDLE_STATE_DE) == 0) 758 seq_puts(m, "\t DE is not idle\n"); 759 if ((idle & VIVS_HI_IDLE_STATE_PE) == 0) 760 seq_puts(m, "\t PE is not idle\n"); 761 if ((idle & VIVS_HI_IDLE_STATE_SH) == 0) 762 seq_puts(m, "\t SH is not idle\n"); 763 if ((idle & VIVS_HI_IDLE_STATE_PA) == 0) 764 seq_puts(m, "\t PA is not idle\n"); 765 if ((idle & VIVS_HI_IDLE_STATE_SE) == 0) 766 seq_puts(m, "\t SE is not idle\n"); 767 if ((idle & VIVS_HI_IDLE_STATE_RA) == 0) 768 seq_puts(m, "\t RA is not idle\n"); 769 if ((idle & VIVS_HI_IDLE_STATE_TX) == 0) 770 seq_puts(m, "\t TX is not idle\n"); 771 if ((idle & VIVS_HI_IDLE_STATE_VG) == 0) 772 seq_puts(m, "\t VG is not idle\n"); 773 if ((idle & VIVS_HI_IDLE_STATE_IM) == 0) 774 seq_puts(m, "\t IM is not idle\n"); 775 if ((idle & VIVS_HI_IDLE_STATE_FP) == 0) 776 seq_puts(m, "\t FP is not idle\n"); 777 if ((idle & VIVS_HI_IDLE_STATE_TS) == 0) 778 seq_puts(m, "\t TS is not idle\n"); 779 if (idle & VIVS_HI_IDLE_STATE_AXI_LP) 780 seq_puts(m, "\t AXI low power mode\n"); 781 782 if (gpu->identity.features & chipFeatures_DEBUG_MODE) { 783 u32 read0 = gpu_read(gpu, VIVS_MC_DEBUG_READ0); 784 u32 read1 = gpu_read(gpu, VIVS_MC_DEBUG_READ1); 785 u32 write = gpu_read(gpu, VIVS_MC_DEBUG_WRITE); 786 787 seq_puts(m, "\tMC\n"); 788 seq_printf(m, "\t read0: 0x%08x\n", read0); 789 seq_printf(m, "\t read1: 0x%08x\n", read1); 790 seq_printf(m, "\t write: 0x%08x\n", write); 791 } 792 793 seq_puts(m, "\tDMA "); 794 795 if (debug.address[0] == debug.address[1] && 796 debug.state[0] == debug.state[1]) { 797 seq_puts(m, "seems to be stuck\n"); 798 } else if (debug.address[0] == debug.address[1]) { 799 seq_puts(m, "address is constant\n"); 800 } else { 801 seq_puts(m, "is running\n"); 802 } 803 804 seq_printf(m, "\t address 0: 0x%08x\n", debug.address[0]); 805 seq_printf(m, "\t address 1: 0x%08x\n", debug.address[1]); 806 seq_printf(m, "\t state 0: 0x%08x\n", debug.state[0]); 807 seq_printf(m, "\t state 1: 0x%08x\n", debug.state[1]); 808 seq_printf(m, "\t last fetch 64 bit word: 0x%08x 0x%08x\n", 809 dma_lo, dma_hi); 810 811 ret = 0; 812 813 pm_runtime_mark_last_busy(gpu->dev); 814 pm_runtime_put_autosuspend(gpu->dev); 815 816 return ret; 817 } 818 #endif 819 820 /* 821 * Power Management: 822 */ 823 static int enable_clk(struct etnaviv_gpu *gpu) 824 { 825 if (gpu->clk_core) 826 clk_prepare_enable(gpu->clk_core); 827 if (gpu->clk_shader) 828 clk_prepare_enable(gpu->clk_shader); 829 830 return 0; 831 } 832 833 static int disable_clk(struct etnaviv_gpu *gpu) 834 { 835 if (gpu->clk_core) 836 clk_disable_unprepare(gpu->clk_core); 837 if (gpu->clk_shader) 838 clk_disable_unprepare(gpu->clk_shader); 839 840 return 0; 841 } 842 843 static int enable_axi(struct etnaviv_gpu *gpu) 844 { 845 if (gpu->clk_bus) 846 clk_prepare_enable(gpu->clk_bus); 847 848 return 0; 849 } 850 851 static int disable_axi(struct etnaviv_gpu *gpu) 852 { 853 if (gpu->clk_bus) 854 clk_disable_unprepare(gpu->clk_bus); 855 856 return 0; 857 } 858 859 /* 860 * Hangcheck detection for locked gpu: 861 */ 862 static void recover_worker(struct work_struct *work) 863 { 864 struct etnaviv_gpu *gpu = container_of(work, struct etnaviv_gpu, 865 recover_work); 866 unsigned long flags; 867 unsigned int i; 868 869 dev_err(gpu->dev, "hangcheck recover!\n"); 870 871 if (pm_runtime_get_sync(gpu->dev) < 0) 872 return; 873 874 mutex_lock(&gpu->lock); 875 876 /* Only catch the first event, or when manually re-armed */ 877 if (etnaviv_dump_core) { 878 etnaviv_core_dump(gpu); 879 etnaviv_dump_core = false; 880 } 881 882 etnaviv_hw_reset(gpu); 883 884 /* complete all events, the GPU won't do it after the reset */ 885 spin_lock_irqsave(&gpu->event_spinlock, flags); 886 for (i = 0; i < ARRAY_SIZE(gpu->event); i++) { 887 if (!gpu->event[i].used) 888 continue; 889 fence_signal(gpu->event[i].fence); 890 gpu->event[i].fence = NULL; 891 gpu->event[i].used = false; 892 complete(&gpu->event_free); 893 } 894 spin_unlock_irqrestore(&gpu->event_spinlock, flags); 895 gpu->completed_fence = gpu->active_fence; 896 897 etnaviv_gpu_hw_init(gpu); 898 gpu->switch_context = true; 899 gpu->exec_state = -1; 900 901 mutex_unlock(&gpu->lock); 902 pm_runtime_mark_last_busy(gpu->dev); 903 pm_runtime_put_autosuspend(gpu->dev); 904 905 /* Retire the buffer objects in a work */ 906 etnaviv_queue_work(gpu->drm, &gpu->retire_work); 907 } 908 909 static void hangcheck_timer_reset(struct etnaviv_gpu *gpu) 910 { 911 DBG("%s", dev_name(gpu->dev)); 912 mod_timer(&gpu->hangcheck_timer, 913 round_jiffies_up(jiffies + DRM_ETNAVIV_HANGCHECK_JIFFIES)); 914 } 915 916 static void hangcheck_handler(unsigned long data) 917 { 918 struct etnaviv_gpu *gpu = (struct etnaviv_gpu *)data; 919 u32 fence = gpu->completed_fence; 920 bool progress = false; 921 922 if (fence != gpu->hangcheck_fence) { 923 gpu->hangcheck_fence = fence; 924 progress = true; 925 } 926 927 if (!progress) { 928 u32 dma_addr = gpu_read(gpu, VIVS_FE_DMA_ADDRESS); 929 int change = dma_addr - gpu->hangcheck_dma_addr; 930 931 if (change < 0 || change > 16) { 932 gpu->hangcheck_dma_addr = dma_addr; 933 progress = true; 934 } 935 } 936 937 if (!progress && fence_after(gpu->active_fence, fence)) { 938 dev_err(gpu->dev, "hangcheck detected gpu lockup!\n"); 939 dev_err(gpu->dev, " completed fence: %u\n", fence); 940 dev_err(gpu->dev, " active fence: %u\n", 941 gpu->active_fence); 942 etnaviv_queue_work(gpu->drm, &gpu->recover_work); 943 } 944 945 /* if still more pending work, reset the hangcheck timer: */ 946 if (fence_after(gpu->active_fence, gpu->hangcheck_fence)) 947 hangcheck_timer_reset(gpu); 948 } 949 950 static void hangcheck_disable(struct etnaviv_gpu *gpu) 951 { 952 del_timer_sync(&gpu->hangcheck_timer); 953 cancel_work_sync(&gpu->recover_work); 954 } 955 956 /* fence object management */ 957 struct etnaviv_fence { 958 struct etnaviv_gpu *gpu; 959 struct fence base; 960 }; 961 962 static inline struct etnaviv_fence *to_etnaviv_fence(struct fence *fence) 963 { 964 return container_of(fence, struct etnaviv_fence, base); 965 } 966 967 static const char *etnaviv_fence_get_driver_name(struct fence *fence) 968 { 969 return "etnaviv"; 970 } 971 972 static const char *etnaviv_fence_get_timeline_name(struct fence *fence) 973 { 974 struct etnaviv_fence *f = to_etnaviv_fence(fence); 975 976 return dev_name(f->gpu->dev); 977 } 978 979 static bool etnaviv_fence_enable_signaling(struct fence *fence) 980 { 981 return true; 982 } 983 984 static bool etnaviv_fence_signaled(struct fence *fence) 985 { 986 struct etnaviv_fence *f = to_etnaviv_fence(fence); 987 988 return fence_completed(f->gpu, f->base.seqno); 989 } 990 991 static void etnaviv_fence_release(struct fence *fence) 992 { 993 struct etnaviv_fence *f = to_etnaviv_fence(fence); 994 995 kfree_rcu(f, base.rcu); 996 } 997 998 static const struct fence_ops etnaviv_fence_ops = { 999 .get_driver_name = etnaviv_fence_get_driver_name, 1000 .get_timeline_name = etnaviv_fence_get_timeline_name, 1001 .enable_signaling = etnaviv_fence_enable_signaling, 1002 .signaled = etnaviv_fence_signaled, 1003 .wait = fence_default_wait, 1004 .release = etnaviv_fence_release, 1005 }; 1006 1007 static struct fence *etnaviv_gpu_fence_alloc(struct etnaviv_gpu *gpu) 1008 { 1009 struct etnaviv_fence *f; 1010 1011 f = kzalloc(sizeof(*f), GFP_KERNEL); 1012 if (!f) 1013 return NULL; 1014 1015 f->gpu = gpu; 1016 1017 fence_init(&f->base, &etnaviv_fence_ops, &gpu->fence_spinlock, 1018 gpu->fence_context, ++gpu->next_fence); 1019 1020 return &f->base; 1021 } 1022 1023 int etnaviv_gpu_fence_sync_obj(struct etnaviv_gem_object *etnaviv_obj, 1024 unsigned int context, bool exclusive) 1025 { 1026 struct reservation_object *robj = etnaviv_obj->resv; 1027 struct reservation_object_list *fobj; 1028 struct fence *fence; 1029 int i, ret; 1030 1031 if (!exclusive) { 1032 ret = reservation_object_reserve_shared(robj); 1033 if (ret) 1034 return ret; 1035 } 1036 1037 /* 1038 * If we have any shared fences, then the exclusive fence 1039 * should be ignored as it will already have been signalled. 1040 */ 1041 fobj = reservation_object_get_list(robj); 1042 if (!fobj || fobj->shared_count == 0) { 1043 /* Wait on any existing exclusive fence which isn't our own */ 1044 fence = reservation_object_get_excl(robj); 1045 if (fence && fence->context != context) { 1046 ret = fence_wait(fence, true); 1047 if (ret) 1048 return ret; 1049 } 1050 } 1051 1052 if (!exclusive || !fobj) 1053 return 0; 1054 1055 for (i = 0; i < fobj->shared_count; i++) { 1056 fence = rcu_dereference_protected(fobj->shared[i], 1057 reservation_object_held(robj)); 1058 if (fence->context != context) { 1059 ret = fence_wait(fence, true); 1060 if (ret) 1061 return ret; 1062 } 1063 } 1064 1065 return 0; 1066 } 1067 1068 /* 1069 * event management: 1070 */ 1071 1072 static unsigned int event_alloc(struct etnaviv_gpu *gpu) 1073 { 1074 unsigned long ret, flags; 1075 unsigned int i, event = ~0U; 1076 1077 ret = wait_for_completion_timeout(&gpu->event_free, 1078 msecs_to_jiffies(10 * 10000)); 1079 if (!ret) 1080 dev_err(gpu->dev, "wait_for_completion_timeout failed"); 1081 1082 spin_lock_irqsave(&gpu->event_spinlock, flags); 1083 1084 /* find first free event */ 1085 for (i = 0; i < ARRAY_SIZE(gpu->event); i++) { 1086 if (gpu->event[i].used == false) { 1087 gpu->event[i].used = true; 1088 event = i; 1089 break; 1090 } 1091 } 1092 1093 spin_unlock_irqrestore(&gpu->event_spinlock, flags); 1094 1095 return event; 1096 } 1097 1098 static void event_free(struct etnaviv_gpu *gpu, unsigned int event) 1099 { 1100 unsigned long flags; 1101 1102 spin_lock_irqsave(&gpu->event_spinlock, flags); 1103 1104 if (gpu->event[event].used == false) { 1105 dev_warn(gpu->dev, "event %u is already marked as free", 1106 event); 1107 spin_unlock_irqrestore(&gpu->event_spinlock, flags); 1108 } else { 1109 gpu->event[event].used = false; 1110 spin_unlock_irqrestore(&gpu->event_spinlock, flags); 1111 1112 complete(&gpu->event_free); 1113 } 1114 } 1115 1116 /* 1117 * Cmdstream submission/retirement: 1118 */ 1119 1120 struct etnaviv_cmdbuf *etnaviv_gpu_cmdbuf_new(struct etnaviv_gpu *gpu, u32 size, 1121 size_t nr_bos) 1122 { 1123 struct etnaviv_cmdbuf *cmdbuf; 1124 size_t sz = size_vstruct(nr_bos, sizeof(cmdbuf->bo_map[0]), 1125 sizeof(*cmdbuf)); 1126 1127 cmdbuf = kzalloc(sz, GFP_KERNEL); 1128 if (!cmdbuf) 1129 return NULL; 1130 1131 cmdbuf->vaddr = dma_alloc_wc(gpu->dev, size, &cmdbuf->paddr, 1132 GFP_KERNEL); 1133 if (!cmdbuf->vaddr) { 1134 kfree(cmdbuf); 1135 return NULL; 1136 } 1137 1138 cmdbuf->gpu = gpu; 1139 cmdbuf->size = size; 1140 1141 return cmdbuf; 1142 } 1143 1144 void etnaviv_gpu_cmdbuf_free(struct etnaviv_cmdbuf *cmdbuf) 1145 { 1146 dma_free_wc(cmdbuf->gpu->dev, cmdbuf->size, cmdbuf->vaddr, 1147 cmdbuf->paddr); 1148 kfree(cmdbuf); 1149 } 1150 1151 static void retire_worker(struct work_struct *work) 1152 { 1153 struct etnaviv_gpu *gpu = container_of(work, struct etnaviv_gpu, 1154 retire_work); 1155 u32 fence = gpu->completed_fence; 1156 struct etnaviv_cmdbuf *cmdbuf, *tmp; 1157 unsigned int i; 1158 1159 mutex_lock(&gpu->lock); 1160 list_for_each_entry_safe(cmdbuf, tmp, &gpu->active_cmd_list, node) { 1161 if (!fence_is_signaled(cmdbuf->fence)) 1162 break; 1163 1164 list_del(&cmdbuf->node); 1165 fence_put(cmdbuf->fence); 1166 1167 for (i = 0; i < cmdbuf->nr_bos; i++) { 1168 struct etnaviv_vram_mapping *mapping = cmdbuf->bo_map[i]; 1169 struct etnaviv_gem_object *etnaviv_obj = mapping->object; 1170 1171 atomic_dec(&etnaviv_obj->gpu_active); 1172 /* drop the refcount taken in etnaviv_gpu_submit */ 1173 etnaviv_gem_mapping_unreference(mapping); 1174 } 1175 1176 etnaviv_gpu_cmdbuf_free(cmdbuf); 1177 /* 1178 * We need to balance the runtime PM count caused by 1179 * each submission. Upon submission, we increment 1180 * the runtime PM counter, and allocate one event. 1181 * So here, we put the runtime PM count for each 1182 * completed event. 1183 */ 1184 pm_runtime_put_autosuspend(gpu->dev); 1185 } 1186 1187 gpu->retired_fence = fence; 1188 1189 mutex_unlock(&gpu->lock); 1190 1191 wake_up_all(&gpu->fence_event); 1192 } 1193 1194 int etnaviv_gpu_wait_fence_interruptible(struct etnaviv_gpu *gpu, 1195 u32 fence, struct timespec *timeout) 1196 { 1197 int ret; 1198 1199 if (fence_after(fence, gpu->next_fence)) { 1200 DRM_ERROR("waiting on invalid fence: %u (of %u)\n", 1201 fence, gpu->next_fence); 1202 return -EINVAL; 1203 } 1204 1205 if (!timeout) { 1206 /* No timeout was requested: just test for completion */ 1207 ret = fence_completed(gpu, fence) ? 0 : -EBUSY; 1208 } else { 1209 unsigned long remaining = etnaviv_timeout_to_jiffies(timeout); 1210 1211 ret = wait_event_interruptible_timeout(gpu->fence_event, 1212 fence_completed(gpu, fence), 1213 remaining); 1214 if (ret == 0) { 1215 DBG("timeout waiting for fence: %u (retired: %u completed: %u)", 1216 fence, gpu->retired_fence, 1217 gpu->completed_fence); 1218 ret = -ETIMEDOUT; 1219 } else if (ret != -ERESTARTSYS) { 1220 ret = 0; 1221 } 1222 } 1223 1224 return ret; 1225 } 1226 1227 /* 1228 * Wait for an object to become inactive. This, on it's own, is not race 1229 * free: the object is moved by the retire worker off the active list, and 1230 * then the iova is put. Moreover, the object could be re-submitted just 1231 * after we notice that it's become inactive. 1232 * 1233 * Although the retirement happens under the gpu lock, we don't want to hold 1234 * that lock in this function while waiting. 1235 */ 1236 int etnaviv_gpu_wait_obj_inactive(struct etnaviv_gpu *gpu, 1237 struct etnaviv_gem_object *etnaviv_obj, struct timespec *timeout) 1238 { 1239 unsigned long remaining; 1240 long ret; 1241 1242 if (!timeout) 1243 return !is_active(etnaviv_obj) ? 0 : -EBUSY; 1244 1245 remaining = etnaviv_timeout_to_jiffies(timeout); 1246 1247 ret = wait_event_interruptible_timeout(gpu->fence_event, 1248 !is_active(etnaviv_obj), 1249 remaining); 1250 if (ret > 0) { 1251 struct etnaviv_drm_private *priv = gpu->drm->dev_private; 1252 1253 /* Synchronise with the retire worker */ 1254 flush_workqueue(priv->wq); 1255 return 0; 1256 } else if (ret == -ERESTARTSYS) { 1257 return -ERESTARTSYS; 1258 } else { 1259 return -ETIMEDOUT; 1260 } 1261 } 1262 1263 int etnaviv_gpu_pm_get_sync(struct etnaviv_gpu *gpu) 1264 { 1265 return pm_runtime_get_sync(gpu->dev); 1266 } 1267 1268 void etnaviv_gpu_pm_put(struct etnaviv_gpu *gpu) 1269 { 1270 pm_runtime_mark_last_busy(gpu->dev); 1271 pm_runtime_put_autosuspend(gpu->dev); 1272 } 1273 1274 /* add bo's to gpu's ring, and kick gpu: */ 1275 int etnaviv_gpu_submit(struct etnaviv_gpu *gpu, 1276 struct etnaviv_gem_submit *submit, struct etnaviv_cmdbuf *cmdbuf) 1277 { 1278 struct fence *fence; 1279 unsigned int event, i; 1280 int ret; 1281 1282 ret = etnaviv_gpu_pm_get_sync(gpu); 1283 if (ret < 0) 1284 return ret; 1285 1286 mutex_lock(&gpu->lock); 1287 1288 /* 1289 * TODO 1290 * 1291 * - flush 1292 * - data endian 1293 * - prefetch 1294 * 1295 */ 1296 1297 event = event_alloc(gpu); 1298 if (unlikely(event == ~0U)) { 1299 DRM_ERROR("no free event\n"); 1300 ret = -EBUSY; 1301 goto out_unlock; 1302 } 1303 1304 fence = etnaviv_gpu_fence_alloc(gpu); 1305 if (!fence) { 1306 event_free(gpu, event); 1307 ret = -ENOMEM; 1308 goto out_unlock; 1309 } 1310 1311 gpu->event[event].fence = fence; 1312 submit->fence = fence->seqno; 1313 gpu->active_fence = submit->fence; 1314 1315 if (gpu->lastctx != cmdbuf->ctx) { 1316 gpu->mmu->need_flush = true; 1317 gpu->switch_context = true; 1318 gpu->lastctx = cmdbuf->ctx; 1319 } 1320 1321 etnaviv_buffer_queue(gpu, event, cmdbuf); 1322 1323 cmdbuf->fence = fence; 1324 list_add_tail(&cmdbuf->node, &gpu->active_cmd_list); 1325 1326 /* We're committed to adding this command buffer, hold a PM reference */ 1327 pm_runtime_get_noresume(gpu->dev); 1328 1329 for (i = 0; i < submit->nr_bos; i++) { 1330 struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; 1331 1332 /* Each cmdbuf takes a refcount on the mapping */ 1333 etnaviv_gem_mapping_reference(submit->bos[i].mapping); 1334 cmdbuf->bo_map[i] = submit->bos[i].mapping; 1335 atomic_inc(&etnaviv_obj->gpu_active); 1336 1337 if (submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE) 1338 reservation_object_add_excl_fence(etnaviv_obj->resv, 1339 fence); 1340 else 1341 reservation_object_add_shared_fence(etnaviv_obj->resv, 1342 fence); 1343 } 1344 cmdbuf->nr_bos = submit->nr_bos; 1345 hangcheck_timer_reset(gpu); 1346 ret = 0; 1347 1348 out_unlock: 1349 mutex_unlock(&gpu->lock); 1350 1351 etnaviv_gpu_pm_put(gpu); 1352 1353 return ret; 1354 } 1355 1356 /* 1357 * Init/Cleanup: 1358 */ 1359 static irqreturn_t irq_handler(int irq, void *data) 1360 { 1361 struct etnaviv_gpu *gpu = data; 1362 irqreturn_t ret = IRQ_NONE; 1363 1364 u32 intr = gpu_read(gpu, VIVS_HI_INTR_ACKNOWLEDGE); 1365 1366 if (intr != 0) { 1367 int event; 1368 1369 pm_runtime_mark_last_busy(gpu->dev); 1370 1371 dev_dbg(gpu->dev, "intr 0x%08x\n", intr); 1372 1373 if (intr & VIVS_HI_INTR_ACKNOWLEDGE_AXI_BUS_ERROR) { 1374 dev_err(gpu->dev, "AXI bus error\n"); 1375 intr &= ~VIVS_HI_INTR_ACKNOWLEDGE_AXI_BUS_ERROR; 1376 } 1377 1378 while ((event = ffs(intr)) != 0) { 1379 struct fence *fence; 1380 1381 event -= 1; 1382 1383 intr &= ~(1 << event); 1384 1385 dev_dbg(gpu->dev, "event %u\n", event); 1386 1387 fence = gpu->event[event].fence; 1388 gpu->event[event].fence = NULL; 1389 fence_signal(fence); 1390 1391 /* 1392 * Events can be processed out of order. Eg, 1393 * - allocate and queue event 0 1394 * - allocate event 1 1395 * - event 0 completes, we process it 1396 * - allocate and queue event 0 1397 * - event 1 and event 0 complete 1398 * we can end up processing event 0 first, then 1. 1399 */ 1400 if (fence_after(fence->seqno, gpu->completed_fence)) 1401 gpu->completed_fence = fence->seqno; 1402 1403 event_free(gpu, event); 1404 } 1405 1406 /* Retire the buffer objects in a work */ 1407 etnaviv_queue_work(gpu->drm, &gpu->retire_work); 1408 1409 ret = IRQ_HANDLED; 1410 } 1411 1412 return ret; 1413 } 1414 1415 static int etnaviv_gpu_clk_enable(struct etnaviv_gpu *gpu) 1416 { 1417 int ret; 1418 1419 ret = enable_clk(gpu); 1420 if (ret) 1421 return ret; 1422 1423 ret = enable_axi(gpu); 1424 if (ret) { 1425 disable_clk(gpu); 1426 return ret; 1427 } 1428 1429 return 0; 1430 } 1431 1432 static int etnaviv_gpu_clk_disable(struct etnaviv_gpu *gpu) 1433 { 1434 int ret; 1435 1436 ret = disable_axi(gpu); 1437 if (ret) 1438 return ret; 1439 1440 ret = disable_clk(gpu); 1441 if (ret) 1442 return ret; 1443 1444 return 0; 1445 } 1446 1447 static int etnaviv_gpu_hw_suspend(struct etnaviv_gpu *gpu) 1448 { 1449 if (gpu->buffer) { 1450 unsigned long timeout; 1451 1452 /* Replace the last WAIT with END */ 1453 etnaviv_buffer_end(gpu); 1454 1455 /* 1456 * We know that only the FE is busy here, this should 1457 * happen quickly (as the WAIT is only 200 cycles). If 1458 * we fail, just warn and continue. 1459 */ 1460 timeout = jiffies + msecs_to_jiffies(100); 1461 do { 1462 u32 idle = gpu_read(gpu, VIVS_HI_IDLE_STATE); 1463 1464 if ((idle & gpu->idle_mask) == gpu->idle_mask) 1465 break; 1466 1467 if (time_is_before_jiffies(timeout)) { 1468 dev_warn(gpu->dev, 1469 "timed out waiting for idle: idle=0x%x\n", 1470 idle); 1471 break; 1472 } 1473 1474 udelay(5); 1475 } while (1); 1476 } 1477 1478 return etnaviv_gpu_clk_disable(gpu); 1479 } 1480 1481 #ifdef CONFIG_PM 1482 static int etnaviv_gpu_hw_resume(struct etnaviv_gpu *gpu) 1483 { 1484 u32 clock; 1485 int ret; 1486 1487 ret = mutex_lock_killable(&gpu->lock); 1488 if (ret) 1489 return ret; 1490 1491 clock = VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS | 1492 VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(0x40); 1493 1494 etnaviv_gpu_load_clock(gpu, clock); 1495 etnaviv_gpu_hw_init(gpu); 1496 1497 gpu->switch_context = true; 1498 gpu->exec_state = -1; 1499 1500 mutex_unlock(&gpu->lock); 1501 1502 return 0; 1503 } 1504 #endif 1505 1506 static int etnaviv_gpu_bind(struct device *dev, struct device *master, 1507 void *data) 1508 { 1509 struct drm_device *drm = data; 1510 struct etnaviv_drm_private *priv = drm->dev_private; 1511 struct etnaviv_gpu *gpu = dev_get_drvdata(dev); 1512 int ret; 1513 1514 #ifdef CONFIG_PM 1515 ret = pm_runtime_get_sync(gpu->dev); 1516 #else 1517 ret = etnaviv_gpu_clk_enable(gpu); 1518 #endif 1519 if (ret < 0) 1520 return ret; 1521 1522 gpu->drm = drm; 1523 gpu->fence_context = fence_context_alloc(1); 1524 spin_lock_init(&gpu->fence_spinlock); 1525 1526 INIT_LIST_HEAD(&gpu->active_cmd_list); 1527 INIT_WORK(&gpu->retire_work, retire_worker); 1528 INIT_WORK(&gpu->recover_work, recover_worker); 1529 init_waitqueue_head(&gpu->fence_event); 1530 1531 setup_deferrable_timer(&gpu->hangcheck_timer, hangcheck_handler, 1532 (unsigned long)gpu); 1533 1534 priv->gpu[priv->num_gpus++] = gpu; 1535 1536 pm_runtime_mark_last_busy(gpu->dev); 1537 pm_runtime_put_autosuspend(gpu->dev); 1538 1539 return 0; 1540 } 1541 1542 static void etnaviv_gpu_unbind(struct device *dev, struct device *master, 1543 void *data) 1544 { 1545 struct etnaviv_gpu *gpu = dev_get_drvdata(dev); 1546 1547 DBG("%s", dev_name(gpu->dev)); 1548 1549 hangcheck_disable(gpu); 1550 1551 #ifdef CONFIG_PM 1552 pm_runtime_get_sync(gpu->dev); 1553 pm_runtime_put_sync_suspend(gpu->dev); 1554 #else 1555 etnaviv_gpu_hw_suspend(gpu); 1556 #endif 1557 1558 if (gpu->buffer) { 1559 etnaviv_gpu_cmdbuf_free(gpu->buffer); 1560 gpu->buffer = NULL; 1561 } 1562 1563 if (gpu->mmu) { 1564 etnaviv_iommu_destroy(gpu->mmu); 1565 gpu->mmu = NULL; 1566 } 1567 1568 gpu->drm = NULL; 1569 } 1570 1571 static const struct component_ops gpu_ops = { 1572 .bind = etnaviv_gpu_bind, 1573 .unbind = etnaviv_gpu_unbind, 1574 }; 1575 1576 static const struct of_device_id etnaviv_gpu_match[] = { 1577 { 1578 .compatible = "vivante,gc" 1579 }, 1580 { /* sentinel */ } 1581 }; 1582 1583 static int etnaviv_gpu_platform_probe(struct platform_device *pdev) 1584 { 1585 struct device *dev = &pdev->dev; 1586 struct etnaviv_gpu *gpu; 1587 int err = 0; 1588 1589 gpu = devm_kzalloc(dev, sizeof(*gpu), GFP_KERNEL); 1590 if (!gpu) 1591 return -ENOMEM; 1592 1593 gpu->dev = &pdev->dev; 1594 mutex_init(&gpu->lock); 1595 1596 /* Map registers: */ 1597 gpu->mmio = etnaviv_ioremap(pdev, NULL, dev_name(gpu->dev)); 1598 if (IS_ERR(gpu->mmio)) 1599 return PTR_ERR(gpu->mmio); 1600 1601 /* Get Interrupt: */ 1602 gpu->irq = platform_get_irq(pdev, 0); 1603 if (gpu->irq < 0) { 1604 err = gpu->irq; 1605 dev_err(dev, "failed to get irq: %d\n", err); 1606 goto fail; 1607 } 1608 1609 err = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, 0, 1610 dev_name(gpu->dev), gpu); 1611 if (err) { 1612 dev_err(dev, "failed to request IRQ%u: %d\n", gpu->irq, err); 1613 goto fail; 1614 } 1615 1616 /* Get Clocks: */ 1617 gpu->clk_bus = devm_clk_get(&pdev->dev, "bus"); 1618 DBG("clk_bus: %p", gpu->clk_bus); 1619 if (IS_ERR(gpu->clk_bus)) 1620 gpu->clk_bus = NULL; 1621 1622 gpu->clk_core = devm_clk_get(&pdev->dev, "core"); 1623 DBG("clk_core: %p", gpu->clk_core); 1624 if (IS_ERR(gpu->clk_core)) 1625 gpu->clk_core = NULL; 1626 1627 gpu->clk_shader = devm_clk_get(&pdev->dev, "shader"); 1628 DBG("clk_shader: %p", gpu->clk_shader); 1629 if (IS_ERR(gpu->clk_shader)) 1630 gpu->clk_shader = NULL; 1631 1632 /* TODO: figure out max mapped size */ 1633 dev_set_drvdata(dev, gpu); 1634 1635 /* 1636 * We treat the device as initially suspended. The runtime PM 1637 * autosuspend delay is rather arbitary: no measurements have 1638 * yet been performed to determine an appropriate value. 1639 */ 1640 pm_runtime_use_autosuspend(gpu->dev); 1641 pm_runtime_set_autosuspend_delay(gpu->dev, 200); 1642 pm_runtime_enable(gpu->dev); 1643 1644 err = component_add(&pdev->dev, &gpu_ops); 1645 if (err < 0) { 1646 dev_err(&pdev->dev, "failed to register component: %d\n", err); 1647 goto fail; 1648 } 1649 1650 return 0; 1651 1652 fail: 1653 return err; 1654 } 1655 1656 static int etnaviv_gpu_platform_remove(struct platform_device *pdev) 1657 { 1658 component_del(&pdev->dev, &gpu_ops); 1659 pm_runtime_disable(&pdev->dev); 1660 return 0; 1661 } 1662 1663 #ifdef CONFIG_PM 1664 static int etnaviv_gpu_rpm_suspend(struct device *dev) 1665 { 1666 struct etnaviv_gpu *gpu = dev_get_drvdata(dev); 1667 u32 idle, mask; 1668 1669 /* If we have outstanding fences, we're not idle */ 1670 if (gpu->completed_fence != gpu->active_fence) 1671 return -EBUSY; 1672 1673 /* Check whether the hardware (except FE) is idle */ 1674 mask = gpu->idle_mask & ~VIVS_HI_IDLE_STATE_FE; 1675 idle = gpu_read(gpu, VIVS_HI_IDLE_STATE) & mask; 1676 if (idle != mask) 1677 return -EBUSY; 1678 1679 return etnaviv_gpu_hw_suspend(gpu); 1680 } 1681 1682 static int etnaviv_gpu_rpm_resume(struct device *dev) 1683 { 1684 struct etnaviv_gpu *gpu = dev_get_drvdata(dev); 1685 int ret; 1686 1687 ret = etnaviv_gpu_clk_enable(gpu); 1688 if (ret) 1689 return ret; 1690 1691 /* Re-initialise the basic hardware state */ 1692 if (gpu->drm && gpu->buffer) { 1693 ret = etnaviv_gpu_hw_resume(gpu); 1694 if (ret) { 1695 etnaviv_gpu_clk_disable(gpu); 1696 return ret; 1697 } 1698 } 1699 1700 return 0; 1701 } 1702 #endif 1703 1704 static const struct dev_pm_ops etnaviv_gpu_pm_ops = { 1705 SET_RUNTIME_PM_OPS(etnaviv_gpu_rpm_suspend, etnaviv_gpu_rpm_resume, 1706 NULL) 1707 }; 1708 1709 struct platform_driver etnaviv_gpu_driver = { 1710 .driver = { 1711 .name = "etnaviv-gpu", 1712 .owner = THIS_MODULE, 1713 .pm = &etnaviv_gpu_pm_ops, 1714 .of_match_table = etnaviv_gpu_match, 1715 }, 1716 .probe = etnaviv_gpu_platform_probe, 1717 .remove = etnaviv_gpu_platform_remove, 1718 .id_table = gpu_ids, 1719 }; 1720