1 // SPDX-License-Identifier: GPL-2.0 or MIT 2 /* Copyright 2023 Collabora ltd. */ 3 4 #ifdef CONFIG_ARM_ARCH_TIMER 5 #include <asm/arch_timer.h> 6 #endif 7 8 #include <linux/clk.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/firmware.h> 11 #include <linux/iopoll.h> 12 #include <linux/iosys-map.h> 13 #include <linux/mutex.h> 14 #include <linux/platform_device.h> 15 #include <linux/pm_runtime.h> 16 17 #include <drm/drm_drv.h> 18 #include <drm/drm_managed.h> 19 #include <drm/drm_print.h> 20 21 #include "panthor_device.h" 22 #include "panthor_fw.h" 23 #include "panthor_fw_regs.h" 24 #include "panthor_gem.h" 25 #include "panthor_gpu.h" 26 #include "panthor_hw.h" 27 #include "panthor_mmu.h" 28 #include "panthor_sched.h" 29 #include "panthor_trace.h" 30 31 #define CSF_FW_NAME "mali_csffw.bin" 32 33 #define PING_INTERVAL_MS 12000 34 #define PROGRESS_TIMEOUT_CYCLES (5ull * 500 * 1024 * 1024) 35 #define PROGRESS_TIMEOUT_SCALE_SHIFT 10 36 #define IDLE_HYSTERESIS_US 800 37 #define PWROFF_HYSTERESIS_US 10000 38 #define MCU_HALT_TIMEOUT_US (1ULL * USEC_PER_SEC) 39 40 /** 41 * struct panthor_fw_binary_hdr - Firmware binary header. 42 */ 43 struct panthor_fw_binary_hdr { 44 /** @magic: Magic value to check binary validity. */ 45 u32 magic; 46 #define CSF_FW_BINARY_HEADER_MAGIC 0xc3f13a6e 47 48 /** @minor: Minor FW version. */ 49 u8 minor; 50 51 /** @major: Major FW version. */ 52 u8 major; 53 #define CSF_FW_BINARY_HEADER_MAJOR_MAX 0 54 55 /** @padding1: MBZ. */ 56 u16 padding1; 57 58 /** @version_hash: FW version hash. */ 59 u32 version_hash; 60 61 /** @padding2: MBZ. */ 62 u32 padding2; 63 64 /** @size: FW binary size. */ 65 u32 size; 66 }; 67 68 /** 69 * enum panthor_fw_binary_entry_type - Firmware binary entry type 70 */ 71 enum panthor_fw_binary_entry_type { 72 /** @CSF_FW_BINARY_ENTRY_TYPE_IFACE: Host <-> FW interface. */ 73 CSF_FW_BINARY_ENTRY_TYPE_IFACE = 0, 74 75 /** @CSF_FW_BINARY_ENTRY_TYPE_CONFIG: FW config. */ 76 CSF_FW_BINARY_ENTRY_TYPE_CONFIG = 1, 77 78 /** @CSF_FW_BINARY_ENTRY_TYPE_FUTF_TEST: Unit-tests. */ 79 CSF_FW_BINARY_ENTRY_TYPE_FUTF_TEST = 2, 80 81 /** @CSF_FW_BINARY_ENTRY_TYPE_TRACE_BUFFER: Trace buffer interface. */ 82 CSF_FW_BINARY_ENTRY_TYPE_TRACE_BUFFER = 3, 83 84 /** @CSF_FW_BINARY_ENTRY_TYPE_TIMELINE_METADATA: Timeline metadata interface. */ 85 CSF_FW_BINARY_ENTRY_TYPE_TIMELINE_METADATA = 4, 86 87 /** 88 * @CSF_FW_BINARY_ENTRY_TYPE_BUILD_INFO_METADATA: Metadata about how 89 * the FW binary was built. 90 */ 91 CSF_FW_BINARY_ENTRY_TYPE_BUILD_INFO_METADATA = 6 92 }; 93 94 #define CSF_FW_BINARY_ENTRY_TYPE(ehdr) ((ehdr) & 0xff) 95 #define CSF_FW_BINARY_ENTRY_SIZE(ehdr) (((ehdr) >> 8) & 0xff) 96 #define CSF_FW_BINARY_ENTRY_UPDATE BIT(30) 97 #define CSF_FW_BINARY_ENTRY_OPTIONAL BIT(31) 98 99 #define CSF_FW_BINARY_IFACE_ENTRY_RD BIT(0) 100 #define CSF_FW_BINARY_IFACE_ENTRY_WR BIT(1) 101 #define CSF_FW_BINARY_IFACE_ENTRY_EX BIT(2) 102 #define CSF_FW_BINARY_IFACE_ENTRY_CACHE_MODE_NONE (0 << 3) 103 #define CSF_FW_BINARY_IFACE_ENTRY_CACHE_MODE_CACHED (1 << 3) 104 #define CSF_FW_BINARY_IFACE_ENTRY_CACHE_MODE_UNCACHED_COHERENT (2 << 3) 105 #define CSF_FW_BINARY_IFACE_ENTRY_CACHE_MODE_CACHED_COHERENT (3 << 3) 106 #define CSF_FW_BINARY_IFACE_ENTRY_CACHE_MODE_MASK GENMASK(4, 3) 107 #define CSF_FW_BINARY_IFACE_ENTRY_PROT BIT(5) 108 #define CSF_FW_BINARY_IFACE_ENTRY_SHARED BIT(30) 109 #define CSF_FW_BINARY_IFACE_ENTRY_ZERO BIT(31) 110 111 #define CSF_FW_BINARY_IFACE_ENTRY_SUPPORTED_FLAGS \ 112 (CSF_FW_BINARY_IFACE_ENTRY_RD | \ 113 CSF_FW_BINARY_IFACE_ENTRY_WR | \ 114 CSF_FW_BINARY_IFACE_ENTRY_EX | \ 115 CSF_FW_BINARY_IFACE_ENTRY_CACHE_MODE_MASK | \ 116 CSF_FW_BINARY_IFACE_ENTRY_PROT | \ 117 CSF_FW_BINARY_IFACE_ENTRY_SHARED | \ 118 CSF_FW_BINARY_IFACE_ENTRY_ZERO) 119 120 /** 121 * struct panthor_fw_binary_section_entry_hdr - Describes a section of FW binary 122 */ 123 struct panthor_fw_binary_section_entry_hdr { 124 /** @flags: Section flags. */ 125 u32 flags; 126 127 /** @va: MCU virtual range to map this binary section to. */ 128 struct { 129 /** @start: Start address. */ 130 u32 start; 131 132 /** @end: End address. */ 133 u32 end; 134 } va; 135 136 /** @data: Data to initialize the FW section with. */ 137 struct { 138 /** @start: Start offset in the FW binary. */ 139 u32 start; 140 141 /** @end: End offset in the FW binary. */ 142 u32 end; 143 } data; 144 }; 145 146 struct panthor_fw_build_info_hdr { 147 /** @meta_start: Offset of the build info data in the FW binary */ 148 u32 meta_start; 149 /** @meta_size: Size of the build info data in the FW binary */ 150 u32 meta_size; 151 }; 152 153 /** 154 * struct panthor_fw_binary_iter - Firmware binary iterator 155 * 156 * Used to parse a firmware binary. 157 */ 158 struct panthor_fw_binary_iter { 159 /** @data: FW binary data. */ 160 const void *data; 161 162 /** @size: FW binary size. */ 163 size_t size; 164 165 /** @offset: Iterator offset. */ 166 size_t offset; 167 }; 168 169 /** 170 * struct panthor_fw_section - FW section 171 */ 172 struct panthor_fw_section { 173 /** @node: Used to keep track of FW sections. */ 174 struct list_head node; 175 176 /** @flags: Section flags, as encoded in the FW binary. */ 177 u32 flags; 178 179 /** @mem: Section memory. */ 180 struct panthor_kernel_bo *mem; 181 182 /** 183 * @name: Name of the section, as specified in the binary. 184 * 185 * Can be NULL. 186 */ 187 const char *name; 188 189 /** 190 * @data: Initial data copied to the FW memory. 191 * 192 * We keep data around so we can reload sections after a reset. 193 */ 194 struct { 195 /** @buf: Buffed used to store init data. */ 196 const void *buf; 197 198 /** @size: Size of @buf in bytes. */ 199 size_t size; 200 } data; 201 }; 202 203 #define CSF_MCU_SHARED_REGION_START 0x04000000ULL 204 #define CSF_MCU_SHARED_REGION_SIZE 0x04000000ULL 205 206 #define MIN_CS_PER_CSG 8 207 #define MIN_CSGS 3 208 209 #define CSF_IFACE_VERSION(major, minor, patch) \ 210 (((major) << 24) | ((minor) << 16) | (patch)) 211 #define CSF_IFACE_VERSION_MAJOR(v) ((v) >> 24) 212 #define CSF_IFACE_VERSION_MINOR(v) (((v) >> 16) & 0xff) 213 #define CSF_IFACE_VERSION_PATCH(v) ((v) & 0xffff) 214 215 #define CSF_GROUP_CONTROL_OFFSET 0x1000 216 #define CSF_STREAM_CONTROL_OFFSET 0x40 217 #define CSF_UNPRESERVED_REG_COUNT 4 218 219 /** 220 * struct panthor_fw_iface - FW interfaces 221 */ 222 struct panthor_fw_iface { 223 /** @global: Global interface. */ 224 struct panthor_fw_global_iface global; 225 226 /** @groups: Group slot interfaces. */ 227 struct panthor_fw_csg_iface groups[MAX_CSGS]; 228 229 /** @streams: Command stream slot interfaces. */ 230 struct panthor_fw_cs_iface streams[MAX_CSGS][MAX_CS_PER_CSG]; 231 }; 232 233 /** 234 * struct panthor_fw - Firmware management 235 */ 236 struct panthor_fw { 237 /** @iomem: CPU mapping of MCU_CONTROL iomem region */ 238 void __iomem *iomem; 239 240 /** @vm: MCU VM. */ 241 struct panthor_vm *vm; 242 243 /** @sections: List of FW sections. */ 244 struct list_head sections; 245 246 /** @shared_section: The section containing the FW interfaces. */ 247 struct panthor_fw_section *shared_section; 248 249 /** @iface: FW interfaces. */ 250 struct panthor_fw_iface iface; 251 252 /** @watchdog: Collection of fields relating to the FW watchdog. */ 253 struct { 254 /** @ping_work: Delayed work used to ping the FW. */ 255 struct delayed_work ping_work; 256 } watchdog; 257 258 /** 259 * @req_waitqueue: FW request waitqueue. 260 * 261 * Everytime a request is sent to a command stream group or the global 262 * interface, the caller will first busy wait for the request to be 263 * acknowledged, and then fallback to a sleeping wait. 264 * 265 * This wait queue is here to support the sleeping wait flavor. 266 */ 267 wait_queue_head_t req_waitqueue; 268 269 /** @booted: True is the FW is booted */ 270 bool booted; 271 272 /** @irq: Job irq data. */ 273 struct panthor_irq irq; 274 }; 275 276 struct panthor_vm *panthor_fw_vm(struct panthor_device *ptdev) 277 { 278 return ptdev->fw->vm; 279 } 280 281 /** 282 * panthor_fw_get_glb_iface() - Get the global interface 283 * @ptdev: Device. 284 * 285 * Return: The global interface. 286 */ 287 struct panthor_fw_global_iface * 288 panthor_fw_get_glb_iface(struct panthor_device *ptdev) 289 { 290 return &ptdev->fw->iface.global; 291 } 292 293 /** 294 * panthor_fw_get_csg_iface() - Get a command stream group slot interface 295 * @ptdev: Device. 296 * @csg_slot: Index of the command stream group slot. 297 * 298 * Return: The command stream group slot interface. 299 */ 300 struct panthor_fw_csg_iface * 301 panthor_fw_get_csg_iface(struct panthor_device *ptdev, u32 csg_slot) 302 { 303 if (drm_WARN_ON(&ptdev->base, csg_slot >= MAX_CSGS)) 304 return NULL; 305 306 return &ptdev->fw->iface.groups[csg_slot]; 307 } 308 309 /** 310 * panthor_fw_get_cs_iface() - Get a command stream slot interface 311 * @ptdev: Device. 312 * @csg_slot: Index of the command stream group slot. 313 * @cs_slot: Index of the command stream slot. 314 * 315 * Return: The command stream slot interface. 316 */ 317 struct panthor_fw_cs_iface * 318 panthor_fw_get_cs_iface(struct panthor_device *ptdev, u32 csg_slot, u32 cs_slot) 319 { 320 if (drm_WARN_ON(&ptdev->base, csg_slot >= MAX_CSGS || cs_slot >= MAX_CS_PER_CSG)) 321 return NULL; 322 323 return &ptdev->fw->iface.streams[csg_slot][cs_slot]; 324 } 325 326 static bool panthor_fw_has_glb_state(struct panthor_device *ptdev) 327 { 328 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 329 330 return glb_iface->control->version >= CSF_IFACE_VERSION(4, 1, 0); 331 } 332 333 static bool panthor_fw_has_64bit_ep_req(struct panthor_device *ptdev) 334 { 335 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 336 337 return glb_iface->control->version >= CSF_IFACE_VERSION(4, 0, 0); 338 } 339 340 u64 panthor_fw_csg_endpoint_req_get(struct panthor_device *ptdev, 341 struct panthor_fw_csg_iface *csg_iface) 342 { 343 if (panthor_fw_has_64bit_ep_req(ptdev)) 344 return csg_iface->input->endpoint_req2; 345 else 346 return csg_iface->input->endpoint_req; 347 } 348 349 void panthor_fw_csg_endpoint_req_set(struct panthor_device *ptdev, 350 struct panthor_fw_csg_iface *csg_iface, u64 value) 351 { 352 if (panthor_fw_has_64bit_ep_req(ptdev)) 353 csg_iface->input->endpoint_req2 = value; 354 else 355 csg_iface->input->endpoint_req = lower_32_bits(value); 356 } 357 358 void panthor_fw_csg_endpoint_req_update(struct panthor_device *ptdev, 359 struct panthor_fw_csg_iface *csg_iface, u64 value, 360 u64 mask) 361 { 362 if (panthor_fw_has_64bit_ep_req(ptdev)) 363 panthor_fw_update_reqs64(csg_iface, endpoint_req2, value, mask); 364 else 365 panthor_fw_update_reqs(csg_iface, endpoint_req, lower_32_bits(value), 366 lower_32_bits(mask)); 367 } 368 369 /** 370 * panthor_fw_conv_timeout() - Convert a timeout into a cycle-count 371 * @ptdev: Device. 372 * @timeout_us: Timeout expressed in micro-seconds. 373 * 374 * The FW has two timer sources: the GPU counter or arch-timer. We need 375 * to express timeouts in term of number of cycles and specify which 376 * timer source should be used. 377 * 378 * Return: A value suitable for timeout fields in the global interface. 379 */ 380 static u32 panthor_fw_conv_timeout(struct panthor_device *ptdev, u32 timeout_us) 381 { 382 bool use_cycle_counter = false; 383 u32 timer_rate = 0; 384 u64 mod_cycles; 385 386 #ifdef CONFIG_ARM_ARCH_TIMER 387 timer_rate = arch_timer_get_cntfrq(); 388 #endif 389 390 if (!timer_rate) { 391 use_cycle_counter = true; 392 timer_rate = clk_get_rate(ptdev->clks.core); 393 } 394 395 if (drm_WARN_ON(&ptdev->base, !timer_rate)) { 396 /* We couldn't get a valid clock rate, let's just pick the 397 * maximum value so the FW still handles the core 398 * power on/off requests. 399 */ 400 return GLB_TIMER_VAL(~0) | 401 GLB_TIMER_SOURCE_GPU_COUNTER; 402 } 403 404 mod_cycles = DIV_ROUND_UP_ULL((u64)timeout_us * timer_rate, 405 1000000ull << 10); 406 if (drm_WARN_ON(&ptdev->base, mod_cycles > GLB_TIMER_VAL(~0))) 407 mod_cycles = GLB_TIMER_VAL(~0); 408 409 return GLB_TIMER_VAL(mod_cycles) | 410 (use_cycle_counter ? GLB_TIMER_SOURCE_GPU_COUNTER : 0); 411 } 412 413 static int panthor_fw_binary_iter_read(struct panthor_device *ptdev, 414 struct panthor_fw_binary_iter *iter, 415 void *out, size_t size) 416 { 417 size_t new_offset = iter->offset + size; 418 419 if (new_offset > iter->size || new_offset < iter->offset) { 420 drm_err(&ptdev->base, "Firmware too small\n"); 421 return -EINVAL; 422 } 423 424 memcpy(out, iter->data + iter->offset, size); 425 iter->offset = new_offset; 426 return 0; 427 } 428 429 static int panthor_fw_binary_sub_iter_init(struct panthor_device *ptdev, 430 struct panthor_fw_binary_iter *iter, 431 struct panthor_fw_binary_iter *sub_iter, 432 size_t size) 433 { 434 size_t new_offset = iter->offset + size; 435 436 if (new_offset > iter->size || new_offset < iter->offset) { 437 drm_err(&ptdev->base, "Firmware entry too long\n"); 438 return -EINVAL; 439 } 440 441 sub_iter->offset = 0; 442 sub_iter->data = iter->data + iter->offset; 443 sub_iter->size = size; 444 iter->offset = new_offset; 445 return 0; 446 } 447 448 static void panthor_fw_init_section_mem(struct panthor_device *ptdev, 449 struct panthor_fw_section *section) 450 { 451 bool was_mapped = !!section->mem->kmap; 452 int ret; 453 454 if (!section->data.size && 455 !(section->flags & CSF_FW_BINARY_IFACE_ENTRY_ZERO)) 456 return; 457 458 ret = panthor_kernel_bo_vmap(section->mem); 459 if (drm_WARN_ON(&ptdev->base, ret)) 460 return; 461 462 memcpy(section->mem->kmap, section->data.buf, section->data.size); 463 if (section->flags & CSF_FW_BINARY_IFACE_ENTRY_ZERO) { 464 memset(section->mem->kmap + section->data.size, 0, 465 panthor_kernel_bo_size(section->mem) - section->data.size); 466 } 467 468 if (!was_mapped) 469 panthor_kernel_bo_vunmap(section->mem); 470 } 471 472 /** 473 * panthor_fw_alloc_queue_iface_mem() - Allocate a ring-buffer interfaces. 474 * @ptdev: Device. 475 * @input: Pointer holding the input interface on success. 476 * Should be ignored on failure. 477 * @output: Pointer holding the output interface on success. 478 * Should be ignored on failure. 479 * @input_fw_va: Pointer holding the input interface FW VA on success. 480 * Should be ignored on failure. 481 * @output_fw_va: Pointer holding the output interface FW VA on success. 482 * Should be ignored on failure. 483 * 484 * Allocates panthor_fw_ringbuf_{input,out}_iface interfaces. The input 485 * interface is at offset 0, and the output interface at offset 4096. 486 * 487 * Return: A valid pointer in case of success, an ERR_PTR() otherwise. 488 */ 489 struct panthor_kernel_bo * 490 panthor_fw_alloc_queue_iface_mem(struct panthor_device *ptdev, 491 struct panthor_fw_ringbuf_input_iface **input, 492 const struct panthor_fw_ringbuf_output_iface **output, 493 u32 *input_fw_va, u32 *output_fw_va) 494 { 495 struct panthor_kernel_bo *mem; 496 int ret; 497 498 mem = panthor_kernel_bo_create(ptdev, ptdev->fw->vm, SZ_8K, 499 DRM_PANTHOR_BO_NO_MMAP, 500 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | 501 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED, 502 PANTHOR_VM_KERNEL_AUTO_VA, 503 "Queue FW interface"); 504 if (IS_ERR(mem)) 505 return mem; 506 507 ret = panthor_kernel_bo_vmap(mem); 508 if (ret) { 509 panthor_kernel_bo_destroy(mem); 510 return ERR_PTR(ret); 511 } 512 513 memset(mem->kmap, 0, panthor_kernel_bo_size(mem)); 514 *input = mem->kmap; 515 *output = mem->kmap + SZ_4K; 516 *input_fw_va = panthor_kernel_bo_gpuva(mem); 517 *output_fw_va = *input_fw_va + SZ_4K; 518 519 return mem; 520 } 521 522 /** 523 * panthor_fw_alloc_suspend_buf_mem() - Allocate a suspend buffer for a command stream group. 524 * @ptdev: Device. 525 * @size: Size of the suspend buffer. 526 * 527 * Return: A valid pointer in case of success, an ERR_PTR() otherwise. 528 */ 529 struct panthor_kernel_bo * 530 panthor_fw_alloc_suspend_buf_mem(struct panthor_device *ptdev, size_t size) 531 { 532 return panthor_kernel_bo_create(ptdev, panthor_fw_vm(ptdev), size, 533 DRM_PANTHOR_BO_NO_MMAP, 534 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC, 535 PANTHOR_VM_KERNEL_AUTO_VA, 536 "FW suspend buffer"); 537 } 538 539 static int panthor_fw_load_section_entry(struct panthor_device *ptdev, 540 const struct firmware *fw, 541 struct panthor_fw_binary_iter *iter, 542 u32 ehdr) 543 { 544 ssize_t vm_pgsz = panthor_vm_page_size(ptdev->fw->vm); 545 struct panthor_fw_binary_section_entry_hdr hdr; 546 struct panthor_fw_section *section; 547 u32 section_size; 548 u32 data_size; 549 u32 name_len; 550 int ret; 551 552 ret = panthor_fw_binary_iter_read(ptdev, iter, &hdr, sizeof(hdr)); 553 if (ret) 554 return ret; 555 556 if (hdr.data.end < hdr.data.start) { 557 drm_err(&ptdev->base, "Firmware corrupted, data.end < data.start (0x%x < 0x%x)\n", 558 hdr.data.end, hdr.data.start); 559 return -EINVAL; 560 } 561 562 if (hdr.va.end < hdr.va.start) { 563 drm_err(&ptdev->base, "Firmware corrupted, hdr.va.end < hdr.va.start (0x%x < 0x%x)\n", 564 hdr.va.end, hdr.va.start); 565 return -EINVAL; 566 } 567 568 if (hdr.data.end > fw->size) { 569 drm_err(&ptdev->base, "Firmware corrupted, file truncated? data_end=0x%x > fw size=0x%zx\n", 570 hdr.data.end, fw->size); 571 return -EINVAL; 572 } 573 574 if (!IS_ALIGNED(hdr.va.start, vm_pgsz) || !IS_ALIGNED(hdr.va.end, vm_pgsz)) { 575 drm_err(&ptdev->base, "Firmware corrupted, virtual addresses not page aligned: 0x%x-0x%x\n", 576 hdr.va.start, hdr.va.end); 577 return -EINVAL; 578 } 579 580 if (hdr.flags & ~CSF_FW_BINARY_IFACE_ENTRY_SUPPORTED_FLAGS) { 581 drm_err(&ptdev->base, "Firmware contains interface with unsupported flags (0x%x)\n", 582 hdr.flags); 583 return -EINVAL; 584 } 585 586 if (hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_PROT) { 587 drm_warn(&ptdev->base, 588 "Firmware protected mode entry is not supported, ignoring"); 589 return 0; 590 } 591 592 if (hdr.va.start == CSF_MCU_SHARED_REGION_START && 593 !(hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_SHARED)) { 594 drm_err(&ptdev->base, 595 "Interface at 0x%llx must be shared", CSF_MCU_SHARED_REGION_START); 596 return -EINVAL; 597 } 598 599 section_size = hdr.va.end - hdr.va.start; 600 data_size = hdr.data.end - hdr.data.start; 601 if (data_size > section_size) { 602 drm_err(&ptdev->base, "Firmware corrupted, section data exceeds section size\n"); 603 return -EINVAL; 604 } 605 606 if (!section_size) 607 return 0; 608 609 name_len = iter->size - iter->offset; 610 611 section = drmm_kzalloc(&ptdev->base, sizeof(*section), GFP_KERNEL); 612 if (!section) 613 return -ENOMEM; 614 615 list_add_tail(§ion->node, &ptdev->fw->sections); 616 section->flags = hdr.flags; 617 section->data.size = data_size; 618 619 if (section->data.size > 0) { 620 void *data = drmm_kmalloc(&ptdev->base, section->data.size, GFP_KERNEL); 621 622 if (!data) 623 return -ENOMEM; 624 625 memcpy(data, fw->data + hdr.data.start, section->data.size); 626 section->data.buf = data; 627 } 628 629 if (name_len > 0) { 630 char *name = drmm_kmalloc(&ptdev->base, name_len + 1, GFP_KERNEL); 631 632 if (!name) 633 return -ENOMEM; 634 635 memcpy(name, iter->data + iter->offset, name_len); 636 name[name_len] = '\0'; 637 section->name = name; 638 } 639 640 if (section_size) { 641 u32 cache_mode = hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_CACHE_MODE_MASK; 642 struct panthor_gem_object *bo; 643 u32 vm_map_flags = 0; 644 u64 va = hdr.va.start; 645 646 if (!(hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_WR)) 647 vm_map_flags |= DRM_PANTHOR_VM_BIND_OP_MAP_READONLY; 648 649 if (!(hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_EX)) 650 vm_map_flags |= DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC; 651 652 /* TODO: CSF_FW_BINARY_IFACE_ENTRY_CACHE_MODE_*_COHERENT are mapped to 653 * non-cacheable for now. We might want to introduce a new 654 * IOMMU_xxx flag (or abuse IOMMU_MMIO, which maps to device 655 * memory and is currently not used by our driver) for 656 * AS_MEMATTR_AARCH64_SHARED memory, so we can take benefit 657 * of IO-coherent systems. 658 */ 659 if (cache_mode != CSF_FW_BINARY_IFACE_ENTRY_CACHE_MODE_CACHED) 660 vm_map_flags |= DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED; 661 662 section->mem = panthor_kernel_bo_create(ptdev, panthor_fw_vm(ptdev), 663 section_size, 664 DRM_PANTHOR_BO_NO_MMAP, 665 vm_map_flags, va, "FW section"); 666 if (IS_ERR(section->mem)) 667 return PTR_ERR(section->mem); 668 669 if (drm_WARN_ON(&ptdev->base, section->mem->va_node.start != hdr.va.start)) 670 return -EINVAL; 671 672 if (section->flags & CSF_FW_BINARY_IFACE_ENTRY_SHARED) { 673 ret = panthor_kernel_bo_vmap(section->mem); 674 if (ret) 675 return ret; 676 } 677 678 panthor_fw_init_section_mem(ptdev, section); 679 680 bo = to_panthor_bo(section->mem->obj); 681 682 /* An sgt should have been requested when the kernel BO was GPU-mapped. */ 683 if (drm_WARN_ON_ONCE(&ptdev->base, !bo->dmap.sgt)) 684 return -EINVAL; 685 686 dma_sync_sgtable_for_device(ptdev->base.dev, bo->dmap.sgt, DMA_TO_DEVICE); 687 } 688 689 if (hdr.va.start == CSF_MCU_SHARED_REGION_START) 690 ptdev->fw->shared_section = section; 691 692 return 0; 693 } 694 695 static int panthor_fw_read_build_info(struct panthor_device *ptdev, 696 const struct firmware *fw, 697 struct panthor_fw_binary_iter *iter, 698 u32 ehdr) 699 { 700 struct panthor_fw_build_info_hdr hdr; 701 static const char git_sha_header[] = "git_sha: "; 702 const int header_len = sizeof(git_sha_header) - 1; 703 int ret; 704 705 ret = panthor_fw_binary_iter_read(ptdev, iter, &hdr, sizeof(hdr)); 706 if (ret) 707 return ret; 708 709 if (hdr.meta_start > fw->size || 710 hdr.meta_size > fw->size - hdr.meta_start || 711 hdr.meta_size <= header_len) { 712 drm_err(&ptdev->base, "Firmware build info corrupt\n"); 713 /* We don't need the build info, so continue */ 714 return 0; 715 } 716 717 if (memcmp(git_sha_header, fw->data + hdr.meta_start, header_len)) { 718 /* Not the expected header, this isn't metadata we understand */ 719 return 0; 720 } 721 722 /* Check that the git SHA is NULL terminated as expected */ 723 if (fw->data[hdr.meta_start + hdr.meta_size - 1] != '\0') { 724 drm_warn(&ptdev->base, "Firmware's git sha is not NULL terminated\n"); 725 /* Don't treat as fatal */ 726 return 0; 727 } 728 729 drm_info(&ptdev->base, "Firmware git sha: %s\n", 730 fw->data + hdr.meta_start + header_len); 731 732 return 0; 733 } 734 735 static void 736 panthor_reload_fw_sections(struct panthor_device *ptdev, bool full_reload) 737 { 738 struct panthor_fw_section *section; 739 740 list_for_each_entry(section, &ptdev->fw->sections, node) { 741 struct sg_table *sgt; 742 743 if (!full_reload && !(section->flags & CSF_FW_BINARY_IFACE_ENTRY_WR)) 744 continue; 745 746 panthor_fw_init_section_mem(ptdev, section); 747 748 /* An sgt should have been requested when the kernel BO was GPU-mapped. */ 749 sgt = to_panthor_bo(section->mem->obj)->dmap.sgt; 750 if (!drm_WARN_ON_ONCE(&ptdev->base, !sgt)) 751 dma_sync_sgtable_for_device(ptdev->base.dev, sgt, DMA_TO_DEVICE); 752 } 753 } 754 755 static int panthor_fw_load_entry(struct panthor_device *ptdev, 756 const struct firmware *fw, 757 struct panthor_fw_binary_iter *iter) 758 { 759 struct panthor_fw_binary_iter eiter; 760 u32 ehdr; 761 int ret; 762 763 ret = panthor_fw_binary_iter_read(ptdev, iter, &ehdr, sizeof(ehdr)); 764 if (ret) 765 return ret; 766 767 if ((iter->offset % sizeof(u32)) || 768 (CSF_FW_BINARY_ENTRY_SIZE(ehdr) % sizeof(u32))) { 769 drm_err(&ptdev->base, "Firmware entry is not 32-bit aligned, offset=0x%x size=0x%x\n", 770 (u32)(iter->offset - sizeof(u32)), CSF_FW_BINARY_ENTRY_SIZE(ehdr)); 771 return -EINVAL; 772 } 773 774 if (panthor_fw_binary_sub_iter_init(ptdev, iter, &eiter, 775 CSF_FW_BINARY_ENTRY_SIZE(ehdr) - sizeof(ehdr))) 776 return -EINVAL; 777 778 switch (CSF_FW_BINARY_ENTRY_TYPE(ehdr)) { 779 case CSF_FW_BINARY_ENTRY_TYPE_IFACE: 780 return panthor_fw_load_section_entry(ptdev, fw, &eiter, ehdr); 781 case CSF_FW_BINARY_ENTRY_TYPE_BUILD_INFO_METADATA: 782 return panthor_fw_read_build_info(ptdev, fw, &eiter, ehdr); 783 784 /* FIXME: handle those entry types? */ 785 case CSF_FW_BINARY_ENTRY_TYPE_CONFIG: 786 case CSF_FW_BINARY_ENTRY_TYPE_FUTF_TEST: 787 case CSF_FW_BINARY_ENTRY_TYPE_TRACE_BUFFER: 788 case CSF_FW_BINARY_ENTRY_TYPE_TIMELINE_METADATA: 789 return 0; 790 default: 791 break; 792 } 793 794 if (ehdr & CSF_FW_BINARY_ENTRY_OPTIONAL) 795 return 0; 796 797 drm_err(&ptdev->base, 798 "Unsupported non-optional entry type %u in firmware\n", 799 CSF_FW_BINARY_ENTRY_TYPE(ehdr)); 800 return -EINVAL; 801 } 802 803 static int panthor_fw_load(struct panthor_device *ptdev) 804 { 805 const struct firmware *fw = NULL; 806 struct panthor_fw_binary_iter iter = {}; 807 struct panthor_fw_binary_hdr hdr; 808 char fw_path[128]; 809 int ret; 810 811 snprintf(fw_path, sizeof(fw_path), "arm/mali/arch%d.%d/%s", 812 (u32)GPU_ARCH_MAJOR(ptdev->gpu_info.gpu_id), 813 (u32)GPU_ARCH_MINOR(ptdev->gpu_info.gpu_id), 814 CSF_FW_NAME); 815 816 ret = request_firmware(&fw, fw_path, ptdev->base.dev); 817 if (ret) { 818 drm_err(&ptdev->base, "Failed to load firmware image '%s'\n", 819 CSF_FW_NAME); 820 return ret; 821 } 822 823 iter.data = fw->data; 824 iter.size = fw->size; 825 ret = panthor_fw_binary_iter_read(ptdev, &iter, &hdr, sizeof(hdr)); 826 if (ret) 827 goto out; 828 829 if (hdr.magic != CSF_FW_BINARY_HEADER_MAGIC) { 830 ret = -EINVAL; 831 drm_err(&ptdev->base, "Invalid firmware magic\n"); 832 goto out; 833 } 834 835 if (hdr.major != CSF_FW_BINARY_HEADER_MAJOR_MAX) { 836 ret = -EINVAL; 837 drm_err(&ptdev->base, "Unsupported firmware binary header version %d.%d (expected %d.x)\n", 838 hdr.major, hdr.minor, CSF_FW_BINARY_HEADER_MAJOR_MAX); 839 goto out; 840 } 841 842 if (hdr.size > iter.size) { 843 ret = -EINVAL; 844 drm_err(&ptdev->base, "Firmware image is truncated\n"); 845 goto out; 846 } 847 848 iter.size = hdr.size; 849 850 while (iter.offset < hdr.size) { 851 ret = panthor_fw_load_entry(ptdev, fw, &iter); 852 if (ret) 853 goto out; 854 } 855 856 if (!ptdev->fw->shared_section) { 857 drm_err(&ptdev->base, "Shared interface region not found\n"); 858 ret = -EINVAL; 859 goto out; 860 } 861 862 out: 863 release_firmware(fw); 864 return ret; 865 } 866 867 /** 868 * iface_fw_to_cpu_addr() - Turn an MCU address into a CPU address 869 * @ptdev: Device. 870 * @mcu_va: MCU address. 871 * @size: Size of the object pointed to by @mcu_va. 872 * 873 * Return: NULL if the object is not part of the shared section, non-NULL otherwise. 874 */ 875 static void *iface_fw_to_cpu_addr(struct panthor_device *ptdev, u32 mcu_va, size_t size) 876 { 877 u64 shared_mem_start = panthor_kernel_bo_gpuva(ptdev->fw->shared_section->mem); 878 size_t shared_mem_size = panthor_kernel_bo_size(ptdev->fw->shared_section->mem); 879 u64 offset; 880 881 if (mcu_va < shared_mem_start) 882 return NULL; 883 884 offset = mcu_va - shared_mem_start; 885 if (offset > shared_mem_size || size > shared_mem_size - offset) 886 return NULL; 887 888 return ptdev->fw->shared_section->mem->kmap + offset; 889 } 890 891 static int panthor_init_cs_iface(struct panthor_device *ptdev, 892 unsigned int csg_idx, unsigned int cs_idx) 893 { 894 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 895 struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, csg_idx); 896 struct panthor_fw_cs_iface *cs_iface = &ptdev->fw->iface.streams[csg_idx][cs_idx]; 897 u64 shared_section_sz = panthor_kernel_bo_size(ptdev->fw->shared_section->mem); 898 u64 iface_offset = CSF_GROUP_CONTROL_OFFSET + 899 ((u64)csg_idx * glb_iface->control->group_stride) + 900 CSF_STREAM_CONTROL_OFFSET + 901 ((u64)cs_idx * csg_iface->control->stream_stride); 902 struct panthor_fw_cs_iface *first_cs_iface = 903 panthor_fw_get_cs_iface(ptdev, 0, 0); 904 905 if (iface_offset > shared_section_sz || 906 sizeof(*cs_iface->control) > shared_section_sz - iface_offset) 907 return -EINVAL; 908 909 spin_lock_init(&cs_iface->lock); 910 cs_iface->control = ptdev->fw->shared_section->mem->kmap + iface_offset; 911 cs_iface->input = iface_fw_to_cpu_addr(ptdev, cs_iface->control->input_va, 912 sizeof(*cs_iface->input)); 913 cs_iface->output = iface_fw_to_cpu_addr(ptdev, cs_iface->control->output_va, 914 sizeof(*cs_iface->output)); 915 916 if (!cs_iface->input || !cs_iface->output) { 917 drm_err(&ptdev->base, "Invalid stream control interface input/output VA"); 918 return -EINVAL; 919 } 920 921 if (cs_iface != first_cs_iface) { 922 if (cs_iface->control->features != first_cs_iface->control->features) { 923 drm_err(&ptdev->base, "Expecting identical CS slots"); 924 return -EINVAL; 925 } 926 } else { 927 u32 reg_count = CS_FEATURES_WORK_REGS(cs_iface->control->features); 928 929 ptdev->csif_info.cs_reg_count = reg_count; 930 ptdev->csif_info.unpreserved_cs_reg_count = CSF_UNPRESERVED_REG_COUNT; 931 } 932 933 return 0; 934 } 935 936 static bool compare_csg(const struct panthor_fw_csg_control_iface *a, 937 const struct panthor_fw_csg_control_iface *b) 938 { 939 if (a->features != b->features) 940 return false; 941 if (a->suspend_size != b->suspend_size) 942 return false; 943 if (a->protm_suspend_size != b->protm_suspend_size) 944 return false; 945 if (a->stream_num != b->stream_num) 946 return false; 947 return true; 948 } 949 950 static int panthor_init_csg_iface(struct panthor_device *ptdev, 951 unsigned int csg_idx) 952 { 953 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 954 struct panthor_fw_csg_iface *csg_iface = &ptdev->fw->iface.groups[csg_idx]; 955 u64 shared_section_sz = panthor_kernel_bo_size(ptdev->fw->shared_section->mem); 956 u64 iface_offset = CSF_GROUP_CONTROL_OFFSET + 957 ((u64)csg_idx * glb_iface->control->group_stride); 958 unsigned int i; 959 960 if (iface_offset > shared_section_sz || 961 sizeof(*csg_iface->control) > shared_section_sz - iface_offset) 962 return -EINVAL; 963 964 spin_lock_init(&csg_iface->lock); 965 csg_iface->control = ptdev->fw->shared_section->mem->kmap + iface_offset; 966 csg_iface->input = iface_fw_to_cpu_addr(ptdev, csg_iface->control->input_va, 967 sizeof(*csg_iface->input)); 968 csg_iface->output = iface_fw_to_cpu_addr(ptdev, csg_iface->control->output_va, 969 sizeof(*csg_iface->output)); 970 971 if (csg_iface->control->stream_num < MIN_CS_PER_CSG || 972 csg_iface->control->stream_num > MAX_CS_PER_CSG) 973 return -EINVAL; 974 975 if (!csg_iface->input || !csg_iface->output) { 976 drm_err(&ptdev->base, "Invalid group control interface input/output VA"); 977 return -EINVAL; 978 } 979 980 if (csg_idx > 0) { 981 struct panthor_fw_csg_iface *first_csg_iface = 982 panthor_fw_get_csg_iface(ptdev, 0); 983 984 if (!compare_csg(first_csg_iface->control, csg_iface->control)) { 985 drm_err(&ptdev->base, "Expecting identical CSG slots"); 986 return -EINVAL; 987 } 988 } 989 990 for (i = 0; i < csg_iface->control->stream_num; i++) { 991 int ret = panthor_init_cs_iface(ptdev, csg_idx, i); 992 993 if (ret) 994 return ret; 995 } 996 997 return 0; 998 } 999 1000 static u32 panthor_get_instr_features(struct panthor_device *ptdev) 1001 { 1002 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 1003 1004 if (glb_iface->control->version < CSF_IFACE_VERSION(1, 1, 0)) 1005 return 0; 1006 1007 return glb_iface->control->instr_features; 1008 } 1009 1010 static int panthor_fw_init_ifaces(struct panthor_device *ptdev) 1011 { 1012 struct panthor_fw_global_iface *glb_iface = &ptdev->fw->iface.global; 1013 u64 shared_section_sz = panthor_kernel_bo_size(ptdev->fw->shared_section->mem); 1014 unsigned int i; 1015 1016 if (!ptdev->fw->shared_section->mem->kmap) 1017 return -EINVAL; 1018 1019 if (sizeof(*glb_iface->control) > shared_section_sz) 1020 return -EINVAL; 1021 1022 spin_lock_init(&glb_iface->lock); 1023 glb_iface->control = ptdev->fw->shared_section->mem->kmap; 1024 1025 if (!glb_iface->control->version) { 1026 drm_err(&ptdev->base, "Firmware version is 0. Firmware may have failed to boot"); 1027 return -EINVAL; 1028 } 1029 1030 glb_iface->input = iface_fw_to_cpu_addr(ptdev, glb_iface->control->input_va, 1031 sizeof(*glb_iface->input)); 1032 glb_iface->output = iface_fw_to_cpu_addr(ptdev, glb_iface->control->output_va, 1033 sizeof(*glb_iface->output)); 1034 if (!glb_iface->input || !glb_iface->output) { 1035 drm_err(&ptdev->base, "Invalid global control interface input/output VA"); 1036 return -EINVAL; 1037 } 1038 1039 if (glb_iface->control->group_num > MAX_CSGS || 1040 glb_iface->control->group_num < MIN_CSGS) { 1041 drm_err(&ptdev->base, "Invalid number of control groups"); 1042 return -EINVAL; 1043 } 1044 1045 for (i = 0; i < glb_iface->control->group_num; i++) { 1046 int ret = panthor_init_csg_iface(ptdev, i); 1047 1048 if (ret) 1049 return ret; 1050 } 1051 1052 drm_info(&ptdev->base, "CSF FW using interface v%d.%d.%d, Features %#x Instrumentation features %#x", 1053 CSF_IFACE_VERSION_MAJOR(glb_iface->control->version), 1054 CSF_IFACE_VERSION_MINOR(glb_iface->control->version), 1055 CSF_IFACE_VERSION_PATCH(glb_iface->control->version), 1056 glb_iface->control->features, 1057 panthor_get_instr_features(ptdev)); 1058 return 0; 1059 } 1060 1061 static void panthor_fw_init_global_iface(struct panthor_device *ptdev) 1062 { 1063 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 1064 1065 /* Enable all cores. */ 1066 glb_iface->input->core_en_mask = ptdev->gpu_info.shader_present; 1067 1068 /* Setup timers. */ 1069 glb_iface->input->poweroff_timer = panthor_fw_conv_timeout(ptdev, PWROFF_HYSTERESIS_US); 1070 glb_iface->input->progress_timer = PROGRESS_TIMEOUT_CYCLES >> PROGRESS_TIMEOUT_SCALE_SHIFT; 1071 glb_iface->input->idle_timer = panthor_fw_conv_timeout(ptdev, IDLE_HYSTERESIS_US); 1072 1073 /* Enable interrupts we care about. */ 1074 glb_iface->input->ack_irq_mask = GLB_CFG_ALLOC_EN | 1075 GLB_PING | 1076 GLB_CFG_PROGRESS_TIMER | 1077 GLB_CFG_POWEROFF_TIMER | 1078 GLB_IDLE_EN | 1079 GLB_IDLE; 1080 1081 if (panthor_fw_has_glb_state(ptdev)) 1082 glb_iface->input->ack_irq_mask |= GLB_STATE_MASK; 1083 1084 panthor_fw_update_reqs(glb_iface, req, GLB_IDLE_EN | GLB_COUNTER_EN, 1085 GLB_IDLE_EN | GLB_COUNTER_EN); 1086 panthor_fw_toggle_reqs(glb_iface, req, ack, 1087 GLB_CFG_ALLOC_EN | 1088 GLB_CFG_POWEROFF_TIMER | 1089 GLB_CFG_PROGRESS_TIMER); 1090 1091 panthor_fw_ring_doorbell(ptdev, CSF_GLB_DOORBELL_ID); 1092 1093 /* Kick the watchdog. */ 1094 mod_delayed_work(ptdev->reset.wq, &ptdev->fw->watchdog.ping_work, 1095 msecs_to_jiffies(PING_INTERVAL_MS)); 1096 } 1097 1098 static void panthor_job_irq_handler(struct panthor_device *ptdev, u32 status) 1099 { 1100 u32 duration; 1101 u64 start = 0; 1102 1103 if (tracepoint_enabled(gpu_job_irq)) 1104 start = ktime_get_ns(); 1105 1106 gpu_write(ptdev->fw->irq.iomem, INT_CLEAR, status); 1107 1108 if (!ptdev->fw->booted && (status & JOB_INT_GLOBAL_IF)) 1109 ptdev->fw->booted = true; 1110 1111 wake_up_all(&ptdev->fw->req_waitqueue); 1112 1113 /* If the FW is not booted, don't process IRQs, just flag the FW as booted. */ 1114 if (!ptdev->fw->booted) 1115 return; 1116 1117 panthor_sched_report_fw_events(ptdev, status); 1118 1119 if (tracepoint_enabled(gpu_job_irq) && start) { 1120 if (check_sub_overflow(ktime_get_ns(), start, &duration)) 1121 duration = U32_MAX; 1122 trace_gpu_job_irq(ptdev->base.dev, status, duration); 1123 } 1124 } 1125 PANTHOR_IRQ_HANDLER(job, panthor_job_irq_handler); 1126 1127 static int panthor_fw_start(struct panthor_device *ptdev) 1128 { 1129 struct panthor_fw *fw = ptdev->fw; 1130 bool timedout = false; 1131 1132 ptdev->fw->booted = false; 1133 panthor_job_irq_enable_events(&ptdev->fw->irq, ~0); 1134 panthor_job_irq_resume(&ptdev->fw->irq); 1135 gpu_write(fw->iomem, MCU_CONTROL, MCU_CONTROL_AUTO); 1136 1137 if (!wait_event_timeout(ptdev->fw->req_waitqueue, 1138 ptdev->fw->booted, 1139 msecs_to_jiffies(1000))) { 1140 if (!ptdev->fw->booted && 1141 !(gpu_read(fw->irq.iomem, INT_STAT) & JOB_INT_GLOBAL_IF)) 1142 timedout = true; 1143 } 1144 1145 if (timedout) { 1146 static const char * const status_str[] = { 1147 [MCU_STATUS_DISABLED] = "disabled", 1148 [MCU_STATUS_ENABLED] = "enabled", 1149 [MCU_STATUS_HALT] = "halt", 1150 [MCU_STATUS_FATAL] = "fatal", 1151 }; 1152 u32 status = gpu_read(fw->iomem, MCU_STATUS); 1153 1154 drm_err(&ptdev->base, "Failed to boot MCU (status=%s)", 1155 status < ARRAY_SIZE(status_str) ? status_str[status] : "unknown"); 1156 return -ETIMEDOUT; 1157 } 1158 1159 return 0; 1160 } 1161 1162 static void panthor_fw_stop(struct panthor_device *ptdev) 1163 { 1164 struct panthor_fw *fw = ptdev->fw; 1165 u32 status; 1166 1167 gpu_write(fw->iomem, MCU_CONTROL, MCU_CONTROL_DISABLE); 1168 if (gpu_read_poll_timeout(fw->iomem, MCU_STATUS, status, 1169 status == MCU_STATUS_DISABLED, 10, 100000)) 1170 drm_err(&ptdev->base, "Failed to stop MCU"); 1171 } 1172 1173 static bool panthor_fw_mcu_halted(struct panthor_device *ptdev) 1174 { 1175 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 1176 bool halted; 1177 1178 halted = gpu_read(ptdev->fw->iomem, MCU_STATUS) == MCU_STATUS_HALT; 1179 1180 if (panthor_fw_has_glb_state(ptdev)) 1181 halted &= (GLB_STATE_GET(glb_iface->output->ack) == GLB_STATE_HALT); 1182 1183 return halted; 1184 } 1185 1186 static void panthor_fw_halt_mcu(struct panthor_device *ptdev) 1187 { 1188 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 1189 1190 if (panthor_fw_has_glb_state(ptdev)) 1191 panthor_fw_update_reqs(glb_iface, req, GLB_STATE(GLB_STATE_HALT), GLB_STATE_MASK); 1192 else 1193 panthor_fw_update_reqs(glb_iface, req, GLB_HALT, GLB_HALT); 1194 1195 panthor_fw_ring_doorbell(ptdev, CSF_GLB_DOORBELL_ID); 1196 } 1197 1198 static bool panthor_fw_wait_mcu_halted(struct panthor_device *ptdev) 1199 { 1200 bool halted = false; 1201 1202 if (read_poll_timeout_atomic(panthor_fw_mcu_halted, halted, halted, 10, 1203 MCU_HALT_TIMEOUT_US, 0, ptdev)) { 1204 drm_warn(&ptdev->base, "Timed out waiting for MCU to halt"); 1205 return false; 1206 } 1207 1208 return true; 1209 } 1210 1211 static void panthor_fw_mcu_set_active(struct panthor_device *ptdev) 1212 { 1213 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 1214 1215 if (panthor_fw_has_glb_state(ptdev)) 1216 panthor_fw_update_reqs(glb_iface, req, GLB_STATE(GLB_STATE_ACTIVE), GLB_STATE_MASK); 1217 else 1218 panthor_fw_update_reqs(glb_iface, req, 0, GLB_HALT); 1219 } 1220 1221 /** 1222 * panthor_fw_pre_reset() - Call before a reset. 1223 * @ptdev: Device. 1224 * @on_hang: true if the reset was triggered on a GPU hang. 1225 * 1226 * If the reset is not triggered on a hang, we try to gracefully halt the 1227 * MCU, so we can do a fast-reset when panthor_fw_post_reset() is called. 1228 */ 1229 void panthor_fw_pre_reset(struct panthor_device *ptdev, bool on_hang) 1230 { 1231 /* Make sure we won't be woken up by a ping. */ 1232 cancel_delayed_work_sync(&ptdev->fw->watchdog.ping_work); 1233 1234 ptdev->reset.fast = false; 1235 1236 if (!on_hang) { 1237 panthor_fw_halt_mcu(ptdev); 1238 if (!panthor_fw_wait_mcu_halted(ptdev)) 1239 drm_warn(&ptdev->base, "Failed to cleanly suspend MCU"); 1240 else 1241 ptdev->reset.fast = true; 1242 } 1243 1244 panthor_job_irq_suspend(&ptdev->fw->irq); 1245 panthor_fw_stop(ptdev); 1246 } 1247 1248 /** 1249 * panthor_fw_post_reset() - Call after a reset. 1250 * @ptdev: Device. 1251 * 1252 * Start the FW. If this is not a fast reset, all FW sections are reloaded to 1253 * make sure we can recover from a memory corruption. 1254 */ 1255 int panthor_fw_post_reset(struct panthor_device *ptdev) 1256 { 1257 int ret; 1258 1259 /* Make the MCU VM active. */ 1260 ret = panthor_vm_active(ptdev->fw->vm); 1261 if (ret) 1262 return ret; 1263 1264 if (!ptdev->reset.fast) { 1265 /* On a slow reset, reload all sections, including RO ones. 1266 * We're not supposed to end up here anyway, let's just assume 1267 * the overhead of reloading everything is acceptable. 1268 */ 1269 panthor_reload_fw_sections(ptdev, true); 1270 } else { 1271 /* 1272 * If the FW was previously successfully halted in the pre-reset 1273 * operation, we need to transition it to active again before 1274 * the FW is rebooted. 1275 * This is not needed on a slow reset because FW sections are 1276 * re-initialized. 1277 */ 1278 panthor_fw_mcu_set_active(ptdev); 1279 } 1280 1281 ret = panthor_fw_start(ptdev); 1282 if (ret) { 1283 drm_err(&ptdev->base, "FW %s reset failed", 1284 ptdev->reset.fast ? "fast" : "slow"); 1285 return ret; 1286 } 1287 1288 /* We must re-initialize the global interface even on fast-reset. */ 1289 panthor_fw_init_global_iface(ptdev); 1290 return 0; 1291 } 1292 1293 /** 1294 * panthor_fw_unplug() - Called when the device is unplugged. 1295 * @ptdev: Device. 1296 * 1297 * This function must make sure all pending operations are flushed before 1298 * will release device resources, thus preventing any interaction with 1299 * the HW. 1300 * 1301 * If there is still FW-related work running after this function returns, 1302 * they must use drm_dev_{enter,exit}() and skip any HW access when 1303 * drm_dev_enter() returns false. 1304 */ 1305 void panthor_fw_unplug(struct panthor_device *ptdev) 1306 { 1307 struct panthor_fw_section *section; 1308 1309 disable_delayed_work_sync(&ptdev->fw->watchdog.ping_work); 1310 1311 if (!IS_ENABLED(CONFIG_PM) || pm_runtime_active(ptdev->base.dev)) { 1312 /* Make sure the IRQ handler cannot be called after that point. */ 1313 panthor_job_irq_suspend(&ptdev->fw->irq); 1314 panthor_fw_stop(ptdev); 1315 } 1316 1317 list_for_each_entry(section, &ptdev->fw->sections, node) 1318 panthor_kernel_bo_destroy(section->mem); 1319 1320 /* We intentionally don't call panthor_vm_idle() and let 1321 * panthor_mmu_unplug() release the AS we acquired with 1322 * panthor_vm_active() so we don't have to track the VM active/idle 1323 * state to keep the active_refcnt balanced. 1324 */ 1325 panthor_vm_put(ptdev->fw->vm); 1326 ptdev->fw->vm = NULL; 1327 1328 if (!IS_ENABLED(CONFIG_PM) || pm_runtime_active(ptdev->base.dev)) 1329 panthor_hw_l2_power_off(ptdev); 1330 } 1331 1332 /** 1333 * panthor_fw_wait_acks() - Wait for requests to be acknowledged by the FW. 1334 * @req_ptr: Pointer to the req register. 1335 * @ack_ptr: Pointer to the ack register. 1336 * @wq: Wait queue to use for the sleeping wait. 1337 * @req_mask: Mask of requests to wait for. 1338 * @acked: Pointer to field that's updated with the acked requests. 1339 * If the function returns 0, *acked == req_mask. 1340 * @timeout_ms: Timeout expressed in milliseconds. 1341 * 1342 * Return: 0 on success, -ETIMEDOUT otherwise. 1343 */ 1344 static int panthor_fw_wait_acks(const u32 *req_ptr, const u32 *ack_ptr, 1345 wait_queue_head_t *wq, 1346 u32 req_mask, u32 *acked, 1347 u32 timeout_ms) 1348 { 1349 u32 ack, req = READ_ONCE(*req_ptr) & req_mask; 1350 int ret; 1351 1352 /* Busy wait for a few µsecs before falling back to a sleeping wait. */ 1353 *acked = req_mask; 1354 ret = read_poll_timeout_atomic(READ_ONCE, ack, 1355 (ack & req_mask) == req, 1356 0, 10, 0, 1357 *ack_ptr); 1358 if (!ret) 1359 return 0; 1360 1361 if (wait_event_timeout(*wq, (READ_ONCE(*ack_ptr) & req_mask) == req, 1362 msecs_to_jiffies(timeout_ms))) 1363 return 0; 1364 1365 /* Check one last time, in case we were not woken up for some reason. */ 1366 ack = READ_ONCE(*ack_ptr); 1367 if ((ack & req_mask) == req) 1368 return 0; 1369 1370 *acked = ~(req ^ ack) & req_mask; 1371 return -ETIMEDOUT; 1372 } 1373 1374 /** 1375 * panthor_fw_glb_wait_acks() - Wait for global requests to be acknowledged. 1376 * @ptdev: Device. 1377 * @req_mask: Mask of requests to wait for. 1378 * @acked: Pointer to field that's updated with the acked requests. 1379 * If the function returns 0, *acked == req_mask. 1380 * @timeout_ms: Timeout expressed in milliseconds. 1381 * 1382 * Return: 0 on success, -ETIMEDOUT otherwise. 1383 */ 1384 int panthor_fw_glb_wait_acks(struct panthor_device *ptdev, 1385 u32 req_mask, u32 *acked, 1386 u32 timeout_ms) 1387 { 1388 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 1389 1390 /* GLB_HALT doesn't get acked through the FW interface. */ 1391 if (drm_WARN_ON(&ptdev->base, req_mask & (~GLB_REQ_MASK | GLB_HALT))) 1392 return -EINVAL; 1393 1394 return panthor_fw_wait_acks(&glb_iface->input->req, 1395 &glb_iface->output->ack, 1396 &ptdev->fw->req_waitqueue, 1397 req_mask, acked, timeout_ms); 1398 } 1399 1400 /** 1401 * panthor_fw_csg_wait_acks() - Wait for command stream group requests to be acknowledged. 1402 * @ptdev: Device. 1403 * @csg_slot: CSG slot ID. 1404 * @req_mask: Mask of requests to wait for. 1405 * @acked: Pointer to field that's updated with the acked requests. 1406 * If the function returns 0, *acked == req_mask. 1407 * @timeout_ms: Timeout expressed in milliseconds. 1408 * 1409 * Return: 0 on success, -ETIMEDOUT otherwise. 1410 */ 1411 int panthor_fw_csg_wait_acks(struct panthor_device *ptdev, u32 csg_slot, 1412 u32 req_mask, u32 *acked, u32 timeout_ms) 1413 { 1414 struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, csg_slot); 1415 int ret; 1416 1417 if (drm_WARN_ON(&ptdev->base, req_mask & ~CSG_REQ_MASK)) 1418 return -EINVAL; 1419 1420 ret = panthor_fw_wait_acks(&csg_iface->input->req, 1421 &csg_iface->output->ack, 1422 &ptdev->fw->req_waitqueue, 1423 req_mask, acked, timeout_ms); 1424 1425 /* 1426 * Check that all bits in the state field were updated, if any mismatch 1427 * then clear all bits in the state field. This allows code to do 1428 * (acked & CSG_STATE_MASK) and get the right value. 1429 */ 1430 1431 if ((*acked & CSG_STATE_MASK) != CSG_STATE_MASK) 1432 *acked &= ~CSG_STATE_MASK; 1433 1434 return ret; 1435 } 1436 1437 void panthor_fw_ring_doorbell(struct panthor_device *ptdev, u32 doorbell_id) 1438 { 1439 gpu_write(ptdev->iomem, CSF_DOORBELL(doorbell_id), 1); 1440 } 1441 1442 /** 1443 * panthor_fw_ring_csg_doorbells() - Ring command stream group doorbells. 1444 * @ptdev: Device. 1445 * @csg_mask: Bitmask encoding the command stream group doorbells to ring. 1446 * 1447 * This function is toggling bits in the doorbell_req and ringing the 1448 * global doorbell. It doesn't require a user doorbell to be attached to 1449 * the group. 1450 */ 1451 void panthor_fw_ring_csg_doorbells(struct panthor_device *ptdev, u32 csg_mask) 1452 { 1453 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 1454 1455 panthor_fw_toggle_reqs(glb_iface, doorbell_req, doorbell_ack, csg_mask); 1456 panthor_fw_ring_doorbell(ptdev, CSF_GLB_DOORBELL_ID); 1457 } 1458 1459 static void panthor_fw_ping_work(struct work_struct *work) 1460 { 1461 struct panthor_fw *fw = container_of(work, struct panthor_fw, watchdog.ping_work.work); 1462 struct panthor_device *ptdev = fw->irq.ptdev; 1463 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); 1464 u32 acked; 1465 int ret; 1466 1467 if (panthor_device_reset_is_pending(ptdev)) 1468 return; 1469 1470 panthor_fw_toggle_reqs(glb_iface, req, ack, GLB_PING); 1471 panthor_fw_ring_doorbell(ptdev, CSF_GLB_DOORBELL_ID); 1472 1473 ret = panthor_fw_glb_wait_acks(ptdev, GLB_PING, &acked, 100); 1474 if (ret) { 1475 panthor_device_schedule_reset(ptdev); 1476 drm_err(&ptdev->base, "FW ping timeout, scheduling a reset"); 1477 } else { 1478 mod_delayed_work(ptdev->reset.wq, &fw->watchdog.ping_work, 1479 msecs_to_jiffies(PING_INTERVAL_MS)); 1480 } 1481 } 1482 1483 /** 1484 * panthor_fw_init() - Initialize FW related data. 1485 * @ptdev: Device. 1486 * 1487 * Return: 0 on success, a negative error code otherwise. 1488 */ 1489 int panthor_fw_init(struct panthor_device *ptdev) 1490 { 1491 struct panthor_fw *fw; 1492 int ret, irq; 1493 1494 fw = drmm_kzalloc(&ptdev->base, sizeof(*fw), GFP_KERNEL); 1495 if (!fw) 1496 return -ENOMEM; 1497 1498 fw->iomem = ptdev->iomem + MCU_CONTROL_BASE; 1499 ptdev->fw = fw; 1500 init_waitqueue_head(&fw->req_waitqueue); 1501 INIT_LIST_HEAD(&fw->sections); 1502 INIT_DELAYED_WORK(&fw->watchdog.ping_work, panthor_fw_ping_work); 1503 1504 irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "job"); 1505 if (irq <= 0) 1506 return -ENODEV; 1507 1508 ret = panthor_request_job_irq(ptdev, &fw->irq, irq, 1509 ptdev->iomem + JOB_INT_BASE); 1510 if (ret) { 1511 drm_err(&ptdev->base, "failed to request job irq"); 1512 return ret; 1513 } 1514 1515 ret = panthor_hw_l2_power_on(ptdev); 1516 if (ret) 1517 return ret; 1518 1519 fw->vm = panthor_vm_create(ptdev, true, 1520 0, SZ_4G, 1521 CSF_MCU_SHARED_REGION_START, 1522 CSF_MCU_SHARED_REGION_SIZE); 1523 if (IS_ERR(fw->vm)) { 1524 ret = PTR_ERR(fw->vm); 1525 fw->vm = NULL; 1526 goto err_unplug_fw; 1527 } 1528 1529 ret = panthor_fw_load(ptdev); 1530 if (ret) 1531 goto err_unplug_fw; 1532 1533 ret = panthor_vm_active(fw->vm); 1534 if (ret) 1535 goto err_unplug_fw; 1536 1537 ret = panthor_fw_start(ptdev); 1538 if (ret) 1539 goto err_unplug_fw; 1540 1541 ret = panthor_fw_init_ifaces(ptdev); 1542 if (ret) 1543 goto err_unplug_fw; 1544 1545 panthor_fw_init_global_iface(ptdev); 1546 return 0; 1547 1548 err_unplug_fw: 1549 panthor_fw_unplug(ptdev); 1550 return ret; 1551 } 1552 1553 MODULE_FIRMWARE("arm/mali/arch10.8/mali_csffw.bin"); 1554 MODULE_FIRMWARE("arm/mali/arch10.10/mali_csffw.bin"); 1555 MODULE_FIRMWARE("arm/mali/arch10.12/mali_csffw.bin"); 1556 MODULE_FIRMWARE("arm/mali/arch11.8/mali_csffw.bin"); 1557 MODULE_FIRMWARE("arm/mali/arch12.8/mali_csffw.bin"); 1558 MODULE_FIRMWARE("arm/mali/arch13.8/mali_csffw.bin"); 1559 MODULE_FIRMWARE("arm/mali/arch14.8/mali_csffw.bin"); 1560