24 	struct panfrost_device *pfdev = data;  in panfrost_gpu_irq_handler()  local
27 	if (test_bit(PANFROST_COMP_BIT_GPU, pfdev->is_suspended))  in panfrost_gpu_irq_handler()
30 	fault_status = gpu_read(pfdev, GPU_FAULT_STATUS);  in panfrost_gpu_irq_handler()
31 	state = gpu_read(pfdev, GPU_INT_STAT);  in panfrost_gpu_irq_handler()
36 		u64 address = (u64) gpu_read(pfdev, GPU_FAULT_ADDRESS_HI) << 32;  in panfrost_gpu_irq_handler()
37 		address |= gpu_read(pfdev, GPU_FAULT_ADDRESS_LO);  in panfrost_gpu_irq_handler()
39 		dev_warn(pfdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx\n",  in panfrost_gpu_irq_handler()
44 			dev_warn(pfdev->dev, "There were multiple GPU faults - some have not been reported\n");  in panfrost_gpu_irq_handler()
46 		gpu_write(pfdev, GPU_INT_MASK, 0);  in panfrost_gpu_irq_handler()
50 		panfrost_perfcnt_sample_done(pfdev);  in panfrost_gpu_irq_handler()
53 		panfrost_perfcnt_clean_cache_done(pfdev);  in panfrost_gpu_irq_handler()
55 	gpu_write(pfdev, GPU_INT_CLEAR, state);  in panfrost_gpu_irq_handler()
60 int panfrost_gpu_soft_reset(struct panfrost_device *pfdev)  in panfrost_gpu_soft_reset()  argument
65 	gpu_write(pfdev, GPU_INT_MASK, 0);  in panfrost_gpu_soft_reset()
66 	gpu_write(pfdev, GPU_INT_CLEAR, GPU_IRQ_RESET_COMPLETED);  in panfrost_gpu_soft_reset()
68 	clear_bit(PANFROST_COMP_BIT_GPU, pfdev->is_suspended);  in panfrost_gpu_soft_reset()
70 	gpu_write(pfdev, GPU_CMD, GPU_CMD_SOFT_RESET);  in panfrost_gpu_soft_reset()
71 	ret = readl_relaxed_poll_timeout(pfdev->iomem + GPU_INT_RAWSTAT,  in panfrost_gpu_soft_reset()
75 		dev_err(pfdev->dev, "gpu soft reset timed out, attempting hard reset\n");  in panfrost_gpu_soft_reset()
77 		gpu_write(pfdev, GPU_CMD, GPU_CMD_HARD_RESET);  in panfrost_gpu_soft_reset()
78 		ret = readl_relaxed_poll_timeout(pfdev->iomem + GPU_INT_RAWSTAT, val,  in panfrost_gpu_soft_reset()
81 			dev_err(pfdev->dev, "gpu hard reset timed out\n");  in panfrost_gpu_soft_reset()
86 	gpu_write(pfdev, GPU_INT_CLEAR, GPU_IRQ_MASK_ALL);  in panfrost_gpu_soft_reset()
89 	gpu_write(pfdev, GPU_INT_MASK,  in panfrost_gpu_soft_reset()
98 	if (drm_WARN_ON(pfdev->ddev, atomic_read(&pfdev->cycle_counter.use_count) != 0))  in panfrost_gpu_soft_reset()
99 		atomic_set(&pfdev->cycle_counter.use_count, 0);  in panfrost_gpu_soft_reset()
104 void panfrost_gpu_amlogic_quirk(struct panfrost_device *pfdev)  in panfrost_gpu_amlogic_quirk()  argument
111 	gpu_write(pfdev, GPU_PWR_KEY, GPU_PWR_KEY_UNLOCK);  in panfrost_gpu_amlogic_quirk()
112 	gpu_write(pfdev, GPU_PWR_OVERRIDE1, 0xfff | (0x20 << 16));  in panfrost_gpu_amlogic_quirk()
115 static void panfrost_gpu_init_quirks(struct panfrost_device *pfdev)  in panfrost_gpu_init_quirks()  argument
119 	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8443) ||  in panfrost_gpu_init_quirks()
120 	    panfrost_has_hw_issue(pfdev, HW_ISSUE_11035))  in panfrost_gpu_init_quirks()
123 	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10327))  in panfrost_gpu_init_quirks()
126 	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10797))  in panfrost_gpu_init_quirks()
129 	if (!panfrost_has_hw_issue(pfdev, GPUCORE_1619)) {  in panfrost_gpu_init_quirks()
130 		if (panfrost_model_cmp(pfdev, 0x750) < 0) /* T60x, T62x, T72x */  in panfrost_gpu_init_quirks()
132 		else if (panfrost_model_cmp(pfdev, 0x880) <= 0) /* T76x, T8xx */  in panfrost_gpu_init_quirks()
136 	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_TTRX_2968_TTRX_3162))  in panfrost_gpu_init_quirks()
139 	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_TLS_HASHING))  in panfrost_gpu_init_quirks()
143 		gpu_write(pfdev, GPU_SHADER_CONFIG, quirks);  in panfrost_gpu_init_quirks()
146 	quirks = gpu_read(pfdev, GPU_TILER_CONFIG);  in panfrost_gpu_init_quirks()
149 	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_T76X_3953))  in panfrost_gpu_init_quirks()
152 	gpu_write(pfdev, GPU_TILER_CONFIG, quirks);  in panfrost_gpu_init_quirks()
156 	if ((panfrost_model_eq(pfdev, 0x860) || panfrost_model_eq(pfdev, 0x880)) &&  in panfrost_gpu_init_quirks()
157 	    pfdev->features.revision >= 0x2000)  in panfrost_gpu_init_quirks()
159 	else if (panfrost_model_eq(pfdev, 0x6000) &&  in panfrost_gpu_init_quirks()
160 		 pfdev->features.coherency_features == COHERENCY_ACE)  in panfrost_gpu_init_quirks()
164 	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_IDVS_GROUP_SIZE))  in panfrost_gpu_init_quirks()
168 		gpu_write(pfdev, GPU_JM_CONFIG, quirks);  in panfrost_gpu_init_quirks()
171 	if (pfdev->comp->vendor_quirk)  in panfrost_gpu_init_quirks()
172 		pfdev->comp->vendor_quirk(pfdev);  in panfrost_gpu_init_quirks()
245 static void panfrost_gpu_init_features(struct panfrost_device *pfdev)  in panfrost_gpu_init_features()  argument
254 	pfdev->features.l2_features = gpu_read(pfdev, GPU_L2_FEATURES);  in panfrost_gpu_init_features()
255 	pfdev->features.core_features = gpu_read(pfdev, GPU_CORE_FEATURES);  in panfrost_gpu_init_features()
256 	pfdev->features.tiler_features = gpu_read(pfdev, GPU_TILER_FEATURES);  in panfrost_gpu_init_features()
257 	pfdev->features.mem_features = gpu_read(pfdev, GPU_MEM_FEATURES);  in panfrost_gpu_init_features()
258 	pfdev->features.mmu_features = gpu_read(pfdev, GPU_MMU_FEATURES);  in panfrost_gpu_init_features()
259 	pfdev->features.thread_features = gpu_read(pfdev, GPU_THREAD_FEATURES);  in panfrost_gpu_init_features()
260 	pfdev->features.max_threads = gpu_read(pfdev, GPU_THREAD_MAX_THREADS);  in panfrost_gpu_init_features()
261 	pfdev->features.thread_max_workgroup_sz = gpu_read(pfdev, GPU_THREAD_MAX_WORKGROUP_SIZE);  in panfrost_gpu_init_features()
262 	pfdev->features.thread_max_barrier_sz = gpu_read(pfdev, GPU_THREAD_MAX_BARRIER_SIZE);  in panfrost_gpu_init_features()
263 	pfdev->features.coherency_features = gpu_read(pfdev, GPU_COHERENCY_FEATURES);  in panfrost_gpu_init_features()
264 	pfdev->features.afbc_features = gpu_read(pfdev, GPU_AFBC_FEATURES);  in panfrost_gpu_init_features()
266 		pfdev->features.texture_features[i] = gpu_read(pfdev, GPU_TEXTURE_FEATURES(i));  in panfrost_gpu_init_features()
268 	pfdev->features.as_present = gpu_read(pfdev, GPU_AS_PRESENT);  in panfrost_gpu_init_features()
270 	pfdev->features.js_present = gpu_read(pfdev, GPU_JS_PRESENT);  in panfrost_gpu_init_features()
271 	num_js = hweight32(pfdev->features.js_present);  in panfrost_gpu_init_features()
273 		pfdev->features.js_features[i] = gpu_read(pfdev, GPU_JS_FEATURES(i));  in panfrost_gpu_init_features()
275 	pfdev->features.shader_present = gpu_read(pfdev, GPU_SHADER_PRESENT_LO);  in panfrost_gpu_init_features()
276 	pfdev->features.shader_present |= (u64)gpu_read(pfdev, GPU_SHADER_PRESENT_HI) << 32;  in panfrost_gpu_init_features()
278 	pfdev->features.tiler_present = gpu_read(pfdev, GPU_TILER_PRESENT_LO);  in panfrost_gpu_init_features()
279 	pfdev->features.tiler_present |= (u64)gpu_read(pfdev, GPU_TILER_PRESENT_HI) << 32;  in panfrost_gpu_init_features()
281 	pfdev->features.l2_present = gpu_read(pfdev, GPU_L2_PRESENT_LO);  in panfrost_gpu_init_features()
282 	pfdev->features.l2_present |= (u64)gpu_read(pfdev, GPU_L2_PRESENT_HI) << 32;  in panfrost_gpu_init_features()
283 	pfdev->features.nr_core_groups = hweight64(pfdev->features.l2_present);  in panfrost_gpu_init_features()
285 	pfdev->features.stack_present = gpu_read(pfdev, GPU_STACK_PRESENT_LO);  in panfrost_gpu_init_features()
286 	pfdev->features.stack_present |= (u64)gpu_read(pfdev, GPU_STACK_PRESENT_HI) << 32;  in panfrost_gpu_init_features()
288 	pfdev->features.thread_tls_alloc = gpu_read(pfdev, GPU_THREAD_TLS_ALLOC);  in panfrost_gpu_init_features()
290 	gpu_id = gpu_read(pfdev, GPU_ID);  in panfrost_gpu_init_features()
291 	pfdev->features.revision = gpu_id & 0xffff;  in panfrost_gpu_init_features()
292 	pfdev->features.id = gpu_id >> 16;  in panfrost_gpu_init_features()
297 	if (pfdev->features.id == 0x6956)  in panfrost_gpu_init_features()
298 		pfdev->features.id = 0x0600;  in panfrost_gpu_init_features()
300 	major = (pfdev->features.revision >> 12) & 0xf;  in panfrost_gpu_init_features()
301 	minor = (pfdev->features.revision >> 4) & 0xff;  in panfrost_gpu_init_features()
302 	status = pfdev->features.revision & 0xf;  in panfrost_gpu_init_features()
303 	rev = pfdev->features.revision;  in panfrost_gpu_init_features()
305 	gpu_id = pfdev->features.id;  in panfrost_gpu_init_features()
310 		if (!panfrost_model_eq(pfdev, model->id))  in panfrost_gpu_init_features()
330 	bitmap_from_u64(pfdev->features.hw_features, hw_feat);  in panfrost_gpu_init_features()
331 	bitmap_from_u64(pfdev->features.hw_issues, hw_issues);  in panfrost_gpu_init_features()
333 	dev_info(pfdev->dev, "mali-%s id 0x%x major 0x%x minor 0x%x status 0x%x",  in panfrost_gpu_init_features()
335 	dev_info(pfdev->dev, "features: %64pb, issues: %64pb",  in panfrost_gpu_init_features()
336 		 pfdev->features.hw_features,  in panfrost_gpu_init_features()
337 		 pfdev->features.hw_issues);  in panfrost_gpu_init_features()
339 …dev_info(pfdev->dev, "Features: L2:0x%08x Shader:0x%08x Tiler:0x%08x Mem:0x%0x MMU:0x%08x AS:0x%x …  in panfrost_gpu_init_features()
340 		 pfdev->features.l2_features,  in panfrost_gpu_init_features()
341 		 pfdev->features.core_features,  in panfrost_gpu_init_features()
342 		 pfdev->features.tiler_features,  in panfrost_gpu_init_features()
343 		 pfdev->features.mem_features,  in panfrost_gpu_init_features()
344 		 pfdev->features.mmu_features,  in panfrost_gpu_init_features()
345 		 pfdev->features.as_present,  in panfrost_gpu_init_features()
346 		 pfdev->features.js_present);  in panfrost_gpu_init_features()
348 	dev_info(pfdev->dev, "shader_present=0x%0llx l2_present=0x%0llx",  in panfrost_gpu_init_features()
349 		 pfdev->features.shader_present, pfdev->features.l2_present);  in panfrost_gpu_init_features()
352 void panfrost_cycle_counter_get(struct panfrost_device *pfdev)  in panfrost_cycle_counter_get()  argument
354 	if (atomic_inc_not_zero(&pfdev->cycle_counter.use_count))  in panfrost_cycle_counter_get()
357 	spin_lock(&pfdev->cycle_counter.lock);  in panfrost_cycle_counter_get()
358 	if (atomic_inc_return(&pfdev->cycle_counter.use_count) == 1)  in panfrost_cycle_counter_get()
359 		gpu_write(pfdev, GPU_CMD, GPU_CMD_CYCLE_COUNT_START);  in panfrost_cycle_counter_get()
360 	spin_unlock(&pfdev->cycle_counter.lock);  in panfrost_cycle_counter_get()
363 void panfrost_cycle_counter_put(struct panfrost_device *pfdev)  in panfrost_cycle_counter_put()  argument
365 	if (atomic_add_unless(&pfdev->cycle_counter.use_count, -1, 1))  in panfrost_cycle_counter_put()
368 	spin_lock(&pfdev->cycle_counter.lock);  in panfrost_cycle_counter_put()
369 	if (atomic_dec_return(&pfdev->cycle_counter.use_count) == 0)  in panfrost_cycle_counter_put()
370 		gpu_write(pfdev, GPU_CMD, GPU_CMD_CYCLE_COUNT_STOP);  in panfrost_cycle_counter_put()
371 	spin_unlock(&pfdev->cycle_counter.lock);  in panfrost_cycle_counter_put()
374 unsigned long long panfrost_cycle_counter_read(struct panfrost_device *pfdev)  in panfrost_cycle_counter_read()  argument
379 		hi = gpu_read(pfdev, GPU_CYCLE_COUNT_HI);  in panfrost_cycle_counter_read()
380 		lo = gpu_read(pfdev, GPU_CYCLE_COUNT_LO);  in panfrost_cycle_counter_read()
381 	} while (hi != gpu_read(pfdev, GPU_CYCLE_COUNT_HI));  in panfrost_cycle_counter_read()
386 unsigned long long panfrost_timestamp_read(struct panfrost_device *pfdev)  in panfrost_timestamp_read()  argument
391 		hi = gpu_read(pfdev, GPU_TIMESTAMP_HI);  in panfrost_timestamp_read()
392 		lo = gpu_read(pfdev, GPU_TIMESTAMP_LO);  in panfrost_timestamp_read()
393 	} while (hi != gpu_read(pfdev, GPU_TIMESTAMP_HI));  in panfrost_timestamp_read()
398 static u64 panfrost_get_core_mask(struct panfrost_device *pfdev)  in panfrost_get_core_mask()  argument
402 	if (pfdev->features.l2_present == 1)  in panfrost_get_core_mask()
412 	core_mask = ~(pfdev->features.l2_present - 1) &  in panfrost_get_core_mask()
413 		     (pfdev->features.l2_present - 2);  in panfrost_get_core_mask()
414 	dev_info_once(pfdev->dev, "using only 1st core group (%lu cores from %lu)\n",  in panfrost_get_core_mask()
416 		      hweight64(pfdev->features.shader_present));  in panfrost_get_core_mask()
421 void panfrost_gpu_power_on(struct panfrost_device *pfdev)  in panfrost_gpu_power_on()  argument
427 	panfrost_gpu_init_quirks(pfdev);  in panfrost_gpu_power_on()
428 	core_mask = panfrost_get_core_mask(pfdev);  in panfrost_gpu_power_on()
430 	gpu_write(pfdev, L2_PWRON_LO, pfdev->features.l2_present & core_mask);  in panfrost_gpu_power_on()
431 	ret = readl_relaxed_poll_timeout(pfdev->iomem + L2_READY_LO,  in panfrost_gpu_power_on()
432 		val, val == (pfdev->features.l2_present & core_mask),  in panfrost_gpu_power_on()
435 		dev_err(pfdev->dev, "error powering up gpu L2");  in panfrost_gpu_power_on()
437 	gpu_write(pfdev, SHADER_PWRON_LO,  in panfrost_gpu_power_on()
438 		  pfdev->features.shader_present & core_mask);  in panfrost_gpu_power_on()
439 	ret = readl_relaxed_poll_timeout(pfdev->iomem + SHADER_READY_LO,  in panfrost_gpu_power_on()
440 		val, val == (pfdev->features.shader_present & core_mask),  in panfrost_gpu_power_on()
443 		dev_err(pfdev->dev, "error powering up gpu shader");  in panfrost_gpu_power_on()
445 	gpu_write(pfdev, TILER_PWRON_LO, pfdev->features.tiler_present);  in panfrost_gpu_power_on()
446 	ret = readl_relaxed_poll_timeout(pfdev->iomem + TILER_READY_LO,  in panfrost_gpu_power_on()
447 		val, val == pfdev->features.tiler_present, 10, 1000);  in panfrost_gpu_power_on()
449 		dev_err(pfdev->dev, "error powering up gpu tiler");  in panfrost_gpu_power_on()
452 void panfrost_gpu_power_off(struct panfrost_device *pfdev)  in panfrost_gpu_power_off()  argument
457 	gpu_write(pfdev, SHADER_PWROFF_LO, pfdev->features.shader_present);  in panfrost_gpu_power_off()
458 	ret = readl_relaxed_poll_timeout(pfdev->iomem + SHADER_PWRTRANS_LO,  in panfrost_gpu_power_off()
461 		dev_err(pfdev->dev, "shader power transition timeout");  in panfrost_gpu_power_off()
463 	gpu_write(pfdev, TILER_PWROFF_LO, pfdev->features.tiler_present);  in panfrost_gpu_power_off()
464 	ret = readl_relaxed_poll_timeout(pfdev->iomem + TILER_PWRTRANS_LO,  in panfrost_gpu_power_off()
467 		dev_err(pfdev->dev, "tiler power transition timeout");  in panfrost_gpu_power_off()
469 	gpu_write(pfdev, L2_PWROFF_LO, pfdev->features.l2_present);  in panfrost_gpu_power_off()
470 	ret = readl_poll_timeout(pfdev->iomem + L2_PWRTRANS_LO,  in panfrost_gpu_power_off()
473 		dev_err(pfdev->dev, "l2 power transition timeout");  in panfrost_gpu_power_off()
476 void panfrost_gpu_suspend_irq(struct panfrost_device *pfdev)  in panfrost_gpu_suspend_irq()  argument
478 	set_bit(PANFROST_COMP_BIT_GPU, pfdev->is_suspended);  in panfrost_gpu_suspend_irq()
480 	gpu_write(pfdev, GPU_INT_MASK, 0);  in panfrost_gpu_suspend_irq()
481 	synchronize_irq(pfdev->gpu_irq);  in panfrost_gpu_suspend_irq()
484 int panfrost_gpu_init(struct panfrost_device *pfdev)  in panfrost_gpu_init()  argument
488 	err = panfrost_gpu_soft_reset(pfdev);  in panfrost_gpu_init()
492 	panfrost_gpu_init_features(pfdev);  in panfrost_gpu_init()
494 	err = dma_set_mask_and_coherent(pfdev->dev,  in panfrost_gpu_init()
495 		DMA_BIT_MASK(FIELD_GET(0xff00, pfdev->features.mmu_features)));  in panfrost_gpu_init()
499 	dma_set_max_seg_size(pfdev->dev, UINT_MAX);  in panfrost_gpu_init()
501 	pfdev->gpu_irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "gpu");  in panfrost_gpu_init()
502 	if (pfdev->gpu_irq < 0)  in panfrost_gpu_init()
503 		return pfdev->gpu_irq;  in panfrost_gpu_init()
505 	err = devm_request_irq(pfdev->dev, pfdev->gpu_irq, panfrost_gpu_irq_handler,  in panfrost_gpu_init()
506 			       IRQF_SHARED, KBUILD_MODNAME "-gpu", pfdev);  in panfrost_gpu_init()
508 		dev_err(pfdev->dev, "failed to request gpu irq");  in panfrost_gpu_init()
512 	panfrost_gpu_power_on(pfdev);  in panfrost_gpu_init()
517 void panfrost_gpu_fini(struct panfrost_device *pfdev)  in panfrost_gpu_fini()  argument
519 	panfrost_gpu_power_off(pfdev);  in panfrost_gpu_fini()
522 u32 panfrost_gpu_get_latest_flush_id(struct panfrost_device *pfdev)  in panfrost_gpu_get_latest_flush_id()  argument
526 	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION)) {  in panfrost_gpu_get_latest_flush_id()
528 		if (pm_runtime_get_if_in_use(pfdev->dev)) {  in panfrost_gpu_get_latest_flush_id()
529 			flush_id = gpu_read(pfdev, GPU_LATEST_FLUSH_ID);  in panfrost_gpu_get_latest_flush_id()
530 			pm_runtime_put(pfdev->dev);  in panfrost_gpu_get_latest_flush_id()