xref: /linux/drivers/gpu/drm/imagination/pvr_ccb.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
2 /* Copyright (c) 2023 Imagination Technologies Ltd. */
3 
4 #include "pvr_ccb.h"
5 #include "pvr_device.h"
6 #include "pvr_drv.h"
7 #include "pvr_dump.h"
8 #include "pvr_free_list.h"
9 #include "pvr_fw.h"
10 #include "pvr_gem.h"
11 #include "pvr_power.h"
12 
13 #include <drm/drm_managed.h>
14 #include <drm/drm_print.h>
15 #include <linux/compiler.h>
16 #include <linux/delay.h>
17 #include <linux/jiffies.h>
18 #include <linux/kernel.h>
19 #include <linux/mutex.h>
20 #include <linux/overflow.h>
21 #include <linux/types.h>
22 #include <linux/workqueue.h>
23 
24 #define RESERVE_SLOT_TIMEOUT (1 * HZ) /* 1s */
25 #define RESERVE_SLOT_MIN_RETRIES 10
26 
27 static void
28 ccb_ctrl_init(void *cpu_ptr, void *priv)
29 {
30 	struct rogue_fwif_ccb_ctl *ctrl = cpu_ptr;
31 	struct pvr_ccb *pvr_ccb = priv;
32 
33 	ctrl->write_offset = 0;
34 	ctrl->read_offset = 0;
35 	ctrl->wrap_mask = pvr_ccb->num_cmds - 1;
36 	ctrl->cmd_size = pvr_ccb->cmd_size;
37 }
38 
39 /**
40  * pvr_ccb_init() - Initialise a CCB
41  * @pvr_dev: Device pointer.
42  * @pvr_ccb: Pointer to CCB structure to initialise.
43  * @num_cmds_log2: Log2 of number of commands in this CCB.
44  * @cmd_size: Command size for this CCB.
45  *
46  * Return:
47  *  * Zero on success, or
48  *  * Any error code returned by pvr_fw_object_create_and_map().
49  */
50 static int
51 pvr_ccb_init(struct pvr_device *pvr_dev, struct pvr_ccb *pvr_ccb,
52 	     u32 num_cmds_log2, size_t cmd_size)
53 {
54 	u32 num_cmds = 1 << num_cmds_log2;
55 	u32 ccb_size = num_cmds * cmd_size;
56 	int err;
57 
58 	pvr_ccb->num_cmds = num_cmds;
59 	pvr_ccb->cmd_size = cmd_size;
60 
61 	err = drmm_mutex_init(from_pvr_device(pvr_dev), &pvr_ccb->lock);
62 	if (err)
63 		return err;
64 
65 	/*
66 	 * Map CCB and control structure as uncached, so we don't have to flush
67 	 * CPU cache repeatedly when polling for space.
68 	 */
69 	pvr_ccb->ctrl = pvr_fw_object_create_and_map(pvr_dev, sizeof(*pvr_ccb->ctrl),
70 						     PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
71 						     ccb_ctrl_init, pvr_ccb, &pvr_ccb->ctrl_obj);
72 	if (IS_ERR(pvr_ccb->ctrl))
73 		return PTR_ERR(pvr_ccb->ctrl);
74 
75 	pvr_ccb->ccb = pvr_fw_object_create_and_map(pvr_dev, ccb_size,
76 						    PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
77 						    NULL, NULL, &pvr_ccb->ccb_obj);
78 	if (IS_ERR(pvr_ccb->ccb)) {
79 		err = PTR_ERR(pvr_ccb->ccb);
80 		goto err_free_ctrl;
81 	}
82 
83 	pvr_fw_object_get_fw_addr(pvr_ccb->ctrl_obj, &pvr_ccb->ctrl_fw_addr);
84 	pvr_fw_object_get_fw_addr(pvr_ccb->ccb_obj, &pvr_ccb->ccb_fw_addr);
85 
86 	return 0;
87 
88 err_free_ctrl:
89 	pvr_fw_object_unmap_and_destroy(pvr_ccb->ctrl_obj);
90 
91 	return err;
92 }
93 
94 /**
95  * pvr_ccb_fini() - Release CCB structure
96  * @pvr_ccb: CCB to release.
97  */
98 void
99 pvr_ccb_fini(struct pvr_ccb *pvr_ccb)
100 {
101 	pvr_fw_object_unmap_and_destroy(pvr_ccb->ccb_obj);
102 	pvr_fw_object_unmap_and_destroy(pvr_ccb->ctrl_obj);
103 }
104 
105 /**
106  * pvr_ccb_slot_available_locked() - Test whether any slots are available in CCB
107  * @pvr_ccb: CCB to test.
108  * @write_offset: Address to store number of next available slot. May be %NULL.
109  *
110  * Caller must hold @pvr_ccb->lock.
111  *
112  * Return:
113  *  * %true if a slot is available, or
114  *  * %false if no slot is available.
115  */
116 static __always_inline bool
117 pvr_ccb_slot_available_locked(struct pvr_ccb *pvr_ccb, u32 *write_offset)
118 {
119 	struct rogue_fwif_ccb_ctl *ctrl = pvr_ccb->ctrl;
120 	u32 next_write_offset = (READ_ONCE(ctrl->write_offset) + 1) & READ_ONCE(ctrl->wrap_mask);
121 
122 	lockdep_assert_held(&pvr_ccb->lock);
123 
124 	if (READ_ONCE(ctrl->read_offset) != next_write_offset) {
125 		if (write_offset)
126 			*write_offset = next_write_offset;
127 		return true;
128 	}
129 
130 	return false;
131 }
132 
133 static void
134 process_fwccb_command(struct pvr_device *pvr_dev, struct rogue_fwif_fwccb_cmd *cmd)
135 {
136 	struct drm_device *drm_dev = from_pvr_device(pvr_dev);
137 
138 	if ((cmd->cmd_type & ROGUE_CMD_MAGIC_DWORD_MASK) != ROGUE_CMD_MAGIC_DWORD_SHIFTED) {
139 		drm_warn_once(drm_dev, "Received FWCCB command with bad magic value; ignoring (type=0x%08x)\n",
140 			      cmd->cmd_type);
141 		return;
142 	}
143 
144 	switch (cmd->cmd_type) {
145 	case ROGUE_FWIF_FWCCB_CMD_REQUEST_GPU_RESTART:
146 		pvr_power_reset(pvr_dev, false);
147 		break;
148 
149 	case ROGUE_FWIF_FWCCB_CMD_FREELISTS_RECONSTRUCTION:
150 		pvr_free_list_process_reconstruct_req(pvr_dev,
151 						      &cmd->cmd_data.cmd_freelists_reconstruction);
152 		break;
153 
154 	case ROGUE_FWIF_FWCCB_CMD_FREELIST_GROW:
155 		pvr_free_list_process_grow_req(pvr_dev, &cmd->cmd_data.cmd_free_list_gs);
156 		break;
157 
158 	case ROGUE_FWIF_FWCCB_CMD_UPDATE_STATS:
159 		/*
160 		 * We currently have no infrastructure for processing these
161 		 * stats. It may be added in the future, but for now just
162 		 * suppress the "unknown" warning when receiving this command.
163 		 */
164 		break;
165 	case ROGUE_FWIF_FWCCB_CMD_CONTEXT_RESET_NOTIFICATION:
166 		pvr_dump_context_reset_notification(pvr_dev,
167 						    &cmd->cmd_data.cmd_context_reset_notification);
168 		break;
169 
170 	default:
171 		drm_info(drm_dev, "Received unknown FWCCB command (type=%d)\n",
172 			 cmd->cmd_type & ~ROGUE_CMD_MAGIC_DWORD_MASK);
173 		break;
174 	}
175 }
176 
177 /**
178  * pvr_fwccb_process() - Process any pending FWCCB commands
179  * @pvr_dev: Target PowerVR device
180  */
181 void pvr_fwccb_process(struct pvr_device *pvr_dev)
182 {
183 	struct rogue_fwif_fwccb_cmd *fwccb = pvr_dev->fwccb.ccb;
184 	struct rogue_fwif_ccb_ctl *ctrl = pvr_dev->fwccb.ctrl;
185 	u32 read_offset;
186 
187 	mutex_lock(&pvr_dev->fwccb.lock);
188 
189 	while ((read_offset = READ_ONCE(ctrl->read_offset)) != READ_ONCE(ctrl->write_offset)) {
190 		struct rogue_fwif_fwccb_cmd cmd = fwccb[read_offset];
191 
192 		WRITE_ONCE(ctrl->read_offset, (read_offset + 1) & READ_ONCE(ctrl->wrap_mask));
193 
194 		/* Drop FWCCB lock while we process command. */
195 		mutex_unlock(&pvr_dev->fwccb.lock);
196 
197 		process_fwccb_command(pvr_dev, &cmd);
198 
199 		mutex_lock(&pvr_dev->fwccb.lock);
200 	}
201 
202 	mutex_unlock(&pvr_dev->fwccb.lock);
203 }
204 
205 /**
206  * pvr_kccb_capacity() - Returns the maximum number of usable KCCB slots.
207  * @pvr_dev: Target PowerVR device
208  *
209  * Return:
210  *  * The maximum number of active slots.
211  */
212 static u32 pvr_kccb_capacity(struct pvr_device *pvr_dev)
213 {
214 	/* Capacity is the number of slot minus one to cope with the wrapping
215 	 * mechanisms. If we were to use all slots, we might end up with
216 	 * read_offset == write_offset, which the FW considers as a KCCB-is-empty
217 	 * condition.
218 	 */
219 	return pvr_dev->kccb.slot_count - 1;
220 }
221 
222 /**
223  * pvr_kccb_used_slot_count_locked() - Get the number of used slots
224  * @pvr_dev: Device pointer.
225  *
226  * KCCB lock must be held.
227  *
228  * Return:
229  *  * The number of slots currently used.
230  */
231 static u32
232 pvr_kccb_used_slot_count_locked(struct pvr_device *pvr_dev)
233 {
234 	struct pvr_ccb *pvr_ccb = &pvr_dev->kccb.ccb;
235 	struct rogue_fwif_ccb_ctl *ctrl = pvr_ccb->ctrl;
236 	u32 wr_offset = READ_ONCE(ctrl->write_offset);
237 	u32 rd_offset = READ_ONCE(ctrl->read_offset);
238 	u32 used_count;
239 
240 	lockdep_assert_held(&pvr_ccb->lock);
241 
242 	if (wr_offset >= rd_offset)
243 		used_count = wr_offset - rd_offset;
244 	else
245 		used_count = wr_offset + pvr_dev->kccb.slot_count - rd_offset;
246 
247 	return used_count;
248 }
249 
250 /**
251  * pvr_kccb_send_cmd_reserved_powered() - Send command to the KCCB, with the PM ref
252  * held and a slot pre-reserved
253  * @pvr_dev: Device pointer.
254  * @cmd: Command to send.
255  * @kccb_slot: Address to store the KCCB slot for this command. May be %NULL.
256  */
257 void
258 pvr_kccb_send_cmd_reserved_powered(struct pvr_device *pvr_dev,
259 				   struct rogue_fwif_kccb_cmd *cmd,
260 				   u32 *kccb_slot)
261 {
262 	struct pvr_ccb *pvr_ccb = &pvr_dev->kccb.ccb;
263 	struct rogue_fwif_kccb_cmd *kccb = pvr_ccb->ccb;
264 	struct rogue_fwif_ccb_ctl *ctrl = pvr_ccb->ctrl;
265 	u32 old_write_offset;
266 	u32 new_write_offset;
267 
268 	WARN_ON(pvr_dev->lost);
269 
270 	mutex_lock(&pvr_ccb->lock);
271 
272 	if (WARN_ON(!pvr_dev->kccb.reserved_count))
273 		goto out_unlock;
274 
275 	old_write_offset = READ_ONCE(ctrl->write_offset);
276 
277 	/* We reserved the slot, we should have one available. */
278 	if (WARN_ON(!pvr_ccb_slot_available_locked(pvr_ccb, &new_write_offset)))
279 		goto out_unlock;
280 
281 	memcpy(&kccb[old_write_offset], cmd,
282 	       sizeof(struct rogue_fwif_kccb_cmd));
283 	if (kccb_slot) {
284 		*kccb_slot = old_write_offset;
285 		/* Clear return status for this slot. */
286 		WRITE_ONCE(pvr_dev->kccb.rtn[old_write_offset],
287 			   ROGUE_FWIF_KCCB_RTN_SLOT_NO_RESPONSE);
288 	}
289 	mb(); /* memory barrier */
290 	WRITE_ONCE(ctrl->write_offset, new_write_offset);
291 	pvr_dev->kccb.reserved_count--;
292 
293 	/* Kick MTS */
294 	pvr_fw_mts_schedule(pvr_dev,
295 			    PVR_FWIF_DM_GP & ~ROGUE_CR_MTS_SCHEDULE_DM_CLRMSK);
296 
297 out_unlock:
298 	mutex_unlock(&pvr_ccb->lock);
299 }
300 
301 /**
302  * pvr_kccb_try_reserve_slot() - Try to reserve a KCCB slot
303  * @pvr_dev: Device pointer.
304  *
305  * Return:
306  *  * true if a KCCB slot was reserved, or
307  *  * false otherwise.
308  */
309 static bool pvr_kccb_try_reserve_slot(struct pvr_device *pvr_dev)
310 {
311 	bool reserved = false;
312 	u32 used_count;
313 
314 	mutex_lock(&pvr_dev->kccb.ccb.lock);
315 
316 	used_count = pvr_kccb_used_slot_count_locked(pvr_dev);
317 	if (pvr_dev->kccb.reserved_count < pvr_kccb_capacity(pvr_dev) - used_count) {
318 		pvr_dev->kccb.reserved_count++;
319 		reserved = true;
320 	}
321 
322 	mutex_unlock(&pvr_dev->kccb.ccb.lock);
323 
324 	return reserved;
325 }
326 
327 /**
328  * pvr_kccb_reserve_slot_sync() - Try to reserve a slot synchronously
329  * @pvr_dev: Device pointer.
330  *
331  * Return:
332  *  * 0 on success, or
333  *  * -EBUSY if no slots were reserved after %RESERVE_SLOT_TIMEOUT, with a minimum of
334  *    %RESERVE_SLOT_MIN_RETRIES retries.
335  */
336 static int pvr_kccb_reserve_slot_sync(struct pvr_device *pvr_dev)
337 {
338 	unsigned long start_timestamp = jiffies;
339 	bool reserved = false;
340 	u32 retries = 0;
341 
342 	while (time_before(jiffies, start_timestamp + RESERVE_SLOT_TIMEOUT) ||
343 	       retries < RESERVE_SLOT_MIN_RETRIES) {
344 		reserved = pvr_kccb_try_reserve_slot(pvr_dev);
345 		if (reserved)
346 			break;
347 
348 		usleep_range(1, 50);
349 
350 		if (retries < U32_MAX)
351 			retries++;
352 	}
353 
354 	return reserved ? 0 : -EBUSY;
355 }
356 
357 /**
358  * pvr_kccb_send_cmd_powered() - Send command to the KCCB, with a PM ref held
359  * @pvr_dev: Device pointer.
360  * @cmd: Command to send.
361  * @kccb_slot: Address to store the KCCB slot for this command. May be %NULL.
362  *
363  * Returns:
364  *  * Zero on success, or
365  *  * -EBUSY if timeout while waiting for a free KCCB slot.
366  */
367 int
368 pvr_kccb_send_cmd_powered(struct pvr_device *pvr_dev, struct rogue_fwif_kccb_cmd *cmd,
369 			  u32 *kccb_slot)
370 {
371 	int err;
372 
373 	err = pvr_kccb_reserve_slot_sync(pvr_dev);
374 	if (err)
375 		return err;
376 
377 	pvr_kccb_send_cmd_reserved_powered(pvr_dev, cmd, kccb_slot);
378 	return 0;
379 }
380 
381 /**
382  * pvr_kccb_send_cmd() - Send command to the KCCB
383  * @pvr_dev: Device pointer.
384  * @cmd: Command to send.
385  * @kccb_slot: Address to store the KCCB slot for this command. May be %NULL.
386  *
387  * Returns:
388  *  * Zero on success, or
389  *  * -EBUSY if timeout while waiting for a free KCCB slot.
390  */
391 int
392 pvr_kccb_send_cmd(struct pvr_device *pvr_dev, struct rogue_fwif_kccb_cmd *cmd,
393 		  u32 *kccb_slot)
394 {
395 	int err;
396 
397 	err = pvr_power_get(pvr_dev);
398 	if (err)
399 		return err;
400 
401 	err = pvr_kccb_send_cmd_powered(pvr_dev, cmd, kccb_slot);
402 
403 	pvr_power_put(pvr_dev);
404 
405 	return err;
406 }
407 
408 /**
409  * pvr_kccb_wait_for_completion() - Wait for a KCCB command to complete
410  * @pvr_dev: Device pointer.
411  * @slot_nr: KCCB slot to wait on.
412  * @timeout: Timeout length (in jiffies).
413  * @rtn_out: Location to store KCCB command result. May be %NULL.
414  *
415  * Returns:
416  *  * Zero on success, or
417  *  * -ETIMEDOUT on timeout.
418  */
419 int
420 pvr_kccb_wait_for_completion(struct pvr_device *pvr_dev, u32 slot_nr,
421 			     u32 timeout, u32 *rtn_out)
422 {
423 	int ret = wait_event_timeout(pvr_dev->kccb.rtn_q, READ_ONCE(pvr_dev->kccb.rtn[slot_nr]) &
424 				     ROGUE_FWIF_KCCB_RTN_SLOT_CMD_EXECUTED, timeout);
425 
426 	if (ret && rtn_out)
427 		*rtn_out = READ_ONCE(pvr_dev->kccb.rtn[slot_nr]);
428 
429 	return ret ? 0 : -ETIMEDOUT;
430 }
431 
432 /**
433  * pvr_kccb_is_idle() - Returns whether the device's KCCB is idle
434  * @pvr_dev: Device pointer
435  *
436  * Returns:
437  *  * %true if the KCCB is idle (contains no commands), or
438  *  * %false if the KCCB contains pending commands.
439  */
440 bool
441 pvr_kccb_is_idle(struct pvr_device *pvr_dev)
442 {
443 	struct rogue_fwif_ccb_ctl *ctrl = pvr_dev->kccb.ccb.ctrl;
444 	bool idle;
445 
446 	mutex_lock(&pvr_dev->kccb.ccb.lock);
447 
448 	idle = (READ_ONCE(ctrl->write_offset) == READ_ONCE(ctrl->read_offset));
449 
450 	mutex_unlock(&pvr_dev->kccb.ccb.lock);
451 
452 	return idle;
453 }
454 
455 static const char *
456 pvr_kccb_fence_get_driver_name(struct dma_fence *f)
457 {
458 	return PVR_DRIVER_NAME;
459 }
460 
461 static const char *
462 pvr_kccb_fence_get_timeline_name(struct dma_fence *f)
463 {
464 	return "kccb";
465 }
466 
467 static const struct dma_fence_ops pvr_kccb_fence_ops = {
468 	.get_driver_name = pvr_kccb_fence_get_driver_name,
469 	.get_timeline_name = pvr_kccb_fence_get_timeline_name,
470 };
471 
472 /**
473  * struct pvr_kccb_fence - Fence object used to wait for a KCCB slot
474  */
475 struct pvr_kccb_fence {
476 	/** @base: Base dma_fence object. */
477 	struct dma_fence base;
478 
479 	/** @node: Node used to insert the fence in the pvr_device::kccb::waiters list. */
480 	struct list_head node;
481 };
482 
483 /**
484  * pvr_kccb_wake_up_waiters() - Check the KCCB waiters
485  * @pvr_dev: Target PowerVR device
486  *
487  * Signal as many KCCB fences as we have slots available.
488  */
489 void pvr_kccb_wake_up_waiters(struct pvr_device *pvr_dev)
490 {
491 	struct pvr_kccb_fence *fence, *tmp_fence;
492 	u32 used_count, available_count;
493 
494 	/* Wake up those waiting for KCCB slot execution. */
495 	wake_up_all(&pvr_dev->kccb.rtn_q);
496 
497 	/* Then iterate over all KCCB fences and signal as many as we can. */
498 	mutex_lock(&pvr_dev->kccb.ccb.lock);
499 	used_count = pvr_kccb_used_slot_count_locked(pvr_dev);
500 
501 	if (WARN_ON(used_count + pvr_dev->kccb.reserved_count > pvr_kccb_capacity(pvr_dev)))
502 		goto out_unlock;
503 
504 	available_count = pvr_kccb_capacity(pvr_dev) - used_count - pvr_dev->kccb.reserved_count;
505 	list_for_each_entry_safe(fence, tmp_fence, &pvr_dev->kccb.waiters, node) {
506 		if (!available_count)
507 			break;
508 
509 		list_del(&fence->node);
510 		pvr_dev->kccb.reserved_count++;
511 		available_count--;
512 		dma_fence_signal(&fence->base);
513 		dma_fence_put(&fence->base);
514 	}
515 
516 out_unlock:
517 	mutex_unlock(&pvr_dev->kccb.ccb.lock);
518 }
519 
520 /**
521  * pvr_kccb_fini() - Cleanup device KCCB
522  * @pvr_dev: Target PowerVR device
523  */
524 void pvr_kccb_fini(struct pvr_device *pvr_dev)
525 {
526 	pvr_fw_object_unmap_and_destroy(pvr_dev->kccb.rtn_obj);
527 	pvr_ccb_fini(&pvr_dev->kccb.ccb);
528 	WARN_ON(!list_empty(&pvr_dev->kccb.waiters));
529 	WARN_ON(pvr_dev->kccb.reserved_count);
530 }
531 
532 /**
533  * pvr_kccb_init() - Initialise device KCCB
534  * @pvr_dev: Target PowerVR device
535  *
536  * Returns:
537  *  * 0 on success, or
538  *  * Any error returned by pvr_ccb_init().
539  */
540 int
541 pvr_kccb_init(struct pvr_device *pvr_dev)
542 {
543 	const u32 num_slots_log2 = ROGUE_FWIF_KCCB_NUMCMDS_LOG2_DEFAULT;
544 	const u32 num_slots = 1 << num_slots_log2;
545 	u32 rtn_size;
546 	int err;
547 
548 	/*
549 	 * The inputs here are compile-time constants; there's no reason to try
550 	 * to gracefully handle overflow at runtime.
551 	 */
552 	BUILD_BUG_ON(check_mul_overflow(num_slots, sizeof(*pvr_dev->kccb.rtn), &rtn_size));
553 
554 	pvr_dev->kccb.slot_count = num_slots;
555 	INIT_LIST_HEAD(&pvr_dev->kccb.waiters);
556 	pvr_dev->kccb.fence_ctx.id = dma_fence_context_alloc(1);
557 	spin_lock_init(&pvr_dev->kccb.fence_ctx.lock);
558 
559 	err = pvr_ccb_init(pvr_dev, &pvr_dev->kccb.ccb, num_slots_log2,
560 			   sizeof(struct rogue_fwif_kccb_cmd));
561 	if (err)
562 		return err;
563 
564 	/* Allocate memory for KCCB return slots. */
565 	pvr_dev->kccb.rtn = pvr_fw_object_create_and_map(pvr_dev, rtn_size,
566 							 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
567 							 NULL, NULL, &pvr_dev->kccb.rtn_obj);
568 	if (IS_ERR(pvr_dev->kccb.rtn)) {
569 		err = PTR_ERR(pvr_dev->kccb.rtn);
570 		goto err_ccb_fini;
571 	}
572 
573 	return 0;
574 
575 err_ccb_fini:
576 	pvr_ccb_fini(&pvr_dev->kccb.ccb);
577 
578 	return err;
579 }
580 
581 /**
582  * pvr_kccb_fence_alloc() - Allocate a pvr_kccb_fence object
583  *
584  * Return:
585  *  * NULL if the allocation fails, or
586  *  * A valid dma_fence pointer otherwise.
587  */
588 struct dma_fence *pvr_kccb_fence_alloc(void)
589 {
590 	struct pvr_kccb_fence *kccb_fence;
591 
592 	kccb_fence = kzalloc_obj(*kccb_fence);
593 	if (!kccb_fence)
594 		return NULL;
595 
596 	return &kccb_fence->base;
597 }
598 
599 /**
600  * pvr_kccb_fence_put() - Drop a KCCB fence reference
601  * @fence: The fence to drop the reference on.
602  *
603  * If the fence hasn't been initialized yet, dma_fence_free() is called. This
604  * way we have a single function taking care of both cases.
605  */
606 void pvr_kccb_fence_put(struct dma_fence *fence)
607 {
608 	if (!fence)
609 		return;
610 
611 	if (!fence->ops) {
612 		dma_fence_free(fence);
613 	} else {
614 		WARN_ON(fence->ops != &pvr_kccb_fence_ops);
615 		dma_fence_put(fence);
616 	}
617 }
618 
619 /**
620  * pvr_kccb_reserve_slot() - Reserve a KCCB slot for later use
621  * @pvr_dev: Target PowerVR device
622  * @f: KCCB fence object previously allocated with pvr_kccb_fence_alloc()
623  *
624  * Try to reserve a KCCB slot, and if there's no slot available,
625  * initializes the fence object and queue it to the waiters list.
626  *
627  * If NULL is returned, that means the slot is reserved. In that case,
628  * the @f is freed and shouldn't be accessed after that point.
629  *
630  * Return:
631  *  * NULL if a slot was available directly, or
632  *  * A valid dma_fence object to wait on if no slot was available.
633  */
634 struct dma_fence *
635 pvr_kccb_reserve_slot(struct pvr_device *pvr_dev, struct dma_fence *f)
636 {
637 	struct pvr_kccb_fence *fence = container_of(f, struct pvr_kccb_fence, base);
638 	struct dma_fence *out_fence = NULL;
639 	u32 used_count;
640 
641 	mutex_lock(&pvr_dev->kccb.ccb.lock);
642 
643 	used_count = pvr_kccb_used_slot_count_locked(pvr_dev);
644 	if (pvr_dev->kccb.reserved_count >= pvr_kccb_capacity(pvr_dev) - used_count) {
645 		dma_fence_init(&fence->base, &pvr_kccb_fence_ops,
646 			       &pvr_dev->kccb.fence_ctx.lock,
647 			       pvr_dev->kccb.fence_ctx.id,
648 			       atomic_inc_return(&pvr_dev->kccb.fence_ctx.seqno));
649 		out_fence = dma_fence_get(&fence->base);
650 		list_add_tail(&fence->node, &pvr_dev->kccb.waiters);
651 	} else {
652 		pvr_kccb_fence_put(f);
653 		pvr_dev->kccb.reserved_count++;
654 	}
655 
656 	mutex_unlock(&pvr_dev->kccb.ccb.lock);
657 
658 	return out_fence;
659 }
660 
661 /**
662  * pvr_kccb_release_slot() - Release a KCCB slot reserved with
663  * pvr_kccb_reserve_slot()
664  * @pvr_dev: Target PowerVR device
665  *
666  * Should only be called if something failed after the
667  * pvr_kccb_reserve_slot() call and you know you won't call
668  * pvr_kccb_send_cmd_reserved().
669  */
670 void pvr_kccb_release_slot(struct pvr_device *pvr_dev)
671 {
672 	mutex_lock(&pvr_dev->kccb.ccb.lock);
673 	if (!WARN_ON(!pvr_dev->kccb.reserved_count))
674 		pvr_dev->kccb.reserved_count--;
675 	mutex_unlock(&pvr_dev->kccb.ccb.lock);
676 }
677 
678 /**
679  * pvr_fwccb_init() - Initialise device FWCCB
680  * @pvr_dev: Target PowerVR device
681  *
682  * Returns:
683  *  * 0 on success, or
684  *  * Any error returned by pvr_ccb_init().
685  */
686 int
687 pvr_fwccb_init(struct pvr_device *pvr_dev)
688 {
689 	return pvr_ccb_init(pvr_dev, &pvr_dev->fwccb,
690 			    ROGUE_FWIF_FWCCB_NUMCMDS_LOG2,
691 			    sizeof(struct rogue_fwif_fwccb_cmd));
692 }
693