xref: /linux/drivers/crypto/ccree/cc_request_mgr.c (revision e9f0878c4b2004ac19581274c1ae4c61ae3ca70e)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2012-2018 ARM Limited or its affiliates. */
3 
4 #include <linux/kernel.h>
5 #include "cc_driver.h"
6 #include "cc_buffer_mgr.h"
7 #include "cc_request_mgr.h"
8 #include "cc_ivgen.h"
9 #include "cc_pm.h"
10 
11 #define CC_MAX_POLL_ITER	10
12 /* The highest descriptor count in used */
13 #define CC_MAX_DESC_SEQ_LEN	23
14 
15 struct cc_req_mgr_handle {
16 	/* Request manager resources */
17 	unsigned int hw_queue_size; /* HW capability */
18 	unsigned int min_free_hw_slots;
19 	unsigned int max_used_sw_slots;
20 	struct cc_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE];
21 	u32 req_queue_head;
22 	u32 req_queue_tail;
23 	u32 axi_completed;
24 	u32 q_free_slots;
25 	/* This lock protects access to HW register
26 	 * that must be single request at a time
27 	 */
28 	spinlock_t hw_lock;
29 	struct cc_hw_desc compl_desc;
30 	u8 *dummy_comp_buff;
31 	dma_addr_t dummy_comp_buff_dma;
32 
33 	/* backlog queue */
34 	struct list_head backlog;
35 	unsigned int bl_len;
36 	spinlock_t bl_lock; /* protect backlog queue */
37 
38 #ifdef COMP_IN_WQ
39 	struct workqueue_struct *workq;
40 	struct delayed_work compwork;
41 #else
42 	struct tasklet_struct comptask;
43 #endif
44 	bool is_runtime_suspended;
45 };
46 
47 struct cc_bl_item {
48 	struct cc_crypto_req creq;
49 	struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN];
50 	unsigned int len;
51 	struct list_head list;
52 	bool notif;
53 };
54 
55 static void comp_handler(unsigned long devarg);
56 #ifdef COMP_IN_WQ
57 static void comp_work_handler(struct work_struct *work);
58 #endif
59 
60 void cc_req_mgr_fini(struct cc_drvdata *drvdata)
61 {
62 	struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
63 	struct device *dev = drvdata_to_dev(drvdata);
64 
65 	if (!req_mgr_h)
66 		return; /* Not allocated */
67 
68 	if (req_mgr_h->dummy_comp_buff_dma) {
69 		dma_free_coherent(dev, sizeof(u32), req_mgr_h->dummy_comp_buff,
70 				  req_mgr_h->dummy_comp_buff_dma);
71 	}
72 
73 	dev_dbg(dev, "max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size -
74 						req_mgr_h->min_free_hw_slots));
75 	dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots);
76 
77 #ifdef COMP_IN_WQ
78 	flush_workqueue(req_mgr_h->workq);
79 	destroy_workqueue(req_mgr_h->workq);
80 #else
81 	/* Kill tasklet */
82 	tasklet_kill(&req_mgr_h->comptask);
83 #endif
84 	kzfree(req_mgr_h);
85 	drvdata->request_mgr_handle = NULL;
86 }
87 
88 int cc_req_mgr_init(struct cc_drvdata *drvdata)
89 {
90 	struct cc_req_mgr_handle *req_mgr_h;
91 	struct device *dev = drvdata_to_dev(drvdata);
92 	int rc = 0;
93 
94 	req_mgr_h = kzalloc(sizeof(*req_mgr_h), GFP_KERNEL);
95 	if (!req_mgr_h) {
96 		rc = -ENOMEM;
97 		goto req_mgr_init_err;
98 	}
99 
100 	drvdata->request_mgr_handle = req_mgr_h;
101 
102 	spin_lock_init(&req_mgr_h->hw_lock);
103 	spin_lock_init(&req_mgr_h->bl_lock);
104 	INIT_LIST_HEAD(&req_mgr_h->backlog);
105 
106 #ifdef COMP_IN_WQ
107 	dev_dbg(dev, "Initializing completion workqueue\n");
108 	req_mgr_h->workq = create_singlethread_workqueue("ccree");
109 	if (!req_mgr_h->workq) {
110 		dev_err(dev, "Failed creating work queue\n");
111 		rc = -ENOMEM;
112 		goto req_mgr_init_err;
113 	}
114 	INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler);
115 #else
116 	dev_dbg(dev, "Initializing completion tasklet\n");
117 	tasklet_init(&req_mgr_h->comptask, comp_handler,
118 		     (unsigned long)drvdata);
119 #endif
120 	req_mgr_h->hw_queue_size = cc_ioread(drvdata,
121 					     CC_REG(DSCRPTR_QUEUE_SRAM_SIZE));
122 	dev_dbg(dev, "hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size);
123 	if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) {
124 		dev_err(dev, "Invalid HW queue size = %u (Min. required is %u)\n",
125 			req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE);
126 		rc = -ENOMEM;
127 		goto req_mgr_init_err;
128 	}
129 	req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size;
130 	req_mgr_h->max_used_sw_slots = 0;
131 
132 	/* Allocate DMA word for "dummy" completion descriptor use */
133 	req_mgr_h->dummy_comp_buff =
134 		dma_alloc_coherent(dev, sizeof(u32),
135 				   &req_mgr_h->dummy_comp_buff_dma,
136 				   GFP_KERNEL);
137 	if (!req_mgr_h->dummy_comp_buff) {
138 		dev_err(dev, "Not enough memory to allocate DMA (%zu) dropped buffer\n",
139 			sizeof(u32));
140 		rc = -ENOMEM;
141 		goto req_mgr_init_err;
142 	}
143 
144 	/* Init. "dummy" completion descriptor */
145 	hw_desc_init(&req_mgr_h->compl_desc);
146 	set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32));
147 	set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma,
148 		      sizeof(u32), NS_BIT, 1);
149 	set_flow_mode(&req_mgr_h->compl_desc, BYPASS);
150 	set_queue_last_ind(drvdata, &req_mgr_h->compl_desc);
151 
152 	return 0;
153 
154 req_mgr_init_err:
155 	cc_req_mgr_fini(drvdata);
156 	return rc;
157 }
158 
159 static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[],
160 			unsigned int seq_len)
161 {
162 	int i, w;
163 	void __iomem *reg = drvdata->cc_base + CC_REG(DSCRPTR_QUEUE_WORD0);
164 	struct device *dev = drvdata_to_dev(drvdata);
165 
166 	/*
167 	 * We do indeed write all 6 command words to the same
168 	 * register. The HW supports this.
169 	 */
170 
171 	for (i = 0; i < seq_len; i++) {
172 		for (w = 0; w <= 5; w++)
173 			writel_relaxed(seq[i].word[w], reg);
174 
175 		if (cc_dump_desc)
176 			dev_dbg(dev, "desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
177 				i, seq[i].word[0], seq[i].word[1],
178 				seq[i].word[2], seq[i].word[3],
179 				seq[i].word[4], seq[i].word[5]);
180 	}
181 }
182 
183 /*!
184  * Completion will take place if and only if user requested completion
185  * by cc_send_sync_request().
186  *
187  * \param dev
188  * \param dx_compl_h The completion event to signal
189  */
190 static void request_mgr_complete(struct device *dev, void *dx_compl_h,
191 				 int dummy)
192 {
193 	struct completion *this_compl = dx_compl_h;
194 
195 	complete(this_compl);
196 }
197 
198 static int cc_queues_status(struct cc_drvdata *drvdata,
199 			    struct cc_req_mgr_handle *req_mgr_h,
200 			    unsigned int total_seq_len)
201 {
202 	unsigned long poll_queue;
203 	struct device *dev = drvdata_to_dev(drvdata);
204 
205 	/* SW queue is checked only once as it will not
206 	 * be chaned during the poll because the spinlock_bh
207 	 * is held by the thread
208 	 */
209 	if (((req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1)) ==
210 	    req_mgr_h->req_queue_tail) {
211 		dev_err(dev, "SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n",
212 			req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE);
213 		return -ENOSPC;
214 	}
215 
216 	if (req_mgr_h->q_free_slots >= total_seq_len)
217 		return 0;
218 
219 	/* Wait for space in HW queue. Poll constant num of iterations. */
220 	for (poll_queue = 0; poll_queue < CC_MAX_POLL_ITER ; poll_queue++) {
221 		req_mgr_h->q_free_slots =
222 			cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
223 		if (req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots)
224 			req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots;
225 
226 		if (req_mgr_h->q_free_slots >= total_seq_len) {
227 			/* If there is enough place return */
228 			return 0;
229 		}
230 
231 		dev_dbg(dev, "HW FIFO is full. q_free_slots=%d total_seq_len=%d\n",
232 			req_mgr_h->q_free_slots, total_seq_len);
233 	}
234 	/* No room in the HW queue try again later */
235 	dev_dbg(dev, "HW FIFO full, timeout. req_queue_head=%d sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n",
236 		req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE,
237 		req_mgr_h->q_free_slots, total_seq_len);
238 	return -ENOSPC;
239 }
240 
241 /*!
242  * Enqueue caller request to crypto hardware.
243  * Need to be called with HW lock held and PM running
244  *
245  * \param drvdata
246  * \param cc_req The request to enqueue
247  * \param desc The crypto sequence
248  * \param len The crypto sequence length
249  * \param add_comp If "true": add an artificial dout DMA to mark completion
250  *
251  * \return int Returns -EINPROGRESS or error code
252  */
253 static int cc_do_send_request(struct cc_drvdata *drvdata,
254 			      struct cc_crypto_req *cc_req,
255 			      struct cc_hw_desc *desc, unsigned int len,
256 				bool add_comp, bool ivgen)
257 {
258 	struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
259 	unsigned int used_sw_slots;
260 	unsigned int iv_seq_len = 0;
261 	unsigned int total_seq_len = len; /*initial sequence length*/
262 	struct cc_hw_desc iv_seq[CC_IVPOOL_SEQ_LEN];
263 	struct device *dev = drvdata_to_dev(drvdata);
264 	int rc;
265 
266 	if (ivgen) {
267 		dev_dbg(dev, "Acquire IV from pool into %d DMA addresses %pad, %pad, %pad, IV-size=%u\n",
268 			cc_req->ivgen_dma_addr_len,
269 			&cc_req->ivgen_dma_addr[0],
270 			&cc_req->ivgen_dma_addr[1],
271 			&cc_req->ivgen_dma_addr[2],
272 			cc_req->ivgen_size);
273 
274 		/* Acquire IV from pool */
275 		rc = cc_get_iv(drvdata, cc_req->ivgen_dma_addr,
276 			       cc_req->ivgen_dma_addr_len,
277 			       cc_req->ivgen_size, iv_seq, &iv_seq_len);
278 
279 		if (rc) {
280 			dev_err(dev, "Failed to generate IV (rc=%d)\n", rc);
281 			return rc;
282 		}
283 
284 		total_seq_len += iv_seq_len;
285 	}
286 
287 	used_sw_slots = ((req_mgr_h->req_queue_head -
288 			  req_mgr_h->req_queue_tail) &
289 			 (MAX_REQUEST_QUEUE_SIZE - 1));
290 	if (used_sw_slots > req_mgr_h->max_used_sw_slots)
291 		req_mgr_h->max_used_sw_slots = used_sw_slots;
292 
293 	/* Enqueue request - must be locked with HW lock*/
294 	req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *cc_req;
295 	req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) &
296 				    (MAX_REQUEST_QUEUE_SIZE - 1);
297 	/* TODO: Use circ_buf.h ? */
298 
299 	dev_dbg(dev, "Enqueue request head=%u\n", req_mgr_h->req_queue_head);
300 
301 	/*
302 	 * We are about to push command to the HW via the command registers
303 	 * that may refernece hsot memory. We need to issue a memory barrier
304 	 * to make sure there are no outstnading memory writes
305 	 */
306 	wmb();
307 
308 	/* STAT_PHASE_4: Push sequence */
309 	if (ivgen)
310 		enqueue_seq(drvdata, iv_seq, iv_seq_len);
311 
312 	enqueue_seq(drvdata, desc, len);
313 
314 	if (add_comp) {
315 		enqueue_seq(drvdata, &req_mgr_h->compl_desc, 1);
316 		total_seq_len++;
317 	}
318 
319 	if (req_mgr_h->q_free_slots < total_seq_len) {
320 		/* This situation should never occur. Maybe indicating problem
321 		 * with resuming power. Set the free slot count to 0 and hope
322 		 * for the best.
323 		 */
324 		dev_err(dev, "HW free slot count mismatch.");
325 		req_mgr_h->q_free_slots = 0;
326 	} else {
327 		/* Update the free slots in HW queue */
328 		req_mgr_h->q_free_slots -= total_seq_len;
329 	}
330 
331 	/* Operation still in process */
332 	return -EINPROGRESS;
333 }
334 
335 static void cc_enqueue_backlog(struct cc_drvdata *drvdata,
336 			       struct cc_bl_item *bli)
337 {
338 	struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
339 
340 	spin_lock_bh(&mgr->bl_lock);
341 	list_add_tail(&bli->list, &mgr->backlog);
342 	++mgr->bl_len;
343 	spin_unlock_bh(&mgr->bl_lock);
344 	tasklet_schedule(&mgr->comptask);
345 }
346 
347 static void cc_proc_backlog(struct cc_drvdata *drvdata)
348 {
349 	struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
350 	struct cc_bl_item *bli;
351 	struct cc_crypto_req *creq;
352 	struct crypto_async_request *req;
353 	bool ivgen;
354 	unsigned int total_len;
355 	struct device *dev = drvdata_to_dev(drvdata);
356 	int rc;
357 
358 	spin_lock(&mgr->bl_lock);
359 
360 	while (mgr->bl_len) {
361 		bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list);
362 		spin_unlock(&mgr->bl_lock);
363 
364 		creq = &bli->creq;
365 		req = (struct crypto_async_request *)creq->user_arg;
366 
367 		/*
368 		 * Notify the request we're moving out of the backlog
369 		 * but only if we haven't done so already.
370 		 */
371 		if (!bli->notif) {
372 			req->complete(req, -EINPROGRESS);
373 			bli->notif = true;
374 		}
375 
376 		ivgen = !!creq->ivgen_dma_addr_len;
377 		total_len = bli->len + (ivgen ? CC_IVPOOL_SEQ_LEN : 0);
378 
379 		spin_lock(&mgr->hw_lock);
380 
381 		rc = cc_queues_status(drvdata, mgr, total_len);
382 		if (rc) {
383 			/*
384 			 * There is still not room in the FIFO for
385 			 * this request. Bail out. We'll return here
386 			 * on the next completion irq.
387 			 */
388 			spin_unlock(&mgr->hw_lock);
389 			return;
390 		}
391 
392 		rc = cc_do_send_request(drvdata, &bli->creq, bli->desc,
393 					bli->len, false, ivgen);
394 
395 		spin_unlock(&mgr->hw_lock);
396 
397 		if (rc != -EINPROGRESS) {
398 			cc_pm_put_suspend(dev);
399 			creq->user_cb(dev, req, rc);
400 		}
401 
402 		/* Remove ourselves from the backlog list */
403 		spin_lock(&mgr->bl_lock);
404 		list_del(&bli->list);
405 		--mgr->bl_len;
406 	}
407 
408 	spin_unlock(&mgr->bl_lock);
409 }
410 
411 int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,
412 		    struct cc_hw_desc *desc, unsigned int len,
413 		    struct crypto_async_request *req)
414 {
415 	int rc;
416 	struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
417 	bool ivgen = !!cc_req->ivgen_dma_addr_len;
418 	unsigned int total_len = len + (ivgen ? CC_IVPOOL_SEQ_LEN : 0);
419 	struct device *dev = drvdata_to_dev(drvdata);
420 	bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
421 	gfp_t flags = cc_gfp_flags(req);
422 	struct cc_bl_item *bli;
423 
424 	rc = cc_pm_get(dev);
425 	if (rc) {
426 		dev_err(dev, "ssi_power_mgr_runtime_get returned %x\n", rc);
427 		return rc;
428 	}
429 
430 	spin_lock_bh(&mgr->hw_lock);
431 	rc = cc_queues_status(drvdata, mgr, total_len);
432 
433 #ifdef CC_DEBUG_FORCE_BACKLOG
434 	if (backlog_ok)
435 		rc = -ENOSPC;
436 #endif /* CC_DEBUG_FORCE_BACKLOG */
437 
438 	if (rc == -ENOSPC && backlog_ok) {
439 		spin_unlock_bh(&mgr->hw_lock);
440 
441 		bli = kmalloc(sizeof(*bli), flags);
442 		if (!bli) {
443 			cc_pm_put_suspend(dev);
444 			return -ENOMEM;
445 		}
446 
447 		memcpy(&bli->creq, cc_req, sizeof(*cc_req));
448 		memcpy(&bli->desc, desc, len * sizeof(*desc));
449 		bli->len = len;
450 		bli->notif = false;
451 		cc_enqueue_backlog(drvdata, bli);
452 		return -EBUSY;
453 	}
454 
455 	if (!rc)
456 		rc = cc_do_send_request(drvdata, cc_req, desc, len, false,
457 					ivgen);
458 
459 	spin_unlock_bh(&mgr->hw_lock);
460 	return rc;
461 }
462 
463 int cc_send_sync_request(struct cc_drvdata *drvdata,
464 			 struct cc_crypto_req *cc_req, struct cc_hw_desc *desc,
465 			 unsigned int len)
466 {
467 	int rc;
468 	struct device *dev = drvdata_to_dev(drvdata);
469 	struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
470 
471 	init_completion(&cc_req->seq_compl);
472 	cc_req->user_cb = request_mgr_complete;
473 	cc_req->user_arg = &cc_req->seq_compl;
474 
475 	rc = cc_pm_get(dev);
476 	if (rc) {
477 		dev_err(dev, "ssi_power_mgr_runtime_get returned %x\n", rc);
478 		return rc;
479 	}
480 
481 	while (true) {
482 		spin_lock_bh(&mgr->hw_lock);
483 		rc = cc_queues_status(drvdata, mgr, len + 1);
484 
485 		if (!rc)
486 			break;
487 
488 		spin_unlock_bh(&mgr->hw_lock);
489 		if (rc != -EAGAIN) {
490 			cc_pm_put_suspend(dev);
491 			return rc;
492 		}
493 		wait_for_completion_interruptible(&drvdata->hw_queue_avail);
494 		reinit_completion(&drvdata->hw_queue_avail);
495 	}
496 
497 	rc = cc_do_send_request(drvdata, cc_req, desc, len, true, false);
498 	spin_unlock_bh(&mgr->hw_lock);
499 
500 	if (rc != -EINPROGRESS) {
501 		cc_pm_put_suspend(dev);
502 		return rc;
503 	}
504 
505 	wait_for_completion(&cc_req->seq_compl);
506 	return 0;
507 }
508 
509 /*!
510  * Enqueue caller request to crypto hardware during init process.
511  * assume this function is not called in middle of a flow,
512  * since we set QUEUE_LAST_IND flag in the last descriptor.
513  *
514  * \param drvdata
515  * \param desc The crypto sequence
516  * \param len The crypto sequence length
517  *
518  * \return int Returns "0" upon success
519  */
520 int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc,
521 		      unsigned int len)
522 {
523 	struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
524 	unsigned int total_seq_len = len; /*initial sequence length*/
525 	int rc = 0;
526 
527 	/* Wait for space in HW and SW FIFO. Poll for as much as FIFO_TIMEOUT.
528 	 */
529 	rc = cc_queues_status(drvdata, req_mgr_h, total_seq_len);
530 	if (rc)
531 		return rc;
532 
533 	set_queue_last_ind(drvdata, &desc[(len - 1)]);
534 
535 	/*
536 	 * We are about to push command to the HW via the command registers
537 	 * that may refernece hsot memory. We need to issue a memory barrier
538 	 * to make sure there are no outstnading memory writes
539 	 */
540 	wmb();
541 	enqueue_seq(drvdata, desc, len);
542 
543 	/* Update the free slots in HW queue */
544 	req_mgr_h->q_free_slots =
545 		cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
546 
547 	return 0;
548 }
549 
550 void complete_request(struct cc_drvdata *drvdata)
551 {
552 	struct cc_req_mgr_handle *request_mgr_handle =
553 						drvdata->request_mgr_handle;
554 
555 	complete(&drvdata->hw_queue_avail);
556 #ifdef COMP_IN_WQ
557 	queue_delayed_work(request_mgr_handle->workq,
558 			   &request_mgr_handle->compwork, 0);
559 #else
560 	tasklet_schedule(&request_mgr_handle->comptask);
561 #endif
562 }
563 
564 #ifdef COMP_IN_WQ
565 static void comp_work_handler(struct work_struct *work)
566 {
567 	struct cc_drvdata *drvdata =
568 		container_of(work, struct cc_drvdata, compwork.work);
569 
570 	comp_handler((unsigned long)drvdata);
571 }
572 #endif
573 
574 static void proc_completions(struct cc_drvdata *drvdata)
575 {
576 	struct cc_crypto_req *cc_req;
577 	struct device *dev = drvdata_to_dev(drvdata);
578 	struct cc_req_mgr_handle *request_mgr_handle =
579 						drvdata->request_mgr_handle;
580 	unsigned int *tail = &request_mgr_handle->req_queue_tail;
581 	unsigned int *head = &request_mgr_handle->req_queue_head;
582 
583 	while (request_mgr_handle->axi_completed) {
584 		request_mgr_handle->axi_completed--;
585 
586 		/* Dequeue request */
587 		if (*head == *tail) {
588 			/* We are supposed to handle a completion but our
589 			 * queue is empty. This is not normal. Return and
590 			 * hope for the best.
591 			 */
592 			dev_err(dev, "Request queue is empty head == tail %u\n",
593 				*head);
594 			break;
595 		}
596 
597 		cc_req = &request_mgr_handle->req_queue[*tail];
598 
599 		if (cc_req->user_cb)
600 			cc_req->user_cb(dev, cc_req->user_arg, 0);
601 		*tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1);
602 		dev_dbg(dev, "Dequeue request tail=%u\n", *tail);
603 		dev_dbg(dev, "Request completed. axi_completed=%d\n",
604 			request_mgr_handle->axi_completed);
605 		cc_pm_put_suspend(dev);
606 	}
607 }
608 
609 static inline u32 cc_axi_comp_count(struct cc_drvdata *drvdata)
610 {
611 	return FIELD_GET(AXIM_MON_COMP_VALUE,
612 			 cc_ioread(drvdata, drvdata->axim_mon_offset));
613 }
614 
615 /* Deferred service handler, run as interrupt-fired tasklet */
616 static void comp_handler(unsigned long devarg)
617 {
618 	struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg;
619 	struct cc_req_mgr_handle *request_mgr_handle =
620 						drvdata->request_mgr_handle;
621 
622 	u32 irq;
623 
624 	irq = (drvdata->irq & CC_COMP_IRQ_MASK);
625 
626 	if (irq & CC_COMP_IRQ_MASK) {
627 		/* To avoid the interrupt from firing as we unmask it,
628 		 * we clear it now
629 		 */
630 		cc_iowrite(drvdata, CC_REG(HOST_ICR), CC_COMP_IRQ_MASK);
631 
632 		/* Avoid race with above clear: Test completion counter
633 		 * once more
634 		 */
635 		request_mgr_handle->axi_completed +=
636 				cc_axi_comp_count(drvdata);
637 
638 		while (request_mgr_handle->axi_completed) {
639 			do {
640 				proc_completions(drvdata);
641 				/* At this point (after proc_completions()),
642 				 * request_mgr_handle->axi_completed is 0.
643 				 */
644 				request_mgr_handle->axi_completed =
645 						cc_axi_comp_count(drvdata);
646 			} while (request_mgr_handle->axi_completed > 0);
647 
648 			cc_iowrite(drvdata, CC_REG(HOST_ICR),
649 				   CC_COMP_IRQ_MASK);
650 
651 			request_mgr_handle->axi_completed +=
652 					cc_axi_comp_count(drvdata);
653 		}
654 	}
655 	/* after verifing that there is nothing to do,
656 	 * unmask AXI completion interrupt
657 	 */
658 	cc_iowrite(drvdata, CC_REG(HOST_IMR),
659 		   cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~irq);
660 
661 	cc_proc_backlog(drvdata);
662 }
663 
664 /*
665  * resume the queue configuration - no need to take the lock as this happens
666  * inside the spin lock protection
667  */
668 #if defined(CONFIG_PM)
669 int cc_resume_req_queue(struct cc_drvdata *drvdata)
670 {
671 	struct cc_req_mgr_handle *request_mgr_handle =
672 		drvdata->request_mgr_handle;
673 
674 	spin_lock_bh(&request_mgr_handle->hw_lock);
675 	request_mgr_handle->is_runtime_suspended = false;
676 	spin_unlock_bh(&request_mgr_handle->hw_lock);
677 
678 	return 0;
679 }
680 
681 /*
682  * suspend the queue configuration. Since it is used for the runtime suspend
683  * only verify that the queue can be suspended.
684  */
685 int cc_suspend_req_queue(struct cc_drvdata *drvdata)
686 {
687 	struct cc_req_mgr_handle *request_mgr_handle =
688 						drvdata->request_mgr_handle;
689 
690 	/* lock the send_request */
691 	spin_lock_bh(&request_mgr_handle->hw_lock);
692 	if (request_mgr_handle->req_queue_head !=
693 	    request_mgr_handle->req_queue_tail) {
694 		spin_unlock_bh(&request_mgr_handle->hw_lock);
695 		return -EBUSY;
696 	}
697 	request_mgr_handle->is_runtime_suspended = true;
698 	spin_unlock_bh(&request_mgr_handle->hw_lock);
699 
700 	return 0;
701 }
702 
703 bool cc_req_queue_suspended(struct cc_drvdata *drvdata)
704 {
705 	struct cc_req_mgr_handle *request_mgr_handle =
706 						drvdata->request_mgr_handle;
707 
708 	return	request_mgr_handle->is_runtime_suspended;
709 }
710 
711 #endif
712