1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 3 4 #include <linux/kernel.h> 5 #include <linux/nospec.h> 6 #include "cc_driver.h" 7 #include "cc_buffer_mgr.h" 8 #include "cc_request_mgr.h" 9 #include "cc_pm.h" 10 11 #define CC_MAX_POLL_ITER 10 12 /* The highest descriptor count in used */ 13 #define CC_MAX_DESC_SEQ_LEN 23 14 15 struct cc_req_mgr_handle { 16 /* Request manager resources */ 17 unsigned int hw_queue_size; /* HW capability */ 18 unsigned int min_free_hw_slots; 19 unsigned int max_used_sw_slots; 20 struct cc_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE]; 21 u32 req_queue_head; 22 u32 req_queue_tail; 23 u32 axi_completed; 24 u32 q_free_slots; 25 /* This lock protects access to HW register 26 * that must be single request at a time 27 */ 28 spinlock_t hw_lock; 29 struct cc_hw_desc compl_desc; 30 u8 *dummy_comp_buff; 31 dma_addr_t dummy_comp_buff_dma; 32 33 /* backlog queue */ 34 struct list_head backlog; 35 unsigned int bl_len; 36 spinlock_t bl_lock; /* protect backlog queue */ 37 38 #ifdef COMP_IN_WQ 39 struct workqueue_struct *workq; 40 struct delayed_work compwork; 41 #else 42 struct tasklet_struct comptask; 43 #endif 44 }; 45 46 struct cc_bl_item { 47 struct cc_crypto_req creq; 48 struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN]; 49 unsigned int len; 50 struct list_head list; 51 bool notif; 52 }; 53 54 static const u32 cc_cpp_int_masks[CC_CPP_NUM_ALGS][CC_CPP_NUM_SLOTS] = { 55 { BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT), 56 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT), 57 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT), 58 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT), 59 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT), 60 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT), 61 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT), 62 BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT) }, 63 { BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT), 64 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT), 65 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT), 66 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT), 67 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT), 68 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT), 69 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT), 70 BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT) } 71 }; 72 73 static void comp_handler(unsigned long devarg); 74 #ifdef COMP_IN_WQ 75 static void comp_work_handler(struct work_struct *work); 76 #endif 77 78 static inline u32 cc_cpp_int_mask(enum cc_cpp_alg alg, int slot) 79 { 80 alg = array_index_nospec(alg, CC_CPP_NUM_ALGS); 81 slot = array_index_nospec(slot, CC_CPP_NUM_SLOTS); 82 83 return cc_cpp_int_masks[alg][slot]; 84 } 85 86 void cc_req_mgr_fini(struct cc_drvdata *drvdata) 87 { 88 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; 89 struct device *dev = drvdata_to_dev(drvdata); 90 91 if (!req_mgr_h) 92 return; /* Not allocated */ 93 94 if (req_mgr_h->dummy_comp_buff_dma) { 95 dma_free_coherent(dev, sizeof(u32), req_mgr_h->dummy_comp_buff, 96 req_mgr_h->dummy_comp_buff_dma); 97 } 98 99 dev_dbg(dev, "max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size - 100 req_mgr_h->min_free_hw_slots)); 101 dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots); 102 103 #ifdef COMP_IN_WQ 104 flush_workqueue(req_mgr_h->workq); 105 destroy_workqueue(req_mgr_h->workq); 106 #else 107 /* Kill tasklet */ 108 tasklet_kill(&req_mgr_h->comptask); 109 #endif 110 kfree_sensitive(req_mgr_h); 111 drvdata->request_mgr_handle = NULL; 112 } 113 114 int cc_req_mgr_init(struct cc_drvdata *drvdata) 115 { 116 struct cc_req_mgr_handle *req_mgr_h; 117 struct device *dev = drvdata_to_dev(drvdata); 118 int rc = 0; 119 120 req_mgr_h = kzalloc(sizeof(*req_mgr_h), GFP_KERNEL); 121 if (!req_mgr_h) { 122 rc = -ENOMEM; 123 goto req_mgr_init_err; 124 } 125 126 drvdata->request_mgr_handle = req_mgr_h; 127 128 spin_lock_init(&req_mgr_h->hw_lock); 129 spin_lock_init(&req_mgr_h->bl_lock); 130 INIT_LIST_HEAD(&req_mgr_h->backlog); 131 132 #ifdef COMP_IN_WQ 133 dev_dbg(dev, "Initializing completion workqueue\n"); 134 req_mgr_h->workq = create_singlethread_workqueue("ccree"); 135 if (!req_mgr_h->workq) { 136 dev_err(dev, "Failed creating work queue\n"); 137 rc = -ENOMEM; 138 goto req_mgr_init_err; 139 } 140 INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler); 141 #else 142 dev_dbg(dev, "Initializing completion tasklet\n"); 143 tasklet_init(&req_mgr_h->comptask, comp_handler, 144 (unsigned long)drvdata); 145 #endif 146 req_mgr_h->hw_queue_size = cc_ioread(drvdata, 147 CC_REG(DSCRPTR_QUEUE_SRAM_SIZE)); 148 dev_dbg(dev, "hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size); 149 if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) { 150 dev_err(dev, "Invalid HW queue size = %u (Min. required is %u)\n", 151 req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE); 152 rc = -ENOMEM; 153 goto req_mgr_init_err; 154 } 155 req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size; 156 req_mgr_h->max_used_sw_slots = 0; 157 158 /* Allocate DMA word for "dummy" completion descriptor use */ 159 req_mgr_h->dummy_comp_buff = 160 dma_alloc_coherent(dev, sizeof(u32), 161 &req_mgr_h->dummy_comp_buff_dma, 162 GFP_KERNEL); 163 if (!req_mgr_h->dummy_comp_buff) { 164 dev_err(dev, "Not enough memory to allocate DMA (%zu) dropped buffer\n", 165 sizeof(u32)); 166 rc = -ENOMEM; 167 goto req_mgr_init_err; 168 } 169 170 /* Init. "dummy" completion descriptor */ 171 hw_desc_init(&req_mgr_h->compl_desc); 172 set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32)); 173 set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma, 174 sizeof(u32), NS_BIT, 1); 175 set_flow_mode(&req_mgr_h->compl_desc, BYPASS); 176 set_queue_last_ind(drvdata, &req_mgr_h->compl_desc); 177 178 return 0; 179 180 req_mgr_init_err: 181 cc_req_mgr_fini(drvdata); 182 return rc; 183 } 184 185 static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[], 186 unsigned int seq_len) 187 { 188 int i, w; 189 void __iomem *reg = drvdata->cc_base + CC_REG(DSCRPTR_QUEUE_WORD0); 190 struct device *dev = drvdata_to_dev(drvdata); 191 192 /* 193 * We do indeed write all 6 command words to the same 194 * register. The HW supports this. 195 */ 196 197 for (i = 0; i < seq_len; i++) { 198 for (w = 0; w <= 5; w++) 199 writel_relaxed(seq[i].word[w], reg); 200 201 if (cc_dump_desc) 202 dev_dbg(dev, "desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n", 203 i, seq[i].word[0], seq[i].word[1], 204 seq[i].word[2], seq[i].word[3], 205 seq[i].word[4], seq[i].word[5]); 206 } 207 } 208 209 /** 210 * request_mgr_complete() - Completion will take place if and only if user 211 * requested completion by cc_send_sync_request(). 212 * 213 * @dev: Device pointer 214 * @dx_compl_h: The completion event to signal 215 * @dummy: unused error code 216 */ 217 static void request_mgr_complete(struct device *dev, void *dx_compl_h, 218 int dummy) 219 { 220 struct completion *this_compl = dx_compl_h; 221 222 complete(this_compl); 223 } 224 225 static int cc_queues_status(struct cc_drvdata *drvdata, 226 struct cc_req_mgr_handle *req_mgr_h, 227 unsigned int total_seq_len) 228 { 229 unsigned long poll_queue; 230 struct device *dev = drvdata_to_dev(drvdata); 231 232 /* SW queue is checked only once as it will not 233 * be changed during the poll because the spinlock_bh 234 * is held by the thread 235 */ 236 if (((req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1)) == 237 req_mgr_h->req_queue_tail) { 238 dev_err(dev, "SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n", 239 req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE); 240 return -ENOSPC; 241 } 242 243 if (req_mgr_h->q_free_slots >= total_seq_len) 244 return 0; 245 246 /* Wait for space in HW queue. Poll constant num of iterations. */ 247 for (poll_queue = 0; poll_queue < CC_MAX_POLL_ITER ; poll_queue++) { 248 req_mgr_h->q_free_slots = 249 cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT)); 250 if (req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots) 251 req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots; 252 253 if (req_mgr_h->q_free_slots >= total_seq_len) { 254 /* If there is enough place return */ 255 return 0; 256 } 257 258 dev_dbg(dev, "HW FIFO is full. q_free_slots=%d total_seq_len=%d\n", 259 req_mgr_h->q_free_slots, total_seq_len); 260 } 261 /* No room in the HW queue try again later */ 262 dev_dbg(dev, "HW FIFO full, timeout. req_queue_head=%d sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n", 263 req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE, 264 req_mgr_h->q_free_slots, total_seq_len); 265 return -ENOSPC; 266 } 267 268 /** 269 * cc_do_send_request() - Enqueue caller request to crypto hardware. 270 * Need to be called with HW lock held and PM running 271 * 272 * @drvdata: Associated device driver context 273 * @cc_req: The request to enqueue 274 * @desc: The crypto sequence 275 * @len: The crypto sequence length 276 * @add_comp: If "true": add an artificial dout DMA to mark completion 277 * 278 */ 279 static void cc_do_send_request(struct cc_drvdata *drvdata, 280 struct cc_crypto_req *cc_req, 281 struct cc_hw_desc *desc, unsigned int len, 282 bool add_comp) 283 { 284 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; 285 unsigned int used_sw_slots; 286 unsigned int total_seq_len = len; /*initial sequence length*/ 287 struct device *dev = drvdata_to_dev(drvdata); 288 289 used_sw_slots = ((req_mgr_h->req_queue_head - 290 req_mgr_h->req_queue_tail) & 291 (MAX_REQUEST_QUEUE_SIZE - 1)); 292 if (used_sw_slots > req_mgr_h->max_used_sw_slots) 293 req_mgr_h->max_used_sw_slots = used_sw_slots; 294 295 /* Enqueue request - must be locked with HW lock*/ 296 req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *cc_req; 297 req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) & 298 (MAX_REQUEST_QUEUE_SIZE - 1); 299 300 dev_dbg(dev, "Enqueue request head=%u\n", req_mgr_h->req_queue_head); 301 302 /* 303 * We are about to push command to the HW via the command registers 304 * that may reference host memory. We need to issue a memory barrier 305 * to make sure there are no outstanding memory writes 306 */ 307 wmb(); 308 309 /* STAT_PHASE_4: Push sequence */ 310 311 enqueue_seq(drvdata, desc, len); 312 313 if (add_comp) { 314 enqueue_seq(drvdata, &req_mgr_h->compl_desc, 1); 315 total_seq_len++; 316 } 317 318 if (req_mgr_h->q_free_slots < total_seq_len) { 319 /* This situation should never occur. Maybe indicating problem 320 * with resuming power. Set the free slot count to 0 and hope 321 * for the best. 322 */ 323 dev_err(dev, "HW free slot count mismatch."); 324 req_mgr_h->q_free_slots = 0; 325 } else { 326 /* Update the free slots in HW queue */ 327 req_mgr_h->q_free_slots -= total_seq_len; 328 } 329 } 330 331 static void cc_enqueue_backlog(struct cc_drvdata *drvdata, 332 struct cc_bl_item *bli) 333 { 334 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 335 struct device *dev = drvdata_to_dev(drvdata); 336 337 spin_lock_bh(&mgr->bl_lock); 338 list_add_tail(&bli->list, &mgr->backlog); 339 ++mgr->bl_len; 340 dev_dbg(dev, "+++bl len: %d\n", mgr->bl_len); 341 spin_unlock_bh(&mgr->bl_lock); 342 tasklet_schedule(&mgr->comptask); 343 } 344 345 static void cc_proc_backlog(struct cc_drvdata *drvdata) 346 { 347 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 348 struct cc_bl_item *bli; 349 struct cc_crypto_req *creq; 350 void *req; 351 struct device *dev = drvdata_to_dev(drvdata); 352 int rc; 353 354 spin_lock(&mgr->bl_lock); 355 356 while (mgr->bl_len) { 357 bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list); 358 dev_dbg(dev, "---bl len: %d\n", mgr->bl_len); 359 360 spin_unlock(&mgr->bl_lock); 361 362 363 creq = &bli->creq; 364 req = creq->user_arg; 365 366 /* 367 * Notify the request we're moving out of the backlog 368 * but only if we haven't done so already. 369 */ 370 if (!bli->notif) { 371 creq->user_cb(dev, req, -EINPROGRESS); 372 bli->notif = true; 373 } 374 375 spin_lock(&mgr->hw_lock); 376 377 rc = cc_queues_status(drvdata, mgr, bli->len); 378 if (rc) { 379 /* 380 * There is still no room in the FIFO for 381 * this request. Bail out. We'll return here 382 * on the next completion irq. 383 */ 384 spin_unlock(&mgr->hw_lock); 385 return; 386 } 387 388 cc_do_send_request(drvdata, &bli->creq, bli->desc, bli->len, 389 false); 390 spin_unlock(&mgr->hw_lock); 391 392 /* Remove ourselves from the backlog list */ 393 spin_lock(&mgr->bl_lock); 394 list_del(&bli->list); 395 --mgr->bl_len; 396 kfree(bli); 397 } 398 399 spin_unlock(&mgr->bl_lock); 400 } 401 402 int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req, 403 struct cc_hw_desc *desc, unsigned int len, 404 struct crypto_async_request *req) 405 { 406 int rc; 407 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 408 struct device *dev = drvdata_to_dev(drvdata); 409 bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG; 410 gfp_t flags = cc_gfp_flags(req); 411 struct cc_bl_item *bli; 412 413 rc = cc_pm_get(dev); 414 if (rc) { 415 dev_err(dev, "cc_pm_get returned %x\n", rc); 416 return rc; 417 } 418 419 spin_lock_bh(&mgr->hw_lock); 420 rc = cc_queues_status(drvdata, mgr, len); 421 422 #ifdef CC_DEBUG_FORCE_BACKLOG 423 if (backlog_ok) 424 rc = -ENOSPC; 425 #endif /* CC_DEBUG_FORCE_BACKLOG */ 426 427 if (rc == -ENOSPC && backlog_ok) { 428 spin_unlock_bh(&mgr->hw_lock); 429 430 bli = kmalloc(sizeof(*bli), flags); 431 if (!bli) { 432 cc_pm_put_suspend(dev); 433 return -ENOMEM; 434 } 435 436 memcpy(&bli->creq, cc_req, sizeof(*cc_req)); 437 memcpy(&bli->desc, desc, len * sizeof(*desc)); 438 bli->len = len; 439 bli->notif = false; 440 cc_enqueue_backlog(drvdata, bli); 441 return -EBUSY; 442 } 443 444 if (!rc) { 445 cc_do_send_request(drvdata, cc_req, desc, len, false); 446 rc = -EINPROGRESS; 447 } 448 449 spin_unlock_bh(&mgr->hw_lock); 450 return rc; 451 } 452 453 int cc_send_sync_request(struct cc_drvdata *drvdata, 454 struct cc_crypto_req *cc_req, struct cc_hw_desc *desc, 455 unsigned int len) 456 { 457 int rc; 458 struct device *dev = drvdata_to_dev(drvdata); 459 struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle; 460 461 init_completion(&cc_req->seq_compl); 462 cc_req->user_cb = request_mgr_complete; 463 cc_req->user_arg = &cc_req->seq_compl; 464 465 rc = cc_pm_get(dev); 466 if (rc) { 467 dev_err(dev, "cc_pm_get returned %x\n", rc); 468 return rc; 469 } 470 471 while (true) { 472 spin_lock_bh(&mgr->hw_lock); 473 rc = cc_queues_status(drvdata, mgr, len + 1); 474 475 if (!rc) 476 break; 477 478 spin_unlock_bh(&mgr->hw_lock); 479 wait_for_completion_interruptible(&drvdata->hw_queue_avail); 480 reinit_completion(&drvdata->hw_queue_avail); 481 } 482 483 cc_do_send_request(drvdata, cc_req, desc, len, true); 484 spin_unlock_bh(&mgr->hw_lock); 485 wait_for_completion(&cc_req->seq_compl); 486 return 0; 487 } 488 489 /** 490 * send_request_init() - Enqueue caller request to crypto hardware during init 491 * process. 492 * Assume this function is not called in the middle of a flow, 493 * since we set QUEUE_LAST_IND flag in the last descriptor. 494 * 495 * @drvdata: Associated device driver context 496 * @desc: The crypto sequence 497 * @len: The crypto sequence length 498 * 499 * Return: 500 * Returns "0" upon success 501 */ 502 int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc, 503 unsigned int len) 504 { 505 struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle; 506 unsigned int total_seq_len = len; /*initial sequence length*/ 507 int rc = 0; 508 509 /* Wait for space in HW and SW FIFO. Poll for as much as FIFO_TIMEOUT. 510 */ 511 rc = cc_queues_status(drvdata, req_mgr_h, total_seq_len); 512 if (rc) 513 return rc; 514 515 set_queue_last_ind(drvdata, &desc[(len - 1)]); 516 517 /* 518 * We are about to push command to the HW via the command registers 519 * that may reference host memory. We need to issue a memory barrier 520 * to make sure there are no outstanding memory writes 521 */ 522 wmb(); 523 enqueue_seq(drvdata, desc, len); 524 525 /* Update the free slots in HW queue */ 526 req_mgr_h->q_free_slots = 527 cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT)); 528 529 return 0; 530 } 531 532 void complete_request(struct cc_drvdata *drvdata) 533 { 534 struct cc_req_mgr_handle *request_mgr_handle = 535 drvdata->request_mgr_handle; 536 537 complete(&drvdata->hw_queue_avail); 538 #ifdef COMP_IN_WQ 539 queue_delayed_work(request_mgr_handle->workq, 540 &request_mgr_handle->compwork, 0); 541 #else 542 tasklet_schedule(&request_mgr_handle->comptask); 543 #endif 544 } 545 546 #ifdef COMP_IN_WQ 547 static void comp_work_handler(struct work_struct *work) 548 { 549 struct cc_drvdata *drvdata = 550 container_of(work, struct cc_drvdata, compwork.work); 551 552 comp_handler((unsigned long)drvdata); 553 } 554 #endif 555 556 static void proc_completions(struct cc_drvdata *drvdata) 557 { 558 struct cc_crypto_req *cc_req; 559 struct device *dev = drvdata_to_dev(drvdata); 560 struct cc_req_mgr_handle *request_mgr_handle = 561 drvdata->request_mgr_handle; 562 unsigned int *tail = &request_mgr_handle->req_queue_tail; 563 unsigned int *head = &request_mgr_handle->req_queue_head; 564 int rc; 565 u32 mask; 566 567 while (request_mgr_handle->axi_completed) { 568 request_mgr_handle->axi_completed--; 569 570 /* Dequeue request */ 571 if (*head == *tail) { 572 /* We are supposed to handle a completion but our 573 * queue is empty. This is not normal. Return and 574 * hope for the best. 575 */ 576 dev_err(dev, "Request queue is empty head == tail %u\n", 577 *head); 578 break; 579 } 580 581 cc_req = &request_mgr_handle->req_queue[*tail]; 582 583 if (cc_req->cpp.is_cpp) { 584 585 dev_dbg(dev, "CPP request completion slot: %d alg:%d\n", 586 cc_req->cpp.slot, cc_req->cpp.alg); 587 mask = cc_cpp_int_mask(cc_req->cpp.alg, 588 cc_req->cpp.slot); 589 rc = (drvdata->irq & mask ? -EPERM : 0); 590 dev_dbg(dev, "Got mask: %x irq: %x rc: %d\n", mask, 591 drvdata->irq, rc); 592 } else { 593 dev_dbg(dev, "None CPP request completion\n"); 594 rc = 0; 595 } 596 597 if (cc_req->user_cb) 598 cc_req->user_cb(dev, cc_req->user_arg, rc); 599 *tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1); 600 dev_dbg(dev, "Dequeue request tail=%u\n", *tail); 601 dev_dbg(dev, "Request completed. axi_completed=%d\n", 602 request_mgr_handle->axi_completed); 603 cc_pm_put_suspend(dev); 604 } 605 } 606 607 static inline u32 cc_axi_comp_count(struct cc_drvdata *drvdata) 608 { 609 return FIELD_GET(AXIM_MON_COMP_VALUE, 610 cc_ioread(drvdata, drvdata->axim_mon_offset)); 611 } 612 613 /* Deferred service handler, run as interrupt-fired tasklet */ 614 static void comp_handler(unsigned long devarg) 615 { 616 struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg; 617 struct cc_req_mgr_handle *request_mgr_handle = 618 drvdata->request_mgr_handle; 619 struct device *dev = drvdata_to_dev(drvdata); 620 u32 irq; 621 622 dev_dbg(dev, "Completion handler called!\n"); 623 irq = (drvdata->irq & drvdata->comp_mask); 624 625 /* To avoid the interrupt from firing as we unmask it, 626 * we clear it now 627 */ 628 cc_iowrite(drvdata, CC_REG(HOST_ICR), irq); 629 630 /* Avoid race with above clear: Test completion counter once more */ 631 632 request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata); 633 634 dev_dbg(dev, "AXI completion after updated: %d\n", 635 request_mgr_handle->axi_completed); 636 637 while (request_mgr_handle->axi_completed) { 638 do { 639 drvdata->irq |= cc_ioread(drvdata, CC_REG(HOST_IRR)); 640 irq = (drvdata->irq & drvdata->comp_mask); 641 proc_completions(drvdata); 642 643 /* At this point (after proc_completions()), 644 * request_mgr_handle->axi_completed is 0. 645 */ 646 request_mgr_handle->axi_completed += 647 cc_axi_comp_count(drvdata); 648 } while (request_mgr_handle->axi_completed > 0); 649 650 cc_iowrite(drvdata, CC_REG(HOST_ICR), irq); 651 652 request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata); 653 } 654 655 /* after verifying that there is nothing to do, 656 * unmask AXI completion interrupt 657 */ 658 cc_iowrite(drvdata, CC_REG(HOST_IMR), 659 cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~drvdata->comp_mask); 660 661 cc_proc_backlog(drvdata); 662 dev_dbg(dev, "Comp. handler done.\n"); 663 } 664