1 /* 2 * CAAM/SEC 4.x transport/backend driver 3 * JobR backend functionality 4 * 5 * Copyright 2008-2012 Freescale Semiconductor, Inc. 6 */ 7 8 #include "compat.h" 9 #include "regs.h" 10 #include "jr.h" 11 #include "desc.h" 12 #include "intern.h" 13 14 /* Main per-ring interrupt handler */ 15 static irqreturn_t caam_jr_interrupt(int irq, void *st_dev) 16 { 17 struct device *dev = st_dev; 18 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); 19 u32 irqstate; 20 21 /* 22 * Check the output ring for ready responses, kick 23 * tasklet if jobs done. 24 */ 25 irqstate = rd_reg32(&jrp->rregs->jrintstatus); 26 if (!irqstate) 27 return IRQ_NONE; 28 29 /* 30 * If JobR error, we got more development work to do 31 * Flag a bug now, but we really need to shut down and 32 * restart the queue (and fix code). 33 */ 34 if (irqstate & JRINT_JR_ERROR) { 35 dev_err(dev, "job ring error: irqstate: %08x\n", irqstate); 36 BUG(); 37 } 38 39 /* mask valid interrupts */ 40 setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); 41 42 /* Have valid interrupt at this point, just ACK and trigger */ 43 wr_reg32(&jrp->rregs->jrintstatus, irqstate); 44 45 preempt_disable(); 46 tasklet_schedule(&jrp->irqtask); 47 preempt_enable(); 48 49 return IRQ_HANDLED; 50 } 51 52 /* Deferred service handler, run as interrupt-fired tasklet */ 53 static void caam_jr_dequeue(unsigned long devarg) 54 { 55 int hw_idx, sw_idx, i, head, tail; 56 struct device *dev = (struct device *)devarg; 57 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); 58 void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg); 59 u32 *userdesc, userstatus; 60 void *userarg; 61 62 while (rd_reg32(&jrp->rregs->outring_used)) { 63 64 head = ACCESS_ONCE(jrp->head); 65 66 spin_lock(&jrp->outlock); 67 68 sw_idx = tail = jrp->tail; 69 hw_idx = jrp->out_ring_read_index; 70 71 for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) { 72 sw_idx = (tail + i) & (JOBR_DEPTH - 1); 73 74 smp_read_barrier_depends(); 75 76 if (jrp->outring[hw_idx].desc == 77 jrp->entinfo[sw_idx].desc_addr_dma) 78 break; /* found */ 79 } 80 /* we should never fail to find a matching descriptor */ 81 BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0); 82 83 /* Unmap just-run descriptor so we can post-process */ 84 dma_unmap_single(dev, jrp->outring[hw_idx].desc, 85 jrp->entinfo[sw_idx].desc_size, 86 DMA_TO_DEVICE); 87 88 /* mark completed, avoid matching on a recycled desc addr */ 89 jrp->entinfo[sw_idx].desc_addr_dma = 0; 90 91 /* Stash callback params for use outside of lock */ 92 usercall = jrp->entinfo[sw_idx].callbk; 93 userarg = jrp->entinfo[sw_idx].cbkarg; 94 userdesc = jrp->entinfo[sw_idx].desc_addr_virt; 95 userstatus = jrp->outring[hw_idx].jrstatus; 96 97 /* set done */ 98 wr_reg32(&jrp->rregs->outring_rmvd, 1); 99 100 jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) & 101 (JOBR_DEPTH - 1); 102 103 /* 104 * if this job completed out-of-order, do not increment 105 * the tail. Otherwise, increment tail by 1 plus the 106 * number of subsequent jobs already completed out-of-order 107 */ 108 if (sw_idx == tail) { 109 do { 110 tail = (tail + 1) & (JOBR_DEPTH - 1); 111 smp_read_barrier_depends(); 112 } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 && 113 jrp->entinfo[tail].desc_addr_dma == 0); 114 115 jrp->tail = tail; 116 } 117 118 spin_unlock(&jrp->outlock); 119 120 /* Finally, execute user's callback */ 121 usercall(dev, userdesc, userstatus, userarg); 122 } 123 124 /* reenable / unmask IRQs */ 125 clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); 126 } 127 128 /** 129 * caam_jr_register() - Alloc a ring for someone to use as needed. Returns 130 * an ordinal of the rings allocated, else returns -ENODEV if no rings 131 * are available. 132 * @ctrldev: points to the controller level dev (parent) that 133 * owns rings available for use. 134 * @dev: points to where a pointer to the newly allocated queue's 135 * dev can be written to if successful. 136 **/ 137 int caam_jr_register(struct device *ctrldev, struct device **rdev) 138 { 139 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); 140 struct caam_drv_private_jr *jrpriv = NULL; 141 int ring; 142 143 /* Lock, if free ring - assign, unlock */ 144 spin_lock(&ctrlpriv->jr_alloc_lock); 145 for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) { 146 jrpriv = dev_get_drvdata(ctrlpriv->jrdev[ring]); 147 if (jrpriv->assign == JOBR_UNASSIGNED) { 148 jrpriv->assign = JOBR_ASSIGNED; 149 *rdev = ctrlpriv->jrdev[ring]; 150 spin_unlock(&ctrlpriv->jr_alloc_lock); 151 return ring; 152 } 153 } 154 155 /* If assigned, write dev where caller needs it */ 156 spin_unlock(&ctrlpriv->jr_alloc_lock); 157 *rdev = NULL; 158 159 return -ENODEV; 160 } 161 EXPORT_SYMBOL(caam_jr_register); 162 163 /** 164 * caam_jr_deregister() - Deregister an API and release the queue. 165 * Returns 0 if OK, -EBUSY if queue still contains pending entries 166 * or unprocessed results at the time of the call 167 * @dev - points to the dev that identifies the queue to 168 * be released. 169 **/ 170 int caam_jr_deregister(struct device *rdev) 171 { 172 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev); 173 struct caam_drv_private *ctrlpriv; 174 175 /* Get the owning controller's private space */ 176 ctrlpriv = dev_get_drvdata(jrpriv->parentdev); 177 178 /* 179 * Make sure ring empty before release 180 */ 181 if (rd_reg32(&jrpriv->rregs->outring_used) || 182 (rd_reg32(&jrpriv->rregs->inpring_avail) != JOBR_DEPTH)) 183 return -EBUSY; 184 185 /* Release ring */ 186 spin_lock(&ctrlpriv->jr_alloc_lock); 187 jrpriv->assign = JOBR_UNASSIGNED; 188 spin_unlock(&ctrlpriv->jr_alloc_lock); 189 190 return 0; 191 } 192 EXPORT_SYMBOL(caam_jr_deregister); 193 194 /** 195 * caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK, 196 * -EBUSY if the queue is full, -EIO if it cannot map the caller's 197 * descriptor. 198 * @dev: device of the job ring to be used. This device should have 199 * been assigned prior by caam_jr_register(). 200 * @desc: points to a job descriptor that execute our request. All 201 * descriptors (and all referenced data) must be in a DMAable 202 * region, and all data references must be physical addresses 203 * accessible to CAAM (i.e. within a PAMU window granted 204 * to it). 205 * @cbk: pointer to a callback function to be invoked upon completion 206 * of this request. This has the form: 207 * callback(struct device *dev, u32 *desc, u32 stat, void *arg) 208 * where: 209 * @dev: contains the job ring device that processed this 210 * response. 211 * @desc: descriptor that initiated the request, same as 212 * "desc" being argued to caam_jr_enqueue(). 213 * @status: untranslated status received from CAAM. See the 214 * reference manual for a detailed description of 215 * error meaning, or see the JRSTA definitions in the 216 * register header file 217 * @areq: optional pointer to an argument passed with the 218 * original request 219 * @areq: optional pointer to a user argument for use at callback 220 * time. 221 **/ 222 int caam_jr_enqueue(struct device *dev, u32 *desc, 223 void (*cbk)(struct device *dev, u32 *desc, 224 u32 status, void *areq), 225 void *areq) 226 { 227 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); 228 struct caam_jrentry_info *head_entry; 229 int head, tail, desc_size; 230 dma_addr_t desc_dma; 231 232 desc_size = (*desc & HDR_JD_LENGTH_MASK) * sizeof(u32); 233 desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE); 234 if (dma_mapping_error(dev, desc_dma)) { 235 dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n"); 236 return -EIO; 237 } 238 239 spin_lock_bh(&jrp->inplock); 240 241 head = jrp->head; 242 tail = ACCESS_ONCE(jrp->tail); 243 244 if (!rd_reg32(&jrp->rregs->inpring_avail) || 245 CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) { 246 spin_unlock_bh(&jrp->inplock); 247 dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE); 248 return -EBUSY; 249 } 250 251 head_entry = &jrp->entinfo[head]; 252 head_entry->desc_addr_virt = desc; 253 head_entry->desc_size = desc_size; 254 head_entry->callbk = (void *)cbk; 255 head_entry->cbkarg = areq; 256 head_entry->desc_addr_dma = desc_dma; 257 258 jrp->inpring[jrp->inp_ring_write_index] = desc_dma; 259 260 smp_wmb(); 261 262 jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) & 263 (JOBR_DEPTH - 1); 264 jrp->head = (head + 1) & (JOBR_DEPTH - 1); 265 266 wr_reg32(&jrp->rregs->inpring_jobadd, 1); 267 268 spin_unlock_bh(&jrp->inplock); 269 270 return 0; 271 } 272 EXPORT_SYMBOL(caam_jr_enqueue); 273 274 static int caam_reset_hw_jr(struct device *dev) 275 { 276 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); 277 unsigned int timeout = 100000; 278 279 /* 280 * mask interrupts since we are going to poll 281 * for reset completion status 282 */ 283 setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); 284 285 /* initiate flush (required prior to reset) */ 286 wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); 287 while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) == 288 JRINT_ERR_HALT_INPROGRESS) && --timeout) 289 cpu_relax(); 290 291 if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) != 292 JRINT_ERR_HALT_COMPLETE || timeout == 0) { 293 dev_err(dev, "failed to flush job ring %d\n", jrp->ridx); 294 return -EIO; 295 } 296 297 /* initiate reset */ 298 timeout = 100000; 299 wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET); 300 while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout) 301 cpu_relax(); 302 303 if (timeout == 0) { 304 dev_err(dev, "failed to reset job ring %d\n", jrp->ridx); 305 return -EIO; 306 } 307 308 /* unmask interrupts */ 309 clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK); 310 311 return 0; 312 } 313 314 /* 315 * Init JobR independent of platform property detection 316 */ 317 static int caam_jr_init(struct device *dev) 318 { 319 struct caam_drv_private_jr *jrp; 320 dma_addr_t inpbusaddr, outbusaddr; 321 int i, error; 322 323 jrp = dev_get_drvdata(dev); 324 325 tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev); 326 327 /* Connect job ring interrupt handler. */ 328 error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED, 329 "caam-jobr", dev); 330 if (error) { 331 dev_err(dev, "can't connect JobR %d interrupt (%d)\n", 332 jrp->ridx, jrp->irq); 333 irq_dispose_mapping(jrp->irq); 334 jrp->irq = 0; 335 return -EINVAL; 336 } 337 338 error = caam_reset_hw_jr(dev); 339 if (error) 340 return error; 341 342 jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, 343 &inpbusaddr, GFP_KERNEL); 344 345 jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) * 346 JOBR_DEPTH, &outbusaddr, GFP_KERNEL); 347 348 jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH, 349 GFP_KERNEL); 350 351 if ((jrp->inpring == NULL) || (jrp->outring == NULL) || 352 (jrp->entinfo == NULL)) { 353 dev_err(dev, "can't allocate job rings for %d\n", 354 jrp->ridx); 355 return -ENOMEM; 356 } 357 358 for (i = 0; i < JOBR_DEPTH; i++) 359 jrp->entinfo[i].desc_addr_dma = !0; 360 361 /* Setup rings */ 362 jrp->inp_ring_write_index = 0; 363 jrp->out_ring_read_index = 0; 364 jrp->head = 0; 365 jrp->tail = 0; 366 367 wr_reg64(&jrp->rregs->inpring_base, inpbusaddr); 368 wr_reg64(&jrp->rregs->outring_base, outbusaddr); 369 wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH); 370 wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH); 371 372 jrp->ringsize = JOBR_DEPTH; 373 374 spin_lock_init(&jrp->inplock); 375 spin_lock_init(&jrp->outlock); 376 377 /* Select interrupt coalescing parameters */ 378 setbits32(&jrp->rregs->rconfig_lo, JOBR_INTC | 379 (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) | 380 (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT)); 381 382 jrp->assign = JOBR_UNASSIGNED; 383 return 0; 384 } 385 386 /* 387 * Shutdown JobR independent of platform property code 388 */ 389 int caam_jr_shutdown(struct device *dev) 390 { 391 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); 392 dma_addr_t inpbusaddr, outbusaddr; 393 int ret; 394 395 ret = caam_reset_hw_jr(dev); 396 397 tasklet_kill(&jrp->irqtask); 398 399 /* Release interrupt */ 400 free_irq(jrp->irq, dev); 401 402 /* Free rings */ 403 inpbusaddr = rd_reg64(&jrp->rregs->inpring_base); 404 outbusaddr = rd_reg64(&jrp->rregs->outring_base); 405 dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, 406 jrp->inpring, inpbusaddr); 407 dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH, 408 jrp->outring, outbusaddr); 409 kfree(jrp->entinfo); 410 411 return ret; 412 } 413 414 /* 415 * Probe routine for each detected JobR subsystem. It assumes that 416 * property detection was picked up externally. 417 */ 418 int caam_jr_probe(struct platform_device *pdev, struct device_node *np, 419 int ring) 420 { 421 struct device *ctrldev, *jrdev; 422 struct platform_device *jr_pdev; 423 struct caam_drv_private *ctrlpriv; 424 struct caam_drv_private_jr *jrpriv; 425 u32 *jroffset; 426 int error; 427 428 ctrldev = &pdev->dev; 429 ctrlpriv = dev_get_drvdata(ctrldev); 430 431 jrpriv = kmalloc(sizeof(struct caam_drv_private_jr), 432 GFP_KERNEL); 433 if (jrpriv == NULL) { 434 dev_err(ctrldev, "can't alloc private mem for job ring %d\n", 435 ring); 436 return -ENOMEM; 437 } 438 jrpriv->parentdev = ctrldev; /* point back to parent */ 439 jrpriv->ridx = ring; /* save ring identity relative to detection */ 440 441 /* 442 * Derive a pointer to the detected JobRs regs 443 * Driver has already iomapped the entire space, we just 444 * need to add in the offset to this JobR. Don't know if I 445 * like this long-term, but it'll run 446 */ 447 jroffset = (u32 *)of_get_property(np, "reg", NULL); 448 jrpriv->rregs = (struct caam_job_ring __iomem *)((void *)ctrlpriv->ctrl 449 + *jroffset); 450 451 /* Build a local dev for each detected queue */ 452 jr_pdev = of_platform_device_create(np, NULL, ctrldev); 453 if (jr_pdev == NULL) { 454 kfree(jrpriv); 455 return -EINVAL; 456 } 457 jrdev = &jr_pdev->dev; 458 dev_set_drvdata(jrdev, jrpriv); 459 ctrlpriv->jrdev[ring] = jrdev; 460 461 if (sizeof(dma_addr_t) == sizeof(u64)) 462 if (of_device_is_compatible(np, "fsl,sec-v5.0-job-ring")) 463 dma_set_mask(jrdev, DMA_BIT_MASK(40)); 464 else 465 dma_set_mask(jrdev, DMA_BIT_MASK(36)); 466 else 467 dma_set_mask(jrdev, DMA_BIT_MASK(32)); 468 469 /* Identify the interrupt */ 470 jrpriv->irq = of_irq_to_resource(np, 0, NULL); 471 472 /* Now do the platform independent part */ 473 error = caam_jr_init(jrdev); /* now turn on hardware */ 474 if (error) { 475 kfree(jrpriv); 476 return error; 477 } 478 479 return error; 480 } 481