1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
19 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
23 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
24 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 /*
29 * Freescale integrated Security Engine (SEC) driver. Currently SEC 2.0 and
30 * 3.0 are supported.
31 */
32
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/bus.h>
36 #include <sys/endian.h>
37 #include <sys/kernel.h>
38 #include <sys/lock.h>
39 #include <sys/malloc.h>
40 #include <sys/mbuf.h>
41 #include <sys/module.h>
42 #include <sys/mutex.h>
43 #include <sys/random.h>
44 #include <sys/rman.h>
45
46 #include <machine/_inttypes.h>
47 #include <machine/bus.h>
48 #include <machine/resource.h>
49
50 #include <opencrypto/cryptodev.h>
51 #include <opencrypto/xform_auth.h>
52 #include "cryptodev_if.h"
53
54 #include <dev/ofw/ofw_bus_subr.h>
55 #include <dev/sec/sec.h>
56
57 static int sec_probe(device_t dev);
58 static int sec_attach(device_t dev);
59 static int sec_detach(device_t dev);
60 static int sec_suspend(device_t dev);
61 static int sec_resume(device_t dev);
62 static int sec_shutdown(device_t dev);
63 static void sec_primary_intr(void *arg);
64 static void sec_secondary_intr(void *arg);
65 static int sec_setup_intr(struct sec_softc *sc, struct resource **ires,
66 void **ihand, int *irid, driver_intr_t handler, const char *iname);
67 static void sec_release_intr(struct sec_softc *sc, struct resource *ires,
68 void *ihand, int irid, const char *iname);
69 static int sec_controller_reset(struct sec_softc *sc);
70 static int sec_channel_reset(struct sec_softc *sc, int channel, int full);
71 static int sec_init(struct sec_softc *sc);
72 static int sec_alloc_dma_mem(struct sec_softc *sc,
73 struct sec_dma_mem *dma_mem, bus_size_t size);
74 static int sec_desc_map_dma(struct sec_softc *sc,
75 struct sec_dma_mem *dma_mem, struct cryptop *crp, bus_size_t size,
76 struct sec_desc_map_info *sdmi);
77 static void sec_free_dma_mem(struct sec_dma_mem *dma_mem);
78 static void sec_enqueue(struct sec_softc *sc);
79 static int sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc,
80 int channel);
81 static int sec_eu_channel(struct sec_softc *sc, int eu);
82 static int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
83 u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize);
84 static int sec_make_pointer_direct(struct sec_softc *sc,
85 struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize);
86 static int sec_probesession(device_t dev,
87 const struct crypto_session_params *csp);
88 static int sec_newsession(device_t dev, crypto_session_t cses,
89 const struct crypto_session_params *csp);
90 static int sec_process(device_t dev, struct cryptop *crp, int hint);
91 static int sec_build_common_ns_desc(struct sec_softc *sc,
92 struct sec_desc *desc, const struct crypto_session_params *csp,
93 struct cryptop *crp);
94 static int sec_build_common_s_desc(struct sec_softc *sc,
95 struct sec_desc *desc, const struct crypto_session_params *csp,
96 struct cryptop *crp);
97
98 static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr);
99
100 /* AESU */
101 static bool sec_aesu_newsession(const struct crypto_session_params *csp);
102 static int sec_aesu_make_desc(struct sec_softc *sc,
103 const struct crypto_session_params *csp, struct sec_desc *desc,
104 struct cryptop *crp);
105
106 /* MDEU */
107 static bool sec_mdeu_can_handle(u_int alg);
108 static int sec_mdeu_config(const struct crypto_session_params *csp,
109 u_int *eu, u_int *mode, u_int *hashlen);
110 static bool sec_mdeu_newsession(const struct crypto_session_params *csp);
111 static int sec_mdeu_make_desc(struct sec_softc *sc,
112 const struct crypto_session_params *csp, struct sec_desc *desc,
113 struct cryptop *crp);
114
115 static device_method_t sec_methods[] = {
116 /* Device interface */
117 DEVMETHOD(device_probe, sec_probe),
118 DEVMETHOD(device_attach, sec_attach),
119 DEVMETHOD(device_detach, sec_detach),
120
121 DEVMETHOD(device_suspend, sec_suspend),
122 DEVMETHOD(device_resume, sec_resume),
123 DEVMETHOD(device_shutdown, sec_shutdown),
124
125 /* Crypto methods */
126 DEVMETHOD(cryptodev_probesession, sec_probesession),
127 DEVMETHOD(cryptodev_newsession, sec_newsession),
128 DEVMETHOD(cryptodev_process, sec_process),
129
130 DEVMETHOD_END
131 };
132 static driver_t sec_driver = {
133 "sec",
134 sec_methods,
135 sizeof(struct sec_softc),
136 };
137
138 DRIVER_MODULE(sec, simplebus, sec_driver, 0, 0);
139 MODULE_DEPEND(sec, crypto, 1, 1, 1);
140
141 static struct sec_eu_methods sec_eus[] = {
142 {
143 sec_aesu_newsession,
144 sec_aesu_make_desc,
145 },
146 {
147 sec_mdeu_newsession,
148 sec_mdeu_make_desc,
149 },
150 { NULL, NULL }
151 };
152
153 static inline void
sec_sync_dma_mem(struct sec_dma_mem * dma_mem,bus_dmasync_op_t op)154 sec_sync_dma_mem(struct sec_dma_mem *dma_mem, bus_dmasync_op_t op)
155 {
156
157 /* Sync only if dma memory is valid */
158 if (dma_mem->dma_vaddr != NULL)
159 bus_dmamap_sync(dma_mem->dma_tag, dma_mem->dma_map, op);
160 }
161
162 static inline void *
sec_get_pointer_data(struct sec_desc * desc,u_int n)163 sec_get_pointer_data(struct sec_desc *desc, u_int n)
164 {
165
166 return (desc->sd_ptr_dmem[n].dma_vaddr);
167 }
168
169 static int
sec_probe(device_t dev)170 sec_probe(device_t dev)
171 {
172 struct sec_softc *sc;
173 uint64_t id;
174
175 if (!ofw_bus_status_okay(dev))
176 return (ENXIO);
177
178 if (!ofw_bus_is_compatible(dev, "fsl,sec2.0"))
179 return (ENXIO);
180
181 sc = device_get_softc(dev);
182
183 sc->sc_rrid = 0;
184 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid,
185 RF_ACTIVE);
186
187 if (sc->sc_rres == NULL)
188 return (ENXIO);
189
190 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres);
191 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres);
192
193 id = SEC_READ(sc, SEC_ID);
194
195 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres);
196
197 switch (id) {
198 case SEC_20_ID:
199 device_set_desc(dev, "Freescale Security Engine 2.0");
200 sc->sc_version = 2;
201 break;
202 case SEC_30_ID:
203 device_set_desc(dev, "Freescale Security Engine 3.0");
204 sc->sc_version = 3;
205 break;
206 case SEC_31_ID:
207 device_set_desc(dev, "Freescale Security Engine 3.1");
208 sc->sc_version = 3;
209 break;
210 default:
211 device_printf(dev, "unknown SEC ID 0x%016"PRIx64"!\n", id);
212 return (ENXIO);
213 }
214
215 return (0);
216 }
217
218 static int
sec_attach(device_t dev)219 sec_attach(device_t dev)
220 {
221 struct sec_softc *sc;
222 struct sec_hw_lt *lt;
223 int error = 0;
224 int i;
225
226 sc = device_get_softc(dev);
227 sc->sc_dev = dev;
228 sc->sc_blocked = 0;
229 sc->sc_shutdown = 0;
230
231 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct sec_session),
232 CRYPTOCAP_F_HARDWARE);
233 if (sc->sc_cid < 0) {
234 device_printf(dev, "could not get crypto driver ID!\n");
235 return (ENXIO);
236 }
237
238 /* Init locks */
239 mtx_init(&sc->sc_controller_lock, device_get_nameunit(dev),
240 "SEC Controller lock", MTX_DEF);
241 mtx_init(&sc->sc_descriptors_lock, device_get_nameunit(dev),
242 "SEC Descriptors lock", MTX_DEF);
243
244 /* Allocate I/O memory for SEC registers */
245 sc->sc_rrid = 0;
246 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid,
247 RF_ACTIVE);
248
249 if (sc->sc_rres == NULL) {
250 device_printf(dev, "could not allocate I/O memory!\n");
251 goto fail1;
252 }
253
254 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres);
255 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres);
256
257 /* Setup interrupts */
258 sc->sc_pri_irid = 0;
259 error = sec_setup_intr(sc, &sc->sc_pri_ires, &sc->sc_pri_ihand,
260 &sc->sc_pri_irid, sec_primary_intr, "primary");
261
262 if (error)
263 goto fail2;
264
265 if (sc->sc_version == 3) {
266 sc->sc_sec_irid = 1;
267 error = sec_setup_intr(sc, &sc->sc_sec_ires, &sc->sc_sec_ihand,
268 &sc->sc_sec_irid, sec_secondary_intr, "secondary");
269
270 if (error)
271 goto fail3;
272 }
273
274 /* Alloc DMA memory for descriptors and link tables */
275 error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem),
276 SEC_DESCRIPTORS * sizeof(struct sec_hw_desc));
277
278 if (error)
279 goto fail4;
280
281 error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem),
282 (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt));
283
284 if (error)
285 goto fail5;
286
287 /* Fill in descriptors and link tables */
288 for (i = 0; i < SEC_DESCRIPTORS; i++) {
289 sc->sc_desc[i].sd_desc =
290 (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i;
291 sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr +
292 (i * sizeof(struct sec_hw_desc));
293 }
294
295 for (i = 0; i < SEC_LT_ENTRIES + 1; i++) {
296 sc->sc_lt[i].sl_lt =
297 (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i;
298 sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr +
299 (i * sizeof(struct sec_hw_lt));
300 }
301
302 /* Last entry in link table is used to create a circle */
303 lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt;
304 lt->shl_length = 0;
305 lt->shl_r = 0;
306 lt->shl_n = 1;
307 lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr;
308
309 /* Init descriptor and link table queues pointers */
310 SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS);
311 SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS);
312 SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS);
313 SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS);
314 SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS);
315 SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS);
316 SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES);
317 SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES);
318
319 /* Create masks for fast checks */
320 sc->sc_int_error_mask = 0;
321 for (i = 0; i < SEC_CHANNELS; i++)
322 sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i));
323
324 switch (sc->sc_version) {
325 case 2:
326 sc->sc_channel_idle_mask =
327 (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) |
328 (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) |
329 (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) |
330 (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S);
331 break;
332 case 3:
333 sc->sc_channel_idle_mask =
334 (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) |
335 (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) |
336 (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) |
337 (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S);
338 break;
339 }
340
341 /* Init hardware */
342 error = sec_init(sc);
343
344 if (error)
345 goto fail6;
346
347 return (0);
348
349 fail6:
350 sec_free_dma_mem(&(sc->sc_lt_dmem));
351 fail5:
352 sec_free_dma_mem(&(sc->sc_desc_dmem));
353 fail4:
354 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand,
355 sc->sc_sec_irid, "secondary");
356 fail3:
357 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand,
358 sc->sc_pri_irid, "primary");
359 fail2:
360 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres);
361 fail1:
362 mtx_destroy(&sc->sc_controller_lock);
363 mtx_destroy(&sc->sc_descriptors_lock);
364
365 return (ENXIO);
366 }
367
368 static int
sec_detach(device_t dev)369 sec_detach(device_t dev)
370 {
371 struct sec_softc *sc = device_get_softc(dev);
372 int i, error, timeout = SEC_TIMEOUT;
373
374 /* Prepare driver to shutdown */
375 SEC_LOCK(sc, descriptors);
376 sc->sc_shutdown = 1;
377 SEC_UNLOCK(sc, descriptors);
378
379 /* Wait until all queued processing finishes */
380 while (1) {
381 SEC_LOCK(sc, descriptors);
382 i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc);
383 SEC_UNLOCK(sc, descriptors);
384
385 if (i == 0)
386 break;
387
388 if (timeout < 0) {
389 device_printf(dev, "queue flush timeout!\n");
390
391 /* DMA can be still active - stop it */
392 for (i = 0; i < SEC_CHANNELS; i++)
393 sec_channel_reset(sc, i, 1);
394
395 break;
396 }
397
398 timeout -= 1000;
399 DELAY(1000);
400 }
401
402 /* Disable interrupts */
403 SEC_WRITE(sc, SEC_IER, 0);
404
405 /* Unregister from OCF */
406 crypto_unregister_all(sc->sc_cid);
407
408 /* Free DMA memory */
409 for (i = 0; i < SEC_DESCRIPTORS; i++)
410 SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i]));
411
412 sec_free_dma_mem(&(sc->sc_lt_dmem));
413 sec_free_dma_mem(&(sc->sc_desc_dmem));
414
415 /* Release interrupts */
416 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand,
417 sc->sc_pri_irid, "primary");
418 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand,
419 sc->sc_sec_irid, "secondary");
420
421 /* Release memory */
422 if (sc->sc_rres) {
423 error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid,
424 sc->sc_rres);
425 if (error)
426 device_printf(dev, "bus_release_resource() failed for"
427 " I/O memory, error %d\n", error);
428
429 sc->sc_rres = NULL;
430 }
431
432 mtx_destroy(&sc->sc_controller_lock);
433 mtx_destroy(&sc->sc_descriptors_lock);
434
435 return (0);
436 }
437
438 static int
sec_suspend(device_t dev)439 sec_suspend(device_t dev)
440 {
441
442 return (0);
443 }
444
445 static int
sec_resume(device_t dev)446 sec_resume(device_t dev)
447 {
448
449 return (0);
450 }
451
452 static int
sec_shutdown(device_t dev)453 sec_shutdown(device_t dev)
454 {
455
456 return (0);
457 }
458
459 static int
sec_setup_intr(struct sec_softc * sc,struct resource ** ires,void ** ihand,int * irid,driver_intr_t handler,const char * iname)460 sec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand,
461 int *irid, driver_intr_t handler, const char *iname)
462 {
463 int error;
464
465 (*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid,
466 RF_ACTIVE);
467
468 if ((*ires) == NULL) {
469 device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname);
470 return (ENXIO);
471 }
472
473 error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET,
474 NULL, handler, sc, ihand);
475
476 if (error) {
477 device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname);
478 if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires))
479 device_printf(sc->sc_dev, "could not release %s IRQ\n",
480 iname);
481
482 (*ires) = NULL;
483 return (error);
484 }
485
486 return (0);
487 }
488
489 static void
sec_release_intr(struct sec_softc * sc,struct resource * ires,void * ihand,int irid,const char * iname)490 sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand,
491 int irid, const char *iname)
492 {
493 int error;
494
495 if (ires == NULL)
496 return;
497
498 error = bus_teardown_intr(sc->sc_dev, ires, ihand);
499 if (error)
500 device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s"
501 " IRQ, error %d\n", iname, error);
502
503 error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires);
504 if (error)
505 device_printf(sc->sc_dev, "bus_release_resource() failed for %s"
506 " IRQ, error %d\n", iname, error);
507 }
508
509 static void
sec_primary_intr(void * arg)510 sec_primary_intr(void *arg)
511 {
512 struct sec_session *ses;
513 struct sec_softc *sc = arg;
514 struct sec_desc *desc;
515 struct cryptop *crp;
516 uint64_t isr;
517 uint8_t hash[HASH_MAX_LEN];
518 int i, wakeup = 0;
519
520 SEC_LOCK(sc, controller);
521
522 /* Check for errors */
523 isr = SEC_READ(sc, SEC_ISR);
524 if (isr & sc->sc_int_error_mask) {
525 /* Check each channel for error */
526 for (i = 0; i < SEC_CHANNELS; i++) {
527 if ((isr & SEC_INT_CH_ERR(i)) == 0)
528 continue;
529
530 device_printf(sc->sc_dev,
531 "I/O error on channel %i!\n", i);
532
533 /* Find and mark problematic descriptor */
534 desc = sec_find_desc(sc, SEC_READ(sc,
535 SEC_CHAN_CDPR(i)));
536
537 if (desc != NULL)
538 desc->sd_error = EIO;
539
540 /* Do partial channel reset */
541 sec_channel_reset(sc, i, 0);
542 }
543 }
544
545 /* ACK interrupt */
546 SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL);
547
548 SEC_UNLOCK(sc, controller);
549 SEC_LOCK(sc, descriptors);
550
551 /* Handle processed descriptors */
552 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
553
554 while (SEC_QUEUED_DESC_CNT(sc) > 0) {
555 desc = SEC_GET_QUEUED_DESC(sc);
556
557 if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) {
558 SEC_PUT_BACK_QUEUED_DESC(sc);
559 break;
560 }
561
562 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD |
563 BUS_DMASYNC_PREWRITE);
564
565 crp = desc->sd_crp;
566 crp->crp_etype = desc->sd_error;
567 if (crp->crp_etype == 0) {
568 ses = crypto_get_driver_session(crp->crp_session);
569 if (ses->ss_mlen != 0) {
570 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
571 crypto_copydata(crp,
572 crp->crp_digest_start,
573 ses->ss_mlen, hash);
574 if (timingsafe_bcmp(
575 desc->sd_desc->shd_digest,
576 hash, ses->ss_mlen) != 0)
577 crp->crp_etype = EBADMSG;
578 } else
579 crypto_copyback(crp,
580 crp->crp_digest_start,
581 ses->ss_mlen,
582 desc->sd_desc->shd_digest);
583 }
584 }
585 crypto_done(desc->sd_crp);
586
587 SEC_DESC_FREE_POINTERS(desc);
588 SEC_DESC_FREE_LT(sc, desc);
589 SEC_DESC_QUEUED2FREE(sc);
590 }
591
592 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
593
594 if (!sc->sc_shutdown) {
595 wakeup = sc->sc_blocked;
596 sc->sc_blocked = 0;
597 }
598
599 SEC_UNLOCK(sc, descriptors);
600
601 /* Enqueue ready descriptors in hardware */
602 sec_enqueue(sc);
603
604 if (wakeup)
605 crypto_unblock(sc->sc_cid, wakeup);
606 }
607
608 static void
sec_secondary_intr(void * arg)609 sec_secondary_intr(void *arg)
610 {
611 struct sec_softc *sc = arg;
612
613 device_printf(sc->sc_dev, "spurious secondary interrupt!\n");
614 sec_primary_intr(arg);
615 }
616
617 static int
sec_controller_reset(struct sec_softc * sc)618 sec_controller_reset(struct sec_softc *sc)
619 {
620 int timeout = SEC_TIMEOUT;
621
622 /* Reset Controller */
623 SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR);
624
625 while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) {
626 DELAY(1000);
627 timeout -= 1000;
628
629 if (timeout < 0) {
630 device_printf(sc->sc_dev, "timeout while waiting for "
631 "device reset!\n");
632 return (ETIMEDOUT);
633 }
634 }
635
636 return (0);
637 }
638
639 static int
sec_channel_reset(struct sec_softc * sc,int channel,int full)640 sec_channel_reset(struct sec_softc *sc, int channel, int full)
641 {
642 int timeout = SEC_TIMEOUT;
643 uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON;
644 uint64_t reg;
645
646 /* Reset Channel */
647 reg = SEC_READ(sc, SEC_CHAN_CCR(channel));
648 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit);
649
650 while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) {
651 DELAY(1000);
652 timeout -= 1000;
653
654 if (timeout < 0) {
655 device_printf(sc->sc_dev, "timeout while waiting for "
656 "channel reset!\n");
657 return (ETIMEDOUT);
658 }
659 }
660
661 if (full) {
662 reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS;
663
664 switch(sc->sc_version) {
665 case 2:
666 reg |= SEC_CHAN_CCR_CDWE;
667 break;
668 case 3:
669 reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN;
670 break;
671 }
672
673 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg);
674 }
675
676 return (0);
677 }
678
679 static int
sec_init(struct sec_softc * sc)680 sec_init(struct sec_softc *sc)
681 {
682 uint64_t reg;
683 int error, i;
684
685 /* Reset controller twice to clear all pending interrupts */
686 error = sec_controller_reset(sc);
687 if (error)
688 return (error);
689
690 error = sec_controller_reset(sc);
691 if (error)
692 return (error);
693
694 /* Reset channels */
695 for (i = 0; i < SEC_CHANNELS; i++) {
696 error = sec_channel_reset(sc, i, 1);
697 if (error)
698 return (error);
699 }
700
701 /* Enable Interrupts */
702 reg = SEC_INT_ITO;
703 for (i = 0; i < SEC_CHANNELS; i++)
704 reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i);
705
706 SEC_WRITE(sc, SEC_IER, reg);
707
708 return (error);
709 }
710
711 static void
sec_alloc_dma_mem_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)712 sec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
713 {
714 struct sec_dma_mem *dma_mem = arg;
715
716 if (error)
717 return;
718
719 KASSERT(nseg == 1, ("Wrong number of segments, should be 1"));
720 dma_mem->dma_paddr = segs->ds_addr;
721 }
722
723 static void
sec_dma_map_desc_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)724 sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg,
725 int error)
726 {
727 struct sec_desc_map_info *sdmi = arg;
728 struct sec_softc *sc = sdmi->sdmi_sc;
729 struct sec_lt *lt = NULL;
730 bus_addr_t addr;
731 bus_size_t size;
732 int i;
733
734 SEC_LOCK_ASSERT(sc, descriptors);
735
736 if (error)
737 return;
738
739 for (i = 0; i < nseg; i++) {
740 addr = segs[i].ds_addr;
741 size = segs[i].ds_len;
742
743 /* Skip requested offset */
744 if (sdmi->sdmi_offset >= size) {
745 sdmi->sdmi_offset -= size;
746 continue;
747 }
748
749 addr += sdmi->sdmi_offset;
750 size -= sdmi->sdmi_offset;
751 sdmi->sdmi_offset = 0;
752
753 /* Do not link more than requested */
754 if (sdmi->sdmi_size < size)
755 size = sdmi->sdmi_size;
756
757 lt = SEC_ALLOC_LT_ENTRY(sc);
758 lt->sl_lt->shl_length = size;
759 lt->sl_lt->shl_r = 0;
760 lt->sl_lt->shl_n = 0;
761 lt->sl_lt->shl_ptr = addr;
762
763 if (sdmi->sdmi_lt_first == NULL)
764 sdmi->sdmi_lt_first = lt;
765
766 sdmi->sdmi_lt_used += 1;
767
768 if ((sdmi->sdmi_size -= size) == 0)
769 break;
770 }
771
772 sdmi->sdmi_lt_last = lt;
773 }
774
775 static int
sec_alloc_dma_mem(struct sec_softc * sc,struct sec_dma_mem * dma_mem,bus_size_t size)776 sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
777 bus_size_t size)
778 {
779 int error;
780
781 if (dma_mem->dma_vaddr != NULL)
782 return (EBUSY);
783
784 error = bus_dma_tag_create(NULL, /* parent */
785 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */
786 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
787 BUS_SPACE_MAXADDR, /* highaddr */
788 NULL, NULL, /* filtfunc, filtfuncarg */
789 size, 1, /* maxsize, nsegments */
790 size, 0, /* maxsegsz, flags */
791 NULL, NULL, /* lockfunc, lockfuncarg */
792 &(dma_mem->dma_tag)); /* dmat */
793
794 if (error) {
795 device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
796 " %i!\n", error);
797 goto err1;
798 }
799
800 error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr),
801 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map));
802
803 if (error) {
804 device_printf(sc->sc_dev, "failed to allocate DMA safe"
805 " memory, error %i!\n", error);
806 goto err2;
807 }
808
809 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map,
810 dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem,
811 BUS_DMA_NOWAIT);
812
813 if (error) {
814 device_printf(sc->sc_dev, "cannot get address of the DMA"
815 " memory, error %i\n", error);
816 goto err3;
817 }
818
819 dma_mem->dma_is_map = 0;
820 return (0);
821
822 err3:
823 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map);
824 err2:
825 bus_dma_tag_destroy(dma_mem->dma_tag);
826 err1:
827 dma_mem->dma_vaddr = NULL;
828 return(error);
829 }
830
831 static int
sec_desc_map_dma(struct sec_softc * sc,struct sec_dma_mem * dma_mem,struct cryptop * crp,bus_size_t size,struct sec_desc_map_info * sdmi)832 sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
833 struct cryptop *crp, bus_size_t size, struct sec_desc_map_info *sdmi)
834 {
835 int error;
836
837 if (dma_mem->dma_vaddr != NULL)
838 return (EBUSY);
839
840 switch (crp->crp_buf.cb_type) {
841 case CRYPTO_BUF_CONTIG:
842 break;
843 case CRYPTO_BUF_UIO:
844 size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE;
845 break;
846 case CRYPTO_BUF_MBUF:
847 size = m_length(crp->crp_buf.cb_mbuf, NULL);
848 break;
849 case CRYPTO_BUF_SINGLE_MBUF:
850 size = crp->crp_buf.cb_mbuf->m_len;
851 break;
852 case CRYPTO_BUF_VMPAGE:
853 size = PAGE_SIZE - crp->crp_buf.cb_vm_page_offset;
854 break;
855 default:
856 return (EINVAL);
857 }
858
859 error = bus_dma_tag_create(NULL, /* parent */
860 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */
861 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
862 BUS_SPACE_MAXADDR, /* highaddr */
863 NULL, NULL, /* filtfunc, filtfuncarg */
864 size, /* maxsize */
865 SEC_FREE_LT_CNT(sc), /* nsegments */
866 SEC_MAX_DMA_BLOCK_SIZE, 0, /* maxsegsz, flags */
867 NULL, NULL, /* lockfunc, lockfuncarg */
868 &(dma_mem->dma_tag)); /* dmat */
869
870 if (error) {
871 device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
872 " %i!\n", error);
873 dma_mem->dma_vaddr = NULL;
874 return (error);
875 }
876
877 error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map));
878
879 if (error) {
880 device_printf(sc->sc_dev, "failed to create DMA map, error %i!"
881 "\n", error);
882 bus_dma_tag_destroy(dma_mem->dma_tag);
883 return (error);
884 }
885
886 error = bus_dmamap_load_crp(dma_mem->dma_tag, dma_mem->dma_map, crp,
887 sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT);
888
889 if (error) {
890 device_printf(sc->sc_dev, "cannot get address of the DMA"
891 " memory, error %i!\n", error);
892 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map);
893 bus_dma_tag_destroy(dma_mem->dma_tag);
894 return (error);
895 }
896
897 dma_mem->dma_is_map = 1;
898 dma_mem->dma_vaddr = crp;
899
900 return (0);
901 }
902
903 static void
sec_free_dma_mem(struct sec_dma_mem * dma_mem)904 sec_free_dma_mem(struct sec_dma_mem *dma_mem)
905 {
906
907 /* Check for double free */
908 if (dma_mem->dma_vaddr == NULL)
909 return;
910
911 bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map);
912
913 if (dma_mem->dma_is_map)
914 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map);
915 else
916 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr,
917 dma_mem->dma_map);
918
919 bus_dma_tag_destroy(dma_mem->dma_tag);
920 dma_mem->dma_vaddr = NULL;
921 }
922
923 static int
sec_eu_channel(struct sec_softc * sc,int eu)924 sec_eu_channel(struct sec_softc *sc, int eu)
925 {
926 uint64_t reg;
927 int channel = 0;
928
929 SEC_LOCK_ASSERT(sc, controller);
930
931 reg = SEC_READ(sc, SEC_EUASR);
932
933 switch (eu) {
934 case SEC_EU_AFEU:
935 channel = SEC_EUASR_AFEU(reg);
936 break;
937 case SEC_EU_DEU:
938 channel = SEC_EUASR_DEU(reg);
939 break;
940 case SEC_EU_MDEU_A:
941 case SEC_EU_MDEU_B:
942 channel = SEC_EUASR_MDEU(reg);
943 break;
944 case SEC_EU_RNGU:
945 channel = SEC_EUASR_RNGU(reg);
946 break;
947 case SEC_EU_PKEU:
948 channel = SEC_EUASR_PKEU(reg);
949 break;
950 case SEC_EU_AESU:
951 channel = SEC_EUASR_AESU(reg);
952 break;
953 case SEC_EU_KEU:
954 channel = SEC_EUASR_KEU(reg);
955 break;
956 case SEC_EU_CRCU:
957 channel = SEC_EUASR_CRCU(reg);
958 break;
959 }
960
961 return (channel - 1);
962 }
963
964 static int
sec_enqueue_desc(struct sec_softc * sc,struct sec_desc * desc,int channel)965 sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel)
966 {
967 u_int fflvl = SEC_MAX_FIFO_LEVEL;
968 uint64_t reg;
969 int i;
970
971 SEC_LOCK_ASSERT(sc, controller);
972
973 /* Find free channel if have not got one */
974 if (channel < 0) {
975 for (i = 0; i < SEC_CHANNELS; i++) {
976 reg = SEC_READ(sc, SEC_CHAN_CSR(channel));
977
978 if ((reg & sc->sc_channel_idle_mask) == 0) {
979 channel = i;
980 break;
981 }
982 }
983 }
984
985 /* There is no free channel */
986 if (channel < 0)
987 return (-1);
988
989 /* Check FIFO level on selected channel */
990 reg = SEC_READ(sc, SEC_CHAN_CSR(channel));
991
992 switch(sc->sc_version) {
993 case 2:
994 fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M;
995 break;
996 case 3:
997 fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M;
998 break;
999 }
1000
1001 if (fflvl >= SEC_MAX_FIFO_LEVEL)
1002 return (-1);
1003
1004 /* Enqueue descriptor in channel */
1005 SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr);
1006
1007 return (channel);
1008 }
1009
1010 static void
sec_enqueue(struct sec_softc * sc)1011 sec_enqueue(struct sec_softc *sc)
1012 {
1013 struct sec_desc *desc;
1014 int ch0, ch1;
1015
1016 SEC_LOCK(sc, descriptors);
1017 SEC_LOCK(sc, controller);
1018
1019 while (SEC_READY_DESC_CNT(sc) > 0) {
1020 desc = SEC_GET_READY_DESC(sc);
1021
1022 ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0);
1023 ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1);
1024
1025 /*
1026 * Both EU are used by the same channel.
1027 * Enqueue descriptor in channel used by busy EUs.
1028 */
1029 if (ch0 >= 0 && ch0 == ch1) {
1030 if (sec_enqueue_desc(sc, desc, ch0) >= 0) {
1031 SEC_DESC_READY2QUEUED(sc);
1032 continue;
1033 }
1034 }
1035
1036 /*
1037 * Only one EU is free.
1038 * Enqueue descriptor in channel used by busy EU.
1039 */
1040 if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) {
1041 if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1)
1042 >= 0) {
1043 SEC_DESC_READY2QUEUED(sc);
1044 continue;
1045 }
1046 }
1047
1048 /*
1049 * Both EU are free.
1050 * Enqueue descriptor in first free channel.
1051 */
1052 if (ch0 < 0 && ch1 < 0) {
1053 if (sec_enqueue_desc(sc, desc, -1) >= 0) {
1054 SEC_DESC_READY2QUEUED(sc);
1055 continue;
1056 }
1057 }
1058
1059 /* Current descriptor can not be queued at the moment */
1060 SEC_PUT_BACK_READY_DESC(sc);
1061 break;
1062 }
1063
1064 SEC_UNLOCK(sc, controller);
1065 SEC_UNLOCK(sc, descriptors);
1066 }
1067
1068 static struct sec_desc *
sec_find_desc(struct sec_softc * sc,bus_addr_t paddr)1069 sec_find_desc(struct sec_softc *sc, bus_addr_t paddr)
1070 {
1071 struct sec_desc *desc = NULL;
1072 int i;
1073
1074 SEC_LOCK_ASSERT(sc, descriptors);
1075
1076 for (i = 0; i < SEC_CHANNELS; i++) {
1077 if (sc->sc_desc[i].sd_desc_paddr == paddr) {
1078 desc = &(sc->sc_desc[i]);
1079 break;
1080 }
1081 }
1082
1083 return (desc);
1084 }
1085
1086 static int
sec_make_pointer_direct(struct sec_softc * sc,struct sec_desc * desc,u_int n,bus_addr_t data,bus_size_t dsize)1087 sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n,
1088 bus_addr_t data, bus_size_t dsize)
1089 {
1090 struct sec_hw_desc_ptr *ptr;
1091
1092 SEC_LOCK_ASSERT(sc, descriptors);
1093
1094 ptr = &(desc->sd_desc->shd_pointer[n]);
1095 ptr->shdp_length = dsize;
1096 ptr->shdp_extent = 0;
1097 ptr->shdp_j = 0;
1098 ptr->shdp_ptr = data;
1099
1100 return (0);
1101 }
1102
1103 static int
sec_make_pointer(struct sec_softc * sc,struct sec_desc * desc,u_int n,struct cryptop * crp,bus_size_t doffset,bus_size_t dsize)1104 sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
1105 u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize)
1106 {
1107 struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 };
1108 struct sec_hw_desc_ptr *ptr;
1109 int error;
1110
1111 SEC_LOCK_ASSERT(sc, descriptors);
1112
1113 error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), crp, dsize,
1114 &sdmi);
1115
1116 if (error)
1117 return (error);
1118
1119 sdmi.sdmi_lt_last->sl_lt->shl_r = 1;
1120 desc->sd_lt_used += sdmi.sdmi_lt_used;
1121
1122 ptr = &(desc->sd_desc->shd_pointer[n]);
1123 ptr->shdp_length = dsize;
1124 ptr->shdp_extent = 0;
1125 ptr->shdp_j = 1;
1126 ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr;
1127
1128 return (0);
1129 }
1130
1131 static bool
sec_cipher_supported(const struct crypto_session_params * csp)1132 sec_cipher_supported(const struct crypto_session_params *csp)
1133 {
1134
1135 switch (csp->csp_cipher_alg) {
1136 case CRYPTO_AES_CBC:
1137 /* AESU */
1138 if (csp->csp_ivlen != AES_BLOCK_LEN)
1139 return (false);
1140 break;
1141 default:
1142 return (false);
1143 }
1144
1145 if (csp->csp_cipher_klen == 0 || csp->csp_cipher_klen > SEC_MAX_KEY_LEN)
1146 return (false);
1147
1148 return (true);
1149 }
1150
1151 static bool
sec_auth_supported(struct sec_softc * sc,const struct crypto_session_params * csp)1152 sec_auth_supported(struct sec_softc *sc,
1153 const struct crypto_session_params *csp)
1154 {
1155
1156 switch (csp->csp_auth_alg) {
1157 case CRYPTO_SHA2_384_HMAC:
1158 case CRYPTO_SHA2_512_HMAC:
1159 if (sc->sc_version < 3)
1160 return (false);
1161 /* FALLTHROUGH */
1162 case CRYPTO_SHA1_HMAC:
1163 case CRYPTO_SHA2_256_HMAC:
1164 if (csp->csp_auth_klen > SEC_MAX_KEY_LEN)
1165 return (false);
1166 break;
1167 case CRYPTO_SHA1:
1168 break;
1169 default:
1170 return (false);
1171 }
1172 return (true);
1173 }
1174
1175 static int
sec_probesession(device_t dev,const struct crypto_session_params * csp)1176 sec_probesession(device_t dev, const struct crypto_session_params *csp)
1177 {
1178 struct sec_softc *sc = device_get_softc(dev);
1179
1180 if (csp->csp_flags != 0)
1181 return (EINVAL);
1182 switch (csp->csp_mode) {
1183 case CSP_MODE_DIGEST:
1184 if (!sec_auth_supported(sc, csp))
1185 return (EINVAL);
1186 break;
1187 case CSP_MODE_CIPHER:
1188 if (!sec_cipher_supported(csp))
1189 return (EINVAL);
1190 break;
1191 case CSP_MODE_ETA:
1192 if (!sec_auth_supported(sc, csp) || !sec_cipher_supported(csp))
1193 return (EINVAL);
1194 break;
1195 default:
1196 return (EINVAL);
1197 }
1198 return (CRYPTODEV_PROBE_HARDWARE);
1199 }
1200
1201 static int
sec_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)1202 sec_newsession(device_t dev, crypto_session_t cses,
1203 const struct crypto_session_params *csp)
1204 {
1205 struct sec_eu_methods *eu = sec_eus;
1206 struct sec_session *ses;
1207
1208 ses = crypto_get_driver_session(cses);
1209
1210 /* Find EU for this session */
1211 while (eu->sem_make_desc != NULL) {
1212 if (eu->sem_newsession(csp))
1213 break;
1214 eu++;
1215 }
1216 KASSERT(eu->sem_make_desc != NULL, ("failed to find eu for session"));
1217
1218 /* Save cipher key */
1219 if (csp->csp_cipher_key != NULL)
1220 memcpy(ses->ss_key, csp->csp_cipher_key, csp->csp_cipher_klen);
1221
1222 /* Save digest key */
1223 if (csp->csp_auth_key != NULL)
1224 memcpy(ses->ss_mkey, csp->csp_auth_key, csp->csp_auth_klen);
1225
1226 if (csp->csp_auth_alg != 0) {
1227 if (csp->csp_auth_mlen == 0)
1228 ses->ss_mlen = crypto_auth_hash(csp)->hashsize;
1229 else
1230 ses->ss_mlen = csp->csp_auth_mlen;
1231 }
1232
1233 return (0);
1234 }
1235
1236 static int
sec_process(device_t dev,struct cryptop * crp,int hint)1237 sec_process(device_t dev, struct cryptop *crp, int hint)
1238 {
1239 struct sec_softc *sc = device_get_softc(dev);
1240 struct sec_desc *desc = NULL;
1241 const struct crypto_session_params *csp;
1242 struct sec_session *ses;
1243 int error = 0;
1244
1245 ses = crypto_get_driver_session(crp->crp_session);
1246 csp = crypto_get_params(crp->crp_session);
1247
1248 /* Check for input length */
1249 if (crypto_buffer_len(&crp->crp_buf) > SEC_MAX_DMA_BLOCK_SIZE) {
1250 crp->crp_etype = E2BIG;
1251 crypto_done(crp);
1252 return (0);
1253 }
1254
1255 SEC_LOCK(sc, descriptors);
1256 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1257
1258 /* Block driver if there is no free descriptors or we are going down */
1259 if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) {
1260 sc->sc_blocked |= CRYPTO_SYMQ;
1261 SEC_UNLOCK(sc, descriptors);
1262 return (ERESTART);
1263 }
1264
1265 /* Prepare descriptor */
1266 desc = SEC_GET_FREE_DESC(sc);
1267 desc->sd_lt_used = 0;
1268 desc->sd_error = 0;
1269 desc->sd_crp = crp;
1270
1271 if (csp->csp_cipher_alg != 0)
1272 crypto_read_iv(crp, desc->sd_desc->shd_iv);
1273
1274 if (crp->crp_cipher_key != NULL)
1275 memcpy(ses->ss_key, crp->crp_cipher_key, csp->csp_cipher_klen);
1276
1277 if (crp->crp_auth_key != NULL)
1278 memcpy(ses->ss_mkey, crp->crp_auth_key, csp->csp_auth_klen);
1279
1280 memcpy(desc->sd_desc->shd_key, ses->ss_key, csp->csp_cipher_klen);
1281 memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, csp->csp_auth_klen);
1282
1283 error = ses->ss_eu->sem_make_desc(sc, csp, desc, crp);
1284
1285 if (error) {
1286 SEC_DESC_FREE_POINTERS(desc);
1287 SEC_DESC_PUT_BACK_LT(sc, desc);
1288 SEC_PUT_BACK_FREE_DESC(sc);
1289 SEC_UNLOCK(sc, descriptors);
1290 crp->crp_etype = error;
1291 crypto_done(crp);
1292 return (0);
1293 }
1294
1295 /*
1296 * Skip DONE interrupt if this is not last request in burst, but only
1297 * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE
1298 * signaling on each descriptor.
1299 */
1300 if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3)
1301 desc->sd_desc->shd_dn = 0;
1302 else
1303 desc->sd_desc->shd_dn = 1;
1304
1305 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1306 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD |
1307 BUS_DMASYNC_POSTWRITE);
1308 SEC_DESC_FREE2READY(sc);
1309 SEC_UNLOCK(sc, descriptors);
1310
1311 /* Enqueue ready descriptors in hardware */
1312 sec_enqueue(sc);
1313
1314 return (0);
1315 }
1316
1317 static int
sec_build_common_ns_desc(struct sec_softc * sc,struct sec_desc * desc,const struct crypto_session_params * csp,struct cryptop * crp)1318 sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc,
1319 const struct crypto_session_params *csp, struct cryptop *crp)
1320 {
1321 struct sec_hw_desc *hd = desc->sd_desc;
1322 int error;
1323
1324 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP;
1325 hd->shd_eu_sel1 = SEC_EU_NONE;
1326 hd->shd_mode1 = 0;
1327
1328 /* Pointer 0: NULL */
1329 error = sec_make_pointer_direct(sc, desc, 0, 0, 0);
1330 if (error)
1331 return (error);
1332
1333 /* Pointer 1: IV IN */
1334 error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr +
1335 offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen);
1336 if (error)
1337 return (error);
1338
1339 /* Pointer 2: Cipher Key */
1340 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
1341 offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen);
1342 if (error)
1343 return (error);
1344
1345 /* Pointer 3: Data IN */
1346 error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start,
1347 crp->crp_payload_length);
1348 if (error)
1349 return (error);
1350
1351 /* Pointer 4: Data OUT */
1352 error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start,
1353 crp->crp_payload_length);
1354 if (error)
1355 return (error);
1356
1357 /* Pointer 5: IV OUT (Not used: NULL) */
1358 error = sec_make_pointer_direct(sc, desc, 5, 0, 0);
1359 if (error)
1360 return (error);
1361
1362 /* Pointer 6: NULL */
1363 error = sec_make_pointer_direct(sc, desc, 6, 0, 0);
1364
1365 return (error);
1366 }
1367
1368 static int
sec_build_common_s_desc(struct sec_softc * sc,struct sec_desc * desc,const struct crypto_session_params * csp,struct cryptop * crp)1369 sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc,
1370 const struct crypto_session_params *csp, struct cryptop *crp)
1371 {
1372 struct sec_hw_desc *hd = desc->sd_desc;
1373 u_int eu, mode, hashlen;
1374 int error;
1375
1376 error = sec_mdeu_config(csp, &eu, &mode, &hashlen);
1377 if (error)
1378 return (error);
1379
1380 hd->shd_desc_type = SEC_DT_HMAC_SNOOP;
1381 hd->shd_eu_sel1 = eu;
1382 hd->shd_mode1 = mode;
1383
1384 /* Pointer 0: HMAC Key */
1385 error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr +
1386 offsetof(struct sec_hw_desc, shd_mkey), csp->csp_auth_klen);
1387 if (error)
1388 return (error);
1389
1390 /* Pointer 1: HMAC-Only Data IN */
1391 error = sec_make_pointer(sc, desc, 1, crp, crp->crp_aad_start,
1392 crp->crp_aad_length);
1393 if (error)
1394 return (error);
1395
1396 /* Pointer 2: Cipher Key */
1397 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
1398 offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen);
1399 if (error)
1400 return (error);
1401
1402 /* Pointer 3: IV IN */
1403 error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr +
1404 offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen);
1405 if (error)
1406 return (error);
1407
1408 /* Pointer 4: Data IN */
1409 error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start,
1410 crp->crp_payload_length);
1411 if (error)
1412 return (error);
1413
1414 /* Pointer 5: Data OUT */
1415 error = sec_make_pointer(sc, desc, 5, crp, crp->crp_payload_start,
1416 crp->crp_payload_length);
1417 if (error)
1418 return (error);
1419
1420 /* Pointer 6: HMAC OUT */
1421 error = sec_make_pointer_direct(sc, desc, 6, desc->sd_desc_paddr +
1422 offsetof(struct sec_hw_desc, shd_digest), hashlen);
1423
1424 return (error);
1425 }
1426
1427 /* AESU */
1428
1429 static bool
sec_aesu_newsession(const struct crypto_session_params * csp)1430 sec_aesu_newsession(const struct crypto_session_params *csp)
1431 {
1432
1433 return (csp->csp_cipher_alg == CRYPTO_AES_CBC);
1434 }
1435
1436 static int
sec_aesu_make_desc(struct sec_softc * sc,const struct crypto_session_params * csp,struct sec_desc * desc,struct cryptop * crp)1437 sec_aesu_make_desc(struct sec_softc *sc,
1438 const struct crypto_session_params *csp, struct sec_desc *desc,
1439 struct cryptop *crp)
1440 {
1441 struct sec_hw_desc *hd = desc->sd_desc;
1442 int error;
1443
1444 hd->shd_eu_sel0 = SEC_EU_AESU;
1445 hd->shd_mode0 = SEC_AESU_MODE_CBC;
1446
1447 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
1448 hd->shd_mode0 |= SEC_AESU_MODE_ED;
1449 hd->shd_dir = 0;
1450 } else
1451 hd->shd_dir = 1;
1452
1453 if (csp->csp_mode == CSP_MODE_ETA)
1454 error = sec_build_common_s_desc(sc, desc, csp, crp);
1455 else
1456 error = sec_build_common_ns_desc(sc, desc, csp, crp);
1457
1458 return (error);
1459 }
1460
1461 /* MDEU */
1462
1463 static bool
sec_mdeu_can_handle(u_int alg)1464 sec_mdeu_can_handle(u_int alg)
1465 {
1466 switch (alg) {
1467 case CRYPTO_SHA1:
1468 case CRYPTO_SHA1_HMAC:
1469 case CRYPTO_SHA2_256_HMAC:
1470 case CRYPTO_SHA2_384_HMAC:
1471 case CRYPTO_SHA2_512_HMAC:
1472 return (true);
1473 default:
1474 return (false);
1475 }
1476 }
1477
1478 static int
sec_mdeu_config(const struct crypto_session_params * csp,u_int * eu,u_int * mode,u_int * hashlen)1479 sec_mdeu_config(const struct crypto_session_params *csp, u_int *eu, u_int *mode,
1480 u_int *hashlen)
1481 {
1482
1483 *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT;
1484 *eu = SEC_EU_NONE;
1485
1486 switch (csp->csp_auth_alg) {
1487 case CRYPTO_SHA1_HMAC:
1488 *mode |= SEC_MDEU_MODE_HMAC;
1489 /* FALLTHROUGH */
1490 case CRYPTO_SHA1:
1491 *eu = SEC_EU_MDEU_A;
1492 *mode |= SEC_MDEU_MODE_SHA1;
1493 *hashlen = SHA1_HASH_LEN;
1494 break;
1495 case CRYPTO_SHA2_256_HMAC:
1496 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256;
1497 *eu = SEC_EU_MDEU_A;
1498 break;
1499 case CRYPTO_SHA2_384_HMAC:
1500 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384;
1501 *eu = SEC_EU_MDEU_B;
1502 break;
1503 case CRYPTO_SHA2_512_HMAC:
1504 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512;
1505 *eu = SEC_EU_MDEU_B;
1506 break;
1507 default:
1508 return (EINVAL);
1509 }
1510
1511 if (*mode & SEC_MDEU_MODE_HMAC)
1512 *hashlen = SEC_HMAC_HASH_LEN;
1513
1514 return (0);
1515 }
1516
1517 static bool
sec_mdeu_newsession(const struct crypto_session_params * csp)1518 sec_mdeu_newsession(const struct crypto_session_params *csp)
1519 {
1520
1521 return (sec_mdeu_can_handle(csp->csp_auth_alg));
1522 }
1523
1524 static int
sec_mdeu_make_desc(struct sec_softc * sc,const struct crypto_session_params * csp,struct sec_desc * desc,struct cryptop * crp)1525 sec_mdeu_make_desc(struct sec_softc *sc,
1526 const struct crypto_session_params *csp,
1527 struct sec_desc *desc, struct cryptop *crp)
1528 {
1529 struct sec_hw_desc *hd = desc->sd_desc;
1530 u_int eu, mode, hashlen;
1531 int error;
1532
1533 error = sec_mdeu_config(csp, &eu, &mode, &hashlen);
1534 if (error)
1535 return (error);
1536
1537 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP;
1538 hd->shd_eu_sel0 = eu;
1539 hd->shd_mode0 = mode;
1540 hd->shd_eu_sel1 = SEC_EU_NONE;
1541 hd->shd_mode1 = 0;
1542
1543 /* Pointer 0: NULL */
1544 error = sec_make_pointer_direct(sc, desc, 0, 0, 0);
1545 if (error)
1546 return (error);
1547
1548 /* Pointer 1: Context In (Not used: NULL) */
1549 error = sec_make_pointer_direct(sc, desc, 1, 0, 0);
1550 if (error)
1551 return (error);
1552
1553 /* Pointer 2: HMAC Key (or NULL, depending on digest type) */
1554 if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC)
1555 error = sec_make_pointer_direct(sc, desc, 2,
1556 desc->sd_desc_paddr + offsetof(struct sec_hw_desc,
1557 shd_mkey), csp->csp_auth_klen);
1558 else
1559 error = sec_make_pointer_direct(sc, desc, 2, 0, 0);
1560
1561 if (error)
1562 return (error);
1563
1564 /* Pointer 3: Input Data */
1565 error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start,
1566 crp->crp_payload_length);
1567 if (error)
1568 return (error);
1569
1570 /* Pointer 4: NULL */
1571 error = sec_make_pointer_direct(sc, desc, 4, 0, 0);
1572 if (error)
1573 return (error);
1574
1575 /* Pointer 5: Hash out */
1576 error = sec_make_pointer_direct(sc, desc, 5, desc->sd_desc_paddr +
1577 offsetof(struct sec_hw_desc, shd_digest), hashlen);
1578 if (error)
1579 return (error);
1580
1581 /* Pointer 6: NULL */
1582 error = sec_make_pointer_direct(sc, desc, 6, 0, 0);
1583
1584 return (0);
1585 }
1586