xref: /titanic_41/usr/src/uts/common/crypto/io/dca.c (revision 95014fbbfdc010ab9f3ed20db2154dc3322e9270)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 
28 /*
29  * Deimos - cryptographic acceleration based upon Broadcom 582x.
30  */
31 
32 #include <sys/types.h>
33 #include <sys/modctl.h>
34 #include <sys/conf.h>
35 #include <sys/devops.h>
36 #include <sys/ddi.h>
37 #include <sys/sunddi.h>
38 #include <sys/cmn_err.h>
39 #include <sys/varargs.h>
40 #include <sys/file.h>
41 #include <sys/stat.h>
42 #include <sys/kmem.h>
43 #include <sys/ioccom.h>
44 #include <sys/open.h>
45 #include <sys/cred.h>
46 #include <sys/kstat.h>
47 #include <sys/strsun.h>
48 #include <sys/note.h>
49 #include <sys/crypto/common.h>
50 #include <sys/crypto/spi.h>
51 #include <sys/ddifm.h>
52 #include <sys/fm/protocol.h>
53 #include <sys/fm/util.h>
54 #include <sys/fm/io/ddi.h>
55 #include <sys/crypto/dca.h>
56 
57 /*
58  * Core Deimos driver.
59  */
60 
61 static void		dca_enlist2(dca_listnode_t *, dca_listnode_t *,
62     kmutex_t *);
63 static void		dca_rmlist2(dca_listnode_t *node, kmutex_t *);
64 static dca_listnode_t	*dca_delist2(dca_listnode_t *q, kmutex_t *);
65 static void		dca_free_context_list(dca_t *dca);
66 static int		dca_free_context_low(crypto_ctx_t *ctx);
67 static int		dca_attach(dev_info_t *, ddi_attach_cmd_t);
68 static int		dca_detach(dev_info_t *, ddi_detach_cmd_t);
69 static int		dca_suspend(dca_t *);
70 static int		dca_resume(dca_t *);
71 static int		dca_init(dca_t *);
72 static int		dca_reset(dca_t *, int);
73 static int		dca_initworklist(dca_t *, dca_worklist_t *);
74 static void		dca_uninit(dca_t *);
75 static void		dca_initq(dca_listnode_t *);
76 static void		dca_enqueue(dca_listnode_t *, dca_listnode_t *);
77 static dca_listnode_t	*dca_dequeue(dca_listnode_t *);
78 static dca_listnode_t	*dca_unqueue(dca_listnode_t *);
79 static dca_request_t	*dca_newreq(dca_t *);
80 static dca_work_t	*dca_getwork(dca_t *, int);
81 static void		dca_freework(dca_work_t *);
82 static dca_work_t	*dca_newwork(dca_t *);
83 static void		dca_destroywork(dca_work_t *);
84 static void		dca_schedule(dca_t *, int);
85 static void		dca_reclaim(dca_t *, int);
86 static uint_t		dca_intr(char *);
87 static void		dca_failure(dca_t *, ddi_fault_location_t,
88 			    dca_fma_eclass_t index, uint64_t, int, char *, ...);
89 static void		dca_jobtimeout(void *);
90 static int		dca_drain(dca_t *);
91 static void		dca_undrain(dca_t *);
92 static void		dca_rejectjobs(dca_t *);
93 
94 #ifdef	SCHEDDELAY
95 static void		dca_schedtimeout(void *);
96 #endif
97 
98 /*
99  * We want these inlined for performance.
100  */
101 #ifndef	DEBUG
102 #pragma inline(dca_freereq, dca_getreq, dca_freework, dca_getwork)
103 #pragma inline(dca_enqueue, dca_dequeue, dca_rmqueue, dca_done)
104 #pragma inline(dca_reverse, dca_length)
105 #endif
106 
107 /*
108  * Device operations.
109  */
110 static struct dev_ops devops = {
111 	DEVO_REV,		/* devo_rev */
112 	0,			/* devo_refcnt */
113 	nodev,			/* devo_getinfo */
114 	nulldev,		/* devo_identify */
115 	nulldev,		/* devo_probe */
116 	dca_attach,		/* devo_attach */
117 	dca_detach,		/* devo_detach */
118 	nodev,			/* devo_reset */
119 	NULL,			/* devo_cb_ops */
120 	NULL,			/* devo_bus_ops */
121 	ddi_power,		/* devo_power */
122 	ddi_quiesce_not_supported,	/* devo_quiesce */
123 };
124 
125 #define	IDENT		"PCI Crypto Accelerator"
126 #define	IDENT_SYM	"Crypto Accel Sym 2.0"
127 #define	IDENT_ASYM	"Crypto Accel Asym 2.0"
128 
129 /* Space-padded, will be filled in dynamically during registration */
130 #define	IDENT3	"PCI Crypto Accelerator Mod 2.0"
131 
132 #define	VENDOR	"Sun Microsystems, Inc."
133 
134 #define	STALETIME	(30 * SECOND)
135 
136 #define	crypto_prov_notify	crypto_provider_notification
137 		/* A 28 char function name doesn't leave much line space */
138 
139 /*
140  * Module linkage.
141  */
142 static struct modldrv modldrv = {
143 	&mod_driverops,		/* drv_modops */
144 	IDENT,			/* drv_linkinfo */
145 	&devops,		/* drv_dev_ops */
146 };
147 
148 extern struct mod_ops mod_cryptoops;
149 
150 static struct modlcrypto modlcrypto = {
151 	&mod_cryptoops,
152 	IDENT3
153 };
154 
155 static struct modlinkage modlinkage = {
156 	MODREV_1,		/* ml_rev */
157 	&modldrv,		/* ml_linkage */
158 	&modlcrypto,
159 	NULL
160 };
161 
162 /*
163  * CSPI information (entry points, provider info, etc.)
164  */
165 
166 /* Mechanisms for the symmetric cipher provider */
167 static crypto_mech_info_t dca_mech_info_tab1[] = {
168 	/* DES-CBC */
169 	{SUN_CKM_DES_CBC, DES_CBC_MECH_INFO_TYPE,
170 	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT |
171 	    CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC,
172 	    DES_KEY_LEN, DES_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
173 	/* 3DES-CBC */
174 	{SUN_CKM_DES3_CBC, DES3_CBC_MECH_INFO_TYPE,
175 	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT |
176 	    CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC,
177 	    DES3_MIN_KEY_LEN, DES3_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BYTES}
178 };
179 
180 /* Mechanisms for the asymmetric cipher provider */
181 static crypto_mech_info_t dca_mech_info_tab2[] = {
182 	/* DSA */
183 	{SUN_CKM_DSA, DSA_MECH_INFO_TYPE,
184 	    CRYPTO_FG_SIGN | CRYPTO_FG_VERIFY |
185 	    CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_VERIFY_ATOMIC,
186 	    CRYPTO_BYTES2BITS(DSA_MIN_KEY_LEN),
187 	    CRYPTO_BYTES2BITS(DSA_MAX_KEY_LEN),
188 	    CRYPTO_KEYSIZE_UNIT_IN_BITS},
189 
190 	/* RSA */
191 	{SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE,
192 	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_SIGN |
193 	    CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_VERIFY |
194 	    CRYPTO_FG_VERIFY_RECOVER |
195 	    CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC |
196 	    CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
197 	    CRYPTO_FG_VERIFY_ATOMIC | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
198 	    CRYPTO_BYTES2BITS(RSA_MIN_KEY_LEN),
199 	    CRYPTO_BYTES2BITS(RSA_MAX_KEY_LEN),
200 	    CRYPTO_KEYSIZE_UNIT_IN_BITS},
201 	{SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE,
202 	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | CRYPTO_FG_SIGN |
203 	    CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_VERIFY |
204 	    CRYPTO_FG_VERIFY_RECOVER |
205 	    CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC |
206 	    CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
207 	    CRYPTO_FG_VERIFY_ATOMIC | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
208 	    CRYPTO_BYTES2BITS(RSA_MIN_KEY_LEN),
209 	    CRYPTO_BYTES2BITS(RSA_MAX_KEY_LEN),
210 	    CRYPTO_KEYSIZE_UNIT_IN_BITS}
211 };
212 
213 static void dca_provider_status(crypto_provider_handle_t, uint_t *);
214 
215 static crypto_control_ops_t dca_control_ops = {
216 	dca_provider_status
217 };
218 
219 static int dca_encrypt_init(crypto_ctx_t *, crypto_mechanism_t *,
220     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
221 static int dca_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
222     crypto_req_handle_t);
223 static int dca_encrypt_update(crypto_ctx_t *, crypto_data_t *,
224     crypto_data_t *, crypto_req_handle_t);
225 static int dca_encrypt_final(crypto_ctx_t *, crypto_data_t *,
226     crypto_req_handle_t);
227 static int dca_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
228     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
229     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
230 
231 static int dca_decrypt_init(crypto_ctx_t *, crypto_mechanism_t *,
232     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
233 static int dca_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
234     crypto_req_handle_t);
235 static int dca_decrypt_update(crypto_ctx_t *, crypto_data_t *,
236     crypto_data_t *, crypto_req_handle_t);
237 static int dca_decrypt_final(crypto_ctx_t *, crypto_data_t *,
238     crypto_req_handle_t);
239 static int dca_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
240     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
241     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
242 
243 static crypto_cipher_ops_t dca_cipher_ops = {
244 	dca_encrypt_init,
245 	dca_encrypt,
246 	dca_encrypt_update,
247 	dca_encrypt_final,
248 	dca_encrypt_atomic,
249 	dca_decrypt_init,
250 	dca_decrypt,
251 	dca_decrypt_update,
252 	dca_decrypt_final,
253 	dca_decrypt_atomic
254 };
255 
256 static int dca_sign_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *,
257     crypto_spi_ctx_template_t, crypto_req_handle_t);
258 static int dca_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
259     crypto_req_handle_t);
260 static int dca_sign_update(crypto_ctx_t *, crypto_data_t *,
261     crypto_req_handle_t);
262 static int dca_sign_final(crypto_ctx_t *, crypto_data_t *,
263     crypto_req_handle_t);
264 static int dca_sign_atomic(crypto_provider_handle_t, crypto_session_id_t,
265     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
266     crypto_spi_ctx_template_t, crypto_req_handle_t);
267 static int dca_sign_recover_init(crypto_ctx_t *, crypto_mechanism_t *,
268     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
269 static int dca_sign_recover(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
270     crypto_req_handle_t);
271 static int dca_sign_recover_atomic(crypto_provider_handle_t,
272     crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
273     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
274 
275 static crypto_sign_ops_t dca_sign_ops = {
276 	dca_sign_init,
277 	dca_sign,
278 	dca_sign_update,
279 	dca_sign_final,
280 	dca_sign_atomic,
281 	dca_sign_recover_init,
282 	dca_sign_recover,
283 	dca_sign_recover_atomic
284 };
285 
286 static int dca_verify_init(crypto_ctx_t *, crypto_mechanism_t *,
287     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
288 static int dca_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
289     crypto_req_handle_t);
290 static int dca_verify_update(crypto_ctx_t *, crypto_data_t *,
291     crypto_req_handle_t);
292 static int dca_verify_final(crypto_ctx_t *, crypto_data_t *,
293     crypto_req_handle_t);
294 static int dca_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
295     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
296     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
297 static int dca_verify_recover_init(crypto_ctx_t *, crypto_mechanism_t *,
298     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
299 static int dca_verify_recover(crypto_ctx_t *, crypto_data_t *,
300     crypto_data_t *, crypto_req_handle_t);
301 static int dca_verify_recover_atomic(crypto_provider_handle_t,
302     crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
303     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
304 
305 static crypto_verify_ops_t dca_verify_ops = {
306 	dca_verify_init,
307 	dca_verify,
308 	dca_verify_update,
309 	dca_verify_final,
310 	dca_verify_atomic,
311 	dca_verify_recover_init,
312 	dca_verify_recover,
313 	dca_verify_recover_atomic
314 };
315 
316 static int dca_generate_random(crypto_provider_handle_t, crypto_session_id_t,
317     uchar_t *, size_t, crypto_req_handle_t);
318 
319 static crypto_random_number_ops_t dca_random_number_ops = {
320 	NULL,
321 	dca_generate_random
322 };
323 
324 static int ext_info_sym(crypto_provider_handle_t prov,
325     crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq);
326 static int ext_info_asym(crypto_provider_handle_t prov,
327     crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq);
328 static int ext_info_base(crypto_provider_handle_t prov,
329     crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq, char *id);
330 
331 static crypto_provider_management_ops_t dca_provmanage_ops_1 = {
332 	ext_info_sym,		/* ext_info */
333 	NULL,			/* init_token */
334 	NULL,			/* init_pin */
335 	NULL			/* set_pin */
336 };
337 
338 static crypto_provider_management_ops_t dca_provmanage_ops_2 = {
339 	ext_info_asym,		/* ext_info */
340 	NULL,			/* init_token */
341 	NULL,			/* init_pin */
342 	NULL			/* set_pin */
343 };
344 
345 int dca_free_context(crypto_ctx_t *);
346 
347 static crypto_ctx_ops_t dca_ctx_ops = {
348 	NULL,
349 	dca_free_context
350 };
351 
352 /* Operations for the symmetric cipher provider */
353 static crypto_ops_t dca_crypto_ops1 = {
354 	&dca_control_ops,
355 	NULL,				/* digest_ops */
356 	&dca_cipher_ops,
357 	NULL,				/* mac_ops */
358 	NULL,				/* sign_ops */
359 	NULL,				/* verify_ops */
360 	NULL,				/* dual_ops */
361 	NULL,				/* cipher_mac_ops */
362 	NULL,				/* random_number_ops */
363 	NULL,				/* session_ops */
364 	NULL,				/* object_ops */
365 	NULL,				/* key_ops */
366 	&dca_provmanage_ops_1,		/* management_ops */
367 	&dca_ctx_ops
368 };
369 
370 /* Operations for the asymmetric cipher provider */
371 static crypto_ops_t dca_crypto_ops2 = {
372 	&dca_control_ops,
373 	NULL,				/* digest_ops */
374 	&dca_cipher_ops,
375 	NULL,				/* mac_ops */
376 	&dca_sign_ops,
377 	&dca_verify_ops,
378 	NULL,				/* dual_ops */
379 	NULL,				/* cipher_mac_ops */
380 	&dca_random_number_ops,
381 	NULL,				/* session_ops */
382 	NULL,				/* object_ops */
383 	NULL,				/* key_ops */
384 	&dca_provmanage_ops_2,		/* management_ops */
385 	&dca_ctx_ops
386 };
387 
388 /* Provider information for the symmetric cipher provider */
389 static crypto_provider_info_t dca_prov_info1 = {
390 	CRYPTO_SPI_VERSION_1,
391 	NULL,				/* pi_provider_description */
392 	CRYPTO_HW_PROVIDER,
393 	NULL,				/* pi_provider_dev */
394 	NULL,				/* pi_provider_handle */
395 	&dca_crypto_ops1,
396 	sizeof (dca_mech_info_tab1)/sizeof (crypto_mech_info_t),
397 	dca_mech_info_tab1,
398 	0,				/* pi_logical_provider_count */
399 	NULL				/* pi_logical_providers */
400 };
401 
402 /* Provider information for the asymmetric cipher provider */
403 static crypto_provider_info_t dca_prov_info2 = {
404 	CRYPTO_SPI_VERSION_1,
405 	NULL,				/* pi_provider_description */
406 	CRYPTO_HW_PROVIDER,
407 	NULL,				/* pi_provider_dev */
408 	NULL,				/* pi_provider_handle */
409 	&dca_crypto_ops2,
410 	sizeof (dca_mech_info_tab2)/sizeof (crypto_mech_info_t),
411 	dca_mech_info_tab2,
412 	0,				/* pi_logical_provider_count */
413 	NULL				/* pi_logical_providers */
414 };
415 
416 /* Convenience macros */
417 /* Retrieve the softc and instance number from a SPI crypto context */
418 #define	DCA_SOFTC_FROM_CTX(ctx, softc, instance) {		\
419 	(softc) = (dca_t *)(ctx)->cc_provider;			\
420 	(instance) = ddi_get_instance((softc)->dca_dip);	\
421 }
422 
423 #define	DCA_MECH_FROM_CTX(ctx) \
424 	(((dca_request_t *)(ctx)->cc_provider_private)->dr_ctx.ctx_cm_type)
425 
426 static int dca_bindchains_one(dca_request_t *reqp, size_t cnt, int dr_offset,
427     caddr_t kaddr, ddi_dma_handle_t handle, uint_t flags,
428     dca_chain_t *head, int *n_chain);
429 static uint64_t dca_ena(uint64_t ena);
430 static caddr_t dca_bufdaddr_out(crypto_data_t *data);
431 static char *dca_fma_eclass_string(char *model, dca_fma_eclass_t index);
432 static int dca_check_acc_handle(dca_t *dca, ddi_acc_handle_t handle,
433     dca_fma_eclass_t eclass_index);
434 
435 static void dca_fma_init(dca_t *dca);
436 static void dca_fma_fini(dca_t *dca);
437 static int dca_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
438     const void *impl_data);
439 
440 
441 static dca_device_t dca_devices[] = {
442 	/* Broadcom vanilla variants */
443 	{	0x14e4, 0x5820, "Broadcom 5820" },
444 	{	0x14e4, 0x5821, "Broadcom 5821" },
445 	{	0x14e4, 0x5822, "Broadcom 5822" },
446 	{	0x14e4, 0x5825, "Broadcom 5825" },
447 	/* Sun specific OEMd variants */
448 	{	0x108e, 0x5454, "SCA" },
449 	{	0x108e, 0x5455, "SCA 1000" },
450 	{	0x108e, 0x5457, "SCA 500" },
451 	/* subsysid should be 0x5457, but got 0x1 from HW. Assume both here. */
452 	{	0x108e, 0x1, "SCA 500" },
453 };
454 
455 /*
456  * Device attributes.
457  */
458 static struct ddi_device_acc_attr dca_regsattr = {
459 	DDI_DEVICE_ATTR_V1,
460 	DDI_STRUCTURE_LE_ACC,
461 	DDI_STRICTORDER_ACC,
462 	DDI_FLAGERR_ACC
463 };
464 
465 static struct ddi_device_acc_attr dca_devattr = {
466 	DDI_DEVICE_ATTR_V0,
467 	DDI_STRUCTURE_LE_ACC,
468 	DDI_STRICTORDER_ACC
469 };
470 
471 #if !defined(i386) && !defined(__i386)
472 static struct ddi_device_acc_attr dca_bufattr = {
473 	DDI_DEVICE_ATTR_V0,
474 	DDI_NEVERSWAP_ACC,
475 	DDI_STRICTORDER_ACC
476 };
477 #endif
478 
479 static struct ddi_dma_attr dca_dmaattr = {
480 	DMA_ATTR_V0,		/* dma_attr_version */
481 	0x0,			/* dma_attr_addr_lo */
482 	0xffffffffUL,		/* dma_attr_addr_hi */
483 	0x00ffffffUL,		/* dma_attr_count_max */
484 	0x40,			/* dma_attr_align */
485 	0x40,			/* dma_attr_burstsizes */
486 	0x1,			/* dma_attr_minxfer */
487 	0x00ffffffUL,		/* dma_attr_maxxfer */
488 	0xffffffffUL,		/* dma_attr_seg */
489 #if defined(i386) || defined(__i386) || defined(__amd64)
490 	512,			/* dma_attr_sgllen */
491 #else
492 	1,			/* dma_attr_sgllen */
493 #endif
494 	1,			/* dma_attr_granular */
495 	DDI_DMA_FLAGERR		/* dma_attr_flags */
496 };
497 
498 static void	*dca_state = NULL;
499 int	dca_mindma = 2500;
500 
501 /*
502  * FMA eclass string definitions. Note that these string arrays must be
503  * consistent with the dca_fma_eclass_t enum.
504  */
505 static char *dca_fma_eclass_sca1000[] = {
506 	"sca1000.hw.device",
507 	"sca1000.hw.timeout",
508 	"sca1000.none"
509 };
510 
511 static char *dca_fma_eclass_sca500[] = {
512 	"sca500.hw.device",
513 	"sca500.hw.timeout",
514 	"sca500.none"
515 };
516 
517 /*
518  * DDI entry points.
519  */
520 int
_init(void)521 _init(void)
522 {
523 	int rv;
524 
525 	DBG(NULL, DMOD, "dca: in _init");
526 
527 	if ((rv = ddi_soft_state_init(&dca_state, sizeof (dca_t), 1)) != 0) {
528 		/* this should *never* happen! */
529 		return (rv);
530 	}
531 
532 	if ((rv = mod_install(&modlinkage)) != 0) {
533 		/* cleanup here */
534 		ddi_soft_state_fini(&dca_state);
535 		return (rv);
536 	}
537 
538 	return (0);
539 }
540 
541 int
_fini(void)542 _fini(void)
543 {
544 	int rv;
545 
546 	DBG(NULL, DMOD, "dca: in _fini");
547 
548 	if ((rv = mod_remove(&modlinkage)) == 0) {
549 		/* cleanup here */
550 		ddi_soft_state_fini(&dca_state);
551 	}
552 	return (rv);
553 }
554 
555 int
_info(struct modinfo * modinfop)556 _info(struct modinfo *modinfop)
557 {
558 	DBG(NULL, DMOD, "dca: in _info");
559 
560 	return (mod_info(&modlinkage, modinfop));
561 }
562 
563 int
dca_attach(dev_info_t * dip,ddi_attach_cmd_t cmd)564 dca_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
565 {
566 	ddi_acc_handle_t	pci;
567 	int			instance;
568 	ddi_iblock_cookie_t	ibc;
569 	int			intr_added = 0;
570 	dca_t			*dca;
571 	ushort_t		venid;
572 	ushort_t		devid;
573 	ushort_t		revid;
574 	ushort_t		subsysid;
575 	ushort_t		subvenid;
576 	int			i;
577 	int			ret;
578 	char			ID[64];
579 	static char		*unknowndev = "Unknown device";
580 
581 #if DEBUG
582 	/* these are only used for debugging */
583 	ushort_t		pcicomm;
584 	ushort_t		pcistat;
585 	uchar_t			cachelinesz;
586 	uchar_t			mingnt;
587 	uchar_t			maxlat;
588 	uchar_t			lattmr;
589 #endif
590 
591 	instance = ddi_get_instance(dip);
592 
593 	DBG(NULL, DMOD, "dca: in dca_attach() for %d", instance);
594 
595 	switch (cmd) {
596 	case DDI_RESUME:
597 		if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) {
598 			dca_diperror(dip, "no soft state in detach");
599 			return (DDI_FAILURE);
600 		}
601 		/* assumption: we won't be DDI_DETACHed until we return */
602 		return (dca_resume(dca));
603 	case DDI_ATTACH:
604 		break;
605 	default:
606 		return (DDI_FAILURE);
607 	}
608 
609 	if (ddi_slaveonly(dip) == DDI_SUCCESS) {
610 		dca_diperror(dip, "slot does not support PCI bus-master");
611 		return (DDI_FAILURE);
612 	}
613 
614 	if (ddi_intr_hilevel(dip, 0) != 0) {
615 		dca_diperror(dip, "hilevel interrupts not supported");
616 		return (DDI_FAILURE);
617 	}
618 
619 	if (pci_config_setup(dip, &pci) != DDI_SUCCESS) {
620 		dca_diperror(dip, "unable to setup PCI config handle");
621 		return (DDI_FAILURE);
622 	}
623 
624 	/* common PCI attributes */
625 	venid = pci_config_get16(pci, PCI_VENID);
626 	devid = pci_config_get16(pci, PCI_DEVID);
627 	revid = pci_config_get8(pci, PCI_REVID);
628 	subvenid = pci_config_get16(pci, PCI_SUBVENID);
629 	subsysid = pci_config_get16(pci, PCI_SUBSYSID);
630 
631 	/*
632 	 * Broadcom-specific timings.
633 	 * We disable these timers/counters since they can cause
634 	 * incorrect false failures when the bus is just a little
635 	 * bit slow, or busy.
636 	 */
637 	pci_config_put8(pci, PCI_TRDYTO, 0);
638 	pci_config_put8(pci, PCI_RETRIES, 0);
639 
640 	/* initialize PCI access settings */
641 	pci_config_put16(pci, PCI_COMM, PCICOMM_SEE |
642 	    PCICOMM_PEE | PCICOMM_BME | PCICOMM_MAE);
643 
644 	/* set up our PCI latency timer */
645 	pci_config_put8(pci, PCI_LATTMR, 0x40);
646 
647 #if DEBUG
648 	/* read registers (for debugging) */
649 	pcicomm = pci_config_get16(pci, PCI_COMM);
650 	pcistat = pci_config_get16(pci, PCI_STATUS);
651 	cachelinesz = pci_config_get8(pci, PCI_CACHELINESZ);
652 	mingnt = pci_config_get8(pci, PCI_MINGNT);
653 	maxlat = pci_config_get8(pci, PCI_MAXLAT);
654 	lattmr = pci_config_get8(pci, PCI_LATTMR);
655 #endif
656 
657 	pci_config_teardown(&pci);
658 
659 	if (ddi_get_iblock_cookie(dip, 0, &ibc) != DDI_SUCCESS) {
660 		dca_diperror(dip, "unable to get iblock cookie");
661 		return (DDI_FAILURE);
662 	}
663 
664 	if (ddi_soft_state_zalloc(dca_state, instance) != DDI_SUCCESS) {
665 		dca_diperror(dip, "unable to allocate soft state");
666 		return (DDI_FAILURE);
667 	}
668 
669 	dca = ddi_get_soft_state(dca_state, instance);
670 	ASSERT(dca != NULL);
671 	dca->dca_dip = dip;
672 	WORKLIST(dca, MCR1)->dwl_prov = NULL;
673 	WORKLIST(dca, MCR2)->dwl_prov = NULL;
674 	/* figure pagesize */
675 	dca->dca_pagesize = ddi_ptob(dip, 1);
676 
677 	/*
678 	 * Search for the device in our supported devices table.  This
679 	 * is here for two reasons.  First, we want to ensure that
680 	 * only Sun-qualified (and presumably Sun-labeled) devices can
681 	 * be used with this driver.  Second, some devices have
682 	 * specific differences.  E.g. the 5821 has support for a
683 	 * special mode of RC4, deeper queues, power management, and
684 	 * other changes.  Also, the export versions of some of these
685 	 * chips don't support RC4 or 3DES, so we catch that here.
686 	 *
687 	 * Note that we only look at the upper nibble of the device
688 	 * id, which is used to distinguish export vs. domestic
689 	 * versions of the chip.  (The lower nibble is used for
690 	 * stepping information.)
691 	 */
692 	for (i = 0; i < (sizeof (dca_devices) / sizeof (dca_device_t)); i++) {
693 		/*
694 		 * Try to match the subsystem information first.
695 		 */
696 		if (subvenid && (subvenid == dca_devices[i].dd_vendor_id) &&
697 		    subsysid && (subsysid == dca_devices[i].dd_device_id)) {
698 			dca->dca_model = dca_devices[i].dd_model;
699 			dca->dca_devid = dca_devices[i].dd_device_id;
700 			break;
701 		}
702 		/*
703 		 * Failing that, try the generic vendor and device id.
704 		 * Even if we find a match, we keep searching anyway,
705 		 * since we would prefer to find a match based on the
706 		 * subsystem ids.
707 		 */
708 		if ((venid == dca_devices[i].dd_vendor_id) &&
709 		    (devid == dca_devices[i].dd_device_id)) {
710 			dca->dca_model = dca_devices[i].dd_model;
711 			dca->dca_devid = dca_devices[i].dd_device_id;
712 		}
713 	}
714 	/* try and handle an unrecognized device */
715 	if (dca->dca_model == NULL) {
716 		dca->dca_model = unknowndev;
717 		dca_error(dca, "device not recognized, not supported");
718 		DBG(dca, DPCI, "i=%d venid=%x devid=%x rev=%d",
719 		    i, venid, devid, revid);
720 	}
721 
722 	if (ddi_prop_update_string(DDI_DEV_T_NONE, dip, "description",
723 	    dca->dca_model) != DDI_SUCCESS) {
724 		dca_error(dca, "unable to create description property");
725 		return (DDI_FAILURE);
726 	}
727 
728 	DBG(dca, DPCI, "PCI command=0x%x status=%x cachelinesz=%x",
729 	    pcicomm, pcistat, cachelinesz);
730 	DBG(dca, DPCI, "mingnt=0x%x maxlat=0x%x lattmr=0x%x",
731 	    mingnt, maxlat, lattmr);
732 
733 	/*
734 	 * initialize locks, etc.
735 	 */
736 	(void) mutex_init(&dca->dca_intrlock, NULL, MUTEX_DRIVER, ibc);
737 
738 	/* use RNGSHA1 by default */
739 	if (ddi_getprop(DDI_DEV_T_ANY, dip,
740 	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "rngdirect", 0) == 0) {
741 		dca->dca_flags |= DCA_RNGSHA1;
742 	}
743 
744 	/* initialize FMA */
745 	dca_fma_init(dca);
746 
747 	/* initialize some key data structures */
748 	if (dca_init(dca) != DDI_SUCCESS) {
749 		goto failed;
750 	}
751 
752 	/* initialize kstats */
753 	dca_ksinit(dca);
754 
755 	/* setup access to registers */
756 	if (ddi_regs_map_setup(dip, 1, (caddr_t *)&dca->dca_regs,
757 	    0, 0, &dca_regsattr, &dca->dca_regs_handle) != DDI_SUCCESS) {
758 		dca_error(dca, "unable to map registers");
759 		goto failed;
760 	}
761 
762 	DBG(dca, DCHATTY, "MCR1 = %x", GETCSR(dca, CSR_MCR1));
763 	DBG(dca, DCHATTY, "CONTROL = %x", GETCSR(dca, CSR_DMACTL));
764 	DBG(dca, DCHATTY, "STATUS = %x", GETCSR(dca, CSR_DMASTAT));
765 	DBG(dca, DCHATTY, "DMAEA = %x", GETCSR(dca, CSR_DMAEA));
766 	DBG(dca, DCHATTY, "MCR2 = %x", GETCSR(dca, CSR_MCR2));
767 
768 	/* reset the chip */
769 	if (dca_reset(dca, 0) < 0) {
770 		goto failed;
771 	}
772 
773 	/* initialize the chip */
774 	PUTCSR(dca, CSR_DMACTL, DMACTL_BE32 | DMACTL_BE64);
775 	if (dca_check_acc_handle(dca, dca->dca_regs_handle,
776 	    DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
777 		goto failed;
778 	}
779 
780 	/* add the interrupt */
781 	if (ddi_add_intr(dip, 0, &dca->dca_icookie, NULL, dca_intr,
782 	    (void *)dca) != DDI_SUCCESS) {
783 		DBG(dca, DWARN, "ddi_add_intr failed");
784 		goto failed;
785 	} else {
786 		intr_added = 1;
787 	}
788 
789 	/* enable interrupts on the device */
790 	/*
791 	 * XXX: Note, 5820A1 errata indicates that this may clobber
792 	 * bits 24 and 23, which affect the speed of the RNG.  Since
793 	 * we always want to run in full-speed mode, this should be
794 	 * harmless.
795 	 */
796 	if (dca->dca_devid == 0x5825) {
797 		/* for 5825 - increase the DMA read size */
798 		SETBIT(dca, CSR_DMACTL,
799 		    DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE | DMACTL_RD256);
800 	} else {
801 		SETBIT(dca, CSR_DMACTL,
802 		    DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
803 	}
804 	if (dca_check_acc_handle(dca, dca->dca_regs_handle,
805 	    DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
806 		goto failed;
807 	}
808 
809 	/* register MCR1 with the crypto framework */
810 	/* Be careful not to exceed 32 chars */
811 	(void) sprintf(ID, "%s/%d %s",
812 	    ddi_driver_name(dip), ddi_get_instance(dip), IDENT_SYM);
813 	dca_prov_info1.pi_provider_description = ID;
814 	dca_prov_info1.pi_provider_dev.pd_hw = dip;
815 	dca_prov_info1.pi_provider_handle = dca;
816 	if ((ret = crypto_register_provider(&dca_prov_info1,
817 	    &WORKLIST(dca, MCR1)->dwl_prov)) != CRYPTO_SUCCESS) {
818 		cmn_err(CE_WARN,
819 		    "crypto_register_provider() failed (%d) for MCR1", ret);
820 		goto failed;
821 	}
822 
823 	/* register MCR2 with the crypto framework */
824 	/* Be careful not to exceed 32 chars */
825 	(void) sprintf(ID, "%s/%d %s",
826 	    ddi_driver_name(dip), ddi_get_instance(dip), IDENT_ASYM);
827 	dca_prov_info2.pi_provider_description = ID;
828 	dca_prov_info2.pi_provider_dev.pd_hw = dip;
829 	dca_prov_info2.pi_provider_handle = dca;
830 	if ((ret = crypto_register_provider(&dca_prov_info2,
831 	    &WORKLIST(dca, MCR2)->dwl_prov)) != CRYPTO_SUCCESS) {
832 		cmn_err(CE_WARN,
833 		    "crypto_register_provider() failed (%d) for MCR2", ret);
834 		goto failed;
835 	}
836 
837 	crypto_prov_notify(WORKLIST(dca, MCR1)->dwl_prov,
838 	    CRYPTO_PROVIDER_READY);
839 	crypto_prov_notify(WORKLIST(dca, MCR2)->dwl_prov,
840 	    CRYPTO_PROVIDER_READY);
841 
842 	/* Initialize the local random number pool for this instance */
843 	if ((ret = dca_random_init(dca)) != CRYPTO_SUCCESS) {
844 		goto failed;
845 	}
846 
847 	mutex_enter(&dca->dca_intrlock);
848 	dca->dca_jobtid = timeout(dca_jobtimeout, (void *)dca,
849 	    drv_usectohz(SECOND));
850 	mutex_exit(&dca->dca_intrlock);
851 
852 	ddi_set_driver_private(dip, (caddr_t)dca);
853 
854 	ddi_report_dev(dip);
855 
856 	if (ddi_get_devstate(dca->dca_dip) != DDI_DEVSTATE_UP) {
857 		ddi_fm_service_impact(dca->dca_dip, DDI_SERVICE_RESTORED);
858 	}
859 
860 	return (DDI_SUCCESS);
861 
862 failed:
863 	/* unregister from the crypto framework */
864 	if (WORKLIST(dca, MCR1)->dwl_prov != NULL) {
865 		(void) crypto_unregister_provider(
866 		    WORKLIST(dca, MCR1)->dwl_prov);
867 	}
868 	if (WORKLIST(dca, MCR2)->dwl_prov != NULL) {
869 		(void) crypto_unregister_provider(
870 		    WORKLIST(dca, MCR2)->dwl_prov);
871 	}
872 	if (intr_added) {
873 		CLRBIT(dca, CSR_DMACTL,
874 		    DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
875 		/* unregister intr handler */
876 		ddi_remove_intr(dip, 0, dca->dca_icookie);
877 	}
878 	if (dca->dca_regs_handle) {
879 		ddi_regs_map_free(&dca->dca_regs_handle);
880 	}
881 	if (dca->dca_intrstats) {
882 		kstat_delete(dca->dca_intrstats);
883 	}
884 	if (dca->dca_ksp) {
885 		kstat_delete(dca->dca_ksp);
886 	}
887 	dca_uninit(dca);
888 
889 	/* finalize FMA */
890 	dca_fma_fini(dca);
891 
892 	mutex_destroy(&dca->dca_intrlock);
893 	ddi_soft_state_free(dca_state, instance);
894 	return (DDI_FAILURE);
895 
896 }
897 
898 int
dca_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)899 dca_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
900 {
901 	int		instance;
902 	dca_t		*dca;
903 	timeout_id_t	tid;
904 
905 	instance = ddi_get_instance(dip);
906 
907 	DBG(NULL, DMOD, "dca: in dca_detach() for %d", instance);
908 
909 	switch (cmd) {
910 	case DDI_SUSPEND:
911 		if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) {
912 			dca_diperror(dip, "no soft state in detach");
913 			return (DDI_FAILURE);
914 		}
915 		/* assumption: we won't be DDI_DETACHed until we return */
916 		return (dca_suspend(dca));
917 
918 	case DDI_DETACH:
919 		break;
920 	default:
921 		return (DDI_FAILURE);
922 	}
923 
924 	if ((dca = (dca_t *)ddi_get_driver_private(dip)) == NULL) {
925 		dca_diperror(dip, "no soft state in detach");
926 		return (DDI_FAILURE);
927 	}
928 
929 	/*
930 	 * Unregister from kCF.
931 	 * This needs to be done at the beginning of detach.
932 	 */
933 	if (WORKLIST(dca, MCR1)->dwl_prov != NULL) {
934 		if (crypto_unregister_provider(
935 		    WORKLIST(dca, MCR1)->dwl_prov) != CRYPTO_SUCCESS) {
936 			dca_error(dca, "unable to unregister MCR1 from kcf");
937 			return (DDI_FAILURE);
938 		}
939 	}
940 
941 	if (WORKLIST(dca, MCR2)->dwl_prov != NULL) {
942 		if (crypto_unregister_provider(
943 		    WORKLIST(dca, MCR2)->dwl_prov) != CRYPTO_SUCCESS) {
944 			dca_error(dca, "unable to unregister MCR2 from kcf");
945 			return (DDI_FAILURE);
946 		}
947 	}
948 
949 	/*
950 	 * Cleanup the private context list. Once the
951 	 * crypto_unregister_provider returns, it is safe to do so.
952 	 */
953 	dca_free_context_list(dca);
954 
955 	/* Cleanup the local random number pool */
956 	dca_random_fini(dca);
957 
958 	/* send any jobs in the waitq back to kCF */
959 	dca_rejectjobs(dca);
960 
961 	/* untimeout the timeouts */
962 	mutex_enter(&dca->dca_intrlock);
963 	tid = dca->dca_jobtid;
964 	dca->dca_jobtid = 0;
965 	mutex_exit(&dca->dca_intrlock);
966 	if (tid) {
967 		(void) untimeout(tid);
968 	}
969 
970 	/* disable device interrupts */
971 	CLRBIT(dca, CSR_DMACTL, DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
972 
973 	/* unregister interrupt handlers */
974 	ddi_remove_intr(dip, 0, dca->dca_icookie);
975 
976 	/* release our regs handle */
977 	ddi_regs_map_free(&dca->dca_regs_handle);
978 
979 	/* toss out kstats */
980 	if (dca->dca_intrstats) {
981 		kstat_delete(dca->dca_intrstats);
982 	}
983 	if (dca->dca_ksp) {
984 		kstat_delete(dca->dca_ksp);
985 	}
986 
987 	mutex_destroy(&dca->dca_intrlock);
988 	dca_uninit(dca);
989 
990 	/* finalize FMA */
991 	dca_fma_fini(dca);
992 
993 	ddi_soft_state_free(dca_state, instance);
994 
995 	return (DDI_SUCCESS);
996 }
997 
998 int
dca_resume(dca_t * dca)999 dca_resume(dca_t *dca)
1000 {
1001 	ddi_acc_handle_t	pci;
1002 
1003 	if (pci_config_setup(dca->dca_dip, &pci) != DDI_SUCCESS) {
1004 		dca_error(dca, "unable to setup PCI config handle");
1005 		return (DDI_FAILURE);
1006 	}
1007 
1008 	/*
1009 	 * Reprogram registers in PCI configuration space.
1010 	 */
1011 
1012 	/* Broadcom-specific timers -- we disable them. */
1013 	pci_config_put8(pci, PCI_TRDYTO, 0);
1014 	pci_config_put8(pci, PCI_RETRIES, 0);
1015 
1016 	/* initialize PCI access settings */
1017 	pci_config_put16(pci, PCI_COMM, PCICOMM_SEE |
1018 	    PCICOMM_PEE | PCICOMM_BME | PCICOMM_MAE);
1019 
1020 	/* set up our PCI latency timer */
1021 	pci_config_put8(pci, PCI_LATTMR, 0x40);
1022 
1023 	pci_config_teardown(&pci);
1024 
1025 	if (dca_reset(dca, 0) < 0) {
1026 		dca_error(dca, "unable to reset device during resume");
1027 		return (DDI_FAILURE);
1028 	}
1029 
1030 	/*
1031 	 * Now restore the card-specific CSRs.
1032 	 */
1033 
1034 	/* restore endianness settings */
1035 	PUTCSR(dca, CSR_DMACTL, DMACTL_BE32 | DMACTL_BE64);
1036 	if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1037 	    DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
1038 		return (DDI_FAILURE);
1039 
1040 	/* restore interrupt enables */
1041 	if (dca->dca_devid == 0x5825) {
1042 		/* for 5825 set 256 byte read size to improve performance */
1043 		SETBIT(dca, CSR_DMACTL,
1044 		    DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE | DMACTL_RD256);
1045 	} else {
1046 		SETBIT(dca, CSR_DMACTL,
1047 		    DMACTL_MCR1IE | DMACTL_MCR2IE | DMACTL_EIE);
1048 	}
1049 	if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1050 	    DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
1051 		return (DDI_FAILURE);
1052 
1053 	/* resume scheduling jobs on the device */
1054 	dca_undrain(dca);
1055 
1056 	return (DDI_SUCCESS);
1057 }
1058 
1059 int
dca_suspend(dca_t * dca)1060 dca_suspend(dca_t *dca)
1061 {
1062 	if ((dca_drain(dca)) != 0) {
1063 		return (DDI_FAILURE);
1064 	}
1065 	if (dca_reset(dca, 0) < 0) {
1066 		dca_error(dca, "unable to reset device during suspend");
1067 		return (DDI_FAILURE);
1068 	}
1069 	return (DDI_SUCCESS);
1070 }
1071 
1072 /*
1073  * Hardware access stuff.
1074  */
1075 int
dca_reset(dca_t * dca,int failreset)1076 dca_reset(dca_t *dca, int failreset)
1077 {
1078 	int i;
1079 
1080 	if (dca->dca_regs_handle == NULL) {
1081 		return (-1);
1082 	}
1083 
1084 	PUTCSR(dca, CSR_DMACTL, DMACTL_RESET);
1085 	if (!failreset) {
1086 		if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1087 		    DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
1088 			return (-1);
1089 	}
1090 
1091 	/* now wait for a reset */
1092 	for (i = 1; i < 100; i++) {
1093 		uint32_t	dmactl;
1094 		drv_usecwait(100);
1095 		dmactl = GETCSR(dca, CSR_DMACTL);
1096 		if (!failreset) {
1097 			if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1098 			    DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
1099 				return (-1);
1100 		}
1101 		if ((dmactl & DMACTL_RESET) == 0) {
1102 			DBG(dca, DCHATTY, "reset in %d usec", i * 100);
1103 			return (0);
1104 		}
1105 	}
1106 	if (!failreset) {
1107 		dca_failure(dca, DDI_DEVICE_FAULT,
1108 		    DCA_FM_ECLASS_NONE, dca_ena(0), CRYPTO_DEVICE_ERROR,
1109 		    "timeout waiting for reset after %d usec", i * 100);
1110 	}
1111 	return (-1);
1112 }
1113 
1114 int
dca_initworklist(dca_t * dca,dca_worklist_t * wlp)1115 dca_initworklist(dca_t *dca, dca_worklist_t *wlp)
1116 {
1117 	int	i;
1118 	int	reqprealloc = wlp->dwl_hiwater + (MAXWORK * MAXREQSPERMCR);
1119 
1120 	/*
1121 	 * Set up work queue.
1122 	 */
1123 	mutex_init(&wlp->dwl_lock, NULL, MUTEX_DRIVER, dca->dca_icookie);
1124 	mutex_init(&wlp->dwl_freereqslock, NULL, MUTEX_DRIVER,
1125 	    dca->dca_icookie);
1126 	mutex_init(&wlp->dwl_freelock, NULL, MUTEX_DRIVER, dca->dca_icookie);
1127 	cv_init(&wlp->dwl_cv, NULL, CV_DRIVER, NULL);
1128 
1129 	mutex_enter(&wlp->dwl_lock);
1130 
1131 	dca_initq(&wlp->dwl_freereqs);
1132 	dca_initq(&wlp->dwl_waitq);
1133 	dca_initq(&wlp->dwl_freework);
1134 	dca_initq(&wlp->dwl_runq);
1135 
1136 	for (i = 0; i < MAXWORK; i++) {
1137 		dca_work_t		*workp;
1138 
1139 		if ((workp = dca_newwork(dca)) == NULL) {
1140 			dca_error(dca, "unable to allocate work");
1141 			mutex_exit(&wlp->dwl_lock);
1142 			return (DDI_FAILURE);
1143 		}
1144 		workp->dw_wlp = wlp;
1145 		dca_freework(workp);
1146 	}
1147 	mutex_exit(&wlp->dwl_lock);
1148 
1149 	for (i = 0; i < reqprealloc; i++) {
1150 		dca_request_t *reqp;
1151 
1152 		if ((reqp = dca_newreq(dca)) == NULL) {
1153 			dca_error(dca, "unable to allocate request");
1154 			return (DDI_FAILURE);
1155 		}
1156 		reqp->dr_dca = dca;
1157 		reqp->dr_wlp = wlp;
1158 		dca_freereq(reqp);
1159 	}
1160 	return (DDI_SUCCESS);
1161 }
1162 
1163 int
dca_init(dca_t * dca)1164 dca_init(dca_t *dca)
1165 {
1166 	dca_worklist_t		*wlp;
1167 
1168 	/* Initialize the private context list and the corresponding lock. */
1169 	mutex_init(&dca->dca_ctx_list_lock, NULL, MUTEX_DRIVER, NULL);
1170 	dca_initq(&dca->dca_ctx_list);
1171 
1172 	/*
1173 	 * MCR1 algorithms.
1174 	 */
1175 	wlp = WORKLIST(dca, MCR1);
1176 	(void) sprintf(wlp->dwl_name, "dca%d:mcr1",
1177 	    ddi_get_instance(dca->dca_dip));
1178 	wlp->dwl_lowater = ddi_getprop(DDI_DEV_T_ANY,
1179 	    dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1180 	    "mcr1_lowater", MCR1LOWATER);
1181 	wlp->dwl_hiwater = ddi_getprop(DDI_DEV_T_ANY,
1182 	    dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1183 	    "mcr1_hiwater", MCR1HIWATER);
1184 	wlp->dwl_reqspermcr = min(ddi_getprop(DDI_DEV_T_ANY,
1185 	    dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1186 	    "mcr1_maxreqs", MCR1MAXREQS), MAXREQSPERMCR);
1187 	wlp->dwl_dca = dca;
1188 	wlp->dwl_mcr = MCR1;
1189 	if (dca_initworklist(dca, wlp) != DDI_SUCCESS) {
1190 		return (DDI_FAILURE);
1191 	}
1192 
1193 	/*
1194 	 * MCR2 algorithms.
1195 	 */
1196 	wlp = WORKLIST(dca, MCR2);
1197 	(void) sprintf(wlp->dwl_name, "dca%d:mcr2",
1198 	    ddi_get_instance(dca->dca_dip));
1199 	wlp->dwl_lowater = ddi_getprop(DDI_DEV_T_ANY,
1200 	    dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1201 	    "mcr2_lowater", MCR2LOWATER);
1202 	wlp->dwl_hiwater = ddi_getprop(DDI_DEV_T_ANY,
1203 	    dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1204 	    "mcr2_hiwater", MCR2HIWATER);
1205 	wlp->dwl_reqspermcr = min(ddi_getprop(DDI_DEV_T_ANY,
1206 	    dca->dca_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
1207 	    "mcr2_maxreqs", MCR2MAXREQS), MAXREQSPERMCR);
1208 	wlp->dwl_dca = dca;
1209 	wlp->dwl_mcr = MCR2;
1210 	if (dca_initworklist(dca, wlp) != DDI_SUCCESS) {
1211 		return (DDI_FAILURE);
1212 	}
1213 	return (DDI_SUCCESS);
1214 }
1215 
1216 /*
1217  * Uninitialize worklists.  This routine should only be called when no
1218  * active jobs (hence DMA mappings) exist.  One way to ensure this is
1219  * to unregister from kCF before calling this routine.  (This is done
1220  * e.g. in detach(9e).)
1221  */
1222 void
dca_uninit(dca_t * dca)1223 dca_uninit(dca_t *dca)
1224 {
1225 	int	mcr;
1226 
1227 	mutex_destroy(&dca->dca_ctx_list_lock);
1228 
1229 	for (mcr = MCR1; mcr <= MCR2; mcr++) {
1230 		dca_worklist_t	*wlp = WORKLIST(dca, mcr);
1231 		dca_work_t	*workp;
1232 		dca_request_t	*reqp;
1233 
1234 		if (dca->dca_regs_handle == NULL) {
1235 			continue;
1236 		}
1237 
1238 		mutex_enter(&wlp->dwl_lock);
1239 		while ((workp = dca_getwork(dca, mcr)) != NULL) {
1240 			dca_destroywork(workp);
1241 		}
1242 		mutex_exit(&wlp->dwl_lock);
1243 		while ((reqp = dca_getreq(dca, mcr, 0)) != NULL) {
1244 			dca_destroyreq(reqp);
1245 		}
1246 
1247 		mutex_destroy(&wlp->dwl_lock);
1248 		mutex_destroy(&wlp->dwl_freereqslock);
1249 		mutex_destroy(&wlp->dwl_freelock);
1250 		cv_destroy(&wlp->dwl_cv);
1251 		wlp->dwl_prov = NULL;
1252 	}
1253 }
1254 
1255 static void
dca_enlist2(dca_listnode_t * q,dca_listnode_t * node,kmutex_t * lock)1256 dca_enlist2(dca_listnode_t *q, dca_listnode_t *node, kmutex_t *lock)
1257 {
1258 	if (!q || !node)
1259 		return;
1260 
1261 	mutex_enter(lock);
1262 	node->dl_next2 = q;
1263 	node->dl_prev2 = q->dl_prev2;
1264 	node->dl_next2->dl_prev2 = node;
1265 	node->dl_prev2->dl_next2 = node;
1266 	mutex_exit(lock);
1267 }
1268 
1269 static void
dca_rmlist2(dca_listnode_t * node,kmutex_t * lock)1270 dca_rmlist2(dca_listnode_t *node, kmutex_t *lock)
1271 {
1272 	if (!node)
1273 		return;
1274 
1275 	mutex_enter(lock);
1276 	node->dl_next2->dl_prev2 = node->dl_prev2;
1277 	node->dl_prev2->dl_next2 = node->dl_next2;
1278 	node->dl_next2 = NULL;
1279 	node->dl_prev2 = NULL;
1280 	mutex_exit(lock);
1281 }
1282 
1283 static dca_listnode_t *
dca_delist2(dca_listnode_t * q,kmutex_t * lock)1284 dca_delist2(dca_listnode_t *q, kmutex_t *lock)
1285 {
1286 	dca_listnode_t *node;
1287 
1288 	mutex_enter(lock);
1289 	if ((node = q->dl_next2) == q) {
1290 		mutex_exit(lock);
1291 		return (NULL);
1292 	}
1293 
1294 	node->dl_next2->dl_prev2 = node->dl_prev2;
1295 	node->dl_prev2->dl_next2 = node->dl_next2;
1296 	node->dl_next2 = NULL;
1297 	node->dl_prev2 = NULL;
1298 	mutex_exit(lock);
1299 
1300 	return (node);
1301 }
1302 
1303 void
dca_initq(dca_listnode_t * q)1304 dca_initq(dca_listnode_t *q)
1305 {
1306 	q->dl_next = q;
1307 	q->dl_prev = q;
1308 	q->dl_next2 = q;
1309 	q->dl_prev2 = q;
1310 }
1311 
1312 void
dca_enqueue(dca_listnode_t * q,dca_listnode_t * node)1313 dca_enqueue(dca_listnode_t *q, dca_listnode_t *node)
1314 {
1315 	/*
1316 	 * Enqueue submits at the "tail" of the list, i.e. just
1317 	 * behind the sentinel.
1318 	 */
1319 	node->dl_next = q;
1320 	node->dl_prev = q->dl_prev;
1321 	node->dl_next->dl_prev = node;
1322 	node->dl_prev->dl_next = node;
1323 }
1324 
1325 void
dca_rmqueue(dca_listnode_t * node)1326 dca_rmqueue(dca_listnode_t *node)
1327 {
1328 	node->dl_next->dl_prev = node->dl_prev;
1329 	node->dl_prev->dl_next = node->dl_next;
1330 	node->dl_next = NULL;
1331 	node->dl_prev = NULL;
1332 }
1333 
1334 dca_listnode_t *
dca_dequeue(dca_listnode_t * q)1335 dca_dequeue(dca_listnode_t *q)
1336 {
1337 	dca_listnode_t *node;
1338 	/*
1339 	 * Dequeue takes from the "head" of the list, i.e. just after
1340 	 * the sentinel.
1341 	 */
1342 	if ((node = q->dl_next) == q) {
1343 		/* queue is empty */
1344 		return (NULL);
1345 	}
1346 	dca_rmqueue(node);
1347 	return (node);
1348 }
1349 
1350 /* this is the opposite of dequeue, it takes things off in LIFO order */
1351 dca_listnode_t *
dca_unqueue(dca_listnode_t * q)1352 dca_unqueue(dca_listnode_t *q)
1353 {
1354 	dca_listnode_t *node;
1355 	/*
1356 	 * unqueue takes from the "tail" of the list, i.e. just before
1357 	 * the sentinel.
1358 	 */
1359 	if ((node = q->dl_prev) == q) {
1360 		/* queue is empty */
1361 		return (NULL);
1362 	}
1363 	dca_rmqueue(node);
1364 	return (node);
1365 }
1366 
1367 dca_listnode_t *
dca_peekqueue(dca_listnode_t * q)1368 dca_peekqueue(dca_listnode_t *q)
1369 {
1370 	dca_listnode_t *node;
1371 
1372 	if ((node = q->dl_next) == q) {
1373 		return (NULL);
1374 	} else {
1375 		return (node);
1376 	}
1377 }
1378 
1379 /*
1380  * Interrupt service routine.
1381  */
1382 uint_t
dca_intr(char * arg)1383 dca_intr(char *arg)
1384 {
1385 	dca_t		*dca = (dca_t *)arg;
1386 	uint32_t	status;
1387 
1388 	mutex_enter(&dca->dca_intrlock);
1389 	status = GETCSR(dca, CSR_DMASTAT);
1390 	PUTCSR(dca, CSR_DMASTAT, status & DMASTAT_INTERRUPTS);
1391 	if (dca_check_acc_handle(dca, dca->dca_regs_handle,
1392 	    DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
1393 		mutex_exit(&dca->dca_intrlock);
1394 		return ((uint_t)DDI_FAILURE);
1395 	}
1396 
1397 	DBG(dca, DINTR, "interrupted, status = 0x%x!", status);
1398 
1399 	if ((status & DMASTAT_INTERRUPTS) == 0) {
1400 		/* increment spurious interrupt kstat */
1401 		if (dca->dca_intrstats) {
1402 			KIOIP(dca)->intrs[KSTAT_INTR_SPURIOUS]++;
1403 		}
1404 		mutex_exit(&dca->dca_intrlock);
1405 		return (DDI_INTR_UNCLAIMED);
1406 	}
1407 
1408 	if (dca->dca_intrstats) {
1409 		KIOIP(dca)->intrs[KSTAT_INTR_HARD]++;
1410 	}
1411 	if (status & DMASTAT_MCR1INT) {
1412 		DBG(dca, DINTR, "MCR1 interrupted");
1413 		mutex_enter(&(WORKLIST(dca, MCR1)->dwl_lock));
1414 		dca_schedule(dca, MCR1);
1415 		dca_reclaim(dca, MCR1);
1416 		mutex_exit(&(WORKLIST(dca, MCR1)->dwl_lock));
1417 	}
1418 
1419 	if (status & DMASTAT_MCR2INT) {
1420 		DBG(dca, DINTR, "MCR2 interrupted");
1421 		mutex_enter(&(WORKLIST(dca, MCR2)->dwl_lock));
1422 		dca_schedule(dca, MCR2);
1423 		dca_reclaim(dca, MCR2);
1424 		mutex_exit(&(WORKLIST(dca, MCR2)->dwl_lock));
1425 	}
1426 
1427 	if (status & DMASTAT_ERRINT) {
1428 		uint32_t	erraddr;
1429 		erraddr = GETCSR(dca, CSR_DMAEA);
1430 		mutex_exit(&dca->dca_intrlock);
1431 
1432 		/*
1433 		 * bit 1 of the error address indicates failure during
1434 		 * read if set, during write otherwise.
1435 		 */
1436 		dca_failure(dca, DDI_DEVICE_FAULT,
1437 		    DCA_FM_ECLASS_HW_DEVICE, dca_ena(0), CRYPTO_DEVICE_ERROR,
1438 		    "DMA master access error %s address 0x%x",
1439 		    erraddr & 0x1 ? "reading" : "writing", erraddr & ~1);
1440 		return (DDI_INTR_CLAIMED);
1441 	}
1442 
1443 	mutex_exit(&dca->dca_intrlock);
1444 
1445 	return (DDI_INTR_CLAIMED);
1446 }
1447 
1448 /*
1449  * Reverse a string of bytes from s1 into s2.  The reversal happens
1450  * from the tail of s1.  If len1 < len2, then null bytes will be
1451  * padded to the end of s2.  If len2 < len1, then (presumably null)
1452  * bytes will be dropped from the start of s1.
1453  *
1454  * The rationale here is that when s1 (source) is shorter, then we
1455  * are reversing from big-endian ordering, into device ordering, and
1456  * want to add some extra nulls to the tail (MSB) side of the device.
1457  *
1458  * Similarly, when s2 (dest) is shorter, then we are truncating what
1459  * are presumably null MSB bits from the device.
1460  *
1461  * There is an expectation when reversing from the device back into
1462  * big-endian, that the number of bytes to reverse and the target size
1463  * will match, and no truncation or padding occurs.
1464  */
1465 void
dca_reverse(void * s1,void * s2,int len1,int len2)1466 dca_reverse(void *s1, void *s2, int len1, int len2)
1467 {
1468 	caddr_t	src, dst;
1469 
1470 	if (len1 == 0) {
1471 		if (len2) {
1472 			bzero(s2, len2);
1473 		}
1474 		return;
1475 	}
1476 	src = (caddr_t)s1 + len1 - 1;
1477 	dst = s2;
1478 	while ((src >= (caddr_t)s1) && (len2)) {
1479 		*dst++ = *src--;
1480 		len2--;
1481 	}
1482 	while (len2 > 0) {
1483 		*dst++ = 0;
1484 		len2--;
1485 	}
1486 }
1487 
1488 uint16_t
dca_padfull(int num)1489 dca_padfull(int num)
1490 {
1491 	if (num <= 512) {
1492 		return (BITS2BYTES(512));
1493 	}
1494 	if (num <= 768) {
1495 		return (BITS2BYTES(768));
1496 	}
1497 	if (num <= 1024) {
1498 		return (BITS2BYTES(1024));
1499 	}
1500 	if (num <= 1536) {
1501 		return (BITS2BYTES(1536));
1502 	}
1503 	if (num <= 2048) {
1504 		return (BITS2BYTES(2048));
1505 	}
1506 	return (0);
1507 }
1508 
1509 uint16_t
dca_padhalf(int num)1510 dca_padhalf(int num)
1511 {
1512 	if (num <= 256) {
1513 		return (BITS2BYTES(256));
1514 	}
1515 	if (num <= 384) {
1516 		return (BITS2BYTES(384));
1517 	}
1518 	if (num <= 512) {
1519 		return (BITS2BYTES(512));
1520 	}
1521 	if (num <= 768) {
1522 		return (BITS2BYTES(768));
1523 	}
1524 	if (num <= 1024) {
1525 		return (BITS2BYTES(1024));
1526 	}
1527 	return (0);
1528 }
1529 
1530 dca_work_t *
dca_newwork(dca_t * dca)1531 dca_newwork(dca_t *dca)
1532 {
1533 	dca_work_t		*workp;
1534 	size_t			size;
1535 	ddi_dma_cookie_t	c;
1536 	unsigned		nc;
1537 	int			rv;
1538 
1539 	workp = kmem_zalloc(sizeof (dca_work_t), KM_SLEEP);
1540 
1541 	rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1542 	    DDI_DMA_SLEEP, NULL, &workp->dw_mcr_dmah);
1543 	if (rv != 0) {
1544 		dca_error(dca, "unable to alloc MCR DMA handle");
1545 		dca_destroywork(workp);
1546 		return (NULL);
1547 	}
1548 
1549 	rv = ddi_dma_mem_alloc(workp->dw_mcr_dmah,
1550 	    ROUNDUP(MCR_SIZE, dca->dca_pagesize),
1551 	    &dca_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
1552 	    &workp->dw_mcr_kaddr, &size, &workp->dw_mcr_acch);
1553 	if (rv != 0) {
1554 		dca_error(dca, "unable to alloc MCR DMA memory");
1555 		dca_destroywork(workp);
1556 		return (NULL);
1557 	}
1558 
1559 	rv = ddi_dma_addr_bind_handle(workp->dw_mcr_dmah, NULL,
1560 	    workp->dw_mcr_kaddr, size, DDI_DMA_CONSISTENT | DDI_DMA_RDWR,
1561 	    DDI_DMA_SLEEP, NULL, &c, &nc);
1562 	if (rv != DDI_DMA_MAPPED) {
1563 		dca_error(dca, "unable to map MCR DMA memory");
1564 		dca_destroywork(workp);
1565 		return (NULL);
1566 	}
1567 
1568 	workp->dw_mcr_paddr = c.dmac_address;
1569 	return (workp);
1570 }
1571 
1572 void
dca_destroywork(dca_work_t * workp)1573 dca_destroywork(dca_work_t *workp)
1574 {
1575 	if (workp->dw_mcr_paddr) {
1576 		(void) ddi_dma_unbind_handle(workp->dw_mcr_dmah);
1577 	}
1578 	if (workp->dw_mcr_acch) {
1579 		ddi_dma_mem_free(&workp->dw_mcr_acch);
1580 	}
1581 	if (workp->dw_mcr_dmah) {
1582 		ddi_dma_free_handle(&workp->dw_mcr_dmah);
1583 	}
1584 	kmem_free(workp, sizeof (dca_work_t));
1585 }
1586 
1587 dca_request_t *
dca_newreq(dca_t * dca)1588 dca_newreq(dca_t *dca)
1589 {
1590 	dca_request_t		*reqp;
1591 	size_t			size;
1592 	ddi_dma_cookie_t	c;
1593 	unsigned		nc;
1594 	int			rv;
1595 	int			n_chain = 0;
1596 
1597 	size = (DESC_SIZE * MAXFRAGS) + CTX_MAXLENGTH;
1598 
1599 	reqp = kmem_zalloc(sizeof (dca_request_t), KM_SLEEP);
1600 
1601 	reqp->dr_dca = dca;
1602 
1603 	/*
1604 	 * Setup the DMA region for the context and descriptors.
1605 	 */
1606 	rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr, DDI_DMA_SLEEP,
1607 	    NULL, &reqp->dr_ctx_dmah);
1608 	if (rv != DDI_SUCCESS) {
1609 		dca_error(dca, "failure allocating request DMA handle");
1610 		dca_destroyreq(reqp);
1611 		return (NULL);
1612 	}
1613 
1614 	/* for driver hardening, allocate in whole pages */
1615 	rv = ddi_dma_mem_alloc(reqp->dr_ctx_dmah,
1616 	    ROUNDUP(size, dca->dca_pagesize), &dca_devattr, DDI_DMA_CONSISTENT,
1617 	    DDI_DMA_SLEEP, NULL, &reqp->dr_ctx_kaddr, &size,
1618 	    &reqp->dr_ctx_acch);
1619 	if (rv != DDI_SUCCESS) {
1620 		dca_error(dca, "unable to alloc request DMA memory");
1621 		dca_destroyreq(reqp);
1622 		return (NULL);
1623 	}
1624 
1625 	rv = ddi_dma_addr_bind_handle(reqp->dr_ctx_dmah, NULL,
1626 	    reqp->dr_ctx_kaddr, size, DDI_DMA_CONSISTENT | DDI_DMA_WRITE,
1627 	    DDI_DMA_SLEEP, 0, &c, &nc);
1628 	if (rv != DDI_DMA_MAPPED) {
1629 		dca_error(dca, "failed binding request DMA handle");
1630 		dca_destroyreq(reqp);
1631 		return (NULL);
1632 	}
1633 	reqp->dr_ctx_paddr = c.dmac_address;
1634 
1635 	reqp->dr_dma_size = size;
1636 
1637 	/*
1638 	 * Set up the dma for our scratch/shared buffers.
1639 	 */
1640 	rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1641 	    DDI_DMA_SLEEP, NULL, &reqp->dr_ibuf_dmah);
1642 	if (rv != DDI_SUCCESS) {
1643 		dca_error(dca, "failure allocating ibuf DMA handle");
1644 		dca_destroyreq(reqp);
1645 		return (NULL);
1646 	}
1647 	rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1648 	    DDI_DMA_SLEEP, NULL, &reqp->dr_obuf_dmah);
1649 	if (rv != DDI_SUCCESS) {
1650 		dca_error(dca, "failure allocating obuf DMA handle");
1651 		dca_destroyreq(reqp);
1652 		return (NULL);
1653 	}
1654 
1655 	rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1656 	    DDI_DMA_SLEEP, NULL, &reqp->dr_chain_in_dmah);
1657 	if (rv != DDI_SUCCESS) {
1658 		dca_error(dca, "failure allocating chain_in DMA handle");
1659 		dca_destroyreq(reqp);
1660 		return (NULL);
1661 	}
1662 
1663 	rv = ddi_dma_alloc_handle(dca->dca_dip, &dca_dmaattr,
1664 	    DDI_DMA_SLEEP, NULL, &reqp->dr_chain_out_dmah);
1665 	if (rv != DDI_SUCCESS) {
1666 		dca_error(dca, "failure allocating chain_out DMA handle");
1667 		dca_destroyreq(reqp);
1668 		return (NULL);
1669 	}
1670 
1671 	/*
1672 	 * for driver hardening, allocate in whole pages.
1673 	 */
1674 	size = ROUNDUP(MAXPACKET, dca->dca_pagesize);
1675 #if defined(i386) || defined(__i386)
1676 	/*
1677 	 * Use kmem_alloc instead of ddi_dma_mem_alloc here since the latter
1678 	 * may fail on x86 platform if a physically contiguous memory chunk
1679 	 * cannot be found. From initial testing, we did not see performance
1680 	 * degradation as seen on Sparc.
1681 	 */
1682 	if ((reqp->dr_ibuf_kaddr = kmem_alloc(size, KM_SLEEP)) == NULL) {
1683 		dca_error(dca, "unable to alloc request ibuf memory");
1684 		dca_destroyreq(reqp);
1685 		return (NULL);
1686 	}
1687 	if ((reqp->dr_obuf_kaddr = kmem_alloc(size, KM_SLEEP)) == NULL) {
1688 		dca_error(dca, "unable to alloc request obuf memory");
1689 		dca_destroyreq(reqp);
1690 		return (NULL);
1691 	}
1692 #else
1693 	/*
1694 	 * We could kmem_alloc for Sparc too. However, it gives worse
1695 	 * performance when transferring more than one page data. For example,
1696 	 * using 4 threads and 12032 byte data and 3DES on 900MHZ Sparc system,
1697 	 * kmem_alloc uses 80% CPU and ddi_dma_mem_alloc uses 50% CPU for
1698 	 * the same throughput.
1699 	 */
1700 	rv = ddi_dma_mem_alloc(reqp->dr_ibuf_dmah,
1701 	    size, &dca_bufattr,
1702 	    DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL, &reqp->dr_ibuf_kaddr,
1703 	    &size, &reqp->dr_ibuf_acch);
1704 	if (rv != DDI_SUCCESS) {
1705 		dca_error(dca, "unable to alloc request DMA memory");
1706 		dca_destroyreq(reqp);
1707 		return (NULL);
1708 	}
1709 
1710 	rv = ddi_dma_mem_alloc(reqp->dr_obuf_dmah,
1711 	    size, &dca_bufattr,
1712 	    DDI_DMA_STREAMING, DDI_DMA_SLEEP, NULL, &reqp->dr_obuf_kaddr,
1713 	    &size, &reqp->dr_obuf_acch);
1714 	if (rv != DDI_SUCCESS) {
1715 		dca_error(dca, "unable to alloc request DMA memory");
1716 		dca_destroyreq(reqp);
1717 		return (NULL);
1718 	}
1719 #endif
1720 
1721 	/* Skip the used portion in the context page */
1722 	reqp->dr_offset = CTX_MAXLENGTH;
1723 	if ((rv = dca_bindchains_one(reqp, size, reqp->dr_offset,
1724 	    reqp->dr_ibuf_kaddr, reqp->dr_ibuf_dmah,
1725 	    DDI_DMA_WRITE | DDI_DMA_STREAMING,
1726 	    &reqp->dr_ibuf_head, &n_chain)) != DDI_SUCCESS) {
1727 		(void) dca_destroyreq(reqp);
1728 		return (NULL);
1729 	}
1730 	reqp->dr_ibuf_paddr = reqp->dr_ibuf_head.dc_buffer_paddr;
1731 	/* Skip the space used by the input buffer */
1732 	reqp->dr_offset += DESC_SIZE * n_chain;
1733 
1734 	if ((rv = dca_bindchains_one(reqp, size, reqp->dr_offset,
1735 	    reqp->dr_obuf_kaddr, reqp->dr_obuf_dmah,
1736 	    DDI_DMA_READ | DDI_DMA_STREAMING,
1737 	    &reqp->dr_obuf_head, &n_chain)) != DDI_SUCCESS) {
1738 		(void) dca_destroyreq(reqp);
1739 		return (NULL);
1740 	}
1741 	reqp->dr_obuf_paddr = reqp->dr_obuf_head.dc_buffer_paddr;
1742 	/* Skip the space used by the output buffer */
1743 	reqp->dr_offset += DESC_SIZE * n_chain;
1744 
1745 	DBG(dca, DCHATTY, "CTX is 0x%p, phys 0x%x, len %d",
1746 	    reqp->dr_ctx_kaddr, reqp->dr_ctx_paddr, CTX_MAXLENGTH);
1747 	return (reqp);
1748 }
1749 
1750 void
dca_destroyreq(dca_request_t * reqp)1751 dca_destroyreq(dca_request_t *reqp)
1752 {
1753 #if defined(i386) || defined(__i386)
1754 	dca_t		*dca = reqp->dr_dca;
1755 	size_t		size = ROUNDUP(MAXPACKET, dca->dca_pagesize);
1756 #endif
1757 
1758 	/*
1759 	 * Clean up DMA for the context structure.
1760 	 */
1761 	if (reqp->dr_ctx_paddr) {
1762 		(void) ddi_dma_unbind_handle(reqp->dr_ctx_dmah);
1763 	}
1764 
1765 	if (reqp->dr_ctx_acch) {
1766 		ddi_dma_mem_free(&reqp->dr_ctx_acch);
1767 	}
1768 
1769 	if (reqp->dr_ctx_dmah) {
1770 		ddi_dma_free_handle(&reqp->dr_ctx_dmah);
1771 	}
1772 
1773 	/*
1774 	 * Clean up DMA for the scratch buffer.
1775 	 */
1776 #if defined(i386) || defined(__i386)
1777 	if (reqp->dr_ibuf_dmah) {
1778 		(void) ddi_dma_unbind_handle(reqp->dr_ibuf_dmah);
1779 		ddi_dma_free_handle(&reqp->dr_ibuf_dmah);
1780 	}
1781 	if (reqp->dr_obuf_dmah) {
1782 		(void) ddi_dma_unbind_handle(reqp->dr_obuf_dmah);
1783 		ddi_dma_free_handle(&reqp->dr_obuf_dmah);
1784 	}
1785 
1786 	kmem_free(reqp->dr_ibuf_kaddr, size);
1787 	kmem_free(reqp->dr_obuf_kaddr, size);
1788 #else
1789 	if (reqp->dr_ibuf_paddr) {
1790 		(void) ddi_dma_unbind_handle(reqp->dr_ibuf_dmah);
1791 	}
1792 	if (reqp->dr_obuf_paddr) {
1793 		(void) ddi_dma_unbind_handle(reqp->dr_obuf_dmah);
1794 	}
1795 
1796 	if (reqp->dr_ibuf_acch) {
1797 		ddi_dma_mem_free(&reqp->dr_ibuf_acch);
1798 	}
1799 	if (reqp->dr_obuf_acch) {
1800 		ddi_dma_mem_free(&reqp->dr_obuf_acch);
1801 	}
1802 
1803 	if (reqp->dr_ibuf_dmah) {
1804 		ddi_dma_free_handle(&reqp->dr_ibuf_dmah);
1805 	}
1806 	if (reqp->dr_obuf_dmah) {
1807 		ddi_dma_free_handle(&reqp->dr_obuf_dmah);
1808 	}
1809 #endif
1810 	/*
1811 	 * These two DMA handles should have been unbinded in
1812 	 * dca_unbindchains() function
1813 	 */
1814 	if (reqp->dr_chain_in_dmah) {
1815 		ddi_dma_free_handle(&reqp->dr_chain_in_dmah);
1816 	}
1817 	if (reqp->dr_chain_out_dmah) {
1818 		ddi_dma_free_handle(&reqp->dr_chain_out_dmah);
1819 	}
1820 
1821 	kmem_free(reqp, sizeof (dca_request_t));
1822 }
1823 
1824 dca_work_t *
dca_getwork(dca_t * dca,int mcr)1825 dca_getwork(dca_t *dca, int mcr)
1826 {
1827 	dca_worklist_t	*wlp = WORKLIST(dca, mcr);
1828 	dca_work_t	*workp;
1829 
1830 	mutex_enter(&wlp->dwl_freelock);
1831 	workp = (dca_work_t *)dca_dequeue(&wlp->dwl_freework);
1832 	mutex_exit(&wlp->dwl_freelock);
1833 	if (workp) {
1834 		int	nreqs;
1835 		bzero(workp->dw_mcr_kaddr, 8);
1836 
1837 		/* clear out old requests */
1838 		for (nreqs = 0; nreqs < MAXREQSPERMCR; nreqs++) {
1839 			workp->dw_reqs[nreqs] = NULL;
1840 		}
1841 	}
1842 	return (workp);
1843 }
1844 
1845 void
dca_freework(dca_work_t * workp)1846 dca_freework(dca_work_t *workp)
1847 {
1848 	mutex_enter(&workp->dw_wlp->dwl_freelock);
1849 	dca_enqueue(&workp->dw_wlp->dwl_freework, (dca_listnode_t *)workp);
1850 	mutex_exit(&workp->dw_wlp->dwl_freelock);
1851 }
1852 
1853 dca_request_t *
dca_getreq(dca_t * dca,int mcr,int tryhard)1854 dca_getreq(dca_t *dca, int mcr, int tryhard)
1855 {
1856 	dca_worklist_t	*wlp = WORKLIST(dca, mcr);
1857 	dca_request_t	*reqp;
1858 
1859 	mutex_enter(&wlp->dwl_freereqslock);
1860 	reqp = (dca_request_t *)dca_dequeue(&wlp->dwl_freereqs);
1861 	mutex_exit(&wlp->dwl_freereqslock);
1862 	if (reqp) {
1863 		reqp->dr_flags = 0;
1864 		reqp->dr_callback = NULL;
1865 	} else if (tryhard) {
1866 		/*
1867 		 * failed to get a free one, try an allocation, the hard way.
1868 		 * XXX: Kstat desired here.
1869 		 */
1870 		if ((reqp = dca_newreq(dca)) != NULL) {
1871 			reqp->dr_wlp = wlp;
1872 			reqp->dr_dca = dca;
1873 			reqp->dr_flags = 0;
1874 			reqp->dr_callback = NULL;
1875 		}
1876 	}
1877 	return (reqp);
1878 }
1879 
1880 void
dca_freereq(dca_request_t * reqp)1881 dca_freereq(dca_request_t *reqp)
1882 {
1883 	reqp->dr_kcf_req = NULL;
1884 	if (!(reqp->dr_flags & DR_NOCACHE)) {
1885 		mutex_enter(&reqp->dr_wlp->dwl_freereqslock);
1886 		dca_enqueue(&reqp->dr_wlp->dwl_freereqs,
1887 		    (dca_listnode_t *)reqp);
1888 		mutex_exit(&reqp->dr_wlp->dwl_freereqslock);
1889 	}
1890 }
1891 
1892 /*
1893  * Binds user buffers to DMA handles dynamically. On Sparc, a user buffer
1894  * is mapped to a single physical address. On x86, a user buffer is mapped
1895  * to multiple physical addresses. These physical addresses are chained
1896  * using the method specified in Broadcom BCM5820 specification.
1897  */
1898 int
dca_bindchains(dca_request_t * reqp,size_t incnt,size_t outcnt)1899 dca_bindchains(dca_request_t *reqp, size_t incnt, size_t outcnt)
1900 {
1901 	int			rv;
1902 	caddr_t			kaddr;
1903 	uint_t			flags;
1904 	int			n_chain = 0;
1905 
1906 	if (reqp->dr_flags & DR_INPLACE) {
1907 		flags = DDI_DMA_RDWR | DDI_DMA_CONSISTENT;
1908 	} else {
1909 		flags = DDI_DMA_WRITE | DDI_DMA_STREAMING;
1910 	}
1911 
1912 	/* first the input */
1913 	if (incnt) {
1914 		if ((kaddr = dca_bufdaddr(reqp->dr_in)) == NULL) {
1915 			DBG(NULL, DWARN, "unrecognised crypto data format");
1916 			return (DDI_FAILURE);
1917 		}
1918 		if ((rv = dca_bindchains_one(reqp, incnt, reqp->dr_offset,
1919 		    kaddr, reqp->dr_chain_in_dmah, flags,
1920 		    &reqp->dr_chain_in_head, &n_chain)) != DDI_SUCCESS) {
1921 			(void) dca_unbindchains(reqp);
1922 			return (rv);
1923 		}
1924 
1925 		/*
1926 		 * The offset and length are altered by the calling routine
1927 		 * reqp->dr_in->cd_offset += incnt;
1928 		 * reqp->dr_in->cd_length -= incnt;
1929 		 */
1930 		/* Save the first one in the chain for MCR */
1931 		reqp->dr_in_paddr = reqp->dr_chain_in_head.dc_buffer_paddr;
1932 		reqp->dr_in_next = reqp->dr_chain_in_head.dc_next_paddr;
1933 		reqp->dr_in_len = reqp->dr_chain_in_head.dc_buffer_length;
1934 	} else {
1935 		reqp->dr_in_paddr = NULL;
1936 		reqp->dr_in_next = 0;
1937 		reqp->dr_in_len = 0;
1938 	}
1939 
1940 	if (reqp->dr_flags & DR_INPLACE) {
1941 		reqp->dr_out_paddr = reqp->dr_in_paddr;
1942 		reqp->dr_out_len = reqp->dr_in_len;
1943 		reqp->dr_out_next = reqp->dr_in_next;
1944 		return (DDI_SUCCESS);
1945 	}
1946 
1947 	/* then the output */
1948 	if (outcnt) {
1949 		flags = DDI_DMA_READ | DDI_DMA_STREAMING;
1950 		if ((kaddr = dca_bufdaddr_out(reqp->dr_out)) == NULL) {
1951 			DBG(NULL, DWARN, "unrecognised crypto data format");
1952 			(void) dca_unbindchains(reqp);
1953 			return (DDI_FAILURE);
1954 		}
1955 		rv = dca_bindchains_one(reqp, outcnt, reqp->dr_offset +
1956 		    n_chain * DESC_SIZE, kaddr, reqp->dr_chain_out_dmah,
1957 		    flags, &reqp->dr_chain_out_head, &n_chain);
1958 		if (rv != DDI_SUCCESS) {
1959 			(void) dca_unbindchains(reqp);
1960 			return (DDI_FAILURE);
1961 		}
1962 
1963 		/* Save the first one in the chain for MCR */
1964 		reqp->dr_out_paddr = reqp->dr_chain_out_head.dc_buffer_paddr;
1965 		reqp->dr_out_next = reqp->dr_chain_out_head.dc_next_paddr;
1966 		reqp->dr_out_len = reqp->dr_chain_out_head.dc_buffer_length;
1967 	} else {
1968 		reqp->dr_out_paddr = NULL;
1969 		reqp->dr_out_next = 0;
1970 		reqp->dr_out_len = 0;
1971 	}
1972 
1973 	return (DDI_SUCCESS);
1974 }
1975 
1976 /*
1977  * Unbind the user buffers from the DMA handles.
1978  */
1979 int
dca_unbindchains(dca_request_t * reqp)1980 dca_unbindchains(dca_request_t *reqp)
1981 {
1982 	int rv = DDI_SUCCESS;
1983 	int rv1 = DDI_SUCCESS;
1984 
1985 	/* Clear the input chain */
1986 	if (reqp->dr_chain_in_head.dc_buffer_paddr != NULL) {
1987 		(void) ddi_dma_unbind_handle(reqp->dr_chain_in_dmah);
1988 		reqp->dr_chain_in_head.dc_buffer_paddr = 0;
1989 	}
1990 
1991 	if (reqp->dr_flags & DR_INPLACE) {
1992 		return (rv);
1993 	}
1994 
1995 	/* Clear the output chain */
1996 	if (reqp->dr_chain_out_head.dc_buffer_paddr != NULL) {
1997 		(void) ddi_dma_unbind_handle(reqp->dr_chain_out_dmah);
1998 		reqp->dr_chain_out_head.dc_buffer_paddr = 0;
1999 	}
2000 
2001 	return ((rv != DDI_SUCCESS)? rv : rv1);
2002 }
2003 
2004 /*
2005  * Build either input chain or output chain. It is single-item chain for Sparc,
2006  * and possible mutiple-item chain for x86.
2007  */
2008 static int
dca_bindchains_one(dca_request_t * reqp,size_t cnt,int dr_offset,caddr_t kaddr,ddi_dma_handle_t handle,uint_t flags,dca_chain_t * head,int * n_chain)2009 dca_bindchains_one(dca_request_t *reqp, size_t cnt, int dr_offset,
2010     caddr_t kaddr, ddi_dma_handle_t handle, uint_t flags,
2011     dca_chain_t *head, int *n_chain)
2012 {
2013 	ddi_dma_cookie_t	c;
2014 	uint_t			nc;
2015 	int			rv;
2016 	caddr_t			chain_kaddr_pre;
2017 	caddr_t			chain_kaddr;
2018 	uint32_t		chain_paddr;
2019 	int 			i;
2020 
2021 	/* Advance past the context structure to the starting address */
2022 	chain_paddr = reqp->dr_ctx_paddr + dr_offset;
2023 	chain_kaddr = reqp->dr_ctx_kaddr + dr_offset;
2024 
2025 	/*
2026 	 * Bind the kernel address to the DMA handle. On x86, the actual
2027 	 * buffer is mapped into multiple physical addresses. On Sparc,
2028 	 * the actual buffer is mapped into a single address.
2029 	 */
2030 	rv = ddi_dma_addr_bind_handle(handle,
2031 	    NULL, kaddr, cnt, flags, DDI_DMA_DONTWAIT, NULL, &c, &nc);
2032 	if (rv != DDI_DMA_MAPPED) {
2033 		return (DDI_FAILURE);
2034 	}
2035 
2036 	(void) ddi_dma_sync(handle, 0, cnt, DDI_DMA_SYNC_FORDEV);
2037 	if ((rv = dca_check_dma_handle(reqp->dr_dca, handle,
2038 	    DCA_FM_ECLASS_NONE)) != DDI_SUCCESS) {
2039 		reqp->destroy = TRUE;
2040 		return (rv);
2041 	}
2042 
2043 	*n_chain = nc;
2044 
2045 	/* Setup the data buffer chain for DMA transfer */
2046 	chain_kaddr_pre = NULL;
2047 	head->dc_buffer_paddr = 0;
2048 	head->dc_next_paddr = 0;
2049 	head->dc_buffer_length = 0;
2050 	for (i = 0; i < nc; i++) {
2051 		/* PIO */
2052 		PUTDESC32(reqp, chain_kaddr, DESC_BUFADDR, c.dmac_address);
2053 		PUTDESC16(reqp, chain_kaddr, DESC_RSVD, 0);
2054 		PUTDESC16(reqp, chain_kaddr, DESC_LENGTH, c.dmac_size);
2055 
2056 		/* Remember the head of the chain */
2057 		if (head->dc_buffer_paddr == 0) {
2058 			head->dc_buffer_paddr = c.dmac_address;
2059 			head->dc_buffer_length = c.dmac_size;
2060 		}
2061 
2062 		/* Link to the previous one if one exists */
2063 		if (chain_kaddr_pre) {
2064 			PUTDESC32(reqp, chain_kaddr_pre, DESC_NEXT,
2065 			    chain_paddr);
2066 			if (head->dc_next_paddr == 0)
2067 				head->dc_next_paddr = chain_paddr;
2068 		}
2069 		chain_kaddr_pre = chain_kaddr;
2070 
2071 		/* Maintain pointers */
2072 		chain_paddr += DESC_SIZE;
2073 		chain_kaddr += DESC_SIZE;
2074 
2075 		/* Retrieve the next cookie if there is one */
2076 		if (i < nc-1)
2077 			ddi_dma_nextcookie(handle, &c);
2078 	}
2079 
2080 	/* Set the next pointer in the last entry to NULL */
2081 	PUTDESC32(reqp, chain_kaddr_pre, DESC_NEXT, 0);
2082 
2083 	return (DDI_SUCCESS);
2084 }
2085 
2086 /*
2087  * Schedule some work.
2088  */
2089 int
dca_start(dca_t * dca,dca_request_t * reqp,int mcr,int dosched)2090 dca_start(dca_t *dca, dca_request_t *reqp, int mcr, int dosched)
2091 {
2092 	dca_worklist_t	*wlp = WORKLIST(dca, mcr);
2093 
2094 	mutex_enter(&wlp->dwl_lock);
2095 
2096 	DBG(dca, DCHATTY, "req=%p, in=%p, out=%p, ctx=%p, ibuf=%p, obuf=%p",
2097 	    reqp, reqp->dr_in, reqp->dr_out, reqp->dr_ctx_kaddr,
2098 	    reqp->dr_ibuf_kaddr, reqp->dr_obuf_kaddr);
2099 	DBG(dca, DCHATTY, "ctx paddr = %x, ibuf paddr = %x, obuf paddr = %x",
2100 	    reqp->dr_ctx_paddr, reqp->dr_ibuf_paddr, reqp->dr_obuf_paddr);
2101 	/* sync out the entire context and descriptor chains */
2102 	(void) ddi_dma_sync(reqp->dr_ctx_dmah, 0, 0, DDI_DMA_SYNC_FORDEV);
2103 	if (dca_check_dma_handle(dca, reqp->dr_ctx_dmah,
2104 	    DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
2105 		reqp->destroy = TRUE;
2106 		mutex_exit(&wlp->dwl_lock);
2107 		return (CRYPTO_DEVICE_ERROR);
2108 	}
2109 
2110 	dca_enqueue(&wlp->dwl_waitq, (dca_listnode_t *)reqp);
2111 	wlp->dwl_count++;
2112 	wlp->dwl_lastsubmit = ddi_get_lbolt();
2113 	reqp->dr_wlp = wlp;
2114 
2115 	if ((wlp->dwl_count == wlp->dwl_hiwater) && (wlp->dwl_busy == 0)) {
2116 		/* we are fully loaded now, let kCF know */
2117 
2118 		wlp->dwl_flowctl++;
2119 		wlp->dwl_busy = 1;
2120 
2121 		crypto_prov_notify(wlp->dwl_prov, CRYPTO_PROVIDER_BUSY);
2122 	}
2123 
2124 	if (dosched) {
2125 #ifdef	SCHEDDELAY
2126 		/* possibly wait for more work to arrive */
2127 		if (wlp->dwl_count >= wlp->dwl_reqspermcr) {
2128 			dca_schedule(dca, mcr);
2129 		} else if (!wlp->dwl_schedtid) {
2130 			/* wait 1 msec for more work before doing it */
2131 			wlp->dwl_schedtid = timeout(dca_schedtimeout,
2132 			    (void *)wlp, drv_usectohz(MSEC));
2133 		}
2134 #else
2135 		dca_schedule(dca, mcr);
2136 #endif
2137 	}
2138 	mutex_exit(&wlp->dwl_lock);
2139 
2140 	return (CRYPTO_QUEUED);
2141 }
2142 
2143 void
dca_schedule(dca_t * dca,int mcr)2144 dca_schedule(dca_t *dca, int mcr)
2145 {
2146 	dca_worklist_t	*wlp = WORKLIST(dca, mcr);
2147 	int		csr;
2148 	int		full;
2149 	uint32_t	status;
2150 
2151 	ASSERT(mutex_owned(&wlp->dwl_lock));
2152 	/*
2153 	 * If the card is draining or has an outstanding failure,
2154 	 * don't schedule any more work on it right now
2155 	 */
2156 	if (wlp->dwl_drain || (dca->dca_flags & DCA_FAILED)) {
2157 		return;
2158 	}
2159 
2160 	if (mcr == MCR2) {
2161 		csr = CSR_MCR2;
2162 		full = DMASTAT_MCR2FULL;
2163 	} else {
2164 		csr = CSR_MCR1;
2165 		full = DMASTAT_MCR1FULL;
2166 	}
2167 
2168 	for (;;) {
2169 		dca_work_t	*workp;
2170 		uint32_t	offset;
2171 		int		nreqs;
2172 
2173 		status = GETCSR(dca, CSR_DMASTAT);
2174 		if (dca_check_acc_handle(dca, dca->dca_regs_handle,
2175 		    DCA_FM_ECLASS_NONE) != DDI_SUCCESS)
2176 			return;
2177 
2178 		if ((status & full) != 0)
2179 			break;
2180 
2181 #ifdef	SCHEDDELAY
2182 		/* if there isn't enough to do, don't bother now */
2183 		if ((wlp->dwl_count < wlp->dwl_reqspermcr) &&
2184 		    (ddi_get_lbolt() < (wlp->dwl_lastsubmit +
2185 		    drv_usectohz(MSEC)))) {
2186 			/* wait a bit longer... */
2187 			if (wlp->dwl_schedtid == 0) {
2188 				wlp->dwl_schedtid = timeout(dca_schedtimeout,
2189 				    (void *)wlp, drv_usectohz(MSEC));
2190 			}
2191 			return;
2192 		}
2193 #endif
2194 
2195 		/* grab a work structure */
2196 		workp = dca_getwork(dca, mcr);
2197 
2198 		if (workp == NULL) {
2199 			/*
2200 			 * There must be work ready to be reclaimed,
2201 			 * in this case, since the chip can only hold
2202 			 * less work outstanding than there are total.
2203 			 */
2204 			dca_reclaim(dca, mcr);
2205 			continue;
2206 		}
2207 
2208 		nreqs = 0;
2209 		offset = MCR_CTXADDR;
2210 
2211 		while (nreqs < wlp->dwl_reqspermcr) {
2212 			dca_request_t	*reqp;
2213 
2214 			reqp = (dca_request_t *)dca_dequeue(&wlp->dwl_waitq);
2215 			if (reqp == NULL) {
2216 				/* nothing left to process */
2217 				break;
2218 			}
2219 			/*
2220 			 * Update flow control.
2221 			 */
2222 			wlp->dwl_count--;
2223 			if ((wlp->dwl_count == wlp->dwl_lowater) &&
2224 			    (wlp->dwl_busy))  {
2225 				wlp->dwl_busy = 0;
2226 				crypto_prov_notify(wlp->dwl_prov,
2227 				    CRYPTO_PROVIDER_READY);
2228 			}
2229 
2230 			/*
2231 			 * Context address.
2232 			 */
2233 			PUTMCR32(workp, offset, reqp->dr_ctx_paddr);
2234 			offset += 4;
2235 
2236 			/*
2237 			 * Input chain.
2238 			 */
2239 			/* input buffer address */
2240 			PUTMCR32(workp, offset, reqp->dr_in_paddr);
2241 			offset += 4;
2242 			/* next input buffer entry */
2243 			PUTMCR32(workp, offset, reqp->dr_in_next);
2244 			offset += 4;
2245 			/* input buffer length */
2246 			PUTMCR16(workp, offset, reqp->dr_in_len);
2247 			offset += 2;
2248 			/* zero the reserved field */
2249 			PUTMCR16(workp, offset, 0);
2250 			offset += 2;
2251 
2252 			/*
2253 			 * Overall length.
2254 			 */
2255 			/* reserved field */
2256 			PUTMCR16(workp, offset, 0);
2257 			offset += 2;
2258 			/* total packet length */
2259 			PUTMCR16(workp, offset, reqp->dr_pkt_length);
2260 			offset += 2;
2261 
2262 			/*
2263 			 * Output chain.
2264 			 */
2265 			/* output buffer address */
2266 			PUTMCR32(workp, offset, reqp->dr_out_paddr);
2267 			offset += 4;
2268 			/* next output buffer entry */
2269 			PUTMCR32(workp, offset, reqp->dr_out_next);
2270 			offset += 4;
2271 			/* output buffer length */
2272 			PUTMCR16(workp, offset, reqp->dr_out_len);
2273 			offset += 2;
2274 			/* zero the reserved field */
2275 			PUTMCR16(workp, offset, 0);
2276 			offset += 2;
2277 
2278 			/*
2279 			 * Note submission.
2280 			 */
2281 			workp->dw_reqs[nreqs] = reqp;
2282 			nreqs++;
2283 		}
2284 
2285 		if (nreqs == 0) {
2286 			/* nothing in the queue! */
2287 			dca_freework(workp);
2288 			return;
2289 		}
2290 
2291 		wlp->dwl_submit++;
2292 
2293 		PUTMCR16(workp, MCR_FLAGS, 0);
2294 		PUTMCR16(workp, MCR_COUNT, nreqs);
2295 
2296 		DBG(dca, DCHATTY,
2297 		    "posting work (phys %x, virt 0x%p) (%d reqs) to MCR%d",
2298 		    workp->dw_mcr_paddr, workp->dw_mcr_kaddr,
2299 		    nreqs, mcr);
2300 
2301 		workp->dw_lbolt = ddi_get_lbolt();
2302 		/* Make sure MCR is synced out to device. */
2303 		(void) ddi_dma_sync(workp->dw_mcr_dmah, 0, 0,
2304 		    DDI_DMA_SYNC_FORDEV);
2305 		if (dca_check_dma_handle(dca, workp->dw_mcr_dmah,
2306 		    DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
2307 			dca_destroywork(workp);
2308 			return;
2309 		}
2310 
2311 		PUTCSR(dca, csr, workp->dw_mcr_paddr);
2312 		if (dca_check_acc_handle(dca, dca->dca_regs_handle,
2313 		    DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
2314 			dca_destroywork(workp);
2315 			return;
2316 		} else {
2317 			dca_enqueue(&wlp->dwl_runq, (dca_listnode_t *)workp);
2318 		}
2319 
2320 		DBG(dca, DCHATTY, "posted");
2321 	}
2322 }
2323 
2324 /*
2325  * Reclaim completed work, called in interrupt context.
2326  */
2327 void
dca_reclaim(dca_t * dca,int mcr)2328 dca_reclaim(dca_t *dca, int mcr)
2329 {
2330 	dca_worklist_t	*wlp = WORKLIST(dca, mcr);
2331 	dca_work_t	*workp;
2332 	ushort_t	flags;
2333 	int		nreclaimed = 0;
2334 	int		i;
2335 
2336 	DBG(dca, DRECLAIM, "worklist = 0x%p (MCR%d)", wlp, mcr);
2337 	ASSERT(mutex_owned(&wlp->dwl_lock));
2338 	/*
2339 	 * For each MCR in the submitted (runq), we check to see if
2340 	 * it has been processed.  If so, then we note each individual
2341 	 * job in the MCR, and and do the completion processing for
2342 	 * each of such job.
2343 	 */
2344 	for (;;) {
2345 
2346 		workp = (dca_work_t *)dca_peekqueue(&wlp->dwl_runq);
2347 		if (workp == NULL) {
2348 			break;
2349 		}
2350 
2351 		/* only sync the MCR flags, since that's all we need */
2352 		(void) ddi_dma_sync(workp->dw_mcr_dmah, 0, 4,
2353 		    DDI_DMA_SYNC_FORKERNEL);
2354 		if (dca_check_dma_handle(dca, workp->dw_mcr_dmah,
2355 		    DCA_FM_ECLASS_NONE) != DDI_SUCCESS) {
2356 			dca_rmqueue((dca_listnode_t *)workp);
2357 			dca_destroywork(workp);
2358 			return;
2359 		}
2360 
2361 		flags = GETMCR16(workp, MCR_FLAGS);
2362 		if ((flags & MCRFLAG_FINISHED) == 0) {
2363 			/* chip is still working on it */
2364 			DBG(dca, DRECLAIM,
2365 			    "chip still working on it (MCR%d)", mcr);
2366 			break;
2367 		}
2368 
2369 		/* its really for us, so remove it from the queue */
2370 		dca_rmqueue((dca_listnode_t *)workp);
2371 
2372 		/* if we were draining, signal on the cv */
2373 		if (wlp->dwl_drain && QEMPTY(&wlp->dwl_runq)) {
2374 			cv_signal(&wlp->dwl_cv);
2375 		}
2376 
2377 		/* update statistics, done under the lock */
2378 		for (i = 0; i < wlp->dwl_reqspermcr; i++) {
2379 			dca_request_t *reqp = workp->dw_reqs[i];
2380 			if (reqp == NULL) {
2381 				continue;
2382 			}
2383 			if (reqp->dr_byte_stat >= 0) {
2384 				dca->dca_stats[reqp->dr_byte_stat] +=
2385 				    reqp->dr_pkt_length;
2386 			}
2387 			if (reqp->dr_job_stat >= 0) {
2388 				dca->dca_stats[reqp->dr_job_stat]++;
2389 			}
2390 		}
2391 		mutex_exit(&wlp->dwl_lock);
2392 
2393 		for (i = 0; i < wlp->dwl_reqspermcr; i++) {
2394 			dca_request_t *reqp = workp->dw_reqs[i];
2395 
2396 			if (reqp == NULL) {
2397 				continue;
2398 			}
2399 
2400 			/* Do the callback. */
2401 			workp->dw_reqs[i] = NULL;
2402 			dca_done(reqp, CRYPTO_SUCCESS);
2403 
2404 			nreclaimed++;
2405 		}
2406 
2407 		/* now we can release the work */
2408 		dca_freework(workp);
2409 
2410 		mutex_enter(&wlp->dwl_lock);
2411 	}
2412 	DBG(dca, DRECLAIM, "reclaimed %d cmds", nreclaimed);
2413 }
2414 
2415 int
dca_length(crypto_data_t * cdata)2416 dca_length(crypto_data_t *cdata)
2417 {
2418 	return (cdata->cd_length);
2419 }
2420 
2421 /*
2422  * This is the callback function called from the interrupt when a kCF job
2423  * completes.  It does some driver-specific things, and then calls the
2424  * kCF-provided callback.  Finally, it cleans up the state for the work
2425  * request and drops the reference count to allow for DR.
2426  */
2427 void
dca_done(dca_request_t * reqp,int err)2428 dca_done(dca_request_t *reqp, int err)
2429 {
2430 	uint64_t	ena = 0;
2431 
2432 	/* unbind any chains we were using */
2433 	if (dca_unbindchains(reqp) != DDI_SUCCESS) {
2434 		/* DMA failure */
2435 		ena = dca_ena(ena);
2436 		dca_failure(reqp->dr_dca, DDI_DATAPATH_FAULT,
2437 		    DCA_FM_ECLASS_NONE, ena, CRYPTO_DEVICE_ERROR,
2438 		    "fault on buffer DMA handle");
2439 		if (err == CRYPTO_SUCCESS) {
2440 			err = CRYPTO_DEVICE_ERROR;
2441 		}
2442 	}
2443 
2444 	if (reqp->dr_callback != NULL) {
2445 		reqp->dr_callback(reqp, err);
2446 	} else {
2447 		dca_freereq(reqp);
2448 	}
2449 }
2450 
2451 /*
2452  * Call this when a failure is detected.  It will reset the chip,
2453  * log a message, alert kCF, and mark jobs in the runq as failed.
2454  */
2455 /* ARGSUSED */
2456 void
dca_failure(dca_t * dca,ddi_fault_location_t loc,dca_fma_eclass_t index,uint64_t ena,int errno,char * mess,...)2457 dca_failure(dca_t *dca, ddi_fault_location_t loc, dca_fma_eclass_t index,
2458     uint64_t ena, int errno, char *mess, ...)
2459 {
2460 	va_list	ap;
2461 	char	buf[256];
2462 	int	mcr;
2463 	char	*eclass;
2464 	int	have_mutex;
2465 
2466 	va_start(ap, mess);
2467 	(void) vsprintf(buf, mess, ap);
2468 	va_end(ap);
2469 
2470 	eclass = dca_fma_eclass_string(dca->dca_model, index);
2471 
2472 	if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) &&
2473 	    index != DCA_FM_ECLASS_NONE) {
2474 		ddi_fm_ereport_post(dca->dca_dip, eclass, ena,
2475 		    DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
2476 		    FM_EREPORT_VERS0, NULL);
2477 
2478 		/* Report the impact of the failure to the DDI. */
2479 		ddi_fm_service_impact(dca->dca_dip, DDI_SERVICE_LOST);
2480 	} else {
2481 		/* Just log the error string to the message log */
2482 		dca_error(dca, buf);
2483 	}
2484 
2485 	/*
2486 	 * Indicate a failure (keeps schedule from running).
2487 	 */
2488 	dca->dca_flags |= DCA_FAILED;
2489 
2490 	/*
2491 	 * Reset the chip.  This should also have as a side effect, the
2492 	 * disabling of all interrupts from the device.
2493 	 */
2494 	(void) dca_reset(dca, 1);
2495 
2496 	/*
2497 	 * Report the failure to kCF.
2498 	 */
2499 	for (mcr = MCR1; mcr <= MCR2; mcr++) {
2500 		if (WORKLIST(dca, mcr)->dwl_prov) {
2501 			crypto_prov_notify(WORKLIST(dca, mcr)->dwl_prov,
2502 			    CRYPTO_PROVIDER_FAILED);
2503 		}
2504 	}
2505 
2506 	/*
2507 	 * Return jobs not sent to hardware back to kCF.
2508 	 */
2509 	dca_rejectjobs(dca);
2510 
2511 	/*
2512 	 * From this point on, no new work should be arriving, and the
2513 	 * chip should not be doing any active DMA.
2514 	 */
2515 
2516 	/*
2517 	 * Now find all the work submitted to the device and fail
2518 	 * them.
2519 	 */
2520 	for (mcr = MCR1; mcr <= MCR2; mcr++) {
2521 		dca_worklist_t	*wlp;
2522 		int		i;
2523 
2524 		wlp = WORKLIST(dca, mcr);
2525 
2526 		if (wlp == NULL || wlp->dwl_waitq.dl_prev == NULL) {
2527 			continue;
2528 		}
2529 		for (;;) {
2530 			dca_work_t	*workp;
2531 
2532 			have_mutex = mutex_tryenter(&wlp->dwl_lock);
2533 			workp = (dca_work_t *)dca_dequeue(&wlp->dwl_runq);
2534 			if (workp == NULL) {
2535 				if (have_mutex)
2536 					mutex_exit(&wlp->dwl_lock);
2537 				break;
2538 			}
2539 			mutex_exit(&wlp->dwl_lock);
2540 
2541 			/*
2542 			 * Free up requests
2543 			 */
2544 			for (i = 0; i < wlp->dwl_reqspermcr; i++) {
2545 				dca_request_t *reqp = workp->dw_reqs[i];
2546 				if (reqp) {
2547 					dca_done(reqp, errno);
2548 					workp->dw_reqs[i] = NULL;
2549 				}
2550 			}
2551 
2552 			mutex_enter(&wlp->dwl_lock);
2553 			/*
2554 			 * If waiting to drain, signal on the waiter.
2555 			 */
2556 			if (wlp->dwl_drain && QEMPTY(&wlp->dwl_runq)) {
2557 				cv_signal(&wlp->dwl_cv);
2558 			}
2559 
2560 			/*
2561 			 * Return the work and request structures to
2562 			 * the free pool.
2563 			 */
2564 			dca_freework(workp);
2565 			if (have_mutex)
2566 				mutex_exit(&wlp->dwl_lock);
2567 		}
2568 	}
2569 
2570 }
2571 
2572 #ifdef	SCHEDDELAY
2573 /*
2574  * Reschedule worklist as needed.
2575  */
2576 void
dca_schedtimeout(void * arg)2577 dca_schedtimeout(void *arg)
2578 {
2579 	dca_worklist_t	*wlp = (dca_worklist_t *)arg;
2580 	mutex_enter(&wlp->dwl_lock);
2581 	wlp->dwl_schedtid = 0;
2582 	dca_schedule(wlp->dwl_dca, wlp->dwl_mcr);
2583 	mutex_exit(&wlp->dwl_lock);
2584 }
2585 #endif
2586 
2587 /*
2588  * Check for stalled jobs.
2589  */
2590 void
dca_jobtimeout(void * arg)2591 dca_jobtimeout(void *arg)
2592 {
2593 	int		mcr;
2594 	dca_t		*dca = (dca_t *)arg;
2595 	int		hung = 0;
2596 
2597 	for (mcr = MCR1; mcr <= MCR2; mcr++) {
2598 		dca_worklist_t	*wlp = WORKLIST(dca, mcr);
2599 		dca_work_t	*workp;
2600 		clock_t		when;
2601 
2602 		mutex_enter(&wlp->dwl_lock);
2603 		when = ddi_get_lbolt();
2604 
2605 		workp = (dca_work_t *)dca_peekqueue(&wlp->dwl_runq);
2606 		if (workp == NULL) {
2607 			/* nothing sitting in the queue */
2608 			mutex_exit(&wlp->dwl_lock);
2609 			continue;
2610 		}
2611 
2612 		if ((when - workp->dw_lbolt) < drv_usectohz(STALETIME)) {
2613 			/* request has been queued for less than STALETIME */
2614 			mutex_exit(&wlp->dwl_lock);
2615 			continue;
2616 		}
2617 
2618 		/* job has been sitting around for over 1 second, badness */
2619 		DBG(dca, DWARN, "stale job (0x%p) found in MCR%d!", workp,
2620 		    mcr);
2621 
2622 		/* put it back in the queue, until we reset the chip */
2623 		hung++;
2624 		mutex_exit(&wlp->dwl_lock);
2625 	}
2626 
2627 	if (hung) {
2628 		dca_failure(dca, DDI_DEVICE_FAULT,
2629 		    DCA_FM_ECLASS_HW_TIMEOUT, dca_ena(0), CRYPTO_DEVICE_ERROR,
2630 		    "timeout processing job.)");
2631 	}
2632 
2633 	/* reschedule ourself */
2634 	mutex_enter(&dca->dca_intrlock);
2635 	if (dca->dca_jobtid == 0) {
2636 		/* timeout has been canceled, prior to DR */
2637 		mutex_exit(&dca->dca_intrlock);
2638 		return;
2639 	}
2640 
2641 	/* check again in 1 second */
2642 	dca->dca_jobtid = timeout(dca_jobtimeout, arg,
2643 	    drv_usectohz(SECOND));
2644 	mutex_exit(&dca->dca_intrlock);
2645 }
2646 
2647 /*
2648  * This returns all jobs back to kCF.  It assumes that processing
2649  * on the worklist has halted.
2650  */
2651 void
dca_rejectjobs(dca_t * dca)2652 dca_rejectjobs(dca_t *dca)
2653 {
2654 	int mcr;
2655 	int have_mutex;
2656 	for (mcr = MCR1; mcr <= MCR2; mcr++) {
2657 		dca_worklist_t	*wlp = WORKLIST(dca, mcr);
2658 		dca_request_t	*reqp;
2659 
2660 		if (wlp == NULL || wlp->dwl_waitq.dl_prev == NULL) {
2661 			continue;
2662 		}
2663 		have_mutex = mutex_tryenter(&wlp->dwl_lock);
2664 		for (;;) {
2665 			reqp = (dca_request_t *)dca_unqueue(&wlp->dwl_waitq);
2666 			if (reqp == NULL) {
2667 				break;
2668 			}
2669 			/* update flow control */
2670 			wlp->dwl_count--;
2671 			if ((wlp->dwl_count == wlp->dwl_lowater) &&
2672 			    (wlp->dwl_busy))  {
2673 				wlp->dwl_busy = 0;
2674 				crypto_prov_notify(wlp->dwl_prov,
2675 				    CRYPTO_PROVIDER_READY);
2676 			}
2677 			mutex_exit(&wlp->dwl_lock);
2678 
2679 			(void) dca_unbindchains(reqp);
2680 			reqp->dr_callback(reqp, EAGAIN);
2681 			mutex_enter(&wlp->dwl_lock);
2682 		}
2683 		if (have_mutex)
2684 			mutex_exit(&wlp->dwl_lock);
2685 	}
2686 }
2687 
2688 int
dca_drain(dca_t * dca)2689 dca_drain(dca_t *dca)
2690 {
2691 	int mcr;
2692 	for (mcr = MCR1; mcr <= MCR2; mcr++) {
2693 #ifdef	SCHEDDELAY
2694 		timeout_id_t	tid;
2695 #endif
2696 		dca_worklist_t *wlp = WORKLIST(dca, mcr);
2697 
2698 		mutex_enter(&wlp->dwl_lock);
2699 		wlp->dwl_drain = 1;
2700 
2701 		/* give it up to a second to drain from the chip */
2702 		if (!QEMPTY(&wlp->dwl_runq)) {
2703 			(void) cv_reltimedwait(&wlp->dwl_cv, &wlp->dwl_lock,
2704 			    drv_usectohz(STALETIME), TR_CLOCK_TICK);
2705 
2706 			if (!QEMPTY(&wlp->dwl_runq)) {
2707 				dca_error(dca, "unable to drain device");
2708 				mutex_exit(&wlp->dwl_lock);
2709 				dca_undrain(dca);
2710 				return (EBUSY);
2711 			}
2712 		}
2713 
2714 #ifdef	SCHEDDELAY
2715 		tid = wlp->dwl_schedtid;
2716 		mutex_exit(&wlp->dwl_lock);
2717 
2718 		/*
2719 		 * untimeout outside the lock -- this is safe because we
2720 		 * have set the drain flag, so dca_schedule() will not
2721 		 * reschedule another timeout
2722 		 */
2723 		if (tid) {
2724 			untimeout(tid);
2725 		}
2726 #else
2727 		mutex_exit(&wlp->dwl_lock);
2728 #endif
2729 	}
2730 	return (0);
2731 }
2732 
2733 void
dca_undrain(dca_t * dca)2734 dca_undrain(dca_t *dca)
2735 {
2736 	int	mcr;
2737 
2738 	for (mcr = MCR1; mcr <= MCR2; mcr++) {
2739 		dca_worklist_t	*wlp = WORKLIST(dca, mcr);
2740 		mutex_enter(&wlp->dwl_lock);
2741 		wlp->dwl_drain = 0;
2742 		dca_schedule(dca, mcr);
2743 		mutex_exit(&wlp->dwl_lock);
2744 	}
2745 }
2746 
2747 /*
2748  * Duplicate the crypto_data_t structure, but point to the original
2749  * buffers.
2750  */
2751 int
dca_dupcrypto(crypto_data_t * input,crypto_data_t * ninput)2752 dca_dupcrypto(crypto_data_t *input, crypto_data_t *ninput)
2753 {
2754 	ninput->cd_format = input->cd_format;
2755 	ninput->cd_offset = input->cd_offset;
2756 	ninput->cd_length = input->cd_length;
2757 	ninput->cd_miscdata = input->cd_miscdata;
2758 
2759 	switch (input->cd_format) {
2760 	case CRYPTO_DATA_RAW:
2761 		ninput->cd_raw.iov_base = input->cd_raw.iov_base;
2762 		ninput->cd_raw.iov_len = input->cd_raw.iov_len;
2763 		break;
2764 
2765 	case CRYPTO_DATA_UIO:
2766 		ninput->cd_uio = input->cd_uio;
2767 		break;
2768 
2769 	case CRYPTO_DATA_MBLK:
2770 		ninput->cd_mp = input->cd_mp;
2771 		break;
2772 
2773 	default:
2774 		DBG(NULL, DWARN,
2775 		    "dca_dupcrypto: unrecognised crypto data format");
2776 		return (CRYPTO_FAILED);
2777 	}
2778 
2779 	return (CRYPTO_SUCCESS);
2780 }
2781 
2782 /*
2783  * Performs validation checks on the input and output data structures.
2784  */
2785 int
dca_verifyio(crypto_data_t * input,crypto_data_t * output)2786 dca_verifyio(crypto_data_t *input, crypto_data_t *output)
2787 {
2788 	int	rv = CRYPTO_SUCCESS;
2789 
2790 	switch (input->cd_format) {
2791 	case CRYPTO_DATA_RAW:
2792 		break;
2793 
2794 	case CRYPTO_DATA_UIO:
2795 		/* we support only kernel buffer */
2796 		if (input->cd_uio->uio_segflg != UIO_SYSSPACE) {
2797 			DBG(NULL, DWARN, "non kernel input uio buffer");
2798 			rv = CRYPTO_ARGUMENTS_BAD;
2799 		}
2800 		break;
2801 
2802 	case CRYPTO_DATA_MBLK:
2803 		break;
2804 
2805 	default:
2806 		DBG(NULL, DWARN, "unrecognised input crypto data format");
2807 		rv = CRYPTO_ARGUMENTS_BAD;
2808 	}
2809 
2810 	switch (output->cd_format) {
2811 	case CRYPTO_DATA_RAW:
2812 		break;
2813 
2814 	case CRYPTO_DATA_UIO:
2815 		/* we support only kernel buffer */
2816 		if (output->cd_uio->uio_segflg != UIO_SYSSPACE) {
2817 			DBG(NULL, DWARN, "non kernel output uio buffer");
2818 			rv = CRYPTO_ARGUMENTS_BAD;
2819 		}
2820 		break;
2821 
2822 	case CRYPTO_DATA_MBLK:
2823 		break;
2824 
2825 	default:
2826 		DBG(NULL, DWARN, "unrecognised output crypto data format");
2827 		rv = CRYPTO_ARGUMENTS_BAD;
2828 	}
2829 
2830 	return (rv);
2831 }
2832 
2833 /*
2834  * data: source crypto_data_t struct
2835  * off:	offset into the source before commencing copy
2836  * count: the amount of data to copy
2837  * dest: destination buffer
2838  */
2839 int
dca_getbufbytes(crypto_data_t * data,size_t off,int count,uchar_t * dest)2840 dca_getbufbytes(crypto_data_t *data, size_t off, int count, uchar_t *dest)
2841 {
2842 	int rv = CRYPTO_SUCCESS;
2843 	uio_t *uiop;
2844 	uint_t vec_idx;
2845 	size_t cur_len;
2846 	mblk_t *mp;
2847 
2848 	if (count == 0) {
2849 		/* We don't want anything so we're done. */
2850 		return (rv);
2851 	}
2852 
2853 	/*
2854 	 * Sanity check that we haven't specified a length greater than the
2855 	 * offset adjusted size of the buffer.
2856 	 */
2857 	if (count > (data->cd_length - off)) {
2858 		return (CRYPTO_DATA_LEN_RANGE);
2859 	}
2860 
2861 	/* Add the internal crypto_data offset to the requested offset. */
2862 	off += data->cd_offset;
2863 
2864 	switch (data->cd_format) {
2865 	case CRYPTO_DATA_RAW:
2866 		bcopy(data->cd_raw.iov_base + off, dest, count);
2867 		break;
2868 
2869 	case CRYPTO_DATA_UIO:
2870 		/*
2871 		 * Jump to the first iovec containing data to be
2872 		 * processed.
2873 		 */
2874 		uiop = data->cd_uio;
2875 		for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
2876 		    off >= uiop->uio_iov[vec_idx].iov_len;
2877 		    off -= uiop->uio_iov[vec_idx++].iov_len)
2878 			;
2879 		if (vec_idx == uiop->uio_iovcnt) {
2880 			/*
2881 			 * The caller specified an offset that is larger than
2882 			 * the total size of the buffers it provided.
2883 			 */
2884 			return (CRYPTO_DATA_LEN_RANGE);
2885 		}
2886 
2887 		/*
2888 		 * Now process the iovecs.
2889 		 */
2890 		while (vec_idx < uiop->uio_iovcnt && count > 0) {
2891 			cur_len = min(uiop->uio_iov[vec_idx].iov_len -
2892 			    off, count);
2893 			bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest,
2894 			    cur_len);
2895 			count -= cur_len;
2896 			dest += cur_len;
2897 			vec_idx++;
2898 			off = 0;
2899 		}
2900 
2901 		if (vec_idx == uiop->uio_iovcnt && count > 0) {
2902 			/*
2903 			 * The end of the specified iovec's was reached but
2904 			 * the length requested could not be processed
2905 			 * (requested to digest more data than it provided).
2906 			 */
2907 			return (CRYPTO_DATA_LEN_RANGE);
2908 		}
2909 		break;
2910 
2911 	case CRYPTO_DATA_MBLK:
2912 		/*
2913 		 * Jump to the first mblk_t containing data to be processed.
2914 		 */
2915 		for (mp = data->cd_mp; mp != NULL && off >= MBLKL(mp);
2916 		    off -= MBLKL(mp), mp = mp->b_cont)
2917 			;
2918 		if (mp == NULL) {
2919 			/*
2920 			 * The caller specified an offset that is larger than
2921 			 * the total size of the buffers it provided.
2922 			 */
2923 			return (CRYPTO_DATA_LEN_RANGE);
2924 		}
2925 
2926 		/*
2927 		 * Now do the processing on the mblk chain.
2928 		 */
2929 		while (mp != NULL && count > 0) {
2930 			cur_len = min(MBLKL(mp) - off, count);
2931 			bcopy((char *)(mp->b_rptr + off), dest, cur_len);
2932 			count -= cur_len;
2933 			dest += cur_len;
2934 			mp = mp->b_cont;
2935 			off = 0;
2936 		}
2937 
2938 		if (mp == NULL && count > 0) {
2939 			/*
2940 			 * The end of the mblk was reached but the length
2941 			 * requested could not be processed, (requested to
2942 			 * digest more data than it provided).
2943 			 */
2944 			return (CRYPTO_DATA_LEN_RANGE);
2945 		}
2946 		break;
2947 
2948 	default:
2949 		DBG(NULL, DWARN, "unrecognised crypto data format");
2950 		rv = CRYPTO_ARGUMENTS_BAD;
2951 	}
2952 	return (rv);
2953 }
2954 
2955 
2956 /*
2957  * Performs the input, output or hard scatter/gather checks on the specified
2958  * crypto_data_t struct. Returns true if the data is scatter/gather in nature
2959  * ie fails the test.
2960  */
2961 int
dca_sgcheck(dca_t * dca,crypto_data_t * data,dca_sg_param_t val)2962 dca_sgcheck(dca_t *dca, crypto_data_t *data, dca_sg_param_t val)
2963 {
2964 	uio_t *uiop;
2965 	mblk_t *mp;
2966 	int rv = FALSE;
2967 
2968 	switch (val) {
2969 	case DCA_SG_CONTIG:
2970 		/*
2971 		 * Check for a contiguous data buffer.
2972 		 */
2973 		switch (data->cd_format) {
2974 		case CRYPTO_DATA_RAW:
2975 			/* Contiguous in nature */
2976 			break;
2977 
2978 		case CRYPTO_DATA_UIO:
2979 			if (data->cd_uio->uio_iovcnt > 1)
2980 				rv = TRUE;
2981 			break;
2982 
2983 		case CRYPTO_DATA_MBLK:
2984 			mp = data->cd_mp;
2985 			if (mp->b_cont != NULL)
2986 				rv = TRUE;
2987 			break;
2988 
2989 		default:
2990 			DBG(NULL, DWARN, "unrecognised crypto data format");
2991 		}
2992 		break;
2993 
2994 	case DCA_SG_WALIGN:
2995 		/*
2996 		 * Check for a contiguous data buffer that is 32-bit word
2997 		 * aligned and is of word multiples in size.
2998 		 */
2999 		switch (data->cd_format) {
3000 		case CRYPTO_DATA_RAW:
3001 			if ((data->cd_raw.iov_len % sizeof (uint32_t)) ||
3002 			    ((uintptr_t)data->cd_raw.iov_base %
3003 			    sizeof (uint32_t))) {
3004 				rv = TRUE;
3005 			}
3006 			break;
3007 
3008 		case CRYPTO_DATA_UIO:
3009 			uiop = data->cd_uio;
3010 			if (uiop->uio_iovcnt > 1) {
3011 				return (TRUE);
3012 			}
3013 			/* So there is only one iovec */
3014 			if ((uiop->uio_iov[0].iov_len % sizeof (uint32_t)) ||
3015 			    ((uintptr_t)uiop->uio_iov[0].iov_base %
3016 			    sizeof (uint32_t))) {
3017 				rv = TRUE;
3018 			}
3019 			break;
3020 
3021 		case CRYPTO_DATA_MBLK:
3022 			mp = data->cd_mp;
3023 			if (mp->b_cont != NULL) {
3024 				return (TRUE);
3025 			}
3026 			/* So there is only one mblk in the chain */
3027 			if ((MBLKL(mp) % sizeof (uint32_t)) ||
3028 			    ((uintptr_t)mp->b_rptr % sizeof (uint32_t))) {
3029 				rv = TRUE;
3030 			}
3031 			break;
3032 
3033 		default:
3034 			DBG(NULL, DWARN, "unrecognised crypto data format");
3035 		}
3036 		break;
3037 
3038 	case DCA_SG_PALIGN:
3039 		/*
3040 		 * Check that the data buffer is page aligned and is of
3041 		 * page multiples in size.
3042 		 */
3043 		switch (data->cd_format) {
3044 		case CRYPTO_DATA_RAW:
3045 			if ((data->cd_length % dca->dca_pagesize) ||
3046 			    ((uintptr_t)data->cd_raw.iov_base %
3047 			    dca->dca_pagesize)) {
3048 				rv = TRUE;
3049 			}
3050 			break;
3051 
3052 		case CRYPTO_DATA_UIO:
3053 			uiop = data->cd_uio;
3054 			if ((uiop->uio_iov[0].iov_len % dca->dca_pagesize) ||
3055 			    ((uintptr_t)uiop->uio_iov[0].iov_base %
3056 			    dca->dca_pagesize)) {
3057 				rv = TRUE;
3058 			}
3059 			break;
3060 
3061 		case CRYPTO_DATA_MBLK:
3062 			mp = data->cd_mp;
3063 			if ((MBLKL(mp) % dca->dca_pagesize) ||
3064 			    ((uintptr_t)mp->b_rptr % dca->dca_pagesize)) {
3065 				rv = TRUE;
3066 			}
3067 			break;
3068 
3069 		default:
3070 			DBG(NULL, DWARN, "unrecognised crypto data format");
3071 		}
3072 		break;
3073 
3074 	default:
3075 		DBG(NULL, DWARN, "unrecognised scatter/gather param type");
3076 	}
3077 
3078 	return (rv);
3079 }
3080 
3081 /*
3082  * Increments the cd_offset and decrements the cd_length as the data is
3083  * gathered from the crypto_data_t struct.
3084  * The data is reverse-copied into the dest buffer if the flag is true.
3085  */
3086 int
dca_gather(crypto_data_t * in,char * dest,int count,int reverse)3087 dca_gather(crypto_data_t *in, char *dest, int count, int reverse)
3088 {
3089 	int	rv = CRYPTO_SUCCESS;
3090 	uint_t	vec_idx;
3091 	uio_t	*uiop;
3092 	off_t	off = in->cd_offset;
3093 	size_t	cur_len;
3094 	mblk_t	*mp;
3095 
3096 	switch (in->cd_format) {
3097 	case CRYPTO_DATA_RAW:
3098 		if (count > in->cd_length) {
3099 			/*
3100 			 * The caller specified a length greater than the
3101 			 * size of the buffer.
3102 			 */
3103 			return (CRYPTO_DATA_LEN_RANGE);
3104 		}
3105 		if (reverse)
3106 			dca_reverse(in->cd_raw.iov_base + off, dest, count,
3107 			    count);
3108 		else
3109 			bcopy(in->cd_raw.iov_base + in->cd_offset, dest, count);
3110 		in->cd_offset += count;
3111 		in->cd_length -= count;
3112 		break;
3113 
3114 	case CRYPTO_DATA_UIO:
3115 		/*
3116 		 * Jump to the first iovec containing data to be processed.
3117 		 */
3118 		uiop = in->cd_uio;
3119 		for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
3120 		    off >= uiop->uio_iov[vec_idx].iov_len;
3121 		    off -= uiop->uio_iov[vec_idx++].iov_len)
3122 			;
3123 		if (vec_idx == uiop->uio_iovcnt) {
3124 			/*
3125 			 * The caller specified an offset that is larger than
3126 			 * the total size of the buffers it provided.
3127 			 */
3128 			return (CRYPTO_DATA_LEN_RANGE);
3129 		}
3130 
3131 		/*
3132 		 * Now process the iovecs.
3133 		 */
3134 		while (vec_idx < uiop->uio_iovcnt && count > 0) {
3135 			cur_len = min(uiop->uio_iov[vec_idx].iov_len -
3136 			    off, count);
3137 			count -= cur_len;
3138 			if (reverse) {
3139 				/* Fill the dest buffer from the end */
3140 				dca_reverse(uiop->uio_iov[vec_idx].iov_base +
3141 				    off, dest+count, cur_len, cur_len);
3142 			} else {
3143 				bcopy(uiop->uio_iov[vec_idx].iov_base + off,
3144 				    dest, cur_len);
3145 				dest += cur_len;
3146 			}
3147 			in->cd_offset += cur_len;
3148 			in->cd_length -= cur_len;
3149 			vec_idx++;
3150 			off = 0;
3151 		}
3152 
3153 		if (vec_idx == uiop->uio_iovcnt && count > 0) {
3154 			/*
3155 			 * The end of the specified iovec's was reached but
3156 			 * the length requested could not be processed
3157 			 * (requested to digest more data than it provided).
3158 			 */
3159 			return (CRYPTO_DATA_LEN_RANGE);
3160 		}
3161 		break;
3162 
3163 	case CRYPTO_DATA_MBLK:
3164 		/*
3165 		 * Jump to the first mblk_t containing data to be processed.
3166 		 */
3167 		for (mp = in->cd_mp; mp != NULL && off >= MBLKL(mp);
3168 		    off -= MBLKL(mp), mp = mp->b_cont)
3169 			;
3170 		if (mp == NULL) {
3171 			/*
3172 			 * The caller specified an offset that is larger than
3173 			 * the total size of the buffers it provided.
3174 			 */
3175 			return (CRYPTO_DATA_LEN_RANGE);
3176 		}
3177 
3178 		/*
3179 		 * Now do the processing on the mblk chain.
3180 		 */
3181 		while (mp != NULL && count > 0) {
3182 			cur_len = min(MBLKL(mp) - off, count);
3183 			count -= cur_len;
3184 			if (reverse) {
3185 				/* Fill the dest buffer from the end */
3186 				dca_reverse((char *)(mp->b_rptr + off),
3187 				    dest+count, cur_len, cur_len);
3188 			} else {
3189 				bcopy((char *)(mp->b_rptr + off), dest,
3190 				    cur_len);
3191 				dest += cur_len;
3192 			}
3193 			in->cd_offset += cur_len;
3194 			in->cd_length -= cur_len;
3195 			mp = mp->b_cont;
3196 			off = 0;
3197 		}
3198 
3199 		if (mp == NULL && count > 0) {
3200 			/*
3201 			 * The end of the mblk was reached but the length
3202 			 * requested could not be processed, (requested to
3203 			 * digest more data than it provided).
3204 			 */
3205 			return (CRYPTO_DATA_LEN_RANGE);
3206 		}
3207 		break;
3208 
3209 	default:
3210 		DBG(NULL, DWARN, "dca_gather: unrecognised crypto data format");
3211 		rv = CRYPTO_ARGUMENTS_BAD;
3212 	}
3213 	return (rv);
3214 }
3215 
3216 /*
3217  * Increments the cd_offset and decrements the cd_length as the data is
3218  * gathered from the crypto_data_t struct.
3219  */
3220 int
dca_resid_gather(crypto_data_t * in,char * resid,int * residlen,char * dest,int count)3221 dca_resid_gather(crypto_data_t *in, char *resid, int *residlen, char *dest,
3222     int count)
3223 {
3224 	int	rv = CRYPTO_SUCCESS;
3225 	caddr_t	baddr;
3226 	uint_t	vec_idx;
3227 	uio_t	*uiop;
3228 	off_t	off = in->cd_offset;
3229 	size_t	cur_len;
3230 	mblk_t	*mp;
3231 
3232 	/* Process the residual first */
3233 	if (*residlen > 0) {
3234 		uint_t	num = min(count, *residlen);
3235 		bcopy(resid, dest, num);
3236 		*residlen -= num;
3237 		if (*residlen > 0) {
3238 			/*
3239 			 * Requested amount 'count' is less than what's in
3240 			 * the residual, so shuffle any remaining resid to
3241 			 * the front.
3242 			 */
3243 			baddr = resid + num;
3244 			bcopy(baddr, resid, *residlen);
3245 		}
3246 		dest += num;
3247 		count -= num;
3248 	}
3249 
3250 	/* Now process what's in the crypto_data_t structs */
3251 	switch (in->cd_format) {
3252 	case CRYPTO_DATA_RAW:
3253 		if (count > in->cd_length) {
3254 			/*
3255 			 * The caller specified a length greater than the
3256 			 * size of the buffer.
3257 			 */
3258 			return (CRYPTO_DATA_LEN_RANGE);
3259 		}
3260 		bcopy(in->cd_raw.iov_base + in->cd_offset, dest, count);
3261 		in->cd_offset += count;
3262 		in->cd_length -= count;
3263 		break;
3264 
3265 	case CRYPTO_DATA_UIO:
3266 		/*
3267 		 * Jump to the first iovec containing data to be processed.
3268 		 */
3269 		uiop = in->cd_uio;
3270 		for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
3271 		    off >= uiop->uio_iov[vec_idx].iov_len;
3272 		    off -= uiop->uio_iov[vec_idx++].iov_len)
3273 			;
3274 		if (vec_idx == uiop->uio_iovcnt) {
3275 			/*
3276 			 * The caller specified an offset that is larger than
3277 			 * the total size of the buffers it provided.
3278 			 */
3279 			return (CRYPTO_DATA_LEN_RANGE);
3280 		}
3281 
3282 		/*
3283 		 * Now process the iovecs.
3284 		 */
3285 		while (vec_idx < uiop->uio_iovcnt && count > 0) {
3286 			cur_len = min(uiop->uio_iov[vec_idx].iov_len -
3287 			    off, count);
3288 			bcopy(uiop->uio_iov[vec_idx].iov_base + off, dest,
3289 			    cur_len);
3290 			count -= cur_len;
3291 			dest += cur_len;
3292 			in->cd_offset += cur_len;
3293 			in->cd_length -= cur_len;
3294 			vec_idx++;
3295 			off = 0;
3296 		}
3297 
3298 		if (vec_idx == uiop->uio_iovcnt && count > 0) {
3299 			/*
3300 			 * The end of the specified iovec's was reached but
3301 			 * the length requested could not be processed
3302 			 * (requested to digest more data than it provided).
3303 			 */
3304 			return (CRYPTO_DATA_LEN_RANGE);
3305 		}
3306 		break;
3307 
3308 	case CRYPTO_DATA_MBLK:
3309 		/*
3310 		 * Jump to the first mblk_t containing data to be processed.
3311 		 */
3312 		for (mp = in->cd_mp; mp != NULL && off >= MBLKL(mp);
3313 		    off -= MBLKL(mp), mp = mp->b_cont)
3314 			;
3315 		if (mp == NULL) {
3316 			/*
3317 			 * The caller specified an offset that is larger than
3318 			 * the total size of the buffers it provided.
3319 			 */
3320 			return (CRYPTO_DATA_LEN_RANGE);
3321 		}
3322 
3323 		/*
3324 		 * Now do the processing on the mblk chain.
3325 		 */
3326 		while (mp != NULL && count > 0) {
3327 			cur_len = min(MBLKL(mp) - off, count);
3328 			bcopy((char *)(mp->b_rptr + off), dest, cur_len);
3329 			count -= cur_len;
3330 			dest += cur_len;
3331 			in->cd_offset += cur_len;
3332 			in->cd_length -= cur_len;
3333 			mp = mp->b_cont;
3334 			off = 0;
3335 		}
3336 
3337 		if (mp == NULL && count > 0) {
3338 			/*
3339 			 * The end of the mblk was reached but the length
3340 			 * requested could not be processed, (requested to
3341 			 * digest more data than it provided).
3342 			 */
3343 			return (CRYPTO_DATA_LEN_RANGE);
3344 		}
3345 		break;
3346 
3347 	default:
3348 		DBG(NULL, DWARN,
3349 		    "dca_resid_gather: unrecognised crypto data format");
3350 		rv = CRYPTO_ARGUMENTS_BAD;
3351 	}
3352 	return (rv);
3353 }
3354 
3355 /*
3356  * Appends the data to the crypto_data_t struct increasing cd_length.
3357  * cd_offset is left unchanged.
3358  * Data is reverse-copied if the flag is TRUE.
3359  */
3360 int
dca_scatter(const char * src,crypto_data_t * out,int count,int reverse)3361 dca_scatter(const char *src, crypto_data_t *out, int count, int reverse)
3362 {
3363 	int	rv = CRYPTO_SUCCESS;
3364 	off_t	offset = out->cd_offset + out->cd_length;
3365 	uint_t	vec_idx;
3366 	uio_t	*uiop;
3367 	size_t	cur_len;
3368 	mblk_t	*mp;
3369 
3370 	switch (out->cd_format) {
3371 	case CRYPTO_DATA_RAW:
3372 		if (out->cd_raw.iov_len - offset < count) {
3373 			/* Trying to write out more than space available. */
3374 			return (CRYPTO_DATA_LEN_RANGE);
3375 		}
3376 		if (reverse)
3377 			dca_reverse((void*) src, out->cd_raw.iov_base + offset,
3378 			    count, count);
3379 		else
3380 			bcopy(src, out->cd_raw.iov_base + offset, count);
3381 		out->cd_length += count;
3382 		break;
3383 
3384 	case CRYPTO_DATA_UIO:
3385 		/*
3386 		 * Jump to the first iovec that can be written to.
3387 		 */
3388 		uiop = out->cd_uio;
3389 		for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
3390 		    offset >= uiop->uio_iov[vec_idx].iov_len;
3391 		    offset -= uiop->uio_iov[vec_idx++].iov_len)
3392 			;
3393 		if (vec_idx == uiop->uio_iovcnt) {
3394 			/*
3395 			 * The caller specified an offset that is larger than
3396 			 * the total size of the buffers it provided.
3397 			 */
3398 			return (CRYPTO_DATA_LEN_RANGE);
3399 		}
3400 
3401 		/*
3402 		 * Now process the iovecs.
3403 		 */
3404 		while (vec_idx < uiop->uio_iovcnt && count > 0) {
3405 			cur_len = min(uiop->uio_iov[vec_idx].iov_len -
3406 			    offset, count);
3407 			count -= cur_len;
3408 			if (reverse) {
3409 				dca_reverse((void*) (src+count),
3410 				    uiop->uio_iov[vec_idx].iov_base +
3411 				    offset, cur_len, cur_len);
3412 			} else {
3413 				bcopy(src, uiop->uio_iov[vec_idx].iov_base +
3414 				    offset, cur_len);
3415 				src += cur_len;
3416 			}
3417 			out->cd_length += cur_len;
3418 			vec_idx++;
3419 			offset = 0;
3420 		}
3421 
3422 		if (vec_idx == uiop->uio_iovcnt && count > 0) {
3423 			/*
3424 			 * The end of the specified iovec's was reached but
3425 			 * the length requested could not be processed
3426 			 * (requested to write more data than space provided).
3427 			 */
3428 			return (CRYPTO_DATA_LEN_RANGE);
3429 		}
3430 		break;
3431 
3432 	case CRYPTO_DATA_MBLK:
3433 		/*
3434 		 * Jump to the first mblk_t that can be written to.
3435 		 */
3436 		for (mp = out->cd_mp; mp != NULL && offset >= MBLKL(mp);
3437 		    offset -= MBLKL(mp), mp = mp->b_cont)
3438 			;
3439 		if (mp == NULL) {
3440 			/*
3441 			 * The caller specified an offset that is larger than
3442 			 * the total size of the buffers it provided.
3443 			 */
3444 			return (CRYPTO_DATA_LEN_RANGE);
3445 		}
3446 
3447 		/*
3448 		 * Now do the processing on the mblk chain.
3449 		 */
3450 		while (mp != NULL && count > 0) {
3451 			cur_len = min(MBLKL(mp) - offset, count);
3452 			count -= cur_len;
3453 			if (reverse) {
3454 				dca_reverse((void*) (src+count),
3455 				    (char *)(mp->b_rptr + offset), cur_len,
3456 				    cur_len);
3457 			} else {
3458 				bcopy(src, (char *)(mp->b_rptr + offset),
3459 				    cur_len);
3460 				src += cur_len;
3461 			}
3462 			out->cd_length += cur_len;
3463 			mp = mp->b_cont;
3464 			offset = 0;
3465 		}
3466 
3467 		if (mp == NULL && count > 0) {
3468 			/*
3469 			 * The end of the mblk was reached but the length
3470 			 * requested could not be processed, (requested to
3471 			 * digest more data than it provided).
3472 			 */
3473 			return (CRYPTO_DATA_LEN_RANGE);
3474 		}
3475 		break;
3476 
3477 	default:
3478 		DBG(NULL, DWARN, "unrecognised crypto data format");
3479 		rv = CRYPTO_ARGUMENTS_BAD;
3480 	}
3481 	return (rv);
3482 }
3483 
3484 /*
3485  * Compare two byte arrays in reverse order.
3486  * Return 0 if they are identical, 1 otherwise.
3487  */
3488 int
dca_bcmp_reverse(const void * s1,const void * s2,size_t n)3489 dca_bcmp_reverse(const void *s1, const void *s2, size_t n)
3490 {
3491 	int i;
3492 	caddr_t src, dst;
3493 
3494 	if (!n)
3495 		return (0);
3496 
3497 	src = ((caddr_t)s1) + n - 1;
3498 	dst = (caddr_t)s2;
3499 	for (i = 0; i < n; i++) {
3500 		if (*src != *dst)
3501 			return (1);
3502 		src--;
3503 		dst++;
3504 	}
3505 
3506 	return (0);
3507 }
3508 
3509 
3510 /*
3511  * This calculates the size of a bignum in bits, specifically not counting
3512  * leading zero bits.  This size calculation must be done *before* any
3513  * endian reversal takes place (i.e. the numbers are in absolute big-endian
3514  * order.)
3515  */
3516 int
dca_bitlen(unsigned char * bignum,int bytelen)3517 dca_bitlen(unsigned char *bignum, int bytelen)
3518 {
3519 	unsigned char	msbyte;
3520 	int		i, j;
3521 
3522 	for (i = 0; i < bytelen - 1; i++) {
3523 		if (bignum[i] != 0) {
3524 			break;
3525 		}
3526 	}
3527 	msbyte = bignum[i];
3528 	for (j = 8; j > 1; j--) {
3529 		if (msbyte & 0x80) {
3530 			break;
3531 		}
3532 		msbyte <<= 1;
3533 	}
3534 	return ((8 * (bytelen - i - 1)) + j);
3535 }
3536 
3537 /*
3538  * This compares to bignums (in big-endian order).  It ignores leading
3539  * null bytes.  The result semantics follow bcmp, mempcmp, strcmp, etc.
3540  */
3541 int
dca_numcmp(caddr_t n1,int n1len,caddr_t n2,int n2len)3542 dca_numcmp(caddr_t n1, int n1len, caddr_t n2, int n2len)
3543 {
3544 	while ((n1len > 1) && (*n1 == 0)) {
3545 		n1len--;
3546 		n1++;
3547 	}
3548 	while ((n2len > 1) && (*n2 == 0)) {
3549 		n2len--;
3550 		n2++;
3551 	}
3552 	if (n1len != n2len) {
3553 		return (n1len - n2len);
3554 	}
3555 	while ((n1len > 1) && (*n1 == *n2)) {
3556 		n1++;
3557 		n2++;
3558 		n1len--;
3559 	}
3560 	return ((int)(*(uchar_t *)n1) - (int)(*(uchar_t *)n2));
3561 }
3562 
3563 /*
3564  * Return array of key attributes.
3565  */
3566 crypto_object_attribute_t *
dca_get_key_attr(crypto_key_t * key)3567 dca_get_key_attr(crypto_key_t *key)
3568 {
3569 	if ((key->ck_format != CRYPTO_KEY_ATTR_LIST) ||
3570 	    (key->ck_count == 0)) {
3571 		return (NULL);
3572 	}
3573 
3574 	return (key->ck_attrs);
3575 }
3576 
3577 /*
3578  * If attribute type exists valp points to it's 32-bit value.
3579  */
3580 int
dca_attr_lookup_uint32(crypto_object_attribute_t * attrp,uint_t atnum,uint64_t atype,uint32_t * valp)3581 dca_attr_lookup_uint32(crypto_object_attribute_t *attrp, uint_t atnum,
3582     uint64_t atype, uint32_t *valp)
3583 {
3584 	crypto_object_attribute_t	*bap;
3585 
3586 	bap = dca_find_attribute(attrp, atnum, atype);
3587 	if (bap == NULL) {
3588 		return (CRYPTO_ATTRIBUTE_TYPE_INVALID);
3589 	}
3590 
3591 	*valp = *bap->oa_value;
3592 
3593 	return (CRYPTO_SUCCESS);
3594 }
3595 
3596 /*
3597  * If attribute type exists data contains the start address of the value,
3598  * and numelems contains it's length.
3599  */
3600 int
dca_attr_lookup_uint8_array(crypto_object_attribute_t * attrp,uint_t atnum,uint64_t atype,void ** data,unsigned int * numelems)3601 dca_attr_lookup_uint8_array(crypto_object_attribute_t *attrp, uint_t atnum,
3602     uint64_t atype, void **data, unsigned int *numelems)
3603 {
3604 	crypto_object_attribute_t	*bap;
3605 
3606 	bap = dca_find_attribute(attrp, atnum, atype);
3607 	if (bap == NULL) {
3608 		return (CRYPTO_ATTRIBUTE_TYPE_INVALID);
3609 	}
3610 
3611 	*data = bap->oa_value;
3612 	*numelems = bap->oa_value_len;
3613 
3614 	return (CRYPTO_SUCCESS);
3615 }
3616 
3617 /*
3618  * Finds entry of specified name. If it is not found dca_find_attribute returns
3619  * NULL.
3620  */
3621 crypto_object_attribute_t *
dca_find_attribute(crypto_object_attribute_t * attrp,uint_t atnum,uint64_t atype)3622 dca_find_attribute(crypto_object_attribute_t *attrp, uint_t atnum,
3623     uint64_t atype)
3624 {
3625 	while (atnum) {
3626 		if (attrp->oa_type == atype)
3627 			return (attrp);
3628 		atnum--;
3629 		attrp++;
3630 	}
3631 	return (NULL);
3632 }
3633 
3634 /*
3635  * Return the address of the first data buffer. If the data format is
3636  * unrecognised return NULL.
3637  */
3638 caddr_t
dca_bufdaddr(crypto_data_t * data)3639 dca_bufdaddr(crypto_data_t *data)
3640 {
3641 	switch (data->cd_format) {
3642 	case CRYPTO_DATA_RAW:
3643 		return (data->cd_raw.iov_base + data->cd_offset);
3644 	case CRYPTO_DATA_UIO:
3645 		return (data->cd_uio->uio_iov[0].iov_base + data->cd_offset);
3646 	case CRYPTO_DATA_MBLK:
3647 		return ((char *)data->cd_mp->b_rptr + data->cd_offset);
3648 	default:
3649 		DBG(NULL, DWARN,
3650 		    "dca_bufdaddr: unrecognised crypto data format");
3651 		return (NULL);
3652 	}
3653 }
3654 
3655 static caddr_t
dca_bufdaddr_out(crypto_data_t * data)3656 dca_bufdaddr_out(crypto_data_t *data)
3657 {
3658 	size_t offset = data->cd_offset + data->cd_length;
3659 
3660 	switch (data->cd_format) {
3661 	case CRYPTO_DATA_RAW:
3662 		return (data->cd_raw.iov_base + offset);
3663 	case CRYPTO_DATA_UIO:
3664 		return (data->cd_uio->uio_iov[0].iov_base + offset);
3665 	case CRYPTO_DATA_MBLK:
3666 		return ((char *)data->cd_mp->b_rptr + offset);
3667 	default:
3668 		DBG(NULL, DWARN,
3669 		    "dca_bufdaddr_out: unrecognised crypto data format");
3670 		return (NULL);
3671 	}
3672 }
3673 
3674 /*
3675  * Control entry points.
3676  */
3677 
3678 /* ARGSUSED */
3679 static void
dca_provider_status(crypto_provider_handle_t provider,uint_t * status)3680 dca_provider_status(crypto_provider_handle_t provider, uint_t *status)
3681 {
3682 	*status = CRYPTO_PROVIDER_READY;
3683 }
3684 
3685 /*
3686  * Cipher (encrypt/decrypt) entry points.
3687  */
3688 
3689 /* ARGSUSED */
3690 static int
dca_encrypt_init(crypto_ctx_t * ctx,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)3691 dca_encrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
3692     crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
3693     crypto_req_handle_t req)
3694 {
3695 	int error = CRYPTO_FAILED;
3696 	dca_t *softc;
3697 	/* LINTED E_FUNC_SET_NOT_USED */
3698 	int instance;
3699 
3700 	/* extract softc and instance number from context */
3701 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
3702 	DBG(softc, DENTRY, "dca_encrypt_init: started");
3703 
3704 	/* check mechanism */
3705 	switch (mechanism->cm_type) {
3706 	case DES_CBC_MECH_INFO_TYPE:
3707 		error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
3708 		    DR_ENCRYPT);
3709 		break;
3710 	case DES3_CBC_MECH_INFO_TYPE:
3711 		error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
3712 		    DR_ENCRYPT | DR_TRIPLE);
3713 		break;
3714 	case RSA_PKCS_MECH_INFO_TYPE:
3715 	case RSA_X_509_MECH_INFO_TYPE:
3716 		error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
3717 		break;
3718 	default:
3719 		cmn_err(CE_WARN, "dca_encrypt_init: unexpected mech type "
3720 		    "0x%llx\n", (unsigned long long)mechanism->cm_type);
3721 		error = CRYPTO_MECHANISM_INVALID;
3722 	}
3723 
3724 	DBG(softc, DENTRY, "dca_encrypt_init: done, err = 0x%x", error);
3725 
3726 	if (error == CRYPTO_SUCCESS)
3727 		dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
3728 		    &softc->dca_ctx_list_lock);
3729 
3730 	return (error);
3731 }
3732 
3733 /* ARGSUSED */
3734 static int
dca_encrypt(crypto_ctx_t * ctx,crypto_data_t * plaintext,crypto_data_t * ciphertext,crypto_req_handle_t req)3735 dca_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
3736     crypto_data_t *ciphertext, crypto_req_handle_t req)
3737 {
3738 	int error = CRYPTO_FAILED;
3739 	dca_t *softc;
3740 	/* LINTED E_FUNC_SET_NOT_USED */
3741 	int instance;
3742 
3743 	if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
3744 		return (CRYPTO_OPERATION_NOT_INITIALIZED);
3745 
3746 	/* extract softc and instance number from context */
3747 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
3748 	DBG(softc, DENTRY, "dca_encrypt: started");
3749 
3750 	/* handle inplace ops */
3751 	if (!ciphertext) {
3752 		dca_request_t *reqp = ctx->cc_provider_private;
3753 		reqp->dr_flags |= DR_INPLACE;
3754 		ciphertext = plaintext;
3755 	}
3756 
3757 	/* check mechanism */
3758 	switch (DCA_MECH_FROM_CTX(ctx)) {
3759 	case DES_CBC_MECH_INFO_TYPE:
3760 		error = dca_3des(ctx, plaintext, ciphertext, req, DR_ENCRYPT);
3761 		break;
3762 	case DES3_CBC_MECH_INFO_TYPE:
3763 		error = dca_3des(ctx, plaintext, ciphertext, req,
3764 		    DR_ENCRYPT | DR_TRIPLE);
3765 		break;
3766 	case RSA_PKCS_MECH_INFO_TYPE:
3767 	case RSA_X_509_MECH_INFO_TYPE:
3768 		error = dca_rsastart(ctx, plaintext, ciphertext, req,
3769 		    DCA_RSA_ENC);
3770 		break;
3771 	default:
3772 		/* Should never reach here */
3773 		cmn_err(CE_WARN, "dca_encrypt: unexpected mech type "
3774 		    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
3775 		error = CRYPTO_MECHANISM_INVALID;
3776 	}
3777 
3778 	if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS) &&
3779 	    (error != CRYPTO_BUFFER_TOO_SMALL)) {
3780 		ciphertext->cd_length = 0;
3781 	}
3782 
3783 	DBG(softc, DENTRY, "dca_encrypt: done, err = 0x%x", error);
3784 
3785 	return (error);
3786 }
3787 
3788 /* ARGSUSED */
3789 static int
dca_encrypt_update(crypto_ctx_t * ctx,crypto_data_t * plaintext,crypto_data_t * ciphertext,crypto_req_handle_t req)3790 dca_encrypt_update(crypto_ctx_t *ctx, crypto_data_t *plaintext,
3791     crypto_data_t *ciphertext, crypto_req_handle_t req)
3792 {
3793 	int error = CRYPTO_FAILED;
3794 	dca_t *softc;
3795 	/* LINTED E_FUNC_SET_NOT_USED */
3796 	int instance;
3797 
3798 	if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
3799 		return (CRYPTO_OPERATION_NOT_INITIALIZED);
3800 
3801 	/* extract softc and instance number from context */
3802 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
3803 	DBG(softc, DENTRY, "dca_encrypt_update: started");
3804 
3805 	/* handle inplace ops */
3806 	if (!ciphertext) {
3807 		dca_request_t *reqp = ctx->cc_provider_private;
3808 		reqp->dr_flags |= DR_INPLACE;
3809 		ciphertext = plaintext;
3810 	}
3811 
3812 	/* check mechanism */
3813 	switch (DCA_MECH_FROM_CTX(ctx)) {
3814 	case DES_CBC_MECH_INFO_TYPE:
3815 		error = dca_3desupdate(ctx, plaintext, ciphertext, req,
3816 		    DR_ENCRYPT);
3817 		break;
3818 	case DES3_CBC_MECH_INFO_TYPE:
3819 		error = dca_3desupdate(ctx, plaintext, ciphertext, req,
3820 		    DR_ENCRYPT | DR_TRIPLE);
3821 		break;
3822 	default:
3823 		/* Should never reach here */
3824 		cmn_err(CE_WARN, "dca_encrypt_update: unexpected mech type "
3825 		    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
3826 		error = CRYPTO_MECHANISM_INVALID;
3827 	}
3828 
3829 	DBG(softc, DENTRY, "dca_encrypt_update: done, err = 0x%x", error);
3830 
3831 	return (error);
3832 }
3833 
3834 /* ARGSUSED */
3835 static int
dca_encrypt_final(crypto_ctx_t * ctx,crypto_data_t * ciphertext,crypto_req_handle_t req)3836 dca_encrypt_final(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
3837     crypto_req_handle_t req)
3838 {
3839 	int error = CRYPTO_FAILED;
3840 	dca_t *softc;
3841 	/* LINTED E_FUNC_SET_NOT_USED */
3842 	int instance;
3843 
3844 	if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
3845 		return (CRYPTO_OPERATION_NOT_INITIALIZED);
3846 
3847 	/* extract softc and instance number from context */
3848 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
3849 	DBG(softc, DENTRY, "dca_encrypt_final: started");
3850 
3851 	/* check mechanism */
3852 	switch (DCA_MECH_FROM_CTX(ctx)) {
3853 	case DES_CBC_MECH_INFO_TYPE:
3854 		error = dca_3desfinal(ctx, ciphertext, DR_ENCRYPT);
3855 		break;
3856 	case DES3_CBC_MECH_INFO_TYPE:
3857 		error = dca_3desfinal(ctx, ciphertext, DR_ENCRYPT | DR_TRIPLE);
3858 		break;
3859 	default:
3860 		/* Should never reach here */
3861 		cmn_err(CE_WARN, "dca_encrypt_final: unexpected mech type "
3862 		    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
3863 		error = CRYPTO_MECHANISM_INVALID;
3864 	}
3865 
3866 	DBG(softc, DENTRY, "dca_encrypt_final: done, err = 0x%x", error);
3867 
3868 	return (error);
3869 }
3870 
3871 /* ARGSUSED */
3872 static int
dca_encrypt_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * plaintext,crypto_data_t * ciphertext,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)3873 dca_encrypt_atomic(crypto_provider_handle_t provider,
3874     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
3875     crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext,
3876     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
3877 {
3878 	int error = CRYPTO_FAILED;
3879 	dca_t *softc = (dca_t *)provider;
3880 
3881 	DBG(softc, DENTRY, "dca_encrypt_atomic: started");
3882 
3883 	if (ctx_template != NULL)
3884 		return (CRYPTO_ARGUMENTS_BAD);
3885 
3886 	/* handle inplace ops */
3887 	if (!ciphertext) {
3888 		ciphertext = plaintext;
3889 	}
3890 
3891 	/* check mechanism */
3892 	switch (mechanism->cm_type) {
3893 	case DES_CBC_MECH_INFO_TYPE:
3894 		error = dca_3desatomic(provider, session_id, mechanism, key,
3895 		    plaintext, ciphertext, KM_SLEEP, req,
3896 		    DR_ENCRYPT | DR_ATOMIC);
3897 		break;
3898 	case DES3_CBC_MECH_INFO_TYPE:
3899 		error = dca_3desatomic(provider, session_id, mechanism, key,
3900 		    plaintext, ciphertext, KM_SLEEP, req,
3901 		    DR_ENCRYPT | DR_TRIPLE | DR_ATOMIC);
3902 		break;
3903 	case RSA_PKCS_MECH_INFO_TYPE:
3904 	case RSA_X_509_MECH_INFO_TYPE:
3905 		error = dca_rsaatomic(provider, session_id, mechanism, key,
3906 		    plaintext, ciphertext, KM_SLEEP, req, DCA_RSA_ENC);
3907 		break;
3908 	default:
3909 		cmn_err(CE_WARN, "dca_encrypt_atomic: unexpected mech type "
3910 		    "0x%llx\n", (unsigned long long)mechanism->cm_type);
3911 		error = CRYPTO_MECHANISM_INVALID;
3912 	}
3913 
3914 	if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS)) {
3915 		ciphertext->cd_length = 0;
3916 	}
3917 
3918 	DBG(softc, DENTRY, "dca_encrypt_atomic: done, err = 0x%x", error);
3919 
3920 	return (error);
3921 }
3922 
3923 /* ARGSUSED */
3924 static int
dca_decrypt_init(crypto_ctx_t * ctx,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)3925 dca_decrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
3926     crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
3927     crypto_req_handle_t req)
3928 {
3929 	int error = CRYPTO_FAILED;
3930 	dca_t *softc;
3931 	/* LINTED E_FUNC_SET_NOT_USED */
3932 	int instance;
3933 
3934 	/* extract softc and instance number from context */
3935 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
3936 	DBG(softc, DENTRY, "dca_decrypt_init: started");
3937 
3938 	/* check mechanism */
3939 	switch (mechanism->cm_type) {
3940 	case DES_CBC_MECH_INFO_TYPE:
3941 		error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
3942 		    DR_DECRYPT);
3943 		break;
3944 	case DES3_CBC_MECH_INFO_TYPE:
3945 		error = dca_3desctxinit(ctx, mechanism, key, KM_SLEEP,
3946 		    DR_DECRYPT | DR_TRIPLE);
3947 		break;
3948 	case RSA_PKCS_MECH_INFO_TYPE:
3949 	case RSA_X_509_MECH_INFO_TYPE:
3950 		error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
3951 		break;
3952 	default:
3953 		cmn_err(CE_WARN, "dca_decrypt_init: unexpected mech type "
3954 		    "0x%llx\n", (unsigned long long)mechanism->cm_type);
3955 		error = CRYPTO_MECHANISM_INVALID;
3956 	}
3957 
3958 	DBG(softc, DENTRY, "dca_decrypt_init: done, err = 0x%x", error);
3959 
3960 	if (error == CRYPTO_SUCCESS)
3961 		dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
3962 		    &softc->dca_ctx_list_lock);
3963 
3964 	return (error);
3965 }
3966 
3967 /* ARGSUSED */
3968 static int
dca_decrypt(crypto_ctx_t * ctx,crypto_data_t * ciphertext,crypto_data_t * plaintext,crypto_req_handle_t req)3969 dca_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
3970     crypto_data_t *plaintext, crypto_req_handle_t req)
3971 {
3972 	int error = CRYPTO_FAILED;
3973 	dca_t *softc;
3974 	/* LINTED E_FUNC_SET_NOT_USED */
3975 	int instance;
3976 
3977 	if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
3978 		return (CRYPTO_OPERATION_NOT_INITIALIZED);
3979 
3980 	/* extract softc and instance number from context */
3981 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
3982 	DBG(softc, DENTRY, "dca_decrypt: started");
3983 
3984 	/* handle inplace ops */
3985 	if (!plaintext) {
3986 		dca_request_t *reqp = ctx->cc_provider_private;
3987 		reqp->dr_flags |= DR_INPLACE;
3988 		plaintext = ciphertext;
3989 	}
3990 
3991 	/* check mechanism */
3992 	switch (DCA_MECH_FROM_CTX(ctx)) {
3993 	case DES_CBC_MECH_INFO_TYPE:
3994 		error = dca_3des(ctx, ciphertext, plaintext, req, DR_DECRYPT);
3995 		break;
3996 	case DES3_CBC_MECH_INFO_TYPE:
3997 		error = dca_3des(ctx, ciphertext, plaintext, req,
3998 		    DR_DECRYPT | DR_TRIPLE);
3999 		break;
4000 	case RSA_PKCS_MECH_INFO_TYPE:
4001 	case RSA_X_509_MECH_INFO_TYPE:
4002 		error = dca_rsastart(ctx, ciphertext, plaintext, req,
4003 		    DCA_RSA_DEC);
4004 		break;
4005 	default:
4006 		/* Should never reach here */
4007 		cmn_err(CE_WARN, "dca_decrypt: unexpected mech type "
4008 		    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4009 		error = CRYPTO_MECHANISM_INVALID;
4010 	}
4011 
4012 	if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS) &&
4013 	    (error != CRYPTO_BUFFER_TOO_SMALL)) {
4014 		if (plaintext)
4015 			plaintext->cd_length = 0;
4016 	}
4017 
4018 	DBG(softc, DENTRY, "dca_decrypt: done, err = 0x%x", error);
4019 
4020 	return (error);
4021 }
4022 
4023 /* ARGSUSED */
4024 static int
dca_decrypt_update(crypto_ctx_t * ctx,crypto_data_t * ciphertext,crypto_data_t * plaintext,crypto_req_handle_t req)4025 dca_decrypt_update(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
4026     crypto_data_t *plaintext, crypto_req_handle_t req)
4027 {
4028 	int error = CRYPTO_FAILED;
4029 	dca_t *softc;
4030 	/* LINTED E_FUNC_SET_NOT_USED */
4031 	int instance;
4032 
4033 	if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4034 		return (CRYPTO_OPERATION_NOT_INITIALIZED);
4035 
4036 	/* extract softc and instance number from context */
4037 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4038 	DBG(softc, DENTRY, "dca_decrypt_update: started");
4039 
4040 	/* handle inplace ops */
4041 	if (!plaintext) {
4042 		dca_request_t *reqp = ctx->cc_provider_private;
4043 		reqp->dr_flags |= DR_INPLACE;
4044 		plaintext = ciphertext;
4045 	}
4046 
4047 	/* check mechanism */
4048 	switch (DCA_MECH_FROM_CTX(ctx)) {
4049 	case DES_CBC_MECH_INFO_TYPE:
4050 		error = dca_3desupdate(ctx, ciphertext, plaintext, req,
4051 		    DR_DECRYPT);
4052 		break;
4053 	case DES3_CBC_MECH_INFO_TYPE:
4054 		error = dca_3desupdate(ctx, ciphertext, plaintext, req,
4055 		    DR_DECRYPT | DR_TRIPLE);
4056 		break;
4057 	default:
4058 		/* Should never reach here */
4059 		cmn_err(CE_WARN, "dca_decrypt_update: unexpected mech type "
4060 		    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4061 		error = CRYPTO_MECHANISM_INVALID;
4062 	}
4063 
4064 	DBG(softc, DENTRY, "dca_decrypt_update: done, err = 0x%x", error);
4065 
4066 	return (error);
4067 }
4068 
4069 /* ARGSUSED */
4070 static int
dca_decrypt_final(crypto_ctx_t * ctx,crypto_data_t * plaintext,crypto_req_handle_t req)4071 dca_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *plaintext,
4072     crypto_req_handle_t req)
4073 {
4074 	int error = CRYPTO_FAILED;
4075 	dca_t *softc;
4076 	/* LINTED E_FUNC_SET_NOT_USED */
4077 	int instance;
4078 
4079 	if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4080 		return (CRYPTO_OPERATION_NOT_INITIALIZED);
4081 
4082 	/* extract softc and instance number from context */
4083 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4084 	DBG(softc, DENTRY, "dca_decrypt_final: started");
4085 
4086 	/* check mechanism */
4087 	switch (DCA_MECH_FROM_CTX(ctx)) {
4088 	case DES_CBC_MECH_INFO_TYPE:
4089 		error = dca_3desfinal(ctx, plaintext, DR_DECRYPT);
4090 		break;
4091 	case DES3_CBC_MECH_INFO_TYPE:
4092 		error = dca_3desfinal(ctx, plaintext, DR_DECRYPT | DR_TRIPLE);
4093 		break;
4094 	default:
4095 		/* Should never reach here */
4096 		cmn_err(CE_WARN, "dca_decrypt_final: unexpected mech type "
4097 		    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4098 		error = CRYPTO_MECHANISM_INVALID;
4099 	}
4100 
4101 	DBG(softc, DENTRY, "dca_decrypt_final: done, err = 0x%x", error);
4102 
4103 	return (error);
4104 }
4105 
4106 /* ARGSUSED */
4107 static int
dca_decrypt_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * ciphertext,crypto_data_t * plaintext,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)4108 dca_decrypt_atomic(crypto_provider_handle_t provider,
4109     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4110     crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext,
4111     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4112 {
4113 	int error = CRYPTO_FAILED;
4114 	dca_t *softc = (dca_t *)provider;
4115 
4116 	DBG(softc, DENTRY, "dca_decrypt_atomic: started");
4117 
4118 	if (ctx_template != NULL)
4119 		return (CRYPTO_ARGUMENTS_BAD);
4120 
4121 	/* handle inplace ops */
4122 	if (!plaintext) {
4123 		plaintext = ciphertext;
4124 	}
4125 
4126 	/* check mechanism */
4127 	switch (mechanism->cm_type) {
4128 	case DES_CBC_MECH_INFO_TYPE:
4129 		error = dca_3desatomic(provider, session_id, mechanism, key,
4130 		    ciphertext, plaintext, KM_SLEEP, req,
4131 		    DR_DECRYPT | DR_ATOMIC);
4132 		break;
4133 	case DES3_CBC_MECH_INFO_TYPE:
4134 		error = dca_3desatomic(provider, session_id, mechanism, key,
4135 		    ciphertext, plaintext, KM_SLEEP, req,
4136 		    DR_DECRYPT | DR_TRIPLE | DR_ATOMIC);
4137 		break;
4138 	case RSA_PKCS_MECH_INFO_TYPE:
4139 	case RSA_X_509_MECH_INFO_TYPE:
4140 		error = dca_rsaatomic(provider, session_id, mechanism, key,
4141 		    ciphertext, plaintext, KM_SLEEP, req, DCA_RSA_DEC);
4142 		break;
4143 	default:
4144 		cmn_err(CE_WARN, "dca_decrypt_atomic: unexpected mech type "
4145 		    "0x%llx\n", (unsigned long long)mechanism->cm_type);
4146 		error = CRYPTO_MECHANISM_INVALID;
4147 	}
4148 
4149 	if ((error != CRYPTO_QUEUED) && (error != CRYPTO_SUCCESS)) {
4150 		plaintext->cd_length = 0;
4151 	}
4152 
4153 	DBG(softc, DENTRY, "dca_decrypt_atomic: done, err = 0x%x", error);
4154 
4155 	return (error);
4156 }
4157 
4158 /*
4159  * Sign entry points.
4160  */
4161 
4162 /* ARGSUSED */
4163 static int
dca_sign_init(crypto_ctx_t * ctx,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)4164 dca_sign_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
4165     crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
4166     crypto_req_handle_t req)
4167 {
4168 	int error = CRYPTO_FAILED;
4169 	dca_t *softc;
4170 	/* LINTED E_FUNC_SET_NOT_USED */
4171 	int instance;
4172 
4173 	/* extract softc and instance number from context */
4174 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4175 	DBG(softc, DENTRY, "dca_sign_init: started\n");
4176 
4177 	if (ctx_template != NULL)
4178 		return (CRYPTO_ARGUMENTS_BAD);
4179 
4180 	/* check mechanism */
4181 	switch (mechanism->cm_type) {
4182 	case RSA_PKCS_MECH_INFO_TYPE:
4183 	case RSA_X_509_MECH_INFO_TYPE:
4184 		error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
4185 		break;
4186 	case DSA_MECH_INFO_TYPE:
4187 		error = dca_dsainit(ctx, mechanism, key, KM_SLEEP,
4188 		    DCA_DSA_SIGN);
4189 		break;
4190 	default:
4191 		cmn_err(CE_WARN, "dca_sign_init: unexpected mech type "
4192 		    "0x%llx\n", (unsigned long long)mechanism->cm_type);
4193 		error = CRYPTO_MECHANISM_INVALID;
4194 	}
4195 
4196 	DBG(softc, DENTRY, "dca_sign_init: done, err = 0x%x", error);
4197 
4198 	if (error == CRYPTO_SUCCESS)
4199 		dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
4200 		    &softc->dca_ctx_list_lock);
4201 
4202 	return (error);
4203 }
4204 
4205 static int
dca_sign(crypto_ctx_t * ctx,crypto_data_t * data,crypto_data_t * signature,crypto_req_handle_t req)4206 dca_sign(crypto_ctx_t *ctx, crypto_data_t *data,
4207     crypto_data_t *signature, crypto_req_handle_t req)
4208 {
4209 	int error = CRYPTO_FAILED;
4210 	dca_t *softc;
4211 	/* LINTED E_FUNC_SET_NOT_USED */
4212 	int instance;
4213 
4214 	if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4215 		return (CRYPTO_OPERATION_NOT_INITIALIZED);
4216 
4217 	/* extract softc and instance number from context */
4218 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4219 	DBG(softc, DENTRY, "dca_sign: started\n");
4220 
4221 	/* check mechanism */
4222 	switch (DCA_MECH_FROM_CTX(ctx)) {
4223 	case RSA_PKCS_MECH_INFO_TYPE:
4224 	case RSA_X_509_MECH_INFO_TYPE:
4225 		error = dca_rsastart(ctx, data, signature, req, DCA_RSA_SIGN);
4226 		break;
4227 	case DSA_MECH_INFO_TYPE:
4228 		error = dca_dsa_sign(ctx, data, signature, req);
4229 		break;
4230 	default:
4231 		cmn_err(CE_WARN, "dca_sign: unexpected mech type "
4232 		    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4233 		error = CRYPTO_MECHANISM_INVALID;
4234 	}
4235 
4236 	DBG(softc, DENTRY, "dca_sign: done, err = 0x%x", error);
4237 
4238 	return (error);
4239 }
4240 
4241 /* ARGSUSED */
4242 static int
dca_sign_update(crypto_ctx_t * ctx,crypto_data_t * data,crypto_req_handle_t req)4243 dca_sign_update(crypto_ctx_t *ctx, crypto_data_t *data,
4244     crypto_req_handle_t req)
4245 {
4246 	int error = CRYPTO_MECHANISM_INVALID;
4247 	dca_t *softc;
4248 	/* LINTED E_FUNC_SET_NOT_USED */
4249 	int instance;
4250 
4251 	if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4252 		return (CRYPTO_OPERATION_NOT_INITIALIZED);
4253 
4254 	/* extract softc and instance number from context */
4255 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4256 	DBG(softc, DENTRY, "dca_sign_update: started\n");
4257 
4258 	cmn_err(CE_WARN, "dca_sign_update: unexpected mech type "
4259 	    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4260 
4261 	DBG(softc, DENTRY, "dca_sign_update: done, err = 0x%x", error);
4262 
4263 	return (error);
4264 }
4265 
4266 /* ARGSUSED */
4267 static int
dca_sign_final(crypto_ctx_t * ctx,crypto_data_t * signature,crypto_req_handle_t req)4268 dca_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature,
4269     crypto_req_handle_t req)
4270 {
4271 	int error = CRYPTO_MECHANISM_INVALID;
4272 	dca_t *softc;
4273 	/* LINTED E_FUNC_SET_NOT_USED */
4274 	int instance;
4275 
4276 	if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4277 		return (CRYPTO_OPERATION_NOT_INITIALIZED);
4278 
4279 	/* extract softc and instance number from context */
4280 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4281 	DBG(softc, DENTRY, "dca_sign_final: started\n");
4282 
4283 	cmn_err(CE_WARN, "dca_sign_final: unexpected mech type "
4284 	    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4285 
4286 	DBG(softc, DENTRY, "dca_sign_final: done, err = 0x%x", error);
4287 
4288 	return (error);
4289 }
4290 
4291 static int
dca_sign_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * data,crypto_data_t * signature,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)4292 dca_sign_atomic(crypto_provider_handle_t provider,
4293     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4294     crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
4295     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4296 {
4297 	int error = CRYPTO_FAILED;
4298 	dca_t *softc = (dca_t *)provider;
4299 
4300 	DBG(softc, DENTRY, "dca_sign_atomic: started\n");
4301 
4302 	if (ctx_template != NULL)
4303 		return (CRYPTO_ARGUMENTS_BAD);
4304 
4305 	/* check mechanism */
4306 	switch (mechanism->cm_type) {
4307 	case RSA_PKCS_MECH_INFO_TYPE:
4308 	case RSA_X_509_MECH_INFO_TYPE:
4309 		error = dca_rsaatomic(provider, session_id, mechanism, key,
4310 		    data, signature, KM_SLEEP, req, DCA_RSA_SIGN);
4311 		break;
4312 	case DSA_MECH_INFO_TYPE:
4313 		error = dca_dsaatomic(provider, session_id, mechanism, key,
4314 		    data, signature, KM_SLEEP, req, DCA_DSA_SIGN);
4315 		break;
4316 	default:
4317 		cmn_err(CE_WARN, "dca_sign_atomic: unexpected mech type "
4318 		    "0x%llx\n", (unsigned long long)mechanism->cm_type);
4319 		error = CRYPTO_MECHANISM_INVALID;
4320 	}
4321 
4322 	DBG(softc, DENTRY, "dca_sign_atomic: done, err = 0x%x", error);
4323 
4324 	return (error);
4325 }
4326 
4327 /* ARGSUSED */
4328 static int
dca_sign_recover_init(crypto_ctx_t * ctx,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)4329 dca_sign_recover_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
4330     crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
4331     crypto_req_handle_t req)
4332 {
4333 	int error = CRYPTO_FAILED;
4334 	dca_t *softc;
4335 	/* LINTED E_FUNC_SET_NOT_USED */
4336 	int instance;
4337 
4338 	/* extract softc and instance number from context */
4339 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4340 	DBG(softc, DENTRY, "dca_sign_recover_init: started\n");
4341 
4342 	if (ctx_template != NULL)
4343 		return (CRYPTO_ARGUMENTS_BAD);
4344 
4345 	/* check mechanism */
4346 	switch (mechanism->cm_type) {
4347 	case RSA_PKCS_MECH_INFO_TYPE:
4348 	case RSA_X_509_MECH_INFO_TYPE:
4349 		error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
4350 		break;
4351 	default:
4352 		cmn_err(CE_WARN, "dca_sign_recover_init: unexpected mech type "
4353 		    "0x%llx\n", (unsigned long long)mechanism->cm_type);
4354 		error = CRYPTO_MECHANISM_INVALID;
4355 	}
4356 
4357 	DBG(softc, DENTRY, "dca_sign_recover_init: done, err = 0x%x", error);
4358 
4359 	if (error == CRYPTO_SUCCESS)
4360 		dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
4361 		    &softc->dca_ctx_list_lock);
4362 
4363 	return (error);
4364 }
4365 
4366 static int
dca_sign_recover(crypto_ctx_t * ctx,crypto_data_t * data,crypto_data_t * signature,crypto_req_handle_t req)4367 dca_sign_recover(crypto_ctx_t *ctx, crypto_data_t *data,
4368     crypto_data_t *signature, crypto_req_handle_t req)
4369 {
4370 	int error = CRYPTO_FAILED;
4371 	dca_t *softc;
4372 	/* LINTED E_FUNC_SET_NOT_USED */
4373 	int instance;
4374 
4375 	if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4376 		return (CRYPTO_OPERATION_NOT_INITIALIZED);
4377 
4378 	/* extract softc and instance number from context */
4379 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4380 	DBG(softc, DENTRY, "dca_sign_recover: started\n");
4381 
4382 	/* check mechanism */
4383 	switch (DCA_MECH_FROM_CTX(ctx)) {
4384 	case RSA_PKCS_MECH_INFO_TYPE:
4385 	case RSA_X_509_MECH_INFO_TYPE:
4386 		error = dca_rsastart(ctx, data, signature, req, DCA_RSA_SIGNR);
4387 		break;
4388 	default:
4389 		cmn_err(CE_WARN, "dca_sign_recover: unexpected mech type "
4390 		    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4391 		error = CRYPTO_MECHANISM_INVALID;
4392 	}
4393 
4394 	DBG(softc, DENTRY, "dca_sign_recover: done, err = 0x%x", error);
4395 
4396 	return (error);
4397 }
4398 
4399 static int
dca_sign_recover_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * data,crypto_data_t * signature,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)4400 dca_sign_recover_atomic(crypto_provider_handle_t provider,
4401     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4402     crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
4403     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4404 {
4405 	int error = CRYPTO_FAILED;
4406 	dca_t *softc = (dca_t *)provider;
4407 	/* LINTED E_FUNC_SET_NOT_USED */
4408 	int instance;
4409 
4410 	instance = ddi_get_instance(softc->dca_dip);
4411 	DBG(softc, DENTRY, "dca_sign_recover_atomic: started\n");
4412 
4413 	if (ctx_template != NULL)
4414 		return (CRYPTO_ARGUMENTS_BAD);
4415 
4416 	/* check mechanism */
4417 	switch (mechanism->cm_type) {
4418 	case RSA_PKCS_MECH_INFO_TYPE:
4419 	case RSA_X_509_MECH_INFO_TYPE:
4420 		error = dca_rsaatomic(provider, session_id, mechanism, key,
4421 		    data, signature, KM_SLEEP, req, DCA_RSA_SIGNR);
4422 		break;
4423 	default:
4424 		cmn_err(CE_WARN, "dca_sign_recover_atomic: unexpected mech type"
4425 		    " 0x%llx\n", (unsigned long long)mechanism->cm_type);
4426 		error = CRYPTO_MECHANISM_INVALID;
4427 	}
4428 
4429 	DBG(softc, DENTRY, "dca_sign_recover_atomic: done, err = 0x%x", error);
4430 
4431 	return (error);
4432 }
4433 
4434 /*
4435  * Verify entry points.
4436  */
4437 
4438 /* ARGSUSED */
4439 static int
dca_verify_init(crypto_ctx_t * ctx,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)4440 dca_verify_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
4441     crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
4442     crypto_req_handle_t req)
4443 {
4444 	int error = CRYPTO_FAILED;
4445 	dca_t *softc;
4446 	/* LINTED E_FUNC_SET_NOT_USED */
4447 	int instance;
4448 
4449 	/* extract softc and instance number from context */
4450 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4451 	DBG(softc, DENTRY, "dca_verify_init: started\n");
4452 
4453 	if (ctx_template != NULL)
4454 		return (CRYPTO_ARGUMENTS_BAD);
4455 
4456 	/* check mechanism */
4457 	switch (mechanism->cm_type) {
4458 	case RSA_PKCS_MECH_INFO_TYPE:
4459 	case RSA_X_509_MECH_INFO_TYPE:
4460 		error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
4461 		break;
4462 	case DSA_MECH_INFO_TYPE:
4463 		error = dca_dsainit(ctx, mechanism, key, KM_SLEEP,
4464 		    DCA_DSA_VRFY);
4465 		break;
4466 	default:
4467 		cmn_err(CE_WARN, "dca_verify_init: unexpected mech type "
4468 		    "0x%llx\n", (unsigned long long)mechanism->cm_type);
4469 		error = CRYPTO_MECHANISM_INVALID;
4470 	}
4471 
4472 	DBG(softc, DENTRY, "dca_verify_init: done, err = 0x%x", error);
4473 
4474 	if (error == CRYPTO_SUCCESS)
4475 		dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
4476 		    &softc->dca_ctx_list_lock);
4477 
4478 	return (error);
4479 }
4480 
4481 static int
dca_verify(crypto_ctx_t * ctx,crypto_data_t * data,crypto_data_t * signature,crypto_req_handle_t req)4482 dca_verify(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
4483     crypto_req_handle_t req)
4484 {
4485 	int error = CRYPTO_FAILED;
4486 	dca_t *softc;
4487 	/* LINTED E_FUNC_SET_NOT_USED */
4488 	int instance;
4489 
4490 	if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4491 		return (CRYPTO_OPERATION_NOT_INITIALIZED);
4492 
4493 	/* extract softc and instance number from context */
4494 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4495 	DBG(softc, DENTRY, "dca_verify: started\n");
4496 
4497 	/* check mechanism */
4498 	switch (DCA_MECH_FROM_CTX(ctx)) {
4499 	case RSA_PKCS_MECH_INFO_TYPE:
4500 	case RSA_X_509_MECH_INFO_TYPE:
4501 		error = dca_rsastart(ctx, signature, data, req, DCA_RSA_VRFY);
4502 		break;
4503 	case DSA_MECH_INFO_TYPE:
4504 		error = dca_dsa_verify(ctx, data, signature, req);
4505 		break;
4506 	default:
4507 		cmn_err(CE_WARN, "dca_verify: unexpected mech type "
4508 		    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4509 		error = CRYPTO_MECHANISM_INVALID;
4510 	}
4511 
4512 	DBG(softc, DENTRY, "dca_verify: done, err = 0x%x", error);
4513 
4514 	return (error);
4515 }
4516 
4517 /* ARGSUSED */
4518 static int
dca_verify_update(crypto_ctx_t * ctx,crypto_data_t * data,crypto_req_handle_t req)4519 dca_verify_update(crypto_ctx_t *ctx, crypto_data_t *data,
4520     crypto_req_handle_t req)
4521 {
4522 	int error = CRYPTO_MECHANISM_INVALID;
4523 	dca_t *softc;
4524 	/* LINTED E_FUNC_SET_NOT_USED */
4525 	int instance;
4526 
4527 	if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4528 		return (CRYPTO_OPERATION_NOT_INITIALIZED);
4529 
4530 	/* extract softc and instance number from context */
4531 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4532 	DBG(softc, DENTRY, "dca_verify_update: started\n");
4533 
4534 	cmn_err(CE_WARN, "dca_verify_update: unexpected mech type "
4535 	    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4536 
4537 	DBG(softc, DENTRY, "dca_verify_update: done, err = 0x%x", error);
4538 
4539 	return (error);
4540 }
4541 
4542 /* ARGSUSED */
4543 static int
dca_verify_final(crypto_ctx_t * ctx,crypto_data_t * signature,crypto_req_handle_t req)4544 dca_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature,
4545     crypto_req_handle_t req)
4546 {
4547 	int error = CRYPTO_MECHANISM_INVALID;
4548 	dca_t *softc;
4549 	/* LINTED E_FUNC_SET_NOT_USED */
4550 	int instance;
4551 
4552 	if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4553 		return (CRYPTO_OPERATION_NOT_INITIALIZED);
4554 
4555 	/* extract softc and instance number from context */
4556 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4557 	DBG(softc, DENTRY, "dca_verify_final: started\n");
4558 
4559 	cmn_err(CE_WARN, "dca_verify_final: unexpected mech type "
4560 	    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4561 
4562 	DBG(softc, DENTRY, "dca_verify_final: done, err = 0x%x", error);
4563 
4564 	return (error);
4565 }
4566 
4567 static int
dca_verify_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * data,crypto_data_t * signature,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)4568 dca_verify_atomic(crypto_provider_handle_t provider,
4569     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4570     crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
4571     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4572 {
4573 	int error = CRYPTO_FAILED;
4574 	dca_t *softc = (dca_t *)provider;
4575 
4576 	DBG(softc, DENTRY, "dca_verify_atomic: started\n");
4577 
4578 	if (ctx_template != NULL)
4579 		return (CRYPTO_ARGUMENTS_BAD);
4580 
4581 	/* check mechanism */
4582 	switch (mechanism->cm_type) {
4583 	case RSA_PKCS_MECH_INFO_TYPE:
4584 	case RSA_X_509_MECH_INFO_TYPE:
4585 		error = dca_rsaatomic(provider, session_id, mechanism, key,
4586 		    signature, data, KM_SLEEP, req, DCA_RSA_VRFY);
4587 		break;
4588 	case DSA_MECH_INFO_TYPE:
4589 		error = dca_dsaatomic(provider, session_id, mechanism, key,
4590 		    data, signature, KM_SLEEP, req, DCA_DSA_VRFY);
4591 		break;
4592 	default:
4593 		cmn_err(CE_WARN, "dca_verify_atomic: unexpected mech type "
4594 		    "0x%llx\n", (unsigned long long)mechanism->cm_type);
4595 		error = CRYPTO_MECHANISM_INVALID;
4596 	}
4597 
4598 	DBG(softc, DENTRY, "dca_verify_atomic: done, err = 0x%x", error);
4599 
4600 	return (error);
4601 }
4602 
4603 /* ARGSUSED */
4604 static int
dca_verify_recover_init(crypto_ctx_t * ctx,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)4605 dca_verify_recover_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
4606     crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
4607     crypto_req_handle_t req)
4608 {
4609 	int error = CRYPTO_MECHANISM_INVALID;
4610 	dca_t *softc;
4611 	/* LINTED E_FUNC_SET_NOT_USED */
4612 	int instance;
4613 
4614 	/* extract softc and instance number from context */
4615 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4616 	DBG(softc, DENTRY, "dca_verify_recover_init: started\n");
4617 
4618 	if (ctx_template != NULL)
4619 		return (CRYPTO_ARGUMENTS_BAD);
4620 
4621 	/* check mechanism */
4622 	switch (mechanism->cm_type) {
4623 	case RSA_PKCS_MECH_INFO_TYPE:
4624 	case RSA_X_509_MECH_INFO_TYPE:
4625 		error = dca_rsainit(ctx, mechanism, key, KM_SLEEP);
4626 		break;
4627 	default:
4628 		cmn_err(CE_WARN, "dca_verify_recover_init: unexpected mech type"
4629 		    " 0x%llx\n", (unsigned long long)mechanism->cm_type);
4630 	}
4631 
4632 	DBG(softc, DENTRY, "dca_verify_recover_init: done, err = 0x%x", error);
4633 
4634 	if (error == CRYPTO_SUCCESS)
4635 		dca_enlist2(&softc->dca_ctx_list, ctx->cc_provider_private,
4636 		    &softc->dca_ctx_list_lock);
4637 
4638 	return (error);
4639 }
4640 
4641 static int
dca_verify_recover(crypto_ctx_t * ctx,crypto_data_t * signature,crypto_data_t * data,crypto_req_handle_t req)4642 dca_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature,
4643     crypto_data_t *data, crypto_req_handle_t req)
4644 {
4645 	int error = CRYPTO_MECHANISM_INVALID;
4646 	dca_t *softc;
4647 	/* LINTED E_FUNC_SET_NOT_USED */
4648 	int instance;
4649 
4650 	if (!ctx || !ctx->cc_provider || !ctx->cc_provider_private)
4651 		return (CRYPTO_OPERATION_NOT_INITIALIZED);
4652 
4653 	/* extract softc and instance number from context */
4654 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4655 	DBG(softc, DENTRY, "dca_verify_recover: started\n");
4656 
4657 	/* check mechanism */
4658 	switch (DCA_MECH_FROM_CTX(ctx)) {
4659 	case RSA_PKCS_MECH_INFO_TYPE:
4660 	case RSA_X_509_MECH_INFO_TYPE:
4661 		error = dca_rsastart(ctx, signature, data, req, DCA_RSA_VRFYR);
4662 		break;
4663 	default:
4664 		cmn_err(CE_WARN, "dca_verify_recover: unexpected mech type "
4665 		    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4666 	}
4667 
4668 	DBG(softc, DENTRY, "dca_verify_recover: done, err = 0x%x", error);
4669 
4670 	return (error);
4671 }
4672 
4673 static int
dca_verify_recover_atomic(crypto_provider_handle_t provider,crypto_session_id_t session_id,crypto_mechanism_t * mechanism,crypto_key_t * key,crypto_data_t * data,crypto_data_t * signature,crypto_spi_ctx_template_t ctx_template,crypto_req_handle_t req)4674 dca_verify_recover_atomic(crypto_provider_handle_t provider,
4675     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
4676     crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
4677     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
4678 {
4679 	int error = CRYPTO_MECHANISM_INVALID;
4680 	dca_t *softc = (dca_t *)provider;
4681 
4682 	DBG(softc, DENTRY, "dca_verify_recover_atomic: started\n");
4683 
4684 	if (ctx_template != NULL)
4685 		return (CRYPTO_ARGUMENTS_BAD);
4686 
4687 	/* check mechanism */
4688 	switch (mechanism->cm_type) {
4689 	case RSA_PKCS_MECH_INFO_TYPE:
4690 	case RSA_X_509_MECH_INFO_TYPE:
4691 		error = dca_rsaatomic(provider, session_id, mechanism, key,
4692 		    signature, data, KM_SLEEP, req, DCA_RSA_VRFYR);
4693 		break;
4694 	default:
4695 		cmn_err(CE_WARN, "dca_verify_recover_atomic: unexpected mech "
4696 		    "type 0x%llx\n", (unsigned long long)mechanism->cm_type);
4697 		error = CRYPTO_MECHANISM_INVALID;
4698 	}
4699 
4700 	DBG(softc, DENTRY,
4701 	    "dca_verify_recover_atomic: done, err = 0x%x", error);
4702 
4703 	return (error);
4704 }
4705 
4706 /*
4707  * Random number entry points.
4708  */
4709 
4710 /* ARGSUSED */
4711 static int
dca_generate_random(crypto_provider_handle_t provider,crypto_session_id_t session_id,uchar_t * buf,size_t len,crypto_req_handle_t req)4712 dca_generate_random(crypto_provider_handle_t provider,
4713     crypto_session_id_t session_id,
4714     uchar_t *buf, size_t len, crypto_req_handle_t req)
4715 {
4716 	int error = CRYPTO_FAILED;
4717 	dca_t *softc = (dca_t *)provider;
4718 	/* LINTED E_FUNC_SET_NOT_USED */
4719 	int instance;
4720 
4721 	instance = ddi_get_instance(softc->dca_dip);
4722 	DBG(softc, DENTRY, "dca_generate_random: started");
4723 
4724 	error = dca_rng(softc, buf, len, req);
4725 
4726 	DBG(softc, DENTRY, "dca_generate_random: done, err = 0x%x", error);
4727 
4728 	return (error);
4729 }
4730 
4731 /*
4732  * Context management entry points.
4733  */
4734 
4735 int
dca_free_context(crypto_ctx_t * ctx)4736 dca_free_context(crypto_ctx_t *ctx)
4737 {
4738 	int error = CRYPTO_SUCCESS;
4739 	dca_t *softc;
4740 	/* LINTED E_FUNC_SET_NOT_USED */
4741 	int instance;
4742 
4743 	/* extract softc and instance number from context */
4744 	DCA_SOFTC_FROM_CTX(ctx, softc, instance);
4745 	DBG(softc, DENTRY, "dca_free_context: entered");
4746 
4747 	if (ctx->cc_provider_private == NULL)
4748 		return (error);
4749 
4750 	dca_rmlist2(ctx->cc_provider_private, &softc->dca_ctx_list_lock);
4751 
4752 	error = dca_free_context_low(ctx);
4753 
4754 	DBG(softc, DENTRY, "dca_free_context: done, err = 0x%x", error);
4755 
4756 	return (error);
4757 }
4758 
4759 static int
dca_free_context_low(crypto_ctx_t * ctx)4760 dca_free_context_low(crypto_ctx_t *ctx)
4761 {
4762 	int error = CRYPTO_SUCCESS;
4763 
4764 	/* check mechanism */
4765 	switch (DCA_MECH_FROM_CTX(ctx)) {
4766 	case DES_CBC_MECH_INFO_TYPE:
4767 	case DES3_CBC_MECH_INFO_TYPE:
4768 		dca_3desctxfree(ctx);
4769 		break;
4770 	case RSA_PKCS_MECH_INFO_TYPE:
4771 	case RSA_X_509_MECH_INFO_TYPE:
4772 		dca_rsactxfree(ctx);
4773 		break;
4774 	case DSA_MECH_INFO_TYPE:
4775 		dca_dsactxfree(ctx);
4776 		break;
4777 	default:
4778 		/* Should never reach here */
4779 		cmn_err(CE_WARN, "dca_free_context_low: unexpected mech type "
4780 		    "0x%llx\n", (unsigned long long)DCA_MECH_FROM_CTX(ctx));
4781 		error = CRYPTO_MECHANISM_INVALID;
4782 	}
4783 
4784 	return (error);
4785 }
4786 
4787 
4788 /* Free any unfreed private context. It is called in detach. */
4789 static void
dca_free_context_list(dca_t * dca)4790 dca_free_context_list(dca_t *dca)
4791 {
4792 	dca_listnode_t	*node;
4793 	crypto_ctx_t	ctx;
4794 
4795 	(void) memset(&ctx, 0, sizeof (ctx));
4796 	ctx.cc_provider = dca;
4797 
4798 	while ((node = dca_delist2(&dca->dca_ctx_list,
4799 	    &dca->dca_ctx_list_lock)) != NULL) {
4800 		ctx.cc_provider_private = node;
4801 		(void) dca_free_context_low(&ctx);
4802 	}
4803 }
4804 
4805 static int
ext_info_sym(crypto_provider_handle_t prov,crypto_provider_ext_info_t * ext_info,crypto_req_handle_t cfreq)4806 ext_info_sym(crypto_provider_handle_t prov,
4807     crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq)
4808 {
4809 	return (ext_info_base(prov, ext_info, cfreq, IDENT_SYM));
4810 }
4811 
4812 static int
ext_info_asym(crypto_provider_handle_t prov,crypto_provider_ext_info_t * ext_info,crypto_req_handle_t cfreq)4813 ext_info_asym(crypto_provider_handle_t prov,
4814     crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq)
4815 {
4816 	int rv;
4817 
4818 	rv = ext_info_base(prov, ext_info, cfreq, IDENT_ASYM);
4819 	/* The asymmetric cipher slot supports random */
4820 	ext_info->ei_flags |= CRYPTO_EXTF_RNG;
4821 
4822 	return (rv);
4823 }
4824 
4825 /* ARGSUSED */
4826 static int
ext_info_base(crypto_provider_handle_t prov,crypto_provider_ext_info_t * ext_info,crypto_req_handle_t cfreq,char * id)4827 ext_info_base(crypto_provider_handle_t prov,
4828     crypto_provider_ext_info_t *ext_info, crypto_req_handle_t cfreq, char *id)
4829 {
4830 	dca_t   *dca = (dca_t *)prov;
4831 	int len;
4832 
4833 	/* Label */
4834 	(void) sprintf((char *)ext_info->ei_label, "%s/%d %s",
4835 	    ddi_driver_name(dca->dca_dip), ddi_get_instance(dca->dca_dip), id);
4836 	len = strlen((char *)ext_info->ei_label);
4837 	(void) memset(ext_info->ei_label + len, ' ',
4838 	    CRYPTO_EXT_SIZE_LABEL - len);
4839 
4840 	/* Manufacturer ID */
4841 	(void) sprintf((char *)ext_info->ei_manufacturerID, "%s",
4842 	    DCA_MANUFACTURER_ID);
4843 	len = strlen((char *)ext_info->ei_manufacturerID);
4844 	(void) memset(ext_info->ei_manufacturerID + len, ' ',
4845 	    CRYPTO_EXT_SIZE_MANUF - len);
4846 
4847 	/* Model */
4848 	(void) sprintf((char *)ext_info->ei_model, dca->dca_model);
4849 
4850 	DBG(dca, DWARN, "kCF MODEL: %s", (char *)ext_info->ei_model);
4851 
4852 	len = strlen((char *)ext_info->ei_model);
4853 	(void) memset(ext_info->ei_model + len, ' ',
4854 	    CRYPTO_EXT_SIZE_MODEL - len);
4855 
4856 	/* Serial Number. Blank for Deimos */
4857 	(void) memset(ext_info->ei_serial_number, ' ', CRYPTO_EXT_SIZE_SERIAL);
4858 
4859 	ext_info->ei_flags = CRYPTO_EXTF_WRITE_PROTECTED;
4860 
4861 	ext_info->ei_max_session_count = CRYPTO_UNAVAILABLE_INFO;
4862 	ext_info->ei_max_pin_len = CRYPTO_UNAVAILABLE_INFO;
4863 	ext_info->ei_min_pin_len = CRYPTO_UNAVAILABLE_INFO;
4864 	ext_info->ei_total_public_memory = CRYPTO_UNAVAILABLE_INFO;
4865 	ext_info->ei_free_public_memory = CRYPTO_UNAVAILABLE_INFO;
4866 	ext_info->ei_total_private_memory = CRYPTO_UNAVAILABLE_INFO;
4867 	ext_info->ei_free_private_memory = CRYPTO_UNAVAILABLE_INFO;
4868 	ext_info->ei_hardware_version.cv_major = 0;
4869 	ext_info->ei_hardware_version.cv_minor = 0;
4870 	ext_info->ei_firmware_version.cv_major = 0;
4871 	ext_info->ei_firmware_version.cv_minor = 0;
4872 
4873 	/* Time. No need to be supplied for token without a clock */
4874 	ext_info->ei_time[0] = '\000';
4875 
4876 	return (CRYPTO_SUCCESS);
4877 }
4878 
4879 static void
dca_fma_init(dca_t * dca)4880 dca_fma_init(dca_t *dca)
4881 {
4882 	ddi_iblock_cookie_t fm_ibc;
4883 	int fm_capabilities = DDI_FM_EREPORT_CAPABLE |
4884 	    DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE |
4885 	    DDI_FM_ERRCB_CAPABLE;
4886 
4887 	/* Read FMA capabilities from dca.conf file (if present) */
4888 	dca->fm_capabilities = ddi_getprop(DDI_DEV_T_ANY, dca->dca_dip,
4889 	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
4890 	    fm_capabilities);
4891 
4892 	DBG(dca, DWARN, "dca->fm_capabilities = 0x%x", dca->fm_capabilities);
4893 
4894 	/* Only register with IO Fault Services if we have some capability */
4895 	if (dca->fm_capabilities) {
4896 		dca_regsattr.devacc_attr_access = DDI_FLAGERR_ACC;
4897 		dca_dmaattr.dma_attr_flags = DDI_DMA_FLAGERR;
4898 
4899 		/* Register capabilities with IO Fault Services */
4900 		ddi_fm_init(dca->dca_dip, &dca->fm_capabilities, &fm_ibc);
4901 		DBG(dca, DWARN, "fm_capable() =  0x%x",
4902 		    ddi_fm_capable(dca->dca_dip));
4903 
4904 		/*
4905 		 * Initialize pci ereport capabilities if ereport capable
4906 		 */
4907 		if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) ||
4908 		    DDI_FM_ERRCB_CAP(dca->fm_capabilities))
4909 			pci_ereport_setup(dca->dca_dip);
4910 
4911 		/*
4912 		 * Initialize callback mutex and register error callback if
4913 		 * error callback capable.
4914 		 */
4915 		if (DDI_FM_ERRCB_CAP(dca->fm_capabilities)) {
4916 			ddi_fm_handler_register(dca->dca_dip, dca_fm_error_cb,
4917 			    (void *)dca);
4918 		}
4919 	} else {
4920 		/*
4921 		 * These fields have to be cleared of FMA if there are no
4922 		 * FMA capabilities at runtime.
4923 		 */
4924 		dca_regsattr.devacc_attr_access = DDI_DEFAULT_ACC;
4925 		dca_dmaattr.dma_attr_flags = 0;
4926 	}
4927 }
4928 
4929 
4930 static void
dca_fma_fini(dca_t * dca)4931 dca_fma_fini(dca_t *dca)
4932 {
4933 	/* Only unregister FMA capabilities if we registered some */
4934 	if (dca->fm_capabilities) {
4935 
4936 		/*
4937 		 * Release any resources allocated by pci_ereport_setup()
4938 		 */
4939 		if (DDI_FM_EREPORT_CAP(dca->fm_capabilities) ||
4940 		    DDI_FM_ERRCB_CAP(dca->fm_capabilities)) {
4941 			pci_ereport_teardown(dca->dca_dip);
4942 		}
4943 
4944 		/*
4945 		 * Free callback mutex and un-register error callback if
4946 		 * error callback capable.
4947 		 */
4948 		if (DDI_FM_ERRCB_CAP(dca->fm_capabilities)) {
4949 			ddi_fm_handler_unregister(dca->dca_dip);
4950 		}
4951 
4952 		/* Unregister from IO Fault Services */
4953 		ddi_fm_fini(dca->dca_dip);
4954 		DBG(dca, DWARN, "fm_capable() = 0x%x",
4955 		    ddi_fm_capable(dca->dca_dip));
4956 	}
4957 }
4958 
4959 
4960 /*
4961  * The IO fault service error handling callback function
4962  */
4963 /*ARGSUSED*/
4964 static int
dca_fm_error_cb(dev_info_t * dip,ddi_fm_error_t * err,const void * impl_data)4965 dca_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
4966 {
4967 	dca_t		*dca = (dca_t *)impl_data;
4968 
4969 	pci_ereport_post(dip, err, NULL);
4970 	if (err->fme_status == DDI_FM_FATAL) {
4971 		dca_failure(dca, DDI_DATAPATH_FAULT,
4972 		    DCA_FM_ECLASS_NONE, dca_ena(0), CRYPTO_DEVICE_ERROR,
4973 		    "fault PCI in FMA callback.");
4974 	}
4975 	return (err->fme_status);
4976 }
4977 
4978 
4979 static int
dca_check_acc_handle(dca_t * dca,ddi_acc_handle_t handle,dca_fma_eclass_t eclass_index)4980 dca_check_acc_handle(dca_t *dca, ddi_acc_handle_t handle,
4981     dca_fma_eclass_t eclass_index)
4982 {
4983 	ddi_fm_error_t	de;
4984 	int		version = 0;
4985 
4986 	ddi_fm_acc_err_get(handle, &de, version);
4987 	if (de.fme_status != DDI_FM_OK) {
4988 		dca_failure(dca, DDI_DATAPATH_FAULT,
4989 		    eclass_index, fm_ena_increment(de.fme_ena),
4990 		    CRYPTO_DEVICE_ERROR, "");
4991 		return (DDI_FAILURE);
4992 	}
4993 
4994 	return (DDI_SUCCESS);
4995 }
4996 
4997 int
dca_check_dma_handle(dca_t * dca,ddi_dma_handle_t handle,dca_fma_eclass_t eclass_index)4998 dca_check_dma_handle(dca_t *dca, ddi_dma_handle_t handle,
4999     dca_fma_eclass_t eclass_index)
5000 {
5001 	ddi_fm_error_t	de;
5002 	int		version = 0;
5003 
5004 	ddi_fm_dma_err_get(handle, &de, version);
5005 	if (de.fme_status != DDI_FM_OK) {
5006 		dca_failure(dca, DDI_DATAPATH_FAULT,
5007 		    eclass_index, fm_ena_increment(de.fme_ena),
5008 		    CRYPTO_DEVICE_ERROR, "");
5009 		return (DDI_FAILURE);
5010 	}
5011 	return (DDI_SUCCESS);
5012 }
5013 
5014 static uint64_t
dca_ena(uint64_t ena)5015 dca_ena(uint64_t ena)
5016 {
5017 	if (ena == 0)
5018 		ena = fm_ena_generate(0, FM_ENA_FMT1);
5019 	else
5020 		ena = fm_ena_increment(ena);
5021 	return (ena);
5022 }
5023 
5024 static char *
dca_fma_eclass_string(char * model,dca_fma_eclass_t index)5025 dca_fma_eclass_string(char *model, dca_fma_eclass_t index)
5026 {
5027 	if (strstr(model, "500"))
5028 		return (dca_fma_eclass_sca500[index]);
5029 	else
5030 		return (dca_fma_eclass_sca1000[index]);
5031 }
5032