1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2025 Intel Corporation */
3 /* System headers */
4 #include <sys/param.h>
5 #include <sys/systm.h>
6 #include <sys/bus.h>
7 #include <sys/cpu.h>
8 #include <sys/kernel.h>
9 #include <sys/mbuf.h>
10 #include <sys/module.h>
11 #include <sys/mutex.h>
12 #include <sys/sysctl.h>
13
14 /* Cryptodev headers */
15 #include <opencrypto/cryptodev.h>
16 #include "cryptodev_if.h"
17
18 /* QAT specific headers */
19 #include "cpa.h"
20 #include "cpa_cy_im.h"
21 #include "cpa_cy_sym_dp.h"
22 #include "adf_accel_devices.h"
23 #include "adf_common_drv.h"
24 #include "lac_sym_hash_defs.h"
25 #include "lac_sym_qat_hash_defs_lookup.h"
26
27 /* To get only IRQ instances */
28 #include "icp_accel_devices.h"
29 #include "icp_adf_accel_mgr.h"
30 #include "lac_sal_types.h"
31
32 /* To disable AEAD HW MAC verification */
33 #include "icp_sal_user.h"
34
35 /* QAT OCF specific headers */
36 #include "qat_ocf_mem_pool.h"
37 #include "qat_ocf_utils.h"
38
39 #define QAT_OCF_MAX_INSTANCES (256)
40 #define QAT_OCF_SESSION_WAIT_TIMEOUT_MS (1000)
41
42 MALLOC_DEFINE(M_QAT_OCF, "qat_ocf", "qat_ocf(4) memory allocations");
43
44 /* QAT OCF internal structures */
45 struct qat_ocf_softc {
46 device_t sc_dev;
47 struct sysctl_oid *rc;
48 uint32_t enabled;
49 int32_t cryptodev_id;
50 struct qat_ocf_instance cyInstHandles[QAT_OCF_MAX_INSTANCES];
51 int32_t numCyInstances;
52 };
53
54 /* Function definitions */
55 static void qat_ocf_freesession(device_t dev, crypto_session_t cses);
56 static int qat_ocf_probesession(device_t dev,
57 const struct crypto_session_params *csp);
58 static int qat_ocf_newsession(device_t dev,
59 crypto_session_t cses,
60 const struct crypto_session_params *csp);
61 static int qat_ocf_attach(device_t dev);
62 static int qat_ocf_detach(device_t dev);
63
64 static void
symDpCallback(CpaCySymDpOpData * pOpData,CpaStatus result,CpaBoolean verifyResult)65 symDpCallback(CpaCySymDpOpData *pOpData,
66 CpaStatus result,
67 CpaBoolean verifyResult)
68 {
69 struct qat_ocf_cookie *qat_cookie;
70 struct cryptop *crp;
71 struct qat_ocf_dsession *qat_dsession = NULL;
72 struct qat_ocf_session *qat_session = NULL;
73 struct qat_ocf_instance *qat_instance = NULL;
74 CpaStatus status;
75 int rc = 0;
76
77 qat_cookie = (struct qat_ocf_cookie *)pOpData->pCallbackTag;
78 if (!qat_cookie)
79 return;
80
81 crp = qat_cookie->crp_op;
82
83 qat_dsession = crypto_get_driver_session(crp->crp_session);
84 qat_instance = qat_dsession->qatInstance;
85
86 status = qat_ocf_cookie_dma_post_sync(crp, pOpData);
87 if (CPA_STATUS_SUCCESS != status) {
88 rc = EIO;
89 goto exit;
90 }
91
92 status = qat_ocf_cookie_dma_unload(crp, pOpData);
93 if (CPA_STATUS_SUCCESS != status) {
94 rc = EIO;
95 goto exit;
96 }
97
98 /* Verify result */
99 if (CPA_STATUS_SUCCESS != result) {
100 rc = EBADMSG;
101 goto exit;
102 }
103
104 /* Verify digest by FW (GCM and CCM only) */
105 if (CPA_TRUE != verifyResult) {
106 rc = EBADMSG;
107 goto exit;
108 }
109
110 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
111 qat_session = &qat_dsession->encSession;
112 else
113 qat_session = &qat_dsession->decSession;
114
115 /* Copy back digest result if it's stored in separated buffer */
116 if (pOpData->digestResult && qat_session->authLen > 0) {
117 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) {
118 char icv[QAT_OCF_MAX_DIGEST] = { 0 };
119 crypto_copydata(crp,
120 crp->crp_digest_start,
121 qat_session->authLen,
122 icv);
123 if (timingsafe_bcmp(icv,
124 qat_cookie->qat_ocf_digest,
125 qat_session->authLen) != 0) {
126 rc = EBADMSG;
127 goto exit;
128 }
129 } else {
130 crypto_copyback(crp,
131 crp->crp_digest_start,
132 qat_session->authLen,
133 qat_cookie->qat_ocf_digest);
134 }
135 }
136
137 exit:
138 qat_ocf_cookie_free(qat_instance, qat_cookie);
139 crp->crp_etype = rc;
140 crypto_done(crp);
141
142 return;
143 }
144
145 static inline CpaPhysicalAddr
qatVirtToPhys(void * virtAddr)146 qatVirtToPhys(void *virtAddr)
147 {
148 return (CpaPhysicalAddr)vtophys(virtAddr);
149 }
150
151 static int
qat_ocf_probesession(device_t dev,const struct crypto_session_params * csp)152 qat_ocf_probesession(device_t dev, const struct crypto_session_params *csp)
153 {
154 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) !=
155 0) {
156 return EINVAL;
157 }
158
159 switch (csp->csp_mode) {
160 case CSP_MODE_CIPHER:
161 switch (csp->csp_cipher_alg) {
162 case CRYPTO_AES_CBC:
163 case CRYPTO_AES_ICM:
164 if (csp->csp_ivlen != AES_BLOCK_LEN)
165 return EINVAL;
166 break;
167 case CRYPTO_AES_XTS:
168 if (csp->csp_ivlen != AES_XTS_IV_LEN)
169 return EINVAL;
170 break;
171 default:
172 return EINVAL;
173 }
174 break;
175 case CSP_MODE_DIGEST:
176 switch (csp->csp_auth_alg) {
177 case CRYPTO_SHA1:
178 case CRYPTO_SHA1_HMAC:
179 case CRYPTO_SHA2_256:
180 case CRYPTO_SHA2_256_HMAC:
181 case CRYPTO_SHA2_384:
182 case CRYPTO_SHA2_384_HMAC:
183 case CRYPTO_SHA2_512:
184 case CRYPTO_SHA2_512_HMAC:
185 break;
186 case CRYPTO_AES_NIST_GMAC:
187 if (csp->csp_ivlen != AES_GCM_IV_LEN)
188 return EINVAL;
189 break;
190 default:
191 return EINVAL;
192 }
193 break;
194 case CSP_MODE_AEAD:
195 switch (csp->csp_cipher_alg) {
196 case CRYPTO_AES_NIST_GCM_16:
197 if (csp->csp_ivlen != AES_GCM_IV_LEN)
198 return EINVAL;
199 break;
200 default:
201 return EINVAL;
202 }
203 break;
204 case CSP_MODE_ETA:
205 switch (csp->csp_auth_alg) {
206 case CRYPTO_SHA1_HMAC:
207 case CRYPTO_SHA2_256_HMAC:
208 case CRYPTO_SHA2_384_HMAC:
209 case CRYPTO_SHA2_512_HMAC:
210 switch (csp->csp_cipher_alg) {
211 case CRYPTO_AES_CBC:
212 case CRYPTO_AES_ICM:
213 if (csp->csp_ivlen != AES_BLOCK_LEN)
214 return EINVAL;
215 break;
216 case CRYPTO_AES_XTS:
217 if (csp->csp_ivlen != AES_XTS_IV_LEN)
218 return EINVAL;
219 break;
220 default:
221 return EINVAL;
222 }
223 break;
224 default:
225 return EINVAL;
226 }
227 break;
228 default:
229 return EINVAL;
230 }
231
232 return CRYPTODEV_PROBE_HARDWARE;
233 }
234
235 static CpaStatus
qat_ocf_session_init(device_t dev,struct cryptop * crp,struct qat_ocf_instance * qat_instance,struct qat_ocf_session * qat_ssession)236 qat_ocf_session_init(device_t dev,
237 struct cryptop *crp,
238 struct qat_ocf_instance *qat_instance,
239 struct qat_ocf_session *qat_ssession)
240 {
241 CpaStatus status = CPA_STATUS_SUCCESS;
242 /* Crytpodev structures */
243 crypto_session_t cses;
244 const struct crypto_session_params *csp;
245 /* DP API Session configuration */
246 CpaCySymSessionSetupData sessionSetupData = { 0 };
247 CpaCySymSessionCtx sessionCtx = NULL;
248 Cpa32U sessionCtxSize = 0;
249
250 cses = crp->crp_session;
251 if (NULL == cses) {
252 device_printf(dev, "no crypto session in cryptodev request\n");
253 return CPA_STATUS_FAIL;
254 }
255
256 csp = crypto_get_params(cses);
257 if (NULL == csp) {
258 device_printf(dev, "no session in cryptodev session\n");
259 return CPA_STATUS_FAIL;
260 }
261
262 /* Common fields */
263 sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH;
264 /* Cipher key */
265 if (crp->crp_cipher_key)
266 sessionSetupData.cipherSetupData.pCipherKey =
267 crp->crp_cipher_key;
268 else
269 sessionSetupData.cipherSetupData.pCipherKey =
270 csp->csp_cipher_key;
271 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
272 csp->csp_cipher_klen;
273
274 /* Auth key */
275 if (crp->crp_auth_key)
276 sessionSetupData.hashSetupData.authModeSetupData.authKey =
277 crp->crp_auth_key;
278 else
279 sessionSetupData.hashSetupData.authModeSetupData.authKey =
280 csp->csp_auth_key;
281 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
282 csp->csp_auth_klen;
283
284 qat_ssession->aadLen = crp->crp_aad_length;
285 if (CPA_TRUE == is_sep_aad_supported(csp))
286 sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes =
287 crp->crp_aad_length;
288 else
289 sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes =
290 0;
291
292 /* Just setup algorithm - regardless of mode */
293 if (csp->csp_cipher_alg) {
294 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER;
295
296 switch (csp->csp_cipher_alg) {
297 case CRYPTO_AES_CBC:
298 sessionSetupData.cipherSetupData.cipherAlgorithm =
299 CPA_CY_SYM_CIPHER_AES_CBC;
300 break;
301 case CRYPTO_AES_ICM:
302 sessionSetupData.cipherSetupData.cipherAlgorithm =
303 CPA_CY_SYM_CIPHER_AES_CTR;
304 break;
305 case CRYPTO_AES_XTS:
306 sessionSetupData.cipherSetupData.cipherAlgorithm =
307 CPA_CY_SYM_CIPHER_AES_XTS;
308 break;
309 case CRYPTO_AES_NIST_GCM_16:
310 sessionSetupData.cipherSetupData.cipherAlgorithm =
311 CPA_CY_SYM_CIPHER_AES_GCM;
312 sessionSetupData.hashSetupData.hashAlgorithm =
313 CPA_CY_SYM_HASH_AES_GCM;
314 sessionSetupData.hashSetupData.hashMode =
315 CPA_CY_SYM_HASH_MODE_AUTH;
316 break;
317 default:
318 device_printf(dev,
319 "cipher_alg: %d not supported\n",
320 csp->csp_cipher_alg);
321 status = CPA_STATUS_UNSUPPORTED;
322 goto fail;
323 }
324 }
325
326 if (csp->csp_auth_alg) {
327 switch (csp->csp_auth_alg) {
328 case CRYPTO_SHA1_HMAC:
329 sessionSetupData.hashSetupData.hashAlgorithm =
330 CPA_CY_SYM_HASH_SHA1;
331 sessionSetupData.hashSetupData.hashMode =
332 CPA_CY_SYM_HASH_MODE_AUTH;
333 break;
334 case CRYPTO_SHA1:
335 sessionSetupData.hashSetupData.hashAlgorithm =
336 CPA_CY_SYM_HASH_SHA1;
337 sessionSetupData.hashSetupData.hashMode =
338 CPA_CY_SYM_HASH_MODE_PLAIN;
339 break;
340
341 case CRYPTO_SHA2_256_HMAC:
342 sessionSetupData.hashSetupData.hashAlgorithm =
343 CPA_CY_SYM_HASH_SHA256;
344 sessionSetupData.hashSetupData.hashMode =
345 CPA_CY_SYM_HASH_MODE_AUTH;
346 break;
347 case CRYPTO_SHA2_256:
348 sessionSetupData.hashSetupData.hashAlgorithm =
349 CPA_CY_SYM_HASH_SHA256;
350 sessionSetupData.hashSetupData.hashMode =
351 CPA_CY_SYM_HASH_MODE_PLAIN;
352 break;
353
354 case CRYPTO_SHA2_224_HMAC:
355 sessionSetupData.hashSetupData.hashAlgorithm =
356 CPA_CY_SYM_HASH_SHA224;
357 sessionSetupData.hashSetupData.hashMode =
358 CPA_CY_SYM_HASH_MODE_AUTH;
359 break;
360 case CRYPTO_SHA2_224:
361 sessionSetupData.hashSetupData.hashAlgorithm =
362 CPA_CY_SYM_HASH_SHA224;
363 sessionSetupData.hashSetupData.hashMode =
364 CPA_CY_SYM_HASH_MODE_PLAIN;
365 break;
366
367 case CRYPTO_SHA2_384_HMAC:
368 sessionSetupData.hashSetupData.hashAlgorithm =
369 CPA_CY_SYM_HASH_SHA384;
370 sessionSetupData.hashSetupData.hashMode =
371 CPA_CY_SYM_HASH_MODE_AUTH;
372 break;
373 case CRYPTO_SHA2_384:
374 sessionSetupData.hashSetupData.hashAlgorithm =
375 CPA_CY_SYM_HASH_SHA384;
376 sessionSetupData.hashSetupData.hashMode =
377 CPA_CY_SYM_HASH_MODE_PLAIN;
378 break;
379
380 case CRYPTO_SHA2_512_HMAC:
381 sessionSetupData.hashSetupData.hashAlgorithm =
382 CPA_CY_SYM_HASH_SHA512;
383 sessionSetupData.hashSetupData.hashMode =
384 CPA_CY_SYM_HASH_MODE_AUTH;
385 break;
386 case CRYPTO_SHA2_512:
387 sessionSetupData.hashSetupData.hashAlgorithm =
388 CPA_CY_SYM_HASH_SHA512;
389 sessionSetupData.hashSetupData.hashMode =
390 CPA_CY_SYM_HASH_MODE_PLAIN;
391 break;
392 case CRYPTO_AES_NIST_GMAC:
393 sessionSetupData.hashSetupData.hashAlgorithm =
394 CPA_CY_SYM_HASH_AES_GMAC;
395 break;
396 default:
397 status = CPA_STATUS_UNSUPPORTED;
398 goto fail;
399 }
400 } /* csp->csp_auth_alg */
401
402 /* Setting digest-length if no cipher-only mode is set */
403 if (csp->csp_mode != CSP_MODE_CIPHER) {
404 lac_sym_qat_hash_defs_t *pHashDefsInfo = NULL;
405 if (csp->csp_auth_mlen) {
406 sessionSetupData.hashSetupData.digestResultLenInBytes =
407 csp->csp_auth_mlen;
408 qat_ssession->authLen = csp->csp_auth_mlen;
409 } else {
410 LacSymQat_HashDefsLookupGet(
411 qat_instance->cyInstHandle,
412 sessionSetupData.hashSetupData.hashAlgorithm,
413 &pHashDefsInfo);
414 if (NULL == pHashDefsInfo) {
415 device_printf(
416 dev,
417 "unable to find corresponding hash data\n");
418 status = CPA_STATUS_UNSUPPORTED;
419 goto fail;
420 }
421 sessionSetupData.hashSetupData.digestResultLenInBytes =
422 pHashDefsInfo->algInfo->digestLength;
423 qat_ssession->authLen =
424 pHashDefsInfo->algInfo->digestLength;
425 }
426 sessionSetupData.verifyDigest = CPA_FALSE;
427 }
428
429 switch (csp->csp_mode) {
430 case CSP_MODE_AEAD:
431 case CSP_MODE_ETA:
432 sessionSetupData.symOperation =
433 CPA_CY_SYM_OP_ALGORITHM_CHAINING;
434 /* Place the digest result in a buffer unrelated to srcBuffer */
435 sessionSetupData.digestIsAppended = CPA_FALSE;
436 /* Due to FW limitation to verify only appended MACs */
437 sessionSetupData.verifyDigest = CPA_FALSE;
438 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
439 sessionSetupData.cipherSetupData.cipherDirection =
440 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
441 sessionSetupData.algChainOrder =
442 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
443 } else {
444 sessionSetupData.cipherSetupData.cipherDirection =
445 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
446 sessionSetupData.algChainOrder =
447 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
448 }
449 break;
450 case CSP_MODE_CIPHER:
451 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
452 sessionSetupData.cipherSetupData.cipherDirection =
453 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
454 } else {
455 sessionSetupData.cipherSetupData.cipherDirection =
456 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
457 }
458 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER;
459 break;
460 case CSP_MODE_DIGEST:
461 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH;
462 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) {
463 sessionSetupData.symOperation =
464 CPA_CY_SYM_OP_ALGORITHM_CHAINING;
465 /* GMAC is always encrypt */
466 sessionSetupData.cipherSetupData.cipherDirection =
467 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
468 sessionSetupData.algChainOrder =
469 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
470 sessionSetupData.cipherSetupData.cipherAlgorithm =
471 CPA_CY_SYM_CIPHER_AES_GCM;
472 sessionSetupData.hashSetupData.hashAlgorithm =
473 CPA_CY_SYM_HASH_AES_GMAC;
474 sessionSetupData.hashSetupData.hashMode =
475 CPA_CY_SYM_HASH_MODE_AUTH;
476 /* Same key for cipher and auth */
477 sessionSetupData.cipherSetupData.pCipherKey =
478 csp->csp_auth_key;
479 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
480 csp->csp_auth_klen;
481 /* Generated GMAC stored in separated buffer */
482 sessionSetupData.digestIsAppended = CPA_FALSE;
483 /* Digest verification not allowed in GMAC case */
484 sessionSetupData.verifyDigest = CPA_FALSE;
485 /* No AAD allowed */
486 sessionSetupData.hashSetupData.authModeSetupData
487 .aadLenInBytes = 0;
488 } else {
489 sessionSetupData.cipherSetupData.cipherDirection =
490 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
491 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH;
492 sessionSetupData.digestIsAppended = CPA_FALSE;
493 }
494 break;
495 default:
496 device_printf(dev,
497 "%s: unhandled crypto algorithm %d, %d\n",
498 __func__,
499 csp->csp_cipher_alg,
500 csp->csp_auth_alg);
501 status = CPA_STATUS_FAIL;
502 goto fail;
503 }
504
505 /* Extracting session size */
506 status = cpaCySymSessionCtxGetSize(qat_instance->cyInstHandle,
507 &sessionSetupData,
508 &sessionCtxSize);
509 if (CPA_STATUS_SUCCESS != status) {
510 device_printf(dev, "unable to get session size\n");
511 goto fail;
512 }
513
514 /* Allocating contiguous memory for session */
515 sessionCtx = contigmalloc(sessionCtxSize,
516 M_QAT_OCF,
517 M_NOWAIT,
518 0,
519 ~1UL,
520 1 << (ilog2(sessionCtxSize - 1) + 1),
521 0);
522 if (NULL == sessionCtx) {
523 device_printf(dev, "unable to allocate memory for session\n");
524 status = CPA_STATUS_RESOURCE;
525 goto fail;
526 }
527
528 status = cpaCySymDpInitSession(qat_instance->cyInstHandle,
529 &sessionSetupData,
530 sessionCtx);
531 if (CPA_STATUS_SUCCESS != status) {
532 device_printf(dev, "session initialization failed\n");
533 goto fail;
534 }
535
536 /* NOTE: lets keep double session (both directions) approach to overcome
537 * lack of direction update in FBSD QAT.
538 */
539 qat_ssession->sessionCtx = sessionCtx;
540 qat_ssession->sessionCtxSize = sessionCtxSize;
541
542 return CPA_STATUS_SUCCESS;
543
544 fail:
545 /* Release resources if any */
546 if (sessionCtx)
547 free(sessionCtx, M_QAT_OCF);
548
549 return status;
550 }
551
552 static int
qat_ocf_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)553 qat_ocf_newsession(device_t dev,
554 crypto_session_t cses,
555 const struct crypto_session_params *csp)
556 {
557 /* Cryptodev QAT structures */
558 struct qat_ocf_softc *qat_softc;
559 struct qat_ocf_dsession *qat_dsession;
560 struct qat_ocf_instance *qat_instance;
561 u_int cpu_id = PCPU_GET(cpuid);
562
563 /* Create cryptodev session */
564 qat_softc = device_get_softc(dev);
565 if (qat_softc->numCyInstances > 0) {
566 qat_instance =
567 &qat_softc
568 ->cyInstHandles[cpu_id % qat_softc->numCyInstances];
569 qat_dsession = crypto_get_driver_session(cses);
570 if (NULL == qat_dsession) {
571 device_printf(dev, "Unable to create new session\n");
572 return (EINVAL);
573 }
574
575 /* Add only instance at this point remaining operations moved to
576 * lazy session init */
577 qat_dsession->qatInstance = qat_instance;
578 } else {
579 return ENXIO;
580 }
581
582 return 0;
583 }
584
585 static CpaStatus
qat_ocf_remove_session(device_t dev,CpaInstanceHandle cyInstHandle,struct qat_ocf_session * qat_session)586 qat_ocf_remove_session(device_t dev,
587 CpaInstanceHandle cyInstHandle,
588 struct qat_ocf_session *qat_session)
589 {
590 CpaStatus status = CPA_STATUS_SUCCESS;
591
592 if (NULL == qat_session->sessionCtx)
593 return CPA_STATUS_SUCCESS;
594
595 /* User callback is executed right before decrementing pending
596 * callback atomic counter. To avoid removing session rejection
597 * we have to wait a very short while for counter update
598 * after call back execution. */
599 status = qat_ocf_wait_for_session(qat_session->sessionCtx,
600 QAT_OCF_SESSION_WAIT_TIMEOUT_MS);
601 if (CPA_STATUS_SUCCESS != status) {
602 device_printf(dev, "waiting for session un-busy failed\n");
603 return CPA_STATUS_FAIL;
604 }
605
606 status = cpaCySymDpRemoveSession(cyInstHandle, qat_session->sessionCtx);
607 if (CPA_STATUS_SUCCESS != status) {
608 device_printf(dev, "error while removing session\n");
609 return CPA_STATUS_FAIL;
610 }
611
612 explicit_bzero(qat_session->sessionCtx, qat_session->sessionCtxSize);
613 free(qat_session->sessionCtx, M_QAT_OCF);
614 qat_session->sessionCtx = NULL;
615 qat_session->sessionCtxSize = 0;
616
617 return CPA_STATUS_SUCCESS;
618 }
619
620 static void
qat_ocf_freesession(device_t dev,crypto_session_t cses)621 qat_ocf_freesession(device_t dev, crypto_session_t cses)
622 {
623 CpaStatus status = CPA_STATUS_SUCCESS;
624 struct qat_ocf_dsession *qat_dsession = NULL;
625 struct qat_ocf_instance *qat_instance = NULL;
626
627 qat_dsession = crypto_get_driver_session(cses);
628 qat_instance = qat_dsession->qatInstance;
629 mtx_lock(&qat_instance->cyInstMtx);
630 status = qat_ocf_remove_session(dev,
631 qat_dsession->qatInstance->cyInstHandle,
632 &qat_dsession->encSession);
633 if (CPA_STATUS_SUCCESS != status)
634 device_printf(dev, "unable to remove encrypt session\n");
635 status = qat_ocf_remove_session(dev,
636 qat_dsession->qatInstance->cyInstHandle,
637 &qat_dsession->decSession);
638 if (CPA_STATUS_SUCCESS != status)
639 device_printf(dev, "unable to remove decrypt session\n");
640 mtx_unlock(&qat_instance->cyInstMtx);
641 }
642
643 /* QAT GCM/CCM FW API are only algorithms which support separated AAD. */
644 static CpaStatus
qat_ocf_load_aad_gcm(struct cryptop * crp,struct qat_ocf_cookie * qat_cookie)645 qat_ocf_load_aad_gcm(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie)
646 {
647 CpaCySymDpOpData *pOpData;
648
649 pOpData = &qat_cookie->pOpdata;
650
651 if (NULL != crp->crp_aad)
652 memcpy(qat_cookie->qat_ocf_gcm_aad,
653 crp->crp_aad,
654 crp->crp_aad_length);
655 else
656 crypto_copydata(crp,
657 crp->crp_aad_start,
658 crp->crp_aad_length,
659 qat_cookie->qat_ocf_gcm_aad);
660
661 pOpData->pAdditionalAuthData = qat_cookie->qat_ocf_gcm_aad;
662 pOpData->additionalAuthData = qat_cookie->qat_ocf_gcm_aad_paddr;
663
664 return CPA_STATUS_SUCCESS;
665 }
666
667 static CpaStatus
qat_ocf_load_aad(struct cryptop * crp,struct qat_ocf_cookie * qat_cookie)668 qat_ocf_load_aad(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie)
669 {
670 CpaStatus status = CPA_STATUS_SUCCESS;
671 const struct crypto_session_params *csp;
672 CpaCySymDpOpData *pOpData;
673 struct qat_ocf_load_cb_arg args;
674
675 pOpData = &qat_cookie->pOpdata;
676 pOpData->pAdditionalAuthData = NULL;
677 pOpData->additionalAuthData = 0UL;
678
679 if (crp->crp_aad_length == 0)
680 return CPA_STATUS_SUCCESS;
681
682 if (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX)
683 return CPA_STATUS_FAIL;
684
685 csp = crypto_get_params(crp->crp_session);
686
687 /* Handle GCM/CCM case */
688 if (CPA_TRUE == is_sep_aad_supported(csp))
689 return qat_ocf_load_aad_gcm(crp, qat_cookie);
690
691 if (NULL == crp->crp_aad) {
692 /* AAD already embedded in source buffer */
693 pOpData->messageLenToCipherInBytes = crp->crp_payload_length;
694 pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start;
695
696 pOpData->messageLenToHashInBytes =
697 crp->crp_aad_length + crp->crp_payload_length;
698 pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start;
699
700 return CPA_STATUS_SUCCESS;
701 }
702
703 /* Separated AAD not supported by QAT - lets place the content
704 * of ADD buffer at the very beginning of source SGL */
705 args.crp_op = crp;
706 args.qat_cookie = qat_cookie;
707 args.pOpData = pOpData;
708 args.error = 0;
709 status = bus_dmamap_load(qat_cookie->gcm_aad_dma_mem.dma_tag,
710 qat_cookie->gcm_aad_dma_mem.dma_map,
711 crp->crp_aad,
712 crp->crp_aad_length,
713 qat_ocf_crypto_load_aadbuf_cb,
714 &args,
715 BUS_DMA_NOWAIT);
716 qat_cookie->is_sep_aad_used = CPA_TRUE;
717
718 /* Right after this step we have AAD placed in the first flat buffer
719 * in source SGL */
720 pOpData->messageLenToCipherInBytes = crp->crp_payload_length;
721 pOpData->cryptoStartSrcOffsetInBytes =
722 crp->crp_aad_length + crp->crp_aad_start + crp->crp_payload_start;
723
724 pOpData->messageLenToHashInBytes =
725 crp->crp_aad_length + crp->crp_payload_length;
726 pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start;
727
728 return status;
729 }
730
731 static CpaStatus
qat_ocf_load(struct cryptop * crp,struct qat_ocf_cookie * qat_cookie)732 qat_ocf_load(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie)
733 {
734 CpaStatus status = CPA_STATUS_SUCCESS;
735 CpaCySymDpOpData *pOpData;
736 struct qat_ocf_load_cb_arg args;
737 /* cryptodev internals */
738 const struct crypto_session_params *csp;
739
740 pOpData = &qat_cookie->pOpdata;
741
742 csp = crypto_get_params(crp->crp_session);
743
744 /* Load IV buffer if present */
745 if (csp->csp_ivlen > 0) {
746 memset(qat_cookie->qat_ocf_iv_buf,
747 0,
748 sizeof(qat_cookie->qat_ocf_iv_buf));
749 crypto_read_iv(crp, qat_cookie->qat_ocf_iv_buf);
750 pOpData->iv = qat_cookie->qat_ocf_iv_buf_paddr;
751 pOpData->pIv = qat_cookie->qat_ocf_iv_buf;
752 pOpData->ivLenInBytes = csp->csp_ivlen;
753 }
754
755 /* GCM/CCM - load AAD to separated buffer
756 * AES+SHA - load AAD to first flat in SGL */
757 status = qat_ocf_load_aad(crp, qat_cookie);
758 if (CPA_STATUS_SUCCESS != status)
759 goto fail;
760
761 /* Load source buffer */
762 args.crp_op = crp;
763 args.qat_cookie = qat_cookie;
764 args.pOpData = pOpData;
765 args.error = 0;
766 status = bus_dmamap_load_crp_buffer(qat_cookie->src_dma_mem.dma_tag,
767 qat_cookie->src_dma_mem.dma_map,
768 &crp->crp_buf,
769 qat_ocf_crypto_load_buf_cb,
770 &args,
771 BUS_DMA_NOWAIT);
772 if (CPA_STATUS_SUCCESS != status)
773 goto fail;
774 pOpData->srcBuffer = qat_cookie->src_buffer_list_paddr;
775 pOpData->srcBufferLen = CPA_DP_BUFLIST;
776
777 /* Load destination buffer */
778 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
779 status =
780 bus_dmamap_load_crp_buffer(qat_cookie->dst_dma_mem.dma_tag,
781 qat_cookie->dst_dma_mem.dma_map,
782 &crp->crp_obuf,
783 qat_ocf_crypto_load_obuf_cb,
784 &args,
785 BUS_DMA_NOWAIT);
786 if (CPA_STATUS_SUCCESS != status)
787 goto fail;
788 pOpData->dstBuffer = qat_cookie->dst_buffer_list_paddr;
789 pOpData->dstBufferLen = CPA_DP_BUFLIST;
790 } else {
791 pOpData->dstBuffer = pOpData->srcBuffer;
792 pOpData->dstBufferLen = pOpData->srcBufferLen;
793 }
794
795 if (CPA_TRUE == is_use_sep_digest(csp))
796 pOpData->digestResult = qat_cookie->qat_ocf_digest_paddr;
797 else
798 pOpData->digestResult = 0UL;
799
800 /* GMAC - aka zero length buffer */
801 if (CPA_TRUE == is_gmac_exception(csp))
802 pOpData->messageLenToCipherInBytes = 0;
803
804 fail:
805 return status;
806 }
807
808 static int
qat_ocf_check_input(device_t dev,struct cryptop * crp)809 qat_ocf_check_input(device_t dev, struct cryptop *crp)
810 {
811 const struct crypto_session_params *csp;
812 csp = crypto_get_params(crp->crp_session);
813
814 if (crypto_buffer_len(&crp->crp_buf) > QAT_OCF_MAX_LEN)
815 return E2BIG;
816
817 if (CPA_TRUE == is_sep_aad_supported(csp) &&
818 (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX))
819 return EBADMSG;
820
821 return 0;
822 }
823
824 static int
qat_ocf_process(device_t dev,struct cryptop * crp,int hint)825 qat_ocf_process(device_t dev, struct cryptop *crp, int hint)
826 {
827 CpaStatus status = CPA_STATUS_SUCCESS;
828 int rc = 0;
829 struct qat_ocf_dsession *qat_dsession = NULL;
830 struct qat_ocf_session *qat_session = NULL;
831 struct qat_ocf_instance *qat_instance = NULL;
832 CpaCySymDpOpData *pOpData = NULL;
833 struct qat_ocf_cookie *qat_cookie = NULL;
834 CpaBoolean memLoaded = CPA_FALSE;
835
836 rc = qat_ocf_check_input(dev, crp);
837 if (rc)
838 goto fail;
839
840 qat_dsession = crypto_get_driver_session(crp->crp_session);
841
842 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
843 qat_session = &qat_dsession->encSession;
844 else
845 qat_session = &qat_dsession->decSession;
846 qat_instance = qat_dsession->qatInstance;
847
848 status = qat_ocf_cookie_alloc(qat_instance, &qat_cookie);
849 if (CPA_STATUS_SUCCESS != status) {
850 rc = EAGAIN;
851 goto fail;
852 }
853
854 qat_cookie->crp_op = crp;
855
856 /* Common request fields */
857 pOpData = &qat_cookie->pOpdata;
858 pOpData->instanceHandle = qat_instance->cyInstHandle;
859 pOpData->sessionCtx = NULL;
860
861 /* Cipher fields */
862 pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start;
863 pOpData->messageLenToCipherInBytes = crp->crp_payload_length;
864 /* Digest fields - any exceptions from this basic rules are covered
865 * in qat_ocf_load */
866 pOpData->hashStartSrcOffsetInBytes = crp->crp_payload_start;
867 pOpData->messageLenToHashInBytes = crp->crp_payload_length;
868
869 status = qat_ocf_load(crp, qat_cookie);
870 if (CPA_STATUS_SUCCESS != status) {
871 device_printf(dev,
872 "unable to load OCF buffers to QAT DMA "
873 "transaction\n");
874 rc = EIO;
875 goto fail;
876 }
877 memLoaded = CPA_TRUE;
878
879 status = qat_ocf_cookie_dma_pre_sync(crp, pOpData);
880 if (CPA_STATUS_SUCCESS != status) {
881 device_printf(dev, "unable to sync DMA buffers\n");
882 rc = EIO;
883 goto fail;
884 }
885
886 mtx_lock(&qat_instance->cyInstMtx);
887 /* Session initialization at the first request. It's done
888 * in such way to overcome missing QAT specific session data
889 * such like AAD length and limited possibility to update
890 * QAT session while handling traffic.
891 */
892 if (NULL == qat_session->sessionCtx) {
893 status =
894 qat_ocf_session_init(dev, crp, qat_instance, qat_session);
895 if (CPA_STATUS_SUCCESS != status) {
896 mtx_unlock(&qat_instance->cyInstMtx);
897 device_printf(dev, "unable to init session\n");
898 rc = EIO;
899 goto fail;
900 }
901 } else {
902 status = qat_ocf_handle_session_update(qat_dsession, crp);
903 if (CPA_STATUS_RESOURCE == status) {
904 mtx_unlock(&qat_instance->cyInstMtx);
905 rc = EAGAIN;
906 goto fail;
907 } else if (CPA_STATUS_SUCCESS != status) {
908 mtx_unlock(&qat_instance->cyInstMtx);
909 rc = EIO;
910 goto fail;
911 }
912 }
913 pOpData->sessionCtx = qat_session->sessionCtx;
914 status = cpaCySymDpEnqueueOp(pOpData, CPA_TRUE);
915 mtx_unlock(&qat_instance->cyInstMtx);
916 if (CPA_STATUS_SUCCESS != status) {
917 if (CPA_STATUS_RETRY == status) {
918 rc = EAGAIN;
919 goto fail;
920 }
921 device_printf(dev,
922 "unable to send request. Status: %d\n",
923 status);
924 rc = EIO;
925 goto fail;
926 }
927
928 return 0;
929 fail:
930 if (qat_cookie) {
931 if (memLoaded)
932 qat_ocf_cookie_dma_unload(crp, pOpData);
933 qat_ocf_cookie_free(qat_instance, qat_cookie);
934 }
935 crp->crp_etype = rc;
936 crypto_done(crp);
937
938 return 0;
939 }
940
941 static void
qat_ocf_identify(driver_t * drv,device_t parent)942 qat_ocf_identify(driver_t *drv, device_t parent)
943 {
944 if (device_find_child(parent, "qat_ocf", DEVICE_UNIT_ANY) == NULL &&
945 BUS_ADD_CHILD(parent, 200, "qat_ocf", DEVICE_UNIT_ANY) == 0)
946 device_printf(parent, "qat_ocf: could not attach!");
947 }
948
949 static int
qat_ocf_probe(device_t dev)950 qat_ocf_probe(device_t dev)
951 {
952 device_set_desc(dev, "QAT engine");
953 return (BUS_PROBE_NOWILDCARD);
954 }
955
956 static CpaStatus
qat_ocf_get_irq_instances(CpaInstanceHandle * cyInstHandles,Cpa16U cyInstHandlesSize,Cpa16U * foundInstances)957 qat_ocf_get_irq_instances(CpaInstanceHandle *cyInstHandles,
958 Cpa16U cyInstHandlesSize,
959 Cpa16U *foundInstances)
960 {
961 CpaStatus status = CPA_STATUS_SUCCESS;
962 icp_accel_dev_t **pAdfInsts = NULL;
963 icp_accel_dev_t *dev_addr = NULL;
964 sal_t *baseAddr = NULL;
965 sal_list_t *listTemp = NULL;
966 CpaInstanceHandle cyInstHandle;
967 CpaInstanceInfo2 info;
968 Cpa16U numDevices;
969 Cpa32U instCtr = 0;
970 Cpa32U i;
971
972 /* Get the number of devices */
973 status = icp_amgr_getNumInstances(&numDevices);
974 if (CPA_STATUS_SUCCESS != status)
975 return status;
976
977 /* Allocate memory to store addr of accel_devs */
978 pAdfInsts =
979 malloc(numDevices * sizeof(icp_accel_dev_t *), M_QAT_OCF, M_WAITOK);
980
981 /* Get ADF to return all accel_devs that support either
982 * symmetric or asymmetric crypto */
983 status = icp_amgr_getAllAccelDevByCapabilities(
984 (ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC), pAdfInsts, &numDevices);
985 if (CPA_STATUS_SUCCESS != status) {
986 free(pAdfInsts, M_QAT_OCF);
987 return status;
988 }
989
990 for (i = 0; i < numDevices; i++) {
991 dev_addr = (icp_accel_dev_t *)pAdfInsts[i];
992 baseAddr = dev_addr->pSalHandle;
993 if (NULL == baseAddr)
994 continue;
995 listTemp = baseAddr->sym_services;
996 if (NULL == listTemp) {
997 listTemp = baseAddr->crypto_services;
998 }
999
1000 while (NULL != listTemp) {
1001 cyInstHandle = SalList_getObject(listTemp);
1002 status = cpaCyInstanceGetInfo2(cyInstHandle, &info);
1003 if (CPA_STATUS_SUCCESS != status)
1004 continue;
1005 listTemp = SalList_next(listTemp);
1006 if (CPA_TRUE == info.isPolled)
1007 continue;
1008 if (instCtr >= cyInstHandlesSize)
1009 break;
1010 cyInstHandles[instCtr++] = cyInstHandle;
1011 }
1012 }
1013 free(pAdfInsts, M_QAT_OCF);
1014 *foundInstances = instCtr;
1015
1016 return CPA_STATUS_SUCCESS;
1017 }
1018
1019 static CpaStatus
qat_ocf_start_instances(struct qat_ocf_softc * qat_softc,device_t dev)1020 qat_ocf_start_instances(struct qat_ocf_softc *qat_softc, device_t dev)
1021 {
1022 CpaStatus status = CPA_STATUS_SUCCESS;
1023 Cpa16U numInstances = 0;
1024 CpaInstanceHandle cyInstHandles[QAT_OCF_MAX_INSTANCES] = { 0 };
1025 CpaInstanceHandle cyInstHandle = NULL;
1026 Cpa32U startedInstances = 0;
1027 Cpa32U i;
1028
1029 qat_softc->numCyInstances = 0;
1030 status = qat_ocf_get_irq_instances(cyInstHandles,
1031 QAT_OCF_MAX_INSTANCES,
1032 &numInstances);
1033 if (CPA_STATUS_SUCCESS != status)
1034 return status;
1035
1036 for (i = 0; i < numInstances; i++) {
1037 struct qat_ocf_instance *qat_ocf_instance;
1038
1039 cyInstHandle = cyInstHandles[i];
1040 if (!cyInstHandle)
1041 continue;
1042
1043 /* Starting instance */
1044 status = cpaCyStartInstance(cyInstHandle);
1045 if (CPA_STATUS_SUCCESS != status) {
1046 device_printf(qat_softc->sc_dev,
1047 "unable to get start instance\n");
1048 continue;
1049 }
1050
1051 qat_ocf_instance = &qat_softc->cyInstHandles[startedInstances];
1052 qat_ocf_instance->cyInstHandle = cyInstHandle;
1053 mtx_init(&qat_ocf_instance->cyInstMtx,
1054 "Instance MTX",
1055 NULL,
1056 MTX_DEF);
1057
1058 status =
1059 cpaCySetAddressTranslation(cyInstHandle, qatVirtToPhys);
1060 if (CPA_STATUS_SUCCESS != status) {
1061 device_printf(qat_softc->sc_dev,
1062 "unable to add virt to phys callback\n");
1063 goto fail;
1064 }
1065
1066 status = cpaCySymDpRegCbFunc(cyInstHandle, symDpCallback);
1067 if (CPA_STATUS_SUCCESS != status) {
1068 device_printf(qat_softc->sc_dev,
1069 "unable to add user callback\n");
1070 goto fail;
1071 }
1072
1073 /* Initialize cookie pool */
1074 status = qat_ocf_cookie_pool_init(qat_ocf_instance, dev);
1075 if (CPA_STATUS_SUCCESS != status) {
1076 device_printf(qat_softc->sc_dev,
1077 "unable to create cookie pool\n");
1078 goto fail;
1079 }
1080
1081 /* Disable forcing HW MAC validation for AEAD */
1082 status = icp_sal_setForceAEADMACVerify(cyInstHandle, CPA_FALSE);
1083 if (CPA_STATUS_SUCCESS != status) {
1084 device_printf(
1085 qat_softc->sc_dev,
1086 "unable to disable AEAD HW MAC verification\n");
1087 goto fail;
1088 }
1089
1090 qat_ocf_instance->driver_id = qat_softc->cryptodev_id;
1091
1092 startedInstances++;
1093 continue;
1094 fail:
1095 mtx_destroy(&qat_ocf_instance->cyInstMtx);
1096
1097 /* Stop instance */
1098 status = cpaCyStopInstance(cyInstHandle);
1099 if (CPA_STATUS_SUCCESS != status)
1100 device_printf(qat_softc->sc_dev,
1101 "unable to stop the instance\n");
1102 }
1103 qat_softc->numCyInstances = startedInstances;
1104
1105 return CPA_STATUS_SUCCESS;
1106 }
1107
1108 static CpaStatus
qat_ocf_stop_instances(struct qat_ocf_softc * qat_softc)1109 qat_ocf_stop_instances(struct qat_ocf_softc *qat_softc)
1110 {
1111 CpaStatus status = CPA_STATUS_SUCCESS;
1112 int i;
1113
1114 for (i = 0; i < qat_softc->numCyInstances; i++) {
1115 struct qat_ocf_instance *qat_instance;
1116
1117 qat_instance = &qat_softc->cyInstHandles[i];
1118 status = cpaCyStopInstance(qat_instance->cyInstHandle);
1119 if (CPA_STATUS_SUCCESS != status) {
1120 pr_err("QAT: stopping instance id: %d failed\n", i);
1121 continue;
1122 }
1123 qat_ocf_cookie_pool_deinit(qat_instance);
1124 mtx_destroy(&qat_instance->cyInstMtx);
1125 }
1126
1127 qat_softc->numCyInstances = 0;
1128
1129 return status;
1130 }
1131
1132 static int
qat_ocf_deinit(struct qat_ocf_softc * qat_softc)1133 qat_ocf_deinit(struct qat_ocf_softc *qat_softc)
1134 {
1135 int status = 0;
1136 CpaStatus cpaStatus;
1137
1138 if (qat_softc->cryptodev_id >= 0) {
1139 crypto_unregister_all(qat_softc->cryptodev_id);
1140 qat_softc->cryptodev_id = -1;
1141 }
1142
1143 /* Stop QAT instances */
1144 cpaStatus = qat_ocf_stop_instances(qat_softc);
1145 if (CPA_STATUS_SUCCESS != cpaStatus) {
1146 device_printf(qat_softc->sc_dev, "unable to stop instances\n");
1147 status = EIO;
1148 }
1149
1150 return status;
1151 }
1152
1153 static int
qat_ocf_init(struct qat_ocf_softc * qat_softc)1154 qat_ocf_init(struct qat_ocf_softc *qat_softc)
1155 {
1156 int32_t cryptodev_id;
1157
1158 /* Starting instances for OCF */
1159 if (qat_ocf_start_instances(qat_softc, qat_softc->sc_dev)) {
1160 device_printf(qat_softc->sc_dev,
1161 "unable to get QAT IRQ instances\n");
1162 goto fail;
1163 }
1164
1165 /* Register only if instances available */
1166 if (qat_softc->numCyInstances) {
1167 cryptodev_id =
1168 crypto_get_driverid(qat_softc->sc_dev,
1169 sizeof(struct qat_ocf_dsession),
1170 CRYPTOCAP_F_HARDWARE);
1171 if (cryptodev_id < 0) {
1172 device_printf(qat_softc->sc_dev,
1173 "cannot initialize!\n");
1174 goto fail;
1175 }
1176 qat_softc->cryptodev_id = cryptodev_id;
1177 }
1178
1179 return 0;
1180 fail:
1181 qat_ocf_deinit(qat_softc);
1182
1183 return ENXIO;
1184 }
1185
qat_ocf_sysctl_handle(SYSCTL_HANDLER_ARGS)1186 static int qat_ocf_sysctl_handle(SYSCTL_HANDLER_ARGS)
1187 {
1188 struct qat_ocf_softc *qat_softc = NULL;
1189 int ret = 0;
1190 device_t dev = arg1;
1191 u_int enabled;
1192
1193 qat_softc = device_get_softc(dev);
1194 enabled = qat_softc->enabled;
1195
1196 ret = sysctl_handle_int(oidp, &enabled, 0, req);
1197 if (ret || !req->newptr)
1198 return (ret);
1199
1200 if (qat_softc->enabled != enabled) {
1201 if (enabled) {
1202 ret = qat_ocf_init(qat_softc);
1203
1204 } else {
1205 ret = qat_ocf_deinit(qat_softc);
1206 }
1207
1208 if (!ret)
1209 qat_softc->enabled = enabled;
1210 }
1211
1212 return ret;
1213 }
1214
1215 static int
qat_ocf_attach(device_t dev)1216 qat_ocf_attach(device_t dev)
1217 {
1218 int status;
1219 struct qat_ocf_softc *qat_softc;
1220
1221 qat_softc = device_get_softc(dev);
1222 qat_softc->sc_dev = dev;
1223 qat_softc->cryptodev_id = -1;
1224 qat_softc->enabled = 1;
1225
1226 qat_softc->rc =
1227 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
1228 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
1229 OID_AUTO,
1230 "enable",
1231 CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE,
1232 dev,
1233 0,
1234 qat_ocf_sysctl_handle,
1235 "I",
1236 "QAT OCF support enablement");
1237
1238 if (!qat_softc->rc)
1239 return ENOMEM;
1240 if (qat_softc->enabled) {
1241 status = qat_ocf_init(qat_softc);
1242 if (status) {
1243 device_printf(dev, "qat_ocf init failed\n");
1244 goto fail;
1245 }
1246 }
1247
1248 return 0;
1249 fail:
1250 qat_ocf_deinit(qat_softc);
1251
1252 return (ENXIO);
1253 }
1254
1255 static int
qat_ocf_detach(device_t dev)1256 qat_ocf_detach(device_t dev)
1257 {
1258 struct qat_ocf_softc *qat_softc = device_get_softc(dev);
1259
1260 return qat_ocf_deinit(qat_softc);
1261 }
1262
1263 static device_method_t qat_ocf_methods[] =
1264 { DEVMETHOD(device_identify, qat_ocf_identify),
1265 DEVMETHOD(device_probe, qat_ocf_probe),
1266 DEVMETHOD(device_attach, qat_ocf_attach),
1267 DEVMETHOD(device_detach, qat_ocf_detach),
1268
1269 /* Cryptodev interface */
1270 DEVMETHOD(cryptodev_probesession, qat_ocf_probesession),
1271 DEVMETHOD(cryptodev_newsession, qat_ocf_newsession),
1272 DEVMETHOD(cryptodev_freesession, qat_ocf_freesession),
1273 DEVMETHOD(cryptodev_process, qat_ocf_process),
1274
1275 DEVMETHOD_END };
1276
1277 static driver_t qat_ocf_driver = {
1278 .name = "qat_ocf",
1279 .methods = qat_ocf_methods,
1280 .size = sizeof(struct qat_ocf_softc),
1281 };
1282
1283 DRIVER_MODULE_ORDERED(qat,
1284 nexus,
1285 qat_ocf_driver,
1286 NULL,
1287 NULL,
1288 SI_ORDER_ANY);
1289 MODULE_VERSION(qat, 1);
1290 MODULE_DEPEND(qat, qat_c62x, 1, 1, 1);
1291 MODULE_DEPEND(qat, qat_200xx, 1, 1, 1);
1292 MODULE_DEPEND(qat, qat_c3xxx, 1, 1, 1);
1293 MODULE_DEPEND(qat, qat_c4xxx, 1, 1, 1);
1294 MODULE_DEPEND(qat, qat_dh895xcc, 1, 1, 1);
1295 MODULE_DEPEND(qat, qat_4xxx, 1, 1, 1);
1296 MODULE_DEPEND(qat, crypto, 1, 1, 1);
1297 MODULE_DEPEND(qat, qat_common, 1, 1, 1);
1298 MODULE_DEPEND(qat, qat_api, 1, 1, 1);
1299 MODULE_DEPEND(qat, linuxkpi, 1, 1, 1);
1300