1 /* SPDX-License-Identifier: BSD-2-Clause AND BSD-3-Clause */
2 /* $NetBSD: qat_hw15.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */
3
4 /*
5 * Copyright (c) 2019 Internet Initiative Japan, Inc.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 * POSSIBILITY OF SUCH DAMAGE.
28 */
29
30 /*
31 * Copyright(c) 2007-2013 Intel Corporation. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 *
37 * * Redistributions of source code must retain the above copyright
38 * notice, this list of conditions and the following disclaimer.
39 * * Redistributions in binary form must reproduce the above copyright
40 * notice, this list of conditions and the following disclaimer in
41 * the documentation and/or other materials provided with the
42 * distribution.
43 * * Neither the name of Intel Corporation nor the names of its
44 * contributors may be used to endorse or promote products derived
45 * from this software without specific prior written permission.
46 *
47 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
48 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
49 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
50 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
51 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
52 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
53 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
54 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
55 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
56 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
57 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
58 */
59
60 #include <sys/cdefs.h>
61 #if 0
62 __KERNEL_RCSID(0, "$NetBSD: qat_hw15.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $");
63 #endif
64
65 #include <sys/param.h>
66 #include <sys/bus.h>
67 #include <sys/kernel.h>
68 #include <sys/proc.h>
69 #include <sys/systm.h>
70
71 #include <machine/bus.h>
72
73 #include <opencrypto/xform.h>
74
75 #include <dev/pci/pcireg.h>
76 #include <dev/pci/pcivar.h>
77
78 #include "qatreg.h"
79 #include "qat_hw15reg.h"
80 #include "qatvar.h"
81 #include "qat_hw15var.h"
82
83 static int qat_adm_ring_init_ring_table(struct qat_softc *);
84 static void qat_adm_ring_build_slice_mask(uint16_t *, uint32_t, uint32_t);
85 static void qat_adm_ring_build_shram_mask(uint64_t *, uint32_t, uint32_t);
86 static int qat_adm_ring_build_ring_table(struct qat_softc *, uint32_t);
87 static int qat_adm_ring_build_init_msg(struct qat_softc *,
88 struct fw_init_req *, enum fw_init_cmd_id, uint32_t,
89 struct qat_accel_init_cb *);
90 static int qat_adm_ring_send_init_msg_sync(struct qat_softc *,
91 enum fw_init_cmd_id, uint32_t);
92 static int qat_adm_ring_send_init_msg(struct qat_softc *,
93 enum fw_init_cmd_id);
94 static int qat_adm_ring_intr(struct qat_softc *, void *, void *);
95
96 void
qat_msg_req_type_populate(struct arch_if_req_hdr * msg,enum arch_if_req type,uint32_t rxring)97 qat_msg_req_type_populate(struct arch_if_req_hdr *msg, enum arch_if_req type,
98 uint32_t rxring)
99 {
100
101 memset(msg, 0, sizeof(struct arch_if_req_hdr));
102 msg->flags = ARCH_IF_FLAGS_VALID_FLAG |
103 ARCH_IF_FLAGS_RESP_RING_TYPE_ET | ARCH_IF_FLAGS_RESP_TYPE_S;
104 msg->req_type = type;
105 msg->resp_pipe_id = rxring;
106 }
107
108 void
qat_msg_cmn_hdr_populate(struct fw_la_bulk_req * msg,bus_addr_t desc_paddr,uint8_t hdrsz,uint8_t hwblksz,uint16_t comn_req_flags,uint32_t flow_id)109 qat_msg_cmn_hdr_populate(struct fw_la_bulk_req *msg, bus_addr_t desc_paddr,
110 uint8_t hdrsz, uint8_t hwblksz, uint16_t comn_req_flags, uint32_t flow_id)
111 {
112 struct fw_comn_req_hdr *hdr = &msg->comn_hdr;
113
114 hdr->comn_req_flags = comn_req_flags;
115 hdr->content_desc_params_sz = hwblksz;
116 hdr->content_desc_hdr_sz = hdrsz;
117 hdr->content_desc_addr = desc_paddr;
118 msg->flow_id = flow_id;
119 }
120
121 void
qat_msg_service_cmd_populate(struct fw_la_bulk_req * msg,enum fw_la_cmd_id cmdid,uint16_t cmd_flags)122 qat_msg_service_cmd_populate(struct fw_la_bulk_req *msg, enum fw_la_cmd_id cmdid,
123 uint16_t cmd_flags)
124 {
125 msg->comn_la_req.la_cmd_id = cmdid;
126 msg->comn_la_req.u.la_flags = cmd_flags;
127 }
128
129 void
qat_msg_cmn_mid_populate(struct fw_comn_req_mid * msg,void * cookie,uint64_t src,uint64_t dst)130 qat_msg_cmn_mid_populate(struct fw_comn_req_mid *msg, void *cookie,
131 uint64_t src, uint64_t dst)
132 {
133
134 msg->opaque_data = (uint64_t)(uintptr_t)cookie;
135 msg->src_data_addr = src;
136 if (dst == 0)
137 msg->dest_data_addr = src;
138 else
139 msg->dest_data_addr = dst;
140 }
141
142 void
qat_msg_req_params_populate(struct fw_la_bulk_req * msg,bus_addr_t req_params_paddr,uint8_t req_params_sz)143 qat_msg_req_params_populate(struct fw_la_bulk_req *msg,
144 bus_addr_t req_params_paddr, uint8_t req_params_sz)
145 {
146 msg->req_params_addr = req_params_paddr;
147 msg->comn_la_req.u1.req_params_blk_sz = req_params_sz / 8;
148 }
149
150 void
qat_msg_cmn_footer_populate(union fw_comn_req_ftr * msg,uint64_t next_addr)151 qat_msg_cmn_footer_populate(union fw_comn_req_ftr *msg, uint64_t next_addr)
152 {
153 msg->next_request_addr = next_addr;
154 }
155
156 void
qat_msg_params_populate(struct fw_la_bulk_req * msg,struct qat_crypto_desc * desc,uint8_t req_params_sz,uint16_t service_cmd_flags,uint16_t comn_req_flags)157 qat_msg_params_populate(struct fw_la_bulk_req *msg,
158 struct qat_crypto_desc *desc, uint8_t req_params_sz,
159 uint16_t service_cmd_flags, uint16_t comn_req_flags)
160 {
161 qat_msg_cmn_hdr_populate(msg, desc->qcd_desc_paddr,
162 desc->qcd_hdr_sz, desc->qcd_hw_blk_sz, comn_req_flags, 0);
163 qat_msg_service_cmd_populate(msg, desc->qcd_cmd_id, service_cmd_flags);
164 qat_msg_cmn_mid_populate(&msg->comn_mid, NULL, 0, 0);
165 qat_msg_req_params_populate(msg, 0, req_params_sz);
166 qat_msg_cmn_footer_populate(&msg->comn_ftr, 0);
167 }
168
169 static int
qat_adm_ring_init_ring_table(struct qat_softc * sc)170 qat_adm_ring_init_ring_table(struct qat_softc *sc)
171 {
172 struct qat_admin_rings *qadr = &sc->sc_admin_rings;
173
174 if (sc->sc_ae_num == 1) {
175 qadr->qadr_cya_ring_tbl =
176 &qadr->qadr_master_ring_tbl[0];
177 qadr->qadr_srv_mask[0] = QAT_SERVICE_CRYPTO_A;
178 } else if (sc->sc_ae_num == 2 || sc->sc_ae_num == 4) {
179 qadr->qadr_cya_ring_tbl =
180 &qadr->qadr_master_ring_tbl[0];
181 qadr->qadr_srv_mask[0] = QAT_SERVICE_CRYPTO_A;
182 qadr->qadr_cyb_ring_tbl =
183 &qadr->qadr_master_ring_tbl[1];
184 qadr->qadr_srv_mask[1] = QAT_SERVICE_CRYPTO_B;
185 }
186
187 return 0;
188 }
189
190 int
qat_adm_ring_init(struct qat_softc * sc)191 qat_adm_ring_init(struct qat_softc *sc)
192 {
193 struct qat_admin_rings *qadr = &sc->sc_admin_rings;
194 int error, i, j;
195
196 error = qat_alloc_dmamem(sc, &qadr->qadr_dma, 1, PAGE_SIZE, PAGE_SIZE);
197 if (error)
198 return error;
199
200 qadr->qadr_master_ring_tbl = qadr->qadr_dma.qdm_dma_vaddr;
201
202 MPASS(sc->sc_ae_num *
203 sizeof(struct fw_init_ring_table) <= PAGE_SIZE);
204
205 /* Initialize the Master Ring Table */
206 for (i = 0; i < sc->sc_ae_num; i++) {
207 struct fw_init_ring_table *firt =
208 &qadr->qadr_master_ring_tbl[i];
209
210 for (j = 0; j < INIT_RING_TABLE_SZ; j++) {
211 struct fw_init_ring_params *firp =
212 &firt->firt_bulk_rings[j];
213
214 firp->firp_reserved = 0;
215 firp->firp_curr_weight = QAT_DEFAULT_RING_WEIGHT;
216 firp->firp_init_weight = QAT_DEFAULT_RING_WEIGHT;
217 firp->firp_ring_pvl = QAT_DEFAULT_PVL;
218 }
219 memset(firt->firt_ring_mask, 0, sizeof(firt->firt_ring_mask));
220 }
221
222 error = qat_etr_setup_ring(sc, 0, RING_NUM_ADMIN_TX,
223 ADMIN_RING_SIZE, sc->sc_hw.qhw_fw_req_size,
224 NULL, NULL, "admin_tx", &qadr->qadr_admin_tx);
225 if (error)
226 return error;
227
228 error = qat_etr_setup_ring(sc, 0, RING_NUM_ADMIN_RX,
229 ADMIN_RING_SIZE, sc->sc_hw.qhw_fw_resp_size,
230 qat_adm_ring_intr, qadr, "admin_rx", &qadr->qadr_admin_rx);
231 if (error)
232 return error;
233
234 /*
235 * Finally set up the service indices into the Master Ring Table
236 * and convenient ring table pointers for each service enabled.
237 * Only the Admin rings are initialized.
238 */
239 error = qat_adm_ring_init_ring_table(sc);
240 if (error)
241 return error;
242
243 /*
244 * Calculate the number of active AEs per QAT
245 * needed for Shram partitioning.
246 */
247 for (i = 0; i < sc->sc_ae_num; i++) {
248 if (qadr->qadr_srv_mask[i])
249 qadr->qadr_active_aes_per_accel++;
250 }
251
252 return 0;
253 }
254
255 static void
qat_adm_ring_build_slice_mask(uint16_t * slice_mask,uint32_t srv_mask,uint32_t init_shram)256 qat_adm_ring_build_slice_mask(uint16_t *slice_mask, uint32_t srv_mask,
257 uint32_t init_shram)
258 {
259 uint16_t shram = 0, comn_req = 0;
260
261 if (init_shram)
262 shram = COMN_REQ_SHRAM_INIT_REQUIRED;
263
264 if (srv_mask & QAT_SERVICE_CRYPTO_A)
265 comn_req |= COMN_REQ_CY0_ONLY(shram);
266 if (srv_mask & QAT_SERVICE_CRYPTO_B)
267 comn_req |= COMN_REQ_CY1_ONLY(shram);
268
269 *slice_mask = comn_req;
270 }
271
272 static void
qat_adm_ring_build_shram_mask(uint64_t * shram_mask,uint32_t active_aes,uint32_t ae)273 qat_adm_ring_build_shram_mask(uint64_t *shram_mask, uint32_t active_aes,
274 uint32_t ae)
275 {
276 *shram_mask = 0;
277
278 if (active_aes == 1) {
279 *shram_mask = ~(*shram_mask);
280 } else if (active_aes == 2) {
281 if (ae == 1)
282 *shram_mask = ((~(*shram_mask)) & 0xffffffff);
283 else
284 *shram_mask = ((~(*shram_mask)) & 0xffffffff00000000ull);
285 } else if (active_aes == 3) {
286 if (ae == 0)
287 *shram_mask = ((~(*shram_mask)) & 0x7fffff);
288 else if (ae == 1)
289 *shram_mask = ((~(*shram_mask)) & 0x3fffff800000ull);
290 else
291 *shram_mask = ((~(*shram_mask)) & 0xffffc00000000000ull);
292 } else {
293 panic("Only three services are supported in current version");
294 }
295 }
296
297 static int
qat_adm_ring_build_ring_table(struct qat_softc * sc,uint32_t ae)298 qat_adm_ring_build_ring_table(struct qat_softc *sc, uint32_t ae)
299 {
300 struct qat_admin_rings *qadr = &sc->sc_admin_rings;
301 struct fw_init_ring_table *tbl;
302 struct fw_init_ring_params *param;
303 uint8_t srv_mask = sc->sc_admin_rings.qadr_srv_mask[ae];
304
305 if ((srv_mask & QAT_SERVICE_CRYPTO_A)) {
306 tbl = qadr->qadr_cya_ring_tbl;
307 } else if ((srv_mask & QAT_SERVICE_CRYPTO_B)) {
308 tbl = qadr->qadr_cyb_ring_tbl;
309 } else {
310 device_printf(sc->sc_dev,
311 "Invalid execution engine %d\n", ae);
312 return EINVAL;
313 }
314
315 param = &tbl->firt_bulk_rings[sc->sc_hw.qhw_ring_sym_tx];
316 param->firp_curr_weight = QAT_HI_PRIO_RING_WEIGHT;
317 param->firp_init_weight = QAT_HI_PRIO_RING_WEIGHT;
318 FW_INIT_RING_MASK_SET(tbl, sc->sc_hw.qhw_ring_sym_tx);
319
320 return 0;
321 }
322
323 static int
qat_adm_ring_build_init_msg(struct qat_softc * sc,struct fw_init_req * initmsg,enum fw_init_cmd_id cmd,uint32_t ae,struct qat_accel_init_cb * cb)324 qat_adm_ring_build_init_msg(struct qat_softc *sc,
325 struct fw_init_req *initmsg, enum fw_init_cmd_id cmd, uint32_t ae,
326 struct qat_accel_init_cb *cb)
327 {
328 struct fw_init_set_ae_info_hdr *aehdr;
329 struct fw_init_set_ae_info *aeinfo;
330 struct fw_init_set_ring_info_hdr *ringhdr;
331 struct fw_init_set_ring_info *ringinfo;
332 int init_shram = 0, tgt_id, cluster_id;
333 uint32_t srv_mask;
334
335 srv_mask = sc->sc_admin_rings.qadr_srv_mask[
336 ae % sc->sc_ae_num];
337
338 memset(initmsg, 0, sizeof(struct fw_init_req));
339
340 qat_msg_req_type_populate(&initmsg->comn_hdr.arch_if,
341 ARCH_IF_REQ_QAT_FW_INIT,
342 sc->sc_admin_rings.qadr_admin_rx->qr_ring_id);
343
344 qat_msg_cmn_mid_populate(&initmsg->comn_mid, cb, 0, 0);
345
346 switch (cmd) {
347 case FW_INIT_CMD_SET_AE_INFO:
348 if (ae % sc->sc_ae_num == 0)
349 init_shram = 1;
350 if (ae >= sc->sc_ae_num) {
351 tgt_id = 1;
352 cluster_id = 1;
353 } else {
354 cluster_id = 0;
355 if (sc->sc_ae_mask)
356 tgt_id = 0;
357 else
358 tgt_id = 1;
359 }
360 aehdr = &initmsg->u.set_ae_info;
361 aeinfo = &initmsg->u1.set_ae_info;
362
363 aehdr->init_cmd_id = cmd;
364 /* XXX that does not support sparse ae_mask */
365 aehdr->init_trgt_id = ae;
366 aehdr->init_ring_cluster_id = cluster_id;
367 aehdr->init_qat_id = tgt_id;
368
369 qat_adm_ring_build_slice_mask(&aehdr->init_slice_mask, srv_mask,
370 init_shram);
371
372 qat_adm_ring_build_shram_mask(&aeinfo->init_shram_mask,
373 sc->sc_admin_rings.qadr_active_aes_per_accel,
374 ae % sc->sc_ae_num);
375
376 break;
377 case FW_INIT_CMD_SET_RING_INFO:
378 ringhdr = &initmsg->u.set_ring_info;
379 ringinfo = &initmsg->u1.set_ring_info;
380
381 ringhdr->init_cmd_id = cmd;
382 /* XXX that does not support sparse ae_mask */
383 ringhdr->init_trgt_id = ae;
384
385 /* XXX */
386 qat_adm_ring_build_ring_table(sc,
387 ae % sc->sc_ae_num);
388
389 ringhdr->init_ring_tbl_sz = sizeof(struct fw_init_ring_table);
390
391 ringinfo->init_ring_table_ptr =
392 sc->sc_admin_rings.qadr_dma.qdm_dma_seg.ds_addr +
393 ((ae % sc->sc_ae_num) *
394 sizeof(struct fw_init_ring_table));
395
396 break;
397 default:
398 return ENOTSUP;
399 }
400
401 return 0;
402 }
403
404 static int
qat_adm_ring_send_init_msg_sync(struct qat_softc * sc,enum fw_init_cmd_id cmd,uint32_t ae)405 qat_adm_ring_send_init_msg_sync(struct qat_softc *sc,
406 enum fw_init_cmd_id cmd, uint32_t ae)
407 {
408 struct fw_init_req initmsg;
409 struct qat_accel_init_cb cb;
410 int error;
411
412 error = qat_adm_ring_build_init_msg(sc, &initmsg, cmd, ae, &cb);
413 if (error)
414 return error;
415
416 error = qat_etr_put_msg(sc, sc->sc_admin_rings.qadr_admin_tx,
417 (uint32_t *)&initmsg);
418 if (error)
419 return error;
420
421 error = tsleep(&cb, PZERO, "qat_init", hz * 3 / 2);
422 if (error) {
423 device_printf(sc->sc_dev,
424 "Timed out initialization firmware: %d\n", error);
425 return error;
426 }
427 if (cb.qaic_status) {
428 device_printf(sc->sc_dev, "Failed to initialize firmware\n");
429 return EIO;
430 }
431
432 return error;
433 }
434
435 static int
qat_adm_ring_send_init_msg(struct qat_softc * sc,enum fw_init_cmd_id cmd)436 qat_adm_ring_send_init_msg(struct qat_softc *sc,
437 enum fw_init_cmd_id cmd)
438 {
439 struct qat_admin_rings *qadr = &sc->sc_admin_rings;
440 uint32_t error, ae;
441
442 for (ae = 0; ae < sc->sc_ae_num; ae++) {
443 uint8_t srv_mask = qadr->qadr_srv_mask[ae];
444 switch (cmd) {
445 case FW_INIT_CMD_SET_AE_INFO:
446 case FW_INIT_CMD_SET_RING_INFO:
447 if (!srv_mask)
448 continue;
449 break;
450 case FW_INIT_CMD_TRNG_ENABLE:
451 case FW_INIT_CMD_TRNG_DISABLE:
452 if (!(srv_mask & QAT_SERVICE_CRYPTO_A))
453 continue;
454 break;
455 default:
456 return ENOTSUP;
457 }
458
459 error = qat_adm_ring_send_init_msg_sync(sc, cmd, ae);
460 if (error)
461 return error;
462 }
463
464 return 0;
465 }
466
467 int
qat_adm_ring_send_init(struct qat_softc * sc)468 qat_adm_ring_send_init(struct qat_softc *sc)
469 {
470 int error;
471
472 error = qat_adm_ring_send_init_msg(sc, FW_INIT_CMD_SET_AE_INFO);
473 if (error)
474 return error;
475
476 error = qat_adm_ring_send_init_msg(sc, FW_INIT_CMD_SET_RING_INFO);
477 if (error)
478 return error;
479
480 return 0;
481 }
482
483 static int
qat_adm_ring_intr(struct qat_softc * sc,void * arg,void * msg)484 qat_adm_ring_intr(struct qat_softc *sc, void *arg, void *msg)
485 {
486 struct arch_if_resp_hdr *resp;
487 struct fw_init_resp *init_resp;
488 struct qat_accel_init_cb *init_cb;
489 int handled = 0;
490
491 resp = (struct arch_if_resp_hdr *)msg;
492
493 switch (resp->resp_type) {
494 case ARCH_IF_REQ_QAT_FW_INIT:
495 init_resp = (struct fw_init_resp *)msg;
496 init_cb = (struct qat_accel_init_cb *)
497 (uintptr_t)init_resp->comn_resp.opaque_data;
498 init_cb->qaic_status =
499 __SHIFTOUT(init_resp->comn_resp.comn_status,
500 COMN_RESP_INIT_ADMIN_STATUS);
501 wakeup(init_cb);
502 break;
503 default:
504 device_printf(sc->sc_dev,
505 "unknown resp type %d\n", resp->resp_type);
506 break;
507 }
508
509 return handled;
510 }
511
512 static inline uint16_t
qat_hw15_get_comn_req_flags(uint8_t ae)513 qat_hw15_get_comn_req_flags(uint8_t ae)
514 {
515 if (ae == 0) {
516 return COMN_REQ_ORD_STRICT | COMN_REQ_PTR_TYPE_SGL |
517 COMN_REQ_AUTH0_SLICE_REQUIRED |
518 COMN_REQ_CIPHER0_SLICE_REQUIRED;
519 } else {
520 return COMN_REQ_ORD_STRICT | COMN_REQ_PTR_TYPE_SGL |
521 COMN_REQ_AUTH1_SLICE_REQUIRED |
522 COMN_REQ_CIPHER1_SLICE_REQUIRED;
523 }
524 }
525
526 static uint32_t
qat_hw15_crypto_setup_cipher_desc(struct qat_crypto_desc * desc,struct qat_session * qs,struct fw_cipher_hdr * cipher_hdr,uint32_t hw_blk_offset,enum fw_slice next_slice)527 qat_hw15_crypto_setup_cipher_desc(struct qat_crypto_desc *desc,
528 struct qat_session *qs, struct fw_cipher_hdr *cipher_hdr,
529 uint32_t hw_blk_offset, enum fw_slice next_slice)
530 {
531 desc->qcd_cipher_blk_sz = HW_AES_BLK_SZ;
532
533 cipher_hdr->state_padding_sz = 0;
534 cipher_hdr->key_sz = qs->qs_cipher_klen / 8;
535
536 cipher_hdr->state_sz = desc->qcd_cipher_blk_sz / 8;
537
538 cipher_hdr->next_id = next_slice;
539 cipher_hdr->curr_id = FW_SLICE_CIPHER;
540 cipher_hdr->offset = hw_blk_offset / 8;
541 cipher_hdr->resrvd = 0;
542
543 return sizeof(struct hw_cipher_config) + qs->qs_cipher_klen;
544 }
545
546 static void
qat_hw15_crypto_setup_cipher_config(const struct qat_crypto_desc * desc,const struct qat_session * qs,const struct cryptop * crp,struct hw_cipher_config * cipher_config)547 qat_hw15_crypto_setup_cipher_config(const struct qat_crypto_desc *desc,
548 const struct qat_session *qs, const struct cryptop *crp,
549 struct hw_cipher_config *cipher_config)
550 {
551 const uint8_t *key;
552 uint8_t *cipher_key;
553
554 cipher_config->val = qat_crypto_load_cipher_session(desc, qs);
555 cipher_config->reserved = 0;
556
557 cipher_key = (uint8_t *)(cipher_config + 1);
558 if (crp != NULL && crp->crp_cipher_key != NULL)
559 key = crp->crp_cipher_key;
560 else
561 key = qs->qs_cipher_key;
562 memcpy(cipher_key, key, qs->qs_cipher_klen);
563 }
564
565 static uint32_t
qat_hw15_crypto_setup_auth_desc(struct qat_crypto_desc * desc,struct qat_session * qs,struct fw_auth_hdr * auth_hdr,uint32_t ctrl_blk_offset,uint32_t hw_blk_offset,enum fw_slice next_slice)566 qat_hw15_crypto_setup_auth_desc(struct qat_crypto_desc *desc,
567 struct qat_session *qs, struct fw_auth_hdr *auth_hdr,
568 uint32_t ctrl_blk_offset, uint32_t hw_blk_offset,
569 enum fw_slice next_slice)
570 {
571 const struct qat_sym_hash_def *hash_def;
572
573 (void)qat_crypto_load_auth_session(desc, qs, &hash_def);
574
575 auth_hdr->next_id = next_slice;
576 auth_hdr->curr_id = FW_SLICE_AUTH;
577 auth_hdr->offset = hw_blk_offset / 8;
578 auth_hdr->resrvd = 0;
579
580 auth_hdr->hash_flags = FW_AUTH_HDR_FLAG_NO_NESTED;
581 auth_hdr->u.inner_prefix_sz = 0;
582 auth_hdr->outer_prefix_sz = 0;
583 auth_hdr->final_sz = hash_def->qshd_alg->qshai_digest_len;
584 auth_hdr->inner_state1_sz =
585 roundup(hash_def->qshd_qat->qshqi_state1_len, 8);
586 auth_hdr->inner_res_sz = hash_def->qshd_alg->qshai_digest_len;
587 auth_hdr->inner_state2_sz =
588 roundup(hash_def->qshd_qat->qshqi_state2_len, 8);
589 auth_hdr->inner_state2_off = auth_hdr->offset +
590 ((sizeof(struct hw_auth_setup) + auth_hdr->inner_state1_sz) / 8);
591
592 auth_hdr->outer_config_off = 0;
593 auth_hdr->outer_state1_sz = 0;
594 auth_hdr->outer_res_sz = 0;
595 auth_hdr->outer_prefix_off = 0;
596
597 desc->qcd_auth_sz = hash_def->qshd_alg->qshai_sah->hashsize;
598 desc->qcd_state_storage_sz = (sizeof(struct hw_auth_counter) +
599 roundup(hash_def->qshd_alg->qshai_state_size, 8)) / 8;
600 desc->qcd_gcm_aad_sz_offset1 = desc->qcd_auth_offset +
601 sizeof(struct hw_auth_setup) + auth_hdr->inner_state1_sz +
602 AES_BLOCK_LEN;
603 desc->qcd_gcm_aad_sz_offset2 = ctrl_blk_offset +
604 offsetof(struct fw_auth_hdr, u.aad_sz);
605
606 return sizeof(struct hw_auth_setup) + auth_hdr->inner_state1_sz +
607 auth_hdr->inner_state2_sz;
608 }
609
610 static void
qat_hw15_crypto_setup_auth_setup(const struct qat_crypto_desc * desc,const struct qat_session * qs,const struct cryptop * crp,struct hw_auth_setup * auth_setup)611 qat_hw15_crypto_setup_auth_setup(const struct qat_crypto_desc *desc,
612 const struct qat_session *qs, const struct cryptop *crp,
613 struct hw_auth_setup *auth_setup)
614 {
615 const struct qat_sym_hash_def *hash_def;
616 const uint8_t *key;
617 uint8_t *state1, *state2;
618 uint32_t state_sz, state1_sz, state2_sz, state1_pad_len, state2_pad_len;
619
620 auth_setup->auth_config.config = qat_crypto_load_auth_session(desc, qs,
621 &hash_def);
622 auth_setup->auth_config.reserved = 0;
623
624 auth_setup->auth_counter.counter =
625 htobe32(hash_def->qshd_qat->qshqi_auth_counter);
626 auth_setup->auth_counter.reserved = 0;
627
628 state1 = (uint8_t *)(auth_setup + 1);
629 state2 = state1 + roundup(hash_def->qshd_qat->qshqi_state1_len, 8);
630 switch (qs->qs_auth_algo) {
631 case HW_AUTH_ALGO_GALOIS_128:
632 qat_crypto_gmac_precompute(desc, qs->qs_cipher_key,
633 qs->qs_cipher_klen, hash_def, state2);
634 break;
635 case HW_AUTH_ALGO_SHA1:
636 state_sz = hash_def->qshd_alg->qshai_state_size;
637 state1_sz = roundup(hash_def->qshd_qat->qshqi_state1_len, 8);
638 state2_sz = roundup(hash_def->qshd_qat->qshqi_state2_len, 8);
639 if (qs->qs_auth_mode == HW_AUTH_MODE1) {
640 state1_pad_len = state1_sz - state_sz;
641 state2_pad_len = state2_sz - state_sz;
642 if (state1_pad_len > 0)
643 memset(state1 + state_sz, 0, state1_pad_len);
644 if (state2_pad_len > 0)
645 memset(state2 + state_sz, 0, state2_pad_len);
646 }
647 /* FALLTHROUGH */
648 case HW_AUTH_ALGO_SHA256:
649 case HW_AUTH_ALGO_SHA384:
650 case HW_AUTH_ALGO_SHA512:
651 switch (qs->qs_auth_mode) {
652 case HW_AUTH_MODE0:
653 memcpy(state1, hash_def->qshd_alg->qshai_init_state,
654 state1_sz);
655 /* Override for mode 0 hashes. */
656 auth_setup->auth_counter.counter = 0;
657 break;
658 case HW_AUTH_MODE1:
659 if (crp != NULL && crp->crp_auth_key != NULL)
660 key = crp->crp_auth_key;
661 else
662 key = qs->qs_auth_key;
663 if (key != NULL) {
664 qat_crypto_hmac_precompute(desc, key,
665 qs->qs_auth_klen, hash_def, state1, state2);
666 }
667 break;
668 default:
669 panic("%s: unhandled auth mode %d", __func__,
670 qs->qs_auth_mode);
671 }
672 break;
673 default:
674 panic("%s: unhandled auth algorithm %d", __func__,
675 qs->qs_auth_algo);
676 }
677 }
678
679 void
qat_hw15_crypto_setup_desc(struct qat_crypto * qcy,struct qat_session * qs,struct qat_crypto_desc * desc)680 qat_hw15_crypto_setup_desc(struct qat_crypto *qcy, struct qat_session *qs,
681 struct qat_crypto_desc *desc)
682 {
683 struct fw_cipher_hdr *cipher_hdr;
684 struct fw_auth_hdr *auth_hdr;
685 struct fw_la_bulk_req *req_cache;
686 struct hw_auth_setup *auth_setup;
687 struct hw_cipher_config *cipher_config;
688 uint32_t ctrl_blk_sz, ctrl_blk_offset, hw_blk_offset;
689 int i;
690 uint16_t la_cmd_flags;
691 uint8_t req_params_sz;
692 uint8_t *ctrl_blk_ptr, *hw_blk_ptr;
693
694 ctrl_blk_sz = 0;
695 if (qs->qs_cipher_algo != HW_CIPHER_ALGO_NULL)
696 ctrl_blk_sz += sizeof(struct fw_cipher_hdr);
697 if (qs->qs_auth_algo != HW_AUTH_ALGO_NULL)
698 ctrl_blk_sz += sizeof(struct fw_auth_hdr);
699
700 ctrl_blk_ptr = desc->qcd_content_desc;
701 ctrl_blk_offset = 0;
702 hw_blk_ptr = ctrl_blk_ptr + ctrl_blk_sz;
703 hw_blk_offset = 0;
704
705 la_cmd_flags = 0;
706 req_params_sz = 0;
707 for (i = 0; i < MAX_FW_SLICE; i++) {
708 switch (desc->qcd_slices[i]) {
709 case FW_SLICE_CIPHER:
710 cipher_hdr = (struct fw_cipher_hdr *)(ctrl_blk_ptr +
711 ctrl_blk_offset);
712 cipher_config = (struct hw_cipher_config *)(hw_blk_ptr +
713 hw_blk_offset);
714 desc->qcd_cipher_offset = ctrl_blk_sz + hw_blk_offset;
715 hw_blk_offset += qat_hw15_crypto_setup_cipher_desc(desc,
716 qs, cipher_hdr, hw_blk_offset,
717 desc->qcd_slices[i + 1]);
718 qat_hw15_crypto_setup_cipher_config(desc, qs, NULL,
719 cipher_config);
720 ctrl_blk_offset += sizeof(struct fw_cipher_hdr);
721 req_params_sz += sizeof(struct fw_la_cipher_req_params);
722 break;
723 case FW_SLICE_AUTH:
724 auth_hdr = (struct fw_auth_hdr *)(ctrl_blk_ptr +
725 ctrl_blk_offset);
726 auth_setup = (struct hw_auth_setup *)(hw_blk_ptr +
727 hw_blk_offset);
728 desc->qcd_auth_offset = ctrl_blk_sz + hw_blk_offset;
729 hw_blk_offset += qat_hw15_crypto_setup_auth_desc(desc,
730 qs, auth_hdr, ctrl_blk_offset, hw_blk_offset,
731 desc->qcd_slices[i + 1]);
732 qat_hw15_crypto_setup_auth_setup(desc, qs, NULL,
733 auth_setup);
734 ctrl_blk_offset += sizeof(struct fw_auth_hdr);
735 req_params_sz += sizeof(struct fw_la_auth_req_params);
736 la_cmd_flags |= LA_FLAGS_RET_AUTH_RES;
737 /* no digest verify */
738 break;
739 case FW_SLICE_DRAM_WR:
740 i = MAX_FW_SLICE; /* end of chain */
741 break;
742 default:
743 MPASS(0);
744 break;
745 }
746 }
747
748 desc->qcd_hdr_sz = ctrl_blk_offset / 8;
749 desc->qcd_hw_blk_sz = hw_blk_offset / 8;
750
751 req_cache = (struct fw_la_bulk_req *)desc->qcd_req_cache;
752 qat_msg_req_type_populate(
753 &req_cache->comn_hdr.arch_if,
754 ARCH_IF_REQ_QAT_FW_LA, 0);
755
756 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128)
757 la_cmd_flags |= LA_FLAGS_PROTO_GCM | LA_FLAGS_GCM_IV_LEN_FLAG;
758 else
759 la_cmd_flags |= LA_FLAGS_PROTO_NO;
760
761 qat_msg_params_populate(req_cache, desc, req_params_sz,
762 la_cmd_flags, 0);
763
764 bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag,
765 qs->qs_desc_mem.qdm_dma_map, BUS_DMASYNC_PREWRITE);
766 }
767
768 static void
qat_hw15_crypto_req_setkey(const struct qat_crypto_desc * desc,const struct qat_session * qs,struct qat_sym_cookie * qsc,struct fw_la_bulk_req * bulk_req,struct cryptop * crp)769 qat_hw15_crypto_req_setkey(const struct qat_crypto_desc *desc,
770 const struct qat_session *qs, struct qat_sym_cookie *qsc,
771 struct fw_la_bulk_req *bulk_req, struct cryptop *crp)
772 {
773 struct hw_auth_setup *auth_setup;
774 struct hw_cipher_config *cipher_config;
775 uint8_t *cdesc;
776 int i;
777
778 cdesc = qsc->qsc_content_desc;
779 memcpy(cdesc, desc->qcd_content_desc, CONTENT_DESC_MAX_SIZE);
780 for (i = 0; i < MAX_FW_SLICE; i++) {
781 switch (desc->qcd_slices[i]) {
782 case FW_SLICE_CIPHER:
783 cipher_config = (struct hw_cipher_config *)
784 (cdesc + desc->qcd_cipher_offset);
785 qat_hw15_crypto_setup_cipher_config(desc, qs, crp,
786 cipher_config);
787 break;
788 case FW_SLICE_AUTH:
789 auth_setup = (struct hw_auth_setup *)
790 (cdesc + desc->qcd_auth_offset);
791 qat_hw15_crypto_setup_auth_setup(desc, qs, crp,
792 auth_setup);
793 break;
794 case FW_SLICE_DRAM_WR:
795 i = MAX_FW_SLICE; /* end of chain */
796 break;
797 default:
798 MPASS(0);
799 }
800 }
801
802 bulk_req->comn_hdr.content_desc_addr = qsc->qsc_content_desc_paddr;
803 }
804
805 void
qat_hw15_crypto_setup_req_params(struct qat_crypto_bank * qcb,struct qat_session * qs,struct qat_crypto_desc const * desc,struct qat_sym_cookie * qsc,struct cryptop * crp)806 qat_hw15_crypto_setup_req_params(struct qat_crypto_bank *qcb,
807 struct qat_session *qs, struct qat_crypto_desc const *desc,
808 struct qat_sym_cookie *qsc, struct cryptop *crp)
809 {
810 struct qat_sym_bulk_cookie *qsbc;
811 struct fw_la_bulk_req *bulk_req;
812 struct fw_la_cipher_req_params *cipher_req;
813 struct fw_la_auth_req_params *auth_req;
814 bus_addr_t digest_paddr;
815 uint8_t *aad_szp2, *req_params_ptr;
816 uint32_t aad_sz, *aad_szp1;
817 enum fw_la_cmd_id cmd_id = desc->qcd_cmd_id;
818 enum fw_slice next_slice;
819
820 qsbc = &qsc->qsc_bulk_cookie;
821
822 bulk_req = (struct fw_la_bulk_req *)qsbc->qsbc_msg;
823 memcpy(bulk_req, &desc->qcd_req_cache, QAT_HW15_SESSION_REQ_CACHE_SIZE);
824 bulk_req->comn_hdr.arch_if.resp_pipe_id = qcb->qcb_sym_rx->qr_ring_id;
825 bulk_req->comn_hdr.comn_req_flags =
826 qat_hw15_get_comn_req_flags(qcb->qcb_bank % 2);
827 bulk_req->comn_mid.src_data_addr = qsc->qsc_buffer_list_desc_paddr;
828 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
829 bulk_req->comn_mid.dest_data_addr =
830 qsc->qsc_obuffer_list_desc_paddr;
831 } else {
832 bulk_req->comn_mid.dest_data_addr =
833 qsc->qsc_buffer_list_desc_paddr;
834 }
835 bulk_req->req_params_addr = qsc->qsc_bulk_req_params_buf_paddr;
836 bulk_req->comn_ftr.next_request_addr = 0;
837 bulk_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)qsc;
838 if (__predict_false(crp->crp_cipher_key != NULL ||
839 crp->crp_auth_key != NULL)) {
840 qat_hw15_crypto_req_setkey(desc, qs, qsc, bulk_req, crp);
841 }
842
843 digest_paddr = 0;
844 if (desc->qcd_auth_sz != 0)
845 digest_paddr = qsc->qsc_auth_res_paddr;
846
847 req_params_ptr = qsbc->qsbc_req_params_buf;
848 memset(req_params_ptr, 0, sizeof(qsbc->qsbc_req_params_buf));
849
850 /*
851 * The SG list layout is a bit different for GCM and GMAC, it's simpler
852 * to handle those cases separately.
853 */
854 if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) {
855 cipher_req = (struct fw_la_cipher_req_params *)req_params_ptr;
856 auth_req = (struct fw_la_auth_req_params *)
857 (req_params_ptr + sizeof(struct fw_la_cipher_req_params));
858
859 cipher_req->cipher_state_sz = desc->qcd_cipher_blk_sz / 8;
860 cipher_req->curr_id = FW_SLICE_CIPHER;
861 if (cmd_id == FW_LA_CMD_HASH_CIPHER || cmd_id == FW_LA_CMD_AUTH)
862 cipher_req->next_id = FW_SLICE_DRAM_WR;
863 else
864 cipher_req->next_id = FW_SLICE_AUTH;
865 cipher_req->state_address = qsc->qsc_iv_buf_paddr;
866
867 if (cmd_id != FW_LA_CMD_AUTH) {
868 /*
869 * Don't fill out the cipher block if we're doing GMAC
870 * only.
871 */
872 cipher_req->cipher_off = 0;
873 cipher_req->cipher_len = crp->crp_payload_length;
874 }
875 auth_req->curr_id = FW_SLICE_AUTH;
876 if (cmd_id == FW_LA_CMD_HASH_CIPHER || cmd_id == FW_LA_CMD_AUTH)
877 auth_req->next_id = FW_SLICE_CIPHER;
878 else
879 auth_req->next_id = FW_SLICE_DRAM_WR;
880
881 auth_req->auth_res_address = digest_paddr;
882 auth_req->auth_res_sz = desc->qcd_auth_sz;
883
884 auth_req->auth_off = 0;
885 auth_req->auth_len = crp->crp_payload_length;
886
887 auth_req->hash_state_sz =
888 roundup2(crp->crp_aad_length, QAT_AES_GCM_AAD_ALIGN) >> 3;
889 auth_req->u1.aad_addr = crp->crp_aad_length > 0 ?
890 qsc->qsc_gcm_aad_paddr : 0;
891
892 /*
893 * Update the hash state block if necessary. This only occurs
894 * when the AAD length changes between requests in a session and
895 * is synchronized by qat_process().
896 */
897 aad_sz = htobe32(crp->crp_aad_length);
898 aad_szp1 = (uint32_t *)(
899 __DECONST(uint8_t *, desc->qcd_content_desc) +
900 desc->qcd_gcm_aad_sz_offset1);
901 aad_szp2 = __DECONST(uint8_t *, desc->qcd_content_desc) +
902 desc->qcd_gcm_aad_sz_offset2;
903 if (__predict_false(*aad_szp1 != aad_sz)) {
904 *aad_szp1 = aad_sz;
905 *aad_szp2 = (uint8_t)roundup2(crp->crp_aad_length,
906 QAT_AES_GCM_AAD_ALIGN);
907 bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag,
908 qs->qs_desc_mem.qdm_dma_map,
909 BUS_DMASYNC_PREWRITE);
910 }
911 } else {
912 cipher_req = (struct fw_la_cipher_req_params *)req_params_ptr;
913 if (cmd_id != FW_LA_CMD_AUTH) {
914 if (cmd_id == FW_LA_CMD_CIPHER ||
915 cmd_id == FW_LA_CMD_HASH_CIPHER)
916 next_slice = FW_SLICE_DRAM_WR;
917 else
918 next_slice = FW_SLICE_AUTH;
919
920 cipher_req->cipher_state_sz =
921 desc->qcd_cipher_blk_sz / 8;
922
923 cipher_req->curr_id = FW_SLICE_CIPHER;
924 cipher_req->next_id = next_slice;
925
926 if (crp->crp_aad_length == 0) {
927 cipher_req->cipher_off = 0;
928 } else if (crp->crp_aad == NULL) {
929 cipher_req->cipher_off =
930 crp->crp_payload_start - crp->crp_aad_start;
931 } else {
932 cipher_req->cipher_off = crp->crp_aad_length;
933 }
934 cipher_req->cipher_len = crp->crp_payload_length;
935 cipher_req->state_address = qsc->qsc_iv_buf_paddr;
936 }
937 if (cmd_id != FW_LA_CMD_CIPHER) {
938 if (cmd_id == FW_LA_CMD_AUTH)
939 auth_req = (struct fw_la_auth_req_params *)
940 req_params_ptr;
941 else
942 auth_req = (struct fw_la_auth_req_params *)
943 (cipher_req + 1);
944 if (cmd_id == FW_LA_CMD_HASH_CIPHER)
945 next_slice = FW_SLICE_CIPHER;
946 else
947 next_slice = FW_SLICE_DRAM_WR;
948
949 auth_req->curr_id = FW_SLICE_AUTH;
950 auth_req->next_id = next_slice;
951
952 auth_req->auth_res_address = digest_paddr;
953 auth_req->auth_res_sz = desc->qcd_auth_sz;
954
955 auth_req->auth_len =
956 crp->crp_payload_length + crp->crp_aad_length;
957 auth_req->auth_off = 0;
958
959 auth_req->hash_state_sz = 0;
960 auth_req->u1.prefix_addr = desc->qcd_hash_state_paddr +
961 desc->qcd_state_storage_sz;
962 }
963 }
964 }
965