1 /* SPDX-License-Identifier: BSD-2-Clause AND BSD-3-Clause */
2 /* $NetBSD: qatvar.h,v 1.2 2020/03/14 18:08:39 ad Exp $ */
3
4 /*
5 * Copyright (c) 2019 Internet Initiative Japan, Inc.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 * POSSIBILITY OF SUCH DAMAGE.
28 */
29
30 /*
31 * Copyright(c) 2007-2019 Intel Corporation. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 *
37 * * Redistributions of source code must retain the above copyright
38 * notice, this list of conditions and the following disclaimer.
39 * * Redistributions in binary form must reproduce the above copyright
40 * notice, this list of conditions and the following disclaimer in
41 * the documentation and/or other materials provided with the
42 * distribution.
43 * * Neither the name of Intel Corporation nor the names of its
44 * contributors may be used to endorse or promote products derived
45 * from this software without specific prior written permission.
46 *
47 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
48 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
49 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
50 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
51 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
52 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
53 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
54 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
55 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
56 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
57 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
58 */
59
60
61 #ifndef _DEV_PCI_QATVAR_H_
62 #define _DEV_PCI_QATVAR_H_
63
64 #include <sys/counter.h>
65 #include <sys/malloc.h>
66
67 #include <opencrypto/cryptodev.h>
68
69 #define QAT_NSYMREQ 256
70 #define QAT_NSYMCOOKIE ((QAT_NSYMREQ * 2 + 1) * 2)
71
72 #define QAT_EV_NAME_SIZE 32
73 #define QAT_RING_NAME_SIZE 32
74
75 #define QAT_MAXSEG HW_MAXSEG /* max segments for sg dma */
76 #define QAT_MAXLEN 65535 /* IP_MAXPACKET */
77
78 #define QAT_HB_INTERVAL 500 /* heartbeat msec */
79 #define QAT_SSM_WDT 100
80
81 enum qat_chip_type {
82 QAT_CHIP_C2XXX = 0, /* NanoQAT: Atom C2000 */
83 QAT_CHIP_C2XXX_IOV,
84 QAT_CHIP_C3XXX, /* Atom C3000 */
85 QAT_CHIP_C3XXX_IOV,
86 QAT_CHIP_C62X,
87 QAT_CHIP_C62X_IOV,
88 QAT_CHIP_D15XX,
89 QAT_CHIP_D15XX_IOV,
90 QAT_CHIP_DH895XCC,
91 QAT_CHIP_DH895XCC_IOV,
92 };
93
94 enum qat_sku {
95 QAT_SKU_UNKNOWN = 0,
96 QAT_SKU_1,
97 QAT_SKU_2,
98 QAT_SKU_3,
99 QAT_SKU_4,
100 QAT_SKU_VF,
101 };
102
103 enum qat_ae_status {
104 QAT_AE_ENABLED = 1,
105 QAT_AE_ACTIVE,
106 QAT_AE_DISABLED
107 };
108
109 #define TIMEOUT_AE_RESET 100
110 #define TIMEOUT_AE_CHECK 10000
111 #define TIMEOUT_AE_CSR 500
112 #define AE_EXEC_CYCLE 20
113
114 #define QAT_UOF_MAX_PAGE 1
115 #define QAT_UOF_MAX_PAGE_REGION 1
116
117 struct qat_dmamem {
118 bus_dma_tag_t qdm_dma_tag;
119 bus_dmamap_t qdm_dma_map;
120 bus_size_t qdm_dma_size;
121 bus_dma_segment_t qdm_dma_seg;
122 void *qdm_dma_vaddr;
123 };
124
125 /* Valid internal ring size values */
126 #define QAT_RING_SIZE_128 0x01
127 #define QAT_RING_SIZE_256 0x02
128 #define QAT_RING_SIZE_512 0x03
129 #define QAT_RING_SIZE_4K 0x06
130 #define QAT_RING_SIZE_16K 0x08
131 #define QAT_RING_SIZE_4M 0x10
132 #define QAT_MIN_RING_SIZE QAT_RING_SIZE_128
133 #define QAT_MAX_RING_SIZE QAT_RING_SIZE_4M
134 #define QAT_DEFAULT_RING_SIZE QAT_RING_SIZE_16K
135
136 /* Valid internal msg size values */
137 #define QAT_MSG_SIZE_32 0x01
138 #define QAT_MSG_SIZE_64 0x02
139 #define QAT_MSG_SIZE_128 0x04
140 #define QAT_MIN_MSG_SIZE QAT_MSG_SIZE_32
141 #define QAT_MAX_MSG_SIZE QAT_MSG_SIZE_128
142
143 /* Size to bytes conversion macros for ring and msg size values */
144 #define QAT_MSG_SIZE_TO_BYTES(SIZE) (SIZE << 5)
145 #define QAT_BYTES_TO_MSG_SIZE(SIZE) (SIZE >> 5)
146 #define QAT_SIZE_TO_RING_SIZE_IN_BYTES(SIZE) ((1 << (SIZE - 1)) << 7)
147 #define QAT_RING_SIZE_IN_BYTES_TO_SIZE(SIZE) ((1 << (SIZE - 1)) >> 7)
148
149 /* Minimum ring buffer size for memory allocation */
150 #define QAT_RING_SIZE_BYTES_MIN(SIZE) \
151 ((SIZE < QAT_SIZE_TO_RING_SIZE_IN_BYTES(QAT_RING_SIZE_4K)) ? \
152 QAT_SIZE_TO_RING_SIZE_IN_BYTES(QAT_RING_SIZE_4K) : SIZE)
153 #define QAT_RING_SIZE_MODULO(SIZE) (SIZE + 0x6)
154 #define QAT_SIZE_TO_POW(SIZE) ((((SIZE & 0x4) >> 1) | ((SIZE & 0x4) >> 2) | \
155 SIZE) & ~0x4)
156 /* Max outstanding requests */
157 #define QAT_MAX_INFLIGHTS(RING_SIZE, MSG_SIZE) \
158 ((((1 << (RING_SIZE - 1)) << 3) >> QAT_SIZE_TO_POW(MSG_SIZE)) - 1)
159
160 #define QAT_RING_PATTERN 0x7f
161
162 struct qat_softc;
163
164 typedef int (*qat_cb_t)(struct qat_softc *, void *, void *);
165
166 struct qat_ring {
167 struct mtx qr_ring_mtx; /* Lock per ring */
168 bool qr_need_wakeup;
169 void *qr_ring_vaddr;
170 uint32_t * volatile qr_inflight; /* tx/rx shared */
171 uint32_t qr_head;
172 uint32_t qr_tail;
173 uint8_t qr_msg_size;
174 uint8_t qr_ring_size;
175 uint32_t qr_ring; /* ring number in bank */
176 uint32_t qr_bank; /* bank number in device */
177 uint32_t qr_ring_id;
178 uint32_t qr_ring_mask;
179 qat_cb_t qr_cb;
180 void *qr_cb_arg;
181 struct qat_dmamem qr_dma;
182 bus_addr_t qr_ring_paddr;
183
184 const char *qr_name;
185 };
186
187 struct qat_bank {
188 struct qat_softc *qb_sc; /* back pointer to softc */
189 uint32_t qb_intr_mask; /* current interrupt mask */
190 uint32_t qb_allocated_rings; /* current allocated ring bitfiled */
191 uint32_t qb_coalescing_time; /* timer in nano sec, 0: disabled */
192 #define COALESCING_TIME_INTERVAL_DEFAULT 10000
193 #define COALESCING_TIME_INTERVAL_MIN 500
194 #define COALESCING_TIME_INTERVAL_MAX 0xfffff
195 uint32_t qb_bank; /* bank index */
196 struct mtx qb_bank_mtx;
197 struct resource *qb_ih;
198 void *qb_ih_cookie;
199
200 struct qat_ring qb_et_rings[MAX_RING_PER_BANK];
201
202 };
203
204 struct qat_ap_bank {
205 uint32_t qab_nf_mask;
206 uint32_t qab_nf_dest;
207 uint32_t qab_ne_mask;
208 uint32_t qab_ne_dest;
209 };
210
211 struct qat_ae_page {
212 struct qat_ae_page *qap_next;
213 struct qat_uof_page *qap_page;
214 struct qat_ae_region *qap_region;
215 u_int qap_flags;
216 };
217
218 #define QAT_AE_PAGA_FLAG_WAITING (1 << 0)
219
220 struct qat_ae_region {
221 struct qat_ae_page *qar_loaded_page;
222 STAILQ_HEAD(, qat_ae_page) qar_waiting_pages;
223 };
224
225 struct qat_ae_slice {
226 u_int qas_assigned_ctx_mask;
227 struct qat_ae_region qas_regions[QAT_UOF_MAX_PAGE_REGION];
228 struct qat_ae_page qas_pages[QAT_UOF_MAX_PAGE];
229 struct qat_ae_page *qas_cur_pages[MAX_AE_CTX];
230 struct qat_uof_image *qas_image;
231 };
232
233 #define QAT_AE(sc, ae) \
234 ((sc)->sc_ae[ae])
235
236 struct qat_ae {
237 u_int qae_state; /* AE state */
238 u_int qae_ustore_size; /* free micro-store address */
239 u_int qae_free_addr; /* free micro-store address */
240 u_int qae_free_size; /* free micro-store size */
241 u_int qae_live_ctx_mask; /* live context mask */
242 u_int qae_ustore_dram_addr; /* micro-store DRAM address */
243 u_int qae_reload_size; /* reloadable code size */
244
245 /* aefw */
246 u_int qae_num_slices;
247 struct qat_ae_slice qae_slices[MAX_AE_CTX];
248 u_int qae_reloc_ustore_dram; /* reloadable ustore-dram address */
249 u_int qae_effect_ustore_size; /* effective AE ustore size */
250 u_int qae_shareable_ustore;
251 };
252
253 struct qat_mof {
254 void *qmf_sym; /* SYM_OBJS in sc_fw_mof */
255 size_t qmf_sym_size;
256 void *qmf_uof_objs; /* UOF_OBJS in sc_fw_mof */
257 size_t qmf_uof_objs_size;
258 void *qmf_suof_objs; /* SUOF_OBJS in sc_fw_mof */
259 size_t qmf_suof_objs_size;
260 };
261
262 struct qat_ae_batch_init {
263 u_int qabi_ae;
264 u_int qabi_addr;
265 u_int *qabi_value;
266 u_int qabi_size;
267 STAILQ_ENTRY(qat_ae_batch_init) qabi_next;
268 };
269
270 STAILQ_HEAD(qat_ae_batch_init_list, qat_ae_batch_init);
271
272 /* overwritten struct uof_uword_block */
273 struct qat_uof_uword_block {
274 u_int quub_start_addr; /* start address */
275 u_int quub_num_words; /* number of microwords */
276 uint64_t quub_micro_words; /* pointer to the uwords */
277 };
278
279 struct qat_uof_page {
280 u_int qup_page_num; /* page number */
281 u_int qup_def_page; /* default page */
282 u_int qup_page_region; /* region of page */
283 u_int qup_beg_vaddr; /* begin virtual address */
284 u_int qup_beg_paddr; /* begin physical address */
285
286 u_int qup_num_uc_var; /* num of uC var in array */
287 struct uof_uword_fixup *qup_uc_var;
288 /* array of import variables */
289 u_int qup_num_imp_var; /* num of import var in array */
290 struct uof_import_var *qup_imp_var;
291 /* array of import variables */
292 u_int qup_num_imp_expr; /* num of import expr in array */
293 struct uof_uword_fixup *qup_imp_expr;
294 /* array of import expressions */
295 u_int qup_num_neigh_reg; /* num of neigh-reg in array */
296 struct uof_uword_fixup *qup_neigh_reg;
297 /* array of neigh-reg assignments */
298 u_int qup_num_micro_words; /* number of microwords in the seg */
299
300 u_int qup_num_uw_blocks; /* number of uword blocks */
301 struct qat_uof_uword_block *qup_uw_blocks;
302 /* array of uword blocks */
303 };
304
305 struct qat_uof_image {
306 struct uof_image *qui_image; /* image pointer */
307 struct qat_uof_page qui_pages[QAT_UOF_MAX_PAGE];
308 /* array of pages */
309
310 u_int qui_num_ae_reg; /* num of registers */
311 struct uof_ae_reg *qui_ae_reg; /* array of registers */
312
313 u_int qui_num_init_reg_sym; /* num of reg/sym init values */
314 struct uof_init_reg_sym *qui_init_reg_sym;
315 /* array of reg/sym init values */
316
317 u_int qui_num_sbreak; /* num of sbreak values */
318 struct qui_sbreak *qui_sbreak; /* array of sbreak values */
319
320 u_int qui_num_uwords_used;
321 /* highest uword addressreferenced + 1 */
322 };
323
324 struct qat_aefw_uof {
325 size_t qafu_size; /* uof size */
326 struct uof_obj_hdr *qafu_obj_hdr; /* UOF_OBJS */
327
328 void *qafu_str_tab;
329 size_t qafu_str_tab_size;
330
331 u_int qafu_num_init_mem;
332 struct uof_init_mem *qafu_init_mem;
333 size_t qafu_init_mem_size;
334
335 struct uof_var_mem_seg *qafu_var_mem_seg;
336
337 struct qat_ae_batch_init_list qafu_lm_init[MAX_AE];
338 size_t qafu_num_lm_init[MAX_AE];
339 size_t qafu_num_lm_init_inst[MAX_AE];
340
341 u_int qafu_num_imgs; /* number of uof image */
342 struct qat_uof_image qafu_imgs[MAX_NUM_AE * MAX_AE_CTX];
343 /* uof images */
344 };
345
346 #define QAT_SERVICE_CRYPTO_A (1 << 0)
347 #define QAT_SERVICE_CRYPTO_B (1 << 1)
348
349 struct qat_admin_rings {
350 uint32_t qadr_active_aes_per_accel;
351 uint8_t qadr_srv_mask[MAX_AE_PER_ACCEL];
352
353 struct qat_dmamem qadr_dma;
354 struct fw_init_ring_table *qadr_master_ring_tbl;
355 struct fw_init_ring_table *qadr_cya_ring_tbl;
356 struct fw_init_ring_table *qadr_cyb_ring_tbl;
357
358 struct qat_ring *qadr_admin_tx;
359 struct qat_ring *qadr_admin_rx;
360 };
361
362 struct qat_accel_init_cb {
363 int qaic_status;
364 };
365
366 struct qat_admin_comms {
367 struct qat_dmamem qadc_dma;
368 struct qat_dmamem qadc_const_tbl_dma;
369 struct qat_dmamem qadc_hb_dma;
370 };
371
372 #define QAT_PID_MINOR_REV 0xf
373 #define QAT_PID_MAJOR_REV (0xf << 4)
374
375 struct qat_suof_image {
376 char *qsi_simg_buf;
377 u_long qsi_simg_len;
378 char *qsi_css_header;
379 char *qsi_css_key;
380 char *qsi_css_signature;
381 char *qsi_css_simg;
382 u_long qsi_simg_size;
383 u_int qsi_ae_num;
384 u_int qsi_ae_mask;
385 u_int qsi_fw_type;
386 u_long qsi_simg_name;
387 u_long qsi_appmeta_data;
388 struct qat_dmamem qsi_dma;
389 };
390
391 struct qat_aefw_suof {
392 u_int qafs_file_id;
393 u_int qafs_check_sum;
394 char qafs_min_ver;
395 char qafs_maj_ver;
396 char qafs_fw_type;
397 char *qafs_suof_buf;
398 u_int qafs_suof_size;
399 char *qafs_sym_str;
400 u_int qafs_sym_size;
401 u_int qafs_num_simgs;
402 struct qat_suof_image *qafs_simg;
403 };
404
405 enum qat_sym_hash_algorithm {
406 QAT_SYM_HASH_NONE = 0,
407 QAT_SYM_HASH_MD5 = 1,
408 QAT_SYM_HASH_SHA1 = 2,
409 QAT_SYM_HASH_SHA224 = 3,
410 QAT_SYM_HASH_SHA256 = 4,
411 QAT_SYM_HASH_SHA384 = 5,
412 QAT_SYM_HASH_SHA512 = 6,
413 QAT_SYM_HASH_AES_XCBC = 7,
414 QAT_SYM_HASH_AES_CCM = 8,
415 QAT_SYM_HASH_AES_GCM = 9,
416 QAT_SYM_HASH_KASUMI_F9 = 10,
417 QAT_SYM_HASH_SNOW3G_UIA2 = 11,
418 QAT_SYM_HASH_AES_CMAC = 12,
419 QAT_SYM_HASH_AES_GMAC = 13,
420 QAT_SYM_HASH_AES_CBC_MAC = 14,
421 };
422
423 #define QAT_HASH_MD5_BLOCK_SIZE 64
424 #define QAT_HASH_MD5_DIGEST_SIZE 16
425 #define QAT_HASH_MD5_STATE_SIZE 16
426 #define QAT_HASH_SHA1_BLOCK_SIZE 64
427 #define QAT_HASH_SHA1_DIGEST_SIZE 20
428 #define QAT_HASH_SHA1_STATE_SIZE 20
429 #define QAT_HASH_SHA224_BLOCK_SIZE 64
430 #define QAT_HASH_SHA224_DIGEST_SIZE 28
431 #define QAT_HASH_SHA224_STATE_SIZE 32
432 #define QAT_HASH_SHA256_BLOCK_SIZE 64
433 #define QAT_HASH_SHA256_DIGEST_SIZE 32
434 #define QAT_HASH_SHA256_STATE_SIZE 32
435 #define QAT_HASH_SHA384_BLOCK_SIZE 128
436 #define QAT_HASH_SHA384_DIGEST_SIZE 48
437 #define QAT_HASH_SHA384_STATE_SIZE 64
438 #define QAT_HASH_SHA512_BLOCK_SIZE 128
439 #define QAT_HASH_SHA512_DIGEST_SIZE 64
440 #define QAT_HASH_SHA512_STATE_SIZE 64
441 #define QAT_HASH_XCBC_PRECOMP_KEY_NUM 3
442 #define QAT_HASH_XCBC_MAC_BLOCK_SIZE 16
443 #define QAT_HASH_XCBC_MAC_128_DIGEST_SIZE 16
444 #define QAT_HASH_CMAC_BLOCK_SIZE 16
445 #define QAT_HASH_CMAC_128_DIGEST_SIZE 16
446 #define QAT_HASH_AES_CCM_BLOCK_SIZE 16
447 #define QAT_HASH_AES_CCM_DIGEST_SIZE 16
448 #define QAT_HASH_AES_GCM_BLOCK_SIZE 16
449 #define QAT_HASH_AES_GCM_DIGEST_SIZE 16
450 #define QAT_HASH_AES_GCM_STATE_SIZE 16
451 #define QAT_HASH_KASUMI_F9_BLOCK_SIZE 8
452 #define QAT_HASH_KASUMI_F9_DIGEST_SIZE 4
453 #define QAT_HASH_SNOW3G_UIA2_BLOCK_SIZE 8
454 #define QAT_HASH_SNOW3G_UIA2_DIGEST_SIZE 4
455 #define QAT_HASH_AES_CBC_MAC_BLOCK_SIZE 16
456 #define QAT_HASH_AES_CBC_MAC_DIGEST_SIZE 16
457 #define QAT_HASH_AES_GCM_ICV_SIZE_8 8
458 #define QAT_HASH_AES_GCM_ICV_SIZE_12 12
459 #define QAT_HASH_AES_GCM_ICV_SIZE_16 16
460 #define QAT_HASH_AES_CCM_ICV_SIZE_MIN 4
461 #define QAT_HASH_AES_CCM_ICV_SIZE_MAX 16
462 #define QAT_HASH_IPAD_BYTE 0x36
463 #define QAT_HASH_OPAD_BYTE 0x5c
464 #define QAT_HASH_IPAD_4_BYTES 0x36363636
465 #define QAT_HASH_OPAD_4_BYTES 0x5c5c5c5c
466 #define QAT_HASH_KASUMI_F9_KEY_MODIFIER_4_BYTES 0xAAAAAAAA
467
468 #define QAT_SYM_XCBC_STATE_SIZE ((QAT_HASH_XCBC_MAC_BLOCK_SIZE) * 3)
469 #define QAT_SYM_CMAC_STATE_SIZE ((QAT_HASH_CMAC_BLOCK_SIZE) * 3)
470
471 struct qat_sym_hash_alg_info {
472 uint32_t qshai_digest_len; /* Digest length in bytes */
473 uint32_t qshai_block_len; /* Block length in bytes */
474 uint32_t qshai_state_size; /* size of above state in bytes */
475 const uint8_t *qshai_init_state; /* Initial state */
476
477 const struct auth_hash *qshai_sah; /* software auth hash */
478 uint32_t qshai_state_offset; /* offset to state in *_CTX */
479 uint32_t qshai_state_word;
480 };
481
482 struct qat_sym_hash_qat_info {
483 uint32_t qshqi_algo_enc; /* QAT Algorithm encoding */
484 uint32_t qshqi_auth_counter; /* Counter value for Auth */
485 uint32_t qshqi_state1_len; /* QAT state1 length in bytes */
486 uint32_t qshqi_state2_len; /* QAT state2 length in bytes */
487 };
488
489 struct qat_sym_hash_def {
490 const struct qat_sym_hash_alg_info *qshd_alg;
491 const struct qat_sym_hash_qat_info *qshd_qat;
492 };
493
494 #define QAT_SYM_REQ_PARAMS_SIZE_MAX (24 + 32)
495 /* Reserve enough space for cipher and authentication request params */
496 /* Basis of values are guaranteed in qat_hw*var.h with CTASSERT */
497
498 #define QAT_SYM_REQ_PARAMS_SIZE_PADDED \
499 roundup(QAT_SYM_REQ_PARAMS_SIZE_MAX, QAT_OPTIMAL_ALIGN)
500 /* Pad out to 64-byte multiple to ensure optimal alignment of next field */
501
502 #define QAT_SYM_KEY_TLS_PREFIX_SIZE (128)
503 /* Hash Prefix size in bytes for TLS (128 = MAX = SHA2 (384, 512)*/
504
505 #define QAT_SYM_KEY_MAX_HASH_STATE_BUFFER \
506 (QAT_SYM_KEY_TLS_PREFIX_SIZE * 2)
507 /* hash state prefix buffer structure that holds the maximum sized secret */
508
509 #define QAT_SYM_HASH_BUFFER_LEN QAT_HASH_SHA512_STATE_SIZE
510 /* Buffer length to hold 16 byte MD5 key and 20 byte SHA1 key */
511
512 #define QAT_GCM_AAD_SIZE_MAX 240
513 /* Maximum AAD size */
514
515 #define QAT_AES_GCM_AAD_ALIGN 16
516
517 struct qat_sym_bulk_cookie {
518 uint8_t qsbc_req_params_buf[QAT_SYM_REQ_PARAMS_SIZE_PADDED];
519 /* memory block reserved for request params, QAT 1.5 only
520 * NOTE: Field must be correctly aligned in memory for access by QAT
521 * engine */
522 struct qat_crypto *qsbc_crypto;
523 struct qat_session *qsbc_session;
524 /* Session context */
525 void *qsbc_cb_tag;
526 /* correlator supplied by the client */
527 uint8_t qsbc_msg[QAT_MSG_SIZE_TO_BYTES(QAT_MAX_MSG_SIZE)];
528 /* QAT request message */
529 } __aligned(QAT_OPTIMAL_ALIGN);
530
531 /* Basis of values are guaranteed in qat_hw*var.h with CTASSERT */
532 #define HASH_CONTENT_DESC_SIZE 176
533 #define CIPHER_CONTENT_DESC_SIZE 64
534
535 #define CONTENT_DESC_MAX_SIZE roundup( \
536 HASH_CONTENT_DESC_SIZE + CIPHER_CONTENT_DESC_SIZE, \
537 QAT_OPTIMAL_ALIGN)
538
539 enum qat_sym_dma {
540 QAT_SYM_DMA_AADBUF = 0,
541 QAT_SYM_DMA_BUF,
542 QAT_SYM_DMA_OBUF,
543 QAT_SYM_DMA_COUNT,
544 };
545
546 struct qat_sym_dmamap {
547 bus_dmamap_t qsd_dmamap;
548 bus_dma_tag_t qsd_dma_tag;
549 };
550
551 struct qat_sym_cookie {
552 struct qat_sym_bulk_cookie qsc_bulk_cookie;
553
554 /* should be 64-byte aligned */
555 struct buffer_list_desc qsc_buf_list;
556 struct buffer_list_desc qsc_obuf_list;
557
558 bus_dmamap_t qsc_self_dmamap;
559 bus_dma_tag_t qsc_self_dma_tag;
560
561 uint8_t qsc_iv_buf[EALG_MAX_BLOCK_LEN];
562 uint8_t qsc_auth_res[QAT_SYM_HASH_BUFFER_LEN];
563 uint8_t qsc_gcm_aad[QAT_GCM_AAD_SIZE_MAX];
564 uint8_t qsc_content_desc[CONTENT_DESC_MAX_SIZE];
565
566 struct qat_sym_dmamap qsc_dma[QAT_SYM_DMA_COUNT];
567
568 bus_addr_t qsc_bulk_req_params_buf_paddr;
569 bus_addr_t qsc_buffer_list_desc_paddr;
570 bus_addr_t qsc_obuffer_list_desc_paddr;
571 bus_addr_t qsc_iv_buf_paddr;
572 bus_addr_t qsc_auth_res_paddr;
573 bus_addr_t qsc_gcm_aad_paddr;
574 bus_addr_t qsc_content_desc_paddr;
575 };
576
577 CTASSERT(offsetof(struct qat_sym_cookie,
578 qsc_bulk_cookie.qsbc_req_params_buf) % QAT_OPTIMAL_ALIGN == 0);
579 CTASSERT(offsetof(struct qat_sym_cookie, qsc_buf_list) % QAT_OPTIMAL_ALIGN == 0);
580
581 #define MAX_CIPHER_SETUP_BLK_SZ \
582 (sizeof(struct hw_cipher_config) + \
583 2 * HW_KASUMI_KEY_SZ + 2 * HW_KASUMI_BLK_SZ)
584 #define MAX_HASH_SETUP_BLK_SZ sizeof(union hw_auth_algo_blk)
585
586 struct qat_crypto_desc {
587 uint8_t qcd_content_desc[CONTENT_DESC_MAX_SIZE]; /* must be first */
588 /* using only for qat 1.5 */
589 uint8_t qcd_hash_state_prefix_buf[QAT_GCM_AAD_SIZE_MAX];
590
591 bus_addr_t qcd_desc_paddr;
592 bus_addr_t qcd_hash_state_paddr;
593
594 enum fw_slice qcd_slices[MAX_FW_SLICE + 1];
595 enum fw_la_cmd_id qcd_cmd_id;
596 enum hw_cipher_dir qcd_cipher_dir;
597
598 /* content desc info */
599 uint8_t qcd_hdr_sz; /* in quad words */
600 uint8_t qcd_hw_blk_sz; /* in quad words */
601 uint32_t qcd_cipher_offset;
602 uint32_t qcd_auth_offset;
603 /* hash info */
604 uint8_t qcd_state_storage_sz; /* in quad words */
605 uint32_t qcd_gcm_aad_sz_offset1;
606 uint32_t qcd_gcm_aad_sz_offset2;
607 /* cipher info */
608 uint16_t qcd_cipher_blk_sz; /* in bytes */
609 uint16_t qcd_auth_sz; /* in bytes */
610
611 uint8_t qcd_req_cache[QAT_MSG_SIZE_TO_BYTES(QAT_MAX_MSG_SIZE)];
612 } __aligned(QAT_OPTIMAL_ALIGN);
613
614 struct qat_session {
615 struct qat_crypto_desc *qs_dec_desc; /* should be at top of struct*/
616 /* decrypt or auth then decrypt or auth */
617
618 struct qat_crypto_desc *qs_enc_desc;
619 /* encrypt or encrypt then auth */
620
621 struct qat_dmamem qs_desc_mem;
622
623 enum hw_cipher_algo qs_cipher_algo;
624 enum hw_cipher_mode qs_cipher_mode;
625 enum hw_auth_algo qs_auth_algo;
626 enum hw_auth_mode qs_auth_mode;
627
628 const uint8_t *qs_cipher_key;
629 int qs_cipher_klen;
630 const uint8_t *qs_auth_key;
631 int qs_auth_klen;
632 int qs_auth_mlen;
633
634 uint32_t qs_status;
635 #define QAT_SESSION_STATUS_ACTIVE (1 << 0)
636 #define QAT_SESSION_STATUS_FREEING (1 << 1)
637 uint32_t qs_inflight;
638 int qs_aad_length;
639 bool qs_need_wakeup;
640
641 struct mtx qs_session_mtx;
642 };
643
644 struct qat_crypto_bank {
645 uint16_t qcb_bank;
646
647 struct qat_ring *qcb_sym_tx;
648 struct qat_ring *qcb_sym_rx;
649
650 struct qat_dmamem qcb_symck_dmamems[QAT_NSYMCOOKIE];
651 struct qat_sym_cookie *qcb_symck_free[QAT_NSYMCOOKIE];
652 uint32_t qcb_symck_free_count;
653
654 struct mtx qcb_bank_mtx;
655
656 char qcb_ring_names[2][QAT_RING_NAME_SIZE]; /* sym tx,rx */
657 };
658
659 struct qat_crypto {
660 struct qat_softc *qcy_sc;
661 uint32_t qcy_bank_mask;
662 uint16_t qcy_num_banks;
663
664 int32_t qcy_cid; /* OpenCrypto driver ID */
665
666 struct qat_crypto_bank *qcy_banks; /* array of qat_crypto_bank */
667
668 uint32_t qcy_session_free_count;
669
670 struct mtx qcy_crypto_mtx;
671 };
672
673 struct qat_hw {
674 int8_t qhw_sram_bar_id;
675 int8_t qhw_misc_bar_id;
676 int8_t qhw_etr_bar_id;
677
678 bus_size_t qhw_cap_global_offset;
679 bus_size_t qhw_ae_offset;
680 bus_size_t qhw_ae_local_offset;
681 bus_size_t qhw_etr_bundle_size;
682
683 /* crypto processing callbacks */
684 size_t qhw_crypto_opaque_offset;
685 void (*qhw_crypto_setup_req_params)(struct qat_crypto_bank *,
686 struct qat_session *, struct qat_crypto_desc const *,
687 struct qat_sym_cookie *, struct cryptop *);
688 void (*qhw_crypto_setup_desc)(struct qat_crypto *, struct qat_session *,
689 struct qat_crypto_desc *);
690
691 uint8_t qhw_num_banks; /* max number of banks */
692 uint8_t qhw_num_ap_banks; /* max number of AutoPush banks */
693 uint8_t qhw_num_rings_per_bank; /* rings per bank */
694 uint8_t qhw_num_accel; /* max number of accelerators */
695 uint8_t qhw_num_engines; /* max number of accelerator engines */
696 uint8_t qhw_tx_rx_gap;
697 uint32_t qhw_tx_rings_mask;
698 uint32_t qhw_clock_per_sec;
699 bool qhw_fw_auth;
700 uint32_t qhw_fw_req_size;
701 uint32_t qhw_fw_resp_size;
702
703 uint8_t qhw_ring_sym_tx;
704 uint8_t qhw_ring_sym_rx;
705 uint8_t qhw_ring_asym_tx;
706 uint8_t qhw_ring_asym_rx;
707
708 /* MSIx */
709 uint32_t qhw_msix_ae_vec_gap; /* gap to ae vec from bank */
710
711 const char *qhw_mof_fwname;
712 const char *qhw_mmp_fwname;
713
714 uint32_t qhw_prod_type; /* cpu type */
715
716 /* setup callbacks */
717 uint32_t (*qhw_get_accel_mask)(struct qat_softc *);
718 uint32_t (*qhw_get_ae_mask)(struct qat_softc *);
719 enum qat_sku (*qhw_get_sku)(struct qat_softc *);
720 uint32_t (*qhw_get_accel_cap)(struct qat_softc *);
721 const char *(*qhw_get_fw_uof_name)(struct qat_softc *);
722 void (*qhw_enable_intr)(struct qat_softc *);
723 void (*qhw_init_etr_intr)(struct qat_softc *, int);
724 int (*qhw_init_admin_comms)(struct qat_softc *);
725 int (*qhw_send_admin_init)(struct qat_softc *);
726 int (*qhw_init_arb)(struct qat_softc *);
727 void (*qhw_get_arb_mapping)(struct qat_softc *, const uint32_t **);
728 void (*qhw_enable_error_correction)(struct qat_softc *);
729 int (*qhw_check_uncorrectable_error)(struct qat_softc *);
730 void (*qhw_print_err_registers)(struct qat_softc *);
731 void (*qhw_disable_error_interrupts)(struct qat_softc *);
732 int (*qhw_check_slice_hang)(struct qat_softc *);
733 int (*qhw_set_ssm_wdtimer)(struct qat_softc *);
734 };
735
736
737 /* sc_flags */
738 #define QAT_FLAG_ESRAM_ENABLE_AUTO_INIT (1 << 0)
739 #define QAT_FLAG_SHRAM_WAIT_READY (1 << 1)
740
741 /* sc_accel_cap */
742 #define QAT_ACCEL_CAP_CRYPTO_SYMMETRIC (1 << 0)
743 #define QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC (1 << 1)
744 #define QAT_ACCEL_CAP_CIPHER (1 << 2)
745 #define QAT_ACCEL_CAP_AUTHENTICATION (1 << 3)
746 #define QAT_ACCEL_CAP_REGEX (1 << 4)
747 #define QAT_ACCEL_CAP_COMPRESSION (1 << 5)
748 #define QAT_ACCEL_CAP_LZS_COMPRESSION (1 << 6)
749 #define QAT_ACCEL_CAP_RANDOM_NUMBER (1 << 7)
750 #define QAT_ACCEL_CAP_ZUC (1 << 8)
751 #define QAT_ACCEL_CAP_SHA3 (1 << 9)
752 #define QAT_ACCEL_CAP_KPT (1 << 10)
753
754 #define QAT_ACCEL_CAP_BITS \
755 "\177\020" \
756 "b\x0a" "KPT\0" \
757 "b\x09" "SHA3\0" \
758 "b\x08" "ZUC\0" \
759 "b\x07" "RANDOM_NUMBER\0" \
760 "b\x06" "LZS_COMPRESSION\0" \
761 "b\x05" "COMPRESSION\0" \
762 "b\x04" "REGEX\0" \
763 "b\x03" "AUTHENTICATION\0" \
764 "b\x02" "CIPHER\0" \
765 "b\x01" "CRYPTO_ASYMMETRIC\0" \
766 "b\x00" "CRYPTO_SYMMETRIC\0"
767
768 #define QAT_HI_PRIO_RING_WEIGHT 0xfc
769 #define QAT_LO_PRIO_RING_WEIGHT 0xfe
770 #define QAT_DEFAULT_RING_WEIGHT 0xff
771 #define QAT_DEFAULT_PVL 0
772
773 struct firmware;
774 struct resource;
775
776 struct qat_softc {
777 device_t sc_dev;
778
779 struct resource *sc_res[MAX_BARS];
780 int sc_rid[MAX_BARS];
781 bus_space_tag_t sc_csrt[MAX_BARS];
782 bus_space_handle_t sc_csrh[MAX_BARS];
783
784 uint32_t sc_ae_num;
785 uint32_t sc_ae_mask;
786
787 struct qat_crypto sc_crypto; /* crypto services */
788
789 struct qat_hw sc_hw;
790
791 uint8_t sc_rev;
792 enum qat_sku sc_sku;
793 uint32_t sc_flags;
794
795 uint32_t sc_accel_num;
796 uint32_t sc_accel_mask;
797 uint32_t sc_accel_cap;
798
799 struct qat_admin_rings sc_admin_rings; /* use only for qat 1.5 */
800 struct qat_admin_comms sc_admin_comms; /* use only for qat 1.7 */
801
802 /* ETR */
803 struct qat_bank *sc_etr_banks; /* array of etr banks */
804 struct qat_ap_bank *sc_etr_ap_banks; /* array of etr auto push banks */
805
806 /* AE */
807 struct qat_ae sc_ae[MAX_NUM_AE];
808
809 /* Interrupt */
810 struct resource *sc_ih; /* ae cluster ih */
811 void *sc_ih_cookie; /* ae cluster ih cookie */
812
813 /* Counters */
814 counter_u64_t sc_gcm_aad_restarts;
815 counter_u64_t sc_gcm_aad_updates;
816 counter_u64_t sc_ring_full_restarts;
817 counter_u64_t sc_sym_alloc_failures;
818
819 /* Firmware */
820 void *sc_fw_mof; /* mof data */
821 size_t sc_fw_mof_size; /* mof size */
822 struct qat_mof sc_mof; /* mof sections */
823
824 const char *sc_fw_uof_name; /* uof/suof name in mof */
825
826 void *sc_fw_uof; /* uof head */
827 size_t sc_fw_uof_size; /* uof size */
828 struct qat_aefw_uof sc_aefw_uof; /* UOF_OBJS in uof */
829
830 void *sc_fw_suof; /* suof head */
831 size_t sc_fw_suof_size; /* suof size */
832 struct qat_aefw_suof sc_aefw_suof; /* suof context */
833
834 void *sc_fw_mmp; /* mmp data */
835 size_t sc_fw_mmp_size; /* mmp size */
836 };
837
838 static inline void
qat_bar_write_4(struct qat_softc * sc,int baroff,bus_size_t offset,uint32_t value)839 qat_bar_write_4(struct qat_softc *sc, int baroff, bus_size_t offset,
840 uint32_t value)
841 {
842
843 MPASS(baroff >= 0 && baroff < MAX_BARS);
844
845 bus_space_write_4(sc->sc_csrt[baroff],
846 sc->sc_csrh[baroff], offset, value);
847 }
848
849 static inline uint32_t
qat_bar_read_4(struct qat_softc * sc,int baroff,bus_size_t offset)850 qat_bar_read_4(struct qat_softc *sc, int baroff, bus_size_t offset)
851 {
852
853 MPASS(baroff >= 0 && baroff < MAX_BARS);
854
855 return bus_space_read_4(sc->sc_csrt[baroff],
856 sc->sc_csrh[baroff], offset);
857 }
858
859 static inline void
qat_misc_write_4(struct qat_softc * sc,bus_size_t offset,uint32_t value)860 qat_misc_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value)
861 {
862
863 qat_bar_write_4(sc, sc->sc_hw.qhw_misc_bar_id, offset, value);
864 }
865
866 static inline uint32_t
qat_misc_read_4(struct qat_softc * sc,bus_size_t offset)867 qat_misc_read_4(struct qat_softc *sc, bus_size_t offset)
868 {
869
870 return qat_bar_read_4(sc, sc->sc_hw.qhw_misc_bar_id, offset);
871 }
872
873 static inline void
qat_misc_read_write_or_4(struct qat_softc * sc,bus_size_t offset,uint32_t value)874 qat_misc_read_write_or_4(struct qat_softc *sc, bus_size_t offset,
875 uint32_t value)
876 {
877 uint32_t reg;
878
879 reg = qat_misc_read_4(sc, offset);
880 reg |= value;
881 qat_misc_write_4(sc, offset, reg);
882 }
883
884 static inline void
qat_misc_read_write_and_4(struct qat_softc * sc,bus_size_t offset,uint32_t mask)885 qat_misc_read_write_and_4(struct qat_softc *sc, bus_size_t offset,
886 uint32_t mask)
887 {
888 uint32_t reg;
889
890 reg = qat_misc_read_4(sc, offset);
891 reg &= mask;
892 qat_misc_write_4(sc, offset, reg);
893 }
894
895 static inline void
qat_etr_write_4(struct qat_softc * sc,bus_size_t offset,uint32_t value)896 qat_etr_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value)
897 {
898
899 qat_bar_write_4(sc, sc->sc_hw.qhw_etr_bar_id, offset, value);
900 }
901
902 static inline uint32_t
qat_etr_read_4(struct qat_softc * sc,bus_size_t offset)903 qat_etr_read_4(struct qat_softc *sc, bus_size_t offset)
904 {
905
906 return qat_bar_read_4(sc, sc->sc_hw.qhw_etr_bar_id, offset);
907 }
908
909 static inline void
qat_ae_local_write_4(struct qat_softc * sc,u_char ae,bus_size_t offset,uint32_t value)910 qat_ae_local_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
911 uint32_t value)
912 {
913
914 offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_LOCAL_AE_MASK) |
915 (offset & AE_LOCAL_CSR_MASK);
916
917 qat_misc_write_4(sc, sc->sc_hw.qhw_ae_local_offset + offset,
918 value);
919 }
920
921 static inline uint32_t
qat_ae_local_read_4(struct qat_softc * sc,u_char ae,bus_size_t offset)922 qat_ae_local_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset)
923 {
924
925 offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_LOCAL_AE_MASK) |
926 (offset & AE_LOCAL_CSR_MASK);
927
928 return qat_misc_read_4(sc, sc->sc_hw.qhw_ae_local_offset + offset);
929 }
930
931 static inline void
qat_ae_xfer_write_4(struct qat_softc * sc,u_char ae,bus_size_t offset,uint32_t value)932 qat_ae_xfer_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset,
933 uint32_t value)
934 {
935 offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_XFER_AE_MASK) |
936 __SHIFTIN(offset, AE_XFER_CSR_MASK);
937
938 qat_misc_write_4(sc, sc->sc_hw.qhw_ae_offset + offset, value);
939 }
940
941 static inline void
qat_cap_global_write_4(struct qat_softc * sc,bus_size_t offset,uint32_t value)942 qat_cap_global_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value)
943 {
944
945 qat_misc_write_4(sc, sc->sc_hw.qhw_cap_global_offset + offset, value);
946 }
947
948 static inline uint32_t
qat_cap_global_read_4(struct qat_softc * sc,bus_size_t offset)949 qat_cap_global_read_4(struct qat_softc *sc, bus_size_t offset)
950 {
951
952 return qat_misc_read_4(sc, sc->sc_hw.qhw_cap_global_offset + offset);
953 }
954
955
956 static inline void
qat_etr_bank_write_4(struct qat_softc * sc,int bank,bus_size_t offset,uint32_t value)957 qat_etr_bank_write_4(struct qat_softc *sc, int bank,
958 bus_size_t offset, uint32_t value)
959 {
960
961 qat_etr_write_4(sc, sc->sc_hw.qhw_etr_bundle_size * bank + offset,
962 value);
963 }
964
965 static inline uint32_t
qat_etr_bank_read_4(struct qat_softc * sc,int bank,bus_size_t offset)966 qat_etr_bank_read_4(struct qat_softc *sc, int bank,
967 bus_size_t offset)
968 {
969
970 return qat_etr_read_4(sc,
971 sc->sc_hw.qhw_etr_bundle_size * bank + offset);
972 }
973
974 static inline void
qat_etr_ap_bank_write_4(struct qat_softc * sc,int ap_bank,bus_size_t offset,uint32_t value)975 qat_etr_ap_bank_write_4(struct qat_softc *sc, int ap_bank,
976 bus_size_t offset, uint32_t value)
977 {
978
979 qat_etr_write_4(sc, ETR_AP_BANK_OFFSET * ap_bank + offset, value);
980 }
981
982 static inline uint32_t
qat_etr_ap_bank_read_4(struct qat_softc * sc,int ap_bank,bus_size_t offset)983 qat_etr_ap_bank_read_4(struct qat_softc *sc, int ap_bank,
984 bus_size_t offset)
985 {
986
987 return qat_etr_read_4(sc, ETR_AP_BANK_OFFSET * ap_bank + offset);
988 }
989
990
991 static inline void
qat_etr_bank_ring_write_4(struct qat_softc * sc,int bank,int ring,bus_size_t offset,uint32_t value)992 qat_etr_bank_ring_write_4(struct qat_softc *sc, int bank, int ring,
993 bus_size_t offset, uint32_t value)
994 {
995
996 qat_etr_bank_write_4(sc, bank, (ring << 2) + offset, value);
997 }
998
999 static inline uint32_t
qat_etr_bank_ring_read_4(struct qat_softc * sc,int bank,int ring,bus_size_t offset)1000 qat_etr_bank_ring_read_4(struct qat_softc *sc, int bank, int ring,
1001 bus_size_t offset)
1002 {
1003
1004 return qat_etr_bank_read_4(sc, bank, (ring << 2) * offset);
1005 }
1006
1007 static inline void
qat_etr_bank_ring_base_write_8(struct qat_softc * sc,int bank,int ring,uint64_t value)1008 qat_etr_bank_ring_base_write_8(struct qat_softc *sc, int bank, int ring,
1009 uint64_t value)
1010 {
1011 uint32_t lo, hi;
1012
1013 lo = (uint32_t)(value & 0xffffffff);
1014 hi = (uint32_t)((value & 0xffffffff00000000ULL) >> 32);
1015 qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_LBASE, lo);
1016 qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_UBASE, hi);
1017 }
1018
1019 static inline void
qat_arb_ringsrvarben_write_4(struct qat_softc * sc,int index,uint32_t value)1020 qat_arb_ringsrvarben_write_4(struct qat_softc *sc, int index, uint32_t value)
1021 {
1022
1023 qat_etr_write_4(sc, ARB_RINGSRVARBEN_OFFSET +
1024 (ARB_REG_SLOT * index), value);
1025 }
1026
1027 static inline void
qat_arb_sarconfig_write_4(struct qat_softc * sc,int index,uint32_t value)1028 qat_arb_sarconfig_write_4(struct qat_softc *sc, int index, uint32_t value)
1029 {
1030
1031 qat_etr_write_4(sc, ARB_OFFSET +
1032 (ARB_REG_SIZE * index), value);
1033 }
1034
1035 static inline void
qat_arb_wrk_2_ser_map_write_4(struct qat_softc * sc,int index,uint32_t value)1036 qat_arb_wrk_2_ser_map_write_4(struct qat_softc *sc, int index, uint32_t value)
1037 {
1038
1039 qat_etr_write_4(sc, ARB_OFFSET + ARB_WRK_2_SER_MAP_OFFSET +
1040 (ARB_REG_SIZE * index), value);
1041 }
1042
1043 void * qat_alloc_mem(size_t);
1044 void qat_free_mem(void *);
1045 void qat_free_dmamem(struct qat_softc *, struct qat_dmamem *);
1046 int qat_alloc_dmamem(struct qat_softc *, struct qat_dmamem *, int,
1047 bus_size_t, bus_size_t);
1048
1049 int qat_etr_setup_ring(struct qat_softc *, int, uint32_t, uint32_t,
1050 uint32_t, qat_cb_t, void *, const char *,
1051 struct qat_ring **);
1052 int qat_etr_put_msg(struct qat_softc *, struct qat_ring *,
1053 uint32_t *);
1054
1055 void qat_memcpy_htobe64(void *, const void *, size_t);
1056 void qat_memcpy_htobe32(void *, const void *, size_t);
1057 void qat_memcpy_htobe(void *, const void *, size_t, uint32_t);
1058 void qat_crypto_gmac_precompute(const struct qat_crypto_desc *,
1059 const uint8_t *key, int klen,
1060 const struct qat_sym_hash_def *, uint8_t *);
1061 void qat_crypto_hmac_precompute(const struct qat_crypto_desc *,
1062 const uint8_t *, int, const struct qat_sym_hash_def *,
1063 uint8_t *, uint8_t *);
1064 uint16_t qat_crypto_load_cipher_session(const struct qat_crypto_desc *,
1065 const struct qat_session *);
1066 uint16_t qat_crypto_load_auth_session(const struct qat_crypto_desc *,
1067 const struct qat_session *,
1068 struct qat_sym_hash_def const **);
1069
1070 #endif
1071