1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2022 Intel Corporation */
3 /**
4 ***************************************************************************
5 * @file lac_mem.h
6 *
7 * @defgroup LacMem Memory
8 *
9 * @ingroup LacCommon
10 *
11 * Memory re-sizing functions and memory accessor macros.
12 *
13 ***************************************************************************/
14
15 #ifndef LAC_MEM_H
16 #define LAC_MEM_H
17
18 /***************************************************************************
19 * Include header files
20 ***************************************************************************/
21 #include "cpa.h"
22 #include "qat_utils.h"
23 #include "lac_common.h"
24
25 /**
26 *******************************************************************************
27 * @ingroup LacMem
28 * These macros are used to Endian swap variables from IA to QAT.
29 *
30 * @param[out] x The variable to be swapped.
31 *
32 * @retval none
33 ******************************************************************************/
34 #if (BYTE_ORDER == LITTLE_ENDIAN)
35 #define LAC_MEM_WR_64(x) QAT_UTILS_HOST_TO_NW_64(x)
36 #define LAC_MEM_WR_32(x) QAT_UTILS_HOST_TO_NW_32(x)
37 #define LAC_MEM_WR_16(x) QAT_UTILS_HOST_TO_NW_16(x)
38 #define LAC_MEM_RD_64(x) QAT_UTILS_NW_TO_HOST_64(x)
39 #define LAC_MEM_RD_32(x) QAT_UTILS_NW_TO_HOST_32(x)
40 #define LAC_MEM_RD_16(x) QAT_UTILS_NW_TO_HOST_16(x)
41 #else
42 #define LAC_MEM_WR_64(x) (x)
43 #define LAC_MEM_WR_32(x) (x)
44 #define LAC_MEM_WR_16(x) (x)
45 #define LAC_MEM_RD_64(x) (x)
46 #define LAC_MEM_RD_32(x) (x)
47 #define LAC_MEM_RD_16(x) (x)
48 #endif
49
50 /*
51 *******************************************************************************
52 * Shared Memory Macros (memory accessible by Acceleration Engines, e.g. QAT)
53 *******************************************************************************
54 */
55
56 /**
57 *******************************************************************************
58 * @ingroup LacMem
59 * This macro can be used to write to a variable that will be read by the
60 * QAT. The macro will automatically detect the size of the target variable and
61 * will select the correct method for performing the write. The data is cast to
62 * the type of the field that it will be written to.
63 * This macro swaps data if required.
64 *
65 * @param[out] var The variable to be written. Can be a field of a struct.
66 *
67 * @param[in] data The value to be written. Will be cast to the size of the
68 * target.
69 *
70 * @retval none
71 ******************************************************************************/
72 #define LAC_MEM_SHARED_WRITE_SWAP(var, data) \
73 do { \
74 switch (sizeof(var)) { \
75 case 1: \
76 (var) = (Cpa8U)(data); \
77 break; \
78 case 2: \
79 (var) = (Cpa16U)(data); \
80 (var) = LAC_MEM_WR_16(((Cpa16U)var)); \
81 break; \
82 case 4: \
83 (var) = (Cpa32U)(data); \
84 (var) = LAC_MEM_WR_32(((Cpa32U)var)); \
85 break; \
86 case 8: \
87 (var) = (Cpa64U)(data); \
88 (var) = LAC_MEM_WR_64(((Cpa64U)var)); \
89 break; \
90 default: \
91 break; \
92 } \
93 } while (0)
94
95 /**
96 *******************************************************************************
97 * @ingroup LacMem
98 * This macro can be used to read a variable that was written by the QAT.
99 * The macro will automatically detect the size of the data to be read and will
100 * select the correct method for performing the read. The value read from the
101 * variable is cast to the size of the data type it will be stored in.
102 * This macro swaps data if required.
103 *
104 * @param[in] var The variable to be read. Can be a field of a struct.
105 *
106 * @param[out] data The variable to hold the result of the read. Data read
107 * will be cast to the size of this variable
108 *
109 * @retval none
110 ******************************************************************************/
111 #define LAC_MEM_SHARED_READ_SWAP(var, data) \
112 do { \
113 switch (sizeof(var)) { \
114 case 1: \
115 (data) = (var); \
116 break; \
117 case 2: \
118 (data) = LAC_MEM_RD_16(((Cpa16U)var)); \
119 break; \
120 case 4: \
121 (data) = LAC_MEM_RD_32(((Cpa32U)var)); \
122 break; \
123 case 8: \
124 (data) = LAC_MEM_RD_64(((Cpa64U)var)); \
125 break; \
126 default: \
127 break; \
128 } \
129 } while (0)
130
131 /**
132 *******************************************************************************
133 * @ingroup LacMem
134 * This macro can be used to write a pointer to a QAT request. The fields
135 * for pointers in the QAT request and response messages are always 64 bits
136 *
137 * @param[out] var The variable to be written to. Can be a field of a struct.
138 *
139 * @param[in] data The value to be written. Will be cast to size of target
140 * variable
141 *
142 * @retval none
143 ******************************************************************************/
144 /* cast pointer to scalar of same size of the native pointer */
145 #define LAC_MEM_SHARED_WRITE_FROM_PTR(var, data) \
146 ((var) = (Cpa64U)(LAC_ARCH_UINT)(data))
147
148 /* Note: any changes to this macro implementation should also be made to the
149 * similar LAC_MEM_CAST_PTR_TO_UINT64 macro
150 */
151
152 /**
153 *******************************************************************************
154 * @ingroup LacMem
155 * This macro can be used to read a pointer from a QAT response. The fields
156 * for pointers in the QAT request and response messages are always 64 bits
157 *
158 * @param[in] var The variable to be read. Can be a field of a struct.
159 *
160 * @param[out] data The variable to hold the result of the read. Data read
161 * will be cast to the size of this variable
162 *
163 * @retval none
164 ******************************************************************************/
165 /* Cast back to native pointer */
166 #define LAC_MEM_SHARED_READ_TO_PTR(var, data) \
167 ((data) = (void *)(LAC_ARCH_UINT)(var))
168
169 /**
170 *******************************************************************************
171 * @ingroup LacMem
172 * This macro safely casts a pointer to a Cpa64U type.
173 *
174 * @param[in] pPtr The pointer to be cast.
175 *
176 * @retval pointer cast to Cpa64U
177 ******************************************************************************/
178 #define LAC_MEM_CAST_PTR_TO_UINT64(pPtr) ((Cpa64U)(pPtr))
179
180 /**
181 *******************************************************************************
182 * @ingroup LacMem
183 * This macro uses an QAT Utils macro to convert from a virtual address to
184 *a
185 * physical address for internally allocated memory.
186 *
187 * @param[in] pVirtAddr The address to be converted.
188 *
189 * @retval The converted physical address
190 ******************************************************************************/
191 #define LAC_OS_VIRT_TO_PHYS_INTERNAL(pVirtAddr) \
192 (QAT_UTILS_MMU_VIRT_TO_PHYS(pVirtAddr))
193
194 /**
195 *******************************************************************************
196 * @ingroup LacMem
197 * This macro should be called on all externally allocated memory it calls
198 * SalMem_virt2PhysExternal function which allows a user
199 * to set the virt2phys function used by an instance.
200 * Defaults to virt to phys for kernel.
201 *
202 * @param[in] genService Generic sal_service_t structure.
203 * @param[in] pVirtAddr The address to be converted.
204 *
205 * @retval The converted physical address
206 ******************************************************************************/
207 #define LAC_OS_VIRT_TO_PHYS_EXTERNAL(genService, pVirtAddr) \
208 ((SalMem_virt2PhysExternal(pVirtAddr, &(genService))))
209
210 /**
211 *******************************************************************************
212 * @ingroup LacMem
213 * This macro can be used to write an address variable that will be read by
214 * the QAT. The macro will perform the necessary virt2phys address translation
215 * This macro is only to be called on memory allocated internally by the driver.
216 *
217 * @param[out] var The address variable to write. Can be a field of a struct.
218 *
219 * @param[in] pPtr The pointer variable to containing the address to be
220 * written
221 *
222 * @retval none
223 ******************************************************************************/
224 #define LAC_MEM_SHARED_WRITE_VIRT_TO_PHYS_PTR_INTERNAL(var, pPtr) \
225 do { \
226 Cpa64U physAddr = 0; \
227 physAddr = LAC_MEM_CAST_PTR_TO_UINT64( \
228 LAC_OS_VIRT_TO_PHYS_INTERNAL(pPtr)); \
229 var = physAddr; \
230 } while (0)
231
232 /**
233 *******************************************************************************
234 * @ingroup LacMem
235 * This macro can be used to write an address variable that will be read by
236 * the QAT. The macro will perform the necessary virt2phys address translation
237 * This macro is to be used on memory allocated externally by the user. It calls
238 * the user supplied virt2phys address translation.
239 *
240 * @param[in] pService The pointer to the service
241 * @param[out] var The address variable to write. Can be a field of a struct
242 * @param[in] pPtr The pointer variable to containing the address to be
243 * written
244 *
245 * @retval none
246 ******************************************************************************/
247 #define LAC_MEM_SHARED_WRITE_VIRT_TO_PHYS_PTR_EXTERNAL(pService, var, pPtr) \
248 do { \
249 Cpa64U physAddr = 0; \
250 physAddr = LAC_MEM_CAST_PTR_TO_UINT64( \
251 LAC_OS_VIRT_TO_PHYS_EXTERNAL(pService, pPtr)); \
252 var = physAddr; \
253 } while (0)
254
255 /*
256 *******************************************************************************
257 * OS Memory Macros
258 *******************************************************************************
259 */
260
261 /**
262 *******************************************************************************
263 * @ingroup LacMem
264 * This function and associated macro allocates the memory for the given
265 * size and stores the address of the memory allocated in the pointer.
266 *
267 * @param[out] ppMemAddr address of pointer where address will be stored
268 * @param[in] sizeBytes the size of the memory to be allocated.
269 *
270 * @retval CPA_STATUS_RESOURCE Macro failed to allocate Memory
271 * @retval CPA_STATUS_SUCCESS Macro executed successfully
272 *
273 ******************************************************************************/
274 static __inline CpaStatus
LacMem_OsMemAlloc(void ** ppMemAddr,Cpa32U sizeBytes)275 LacMem_OsMemAlloc(void **ppMemAddr, Cpa32U sizeBytes)
276 {
277 *ppMemAddr = malloc(sizeBytes, M_QAT, M_WAITOK);
278
279 return CPA_STATUS_SUCCESS;
280 }
281
282 /**
283 *******************************************************************************
284 * @ingroup LacMem
285 * This function and associated macro allocates the contiguous
286 * memory for the given
287 * size and stores the address of the memory allocated in the pointer.
288 *
289 * @param[out] ppMemAddr address of pointer where address will be stored
290 * @param[in] sizeBytes the size of the memory to be allocated.
291 * @param[in] alignmentBytes the alignment
292 * @param[in] node node to allocate from
293 *
294 * @retval CPA_STATUS_RESOURCE Macro failed to allocate Memory
295 * @retval CPA_STATUS_SUCCESS Macro executed successfully
296 *
297 ******************************************************************************/
298 static __inline CpaStatus
LacMem_OsContigAlignMemAlloc(void ** ppMemAddr,Cpa32U sizeBytes,Cpa32U alignmentBytes,Cpa32U node)299 LacMem_OsContigAlignMemAlloc(void **ppMemAddr,
300 Cpa32U sizeBytes,
301 Cpa32U alignmentBytes,
302 Cpa32U node)
303 {
304 if ((alignmentBytes & (alignmentBytes - 1)) !=
305 0) /* if is not power of 2 */
306 {
307 *ppMemAddr = NULL;
308 QAT_UTILS_LOG("alignmentBytes MUST be the power of 2\n");
309 return CPA_STATUS_INVALID_PARAM;
310 }
311
312 *ppMemAddr =
313 qatUtilsMemAllocContiguousNUMA(sizeBytes, node, alignmentBytes);
314
315 if (NULL == *ppMemAddr) {
316 return CPA_STATUS_RESOURCE;
317 }
318
319 return CPA_STATUS_SUCCESS;
320 }
321
322 /**
323 *******************************************************************************
324 * @ingroup LacMem
325 * Macro from the malloc() function
326 *
327 ******************************************************************************/
328 #define LAC_OS_MALLOC(sizeBytes) malloc(sizeBytes, M_QAT, M_WAITOK)
329
330 /**
331 *******************************************************************************
332 * @ingroup LacMem
333 * Macro from the LacMem_OsContigAlignMemAlloc function
334 *
335 ******************************************************************************/
336 #define LAC_OS_CAMALLOC(ppMemAddr, sizeBytes, alignmentBytes, node) \
337 LacMem_OsContigAlignMemAlloc((void *)ppMemAddr, \
338 sizeBytes, \
339 alignmentBytes, \
340 node)
341
342 /**
343 *******************************************************************************
344 * @ingroup LacMem
345 * Macro for declaration static const unsigned int constant. One provides
346 * the compilation time computation with the highest bit set in the
347 * sizeof(TYPE) value. The constant is being put by the linker by default in
348 * .rodata section
349 *
350 * E.g. Statement LAC_DECLARE_HIGHEST_BIT_OF(lac_mem_blk_t)
351 * results in following entry:
352 * static const unsigned int highest_bit_of_lac_mem_blk_t = 3
353 *
354 * CAUTION!!
355 * Macro is prepared only for type names NOT-containing ANY
356 * special characters. Types as amongst others:
357 * - void *
358 * - unsigned long
359 * - unsigned int
360 * are strictly forbidden and will result in compilation error.
361 * Use typedef to provide one-word type name for MACRO's usage.
362 ******************************************************************************/
363 #define LAC_DECLARE_HIGHEST_BIT_OF(TYPE) \
364 static const unsigned int highest_bit_of_##TYPE = \
365 (sizeof(TYPE) & 0x80000000 ? 31 : (sizeof(TYPE) & 0x40000000 ? 30 : (sizeof(TYPE) & 0x20000000 ? 29 : ( \
366 sizeof(TYPE) & 0x10000000 ? 28 : ( \
367 sizeof(TYPE) & 0x08000000 ? 27 : ( \
368 sizeof(TYPE) & 0x04000000 ? 26 : ( \
369 sizeof(TYPE) & 0x02000000 ? 25 : ( \
370 sizeof(TYPE) & 0x01000000 ? 24 : ( \
371 sizeof(TYPE) & 0x00800000 ? \
372 23 : \
373 (sizeof(TYPE) & 0x00400000 ? 22 : ( \
374 sizeof( \
375 TYPE) & \
376 0x00200000 ? \
377 21 : \
378 ( \
379 sizeof(TYPE) & 0x00100000 ? 20 : (sizeof(TYPE) & 0x00080000 ? 19 : ( \
380 sizeof( \
381 TYPE) & \
382 0x00040000 ? \
383 18 : \
384 ( \
385 sizeof(TYPE) & 0x00020000 ? 17 : ( \
386 sizeof(TYPE) & 0x00010000 ? 16 : (sizeof(TYPE) & \
387 0x00008000 ? \
388 15 : \
389 (sizeof(TYPE) & 0x00004000 ? 14 : ( \
390 sizeof(TYPE) & 0x00002000 ? 13 : \
391 ( \
392 sizeof(TYPE) & 0x00001000 ? 12 : ( \
393 sizeof(TYPE) & 0x00000800 ? 11 : ( \
394 sizeof(TYPE) & 0x00000400 ? 10 : \
395 ( \
396 sizeof(TYPE) & \
397 0x00000200 ? \
398 9 : \
399 (sizeof( \
400 TYPE) & \
401 0x00000100 ? \
402 8 : \
403 (sizeof(TYPE) & 0x00000080 ? 7 : \
404 ( \
405 sizeof(TYPE) & 0x00000040 ? \
406 6 : \
407 ( \
408 sizeof(TYPE) & 0x00000020 ? 5 : \
409 ( \
410 sizeof(TYPE) & 0x00000010 ? 4 : \
411 ( \
412 sizeof(TYPE) & 0x00000008 ? 3 : \
413 ( \
414 sizeof(TYPE) & 0x00000004 ? 2 : \
415 ( \
416 sizeof(TYPE) & 0x00000002 ? 1 : ( \
417 sizeof(TYPE) & 0x00000001 ? 0 : 32))))))))))))))))) /*16*/))))))))))))))) /* 31 */
418
419 /**
420 *******************************************************************************
421 * @ingroup LacMem
422 * This function and associated macro frees the memory at the given address
423 * and resets the pointer to NULL
424 *
425 * @param[out] ppMemAddr address of pointer where mem address is stored.
426 * If pointer is NULL, the function will exit silently
427 *
428 * @retval void
429 *
430 ******************************************************************************/
431 static __inline void
LacMem_OsMemFree(void ** ppMemAddr)432 LacMem_OsMemFree(void **ppMemAddr)
433 {
434 free(*ppMemAddr, M_QAT);
435 *ppMemAddr = NULL;
436 }
437
438 /**
439 *******************************************************************************
440 * @ingroup LacMem
441 * This function and associated macro frees the contiguous memory at the
442 * given address and resets the pointer to NULL
443 *
444 * @param[out] ppMemAddr address of pointer where mem address is stored.
445 * If pointer is NULL, the function will exit silently
446 *
447 * @retval void
448 *
449 ******************************************************************************/
450 static __inline void
LacMem_OsContigAlignMemFree(void ** ppMemAddr)451 LacMem_OsContigAlignMemFree(void **ppMemAddr)
452 {
453 if (NULL != *ppMemAddr) {
454 qatUtilsMemFreeNUMA(*ppMemAddr);
455 *ppMemAddr = NULL;
456 }
457 }
458
459 #define LAC_OS_FREE(pMemAddr) LacMem_OsMemFree((void *)&pMemAddr)
460
461 #define LAC_OS_CAFREE(pMemAddr) LacMem_OsContigAlignMemFree((void *)&pMemAddr)
462
463 /**
464 *******************************************************************************
465 * @ingroup LacMem
466 * Copies user data to a working buffer of the correct size (required by
467 * PKE services)
468 *
469 * @description
470 * This function produces a correctly sized working buffer from the input
471 * user buffer. If the original buffer is too small a new buffer shall
472 * be allocated and memory is copied (left padded with zeros to the
473 *required
474 * length).
475 *
476 * The returned working buffer is guaranteed to be of the desired size for
477 * QAT.
478 *
479 * When this function is called pInternalMem describes the user_buffer and
480 * when the function returns pInternalMem describes the working buffer.
481 * This is because pInternalMem describes the memory that will be sent to
482 * QAT.
483 *
484 * The caller must keep the original buffer pointer. The alllocated buffer
485 *is
486 * freed (as necessary) using icp_LacBufferRestore().
487 *
488 * @param[in] instanceHandle Handle to crypto instance so pke_resize mem pool
489 *can
490 * be located
491 * @param[in] pUserBuffer Pointer on the user buffer
492 * @param[in] userLen length of the user buffer
493 * @param[in] workingLen length of the working (correctly sized) buffer
494 * @param[in/out] pInternalMem pointer to boolean if TRUE on input then
495 * user_buffer is internally allocated memory
496 * if false then it is externally allocated.
497 * This value gets updated by the function
498 * if the returned pointer references internally
499 * allocated memory.
500 *
501 * @return a pointer to the working (correctly sized) buffer or NULL if the
502 * allocation failed
503 *
504 * @note the working length cannot be smaller than the user buffer length
505 *
506 * @warning the working buffer may be the same or different from the original
507 * user buffer; the caller should make no assumptions in this regard
508 *
509 * @see icp_LacBufferRestore()
510 *
511 ******************************************************************************/
512 Cpa8U *icp_LacBufferResize(CpaInstanceHandle instanceHandle,
513 Cpa8U *pUserBuffer,
514 Cpa32U userLen,
515 Cpa32U workingLen,
516 CpaBoolean *pInternalMemory);
517
518 /**
519 *******************************************************************************
520 * @ingroup LacMem
521 * Restores a user buffer
522 *
523 * @description
524 * This function restores a user buffer and releases its
525 * corresponding working buffer. The working buffer, assumed to be
526 * previously obtained using icp_LacBufferResize(), is freed as necessary.
527 *
528 * The contents are copied in the process.
529 *
530 * @note the working length cannot be smaller than the user buffer length
531 *
532 * @param[out] pUserBuffer Pointer on the user buffer
533 * @param[in] userLen length of the user buffer
534 * @param[in] pWorkingBuffer Pointer on the working buffer
535 * @param[in] workingLen working buffer length
536 * @param[in] copyBuf if set _TRUE the data in the workingBuffer
537 * will be copied to the userBuffer before the
538 * workingBuffer is freed.
539 *
540 * @return the status of the operation
541 *
542 * @see icp_LacBufferResize()
543 *
544 ******************************************************************************/
545 CpaStatus icp_LacBufferRestore(Cpa8U *pUserBuffer,
546 Cpa32U userLen,
547 Cpa8U *pWorkingBuffer,
548 Cpa32U workingLen,
549 CpaBoolean copyBuf);
550
551 /**
552 *******************************************************************************
553 * @ingroup LacMem
554 * Uses an instance specific user supplied virt2phys function to convert a
555 * virtual address to a physical address.
556 *
557 * @description
558 * Uses an instance specific user supplied virt2phys function to convert a
559 * virtual address to a physical address. A client of QA API can set the
560 * virt2phys function for an instance by using the
561 * cpaXxSetAddressTranslation() function. If the client does not set the
562 * virt2phys function and the instance is in kernel space then OS specific
563 * virt2phys function will be used. In user space the virt2phys function
564 * MUST be set by the user.
565 *
566 * @param[in] pVirtAddr the virtual addr to be converted
567 * @param[in] pServiceGen Pointer on the sal_service_t structure
568 * so client supplied virt2phys function can be
569 * called.
570 *
571 * @return the physical address
572 *
573 ******************************************************************************/
574 CpaPhysicalAddr SalMem_virt2PhysExternal(void *pVirtAddr, void *pServiceGen);
575
576 #endif /* LAC_MEM_H */
577