1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* $FreeBSD$ */ 4 /** 5 *************************************************************************** 6 * @file lac_mem.h 7 * 8 * @defgroup LacMem Memory 9 * 10 * @ingroup LacCommon 11 * 12 * Memory re-sizing functions and memory accessor macros. 13 * 14 ***************************************************************************/ 15 16 #ifndef LAC_MEM_H 17 #define LAC_MEM_H 18 19 /*************************************************************************** 20 * Include header files 21 ***************************************************************************/ 22 #include "cpa.h" 23 #include "qat_utils.h" 24 #include "lac_common.h" 25 26 /** 27 ******************************************************************************* 28 * @ingroup LacMem 29 * These macros are used to Endian swap variables from IA to QAT. 30 * 31 * @param[out] x The variable to be swapped. 32 * 33 * @retval none 34 ******************************************************************************/ 35 #if (BYTE_ORDER == LITTLE_ENDIAN) 36 #define LAC_MEM_WR_64(x) QAT_UTILS_HOST_TO_NW_64(x) 37 #define LAC_MEM_WR_32(x) QAT_UTILS_HOST_TO_NW_32(x) 38 #define LAC_MEM_WR_16(x) QAT_UTILS_HOST_TO_NW_16(x) 39 #define LAC_MEM_RD_64(x) QAT_UTILS_NW_TO_HOST_64(x) 40 #define LAC_MEM_RD_32(x) QAT_UTILS_NW_TO_HOST_32(x) 41 #define LAC_MEM_RD_16(x) QAT_UTILS_NW_TO_HOST_16(x) 42 #else 43 #define LAC_MEM_WR_64(x) (x) 44 #define LAC_MEM_WR_32(x) (x) 45 #define LAC_MEM_WR_16(x) (x) 46 #define LAC_MEM_RD_64(x) (x) 47 #define LAC_MEM_RD_32(x) (x) 48 #define LAC_MEM_RD_16(x) (x) 49 #endif 50 51 /* 52 ******************************************************************************* 53 * Shared Memory Macros (memory accessible by Acceleration Engines, e.g. QAT) 54 ******************************************************************************* 55 */ 56 57 /** 58 ******************************************************************************* 59 * @ingroup LacMem 60 * This macro can be used to write to a variable that will be read by the 61 * QAT. The macro will automatically detect the size of the target variable and 62 * will select the correct method for performing the write. The data is cast to 63 * the type of the field that it will be written to. 64 * This macro swaps data if required. 65 * 66 * @param[out] var The variable to be written. Can be a field of a struct. 67 * 68 * @param[in] data The value to be written. Will be cast to the size of the 69 * target. 70 * 71 * @retval none 72 ******************************************************************************/ 73 #define LAC_MEM_SHARED_WRITE_SWAP(var, data) \ 74 do { \ 75 switch (sizeof(var)) { \ 76 case 1: \ 77 (var) = (Cpa8U)(data); \ 78 break; \ 79 case 2: \ 80 (var) = (Cpa16U)(data); \ 81 (var) = LAC_MEM_WR_16(((Cpa16U)var)); \ 82 break; \ 83 case 4: \ 84 (var) = (Cpa32U)(data); \ 85 (var) = LAC_MEM_WR_32(((Cpa32U)var)); \ 86 break; \ 87 case 8: \ 88 (var) = (Cpa64U)(data); \ 89 (var) = LAC_MEM_WR_64(((Cpa64U)var)); \ 90 break; \ 91 default: \ 92 break; \ 93 } \ 94 } while (0) 95 96 /** 97 ******************************************************************************* 98 * @ingroup LacMem 99 * This macro can be used to read a variable that was written by the QAT. 100 * The macro will automatically detect the size of the data to be read and will 101 * select the correct method for performing the read. The value read from the 102 * variable is cast to the size of the data type it will be stored in. 103 * This macro swaps data if required. 104 * 105 * @param[in] var The variable to be read. Can be a field of a struct. 106 * 107 * @param[out] data The variable to hold the result of the read. Data read 108 * will be cast to the size of this variable 109 * 110 * @retval none 111 ******************************************************************************/ 112 #define LAC_MEM_SHARED_READ_SWAP(var, data) \ 113 do { \ 114 switch (sizeof(var)) { \ 115 case 1: \ 116 (data) = (var); \ 117 break; \ 118 case 2: \ 119 (data) = LAC_MEM_RD_16(((Cpa16U)var)); \ 120 break; \ 121 case 4: \ 122 (data) = LAC_MEM_RD_32(((Cpa32U)var)); \ 123 break; \ 124 case 8: \ 125 (data) = LAC_MEM_RD_64(((Cpa64U)var)); \ 126 break; \ 127 default: \ 128 break; \ 129 } \ 130 } while (0) 131 132 /** 133 ******************************************************************************* 134 * @ingroup LacMem 135 * This macro can be used to write a pointer to a QAT request. The fields 136 * for pointers in the QAT request and response messages are always 64 bits 137 * 138 * @param[out] var The variable to be written to. Can be a field of a struct. 139 * 140 * @param[in] data The value to be written. Will be cast to size of target 141 * variable 142 * 143 * @retval none 144 ******************************************************************************/ 145 /* cast pointer to scalar of same size of the native pointer */ 146 #define LAC_MEM_SHARED_WRITE_FROM_PTR(var, data) \ 147 ((var) = (Cpa64U)(LAC_ARCH_UINT)(data)) 148 149 /* Note: any changes to this macro implementation should also be made to the 150 * similar LAC_MEM_CAST_PTR_TO_UINT64 macro 151 */ 152 153 /** 154 ******************************************************************************* 155 * @ingroup LacMem 156 * This macro can be used to read a pointer from a QAT response. The fields 157 * for pointers in the QAT request and response messages are always 64 bits 158 * 159 * @param[in] var The variable to be read. Can be a field of a struct. 160 * 161 * @param[out] data The variable to hold the result of the read. Data read 162 * will be cast to the size of this variable 163 * 164 * @retval none 165 ******************************************************************************/ 166 /* Cast back to native pointer */ 167 #define LAC_MEM_SHARED_READ_TO_PTR(var, data) \ 168 ((data) = (void *)(LAC_ARCH_UINT)(var)) 169 170 /** 171 ******************************************************************************* 172 * @ingroup LacMem 173 * This macro safely casts a pointer to a Cpa64U type. 174 * 175 * @param[in] pPtr The pointer to be cast. 176 * 177 * @retval pointer cast to Cpa64U 178 ******************************************************************************/ 179 #define LAC_MEM_CAST_PTR_TO_UINT64(pPtr) ((Cpa64U)(pPtr)) 180 181 /** 182 ******************************************************************************* 183 * @ingroup LacMem 184 * This macro uses an QAT Utils macro to convert from a virtual address to 185 *a 186 * physical address for internally allocated memory. 187 * 188 * @param[in] pVirtAddr The address to be converted. 189 * 190 * @retval The converted physical address 191 ******************************************************************************/ 192 #define LAC_OS_VIRT_TO_PHYS_INTERNAL(pVirtAddr) \ 193 (QAT_UTILS_MMU_VIRT_TO_PHYS(pVirtAddr)) 194 195 /** 196 ******************************************************************************* 197 * @ingroup LacMem 198 * This macro should be called on all externally allocated memory it calls 199 * SalMem_virt2PhysExternal function which allows a user 200 * to set the virt2phys function used by an instance. 201 * Defaults to virt to phys for kernel. 202 * 203 * @param[in] genService Generic sal_service_t structure. 204 * @param[in] pVirtAddr The address to be converted. 205 * 206 * @retval The converted physical address 207 ******************************************************************************/ 208 #define LAC_OS_VIRT_TO_PHYS_EXTERNAL(genService, pVirtAddr) \ 209 ((SalMem_virt2PhysExternal(pVirtAddr, &(genService)))) 210 211 /** 212 ******************************************************************************* 213 * @ingroup LacMem 214 * This macro can be used to write an address variable that will be read by 215 * the QAT. The macro will perform the necessary virt2phys address translation 216 * This macro is only to be called on memory allocated internally by the driver. 217 * 218 * @param[out] var The address variable to write. Can be a field of a struct. 219 * 220 * @param[in] pPtr The pointer variable to containing the address to be 221 * written 222 * 223 * @retval none 224 ******************************************************************************/ 225 #define LAC_MEM_SHARED_WRITE_VIRT_TO_PHYS_PTR_INTERNAL(var, pPtr) \ 226 do { \ 227 Cpa64U physAddr = 0; \ 228 physAddr = LAC_MEM_CAST_PTR_TO_UINT64( \ 229 LAC_OS_VIRT_TO_PHYS_INTERNAL(pPtr)); \ 230 var = physAddr; \ 231 } while (0) 232 233 /** 234 ******************************************************************************* 235 * @ingroup LacMem 236 * This macro can be used to write an address variable that will be read by 237 * the QAT. The macro will perform the necessary virt2phys address translation 238 * This macro is to be used on memory allocated externally by the user. It calls 239 * the user supplied virt2phys address translation. 240 * 241 * @param[in] pService The pointer to the service 242 * @param[out] var The address variable to write. Can be a field of a struct 243 * @param[in] pPtr The pointer variable to containing the address to be 244 * written 245 * 246 * @retval none 247 ******************************************************************************/ 248 #define LAC_MEM_SHARED_WRITE_VIRT_TO_PHYS_PTR_EXTERNAL(pService, var, pPtr) \ 249 do { \ 250 Cpa64U physAddr = 0; \ 251 physAddr = LAC_MEM_CAST_PTR_TO_UINT64( \ 252 LAC_OS_VIRT_TO_PHYS_EXTERNAL(pService, pPtr)); \ 253 var = physAddr; \ 254 } while (0) 255 256 /* 257 ******************************************************************************* 258 * OS Memory Macros 259 ******************************************************************************* 260 */ 261 262 /** 263 ******************************************************************************* 264 * @ingroup LacMem 265 * This function and associated macro allocates the memory for the given 266 * size and stores the address of the memory allocated in the pointer. 267 * 268 * @param[out] ppMemAddr address of pointer where address will be stored 269 * @param[in] sizeBytes the size of the memory to be allocated. 270 * 271 * @retval CPA_STATUS_RESOURCE Macro failed to allocate Memory 272 * @retval CPA_STATUS_SUCCESS Macro executed successfully 273 * 274 ******************************************************************************/ 275 static __inline CpaStatus 276 LacMem_OsMemAlloc(void **ppMemAddr, Cpa32U sizeBytes) 277 { 278 *ppMemAddr = malloc(sizeBytes, M_QAT, M_WAITOK); 279 280 return CPA_STATUS_SUCCESS; 281 } 282 283 /** 284 ******************************************************************************* 285 * @ingroup LacMem 286 * This function and associated macro allocates the contiguous 287 * memory for the given 288 * size and stores the address of the memory allocated in the pointer. 289 * 290 * @param[out] ppMemAddr address of pointer where address will be stored 291 * @param[in] sizeBytes the size of the memory to be allocated. 292 * @param[in] alignmentBytes the alignment 293 * @param[in] node node to allocate from 294 * 295 * @retval CPA_STATUS_RESOURCE Macro failed to allocate Memory 296 * @retval CPA_STATUS_SUCCESS Macro executed successfully 297 * 298 ******************************************************************************/ 299 static __inline CpaStatus 300 LacMem_OsContigAlignMemAlloc(void **ppMemAddr, 301 Cpa32U sizeBytes, 302 Cpa32U alignmentBytes, 303 Cpa32U node) 304 { 305 if ((alignmentBytes & (alignmentBytes - 1)) != 306 0) /* if is not power of 2 */ 307 { 308 *ppMemAddr = NULL; 309 QAT_UTILS_LOG("alignmentBytes MUST be the power of 2\n"); 310 return CPA_STATUS_INVALID_PARAM; 311 } 312 313 *ppMemAddr = 314 qatUtilsMemAllocContiguousNUMA(sizeBytes, node, alignmentBytes); 315 316 if (NULL == *ppMemAddr) { 317 return CPA_STATUS_RESOURCE; 318 } 319 320 return CPA_STATUS_SUCCESS; 321 } 322 323 /** 324 ******************************************************************************* 325 * @ingroup LacMem 326 * Macro from the malloc() function 327 * 328 ******************************************************************************/ 329 #define LAC_OS_MALLOC(sizeBytes) malloc(sizeBytes, M_QAT, M_WAITOK) 330 331 /** 332 ******************************************************************************* 333 * @ingroup LacMem 334 * Macro from the LacMem_OsContigAlignMemAlloc function 335 * 336 ******************************************************************************/ 337 #define LAC_OS_CAMALLOC(ppMemAddr, sizeBytes, alignmentBytes, node) \ 338 LacMem_OsContigAlignMemAlloc((void *)ppMemAddr, \ 339 sizeBytes, \ 340 alignmentBytes, \ 341 node) 342 343 /** 344 ******************************************************************************* 345 * @ingroup LacMem 346 * Macro for declaration static const unsigned int constant. One provides 347 * the compilation time computation with the highest bit set in the 348 * sizeof(TYPE) value. The constant is being put by the linker by default in 349 * .rodata section 350 * 351 * E.g. Statement LAC_DECLARE_HIGHEST_BIT_OF(lac_mem_blk_t) 352 * results in following entry: 353 * static const unsigned int highest_bit_of_lac_mem_blk_t = 3 354 * 355 * CAUTION!! 356 * Macro is prepared only for type names NOT-containing ANY 357 * special characters. Types as amongst others: 358 * - void * 359 * - unsigned long 360 * - unsigned int 361 * are strictly forbidden and will result in compilation error. 362 * Use typedef to provide one-word type name for MACRO's usage. 363 ******************************************************************************/ 364 #define LAC_DECLARE_HIGHEST_BIT_OF(TYPE) \ 365 static const unsigned int highest_bit_of_##TYPE = \ 366 (sizeof(TYPE) & 0x80000000 ? 31 : (sizeof(TYPE) & 0x40000000 ? 30 : (sizeof(TYPE) & 0x20000000 ? 29 : ( \ 367 sizeof(TYPE) & 0x10000000 ? 28 : ( \ 368 sizeof(TYPE) & 0x08000000 ? 27 : ( \ 369 sizeof(TYPE) & 0x04000000 ? 26 : ( \ 370 sizeof(TYPE) & 0x02000000 ? 25 : ( \ 371 sizeof(TYPE) & 0x01000000 ? 24 : ( \ 372 sizeof(TYPE) & 0x00800000 ? \ 373 23 : \ 374 (sizeof(TYPE) & 0x00400000 ? 22 : ( \ 375 sizeof( \ 376 TYPE) & \ 377 0x00200000 ? \ 378 21 : \ 379 ( \ 380 sizeof(TYPE) & 0x00100000 ? 20 : (sizeof(TYPE) & 0x00080000 ? 19 : ( \ 381 sizeof( \ 382 TYPE) & \ 383 0x00040000 ? \ 384 18 : \ 385 ( \ 386 sizeof(TYPE) & 0x00020000 ? 17 : ( \ 387 sizeof(TYPE) & 0x00010000 ? 16 : (sizeof(TYPE) & \ 388 0x00008000 ? \ 389 15 : \ 390 (sizeof(TYPE) & 0x00004000 ? 14 : ( \ 391 sizeof(TYPE) & 0x00002000 ? 13 : \ 392 ( \ 393 sizeof(TYPE) & 0x00001000 ? 12 : ( \ 394 sizeof(TYPE) & 0x00000800 ? 11 : ( \ 395 sizeof(TYPE) & 0x00000400 ? 10 : \ 396 ( \ 397 sizeof(TYPE) & \ 398 0x00000200 ? \ 399 9 : \ 400 (sizeof( \ 401 TYPE) & \ 402 0x00000100 ? \ 403 8 : \ 404 (sizeof(TYPE) & 0x00000080 ? 7 : \ 405 ( \ 406 sizeof(TYPE) & 0x00000040 ? \ 407 6 : \ 408 ( \ 409 sizeof(TYPE) & 0x00000020 ? 5 : \ 410 ( \ 411 sizeof(TYPE) & 0x00000010 ? 4 : \ 412 ( \ 413 sizeof(TYPE) & 0x00000008 ? 3 : \ 414 ( \ 415 sizeof(TYPE) & 0x00000004 ? 2 : \ 416 ( \ 417 sizeof(TYPE) & 0x00000002 ? 1 : ( \ 418 sizeof(TYPE) & 0x00000001 ? 0 : 32))))))))))))))))) /*16*/))))))))))))))) /* 31 */ 419 420 /** 421 ******************************************************************************* 422 * @ingroup LacMem 423 * This function and associated macro frees the memory at the given address 424 * and resets the pointer to NULL 425 * 426 * @param[out] ppMemAddr address of pointer where mem address is stored. 427 * If pointer is NULL, the function will exit silently 428 * 429 * @retval void 430 * 431 ******************************************************************************/ 432 static __inline void 433 LacMem_OsMemFree(void **ppMemAddr) 434 { 435 free(*ppMemAddr, M_QAT); 436 *ppMemAddr = NULL; 437 } 438 439 /** 440 ******************************************************************************* 441 * @ingroup LacMem 442 * This function and associated macro frees the contiguous memory at the 443 * given address and resets the pointer to NULL 444 * 445 * @param[out] ppMemAddr address of pointer where mem address is stored. 446 * If pointer is NULL, the function will exit silently 447 * 448 * @retval void 449 * 450 ******************************************************************************/ 451 static __inline void 452 LacMem_OsContigAlignMemFree(void **ppMemAddr) 453 { 454 if (NULL != *ppMemAddr) { 455 qatUtilsMemFreeNUMA(*ppMemAddr); 456 *ppMemAddr = NULL; 457 } 458 } 459 460 #define LAC_OS_FREE(pMemAddr) LacMem_OsMemFree((void *)&pMemAddr) 461 462 #define LAC_OS_CAFREE(pMemAddr) LacMem_OsContigAlignMemFree((void *)&pMemAddr) 463 464 /** 465 ******************************************************************************* 466 * @ingroup LacMem 467 * Copies user data to a working buffer of the correct size (required by 468 * PKE services) 469 * 470 * @description 471 * This function produces a correctly sized working buffer from the input 472 * user buffer. If the original buffer is too small a new buffer shall 473 * be allocated and memory is copied (left padded with zeros to the 474 *required 475 * length). 476 * 477 * The returned working buffer is guaranteed to be of the desired size for 478 * QAT. 479 * 480 * When this function is called pInternalMem describes the user_buffer and 481 * when the function returns pInternalMem describes the working buffer. 482 * This is because pInternalMem describes the memory that will be sent to 483 * QAT. 484 * 485 * The caller must keep the original buffer pointer. The alllocated buffer 486 *is 487 * freed (as necessary) using icp_LacBufferRestore(). 488 * 489 * @param[in] instanceHandle Handle to crypto instance so pke_resize mem pool 490 *can 491 * be located 492 * @param[in] pUserBuffer Pointer on the user buffer 493 * @param[in] userLen length of the user buffer 494 * @param[in] workingLen length of the working (correctly sized) buffer 495 * @param[in/out] pInternalMem pointer to boolean if TRUE on input then 496 * user_buffer is internally allocated memory 497 * if false then it is externally allocated. 498 * This value gets updated by the function 499 * if the returned pointer references internally 500 * allocated memory. 501 * 502 * @return a pointer to the working (correctly sized) buffer or NULL if the 503 * allocation failed 504 * 505 * @note the working length cannot be smaller than the user buffer length 506 * 507 * @warning the working buffer may be the same or different from the original 508 * user buffer; the caller should make no assumptions in this regard 509 * 510 * @see icp_LacBufferRestore() 511 * 512 ******************************************************************************/ 513 Cpa8U *icp_LacBufferResize(CpaInstanceHandle instanceHandle, 514 Cpa8U *pUserBuffer, 515 Cpa32U userLen, 516 Cpa32U workingLen, 517 CpaBoolean *pInternalMemory); 518 519 /** 520 ******************************************************************************* 521 * @ingroup LacMem 522 * Restores a user buffer 523 * 524 * @description 525 * This function restores a user buffer and releases its 526 * corresponding working buffer. The working buffer, assumed to be 527 * previously obtained using icp_LacBufferResize(), is freed as necessary. 528 * 529 * The contents are copied in the process. 530 * 531 * @note the working length cannot be smaller than the user buffer length 532 * 533 * @param[out] pUserBuffer Pointer on the user buffer 534 * @param[in] userLen length of the user buffer 535 * @param[in] pWorkingBuffer Pointer on the working buffer 536 * @param[in] workingLen working buffer length 537 * @param[in] copyBuf if set _TRUE the data in the workingBuffer 538 * will be copied to the userBuffer before the 539 * workingBuffer is freed. 540 * 541 * @return the status of the operation 542 * 543 * @see icp_LacBufferResize() 544 * 545 ******************************************************************************/ 546 CpaStatus icp_LacBufferRestore(Cpa8U *pUserBuffer, 547 Cpa32U userLen, 548 Cpa8U *pWorkingBuffer, 549 Cpa32U workingLen, 550 CpaBoolean copyBuf); 551 552 /** 553 ******************************************************************************* 554 * @ingroup LacMem 555 * Uses an instance specific user supplied virt2phys function to convert a 556 * virtual address to a physical address. 557 * 558 * @description 559 * Uses an instance specific user supplied virt2phys function to convert a 560 * virtual address to a physical address. A client of QA API can set the 561 * virt2phys function for an instance by using the 562 * cpaXxSetAddressTranslation() function. If the client does not set the 563 * virt2phys function and the instance is in kernel space then OS specific 564 * virt2phys function will be used. In user space the virt2phys function 565 * MUST be set by the user. 566 * 567 * @param[in] pVirtAddr the virtual addr to be converted 568 * @param[in] pServiceGen Pointer on the sal_service_t structure 569 * so client supplied virt2phys function can be 570 * called. 571 * 572 * @return the physical address 573 * 574 ******************************************************************************/ 575 CpaPhysicalAddr SalMem_virt2PhysExternal(void *pVirtAddr, void *pServiceGen); 576 577 #endif /* LAC_MEM_H */ 578