1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /** 4 *************************************************************************** 5 * @file lac_mem.h 6 * 7 * @defgroup LacMem Memory 8 * 9 * @ingroup LacCommon 10 * 11 * Memory re-sizing functions and memory accessor macros. 12 * 13 ***************************************************************************/ 14 15 #ifndef LAC_MEM_H 16 #define LAC_MEM_H 17 18 /*************************************************************************** 19 * Include header files 20 ***************************************************************************/ 21 #include "cpa.h" 22 #include "qat_utils.h" 23 #include "lac_common.h" 24 25 /** 26 ******************************************************************************* 27 * @ingroup LacMem 28 * These macros are used to Endian swap variables from IA to QAT. 29 * 30 * @param[out] x The variable to be swapped. 31 * 32 * @retval none 33 ******************************************************************************/ 34 #if (BYTE_ORDER == LITTLE_ENDIAN) 35 #define LAC_MEM_WR_64(x) QAT_UTILS_HOST_TO_NW_64(x) 36 #define LAC_MEM_WR_32(x) QAT_UTILS_HOST_TO_NW_32(x) 37 #define LAC_MEM_WR_16(x) QAT_UTILS_HOST_TO_NW_16(x) 38 #define LAC_MEM_RD_64(x) QAT_UTILS_NW_TO_HOST_64(x) 39 #define LAC_MEM_RD_32(x) QAT_UTILS_NW_TO_HOST_32(x) 40 #define LAC_MEM_RD_16(x) QAT_UTILS_NW_TO_HOST_16(x) 41 #else 42 #define LAC_MEM_WR_64(x) (x) 43 #define LAC_MEM_WR_32(x) (x) 44 #define LAC_MEM_WR_16(x) (x) 45 #define LAC_MEM_RD_64(x) (x) 46 #define LAC_MEM_RD_32(x) (x) 47 #define LAC_MEM_RD_16(x) (x) 48 #endif 49 50 /* 51 ******************************************************************************* 52 * Shared Memory Macros (memory accessible by Acceleration Engines, e.g. QAT) 53 ******************************************************************************* 54 */ 55 56 /** 57 ******************************************************************************* 58 * @ingroup LacMem 59 * This macro can be used to write to a variable that will be read by the 60 * QAT. The macro will automatically detect the size of the target variable and 61 * will select the correct method for performing the write. The data is cast to 62 * the type of the field that it will be written to. 63 * This macro swaps data if required. 64 * 65 * @param[out] var The variable to be written. Can be a field of a struct. 66 * 67 * @param[in] data The value to be written. Will be cast to the size of the 68 * target. 69 * 70 * @retval none 71 ******************************************************************************/ 72 #define LAC_MEM_SHARED_WRITE_SWAP(var, data) \ 73 do { \ 74 switch (sizeof(var)) { \ 75 case 1: \ 76 (var) = (Cpa8U)(data); \ 77 break; \ 78 case 2: \ 79 (var) = (Cpa16U)(data); \ 80 (var) = LAC_MEM_WR_16(((Cpa16U)var)); \ 81 break; \ 82 case 4: \ 83 (var) = (Cpa32U)(data); \ 84 (var) = LAC_MEM_WR_32(((Cpa32U)var)); \ 85 break; \ 86 case 8: \ 87 (var) = (Cpa64U)(data); \ 88 (var) = LAC_MEM_WR_64(((Cpa64U)var)); \ 89 break; \ 90 default: \ 91 break; \ 92 } \ 93 } while (0) 94 95 /** 96 ******************************************************************************* 97 * @ingroup LacMem 98 * This macro can be used to read a variable that was written by the QAT. 99 * The macro will automatically detect the size of the data to be read and will 100 * select the correct method for performing the read. The value read from the 101 * variable is cast to the size of the data type it will be stored in. 102 * This macro swaps data if required. 103 * 104 * @param[in] var The variable to be read. Can be a field of a struct. 105 * 106 * @param[out] data The variable to hold the result of the read. Data read 107 * will be cast to the size of this variable 108 * 109 * @retval none 110 ******************************************************************************/ 111 #define LAC_MEM_SHARED_READ_SWAP(var, data) \ 112 do { \ 113 switch (sizeof(var)) { \ 114 case 1: \ 115 (data) = (var); \ 116 break; \ 117 case 2: \ 118 (data) = LAC_MEM_RD_16(((Cpa16U)var)); \ 119 break; \ 120 case 4: \ 121 (data) = LAC_MEM_RD_32(((Cpa32U)var)); \ 122 break; \ 123 case 8: \ 124 (data) = LAC_MEM_RD_64(((Cpa64U)var)); \ 125 break; \ 126 default: \ 127 break; \ 128 } \ 129 } while (0) 130 131 /** 132 ******************************************************************************* 133 * @ingroup LacMem 134 * This macro can be used to write a pointer to a QAT request. The fields 135 * for pointers in the QAT request and response messages are always 64 bits 136 * 137 * @param[out] var The variable to be written to. Can be a field of a struct. 138 * 139 * @param[in] data The value to be written. Will be cast to size of target 140 * variable 141 * 142 * @retval none 143 ******************************************************************************/ 144 /* cast pointer to scalar of same size of the native pointer */ 145 #define LAC_MEM_SHARED_WRITE_FROM_PTR(var, data) \ 146 ((var) = (Cpa64U)(LAC_ARCH_UINT)(data)) 147 148 /* Note: any changes to this macro implementation should also be made to the 149 * similar LAC_MEM_CAST_PTR_TO_UINT64 macro 150 */ 151 152 /** 153 ******************************************************************************* 154 * @ingroup LacMem 155 * This macro can be used to read a pointer from a QAT response. The fields 156 * for pointers in the QAT request and response messages are always 64 bits 157 * 158 * @param[in] var The variable to be read. Can be a field of a struct. 159 * 160 * @param[out] data The variable to hold the result of the read. Data read 161 * will be cast to the size of this variable 162 * 163 * @retval none 164 ******************************************************************************/ 165 /* Cast back to native pointer */ 166 #define LAC_MEM_SHARED_READ_TO_PTR(var, data) \ 167 ((data) = (void *)(LAC_ARCH_UINT)(var)) 168 169 /** 170 ******************************************************************************* 171 * @ingroup LacMem 172 * This macro safely casts a pointer to a Cpa64U type. 173 * 174 * @param[in] pPtr The pointer to be cast. 175 * 176 * @retval pointer cast to Cpa64U 177 ******************************************************************************/ 178 #define LAC_MEM_CAST_PTR_TO_UINT64(pPtr) ((Cpa64U)(pPtr)) 179 180 /** 181 ******************************************************************************* 182 * @ingroup LacMem 183 * This macro uses an QAT Utils macro to convert from a virtual address to 184 *a 185 * physical address for internally allocated memory. 186 * 187 * @param[in] pVirtAddr The address to be converted. 188 * 189 * @retval The converted physical address 190 ******************************************************************************/ 191 #define LAC_OS_VIRT_TO_PHYS_INTERNAL(pVirtAddr) \ 192 (QAT_UTILS_MMU_VIRT_TO_PHYS(pVirtAddr)) 193 194 /** 195 ******************************************************************************* 196 * @ingroup LacMem 197 * This macro should be called on all externally allocated memory it calls 198 * SalMem_virt2PhysExternal function which allows a user 199 * to set the virt2phys function used by an instance. 200 * Defaults to virt to phys for kernel. 201 * 202 * @param[in] genService Generic sal_service_t structure. 203 * @param[in] pVirtAddr The address to be converted. 204 * 205 * @retval The converted physical address 206 ******************************************************************************/ 207 #define LAC_OS_VIRT_TO_PHYS_EXTERNAL(genService, pVirtAddr) \ 208 ((SalMem_virt2PhysExternal(pVirtAddr, &(genService)))) 209 210 /** 211 ******************************************************************************* 212 * @ingroup LacMem 213 * This macro can be used to write an address variable that will be read by 214 * the QAT. The macro will perform the necessary virt2phys address translation 215 * This macro is only to be called on memory allocated internally by the driver. 216 * 217 * @param[out] var The address variable to write. Can be a field of a struct. 218 * 219 * @param[in] pPtr The pointer variable to containing the address to be 220 * written 221 * 222 * @retval none 223 ******************************************************************************/ 224 #define LAC_MEM_SHARED_WRITE_VIRT_TO_PHYS_PTR_INTERNAL(var, pPtr) \ 225 do { \ 226 Cpa64U physAddr = 0; \ 227 physAddr = LAC_MEM_CAST_PTR_TO_UINT64( \ 228 LAC_OS_VIRT_TO_PHYS_INTERNAL(pPtr)); \ 229 var = physAddr; \ 230 } while (0) 231 232 /** 233 ******************************************************************************* 234 * @ingroup LacMem 235 * This macro can be used to write an address variable that will be read by 236 * the QAT. The macro will perform the necessary virt2phys address translation 237 * This macro is to be used on memory allocated externally by the user. It calls 238 * the user supplied virt2phys address translation. 239 * 240 * @param[in] pService The pointer to the service 241 * @param[out] var The address variable to write. Can be a field of a struct 242 * @param[in] pPtr The pointer variable to containing the address to be 243 * written 244 * 245 * @retval none 246 ******************************************************************************/ 247 #define LAC_MEM_SHARED_WRITE_VIRT_TO_PHYS_PTR_EXTERNAL(pService, var, pPtr) \ 248 do { \ 249 Cpa64U physAddr = 0; \ 250 physAddr = LAC_MEM_CAST_PTR_TO_UINT64( \ 251 LAC_OS_VIRT_TO_PHYS_EXTERNAL(pService, pPtr)); \ 252 var = physAddr; \ 253 } while (0) 254 255 /* 256 ******************************************************************************* 257 * OS Memory Macros 258 ******************************************************************************* 259 */ 260 261 /** 262 ******************************************************************************* 263 * @ingroup LacMem 264 * This function and associated macro allocates the memory for the given 265 * size and stores the address of the memory allocated in the pointer. 266 * 267 * @param[out] ppMemAddr address of pointer where address will be stored 268 * @param[in] sizeBytes the size of the memory to be allocated. 269 * 270 * @retval CPA_STATUS_RESOURCE Macro failed to allocate Memory 271 * @retval CPA_STATUS_SUCCESS Macro executed successfully 272 * 273 ******************************************************************************/ 274 static __inline CpaStatus 275 LacMem_OsMemAlloc(void **ppMemAddr, Cpa32U sizeBytes) 276 { 277 *ppMemAddr = malloc(sizeBytes, M_QAT, M_WAITOK); 278 279 return CPA_STATUS_SUCCESS; 280 } 281 282 /** 283 ******************************************************************************* 284 * @ingroup LacMem 285 * This function and associated macro allocates the contiguous 286 * memory for the given 287 * size and stores the address of the memory allocated in the pointer. 288 * 289 * @param[out] ppMemAddr address of pointer where address will be stored 290 * @param[in] sizeBytes the size of the memory to be allocated. 291 * @param[in] alignmentBytes the alignment 292 * @param[in] node node to allocate from 293 * 294 * @retval CPA_STATUS_RESOURCE Macro failed to allocate Memory 295 * @retval CPA_STATUS_SUCCESS Macro executed successfully 296 * 297 ******************************************************************************/ 298 static __inline CpaStatus 299 LacMem_OsContigAlignMemAlloc(void **ppMemAddr, 300 Cpa32U sizeBytes, 301 Cpa32U alignmentBytes, 302 Cpa32U node) 303 { 304 if ((alignmentBytes & (alignmentBytes - 1)) != 305 0) /* if is not power of 2 */ 306 { 307 *ppMemAddr = NULL; 308 QAT_UTILS_LOG("alignmentBytes MUST be the power of 2\n"); 309 return CPA_STATUS_INVALID_PARAM; 310 } 311 312 *ppMemAddr = 313 qatUtilsMemAllocContiguousNUMA(sizeBytes, node, alignmentBytes); 314 315 if (NULL == *ppMemAddr) { 316 return CPA_STATUS_RESOURCE; 317 } 318 319 return CPA_STATUS_SUCCESS; 320 } 321 322 /** 323 ******************************************************************************* 324 * @ingroup LacMem 325 * Macro from the malloc() function 326 * 327 ******************************************************************************/ 328 #define LAC_OS_MALLOC(sizeBytes) malloc(sizeBytes, M_QAT, M_WAITOK) 329 330 /** 331 ******************************************************************************* 332 * @ingroup LacMem 333 * Macro from the LacMem_OsContigAlignMemAlloc function 334 * 335 ******************************************************************************/ 336 #define LAC_OS_CAMALLOC(ppMemAddr, sizeBytes, alignmentBytes, node) \ 337 LacMem_OsContigAlignMemAlloc((void *)ppMemAddr, \ 338 sizeBytes, \ 339 alignmentBytes, \ 340 node) 341 342 /** 343 ******************************************************************************* 344 * @ingroup LacMem 345 * Macro for declaration static const unsigned int constant. One provides 346 * the compilation time computation with the highest bit set in the 347 * sizeof(TYPE) value. The constant is being put by the linker by default in 348 * .rodata section 349 * 350 * E.g. Statement LAC_DECLARE_HIGHEST_BIT_OF(lac_mem_blk_t) 351 * results in following entry: 352 * static const unsigned int highest_bit_of_lac_mem_blk_t = 3 353 * 354 * CAUTION!! 355 * Macro is prepared only for type names NOT-containing ANY 356 * special characters. Types as amongst others: 357 * - void * 358 * - unsigned long 359 * - unsigned int 360 * are strictly forbidden and will result in compilation error. 361 * Use typedef to provide one-word type name for MACRO's usage. 362 ******************************************************************************/ 363 #define LAC_DECLARE_HIGHEST_BIT_OF(TYPE) \ 364 static const unsigned int highest_bit_of_##TYPE = \ 365 (sizeof(TYPE) & 0x80000000 ? 31 : (sizeof(TYPE) & 0x40000000 ? 30 : (sizeof(TYPE) & 0x20000000 ? 29 : ( \ 366 sizeof(TYPE) & 0x10000000 ? 28 : ( \ 367 sizeof(TYPE) & 0x08000000 ? 27 : ( \ 368 sizeof(TYPE) & 0x04000000 ? 26 : ( \ 369 sizeof(TYPE) & 0x02000000 ? 25 : ( \ 370 sizeof(TYPE) & 0x01000000 ? 24 : ( \ 371 sizeof(TYPE) & 0x00800000 ? \ 372 23 : \ 373 (sizeof(TYPE) & 0x00400000 ? 22 : ( \ 374 sizeof( \ 375 TYPE) & \ 376 0x00200000 ? \ 377 21 : \ 378 ( \ 379 sizeof(TYPE) & 0x00100000 ? 20 : (sizeof(TYPE) & 0x00080000 ? 19 : ( \ 380 sizeof( \ 381 TYPE) & \ 382 0x00040000 ? \ 383 18 : \ 384 ( \ 385 sizeof(TYPE) & 0x00020000 ? 17 : ( \ 386 sizeof(TYPE) & 0x00010000 ? 16 : (sizeof(TYPE) & \ 387 0x00008000 ? \ 388 15 : \ 389 (sizeof(TYPE) & 0x00004000 ? 14 : ( \ 390 sizeof(TYPE) & 0x00002000 ? 13 : \ 391 ( \ 392 sizeof(TYPE) & 0x00001000 ? 12 : ( \ 393 sizeof(TYPE) & 0x00000800 ? 11 : ( \ 394 sizeof(TYPE) & 0x00000400 ? 10 : \ 395 ( \ 396 sizeof(TYPE) & \ 397 0x00000200 ? \ 398 9 : \ 399 (sizeof( \ 400 TYPE) & \ 401 0x00000100 ? \ 402 8 : \ 403 (sizeof(TYPE) & 0x00000080 ? 7 : \ 404 ( \ 405 sizeof(TYPE) & 0x00000040 ? \ 406 6 : \ 407 ( \ 408 sizeof(TYPE) & 0x00000020 ? 5 : \ 409 ( \ 410 sizeof(TYPE) & 0x00000010 ? 4 : \ 411 ( \ 412 sizeof(TYPE) & 0x00000008 ? 3 : \ 413 ( \ 414 sizeof(TYPE) & 0x00000004 ? 2 : \ 415 ( \ 416 sizeof(TYPE) & 0x00000002 ? 1 : ( \ 417 sizeof(TYPE) & 0x00000001 ? 0 : 32))))))))))))))))) /*16*/))))))))))))))) /* 31 */ 418 419 /** 420 ******************************************************************************* 421 * @ingroup LacMem 422 * This function and associated macro frees the memory at the given address 423 * and resets the pointer to NULL 424 * 425 * @param[out] ppMemAddr address of pointer where mem address is stored. 426 * If pointer is NULL, the function will exit silently 427 * 428 * @retval void 429 * 430 ******************************************************************************/ 431 static __inline void 432 LacMem_OsMemFree(void **ppMemAddr) 433 { 434 free(*ppMemAddr, M_QAT); 435 *ppMemAddr = NULL; 436 } 437 438 /** 439 ******************************************************************************* 440 * @ingroup LacMem 441 * This function and associated macro frees the contiguous memory at the 442 * given address and resets the pointer to NULL 443 * 444 * @param[out] ppMemAddr address of pointer where mem address is stored. 445 * If pointer is NULL, the function will exit silently 446 * 447 * @retval void 448 * 449 ******************************************************************************/ 450 static __inline void 451 LacMem_OsContigAlignMemFree(void **ppMemAddr) 452 { 453 if (NULL != *ppMemAddr) { 454 qatUtilsMemFreeNUMA(*ppMemAddr); 455 *ppMemAddr = NULL; 456 } 457 } 458 459 #define LAC_OS_FREE(pMemAddr) LacMem_OsMemFree((void *)&pMemAddr) 460 461 #define LAC_OS_CAFREE(pMemAddr) LacMem_OsContigAlignMemFree((void *)&pMemAddr) 462 463 /** 464 ******************************************************************************* 465 * @ingroup LacMem 466 * Copies user data to a working buffer of the correct size (required by 467 * PKE services) 468 * 469 * @description 470 * This function produces a correctly sized working buffer from the input 471 * user buffer. If the original buffer is too small a new buffer shall 472 * be allocated and memory is copied (left padded with zeros to the 473 *required 474 * length). 475 * 476 * The returned working buffer is guaranteed to be of the desired size for 477 * QAT. 478 * 479 * When this function is called pInternalMem describes the user_buffer and 480 * when the function returns pInternalMem describes the working buffer. 481 * This is because pInternalMem describes the memory that will be sent to 482 * QAT. 483 * 484 * The caller must keep the original buffer pointer. The alllocated buffer 485 *is 486 * freed (as necessary) using icp_LacBufferRestore(). 487 * 488 * @param[in] instanceHandle Handle to crypto instance so pke_resize mem pool 489 *can 490 * be located 491 * @param[in] pUserBuffer Pointer on the user buffer 492 * @param[in] userLen length of the user buffer 493 * @param[in] workingLen length of the working (correctly sized) buffer 494 * @param[in/out] pInternalMem pointer to boolean if TRUE on input then 495 * user_buffer is internally allocated memory 496 * if false then it is externally allocated. 497 * This value gets updated by the function 498 * if the returned pointer references internally 499 * allocated memory. 500 * 501 * @return a pointer to the working (correctly sized) buffer or NULL if the 502 * allocation failed 503 * 504 * @note the working length cannot be smaller than the user buffer length 505 * 506 * @warning the working buffer may be the same or different from the original 507 * user buffer; the caller should make no assumptions in this regard 508 * 509 * @see icp_LacBufferRestore() 510 * 511 ******************************************************************************/ 512 Cpa8U *icp_LacBufferResize(CpaInstanceHandle instanceHandle, 513 Cpa8U *pUserBuffer, 514 Cpa32U userLen, 515 Cpa32U workingLen, 516 CpaBoolean *pInternalMemory); 517 518 /** 519 ******************************************************************************* 520 * @ingroup LacMem 521 * Restores a user buffer 522 * 523 * @description 524 * This function restores a user buffer and releases its 525 * corresponding working buffer. The working buffer, assumed to be 526 * previously obtained using icp_LacBufferResize(), is freed as necessary. 527 * 528 * The contents are copied in the process. 529 * 530 * @note the working length cannot be smaller than the user buffer length 531 * 532 * @param[out] pUserBuffer Pointer on the user buffer 533 * @param[in] userLen length of the user buffer 534 * @param[in] pWorkingBuffer Pointer on the working buffer 535 * @param[in] workingLen working buffer length 536 * @param[in] copyBuf if set _TRUE the data in the workingBuffer 537 * will be copied to the userBuffer before the 538 * workingBuffer is freed. 539 * 540 * @return the status of the operation 541 * 542 * @see icp_LacBufferResize() 543 * 544 ******************************************************************************/ 545 CpaStatus icp_LacBufferRestore(Cpa8U *pUserBuffer, 546 Cpa32U userLen, 547 Cpa8U *pWorkingBuffer, 548 Cpa32U workingLen, 549 CpaBoolean copyBuf); 550 551 /** 552 ******************************************************************************* 553 * @ingroup LacMem 554 * Uses an instance specific user supplied virt2phys function to convert a 555 * virtual address to a physical address. 556 * 557 * @description 558 * Uses an instance specific user supplied virt2phys function to convert a 559 * virtual address to a physical address. A client of QA API can set the 560 * virt2phys function for an instance by using the 561 * cpaXxSetAddressTranslation() function. If the client does not set the 562 * virt2phys function and the instance is in kernel space then OS specific 563 * virt2phys function will be used. In user space the virt2phys function 564 * MUST be set by the user. 565 * 566 * @param[in] pVirtAddr the virtual addr to be converted 567 * @param[in] pServiceGen Pointer on the sal_service_t structure 568 * so client supplied virt2phys function can be 569 * called. 570 * 571 * @return the physical address 572 * 573 ******************************************************************************/ 574 CpaPhysicalAddr SalMem_virt2PhysExternal(void *pVirtAddr, void *pServiceGen); 575 576 #endif /* LAC_MEM_H */ 577