/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
/**
***************************************************************************
* @file lac_mem.h
*
* @defgroup LacMem Memory
*
* @ingroup LacCommon
*
* Memory re-sizing functions and memory accessor macros.
*
***************************************************************************/
#ifndef LAC_MEM_H
#define LAC_MEM_H
/***************************************************************************
* Include header files
***************************************************************************/
#include "cpa.h"
#include "qat_utils.h"
#include "lac_common.h"
/**
*******************************************************************************
* @ingroup LacMem
* These macros are used to Endian swap variables from IA to QAT.
*
* @param[out] x The variable to be swapped.
*
* @retval none
******************************************************************************/
#if (BYTE_ORDER == LITTLE_ENDIAN)
#define LAC_MEM_WR_64(x) QAT_UTILS_HOST_TO_NW_64(x)
#define LAC_MEM_WR_32(x) QAT_UTILS_HOST_TO_NW_32(x)
#define LAC_MEM_WR_16(x) QAT_UTILS_HOST_TO_NW_16(x)
#define LAC_MEM_RD_64(x) QAT_UTILS_NW_TO_HOST_64(x)
#define LAC_MEM_RD_32(x) QAT_UTILS_NW_TO_HOST_32(x)
#define LAC_MEM_RD_16(x) QAT_UTILS_NW_TO_HOST_16(x)
#else
#define LAC_MEM_WR_64(x) (x)
#define LAC_MEM_WR_32(x) (x)
#define LAC_MEM_WR_16(x) (x)
#define LAC_MEM_RD_64(x) (x)
#define LAC_MEM_RD_32(x) (x)
#define LAC_MEM_RD_16(x) (x)
#endif
/*
*******************************************************************************
* Shared Memory Macros (memory accessible by Acceleration Engines, e.g. QAT)
*******************************************************************************
*/
/**
*******************************************************************************
* @ingroup LacMem
* This macro can be used to write to a variable that will be read by the
* QAT. The macro will automatically detect the size of the target variable and
* will select the correct method for performing the write. The data is cast to
* the type of the field that it will be written to.
* This macro swaps data if required.
*
* @param[out] var The variable to be written. Can be a field of a struct.
*
* @param[in] data The value to be written. Will be cast to the size of the
* target.
*
* @retval none
******************************************************************************/
#define LAC_MEM_SHARED_WRITE_SWAP(var, data) \
do { \
switch (sizeof(var)) { \
case 1: \
(var) = (Cpa8U)(data); \
break; \
case 2: \
(var) = (Cpa16U)(data); \
(var) = LAC_MEM_WR_16(((Cpa16U)var)); \
break; \
case 4: \
(var) = (Cpa32U)(data); \
(var) = LAC_MEM_WR_32(((Cpa32U)var)); \
break; \
case 8: \
(var) = (Cpa64U)(data); \
(var) = LAC_MEM_WR_64(((Cpa64U)var)); \
break; \
default: \
break; \
} \
} while (0)
/**
*******************************************************************************
* @ingroup LacMem
* This macro can be used to read a variable that was written by the QAT.
* The macro will automatically detect the size of the data to be read and will
* select the correct method for performing the read. The value read from the
* variable is cast to the size of the data type it will be stored in.
* This macro swaps data if required.
*
* @param[in] var The variable to be read. Can be a field of a struct.
*
* @param[out] data The variable to hold the result of the read. Data read
* will be cast to the size of this variable
*
* @retval none
******************************************************************************/
#define LAC_MEM_SHARED_READ_SWAP(var, data) \
do { \
switch (sizeof(var)) { \
case 1: \
(data) = (var); \
break; \
case 2: \
(data) = LAC_MEM_RD_16(((Cpa16U)var)); \
break; \
case 4: \
(data) = LAC_MEM_RD_32(((Cpa32U)var)); \
break; \
case 8: \
(data) = LAC_MEM_RD_64(((Cpa64U)var)); \
break; \
default: \
break; \
} \
} while (0)
/**
*******************************************************************************
* @ingroup LacMem
* This macro can be used to write a pointer to a QAT request. The fields
* for pointers in the QAT request and response messages are always 64 bits
*
* @param[out] var The variable to be written to. Can be a field of a struct.
*
* @param[in] data The value to be written. Will be cast to size of target
* variable
*
* @retval none
******************************************************************************/
/* cast pointer to scalar of same size of the native pointer */
#define LAC_MEM_SHARED_WRITE_FROM_PTR(var, data) \
((var) = (Cpa64U)(LAC_ARCH_UINT)(data))
/* Note: any changes to this macro implementation should also be made to the
* similar LAC_MEM_CAST_PTR_TO_UINT64 macro
*/
/**
*******************************************************************************
* @ingroup LacMem
* This macro can be used to read a pointer from a QAT response. The fields
* for pointers in the QAT request and response messages are always 64 bits
*
* @param[in] var The variable to be read. Can be a field of a struct.
*
* @param[out] data The variable to hold the result of the read. Data read
* will be cast to the size of this variable
*
* @retval none
******************************************************************************/
/* Cast back to native pointer */
#define LAC_MEM_SHARED_READ_TO_PTR(var, data) \
((data) = (void *)(LAC_ARCH_UINT)(var))
/**
*******************************************************************************
* @ingroup LacMem
* This macro safely casts a pointer to a Cpa64U type.
*
* @param[in] pPtr The pointer to be cast.
*
* @retval pointer cast to Cpa64U
******************************************************************************/
#define LAC_MEM_CAST_PTR_TO_UINT64(pPtr) ((Cpa64U)(pPtr))
/**
*******************************************************************************
* @ingroup LacMem
* This macro uses an QAT Utils macro to convert from a virtual address to
*a
* physical address for internally allocated memory.
*
* @param[in] pVirtAddr The address to be converted.
*
* @retval The converted physical address
******************************************************************************/
#define LAC_OS_VIRT_TO_PHYS_INTERNAL(pVirtAddr) \
(QAT_UTILS_MMU_VIRT_TO_PHYS(pVirtAddr))
/**
*******************************************************************************
* @ingroup LacMem
* This macro should be called on all externally allocated memory it calls
* SalMem_virt2PhysExternal function which allows a user
* to set the virt2phys function used by an instance.
* Defaults to virt to phys for kernel.
*
* @param[in] genService Generic sal_service_t structure.
* @param[in] pVirtAddr The address to be converted.
*
* @retval The converted physical address
******************************************************************************/
#define LAC_OS_VIRT_TO_PHYS_EXTERNAL(genService, pVirtAddr) \
((SalMem_virt2PhysExternal(pVirtAddr, &(genService))))
/**
*******************************************************************************
* @ingroup LacMem
* This macro can be used to write an address variable that will be read by
* the QAT. The macro will perform the necessary virt2phys address translation
* This macro is only to be called on memory allocated internally by the driver.
*
* @param[out] var The address variable to write. Can be a field of a struct.
*
* @param[in] pPtr The pointer variable to containing the address to be
* written
*
* @retval none
******************************************************************************/
#define LAC_MEM_SHARED_WRITE_VIRT_TO_PHYS_PTR_INTERNAL(var, pPtr) \
do { \
Cpa64U physAddr = 0; \
physAddr = LAC_MEM_CAST_PTR_TO_UINT64( \
LAC_OS_VIRT_TO_PHYS_INTERNAL(pPtr)); \
var = physAddr; \
} while (0)
/**
*******************************************************************************
* @ingroup LacMem
* This macro can be used to write an address variable that will be read by
* the QAT. The macro will perform the necessary virt2phys address translation
* This macro is to be used on memory allocated externally by the user. It calls
* the user supplied virt2phys address translation.
*
* @param[in] pService The pointer to the service
* @param[out] var The address variable to write. Can be a field of a struct
* @param[in] pPtr The pointer variable to containing the address to be
* written
*
* @retval none
******************************************************************************/
#define LAC_MEM_SHARED_WRITE_VIRT_TO_PHYS_PTR_EXTERNAL(pService, var, pPtr) \
do { \
Cpa64U physAddr = 0; \
physAddr = LAC_MEM_CAST_PTR_TO_UINT64( \
LAC_OS_VIRT_TO_PHYS_EXTERNAL(pService, pPtr)); \
var = physAddr; \
} while (0)
/*
*******************************************************************************
* OS Memory Macros
*******************************************************************************
*/
/**
*******************************************************************************
* @ingroup LacMem
* This function and associated macro allocates the memory for the given
* size and stores the address of the memory allocated in the pointer.
*
* @param[out] ppMemAddr address of pointer where address will be stored
* @param[in] sizeBytes the size of the memory to be allocated.
*
* @retval CPA_STATUS_RESOURCE Macro failed to allocate Memory
* @retval CPA_STATUS_SUCCESS Macro executed successfully
*
******************************************************************************/
static __inline CpaStatus
LacMem_OsMemAlloc(void **ppMemAddr, Cpa32U sizeBytes)
{
*ppMemAddr = malloc(sizeBytes, M_QAT, M_WAITOK);
return CPA_STATUS_SUCCESS;
}
/**
*******************************************************************************
* @ingroup LacMem
* This function and associated macro allocates the contiguous
* memory for the given
* size and stores the address of the memory allocated in the pointer.
*
* @param[out] ppMemAddr address of pointer where address will be stored
* @param[in] sizeBytes the size of the memory to be allocated.
* @param[in] alignmentBytes the alignment
* @param[in] node node to allocate from
*
* @retval CPA_STATUS_RESOURCE Macro failed to allocate Memory
* @retval CPA_STATUS_SUCCESS Macro executed successfully
*
******************************************************************************/
static __inline CpaStatus
LacMem_OsContigAlignMemAlloc(void **ppMemAddr,
Cpa32U sizeBytes,
Cpa32U alignmentBytes,
Cpa32U node)
{
if ((alignmentBytes & (alignmentBytes - 1)) !=
0) /* if is not power of 2 */
{
*ppMemAddr = NULL;
QAT_UTILS_LOG("alignmentBytes MUST be the power of 2\n");
return CPA_STATUS_INVALID_PARAM;
}
*ppMemAddr =
qatUtilsMemAllocContiguousNUMA(sizeBytes, node, alignmentBytes);
if (NULL == *ppMemAddr) {
return CPA_STATUS_RESOURCE;
}
return CPA_STATUS_SUCCESS;
}
/**
*******************************************************************************
* @ingroup LacMem
* Macro from the malloc() function
*
******************************************************************************/
#define LAC_OS_MALLOC(sizeBytes) malloc(sizeBytes, M_QAT, M_WAITOK)
/**
*******************************************************************************
* @ingroup LacMem
* Macro from the LacMem_OsContigAlignMemAlloc function
*
******************************************************************************/
#define LAC_OS_CAMALLOC(ppMemAddr, sizeBytes, alignmentBytes, node) \
LacMem_OsContigAlignMemAlloc((void *)ppMemAddr, \
sizeBytes, \
alignmentBytes, \
node)
/**
*******************************************************************************
* @ingroup LacMem
* Macro for declaration static const unsigned int constant. One provides
* the compilation time computation with the highest bit set in the
* sizeof(TYPE) value. The constant is being put by the linker by default in
* .rodata section
*
* E.g. Statement LAC_DECLARE_HIGHEST_BIT_OF(lac_mem_blk_t)
* results in following entry:
* static const unsigned int highest_bit_of_lac_mem_blk_t = 3
*
* CAUTION!!
* Macro is prepared only for type names NOT-containing ANY
* special characters. Types as amongst others:
* - void *
* - unsigned long
* - unsigned int
* are strictly forbidden and will result in compilation error.
* Use typedef to provide one-word type name for MACRO's usage.
******************************************************************************/
#define LAC_DECLARE_HIGHEST_BIT_OF(TYPE) \
static const unsigned int highest_bit_of_##TYPE = \
(sizeof(TYPE) & 0x80000000 ? 31 : (sizeof(TYPE) & 0x40000000 ? 30 : (sizeof(TYPE) & 0x20000000 ? 29 : ( \
sizeof(TYPE) & 0x10000000 ? 28 : ( \
sizeof(TYPE) & 0x08000000 ? 27 : ( \
sizeof(TYPE) & 0x04000000 ? 26 : ( \
sizeof(TYPE) & 0x02000000 ? 25 : ( \
sizeof(TYPE) & 0x01000000 ? 24 : ( \
sizeof(TYPE) & 0x00800000 ? \
23 : \
(sizeof(TYPE) & 0x00400000 ? 22 : ( \
sizeof( \
TYPE) & \
0x00200000 ? \
21 : \
( \
sizeof(TYPE) & 0x00100000 ? 20 : (sizeof(TYPE) & 0x00080000 ? 19 : ( \
sizeof( \
TYPE) & \
0x00040000 ? \
18 : \
( \
sizeof(TYPE) & 0x00020000 ? 17 : ( \
sizeof(TYPE) & 0x00010000 ? 16 : (sizeof(TYPE) & \
0x00008000 ? \
15 : \
(sizeof(TYPE) & 0x00004000 ? 14 : ( \
sizeof(TYPE) & 0x00002000 ? 13 : \
( \
sizeof(TYPE) & 0x00001000 ? 12 : ( \
sizeof(TYPE) & 0x00000800 ? 11 : ( \
sizeof(TYPE) & 0x00000400 ? 10 : \
( \
sizeof(TYPE) & \
0x00000200 ? \
9 : \
(sizeof( \
TYPE) & \
0x00000100 ? \
8 : \
(sizeof(TYPE) & 0x00000080 ? 7 : \
( \
sizeof(TYPE) & 0x00000040 ? \
6 : \
( \
sizeof(TYPE) & 0x00000020 ? 5 : \
( \
sizeof(TYPE) & 0x00000010 ? 4 : \
( \
sizeof(TYPE) & 0x00000008 ? 3 : \
( \
sizeof(TYPE) & 0x00000004 ? 2 : \
( \
sizeof(TYPE) & 0x00000002 ? 1 : ( \
sizeof(TYPE) & 0x00000001 ? 0 : 32))))))))))))))))) /*16*/))))))))))))))) /* 31 */
/**
*******************************************************************************
* @ingroup LacMem
* This function and associated macro frees the memory at the given address
* and resets the pointer to NULL
*
* @param[out] ppMemAddr address of pointer where mem address is stored.
* If pointer is NULL, the function will exit silently
*
* @retval void
*
******************************************************************************/
static __inline void
LacMem_OsMemFree(void **ppMemAddr)
{
free(*ppMemAddr, M_QAT);
*ppMemAddr = NULL;
}
/**
*******************************************************************************
* @ingroup LacMem
* This function and associated macro frees the contiguous memory at the
* given address and resets the pointer to NULL
*
* @param[out] ppMemAddr address of pointer where mem address is stored.
* If pointer is NULL, the function will exit silently
*
* @retval void
*
******************************************************************************/
static __inline void
LacMem_OsContigAlignMemFree(void **ppMemAddr)
{
if (NULL != *ppMemAddr) {
qatUtilsMemFreeNUMA(*ppMemAddr);
*ppMemAddr = NULL;
}
}
#define LAC_OS_FREE(pMemAddr) LacMem_OsMemFree((void *)&pMemAddr)
#define LAC_OS_CAFREE(pMemAddr) LacMem_OsContigAlignMemFree((void *)&pMemAddr)
/**
*******************************************************************************
* @ingroup LacMem
* Copies user data to a working buffer of the correct size (required by
* PKE services)
*
* @description
* This function produces a correctly sized working buffer from the input
* user buffer. If the original buffer is too small a new buffer shall
* be allocated and memory is copied (left padded with zeros to the
*required
* length).
*
* The returned working buffer is guaranteed to be of the desired size for
* QAT.
*
* When this function is called pInternalMem describes the user_buffer and
* when the function returns pInternalMem describes the working buffer.
* This is because pInternalMem describes the memory that will be sent to
* QAT.
*
* The caller must keep the original buffer pointer. The alllocated buffer
*is
* freed (as necessary) using icp_LacBufferRestore().
*
* @param[in] instanceHandle Handle to crypto instance so pke_resize mem pool
*can
* be located
* @param[in] pUserBuffer Pointer on the user buffer
* @param[in] userLen length of the user buffer
* @param[in] workingLen length of the working (correctly sized) buffer
* @param[in/out] pInternalMem pointer to boolean if TRUE on input then
* user_buffer is internally allocated memory
* if false then it is externally allocated.
* This value gets updated by the function
* if the returned pointer references internally
* allocated memory.
*
* @return a pointer to the working (correctly sized) buffer or NULL if the
* allocation failed
*
* @note the working length cannot be smaller than the user buffer length
*
* @warning the working buffer may be the same or different from the original
* user buffer; the caller should make no assumptions in this regard
*
* @see icp_LacBufferRestore()
*
******************************************************************************/
Cpa8U *icp_LacBufferResize(CpaInstanceHandle instanceHandle,
Cpa8U *pUserBuffer,
Cpa32U userLen,
Cpa32U workingLen,
CpaBoolean *pInternalMemory);
/**
*******************************************************************************
* @ingroup LacMem
* Restores a user buffer
*
* @description
* This function restores a user buffer and releases its
* corresponding working buffer. The working buffer, assumed to be
* previously obtained using icp_LacBufferResize(), is freed as necessary.
*
* The contents are copied in the process.
*
* @note the working length cannot be smaller than the user buffer length
*
* @param[out] pUserBuffer Pointer on the user buffer
* @param[in] userLen length of the user buffer
* @param[in] pWorkingBuffer Pointer on the working buffer
* @param[in] workingLen working buffer length
* @param[in] copyBuf if set _TRUE the data in the workingBuffer
* will be copied to the userBuffer before the
* workingBuffer is freed.
*
* @return the status of the operation
*
* @see icp_LacBufferResize()
*
******************************************************************************/
CpaStatus icp_LacBufferRestore(Cpa8U *pUserBuffer,
Cpa32U userLen,
Cpa8U *pWorkingBuffer,
Cpa32U workingLen,
CpaBoolean copyBuf);
/**
*******************************************************************************
* @ingroup LacMem
* Uses an instance specific user supplied virt2phys function to convert a
* virtual address to a physical address.
*
* @description
* Uses an instance specific user supplied virt2phys function to convert a
* virtual address to a physical address. A client of QA API can set the
* virt2phys function for an instance by using the
* cpaXxSetAddressTranslation() function. If the client does not set the
* virt2phys function and the instance is in kernel space then OS specific
* virt2phys function will be used. In user space the virt2phys function
* MUST be set by the user.
*
* @param[in] pVirtAddr the virtual addr to be converted
* @param[in] pServiceGen Pointer on the sal_service_t structure
* so client supplied virt2phys function can be
* called.
*
* @return the physical address
*
******************************************************************************/
CpaPhysicalAddr SalMem_virt2PhysExternal(void *pVirtAddr, void *pServiceGen);
#endif /* LAC_MEM_H */