1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2022 Intel Corporation */
3 /**
4 ***************************************************************************
5 * @file lac_mem_pools.c
6 *
7 * @ingroup LacMemPool
8 *
9 * Memory Pool creation and mgmt function implementations
10 *
11 ***************************************************************************/
12
13 #include "cpa.h"
14 #include "qat_utils.h"
15 #include "icp_accel_devices.h"
16 #include "icp_adf_init.h"
17 #include "icp_adf_transport.h"
18 #include "icp_adf_debug.h"
19 #include "lac_lock_free_stack.h"
20 #include "lac_mem_pools.h"
21 #include "lac_mem.h"
22 #include "lac_common.h"
23 #include "cpa_dc.h"
24 #include "dc_session.h"
25 #include "dc_datapath.h"
26 #include "icp_qat_fw_comp.h"
27 #include "icp_buffer_desc.h"
28 #include "lac_sym.h"
29
30 #define LAC_MEM_POOLS_NUM_SUPPORTED 32000
31 /**< @ingroup LacMemPool
32 * Number of mem pools supported */
33
34 #define LAC_MEM_POOLS_NAME_SIZE 17
35 /**< @ingroup LacMemPool
36 * 16 bytes plus '\\0' terminator */
37
38 /**< @ingroup LacMemPool
39 * This structure is used to manage each pool created using this utility
40 * feature. The client will maintain a pointer (identifier) to the created
41 * structure per pool.
42 */
43 typedef struct lac_mem_pool_hdr_s {
44 lock_free_stack_t stack;
45 char poolName[LAC_MEM_POOLS_NAME_SIZE]; /*16 bytes of a pool name */
46 /**< up to 16 bytes of a pool name */
47 unsigned int numElementsInPool;
48 /**< number of elements in the Pool */
49 unsigned int blkSizeInBytes;
50 /**< Block size in bytes */
51 unsigned int blkAlignmentInBytes;
52 /**< block alignment in bytes */
53 lac_mem_blk_t **trackBlks;
54 /* An array of mem block pointers to track the allocated entries in pool
55 */
56 volatile size_t availBlks;
57 /* Number of blocks available for allocation in this pool */
58 } lac_mem_pool_hdr_t;
59
60 static lac_mem_pool_hdr_t *lac_mem_pools[LAC_MEM_POOLS_NUM_SUPPORTED] = {
61 NULL
62 };
63 /**< @ingroup LacMemPool
64 * Array of pointers to the mem pool header structure
65 */
66
67 LAC_DECLARE_HIGHEST_BIT_OF(lac_mem_blk_t);
68 /**< @ingroup LacMemPool
69 * local constant for quickening computation of additional space allocated
70 * for holding lac_mem_blk_t container-structure
71 */
72
73 /**
74 *******************************************************************************
75 * @ingroup LacMemPool
76 * This function cleans up a mem pool.
77 ******************************************************************************/
78 void Lac_MemPoolCleanUpInternal(lac_mem_pool_hdr_t *pPoolID);
79
80 static inline Cpa32U
Lac_MemPoolGetElementRealSize(Cpa32U blkSizeInBytes,Cpa32U blkAlignmentInBytes)81 Lac_MemPoolGetElementRealSize(Cpa32U blkSizeInBytes, Cpa32U blkAlignmentInBytes)
82 {
83 Cpa32U addSize = (blkAlignmentInBytes >= sizeof(lac_mem_blk_t) ?
84 blkAlignmentInBytes :
85 1 << (highest_bit_of_lac_mem_blk_t + 1));
86 return blkSizeInBytes + addSize;
87 }
88
89 CpaStatus
Lac_MemPoolCreate(lac_memory_pool_id_t * pPoolID,char * poolName,unsigned int numElementsInPool,unsigned int blkSizeInBytes,unsigned int blkAlignmentInBytes,CpaBoolean trackMemory,Cpa32U node)90 Lac_MemPoolCreate(lac_memory_pool_id_t *pPoolID,
91 char *poolName,
92 unsigned int numElementsInPool, /*Number of elements*/
93 unsigned int blkSizeInBytes, /*Block Size in bytes*/
94 unsigned int blkAlignmentInBytes, /*Block alignment (bytes)*/
95 CpaBoolean trackMemory,
96 Cpa32U node)
97 {
98 unsigned int poolSearch = 0;
99 unsigned int counter = 0;
100 lac_mem_blk_t *pMemBlkCurrent = NULL;
101
102 void *pMemBlk = NULL;
103
104 if (pPoolID == NULL) {
105 QAT_UTILS_LOG("Invalid Pool ID param\n");
106 return CPA_STATUS_INVALID_PARAM; /*Error*/
107 }
108
109 /* Find First available Pool return error otherwise */
110 while (lac_mem_pools[poolSearch] != NULL) {
111 poolSearch++;
112 if (LAC_MEM_POOLS_NUM_SUPPORTED == poolSearch) {
113 QAT_UTILS_LOG(
114 "No more memory pools available for allocation.\n");
115 return CPA_STATUS_FAIL;
116 }
117 }
118
119 /* Allocate a Pool header */
120 lac_mem_pools[poolSearch] = LAC_OS_MALLOC(sizeof(lac_mem_pool_hdr_t));
121 if (NULL == lac_mem_pools[poolSearch]) {
122 QAT_UTILS_LOG(
123 "Unable to allocate memory for creation of the pool.\n");
124 return CPA_STATUS_RESOURCE; /*Error*/
125 }
126 memset(lac_mem_pools[poolSearch], 0, sizeof(lac_mem_pool_hdr_t));
127
128 /* Copy in Pool Name */
129 if (poolName != NULL) {
130 snprintf(lac_mem_pools[poolSearch]->poolName,
131 LAC_MEM_POOLS_NAME_SIZE,
132 "%s",
133 poolName);
134 } else {
135 LAC_OS_FREE(lac_mem_pools[poolSearch]);
136 lac_mem_pools[poolSearch] = NULL;
137 QAT_UTILS_LOG("Invalid Pool Name pointer\n");
138 return CPA_STATUS_INVALID_PARAM; /*Error*/
139 }
140
141 /* Allocate table for tracking memory blocks */
142 if (CPA_TRUE == trackMemory) {
143 lac_mem_pools[poolSearch]->trackBlks = LAC_OS_MALLOC(
144 (sizeof(lac_mem_blk_t *) * numElementsInPool));
145 if (NULL == lac_mem_pools[poolSearch]->trackBlks) {
146 LAC_OS_FREE(lac_mem_pools[poolSearch]);
147 lac_mem_pools[poolSearch] = NULL;
148 QAT_UTILS_LOG(
149 "Unable to allocate memory for tracking memory blocks.\n");
150 return CPA_STATUS_RESOURCE; /*Error*/
151 }
152 } else {
153 lac_mem_pools[poolSearch]->trackBlks = NULL;
154 }
155
156 lac_mem_pools[poolSearch]->availBlks = 0;
157 lac_mem_pools[poolSearch]->stack = _init_stack();
158
159 /* Calculate alignment needed for allocation */
160 for (counter = 0; counter < numElementsInPool; counter++) {
161 CpaPhysicalAddr physAddr = 0;
162 /* realSize is computed for allocation of blkSize bytes +
163 additional
164 capacity for lac_mem_blk_t structure storage due to the some
165 OSes
166 (BSD) limitations for memory alignment to be power of 2;
167 sizeof(lac_mem_blk_t) is being round up to the closest power
168 of 2 -
169 optimised towards the least CPU overhead but at additional
170 memory
171 cost
172 */
173 Cpa32U realSize =
174 Lac_MemPoolGetElementRealSize(blkSizeInBytes,
175 blkAlignmentInBytes);
176 Cpa32U addSize = realSize - blkSizeInBytes;
177
178 if (CPA_STATUS_SUCCESS != LAC_OS_CAMALLOC(&pMemBlk,
179 realSize,
180 blkAlignmentInBytes,
181 node)) {
182 Lac_MemPoolCleanUpInternal(lac_mem_pools[poolSearch]);
183 lac_mem_pools[poolSearch] = NULL;
184 QAT_UTILS_LOG(
185 "Unable to allocate contiguous chunk of memory.\n");
186 return CPA_STATUS_RESOURCE;
187 }
188
189 /* Calcaulate various offsets */
190 physAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL(
191 (void *)((LAC_ARCH_UINT)pMemBlk + addSize));
192
193 /* physAddr is now already aligned to the greater power of 2:
194 blkAlignmentInBytes or sizeof(lac_mem_blk_t) round up
195 We safely put the structure right before the blkSize
196 real data block
197 */
198 pMemBlkCurrent =
199 (lac_mem_blk_t *)(((LAC_ARCH_UINT)(pMemBlk)) + addSize -
200 sizeof(lac_mem_blk_t));
201
202 pMemBlkCurrent->physDataPtr = physAddr;
203 pMemBlkCurrent->pMemAllocPtr = pMemBlk;
204 pMemBlkCurrent->pPoolID = lac_mem_pools[poolSearch];
205 pMemBlkCurrent->isInUse = CPA_FALSE;
206 pMemBlkCurrent->pNext = NULL;
207
208 push(&lac_mem_pools[poolSearch]->stack, pMemBlkCurrent);
209
210 /* Store allocated memory pointer */
211 if (lac_mem_pools[poolSearch]->trackBlks != NULL) {
212 (lac_mem_pools[poolSearch]->trackBlks[counter]) =
213 (lac_mem_blk_t *)pMemBlkCurrent;
214 }
215 __sync_add_and_fetch(&lac_mem_pools[poolSearch]->availBlks, 1);
216 (lac_mem_pools[poolSearch])->numElementsInPool = counter + 1;
217 }
218
219 /* Set Pool details in the header */
220 (lac_mem_pools[poolSearch])->blkSizeInBytes = blkSizeInBytes;
221 (lac_mem_pools[poolSearch])->blkAlignmentInBytes = blkAlignmentInBytes;
222 /* Set the Pool ID output parameter */
223 *pPoolID = (LAC_ARCH_UINT)(lac_mem_pools[poolSearch]);
224 /* Success */
225 return CPA_STATUS_SUCCESS;
226 }
227
228 void *
Lac_MemPoolEntryAlloc(lac_memory_pool_id_t poolID)229 Lac_MemPoolEntryAlloc(lac_memory_pool_id_t poolID)
230 {
231 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID;
232 lac_mem_blk_t *pMemBlkCurrent = NULL;
233
234 /* Explicitly removing NULL PoolID check for speed */
235 if (pPoolID == NULL) {
236 QAT_UTILS_LOG("Invalid Pool ID");
237 return NULL;
238 }
239
240 /* Remove block from pool */
241 pMemBlkCurrent = pop(&pPoolID->stack);
242 if (NULL == pMemBlkCurrent) {
243 return (void *)CPA_STATUS_RETRY;
244 }
245 __sync_sub_and_fetch(&pPoolID->availBlks, 1);
246 pMemBlkCurrent->isInUse = CPA_TRUE;
247 return (void *)((LAC_ARCH_UINT)(pMemBlkCurrent) +
248 sizeof(lac_mem_blk_t));
249 }
250
251 void
Lac_MemPoolEntryFree(void * pEntry)252 Lac_MemPoolEntryFree(void *pEntry)
253 {
254 lac_mem_blk_t *pMemBlk = NULL;
255
256 /* Explicitly NULL pointer check */
257 if (pEntry == NULL) {
258 QAT_UTILS_LOG("Memory Handle NULL");
259 return;
260 }
261
262 pMemBlk =
263 (lac_mem_blk_t *)((LAC_ARCH_UINT)pEntry - sizeof(lac_mem_blk_t));
264 pMemBlk->isInUse = CPA_FALSE;
265
266 push(&pMemBlk->pPoolID->stack, pMemBlk);
267 __sync_add_and_fetch(&pMemBlk->pPoolID->availBlks, 1);
268 }
269
270 void
Lac_MemPoolDestroy(lac_memory_pool_id_t poolID)271 Lac_MemPoolDestroy(lac_memory_pool_id_t poolID)
272 {
273 unsigned int poolSearch = 0;
274 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID;
275
276 if (pPoolID != NULL) {
277 /*Remove entry from table*/
278 while (lac_mem_pools[poolSearch] != pPoolID) {
279 poolSearch++;
280
281 if (LAC_MEM_POOLS_NUM_SUPPORTED == poolSearch) {
282 QAT_UTILS_LOG("Invalid Pool ID submitted.\n");
283 return;
284 }
285 }
286
287 lac_mem_pools[poolSearch] = NULL; /*Remove handle from pool*/
288
289 Lac_MemPoolCleanUpInternal(pPoolID);
290 }
291 }
292
293 void
Lac_MemPoolCleanUpInternal(lac_mem_pool_hdr_t * pPoolID)294 Lac_MemPoolCleanUpInternal(lac_mem_pool_hdr_t *pPoolID)
295 {
296 lac_mem_blk_t *pCurrentBlk = NULL;
297 void *pFreePtr = NULL;
298 Cpa32U count = 0;
299
300 if (pPoolID->trackBlks == NULL) {
301 pCurrentBlk = pop(&pPoolID->stack);
302
303 while (pCurrentBlk != NULL) {
304 /* Free Data Blocks */
305 pFreePtr = pCurrentBlk->pMemAllocPtr;
306 pCurrentBlk = pop(&pPoolID->stack);
307 LAC_OS_CAFREE(pFreePtr);
308 }
309 } else {
310 for (count = 0; count < pPoolID->numElementsInPool; count++) {
311 pFreePtr = (pPoolID->trackBlks[count])->pMemAllocPtr;
312 LAC_OS_CAFREE(pFreePtr);
313 }
314 LAC_OS_FREE(pPoolID->trackBlks);
315 }
316 LAC_OS_FREE(pPoolID);
317 }
318
319 unsigned int
Lac_MemPoolAvailableEntries(lac_memory_pool_id_t poolID)320 Lac_MemPoolAvailableEntries(lac_memory_pool_id_t poolID)
321 {
322 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID;
323 if (pPoolID == NULL) {
324 QAT_UTILS_LOG("Invalid Pool ID\n");
325 return 0;
326 }
327 return pPoolID->availBlks;
328 }
329
330 void
Lac_MemPoolStatsShow(void)331 Lac_MemPoolStatsShow(void)
332 {
333 unsigned int index = 0;
334 QAT_UTILS_LOG(SEPARATOR BORDER
335 " Memory Pools Stats\n" SEPARATOR);
336
337 while (index < LAC_MEM_POOLS_NUM_SUPPORTED) {
338 if (lac_mem_pools[index] != NULL) {
339 QAT_UTILS_LOG(
340 BORDER " Pool Name: %s \n" BORDER
341 " No. Elements in Pool: %10u \n" BORDER
342 " Element Size in Bytes: %10u \n" BORDER
343 " Alignment in Bytes: %10u \n" BORDER
344 " No. Available Blocks: %10zu \n" SEPARATOR,
345 lac_mem_pools[index]->poolName,
346 lac_mem_pools[index]->numElementsInPool,
347 lac_mem_pools[index]->blkSizeInBytes,
348 lac_mem_pools[index]->blkAlignmentInBytes,
349 lac_mem_pools[index]->availBlks);
350 }
351 index++;
352 }
353 }
354
355 static void
Lac_MemPoolInitSymCookies(lac_sym_cookie_t * pSymCookie)356 Lac_MemPoolInitSymCookies(lac_sym_cookie_t *pSymCookie)
357 {
358 pSymCookie->keyContentDescPhyAddr =
359 LAC_OS_VIRT_TO_PHYS_INTERNAL(pSymCookie->u.keyCookie.contentDesc);
360 pSymCookie->keyHashStateBufferPhyAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL(
361 pSymCookie->u.keyCookie.hashStateBuffer);
362 pSymCookie->keySslKeyInputPhyAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL(
363 &(pSymCookie->u.keyCookie.u.sslKeyInput));
364 pSymCookie->keyTlsKeyInputPhyAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL(
365 &(pSymCookie->u.keyCookie.u.tlsKeyInput));
366 }
367
368 CpaStatus
Lac_MemPoolInitSymCookiesPhyAddr(lac_memory_pool_id_t poolID)369 Lac_MemPoolInitSymCookiesPhyAddr(lac_memory_pool_id_t poolID)
370 {
371 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID;
372 lac_sym_cookie_t *pSymCookie = NULL;
373 lac_mem_blk_t *pCurrentBlk = NULL;
374
375 if (NULL == pPoolID) {
376 QAT_UTILS_LOG("Invalid Pool ID\n");
377 return CPA_STATUS_FAIL;
378 }
379
380 if (pPoolID->trackBlks == NULL) {
381 pCurrentBlk = top(&pPoolID->stack);
382
383 while (pCurrentBlk != NULL) {
384 pSymCookie =
385 (lac_sym_cookie_t *)((LAC_ARCH_UINT)(pCurrentBlk) +
386 sizeof(lac_mem_blk_t));
387 pCurrentBlk = pCurrentBlk->pNext;
388 Lac_MemPoolInitSymCookies(pSymCookie);
389 }
390 } else {
391 Cpa32U count = 0;
392
393 for (count = 0; count < pPoolID->numElementsInPool; count++) {
394 pCurrentBlk = pPoolID->trackBlks[count];
395 pSymCookie =
396 (lac_sym_cookie_t *)((LAC_ARCH_UINT)(pCurrentBlk) +
397 sizeof(lac_mem_blk_t));
398 Lac_MemPoolInitSymCookies(pSymCookie);
399 }
400 }
401 return CPA_STATUS_SUCCESS;
402 }
403
404 CpaStatus
Lac_MemPoolInitDcCookiePhyAddr(lac_memory_pool_id_t poolID)405 Lac_MemPoolInitDcCookiePhyAddr(lac_memory_pool_id_t poolID)
406 {
407 lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID;
408 lac_mem_blk_t *pCurrentBlk = NULL;
409
410 if (NULL == pPoolID) {
411 QAT_UTILS_LOG("Invalid Pool ID\n");
412 return CPA_STATUS_FAIL;
413 }
414
415 if (NULL == pPoolID->trackBlks) {
416 pCurrentBlk = top(&pPoolID->stack);
417
418 while (pCurrentBlk != NULL) {
419 pCurrentBlk = pCurrentBlk->pNext;
420 }
421 } else {
422 Cpa32U count = 0;
423
424 for (count = 0; count < pPoolID->numElementsInPool; count++) {
425 pCurrentBlk = pPoolID->trackBlks[count];
426 }
427 }
428 return CPA_STATUS_SUCCESS;
429 }
430