1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 *
21 * Copyright (c) 2002-2006 Neterion, Inc.
22 */
23
24 #include "xge-os-pal.h"
25 #include "xgehal-mm.h"
26 #include "xge-debug.h"
27
28 /*
29 * __hal_mempool_grow
30 *
31 * Will resize mempool up to %num_allocate value.
32 */
33 xge_hal_status_e
__hal_mempool_grow(xge_hal_mempool_t * mempool,int num_allocate,int * num_allocated)34 __hal_mempool_grow(xge_hal_mempool_t *mempool, int num_allocate,
35 int *num_allocated)
36 {
37 int i, first_time = mempool->memblocks_allocated == 0 ? 1 : 0;
38 int n_items = mempool->items_per_memblock;
39
40 *num_allocated = 0;
41
42 if ((mempool->memblocks_allocated + num_allocate) >
43 mempool->memblocks_max) {
44 xge_debug_mm(XGE_ERR, "%s",
45 "__hal_mempool_grow: can grow anymore");
46 return XGE_HAL_ERR_OUT_OF_MEMORY;
47 }
48
49 for (i = mempool->memblocks_allocated;
50 i < mempool->memblocks_allocated + num_allocate; i++) {
51 int j;
52 int is_last =
53 ((mempool->memblocks_allocated+num_allocate-1) == i);
54 xge_hal_mempool_dma_t *dma_object =
55 mempool->memblocks_dma_arr + i;
56 void *the_memblock;
57 int dma_flags;
58
59 dma_flags = XGE_OS_DMA_CACHELINE_ALIGNED;
60 #ifdef XGE_HAL_DMA_DTR_CONSISTENT
61 dma_flags |= XGE_OS_DMA_CONSISTENT;
62 #else
63 dma_flags |= XGE_OS_DMA_STREAMING;
64 #endif
65
66 /* allocate DMA-capable memblock */
67 mempool->memblocks_arr[i] = xge_os_dma_malloc(mempool->pdev,
68 mempool->memblock_size,
69 dma_flags,
70 &dma_object->handle,
71 &dma_object->acc_handle);
72 if (mempool->memblocks_arr[i] == NULL) {
73 xge_debug_mm(XGE_ERR,
74 "memblock[%d]: out of DMA memory", i);
75 return XGE_HAL_ERR_OUT_OF_MEMORY;
76 }
77 xge_os_memzero(mempool->memblocks_arr[i],
78 mempool->memblock_size);
79 the_memblock = mempool->memblocks_arr[i];
80
81 /* allocate memblock's private part. Each DMA memblock
82 * has a space allocated for item's private usage upon
83 * mempool's user request. Each time mempool grows, it will
84 * allocate new memblock and its private part at once.
85 * This helps to minimize memory usage a lot. */
86 mempool->memblocks_priv_arr[i] = xge_os_malloc(mempool->pdev,
87 mempool->items_priv_size * n_items);
88 if (mempool->memblocks_priv_arr[i] == NULL) {
89 xge_os_dma_free(mempool->pdev,
90 the_memblock,
91 mempool->memblock_size,
92 &dma_object->acc_handle,
93 &dma_object->handle);
94 xge_debug_mm(XGE_ERR,
95 "memblock_priv[%d]: out of virtual memory, "
96 "requested %d(%d:%d) bytes", i,
97 mempool->items_priv_size * n_items,
98 mempool->items_priv_size, n_items);
99 return XGE_HAL_ERR_OUT_OF_MEMORY;
100 }
101 xge_os_memzero(mempool->memblocks_priv_arr[i],
102 mempool->items_priv_size * n_items);
103
104 /* map memblock to physical memory */
105 dma_object->addr = xge_os_dma_map(mempool->pdev,
106 dma_object->handle,
107 the_memblock,
108 mempool->memblock_size,
109 XGE_OS_DMA_DIR_BIDIRECTIONAL,
110 #ifdef XGE_HAL_DMA_DTR_CONSISTENT
111 XGE_OS_DMA_CONSISTENT
112 #else
113 XGE_OS_DMA_STREAMING
114 #endif
115 );
116 if (dma_object->addr == XGE_OS_INVALID_DMA_ADDR) {
117 xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i],
118 mempool->items_priv_size *
119 n_items);
120 xge_os_dma_free(mempool->pdev,
121 the_memblock,
122 mempool->memblock_size,
123 &dma_object->acc_handle,
124 &dma_object->handle);
125 return XGE_HAL_ERR_OUT_OF_MAPPING;
126 }
127
128 /* fill the items hash array */
129 for (j=0; j<n_items; j++) {
130 int index = i*n_items + j;
131
132 if (first_time && index >= mempool->items_initial) {
133 break;
134 }
135
136 mempool->items_arr[index] =
137 ((char *)the_memblock + j*mempool->item_size);
138
139 /* let caller to do more job on each item */
140 if (mempool->item_func_alloc != NULL) {
141 xge_hal_status_e status;
142
143 if ((status = mempool->item_func_alloc(
144 mempool,
145 the_memblock,
146 i,
147 dma_object,
148 mempool->items_arr[index],
149 index,
150 is_last,
151 mempool->userdata)) != XGE_HAL_OK) {
152
153 if (mempool->item_func_free != NULL) {
154 int k;
155
156 for (k=0; k<j; k++) {
157
158 index =i*n_items + k;
159
160 (void)mempool->item_func_free(
161 mempool, the_memblock,
162 i, dma_object,
163 mempool->items_arr[index],
164 index, is_last,
165 mempool->userdata);
166 }
167 }
168
169 xge_os_free(mempool->pdev,
170 mempool->memblocks_priv_arr[i],
171 mempool->items_priv_size *
172 n_items);
173 xge_os_dma_unmap(mempool->pdev,
174 dma_object->handle,
175 dma_object->addr,
176 mempool->memblock_size,
177 XGE_OS_DMA_DIR_BIDIRECTIONAL);
178 xge_os_dma_free(mempool->pdev,
179 the_memblock,
180 mempool->memblock_size,
181 &dma_object->acc_handle,
182 &dma_object->handle);
183 return status;
184 }
185 }
186
187 mempool->items_current = index + 1;
188 }
189
190 xge_debug_mm(XGE_TRACE,
191 "memblock%d: allocated %dk, vaddr 0x"XGE_OS_LLXFMT", "
192 "dma_addr 0x"XGE_OS_LLXFMT, i, mempool->memblock_size / 1024,
193 (unsigned long long)(ulong_t)mempool->memblocks_arr[i],
194 (unsigned long long)dma_object->addr);
195
196 (*num_allocated)++;
197
198 if (first_time && mempool->items_current ==
199 mempool->items_initial) {
200 break;
201 }
202 }
203
204 /* increment actual number of allocated memblocks */
205 mempool->memblocks_allocated += *num_allocated;
206
207 return XGE_HAL_OK;
208 }
209
210 /*
211 * xge_hal_mempool_create
212 * @memblock_size:
213 * @items_initial:
214 * @items_max:
215 * @item_size:
216 * @item_func:
217 *
218 * This function will create memory pool object. Pool may grow but will
219 * never shrink. Pool consists of number of dynamically allocated blocks
220 * with size enough to hold %items_initial number of items. Memory is
221 * DMA-able but client must map/unmap before interoperating with the device.
222 * See also: xge_os_dma_map(), xge_hal_dma_unmap(), xge_hal_status_e{}.
223 */
224 xge_hal_mempool_t*
__hal_mempool_create(pci_dev_h pdev,int memblock_size,int item_size,int items_priv_size,int items_initial,int items_max,xge_hal_mempool_item_f item_func_alloc,xge_hal_mempool_item_f item_func_free,void * userdata)225 __hal_mempool_create(pci_dev_h pdev, int memblock_size, int item_size,
226 int items_priv_size, int items_initial, int items_max,
227 xge_hal_mempool_item_f item_func_alloc,
228 xge_hal_mempool_item_f item_func_free, void *userdata)
229 {
230 xge_hal_status_e status;
231 int memblocks_to_allocate;
232 xge_hal_mempool_t *mempool;
233 int allocated;
234
235 if (memblock_size < item_size) {
236 xge_debug_mm(XGE_ERR,
237 "memblock_size %d < item_size %d: misconfiguration",
238 memblock_size, item_size);
239 return NULL;
240 }
241
242 mempool = (xge_hal_mempool_t *) \
243 xge_os_malloc(pdev, sizeof(xge_hal_mempool_t));
244 if (mempool == NULL) {
245 xge_debug_mm(XGE_ERR, "mempool allocation failure");
246 return NULL;
247 }
248 xge_os_memzero(mempool, sizeof(xge_hal_mempool_t));
249
250 mempool->pdev = pdev;
251 mempool->memblock_size = memblock_size;
252 mempool->items_max = items_max;
253 mempool->items_initial = items_initial;
254 mempool->item_size = item_size;
255 mempool->items_priv_size = items_priv_size;
256 mempool->item_func_alloc = item_func_alloc;
257 mempool->item_func_free = item_func_free;
258 mempool->userdata = userdata;
259
260 mempool->memblocks_allocated = 0;
261
262 mempool->items_per_memblock = memblock_size / item_size;
263
264 mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) /
265 mempool->items_per_memblock;
266
267 /* allocate array of memblocks */
268 mempool->memblocks_arr = (void ** ) xge_os_malloc(mempool->pdev,
269 sizeof(void*) * mempool->memblocks_max);
270 if (mempool->memblocks_arr == NULL) {
271 xge_debug_mm(XGE_ERR, "memblocks_arr allocation failure");
272 __hal_mempool_destroy(mempool);
273 return NULL;
274 }
275 xge_os_memzero(mempool->memblocks_arr,
276 sizeof(void*) * mempool->memblocks_max);
277
278 /* allocate array of private parts of items per memblocks */
279 mempool->memblocks_priv_arr = (void **) xge_os_malloc(mempool->pdev,
280 sizeof(void*) * mempool->memblocks_max);
281 if (mempool->memblocks_priv_arr == NULL) {
282 xge_debug_mm(XGE_ERR, "memblocks_priv_arr allocation failure");
283 __hal_mempool_destroy(mempool);
284 return NULL;
285 }
286 xge_os_memzero(mempool->memblocks_priv_arr,
287 sizeof(void*) * mempool->memblocks_max);
288
289 /* allocate array of memblocks DMA objects */
290 mempool->memblocks_dma_arr =
291 (xge_hal_mempool_dma_t *) xge_os_malloc(mempool->pdev,
292 sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max);
293
294 if (mempool->memblocks_dma_arr == NULL) {
295 xge_debug_mm(XGE_ERR, "memblocks_dma_arr allocation failure");
296 __hal_mempool_destroy(mempool);
297 return NULL;
298 }
299 xge_os_memzero(mempool->memblocks_dma_arr,
300 sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max);
301
302 /* allocate hash array of items */
303 mempool->items_arr = (void **) xge_os_malloc(mempool->pdev,
304 sizeof(void*) * mempool->items_max);
305 if (mempool->items_arr == NULL) {
306 xge_debug_mm(XGE_ERR, "items_arr allocation failure");
307 __hal_mempool_destroy(mempool);
308 return NULL;
309 }
310 xge_os_memzero(mempool->items_arr, sizeof(void *) * mempool->items_max);
311
312 mempool->shadow_items_arr = (void **) xge_os_malloc(mempool->pdev,
313 sizeof(void*) * mempool->items_max);
314 if (mempool->shadow_items_arr == NULL) {
315 xge_debug_mm(XGE_ERR, "shadow_items_arr allocation failure");
316 __hal_mempool_destroy(mempool);
317 return NULL;
318 }
319 xge_os_memzero(mempool->shadow_items_arr,
320 sizeof(void *) * mempool->items_max);
321
322 /* calculate initial number of memblocks */
323 memblocks_to_allocate = (mempool->items_initial +
324 mempool->items_per_memblock - 1) /
325 mempool->items_per_memblock;
326
327 xge_debug_mm(XGE_TRACE, "allocating %d memblocks, "
328 "%d items per memblock", memblocks_to_allocate,
329 mempool->items_per_memblock);
330
331 /* pre-allocate the mempool */
332 status = __hal_mempool_grow(mempool, memblocks_to_allocate, &allocated);
333 xge_os_memcpy(mempool->shadow_items_arr, mempool->items_arr,
334 sizeof(void*) * mempool->items_max);
335 if (status != XGE_HAL_OK) {
336 xge_debug_mm(XGE_ERR, "mempool_grow failure");
337 __hal_mempool_destroy(mempool);
338 return NULL;
339 }
340
341 xge_debug_mm(XGE_TRACE,
342 "total: allocated %dk of DMA-capable memory",
343 mempool->memblock_size * allocated / 1024);
344
345 return mempool;
346 }
347
348 /*
349 * xge_hal_mempool_destroy
350 */
351 void
__hal_mempool_destroy(xge_hal_mempool_t * mempool)352 __hal_mempool_destroy(xge_hal_mempool_t *mempool)
353 {
354 int i, j;
355
356 for (i=0; i<mempool->memblocks_allocated; i++) {
357 xge_hal_mempool_dma_t *dma_object;
358
359 xge_assert(mempool->memblocks_arr[i]);
360 xge_assert(mempool->memblocks_dma_arr + i);
361
362 dma_object = mempool->memblocks_dma_arr + i;
363
364 for (j=0; j<mempool->items_per_memblock; j++) {
365 int index = i*mempool->items_per_memblock + j;
366
367 /* to skip last partially filled(if any) memblock */
368 if (index >= mempool->items_current) {
369 break;
370 }
371
372 /* let caller to do more job on each item */
373 if (mempool->item_func_free != NULL) {
374
375 mempool->item_func_free(mempool,
376 mempool->memblocks_arr[i],
377 i, dma_object,
378 mempool->shadow_items_arr[index],
379 index, /* unused */ -1,
380 mempool->userdata);
381 }
382 }
383
384 xge_os_dma_unmap(mempool->pdev,
385 dma_object->handle, dma_object->addr,
386 mempool->memblock_size, XGE_OS_DMA_DIR_BIDIRECTIONAL);
387
388 xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i],
389 mempool->items_priv_size * mempool->items_per_memblock);
390
391 xge_os_dma_free(mempool->pdev, mempool->memblocks_arr[i],
392 mempool->memblock_size, &dma_object->acc_handle,
393 &dma_object->handle);
394 }
395
396 if (mempool->items_arr) {
397 xge_os_free(mempool->pdev, mempool->items_arr, sizeof(void*) *
398 mempool->items_max);
399 }
400
401 if (mempool->shadow_items_arr) {
402 xge_os_free(mempool->pdev, mempool->shadow_items_arr,
403 sizeof(void*) * mempool->items_max);
404 }
405
406 if (mempool->memblocks_dma_arr) {
407 xge_os_free(mempool->pdev, mempool->memblocks_dma_arr,
408 sizeof(xge_hal_mempool_dma_t) *
409 mempool->memblocks_max);
410 }
411
412 if (mempool->memblocks_priv_arr) {
413 xge_os_free(mempool->pdev, mempool->memblocks_priv_arr,
414 sizeof(void*) * mempool->memblocks_max);
415 }
416
417 if (mempool->memblocks_arr) {
418 xge_os_free(mempool->pdev, mempool->memblocks_arr,
419 sizeof(void*) * mempool->memblocks_max);
420 }
421
422 xge_os_free(mempool->pdev, mempool, sizeof(xge_hal_mempool_t));
423 }
424