1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2002-2005 Neterion, Inc. 24 * All right Reserved. 25 * 26 * FileName : hal-mm.c 27 * 28 * Description: chipset memory pool object implementation 29 * 30 * Created: 10 May 2004 31 */ 32 33 #include "xge-os-pal.h" 34 #include "xgehal-mm.h" 35 #include "xge-debug.h" 36 37 /* 38 * __hal_mempool_grow 39 * 40 * Will resize mempool up to %num_allocate value. 41 */ 42 xge_hal_status_e 43 __hal_mempool_grow(xge_hal_mempool_t *mempool, int num_allocate, 44 int *num_allocated) 45 { 46 int i, first_time = mempool->memblocks_allocated == 0 ? 1 : 0; 47 int n_items = mempool->items_per_memblock; 48 49 *num_allocated = 0; 50 51 if ((mempool->memblocks_allocated + num_allocate) > 52 mempool->memblocks_max) { 53 xge_debug_mm(XGE_ERR, "%s", 54 "__hal_mempool_grow: can grow anymore"); 55 return XGE_HAL_ERR_OUT_OF_MEMORY; 56 } 57 58 for (i = mempool->memblocks_allocated; 59 i < mempool->memblocks_allocated + num_allocate; i++) { 60 int j; 61 int is_last = 62 ((mempool->memblocks_allocated+num_allocate-1) == i); 63 xge_hal_mempool_dma_t *dma_object = 64 mempool->memblocks_dma_arr + i; 65 void *the_memblock; 66 int dma_flags; 67 68 dma_flags = XGE_OS_DMA_CACHELINE_ALIGNED; 69 #ifdef XGE_HAL_DMA_DTR_CONSISTENT 70 dma_flags |= XGE_OS_DMA_CONSISTENT; 71 #else 72 dma_flags |= XGE_OS_DMA_STREAMING; 73 #endif 74 75 /* allocate DMA-capable memblock */ 76 mempool->memblocks_arr[i] = xge_os_dma_malloc(mempool->pdev, 77 mempool->memblock_size, 78 dma_flags, 79 &dma_object->handle, 80 &dma_object->acc_handle); 81 if (mempool->memblocks_arr[i] == NULL) { 82 xge_debug_mm(XGE_ERR, 83 "memblock[%d]: out of DMA memory", i); 84 return XGE_HAL_ERR_OUT_OF_MEMORY; 85 } 86 xge_os_memzero(mempool->memblocks_arr[i], 87 mempool->memblock_size); 88 the_memblock = mempool->memblocks_arr[i]; 89 90 /* allocate memblock's private part. Each DMA memblock 91 * has a space allocated for item's private usage upon 92 * mempool's user request. Each time mempool grows, it will 93 * allocate new memblock and its private part at once. 94 * This helps to minimize memory usage a lot. */ 95 mempool->memblocks_priv_arr[i] = xge_os_malloc(mempool->pdev, 96 mempool->items_priv_size * n_items); 97 if (mempool->memblocks_priv_arr[i] == NULL) { 98 xge_os_dma_free(mempool->pdev, 99 the_memblock, 100 mempool->memblock_size, 101 &dma_object->acc_handle, 102 &dma_object->handle); 103 xge_debug_mm(XGE_ERR, 104 "memblock_priv[%d]: out of virtual memory, " 105 "requested %d(%d:%d) bytes", i, 106 mempool->items_priv_size * n_items, 107 mempool->items_priv_size, n_items); 108 return XGE_HAL_ERR_OUT_OF_MEMORY; 109 } 110 xge_os_memzero(mempool->memblocks_priv_arr[i], 111 mempool->items_priv_size * n_items); 112 113 /* map memblock to physical memory */ 114 dma_object->addr = xge_os_dma_map(mempool->pdev, 115 dma_object->handle, 116 the_memblock, 117 mempool->memblock_size, 118 XGE_OS_DMA_DIR_BIDIRECTIONAL, 119 #ifdef XGE_HAL_DMA_DTR_CONSISTENT 120 XGE_OS_DMA_CONSISTENT 121 #else 122 XGE_OS_DMA_STREAMING 123 #endif 124 ); 125 if (dma_object->addr == XGE_OS_INVALID_DMA_ADDR) { 126 xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i], 127 mempool->items_priv_size * 128 n_items); 129 xge_os_dma_free(mempool->pdev, 130 the_memblock, 131 mempool->memblock_size, 132 &dma_object->acc_handle, 133 &dma_object->handle); 134 return XGE_HAL_ERR_OUT_OF_MAPPING; 135 } 136 137 /* fill the items hash array */ 138 for (j=0; j<n_items; j++) { 139 int index = i*n_items + j; 140 141 if (first_time && index >= mempool->items_initial) { 142 break; 143 } 144 145 mempool->items_arr[index] = 146 ((char *)the_memblock + j*mempool->item_size); 147 148 /* let caller to do more job on each item */ 149 if (mempool->item_func_alloc != NULL) { 150 xge_hal_status_e status; 151 152 if ((status = mempool->item_func_alloc( 153 mempool, 154 the_memblock, 155 i, 156 dma_object, 157 mempool->items_arr[index], 158 index, 159 is_last, 160 mempool->userdata)) != XGE_HAL_OK) { 161 162 if (mempool->item_func_free != NULL) { 163 int k; 164 165 for (k=0; k<j; k++) { 166 167 index =i*n_items + k; 168 169 (void)mempool->item_func_free( 170 mempool, the_memblock, 171 i, dma_object, 172 mempool->items_arr[index], 173 index, is_last, 174 mempool->userdata); 175 } 176 } 177 178 xge_os_free(mempool->pdev, 179 mempool->memblocks_priv_arr[i], 180 mempool->items_priv_size * 181 n_items); 182 xge_os_dma_unmap(mempool->pdev, 183 dma_object->handle, 184 dma_object->addr, 185 mempool->memblock_size, 186 XGE_OS_DMA_DIR_BIDIRECTIONAL); 187 xge_os_dma_free(mempool->pdev, 188 the_memblock, 189 mempool->memblock_size, 190 &dma_object->acc_handle, 191 &dma_object->handle); 192 return status; 193 } 194 } 195 196 mempool->items_current = index + 1; 197 } 198 199 xge_debug_mm(XGE_TRACE, 200 "memblock%d: allocated %dk, vaddr 0x%llx, " 201 "dma_addr 0x%llx", i, mempool->memblock_size / 1024, 202 (unsigned long long)(ulong_t)mempool->memblocks_arr[i], 203 (unsigned long long)dma_object->addr); 204 205 (*num_allocated)++; 206 207 if (first_time && mempool->items_current == 208 mempool->items_initial) { 209 break; 210 } 211 } 212 213 /* increment actual number of allocated memblocks */ 214 mempool->memblocks_allocated += *num_allocated; 215 216 return XGE_HAL_OK; 217 } 218 219 /* 220 * xge_hal_mempool_create 221 * @memblock_size: 222 * @items_initial: 223 * @items_max: 224 * @item_size: 225 * @item_func: 226 * 227 * This function will create memory pool object. Pool may grow but will 228 * never shrink. Pool consists of number of dynamically allocated blocks 229 * with size enough to hold %items_initial number of items. Memory is 230 * DMA-able but client must map/unmap before interoperating with the device. 231 * See also: xge_os_dma_map(), xge_hal_dma_unmap(), xge_hal_status_e{}. 232 */ 233 xge_hal_mempool_t* 234 __hal_mempool_create(pci_dev_h pdev, int memblock_size, int item_size, 235 int items_priv_size, int items_initial, int items_max, 236 xge_hal_mempool_item_f item_func_alloc, 237 xge_hal_mempool_item_f item_func_free, void *userdata) 238 { 239 xge_hal_status_e status; 240 int memblocks_to_allocate; 241 xge_hal_mempool_t *mempool; 242 int allocated; 243 244 if (memblock_size < item_size) { 245 xge_debug_mm(XGE_ERR, 246 "memblock_size %d < item_size %d: misconfiguration", 247 memblock_size, item_size); 248 return NULL; 249 } 250 251 mempool = xge_os_malloc(pdev, sizeof(xge_hal_mempool_t)); 252 if (mempool == NULL) { 253 xge_debug_mm(XGE_ERR, "mempool allocation failure"); 254 return NULL; 255 } 256 xge_os_memzero(mempool, sizeof(xge_hal_mempool_t)); 257 258 mempool->pdev = pdev; 259 mempool->memblock_size = memblock_size; 260 mempool->items_max = items_max; 261 mempool->items_initial = items_initial; 262 mempool->item_size = item_size; 263 mempool->items_priv_size = items_priv_size; 264 mempool->item_func_alloc = item_func_alloc; 265 mempool->item_func_free = item_func_free; 266 mempool->userdata = userdata; 267 268 mempool->memblocks_allocated = 0; 269 270 mempool->items_per_memblock = memblock_size / item_size; 271 272 mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) / 273 mempool->items_per_memblock; 274 275 /* allocate array of memblocks */ 276 mempool->memblocks_arr = xge_os_malloc(mempool->pdev, 277 sizeof(void*) * mempool->memblocks_max); 278 if (mempool->memblocks_arr == NULL) { 279 xge_debug_mm(XGE_ERR, "memblocks_arr allocation failure"); 280 __hal_mempool_destroy(mempool); 281 return NULL; 282 } 283 xge_os_memzero(mempool->memblocks_arr, 284 sizeof(void*) * mempool->memblocks_max); 285 286 /* allocate array of private parts of items per memblocks */ 287 mempool->memblocks_priv_arr = xge_os_malloc(mempool->pdev, 288 sizeof(void*) * mempool->memblocks_max); 289 if (mempool->memblocks_priv_arr == NULL) { 290 xge_debug_mm(XGE_ERR, "memblocks_priv_arr allocation failure"); 291 __hal_mempool_destroy(mempool); 292 return NULL; 293 } 294 xge_os_memzero(mempool->memblocks_priv_arr, 295 sizeof(void*) * mempool->memblocks_max); 296 297 /* allocate array of memblocks DMA objects */ 298 mempool->memblocks_dma_arr = xge_os_malloc(mempool->pdev, 299 sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max); 300 if (mempool->memblocks_dma_arr == NULL) { 301 xge_debug_mm(XGE_ERR, "memblocks_dma_arr allocation failure"); 302 __hal_mempool_destroy(mempool); 303 return NULL; 304 } 305 xge_os_memzero(mempool->memblocks_dma_arr, 306 sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max); 307 308 /* allocate hash array of items */ 309 mempool->items_arr = xge_os_malloc(mempool->pdev, 310 sizeof(void*) * mempool->items_max); 311 if (mempool->items_arr == NULL) { 312 xge_debug_mm(XGE_ERR, "items_arr allocation failure"); 313 __hal_mempool_destroy(mempool); 314 return NULL; 315 } 316 xge_os_memzero(mempool->items_arr, sizeof(void *) * mempool->items_max); 317 318 mempool->shadow_items_arr = xge_os_malloc(mempool->pdev,sizeof(void*) * 319 mempool->items_max); 320 if (mempool->shadow_items_arr == NULL) { 321 xge_debug_mm(XGE_ERR, "shadow_items_arr allocation failure"); 322 __hal_mempool_destroy(mempool); 323 return NULL; 324 } 325 xge_os_memzero(mempool->shadow_items_arr, 326 sizeof(void *) * mempool->items_max); 327 328 /* calculate initial number of memblocks */ 329 memblocks_to_allocate = (mempool->items_initial + 330 mempool->items_per_memblock - 1) / 331 mempool->items_per_memblock; 332 333 xge_debug_mm(XGE_TRACE, "allocating %d memblocks, " 334 "%d items per memblock", memblocks_to_allocate, 335 mempool->items_per_memblock); 336 337 /* pre-allocate the mempool */ 338 status = __hal_mempool_grow(mempool, memblocks_to_allocate, &allocated); 339 xge_os_memcpy(mempool->shadow_items_arr, mempool->items_arr, 340 sizeof(void*) * mempool->items_max); 341 if (status != XGE_HAL_OK) { 342 xge_debug_mm(XGE_ERR, "mempool_grow failure"); 343 __hal_mempool_destroy(mempool); 344 return NULL; 345 } 346 347 xge_debug_mm(XGE_TRACE, 348 "total: allocated %dk of DMA-capable memory", 349 mempool->memblock_size * allocated / 1024); 350 351 return mempool; 352 } 353 354 /* 355 * xge_hal_mempool_destroy 356 */ 357 void 358 __hal_mempool_destroy(xge_hal_mempool_t *mempool) 359 { 360 int i, j; 361 362 for (i=0; i<mempool->memblocks_allocated; i++) { 363 xge_hal_mempool_dma_t *dma_object; 364 365 xge_assert(mempool->memblocks_arr[i]); 366 xge_assert(mempool->memblocks_dma_arr + i); 367 368 dma_object = mempool->memblocks_dma_arr + i; 369 370 for (j=0; j<mempool->items_per_memblock; j++) { 371 int index = i*mempool->items_per_memblock + j; 372 373 /* to skip last partially filled(if any) memblock */ 374 if (index >= mempool->items_current) { 375 break; 376 } 377 378 /* let caller to do more job on each item */ 379 if (mempool->item_func_free != NULL) { 380 381 mempool->item_func_free(mempool, 382 mempool->memblocks_arr[i], 383 i, dma_object, 384 mempool->shadow_items_arr[index], 385 index, /* unused */ -1, 386 mempool->userdata); 387 } 388 } 389 390 xge_os_dma_unmap(mempool->pdev, 391 dma_object->handle, dma_object->addr, 392 mempool->memblock_size, XGE_OS_DMA_DIR_BIDIRECTIONAL); 393 394 xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i], 395 mempool->items_priv_size * mempool->items_per_memblock); 396 397 xge_os_dma_free(mempool->pdev, mempool->memblocks_arr[i], 398 mempool->memblock_size, &dma_object->acc_handle, 399 &dma_object->handle); 400 } 401 402 if (mempool->items_arr) { 403 xge_os_free(mempool->pdev, mempool->items_arr, sizeof(void*) * 404 mempool->items_max); 405 } 406 407 if (mempool->shadow_items_arr) { 408 xge_os_free(mempool->pdev, mempool->shadow_items_arr, 409 sizeof(void*) * mempool->items_max); 410 } 411 412 if (mempool->memblocks_dma_arr) { 413 xge_os_free(mempool->pdev, mempool->memblocks_dma_arr, 414 sizeof(xge_hal_mempool_dma_t) * 415 mempool->memblocks_max); 416 } 417 418 if (mempool->memblocks_priv_arr) { 419 xge_os_free(mempool->pdev, mempool->memblocks_priv_arr, 420 sizeof(void*) * mempool->memblocks_max); 421 } 422 423 if (mempool->memblocks_arr) { 424 xge_os_free(mempool->pdev, mempool->memblocks_arr, 425 sizeof(void*) * mempool->memblocks_max); 426 } 427 428 xge_os_free(mempool->pdev, mempool, sizeof(xge_hal_mempool_t)); 429 } 430