1 /* 2 * Copyright (C) 2012-2014 Matteo Landi, Luigi Rizzo, Giuseppe Lettieri. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #ifdef linux 27 #include "bsd_glue.h" 28 #endif /* linux */ 29 30 #ifdef __APPLE__ 31 #include "osx_glue.h" 32 #endif /* __APPLE__ */ 33 34 #ifdef __FreeBSD__ 35 #include <sys/cdefs.h> /* prerequisite */ 36 __FBSDID("$FreeBSD$"); 37 38 #include <sys/types.h> 39 #include <sys/malloc.h> 40 #include <sys/proc.h> 41 #include <vm/vm.h> /* vtophys */ 42 #include <vm/pmap.h> /* vtophys */ 43 #include <sys/socket.h> /* sockaddrs */ 44 #include <sys/selinfo.h> 45 #include <sys/sysctl.h> 46 #include <net/if.h> 47 #include <net/if_var.h> 48 #include <net/vnet.h> 49 #include <machine/bus.h> /* bus_dmamap_* */ 50 51 #endif /* __FreeBSD__ */ 52 53 #include <net/netmap.h> 54 #include <dev/netmap/netmap_kern.h> 55 #include "netmap_mem2.h" 56 57 #ifdef linux 58 #define NMA_LOCK_INIT(n) sema_init(&(n)->nm_mtx, 1) 59 #define NMA_LOCK_DESTROY(n) 60 #define NMA_LOCK(n) down(&(n)->nm_mtx) 61 #define NMA_UNLOCK(n) up(&(n)->nm_mtx) 62 #else /* !linux */ 63 #define NMA_LOCK_INIT(n) mtx_init(&(n)->nm_mtx, "netmap memory allocator lock", NULL, MTX_DEF) 64 #define NMA_LOCK_DESTROY(n) mtx_destroy(&(n)->nm_mtx) 65 #define NMA_LOCK(n) mtx_lock(&(n)->nm_mtx) 66 #define NMA_UNLOCK(n) mtx_unlock(&(n)->nm_mtx) 67 #endif /* linux */ 68 69 70 struct netmap_obj_params netmap_params[NETMAP_POOLS_NR] = { 71 [NETMAP_IF_POOL] = { 72 .size = 1024, 73 .num = 100, 74 }, 75 [NETMAP_RING_POOL] = { 76 .size = 9*PAGE_SIZE, 77 .num = 200, 78 }, 79 [NETMAP_BUF_POOL] = { 80 .size = 2048, 81 .num = NETMAP_BUF_MAX_NUM, 82 }, 83 }; 84 85 86 /* 87 * nm_mem is the memory allocator used for all physical interfaces 88 * running in netmap mode. 89 * Virtual (VALE) ports will have each its own allocator. 90 */ 91 static int netmap_mem_global_config(struct netmap_mem_d *nmd); 92 static int netmap_mem_global_finalize(struct netmap_mem_d *nmd); 93 static void netmap_mem_global_deref(struct netmap_mem_d *nmd); 94 struct netmap_mem_d nm_mem = { /* Our memory allocator. */ 95 .pools = { 96 [NETMAP_IF_POOL] = { 97 .name = "netmap_if", 98 .objminsize = sizeof(struct netmap_if), 99 .objmaxsize = 4096, 100 .nummin = 10, /* don't be stingy */ 101 .nummax = 10000, /* XXX very large */ 102 }, 103 [NETMAP_RING_POOL] = { 104 .name = "netmap_ring", 105 .objminsize = sizeof(struct netmap_ring), 106 .objmaxsize = 32*PAGE_SIZE, 107 .nummin = 2, 108 .nummax = 1024, 109 }, 110 [NETMAP_BUF_POOL] = { 111 .name = "netmap_buf", 112 .objminsize = 64, 113 .objmaxsize = 65536, 114 .nummin = 4, 115 .nummax = 1000000, /* one million! */ 116 }, 117 }, 118 .config = netmap_mem_global_config, 119 .finalize = netmap_mem_global_finalize, 120 .deref = netmap_mem_global_deref, 121 }; 122 123 124 // XXX logically belongs to nm_mem 125 struct lut_entry *netmap_buffer_lut; /* exported */ 126 127 /* blueprint for the private memory allocators */ 128 static int netmap_mem_private_config(struct netmap_mem_d *nmd); 129 static int netmap_mem_private_finalize(struct netmap_mem_d *nmd); 130 static void netmap_mem_private_deref(struct netmap_mem_d *nmd); 131 const struct netmap_mem_d nm_blueprint = { 132 .pools = { 133 [NETMAP_IF_POOL] = { 134 .name = "%s_if", 135 .objminsize = sizeof(struct netmap_if), 136 .objmaxsize = 4096, 137 .nummin = 1, 138 .nummax = 10, 139 }, 140 [NETMAP_RING_POOL] = { 141 .name = "%s_ring", 142 .objminsize = sizeof(struct netmap_ring), 143 .objmaxsize = 32*PAGE_SIZE, 144 .nummin = 2, 145 .nummax = 1024, 146 }, 147 [NETMAP_BUF_POOL] = { 148 .name = "%s_buf", 149 .objminsize = 64, 150 .objmaxsize = 65536, 151 .nummin = 4, 152 .nummax = 1000000, /* one million! */ 153 }, 154 }, 155 .config = netmap_mem_private_config, 156 .finalize = netmap_mem_private_finalize, 157 .deref = netmap_mem_private_deref, 158 159 .flags = NETMAP_MEM_PRIVATE, 160 }; 161 162 /* memory allocator related sysctls */ 163 164 #define STRINGIFY(x) #x 165 166 167 #define DECLARE_SYSCTLS(id, name) \ 168 SYSCTL_INT(_dev_netmap, OID_AUTO, name##_size, \ 169 CTLFLAG_RW, &netmap_params[id].size, 0, "Requested size of netmap " STRINGIFY(name) "s"); \ 170 SYSCTL_INT(_dev_netmap, OID_AUTO, name##_curr_size, \ 171 CTLFLAG_RD, &nm_mem.pools[id]._objsize, 0, "Current size of netmap " STRINGIFY(name) "s"); \ 172 SYSCTL_INT(_dev_netmap, OID_AUTO, name##_num, \ 173 CTLFLAG_RW, &netmap_params[id].num, 0, "Requested number of netmap " STRINGIFY(name) "s"); \ 174 SYSCTL_INT(_dev_netmap, OID_AUTO, name##_curr_num, \ 175 CTLFLAG_RD, &nm_mem.pools[id].objtotal, 0, "Current number of netmap " STRINGIFY(name) "s") 176 177 SYSCTL_DECL(_dev_netmap); 178 DECLARE_SYSCTLS(NETMAP_IF_POOL, if); 179 DECLARE_SYSCTLS(NETMAP_RING_POOL, ring); 180 DECLARE_SYSCTLS(NETMAP_BUF_POOL, buf); 181 182 /* 183 * First, find the allocator that contains the requested offset, 184 * then locate the cluster through a lookup table. 185 */ 186 vm_paddr_t 187 netmap_mem_ofstophys(struct netmap_mem_d* nmd, vm_ooffset_t offset) 188 { 189 int i; 190 vm_ooffset_t o = offset; 191 vm_paddr_t pa; 192 struct netmap_obj_pool *p; 193 194 NMA_LOCK(nmd); 195 p = nmd->pools; 196 197 for (i = 0; i < NETMAP_POOLS_NR; offset -= p[i].memtotal, i++) { 198 if (offset >= p[i].memtotal) 199 continue; 200 // now lookup the cluster's address 201 pa = p[i].lut[offset / p[i]._objsize].paddr + 202 offset % p[i]._objsize; 203 NMA_UNLOCK(nmd); 204 return pa; 205 } 206 /* this is only in case of errors */ 207 D("invalid ofs 0x%x out of 0x%x 0x%x 0x%x", (u_int)o, 208 p[NETMAP_IF_POOL].memtotal, 209 p[NETMAP_IF_POOL].memtotal 210 + p[NETMAP_RING_POOL].memtotal, 211 p[NETMAP_IF_POOL].memtotal 212 + p[NETMAP_RING_POOL].memtotal 213 + p[NETMAP_BUF_POOL].memtotal); 214 NMA_UNLOCK(nmd); 215 return 0; // XXX bad address 216 } 217 218 int 219 netmap_mem_get_info(struct netmap_mem_d* nmd, u_int* size, u_int *memflags) 220 { 221 int error = 0; 222 NMA_LOCK(nmd); 223 error = nmd->config(nmd); 224 if (error) 225 goto out; 226 if (nmd->flags & NETMAP_MEM_FINALIZED) { 227 *size = nmd->nm_totalsize; 228 } else { 229 int i; 230 *size = 0; 231 for (i = 0; i < NETMAP_POOLS_NR; i++) { 232 struct netmap_obj_pool *p = nmd->pools + i; 233 *size += (p->_numclusters * p->_clustsize); 234 } 235 } 236 *memflags = nmd->flags; 237 out: 238 NMA_UNLOCK(nmd); 239 return error; 240 } 241 242 /* 243 * we store objects by kernel address, need to find the offset 244 * within the pool to export the value to userspace. 245 * Algorithm: scan until we find the cluster, then add the 246 * actual offset in the cluster 247 */ 248 static ssize_t 249 netmap_obj_offset(struct netmap_obj_pool *p, const void *vaddr) 250 { 251 int i, k = p->_clustentries, n = p->objtotal; 252 ssize_t ofs = 0; 253 254 for (i = 0; i < n; i += k, ofs += p->_clustsize) { 255 const char *base = p->lut[i].vaddr; 256 ssize_t relofs = (const char *) vaddr - base; 257 258 if (relofs < 0 || relofs >= p->_clustsize) 259 continue; 260 261 ofs = ofs + relofs; 262 ND("%s: return offset %d (cluster %d) for pointer %p", 263 p->name, ofs, i, vaddr); 264 return ofs; 265 } 266 D("address %p is not contained inside any cluster (%s)", 267 vaddr, p->name); 268 return 0; /* An error occurred */ 269 } 270 271 /* Helper functions which convert virtual addresses to offsets */ 272 #define netmap_if_offset(n, v) \ 273 netmap_obj_offset(&(n)->pools[NETMAP_IF_POOL], (v)) 274 275 #define netmap_ring_offset(n, v) \ 276 ((n)->pools[NETMAP_IF_POOL].memtotal + \ 277 netmap_obj_offset(&(n)->pools[NETMAP_RING_POOL], (v))) 278 279 #define netmap_buf_offset(n, v) \ 280 ((n)->pools[NETMAP_IF_POOL].memtotal + \ 281 (n)->pools[NETMAP_RING_POOL].memtotal + \ 282 netmap_obj_offset(&(n)->pools[NETMAP_BUF_POOL], (v))) 283 284 285 ssize_t 286 netmap_mem_if_offset(struct netmap_mem_d *nmd, const void *addr) 287 { 288 ssize_t v; 289 NMA_LOCK(nmd); 290 v = netmap_if_offset(nmd, addr); 291 NMA_UNLOCK(nmd); 292 return v; 293 } 294 295 /* 296 * report the index, and use start position as a hint, 297 * otherwise buffer allocation becomes terribly expensive. 298 */ 299 static void * 300 netmap_obj_malloc(struct netmap_obj_pool *p, u_int len, uint32_t *start, uint32_t *index) 301 { 302 uint32_t i = 0; /* index in the bitmap */ 303 uint32_t mask, j; /* slot counter */ 304 void *vaddr = NULL; 305 306 if (len > p->_objsize) { 307 D("%s request size %d too large", p->name, len); 308 // XXX cannot reduce the size 309 return NULL; 310 } 311 312 if (p->objfree == 0) { 313 D("no more %s objects", p->name); 314 return NULL; 315 } 316 if (start) 317 i = *start; 318 319 /* termination is guaranteed by p->free, but better check bounds on i */ 320 while (vaddr == NULL && i < p->bitmap_slots) { 321 uint32_t cur = p->bitmap[i]; 322 if (cur == 0) { /* bitmask is fully used */ 323 i++; 324 continue; 325 } 326 /* locate a slot */ 327 for (j = 0, mask = 1; (cur & mask) == 0; j++, mask <<= 1) 328 ; 329 330 p->bitmap[i] &= ~mask; /* mark object as in use */ 331 p->objfree--; 332 333 vaddr = p->lut[i * 32 + j].vaddr; 334 if (index) 335 *index = i * 32 + j; 336 } 337 ND("%s allocator: allocated object @ [%d][%d]: vaddr %p", i, j, vaddr); 338 339 if (start) 340 *start = i; 341 return vaddr; 342 } 343 344 345 /* 346 * free by index, not by address. This is slow, but is only used 347 * for a small number of objects (rings, nifp) 348 */ 349 static void 350 netmap_obj_free(struct netmap_obj_pool *p, uint32_t j) 351 { 352 if (j >= p->objtotal) { 353 D("invalid index %u, max %u", j, p->objtotal); 354 return; 355 } 356 p->bitmap[j / 32] |= (1 << (j % 32)); 357 p->objfree++; 358 return; 359 } 360 361 static void 362 netmap_obj_free_va(struct netmap_obj_pool *p, void *vaddr) 363 { 364 u_int i, j, n = p->numclusters; 365 366 for (i = 0, j = 0; i < n; i++, j += p->_clustentries) { 367 void *base = p->lut[i * p->_clustentries].vaddr; 368 ssize_t relofs = (ssize_t) vaddr - (ssize_t) base; 369 370 /* Given address, is out of the scope of the current cluster.*/ 371 if (vaddr < base || relofs >= p->_clustsize) 372 continue; 373 374 j = j + relofs / p->_objsize; 375 /* KASSERT(j != 0, ("Cannot free object 0")); */ 376 netmap_obj_free(p, j); 377 return; 378 } 379 D("address %p is not contained inside any cluster (%s)", 380 vaddr, p->name); 381 } 382 383 #define netmap_if_malloc(n, len) netmap_obj_malloc(&(n)->pools[NETMAP_IF_POOL], len, NULL, NULL) 384 #define netmap_if_free(n, v) netmap_obj_free_va(&(n)->pools[NETMAP_IF_POOL], (v)) 385 #define netmap_ring_malloc(n, len) netmap_obj_malloc(&(n)->pools[NETMAP_RING_POOL], len, NULL, NULL) 386 #define netmap_ring_free(n, v) netmap_obj_free_va(&(n)->pools[NETMAP_RING_POOL], (v)) 387 #define netmap_buf_malloc(n, _pos, _index) \ 388 netmap_obj_malloc(&(n)->pools[NETMAP_BUF_POOL], NETMAP_BDG_BUF_SIZE(n), _pos, _index) 389 390 391 /* Return the index associated to the given packet buffer */ 392 #define netmap_buf_index(n, v) \ 393 (netmap_obj_offset(&(n)->pools[NETMAP_BUF_POOL], (v)) / NETMAP_BDG_BUF_SIZE(n)) 394 395 396 /* Return nonzero on error */ 397 static int 398 netmap_new_bufs(struct netmap_mem_d *nmd, struct netmap_slot *slot, u_int n) 399 { 400 struct netmap_obj_pool *p = &nmd->pools[NETMAP_BUF_POOL]; 401 u_int i = 0; /* slot counter */ 402 uint32_t pos = 0; /* slot in p->bitmap */ 403 uint32_t index = 0; /* buffer index */ 404 405 for (i = 0; i < n; i++) { 406 void *vaddr = netmap_buf_malloc(nmd, &pos, &index); 407 if (vaddr == NULL) { 408 D("no more buffers after %d of %d", i, n); 409 goto cleanup; 410 } 411 slot[i].buf_idx = index; 412 slot[i].len = p->_objsize; 413 slot[i].flags = 0; 414 } 415 416 ND("allocated %d buffers, %d available, first at %d", n, p->objfree, pos); 417 return (0); 418 419 cleanup: 420 while (i > 0) { 421 i--; 422 netmap_obj_free(p, slot[i].buf_idx); 423 } 424 bzero(slot, n * sizeof(slot[0])); 425 return (ENOMEM); 426 } 427 428 429 static void 430 netmap_free_buf(struct netmap_mem_d *nmd, uint32_t i) 431 { 432 struct netmap_obj_pool *p = &nmd->pools[NETMAP_BUF_POOL]; 433 434 if (i < 2 || i >= p->objtotal) { 435 D("Cannot free buf#%d: should be in [2, %d[", i, p->objtotal); 436 return; 437 } 438 netmap_obj_free(p, i); 439 } 440 441 static void 442 netmap_reset_obj_allocator(struct netmap_obj_pool *p) 443 { 444 445 if (p == NULL) 446 return; 447 if (p->bitmap) 448 free(p->bitmap, M_NETMAP); 449 p->bitmap = NULL; 450 if (p->lut) { 451 u_int i; 452 size_t sz = p->_clustsize; 453 454 for (i = 0; i < p->objtotal; i += p->_clustentries) { 455 if (p->lut[i].vaddr) 456 contigfree(p->lut[i].vaddr, sz, M_NETMAP); 457 } 458 bzero(p->lut, sizeof(struct lut_entry) * p->objtotal); 459 #ifdef linux 460 vfree(p->lut); 461 #else 462 free(p->lut, M_NETMAP); 463 #endif 464 } 465 p->lut = NULL; 466 p->objtotal = 0; 467 p->memtotal = 0; 468 p->numclusters = 0; 469 p->objfree = 0; 470 } 471 472 /* 473 * Free all resources related to an allocator. 474 */ 475 static void 476 netmap_destroy_obj_allocator(struct netmap_obj_pool *p) 477 { 478 if (p == NULL) 479 return; 480 netmap_reset_obj_allocator(p); 481 } 482 483 /* 484 * We receive a request for objtotal objects, of size objsize each. 485 * Internally we may round up both numbers, as we allocate objects 486 * in small clusters multiple of the page size. 487 * We need to keep track of objtotal and clustentries, 488 * as they are needed when freeing memory. 489 * 490 * XXX note -- userspace needs the buffers to be contiguous, 491 * so we cannot afford gaps at the end of a cluster. 492 */ 493 494 495 /* call with NMA_LOCK held */ 496 static int 497 netmap_config_obj_allocator(struct netmap_obj_pool *p, u_int objtotal, u_int objsize) 498 { 499 int i; 500 u_int clustsize; /* the cluster size, multiple of page size */ 501 u_int clustentries; /* how many objects per entry */ 502 503 /* we store the current request, so we can 504 * detect configuration changes later */ 505 p->r_objtotal = objtotal; 506 p->r_objsize = objsize; 507 508 #define MAX_CLUSTSIZE (1<<17) 509 #define LINE_ROUND NM_CACHE_ALIGN // 64 510 if (objsize >= MAX_CLUSTSIZE) { 511 /* we could do it but there is no point */ 512 D("unsupported allocation for %d bytes", objsize); 513 return EINVAL; 514 } 515 /* make sure objsize is a multiple of LINE_ROUND */ 516 i = (objsize & (LINE_ROUND - 1)); 517 if (i) { 518 D("XXX aligning object by %d bytes", LINE_ROUND - i); 519 objsize += LINE_ROUND - i; 520 } 521 if (objsize < p->objminsize || objsize > p->objmaxsize) { 522 D("requested objsize %d out of range [%d, %d]", 523 objsize, p->objminsize, p->objmaxsize); 524 return EINVAL; 525 } 526 if (objtotal < p->nummin || objtotal > p->nummax) { 527 D("requested objtotal %d out of range [%d, %d]", 528 objtotal, p->nummin, p->nummax); 529 return EINVAL; 530 } 531 /* 532 * Compute number of objects using a brute-force approach: 533 * given a max cluster size, 534 * we try to fill it with objects keeping track of the 535 * wasted space to the next page boundary. 536 */ 537 for (clustentries = 0, i = 1;; i++) { 538 u_int delta, used = i * objsize; 539 if (used > MAX_CLUSTSIZE) 540 break; 541 delta = used % PAGE_SIZE; 542 if (delta == 0) { // exact solution 543 clustentries = i; 544 break; 545 } 546 if (delta > ( (clustentries*objsize) % PAGE_SIZE) ) 547 clustentries = i; 548 } 549 // D("XXX --- ouch, delta %d (bad for buffers)", delta); 550 /* compute clustsize and round to the next page */ 551 clustsize = clustentries * objsize; 552 i = (clustsize & (PAGE_SIZE - 1)); 553 if (i) 554 clustsize += PAGE_SIZE - i; 555 if (netmap_verbose) 556 D("objsize %d clustsize %d objects %d", 557 objsize, clustsize, clustentries); 558 559 /* 560 * The number of clusters is n = ceil(objtotal/clustentries) 561 * objtotal' = n * clustentries 562 */ 563 p->_clustentries = clustentries; 564 p->_clustsize = clustsize; 565 p->_numclusters = (objtotal + clustentries - 1) / clustentries; 566 567 /* actual values (may be larger than requested) */ 568 p->_objsize = objsize; 569 p->_objtotal = p->_numclusters * clustentries; 570 571 return 0; 572 } 573 574 575 /* call with NMA_LOCK held */ 576 static int 577 netmap_finalize_obj_allocator(struct netmap_obj_pool *p) 578 { 579 int i; /* must be signed */ 580 size_t n; 581 582 /* optimistically assume we have enough memory */ 583 p->numclusters = p->_numclusters; 584 p->objtotal = p->_objtotal; 585 586 n = sizeof(struct lut_entry) * p->objtotal; 587 #ifdef linux 588 p->lut = vmalloc(n); 589 #else 590 p->lut = malloc(n, M_NETMAP, M_NOWAIT | M_ZERO); 591 #endif 592 if (p->lut == NULL) { 593 D("Unable to create lookup table (%d bytes) for '%s'", (int)n, p->name); 594 goto clean; 595 } 596 597 /* Allocate the bitmap */ 598 n = (p->objtotal + 31) / 32; 599 p->bitmap = malloc(sizeof(uint32_t) * n, M_NETMAP, M_NOWAIT | M_ZERO); 600 if (p->bitmap == NULL) { 601 D("Unable to create bitmap (%d entries) for allocator '%s'", (int)n, 602 p->name); 603 goto clean; 604 } 605 p->bitmap_slots = n; 606 607 /* 608 * Allocate clusters, init pointers and bitmap 609 */ 610 611 n = p->_clustsize; 612 for (i = 0; i < (int)p->objtotal;) { 613 int lim = i + p->_clustentries; 614 char *clust; 615 616 clust = contigmalloc(n, M_NETMAP, M_NOWAIT | M_ZERO, 617 (size_t)0, -1UL, PAGE_SIZE, 0); 618 if (clust == NULL) { 619 /* 620 * If we get here, there is a severe memory shortage, 621 * so halve the allocated memory to reclaim some. 622 */ 623 D("Unable to create cluster at %d for '%s' allocator", 624 i, p->name); 625 if (i < 2) /* nothing to halve */ 626 goto out; 627 lim = i / 2; 628 for (i--; i >= lim; i--) { 629 p->bitmap[ (i>>5) ] &= ~( 1 << (i & 31) ); 630 if (i % p->_clustentries == 0 && p->lut[i].vaddr) 631 contigfree(p->lut[i].vaddr, 632 n, M_NETMAP); 633 } 634 out: 635 p->objtotal = i; 636 /* we may have stopped in the middle of a cluster */ 637 p->numclusters = (i + p->_clustentries - 1) / p->_clustentries; 638 break; 639 } 640 for (; i < lim; i++, clust += p->_objsize) { 641 p->bitmap[ (i>>5) ] |= ( 1 << (i & 31) ); 642 p->lut[i].vaddr = clust; 643 p->lut[i].paddr = vtophys(clust); 644 } 645 } 646 p->objfree = p->objtotal; 647 p->memtotal = p->numclusters * p->_clustsize; 648 if (p->objfree == 0) 649 goto clean; 650 if (netmap_verbose) 651 D("Pre-allocated %d clusters (%d/%dKB) for '%s'", 652 p->numclusters, p->_clustsize >> 10, 653 p->memtotal >> 10, p->name); 654 655 return 0; 656 657 clean: 658 netmap_reset_obj_allocator(p); 659 return ENOMEM; 660 } 661 662 /* call with lock held */ 663 static int 664 netmap_memory_config_changed(struct netmap_mem_d *nmd) 665 { 666 int i; 667 668 for (i = 0; i < NETMAP_POOLS_NR; i++) { 669 if (nmd->pools[i].r_objsize != netmap_params[i].size || 670 nmd->pools[i].r_objtotal != netmap_params[i].num) 671 return 1; 672 } 673 return 0; 674 } 675 676 static void 677 netmap_mem_reset_all(struct netmap_mem_d *nmd) 678 { 679 int i; 680 D("resetting %p", nmd); 681 for (i = 0; i < NETMAP_POOLS_NR; i++) { 682 netmap_reset_obj_allocator(&nmd->pools[i]); 683 } 684 nmd->flags &= ~NETMAP_MEM_FINALIZED; 685 } 686 687 static int 688 netmap_mem_finalize_all(struct netmap_mem_d *nmd) 689 { 690 int i; 691 if (nmd->flags & NETMAP_MEM_FINALIZED) 692 return 0; 693 nmd->lasterr = 0; 694 nmd->nm_totalsize = 0; 695 for (i = 0; i < NETMAP_POOLS_NR; i++) { 696 nmd->lasterr = netmap_finalize_obj_allocator(&nmd->pools[i]); 697 if (nmd->lasterr) 698 goto error; 699 nmd->nm_totalsize += nmd->pools[i].memtotal; 700 } 701 /* buffers 0 and 1 are reserved */ 702 nmd->pools[NETMAP_BUF_POOL].objfree -= 2; 703 nmd->pools[NETMAP_BUF_POOL].bitmap[0] = ~3; 704 nmd->flags |= NETMAP_MEM_FINALIZED; 705 706 D("Have %d KB for interfaces, %d KB for rings and %d MB for buffers", 707 nmd->pools[NETMAP_IF_POOL].memtotal >> 10, 708 nmd->pools[NETMAP_RING_POOL].memtotal >> 10, 709 nmd->pools[NETMAP_BUF_POOL].memtotal >> 20); 710 711 D("Free buffers: %d", nmd->pools[NETMAP_BUF_POOL].objfree); 712 713 714 return 0; 715 error: 716 netmap_mem_reset_all(nmd); 717 return nmd->lasterr; 718 } 719 720 721 722 void 723 netmap_mem_private_delete(struct netmap_mem_d *nmd) 724 { 725 if (nmd == NULL) 726 return; 727 D("deleting %p", nmd); 728 if (nmd->refcount > 0) 729 D("bug: deleting mem allocator with refcount=%d!", nmd->refcount); 730 D("done deleting %p", nmd); 731 NMA_LOCK_DESTROY(nmd); 732 free(nmd, M_DEVBUF); 733 } 734 735 static int 736 netmap_mem_private_config(struct netmap_mem_d *nmd) 737 { 738 /* nothing to do, we are configured on creation 739 * and configuration never changes thereafter 740 */ 741 return 0; 742 } 743 744 static int 745 netmap_mem_private_finalize(struct netmap_mem_d *nmd) 746 { 747 int err; 748 NMA_LOCK(nmd); 749 nmd->refcount++; 750 err = netmap_mem_finalize_all(nmd); 751 NMA_UNLOCK(nmd); 752 return err; 753 754 } 755 756 static void 757 netmap_mem_private_deref(struct netmap_mem_d *nmd) 758 { 759 NMA_LOCK(nmd); 760 if (--nmd->refcount <= 0) 761 netmap_mem_reset_all(nmd); 762 NMA_UNLOCK(nmd); 763 } 764 765 struct netmap_mem_d * 766 netmap_mem_private_new(const char *name, u_int txr, u_int txd, u_int rxr, u_int rxd) 767 { 768 struct netmap_mem_d *d = NULL; 769 struct netmap_obj_params p[NETMAP_POOLS_NR]; 770 int i; 771 u_int maxd; 772 773 d = malloc(sizeof(struct netmap_mem_d), 774 M_DEVBUF, M_NOWAIT | M_ZERO); 775 if (d == NULL) 776 return NULL; 777 778 *d = nm_blueprint; 779 780 /* XXX the rest of the code assumes the stack rings are alwasy present */ 781 txr++; 782 rxr++; 783 p[NETMAP_IF_POOL].size = sizeof(struct netmap_if) + 784 sizeof(ssize_t) * (txr + rxr); 785 p[NETMAP_IF_POOL].num = 2; 786 maxd = (txd > rxd) ? txd : rxd; 787 p[NETMAP_RING_POOL].size = sizeof(struct netmap_ring) + 788 sizeof(struct netmap_slot) * maxd; 789 p[NETMAP_RING_POOL].num = txr + rxr; 790 p[NETMAP_BUF_POOL].size = 2048; /* XXX find a way to let the user choose this */ 791 p[NETMAP_BUF_POOL].num = rxr * (rxd + 2) + txr * (txd + 2); 792 793 D("req if %d*%d ring %d*%d buf %d*%d", 794 p[NETMAP_IF_POOL].num, 795 p[NETMAP_IF_POOL].size, 796 p[NETMAP_RING_POOL].num, 797 p[NETMAP_RING_POOL].size, 798 p[NETMAP_BUF_POOL].num, 799 p[NETMAP_BUF_POOL].size); 800 801 for (i = 0; i < NETMAP_POOLS_NR; i++) { 802 snprintf(d->pools[i].name, NETMAP_POOL_MAX_NAMSZ, 803 nm_blueprint.pools[i].name, 804 name); 805 if (netmap_config_obj_allocator(&d->pools[i], 806 p[i].num, p[i].size)) 807 goto error; 808 } 809 810 d->flags &= ~NETMAP_MEM_FINALIZED; 811 812 NMA_LOCK_INIT(d); 813 814 return d; 815 error: 816 netmap_mem_private_delete(d); 817 return NULL; 818 } 819 820 821 /* call with lock held */ 822 static int 823 netmap_mem_global_config(struct netmap_mem_d *nmd) 824 { 825 int i; 826 827 if (nmd->refcount) 828 /* already in use, we cannot change the configuration */ 829 goto out; 830 831 if (!netmap_memory_config_changed(nmd)) 832 goto out; 833 834 D("reconfiguring"); 835 836 if (nmd->flags & NETMAP_MEM_FINALIZED) { 837 /* reset previous allocation */ 838 for (i = 0; i < NETMAP_POOLS_NR; i++) { 839 netmap_reset_obj_allocator(&nmd->pools[i]); 840 } 841 nmd->flags &= ~NETMAP_MEM_FINALIZED; 842 } 843 844 for (i = 0; i < NETMAP_POOLS_NR; i++) { 845 nmd->lasterr = netmap_config_obj_allocator(&nmd->pools[i], 846 netmap_params[i].num, netmap_params[i].size); 847 if (nmd->lasterr) 848 goto out; 849 } 850 851 out: 852 853 return nmd->lasterr; 854 } 855 856 static int 857 netmap_mem_global_finalize(struct netmap_mem_d *nmd) 858 { 859 int err; 860 861 NMA_LOCK(nmd); 862 863 864 /* update configuration if changed */ 865 if (netmap_mem_global_config(nmd)) 866 goto out; 867 868 nmd->refcount++; 869 870 if (nmd->flags & NETMAP_MEM_FINALIZED) { 871 /* may happen if config is not changed */ 872 ND("nothing to do"); 873 goto out; 874 } 875 876 if (netmap_mem_finalize_all(nmd)) 877 goto out; 878 879 /* backward compatibility */ 880 netmap_buf_size = nmd->pools[NETMAP_BUF_POOL]._objsize; 881 netmap_total_buffers = nmd->pools[NETMAP_BUF_POOL].objtotal; 882 883 netmap_buffer_lut = nmd->pools[NETMAP_BUF_POOL].lut; 884 netmap_buffer_base = nmd->pools[NETMAP_BUF_POOL].lut[0].vaddr; 885 886 nmd->lasterr = 0; 887 888 out: 889 if (nmd->lasterr) 890 nmd->refcount--; 891 err = nmd->lasterr; 892 893 NMA_UNLOCK(nmd); 894 895 return err; 896 897 } 898 899 int 900 netmap_mem_init(void) 901 { 902 NMA_LOCK_INIT(&nm_mem); 903 return (0); 904 } 905 906 void 907 netmap_mem_fini(void) 908 { 909 int i; 910 911 for (i = 0; i < NETMAP_POOLS_NR; i++) { 912 netmap_destroy_obj_allocator(&nm_mem.pools[i]); 913 } 914 NMA_LOCK_DESTROY(&nm_mem); 915 } 916 917 static void 918 netmap_free_rings(struct netmap_adapter *na) 919 { 920 u_int i; 921 if (!na->tx_rings) 922 return; 923 for (i = 0; i < na->num_tx_rings + 1; i++) { 924 if (na->tx_rings[i].ring) { 925 netmap_ring_free(na->nm_mem, na->tx_rings[i].ring); 926 na->tx_rings[i].ring = NULL; 927 } 928 } 929 for (i = 0; i < na->num_rx_rings + 1; i++) { 930 if (na->rx_rings[i].ring) { 931 netmap_ring_free(na->nm_mem, na->rx_rings[i].ring); 932 na->rx_rings[i].ring = NULL; 933 } 934 } 935 } 936 937 /* call with NMA_LOCK held * 938 * 939 * Allocate netmap rings and buffers for this card 940 * The rings are contiguous, but have variable size. 941 */ 942 int 943 netmap_mem_rings_create(struct netmap_adapter *na) 944 { 945 struct netmap_ring *ring; 946 u_int len, ndesc; 947 struct netmap_kring *kring; 948 949 NMA_LOCK(na->nm_mem); 950 951 for (kring = na->tx_rings; kring != na->rx_rings; kring++) { /* Transmit rings */ 952 ndesc = kring->nkr_num_slots; 953 len = sizeof(struct netmap_ring) + 954 ndesc * sizeof(struct netmap_slot); 955 ring = netmap_ring_malloc(na->nm_mem, len); 956 if (ring == NULL) { 957 D("Cannot allocate tx_ring"); 958 goto cleanup; 959 } 960 ND("txring[%d] at %p ofs %d", i, ring); 961 kring->ring = ring; 962 *(uint32_t *)(uintptr_t)&ring->num_slots = ndesc; 963 *(int64_t *)(uintptr_t)&ring->buf_ofs = 964 (na->nm_mem->pools[NETMAP_IF_POOL].memtotal + 965 na->nm_mem->pools[NETMAP_RING_POOL].memtotal) - 966 netmap_ring_offset(na->nm_mem, ring); 967 968 /* copy values from kring */ 969 ring->head = kring->rhead; 970 ring->cur = kring->rcur; 971 ring->tail = kring->rtail; 972 *(uint16_t *)(uintptr_t)&ring->nr_buf_size = 973 NETMAP_BDG_BUF_SIZE(na->nm_mem); 974 ND("initializing slots for txring"); 975 if (netmap_new_bufs(na->nm_mem, ring->slot, ndesc)) { 976 D("Cannot allocate buffers for tx_ring"); 977 goto cleanup; 978 } 979 } 980 981 for ( ; kring != na->tailroom; kring++) { /* Receive rings */ 982 ndesc = kring->nkr_num_slots; 983 len = sizeof(struct netmap_ring) + 984 ndesc * sizeof(struct netmap_slot); 985 ring = netmap_ring_malloc(na->nm_mem, len); 986 if (ring == NULL) { 987 D("Cannot allocate rx_ring"); 988 goto cleanup; 989 } 990 ND("rxring at %p ofs %d", ring); 991 992 kring->ring = ring; 993 *(uint32_t *)(uintptr_t)&ring->num_slots = ndesc; 994 *(int64_t *)(uintptr_t)&ring->buf_ofs = 995 (na->nm_mem->pools[NETMAP_IF_POOL].memtotal + 996 na->nm_mem->pools[NETMAP_RING_POOL].memtotal) - 997 netmap_ring_offset(na->nm_mem, ring); 998 999 /* copy values from kring */ 1000 ring->head = kring->rhead; 1001 ring->cur = kring->rcur; 1002 ring->tail = kring->rtail; 1003 *(int *)(uintptr_t)&ring->nr_buf_size = 1004 NETMAP_BDG_BUF_SIZE(na->nm_mem); 1005 ND("initializing slots for rxring[%d]", i); 1006 if (netmap_new_bufs(na->nm_mem, ring->slot, ndesc)) { 1007 D("Cannot allocate buffers for rx_ring"); 1008 goto cleanup; 1009 } 1010 } 1011 1012 NMA_UNLOCK(na->nm_mem); 1013 1014 return 0; 1015 1016 cleanup: 1017 netmap_free_rings(na); 1018 1019 NMA_UNLOCK(na->nm_mem); 1020 1021 return ENOMEM; 1022 } 1023 1024 void 1025 netmap_mem_rings_delete(struct netmap_adapter *na) 1026 { 1027 /* last instance, release bufs and rings */ 1028 u_int i, lim; 1029 struct netmap_kring *kring; 1030 struct netmap_ring *ring; 1031 1032 NMA_LOCK(na->nm_mem); 1033 1034 for (kring = na->tx_rings; kring != na->tailroom; kring++) { 1035 ring = kring->ring; 1036 if (ring == NULL) 1037 continue; 1038 lim = kring->nkr_num_slots; 1039 for (i = 0; i < lim; i++) 1040 netmap_free_buf(na->nm_mem, ring->slot[i].buf_idx); 1041 } 1042 netmap_free_rings(na); 1043 1044 NMA_UNLOCK(na->nm_mem); 1045 } 1046 1047 1048 /* call with NMA_LOCK held */ 1049 /* 1050 * Allocate the per-fd structure netmap_if. 1051 * 1052 * We assume that the configuration stored in na 1053 * (number of tx/rx rings and descs) does not change while 1054 * the interface is in netmap mode. 1055 */ 1056 struct netmap_if * 1057 netmap_mem_if_new(const char *ifname, struct netmap_adapter *na) 1058 { 1059 struct netmap_if *nifp; 1060 ssize_t base; /* handy for relative offsets between rings and nifp */ 1061 u_int i, len, ntx, nrx; 1062 1063 /* 1064 * verify whether virtual port need the stack ring 1065 */ 1066 ntx = na->num_tx_rings + 1; /* shorthand, include stack ring */ 1067 nrx = na->num_rx_rings + 1; /* shorthand, include stack ring */ 1068 /* 1069 * the descriptor is followed inline by an array of offsets 1070 * to the tx and rx rings in the shared memory region. 1071 * For virtual rx rings we also allocate an array of 1072 * pointers to assign to nkr_leases. 1073 */ 1074 1075 NMA_LOCK(na->nm_mem); 1076 1077 len = sizeof(struct netmap_if) + (nrx + ntx) * sizeof(ssize_t); 1078 nifp = netmap_if_malloc(na->nm_mem, len); 1079 if (nifp == NULL) { 1080 NMA_UNLOCK(na->nm_mem); 1081 return NULL; 1082 } 1083 1084 /* initialize base fields -- override const */ 1085 *(u_int *)(uintptr_t)&nifp->ni_tx_rings = na->num_tx_rings; 1086 *(u_int *)(uintptr_t)&nifp->ni_rx_rings = na->num_rx_rings; 1087 strncpy(nifp->ni_name, ifname, (size_t)IFNAMSIZ); 1088 1089 /* 1090 * fill the slots for the rx and tx rings. They contain the offset 1091 * between the ring and nifp, so the information is usable in 1092 * userspace to reach the ring from the nifp. 1093 */ 1094 base = netmap_if_offset(na->nm_mem, nifp); 1095 for (i = 0; i < ntx; i++) { 1096 *(ssize_t *)(uintptr_t)&nifp->ring_ofs[i] = 1097 netmap_ring_offset(na->nm_mem, na->tx_rings[i].ring) - base; 1098 } 1099 for (i = 0; i < nrx; i++) { 1100 *(ssize_t *)(uintptr_t)&nifp->ring_ofs[i+ntx] = 1101 netmap_ring_offset(na->nm_mem, na->rx_rings[i].ring) - base; 1102 } 1103 1104 NMA_UNLOCK(na->nm_mem); 1105 1106 return (nifp); 1107 } 1108 1109 void 1110 netmap_mem_if_delete(struct netmap_adapter *na, struct netmap_if *nifp) 1111 { 1112 if (nifp == NULL) 1113 /* nothing to do */ 1114 return; 1115 NMA_LOCK(na->nm_mem); 1116 1117 netmap_if_free(na->nm_mem, nifp); 1118 1119 NMA_UNLOCK(na->nm_mem); 1120 } 1121 1122 static void 1123 netmap_mem_global_deref(struct netmap_mem_d *nmd) 1124 { 1125 NMA_LOCK(nmd); 1126 1127 nmd->refcount--; 1128 if (netmap_verbose) 1129 D("refcount = %d", nmd->refcount); 1130 1131 NMA_UNLOCK(nmd); 1132 } 1133 1134 int 1135 netmap_mem_finalize(struct netmap_mem_d *nmd) 1136 { 1137 return nmd->finalize(nmd); 1138 } 1139 1140 void 1141 netmap_mem_deref(struct netmap_mem_d *nmd) 1142 { 1143 return nmd->deref(nmd); 1144 } 1145