1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2013-2016 Universita` di Pisa 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 30 #if defined(__FreeBSD__) 31 #include <sys/cdefs.h> /* prerequisite */ 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/types.h> 35 #include <sys/errno.h> 36 #include <sys/param.h> /* defines used in kernel.h */ 37 #include <sys/kernel.h> /* types used in module initialization */ 38 #include <sys/conf.h> /* cdevsw struct, UID, GID */ 39 #include <sys/sockio.h> 40 #include <sys/socketvar.h> /* struct socket */ 41 #include <sys/malloc.h> 42 #include <sys/poll.h> 43 #include <sys/rwlock.h> 44 #include <sys/socket.h> /* sockaddrs */ 45 #include <sys/selinfo.h> 46 #include <sys/sysctl.h> 47 #include <net/if.h> 48 #include <net/if_var.h> 49 #include <net/bpf.h> /* BIOCIMMEDIATE */ 50 #include <machine/bus.h> /* bus_dmamap_* */ 51 #include <sys/endian.h> 52 #include <sys/refcount.h> 53 #include <sys/smp.h> 54 55 56 #elif defined(linux) 57 58 #include "bsd_glue.h" 59 60 #elif defined(__APPLE__) 61 62 #warning OSX support is only partial 63 #include "osx_glue.h" 64 65 #elif defined(_WIN32) 66 #include "win_glue.h" 67 68 #else 69 70 #error Unsupported platform 71 72 #endif /* unsupported */ 73 74 /* 75 * common headers 76 */ 77 78 #include <net/netmap.h> 79 #include <dev/netmap/netmap_kern.h> 80 #include <dev/netmap/netmap_mem2.h> 81 #include <dev/netmap/netmap_bdg.h> 82 83 #ifdef WITH_VALE 84 85 /* 86 * system parameters (most of them in netmap_kern.h) 87 * NM_BDG_NAME prefix for switch port names, default "vale" 88 * NM_BDG_MAXPORTS number of ports 89 * NM_BRIDGES max number of switches in the system. 90 * 91 * Switch ports are named valeX:Y where X is the switch name and Y 92 * is the port. If Y matches a physical interface name, the port is 93 * connected to a physical device. 94 * 95 * Unlike physical interfaces, switch ports use their own memory region 96 * for rings and buffers. 97 * The virtual interfaces use per-queue lock instead of core lock. 98 * In the tx loop, we aggregate traffic in batches to make all operations 99 * faster. The batch size is bridge_batch. 100 */ 101 #define NM_BDG_MAXRINGS 16 /* XXX unclear how many (must be a pow of 2). */ 102 #define NM_BDG_MAXSLOTS 4096 /* XXX same as above */ 103 #define NM_BRIDGE_RINGSIZE 1024 /* in the device */ 104 #define NM_BDG_BATCH 1024 /* entries in the forwarding buffer */ 105 /* actual size of the tables */ 106 #define NM_BDG_BATCH_MAX (NM_BDG_BATCH + NETMAP_MAX_FRAGS) 107 /* NM_FT_NULL terminates a list of slots in the ft */ 108 #define NM_FT_NULL NM_BDG_BATCH_MAX 109 110 111 /* 112 * bridge_batch is set via sysctl to the max batch size to be 113 * used in the bridge. The actual value may be larger as the 114 * last packet in the block may overflow the size. 115 */ 116 static int bridge_batch = NM_BDG_BATCH; /* bridge batch size */ 117 118 /* Max number of vale bridges (loader tunable). */ 119 unsigned int vale_max_bridges = NM_BRIDGES; 120 121 SYSBEGIN(vars_vale); 122 SYSCTL_DECL(_dev_netmap); 123 SYSCTL_INT(_dev_netmap, OID_AUTO, bridge_batch, CTLFLAG_RW, &bridge_batch, 0, 124 "Max batch size to be used in the bridge"); 125 SYSCTL_UINT(_dev_netmap, OID_AUTO, max_bridges, CTLFLAG_RDTUN, &vale_max_bridges, 0, 126 "Max number of vale bridges"); 127 SYSEND; 128 129 static int netmap_vale_vp_create(struct nmreq_header *hdr, struct ifnet *, 130 struct netmap_mem_d *nmd, struct netmap_vp_adapter **); 131 static int netmap_vale_vp_bdg_attach(const char *, struct netmap_adapter *, 132 struct nm_bridge *); 133 static int netmap_vale_bwrap_attach(const char *, struct netmap_adapter *); 134 135 /* 136 * For each output interface, nm_vale_q is used to construct a list. 137 * bq_len is the number of output buffers (we can have coalescing 138 * during the copy). 139 */ 140 struct nm_vale_q { 141 uint16_t bq_head; 142 uint16_t bq_tail; 143 uint32_t bq_len; /* number of buffers */ 144 }; 145 146 /* Holds the default callbacks */ 147 struct netmap_bdg_ops vale_bdg_ops = { 148 .lookup = netmap_vale_learning, 149 .config = NULL, 150 .dtor = NULL, 151 .vp_create = netmap_vale_vp_create, 152 .bwrap_attach = netmap_vale_bwrap_attach, 153 .name = NM_BDG_NAME, 154 }; 155 156 /* 157 * this is a slightly optimized copy routine which rounds 158 * to multiple of 64 bytes and is often faster than dealing 159 * with other odd sizes. We assume there is enough room 160 * in the source and destination buffers. 161 * 162 * XXX only for multiples of NM_BUF_ALIGN bytes, non overlapped. 163 */ 164 165 static inline void 166 pkt_copy(void *_src, void *_dst, int l) 167 { 168 uint64_t *src = _src; 169 uint64_t *dst = _dst; 170 if (unlikely(l >= 1024)) { 171 memcpy(dst, src, l); 172 return; 173 } 174 for (; likely(l > 0); l -= NM_BUF_ALIGN) { 175 /* XXX NM_BUF_ALIGN/sizeof(uint64_t) statements */ 176 *dst++ = *src++; 177 *dst++ = *src++; 178 *dst++ = *src++; 179 *dst++ = *src++; 180 *dst++ = *src++; 181 *dst++ = *src++; 182 *dst++ = *src++; 183 *dst++ = *src++; 184 } 185 } 186 187 188 /* 189 * Free the forwarding tables for rings attached to switch ports. 190 */ 191 static void 192 nm_free_bdgfwd(struct netmap_adapter *na) 193 { 194 int nrings, i; 195 struct netmap_kring **kring; 196 197 NMG_LOCK_ASSERT(); 198 nrings = na->num_tx_rings; 199 kring = na->tx_rings; 200 for (i = 0; i < nrings; i++) { 201 if (kring[i]->nkr_ft) { 202 nm_os_free(kring[i]->nkr_ft); 203 kring[i]->nkr_ft = NULL; /* protect from freeing twice */ 204 } 205 } 206 } 207 208 209 /* 210 * Allocate the forwarding tables for the rings attached to the bridge ports. 211 */ 212 static int 213 nm_alloc_bdgfwd(struct netmap_adapter *na) 214 { 215 int nrings, l, i, num_dstq; 216 struct netmap_kring **kring; 217 218 NMG_LOCK_ASSERT(); 219 /* all port:rings + broadcast */ 220 num_dstq = NM_BDG_MAXPORTS * NM_BDG_MAXRINGS + 1; 221 l = sizeof(struct nm_bdg_fwd) * NM_BDG_BATCH_MAX; 222 l += sizeof(struct nm_vale_q) * num_dstq; 223 l += sizeof(uint16_t) * NM_BDG_BATCH_MAX; 224 225 nrings = netmap_real_rings(na, NR_TX); 226 kring = na->tx_rings; 227 for (i = 0; i < nrings; i++) { 228 struct nm_bdg_fwd *ft; 229 struct nm_vale_q *dstq; 230 int j; 231 232 ft = nm_os_malloc(l); 233 if (!ft) { 234 nm_free_bdgfwd(na); 235 return ENOMEM; 236 } 237 dstq = (struct nm_vale_q *)(ft + NM_BDG_BATCH_MAX); 238 for (j = 0; j < num_dstq; j++) { 239 dstq[j].bq_head = dstq[j].bq_tail = NM_FT_NULL; 240 dstq[j].bq_len = 0; 241 } 242 kring[i]->nkr_ft = ft; 243 } 244 return 0; 245 } 246 247 /* Allows external modules to create bridges in exclusive mode, 248 * returns an authentication token that the external module will need 249 * to provide during nm_bdg_ctl_{attach, detach}(), netmap_bdg_regops(), 250 * and nm_bdg_update_private_data() operations. 251 * Successfully executed if ret != NULL and *return_status == 0. 252 */ 253 void * 254 netmap_vale_create(const char *bdg_name, int *return_status) 255 { 256 struct nm_bridge *b = NULL; 257 void *ret = NULL; 258 259 NMG_LOCK(); 260 b = nm_find_bridge(bdg_name, 0 /* don't create */, NULL); 261 if (b) { 262 *return_status = EEXIST; 263 goto unlock_bdg_create; 264 } 265 266 b = nm_find_bridge(bdg_name, 1 /* create */, &vale_bdg_ops); 267 if (!b) { 268 *return_status = ENOMEM; 269 goto unlock_bdg_create; 270 } 271 272 b->bdg_flags |= NM_BDG_ACTIVE | NM_BDG_EXCLUSIVE; 273 ret = nm_bdg_get_auth_token(b); 274 *return_status = 0; 275 276 unlock_bdg_create: 277 NMG_UNLOCK(); 278 return ret; 279 } 280 281 /* Allows external modules to destroy a bridge created through 282 * netmap_bdg_create(), the bridge must be empty. 283 */ 284 int 285 netmap_vale_destroy(const char *bdg_name, void *auth_token) 286 { 287 struct nm_bridge *b = NULL; 288 int ret = 0; 289 290 NMG_LOCK(); 291 b = nm_find_bridge(bdg_name, 0 /* don't create */, NULL); 292 if (!b) { 293 ret = ENXIO; 294 goto unlock_bdg_free; 295 } 296 297 if (!nm_bdg_valid_auth_token(b, auth_token)) { 298 ret = EACCES; 299 goto unlock_bdg_free; 300 } 301 if (!(b->bdg_flags & NM_BDG_EXCLUSIVE)) { 302 ret = EINVAL; 303 goto unlock_bdg_free; 304 } 305 306 b->bdg_flags &= ~(NM_BDG_EXCLUSIVE | NM_BDG_ACTIVE); 307 ret = netmap_bdg_free(b); 308 if (ret) { 309 b->bdg_flags |= NM_BDG_EXCLUSIVE | NM_BDG_ACTIVE; 310 } 311 312 unlock_bdg_free: 313 NMG_UNLOCK(); 314 return ret; 315 } 316 317 /* Process NETMAP_REQ_VALE_LIST. */ 318 int 319 netmap_vale_list(struct nmreq_header *hdr) 320 { 321 struct nmreq_vale_list *req = 322 (struct nmreq_vale_list *)(uintptr_t)hdr->nr_body; 323 int namelen = strlen(hdr->nr_name); 324 struct nm_bridge *b, *bridges; 325 struct netmap_vp_adapter *vpna; 326 int error = 0, i, j; 327 u_int num_bridges; 328 329 netmap_bns_getbridges(&bridges, &num_bridges); 330 331 /* this is used to enumerate bridges and ports */ 332 if (namelen) { /* look up indexes of bridge and port */ 333 if (strncmp(hdr->nr_name, NM_BDG_NAME, 334 strlen(NM_BDG_NAME))) { 335 return EINVAL; 336 } 337 NMG_LOCK(); 338 b = nm_find_bridge(hdr->nr_name, 0 /* don't create */, NULL); 339 if (!b) { 340 NMG_UNLOCK(); 341 return ENOENT; 342 } 343 344 req->nr_bridge_idx = b - bridges; /* bridge index */ 345 req->nr_port_idx = NM_BDG_NOPORT; 346 for (j = 0; j < b->bdg_active_ports; j++) { 347 i = b->bdg_port_index[j]; 348 vpna = b->bdg_ports[i]; 349 if (vpna == NULL) { 350 nm_prerr("This should not happen"); 351 continue; 352 } 353 /* the former and the latter identify a 354 * virtual port and a NIC, respectively 355 */ 356 if (!strcmp(vpna->up.name, hdr->nr_name)) { 357 req->nr_port_idx = i; /* port index */ 358 break; 359 } 360 } 361 NMG_UNLOCK(); 362 } else { 363 /* return the first non-empty entry starting from 364 * bridge nr_arg1 and port nr_arg2. 365 * 366 * Users can detect the end of the same bridge by 367 * seeing the new and old value of nr_arg1, and can 368 * detect the end of all the bridge by error != 0 369 */ 370 i = req->nr_bridge_idx; 371 j = req->nr_port_idx; 372 373 NMG_LOCK(); 374 for (error = ENOENT; i < vale_max_bridges; i++) { 375 b = bridges + i; 376 for ( ; j < NM_BDG_MAXPORTS; j++) { 377 if (b->bdg_ports[j] == NULL) 378 continue; 379 vpna = b->bdg_ports[j]; 380 /* write back the VALE switch name */ 381 strlcpy(hdr->nr_name, vpna->up.name, 382 sizeof(hdr->nr_name)); 383 error = 0; 384 goto out; 385 } 386 j = 0; /* following bridges scan from 0 */ 387 } 388 out: 389 req->nr_bridge_idx = i; 390 req->nr_port_idx = j; 391 NMG_UNLOCK(); 392 } 393 394 return error; 395 } 396 397 398 /* nm_dtor callback for ephemeral VALE ports */ 399 static void 400 netmap_vale_vp_dtor(struct netmap_adapter *na) 401 { 402 struct netmap_vp_adapter *vpna = (struct netmap_vp_adapter*)na; 403 struct nm_bridge *b = vpna->na_bdg; 404 405 nm_prdis("%s has %d references", na->name, na->na_refcount); 406 407 if (b) { 408 netmap_bdg_detach_common(b, vpna->bdg_port, -1); 409 } 410 411 if (na->ifp != NULL && !nm_iszombie(na)) { 412 NM_DETACH_NA(na->ifp); 413 if (vpna->autodelete) { 414 nm_prdis("releasing %s", na->ifp->if_xname); 415 NMG_UNLOCK(); 416 nm_os_vi_detach(na->ifp); 417 NMG_LOCK(); 418 } 419 } 420 } 421 422 423 424 /* nm_krings_create callback for VALE ports. 425 * Calls the standard netmap_krings_create, then adds leases on rx 426 * rings and bdgfwd on tx rings. 427 */ 428 static int 429 netmap_vale_vp_krings_create(struct netmap_adapter *na) 430 { 431 u_int tailroom; 432 int error, i; 433 uint32_t *leases; 434 u_int nrx = netmap_real_rings(na, NR_RX); 435 436 /* 437 * Leases are attached to RX rings on vale ports 438 */ 439 tailroom = sizeof(uint32_t) * na->num_rx_desc * nrx; 440 441 error = netmap_krings_create(na, tailroom); 442 if (error) 443 return error; 444 445 leases = na->tailroom; 446 447 for (i = 0; i < nrx; i++) { /* Receive rings */ 448 na->rx_rings[i]->nkr_leases = leases; 449 leases += na->num_rx_desc; 450 } 451 452 error = nm_alloc_bdgfwd(na); 453 if (error) { 454 netmap_krings_delete(na); 455 return error; 456 } 457 458 return 0; 459 } 460 461 462 /* nm_krings_delete callback for VALE ports. */ 463 static void 464 netmap_vale_vp_krings_delete(struct netmap_adapter *na) 465 { 466 nm_free_bdgfwd(na); 467 netmap_krings_delete(na); 468 } 469 470 471 static int 472 nm_vale_flush(struct nm_bdg_fwd *ft, u_int n, 473 struct netmap_vp_adapter *na, u_int ring_nr); 474 475 476 /* 477 * main dispatch routine for the bridge. 478 * Grab packets from a kring, move them into the ft structure 479 * associated to the tx (input) port. Max one instance per port, 480 * filtered on input (ioctl, poll or XXX). 481 * Returns the next position in the ring. 482 */ 483 static int 484 nm_vale_preflush(struct netmap_kring *kring, u_int end) 485 { 486 struct netmap_vp_adapter *na = 487 (struct netmap_vp_adapter*)kring->na; 488 struct netmap_ring *ring = kring->ring; 489 struct nm_bdg_fwd *ft; 490 u_int ring_nr = kring->ring_id; 491 u_int j = kring->nr_hwcur, lim = kring->nkr_num_slots - 1; 492 u_int ft_i = 0; /* start from 0 */ 493 u_int frags = 1; /* how many frags ? */ 494 struct nm_bridge *b = na->na_bdg; 495 496 /* To protect against modifications to the bridge we acquire a 497 * shared lock, waiting if we can sleep (if the source port is 498 * attached to a user process) or with a trylock otherwise (NICs). 499 */ 500 nm_prdis("wait rlock for %d packets", ((j > end ? lim+1 : 0) + end) - j); 501 if (na->up.na_flags & NAF_BDG_MAYSLEEP) 502 BDG_RLOCK(b); 503 else if (!BDG_RTRYLOCK(b)) 504 return j; 505 nm_prdis(5, "rlock acquired for %d packets", ((j > end ? lim+1 : 0) + end) - j); 506 ft = kring->nkr_ft; 507 508 for (; likely(j != end); j = nm_next(j, lim)) { 509 struct netmap_slot *slot = &ring->slot[j]; 510 char *buf; 511 512 ft[ft_i].ft_len = slot->len; 513 ft[ft_i].ft_flags = slot->flags; 514 ft[ft_i].ft_offset = 0; 515 516 nm_prdis("flags is 0x%x", slot->flags); 517 /* we do not use the buf changed flag, but we still need to reset it */ 518 slot->flags &= ~NS_BUF_CHANGED; 519 520 /* this slot goes into a list so initialize the link field */ 521 ft[ft_i].ft_next = NM_FT_NULL; 522 buf = ft[ft_i].ft_buf = (slot->flags & NS_INDIRECT) ? 523 (void *)(uintptr_t)slot->ptr : NMB_O(kring, slot); 524 if (unlikely(buf == NULL || 525 slot->len > NETMAP_BUF_SIZE(&na->up) - nm_get_offset(kring, slot))) { 526 nm_prlim(5, "NULL %s buffer pointer from %s slot %d len %d", 527 (slot->flags & NS_INDIRECT) ? "INDIRECT" : "DIRECT", 528 kring->name, j, ft[ft_i].ft_len); 529 buf = ft[ft_i].ft_buf = NETMAP_BUF_BASE(&na->up); 530 ft[ft_i].ft_len = 0; 531 ft[ft_i].ft_flags = 0; 532 } 533 __builtin_prefetch(buf); 534 ++ft_i; 535 if (slot->flags & NS_MOREFRAG) { 536 frags++; 537 continue; 538 } 539 if (unlikely(netmap_verbose && frags > 1)) 540 nm_prlim(5, "%d frags at %d", frags, ft_i - frags); 541 ft[ft_i - frags].ft_frags = frags; 542 frags = 1; 543 if (unlikely((int)ft_i >= bridge_batch)) 544 ft_i = nm_vale_flush(ft, ft_i, na, ring_nr); 545 } 546 if (frags > 1) { 547 /* Here ft_i > 0, ft[ft_i-1].flags has NS_MOREFRAG, and we 548 * have to fix frags count. */ 549 frags--; 550 ft[ft_i - 1].ft_flags &= ~NS_MOREFRAG; 551 ft[ft_i - frags].ft_frags = frags; 552 nm_prlim(5, "Truncate incomplete fragment at %d (%d frags)", ft_i, frags); 553 } 554 if (ft_i) 555 ft_i = nm_vale_flush(ft, ft_i, na, ring_nr); 556 BDG_RUNLOCK(b); 557 return j; 558 } 559 560 561 /* ----- FreeBSD if_bridge hash function ------- */ 562 563 /* 564 * The following hash function is adapted from "Hash Functions" by Bob Jenkins 565 * ("Algorithm Alley", Dr. Dobbs Journal, September 1997). 566 * 567 * http://www.burtleburtle.net/bob/hash/spooky.html 568 */ 569 #define mix(a, b, c) \ 570 do { \ 571 a -= b; a -= c; a ^= (c >> 13); \ 572 b -= c; b -= a; b ^= (a << 8); \ 573 c -= a; c -= b; c ^= (b >> 13); \ 574 a -= b; a -= c; a ^= (c >> 12); \ 575 b -= c; b -= a; b ^= (a << 16); \ 576 c -= a; c -= b; c ^= (b >> 5); \ 577 a -= b; a -= c; a ^= (c >> 3); \ 578 b -= c; b -= a; b ^= (a << 10); \ 579 c -= a; c -= b; c ^= (b >> 15); \ 580 } while (/*CONSTCOND*/0) 581 582 583 static __inline uint32_t 584 nm_vale_rthash(const uint8_t *addr) 585 { 586 uint32_t a = 0x9e3779b9, b = 0x9e3779b9, c = 0; // hash key 587 588 b += addr[5] << 8; 589 b += addr[4]; 590 a += addr[3] << 24; 591 a += addr[2] << 16; 592 a += addr[1] << 8; 593 a += addr[0]; 594 595 mix(a, b, c); 596 #define BRIDGE_RTHASH_MASK (NM_BDG_HASH-1) 597 return (c & BRIDGE_RTHASH_MASK); 598 } 599 600 #undef mix 601 602 603 /* 604 * Lookup function for a learning bridge. 605 * Update the hash table with the source address, 606 * and then returns the destination port index, and the 607 * ring in *dst_ring (at the moment, always use ring 0) 608 */ 609 uint32_t 610 netmap_vale_learning(struct nm_bdg_fwd *ft, uint8_t *dst_ring, 611 struct netmap_vp_adapter *na, void *private_data) 612 { 613 uint8_t *buf = ((uint8_t *)ft->ft_buf) + ft->ft_offset; 614 u_int buf_len = ft->ft_len - ft->ft_offset; 615 struct nm_hash_ent *ht = private_data; 616 uint32_t sh, dh; 617 u_int dst, mysrc = na->bdg_port; 618 uint64_t smac, dmac; 619 uint8_t indbuf[12]; 620 621 if (buf_len < 14) { 622 return NM_BDG_NOPORT; 623 } 624 625 if (ft->ft_flags & NS_INDIRECT) { 626 if (copyin(buf, indbuf, sizeof(indbuf))) { 627 return NM_BDG_NOPORT; 628 } 629 buf = indbuf; 630 } 631 632 dmac = le64toh(*(uint64_t *)(buf)) & 0xffffffffffff; 633 smac = le64toh(*(uint64_t *)(buf + 4)); 634 smac >>= 16; 635 636 /* 637 * The hash is somewhat expensive, there might be some 638 * worthwhile optimizations here. 639 */ 640 if (((buf[6] & 1) == 0) && (na->last_smac != smac)) { /* valid src */ 641 uint8_t *s = buf+6; 642 sh = nm_vale_rthash(s); /* hash of source */ 643 /* update source port forwarding entry */ 644 na->last_smac = ht[sh].mac = smac; /* XXX expire ? */ 645 ht[sh].ports = mysrc; 646 if (netmap_debug & NM_DEBUG_VALE) 647 nm_prinf("src %02x:%02x:%02x:%02x:%02x:%02x on port %d", 648 s[0], s[1], s[2], s[3], s[4], s[5], mysrc); 649 } 650 dst = NM_BDG_BROADCAST; 651 if ((buf[0] & 1) == 0) { /* unicast */ 652 dh = nm_vale_rthash(buf); /* hash of dst */ 653 if (ht[dh].mac == dmac) { /* found dst */ 654 dst = ht[dh].ports; 655 } 656 } 657 return dst; 658 } 659 660 661 /* 662 * Available space in the ring. Only used in VALE code 663 * and only with is_rx = 1 664 */ 665 static inline uint32_t 666 nm_kr_space(struct netmap_kring *k, int is_rx) 667 { 668 int space; 669 670 if (is_rx) { 671 int busy = k->nkr_hwlease - k->nr_hwcur; 672 if (busy < 0) 673 busy += k->nkr_num_slots; 674 space = k->nkr_num_slots - 1 - busy; 675 } else { 676 /* XXX never used in this branch */ 677 space = k->nr_hwtail - k->nkr_hwlease; 678 if (space < 0) 679 space += k->nkr_num_slots; 680 } 681 #if 0 682 // sanity check 683 if (k->nkr_hwlease >= k->nkr_num_slots || 684 k->nr_hwcur >= k->nkr_num_slots || 685 k->nr_tail >= k->nkr_num_slots || 686 busy < 0 || 687 busy >= k->nkr_num_slots) { 688 nm_prerr("invalid kring, cur %d tail %d lease %d lease_idx %d lim %d", 689 k->nr_hwcur, k->nr_hwtail, k->nkr_hwlease, 690 k->nkr_lease_idx, k->nkr_num_slots); 691 } 692 #endif 693 return space; 694 } 695 696 697 698 699 /* make a lease on the kring for N positions. return the 700 * lease index 701 * XXX only used in VALE code and with is_rx = 1 702 */ 703 static inline uint32_t 704 nm_kr_lease(struct netmap_kring *k, u_int n, int is_rx) 705 { 706 uint32_t lim = k->nkr_num_slots - 1; 707 uint32_t lease_idx = k->nkr_lease_idx; 708 709 k->nkr_leases[lease_idx] = NR_NOSLOT; 710 k->nkr_lease_idx = nm_next(lease_idx, lim); 711 712 #ifdef CONFIG_NETMAP_DEBUG 713 if (n > nm_kr_space(k, is_rx)) { 714 nm_prerr("invalid request for %d slots", n); 715 panic("x"); 716 } 717 #endif /* CONFIG NETMAP_DEBUG */ 718 /* XXX verify that there are n slots */ 719 k->nkr_hwlease += n; 720 if (k->nkr_hwlease > lim) 721 k->nkr_hwlease -= lim + 1; 722 723 #ifdef CONFIG_NETMAP_DEBUG 724 if (k->nkr_hwlease >= k->nkr_num_slots || 725 k->nr_hwcur >= k->nkr_num_slots || 726 k->nr_hwtail >= k->nkr_num_slots || 727 k->nkr_lease_idx >= k->nkr_num_slots) { 728 nm_prerr("invalid kring %s, cur %d tail %d lease %d lease_idx %d lim %d", 729 k->na->name, 730 k->nr_hwcur, k->nr_hwtail, k->nkr_hwlease, 731 k->nkr_lease_idx, k->nkr_num_slots); 732 } 733 #endif /* CONFIG_NETMAP_DEBUG */ 734 return lease_idx; 735 } 736 737 /* 738 * 739 * This flush routine supports only unicast and broadcast but a large 740 * number of ports, and lets us replace the learn and dispatch functions. 741 */ 742 int 743 nm_vale_flush(struct nm_bdg_fwd *ft, u_int n, struct netmap_vp_adapter *na, 744 u_int ring_nr) 745 { 746 struct nm_vale_q *dst_ents, *brddst; 747 uint16_t num_dsts = 0, *dsts; 748 struct nm_bridge *b = na->na_bdg; 749 u_int i, me = na->bdg_port; 750 751 /* 752 * The work area (pointed by ft) is followed by an array of 753 * pointers to queues , dst_ents; there are NM_BDG_MAXRINGS 754 * queues per port plus one for the broadcast traffic. 755 * Then we have an array of destination indexes. 756 */ 757 dst_ents = (struct nm_vale_q *)(ft + NM_BDG_BATCH_MAX); 758 dsts = (uint16_t *)(dst_ents + NM_BDG_MAXPORTS * NM_BDG_MAXRINGS + 1); 759 760 /* first pass: find a destination for each packet in the batch */ 761 for (i = 0; likely(i < n); i += ft[i].ft_frags) { 762 uint8_t dst_ring = ring_nr; /* default, same ring as origin */ 763 uint16_t dst_port, d_i; 764 struct nm_vale_q *d; 765 struct nm_bdg_fwd *start_ft = NULL; 766 767 nm_prdis("slot %d frags %d", i, ft[i].ft_frags); 768 769 if (na->up.virt_hdr_len < ft[i].ft_len) { 770 ft[i].ft_offset = na->up.virt_hdr_len; 771 start_ft = &ft[i]; 772 } else if (na->up.virt_hdr_len == ft[i].ft_len && ft[i].ft_flags & NS_MOREFRAG) { 773 ft[i].ft_offset = ft[i].ft_len; 774 start_ft = &ft[i+1]; 775 } else { 776 /* Drop the packet if the virtio-net header is not into the first 777 * fragment nor at the very beginning of the second. 778 */ 779 continue; 780 } 781 dst_port = b->bdg_ops.lookup(start_ft, &dst_ring, na, b->private_data); 782 if (netmap_verbose > 255) 783 nm_prlim(5, "slot %d port %d -> %d", i, me, dst_port); 784 if (dst_port >= NM_BDG_NOPORT) 785 continue; /* this packet is identified to be dropped */ 786 else if (dst_port == NM_BDG_BROADCAST) 787 dst_ring = 0; /* broadcasts always go to ring 0 */ 788 else if (unlikely(dst_port == me || 789 !b->bdg_ports[dst_port])) 790 continue; 791 792 /* get a position in the scratch pad */ 793 d_i = dst_port * NM_BDG_MAXRINGS + dst_ring; 794 d = dst_ents + d_i; 795 796 /* append the first fragment to the list */ 797 if (d->bq_head == NM_FT_NULL) { /* new destination */ 798 d->bq_head = d->bq_tail = i; 799 /* remember this position to be scanned later */ 800 if (dst_port != NM_BDG_BROADCAST) 801 dsts[num_dsts++] = d_i; 802 } else { 803 ft[d->bq_tail].ft_next = i; 804 d->bq_tail = i; 805 } 806 d->bq_len += ft[i].ft_frags; 807 } 808 809 /* 810 * Broadcast traffic goes to ring 0 on all destinations. 811 * So we need to add these rings to the list of ports to scan. 812 */ 813 brddst = dst_ents + NM_BDG_BROADCAST * NM_BDG_MAXRINGS; 814 if (brddst->bq_head != NM_FT_NULL) { 815 u_int j; 816 for (j = 0; likely(j < b->bdg_active_ports); j++) { 817 uint16_t d_i; 818 i = b->bdg_port_index[j]; 819 if (unlikely(i == me)) 820 continue; 821 d_i = i * NM_BDG_MAXRINGS; 822 if (dst_ents[d_i].bq_head == NM_FT_NULL) 823 dsts[num_dsts++] = d_i; 824 } 825 } 826 827 nm_prdis(5, "pass 1 done %d pkts %d dsts", n, num_dsts); 828 /* second pass: scan destinations */ 829 for (i = 0; i < num_dsts; i++) { 830 struct netmap_vp_adapter *dst_na; 831 struct netmap_kring *kring; 832 struct netmap_ring *ring; 833 u_int dst_nr, lim, j, d_i, next, brd_next; 834 u_int needed, howmany; 835 int retry = netmap_txsync_retry; 836 struct nm_vale_q *d; 837 uint32_t my_start = 0, lease_idx = 0; 838 int nrings; 839 int virt_hdr_mismatch = 0; 840 841 d_i = dsts[i]; 842 nm_prdis("second pass %d port %d", i, d_i); 843 d = dst_ents + d_i; 844 // XXX fix the division 845 dst_na = b->bdg_ports[d_i/NM_BDG_MAXRINGS]; 846 /* protect from the lookup function returning an inactive 847 * destination port 848 */ 849 if (unlikely(dst_na == NULL)) 850 goto cleanup; 851 if (dst_na->up.na_flags & NAF_SW_ONLY) 852 goto cleanup; 853 /* 854 * The interface may be in !netmap mode in two cases: 855 * - when na is attached but not activated yet; 856 * - when na is being deactivated but is still attached. 857 */ 858 if (unlikely(!nm_netmap_on(&dst_na->up))) { 859 nm_prdis("not in netmap mode!"); 860 goto cleanup; 861 } 862 863 /* there is at least one either unicast or broadcast packet */ 864 brd_next = brddst->bq_head; 865 next = d->bq_head; 866 /* we need to reserve this many slots. If fewer are 867 * available, some packets will be dropped. 868 * Packets may have multiple fragments, so 869 * there is a chance that we may not use all of the slots 870 * we have claimed, so we will need to handle the leftover 871 * ones when we regain the lock. 872 */ 873 needed = d->bq_len + brddst->bq_len; 874 875 if (unlikely(dst_na->up.virt_hdr_len != na->up.virt_hdr_len)) { 876 if (netmap_verbose) { 877 nm_prlim(3, "virt_hdr_mismatch, src %d dst %d", na->up.virt_hdr_len, 878 dst_na->up.virt_hdr_len); 879 } 880 /* There is a virtio-net header/offloadings mismatch between 881 * source and destination. The slower mismatch datapath will 882 * be used to cope with all the mismatches. 883 */ 884 virt_hdr_mismatch = 1; 885 if (dst_na->mfs < na->mfs) { 886 /* We may need to do segmentation offloadings, and so 887 * we may need a number of destination slots greater 888 * than the number of input slots ('needed'). 889 * We look for the smallest integer 'x' which satisfies: 890 * needed * na->mfs + x * H <= x * na->mfs 891 * where 'H' is the length of the longest header that may 892 * be replicated in the segmentation process (e.g. for 893 * TCPv4 we must account for ethernet header, IP header 894 * and TCPv4 header). 895 */ 896 KASSERT(dst_na->mfs > 0, ("vpna->mfs is 0")); 897 needed = (needed * na->mfs) / 898 (dst_na->mfs - WORST_CASE_GSO_HEADER) + 1; 899 nm_prdis(3, "srcmtu=%u, dstmtu=%u, x=%u", na->mfs, dst_na->mfs, needed); 900 } 901 } 902 903 nm_prdis(5, "pass 2 dst %d is %x %s", 904 i, d_i, nm_is_bwrap(&dst_na->up) ? "nic/host" : "virtual"); 905 dst_nr = d_i & (NM_BDG_MAXRINGS-1); 906 nrings = dst_na->up.num_rx_rings; 907 if (dst_nr >= nrings) 908 dst_nr = dst_nr % nrings; 909 kring = dst_na->up.rx_rings[dst_nr]; 910 ring = kring->ring; 911 /* the destination ring may have not been opened for RX */ 912 if (unlikely(ring == NULL || kring->nr_mode != NKR_NETMAP_ON)) 913 goto cleanup; 914 lim = kring->nkr_num_slots - 1; 915 916 retry: 917 918 if (dst_na->retry && retry) { 919 /* try to get some free slot from the previous run */ 920 kring->nm_notify(kring, NAF_FORCE_RECLAIM); 921 /* actually useful only for bwraps, since there 922 * the notify will trigger a txsync on the hwna. VALE ports 923 * have dst_na->retry == 0 924 */ 925 } 926 /* reserve the buffers in the queue and an entry 927 * to report completion, and drop lock. 928 * XXX this might become a helper function. 929 */ 930 mtx_lock(&kring->q_lock); 931 if (kring->nkr_stopped) { 932 mtx_unlock(&kring->q_lock); 933 goto cleanup; 934 } 935 my_start = j = kring->nkr_hwlease; 936 howmany = nm_kr_space(kring, 1); 937 if (needed < howmany) 938 howmany = needed; 939 lease_idx = nm_kr_lease(kring, howmany, 1); 940 mtx_unlock(&kring->q_lock); 941 942 /* only retry if we need more than available slots */ 943 if (retry && needed <= howmany) 944 retry = 0; 945 946 /* copy to the destination queue */ 947 while (howmany > 0) { 948 struct netmap_slot *slot; 949 struct nm_bdg_fwd *ft_p, *ft_end; 950 u_int cnt; 951 952 /* find the queue from which we pick next packet. 953 * NM_FT_NULL is always higher than valid indexes 954 * so we never dereference it if the other list 955 * has packets (and if both are empty we never 956 * get here). 957 */ 958 if (next < brd_next) { 959 ft_p = ft + next; 960 next = ft_p->ft_next; 961 } else { /* insert broadcast */ 962 ft_p = ft + brd_next; 963 brd_next = ft_p->ft_next; 964 } 965 cnt = ft_p->ft_frags; // cnt > 0 966 if (unlikely(cnt > howmany)) 967 break; /* no more space */ 968 if (netmap_verbose && cnt > 1) 969 nm_prlim(5, "rx %d frags to %d", cnt, j); 970 ft_end = ft_p + cnt; 971 if (unlikely(virt_hdr_mismatch)) { 972 bdg_mismatch_datapath(na, dst_na, ft_p, ring, &j, lim, &howmany); 973 } else { 974 howmany -= cnt; 975 do { 976 char *dst, *src = ft_p->ft_buf; 977 size_t copy_len = ft_p->ft_len, dst_len = copy_len; 978 uintptr_t src_cb; 979 uint64_t dstoff, dstoff_cb; 980 int src_co, dst_co; 981 const uintptr_t mask = NM_BUF_ALIGN - 1; 982 983 slot = &ring->slot[j]; 984 dst = NMB(&dst_na->up, slot); 985 dstoff = nm_get_offset(kring, slot); 986 dstoff_cb = dstoff & ~mask; 987 src_cb = ((uintptr_t)src) & ~mask; 988 src_co = ((uintptr_t)src) & mask; 989 dst_co = ((uintptr_t)(dst + dstoff)) & mask; 990 if (dst_co < src_co) { 991 dstoff_cb += NM_BUF_ALIGN; 992 } 993 dstoff = dstoff_cb + src_co; 994 copy_len += src_co; 995 996 nm_prdis("send [%d] %d(%d) bytes at %s:%d", 997 i, (int)copy_len, (int)dst_len, 998 NM_IFPNAME(dst_ifp), j); 999 1000 if (unlikely(dstoff > NETMAP_BUF_SIZE(&dst_na->up) || 1001 dst_len > NETMAP_BUF_SIZE(&dst_na->up) - dstoff)) { 1002 nm_prlim(5, "dropping packet/fragment of len %zu, dest offset %llu", 1003 dst_len, (unsigned long long)dstoff); 1004 copy_len = dst_len = 0; 1005 dstoff = nm_get_offset(kring, slot); 1006 } 1007 1008 if (ft_p->ft_flags & NS_INDIRECT) { 1009 if (copyin(src, dst, copy_len)) { 1010 // invalid user pointer, pretend len is 0 1011 dst_len = 0; 1012 } 1013 } else { 1014 //memcpy(dst, src, copy_len); 1015 pkt_copy((char *)src_cb, dst + dstoff_cb, (int)copy_len); 1016 } 1017 slot->len = dst_len; 1018 slot->flags = (cnt << 8)| NS_MOREFRAG; 1019 nm_write_offset(kring, slot, dstoff); 1020 j = nm_next(j, lim); 1021 needed--; 1022 ft_p++; 1023 } while (ft_p != ft_end); 1024 slot->flags = (cnt << 8); /* clear flag on last entry */ 1025 } 1026 /* are we done ? */ 1027 if (next == NM_FT_NULL && brd_next == NM_FT_NULL) 1028 break; 1029 } 1030 { 1031 /* current position */ 1032 uint32_t *p = kring->nkr_leases; /* shorthand */ 1033 uint32_t update_pos; 1034 int still_locked = 1; 1035 1036 mtx_lock(&kring->q_lock); 1037 if (unlikely(howmany > 0)) { 1038 /* not used all bufs. If i am the last one 1039 * i can recover the slots, otherwise must 1040 * fill them with 0 to mark empty packets. 1041 */ 1042 nm_prdis("leftover %d bufs", howmany); 1043 if (nm_next(lease_idx, lim) == kring->nkr_lease_idx) { 1044 /* yes i am the last one */ 1045 nm_prdis("roll back nkr_hwlease to %d", j); 1046 kring->nkr_hwlease = j; 1047 } else { 1048 while (howmany-- > 0) { 1049 ring->slot[j].len = 0; 1050 ring->slot[j].flags = 0; 1051 j = nm_next(j, lim); 1052 } 1053 } 1054 } 1055 p[lease_idx] = j; /* report I am done */ 1056 1057 update_pos = kring->nr_hwtail; 1058 1059 if (my_start == update_pos) { 1060 /* all slots before my_start have been reported, 1061 * so scan subsequent leases to see if other ranges 1062 * have been completed, and to a selwakeup or txsync. 1063 */ 1064 while (lease_idx != kring->nkr_lease_idx && 1065 p[lease_idx] != NR_NOSLOT) { 1066 j = p[lease_idx]; 1067 p[lease_idx] = NR_NOSLOT; 1068 lease_idx = nm_next(lease_idx, lim); 1069 } 1070 /* j is the new 'write' position. j != my_start 1071 * means there are new buffers to report 1072 */ 1073 if (likely(j != my_start)) { 1074 kring->nr_hwtail = j; 1075 still_locked = 0; 1076 mtx_unlock(&kring->q_lock); 1077 kring->nm_notify(kring, 0); 1078 /* this is netmap_notify for VALE ports and 1079 * netmap_bwrap_notify for bwrap. The latter will 1080 * trigger a txsync on the underlying hwna 1081 */ 1082 if (dst_na->retry && retry--) { 1083 /* XXX this is going to call nm_notify again. 1084 * Only useful for bwrap in virtual machines 1085 */ 1086 goto retry; 1087 } 1088 } 1089 } 1090 if (still_locked) 1091 mtx_unlock(&kring->q_lock); 1092 } 1093 cleanup: 1094 d->bq_head = d->bq_tail = NM_FT_NULL; /* cleanup */ 1095 d->bq_len = 0; 1096 } 1097 brddst->bq_head = brddst->bq_tail = NM_FT_NULL; /* cleanup */ 1098 brddst->bq_len = 0; 1099 return 0; 1100 } 1101 1102 /* nm_txsync callback for VALE ports */ 1103 static int 1104 netmap_vale_vp_txsync(struct netmap_kring *kring, int flags) 1105 { 1106 struct netmap_vp_adapter *na = 1107 (struct netmap_vp_adapter *)kring->na; 1108 u_int done; 1109 u_int const lim = kring->nkr_num_slots - 1; 1110 u_int const head = kring->rhead; 1111 1112 if (bridge_batch <= 0) { /* testing only */ 1113 done = head; // used all 1114 goto done; 1115 } 1116 if (!na->na_bdg) { 1117 done = head; 1118 goto done; 1119 } 1120 if (bridge_batch > NM_BDG_BATCH) 1121 bridge_batch = NM_BDG_BATCH; 1122 1123 done = nm_vale_preflush(kring, head); 1124 done: 1125 if (done != head) 1126 nm_prerr("early break at %d/ %d, tail %d", done, head, kring->nr_hwtail); 1127 /* 1128 * packets between 'done' and 'cur' are left unsent. 1129 */ 1130 kring->nr_hwcur = done; 1131 kring->nr_hwtail = nm_prev(done, lim); 1132 if (netmap_debug & NM_DEBUG_TXSYNC) 1133 nm_prinf("%s ring %d flags %d", na->up.name, kring->ring_id, flags); 1134 return 0; 1135 } 1136 1137 1138 /* create a netmap_vp_adapter that describes a VALE port. 1139 * Only persistent VALE ports have a non-null ifp. 1140 */ 1141 static int 1142 netmap_vale_vp_create(struct nmreq_header *hdr, struct ifnet *ifp, 1143 struct netmap_mem_d *nmd, struct netmap_vp_adapter **ret) 1144 { 1145 struct nmreq_register *req = (struct nmreq_register *)(uintptr_t)hdr->nr_body; 1146 struct netmap_vp_adapter *vpna; 1147 struct netmap_adapter *na; 1148 int error = 0; 1149 u_int npipes = 0; 1150 u_int extrabufs = 0; 1151 1152 if (hdr->nr_reqtype != NETMAP_REQ_REGISTER) { 1153 return EINVAL; 1154 } 1155 1156 vpna = nm_os_malloc(sizeof(*vpna)); 1157 if (vpna == NULL) 1158 return ENOMEM; 1159 1160 na = &vpna->up; 1161 1162 na->ifp = ifp; 1163 strlcpy(na->name, hdr->nr_name, sizeof(na->name)); 1164 1165 /* bound checking */ 1166 na->num_tx_rings = req->nr_tx_rings; 1167 nm_bound_var(&na->num_tx_rings, 1, 1, NM_BDG_MAXRINGS, NULL); 1168 req->nr_tx_rings = na->num_tx_rings; /* write back */ 1169 na->num_rx_rings = req->nr_rx_rings; 1170 nm_bound_var(&na->num_rx_rings, 1, 1, NM_BDG_MAXRINGS, NULL); 1171 req->nr_rx_rings = na->num_rx_rings; /* write back */ 1172 nm_bound_var(&req->nr_tx_slots, NM_BRIDGE_RINGSIZE, 1173 1, NM_BDG_MAXSLOTS, NULL); 1174 na->num_tx_desc = req->nr_tx_slots; 1175 nm_bound_var(&req->nr_rx_slots, NM_BRIDGE_RINGSIZE, 1176 1, NM_BDG_MAXSLOTS, NULL); 1177 /* validate number of pipes. We want at least 1, 1178 * but probably can do with some more. 1179 * So let's use 2 as default (when 0 is supplied) 1180 */ 1181 nm_bound_var(&npipes, 2, 1, NM_MAXPIPES, NULL); 1182 /* validate extra bufs */ 1183 extrabufs = req->nr_extra_bufs; 1184 nm_bound_var(&extrabufs, 0, 0, 1185 128*NM_BDG_MAXSLOTS, NULL); 1186 req->nr_extra_bufs = extrabufs; /* write back */ 1187 na->num_rx_desc = req->nr_rx_slots; 1188 /* Set the mfs to a default value, as it is needed on the VALE 1189 * mismatch datapath. XXX We should set it according to the MTU 1190 * known to the kernel. */ 1191 vpna->mfs = NM_BDG_MFS_DEFAULT; 1192 vpna->last_smac = ~0llu; 1193 /*if (vpna->mfs > netmap_buf_size) TODO netmap_buf_size is zero?? 1194 vpna->mfs = netmap_buf_size; */ 1195 if (netmap_verbose) 1196 nm_prinf("max frame size %u", vpna->mfs); 1197 1198 na->na_flags |= (NAF_BDG_MAYSLEEP | NAF_OFFSETS); 1199 /* persistent VALE ports look like hw devices 1200 * with a native netmap adapter 1201 */ 1202 if (ifp) 1203 na->na_flags |= NAF_NATIVE; 1204 na->nm_txsync = netmap_vale_vp_txsync; 1205 na->nm_rxsync = netmap_vp_rxsync; /* use the one provided by bdg */ 1206 na->nm_register = netmap_vp_reg; /* use the one provided by bdg */ 1207 na->nm_krings_create = netmap_vale_vp_krings_create; 1208 na->nm_krings_delete = netmap_vale_vp_krings_delete; 1209 na->nm_dtor = netmap_vale_vp_dtor; 1210 nm_prdis("nr_mem_id %d", req->nr_mem_id); 1211 na->nm_mem = nmd ? 1212 netmap_mem_get(nmd): 1213 netmap_mem_private_new( 1214 na->num_tx_rings, na->num_tx_desc, 1215 na->num_rx_rings, na->num_rx_desc, 1216 req->nr_extra_bufs, npipes, &error); 1217 if (na->nm_mem == NULL) 1218 goto err; 1219 na->nm_bdg_attach = netmap_vale_vp_bdg_attach; 1220 /* other nmd fields are set in the common routine */ 1221 error = netmap_attach_common(na); 1222 if (error) 1223 goto err; 1224 *ret = vpna; 1225 return 0; 1226 1227 err: 1228 if (na->nm_mem != NULL) 1229 netmap_mem_put(na->nm_mem); 1230 nm_os_free(vpna); 1231 return error; 1232 } 1233 1234 /* nm_bdg_attach callback for VALE ports 1235 * The na_vp port is this same netmap_adapter. There is no host port. 1236 */ 1237 static int 1238 netmap_vale_vp_bdg_attach(const char *name, struct netmap_adapter *na, 1239 struct nm_bridge *b) 1240 { 1241 struct netmap_vp_adapter *vpna = (struct netmap_vp_adapter *)na; 1242 1243 if ((b->bdg_flags & NM_BDG_NEED_BWRAP) || vpna->na_bdg) { 1244 return NM_NEED_BWRAP; 1245 } 1246 na->na_vp = vpna; 1247 strlcpy(na->name, name, sizeof(na->name)); 1248 na->na_hostvp = NULL; 1249 return 0; 1250 } 1251 1252 static int 1253 netmap_vale_bwrap_krings_create(struct netmap_adapter *na) 1254 { 1255 int error; 1256 1257 /* impersonate a netmap_vp_adapter */ 1258 error = netmap_vale_vp_krings_create(na); 1259 if (error) 1260 return error; 1261 error = netmap_bwrap_krings_create_common(na); 1262 if (error) { 1263 netmap_vale_vp_krings_delete(na); 1264 } 1265 return error; 1266 } 1267 1268 static void 1269 netmap_vale_bwrap_krings_delete(struct netmap_adapter *na) 1270 { 1271 netmap_bwrap_krings_delete_common(na); 1272 netmap_vale_vp_krings_delete(na); 1273 } 1274 1275 static int 1276 netmap_vale_bwrap_attach(const char *nr_name, struct netmap_adapter *hwna) 1277 { 1278 struct netmap_bwrap_adapter *bna; 1279 struct netmap_adapter *na = NULL; 1280 struct netmap_adapter *hostna = NULL; 1281 int error; 1282 1283 bna = nm_os_malloc(sizeof(*bna)); 1284 if (bna == NULL) { 1285 return ENOMEM; 1286 } 1287 na = &bna->up.up; 1288 strlcpy(na->name, nr_name, sizeof(na->name)); 1289 na->nm_register = netmap_bwrap_reg; 1290 na->nm_txsync = netmap_vale_vp_txsync; 1291 // na->nm_rxsync = netmap_bwrap_rxsync; 1292 na->nm_krings_create = netmap_vale_bwrap_krings_create; 1293 na->nm_krings_delete = netmap_vale_bwrap_krings_delete; 1294 na->nm_notify = netmap_bwrap_notify; 1295 bna->nm_intr_notify = netmap_bwrap_intr_notify; 1296 bna->up.retry = 1; /* XXX maybe this should depend on the hwna */ 1297 /* Set the mfs, needed on the VALE mismatch datapath. */ 1298 bna->up.mfs = NM_BDG_MFS_DEFAULT; 1299 1300 if (hwna->na_flags & NAF_HOST_RINGS) { 1301 hostna = &bna->host.up; 1302 hostna->nm_notify = netmap_bwrap_notify; 1303 bna->host.mfs = NM_BDG_MFS_DEFAULT; 1304 } 1305 1306 error = netmap_bwrap_attach_common(na, hwna); 1307 if (error) { 1308 nm_os_free(bna); 1309 } 1310 return error; 1311 } 1312 1313 int 1314 netmap_get_vale_na(struct nmreq_header *hdr, struct netmap_adapter **na, 1315 struct netmap_mem_d *nmd, int create) 1316 { 1317 return netmap_get_bdg_na(hdr, na, nmd, create, &vale_bdg_ops); 1318 } 1319 1320 1321 /* creates a persistent VALE port */ 1322 int 1323 nm_vi_create(struct nmreq_header *hdr) 1324 { 1325 struct nmreq_vale_newif *req = 1326 (struct nmreq_vale_newif *)(uintptr_t)hdr->nr_body; 1327 int error = 0; 1328 /* Build a nmreq_register out of the nmreq_vale_newif, 1329 * so that we can call netmap_get_bdg_na(). */ 1330 struct nmreq_register regreq; 1331 bzero(®req, sizeof(regreq)); 1332 regreq.nr_tx_slots = req->nr_tx_slots; 1333 regreq.nr_rx_slots = req->nr_rx_slots; 1334 regreq.nr_tx_rings = req->nr_tx_rings; 1335 regreq.nr_rx_rings = req->nr_rx_rings; 1336 regreq.nr_mem_id = req->nr_mem_id; 1337 hdr->nr_reqtype = NETMAP_REQ_REGISTER; 1338 hdr->nr_body = (uintptr_t)®req; 1339 error = netmap_vi_create(hdr, 0 /* no autodelete */); 1340 hdr->nr_reqtype = NETMAP_REQ_VALE_NEWIF; 1341 hdr->nr_body = (uintptr_t)req; 1342 /* Write back to the original struct. */ 1343 req->nr_tx_slots = regreq.nr_tx_slots; 1344 req->nr_rx_slots = regreq.nr_rx_slots; 1345 req->nr_tx_rings = regreq.nr_tx_rings; 1346 req->nr_rx_rings = regreq.nr_rx_rings; 1347 req->nr_mem_id = regreq.nr_mem_id; 1348 return error; 1349 } 1350 1351 /* remove a persistent VALE port from the system */ 1352 int 1353 nm_vi_destroy(const char *name) 1354 { 1355 struct ifnet *ifp; 1356 struct netmap_vp_adapter *vpna; 1357 int error; 1358 1359 ifp = ifunit_ref(name); 1360 if (!ifp) 1361 return ENXIO; 1362 NMG_LOCK(); 1363 /* make sure this is actually a VALE port */ 1364 if (!NM_NA_VALID(ifp) || NA(ifp)->nm_register != netmap_vp_reg) { 1365 error = EINVAL; 1366 goto err; 1367 } 1368 1369 vpna = (struct netmap_vp_adapter *)NA(ifp); 1370 1371 /* we can only destroy ports that were created via NETMAP_BDG_NEWIF */ 1372 if (vpna->autodelete) { 1373 error = EINVAL; 1374 goto err; 1375 } 1376 1377 /* also make sure that nobody is using the interface */ 1378 if (NETMAP_OWNED_BY_ANY(&vpna->up) || 1379 vpna->up.na_refcount > 1 /* any ref besides the one in nm_vi_create()? */) { 1380 error = EBUSY; 1381 goto err; 1382 } 1383 1384 NMG_UNLOCK(); 1385 1386 if (netmap_verbose) 1387 nm_prinf("destroying a persistent vale interface %s", ifp->if_xname); 1388 /* Linux requires all the references are released 1389 * before unregister 1390 */ 1391 netmap_detach(ifp); 1392 if_rele(ifp); 1393 nm_os_vi_detach(ifp); 1394 return 0; 1395 1396 err: 1397 NMG_UNLOCK(); 1398 if_rele(ifp); 1399 return error; 1400 } 1401 1402 static int 1403 nm_update_info(struct nmreq_register *req, struct netmap_adapter *na) 1404 { 1405 req->nr_rx_rings = na->num_rx_rings; 1406 req->nr_tx_rings = na->num_tx_rings; 1407 req->nr_rx_slots = na->num_rx_desc; 1408 req->nr_tx_slots = na->num_tx_desc; 1409 return netmap_mem_get_info(na->nm_mem, &req->nr_memsize, NULL, 1410 &req->nr_mem_id); 1411 } 1412 1413 1414 /* 1415 * Create a virtual interface registered to the system. 1416 * The interface will be attached to a bridge later. 1417 */ 1418 int 1419 netmap_vi_create(struct nmreq_header *hdr, int autodelete) 1420 { 1421 struct nmreq_register *req = (struct nmreq_register *)(uintptr_t)hdr->nr_body; 1422 struct ifnet *ifp; 1423 struct netmap_vp_adapter *vpna; 1424 struct netmap_mem_d *nmd = NULL; 1425 int error; 1426 1427 if (hdr->nr_reqtype != NETMAP_REQ_REGISTER) { 1428 return EINVAL; 1429 } 1430 1431 /* don't include VALE prefix */ 1432 if (!strncmp(hdr->nr_name, NM_BDG_NAME, strlen(NM_BDG_NAME))) 1433 return EINVAL; 1434 if (strlen(hdr->nr_name) >= IFNAMSIZ) { 1435 return EINVAL; 1436 } 1437 ifp = ifunit_ref(hdr->nr_name); 1438 if (ifp) { /* already exist, cannot create new one */ 1439 error = EEXIST; 1440 NMG_LOCK(); 1441 if (NM_NA_VALID(ifp)) { 1442 int update_err = nm_update_info(req, NA(ifp)); 1443 if (update_err) 1444 error = update_err; 1445 } 1446 NMG_UNLOCK(); 1447 if_rele(ifp); 1448 return error; 1449 } 1450 error = nm_os_vi_persist(hdr->nr_name, &ifp); 1451 if (error) 1452 return error; 1453 1454 NMG_LOCK(); 1455 if (req->nr_mem_id) { 1456 nmd = netmap_mem_find(req->nr_mem_id); 1457 if (nmd == NULL) { 1458 error = EINVAL; 1459 goto err_1; 1460 } 1461 } 1462 /* netmap_vp_create creates a struct netmap_vp_adapter */ 1463 error = netmap_vale_vp_create(hdr, ifp, nmd, &vpna); 1464 if (error) { 1465 if (netmap_debug & NM_DEBUG_VALE) 1466 nm_prerr("error %d", error); 1467 goto err_1; 1468 } 1469 /* persist-specific routines */ 1470 vpna->up.nm_bdg_ctl = netmap_vp_bdg_ctl; 1471 if (!autodelete) { 1472 netmap_adapter_get(&vpna->up); 1473 } else { 1474 vpna->autodelete = 1; 1475 } 1476 NM_ATTACH_NA(ifp, &vpna->up); 1477 /* return the updated info */ 1478 error = nm_update_info(req, &vpna->up); 1479 if (error) { 1480 goto err_2; 1481 } 1482 nm_prdis("returning nr_mem_id %d", req->nr_mem_id); 1483 if (nmd) 1484 netmap_mem_put(nmd); 1485 NMG_UNLOCK(); 1486 nm_prdis("created %s", ifp->if_xname); 1487 return 0; 1488 1489 err_2: 1490 netmap_detach(ifp); 1491 err_1: 1492 if (nmd) 1493 netmap_mem_put(nmd); 1494 NMG_UNLOCK(); 1495 nm_os_vi_detach(ifp); 1496 1497 return error; 1498 } 1499 1500 #endif /* WITH_VALE */ 1501