1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2012 Chelsio Communications, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #ifdef TCP_OFFLOAD 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/module.h> 39 #include <sys/bus.h> 40 #include <sys/fnv_hash.h> 41 #include <sys/lock.h> 42 #include <sys/mutex.h> 43 #include <sys/rwlock.h> 44 #include <sys/socket.h> 45 #include <sys/sbuf.h> 46 #include <sys/taskqueue.h> 47 #include <net/if.h> 48 #include <net/if_types.h> 49 #include <net/ethernet.h> 50 #include <net/if_vlan_var.h> 51 #include <net/route.h> 52 #include <netinet/in.h> 53 #include <netinet/toecore.h> 54 55 #include "common/common.h" 56 #include "common/t4_msg.h" 57 #include "tom/t4_tom_l2t.h" 58 #include "tom/t4_tom.h" 59 60 #define VLAN_NONE 0xfff 61 62 static inline void 63 l2t_hold(struct l2t_data *d, struct l2t_entry *e) 64 { 65 66 if (atomic_fetchadd_int(&e->refcnt, 1) == 0) /* 0 -> 1 transition */ 67 atomic_subtract_int(&d->nfree, 1); 68 } 69 70 static inline u_int 71 l2_hash(struct l2t_data *d, const struct sockaddr *sa, int ifindex) 72 { 73 u_int hash, half = d->l2t_size / 2, start = 0; 74 const void *key; 75 size_t len; 76 77 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6, 78 ("%s: sa %p has unexpected sa_family %d", __func__, sa, 79 sa->sa_family)); 80 81 if (sa->sa_family == AF_INET) { 82 const struct sockaddr_in *sin = (const void *)sa; 83 84 key = &sin->sin_addr; 85 len = sizeof(sin->sin_addr); 86 } else { 87 const struct sockaddr_in6 *sin6 = (const void *)sa; 88 89 key = &sin6->sin6_addr; 90 len = sizeof(sin6->sin6_addr); 91 start = half; 92 } 93 94 hash = fnv_32_buf(key, len, FNV1_32_INIT); 95 hash = fnv_32_buf(&ifindex, sizeof(ifindex), hash); 96 hash %= half; 97 98 return (hash + start); 99 } 100 101 static inline int 102 l2_cmp(const struct sockaddr *sa, struct l2t_entry *e) 103 { 104 105 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6, 106 ("%s: sa %p has unexpected sa_family %d", __func__, sa, 107 sa->sa_family)); 108 109 if (sa->sa_family == AF_INET) { 110 const struct sockaddr_in *sin = (const void *)sa; 111 112 return (e->addr[0] != sin->sin_addr.s_addr); 113 } else { 114 const struct sockaddr_in6 *sin6 = (const void *)sa; 115 116 return (memcmp(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr))); 117 } 118 } 119 120 static inline void 121 l2_store(const struct sockaddr *sa, struct l2t_entry *e) 122 { 123 124 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6, 125 ("%s: sa %p has unexpected sa_family %d", __func__, sa, 126 sa->sa_family)); 127 128 if (sa->sa_family == AF_INET) { 129 const struct sockaddr_in *sin = (const void *)sa; 130 131 e->addr[0] = sin->sin_addr.s_addr; 132 e->ipv6 = 0; 133 } else { 134 const struct sockaddr_in6 *sin6 = (const void *)sa; 135 136 memcpy(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr)); 137 e->ipv6 = 1; 138 } 139 } 140 141 /* 142 * Add a WR to an L2T entry's queue of work requests awaiting resolution. 143 * Must be called with the entry's lock held. 144 */ 145 static inline void 146 arpq_enqueue(struct l2t_entry *e, struct wrqe *wr) 147 { 148 mtx_assert(&e->lock, MA_OWNED); 149 150 STAILQ_INSERT_TAIL(&e->wr_list, wr, link); 151 } 152 153 static inline void 154 send_pending(struct adapter *sc, struct l2t_entry *e) 155 { 156 struct wrqe *wr; 157 158 mtx_assert(&e->lock, MA_OWNED); 159 160 while ((wr = STAILQ_FIRST(&e->wr_list)) != NULL) { 161 STAILQ_REMOVE_HEAD(&e->wr_list, link); 162 t4_wrq_tx(sc, wr); 163 } 164 } 165 166 static void 167 resolution_failed(struct adapter *sc, struct l2t_entry *e) 168 { 169 struct tom_data *td = sc->tom_softc; 170 171 mtx_assert(&e->lock, MA_OWNED); 172 173 mtx_lock(&td->unsent_wr_lock); 174 STAILQ_CONCAT(&td->unsent_wr_list, &e->wr_list); 175 mtx_unlock(&td->unsent_wr_lock); 176 177 taskqueue_enqueue(taskqueue_thread, &td->reclaim_wr_resources); 178 } 179 180 static void 181 update_entry(struct adapter *sc, struct l2t_entry *e, uint8_t *lladdr, 182 uint16_t vtag) 183 { 184 185 mtx_assert(&e->lock, MA_OWNED); 186 187 /* 188 * The entry may be in active use (e->refcount > 0) or not. We update 189 * it even when it's not as this simplifies the case where we decide to 190 * reuse the entry later. 191 */ 192 193 if (lladdr == NULL && 194 (e->state == L2T_STATE_RESOLVING || e->state == L2T_STATE_FAILED)) { 195 /* 196 * Never got a valid L2 address for this one. Just mark it as 197 * failed instead of removing it from the hash (for which we'd 198 * need to wlock the table). 199 */ 200 e->state = L2T_STATE_FAILED; 201 resolution_failed(sc, e); 202 return; 203 204 } else if (lladdr == NULL) { 205 206 /* Valid or already-stale entry was deleted (or expired) */ 207 208 KASSERT(e->state == L2T_STATE_VALID || 209 e->state == L2T_STATE_STALE, 210 ("%s: lladdr NULL, state %d", __func__, e->state)); 211 212 e->state = L2T_STATE_STALE; 213 214 } else { 215 216 if (e->state == L2T_STATE_RESOLVING || 217 e->state == L2T_STATE_FAILED || 218 memcmp(e->dmac, lladdr, ETHER_ADDR_LEN)) { 219 220 /* unresolved -> resolved; or dmac changed */ 221 222 memcpy(e->dmac, lladdr, ETHER_ADDR_LEN); 223 e->vlan = vtag; 224 t4_write_l2e(e, 1); 225 } 226 e->state = L2T_STATE_VALID; 227 } 228 } 229 230 static int 231 resolve_entry(struct adapter *sc, struct l2t_entry *e) 232 { 233 struct tom_data *td = sc->tom_softc; 234 struct toedev *tod = &td->tod; 235 struct sockaddr_in sin = {0}; 236 struct sockaddr_in6 sin6 = {0}; 237 struct sockaddr *sa; 238 uint8_t dmac[ETHER_HDR_LEN]; 239 uint16_t vtag; 240 int rc; 241 242 if (e->ipv6 == 0) { 243 sin.sin_family = AF_INET; 244 sin.sin_len = sizeof(struct sockaddr_in); 245 sin.sin_addr.s_addr = e->addr[0]; 246 sa = (void *)&sin; 247 } else { 248 sin6.sin6_family = AF_INET6; 249 sin6.sin6_len = sizeof(struct sockaddr_in6); 250 memcpy(&sin6.sin6_addr, &e->addr[0], sizeof(e->addr)); 251 sa = (void *)&sin6; 252 } 253 254 vtag = EVL_MAKETAG(VLAN_NONE, 0, 0); 255 rc = toe_l2_resolve(tod, e->ifp, sa, dmac, &vtag); 256 if (rc == EWOULDBLOCK) 257 return (rc); 258 259 mtx_lock(&e->lock); 260 update_entry(sc, e, rc == 0 ? dmac : NULL, vtag); 261 mtx_unlock(&e->lock); 262 263 return (rc); 264 } 265 266 int 267 t4_l2t_send_slow(struct adapter *sc, struct wrqe *wr, struct l2t_entry *e) 268 { 269 270 again: 271 switch (e->state) { 272 case L2T_STATE_STALE: /* entry is stale, kick off revalidation */ 273 274 resolve_entry(sc, e); 275 276 /* Fall through */ 277 278 case L2T_STATE_VALID: /* fast-path, send the packet on */ 279 280 t4_wrq_tx(sc, wr); 281 return (0); 282 283 case L2T_STATE_RESOLVING: 284 case L2T_STATE_SYNC_WRITE: 285 286 mtx_lock(&e->lock); 287 if (e->state != L2T_STATE_SYNC_WRITE && 288 e->state != L2T_STATE_RESOLVING) { 289 /* state changed by the time we got here */ 290 mtx_unlock(&e->lock); 291 goto again; 292 } 293 arpq_enqueue(e, wr); 294 mtx_unlock(&e->lock); 295 296 if (resolve_entry(sc, e) == EWOULDBLOCK) 297 break; 298 299 mtx_lock(&e->lock); 300 if (e->state == L2T_STATE_VALID && !STAILQ_EMPTY(&e->wr_list)) 301 send_pending(sc, e); 302 if (e->state == L2T_STATE_FAILED) 303 resolution_failed(sc, e); 304 mtx_unlock(&e->lock); 305 break; 306 307 case L2T_STATE_FAILED: 308 return (EHOSTUNREACH); 309 } 310 311 return (0); 312 } 313 314 int 315 do_l2t_write_rpl2(struct sge_iq *iq, const struct rss_header *rss, 316 struct mbuf *m) 317 { 318 struct adapter *sc = iq->adapter; 319 const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1); 320 unsigned int tid = GET_TID(rpl); 321 unsigned int idx = tid % L2T_SIZE; 322 323 if (__predict_false(rpl->status != CPL_ERR_NONE)) { 324 log(LOG_ERR, 325 "Unexpected L2T_WRITE_RPL (%u) for entry at hw_idx %u\n", 326 rpl->status, idx); 327 return (EINVAL); 328 } 329 330 if (tid & F_SYNC_WR) { 331 struct l2t_entry *e = &sc->l2t->l2tab[idx - sc->vres.l2t.start]; 332 333 mtx_lock(&e->lock); 334 if (e->state != L2T_STATE_SWITCHING) { 335 send_pending(sc, e); 336 e->state = L2T_STATE_VALID; 337 } 338 mtx_unlock(&e->lock); 339 } 340 341 return (0); 342 } 343 344 /* 345 * The TOE wants an L2 table entry that it can use to reach the next hop over 346 * the specified port. Produce such an entry - create one if needed. 347 * 348 * Note that the ifnet could be a pseudo-device like if_vlan, if_lagg, etc. on 349 * top of the real cxgbe interface. 350 */ 351 struct l2t_entry * 352 t4_l2t_get(struct port_info *pi, struct ifnet *ifp, struct sockaddr *sa) 353 { 354 struct l2t_entry *e; 355 struct adapter *sc = pi->adapter; 356 struct l2t_data *d = sc->l2t; 357 u_int hash, smt_idx = pi->port_id; 358 uint16_t vid, pcp, vtag; 359 360 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6, 361 ("%s: sa %p has unexpected sa_family %d", __func__, sa, 362 sa->sa_family)); 363 364 vid = VLAN_NONE; 365 pcp = 0; 366 if (ifp->if_type == IFT_L2VLAN) { 367 VLAN_TAG(ifp, &vid); 368 VLAN_PCP(ifp, &pcp); 369 } else if (ifp->if_pcp != IFNET_PCP_NONE) { 370 vid = 0; 371 pcp = ifp->if_pcp; 372 } 373 vtag = EVL_MAKETAG(vid, pcp, 0); 374 375 hash = l2_hash(d, sa, ifp->if_index); 376 rw_wlock(&d->lock); 377 for (e = d->l2tab[hash].first; e; e = e->next) { 378 if (l2_cmp(sa, e) == 0 && e->ifp == ifp && e->vlan == vtag && 379 e->smt_idx == smt_idx) { 380 l2t_hold(d, e); 381 goto done; 382 } 383 } 384 385 /* Need to allocate a new entry */ 386 e = t4_alloc_l2e(d); 387 if (e) { 388 mtx_lock(&e->lock); /* avoid race with t4_l2t_free */ 389 e->next = d->l2tab[hash].first; 390 d->l2tab[hash].first = e; 391 392 e->state = L2T_STATE_RESOLVING; 393 l2_store(sa, e); 394 e->ifp = ifp; 395 e->smt_idx = smt_idx; 396 e->hash = hash; 397 e->lport = pi->lport; 398 e->wrq = &sc->sge.ctrlq[pi->port_id]; 399 e->iqid = sc->sge.ofld_rxq[pi->vi[0].first_ofld_rxq].iq.abs_id; 400 atomic_store_rel_int(&e->refcnt, 1); 401 e->vlan = vtag; 402 mtx_unlock(&e->lock); 403 } 404 done: 405 rw_wunlock(&d->lock); 406 return e; 407 } 408 409 /* 410 * Called when the host's ARP layer makes a change to some entry that is loaded 411 * into the HW L2 table. 412 */ 413 void 414 t4_l2_update(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa, 415 uint8_t *lladdr, uint16_t vtag) 416 { 417 struct adapter *sc = tod->tod_softc; 418 struct l2t_entry *e; 419 struct l2t_data *d = sc->l2t; 420 u_int hash; 421 422 KASSERT(d != NULL, ("%s: no L2 table", __func__)); 423 424 hash = l2_hash(d, sa, ifp->if_index); 425 rw_rlock(&d->lock); 426 for (e = d->l2tab[hash].first; e; e = e->next) { 427 if (l2_cmp(sa, e) == 0 && e->ifp == ifp) { 428 mtx_lock(&e->lock); 429 if (atomic_load_acq_int(&e->refcnt)) 430 goto found; 431 e->state = L2T_STATE_STALE; 432 mtx_unlock(&e->lock); 433 break; 434 } 435 } 436 rw_runlock(&d->lock); 437 438 /* 439 * This is of no interest to us. We've never had an offloaded 440 * connection to this destination, and we aren't attempting one right 441 * now. 442 */ 443 return; 444 445 found: 446 rw_runlock(&d->lock); 447 448 KASSERT(e->state != L2T_STATE_UNUSED, 449 ("%s: unused entry in the hash.", __func__)); 450 451 update_entry(sc, e, lladdr, vtag); 452 mtx_unlock(&e->lock); 453 } 454 #endif 455