1 /*- 2 * Copyright (c) 2012 Chelsio Communications, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 #include <sys/cdefs.h> 27 __FBSDID("$FreeBSD$"); 28 29 #include "opt_inet.h" 30 #include "opt_inet6.h" 31 32 #ifdef TCP_OFFLOAD 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/kernel.h> 36 #include <sys/module.h> 37 #include <sys/bus.h> 38 #include <sys/fnv_hash.h> 39 #include <sys/lock.h> 40 #include <sys/mutex.h> 41 #include <sys/rwlock.h> 42 #include <sys/socket.h> 43 #include <sys/sbuf.h> 44 #include <sys/taskqueue.h> 45 #include <net/if.h> 46 #include <net/if_types.h> 47 #include <net/ethernet.h> 48 #include <net/if_vlan_var.h> 49 #include <net/route.h> 50 #include <netinet/in.h> 51 #include <netinet/toecore.h> 52 53 #include "common/common.h" 54 #include "common/t4_msg.h" 55 #include "tom/t4_tom_l2t.h" 56 #include "tom/t4_tom.h" 57 58 #define VLAN_NONE 0xfff 59 60 static inline void 61 l2t_hold(struct l2t_data *d, struct l2t_entry *e) 62 { 63 64 if (atomic_fetchadd_int(&e->refcnt, 1) == 0) /* 0 -> 1 transition */ 65 atomic_subtract_int(&d->nfree, 1); 66 } 67 68 static inline u_int 69 l2_hash(struct l2t_data *d, const struct sockaddr *sa, int ifindex) 70 { 71 u_int hash, half = d->l2t_size / 2, start = 0; 72 const void *key; 73 size_t len; 74 75 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6, 76 ("%s: sa %p has unexpected sa_family %d", __func__, sa, 77 sa->sa_family)); 78 79 if (sa->sa_family == AF_INET) { 80 const struct sockaddr_in *sin = (const void *)sa; 81 82 key = &sin->sin_addr; 83 len = sizeof(sin->sin_addr); 84 } else { 85 const struct sockaddr_in6 *sin6 = (const void *)sa; 86 87 key = &sin6->sin6_addr; 88 len = sizeof(sin6->sin6_addr); 89 start = half; 90 } 91 92 hash = fnv_32_buf(key, len, FNV1_32_INIT); 93 hash = fnv_32_buf(&ifindex, sizeof(ifindex), hash); 94 hash %= half; 95 96 return (hash + start); 97 } 98 99 static inline int 100 l2_cmp(const struct sockaddr *sa, struct l2t_entry *e) 101 { 102 103 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6, 104 ("%s: sa %p has unexpected sa_family %d", __func__, sa, 105 sa->sa_family)); 106 107 if (sa->sa_family == AF_INET) { 108 const struct sockaddr_in *sin = (const void *)sa; 109 110 return (e->addr[0] != sin->sin_addr.s_addr); 111 } else { 112 const struct sockaddr_in6 *sin6 = (const void *)sa; 113 114 return (memcmp(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr))); 115 } 116 } 117 118 static inline void 119 l2_store(const struct sockaddr *sa, struct l2t_entry *e) 120 { 121 122 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6, 123 ("%s: sa %p has unexpected sa_family %d", __func__, sa, 124 sa->sa_family)); 125 126 if (sa->sa_family == AF_INET) { 127 const struct sockaddr_in *sin = (const void *)sa; 128 129 e->addr[0] = sin->sin_addr.s_addr; 130 e->ipv6 = 0; 131 } else { 132 const struct sockaddr_in6 *sin6 = (const void *)sa; 133 134 memcpy(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr)); 135 e->ipv6 = 1; 136 } 137 } 138 139 /* 140 * Add a WR to an L2T entry's queue of work requests awaiting resolution. 141 * Must be called with the entry's lock held. 142 */ 143 static inline void 144 arpq_enqueue(struct l2t_entry *e, struct wrqe *wr) 145 { 146 mtx_assert(&e->lock, MA_OWNED); 147 148 STAILQ_INSERT_TAIL(&e->wr_list, wr, link); 149 } 150 151 static inline void 152 send_pending(struct adapter *sc, struct l2t_entry *e) 153 { 154 struct wrqe *wr; 155 156 mtx_assert(&e->lock, MA_OWNED); 157 158 while ((wr = STAILQ_FIRST(&e->wr_list)) != NULL) { 159 STAILQ_REMOVE_HEAD(&e->wr_list, link); 160 t4_wrq_tx(sc, wr); 161 } 162 } 163 164 static void 165 resolution_failed(struct adapter *sc, struct l2t_entry *e) 166 { 167 struct tom_data *td = sc->tom_softc; 168 169 mtx_assert(&e->lock, MA_OWNED); 170 171 mtx_lock(&td->unsent_wr_lock); 172 STAILQ_CONCAT(&td->unsent_wr_list, &e->wr_list); 173 mtx_unlock(&td->unsent_wr_lock); 174 175 taskqueue_enqueue(taskqueue_thread, &td->reclaim_wr_resources); 176 } 177 178 static void 179 update_entry(struct adapter *sc, struct l2t_entry *e, uint8_t *lladdr, 180 uint16_t vtag) 181 { 182 183 mtx_assert(&e->lock, MA_OWNED); 184 185 /* 186 * The entry may be in active use (e->refcount > 0) or not. We update 187 * it even when it's not as this simplifies the case where we decide to 188 * reuse the entry later. 189 */ 190 191 if (lladdr == NULL && 192 (e->state == L2T_STATE_RESOLVING || e->state == L2T_STATE_FAILED)) { 193 /* 194 * Never got a valid L2 address for this one. Just mark it as 195 * failed instead of removing it from the hash (for which we'd 196 * need to wlock the table). 197 */ 198 e->state = L2T_STATE_FAILED; 199 resolution_failed(sc, e); 200 return; 201 202 } else if (lladdr == NULL) { 203 204 /* Valid or already-stale entry was deleted (or expired) */ 205 206 KASSERT(e->state == L2T_STATE_VALID || 207 e->state == L2T_STATE_STALE, 208 ("%s: lladdr NULL, state %d", __func__, e->state)); 209 210 e->state = L2T_STATE_STALE; 211 212 } else { 213 214 if (e->state == L2T_STATE_RESOLVING || 215 e->state == L2T_STATE_FAILED || 216 memcmp(e->dmac, lladdr, ETHER_ADDR_LEN)) { 217 218 /* unresolved -> resolved; or dmac changed */ 219 220 memcpy(e->dmac, lladdr, ETHER_ADDR_LEN); 221 e->vlan = vtag; 222 t4_write_l2e(e, 1); 223 } 224 e->state = L2T_STATE_VALID; 225 } 226 } 227 228 static int 229 resolve_entry(struct adapter *sc, struct l2t_entry *e) 230 { 231 struct tom_data *td = sc->tom_softc; 232 struct toedev *tod = &td->tod; 233 struct sockaddr_in sin = {0}; 234 struct sockaddr_in6 sin6 = {0}; 235 struct sockaddr *sa; 236 uint8_t dmac[ETHER_HDR_LEN]; 237 uint16_t vtag = VLAN_NONE; 238 int rc; 239 240 if (e->ipv6 == 0) { 241 sin.sin_family = AF_INET; 242 sin.sin_len = sizeof(struct sockaddr_in); 243 sin.sin_addr.s_addr = e->addr[0]; 244 sa = (void *)&sin; 245 } else { 246 sin6.sin6_family = AF_INET6; 247 sin6.sin6_len = sizeof(struct sockaddr_in6); 248 memcpy(&sin6.sin6_addr, &e->addr[0], sizeof(e->addr)); 249 sa = (void *)&sin6; 250 } 251 252 rc = toe_l2_resolve(tod, e->ifp, sa, dmac, &vtag); 253 if (rc == EWOULDBLOCK) 254 return (rc); 255 256 mtx_lock(&e->lock); 257 update_entry(sc, e, rc == 0 ? dmac : NULL, vtag); 258 mtx_unlock(&e->lock); 259 260 return (rc); 261 } 262 263 int 264 t4_l2t_send_slow(struct adapter *sc, struct wrqe *wr, struct l2t_entry *e) 265 { 266 267 again: 268 switch (e->state) { 269 case L2T_STATE_STALE: /* entry is stale, kick off revalidation */ 270 271 if (resolve_entry(sc, e) != EWOULDBLOCK) 272 goto again; /* entry updated, re-examine state */ 273 274 /* Fall through */ 275 276 case L2T_STATE_VALID: /* fast-path, send the packet on */ 277 278 t4_wrq_tx(sc, wr); 279 return (0); 280 281 case L2T_STATE_RESOLVING: 282 case L2T_STATE_SYNC_WRITE: 283 284 mtx_lock(&e->lock); 285 if (e->state != L2T_STATE_SYNC_WRITE && 286 e->state != L2T_STATE_RESOLVING) { 287 /* state changed by the time we got here */ 288 mtx_unlock(&e->lock); 289 goto again; 290 } 291 arpq_enqueue(e, wr); 292 mtx_unlock(&e->lock); 293 294 if (resolve_entry(sc, e) == EWOULDBLOCK) 295 break; 296 297 mtx_lock(&e->lock); 298 if (e->state == L2T_STATE_VALID && !STAILQ_EMPTY(&e->wr_list)) 299 send_pending(sc, e); 300 if (e->state == L2T_STATE_FAILED) 301 resolution_failed(sc, e); 302 mtx_unlock(&e->lock); 303 break; 304 305 case L2T_STATE_FAILED: 306 return (EHOSTUNREACH); 307 } 308 309 return (0); 310 } 311 312 int 313 do_l2t_write_rpl2(struct sge_iq *iq, const struct rss_header *rss, 314 struct mbuf *m) 315 { 316 struct adapter *sc = iq->adapter; 317 const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1); 318 unsigned int tid = GET_TID(rpl); 319 unsigned int idx = tid % L2T_SIZE; 320 321 if (__predict_false(rpl->status != CPL_ERR_NONE)) { 322 log(LOG_ERR, 323 "Unexpected L2T_WRITE_RPL (%u) for entry at hw_idx %u\n", 324 rpl->status, idx); 325 return (EINVAL); 326 } 327 328 if (tid & F_SYNC_WR) { 329 struct l2t_entry *e = &sc->l2t->l2tab[idx - sc->vres.l2t.start]; 330 331 mtx_lock(&e->lock); 332 if (e->state != L2T_STATE_SWITCHING) { 333 send_pending(sc, e); 334 e->state = L2T_STATE_VALID; 335 } 336 mtx_unlock(&e->lock); 337 } 338 339 return (0); 340 } 341 342 /* 343 * The TOE wants an L2 table entry that it can use to reach the next hop over 344 * the specified port. Produce such an entry - create one if needed. 345 * 346 * Note that the ifnet could be a pseudo-device like if_vlan, if_lagg, etc. on 347 * top of the real cxgbe interface. 348 */ 349 struct l2t_entry * 350 t4_l2t_get(struct port_info *pi, struct ifnet *ifp, struct sockaddr *sa) 351 { 352 struct l2t_entry *e; 353 struct adapter *sc = pi->adapter; 354 struct l2t_data *d = sc->l2t; 355 u_int hash, smt_idx = pi->port_id; 356 357 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6, 358 ("%s: sa %p has unexpected sa_family %d", __func__, sa, 359 sa->sa_family)); 360 361 #ifndef VLAN_TAG 362 if (ifp->if_type == IFT_L2VLAN) 363 return (NULL); 364 #endif 365 366 hash = l2_hash(d, sa, ifp->if_index); 367 rw_wlock(&d->lock); 368 for (e = d->l2tab[hash].first; e; e = e->next) { 369 if (l2_cmp(sa, e) == 0 && e->ifp == ifp && 370 e->smt_idx == smt_idx) { 371 l2t_hold(d, e); 372 goto done; 373 } 374 } 375 376 /* Need to allocate a new entry */ 377 e = t4_alloc_l2e(d); 378 if (e) { 379 mtx_lock(&e->lock); /* avoid race with t4_l2t_free */ 380 e->next = d->l2tab[hash].first; 381 d->l2tab[hash].first = e; 382 383 e->state = L2T_STATE_RESOLVING; 384 l2_store(sa, e); 385 e->ifp = ifp; 386 e->smt_idx = smt_idx; 387 e->hash = hash; 388 e->lport = pi->lport; 389 e->wrq = &sc->sge.ctrlq[pi->port_id]; 390 e->iqid = sc->sge.ofld_rxq[pi->vi[0].first_ofld_rxq].iq.abs_id; 391 atomic_store_rel_int(&e->refcnt, 1); 392 #ifdef VLAN_TAG 393 if (ifp->if_type == IFT_L2VLAN) 394 VLAN_TAG(ifp, &e->vlan); 395 else 396 e->vlan = VLAN_NONE; 397 #endif 398 mtx_unlock(&e->lock); 399 } 400 done: 401 rw_wunlock(&d->lock); 402 return e; 403 } 404 405 /* 406 * Called when the host's ARP layer makes a change to some entry that is loaded 407 * into the HW L2 table. 408 */ 409 void 410 t4_l2_update(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa, 411 uint8_t *lladdr, uint16_t vtag) 412 { 413 struct adapter *sc = tod->tod_softc; 414 struct l2t_entry *e; 415 struct l2t_data *d = sc->l2t; 416 u_int hash; 417 418 KASSERT(d != NULL, ("%s: no L2 table", __func__)); 419 420 hash = l2_hash(d, sa, ifp->if_index); 421 rw_rlock(&d->lock); 422 for (e = d->l2tab[hash].first; e; e = e->next) { 423 if (l2_cmp(sa, e) == 0 && e->ifp == ifp) { 424 mtx_lock(&e->lock); 425 if (atomic_load_acq_int(&e->refcnt)) 426 goto found; 427 e->state = L2T_STATE_STALE; 428 mtx_unlock(&e->lock); 429 break; 430 } 431 } 432 rw_runlock(&d->lock); 433 434 /* 435 * This is of no interest to us. We've never had an offloaded 436 * connection to this destination, and we aren't attempting one right 437 * now. 438 */ 439 return; 440 441 found: 442 rw_runlock(&d->lock); 443 444 KASSERT(e->state != L2T_STATE_UNUSED, 445 ("%s: unused entry in the hash.", __func__)); 446 447 update_entry(sc, e, lladdr, vtag); 448 mtx_unlock(&e->lock); 449 } 450 #endif 451