xref: /freebsd/sys/dev/cxgbe/tom/t4_tom_l2t.c (revision f5f7c05209ca2c3748fd8b27c5e80ffad49120eb)
1 /*-
2  * Copyright (c) 2012 Chelsio Communications, Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 #include <sys/cdefs.h>
27 __FBSDID("$FreeBSD$");
28 
29 #include "opt_inet.h"
30 #include "opt_inet6.h"
31 
32 #ifdef TCP_OFFLOAD
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
36 #include <sys/module.h>
37 #include <sys/bus.h>
38 #include <sys/fnv_hash.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <sys/rwlock.h>
42 #include <sys/socket.h>
43 #include <sys/sbuf.h>
44 #include <net/if.h>
45 #include <net/if_types.h>
46 #include <net/ethernet.h>
47 #include <net/if_vlan_var.h>
48 #include <net/route.h>
49 #include <netinet/in.h>
50 #include <netinet/toecore.h>
51 
52 #include "common/common.h"
53 #include "common/t4_msg.h"
54 #include "tom/t4_tom_l2t.h"
55 #include "tom/t4_tom.h"
56 
57 #define VLAN_NONE	0xfff
58 
59 static inline void
60 l2t_hold(struct l2t_data *d, struct l2t_entry *e)
61 {
62 
63 	if (atomic_fetchadd_int(&e->refcnt, 1) == 0)  /* 0 -> 1 transition */
64 		atomic_subtract_int(&d->nfree, 1);
65 }
66 
67 static inline u_int
68 l2_hash(struct l2t_data *d, const struct sockaddr *sa, int ifindex)
69 {
70 	u_int hash, half = d->l2t_size / 2, start = 0;
71 	const void *key;
72 	size_t len;
73 
74 	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
75 	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
76 	    sa->sa_family));
77 
78 	if (sa->sa_family == AF_INET) {
79 		const struct sockaddr_in *sin = (const void *)sa;
80 
81 		key = &sin->sin_addr;
82 		len = sizeof(sin->sin_addr);
83 	} else {
84 		const struct sockaddr_in6 *sin6 = (const void *)sa;
85 
86 		key = &sin6->sin6_addr;
87 		len = sizeof(sin6->sin6_addr);
88 		start = half;
89 	}
90 
91 	hash = fnv_32_buf(key, len, FNV1_32_INIT);
92 	hash = fnv_32_buf(&ifindex, sizeof(ifindex), hash);
93 	hash %= half;
94 
95 	return (hash + start);
96 }
97 
98 static inline int
99 l2_cmp(const struct sockaddr *sa, struct l2t_entry *e)
100 {
101 
102 	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
103 	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
104 	    sa->sa_family));
105 
106 	if (sa->sa_family == AF_INET) {
107 		const struct sockaddr_in *sin = (const void *)sa;
108 
109 		return (e->addr[0] != sin->sin_addr.s_addr);
110 	} else {
111 		const struct sockaddr_in6 *sin6 = (const void *)sa;
112 
113 		return (memcmp(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr)));
114 	}
115 }
116 
117 static inline void
118 l2_store(const struct sockaddr *sa, struct l2t_entry *e)
119 {
120 
121 	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
122 	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
123 	    sa->sa_family));
124 
125 	if (sa->sa_family == AF_INET) {
126 		const struct sockaddr_in *sin = (const void *)sa;
127 
128 		e->addr[0] = sin->sin_addr.s_addr;
129 		e->ipv6 = 0;
130 	} else {
131 		const struct sockaddr_in6 *sin6 = (const void *)sa;
132 
133 		memcpy(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr));
134 		e->ipv6 = 1;
135 	}
136 }
137 
138 /*
139  * Add a WR to an L2T entry's queue of work requests awaiting resolution.
140  * Must be called with the entry's lock held.
141  */
142 static inline void
143 arpq_enqueue(struct l2t_entry *e, struct wrqe *wr)
144 {
145 	mtx_assert(&e->lock, MA_OWNED);
146 
147 	STAILQ_INSERT_TAIL(&e->wr_list, wr, link);
148 }
149 
150 static inline void
151 send_pending(struct adapter *sc, struct l2t_entry *e)
152 {
153 	struct wrqe *wr;
154 
155 	mtx_assert(&e->lock, MA_OWNED);
156 
157 	while ((wr = STAILQ_FIRST(&e->wr_list)) != NULL) {
158 		STAILQ_REMOVE_HEAD(&e->wr_list, link);
159 		t4_wrq_tx(sc, wr);
160 	}
161 }
162 
163 static void
164 resolution_failed_for_wr(struct wrqe *wr)
165 {
166 	log(LOG_ERR, "%s: leaked work request %p, wr_len %d\n", __func__, wr,
167 	    wr->wr_len);
168 
169 	/* free(wr, M_CXGBE); */
170 }
171 
172 static void
173 resolution_failed(struct l2t_entry *e)
174 {
175 	struct wrqe *wr;
176 
177 	mtx_assert(&e->lock, MA_OWNED);
178 
179 	while ((wr = STAILQ_FIRST(&e->wr_list)) != NULL) {
180 		STAILQ_REMOVE_HEAD(&e->wr_list, link);
181 		resolution_failed_for_wr(wr);
182 	}
183 }
184 
185 static void
186 update_entry(struct adapter *sc, struct l2t_entry *e, uint8_t *lladdr,
187     uint16_t vtag)
188 {
189 
190 	mtx_assert(&e->lock, MA_OWNED);
191 
192 	/*
193 	 * The entry may be in active use (e->refcount > 0) or not.  We update
194 	 * it even when it's not as this simplifies the case where we decide to
195 	 * reuse the entry later.
196 	 */
197 
198 	if (lladdr == NULL &&
199 	    (e->state == L2T_STATE_RESOLVING || e->state == L2T_STATE_FAILED)) {
200 		/*
201 		 * Never got a valid L2 address for this one.  Just mark it as
202 		 * failed instead of removing it from the hash (for which we'd
203 		 * need to wlock the table).
204 		 */
205 		e->state = L2T_STATE_FAILED;
206 		resolution_failed(e);
207 		return;
208 
209 	} else if (lladdr == NULL) {
210 
211 		/* Valid or already-stale entry was deleted (or expired) */
212 
213 		KASSERT(e->state == L2T_STATE_VALID ||
214 		    e->state == L2T_STATE_STALE,
215 		    ("%s: lladdr NULL, state %d", __func__, e->state));
216 
217 		e->state = L2T_STATE_STALE;
218 
219 	} else {
220 
221 		if (e->state == L2T_STATE_RESOLVING ||
222 		    e->state == L2T_STATE_FAILED ||
223 		    memcmp(e->dmac, lladdr, ETHER_ADDR_LEN)) {
224 
225 			/* unresolved -> resolved; or dmac changed */
226 
227 			memcpy(e->dmac, lladdr, ETHER_ADDR_LEN);
228 			e->vlan = vtag;
229 			t4_write_l2e(sc, e, 1);
230 		}
231 		e->state = L2T_STATE_VALID;
232 	}
233 }
234 
235 static int
236 resolve_entry(struct adapter *sc, struct l2t_entry *e)
237 {
238 	struct tom_data *td = sc->tom_softc;
239 	struct toedev *tod = &td->tod;
240 	struct sockaddr_in sin = {0};
241 	struct sockaddr_in6 sin6 = {0};
242 	struct sockaddr *sa;
243 	uint8_t dmac[ETHER_ADDR_LEN];
244 	uint16_t vtag = VLAN_NONE;
245 	int rc;
246 
247 	if (e->ipv6 == 0) {
248 		sin.sin_family = AF_INET;
249 		sin.sin_len = sizeof(struct sockaddr_in);
250 		sin.sin_addr.s_addr = e->addr[0];
251 		sa = (void *)&sin;
252 	} else {
253 		sin6.sin6_family = AF_INET6;
254 		sin6.sin6_len = sizeof(struct sockaddr_in6);
255 		memcpy(&sin6.sin6_addr, &e->addr[0], sizeof(e->addr));
256 		sa = (void *)&sin6;
257 	}
258 
259 	rc = toe_l2_resolve(tod, e->ifp, sa, dmac, &vtag);
260 	if (rc == EWOULDBLOCK)
261 		return (rc);
262 
263 	mtx_lock(&e->lock);
264 	update_entry(sc, e, rc == 0 ? dmac : NULL, vtag);
265 	mtx_unlock(&e->lock);
266 
267 	return (rc);
268 }
269 
270 int
271 t4_l2t_send_slow(struct adapter *sc, struct wrqe *wr, struct l2t_entry *e)
272 {
273 
274 again:
275 	switch (e->state) {
276 	case L2T_STATE_STALE:     /* entry is stale, kick off revalidation */
277 
278 		if (resolve_entry(sc, e) != EWOULDBLOCK)
279 			goto again;	/* entry updated, re-examine state */
280 
281 		/* Fall through */
282 
283 	case L2T_STATE_VALID:     /* fast-path, send the packet on */
284 
285 		t4_wrq_tx(sc, wr);
286 		return (0);
287 
288 	case L2T_STATE_RESOLVING:
289 	case L2T_STATE_SYNC_WRITE:
290 
291 		mtx_lock(&e->lock);
292 		if (e->state != L2T_STATE_SYNC_WRITE &&
293 		    e->state != L2T_STATE_RESOLVING) {
294 			/* state changed by the time we got here */
295 			mtx_unlock(&e->lock);
296 			goto again;
297 		}
298 		arpq_enqueue(e, wr);
299 		mtx_unlock(&e->lock);
300 
301 		if (resolve_entry(sc, e) == EWOULDBLOCK)
302 			break;
303 
304 		mtx_lock(&e->lock);
305 		if (e->state == L2T_STATE_VALID && !STAILQ_EMPTY(&e->wr_list))
306 			send_pending(sc, e);
307 		if (e->state == L2T_STATE_FAILED)
308 			resolution_failed(e);
309 		mtx_unlock(&e->lock);
310 		break;
311 
312 	case L2T_STATE_FAILED:
313 		resolution_failed_for_wr(wr);
314 		return (EHOSTUNREACH);
315 	}
316 
317 	return (0);
318 }
319 
320 /*
321  * Called when an L2T entry has no more users.  The entry is left in the hash
322  * table since it is likely to be reused but we also bump nfree to indicate
323  * that the entry can be reallocated for a different neighbor.  We also drop
324  * the existing neighbor reference in case the neighbor is going away and is
325  * waiting on our reference.
326  *
327  * Because entries can be reallocated to other neighbors once their ref count
328  * drops to 0 we need to take the entry's lock to avoid races with a new
329  * incarnation.
330  */
331 
332 static int
333 do_l2t_write_rpl2(struct sge_iq *iq, const struct rss_header *rss,
334     struct mbuf *m)
335 {
336 	struct adapter *sc = iq->adapter;
337 	const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
338 	unsigned int tid = GET_TID(rpl);
339 	unsigned int idx = tid % L2T_SIZE;
340 	int rc;
341 
342 	rc = do_l2t_write_rpl(iq, rss, m);
343 	if (rc != 0)
344 		return (rc);
345 
346 	if (tid & F_SYNC_WR) {
347 		struct l2t_entry *e = &sc->l2t->l2tab[idx - sc->vres.l2t.start];
348 
349 		mtx_lock(&e->lock);
350 		if (e->state != L2T_STATE_SWITCHING) {
351 			send_pending(sc, e);
352 			e->state = L2T_STATE_VALID;
353 		}
354 		mtx_unlock(&e->lock);
355 	}
356 
357 	return (0);
358 }
359 
360 void
361 t4_init_l2t_cpl_handlers(struct adapter *sc)
362 {
363 
364 	t4_register_cpl_handler(sc, CPL_L2T_WRITE_RPL, do_l2t_write_rpl2);
365 }
366 
367 void
368 t4_uninit_l2t_cpl_handlers(struct adapter *sc)
369 {
370 
371 	t4_register_cpl_handler(sc, CPL_L2T_WRITE_RPL, do_l2t_write_rpl);
372 }
373 
374 /*
375  * The TOE wants an L2 table entry that it can use to reach the next hop over
376  * the specified port.  Produce such an entry - create one if needed.
377  *
378  * Note that the ifnet could be a pseudo-device like if_vlan, if_lagg, etc. on
379  * top of the real cxgbe interface.
380  */
381 struct l2t_entry *
382 t4_l2t_get(struct port_info *pi, struct ifnet *ifp, struct sockaddr *sa)
383 {
384 	struct l2t_entry *e;
385 	struct l2t_data *d = pi->adapter->l2t;
386 	u_int hash, smt_idx = pi->port_id;
387 
388 	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
389 	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
390 	    sa->sa_family));
391 
392 #ifndef VLAN_TAG
393 	if (ifp->if_type == IFT_L2VLAN)
394 		return (NULL);
395 #endif
396 
397 	hash = l2_hash(d, sa, ifp->if_index);
398 	rw_wlock(&d->lock);
399 	for (e = d->l2tab[hash].first; e; e = e->next) {
400 		if (l2_cmp(sa, e) == 0 && e->ifp == ifp &&
401 		    e->smt_idx == smt_idx) {
402 			l2t_hold(d, e);
403 			goto done;
404 		}
405 	}
406 
407 	/* Need to allocate a new entry */
408 	e = t4_alloc_l2e(d);
409 	if (e) {
410 		mtx_lock(&e->lock);          /* avoid race with t4_l2t_free */
411 		e->next = d->l2tab[hash].first;
412 		d->l2tab[hash].first = e;
413 
414 		e->state = L2T_STATE_RESOLVING;
415 		l2_store(sa, e);
416 		e->ifp = ifp;
417 		e->smt_idx = smt_idx;
418 		e->hash = hash;
419 		e->lport = pi->lport;
420 		atomic_store_rel_int(&e->refcnt, 1);
421 #ifdef VLAN_TAG
422 		if (ifp->if_type == IFT_L2VLAN)
423 			VLAN_TAG(ifp, &e->vlan);
424 		else
425 			e->vlan = VLAN_NONE;
426 #endif
427 		mtx_unlock(&e->lock);
428 	}
429 done:
430 	rw_wunlock(&d->lock);
431 	return e;
432 }
433 
434 /*
435  * Called when the host's ARP layer makes a change to some entry that is loaded
436  * into the HW L2 table.
437  */
438 void
439 t4_l2_update(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa,
440     uint8_t *lladdr, uint16_t vtag)
441 {
442 	struct adapter *sc = tod->tod_softc;
443 	struct l2t_entry *e;
444 	struct l2t_data *d = sc->l2t;
445 	u_int hash;
446 
447 	KASSERT(d != NULL, ("%s: no L2 table", __func__));
448 
449 	hash = l2_hash(d, sa, ifp->if_index);
450 	rw_rlock(&d->lock);
451 	for (e = d->l2tab[hash].first; e; e = e->next) {
452 		if (l2_cmp(sa, e) == 0 && e->ifp == ifp) {
453 			mtx_lock(&e->lock);
454 			if (atomic_load_acq_int(&e->refcnt))
455 				goto found;
456 			e->state = L2T_STATE_STALE;
457 			mtx_unlock(&e->lock);
458 			break;
459 		}
460 	}
461 	rw_runlock(&d->lock);
462 
463 	/*
464 	 * This is of no interest to us.  We've never had an offloaded
465 	 * connection to this destination, and we aren't attempting one right
466 	 * now.
467 	 */
468 	return;
469 
470 found:
471 	rw_runlock(&d->lock);
472 
473 	KASSERT(e->state != L2T_STATE_UNUSED,
474 	    ("%s: unused entry in the hash.", __func__));
475 
476 	update_entry(sc, e, lladdr, vtag);
477 	mtx_unlock(&e->lock);
478 }
479 #endif
480