xref: /illumos-gate/usr/src/uts/common/io/cxgbe/t4nex/t4_l2t.c (revision 94c3dad2979525d0a82595f3d8350a6116aba8ed)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source. A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * This file is part of the Chelsio T4 support code.
14  *
15  * Copyright (C) 2010-2013 Chelsio Communications.  All rights reserved.
16  *
17  * This program is distributed in the hope that it will be useful, but WITHOUT
18  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19  * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
20  * release for licensing terms and conditions.
21  */
22 
23 #include <sys/ddi.h>
24 #include <sys/sunddi.h>
25 #include <sys/sunndi.h>
26 #include <sys/atomic.h>
27 #include <sys/dlpi.h>
28 #include <sys/pattr.h>
29 #include <sys/strsubr.h>
30 #include <sys/stream.h>
31 #include <sys/strsun.h>
32 #include <sys/ethernet.h>
33 #include <sys/containerof.h>
34 #include <inet/ip.h>
35 #include <inet/ipclassifier.h>
36 #include <inet/tcp.h>
37 
38 #include "common/common.h"
39 #include "common/t4_msg.h"
40 #include "common/t4_regs.h"
41 #include "common/t4_regs_values.h"
42 #include "t4_l2t.h"
43 
44 /* identifies sync vs async L2T_WRITE_REQs */
45 #define	S_SYNC_WR	12
46 #define	V_SYNC_WR(x)	((x) << S_SYNC_WR)
47 #define	F_SYNC_WR	V_SYNC_WR(1)
48 #define	VLAN_NONE	0xfff
49 
50 /*
51  * jhash.h: Jenkins hash support.
52  *
53  * Copyright (C) 1996 Bob Jenkins (bob_jenkins@burtleburtle.net)
54  *
55  * http://burtleburtle.net/bob/hash/
56  *
57  * These are the credits from Bob's sources:
58  *
59  * lookup2.c, by Bob Jenkins, December 1996, Public Domain.
60  * hash(), hash2(), hash3, and mix() are externally useful functions.
61  * Routines to test the hash are included if SELF_TEST is defined.
62  * You can use this free for any purpose.  It has no warranty.
63  */
64 
65 /* NOTE: Arguments are modified. */
66 #define	__jhash_mix(a, b, c) \
67 { \
68 	a -= b; a -= c; a ^= (c>>13); \
69 	b -= c; b -= a; b ^= (a<<8); \
70 	c -= a; c -= b; c ^= (b>>13); \
71 	a -= b; a -= c; a ^= (c>>12);  \
72 	b -= c; b -= a; b ^= (a<<16); \
73 	c -= a; c -= b; c ^= (b>>5); \
74 	a -= b; a -= c; a ^= (c>>3);  \
75 	b -= c; b -= a; b ^= (a<<10); \
76 	c -= a; c -= b; c ^= (b>>15); \
77 }
78 
79 /* The golden ration: an arbitrary value */
80 #define	JHASH_GOLDEN_RATIO	0x9e3779b9
81 
82 /*
83  * A special ultra-optimized versions that knows they are hashing exactly
84  * 3, 2 or 1 word(s).
85  *
86  * NOTE: In partilar the "c += length; __jhash_mix(a,b,c);" normally
87  *	 done at the end is not done here.
88  */
89 static inline u32
jhash_3words(u32 a,u32 b,u32 c,u32 initval)90 jhash_3words(u32 a, u32 b, u32 c, u32 initval)
91 {
92 	a += JHASH_GOLDEN_RATIO;
93 	b += JHASH_GOLDEN_RATIO;
94 	c += initval;
95 
96 	__jhash_mix(a, b, c);
97 
98 	return (c);
99 }
100 
101 static inline u32
jhash_2words(u32 a,u32 b,u32 initval)102 jhash_2words(u32 a, u32 b, u32 initval)
103 {
104 	return (jhash_3words(a, b, 0, initval));
105 }
106 
107 #if defined(__GNUC__)
108 #define	likely(x)	__builtin_expect((x), 1)
109 #define	unlikely(x)	__builtin_expect((x), 0)
110 #else
111 #define	likely(x)	(x)
112 #define	unlikely(x)	(x)
113 #endif /* defined(__GNUC__) */
114 
115 enum {
116 	L2T_STATE_VALID,	/* entry is up to date */
117 	L2T_STATE_STALE,	/* entry may be used but needs revalidation */
118 	L2T_STATE_RESOLVING,	/* entry needs address resolution */
119 	L2T_STATE_SYNC_WRITE,	/* synchronous write of entry underway */
120 
121 	/* when state is one of the below the entry is not hashed */
122 	L2T_STATE_SWITCHING,	/* entry is being used by a switching filter */
123 	L2T_STATE_UNUSED	/* entry not in use */
124 };
125 
126 struct l2t_data {
127 	krwlock_t lock;
128 	u_int l2t_size;
129 	volatile uint_t nfree;	 /* number of free entries */
130 	struct l2t_entry *rover; /* starting point for next allocation */
131 	struct l2t_entry l2tab[];
132 };
133 
134 #define	VLAN_NONE	0xfff
135 #define	SA(x)		((struct sockaddr *)(x))
136 #define	SIN(x)		((struct sockaddr_in *)(x))
137 #define	SINADDR(x)	(SIN(x)->sin_addr.s_addr)
138 #define	atomic_read(x) atomic_add_int_nv(x, 0)
139 
140 #ifdef TCP_OFFLOAD_ENABLE
141 /*
142  * Allocate a free L2T entry.
143  * Must be called with l2t_data.lockatomic_load_acq_int held.
144  */
145 static struct l2t_entry *
alloc_l2e(struct l2t_data * d)146 alloc_l2e(struct l2t_data *d)
147 {
148 	struct l2t_entry *end, *e, **p;
149 
150 	ASSERT(rw_write_held(&d->lock));
151 
152 	if (!atomic_read(&d->nfree))
153 		return (NULL);
154 
155 	/* there's definitely a free entry */
156 	for (e = d->rover, end = &d->l2tab[d->l2t_size]; e != end; ++e)
157 		if (atomic_read(&e->refcnt) == 0)
158 			goto found;
159 
160 	for (e = d->l2tab; atomic_read(&e->refcnt); ++e)
161 		/* */;
162 found:
163 	d->rover = e + 1;
164 	atomic_dec_uint(&d->nfree);
165 
166 	/*
167 	 * The entry we found may be an inactive entry that is
168 	 * presently in the hash table.  We need to remove it.
169 	 */
170 	if (e->state < L2T_STATE_SWITCHING) {
171 		for (p = &d->l2tab[e->hash].first; *p; p = &(*p)->next) {
172 			if (*p == e) {
173 				*p = e->next;
174 				e->next = NULL;
175 				break;
176 			}
177 		}
178 	}
179 
180 	e->state = L2T_STATE_UNUSED;
181 	return (e);
182 }
183 
184 /*
185  * Write an L2T entry.  Must be called with the entry locked.
186  * The write may be synchronous or asynchronous.
187  */
188 static int
write_l2e(adapter_t * sc,struct l2t_entry * e,int sync)189 write_l2e(adapter_t *sc, struct l2t_entry *e, int sync)
190 {
191 	mblk_t *m;
192 	struct cpl_l2t_write_req *req;
193 	int idx = e->idx + sc->vres.l2t.start;
194 
195 	ASSERT(MUTEX_HELD(&e->lock));
196 
197 	if ((m = allocb(sizeof (*req), BPRI_HI)) == NULL)
198 		return (ENOMEM);
199 
200 	/* LINTED: E_BAD_PTR_CAST_ALIGN */
201 	req = (struct cpl_l2t_write_req *)m->b_wptr;
202 
203 	/* LINTED: E_CONSTANT_CONDITION */
204 	INIT_TP_WR(req, 0);
205 	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, idx |
206 	    V_SYNC_WR(sync) | V_TID_QID(sc->sge.fwq.abs_id)));
207 	req->params = htons(V_L2T_W_PORT(e->lport) | V_L2T_W_NOREPLY(!sync));
208 	req->l2t_idx = htons(idx);
209 	req->vlan = htons(e->vlan);
210 	(void) memcpy(req->dst_mac, e->dmac, sizeof (req->dst_mac));
211 
212 	m->b_wptr += sizeof (*req);
213 
214 	(void) t4_mgmt_tx(sc, m);
215 
216 	if (sync && e->state != L2T_STATE_SWITCHING)
217 		e->state = L2T_STATE_SYNC_WRITE;
218 
219 	return (0);
220 }
221 #endif
222 
223 struct l2t_data *
t4_init_l2t(struct adapter * sc)224 t4_init_l2t(struct adapter *sc)
225 {
226 	int i, l2t_size;
227 	struct l2t_data *d;
228 
229 	l2t_size = sc->vres.l2t.size;
230 	if(l2t_size < 1)
231 		return (NULL);
232 
233 	d = kmem_zalloc(sizeof(*d) + l2t_size * sizeof (struct l2t_entry), KM_SLEEP);
234 	if (!d)
235 		return (NULL);
236 
237 	d->l2t_size = l2t_size;
238 
239 	d->rover = d->l2tab;
240 	(void) atomic_swap_uint(&d->nfree, l2t_size);
241 	rw_init(&d->lock, NULL, RW_DRIVER, NULL);
242 
243 	for (i = 0; i < l2t_size; i++) {
244 		/* LINTED: E_ASSIGN_NARROW_CONV */
245 		d->l2tab[i].idx = i;
246 		d->l2tab[i].state = L2T_STATE_UNUSED;
247 		mutex_init(&d->l2tab[i].lock, NULL, MUTEX_DRIVER, NULL);
248 		(void) atomic_swap_uint(&d->l2tab[i].refcnt, 0);
249 	}
250 
251 #ifdef TCP_OFFLOAD_ENABLE
252 	(void) t4_register_cpl_handler(sc, CPL_L2T_WRITE_RPL, do_l2t_write_rpl);
253 #endif
254 
255 	return (d);
256 }
257 
258 int
t4_free_l2t(struct l2t_data * d)259 t4_free_l2t(struct l2t_data *d)
260 {
261 	int i;
262 
263 	for (i = 0; i < L2T_SIZE; i++)
264 		mutex_destroy(&d->l2tab[i].lock);
265 	rw_destroy(&d->lock);
266 	kmem_free(d, sizeof (*d));
267 
268 	return (0);
269 }
270 
271 #ifdef TCP_OFFLOAD_ENABLE
272 static inline void
l2t_hold(struct l2t_data * d,struct l2t_entry * e)273 l2t_hold(struct l2t_data *d, struct l2t_entry *e)
274 {
275 	if (atomic_inc_uint_nv(&e->refcnt) == 1)  /* 0 -> 1 transition */
276 		atomic_dec_uint(&d->nfree);
277 }
278 
279 /*
280  * To avoid having to check address families we do not allow v4 and v6
281  * neighbors to be on the same hash chain.  We keep v4 entries in the first
282  * half of available hash buckets and v6 in the second.
283  */
284 enum {
285 	L2T_SZ_HALF = L2T_SIZE / 2,
286 	L2T_HASH_MASK = L2T_SZ_HALF - 1
287 };
288 
289 static inline unsigned int
arp_hash(const uint32_t * key,int ifindex)290 arp_hash(const uint32_t *key, int ifindex)
291 {
292 	return (jhash_2words(*key, ifindex, 0) & L2T_HASH_MASK);
293 }
294 
295 static inline unsigned int
ipv6_hash(const uint32_t * key,int ifindex)296 ipv6_hash(const uint32_t *key, int ifindex)
297 {
298 	uint32_t xor = key[0] ^ key[1] ^ key[2] ^ key[3];
299 
300 	return (L2T_SZ_HALF + (jhash_2words(xor, ifindex, 0) & L2T_HASH_MASK));
301 }
302 
303 static inline unsigned int
addr_hash(const uint32_t * addr,int addr_len,int ifindex)304 addr_hash(const uint32_t *addr, int addr_len, int ifindex)
305 {
306 	return (addr_len == 4 ? arp_hash(addr, ifindex) :
307 	    ipv6_hash(addr, ifindex));
308 }
309 
310 /*
311  * Checks if an L2T entry is for the given IP/IPv6 address.  It does not check
312  * whether the L2T entry and the address are of the same address family.
313  * Callers ensure an address is only checked against L2T entries of the same
314  * family, something made trivial by the separation of IP and IPv6 hash chains
315  * mentioned above.  Returns 0 if there's a match,
316  */
317 static inline int
addreq(const struct l2t_entry * e,const uint32_t * addr)318 addreq(const struct l2t_entry *e, const uint32_t *addr)
319 {
320 	if (e->v6 != 0)
321 		return ((e->addr[0] ^ addr[0]) | (e->addr[1] ^ addr[1]) |
322 		    (e->addr[2] ^ addr[2]) | (e->addr[3] ^ addr[3]));
323 	return (e->addr[0] ^ addr[0]);
324 }
325 
326 /*
327  * Add a packet to an L2T entry's queue of packets awaiting resolution.
328  * Must be called with the entry's lock held.
329  */
330 static inline void
arpq_enqueue(struct l2t_entry * e,mblk_t * m)331 arpq_enqueue(struct l2t_entry *e, mblk_t *m)
332 {
333 	ASSERT(MUTEX_HELD(&e->lock));
334 
335 	ASSERT(m->b_next == NULL);
336 	if (e->arpq_head != NULL)
337 		e->arpq_tail->b_next = m;
338 	else
339 		e->arpq_head = m;
340 	e->arpq_tail = m;
341 }
342 
343 int
t4_l2t_send(struct adapter * sc,mblk_t * m,struct l2t_entry * e)344 t4_l2t_send(struct adapter *sc, mblk_t *m, struct l2t_entry *e)
345 {
346 	sin_t *sin;
347 	ip2mac_t ip2m;
348 
349 	if (e->v6 != 0)
350 		ASSERT(0);
351 again:
352 	switch (e->state) {
353 	case L2T_STATE_STALE:	/* entry is stale, kick off revalidation */
354 
355 	/* Fall through */
356 	case L2T_STATE_VALID:	/* fast-path, send the packet on */
357 		(void) t4_wrq_tx(sc, MBUF_EQ(m), m);
358 		return (0);
359 
360 	case L2T_STATE_RESOLVING:
361 	case L2T_STATE_SYNC_WRITE:
362 		mutex_enter(&e->lock);
363 		if (e->state != L2T_STATE_SYNC_WRITE &&
364 		    e->state != L2T_STATE_RESOLVING) {
365 			/* state changed by the time we got here */
366 			mutex_exit(&e->lock);
367 			goto again;
368 		}
369 		arpq_enqueue(e, m);
370 		mutex_exit(&e->lock);
371 
372 		bzero(&ip2m, sizeof (ip2m));
373 		sin = (sin_t *)&ip2m.ip2mac_pa;
374 		sin->sin_family = AF_INET;
375 		sin->sin_addr.s_addr = e->in_addr;
376 		ip2m.ip2mac_ifindex = e->ifindex;
377 
378 		if (e->state == L2T_STATE_RESOLVING) {
379 			(void) ip2mac(IP2MAC_RESOLVE, &ip2m, t4_l2t_update, e,
380 			    0);
381 			if (ip2m.ip2mac_err == EINPROGRESS)
382 				ASSERT(0);
383 			else if (ip2m.ip2mac_err == 0)
384 				t4_l2t_update(&ip2m, e);
385 			else
386 				ASSERT(0);
387 		}
388 	}
389 
390 	return (0);
391 }
392 
393 /*
394  * Called when an L2T entry has no more users.  The entry is left in the hash
395  * table since it is likely to be reused but we also bump nfree to indicate
396  * that the entry can be reallocated for a different neighbor.  We also drop
397  * the existing neighbor reference in case the neighbor is going away and is
398  * waiting on our reference.
399  *
400  * Because entries can be reallocated to other neighbors once their ref count
401  * drops to 0 we need to take the entry's lock to avoid races with a new
402  * incarnation.
403  */
404 static void
t4_l2e_free(struct l2t_entry * e)405 t4_l2e_free(struct l2t_entry *e)
406 {
407 	struct l2t_data *d;
408 
409 	mutex_enter(&e->lock);
410 	/* LINTED: E_NOP_IF_STMT */
411 	if (atomic_read(&e->refcnt) == 0) {  /* hasn't been recycled */
412 		/*
413 		 * Don't need to worry about the arpq, an L2T entry can't be
414 		 * released if any packets are waiting for resolution as we
415 		 * need to be able to communicate with the device to close a
416 		 * connection.
417 		 */
418 	}
419 	mutex_exit(&e->lock);
420 
421 	d = __containerof(e, struct l2t_data, l2tab[e->idx]);
422 	atomic_inc_uint(&d->nfree);
423 
424 }
425 
426 void
t4_l2t_release(struct l2t_entry * e)427 t4_l2t_release(struct l2t_entry *e)
428 {
429 	if (atomic_dec_uint_nv(&e->refcnt) == 0)
430 		t4_l2e_free(e);
431 }
432 
433 /* ARGSUSED */
434 int
do_l2t_write_rpl(struct sge_iq * iq,const struct rss_header * rss,mblk_t * m)435 do_l2t_write_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
436 {
437 	struct adapter *sc = iq->adapter;
438 	const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
439 	unsigned int tid = GET_TID(rpl);
440 	unsigned int idx = tid % L2T_SIZE;
441 
442 	if (likely(rpl->status != CPL_ERR_NONE)) {
443 		cxgb_printf(sc->dip, CE_WARN,
444 		    "Unexpected L2T_WRITE_RPL status %u for entry %u",
445 		    rpl->status, idx);
446 		return (-EINVAL);
447 	}
448 
449 	return (0);
450 }
451 
452 /*
453  * The TOE wants an L2 table entry that it can use to reach the next hop over
454  * the specified port.  Produce such an entry - create one if needed.
455  *
456  * Note that the ifnet could be a pseudo-device like if_vlan, if_lagg, etc. on
457  * top of the real cxgbe interface.
458  */
459 struct l2t_entry *
t4_l2t_get(struct port_info * pi,conn_t * connp)460 t4_l2t_get(struct port_info *pi, conn_t *connp)
461 {
462 	struct l2t_entry *e;
463 	struct l2t_data *d = pi->adapter->l2t;
464 	int addr_len;
465 	uint32_t *addr;
466 	int hash;
467 	int index = \
468 	    connp->conn_ixa->ixa_ire->ire_ill->ill_phyint->phyint_ifindex;
469 	unsigned int smt_idx = pi->port_id;
470 	addr = (uint32_t *)&connp->conn_faddr_v4;
471 	addr_len  = sizeof (connp->conn_faddr_v4);
472 
473 	hash = addr_hash(addr, addr_len, index);
474 
475 	rw_enter(&d->lock, RW_WRITER);
476 	for (e = d->l2tab[hash].first; e; e = e->next) {
477 		if (!addreq(e, addr) && e->smt_idx == smt_idx) {
478 			l2t_hold(d, e);
479 			goto done;
480 		}
481 	}
482 
483 	/* Need to allocate a new entry */
484 	e = alloc_l2e(d);
485 	if (e != NULL) {
486 		mutex_enter(&e->lock);	/* avoid race with t4_l2t_free */
487 		e->state = L2T_STATE_RESOLVING;
488 		(void) memcpy(e->addr, addr, addr_len);
489 		e->in_addr = connp->conn_faddr_v4;
490 		e->ifindex = index;
491 		/* LINTED: E_ASSIGN_NARROW_CONV */
492 		e->smt_idx = smt_idx;
493 		/* LINTED: E_ASSIGN_NARROW_CONV */
494 		e->hash = hash;
495 		e->lport = pi->lport;
496 		e->arpq_head = e->arpq_tail = NULL;
497 		e->v6 = (addr_len == 16);
498 		e->sc = pi->adapter;
499 		(void) atomic_swap_uint(&e->refcnt, 1);
500 		e->vlan = VLAN_NONE;
501 		e->next = d->l2tab[hash].first;
502 		d->l2tab[hash].first = e;
503 		mutex_exit(&e->lock);
504 	} else {
505 		ASSERT(0);
506 	}
507 
508 done:
509 	rw_exit(&d->lock);
510 	return (e);
511 }
512 
513 /*
514  * Called when the host's neighbor layer makes a change to some entry that is
515  * loaded into the HW L2 table.
516  */
517 void
t4_l2t_update(ip2mac_t * ip2macp,void * arg)518 t4_l2t_update(ip2mac_t *ip2macp, void *arg)
519 {
520 	struct l2t_entry *e = (struct l2t_entry *)arg;
521 	struct adapter *sc = e->sc;
522 	uchar_t *cp;
523 
524 	if (ip2macp->ip2mac_err != 0) {
525 		ASSERT(0); /* Don't know what to do. Needs to be investigated */
526 	}
527 
528 	mutex_enter(&e->lock);
529 	if (atomic_read(&e->refcnt) != 0)
530 		goto found;
531 	e->state = L2T_STATE_STALE;
532 	mutex_exit(&e->lock);
533 
534 	/* The TOE has no interest in this LLE */
535 	return;
536 
537 found:
538 	if (atomic_read(&e->refcnt) != 0) {
539 
540 		/* Entry is referenced by at least 1 offloaded connection. */
541 
542 		cp = (uchar_t *)LLADDR(&ip2macp->ip2mac_ha);
543 		bcopy(cp, e->dmac, 6);
544 		(void) write_l2e(sc, e, 1);
545 		e->state = L2T_STATE_VALID;
546 
547 	}
548 	mutex_exit(&e->lock);
549 }
550 #endif
551