xref: /freebsd/sys/dev/cxgbe/t4_filter.c (revision 721033a7d96f24e80ec18ec9cc17712be55a52b7)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2018 Chelsio Communications, Inc.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 #include <sys/cdefs.h>
29 #include "opt_inet.h"
30 #include "opt_inet6.h"
31 
32 #include <sys/param.h>
33 #include <sys/eventhandler.h>
34 #include <sys/fnv_hash.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/module.h>
38 #include <sys/bus.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <sys/rwlock.h>
42 #include <sys/socket.h>
43 #include <sys/sbuf.h>
44 #include <netinet/in.h>
45 
46 #include "common/common.h"
47 #include "common/t4_msg.h"
48 #include "common/t4_regs.h"
49 #include "common/t4_regs_values.h"
50 #include "common/t4_tcb.h"
51 #include "t4_l2t.h"
52 #include "t4_smt.h"
53 
54 struct filter_entry {
55 	LIST_ENTRY(filter_entry) link_4t;
56 	LIST_ENTRY(filter_entry) link_tid;
57 
58 	uint32_t valid:1;	/* filter allocated and valid */
59 	uint32_t locked:1;	/* filter is administratively locked or busy */
60 	uint32_t pending:1;	/* filter action is pending firmware reply */
61 	int tid;		/* tid of the filter TCB */
62 	struct l2t_entry *l2te;	/* L2 table entry for DMAC rewrite */
63 	struct smt_entry *smt;	/* SMT entry for SMAC rewrite */
64 
65 	struct t4_filter_specification fs;
66 };
67 
68 static void free_filter_resources(struct filter_entry *);
69 static int get_tcamfilter(struct adapter *, struct t4_filter *);
70 static int get_hashfilter(struct adapter *, struct t4_filter *);
71 static int set_hashfilter(struct adapter *, struct t4_filter *, uint64_t,
72     struct l2t_entry *, struct smt_entry *);
73 static int del_hashfilter(struct adapter *, struct t4_filter *);
74 static int configure_hashfilter_tcb(struct adapter *, struct filter_entry *);
75 
76 static inline bool
77 separate_hpfilter_region(struct adapter *sc)
78 {
79 
80 	return (chip_id(sc) >= CHELSIO_T6);
81 }
82 
83 static inline uint32_t
84 hf_hashfn_4t(struct t4_filter_specification *fs)
85 {
86 	struct t4_filter_tuple *ft = &fs->val;
87 	uint32_t hash;
88 
89 	if (fs->type) {
90 		/* IPv6 */
91 		hash = fnv_32_buf(&ft->sip[0], 16, FNV1_32_INIT);
92 		hash = fnv_32_buf(&ft->dip[0], 16, hash);
93 	} else {
94 		hash = fnv_32_buf(&ft->sip[0], 4, FNV1_32_INIT);
95 		hash = fnv_32_buf(&ft->dip[0], 4, hash);
96 	}
97 	hash = fnv_32_buf(&ft->sport, sizeof(ft->sport), hash);
98 	hash = fnv_32_buf(&ft->dport, sizeof(ft->dport), hash);
99 
100 	return (hash);
101 }
102 
103 static inline uint32_t
104 hf_hashfn_tid(int tid)
105 {
106 
107 	return (fnv_32_buf(&tid, sizeof(tid), FNV1_32_INIT));
108 }
109 
110 static int
111 alloc_hftid_hash(struct tid_info *t, int flags)
112 {
113 	int n;
114 
115 	MPASS(t->ntids > 0);
116 	MPASS(t->hftid_hash_4t == NULL);
117 	MPASS(t->hftid_hash_tid == NULL);
118 
119 	n = max(t->ntids / 1024, 16);
120 	t->hftid_hash_4t = hashinit_flags(n, M_CXGBE, &t->hftid_4t_mask, flags);
121 	if (t->hftid_hash_4t == NULL)
122 		return (ENOMEM);
123 	t->hftid_hash_tid = hashinit_flags(n, M_CXGBE, &t->hftid_tid_mask,
124 	    flags);
125 	if (t->hftid_hash_tid == NULL) {
126 		hashdestroy(t->hftid_hash_4t, M_CXGBE, t->hftid_4t_mask);
127 		t->hftid_hash_4t = NULL;
128 		return (ENOMEM);
129 	}
130 
131 	mtx_init(&t->hftid_lock, "T4 hashfilters", 0, MTX_DEF);
132 	cv_init(&t->hftid_cv, "t4hfcv");
133 
134 	return (0);
135 }
136 
137 void
138 free_hftid_hash(struct tid_info *t)
139 {
140 	struct filter_entry *f, *ftmp;
141 	LIST_HEAD(, filter_entry) *head;
142 	int i;
143 #ifdef INVARIANTS
144 	int n = 0;
145 #endif
146 
147 	if (t->tids_in_use > 0) {
148 		/* Remove everything from the tid hash. */
149 		head = t->hftid_hash_tid;
150 		for (i = 0; i <= t->hftid_tid_mask; i++) {
151 			LIST_FOREACH_SAFE(f, &head[i], link_tid, ftmp) {
152 				LIST_REMOVE(f, link_tid);
153 			}
154 		}
155 
156 		/* Remove and then free each filter in the 4t hash. */
157 		head = t->hftid_hash_4t;
158 		for (i = 0; i <= t->hftid_4t_mask; i++) {
159 			LIST_FOREACH_SAFE(f, &head[i], link_4t, ftmp) {
160 #ifdef INVARIANTS
161 				n += f->fs.type ? 2 : 1;
162 #endif
163 				LIST_REMOVE(f, link_4t);
164 				free(f, M_CXGBE);
165 			}
166 		}
167 		MPASS(t->tids_in_use == n);
168 		t->tids_in_use = 0;
169 	}
170 
171 	if (t->hftid_hash_4t) {
172 		hashdestroy(t->hftid_hash_4t, M_CXGBE, t->hftid_4t_mask);
173 		t->hftid_hash_4t = NULL;
174 	}
175 	if (t->hftid_hash_tid) {
176 		hashdestroy(t->hftid_hash_tid, M_CXGBE, t->hftid_tid_mask);
177 		t->hftid_hash_tid = NULL;
178 	}
179 	if (mtx_initialized(&t->hftid_lock)) {
180 		mtx_destroy(&t->hftid_lock);
181 		cv_destroy(&t->hftid_cv);
182 	}
183 }
184 
185 static void
186 insert_hf(struct adapter *sc, struct filter_entry *f, uint32_t hash)
187 {
188 	struct tid_info *t = &sc->tids;
189 	LIST_HEAD(, filter_entry) *head = t->hftid_hash_4t;
190 
191 	MPASS(head != NULL);
192 	if (hash == 0)
193 		hash = hf_hashfn_4t(&f->fs);
194 	LIST_INSERT_HEAD(&head[hash & t->hftid_4t_mask], f, link_4t);
195 	atomic_add_int(&t->tids_in_use, f->fs.type ? 2 : 1);
196 }
197 
198 static void
199 insert_hftid(struct adapter *sc, struct filter_entry *f)
200 {
201 	struct tid_info *t = &sc->tids;
202 	LIST_HEAD(, filter_entry) *head = t->hftid_hash_tid;
203 	uint32_t hash;
204 
205 	MPASS(f->tid >= t->tid_base);
206 	MPASS(f->tid - t->tid_base < t->ntids);
207 	mtx_assert(&t->hftid_lock, MA_OWNED);
208 
209 	hash = hf_hashfn_tid(f->tid);
210 	LIST_INSERT_HEAD(&head[hash & t->hftid_tid_mask], f, link_tid);
211 }
212 
213 static bool
214 filter_eq(struct t4_filter_specification *fs1,
215     struct t4_filter_specification *fs2)
216 {
217 	int n;
218 
219 	MPASS(fs1->hash && fs2->hash);
220 
221 	if (fs1->type != fs2->type)
222 		return (false);
223 
224 	n = fs1->type ? 16 : 4;
225 	if (bcmp(&fs1->val.sip[0], &fs2->val.sip[0], n) ||
226 	    bcmp(&fs1->val.dip[0], &fs2->val.dip[0], n) ||
227 	    fs1->val.sport != fs2->val.sport ||
228 	    fs1->val.dport != fs2->val.dport)
229 		return (false);
230 
231 	/*
232 	 * We know the masks are the same because all hashfilters conform to the
233 	 * global tp->filter_mask and the driver has verified that already.
234 	 */
235 
236 	if ((fs1->mask.pfvf_vld || fs1->mask.ovlan_vld) &&
237 	    fs1->val.vnic != fs2->val.vnic)
238 		return (false);
239 	if (fs1->mask.vlan_vld && fs1->val.vlan != fs2->val.vlan)
240 		return (false);
241 	if (fs1->mask.macidx && fs1->val.macidx != fs2->val.macidx)
242 		return (false);
243 	if (fs1->mask.frag && fs1->val.frag != fs2->val.frag)
244 		return (false);
245 	if (fs1->mask.matchtype && fs1->val.matchtype != fs2->val.matchtype)
246 		return (false);
247 	if (fs1->mask.iport && fs1->val.iport != fs2->val.iport)
248 		return (false);
249 	if (fs1->mask.fcoe && fs1->val.fcoe != fs2->val.fcoe)
250 		return (false);
251 	if (fs1->mask.proto && fs1->val.proto != fs2->val.proto)
252 		return (false);
253 	if (fs1->mask.tos && fs1->val.tos != fs2->val.tos)
254 		return (false);
255 	if (fs1->mask.ethtype && fs1->val.ethtype != fs2->val.ethtype)
256 		return (false);
257 
258 	return (true);
259 }
260 
261 static struct filter_entry *
262 lookup_hf(struct adapter *sc, struct t4_filter_specification *fs, uint32_t hash)
263 {
264 	struct tid_info *t = &sc->tids;
265 	LIST_HEAD(, filter_entry) *head = t->hftid_hash_4t;
266 	struct filter_entry *f;
267 
268 	mtx_assert(&t->hftid_lock, MA_OWNED);
269 	MPASS(head != NULL);
270 
271 	if (hash == 0)
272 		hash = hf_hashfn_4t(fs);
273 
274 	LIST_FOREACH(f, &head[hash & t->hftid_4t_mask], link_4t) {
275 		if (filter_eq(&f->fs, fs))
276 			return (f);
277 	}
278 
279 	return (NULL);
280 }
281 
282 static struct filter_entry *
283 lookup_hftid(struct adapter *sc, int tid)
284 {
285 	struct tid_info *t = &sc->tids;
286 	LIST_HEAD(, filter_entry) *head = t->hftid_hash_tid;
287 	struct filter_entry *f;
288 	uint32_t hash;
289 
290 	mtx_assert(&t->hftid_lock, MA_OWNED);
291 	MPASS(head != NULL);
292 
293 	hash = hf_hashfn_tid(tid);
294 	LIST_FOREACH(f, &head[hash & t->hftid_tid_mask], link_tid) {
295 		if (f->tid == tid)
296 			return (f);
297 	}
298 
299 	return (NULL);
300 }
301 
302 static void
303 remove_hf(struct adapter *sc, struct filter_entry *f)
304 {
305 	struct tid_info *t = &sc->tids;
306 
307 	mtx_assert(&t->hftid_lock, MA_OWNED);
308 
309 	LIST_REMOVE(f, link_4t);
310 	atomic_subtract_int(&t->tids_in_use, f->fs.type ? 2 : 1);
311 }
312 
313 static void
314 remove_hftid(struct adapter *sc, struct filter_entry *f)
315 {
316 #ifdef INVARIANTS
317 	struct tid_info *t = &sc->tids;
318 
319 	mtx_assert(&t->hftid_lock, MA_OWNED);
320 #endif
321 
322 	LIST_REMOVE(f, link_tid);
323 }
324 
325 static uint16_t
326 mode_to_fconf_t4(uint32_t mode)
327 {
328 	uint32_t fconf = 0;
329 
330 	if (mode & T4_FILTER_IP_FRAGMENT)
331 		fconf |= F_FRAGMENTATION;
332 	if (mode & T4_FILTER_MPS_HIT_TYPE)
333 		fconf |= F_MPSHITTYPE;
334 	if (mode & T4_FILTER_MAC_IDX)
335 		fconf |= F_MACMATCH;
336 	if (mode & T4_FILTER_ETH_TYPE)
337 		fconf |= F_ETHERTYPE;
338 	if (mode & T4_FILTER_IP_PROTO)
339 		fconf |= F_PROTOCOL;
340 	if (mode & T4_FILTER_IP_TOS)
341 		fconf |= F_TOS;
342 	if (mode & T4_FILTER_VLAN)
343 		fconf |= F_VLAN;
344 	if (mode & T4_FILTER_VNIC)
345 		fconf |= F_VNIC_ID;
346 	if (mode & T4_FILTER_PORT)
347 		fconf |= F_PORT;
348 	if (mode & T4_FILTER_FCoE)
349 		fconf |= F_FCOE;
350 
351 	return (fconf);
352 }
353 
354 static uint16_t
355 mode_to_fconf_t7(uint32_t mode)
356 {
357 	uint32_t fconf = 0;
358 
359 	if (mode & T4_FILTER_TCPFLAGS)
360 		fconf |= F_TCPFLAGS;
361 	if (mode & T4_FILTER_SYNONLY)
362 		fconf |= F_SYNONLY;
363 	if (mode & T4_FILTER_ROCE)
364 		fconf |= F_ROCE;
365 	if (mode & T4_FILTER_IP_FRAGMENT)
366 		fconf |= F_T7_FRAGMENTATION;
367 	if (mode & T4_FILTER_MPS_HIT_TYPE)
368 		fconf |= F_T7_MPSHITTYPE;
369 	if (mode & T4_FILTER_MAC_IDX)
370 		fconf |= F_T7_MACMATCH;
371 	if (mode & T4_FILTER_ETH_TYPE)
372 		fconf |= F_T7_ETHERTYPE;
373 	if (mode & T4_FILTER_IP_PROTO)
374 		fconf |= F_T7_PROTOCOL;
375 	if (mode & T4_FILTER_IP_TOS)
376 		fconf |= F_T7_TOS;
377 	if (mode & T4_FILTER_VLAN)
378 		fconf |= F_T7_VLAN;
379 	if (mode & T4_FILTER_VNIC)
380 		fconf |= F_T7_VNIC_ID;
381 	if (mode & T4_FILTER_PORT)
382 		fconf |= F_T7_PORT;
383 	if (mode & T4_FILTER_FCoE)
384 		fconf |= F_T7_FCOE;
385 	if (mode & T4_FILTER_IPSECIDX)
386 		fconf |= F_IPSECIDX;
387 
388 	return (fconf);
389 }
390 
391 /*
392  * Input: driver's 32b filter mode.
393  * Returns: hardware filter mode (bits to set in vlan_pri_map) for the input.
394  */
395 static uint16_t
396 mode_to_fconf(struct adapter *sc, uint32_t mode)
397 {
398 	if (chip_id(sc) >= CHELSIO_T7)
399 		return (mode_to_fconf_t7(mode));
400 	else
401 		return (mode_to_fconf_t4(mode));
402 }
403 
404 /*
405  * Input: driver's 32b filter mode.
406  * Returns: hardware vnic mode (ingress config) matching the input.
407  */
408 static int
409 mode_to_iconf(uint32_t mode)
410 {
411 	if ((mode & T4_FILTER_VNIC) == 0)
412 		return (-1);	/* ingress config doesn't matter. */
413 
414 	if (mode & T4_FILTER_IC_VNIC)
415 		return (FW_VNIC_MODE_PF_VF);
416 	else if (mode & T4_FILTER_IC_ENCAP)
417 		return (FW_VNIC_MODE_ENCAP_EN);
418 	else
419 		return (FW_VNIC_MODE_OUTER_VLAN);
420 }
421 
422 static int
423 check_fspec_against_fconf_iconf(struct adapter *sc,
424     struct t4_filter_specification *fs)
425 {
426 	struct tp_params *tpp = &sc->params.tp;
427 	uint32_t fconf = 0;
428 
429 	if (chip_id(sc) >= CHELSIO_T7) {
430 		if (fs->val.tcpflags || fs->mask.tcpflags)
431 			fconf |= F_TCPFLAGS;
432 		if (fs->val.synonly || fs->mask.synonly)
433 			fconf |= F_SYNONLY;
434 		if (fs->val.roce || fs->mask.roce)
435 			fconf |= F_ROCE;
436 		if (fs->val.frag || fs->mask.frag)
437 			fconf |= F_T7_FRAGMENTATION;
438 		if (fs->val.matchtype || fs->mask.matchtype)
439 			fconf |= F_T7_MPSHITTYPE;
440 		if (fs->val.macidx || fs->mask.macidx)
441 			fconf |= F_T7_MACMATCH;
442 		if (fs->val.ethtype || fs->mask.ethtype)
443 			fconf |= F_T7_ETHERTYPE;
444 		if (fs->val.proto || fs->mask.proto)
445 			fconf |= F_T7_PROTOCOL;
446 		if (fs->val.tos || fs->mask.tos)
447 			fconf |= F_T7_TOS;
448 		if (fs->val.vlan_vld || fs->mask.vlan_vld)
449 			fconf |= F_T7_VLAN;
450 		if (fs->val.ovlan_vld || fs->mask.ovlan_vld) {
451 			if (tpp->vnic_mode != FW_VNIC_MODE_OUTER_VLAN)
452 				return (EINVAL);
453 			fconf |= F_T7_VNIC_ID;
454 		}
455 		if (fs->val.pfvf_vld || fs->mask.pfvf_vld) {
456 			if (tpp->vnic_mode != FW_VNIC_MODE_PF_VF)
457 				return (EINVAL);
458 			fconf |= F_T7_VNIC_ID;
459 		}
460 #ifdef notyet
461 		if (fs->val.encap_vld || fs->mask.encap_vld) {
462 			if (tpp->vnic_mode != FW_VNIC_MODE_ENCAP_EN);
463 				return (EINVAL);
464 			fconf |= F_T7_VNIC_ID;
465 		}
466 #endif
467 		if (fs->val.iport || fs->mask.iport)
468 			fconf |= F_T7_PORT;
469 		if (fs->val.fcoe || fs->mask.fcoe)
470 			fconf |= F_T7_FCOE;
471 		if (fs->val.ipsecidx || fs->mask.ipsecidx)
472 			fconf |= F_IPSECIDX;
473 	} else {
474 		if (fs->val.tcpflags || fs->mask.tcpflags ||
475 		    fs->val.synonly || fs->mask.synonly ||
476 		    fs->val.roce || fs->mask.roce ||
477 		    fs->val.ipsecidx || fs->mask.ipsecidx)
478 			return (EINVAL);
479 		if (fs->val.frag || fs->mask.frag)
480 			fconf |= F_FRAGMENTATION;
481 		if (fs->val.matchtype || fs->mask.matchtype)
482 			fconf |= F_MPSHITTYPE;
483 		if (fs->val.macidx || fs->mask.macidx)
484 			fconf |= F_MACMATCH;
485 		if (fs->val.ethtype || fs->mask.ethtype)
486 			fconf |= F_ETHERTYPE;
487 		if (fs->val.proto || fs->mask.proto)
488 			fconf |= F_PROTOCOL;
489 		if (fs->val.tos || fs->mask.tos)
490 			fconf |= F_TOS;
491 		if (fs->val.vlan_vld || fs->mask.vlan_vld)
492 			fconf |= F_VLAN;
493 		if (fs->val.ovlan_vld || fs->mask.ovlan_vld) {
494 			if (tpp->vnic_mode != FW_VNIC_MODE_OUTER_VLAN)
495 				return (EINVAL);
496 			fconf |= F_VNIC_ID;
497 		}
498 		if (fs->val.pfvf_vld || fs->mask.pfvf_vld) {
499 			if (tpp->vnic_mode != FW_VNIC_MODE_PF_VF)
500 				return (EINVAL);
501 			fconf |= F_VNIC_ID;
502 		}
503 #ifdef notyet
504 		if (fs->val.encap_vld || fs->mask.encap_vld) {
505 			if (tpp->vnic_mode != FW_VNIC_MODE_ENCAP_EN);
506 				return (EINVAL);
507 			fconf |= F_VNIC_ID;
508 		}
509 #endif
510 		if (fs->val.iport || fs->mask.iport)
511 			fconf |= F_PORT;
512 		if (fs->val.fcoe || fs->mask.fcoe)
513 			fconf |= F_FCOE;
514 	}
515 	if ((tpp->filter_mode | fconf) != tpp->filter_mode)
516 		return (E2BIG);
517 
518 	return (0);
519 }
520 
521 static uint32_t
522 fconf_to_mode_t4(uint16_t hwmode, int vnic_mode)
523 {
524 	uint32_t mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
525 	    T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
526 
527 	if (hwmode & F_FRAGMENTATION)
528 		mode |= T4_FILTER_IP_FRAGMENT;
529 	if (hwmode & F_MPSHITTYPE)
530 		mode |= T4_FILTER_MPS_HIT_TYPE;
531 	if (hwmode & F_MACMATCH)
532 		mode |= T4_FILTER_MAC_IDX;
533 	if (hwmode & F_ETHERTYPE)
534 		mode |= T4_FILTER_ETH_TYPE;
535 	if (hwmode & F_PROTOCOL)
536 		mode |= T4_FILTER_IP_PROTO;
537 	if (hwmode & F_TOS)
538 		mode |= T4_FILTER_IP_TOS;
539 	if (hwmode & F_VLAN)
540 		mode |= T4_FILTER_VLAN;
541 	if (hwmode & F_VNIC_ID)
542 		mode |= T4_FILTER_VNIC; /* real meaning depends on vnic_mode. */
543 	if (hwmode & F_PORT)
544 		mode |= T4_FILTER_PORT;
545 	if (hwmode & F_FCOE)
546 		mode |= T4_FILTER_FCoE;
547 
548 	switch (vnic_mode) {
549 	case FW_VNIC_MODE_PF_VF:
550 		mode |= T4_FILTER_IC_VNIC;
551 		break;
552 	case FW_VNIC_MODE_ENCAP_EN:
553 		mode |= T4_FILTER_IC_ENCAP;
554 		break;
555 	case FW_VNIC_MODE_OUTER_VLAN:
556 	default:
557 		break;
558 	}
559 
560 	return (mode);
561 }
562 
563 static uint32_t
564 fconf_to_mode_t7(uint16_t hwmode, int vnic_mode)
565 {
566 	uint32_t mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
567 	    T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
568 
569 	if (hwmode & F_TCPFLAGS)
570 		mode |= T4_FILTER_TCPFLAGS;
571 	if (hwmode & F_SYNONLY)
572 		mode |= T4_FILTER_SYNONLY;
573 	if (hwmode & F_ROCE)
574 		mode |= T4_FILTER_ROCE;
575 	if (hwmode & F_T7_FRAGMENTATION)
576 		mode |= T4_FILTER_IP_FRAGMENT;
577 	if (hwmode & F_T7_MPSHITTYPE)
578 		mode |= T4_FILTER_MPS_HIT_TYPE;
579 	if (hwmode & F_T7_MACMATCH)
580 		mode |= T4_FILTER_MAC_IDX;
581 	if (hwmode & F_T7_ETHERTYPE)
582 		mode |= T4_FILTER_ETH_TYPE;
583 	if (hwmode & F_T7_PROTOCOL)
584 		mode |= T4_FILTER_IP_PROTO;
585 	if (hwmode & F_T7_TOS)
586 		mode |= T4_FILTER_IP_TOS;
587 	if (hwmode & F_T7_VLAN)
588 		mode |= T4_FILTER_VLAN;
589 	if (hwmode & F_T7_VNIC_ID)
590 		mode |= T4_FILTER_VNIC; /* real meaning depends on vnic_mode. */
591 	if (hwmode & F_T7_PORT)
592 		mode |= T4_FILTER_PORT;
593 	if (hwmode & F_T7_FCOE)
594 		mode |= T4_FILTER_FCoE;
595 	if (hwmode & F_IPSECIDX)
596 		mode |= T4_FILTER_IPSECIDX;
597 
598 	switch (vnic_mode) {
599 	case FW_VNIC_MODE_PF_VF:
600 		mode |= T4_FILTER_IC_VNIC;
601 		break;
602 	case FW_VNIC_MODE_ENCAP_EN:
603 		mode |= T4_FILTER_IC_ENCAP;
604 		break;
605 	case FW_VNIC_MODE_OUTER_VLAN:
606 	default:
607 		break;
608 	}
609 
610 	return (mode);
611 }
612 
613 /*
614  * Input: hardware filter configuration (filter mode/mask, ingress config).
615  * Output: driver's 32b filter mode matching the input.
616  */
617 static inline uint32_t
618 fconf_to_mode(struct adapter *sc, uint16_t hwmode, int vnic_mode)
619 {
620 	if (chip_id(sc) >= CHELSIO_T7)
621 		return (fconf_to_mode_t7(hwmode, vnic_mode));
622 	else
623 		return (fconf_to_mode_t4(hwmode, vnic_mode));
624 }
625 
626 int
627 get_filter_mode(struct adapter *sc, uint32_t *mode)
628 {
629 	struct tp_params *tp = &sc->params.tp;
630 	uint16_t filter_mode;
631 
632 	/* Filter mask must comply with the global filter mode. */
633 	MPASS((tp->filter_mode | tp->filter_mask) == tp->filter_mode);
634 
635 	/* Non-zero incoming value in mode means "hashfilter mode". */
636 	filter_mode = *mode ? tp->filter_mask : tp->filter_mode;
637 	*mode = fconf_to_mode(sc, filter_mode, tp->vnic_mode);
638 
639 	return (0);
640 }
641 
642 int
643 set_filter_mode(struct adapter *sc, uint32_t mode)
644 {
645 	struct tp_params *tp = &sc->params.tp;
646 	int rc, iconf;
647 	uint16_t fconf;
648 
649 	iconf = mode_to_iconf(mode);
650 	fconf = mode_to_fconf(sc, mode);
651 	if ((iconf == -1 || iconf == tp->vnic_mode) && fconf == tp->filter_mode)
652 		return (0);	/* Nothing to do */
653 
654 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setfm");
655 	if (rc)
656 		return (rc);
657 
658 	if (!hw_all_ok(sc)) {
659 		rc = ENXIO;
660 		goto done;
661 	}
662 
663 	if (sc->tids.ftids_in_use > 0 ||	/* TCAM filters active */
664 	    sc->tids.hpftids_in_use > 0 ||	/* hi-pri TCAM filters active */
665 	    sc->tids.tids_in_use > 0) {		/* TOE or hashfilters active */
666 		rc = EBUSY;
667 		goto done;
668 	}
669 
670 #ifdef TCP_OFFLOAD
671 	if (uld_active(sc, ULD_TOM)) {
672 		rc = EBUSY;
673 		goto done;
674 	}
675 #endif
676 
677 	/* Note that filter mask will get clipped to the new filter mode. */
678 	rc = -t4_set_filter_cfg(sc, fconf, -1, iconf);
679 done:
680 	end_synchronized_op(sc, 0);
681 	return (rc);
682 }
683 
684 int
685 set_filter_mask(struct adapter *sc, uint32_t mode)
686 {
687 	struct tp_params *tp = &sc->params.tp;
688 	int rc, iconf;
689 	uint16_t fmask;
690 
691 	iconf = mode_to_iconf(mode);
692 	fmask = mode_to_fconf(sc, mode);
693 	if ((iconf == -1 || iconf == tp->vnic_mode) && fmask == tp->filter_mask)
694 		return (0);	/* Nothing to do */
695 
696 	/*
697 	 * We aren't going to change the global filter mode or VNIC mode here.
698 	 * The given filter mask must conform to them.
699 	 */
700 	if ((fmask | tp->filter_mode) != tp->filter_mode)
701 		return (EINVAL);
702 	if (iconf != -1 && iconf != tp->vnic_mode)
703 		return (EINVAL);
704 
705 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4sethfm");
706 	if (rc)
707 		return (rc);
708 
709 	if (!hw_all_ok(sc)) {
710 		rc = ENXIO;
711 		goto done;
712 	}
713 
714 	if (sc->tids.tids_in_use > 0) {		/* TOE or hashfilters active */
715 		rc = EBUSY;
716 		goto done;
717 	}
718 
719 #ifdef TCP_OFFLOAD
720 	if (uld_active(sc, ULD_TOM)) {
721 		rc = EBUSY;
722 		goto done;
723 	}
724 #endif
725 	rc = -t4_set_filter_cfg(sc, -1, fmask, -1);
726 done:
727 	end_synchronized_op(sc, 0);
728 	return (rc);
729 }
730 
731 static inline uint64_t
732 get_filter_hits(struct adapter *sc, uint32_t tid)
733 {
734 	uint32_t tcb_addr;
735 	uint64_t hits;
736 
737 	tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE) + tid * TCB_SIZE;
738 
739 	mtx_lock(&sc->reg_lock);
740 	if (!hw_all_ok(sc))
741 		hits = 0;
742 	else if (is_t4(sc)) {
743 		uint64_t t;
744 
745 		read_via_memwin(sc, 0, tcb_addr + 16, (uint32_t *)&t, 8);
746 		hits = be64toh(t);
747 	} else {
748 		uint32_t t;
749 
750 		read_via_memwin(sc, 0, tcb_addr + 24, &t, 4);
751 		hits = be32toh(t);
752 	}
753 	mtx_unlock(&sc->reg_lock);
754 
755 	return (hits);
756 }
757 
758 int
759 get_filter(struct adapter *sc, struct t4_filter *t)
760 {
761 	if (t->fs.hash)
762 		return (get_hashfilter(sc, t));
763 	else
764 		return (get_tcamfilter(sc, t));
765 }
766 
767 static int
768 set_tcamfilter(struct adapter *sc, struct t4_filter *t, struct l2t_entry *l2te,
769     struct smt_entry *smt)
770 {
771 	struct filter_entry *f;
772 	struct fw_filter2_wr *fwr;
773 	u_int vnic_vld, vnic_vld_mask;
774 	struct wrq_cookie cookie;
775 	int i, rc, busy, locked;
776 	u_int tid;
777 	const int ntids = t->fs.type ? 4 : 1;
778 
779 	MPASS(!t->fs.hash);
780 	/* Already validated against fconf, iconf */
781 	MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0);
782 	MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0);
783 
784 	if (separate_hpfilter_region(sc) && t->fs.prio) {
785 		MPASS(t->idx < sc->tids.nhpftids);
786 		f = &sc->tids.hpftid_tab[t->idx];
787 		tid = sc->tids.hpftid_base + t->idx;
788 	} else {
789 		MPASS(t->idx < sc->tids.nftids);
790 		f = &sc->tids.ftid_tab[t->idx];
791 		tid = sc->tids.ftid_base + t->idx;
792 	}
793 	rc = busy = locked = 0;
794 	mtx_lock(&sc->tids.ftid_lock);
795 	for (i = 0; i < ntids; i++) {
796 		busy += f[i].pending + f[i].valid;
797 		locked += f[i].locked;
798 	}
799 	if (locked > 0)
800 		rc = EPERM;
801 	else if (busy > 0)
802 		rc = EBUSY;
803 	else {
804 		int len16;
805 
806 		if (sc->params.filter2_wr_support)
807 			len16 = howmany(sizeof(struct fw_filter2_wr), 16);
808 		else
809 			len16 = howmany(sizeof(struct fw_filter_wr), 16);
810 		fwr = start_wrq_wr(&sc->sge.ctrlq[0], len16, &cookie);
811 		if (__predict_false(fwr == NULL))
812 			rc = ENOMEM;
813 		else {
814 			f->pending = 1;
815 			if (separate_hpfilter_region(sc) && t->fs.prio)
816 				sc->tids.hpftids_in_use++;
817 			else
818 				sc->tids.ftids_in_use++;
819 		}
820 	}
821 	mtx_unlock(&sc->tids.ftid_lock);
822 	if (rc != 0)
823 		return (rc);
824 
825 	/*
826 	 * Can't fail now.  A set-filter WR will definitely be sent.
827 	 */
828 
829 	f->tid = tid;
830 	f->fs = t->fs;
831 	f->l2te = l2te;
832 	f->smt = smt;
833 
834 	if (t->fs.val.pfvf_vld || t->fs.val.ovlan_vld)
835 		vnic_vld = 1;
836 	else
837 		vnic_vld = 0;
838 	if (t->fs.mask.pfvf_vld || t->fs.mask.ovlan_vld)
839 		vnic_vld_mask = 1;
840 	else
841 		vnic_vld_mask = 0;
842 
843 	bzero(fwr, sizeof(*fwr));
844 	if (sc->params.filter2_wr_support)
845 		fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER2_WR));
846 	else
847 		fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
848 	fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
849 	fwr->tid_to_iq =
850 	    htobe32(V_FW_FILTER_WR_TID(f->tid) |
851 		V_FW_FILTER_WR_RQTYPE(f->fs.type) |
852 		V_FW_FILTER_WR_NOREPLY(0) |
853 		V_FW_FILTER_WR_IQ(f->fs.iq));
854 	fwr->del_filter_to_l2tix =
855 	    htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
856 		V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
857 		V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
858 		V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
859 		V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
860 		V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
861 		V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
862 		V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
863 		V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
864 		    f->fs.newvlan == VLAN_REWRITE) |
865 		V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
866 		    f->fs.newvlan == VLAN_REWRITE) |
867 		V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
868 		V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
869 		V_FW_FILTER_WR_PRIO(f->fs.prio) |
870 		V_FW_FILTER_WR_L2TIX(f->l2te ? f->l2te->idx : 0));
871 	fwr->ethtype = htobe16(f->fs.val.ethtype);
872 	fwr->ethtypem = htobe16(f->fs.mask.ethtype);
873 	fwr->frag_to_ovlan_vldm =
874 	    (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
875 		V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
876 		V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
877 		V_FW_FILTER_WR_OVLAN_VLD(vnic_vld) |
878 		V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
879 		V_FW_FILTER_WR_OVLAN_VLDM(vnic_vld_mask));
880 	fwr->smac_sel = 0;
881 	fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
882 	    V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
883 	fwr->maci_to_matchtypem =
884 	    htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
885 		V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
886 		V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
887 		V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
888 		V_FW_FILTER_WR_PORT(f->fs.val.iport) |
889 		V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
890 		V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
891 		V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
892 	fwr->ptcl = f->fs.val.proto;
893 	fwr->ptclm = f->fs.mask.proto;
894 	fwr->ttyp = f->fs.val.tos;
895 	fwr->ttypm = f->fs.mask.tos;
896 	fwr->ivlan = htobe16(f->fs.val.vlan);
897 	fwr->ivlanm = htobe16(f->fs.mask.vlan);
898 	fwr->ovlan = htobe16(f->fs.val.vnic);
899 	fwr->ovlanm = htobe16(f->fs.mask.vnic);
900 	bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
901 	bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
902 	bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
903 	bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
904 	fwr->lp = htobe16(f->fs.val.dport);
905 	fwr->lpm = htobe16(f->fs.mask.dport);
906 	fwr->fp = htobe16(f->fs.val.sport);
907 	fwr->fpm = htobe16(f->fs.mask.sport);
908 	/* sma = 0 tells the fw to use SMAC_SEL for source MAC address */
909 	bzero(fwr->sma, sizeof (fwr->sma));
910 	if (sc->params.filter2_wr_support) {
911 		fwr->filter_type_swapmac =
912 		    V_FW_FILTER2_WR_SWAPMAC(f->fs.swapmac);
913 		fwr->natmode_to_ulp_type =
914 		    V_FW_FILTER2_WR_ULP_TYPE(f->fs.nat_mode ?
915 			ULP_MODE_TCPDDP : ULP_MODE_NONE) |
916 		    V_FW_FILTER2_WR_NATFLAGCHECK(f->fs.nat_flag_chk) |
917 		    V_FW_FILTER2_WR_NATMODE(f->fs.nat_mode);
918 		memcpy(fwr->newlip, f->fs.nat_dip, sizeof(fwr->newlip));
919 		memcpy(fwr->newfip, f->fs.nat_sip, sizeof(fwr->newfip));
920 		fwr->newlport = htobe16(f->fs.nat_dport);
921 		fwr->newfport = htobe16(f->fs.nat_sport);
922 		fwr->natseqcheck = htobe32(f->fs.nat_seq_chk);
923 	}
924 	commit_wrq_wr(&sc->sge.ctrlq[0], fwr, &cookie);
925 
926 	/* Wait for response. */
927 	mtx_lock(&sc->tids.ftid_lock);
928 	for (;;) {
929 		if (f->pending == 0) {
930 			rc = f->valid ? 0 : EIO;
931 			break;
932 		}
933 		if (cv_wait_sig(&sc->tids.ftid_cv, &sc->tids.ftid_lock) != 0) {
934 			rc = EINPROGRESS;
935 			break;
936 		}
937 	}
938 	mtx_unlock(&sc->tids.ftid_lock);
939 	return (rc);
940 }
941 
942 static int
943 hashfilter_ntuple(struct adapter *sc, const struct t4_filter_specification *fs,
944     uint64_t *ftuple)
945 {
946 	struct tp_params *tp = &sc->params.tp;
947 	uint16_t fmask;
948 
949 	/*
950 	 * Initialize each of the fields which we care about which are present
951 	 * in the Compressed Filter Tuple.
952 	 */
953 #define SFF(V, S) ((uint64_t)(V) << S) /* Shifted Filter Field. */
954 	*ftuple = fmask = 0;
955 	if (chip_id(sc) >= CHELSIO_T7) {
956 		if (tp->ipsecidx_shift >= 0 && fs->mask.ipsecidx) {
957 			*ftuple |= SFF(fs->val.ipsecidx, tp->ipsecidx_shift);
958 			fmask |= F_IPSECIDX;
959 		}
960 		if (tp->fcoe_shift >= 0 && fs->mask.fcoe) {
961 			*ftuple |= SFF(fs->val.fcoe, tp->fcoe_shift);
962 			fmask |= F_T7_FCOE;
963 		}
964 		if (tp->port_shift >= 0 && fs->mask.iport) {
965 			*ftuple |= (uint64_t)fs->val.iport << tp->port_shift;
966 			fmask |= F_T7_PORT;
967 		}
968 		if (tp->vnic_shift >= 0 && fs->mask.vnic) {
969 			/* vnic_mode was already validated. */
970 			if (tp->vnic_mode == FW_VNIC_MODE_PF_VF)
971 				MPASS(fs->mask.pfvf_vld);
972 			else if (tp->vnic_mode == FW_VNIC_MODE_OUTER_VLAN)
973 				MPASS(fs->mask.ovlan_vld);
974 #ifdef notyet
975 			else if (tp->vnic_mode == FW_VNIC_MODE_ENCAP_EN)
976 				MPASS(fs->mask.encap_vld);
977 #endif
978 			*ftuple |= SFF(F_FT_VNID_ID_VLD | fs->val.vnic, tp->vnic_shift);
979 			fmask |= F_T7_VNIC_ID;
980 		}
981 		if (tp->vlan_shift >= 0 && fs->mask.vlan) {
982 			*ftuple |= SFF(F_FT_VLAN_VLD | fs->val.vlan, tp->vlan_shift);
983 			fmask |= F_T7_VLAN;
984 		}
985 		if (tp->tos_shift >= 0 && fs->mask.tos) {
986 			*ftuple |= SFF(fs->val.tos, tp->tos_shift);
987 			fmask |= F_T7_TOS;
988 		}
989 		if (tp->protocol_shift >= 0 && fs->mask.proto) {
990 			*ftuple |= SFF(fs->val.proto, tp->protocol_shift);
991 			fmask |= F_T7_PROTOCOL;
992 		}
993 		if (tp->ethertype_shift >= 0 && fs->mask.ethtype) {
994 			*ftuple |= SFF(fs->val.ethtype, tp->ethertype_shift);
995 			fmask |= F_T7_ETHERTYPE;
996 		}
997 		if (tp->macmatch_shift >= 0 && fs->mask.macidx) {
998 			*ftuple |= SFF(fs->val.macidx, tp->macmatch_shift);
999 			fmask |= F_T7_MACMATCH;
1000 		}
1001 		if (tp->matchtype_shift >= 0 && fs->mask.matchtype) {
1002 			*ftuple |= SFF(fs->val.matchtype, tp->matchtype_shift);
1003 			fmask |= F_T7_MPSHITTYPE;
1004 		}
1005 		if (tp->frag_shift >= 0 && fs->mask.frag) {
1006 			*ftuple |= SFF(fs->val.frag, tp->frag_shift);
1007 			fmask |= F_T7_FRAGMENTATION;
1008 		}
1009 		if (tp->roce_shift >= 0 && fs->mask.roce) {
1010 			*ftuple |= SFF(fs->val.roce, tp->roce_shift);
1011 			fmask |= F_ROCE;
1012 		}
1013 		if (tp->synonly_shift >= 0 && fs->mask.synonly) {
1014 			*ftuple |= SFF(fs->val.synonly, tp->synonly_shift);
1015 			fmask |= F_SYNONLY;
1016 		}
1017 		if (tp->tcpflags_shift >= 0 && fs->mask.tcpflags) {
1018 			*ftuple |= SFF(fs->val.tcpflags, tp->synonly_shift);
1019 			fmask |= F_TCPFLAGS;
1020 		}
1021 	} else {
1022 		if (fs->mask.ipsecidx || fs->mask.roce || fs->mask.synonly ||
1023 		    fs->mask.tcpflags) {
1024 			MPASS(tp->ipsecidx_shift == -1);
1025 			MPASS(tp->roce_shift == -1);
1026 			MPASS(tp->synonly_shift == -1);
1027 			MPASS(tp->tcpflags_shift == -1);
1028 			return (EINVAL);
1029 		}
1030 		if (tp->fcoe_shift >= 0 && fs->mask.fcoe) {
1031 			*ftuple |= SFF(fs->val.fcoe, tp->fcoe_shift);
1032 			fmask |= F_FCOE;
1033 		}
1034 		if (tp->port_shift >= 0 && fs->mask.iport) {
1035 			*ftuple |= (uint64_t)fs->val.iport << tp->port_shift;
1036 			fmask |= F_PORT;
1037 		}
1038 		if (tp->vnic_shift >= 0 && fs->mask.vnic) {
1039 			/* vnic_mode was already validated. */
1040 			if (tp->vnic_mode == FW_VNIC_MODE_PF_VF)
1041 				MPASS(fs->mask.pfvf_vld);
1042 			else if (tp->vnic_mode == FW_VNIC_MODE_OUTER_VLAN)
1043 				MPASS(fs->mask.ovlan_vld);
1044 #ifdef notyet
1045 			else if (tp->vnic_mode == FW_VNIC_MODE_ENCAP_EN)
1046 				MPASS(fs->mask.encap_vld);
1047 #endif
1048 			*ftuple |= SFF(F_FT_VNID_ID_VLD | fs->val.vnic, tp->vnic_shift);
1049 			fmask |= F_VNIC_ID;
1050 		}
1051 		if (tp->vlan_shift >= 0 && fs->mask.vlan) {
1052 			*ftuple |= SFF(F_FT_VLAN_VLD | fs->val.vlan, tp->vlan_shift);
1053 			fmask |= F_VLAN;
1054 		}
1055 		if (tp->tos_shift >= 0 && fs->mask.tos) {
1056 			*ftuple |= SFF(fs->val.tos, tp->tos_shift);
1057 			fmask |= F_TOS;
1058 		}
1059 		if (tp->protocol_shift >= 0 && fs->mask.proto) {
1060 			*ftuple |= SFF(fs->val.proto, tp->protocol_shift);
1061 			fmask |= F_PROTOCOL;
1062 		}
1063 		if (tp->ethertype_shift >= 0 && fs->mask.ethtype) {
1064 			*ftuple |= SFF(fs->val.ethtype, tp->ethertype_shift);
1065 			fmask |= F_ETHERTYPE;
1066 		}
1067 		if (tp->macmatch_shift >= 0 && fs->mask.macidx) {
1068 			*ftuple |= SFF(fs->val.macidx, tp->macmatch_shift);
1069 			fmask |= F_MACMATCH;
1070 		}
1071 		if (tp->matchtype_shift >= 0 && fs->mask.matchtype) {
1072 			*ftuple |= SFF(fs->val.matchtype, tp->matchtype_shift);
1073 			fmask |= F_MPSHITTYPE;
1074 		}
1075 		if (tp->frag_shift >= 0 && fs->mask.frag) {
1076 			*ftuple |= SFF(fs->val.frag, tp->frag_shift);
1077 			fmask |= F_FRAGMENTATION;
1078 		}
1079 	}
1080 #undef SFF
1081 
1082 	/* A hashfilter must conform to the hardware filter mask. */
1083 	if (fmask != tp->filter_mask)
1084 		return (EINVAL);
1085 
1086 	return (0);
1087 }
1088 
1089 static bool
1090 is_4tuple_specified(struct t4_filter_specification *fs)
1091 {
1092 	int i;
1093 	const int n = fs->type ? 16 : 4;
1094 
1095 	if (fs->mask.sport != 0xffff || fs->mask.dport != 0xffff)
1096 		return (false);
1097 
1098 	for (i = 0; i < n; i++) {
1099 		if (fs->mask.sip[i] != 0xff)
1100 			return (false);
1101 		if (fs->mask.dip[i] != 0xff)
1102 			return (false);
1103 	}
1104 
1105 	return (true);
1106 }
1107 
1108 int
1109 set_filter(struct adapter *sc, struct t4_filter *t)
1110 {
1111 	struct tid_info *ti = &sc->tids;
1112 	struct l2t_entry *l2te = NULL;
1113 	struct smt_entry *smt = NULL;
1114 	uint64_t ftuple;
1115 	int rc;
1116 
1117 	/*
1118 	 * Basic filter checks first.
1119 	 */
1120 
1121 	if (t->fs.hash) {
1122 		if (!is_hashfilter(sc) || ti->ntids == 0)
1123 			return (ENOTSUP);
1124 		/* Hardware, not user, selects a tid for hashfilters. */
1125 		if (t->idx != (uint32_t)-1)
1126 			return (EINVAL);
1127 		/* T5 can't count hashfilter hits. */
1128 		if (is_t5(sc) && t->fs.hitcnts)
1129 			return (EINVAL);
1130 		if (!is_4tuple_specified(&t->fs))
1131 			return (EINVAL);
1132 		rc = hashfilter_ntuple(sc, &t->fs, &ftuple);
1133 		if (rc != 0)
1134 			return (rc);
1135 	} else {
1136 		if (separate_hpfilter_region(sc) && t->fs.prio) {
1137 			if (ti->nhpftids == 0)
1138 				return (ENOTSUP);
1139 			if (t->idx >= ti->nhpftids)
1140 				return (EINVAL);
1141 		} else {
1142 			if (ti->nftids == 0)
1143 				return (ENOTSUP);
1144 			if (t->idx >= ti->nftids)
1145 				return (EINVAL);
1146 		}
1147 		/* IPv6 filter idx must be 4 aligned */
1148 		if (t->fs.type == 1 &&
1149 		    ((t->idx & 0x3) || t->idx + 4 >= ti->nftids))
1150 			return (EINVAL);
1151 	}
1152 
1153 	/* T4 doesn't support VLAN tag removal or rewrite, swapmac, and NAT. */
1154 	if (is_t4(sc) && t->fs.action == FILTER_SWITCH &&
1155 	    (t->fs.newvlan == VLAN_REMOVE || t->fs.newvlan == VLAN_REWRITE ||
1156 	    t->fs.swapmac || t->fs.nat_mode))
1157 		return (ENOTSUP);
1158 
1159 	if (t->fs.action == FILTER_SWITCH && t->fs.eport >= sc->params.nports)
1160 		return (EINVAL);
1161 	if (t->fs.val.iport >= sc->params.nports)
1162 		return (EINVAL);
1163 
1164 	/* Can't specify an iqid/rss_info if not steering. */
1165 	if (!t->fs.dirsteer && !t->fs.dirsteerhash && !t->fs.maskhash && t->fs.iq)
1166 		return (EINVAL);
1167 
1168 	/* Validate against the global filter mode and ingress config */
1169 	rc = check_fspec_against_fconf_iconf(sc, &t->fs);
1170 	if (rc != 0)
1171 		return (rc);
1172 
1173 	/*
1174 	 * Basic checks passed.  Make sure the queues and tid tables are setup.
1175 	 */
1176 
1177 	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
1178 	if (rc)
1179 		return (rc);
1180 
1181 	if (!hw_all_ok(sc)) {
1182 		rc = ENXIO;
1183 		goto done;
1184 	}
1185 
1186 	if (!(sc->flags & FULL_INIT_DONE) && ((rc = adapter_init(sc)) != 0))
1187 		goto done;
1188 
1189 	if (t->fs.hash) {
1190 		if (__predict_false(ti->hftid_hash_4t == NULL)) {
1191 			rc = alloc_hftid_hash(&sc->tids, HASH_NOWAIT);
1192 			if (rc != 0)
1193 				goto done;
1194 		}
1195 	} else if (separate_hpfilter_region(sc) && t->fs.prio &&
1196 	    __predict_false(ti->hpftid_tab == NULL)) {
1197 		MPASS(ti->nhpftids != 0);
1198 		KASSERT(ti->hpftids_in_use == 0,
1199 		    ("%s: no memory allocated but hpftids_in_use is %u",
1200 		    __func__, ti->hpftids_in_use));
1201 		ti->hpftid_tab = malloc(sizeof(struct filter_entry) *
1202 		    ti->nhpftids, M_CXGBE, M_NOWAIT | M_ZERO);
1203 		if (ti->hpftid_tab == NULL) {
1204 			rc = ENOMEM;
1205 			goto done;
1206 		}
1207 		if (!mtx_initialized(&sc->tids.ftid_lock)) {
1208 			mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF);
1209 			cv_init(&ti->ftid_cv, "t4fcv");
1210 		}
1211 	} else if (__predict_false(ti->ftid_tab == NULL)) {
1212 		MPASS(ti->nftids != 0);
1213 		KASSERT(ti->ftids_in_use == 0,
1214 		    ("%s: no memory allocated but ftids_in_use is %u",
1215 		    __func__, ti->ftids_in_use));
1216 		ti->ftid_tab = malloc(sizeof(struct filter_entry) * ti->nftids,
1217 		    M_CXGBE, M_NOWAIT | M_ZERO);
1218 		if (ti->ftid_tab == NULL) {
1219 			rc = ENOMEM;
1220 			goto done;
1221 		}
1222 		if (!mtx_initialized(&sc->tids.ftid_lock)) {
1223 			mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF);
1224 			cv_init(&ti->ftid_cv, "t4fcv");
1225 		}
1226 	}
1227 done:
1228 	end_synchronized_op(sc, 0);
1229 	if (rc != 0)
1230 		return (rc);
1231 
1232 	/*
1233 	 * Allocate L2T entry, SMT entry, etc.
1234 	 */
1235 
1236 	if (t->fs.newdmac || t->fs.newvlan) {
1237 		/* This filter needs an L2T entry; allocate one. */
1238 		l2te = t4_l2t_alloc_switching(sc, t->fs.vlan, t->fs.eport,
1239 		    t->fs.dmac);
1240 		if (__predict_false(l2te == NULL)) {
1241 			rc = EAGAIN;
1242 			goto error;
1243 		}
1244 	}
1245 
1246 	if (t->fs.newsmac) {
1247 		/* This filter needs an SMT entry; allocate one. */
1248 		smt = t4_smt_alloc_switching(sc->smt, t->fs.smac);
1249 		if (__predict_false(smt == NULL)) {
1250 			rc = EAGAIN;
1251 			goto error;
1252 		}
1253 		rc = t4_smt_set_switching(sc, smt, 0x0, t->fs.smac);
1254 		if (rc)
1255 			goto error;
1256 	}
1257 
1258 	if (t->fs.hash)
1259 		rc = set_hashfilter(sc, t, ftuple, l2te, smt);
1260 	else
1261 		rc = set_tcamfilter(sc, t, l2te, smt);
1262 
1263 	if (rc != 0 && rc != EINPROGRESS) {
1264 error:
1265 		if (l2te)
1266 			t4_l2t_release(l2te);
1267 		if (smt)
1268 			t4_smt_release(smt);
1269 	}
1270 	return (rc);
1271 }
1272 
1273 static int
1274 del_tcamfilter(struct adapter *sc, struct t4_filter *t)
1275 {
1276 	struct filter_entry *f;
1277 	struct fw_filter_wr *fwr;
1278 	struct wrq_cookie cookie;
1279 	int rc, nfilters;
1280 #ifdef INVARIANTS
1281 	u_int tid_base;
1282 #endif
1283 
1284 	mtx_lock(&sc->tids.ftid_lock);
1285 	if (separate_hpfilter_region(sc) && t->fs.prio) {
1286 		nfilters = sc->tids.nhpftids;
1287 		f = sc->tids.hpftid_tab;
1288 #ifdef INVARIANTS
1289 		tid_base = sc->tids.hpftid_base;
1290 #endif
1291 	} else {
1292 		nfilters = sc->tids.nftids;
1293 		f = sc->tids.ftid_tab;
1294 #ifdef INVARIANTS
1295 		tid_base = sc->tids.ftid_base;
1296 #endif
1297 	}
1298 	MPASS(f != NULL);	/* Caller checked this. */
1299 	if (t->idx >= nfilters) {
1300 		rc = EINVAL;
1301 		goto done;
1302 	}
1303 	f += t->idx;
1304 
1305 	if (f->locked) {
1306 		rc = EPERM;
1307 		goto done;
1308 	}
1309 	if (f->pending) {
1310 		rc = EBUSY;
1311 		goto done;
1312 	}
1313 	if (f->valid == 0) {
1314 		rc = EINVAL;
1315 		goto done;
1316 	}
1317 	MPASS(f->tid == tid_base + t->idx);
1318 	fwr = start_wrq_wr(&sc->sge.ctrlq[0], howmany(sizeof(*fwr), 16), &cookie);
1319 	if (fwr == NULL) {
1320 		rc = ENOMEM;
1321 		goto done;
1322 	}
1323 
1324 	bzero(fwr, sizeof (*fwr));
1325 	t4_mk_filtdelwr(f->tid, fwr, sc->sge.fwq.abs_id);
1326 	f->pending = 1;
1327 	commit_wrq_wr(&sc->sge.ctrlq[0], fwr, &cookie);
1328 	t->fs = f->fs;	/* extra info for the caller */
1329 
1330 	for (;;) {
1331 		if (f->pending == 0) {
1332 			rc = f->valid ? EIO : 0;
1333 			break;
1334 		}
1335 		if (cv_wait_sig(&sc->tids.ftid_cv, &sc->tids.ftid_lock) != 0) {
1336 			rc = EINPROGRESS;
1337 			break;
1338 		}
1339 	}
1340 done:
1341 	mtx_unlock(&sc->tids.ftid_lock);
1342 	return (rc);
1343 }
1344 
1345 int
1346 del_filter(struct adapter *sc, struct t4_filter *t)
1347 {
1348 
1349 	/* No filters possible if not initialized yet. */
1350 	if (!(sc->flags & FULL_INIT_DONE))
1351 		return (EINVAL);
1352 
1353 	/*
1354 	 * The checks for tid tables ensure that the locks that del_* will reach
1355 	 * for are initialized.
1356 	 */
1357 	if (t->fs.hash) {
1358 		if (sc->tids.hftid_hash_4t != NULL)
1359 			return (del_hashfilter(sc, t));
1360 	} else if (separate_hpfilter_region(sc) && t->fs.prio) {
1361 		if (sc->tids.hpftid_tab != NULL)
1362 			return (del_tcamfilter(sc, t));
1363 	} else {
1364 		if (sc->tids.ftid_tab != NULL)
1365 			return (del_tcamfilter(sc, t));
1366 	}
1367 
1368 	return (EINVAL);
1369 }
1370 
1371 /*
1372  * Release secondary resources associated with the filter.
1373  */
1374 static void
1375 free_filter_resources(struct filter_entry *f)
1376 {
1377 
1378 	if (f->l2te) {
1379 		t4_l2t_release(f->l2te);
1380 		f->l2te = NULL;
1381 	}
1382 	if (f->smt) {
1383 		t4_smt_release(f->smt);
1384 		f->smt = NULL;
1385 	}
1386 }
1387 
1388 static int
1389 set_tcb_field(struct adapter *sc, u_int tid, uint16_t word, uint64_t mask,
1390     uint64_t val, int no_reply)
1391 {
1392 	struct wrq_cookie cookie;
1393 	struct cpl_set_tcb_field *req;
1394 
1395 	req = start_wrq_wr(&sc->sge.ctrlq[0], howmany(sizeof(*req), 16), &cookie);
1396 	if (req == NULL)
1397 		return (ENOMEM);
1398 	bzero(req, sizeof(*req));
1399 	INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, tid);
1400 	if (no_reply) {
1401 		req->reply_ctrl = htobe16(F_NO_REPLY);
1402 	} else {
1403 		const int qid = sc->sge.fwq.abs_id;
1404 
1405 		if (chip_id(sc) >= CHELSIO_T7) {
1406 			req->reply_ctrl = htobe16(V_T7_QUEUENO(qid) |
1407 			    V_T7_REPLY_CHAN(0) | V_NO_REPLY(0));
1408 		} else {
1409 			req->reply_ctrl = htobe16(V_QUEUENO(qid) |
1410 			    V_REPLY_CHAN(0) | V_NO_REPLY(0));
1411 		}
1412 	}
1413 	req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(CPL_COOKIE_HASHFILTER));
1414 	req->mask = htobe64(mask);
1415 	req->val = htobe64(val);
1416 	commit_wrq_wr(&sc->sge.ctrlq[0], req, &cookie);
1417 
1418 	return (0);
1419 }
1420 
1421 /* Set one of the t_flags bits in the TCB. */
1422 static inline int
1423 set_tcb_tflag(struct adapter *sc, int tid, u_int bit_pos, u_int val,
1424     u_int no_reply)
1425 {
1426 
1427 	return (set_tcb_field(sc, tid,  W_TCB_T_FLAGS, 1ULL << bit_pos,
1428 	    (uint64_t)val << bit_pos, no_reply));
1429 }
1430 
1431 int
1432 t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1433 {
1434 	struct adapter *sc = iq->adapter;
1435 	const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
1436 	u_int tid = GET_TID(rpl);
1437 	u_int rc, idx;
1438 	struct filter_entry *f;
1439 
1440 	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
1441 	    rss->opcode));
1442 
1443 
1444 	if (is_hpftid(sc, tid)) {
1445 		idx = tid - sc->tids.hpftid_base;
1446 		f = &sc->tids.hpftid_tab[idx];
1447 	} else if (is_ftid(sc, tid)) {
1448 		idx = tid - sc->tids.ftid_base;
1449 		f = &sc->tids.ftid_tab[idx];
1450 	} else
1451 		panic("%s: FW reply for invalid TID %d.", __func__, tid);
1452 
1453 	MPASS(f->tid == tid);
1454 	rc = G_COOKIE(rpl->cookie);
1455 
1456 	mtx_lock(&sc->tids.ftid_lock);
1457 	KASSERT(f->pending, ("%s: reply %d for filter[%u] that isn't pending.",
1458 	    __func__, rc, tid));
1459 	switch(rc) {
1460 	case FW_FILTER_WR_FLT_ADDED:
1461 		/* set-filter succeeded */
1462 		f->valid = 1;
1463 		if (f->fs.newsmac) {
1464 			MPASS(f->smt != NULL);
1465 			set_tcb_tflag(sc, f->tid, S_TF_CCTRL_CWR, 1, 1);
1466 			set_tcb_field(sc, f->tid, W_TCB_SMAC_SEL,
1467 			    V_TCB_SMAC_SEL(M_TCB_SMAC_SEL),
1468 			    V_TCB_SMAC_SEL(f->smt->idx), 1);
1469 			/* XXX: wait for reply to TCB update before !pending */
1470 		}
1471 		break;
1472 	case FW_FILTER_WR_FLT_DELETED:
1473 		/* del-filter succeeded */
1474 		MPASS(f->valid == 1);
1475 		f->valid = 0;
1476 		/* Fall through */
1477 	case FW_FILTER_WR_SMT_TBL_FULL:
1478 		/* set-filter failed due to lack of SMT space. */
1479 		MPASS(f->valid == 0);
1480 		free_filter_resources(f);
1481 		if (separate_hpfilter_region(sc) && f->fs.prio)
1482 			sc->tids.hpftids_in_use--;
1483 		else
1484 			sc->tids.ftids_in_use--;
1485 		break;
1486 	case FW_FILTER_WR_SUCCESS:
1487 	case FW_FILTER_WR_EINVAL:
1488 	default:
1489 		panic("%s: unexpected reply %d for filter[%d].", __func__, rc,
1490 		    idx);
1491 	}
1492 	f->pending = 0;
1493 	cv_broadcast(&sc->tids.ftid_cv);
1494 	mtx_unlock(&sc->tids.ftid_lock);
1495 
1496 	return (0);
1497 }
1498 
1499 /*
1500  * This is the reply to the Active Open that created the filter.  Additional TCB
1501  * updates may be required to complete the filter configuration.
1502  */
1503 int
1504 t4_hashfilter_ao_rpl(struct sge_iq *iq, const struct rss_header *rss,
1505     struct mbuf *m)
1506 {
1507 	struct adapter *sc = iq->adapter;
1508 	const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1);
1509 	u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status)));
1510 	u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status));
1511 	struct filter_entry *f = lookup_atid(sc, atid);
1512 
1513 	KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1514 
1515 	mtx_lock(&sc->tids.hftid_lock);
1516 	KASSERT(f->pending, ("%s: hashfilter[%p] isn't pending.", __func__, f));
1517 	KASSERT(f->tid == -1, ("%s: hashfilter[%p] has tid %d already.",
1518 	    __func__, f, f->tid));
1519 	if (status == CPL_ERR_NONE) {
1520 		f->tid = GET_TID(cpl);
1521 		MPASS(lookup_hftid(sc, f->tid) == NULL);
1522 		insert_hftid(sc, f);
1523 		/*
1524 		 * Leave the filter pending until it is fully set up, which will
1525 		 * be indicated by the reply to the last TCB update.  No need to
1526 		 * unblock the ioctl thread either.
1527 		 */
1528 		if (configure_hashfilter_tcb(sc, f) == EINPROGRESS)
1529 			goto done;
1530 		f->valid = 1;
1531 		f->pending = 0;
1532 	} else {
1533 		/* provide errno instead of tid to ioctl */
1534 		f->tid = act_open_rpl_status_to_errno(status);
1535 		f->valid = 0;
1536 		f->pending = 0;
1537 		if (act_open_has_tid(status))
1538 			release_tid(sc, GET_TID(cpl), &sc->sge.ctrlq[0]);
1539 		free_filter_resources(f);
1540 		remove_hf(sc, f);
1541 		if (f->locked == 0)
1542 			free(f, M_CXGBE);
1543 	}
1544 	cv_broadcast(&sc->tids.hftid_cv);
1545 done:
1546 	mtx_unlock(&sc->tids.hftid_lock);
1547 
1548 	free_atid(sc, atid);
1549 	return (0);
1550 }
1551 
1552 int
1553 t4_hashfilter_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss,
1554     struct mbuf *m)
1555 {
1556 	struct adapter *sc = iq->adapter;
1557 	const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
1558 	u_int tid = GET_TID(rpl);
1559 	struct filter_entry *f;
1560 
1561 	mtx_lock(&sc->tids.hftid_lock);
1562 	f = lookup_hftid(sc, tid);
1563 	KASSERT(f->tid == tid, ("%s: filter tid mismatch", __func__));
1564 	KASSERT(f->pending, ("%s: hashfilter %p [%u] isn't pending.", __func__,
1565 	    f, tid));
1566 	KASSERT(f->valid == 0, ("%s: hashfilter %p [%u] is valid already.",
1567 	    __func__, f, tid));
1568 	f->pending = 0;
1569 	if (rpl->status == 0) {
1570 		f->valid = 1;
1571 	} else {
1572 		f->tid = EIO;
1573 		f->valid = 0;
1574 		free_filter_resources(f);
1575 		remove_hftid(sc, f);
1576 		remove_hf(sc, f);
1577 		release_tid(sc, tid, &sc->sge.ctrlq[0]);
1578 		if (f->locked == 0)
1579 			free(f, M_CXGBE);
1580 	}
1581 	cv_broadcast(&sc->tids.hftid_cv);
1582 	mtx_unlock(&sc->tids.hftid_lock);
1583 
1584 	return (0);
1585 }
1586 
1587 int
1588 t4_del_hashfilter_rpl(struct sge_iq *iq, const struct rss_header *rss,
1589     struct mbuf *m)
1590 {
1591 	struct adapter *sc = iq->adapter;
1592 	const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1);
1593 	unsigned int tid = GET_TID(cpl);
1594 	struct filter_entry *f;
1595 
1596 	mtx_lock(&sc->tids.hftid_lock);
1597 	f = lookup_hftid(sc, tid);
1598 	KASSERT(f->tid == tid, ("%s: filter tid mismatch", __func__));
1599 	KASSERT(f->pending, ("%s: hashfilter %p [%u] isn't pending.", __func__,
1600 	    f, tid));
1601 	KASSERT(f->valid, ("%s: hashfilter %p [%u] isn't valid.", __func__, f,
1602 	    tid));
1603 	f->pending = 0;
1604 	if (cpl->status == 0) {
1605 		f->valid = 0;
1606 		free_filter_resources(f);
1607 		remove_hftid(sc, f);
1608 		remove_hf(sc, f);
1609 		release_tid(sc, tid, &sc->sge.ctrlq[0]);
1610 		if (f->locked == 0)
1611 			free(f, M_CXGBE);
1612 	}
1613 	cv_broadcast(&sc->tids.hftid_cv);
1614 	mtx_unlock(&sc->tids.hftid_lock);
1615 
1616 	return (0);
1617 }
1618 
1619 static int
1620 get_tcamfilter(struct adapter *sc, struct t4_filter *t)
1621 {
1622 	int i, nfilters;
1623 	struct filter_entry *f;
1624 	u_int in_use;
1625 #ifdef INVARIANTS
1626 	u_int tid_base;
1627 #endif
1628 
1629 	MPASS(!t->fs.hash);
1630 
1631 	if (separate_hpfilter_region(sc) && t->fs.prio) {
1632 		nfilters = sc->tids.nhpftids;
1633 		f = sc->tids.hpftid_tab;
1634 		in_use = sc->tids.hpftids_in_use;
1635 #ifdef INVARIANTS
1636 		tid_base = sc->tids.hpftid_base;
1637 #endif
1638 	} else {
1639 		nfilters = sc->tids.nftids;
1640 		f = sc->tids.ftid_tab;
1641 		in_use = sc->tids.ftids_in_use;
1642 #ifdef INVARIANTS
1643 		tid_base = sc->tids.ftid_base;
1644 #endif
1645 	}
1646 
1647 	if (in_use == 0 || f == NULL || t->idx >= nfilters) {
1648 		t->idx = 0xffffffff;
1649 		return (0);
1650 	}
1651 
1652 	f += t->idx;
1653 	mtx_lock(&sc->tids.ftid_lock);
1654 	for (i = t->idx; i < nfilters; i++, f++) {
1655 		if (f->valid) {
1656 			MPASS(f->tid == tid_base + i);
1657 			t->idx = i;
1658 			t->l2tidx = f->l2te ? f->l2te->idx : 0;
1659 			t->smtidx = f->smt ? f->smt->idx : 0;
1660 			if (f->fs.hitcnts)
1661 				t->hits = get_filter_hits(sc, f->tid);
1662 			else
1663 				t->hits = UINT64_MAX;
1664 			t->fs = f->fs;
1665 
1666 			goto done;
1667 		}
1668 	}
1669 	t->idx = 0xffffffff;
1670 done:
1671 	mtx_unlock(&sc->tids.ftid_lock);
1672 	return (0);
1673 }
1674 
1675 static int
1676 get_hashfilter(struct adapter *sc, struct t4_filter *t)
1677 {
1678 	struct tid_info *ti = &sc->tids;
1679 	int tid;
1680 	struct filter_entry *f;
1681 	const int inv_tid = ti->ntids + ti->tid_base;
1682 
1683 	MPASS(t->fs.hash);
1684 
1685 	if (ti->tids_in_use == 0 || ti->hftid_hash_tid == NULL ||
1686 	    t->idx >= inv_tid) {
1687 		t->idx = 0xffffffff;
1688 		return (0);
1689 	}
1690 	if (t->idx < ti->tid_base)
1691 		t->idx = ti->tid_base;
1692 
1693 	mtx_lock(&ti->hftid_lock);
1694 	for (tid = t->idx; tid < inv_tid; tid++) {
1695 		f = lookup_hftid(sc, tid);
1696 		if (f != NULL && f->valid) {
1697 			t->idx = tid;
1698 			t->l2tidx = f->l2te ? f->l2te->idx : 0;
1699 			t->smtidx = f->smt ? f->smt->idx : 0;
1700 			if (f->fs.hitcnts)
1701 				t->hits = get_filter_hits(sc, tid);
1702 			else
1703 				t->hits = UINT64_MAX;
1704 			t->fs = f->fs;
1705 
1706 			goto done;
1707 		}
1708 	}
1709 	t->idx = 0xffffffff;
1710 done:
1711 	mtx_unlock(&ti->hftid_lock);
1712 	return (0);
1713 }
1714 
1715 static void
1716 mk_act_open_req6(struct adapter *sc, struct filter_entry *f, int atid,
1717     uint64_t ftuple, struct cpl_act_open_req6 *cpl)
1718 {
1719 	struct cpl_t5_act_open_req6 *cpl5 = (void *)cpl;
1720 	struct cpl_t6_act_open_req6 *cpl6 = (void *)cpl;
1721 
1722 	/* Review changes to CPL after cpl_t6_act_open_req if this goes off. */
1723 	MPASS(chip_id(sc) >= CHELSIO_T5 && chip_id(sc) <= CHELSIO_T6);
1724 	MPASS(atid >= 0);
1725 
1726 	if (chip_id(sc) == CHELSIO_T5) {
1727 		INIT_TP_WR(cpl5, 0);
1728 	} else {
1729 		INIT_TP_WR(cpl6, 0);
1730 		cpl6->rsvd2 = 0;
1731 		cpl6->opt3 = 0;
1732 	}
1733 
1734 	OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
1735 	    V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) |
1736 	    V_TID_COOKIE(CPL_COOKIE_HASHFILTER)));
1737 	cpl->local_port = htobe16(f->fs.val.dport);
1738 	cpl->peer_port = htobe16(f->fs.val.sport);
1739 	cpl->local_ip_hi = *(uint64_t *)(&f->fs.val.dip);
1740 	cpl->local_ip_lo = *(((uint64_t *)&f->fs.val.dip) + 1);
1741 	cpl->peer_ip_hi = *(uint64_t *)(&f->fs.val.sip);
1742 	cpl->peer_ip_lo = *(((uint64_t *)&f->fs.val.sip) + 1);
1743 	cpl->opt0 = htobe64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE ||
1744 	    f->fs.newvlan == VLAN_REWRITE) | V_DELACK(f->fs.hitcnts) |
1745 	    V_L2T_IDX(f->l2te ? f->l2te->idx : 0) | V_TX_CHAN(f->fs.eport) |
1746 	    V_NO_CONG(f->fs.rpttid) |
1747 	    V_ULP_MODE(f->fs.nat_mode ? ULP_MODE_TCPDDP : ULP_MODE_NONE) |
1748 	    F_TCAM_BYPASS | F_NON_OFFLOAD);
1749 
1750 	cpl6->params = htobe64(V_FILTER_TUPLE(ftuple));
1751 	cpl6->opt2 = htobe32(F_RSS_QUEUE_VALID | V_RSS_QUEUE(f->fs.iq) |
1752 	    V_TX_QUEUE(f->fs.nat_mode) | V_WND_SCALE_EN(f->fs.nat_flag_chk) |
1753 	    V_RX_FC_DISABLE(f->fs.nat_seq_chk ? 1 : 0) | F_T5_OPT_2_VALID |
1754 	    F_RX_CHANNEL | V_SACK_EN(f->fs.swapmac) |
1755 	    V_CONG_CNTRL((f->fs.action == FILTER_DROP) | (f->fs.dirsteer << 1)) |
1756 	    V_PACE(f->fs.maskhash | (f->fs.dirsteerhash << 1)));
1757 }
1758 
1759 static void
1760 mk_act_open_req(struct adapter *sc, struct filter_entry *f, int atid,
1761     uint64_t ftuple, struct cpl_act_open_req *cpl)
1762 {
1763 	struct cpl_t5_act_open_req *cpl5 = (void *)cpl;
1764 	struct cpl_t6_act_open_req *cpl6 = (void *)cpl;
1765 
1766 	/* Review changes to CPL after cpl_t6_act_open_req if this goes off. */
1767 	MPASS(chip_id(sc) >= CHELSIO_T5 && chip_id(sc) <= CHELSIO_T6);
1768 	MPASS(atid >= 0);
1769 
1770 	if (chip_id(sc) == CHELSIO_T5) {
1771 		INIT_TP_WR(cpl5, 0);
1772 	} else {
1773 		INIT_TP_WR(cpl6, 0);
1774 		cpl6->rsvd2 = 0;
1775 		cpl6->opt3 = 0;
1776 	}
1777 
1778 	OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
1779 	    V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) |
1780 	    V_TID_COOKIE(CPL_COOKIE_HASHFILTER)));
1781 	cpl->local_port = htobe16(f->fs.val.dport);
1782 	cpl->peer_port = htobe16(f->fs.val.sport);
1783 	cpl->local_ip = f->fs.val.dip[0] | f->fs.val.dip[1] << 8 |
1784 	    f->fs.val.dip[2] << 16 | f->fs.val.dip[3] << 24;
1785 	cpl->peer_ip = f->fs.val.sip[0] | f->fs.val.sip[1] << 8 |
1786 		f->fs.val.sip[2] << 16 | f->fs.val.sip[3] << 24;
1787 	cpl->opt0 = htobe64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE ||
1788 	    f->fs.newvlan == VLAN_REWRITE) | V_DELACK(f->fs.hitcnts) |
1789 	    V_L2T_IDX(f->l2te ? f->l2te->idx : 0) | V_TX_CHAN(f->fs.eport) |
1790 	    V_NO_CONG(f->fs.rpttid) |
1791 	    V_ULP_MODE(f->fs.nat_mode ? ULP_MODE_TCPDDP : ULP_MODE_NONE) |
1792 	    F_TCAM_BYPASS | F_NON_OFFLOAD);
1793 
1794 	cpl6->params = htobe64(V_FILTER_TUPLE(ftuple));
1795 	cpl6->opt2 = htobe32(F_RSS_QUEUE_VALID | V_RSS_QUEUE(f->fs.iq) |
1796 	    V_TX_QUEUE(f->fs.nat_mode) | V_WND_SCALE_EN(f->fs.nat_flag_chk) |
1797 	    V_RX_FC_DISABLE(f->fs.nat_seq_chk ? 1 : 0) | F_T5_OPT_2_VALID |
1798 	    F_RX_CHANNEL | V_SACK_EN(f->fs.swapmac) |
1799 	    V_CONG_CNTRL((f->fs.action == FILTER_DROP) | (f->fs.dirsteer << 1)) |
1800 	    V_PACE(f->fs.maskhash | (f->fs.dirsteerhash << 1)));
1801 }
1802 
1803 static int
1804 act_open_cpl_len16(struct adapter *sc, int isipv6)
1805 {
1806 	int idx;
1807 	static const int sz_table[4][2] = {
1808 		{
1809 			howmany(sizeof (struct cpl_act_open_req), 16),
1810 			howmany(sizeof (struct cpl_act_open_req6), 16)
1811 		},
1812 		{
1813 			howmany(sizeof (struct cpl_t5_act_open_req), 16),
1814 			howmany(sizeof (struct cpl_t5_act_open_req6), 16)
1815 		},
1816 		{
1817 			howmany(sizeof (struct cpl_t6_act_open_req), 16),
1818 			howmany(sizeof (struct cpl_t6_act_open_req6), 16)
1819 		},
1820 		{
1821 			howmany(sizeof (struct cpl_t7_act_open_req), 16),
1822 			howmany(sizeof (struct cpl_t7_act_open_req6), 16)
1823 		},
1824 	};
1825 
1826 	MPASS(chip_id(sc) >= CHELSIO_T4);
1827 	idx = min(chip_id(sc) - CHELSIO_T4, 3);
1828 
1829 	return (sz_table[idx][!!isipv6]);
1830 }
1831 
1832 static int
1833 set_hashfilter(struct adapter *sc, struct t4_filter *t, uint64_t ftuple,
1834     struct l2t_entry *l2te, struct smt_entry *smt)
1835 {
1836 	void *wr;
1837 	struct wrq_cookie cookie;
1838 	struct filter_entry *f;
1839 	int rc, atid = -1;
1840 	uint32_t hash;
1841 
1842 	MPASS(t->fs.hash);
1843 	/* Already validated against fconf, iconf */
1844 	MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0);
1845 	MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0);
1846 
1847 	hash = hf_hashfn_4t(&t->fs);
1848 
1849 	mtx_lock(&sc->tids.hftid_lock);
1850 	if (lookup_hf(sc, &t->fs, hash) != NULL) {
1851 		rc = EEXIST;
1852 		goto done;
1853 	}
1854 
1855 	f = malloc(sizeof(*f), M_CXGBE, M_ZERO | M_NOWAIT);
1856 	if (__predict_false(f == NULL)) {
1857 		rc = ENOMEM;
1858 		goto done;
1859 	}
1860 	f->fs = t->fs;
1861 	f->l2te = l2te;
1862 	f->smt = smt;
1863 
1864 	atid = alloc_atid(sc, f);
1865 	if (__predict_false(atid) == -1) {
1866 		free(f, M_CXGBE);
1867 		rc = EAGAIN;
1868 		goto done;
1869 	}
1870 	MPASS(atid >= 0);
1871 
1872 	wr = start_wrq_wr(&sc->sge.ctrlq[0], act_open_cpl_len16(sc, f->fs.type),
1873 	    &cookie);
1874 	if (wr == NULL) {
1875 		free_atid(sc, atid);
1876 		free(f, M_CXGBE);
1877 		rc = ENOMEM;
1878 		goto done;
1879 	}
1880 	if (f->fs.type)
1881 		mk_act_open_req6(sc, f, atid, ftuple, wr);
1882 	else
1883 		mk_act_open_req(sc, f, atid, ftuple, wr);
1884 
1885 	f->locked = 1; /* ithread mustn't free f if ioctl is still around. */
1886 	f->pending = 1;
1887 	f->tid = -1;
1888 	insert_hf(sc, f, hash);
1889 	commit_wrq_wr(&sc->sge.ctrlq[0], wr, &cookie);
1890 
1891 	for (;;) {
1892 		MPASS(f->locked);
1893 		if (f->pending == 0) {
1894 			if (f->valid) {
1895 				rc = 0;
1896 				f->locked = 0;
1897 				t->idx = f->tid;
1898 			} else {
1899 				rc = f->tid;
1900 				free(f, M_CXGBE);
1901 			}
1902 			break;
1903 		}
1904 		if (cv_wait_sig(&sc->tids.hftid_cv, &sc->tids.hftid_lock) != 0) {
1905 			f->locked = 0;
1906 			rc = EINPROGRESS;
1907 			break;
1908 		}
1909 	}
1910 done:
1911 	mtx_unlock(&sc->tids.hftid_lock);
1912 	return (rc);
1913 }
1914 
1915 /* ABORT_REQ sent as a ULP command looks like this */
1916 #define LEN__ABORT_REQ_ULP (sizeof(struct ulp_txpkt) + \
1917 	sizeof(struct ulptx_idata) + sizeof(struct cpl_abort_req_core))
1918 
1919 static void *
1920 mk_abort_req_ulp(struct ulp_txpkt *ulpmc, uint32_t tid)
1921 {
1922 	struct ulptx_idata *ulpsc;
1923 	struct cpl_abort_req_core *req;
1924 
1925 	ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
1926 	ulpmc->len = htobe32(howmany(LEN__ABORT_REQ_ULP, 16));
1927 
1928 	ulpsc = (struct ulptx_idata *)(ulpmc + 1);
1929 	ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1930 	ulpsc->len = htobe32(sizeof(*req));
1931 
1932 	req = (struct cpl_abort_req_core *)(ulpsc + 1);
1933 	OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
1934 	req->rsvd0 = htonl(0);
1935 	req->rsvd1 = 0;
1936 	req->cmd = CPL_ABORT_NO_RST;
1937 
1938 	ulpsc = (struct ulptx_idata *)(req + 1);
1939 	if (LEN__ABORT_REQ_ULP % 16) {
1940 		ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1941 		ulpsc->len = htobe32(0);
1942 		return (ulpsc + 1);
1943 	}
1944 	return (ulpsc);
1945 }
1946 
1947 /* ABORT_RPL sent as a ULP command looks like this */
1948 #define LEN__ABORT_RPL_ULP (sizeof(struct ulp_txpkt) + \
1949 	sizeof(struct ulptx_idata) + sizeof(struct cpl_abort_rpl_core))
1950 
1951 static void *
1952 mk_abort_rpl_ulp(struct ulp_txpkt *ulpmc, uint32_t tid)
1953 {
1954 	struct ulptx_idata *ulpsc;
1955 	struct cpl_abort_rpl_core *rpl;
1956 
1957 	ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
1958 	ulpmc->len = htobe32(howmany(LEN__ABORT_RPL_ULP, 16));
1959 
1960 	ulpsc = (struct ulptx_idata *)(ulpmc + 1);
1961 	ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1962 	ulpsc->len = htobe32(sizeof(*rpl));
1963 
1964 	rpl = (struct cpl_abort_rpl_core *)(ulpsc + 1);
1965 	OPCODE_TID(rpl) = htobe32(MK_OPCODE_TID(CPL_ABORT_RPL, tid));
1966 	rpl->rsvd0 = htonl(0);
1967 	rpl->rsvd1 = 0;
1968 	rpl->cmd = CPL_ABORT_NO_RST;
1969 
1970 	ulpsc = (struct ulptx_idata *)(rpl + 1);
1971 	if (LEN__ABORT_RPL_ULP % 16) {
1972 		ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1973 		ulpsc->len = htobe32(0);
1974 		return (ulpsc + 1);
1975 	}
1976 	return (ulpsc);
1977 }
1978 
1979 static inline int
1980 del_hashfilter_wrlen(void)
1981 {
1982 
1983 	return (sizeof(struct work_request_hdr) +
1984 	    roundup2(LEN__SET_TCB_FIELD_ULP, 16) +
1985 	    roundup2(LEN__ABORT_REQ_ULP, 16) +
1986 	    roundup2(LEN__ABORT_RPL_ULP, 16));
1987 }
1988 
1989 static void
1990 mk_del_hashfilter_wr(struct adapter *sc, int tid, struct work_request_hdr *wrh,
1991     int wrlen, int qid)
1992 {
1993 	struct ulp_txpkt *ulpmc;
1994 
1995 	INIT_ULPTX_WRH(wrh, wrlen, 0, 0);
1996 	ulpmc = (struct ulp_txpkt *)(wrh + 1);
1997 	ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, tid, W_TCB_RSS_INFO,
1998 	    V_TCB_RSS_INFO(M_TCB_RSS_INFO), V_TCB_RSS_INFO(qid));
1999 	ulpmc = mk_abort_req_ulp(ulpmc, tid);
2000 	ulpmc = mk_abort_rpl_ulp(ulpmc, tid);
2001 }
2002 
2003 static int
2004 del_hashfilter(struct adapter *sc, struct t4_filter *t)
2005 {
2006 	struct tid_info *ti = &sc->tids;
2007 	void *wr;
2008 	struct filter_entry *f;
2009 	struct wrq_cookie cookie;
2010 	int rc;
2011 	const int wrlen = del_hashfilter_wrlen();
2012 	const int inv_tid = ti->ntids + ti->tid_base;
2013 
2014 	MPASS(sc->tids.hftid_hash_4t != NULL);
2015 	MPASS(sc->tids.ntids > 0);
2016 
2017 	if (t->idx < sc->tids.tid_base || t->idx >= inv_tid)
2018 		return (EINVAL);
2019 
2020 	mtx_lock(&ti->hftid_lock);
2021 	f = lookup_hftid(sc, t->idx);
2022 	if (f == NULL || f->valid == 0) {
2023 		rc = EINVAL;
2024 		goto done;
2025 	}
2026 	MPASS(f->tid == t->idx);
2027 	if (f->locked) {
2028 		rc = EPERM;
2029 		goto done;
2030 	}
2031 	if (f->pending) {
2032 		rc = EBUSY;
2033 		goto done;
2034 	}
2035 	wr = start_wrq_wr(&sc->sge.ctrlq[0], howmany(wrlen, 16), &cookie);
2036 	if (wr == NULL) {
2037 		rc = ENOMEM;
2038 		goto done;
2039 	}
2040 
2041 	mk_del_hashfilter_wr(sc, t->idx, wr, wrlen, sc->sge.fwq.abs_id);
2042 	f->locked = 1;
2043 	f->pending = 1;
2044 	commit_wrq_wr(&sc->sge.ctrlq[0], wr, &cookie);
2045 	t->fs = f->fs;	/* extra info for the caller */
2046 
2047 	for (;;) {
2048 		MPASS(f->locked);
2049 		if (f->pending == 0) {
2050 			if (f->valid) {
2051 				f->locked = 0;
2052 				rc = EIO;
2053 			} else {
2054 				rc = 0;
2055 				free(f, M_CXGBE);
2056 			}
2057 			break;
2058 		}
2059 		if (cv_wait_sig(&ti->hftid_cv, &ti->hftid_lock) != 0) {
2060 			f->locked = 0;
2061 			rc = EINPROGRESS;
2062 			break;
2063 		}
2064 	}
2065 done:
2066 	mtx_unlock(&ti->hftid_lock);
2067 	return (rc);
2068 }
2069 
2070 #define WORD_MASK       0xffffffff
2071 static void
2072 set_nat_params(struct adapter *sc, struct filter_entry *f, const bool dip,
2073     const bool sip, const bool dp, const bool sp)
2074 {
2075 
2076 	if (dip) {
2077 		if (f->fs.type) {
2078 			set_tcb_field(sc, f->tid, W_TCB_SND_UNA_RAW, WORD_MASK,
2079 			    f->fs.nat_dip[15] | f->fs.nat_dip[14] << 8 |
2080 			    f->fs.nat_dip[13] << 16 | f->fs.nat_dip[12] << 24, 1);
2081 
2082 			set_tcb_field(sc, f->tid,
2083 			    W_TCB_SND_UNA_RAW + 1, WORD_MASK,
2084 			    f->fs.nat_dip[11] | f->fs.nat_dip[10] << 8 |
2085 			    f->fs.nat_dip[9] << 16 | f->fs.nat_dip[8] << 24, 1);
2086 
2087 			set_tcb_field(sc, f->tid,
2088 			    W_TCB_SND_UNA_RAW + 2, WORD_MASK,
2089 			    f->fs.nat_dip[7] | f->fs.nat_dip[6] << 8 |
2090 			    f->fs.nat_dip[5] << 16 | f->fs.nat_dip[4] << 24, 1);
2091 
2092 			set_tcb_field(sc, f->tid,
2093 			    W_TCB_SND_UNA_RAW + 3, WORD_MASK,
2094 			    f->fs.nat_dip[3] | f->fs.nat_dip[2] << 8 |
2095 			    f->fs.nat_dip[1] << 16 | f->fs.nat_dip[0] << 24, 1);
2096 		} else {
2097 			set_tcb_field(sc, f->tid,
2098 			    W_TCB_RX_FRAG3_LEN_RAW, WORD_MASK,
2099 			    f->fs.nat_dip[3] | f->fs.nat_dip[2] << 8 |
2100 			    f->fs.nat_dip[1] << 16 | f->fs.nat_dip[0] << 24, 1);
2101 		}
2102 	}
2103 
2104 	if (sip) {
2105 		if (f->fs.type) {
2106 			set_tcb_field(sc, f->tid,
2107 			    W_TCB_RX_FRAG2_PTR_RAW, WORD_MASK,
2108 			    f->fs.nat_sip[15] | f->fs.nat_sip[14] << 8 |
2109 			    f->fs.nat_sip[13] << 16 | f->fs.nat_sip[12] << 24, 1);
2110 
2111 			set_tcb_field(sc, f->tid,
2112 			    W_TCB_RX_FRAG2_PTR_RAW + 1, WORD_MASK,
2113 			    f->fs.nat_sip[11] | f->fs.nat_sip[10] << 8 |
2114 			    f->fs.nat_sip[9] << 16 | f->fs.nat_sip[8] << 24, 1);
2115 
2116 			set_tcb_field(sc, f->tid,
2117 			    W_TCB_RX_FRAG2_PTR_RAW + 2, WORD_MASK,
2118 			    f->fs.nat_sip[7] | f->fs.nat_sip[6] << 8 |
2119 			    f->fs.nat_sip[5] << 16 | f->fs.nat_sip[4] << 24, 1);
2120 
2121 			set_tcb_field(sc, f->tid,
2122 			    W_TCB_RX_FRAG2_PTR_RAW + 3, WORD_MASK,
2123 			    f->fs.nat_sip[3] | f->fs.nat_sip[2] << 8 |
2124 			    f->fs.nat_sip[1] << 16 | f->fs.nat_sip[0] << 24, 1);
2125 
2126 		} else {
2127 			set_tcb_field(sc, f->tid,
2128 			    W_TCB_RX_FRAG3_START_IDX_OFFSET_RAW, WORD_MASK,
2129 			    f->fs.nat_sip[3] | f->fs.nat_sip[2] << 8 |
2130 			    f->fs.nat_sip[1] << 16 | f->fs.nat_sip[0] << 24, 1);
2131 		}
2132 	}
2133 
2134 	set_tcb_field(sc, f->tid, W_TCB_PDU_HDR_LEN, WORD_MASK,
2135 	    (dp ? f->fs.nat_dport : 0) | (sp ? f->fs.nat_sport << 16 : 0), 1);
2136 }
2137 
2138 /*
2139  * Returns EINPROGRESS to indicate that at least one TCB update was sent and the
2140  * last of the series of updates requested a reply.  The reply informs the
2141  * driver that the filter is fully setup.
2142  */
2143 static int
2144 configure_hashfilter_tcb(struct adapter *sc, struct filter_entry *f)
2145 {
2146 	int updated = 0;
2147 
2148 	MPASS(f->tid < sc->tids.ntids);
2149 	MPASS(f->fs.hash);
2150 	MPASS(f->pending);
2151 	MPASS(f->valid == 0);
2152 
2153 	if (f->fs.newdmac) {
2154 		set_tcb_tflag(sc, f->tid, S_TF_CCTRL_ECE, 1, 1);
2155 		updated++;
2156 	}
2157 
2158 	if (f->fs.newvlan == VLAN_INSERT || f->fs.newvlan == VLAN_REWRITE) {
2159 		set_tcb_tflag(sc, f->tid, S_TF_CCTRL_RFR, 1, 1);
2160 		updated++;
2161 	}
2162 
2163 	if (f->fs.newsmac) {
2164 		MPASS(f->smt != NULL);
2165 		set_tcb_tflag(sc, f->tid, S_TF_CCTRL_CWR, 1, 1);
2166 		set_tcb_field(sc, f->tid, W_TCB_SMAC_SEL,
2167 		    V_TCB_SMAC_SEL(M_TCB_SMAC_SEL), V_TCB_SMAC_SEL(f->smt->idx),
2168 		    1);
2169 		updated++;
2170 	}
2171 
2172 	switch(f->fs.nat_mode) {
2173 	case NAT_MODE_NONE:
2174 		break;
2175 	case NAT_MODE_DIP:
2176 		set_nat_params(sc, f, true, false, false, false);
2177 		updated++;
2178 		break;
2179 	case NAT_MODE_DIP_DP:
2180 		set_nat_params(sc, f, true, false, true, false);
2181 		updated++;
2182 		break;
2183 	case NAT_MODE_DIP_DP_SIP:
2184 		set_nat_params(sc, f, true, true, true, false);
2185 		updated++;
2186 		break;
2187 	case NAT_MODE_DIP_DP_SP:
2188 		set_nat_params(sc, f, true, false, true, true);
2189 		updated++;
2190 		break;
2191 	case NAT_MODE_SIP_SP:
2192 		set_nat_params(sc, f, false, true, false, true);
2193 		updated++;
2194 		break;
2195 	case NAT_MODE_DIP_SIP_SP:
2196 		set_nat_params(sc, f, true, true, false, true);
2197 		updated++;
2198 		break;
2199 	case NAT_MODE_ALL:
2200 		set_nat_params(sc, f, true, true, true, true);
2201 		updated++;
2202 		break;
2203 	default:
2204 		MPASS(0);	/* should have been validated earlier */
2205 		break;
2206 
2207 	}
2208 
2209 	if (f->fs.nat_seq_chk) {
2210 		set_tcb_field(sc, f->tid, W_TCB_RCV_NXT,
2211 		    V_TCB_RCV_NXT(M_TCB_RCV_NXT),
2212 		    V_TCB_RCV_NXT(f->fs.nat_seq_chk), 1);
2213 		updated++;
2214 	}
2215 
2216 	if (is_t5(sc) && f->fs.action == FILTER_DROP) {
2217 		/*
2218 		 * Migrating = 1, Non-offload = 0 to get a T5 hashfilter to drop.
2219 		 */
2220 		set_tcb_field(sc, f->tid, W_TCB_T_FLAGS, V_TF_NON_OFFLOAD(1) |
2221 		    V_TF_MIGRATING(1), V_TF_MIGRATING(1), 1);
2222 		updated++;
2223 	}
2224 
2225 	/*
2226 	 * Enable switching after all secondary resources (L2T entry, SMT entry,
2227 	 * etc.) are setup so that any switched packet will use correct
2228 	 * values.
2229 	 */
2230 	if (f->fs.action == FILTER_SWITCH) {
2231 		set_tcb_tflag(sc, f->tid, S_TF_CCTRL_ECN, 1, 1);
2232 		updated++;
2233 	}
2234 
2235 	if (f->fs.hitcnts || updated > 0) {
2236 		set_tcb_field(sc, f->tid, W_TCB_TIMESTAMP,
2237 		    V_TCB_TIMESTAMP(M_TCB_TIMESTAMP) |
2238 		    V_TCB_T_RTT_TS_RECENT_AGE(M_TCB_T_RTT_TS_RECENT_AGE),
2239 		    V_TCB_TIMESTAMP(0ULL) | V_TCB_T_RTT_TS_RECENT_AGE(0ULL), 0);
2240 		return (EINPROGRESS);
2241 	}
2242 
2243 	return (0);
2244 }
2245