111 SYSCTL_NODE(_kern_ipc, OID_AUTO, tls, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
306 ktls_copyin_tls_enable(struct sockopt *sopt, struct tls_enable *tls)  in ktls_copyin_tls_enable()  argument
316 		memset(tls, 0, sizeof(*tls));  in ktls_copyin_tls_enable()
317 		tls->cipher_key = tls_v0.cipher_key;  in ktls_copyin_tls_enable()
318 		tls->iv = tls_v0.iv;  in ktls_copyin_tls_enable()
319 		tls->auth_key = tls_v0.auth_key;  in ktls_copyin_tls_enable()
320 		tls->cipher_algorithm = tls_v0.cipher_algorithm;  in ktls_copyin_tls_enable()
321 		tls->cipher_key_len = tls_v0.cipher_key_len;  in ktls_copyin_tls_enable()
322 		tls->iv_len = tls_v0.iv_len;  in ktls_copyin_tls_enable()
323 		tls->auth_algorithm = tls_v0.auth_algorithm;  in ktls_copyin_tls_enable()
324 		tls->auth_key_len = tls_v0.auth_key_len;  in ktls_copyin_tls_enable()
325 		tls->flags = tls_v0.flags;  in ktls_copyin_tls_enable()
326 		tls->tls_vmajor = tls_v0.tls_vmajor;  in ktls_copyin_tls_enable()
327 		tls->tls_vminor = tls_v0.tls_vminor;  in ktls_copyin_tls_enable()
329 		error = sooptcopyin(sopt, tls, sizeof(*tls), sizeof(*tls));  in ktls_copyin_tls_enable()
334 	if (tls->cipher_key_len < 0 || tls->cipher_key_len > TLS_MAX_PARAM_SIZE)  in ktls_copyin_tls_enable()
336 	if (tls->iv_len < 0 || tls->iv_len > sizeof(((struct ktls_session *)NULL)->params.iv))  in ktls_copyin_tls_enable()
338 	if (tls->auth_key_len < 0 || tls->auth_key_len > TLS_MAX_PARAM_SIZE)  in ktls_copyin_tls_enable()
342 	if (tls->cipher_key_len == 0)  in ktls_copyin_tls_enable()
352 	if (tls->cipher_key_len != 0) {  in ktls_copyin_tls_enable()
353 		cipher_key = malloc(tls->cipher_key_len, M_KTLS, M_WAITOK);  in ktls_copyin_tls_enable()
355 			error = copyin(tls->cipher_key, cipher_key, tls->cipher_key_len);  in ktls_copyin_tls_enable()
359 			bcopy(tls->cipher_key, cipher_key, tls->cipher_key_len);  in ktls_copyin_tls_enable()
362 	if (tls->iv_len != 0) {  in ktls_copyin_tls_enable()
363 		iv = malloc(tls->iv_len, M_KTLS, M_WAITOK);  in ktls_copyin_tls_enable()
365 			error = copyin(tls->iv, iv, tls->iv_len);  in ktls_copyin_tls_enable()
369 			bcopy(tls->iv, iv, tls->iv_len);  in ktls_copyin_tls_enable()
372 	if (tls->auth_key_len != 0) {  in ktls_copyin_tls_enable()
373 		auth_key = malloc(tls->auth_key_len, M_KTLS, M_WAITOK);  in ktls_copyin_tls_enable()
375 			error = copyin(tls->auth_key, auth_key, tls->auth_key_len);  in ktls_copyin_tls_enable()
379 			bcopy(tls->auth_key, auth_key, tls->auth_key_len);  in ktls_copyin_tls_enable()
382 	tls->cipher_key = cipher_key;  in ktls_copyin_tls_enable()
383 	tls->iv = iv;  in ktls_copyin_tls_enable()
384 	tls->auth_key = auth_key;  in ktls_copyin_tls_enable()
397 ktls_cleanup_tls_enable(struct tls_enable *tls)  in ktls_cleanup_tls_enable()  argument
399 	zfree(__DECONST(void *, tls->cipher_key), M_KTLS);  in ktls_cleanup_tls_enable()
400 	zfree(__DECONST(void *, tls->iv), M_KTLS);  in ktls_cleanup_tls_enable()
401 	zfree(__DECONST(void *, tls->auth_key), M_KTLS);  in ktls_cleanup_tls_enable()
602 	struct ktls_session *tls;  in ktls_create_session()  local
701 	tls = uma_zalloc(ktls_session_zone, M_WAITOK | M_ZERO);  in ktls_create_session()
705 	refcount_init(&tls->refcount, 1);  in ktls_create_session()
707 		TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_receive_tag, tls);  in ktls_create_session()
709 		TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_send_tag, tls);  in ktls_create_session()
710 		tls->inp = so->so_pcb;  in ktls_create_session()
711 		in_pcbref(tls->inp);  in ktls_create_session()
712 		tls->tx = true;  in ktls_create_session()
715 	tls->wq_index = ktls_get_cpu(so);  in ktls_create_session()
717 	tls->params.cipher_algorithm = en->cipher_algorithm;  in ktls_create_session()
718 	tls->params.auth_algorithm = en->auth_algorithm;  in ktls_create_session()
719 	tls->params.tls_vmajor = en->tls_vmajor;  in ktls_create_session()
720 	tls->params.tls_vminor = en->tls_vminor;  in ktls_create_session()
721 	tls->params.flags = en->flags;  in ktls_create_session()
722 	tls->params.max_frame_len = min(TLS_MAX_MSG_SIZE_V10_2, ktls_maxlen);  in ktls_create_session()
725 	tls->params.tls_hlen = sizeof(struct tls_record_layer);  in ktls_create_session()
733 			tls->params.tls_hlen += sizeof(uint64_t);  in ktls_create_session()
734 		tls->params.tls_tlen = AES_GMAC_HASH_LEN;  in ktls_create_session()
735 		tls->params.tls_bs = 1;  in ktls_create_session()
742 				tls->sequential_records = true;  in ktls_create_session()
743 				tls->next_seqno = be64dec(en->rec_seq);  in ktls_create_session()
744 				STAILQ_INIT(&tls->pending_records);  in ktls_create_session()
746 				tls->params.tls_hlen += AES_BLOCK_LEN;  in ktls_create_session()
748 			tls->params.tls_tlen = AES_BLOCK_LEN +  in ktls_create_session()
752 			tls->params.tls_hlen += AES_BLOCK_LEN;  in ktls_create_session()
753 			tls->params.tls_tlen = AES_BLOCK_LEN +  in ktls_create_session()
757 			tls->params.tls_hlen += AES_BLOCK_LEN;  in ktls_create_session()
758 			tls->params.tls_tlen = AES_BLOCK_LEN +  in ktls_create_session()
764 		tls->params.tls_bs = AES_BLOCK_LEN;  in ktls_create_session()
770 		tls->params.tls_tlen = POLY1305_HASH_LEN;  in ktls_create_session()
771 		tls->params.tls_bs = 1;  in ktls_create_session()
783 		tls->params.tls_tlen += sizeof(uint8_t);  in ktls_create_session()
785 	KASSERT(tls->params.tls_hlen <= MBUF_PEXT_HDR_LEN,  in ktls_create_session()
786 	    ("TLS header length too long: %d", tls->params.tls_hlen));  in ktls_create_session()
787 	KASSERT(tls->params.tls_tlen <= MBUF_PEXT_TRAIL_LEN,  in ktls_create_session()
788 	    ("TLS trailer length too long: %d", tls->params.tls_tlen));  in ktls_create_session()
791 		tls->params.auth_key_len = en->auth_key_len;  in ktls_create_session()
792 		tls->params.auth_key = malloc(en->auth_key_len, M_KTLS,  in ktls_create_session()
794 		bcopy(en->auth_key, tls->params.auth_key, en->auth_key_len);  in ktls_create_session()
797 	tls->params.cipher_key_len = en->cipher_key_len;  in ktls_create_session()
798 	tls->params.cipher_key = malloc(en->cipher_key_len, M_KTLS, M_WAITOK);  in ktls_create_session()
799 	bcopy(en->cipher_key, tls->params.cipher_key, en->cipher_key_len);  in ktls_create_session()
807 		tls->params.iv_len = en->iv_len;  in ktls_create_session()
808 		bcopy(en->iv, tls->params.iv, en->iv_len);  in ktls_create_session()
819 			arc4rand(tls->params.iv + 8, sizeof(uint64_t), 0);  in ktls_create_session()
822 	tls->gen = 0;  in ktls_create_session()
823 	*tlsp = tls;  in ktls_create_session()
828 ktls_clone_session(struct ktls_session *tls, int direction)  in ktls_clone_session()  argument
843 		tls_new->inp = tls->inp;  in ktls_clone_session()
849 	tls_new->params = tls->params;  in ktls_clone_session()
850 	tls_new->wq_index = tls->wq_index;  in ktls_clone_session()
854 		tls_new->params.auth_key = malloc(tls->params.auth_key_len,  in ktls_clone_session()
856 		memcpy(tls_new->params.auth_key, tls->params.auth_key,  in ktls_clone_session()
857 		    tls->params.auth_key_len);  in ktls_clone_session()
860 	tls_new->params.cipher_key = malloc(tls->params.cipher_key_len, M_KTLS,  in ktls_clone_session()
862 	memcpy(tls_new->params.cipher_key, tls->params.cipher_key,  in ktls_clone_session()
863 	    tls->params.cipher_key_len);  in ktls_clone_session()
871 ktls_try_toe(struct socket *so, struct ktls_session *tls, int direction)  in ktls_try_toe()  argument
893 	error = tcp_offload_alloc_tls_session(tp, tls, direction);  in ktls_try_toe()
896 		tls->mode = TCP_TLS_MODE_TOE;  in ktls_try_toe()
897 		switch (tls->params.cipher_algorithm) {  in ktls_try_toe()
920 ktls_alloc_snd_tag(struct inpcb *inp, struct ktls_session *tls, bool force,  in ktls_alloc_snd_tag()  argument
975 		params.tls_rate_limit.tls = tls;  in ktls_alloc_snd_tag()
979 		params.tls.inp = inp;  in ktls_alloc_snd_tag()
980 		params.tls.tls = tls;  in ktls_alloc_snd_tag()
1018 ktls_alloc_rcv_tag(struct inpcb *inp, struct ktls_session *tls,  in ktls_alloc_rcv_tag()  argument
1026 	if (!ktls_ocf_recrypt_supported(tls))  in ktls_alloc_rcv_tag()
1059 	tls->rx_ifp = ifp;  in ktls_alloc_rcv_tag()
1066 	params.tls_rx.tls = tls;  in ktls_alloc_rcv_tag()
1092 		tls->rx_vlan_id = params.tls_rx.vlan_id;  in ktls_alloc_rcv_tag()
1098 ktls_try_ifnet(struct socket *so, struct ktls_session *tls, int direction,  in ktls_try_ifnet()  argument
1106 		error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst);  in ktls_try_ifnet()
1112 		error = ktls_alloc_rcv_tag(so->so_pcb, tls, &mst);  in ktls_try_ifnet()
1120 	tls->mode = TCP_TLS_MODE_IFNET;  in ktls_try_ifnet()
1121 	tls->snd_tag = mst;  in ktls_try_ifnet()
1123 	switch (tls->params.cipher_algorithm) {  in ktls_try_ifnet()
1141 ktls_use_sw(struct ktls_session *tls)  in ktls_use_sw()  argument
1143 	tls->mode = TCP_TLS_MODE_SW;  in ktls_use_sw()
1144 	switch (tls->params.cipher_algorithm) {  in ktls_use_sw()
1158 ktls_try_sw(struct ktls_session *tls, int direction)  in ktls_try_sw()  argument
1162 	error = ktls_ocf_try(tls, direction);  in ktls_try_sw()
1165 	ktls_use_sw(tls);  in ktls_try_sw()
1290 	struct ktls_session *tls;  in ktls_enable_rx()  local
1315 	error = ktls_create_session(so, en, &tls, KTLS_RX);  in ktls_enable_rx()
1319 	error = ktls_ocf_try(tls, KTLS_RX);  in ktls_enable_rx()
1321 		ktls_free(tls);  in ktls_enable_rx()
1331 		ktls_free(tls);  in ktls_enable_rx()
1344 		ktls_free(tls);  in ktls_enable_rx()
1348 	so->so_rcv.sb_tls_info = tls;  in ktls_enable_rx()
1359 	error = ktls_try_toe(so, tls, KTLS_RX);  in ktls_enable_rx()
1362 		error = ktls_try_ifnet(so, tls, KTLS_RX, false);  in ktls_enable_rx()
1364 		ktls_use_sw(tls);  in ktls_enable_rx()
1374 	struct ktls_session *tls;  in ktls_enable_tx()  local
1405 	error = ktls_create_session(so, en, &tls, KTLS_TX);  in ktls_enable_tx()
1411 	error = ktls_try_toe(so, tls, KTLS_TX);  in ktls_enable_tx()
1414 		error = ktls_try_ifnet(so, tls, KTLS_TX, false);  in ktls_enable_tx()
1416 		error = ktls_try_sw(tls, KTLS_TX);  in ktls_enable_tx()
1419 		ktls_free(tls);  in ktls_enable_tx()
1429 		ktls_free(tls);  in ktls_enable_tx()
1449 		ktls_free(tls);  in ktls_enable_tx()
1453 	so->so_snd.sb_tls_info = tls;  in ktls_enable_tx()
1454 	if (tls->mode != TCP_TLS_MODE_SW) {  in ktls_enable_tx()
1473 	struct ktls_session *tls;  in ktls_get_rx_mode()  local
1481 	tls = so->so_rcv.sb_tls_info;  in ktls_get_rx_mode()
1482 	if (tls == NULL)  in ktls_get_rx_mode()
1485 		*modep = tls->mode;  in ktls_get_rx_mode()
1536 	struct ktls_session *tls;  in ktls_get_tx_mode()  local
1544 	tls = so->so_snd.sb_tls_info;  in ktls_get_tx_mode()
1545 	if (tls == NULL)  in ktls_get_tx_mode()
1548 		*modep = tls->mode;  in ktls_get_tx_mode()
1559 	struct ktls_session *tls, *tls_new;  in ktls_set_tx_mode()  local
1592 	tls = so->so_snd.sb_tls_info;  in ktls_set_tx_mode()
1593 	if (tls == NULL) {  in ktls_set_tx_mode()
1598 	if (tls->mode == mode) {  in ktls_set_tx_mode()
1603 	tls = ktls_hold(tls);  in ktls_set_tx_mode()
1607 	tls_new = ktls_clone_session(tls, KTLS_TX);  in ktls_set_tx_mode()
1616 		ktls_free(tls);  in ktls_set_tx_mode()
1625 		ktls_free(tls);  in ktls_set_tx_mode()
1634 	if (tls != so->so_snd.sb_tls_info) {  in ktls_set_tx_mode()
1638 		ktls_free(tls);  in ktls_set_tx_mode()
1660 	KASSERT(tls->refcount >= 2, ("too few references on old session"));  in ktls_set_tx_mode()
1661 	ktls_free(tls);  in ktls_set_tx_mode()
1662 	ktls_free(tls);  in ktls_set_tx_mode()
1682 	struct ktls_session *tls;  in ktls_reset_receive_tag()  local
1691 	tls = context;  in ktls_reset_receive_tag()
1692 	so = tls->so;  in ktls_reset_receive_tag()
1703 	mst = tls->snd_tag;  in ktls_reset_receive_tag()
1704 	tls->snd_tag = NULL;  in ktls_reset_receive_tag()
1708 	ifp = tls->rx_ifp;  in ktls_reset_receive_tag()
1717 	params.tls_rx.tls = tls;  in ktls_reset_receive_tag()
1718 	params.tls_rx.vlan_id = tls->rx_vlan_id;  in ktls_reset_receive_tag()
1732 		tls->snd_tag = mst;  in ktls_reset_receive_tag()
1747 	mtx_pool_lock(mtxpool_sleep, tls);  in ktls_reset_receive_tag()
1748 	tls->reset_pending = false;  in ktls_reset_receive_tag()
1749 	mtx_pool_unlock(mtxpool_sleep, tls);  in ktls_reset_receive_tag()
1756 	ktls_free(tls);  in ktls_reset_receive_tag()
1771 	struct ktls_session *tls;  in ktls_reset_send_tag()  local
1779 	tls = context;  in ktls_reset_send_tag()
1780 	inp = tls->inp;  in ktls_reset_send_tag()
1793 	old = tls->snd_tag;  in ktls_reset_send_tag()
1794 	tls->snd_tag = NULL;  in ktls_reset_send_tag()
1799 	error = ktls_alloc_snd_tag(inp, tls, true, &new);  in ktls_reset_send_tag()
1803 		tls->snd_tag = new;  in ktls_reset_send_tag()
1804 		mtx_pool_lock(mtxpool_sleep, tls);  in ktls_reset_send_tag()
1805 		tls->reset_pending = false;  in ktls_reset_send_tag()
1806 		mtx_pool_unlock(mtxpool_sleep, tls);  in ktls_reset_send_tag()
1839 	ktls_free(tls);  in ktls_reset_send_tag()
1845 	struct ktls_session *tls;  in ktls_input_ifp_mismatch()  local
1853 	tls = sb->sb_tls_info;  in ktls_input_ifp_mismatch()
1854 	if_rele(tls->rx_ifp);  in ktls_input_ifp_mismatch()
1856 	tls->rx_ifp = ifp;  in ktls_input_ifp_mismatch()
1862 	mtx_pool_lock(mtxpool_sleep, tls);  in ktls_input_ifp_mismatch()
1863 	if (!tls->reset_pending) {  in ktls_input_ifp_mismatch()
1864 		(void) ktls_hold(tls);  in ktls_input_ifp_mismatch()
1866 		tls->so = so;  in ktls_input_ifp_mismatch()
1867 		tls->reset_pending = true;  in ktls_input_ifp_mismatch()
1868 		taskqueue_enqueue(taskqueue_thread, &tls->reset_tag_task);  in ktls_input_ifp_mismatch()
1870 	mtx_pool_unlock(mtxpool_sleep, tls);  in ktls_input_ifp_mismatch()
1874 ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls)  in ktls_output_eagain()  argument
1886 	mtx_pool_lock(mtxpool_sleep, tls);  in ktls_output_eagain()
1887 	if (!tls->reset_pending) {  in ktls_output_eagain()
1888 		(void) ktls_hold(tls);  in ktls_output_eagain()
1889 		tls->reset_pending = true;  in ktls_output_eagain()
1890 		taskqueue_enqueue(taskqueue_thread, &tls->reset_tag_task);  in ktls_output_eagain()
1892 	mtx_pool_unlock(mtxpool_sleep, tls);  in ktls_output_eagain()
1898 ktls_modify_txrtlmt(struct ktls_session *tls, uint64_t max_pacing_rate)  in ktls_modify_txrtlmt()  argument
1909 	MPASS(tls->mode == TCP_TLS_MODE_IFNET);  in ktls_modify_txrtlmt()
1911 	if (tls->snd_tag == NULL) {  in ktls_modify_txrtlmt()
1921 	mst = tls->snd_tag;  in ktls_modify_txrtlmt()
1937 ktls_destroy(struct ktls_session *tls)  in ktls_destroy()  argument
1943 	MPASS(tls->refcount == 0);  in ktls_destroy()
1945 	inp = tls->inp;  in ktls_destroy()
1946 	if (tls->tx) {  in ktls_destroy()
1969 				TASK_INIT(&tls->destroy_task, 0,  in ktls_destroy()
1970 				    ktls_destroy_help, tls);  in ktls_destroy()
1972 				    &tls->destroy_task);  in ktls_destroy()
1978 	if (tls->sequential_records) {  in ktls_destroy()
1982 		STAILQ_FOREACH_SAFE(m, &tls->pending_records, m_epg_stailq, n) {  in ktls_destroy()
1994 	switch (tls->mode) {  in ktls_destroy()
1996 		switch (tls->params.cipher_algorithm) {  in ktls_destroy()
2009 		switch (tls->params.cipher_algorithm) {  in ktls_destroy()
2020 		if (tls->snd_tag != NULL)  in ktls_destroy()
2021 			m_snd_tag_rele(tls->snd_tag);  in ktls_destroy()
2022 		if (tls->rx_ifp != NULL)  in ktls_destroy()
2023 			if_rele(tls->rx_ifp);  in ktls_destroy()
2024 		if (tls->tx) {  in ktls_destroy()
2033 		switch (tls->params.cipher_algorithm) {  in ktls_destroy()
2047 	if (tls->ocf_session != NULL)  in ktls_destroy()
2048 		ktls_ocf_free(tls);  in ktls_destroy()
2049 	if (tls->params.auth_key != NULL) {  in ktls_destroy()
2050 		zfree(tls->params.auth_key, M_KTLS);  in ktls_destroy()
2051 		tls->params.auth_key = NULL;  in ktls_destroy()
2052 		tls->params.auth_key_len = 0;  in ktls_destroy()
2054 	if (tls->params.cipher_key != NULL) {  in ktls_destroy()
2055 		zfree(tls->params.cipher_key, M_KTLS);  in ktls_destroy()
2056 		tls->params.cipher_key = NULL;  in ktls_destroy()
2057 		tls->params.cipher_key_len = 0;  in ktls_destroy()
2059 	if (tls->tx) {  in ktls_destroy()
2064 	explicit_bzero(tls->params.iv, sizeof(tls->params.iv));  in ktls_destroy()
2066 	uma_zfree(ktls_session_zone, tls);  in ktls_destroy()
2098 ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,  in ktls_frame()  argument
2107 	maxlen = tls->params.max_frame_len;  in ktls_frame()
2117 		    (m->m_len > 0 || ktls_permit_empty_frames(tls)),  in ktls_frame()
2130 		m->m_epg_tls = ktls_hold(tls);  in ktls_frame()
2132 		m->m_epg_hdrlen = tls->params.tls_hlen;  in ktls_frame()
2133 		m->m_epg_trllen = tls->params.tls_tlen;  in ktls_frame()
2134 		if (tls->params.cipher_algorithm == CRYPTO_AES_CBC) {  in ktls_frame()
2153 			bs = tls->params.tls_bs;  in ktls_frame()
2154 			delta = (tls_len + tls->params.tls_tlen) & (bs - 1);  in ktls_frame()
2161 		tlshdr->tls_vmajor = tls->params.tls_vmajor;  in ktls_frame()
2167 		if (tls->params.tls_vminor == TLS_MINOR_VER_THREE &&  in ktls_frame()
2168 		    tls->params.tls_vmajor == TLS_MAJOR_VER_ONE) {  in ktls_frame()
2175 			tlshdr->tls_vminor = tls->params.tls_vminor;  in ktls_frame()
2190 		if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 &&  in ktls_frame()
2191 		    tls->params.tls_vminor == TLS_MINOR_VER_TWO) {  in ktls_frame()
2192 			noncep = (uint64_t *)(tls->params.iv + 8);  in ktls_frame()
2195 		} else if (tls->params.cipher_algorithm == CRYPTO_AES_CBC &&  in ktls_frame()
2196 		    tls->params.tls_vminor >= TLS_MINOR_VER_ONE)  in ktls_frame()
2207 		if (tls->mode == TCP_TLS_MODE_SW) {  in ktls_frame()
2220 ktls_permit_empty_frames(struct ktls_session *tls)  in ktls_permit_empty_frames()  argument
2222 	return (tls->params.cipher_algorithm == CRYPTO_AES_CBC &&  in ktls_permit_empty_frames()
2223 	    tls->params.tls_vminor == TLS_MINOR_VER_ZERO);  in ktls_permit_empty_frames()
2383 tls13_find_record_type(struct ktls_session *tls, struct mbuf *m, int tls_len,  in tls13_find_record_type()  argument
2407 	if (last_offset < tls->params.tls_hlen)  in tls13_find_record_type()
2524 	struct ktls_session *tls;  in ktls_decrypt()  local
2541 	tls = sb->sb_tls_info;  in ktls_decrypt()
2542 	MPASS(tls != NULL);  in ktls_decrypt()
2544 	tls13 = (tls->params.tls_vminor == TLS_MINOR_VER_THREE);  in ktls_decrypt()
2548 		vminor = tls->params.tls_vminor;  in ktls_decrypt()
2551 		if (sb->sb_tlscc < tls->params.tls_hlen)  in ktls_decrypt()
2554 		m_copydata(sb->sb_mtls, 0, tls->params.tls_hlen, tls_header);  in ktls_decrypt()
2557 		if (hdr->tls_vmajor != tls->params.tls_vmajor ||  in ktls_decrypt()
2562 		else if (tls_len < tls->params.tls_hlen || tls_len >  in ktls_decrypt()
2563 		    tls->params.tls_hlen + TLS_MAX_MSG_SIZE_V10_2 +  in ktls_decrypt()
2564 		    tls->params.tls_tlen)  in ktls_decrypt()
2604 			error = ktls_ocf_recrypt(tls, hdr, data, seqno);  in ktls_decrypt()
2609 			error = ktls_ocf_decrypt(tls, hdr, data, seqno,  in ktls_decrypt()
2613 					error = tls13_find_record_type(tls, data,  in ktls_decrypt()
2627 				trail_len = tls->params.tls_tlen - 1;  in ktls_decrypt()
2628 				error = tls13_find_record_type(tls, data,  in ktls_decrypt()
2631 				trail_len = tls->params.tls_tlen;  in ktls_decrypt()
2678 		tgr.tls_length = htobe16(tls_len - tls->params.tls_hlen -  in ktls_decrypt()
2706 			remain = tls->params.tls_hlen;  in ktls_decrypt()
2822     struct ktls_session *tls, struct ktls_ocf_encrypt_state *state)  in ktls_encrypt_record()  argument
2835 		return (ktls_ocf_encrypt(state, tls, m, NULL, 0));  in ktls_encrypt_record()
2865 	error = ktls_ocf_encrypt(state, tls, m, state->dst_iov, i + 1);  in ktls_encrypt_record()
2902 	struct ktls_session *tls;  in ktls_enqueue()  local
2923 	tls = m->m_epg_tls;  in ktls_enqueue()
2924 	wq = &ktls_wq[tls->wq_index];  in ktls_enqueue()
2926 	if (__predict_false(tls->sequential_records)) {  in ktls_enqueue()
2949 		if (m->m_epg_seqno != tls->next_seqno) {  in ktls_enqueue()
2953 			STAILQ_FOREACH(n, &tls->pending_records, m_epg_stailq) {  in ktls_enqueue()
2959 				STAILQ_INSERT_TAIL(&tls->pending_records, m,  in ktls_enqueue()
2962 				STAILQ_INSERT_HEAD(&tls->pending_records, m,  in ktls_enqueue()
2965 				STAILQ_INSERT_AFTER(&tls->pending_records, p, m,  in ktls_enqueue()
2972 		tls->next_seqno += ktls_batched_records(m);  in ktls_enqueue()
2975 		while (!STAILQ_EMPTY(&tls->pending_records)) {  in ktls_enqueue()
2978 			n = STAILQ_FIRST(&tls->pending_records);  in ktls_enqueue()
2979 			if (n->m_epg_seqno != tls->next_seqno)  in ktls_enqueue()
2983 			STAILQ_REMOVE_HEAD(&tls->pending_records, m_epg_stailq);  in ktls_enqueue()
2984 			tls->next_seqno += ktls_batched_records(n);  in ktls_enqueue()
3036 	struct ktls_session *tls;  in ktls_encrypt()  local
3042 	tls = top->m_epg_tls;  in ktls_encrypt()
3043 	KASSERT(tls != NULL, ("tls = NULL, top = %p\n", top));  in ktls_encrypt()
3070 		KASSERT(m->m_epg_tls == tls,  in ktls_encrypt()
3072 		    tls, m->m_epg_tls));  in ktls_encrypt()
3077 		error = ktls_encrypt_record(wq, m, tls, &state);  in ktls_encrypt()
3097 		ktls_free(tls);  in ktls_encrypt()
3115 	struct ktls_session *tls;  in ktls_encrypt_cb()  local
3135 	tls = m->m_epg_tls;  in ktls_encrypt_cb()
3137 	ktls_free(tls);  in ktls_encrypt_cb()
3165 	struct ktls_session *tls;  in ktls_encrypt_async()  local
3171 	tls = top->m_epg_tls;  in ktls_encrypt_async()
3172 	KASSERT(tls != NULL, ("tls = NULL, top = %p\n", top));  in ktls_encrypt_async()
3182 		KASSERT(m->m_epg_tls == tls,  in ktls_encrypt_async()
3184 		    tls, m->m_epg_tls));  in ktls_encrypt_async()
3197 		error = ktls_encrypt_record(wq, m, tls, state);  in ktls_encrypt_async()
3359 	struct ktls_session *tls;  in ktls_disable_ifnet_help()  local
3365 	tls = context;  in ktls_disable_ifnet_help()
3366 	inp = tls->inp;  in ktls_disable_ifnet_help()
3396 	ktls_free(tls);  in ktls_disable_ifnet_help()
3418 	struct ktls_session *tls;  in ktls_disable_ifnet()  local
3425 	tls = so->so_snd.sb_tls_info;  in ktls_disable_ifnet()
3437 	(void)ktls_hold(tls);  in ktls_disable_ifnet()
3441 	TASK_INIT(&tls->disable_ifnet_task, 0, ktls_disable_ifnet_help, tls);  in ktls_disable_ifnet()
3442 	(void)taskqueue_enqueue(taskqueue_thread, &tls->disable_ifnet_task);  in ktls_disable_ifnet()