49 	/* Encrypt a single outbound TLS record. */
51 	    struct ktls_session *tls, struct mbuf *m,
54 	/* Re-encrypt a received TLS record that is partially decrypted. */
55 	int	(*recrypt)(struct ktls_session *tls,
59 	/* Decrypt a received TLS record. */
60 	int	(*decrypt)(struct ktls_session *tls,
74 	/* Only used for TLS 1.0 with the implicit IV. */
94     "Kernel TLS offload via OCF stats");
99     "Total number of OCF TLS 1.0 CBC encryption operations");
104     "Total number of OCF TLS 1.1/1.2 CBC decryption operations");
109     "Total number of OCF TLS 1.1/1.2 CBC encryption operations");
114     "Total number of OCF TLS 1.2 GCM decryption operations");
119     "Total number of OCF TLS 1.2 GCM encryption operations");
124     "Total number of OCF TLS 1.2 GCM re-encryption operations");
129     "Total number of OCF TLS 1.2 Chacha20-Poly1305 decryption operations");
134     "Total number of OCF TLS 1.2 Chacha20-Poly1305 encryption operations");
139     "Total number of OCF TLS 1.3 GCM decryption operations");
144     "Total number of OCF TLS 1.3 GCM encryption operations");
149     "Total number of OCF TLS 1.3 GCM re-encryption operations");
154     "Total number of OCF TLS 1.3 Chacha20-Poly1305 decryption operations");
159     "Total number of OCF TLS 1.3 Chacha20-Poly1305 encryption operations");
273     struct ktls_session *tls, struct mbuf *m, struct iovec *outiov,  in ktls_ocf_tls_cbc_encrypt()  argument
289 	os = tls->ocf_session;  in ktls_ocf_tls_cbc_encrypt()
293 	MPASS(tls->sync_dispatch);  in ktls_ocf_tls_cbc_encrypt()
301 			printf("KTLS CBC: TLS records out of order.  "  in ktls_ocf_tls_cbc_encrypt()
435 ktls_ocf_tls_cbc_decrypt(struct ktls_session *tls,  in ktls_ocf_tls_cbc_decrypt()  argument
450 	os = tls->ocf_session;  in ktls_ocf_tls_cbc_decrypt()
465 	crp.crp_payload_start = tls->params.tls_hlen;  in ktls_ocf_tls_cbc_decrypt()
493 	 * Allocate and populate the iov.  Have to skip over the TLS  in ktls_ocf_tls_cbc_decrypt()
546     struct ktls_session *tls, struct mbuf *m, struct iovec *outiov,  in ktls_ocf_tls12_aead_encrypt()  argument
557 	os = tls->ocf_session;  in ktls_ocf_tls12_aead_encrypt()
565 	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {  in ktls_ocf_tls12_aead_encrypt()
566 		memcpy(crp->crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);  in ktls_ocf_tls12_aead_encrypt()
571 		 * Chacha20-Poly1305 constructs the IV for TLS 1.2  in ktls_ocf_tls12_aead_encrypt()
572 		 * identically to constructing the IV for AEAD in TLS  in ktls_ocf_tls12_aead_encrypt()
575 		memcpy(crp->crp_iv, tls->params.iv, tls->params.iv_len);  in ktls_ocf_tls12_aead_encrypt()
603 		uio->uio_resid = crp->crp_payload_length + tls->params.tls_tlen;  in ktls_ocf_tls12_aead_encrypt()
611 	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)  in ktls_ocf_tls12_aead_encrypt()
619 	if (tls->sync_dispatch) {  in ktls_ocf_tls12_aead_encrypt()
628 ktls_ocf_tls12_aead_decrypt(struct ktls_session *tls,  in ktls_ocf_tls12_aead_decrypt()  argument
638 	os = tls->ocf_session;  in ktls_ocf_tls12_aead_decrypt()
642 	if (tls_len + sizeof(*hdr) < tls->params.tls_hlen +  in ktls_ocf_tls12_aead_decrypt()
643 	    tls->params.tls_tlen)  in ktls_ocf_tls12_aead_decrypt()
649 	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {  in ktls_ocf_tls12_aead_decrypt()
650 		memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);  in ktls_ocf_tls12_aead_decrypt()
655 		 * Chacha20-Poly1305 constructs the IV for TLS 1.2  in ktls_ocf_tls12_aead_decrypt()
656 		 * identically to constructing the IV for AEAD in TLS  in ktls_ocf_tls12_aead_decrypt()
659 		memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len);  in ktls_ocf_tls12_aead_decrypt()
664 	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)  in ktls_ocf_tls12_aead_decrypt()
677 	crp.crp_payload_start = tls->params.tls_hlen;  in ktls_ocf_tls12_aead_decrypt()
685 	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)  in ktls_ocf_tls12_aead_decrypt()
692 	*trailer_len = tls->params.tls_tlen;  in ktls_ocf_tls12_aead_decrypt()
725 ktls_ocf_tls12_aead_recrypt(struct ktls_session *tls,  in ktls_ocf_tls12_aead_recrypt()  argument
736 	os = tls->ocf_session;  in ktls_ocf_tls12_aead_recrypt()
745 	KASSERT(tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16,  in ktls_ocf_tls12_aead_recrypt()
749 	memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);  in ktls_ocf_tls12_aead_recrypt()
757 	crp.crp_payload_start = tls->params.tls_hlen;  in ktls_ocf_tls12_aead_recrypt()
769 		ktls_ocf_recrypt_fixup(m, tls->params.tls_hlen, payload_len,  in ktls_ocf_tls12_aead_recrypt()
784     struct ktls_session *tls, struct mbuf *m, struct iovec *outiov,  in ktls_ocf_tls13_aead_encrypt()  argument
794 	os = tls->ocf_session;  in ktls_ocf_tls13_aead_encrypt()
802 	memcpy(crp->crp_iv, tls->params.iv, tls->params.iv_len);  in ktls_ocf_tls13_aead_encrypt()
841 	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)  in ktls_ocf_tls13_aead_encrypt()
849 	if (tls->sync_dispatch) {  in ktls_ocf_tls13_aead_encrypt()
858 ktls_ocf_tls13_aead_decrypt(struct ktls_session *tls,  in ktls_ocf_tls13_aead_decrypt()  argument
869 	os = tls->ocf_session;  in ktls_ocf_tls13_aead_decrypt()
871 	tag_len = tls->params.tls_tlen - 1;  in ktls_ocf_tls13_aead_decrypt()
881 	memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len);  in ktls_ocf_tls13_aead_decrypt()
892 	crp.crp_payload_start = tls->params.tls_hlen;  in ktls_ocf_tls13_aead_decrypt()
900 	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)  in ktls_ocf_tls13_aead_decrypt()
912 ktls_ocf_tls13_aead_recrypt(struct ktls_session *tls,  in ktls_ocf_tls13_aead_recrypt()  argument
923 	os = tls->ocf_session;  in ktls_ocf_tls13_aead_recrypt()
932 	KASSERT(tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16,  in ktls_ocf_tls13_aead_recrypt()
936 	memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len);  in ktls_ocf_tls13_aead_recrypt()
944 	crp.crp_payload_start = tls->params.tls_hlen;  in ktls_ocf_tls13_aead_recrypt()
956 		ktls_ocf_recrypt_fixup(m, tls->params.tls_hlen, payload_len,  in ktls_ocf_tls13_aead_recrypt()
970 ktls_ocf_free(struct ktls_session *tls)  in ktls_ocf_free()  argument
974 	os = tls->ocf_session;  in ktls_ocf_free()
983 ktls_ocf_try(struct ktls_session *tls, int direction)  in ktls_ocf_try()  argument
996 	switch (tls->params.cipher_algorithm) {  in ktls_ocf_try()
998 		switch (tls->params.cipher_key_len) {  in ktls_ocf_try()
1006 		/* Only TLS 1.2 and 1.3 are supported. */  in ktls_ocf_try()
1007 		if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||  in ktls_ocf_try()
1008 		    tls->params.tls_vminor < TLS_MINOR_VER_TWO ||  in ktls_ocf_try()
1009 		    tls->params.tls_vminor > TLS_MINOR_VER_THREE)  in ktls_ocf_try()
1015 		csp.csp_cipher_key = tls->params.cipher_key;  in ktls_ocf_try()
1016 		csp.csp_cipher_klen = tls->params.cipher_key_len;  in ktls_ocf_try()
1022 		recrypt_csp.csp_cipher_key = tls->params.cipher_key;  in ktls_ocf_try()
1023 		recrypt_csp.csp_cipher_klen = tls->params.cipher_key_len;  in ktls_ocf_try()
1027 		switch (tls->params.cipher_key_len) {  in ktls_ocf_try()
1035 		switch (tls->params.auth_algorithm) {  in ktls_ocf_try()
1049 		/* Only TLS 1.0-1.2 are supported. */  in ktls_ocf_try()
1050 		if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||  in ktls_ocf_try()
1051 		    tls->params.tls_vminor < TLS_MINOR_VER_ZERO ||  in ktls_ocf_try()
1052 		    tls->params.tls_vminor > TLS_MINOR_VER_TWO)  in ktls_ocf_try()
1055 		/* AES-CBC is not supported for receive for TLS 1.0. */  in ktls_ocf_try()
1057 		    tls->params.tls_vminor == TLS_MINOR_VER_ZERO)  in ktls_ocf_try()
1063 		csp.csp_cipher_key = tls->params.cipher_key;  in ktls_ocf_try()
1064 		csp.csp_cipher_klen = tls->params.cipher_key_len;  in ktls_ocf_try()
1069 		mac_csp.csp_auth_alg = tls->params.auth_algorithm;  in ktls_ocf_try()
1070 		mac_csp.csp_auth_key = tls->params.auth_key;  in ktls_ocf_try()
1071 		mac_csp.csp_auth_klen = tls->params.auth_key_len;  in ktls_ocf_try()
1074 		switch (tls->params.cipher_key_len) {  in ktls_ocf_try()
1081 		/* Only TLS 1.2 and 1.3 are supported. */  in ktls_ocf_try()
1082 		if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||  in ktls_ocf_try()
1083 		    tls->params.tls_vminor < TLS_MINOR_VER_TWO ||  in ktls_ocf_try()
1084 		    tls->params.tls_vminor > TLS_MINOR_VER_THREE)  in ktls_ocf_try()
1090 		csp.csp_cipher_key = tls->params.cipher_key;  in ktls_ocf_try()
1091 		csp.csp_cipher_klen = tls->params.cipher_key_len;  in ktls_ocf_try()
1131 	tls->ocf_session = os;  in ktls_ocf_try()
1132 	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 ||  in ktls_ocf_try()
1133 	    tls->params.cipher_algorithm == CRYPTO_CHACHA20_POLY1305) {  in ktls_ocf_try()
1134 		if (tls->params.tls_vminor == TLS_MINOR_VER_THREE)  in ktls_ocf_try()
1140 		if (tls->params.tls_vminor == TLS_MINOR_VER_ZERO) {  in ktls_ocf_try()
1142 			memcpy(os->iv, tls->params.iv, AES_BLOCK_LEN);  in ktls_ocf_try()
1144 			os->next_seqno = tls->next_seqno;  in ktls_ocf_try()
1154 	tls->sync_dispatch = CRYPTO_SESS_SYNC(os->sid) ||  in ktls_ocf_try()
1155 	    tls->params.cipher_algorithm == CRYPTO_AES_CBC;  in ktls_ocf_try()
1161     struct ktls_session *tls, struct mbuf *m, struct iovec *outiov,  in ktls_ocf_encrypt()  argument
1164 	return (tls->ocf_session->sw->encrypt(state, tls, m, outiov,  in ktls_ocf_encrypt()
1169 ktls_ocf_decrypt(struct ktls_session *tls, const struct tls_record_layer *hdr,  in ktls_ocf_decrypt()  argument
1172 	return (tls->ocf_session->sw->decrypt(tls, hdr, m, seqno, trailer_len));  in ktls_ocf_decrypt()
1176 ktls_ocf_recrypt(struct ktls_session *tls, const struct tls_record_layer *hdr,  in ktls_ocf_recrypt()  argument
1179 	return (tls->ocf_session->sw->recrypt(tls, hdr, m, seqno));  in ktls_ocf_recrypt()
1183 ktls_ocf_recrypt_supported(struct ktls_session *tls)  in ktls_ocf_recrypt_supported()  argument
1185 	return (tls->ocf_session->sw->recrypt != NULL &&  in ktls_ocf_recrypt_supported()
1186 	    tls->ocf_session->recrypt_sid != NULL);  in ktls_ocf_recrypt_supported()