1 /* $FreeBSD$ */ 2 /* $OpenBSD: ip_esp.c,v 1.69 2001/06/26 06:18:59 angelos Exp $ */ 3 /*- 4 * The authors of this code are John Ioannidis (ji@tla.org), 5 * Angelos D. Keromytis (kermit@csd.uch.gr) and 6 * Niels Provos (provos@physnet.uni-hamburg.de). 7 * 8 * The original version of this code was written by John Ioannidis 9 * for BSD/OS in Athens, Greece, in November 1995. 10 * 11 * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, 12 * by Angelos D. Keromytis. 13 * 14 * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis 15 * and Niels Provos. 16 * 17 * Additional features in 1999 by Angelos D. Keromytis. 18 * 19 * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, 20 * Angelos D. Keromytis and Niels Provos. 21 * Copyright (c) 2001 Angelos D. Keromytis. 22 * 23 * Permission to use, copy, and modify this software with or without fee 24 * is hereby granted, provided that this entire notice is included in 25 * all copies of any software which is or includes a copy or 26 * modification of this software. 27 * You may use this code under the GNU public license if you so wish. Please 28 * contribute changes back to the authors under this freer than GPL license 29 * so that we may further the use of strong encryption without limitations to 30 * all. 31 * 32 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 33 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 34 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 35 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 36 * PURPOSE. 37 */ 38 #include "opt_inet.h" 39 #include "opt_inet6.h" 40 #include "opt_ipsec.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/malloc.h> 45 #include <sys/mbuf.h> 46 #include <sys/socket.h> 47 #include <sys/syslog.h> 48 #include <sys/kernel.h> 49 #include <sys/lock.h> 50 #include <sys/random.h> 51 #include <sys/mutex.h> 52 #include <sys/sysctl.h> 53 #include <sys/mutex.h> 54 #include <machine/atomic.h> 55 56 #include <net/if.h> 57 #include <net/vnet.h> 58 59 #include <netinet/in.h> 60 #include <netinet/in_systm.h> 61 #include <netinet/ip.h> 62 #include <netinet/ip_ecn.h> 63 #include <netinet/ip6.h> 64 65 #include <netipsec/ipsec.h> 66 #include <netipsec/ah.h> 67 #include <netipsec/ah_var.h> 68 #include <netipsec/esp.h> 69 #include <netipsec/esp_var.h> 70 #include <netipsec/xform.h> 71 72 #ifdef INET6 73 #include <netinet6/ip6_var.h> 74 #include <netipsec/ipsec6.h> 75 #include <netinet6/ip6_ecn.h> 76 #endif 77 78 #include <netipsec/key.h> 79 #include <netipsec/key_debug.h> 80 81 #include <opencrypto/cryptodev.h> 82 #include <opencrypto/xform.h> 83 84 #define SPI_SIZE 4 85 86 VNET_DEFINE(int, esp_enable) = 1; 87 VNET_DEFINE_STATIC(int, esp_ctr_compatibility) = 1; 88 #define V_esp_ctr_compatibility VNET(esp_ctr_compatibility) 89 VNET_PCPUSTAT_DEFINE(struct espstat, espstat); 90 VNET_PCPUSTAT_SYSINIT(espstat); 91 92 #ifdef VIMAGE 93 VNET_PCPUSTAT_SYSUNINIT(espstat); 94 #endif /* VIMAGE */ 95 96 SYSCTL_DECL(_net_inet_esp); 97 SYSCTL_INT(_net_inet_esp, OID_AUTO, esp_enable, 98 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(esp_enable), 0, ""); 99 SYSCTL_INT(_net_inet_esp, OID_AUTO, ctr_compatibility, 100 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(esp_ctr_compatibility), 0, 101 "Align AES-CTR encrypted transmitted frames to blocksize"); 102 SYSCTL_VNET_PCPUSTAT(_net_inet_esp, IPSECCTL_STATS, stats, 103 struct espstat, espstat, 104 "ESP statistics (struct espstat, netipsec/esp_var.h"); 105 106 static MALLOC_DEFINE(M_ESP, "esp", "IPsec ESP"); 107 108 static int esp_input_cb(struct cryptop *op); 109 static int esp_output_cb(struct cryptop *crp); 110 111 size_t 112 esp_hdrsiz(struct secasvar *sav) 113 { 114 size_t size; 115 116 if (sav != NULL) { 117 /*XXX not right for null algorithm--does it matter??*/ 118 IPSEC_ASSERT(sav->tdb_encalgxform != NULL, 119 ("SA with null xform")); 120 if (sav->flags & SADB_X_EXT_OLD) 121 size = sizeof (struct esp); 122 else 123 size = sizeof (struct newesp); 124 size += sav->tdb_encalgxform->blocksize + 9; 125 /*XXX need alg check???*/ 126 if (sav->tdb_authalgxform != NULL && sav->replay) 127 size += ah_hdrsiz(sav); 128 } else { 129 /* 130 * base header size 131 * + max iv length for CBC mode 132 * + max pad length 133 * + sizeof (pad length field) 134 * + sizeof (next header field) 135 * + max icv supported. 136 */ 137 size = sizeof (struct newesp) + EALG_MAX_BLOCK_LEN + 9 + 16; 138 } 139 return size; 140 } 141 142 /* 143 * esp_init() is called when an SPI is being set up. 144 */ 145 static int 146 esp_init(struct secasvar *sav, struct xformsw *xsp) 147 { 148 const struct enc_xform *txform; 149 struct crypto_session_params csp; 150 int keylen; 151 int error; 152 153 txform = enc_algorithm_lookup(sav->alg_enc); 154 if (txform == NULL) { 155 DPRINTF(("%s: unsupported encryption algorithm %d\n", 156 __func__, sav->alg_enc)); 157 return EINVAL; 158 } 159 if (sav->key_enc == NULL) { 160 DPRINTF(("%s: no encoding key for %s algorithm\n", 161 __func__, txform->name)); 162 return EINVAL; 163 } 164 if ((sav->flags & (SADB_X_EXT_OLD | SADB_X_EXT_IV4B)) == 165 SADB_X_EXT_IV4B) { 166 DPRINTF(("%s: 4-byte IV not supported with protocol\n", 167 __func__)); 168 return EINVAL; 169 } 170 171 /* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */ 172 keylen = _KEYLEN(sav->key_enc) - SAV_ISCTRORGCM(sav) * 4 - 173 SAV_ISCHACHA(sav) * 4; 174 if (txform->minkey > keylen || keylen > txform->maxkey) { 175 DPRINTF(("%s: invalid key length %u, must be in the range " 176 "[%u..%u] for algorithm %s\n", __func__, 177 keylen, txform->minkey, txform->maxkey, 178 txform->name)); 179 return EINVAL; 180 } 181 182 if (SAV_ISCTRORGCM(sav) || SAV_ISCHACHA(sav)) 183 sav->ivlen = 8; /* RFC4106 3.1 and RFC3686 3.1 */ 184 else 185 sav->ivlen = txform->ivsize; 186 187 memset(&csp, 0, sizeof(csp)); 188 189 /* 190 * Setup AH-related state. 191 */ 192 if (sav->alg_auth != 0) { 193 error = ah_init0(sav, xsp, &csp); 194 if (error) 195 return error; 196 } 197 198 /* NB: override anything set in ah_init0 */ 199 sav->tdb_xform = xsp; 200 sav->tdb_encalgxform = txform; 201 202 /* 203 * Whenever AES-GCM is used for encryption, one 204 * of the AES authentication algorithms is chosen 205 * as well, based on the key size. 206 */ 207 if (sav->alg_enc == SADB_X_EALG_AESGCM16) { 208 switch (keylen) { 209 case AES_128_GMAC_KEY_LEN: 210 sav->alg_auth = SADB_X_AALG_AES128GMAC; 211 sav->tdb_authalgxform = &auth_hash_nist_gmac_aes_128; 212 break; 213 case AES_192_GMAC_KEY_LEN: 214 sav->alg_auth = SADB_X_AALG_AES192GMAC; 215 sav->tdb_authalgxform = &auth_hash_nist_gmac_aes_192; 216 break; 217 case AES_256_GMAC_KEY_LEN: 218 sav->alg_auth = SADB_X_AALG_AES256GMAC; 219 sav->tdb_authalgxform = &auth_hash_nist_gmac_aes_256; 220 break; 221 default: 222 DPRINTF(("%s: invalid key length %u" 223 "for algorithm %s\n", __func__, 224 keylen, txform->name)); 225 return EINVAL; 226 } 227 csp.csp_mode = CSP_MODE_AEAD; 228 if (sav->flags & SADB_X_SAFLAGS_ESN) 229 csp.csp_flags |= CSP_F_SEPARATE_AAD; 230 } else if (sav->alg_enc == SADB_X_EALG_CHACHA20POLY1305) { 231 sav->alg_auth = SADB_X_AALG_CHACHA20POLY1305; 232 sav->tdb_authalgxform = &auth_hash_poly1305; 233 csp.csp_mode = CSP_MODE_AEAD; 234 if (sav->flags & SADB_X_SAFLAGS_ESN) 235 csp.csp_flags |= CSP_F_SEPARATE_AAD; 236 } else if (sav->alg_auth != 0) { 237 csp.csp_mode = CSP_MODE_ETA; 238 if (sav->flags & SADB_X_SAFLAGS_ESN) 239 csp.csp_flags |= CSP_F_ESN; 240 } else 241 csp.csp_mode = CSP_MODE_CIPHER; 242 243 /* Initialize crypto session. */ 244 csp.csp_cipher_alg = sav->tdb_encalgxform->type; 245 if (csp.csp_cipher_alg != CRYPTO_NULL_CBC) { 246 csp.csp_cipher_key = sav->key_enc->key_data; 247 csp.csp_cipher_klen = _KEYBITS(sav->key_enc) / 8 - 248 SAV_ISCTRORGCM(sav) * 4 - SAV_ISCHACHA(sav) * 4; 249 }; 250 csp.csp_ivlen = txform->ivsize; 251 252 error = crypto_newsession(&sav->tdb_cryptoid, &csp, V_crypto_support); 253 return error; 254 } 255 256 static void 257 esp_cleanup(struct secasvar *sav) 258 { 259 260 crypto_freesession(sav->tdb_cryptoid); 261 sav->tdb_cryptoid = NULL; 262 sav->tdb_authalgxform = NULL; 263 sav->tdb_encalgxform = NULL; 264 } 265 266 /* 267 * ESP input processing, called (eventually) through the protocol switch. 268 */ 269 static int 270 esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff) 271 { 272 IPSEC_DEBUG_DECLARE(char buf[128]); 273 const struct auth_hash *esph; 274 const struct enc_xform *espx; 275 struct xform_data *xd; 276 struct cryptop *crp; 277 struct newesp *esp; 278 uint8_t *ivp; 279 crypto_session_t cryptoid; 280 int alen, error, hlen, plen; 281 uint32_t seqh; 282 const struct crypto_session_params *csp; 283 284 SECASVAR_RLOCK_TRACKER; 285 286 IPSEC_ASSERT(sav != NULL, ("null SA")); 287 IPSEC_ASSERT(sav->tdb_encalgxform != NULL, ("null encoding xform")); 288 289 error = EINVAL; 290 /* Valid IP Packet length ? */ 291 if ( (skip&3) || (m->m_pkthdr.len&3) ){ 292 DPRINTF(("%s: misaligned packet, skip %u pkt len %u", 293 __func__, skip, m->m_pkthdr.len)); 294 ESPSTAT_INC(esps_badilen); 295 goto bad; 296 } 297 298 if (m->m_len < skip + sizeof(*esp)) { 299 m = m_pullup(m, skip + sizeof(*esp)); 300 if (m == NULL) { 301 DPRINTF(("%s: cannot pullup header\n", __func__)); 302 ESPSTAT_INC(esps_hdrops); /*XXX*/ 303 error = ENOBUFS; 304 goto bad; 305 } 306 } 307 esp = (struct newesp *)(mtod(m, caddr_t) + skip); 308 309 esph = sav->tdb_authalgxform; 310 espx = sav->tdb_encalgxform; 311 312 /* Determine the ESP header and auth length */ 313 if (sav->flags & SADB_X_EXT_OLD) 314 hlen = sizeof (struct esp) + sav->ivlen; 315 else 316 hlen = sizeof (struct newesp) + sav->ivlen; 317 318 alen = xform_ah_authsize(esph); 319 320 /* 321 * Verify payload length is multiple of encryption algorithm 322 * block size. 323 * 324 * NB: This works for the null algorithm because the blocksize 325 * is 4 and all packets must be 4-byte aligned regardless 326 * of the algorithm. 327 */ 328 plen = m->m_pkthdr.len - (skip + hlen + alen); 329 if ((plen & (espx->blocksize - 1)) || (plen <= 0)) { 330 DPRINTF(("%s: payload of %d octets not a multiple of %d octets," 331 " SA %s/%08lx\n", __func__, plen, espx->blocksize, 332 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 333 (u_long)ntohl(sav->spi))); 334 ESPSTAT_INC(esps_badilen); 335 goto bad; 336 } 337 338 /* 339 * Check sequence number. 340 */ 341 SECASVAR_RLOCK(sav); 342 if (esph != NULL && sav->replay != NULL && sav->replay->wsize != 0) { 343 if (ipsec_chkreplay(ntohl(esp->esp_seq), &seqh, sav) == 0) { 344 SECASVAR_RUNLOCK(sav); 345 DPRINTF(("%s: packet replay check for %s\n", __func__, 346 ipsec_sa2str(sav, buf, sizeof(buf)))); 347 ESPSTAT_INC(esps_replay); 348 error = EACCES; 349 goto bad; 350 } 351 seqh = htonl(seqh); 352 } 353 cryptoid = sav->tdb_cryptoid; 354 SECASVAR_RUNLOCK(sav); 355 356 /* Update the counters */ 357 ESPSTAT_ADD(esps_ibytes, m->m_pkthdr.len - (skip + hlen + alen)); 358 359 /* Get crypto descriptors */ 360 crp = crypto_getreq(cryptoid, M_NOWAIT); 361 if (crp == NULL) { 362 DPRINTF(("%s: failed to acquire crypto descriptors\n", 363 __func__)); 364 ESPSTAT_INC(esps_crypto); 365 error = ENOBUFS; 366 goto bad; 367 } 368 369 /* Get IPsec-specific opaque pointer */ 370 xd = malloc(sizeof(*xd), M_ESP, M_NOWAIT | M_ZERO); 371 if (xd == NULL) { 372 DPRINTF(("%s: failed to allocate xform_data\n", __func__)); 373 goto xd_fail; 374 } 375 376 if (esph != NULL) { 377 crp->crp_op = CRYPTO_OP_VERIFY_DIGEST; 378 if (SAV_ISGCM(sav) || SAV_ISCHACHA(sav)) 379 crp->crp_aad_length = 8; /* RFC4106 5, SPI + SN */ 380 else 381 crp->crp_aad_length = hlen; 382 383 csp = crypto_get_params(crp->crp_session); 384 if ((csp->csp_flags & CSP_F_SEPARATE_AAD) && 385 (sav->replay != NULL) && (sav->replay->wsize != 0)) { 386 int aad_skip; 387 388 crp->crp_aad_length += sizeof(seqh); 389 crp->crp_aad = malloc(crp->crp_aad_length, M_ESP, M_NOWAIT); 390 if (crp->crp_aad == NULL) { 391 DPRINTF(("%s: failed to allocate xform_data\n", 392 __func__)); 393 goto crp_aad_fail; 394 } 395 396 /* SPI */ 397 m_copydata(m, skip, SPI_SIZE, crp->crp_aad); 398 aad_skip = SPI_SIZE; 399 400 /* ESN */ 401 bcopy(&seqh, (char *)crp->crp_aad + aad_skip, sizeof(seqh)); 402 aad_skip += sizeof(seqh); 403 404 /* Rest of aad */ 405 if (crp->crp_aad_length - aad_skip > 0) 406 m_copydata(m, skip + SPI_SIZE, 407 crp->crp_aad_length - aad_skip, 408 (char *)crp->crp_aad + aad_skip); 409 } else 410 crp->crp_aad_start = skip; 411 412 if (csp->csp_flags & CSP_F_ESN && 413 sav->replay != NULL && sav->replay->wsize != 0) 414 memcpy(crp->crp_esn, &seqh, sizeof(seqh)); 415 416 crp->crp_digest_start = m->m_pkthdr.len - alen; 417 } 418 419 /* Crypto operation descriptor */ 420 crp->crp_flags = CRYPTO_F_CBIFSYNC; 421 crypto_use_mbuf(crp, m); 422 crp->crp_callback = esp_input_cb; 423 crp->crp_opaque = xd; 424 425 /* These are passed as-is to the callback */ 426 xd->sav = sav; 427 xd->protoff = protoff; 428 xd->skip = skip; 429 xd->cryptoid = cryptoid; 430 xd->vnet = curvnet; 431 432 /* Decryption descriptor */ 433 crp->crp_op |= CRYPTO_OP_DECRYPT; 434 crp->crp_payload_start = skip + hlen; 435 crp->crp_payload_length = m->m_pkthdr.len - (skip + hlen + alen); 436 437 /* Generate or read cipher IV. */ 438 if (SAV_ISCTRORGCM(sav) || SAV_ISCHACHA(sav)) { 439 ivp = &crp->crp_iv[0]; 440 441 /* 442 * AES-GCM and AES-CTR use similar cipher IV formats 443 * defined in RFC 4106 section 4 and RFC 3686 section 444 * 4, respectively. 445 * 446 * The first 4 bytes of the cipher IV contain an 447 * implicit salt, or nonce, obtained from the last 4 448 * bytes of the encryption key. The next 8 bytes hold 449 * an explicit IV unique to each packet. This 450 * explicit IV is used as the ESP IV for the packet. 451 * The last 4 bytes hold a big-endian block counter 452 * incremented for each block. For AES-GCM, the block 453 * counter's initial value is defined as part of the 454 * algorithm. For AES-CTR, the block counter's 455 * initial value for each packet is defined as 1 by 456 * RFC 3686. 457 * 458 * ------------------------------------------ 459 * | Salt | Explicit ESP IV | Block Counter | 460 * ------------------------------------------ 461 * 4 bytes 8 bytes 4 bytes 462 */ 463 memcpy(ivp, sav->key_enc->key_data + 464 _KEYLEN(sav->key_enc) - 4, 4); 465 m_copydata(m, skip + hlen - sav->ivlen, sav->ivlen, &ivp[4]); 466 if (SAV_ISCTR(sav)) { 467 be32enc(&ivp[sav->ivlen + 4], 1); 468 } 469 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 470 } else if (sav->ivlen != 0) 471 crp->crp_iv_start = skip + hlen - sav->ivlen; 472 473 if (V_async_crypto) 474 return (crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED)); 475 else 476 return (crypto_dispatch(crp)); 477 478 crp_aad_fail: 479 free(xd, M_ESP); 480 xd_fail: 481 crypto_freereq(crp); 482 ESPSTAT_INC(esps_crypto); 483 error = ENOBUFS; 484 bad: 485 m_freem(m); 486 key_freesav(&sav); 487 return (error); 488 } 489 490 /* 491 * ESP input callback from the crypto driver. 492 */ 493 static int 494 esp_input_cb(struct cryptop *crp) 495 { 496 IPSEC_DEBUG_DECLARE(char buf[128]); 497 uint8_t lastthree[3]; 498 const struct auth_hash *esph; 499 struct mbuf *m; 500 struct xform_data *xd; 501 struct secasvar *sav; 502 struct secasindex *saidx; 503 crypto_session_t cryptoid; 504 int hlen, skip, protoff, error, alen; 505 506 SECASVAR_RLOCK_TRACKER; 507 508 m = crp->crp_buf.cb_mbuf; 509 xd = crp->crp_opaque; 510 CURVNET_SET(xd->vnet); 511 sav = xd->sav; 512 skip = xd->skip; 513 protoff = xd->protoff; 514 cryptoid = xd->cryptoid; 515 saidx = &sav->sah->saidx; 516 esph = sav->tdb_authalgxform; 517 518 /* Check for crypto errors */ 519 if (crp->crp_etype) { 520 if (crp->crp_etype == EAGAIN) { 521 /* Reset the session ID */ 522 if (ipsec_updateid(sav, &crp->crp_session, &cryptoid) != 0) 523 crypto_freesession(cryptoid); 524 xd->cryptoid = crp->crp_session; 525 CURVNET_RESTORE(); 526 return (crypto_dispatch(crp)); 527 } 528 529 /* EBADMSG indicates authentication failure. */ 530 if (!(crp->crp_etype == EBADMSG && esph != NULL)) { 531 ESPSTAT_INC(esps_noxform); 532 DPRINTF(("%s: crypto error %d\n", __func__, 533 crp->crp_etype)); 534 error = crp->crp_etype; 535 goto bad; 536 } 537 } 538 539 /* Shouldn't happen... */ 540 if (m == NULL) { 541 ESPSTAT_INC(esps_crypto); 542 DPRINTF(("%s: bogus returned buffer from crypto\n", __func__)); 543 error = EINVAL; 544 goto bad; 545 } 546 ESPSTAT_INC(esps_hist[sav->alg_enc]); 547 548 /* If authentication was performed, check now. */ 549 if (esph != NULL) { 550 alen = xform_ah_authsize(esph); 551 AHSTAT_INC(ahs_hist[sav->alg_auth]); 552 if (crp->crp_etype == EBADMSG) { 553 DPRINTF(("%s: authentication hash mismatch for " 554 "packet in SA %s/%08lx\n", __func__, 555 ipsec_address(&saidx->dst, buf, sizeof(buf)), 556 (u_long) ntohl(sav->spi))); 557 ESPSTAT_INC(esps_badauth); 558 error = EACCES; 559 goto bad; 560 } 561 m->m_flags |= M_AUTHIPDGM; 562 /* Remove trailing authenticator */ 563 m_adj(m, -alen); 564 } 565 566 /* Release the crypto descriptors */ 567 free(xd, M_ESP), xd = NULL; 568 free(crp->crp_aad, M_ESP), crp->crp_aad = NULL; 569 crypto_freereq(crp), crp = NULL; 570 571 /* 572 * Packet is now decrypted. 573 */ 574 m->m_flags |= M_DECRYPTED; 575 576 /* 577 * Update replay sequence number, if appropriate. 578 */ 579 if (sav->replay) { 580 u_int32_t seq; 581 582 m_copydata(m, skip + offsetof(struct newesp, esp_seq), 583 sizeof (seq), (caddr_t) &seq); 584 SECASVAR_RLOCK(sav); 585 if (ipsec_updatereplay(ntohl(seq), sav)) { 586 SECASVAR_RUNLOCK(sav); 587 DPRINTF(("%s: packet replay check for %s\n", __func__, 588 ipsec_sa2str(sav, buf, sizeof(buf)))); 589 ESPSTAT_INC(esps_replay); 590 error = EACCES; 591 goto bad; 592 } 593 SECASVAR_RUNLOCK(sav); 594 } 595 596 /* Determine the ESP header length */ 597 if (sav->flags & SADB_X_EXT_OLD) 598 hlen = sizeof (struct esp) + sav->ivlen; 599 else 600 hlen = sizeof (struct newesp) + sav->ivlen; 601 602 /* Remove the ESP header and IV from the mbuf. */ 603 error = m_striphdr(m, skip, hlen); 604 if (error) { 605 ESPSTAT_INC(esps_hdrops); 606 DPRINTF(("%s: bad mbuf chain, SA %s/%08lx\n", __func__, 607 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 608 (u_long) ntohl(sav->spi))); 609 goto bad; 610 } 611 612 /* Save the last three bytes of decrypted data */ 613 m_copydata(m, m->m_pkthdr.len - 3, 3, lastthree); 614 615 /* Verify pad length */ 616 if (lastthree[1] + 2 > m->m_pkthdr.len - skip) { 617 ESPSTAT_INC(esps_badilen); 618 DPRINTF(("%s: invalid padding length %d for %u byte packet " 619 "in SA %s/%08lx\n", __func__, lastthree[1], 620 m->m_pkthdr.len - skip, 621 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 622 (u_long) ntohl(sav->spi))); 623 error = EINVAL; 624 goto bad; 625 } 626 627 /* Verify correct decryption by checking the last padding bytes */ 628 if ((sav->flags & SADB_X_EXT_PMASK) != SADB_X_EXT_PRAND) { 629 if (lastthree[1] != lastthree[0] && lastthree[1] != 0) { 630 ESPSTAT_INC(esps_badenc); 631 DPRINTF(("%s: decryption failed for packet in " 632 "SA %s/%08lx\n", __func__, ipsec_address( 633 &sav->sah->saidx.dst, buf, sizeof(buf)), 634 (u_long) ntohl(sav->spi))); 635 error = EINVAL; 636 goto bad; 637 } 638 } 639 640 /* 641 * RFC4303 2.6: 642 * Silently drop packet if next header field is IPPROTO_NONE. 643 */ 644 if (lastthree[2] == IPPROTO_NONE) 645 goto bad; 646 647 /* Trim the mbuf chain to remove trailing authenticator and padding */ 648 m_adj(m, -(lastthree[1] + 2)); 649 650 /* Restore the Next Protocol field */ 651 m_copyback(m, protoff, sizeof (u_int8_t), lastthree + 2); 652 653 switch (saidx->dst.sa.sa_family) { 654 #ifdef INET6 655 case AF_INET6: 656 error = ipsec6_common_input_cb(m, sav, skip, protoff); 657 break; 658 #endif 659 #ifdef INET 660 case AF_INET: 661 error = ipsec4_common_input_cb(m, sav, skip, protoff); 662 break; 663 #endif 664 default: 665 panic("%s: Unexpected address family: %d saidx=%p", __func__, 666 saidx->dst.sa.sa_family, saidx); 667 } 668 CURVNET_RESTORE(); 669 return error; 670 bad: 671 if (sav != NULL) 672 key_freesav(&sav); 673 if (m != NULL) 674 m_freem(m); 675 if (xd != NULL) 676 free(xd, M_ESP); 677 if (crp != NULL) { 678 free(crp->crp_aad, M_ESP); 679 crypto_freereq(crp); 680 } 681 CURVNET_RESTORE(); 682 return error; 683 } 684 /* 685 * ESP output routine, called by ipsec[46]_perform_request(). 686 */ 687 static int 688 esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav, 689 u_int idx, int skip, int protoff) 690 { 691 IPSEC_DEBUG_DECLARE(char buf[IPSEC_ADDRSTRLEN]); 692 struct cryptop *crp; 693 const struct auth_hash *esph; 694 const struct enc_xform *espx; 695 struct mbuf *mo = NULL; 696 struct xform_data *xd; 697 struct secasindex *saidx; 698 unsigned char *pad; 699 uint8_t *ivp; 700 uint64_t cntr; 701 crypto_session_t cryptoid; 702 int hlen, rlen, padding, blks, alen, i, roff; 703 int error, maxpacketsize; 704 uint8_t prot; 705 uint32_t seqh; 706 const struct crypto_session_params *csp; 707 708 SECASVAR_RLOCK_TRACKER; 709 710 IPSEC_ASSERT(sav != NULL, ("null SA")); 711 esph = sav->tdb_authalgxform; 712 espx = sav->tdb_encalgxform; 713 IPSEC_ASSERT(espx != NULL, ("null encoding xform")); 714 715 if (sav->flags & SADB_X_EXT_OLD) 716 hlen = sizeof (struct esp) + sav->ivlen; 717 else 718 hlen = sizeof (struct newesp) + sav->ivlen; 719 720 rlen = m->m_pkthdr.len - skip; /* Raw payload length. */ 721 /* 722 * RFC4303 2.4 Requires 4 byte alignment. 723 * Old versions of FreeBSD can't decrypt partial blocks encrypted 724 * with AES-CTR. Align payload to native_blocksize (16 bytes) 725 * in order to preserve compatibility. 726 */ 727 if (SAV_ISCTR(sav) && V_esp_ctr_compatibility) 728 blks = MAX(4, espx->native_blocksize); /* Cipher blocksize */ 729 else 730 blks = MAX(4, espx->blocksize); 731 732 /* XXX clamp padding length a la KAME??? */ 733 padding = ((blks - ((rlen + 2) % blks)) % blks) + 2; 734 735 alen = xform_ah_authsize(esph); 736 737 ESPSTAT_INC(esps_output); 738 739 saidx = &sav->sah->saidx; 740 /* Check for maximum packet size violations. */ 741 switch (saidx->dst.sa.sa_family) { 742 #ifdef INET 743 case AF_INET: 744 maxpacketsize = IP_MAXPACKET; 745 break; 746 #endif /* INET */ 747 #ifdef INET6 748 case AF_INET6: 749 maxpacketsize = IPV6_MAXPACKET; 750 break; 751 #endif /* INET6 */ 752 default: 753 DPRINTF(("%s: unknown/unsupported protocol " 754 "family %d, SA %s/%08lx\n", __func__, 755 saidx->dst.sa.sa_family, ipsec_address(&saidx->dst, 756 buf, sizeof(buf)), (u_long) ntohl(sav->spi))); 757 ESPSTAT_INC(esps_nopf); 758 error = EPFNOSUPPORT; 759 goto bad; 760 } 761 /* 762 DPRINTF(("%s: skip %d hlen %d rlen %d padding %d alen %d blksd %d\n", 763 __func__, skip, hlen, rlen, padding, alen, blks)); */ 764 if (skip + hlen + rlen + padding + alen > maxpacketsize) { 765 DPRINTF(("%s: packet in SA %s/%08lx got too big " 766 "(len %u, max len %u)\n", __func__, 767 ipsec_address(&saidx->dst, buf, sizeof(buf)), 768 (u_long) ntohl(sav->spi), 769 skip + hlen + rlen + padding + alen, maxpacketsize)); 770 ESPSTAT_INC(esps_toobig); 771 error = EMSGSIZE; 772 goto bad; 773 } 774 775 /* Update the counters. */ 776 ESPSTAT_ADD(esps_obytes, m->m_pkthdr.len - skip); 777 778 m = m_unshare(m, M_NOWAIT); 779 if (m == NULL) { 780 DPRINTF(("%s: cannot clone mbuf chain, SA %s/%08lx\n", __func__, 781 ipsec_address(&saidx->dst, buf, sizeof(buf)), 782 (u_long) ntohl(sav->spi))); 783 ESPSTAT_INC(esps_hdrops); 784 error = ENOBUFS; 785 goto bad; 786 } 787 788 /* Inject ESP header. */ 789 mo = m_makespace(m, skip, hlen, &roff); 790 if (mo == NULL) { 791 DPRINTF(("%s: %u byte ESP hdr inject failed for SA %s/%08lx\n", 792 __func__, hlen, ipsec_address(&saidx->dst, buf, 793 sizeof(buf)), (u_long) ntohl(sav->spi))); 794 ESPSTAT_INC(esps_hdrops); /* XXX diffs from openbsd */ 795 error = ENOBUFS; 796 goto bad; 797 } 798 799 /* Initialize ESP header. */ 800 bcopy((caddr_t) &sav->spi, mtod(mo, caddr_t) + roff, 801 sizeof(uint32_t)); 802 SECASVAR_RLOCK(sav); 803 if (sav->replay) { 804 uint32_t replay; 805 806 SECREPLAY_LOCK(sav->replay); 807 #ifdef REGRESSION 808 /* Emulate replay attack when ipsec_replay is TRUE. */ 809 if (!V_ipsec_replay) 810 #endif 811 sav->replay->count++; 812 replay = htonl((uint32_t)sav->replay->count); 813 814 bcopy((caddr_t) &replay, mtod(mo, caddr_t) + roff + 815 sizeof(uint32_t), sizeof(uint32_t)); 816 817 seqh = htonl((uint32_t)(sav->replay->count >> IPSEC_SEQH_SHIFT)); 818 SECREPLAY_UNLOCK(sav->replay); 819 } 820 cryptoid = sav->tdb_cryptoid; 821 if (SAV_ISCTRORGCM(sav) || SAV_ISCHACHA(sav)) 822 cntr = sav->cntr++; 823 SECASVAR_RUNLOCK(sav); 824 825 /* 826 * Add padding -- better to do it ourselves than use the crypto engine, 827 * although if/when we support compression, we'd have to do that. 828 */ 829 pad = (u_char *) m_pad(m, padding + alen); 830 if (pad == NULL) { 831 DPRINTF(("%s: m_pad failed for SA %s/%08lx\n", __func__, 832 ipsec_address(&saidx->dst, buf, sizeof(buf)), 833 (u_long) ntohl(sav->spi))); 834 m = NULL; /* NB: free'd by m_pad */ 835 error = ENOBUFS; 836 goto bad; 837 } 838 839 /* 840 * Add padding: random, zero, or self-describing. 841 * XXX catch unexpected setting 842 */ 843 switch (sav->flags & SADB_X_EXT_PMASK) { 844 case SADB_X_EXT_PRAND: 845 arc4random_buf(pad, padding - 2); 846 break; 847 case SADB_X_EXT_PZERO: 848 bzero(pad, padding - 2); 849 break; 850 case SADB_X_EXT_PSEQ: 851 for (i = 0; i < padding - 2; i++) 852 pad[i] = i+1; 853 break; 854 } 855 856 /* Fix padding length and Next Protocol in padding itself. */ 857 pad[padding - 2] = padding - 2; 858 m_copydata(m, protoff, sizeof(u_int8_t), pad + padding - 1); 859 860 /* Fix Next Protocol in IPv4/IPv6 header. */ 861 prot = IPPROTO_ESP; 862 m_copyback(m, protoff, sizeof(u_int8_t), (u_char *) &prot); 863 864 /* Get crypto descriptor. */ 865 crp = crypto_getreq(cryptoid, M_NOWAIT); 866 if (crp == NULL) { 867 DPRINTF(("%s: failed to acquire crypto descriptor\n", 868 __func__)); 869 ESPSTAT_INC(esps_crypto); 870 error = ENOBUFS; 871 goto bad; 872 } 873 874 /* IPsec-specific opaque crypto info. */ 875 xd = malloc(sizeof(struct xform_data), M_ESP, M_NOWAIT | M_ZERO); 876 if (xd == NULL) { 877 DPRINTF(("%s: failed to allocate xform_data\n", __func__)); 878 goto xd_fail; 879 } 880 881 /* Encryption descriptor. */ 882 crp->crp_payload_start = skip + hlen; 883 crp->crp_payload_length = m->m_pkthdr.len - (skip + hlen + alen); 884 crp->crp_op = CRYPTO_OP_ENCRYPT; 885 886 /* Generate cipher and ESP IVs. */ 887 ivp = &crp->crp_iv[0]; 888 if (SAV_ISCTRORGCM(sav) || SAV_ISCHACHA(sav)) { 889 /* 890 * See comment in esp_input() for details on the 891 * cipher IV. A simple per-SA counter stored in 892 * 'cntr' is used as the explicit ESP IV. 893 */ 894 memcpy(ivp, sav->key_enc->key_data + 895 _KEYLEN(sav->key_enc) - 4, 4); 896 be64enc(&ivp[4], cntr); 897 if (SAV_ISCTR(sav)) { 898 be32enc(&ivp[sav->ivlen + 4], 1); 899 } 900 m_copyback(m, skip + hlen - sav->ivlen, sav->ivlen, &ivp[4]); 901 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 902 } else if (sav->ivlen != 0) { 903 arc4rand(ivp, sav->ivlen, 0); 904 crp->crp_iv_start = skip + hlen - sav->ivlen; 905 m_copyback(m, crp->crp_iv_start, sav->ivlen, ivp); 906 } 907 908 /* Callback parameters */ 909 xd->sp = sp; 910 xd->sav = sav; 911 xd->idx = idx; 912 xd->cryptoid = cryptoid; 913 xd->vnet = curvnet; 914 915 /* Crypto operation descriptor. */ 916 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 917 crypto_use_mbuf(crp, m); 918 crp->crp_callback = esp_output_cb; 919 crp->crp_opaque = xd; 920 921 if (esph) { 922 /* Authentication descriptor. */ 923 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST; 924 if (SAV_ISGCM(sav) || SAV_ISCHACHA(sav)) 925 crp->crp_aad_length = 8; /* RFC4106 5, SPI + SN */ 926 else 927 crp->crp_aad_length = hlen; 928 929 csp = crypto_get_params(crp->crp_session); 930 if (csp->csp_flags & CSP_F_SEPARATE_AAD && 931 sav->replay != NULL) { 932 int aad_skip; 933 934 crp->crp_aad_length += sizeof(seqh); 935 crp->crp_aad = malloc(crp->crp_aad_length, M_ESP, M_NOWAIT); 936 if (crp->crp_aad == NULL) { 937 DPRINTF(("%s: failed to allocate xform_data\n", 938 __func__)); 939 goto crp_aad_fail; 940 } 941 942 /* SPI */ 943 m_copydata(m, skip, SPI_SIZE, crp->crp_aad); 944 aad_skip = SPI_SIZE; 945 946 /* ESN */ 947 bcopy(&seqh, (char *)crp->crp_aad + aad_skip, sizeof(seqh)); 948 aad_skip += sizeof(seqh); 949 950 /* Rest of aad */ 951 if (crp->crp_aad_length - aad_skip > 0) 952 m_copydata(m, skip + SPI_SIZE, 953 crp->crp_aad_length - aad_skip, 954 (char *)crp->crp_aad + aad_skip); 955 } else 956 crp->crp_aad_start = skip; 957 958 if (csp->csp_flags & CSP_F_ESN && sav->replay != NULL) 959 memcpy(crp->crp_esn, &seqh, sizeof(seqh)); 960 961 crp->crp_digest_start = m->m_pkthdr.len - alen; 962 } 963 964 if (V_async_crypto) 965 return (crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED)); 966 else 967 return (crypto_dispatch(crp)); 968 969 crp_aad_fail: 970 free(xd, M_ESP); 971 xd_fail: 972 crypto_freereq(crp); 973 ESPSTAT_INC(esps_crypto); 974 error = ENOBUFS; 975 bad: 976 if (m) 977 m_freem(m); 978 key_freesav(&sav); 979 key_freesp(&sp); 980 return (error); 981 } 982 /* 983 * ESP output callback from the crypto driver. 984 */ 985 static int 986 esp_output_cb(struct cryptop *crp) 987 { 988 struct xform_data *xd; 989 struct secpolicy *sp; 990 struct secasvar *sav; 991 struct mbuf *m; 992 crypto_session_t cryptoid; 993 u_int idx; 994 int error; 995 996 xd = (struct xform_data *) crp->crp_opaque; 997 CURVNET_SET(xd->vnet); 998 m = crp->crp_buf.cb_mbuf; 999 sp = xd->sp; 1000 sav = xd->sav; 1001 idx = xd->idx; 1002 cryptoid = xd->cryptoid; 1003 1004 /* Check for crypto errors. */ 1005 if (crp->crp_etype) { 1006 if (crp->crp_etype == EAGAIN) { 1007 /* Reset the session ID */ 1008 if (ipsec_updateid(sav, &crp->crp_session, &cryptoid) != 0) 1009 crypto_freesession(cryptoid); 1010 xd->cryptoid = crp->crp_session; 1011 CURVNET_RESTORE(); 1012 return (crypto_dispatch(crp)); 1013 } 1014 ESPSTAT_INC(esps_noxform); 1015 DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype)); 1016 error = crp->crp_etype; 1017 m_freem(m); 1018 goto bad; 1019 } 1020 1021 /* Shouldn't happen... */ 1022 if (m == NULL) { 1023 ESPSTAT_INC(esps_crypto); 1024 DPRINTF(("%s: bogus returned buffer from crypto\n", __func__)); 1025 error = EINVAL; 1026 goto bad; 1027 } 1028 free(xd, M_ESP); 1029 free(crp->crp_aad, M_ESP); 1030 crypto_freereq(crp); 1031 ESPSTAT_INC(esps_hist[sav->alg_enc]); 1032 if (sav->tdb_authalgxform != NULL) 1033 AHSTAT_INC(ahs_hist[sav->alg_auth]); 1034 1035 #ifdef REGRESSION 1036 /* Emulate man-in-the-middle attack when ipsec_integrity is TRUE. */ 1037 if (V_ipsec_integrity) { 1038 static unsigned char ipseczeroes[AH_HMAC_MAXHASHLEN]; 1039 const struct auth_hash *esph; 1040 1041 /* 1042 * Corrupt HMAC if we want to test integrity verification of 1043 * the other side. 1044 */ 1045 esph = sav->tdb_authalgxform; 1046 if (esph != NULL) { 1047 int alen; 1048 1049 alen = xform_ah_authsize(esph); 1050 m_copyback(m, m->m_pkthdr.len - alen, 1051 alen, ipseczeroes); 1052 } 1053 } 1054 #endif 1055 1056 /* NB: m is reclaimed by ipsec_process_done. */ 1057 error = ipsec_process_done(m, sp, sav, idx); 1058 CURVNET_RESTORE(); 1059 return (error); 1060 bad: 1061 free(xd, M_ESP); 1062 free(crp->crp_aad, M_ESP); 1063 crypto_freereq(crp); 1064 key_freesav(&sav); 1065 key_freesp(&sp); 1066 CURVNET_RESTORE(); 1067 return (error); 1068 } 1069 1070 static struct xformsw esp_xformsw = { 1071 .xf_type = XF_ESP, 1072 .xf_name = "IPsec ESP", 1073 .xf_init = esp_init, 1074 .xf_cleanup = esp_cleanup, 1075 .xf_input = esp_input, 1076 .xf_output = esp_output, 1077 }; 1078 1079 SYSINIT(esp_xform_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE, 1080 xform_attach, &esp_xformsw); 1081 SYSUNINIT(esp_xform_uninit, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE, 1082 xform_detach, &esp_xformsw); 1083