1 /* 2 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995 3 * The Regents of the University of California. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * @(#)tcp_timer.c 8.2 (Berkeley) 5/24/95 30 * $FreeBSD$ 31 */ 32 33 #include "opt_inet6.h" 34 #include "opt_tcpdebug.h" 35 #include "opt_tcp_sack.h" 36 37 #include <sys/param.h> 38 #include <sys/kernel.h> 39 #include <sys/lock.h> 40 #include <sys/mbuf.h> 41 #include <sys/mutex.h> 42 #include <sys/protosw.h> 43 #include <sys/socket.h> 44 #include <sys/socketvar.h> 45 #include <sys/sysctl.h> 46 #include <sys/systm.h> 47 48 #include <net/route.h> 49 50 #include <netinet/in.h> 51 #include <netinet/in_pcb.h> 52 #include <netinet/in_systm.h> 53 #ifdef INET6 54 #include <netinet6/in6_pcb.h> 55 #endif 56 #include <netinet/ip_var.h> 57 #include <netinet/tcp.h> 58 #include <netinet/tcp_fsm.h> 59 #include <netinet/tcp_timer.h> 60 #include <netinet/tcp_var.h> 61 #include <netinet/tcpip.h> 62 #ifdef TCPDEBUG 63 #include <netinet/tcp_debug.h> 64 #endif 65 66 static int 67 sysctl_msec_to_ticks(SYSCTL_HANDLER_ARGS) 68 { 69 int error, s, tt; 70 71 tt = *(int *)oidp->oid_arg1; 72 s = (int)((int64_t)tt * 1000 / hz); 73 74 error = sysctl_handle_int(oidp, &s, 0, req); 75 if (error || !req->newptr) 76 return (error); 77 78 tt = (int)((int64_t)s * hz / 1000); 79 if (tt < 1) 80 return (EINVAL); 81 82 *(int *)oidp->oid_arg1 = tt; 83 return (0); 84 } 85 86 int tcp_keepinit; 87 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit, CTLTYPE_INT|CTLFLAG_RW, 88 &tcp_keepinit, 0, sysctl_msec_to_ticks, "I", ""); 89 90 int tcp_keepidle; 91 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle, CTLTYPE_INT|CTLFLAG_RW, 92 &tcp_keepidle, 0, sysctl_msec_to_ticks, "I", ""); 93 94 int tcp_keepintvl; 95 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl, CTLTYPE_INT|CTLFLAG_RW, 96 &tcp_keepintvl, 0, sysctl_msec_to_ticks, "I", ""); 97 98 int tcp_delacktime; 99 SYSCTL_PROC(_net_inet_tcp, TCPCTL_DELACKTIME, delacktime, 100 CTLTYPE_INT|CTLFLAG_RW, &tcp_delacktime, 0, sysctl_msec_to_ticks, "I", 101 "Time before a delayed ACK is sent"); 102 103 int tcp_msl; 104 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl, CTLTYPE_INT|CTLFLAG_RW, 105 &tcp_msl, 0, sysctl_msec_to_ticks, "I", "Maximum segment lifetime"); 106 107 int tcp_rexmit_min; 108 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, rexmit_min, CTLTYPE_INT|CTLFLAG_RW, 109 &tcp_rexmit_min, 0, sysctl_msec_to_ticks, "I", "Minimum Retransmission Timeout"); 110 111 int tcp_rexmit_slop; 112 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, rexmit_slop, CTLTYPE_INT|CTLFLAG_RW, 113 &tcp_rexmit_slop, 0, sysctl_msec_to_ticks, "I", "Retransmission Timer Slop"); 114 115 static int always_keepalive = 1; 116 SYSCTL_INT(_net_inet_tcp, OID_AUTO, always_keepalive, CTLFLAG_RW, 117 &always_keepalive , 0, "Assume SO_KEEPALIVE on all TCP connections"); 118 119 static int tcp_keepcnt = TCPTV_KEEPCNT; 120 /* max idle probes */ 121 int tcp_maxpersistidle; 122 /* max idle time in persist */ 123 int tcp_maxidle; 124 125 /* 126 * Tcp protocol timeout routine called every 500 ms. 127 * Updates timestamps used for TCP 128 * causes finite state machine actions if timers expire. 129 */ 130 void 131 tcp_slowtimo() 132 { 133 int s; 134 135 s = splnet(); 136 tcp_maxidle = tcp_keepcnt * tcp_keepintvl; 137 splx(s); 138 INP_INFO_WLOCK(&tcbinfo); 139 (void) tcp_timer_2msl_tw(0); 140 INP_INFO_WUNLOCK(&tcbinfo); 141 } 142 143 /* 144 * Cancel all timers for TCP tp. 145 */ 146 void 147 tcp_canceltimers(tp) 148 struct tcpcb *tp; 149 { 150 callout_stop(tp->tt_2msl); 151 callout_stop(tp->tt_persist); 152 callout_stop(tp->tt_keep); 153 callout_stop(tp->tt_rexmt); 154 } 155 156 int tcp_syn_backoff[TCP_MAXRXTSHIFT + 1] = 157 { 1, 1, 1, 1, 1, 2, 4, 8, 16, 32, 64, 64, 64 }; 158 159 int tcp_backoff[TCP_MAXRXTSHIFT + 1] = 160 { 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 512, 512, 512 }; 161 162 static int tcp_totbackoff = 2559; /* sum of tcp_backoff[] */ 163 164 /* 165 * TCP timer processing. 166 */ 167 168 void 169 tcp_timer_delack(xtp) 170 void *xtp; 171 { 172 struct tcpcb *tp = xtp; 173 int s; 174 struct inpcb *inp; 175 176 s = splnet(); 177 INP_INFO_RLOCK(&tcbinfo); 178 inp = tp->t_inpcb; 179 if (!inp) { 180 INP_INFO_RUNLOCK(&tcbinfo); 181 splx(s); 182 return; 183 } 184 INP_LOCK(inp); 185 INP_INFO_RUNLOCK(&tcbinfo); 186 if (callout_pending(tp->tt_delack) || !callout_active(tp->tt_delack)) { 187 INP_UNLOCK(inp); 188 splx(s); 189 return; 190 } 191 callout_deactivate(tp->tt_delack); 192 193 tp->t_flags |= TF_ACKNOW; 194 tcpstat.tcps_delack++; 195 (void) tcp_output(tp); 196 INP_UNLOCK(inp); 197 splx(s); 198 } 199 200 void 201 tcp_timer_2msl(xtp) 202 void *xtp; 203 { 204 struct tcpcb *tp = xtp; 205 int s; 206 struct inpcb *inp; 207 #ifdef TCPDEBUG 208 int ostate; 209 210 ostate = tp->t_state; 211 #endif 212 s = splnet(); 213 INP_INFO_WLOCK(&tcbinfo); 214 inp = tp->t_inpcb; 215 if (!inp) { 216 INP_INFO_WUNLOCK(&tcbinfo); 217 splx(s); 218 return; 219 } 220 INP_LOCK(inp); 221 tcp_free_sackholes(tp); 222 if (callout_pending(tp->tt_2msl) || !callout_active(tp->tt_2msl)) { 223 INP_UNLOCK(tp->t_inpcb); 224 INP_INFO_WUNLOCK(&tcbinfo); 225 splx(s); 226 return; 227 } 228 callout_deactivate(tp->tt_2msl); 229 /* 230 * 2 MSL timeout in shutdown went off. If we're closed but 231 * still waiting for peer to close and connection has been idle 232 * too long, or if 2MSL time is up from TIME_WAIT, delete connection 233 * control block. Otherwise, check again in a bit. 234 */ 235 if (tp->t_state != TCPS_TIME_WAIT && 236 (ticks - tp->t_rcvtime) <= tcp_maxidle) 237 callout_reset(tp->tt_2msl, tcp_keepintvl, 238 tcp_timer_2msl, tp); 239 else 240 tp = tcp_close(tp); 241 242 #ifdef TCPDEBUG 243 if (tp && (tp->t_inpcb->inp_socket->so_options & SO_DEBUG)) 244 tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0, 245 PRU_SLOWTIMO); 246 #endif 247 if (tp) 248 INP_UNLOCK(inp); 249 INP_INFO_WUNLOCK(&tcbinfo); 250 splx(s); 251 } 252 253 struct twlist { 254 LIST_HEAD(, tcptw) tw_list; 255 struct tcptw tw_tail; 256 }; 257 #define TWLIST_NLISTS 2 258 static struct twlist twl_2msl[TWLIST_NLISTS]; 259 static struct twlist *tw_2msl_list[] = { &twl_2msl[0], &twl_2msl[1], NULL }; 260 261 void 262 tcp_timer_init(void) 263 { 264 int i; 265 struct twlist *twl; 266 267 for (i = 0; i < TWLIST_NLISTS; i++) { 268 twl = &twl_2msl[i]; 269 LIST_INIT(&twl->tw_list); 270 LIST_INSERT_HEAD(&twl->tw_list, &twl->tw_tail, tw_2msl); 271 } 272 } 273 274 void 275 tcp_timer_2msl_reset(struct tcptw *tw, int timeo) 276 { 277 int i; 278 struct tcptw *tw_tail; 279 280 if (tw->tw_time != 0) 281 LIST_REMOVE(tw, tw_2msl); 282 tw->tw_time = timeo + ticks; 283 i = timeo > tcp_msl ? 1 : 0; 284 tw_tail = &twl_2msl[i].tw_tail; 285 LIST_INSERT_BEFORE(tw_tail, tw, tw_2msl); 286 } 287 288 void 289 tcp_timer_2msl_stop(struct tcptw *tw) 290 { 291 292 if (tw->tw_time != 0) 293 LIST_REMOVE(tw, tw_2msl); 294 } 295 296 struct tcptw * 297 tcp_timer_2msl_tw(int reuse) 298 { 299 struct tcptw *tw, *tw_tail; 300 struct twlist *twl; 301 int i; 302 303 for (i = 0; i < 2; i++) { 304 twl = tw_2msl_list[i]; 305 tw_tail = &twl->tw_tail; 306 for (;;) { 307 tw = LIST_FIRST(&twl->tw_list); 308 if (tw == tw_tail || (!reuse && tw->tw_time > ticks)) 309 break; 310 INP_LOCK(tw->tw_inpcb); 311 if (tcp_twclose(tw, reuse) != NULL) 312 return (tw); 313 } 314 } 315 return (NULL); 316 } 317 318 void 319 tcp_timer_keep(xtp) 320 void *xtp; 321 { 322 struct tcpcb *tp = xtp; 323 struct tcptemp *t_template; 324 int s; 325 struct inpcb *inp; 326 #ifdef TCPDEBUG 327 int ostate; 328 329 ostate = tp->t_state; 330 #endif 331 s = splnet(); 332 INP_INFO_WLOCK(&tcbinfo); 333 inp = tp->t_inpcb; 334 if (!inp) { 335 INP_INFO_WUNLOCK(&tcbinfo); 336 splx(s); 337 return; 338 } 339 INP_LOCK(inp); 340 if (callout_pending(tp->tt_keep) || !callout_active(tp->tt_keep)) { 341 INP_UNLOCK(inp); 342 INP_INFO_WUNLOCK(&tcbinfo); 343 splx(s); 344 return; 345 } 346 callout_deactivate(tp->tt_keep); 347 /* 348 * Keep-alive timer went off; send something 349 * or drop connection if idle for too long. 350 */ 351 tcpstat.tcps_keeptimeo++; 352 if (tp->t_state < TCPS_ESTABLISHED) 353 goto dropit; 354 if ((always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) && 355 tp->t_state <= TCPS_CLOSING) { 356 if ((ticks - tp->t_rcvtime) >= tcp_keepidle + tcp_maxidle) 357 goto dropit; 358 /* 359 * Send a packet designed to force a response 360 * if the peer is up and reachable: 361 * either an ACK if the connection is still alive, 362 * or an RST if the peer has closed the connection 363 * due to timeout or reboot. 364 * Using sequence number tp->snd_una-1 365 * causes the transmitted zero-length segment 366 * to lie outside the receive window; 367 * by the protocol spec, this requires the 368 * correspondent TCP to respond. 369 */ 370 tcpstat.tcps_keepprobe++; 371 t_template = tcpip_maketemplate(inp); 372 if (t_template) { 373 tcp_respond(tp, t_template->tt_ipgen, 374 &t_template->tt_t, (struct mbuf *)NULL, 375 tp->rcv_nxt, tp->snd_una - 1, 0); 376 (void) m_free(dtom(t_template)); 377 } 378 callout_reset(tp->tt_keep, tcp_keepintvl, tcp_timer_keep, tp); 379 } else 380 callout_reset(tp->tt_keep, tcp_keepidle, tcp_timer_keep, tp); 381 382 #ifdef TCPDEBUG 383 if (inp->inp_socket->so_options & SO_DEBUG) 384 tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0, 385 PRU_SLOWTIMO); 386 #endif 387 INP_UNLOCK(inp); 388 INP_INFO_WUNLOCK(&tcbinfo); 389 splx(s); 390 return; 391 392 dropit: 393 tcpstat.tcps_keepdrops++; 394 tp = tcp_drop(tp, ETIMEDOUT); 395 396 #ifdef TCPDEBUG 397 if (tp && (tp->t_inpcb->inp_socket->so_options & SO_DEBUG)) 398 tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0, 399 PRU_SLOWTIMO); 400 #endif 401 if (tp) 402 INP_UNLOCK(tp->t_inpcb); 403 INP_INFO_WUNLOCK(&tcbinfo); 404 splx(s); 405 } 406 407 void 408 tcp_timer_persist(xtp) 409 void *xtp; 410 { 411 struct tcpcb *tp = xtp; 412 int s; 413 struct inpcb *inp; 414 #ifdef TCPDEBUG 415 int ostate; 416 417 ostate = tp->t_state; 418 #endif 419 s = splnet(); 420 INP_INFO_WLOCK(&tcbinfo); 421 inp = tp->t_inpcb; 422 if (!inp) { 423 INP_INFO_WUNLOCK(&tcbinfo); 424 splx(s); 425 return; 426 } 427 INP_LOCK(inp); 428 if (callout_pending(tp->tt_persist) || !callout_active(tp->tt_persist)){ 429 INP_UNLOCK(inp); 430 INP_INFO_WUNLOCK(&tcbinfo); 431 splx(s); 432 return; 433 } 434 callout_deactivate(tp->tt_persist); 435 /* 436 * Persistance timer into zero window. 437 * Force a byte to be output, if possible. 438 */ 439 tcpstat.tcps_persisttimeo++; 440 /* 441 * Hack: if the peer is dead/unreachable, we do not 442 * time out if the window is closed. After a full 443 * backoff, drop the connection if the idle time 444 * (no responses to probes) reaches the maximum 445 * backoff that we would use if retransmitting. 446 */ 447 if (tp->t_rxtshift == TCP_MAXRXTSHIFT && 448 ((ticks - tp->t_rcvtime) >= tcp_maxpersistidle || 449 (ticks - tp->t_rcvtime) >= TCP_REXMTVAL(tp) * tcp_totbackoff)) { 450 tcpstat.tcps_persistdrop++; 451 tp = tcp_drop(tp, ETIMEDOUT); 452 goto out; 453 } 454 tcp_setpersist(tp); 455 tp->t_force = 1; 456 (void) tcp_output(tp); 457 tp->t_force = 0; 458 459 out: 460 #ifdef TCPDEBUG 461 if (tp && tp->t_inpcb->inp_socket->so_options & SO_DEBUG) 462 tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0, 463 PRU_SLOWTIMO); 464 #endif 465 if (tp) 466 INP_UNLOCK(inp); 467 INP_INFO_WUNLOCK(&tcbinfo); 468 splx(s); 469 } 470 471 void 472 tcp_timer_rexmt(xtp) 473 void *xtp; 474 { 475 struct tcpcb *tp = xtp; 476 int s; 477 int rexmt; 478 int headlocked; 479 struct inpcb *inp; 480 #ifdef TCPDEBUG 481 int ostate; 482 483 ostate = tp->t_state; 484 #endif 485 s = splnet(); 486 INP_INFO_WLOCK(&tcbinfo); 487 headlocked = 1; 488 inp = tp->t_inpcb; 489 if (!inp) { 490 INP_INFO_WUNLOCK(&tcbinfo); 491 splx(s); 492 return; 493 } 494 INP_LOCK(inp); 495 if (callout_pending(tp->tt_rexmt) || !callout_active(tp->tt_rexmt)) { 496 INP_UNLOCK(inp); 497 INP_INFO_WUNLOCK(&tcbinfo); 498 splx(s); 499 return; 500 } 501 callout_deactivate(tp->tt_rexmt); 502 tcp_free_sackholes(tp); 503 /* 504 * Retransmission timer went off. Message has not 505 * been acked within retransmit interval. Back off 506 * to a longer retransmit interval and retransmit one segment. 507 */ 508 if (++tp->t_rxtshift > TCP_MAXRXTSHIFT) { 509 tp->t_rxtshift = TCP_MAXRXTSHIFT; 510 tcpstat.tcps_timeoutdrop++; 511 tp = tcp_drop(tp, tp->t_softerror ? 512 tp->t_softerror : ETIMEDOUT); 513 goto out; 514 } 515 INP_INFO_WUNLOCK(&tcbinfo); 516 headlocked = 0; 517 if (tp->t_rxtshift == 1) { 518 /* 519 * first retransmit; record ssthresh and cwnd so they can 520 * be recovered if this turns out to be a "bad" retransmit. 521 * A retransmit is considered "bad" if an ACK for this 522 * segment is received within RTT/2 interval; the assumption 523 * here is that the ACK was already in flight. See 524 * "On Estimating End-to-End Network Path Properties" by 525 * Allman and Paxson for more details. 526 */ 527 tp->snd_cwnd_prev = tp->snd_cwnd; 528 tp->snd_ssthresh_prev = tp->snd_ssthresh; 529 tp->snd_recover_prev = tp->snd_recover; 530 if (IN_FASTRECOVERY(tp)) 531 tp->t_flags |= TF_WASFRECOVERY; 532 else 533 tp->t_flags &= ~TF_WASFRECOVERY; 534 tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1)); 535 } 536 tcpstat.tcps_rexmttimeo++; 537 if (tp->t_state == TCPS_SYN_SENT) 538 rexmt = TCP_REXMTVAL(tp) * tcp_syn_backoff[tp->t_rxtshift]; 539 else 540 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift]; 541 TCPT_RANGESET(tp->t_rxtcur, rexmt, 542 tp->t_rttmin, TCPTV_REXMTMAX); 543 /* 544 * Disable rfc1323 and rfc1644 if we havn't got any response to 545 * our third SYN to work-around some broken terminal servers 546 * (most of which have hopefully been retired) that have bad VJ 547 * header compression code which trashes TCP segments containing 548 * unknown-to-them TCP options. 549 */ 550 if ((tp->t_state == TCPS_SYN_SENT) && (tp->t_rxtshift == 3)) 551 tp->t_flags &= ~(TF_REQ_SCALE|TF_REQ_TSTMP|TF_REQ_CC); 552 /* 553 * If we backed off this far, our srtt estimate is probably bogus. 554 * Clobber it so we'll take the next rtt measurement as our srtt; 555 * move the current srtt into rttvar to keep the current 556 * retransmit times until then. 557 */ 558 if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) { 559 #ifdef INET6 560 if ((tp->t_inpcb->inp_vflag & INP_IPV6) != 0) 561 in6_losing(tp->t_inpcb); 562 else 563 #endif 564 tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT); 565 tp->t_srtt = 0; 566 } 567 tp->snd_nxt = tp->snd_una; 568 tp->snd_recover = tp->snd_max; 569 /* 570 * Force a segment to be sent. 571 */ 572 tp->t_flags |= TF_ACKNOW; 573 /* 574 * If timing a segment in this window, stop the timer. 575 */ 576 tp->t_rtttime = 0; 577 /* 578 * Close the congestion window down to one segment 579 * (we'll open it by one segment for each ack we get). 580 * Since we probably have a window's worth of unacked 581 * data accumulated, this "slow start" keeps us from 582 * dumping all that data as back-to-back packets (which 583 * might overwhelm an intermediate gateway). 584 * 585 * There are two phases to the opening: Initially we 586 * open by one mss on each ack. This makes the window 587 * size increase exponentially with time. If the 588 * window is larger than the path can handle, this 589 * exponential growth results in dropped packet(s) 590 * almost immediately. To get more time between 591 * drops but still "push" the network to take advantage 592 * of improving conditions, we switch from exponential 593 * to linear window opening at some threshhold size. 594 * For a threshhold, we use half the current window 595 * size, truncated to a multiple of the mss. 596 * 597 * (the minimum cwnd that will give us exponential 598 * growth is 2 mss. We don't allow the threshhold 599 * to go below this.) 600 */ 601 { 602 u_int win = min(tp->snd_wnd, tp->snd_cwnd) / 2 / tp->t_maxseg; 603 if (win < 2) 604 win = 2; 605 tp->snd_cwnd = tp->t_maxseg; 606 tp->snd_ssthresh = win * tp->t_maxseg; 607 tp->t_dupacks = 0; 608 } 609 EXIT_FASTRECOVERY(tp); 610 (void) tcp_output(tp); 611 612 out: 613 #ifdef TCPDEBUG 614 if (tp && (tp->t_inpcb->inp_socket->so_options & SO_DEBUG)) 615 tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0, 616 PRU_SLOWTIMO); 617 #endif 618 if (tp) 619 INP_UNLOCK(inp); 620 if (headlocked) 621 INP_INFO_WUNLOCK(&tcbinfo); 622 splx(s); 623 } 624