1 /*- 2 * Copyright (c) 2012 Adrian Chadd <adrian@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer, 10 * without modification. 11 * 2. Redistributions in binary form must reproduce at minimum a disclaimer 12 * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any 13 * redistribution must be conditioned upon including a substantially 14 * similar Disclaimer requirement for further binary redistribution. 15 * 16 * NO WARRANTY 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY 20 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, 22 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 27 * THE POSSIBILITY OF SUCH DAMAGES. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 /* 34 * Driver for the Atheros Wireless LAN controller. 35 * 36 * This software is derived from work of Atsushi Onoe; his contribution 37 * is greatly appreciated. 38 */ 39 40 #include "opt_inet.h" 41 #include "opt_ath.h" 42 /* 43 * This is needed for register operations which are performed 44 * by the driver - eg, calls to ath_hal_gettsf32(). 45 * 46 * It's also required for any AH_DEBUG checks in here, eg the 47 * module dependencies. 48 */ 49 #include "opt_ah.h" 50 #include "opt_wlan.h" 51 52 #include <sys/param.h> 53 #include <sys/systm.h> 54 #include <sys/sysctl.h> 55 #include <sys/mbuf.h> 56 #include <sys/malloc.h> 57 #include <sys/lock.h> 58 #include <sys/mutex.h> 59 #include <sys/kernel.h> 60 #include <sys/socket.h> 61 #include <sys/sockio.h> 62 #include <sys/errno.h> 63 #include <sys/callout.h> 64 #include <sys/bus.h> 65 #include <sys/endian.h> 66 #include <sys/kthread.h> 67 #include <sys/taskqueue.h> 68 #include <sys/priv.h> 69 #include <sys/module.h> 70 #include <sys/ktr.h> 71 #include <sys/smp.h> /* for mp_ncpus */ 72 73 #include <machine/bus.h> 74 75 #include <net/if.h> 76 #include <net/if_dl.h> 77 #include <net/if_media.h> 78 #include <net/if_types.h> 79 #include <net/if_arp.h> 80 #include <net/ethernet.h> 81 #include <net/if_llc.h> 82 83 #include <net80211/ieee80211_var.h> 84 #include <net80211/ieee80211_regdomain.h> 85 #ifdef IEEE80211_SUPPORT_SUPERG 86 #include <net80211/ieee80211_superg.h> 87 #endif 88 #ifdef IEEE80211_SUPPORT_TDMA 89 #include <net80211/ieee80211_tdma.h> 90 #endif 91 92 #include <net/bpf.h> 93 94 #ifdef INET 95 #include <netinet/in.h> 96 #include <netinet/if_ether.h> 97 #endif 98 99 #include <dev/ath/if_athvar.h> 100 #include <dev/ath/ath_hal/ah_devid.h> /* XXX for softled */ 101 #include <dev/ath/ath_hal/ah_diagcodes.h> 102 103 #include <dev/ath/if_ath_debug.h> 104 #include <dev/ath/if_ath_misc.h> 105 #include <dev/ath/if_ath_tsf.h> 106 #include <dev/ath/if_ath_tx.h> 107 #include <dev/ath/if_ath_sysctl.h> 108 #include <dev/ath/if_ath_led.h> 109 #include <dev/ath/if_ath_keycache.h> 110 #include <dev/ath/if_ath_rx.h> 111 #include <dev/ath/if_ath_beacon.h> 112 #include <dev/ath/if_athdfs.h> 113 114 #ifdef ATH_TX99_DIAG 115 #include <dev/ath/ath_tx99/ath_tx99.h> 116 #endif 117 118 #include <dev/ath/if_ath_tx_edma.h> 119 120 /* 121 * some general macros 122 */ 123 #define INCR(_l, _sz) (_l) ++; (_l) &= ((_sz) - 1) 124 #define DECR(_l, _sz) (_l) --; (_l) &= ((_sz) - 1) 125 126 /* 127 * XXX doesn't belong here, and should be tunable 128 */ 129 #define ATH_TXSTATUS_RING_SIZE 512 130 131 MALLOC_DECLARE(M_ATHDEV); 132 133 static void 134 ath_edma_tx_fifo_fill(struct ath_softc *sc, struct ath_txq *txq) 135 { 136 struct ath_buf *bf; 137 int i = 0; 138 139 ATH_TXQ_LOCK_ASSERT(txq); 140 141 DPRINTF(sc, ATH_DEBUG_TX_PROC, "%s: called\n", __func__); 142 143 TAILQ_FOREACH(bf, &txq->axq_q, bf_list) { 144 if (txq->axq_fifo_depth >= HAL_TXFIFO_DEPTH) 145 break; 146 ath_hal_puttxbuf(sc->sc_ah, txq->axq_qnum, bf->bf_daddr); 147 #ifdef ATH_DEBUG 148 if (sc->sc_debug & ATH_DEBUG_XMIT_DESC) 149 ath_printtxbuf(sc, bf, txq->axq_qnum, i, 0); 150 #endif 151 txq->axq_fifo_depth++; 152 i++; 153 } 154 if (i > 0) 155 ath_hal_txstart(sc->sc_ah, txq->axq_qnum); 156 } 157 158 /* 159 * Re-initialise the DMA FIFO with the current contents of 160 * said TXQ. 161 * 162 * This should only be called as part of the chip reset path, as it 163 * assumes the FIFO is currently empty. 164 * 165 * TODO: verify that a cold/warm reset does clear the TX FIFO, so 166 * writing in a partially-filled FIFO will not cause double-entries 167 * to appear. 168 */ 169 static void 170 ath_edma_dma_restart(struct ath_softc *sc, struct ath_txq *txq) 171 { 172 173 device_printf(sc->sc_dev, "%s: called: txq=%p, qnum=%d\n", 174 __func__, 175 txq, 176 txq->axq_qnum); 177 178 ATH_TXQ_LOCK_ASSERT(txq); 179 ath_edma_tx_fifo_fill(sc, txq); 180 181 } 182 183 /* 184 * Hand off this frame to a hardware queue. 185 * 186 * Things are a bit hairy in the EDMA world. The TX FIFO is only 187 * 8 entries deep, so we need to keep track of exactly what we've 188 * pushed into the FIFO and what's just sitting in the TX queue, 189 * waiting to go out. 190 * 191 * So this is split into two halves - frames get appended to the 192 * TXQ; then a scheduler is called to push some frames into the 193 * actual TX FIFO. 194 */ 195 static void 196 ath_edma_xmit_handoff_hw(struct ath_softc *sc, struct ath_txq *txq, 197 struct ath_buf *bf) 198 { 199 struct ath_hal *ah = sc->sc_ah; 200 201 ATH_TXQ_LOCK_ASSERT(txq); 202 203 KASSERT((bf->bf_flags & ATH_BUF_BUSY) == 0, 204 ("%s: busy status 0x%x", __func__, bf->bf_flags)); 205 206 /* 207 * XXX TODO: write a hard-coded check to ensure that 208 * the queue id in the TX descriptor matches txq->axq_qnum. 209 */ 210 211 /* Update aggr stats */ 212 if (bf->bf_state.bfs_aggr) 213 txq->axq_aggr_depth++; 214 215 /* Push and update frame stats */ 216 ATH_TXQ_INSERT_TAIL(txq, bf, bf_list); 217 218 /* Only schedule to the FIFO if there's space */ 219 if (txq->axq_fifo_depth < HAL_TXFIFO_DEPTH) { 220 #ifdef ATH_DEBUG 221 if (sc->sc_debug & ATH_DEBUG_XMIT_DESC) 222 ath_printtxbuf(sc, bf, txq->axq_qnum, 0, 0); 223 #endif /* ATH_DEBUG */ 224 ath_hal_puttxbuf(ah, txq->axq_qnum, bf->bf_daddr); 225 txq->axq_fifo_depth++; 226 ath_hal_txstart(ah, txq->axq_qnum); 227 } 228 } 229 230 /* 231 * Hand off this frame to a multicast software queue. 232 * 233 * Unlike legacy DMA, this doesn't chain together frames via the 234 * link pointer. Instead, they're just added to the queue. 235 * When it comes time to populate the CABQ, these frames should 236 * be individually pushed into the FIFO as appropriate. 237 * 238 * Yes, this does mean that I'll eventually have to flesh out some 239 * replacement code to handle populating the CABQ, rather than 240 * what's done in ath_beacon_generate(). It'll have to push each 241 * frame from the HW CABQ to the FIFO rather than just appending 242 * it to the existing TXQ and kicking off DMA. 243 */ 244 static void 245 ath_edma_xmit_handoff_mcast(struct ath_softc *sc, struct ath_txq *txq, 246 struct ath_buf *bf) 247 { 248 249 ATH_TXQ_LOCK_ASSERT(txq); 250 KASSERT((bf->bf_flags & ATH_BUF_BUSY) == 0, 251 ("%s: busy status 0x%x", __func__, bf->bf_flags)); 252 253 /* 254 * XXX this is mostly duplicated in ath_tx_handoff_mcast(). 255 */ 256 if (ATH_TXQ_FIRST(txq) != NULL) { 257 struct ath_buf *bf_last = ATH_TXQ_LAST(txq, axq_q_s); 258 struct ieee80211_frame *wh; 259 260 /* mark previous frame */ 261 wh = mtod(bf_last->bf_m, struct ieee80211_frame *); 262 wh->i_fc[1] |= IEEE80211_FC1_MORE_DATA; 263 264 /* sync descriptor to memory */ 265 bus_dmamap_sync(sc->sc_dmat, bf_last->bf_dmamap, 266 BUS_DMASYNC_PREWRITE); 267 } 268 269 ATH_TXQ_INSERT_TAIL(txq, bf, bf_list); 270 } 271 272 /* 273 * Handoff this frame to the hardware. 274 * 275 * For the multicast queue, this will treat it as a software queue 276 * and append it to the list, after updating the MORE_DATA flag 277 * in the previous frame. The cabq processing code will ensure 278 * that the queue contents gets transferred over. 279 * 280 * For the hardware queues, this will queue a frame to the queue 281 * like before, then populate the FIFO from that. Since the 282 * EDMA hardware has 8 FIFO slots per TXQ, this ensures that 283 * frames such as management frames don't get prematurely dropped. 284 * 285 * This does imply that a similar flush-hwq-to-fifoq method will 286 * need to be called from the processq function, before the 287 * per-node software scheduler is called. 288 */ 289 static void 290 ath_edma_xmit_handoff(struct ath_softc *sc, struct ath_txq *txq, 291 struct ath_buf *bf) 292 { 293 294 ATH_TXQ_LOCK_ASSERT(txq); 295 296 DPRINTF(sc, ATH_DEBUG_XMIT_DESC, 297 "%s: called; bf=%p, txq=%p, qnum=%d\n", 298 __func__, 299 bf, 300 txq, 301 txq->axq_qnum); 302 303 if (txq->axq_qnum == ATH_TXQ_SWQ) 304 ath_edma_xmit_handoff_mcast(sc, txq, bf); 305 else 306 ath_edma_xmit_handoff_hw(sc, txq, bf); 307 308 #if 0 309 /* 310 * XXX For now this is a placeholder; free the buffer 311 * and inform the stack that the TX failed. 312 */ 313 ath_tx_default_comp(sc, bf, 1); 314 #endif 315 } 316 317 static int 318 ath_edma_setup_txfifo(struct ath_softc *sc, int qnum) 319 { 320 struct ath_tx_edma_fifo *te = &sc->sc_txedma[qnum]; 321 322 te->m_fifo = malloc(sizeof(struct ath_buf *) * HAL_TXFIFO_DEPTH, 323 M_ATHDEV, 324 M_NOWAIT | M_ZERO); 325 if (te->m_fifo == NULL) { 326 device_printf(sc->sc_dev, "%s: malloc failed\n", 327 __func__); 328 return (-ENOMEM); 329 } 330 331 /* 332 * Set initial "empty" state. 333 */ 334 te->m_fifo_head = te->m_fifo_tail = te->m_fifo_depth = 0; 335 336 return (0); 337 } 338 339 static int 340 ath_edma_free_txfifo(struct ath_softc *sc, int qnum) 341 { 342 struct ath_tx_edma_fifo *te = &sc->sc_txedma[qnum]; 343 344 /* XXX TODO: actually deref the ath_buf entries? */ 345 free(te->m_fifo, M_ATHDEV); 346 return (0); 347 } 348 349 static int 350 ath_edma_dma_txsetup(struct ath_softc *sc) 351 { 352 int error; 353 int i; 354 355 error = ath_descdma_alloc_desc(sc, &sc->sc_txsdma, 356 NULL, "txcomp", sc->sc_tx_statuslen, ATH_TXSTATUS_RING_SIZE); 357 if (error != 0) 358 return (error); 359 360 ath_hal_setuptxstatusring(sc->sc_ah, 361 (void *) sc->sc_txsdma.dd_desc, 362 sc->sc_txsdma.dd_desc_paddr, 363 ATH_TXSTATUS_RING_SIZE); 364 365 for (i = 0; i < HAL_NUM_TX_QUEUES; i++) { 366 ath_edma_setup_txfifo(sc, i); 367 } 368 369 370 return (0); 371 } 372 373 static int 374 ath_edma_dma_txteardown(struct ath_softc *sc) 375 { 376 int i; 377 378 for (i = 0; i < HAL_NUM_TX_QUEUES; i++) { 379 ath_edma_free_txfifo(sc, i); 380 } 381 382 ath_descdma_cleanup(sc, &sc->sc_txsdma, NULL); 383 return (0); 384 } 385 386 /* 387 * Drain all TXQs, potentially after completing the existing completed 388 * frames. 389 */ 390 static void 391 ath_edma_tx_drain(struct ath_softc *sc, ATH_RESET_TYPE reset_type) 392 { 393 struct ifnet *ifp = sc->sc_ifp; 394 int i; 395 396 device_printf(sc->sc_dev, "%s: called\n", __func__); 397 398 (void) ath_stoptxdma(sc); 399 400 /* 401 * If reset type is noloss, the TX FIFO needs to be serviced 402 * and those frames need to be handled. 403 * 404 * Otherwise, just toss everything in each TX queue. 405 */ 406 407 /* XXX dump out the TX completion FIFO contents */ 408 409 /* XXX dump out the frames */ 410 411 /* XXX for now, just drain */ 412 for (i = 0; i < HAL_NUM_TX_QUEUES; i++) { 413 if (ATH_TXQ_SETUP(sc, i)) 414 ath_tx_draintxq(sc, &sc->sc_txq[i]); 415 } 416 417 IF_LOCK(&ifp->if_snd); 418 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 419 IF_UNLOCK(&ifp->if_snd); 420 sc->sc_wd_timer = 0; 421 } 422 423 /* 424 * Process the TX status queue. 425 */ 426 static void 427 ath_edma_tx_proc(void *arg, int npending) 428 { 429 struct ath_softc *sc = (struct ath_softc *) arg; 430 struct ath_hal *ah = sc->sc_ah; 431 HAL_STATUS status; 432 struct ath_tx_status ts; 433 struct ath_txq *txq; 434 struct ath_buf *bf; 435 struct ieee80211_node *ni; 436 int nacked = 0; 437 int idx; 438 439 #ifdef ATH_DEBUG 440 /* XXX */ 441 uint32_t txstatus[32]; 442 #endif 443 444 DPRINTF(sc, ATH_DEBUG_TX_PROC, "%s: called, npending=%d\n", 445 __func__, npending); 446 447 for (idx = 0; ; idx++) { 448 bzero(&ts, sizeof(ts)); 449 450 ATH_TXSTATUS_LOCK(sc); 451 status = ath_hal_txprocdesc(ah, NULL, (void *) &ts); 452 #ifdef ATH_DEBUG 453 ath_hal_gettxrawtxdesc(ah, txstatus); 454 #endif 455 ATH_TXSTATUS_UNLOCK(sc); 456 457 #ifdef ATH_DEBUG 458 if (sc->sc_debug & ATH_DEBUG_TX_PROC) 459 ath_printtxstatbuf(sc, NULL, txstatus, ts.ts_queue_id, 460 idx, (status == HAL_OK)); 461 #endif 462 463 if (status == HAL_EINPROGRESS) 464 break; 465 466 /* 467 * If there is an error with this descriptor, continue 468 * processing. 469 * 470 * XXX TBD: log some statistics? 471 */ 472 if (status == HAL_EIO) { 473 device_printf(sc->sc_dev, "%s: invalid TX status?\n", 474 __func__); 475 continue; 476 } 477 478 /* 479 * At this point we have a valid status descriptor. 480 * The QID and descriptor ID (which currently isn't set) 481 * is part of the status. 482 * 483 * We then assume that the descriptor in question is the 484 * -head- of the given QID. Eventually we should verify 485 * this by using the descriptor ID. 486 */ 487 488 /* 489 * The beacon queue is not currently a "real" queue. 490 * Frames aren't pushed onto it and the lock isn't setup. 491 * So skip it for now; the beacon handling code will 492 * free and alloc more beacon buffers as appropriate. 493 */ 494 if (ts.ts_queue_id == sc->sc_bhalq) 495 continue; 496 497 txq = &sc->sc_txq[ts.ts_queue_id]; 498 499 ATH_TXQ_LOCK(txq); 500 bf = TAILQ_FIRST(&txq->axq_q); 501 502 DPRINTF(sc, ATH_DEBUG_TX_PROC, "%s: qcuid=%d, bf=%p\n", 503 __func__, 504 ts.ts_queue_id, bf); 505 506 /* XXX TODO: actually output debugging info about this */ 507 508 #if 0 509 /* XXX assert the buffer/descriptor matches the status descid */ 510 if (ts.ts_desc_id != bf->bf_descid) { 511 device_printf(sc->sc_dev, 512 "%s: mismatched descid (qid=%d, tsdescid=%d, " 513 "bfdescid=%d\n", 514 __func__, 515 ts.ts_queue_id, 516 ts.ts_desc_id, 517 bf->bf_descid); 518 } 519 #endif 520 521 /* This removes the buffer and decrements the queue depth */ 522 ATH_TXQ_REMOVE(txq, bf, bf_list); 523 if (bf->bf_state.bfs_aggr) 524 txq->axq_aggr_depth--; 525 txq->axq_fifo_depth --; 526 /* XXX assert FIFO depth >= 0 */ 527 ATH_TXQ_UNLOCK(txq); 528 529 /* 530 * First we need to make sure ts_rate is valid. 531 * 532 * Pre-EDMA chips pass the whole TX descriptor to 533 * the proctxdesc function which will then fill out 534 * ts_rate based on the ts_finaltsi (final TX index) 535 * in the TX descriptor. However the TX completion 536 * FIFO doesn't have this information. So here we 537 * do a separate HAL call to populate that information. 538 */ 539 540 /* XXX TODO */ 541 /* XXX faked for now. Ew. */ 542 if (ts.ts_finaltsi < 4) { 543 ts.ts_rate = 544 bf->bf_state.bfs_rc[ts.ts_finaltsi].ratecode; 545 } else { 546 device_printf(sc->sc_dev, "%s: finaltsi=%d\n", 547 __func__, 548 ts.ts_finaltsi); 549 ts.ts_rate = bf->bf_state.bfs_rc[0].ratecode; 550 } 551 552 /* 553 * XXX This is terrible. 554 * 555 * Right now, some code uses the TX status that is 556 * passed in here, but the completion handlers in the 557 * software TX path also use bf_status.ds_txstat. 558 * Ew. That should all go away. 559 * 560 * XXX It's also possible the rate control completion 561 * routine is called twice. 562 */ 563 memcpy(&bf->bf_status, &ts, sizeof(ts)); 564 565 ni = bf->bf_node; 566 567 /* Update RSSI */ 568 /* XXX duplicate from ath_tx_processq */ 569 if (ni != NULL && ts.ts_status == 0 && 570 ((bf->bf_state.bfs_txflags & HAL_TXDESC_NOACK) == 0)) { 571 nacked++; 572 sc->sc_stats.ast_tx_rssi = ts.ts_rssi; 573 ATH_RSSI_LPF(sc->sc_halstats.ns_avgtxrssi, 574 ts.ts_rssi); 575 } 576 577 /* Handle frame completion and rate control update */ 578 ath_tx_process_buf_completion(sc, txq, &ts, bf); 579 580 /* bf is invalid at this point */ 581 582 /* 583 * Now that there's space in the FIFO, let's push some 584 * more frames into it. 585 * 586 * Unfortunately for now, the txq has FIFO and non-FIFO 587 * frames in the same linked list, so there's no way 588 * to quickly/easily populate frames without walking 589 * the queue and skipping 'axq_fifo_depth' frames. 590 * 591 * So for now, let's only repopulate the FIFO once it 592 * is empty. It's sucky for performance but it's enough 593 * to begin validating that things are somewhat 594 * working. 595 */ 596 ATH_TXQ_LOCK(txq); 597 if (txq->axq_fifo_depth == 0) { 598 ath_edma_tx_fifo_fill(sc, txq); 599 } 600 ATH_TXQ_UNLOCK(txq); 601 } 602 603 sc->sc_wd_timer = 0; 604 605 if (idx > 0) { 606 IF_LOCK(&sc->sc_ifp->if_snd); 607 sc->sc_ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 608 IF_UNLOCK(&sc->sc_ifp->if_snd); 609 } 610 611 /* Kick software scheduler */ 612 /* 613 * XXX It's inefficient to do this if the FIFO queue is full, 614 * but there's no easy way right now to only populate 615 * the txq task for _one_ TXQ. This should be fixed. 616 */ 617 taskqueue_enqueue(sc->sc_tq, &sc->sc_txqtask); 618 } 619 620 static void 621 ath_edma_attach_comp_func(struct ath_softc *sc) 622 { 623 624 TASK_INIT(&sc->sc_txtask, 0, ath_edma_tx_proc, sc); 625 } 626 627 void 628 ath_xmit_setup_edma(struct ath_softc *sc) 629 { 630 631 /* Fetch EDMA field and buffer sizes */ 632 (void) ath_hal_gettxdesclen(sc->sc_ah, &sc->sc_tx_desclen); 633 (void) ath_hal_gettxstatuslen(sc->sc_ah, &sc->sc_tx_statuslen); 634 (void) ath_hal_getntxmaps(sc->sc_ah, &sc->sc_tx_nmaps); 635 636 device_printf(sc->sc_dev, "TX descriptor length: %d\n", 637 sc->sc_tx_desclen); 638 device_printf(sc->sc_dev, "TX status length: %d\n", 639 sc->sc_tx_statuslen); 640 device_printf(sc->sc_dev, "TX buffers per descriptor: %d\n", 641 sc->sc_tx_nmaps); 642 643 sc->sc_tx.xmit_setup = ath_edma_dma_txsetup; 644 sc->sc_tx.xmit_teardown = ath_edma_dma_txteardown; 645 sc->sc_tx.xmit_attach_comp_func = ath_edma_attach_comp_func; 646 647 sc->sc_tx.xmit_dma_restart = ath_edma_dma_restart; 648 sc->sc_tx.xmit_handoff = ath_edma_xmit_handoff; 649 sc->sc_tx.xmit_drain = ath_edma_tx_drain; 650 } 651