cxgb_sge.c (4af83c8cff85c89629e629b6becbf9712fa691c0) cxgb_sge.c (25292deb42c1c99213f6d603cf461b950691cc79)
1/**************************************************************************
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8

--- 181 unchanged lines hidden (view full) ---

190 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
191 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
192#else
193# error "SGE_NUM_GENBITS must be 1 or 2"
194#endif
195};
196
197
1/**************************************************************************
2
3Copyright (c) 2007, Chelsio Inc.
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8

--- 181 unchanged lines hidden (view full) ---

190 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
191 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
192#else
193# error "SGE_NUM_GENBITS must be 1 or 2"
194#endif
195};
196
197
198static int lro_default = 0;
199int cxgb_debug = 0;
200
201static void sge_timer_cb(void *arg);
202static void sge_timer_reclaim(void *arg, int ncount);
203static void sge_txq_reclaim_handler(void *arg, int ncount);
204
205/**
206 * reclaim_completed_tx - reclaims completed Tx descriptors

--- 1557 unchanged lines hidden (view full) ---

1764
1765 bus_dmamap_unload(q->rspq.desc_tag, q->rspq.desc_map);
1766 bus_dmamem_free(q->rspq.desc_tag, q->rspq.desc,
1767 q->rspq.desc_map);
1768 bus_dma_tag_destroy(q->rspq.desc_tag);
1769 MTX_DESTROY(&q->rspq.lock);
1770 }
1771
198int cxgb_debug = 0;
199
200static void sge_timer_cb(void *arg);
201static void sge_timer_reclaim(void *arg, int ncount);
202static void sge_txq_reclaim_handler(void *arg, int ncount);
203
204/**
205 * reclaim_completed_tx - reclaims completed Tx descriptors

--- 1557 unchanged lines hidden (view full) ---

1763
1764 bus_dmamap_unload(q->rspq.desc_tag, q->rspq.desc_map);
1765 bus_dmamem_free(q->rspq.desc_tag, q->rspq.desc,
1766 q->rspq.desc_map);
1767 bus_dma_tag_destroy(q->rspq.desc_tag);
1768 MTX_DESTROY(&q->rspq.lock);
1769 }
1770
1771 tcp_lro_free(&q->lro.ctrl);
1772
1772 bzero(q, sizeof(*q));
1773}
1774
1775/**
1776 * t3_free_sge_resources - free SGE resources
1777 * @sc: the adapter softc
1778 *
1779 * Frees resources used by the SGE queue sets.

--- 596 unchanged lines hidden (view full) ---

2376 q->fl[1].zone = zone_jumbo9;
2377 q->fl[1].type = EXT_JUMBO9;
2378 }
2379#else
2380 q->fl[1].buf_size = MJUMPAGESIZE - header_size;
2381 q->fl[1].zone = zone_jumbop;
2382 q->fl[1].type = EXT_JUMBOP;
2383#endif
1773 bzero(q, sizeof(*q));
1774}
1775
1776/**
1777 * t3_free_sge_resources - free SGE resources
1778 * @sc: the adapter softc
1779 *
1780 * Frees resources used by the SGE queue sets.

--- 596 unchanged lines hidden (view full) ---

2377 q->fl[1].zone = zone_jumbo9;
2378 q->fl[1].type = EXT_JUMBO9;
2379 }
2380#else
2381 q->fl[1].buf_size = MJUMPAGESIZE - header_size;
2382 q->fl[1].zone = zone_jumbop;
2383 q->fl[1].type = EXT_JUMBOP;
2384#endif
2384 q->lro.enabled = lro_default;
2385
2385
2386 /*
2387 * We allocate and setup the lro_ctrl structure irrespective of whether
2388 * lro is available and/or enabled.
2389 */
2390 q->lro.enabled = !!(pi->ifp->if_capenable & IFCAP_LRO);
2391 ret = tcp_lro_init(&q->lro.ctrl);
2392 if (ret) {
2393 printf("error %d from tcp_lro_init\n", ret);
2394 goto err;
2395 }
2396 q->lro.ctrl.ifp = pi->ifp;
2397
2386 mtx_lock_spin(&sc->sge.reg_lock);
2387 ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx,
2388 q->rspq.phys_addr, q->rspq.size,
2389 q->fl[0].buf_size, 1, 0);
2390 if (ret) {
2391 printf("error %d from t3_sge_init_rspcntxt\n", ret);
2392 goto err_unlock;
2393 }

--- 61 unchanged lines hidden (view full) ---

2455err_unlock:
2456 mtx_unlock_spin(&sc->sge.reg_lock);
2457err:
2458 t3_free_qset(sc, q);
2459
2460 return (ret);
2461}
2462
2398 mtx_lock_spin(&sc->sge.reg_lock);
2399 ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx,
2400 q->rspq.phys_addr, q->rspq.size,
2401 q->fl[0].buf_size, 1, 0);
2402 if (ret) {
2403 printf("error %d from t3_sge_init_rspcntxt\n", ret);
2404 goto err_unlock;
2405 }

--- 61 unchanged lines hidden (view full) ---

2467err_unlock:
2468 mtx_unlock_spin(&sc->sge.reg_lock);
2469err:
2470 t3_free_qset(sc, q);
2471
2472 return (ret);
2473}
2474
2475/*
2476 * Remove CPL_RX_PKT headers from the mbuf and reduce it to a regular mbuf with
2477 * ethernet data. Hardware assistance with various checksums and any vlan tag
2478 * will also be taken into account here.
2479 */
2463void
2464t3_rx_eth(struct adapter *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad)
2465{
2466 struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + ethpad);
2467 struct port_info *pi = &adap->port[adap->rxpkt_map[cpl->iff]];
2468 struct ifnet *ifp = pi->ifp;
2469
2470 DPRINTF("rx_eth m=%p m->m_data=%p p->iff=%d\n", m, mtod(m, uint8_t *), cpl->iff);

--- 21 unchanged lines hidden (view full) ---

2492 m_explode(m);
2493#endif
2494 /*
2495 * adjust after conversion to mbuf chain
2496 */
2497 m->m_pkthdr.len -= (sizeof(*cpl) + ethpad);
2498 m->m_len -= (sizeof(*cpl) + ethpad);
2499 m->m_data += (sizeof(*cpl) + ethpad);
2480void
2481t3_rx_eth(struct adapter *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad)
2482{
2483 struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + ethpad);
2484 struct port_info *pi = &adap->port[adap->rxpkt_map[cpl->iff]];
2485 struct ifnet *ifp = pi->ifp;
2486
2487 DPRINTF("rx_eth m=%p m->m_data=%p p->iff=%d\n", m, mtod(m, uint8_t *), cpl->iff);

--- 21 unchanged lines hidden (view full) ---

2509 m_explode(m);
2510#endif
2511 /*
2512 * adjust after conversion to mbuf chain
2513 */
2514 m->m_pkthdr.len -= (sizeof(*cpl) + ethpad);
2515 m->m_len -= (sizeof(*cpl) + ethpad);
2516 m->m_data += (sizeof(*cpl) + ethpad);
2500
2501 (*ifp->if_input)(ifp, m);
2502}
2503
2504static void
2505ext_free_handler(void *arg1, void * arg2)
2506{
2507 uintptr_t type = (uintptr_t)arg2;
2508 uma_zone_t zone;
2509 struct mbuf *m;

--- 269 unchanged lines hidden (view full) ---

2779 */
2780int
2781process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
2782{
2783 struct sge_rspq *rspq = &qs->rspq;
2784 struct rsp_desc *r = &rspq->desc[rspq->cidx];
2785 int budget_left = budget;
2786 unsigned int sleeping = 0;
2517}
2518
2519static void
2520ext_free_handler(void *arg1, void * arg2)
2521{
2522 uintptr_t type = (uintptr_t)arg2;
2523 uma_zone_t zone;
2524 struct mbuf *m;

--- 269 unchanged lines hidden (view full) ---

2794 */
2795int
2796process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
2797{
2798 struct sge_rspq *rspq = &qs->rspq;
2799 struct rsp_desc *r = &rspq->desc[rspq->cidx];
2800 int budget_left = budget;
2801 unsigned int sleeping = 0;
2787 int lro = qs->lro.enabled;
2802 int lro_enabled = qs->lro.enabled;
2803 struct lro_ctrl *lro_ctrl = &qs->lro.ctrl;
2788 struct mbuf *offload_mbufs[RX_BUNDLE_SIZE];
2789 int ngathered = 0;
2790#ifdef DEBUG
2791 static int last_holdoff = 0;
2792 if (cxgb_debug && rspq->holdoff_tmr != last_holdoff) {
2793 printf("next_holdoff=%d\n", rspq->holdoff_tmr);
2794 last_holdoff = rspq->holdoff_tmr;
2795 }

--- 96 unchanged lines hidden (view full) ---

2892
2893
2894 ngathered = rx_offload(&adap->tdev, rspq,
2895 rspq->rspq_mh.mh_head, offload_mbufs, ngathered);
2896 rspq->rspq_mh.mh_head = NULL;
2897 DPRINTF("received offload packet\n");
2898
2899 } else if (eth && eop) {
2804 struct mbuf *offload_mbufs[RX_BUNDLE_SIZE];
2805 int ngathered = 0;
2806#ifdef DEBUG
2807 static int last_holdoff = 0;
2808 if (cxgb_debug && rspq->holdoff_tmr != last_holdoff) {
2809 printf("next_holdoff=%d\n", rspq->holdoff_tmr);
2810 last_holdoff = rspq->holdoff_tmr;
2811 }

--- 96 unchanged lines hidden (view full) ---

2908
2909
2910 ngathered = rx_offload(&adap->tdev, rspq,
2911 rspq->rspq_mh.mh_head, offload_mbufs, ngathered);
2912 rspq->rspq_mh.mh_head = NULL;
2913 DPRINTF("received offload packet\n");
2914
2915 } else if (eth && eop) {
2900 prefetch(mtod(rspq->rspq_mh.mh_head, uint8_t *));
2901 prefetch(mtod(rspq->rspq_mh.mh_head, uint8_t *) + L1_CACHE_BYTES);
2916 struct mbuf *m = rspq->rspq_mh.mh_head;
2917 prefetch(mtod(m, uint8_t *));
2918 prefetch(mtod(m, uint8_t *) + L1_CACHE_BYTES);
2902
2919
2903 t3_rx_eth_lro(adap, rspq, rspq->rspq_mh.mh_head, ethpad,
2904 rss_hash, rss_csum, lro);
2920 t3_rx_eth(adap, rspq, m, ethpad);
2921 if (lro_enabled && lro_ctrl->lro_cnt &&
2922 (tcp_lro_rx(lro_ctrl, m, 0) == 0)) {
2923 /* successfully queue'd for LRO */
2924 } else {
2925 /*
2926 * LRO not enabled, packet unsuitable for LRO,
2927 * or unable to queue. Pass it up right now in
2928 * either case.
2929 */
2930 struct ifnet *ifp = m->m_pkthdr.rcvif;
2931 (*ifp->if_input)(ifp, m);
2932 }
2905 DPRINTF("received tunnel packet\n");
2933 DPRINTF("received tunnel packet\n");
2906 rspq->rspq_mh.mh_head = NULL;
2934 rspq->rspq_mh.mh_head = NULL;
2907
2908 }
2909 __refill_fl_lt(adap, &qs->fl[0], 32);
2910 __refill_fl_lt(adap, &qs->fl[1], 32);
2911 --budget_left;
2912 }
2913
2914 deliver_partial_bundle(&adap->tdev, rspq, offload_mbufs, ngathered);
2935
2936 }
2937 __refill_fl_lt(adap, &qs->fl[0], 32);
2938 __refill_fl_lt(adap, &qs->fl[1], 32);
2939 --budget_left;
2940 }
2941
2942 deliver_partial_bundle(&adap->tdev, rspq, offload_mbufs, ngathered);
2915 t3_lro_flush(adap, qs, &qs->lro);
2916
2943
2944 /* Flush LRO */
2945 while (!SLIST_EMPTY(&lro_ctrl->lro_active)) {
2946 struct lro_entry *queued = SLIST_FIRST(&lro_ctrl->lro_active);
2947 SLIST_REMOVE_HEAD(&lro_ctrl->lro_active, next);
2948 tcp_lro_flush(lro_ctrl, queued);
2949 }
2950
2917 if (sleeping)
2918 check_ring_db(adap, qs, sleeping);
2919
2920 smp_mb(); /* commit Tx queue processed updates */
2921 if (__predict_false(qs->txq_stopped > 1)) {
2922 printf("restarting tx on %p\n", qs);
2923
2924 restart_tx(qs);

--- 298 unchanged lines hidden (view full) ---

3223 }
3224 sbuf_finish(sb);
3225 err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1);
3226 sbuf_delete(sb);
3227 return (err);
3228}
3229
3230static int
2951 if (sleeping)
2952 check_ring_db(adap, qs, sleeping);
2953
2954 smp_mb(); /* commit Tx queue processed updates */
2955 if (__predict_false(qs->txq_stopped > 1)) {
2956 printf("restarting tx on %p\n", qs);
2957
2958 restart_tx(qs);

--- 298 unchanged lines hidden (view full) ---

3257 }
3258 sbuf_finish(sb);
3259 err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1);
3260 sbuf_delete(sb);
3261 return (err);
3262}
3263
3264static int
3231t3_lro_enable(SYSCTL_HANDLER_ARGS)
3232{
3233 adapter_t *sc;
3234 int i, j, enabled, err, nqsets = 0;
3235
3236#ifndef LRO_WORKING
3237 return (0);
3238#endif
3239 sc = arg1;
3240 enabled = sc->sge.qs[0].lro.enabled;
3241 err = sysctl_handle_int(oidp, &enabled, arg2, req);
3242
3243 if (err != 0)
3244 return (err);
3245 if (enabled == sc->sge.qs[0].lro.enabled)
3246 return (0);
3247
3248 for (i = 0; i < sc->params.nports; i++)
3249 for (j = 0; j < sc->port[i].nqsets; j++)
3250 nqsets++;
3251
3252 for (i = 0; i < nqsets; i++)
3253 sc->sge.qs[i].lro.enabled = enabled;
3254
3255 return (0);
3256}
3257
3258static int
3259t3_set_coalesce_usecs(SYSCTL_HANDLER_ARGS)
3260{
3261 adapter_t *sc = arg1;
3262 struct qset_params *qsp = &sc->params.sge.qset[0];
3263 int coalesce_usecs;
3264 struct sge_qset *qs;
3265 int i, j, err, nqsets = 0;
3266 struct mtx *lock;

--- 44 unchanged lines hidden (view full) ---

3311 ctx = device_get_sysctl_ctx(sc->dev);
3312 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
3313
3314 /* random information */
3315 SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
3316 "firmware_version",
3317 CTLFLAG_RD, &sc->fw_version,
3318 0, "firmware version");
3265t3_set_coalesce_usecs(SYSCTL_HANDLER_ARGS)
3266{
3267 adapter_t *sc = arg1;
3268 struct qset_params *qsp = &sc->params.sge.qset[0];
3269 int coalesce_usecs;
3270 struct sge_qset *qs;
3271 int i, j, err, nqsets = 0;
3272 struct mtx *lock;

--- 44 unchanged lines hidden (view full) ---

3317 ctx = device_get_sysctl_ctx(sc->dev);
3318 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
3319
3320 /* random information */
3321 SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
3322 "firmware_version",
3323 CTLFLAG_RD, &sc->fw_version,
3324 0, "firmware version");
3319
3320 SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
3321 "enable_lro",
3322 CTLTYPE_INT|CTLFLAG_RW, sc,
3323 0, t3_lro_enable,
3324 "I", "enable large receive offload");
3325 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
3326 "hw_revision",
3327 CTLFLAG_RD, &sc->params.rev,
3328 0, "chip model");
3329 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
3330 "enable_debug",
3331 CTLFLAG_RW, &cxgb_debug,
3332 0, "enable verbose debugging output");

--- 235 unchanged lines hidden --- 		3325 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
3326 "hw_revision",
3327 CTLFLAG_RD, &sc->params.rev,
3328 0, "chip model");
3329 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
3330 "enable_debug",
3331 CTLFLAG_RW, &cxgb_debug,
3332 0, "enable verbose debugging output");

--- 235 unchanged lines hidden ---