1a23fd118Syl150051 /*
2a23fd118Syl150051 * CDDL HEADER START
3a23fd118Syl150051 *
4a23fd118Syl150051 * The contents of this file are subject to the terms of the
5a23fd118Syl150051 * Common Development and Distribution License (the "License").
6a23fd118Syl150051 * You may not use this file except in compliance with the License.
7a23fd118Syl150051 *
8a23fd118Syl150051 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9a23fd118Syl150051 * or http://www.opensolaris.org/os/licensing.
10a23fd118Syl150051 * See the License for the specific language governing permissions
11a23fd118Syl150051 * and limitations under the License.
12a23fd118Syl150051 *
13a23fd118Syl150051 * When distributing Covered Code, include this CDDL HEADER in each
14a23fd118Syl150051 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15a23fd118Syl150051 * If applicable, add the following below this CDDL HEADER, with the
16a23fd118Syl150051 * fields enclosed by brackets "[]" replaced with your own identifying
17a23fd118Syl150051 * information: Portions Copyright [yyyy] [name of copyright owner]
18a23fd118Syl150051 *
19a23fd118Syl150051 * CDDL HEADER END
20a23fd118Syl150051 *
218347601bSyl150051 * Copyright (c) 2002-2006 Neterion, Inc.
22a23fd118Syl150051 */
23a23fd118Syl150051
24*da14cebeSEric Cheng /*
25*da14cebeSEric Cheng * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
26*da14cebeSEric Cheng * Use is subject to license terms.
27*da14cebeSEric Cheng */
28*da14cebeSEric Cheng
29a23fd118Syl150051 #ifdef XGE_DEBUG_FP
30a23fd118Syl150051 #include "xgehal-device.h"
31a23fd118Syl150051 #endif
32a23fd118Syl150051
33a23fd118Syl150051 #include "xgehal-ring.h"
34a23fd118Syl150051 #include "xgehal-fifo.h"
35a23fd118Syl150051
36a23fd118Syl150051 /**
37a23fd118Syl150051 * xge_hal_device_bar0 - Get BAR0 mapped address.
38a23fd118Syl150051 * @hldev: HAL device handle.
39a23fd118Syl150051 *
40a23fd118Syl150051 * Returns: BAR0 address of the specified device.
41a23fd118Syl150051 */
42a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char *
xge_hal_device_bar0(xge_hal_device_t * hldev)43a23fd118Syl150051 xge_hal_device_bar0(xge_hal_device_t *hldev)
44a23fd118Syl150051 {
45a23fd118Syl150051 return hldev->bar0;
46a23fd118Syl150051 }
47a23fd118Syl150051
48a23fd118Syl150051 /**
49a23fd118Syl150051 * xge_hal_device_isrbar0 - Get BAR0 mapped address.
50a23fd118Syl150051 * @hldev: HAL device handle.
51a23fd118Syl150051 *
52a23fd118Syl150051 * Returns: BAR0 address of the specified device.
53a23fd118Syl150051 */
54a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char *
xge_hal_device_isrbar0(xge_hal_device_t * hldev)55a23fd118Syl150051 xge_hal_device_isrbar0(xge_hal_device_t *hldev)
56a23fd118Syl150051 {
57a23fd118Syl150051 return hldev->isrbar0;
58a23fd118Syl150051 }
59a23fd118Syl150051
60a23fd118Syl150051 /**
61a23fd118Syl150051 * xge_hal_device_bar1 - Get BAR1 mapped address.
62a23fd118Syl150051 * @hldev: HAL device handle.
63a23fd118Syl150051 *
64a23fd118Syl150051 * Returns: BAR1 address of the specified device.
65a23fd118Syl150051 */
66a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char *
xge_hal_device_bar1(xge_hal_device_t * hldev)67a23fd118Syl150051 xge_hal_device_bar1(xge_hal_device_t *hldev)
68a23fd118Syl150051 {
69a23fd118Syl150051 return hldev->bar1;
70a23fd118Syl150051 }
71a23fd118Syl150051
72a23fd118Syl150051 /**
73a23fd118Syl150051 * xge_hal_device_bar0_set - Set BAR0 mapped address.
74a23fd118Syl150051 * @hldev: HAL device handle.
75a23fd118Syl150051 * @bar0: BAR0 mapped address.
76a23fd118Syl150051 * * Set BAR0 address in the HAL device object.
77a23fd118Syl150051 */
78a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_bar0_set(xge_hal_device_t * hldev,char * bar0)79a23fd118Syl150051 xge_hal_device_bar0_set(xge_hal_device_t *hldev, char *bar0)
80a23fd118Syl150051 {
81a23fd118Syl150051 xge_assert(bar0);
82a23fd118Syl150051 hldev->bar0 = bar0;
83a23fd118Syl150051 }
84a23fd118Syl150051
85a23fd118Syl150051 /**
86a23fd118Syl150051 * xge_hal_device_isrbar0_set - Set BAR0 mapped address.
87a23fd118Syl150051 * @hldev: HAL device handle.
88a23fd118Syl150051 * @isrbar0: BAR0 mapped address.
89a23fd118Syl150051 * * Set BAR0 address in the HAL device object.
90a23fd118Syl150051 */
91a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_isrbar0_set(xge_hal_device_t * hldev,char * isrbar0)92a23fd118Syl150051 xge_hal_device_isrbar0_set(xge_hal_device_t *hldev, char *isrbar0)
93a23fd118Syl150051 {
94a23fd118Syl150051 xge_assert(isrbar0);
95a23fd118Syl150051 hldev->isrbar0 = isrbar0;
96a23fd118Syl150051 }
97a23fd118Syl150051
98a23fd118Syl150051 /**
99a23fd118Syl150051 * xge_hal_device_bar1_set - Set BAR1 mapped address.
100a23fd118Syl150051 * @hldev: HAL device handle.
101a23fd118Syl150051 * @channelh: Channel handle.
102a23fd118Syl150051 * @bar1: BAR1 mapped address.
103a23fd118Syl150051 *
104a23fd118Syl150051 * Set BAR1 address for the given channel.
105a23fd118Syl150051 */
106a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_bar1_set(xge_hal_device_t * hldev,xge_hal_channel_h channelh,char * bar1)107a23fd118Syl150051 xge_hal_device_bar1_set(xge_hal_device_t *hldev, xge_hal_channel_h channelh,
108a23fd118Syl150051 char *bar1)
109a23fd118Syl150051 {
110a23fd118Syl150051 xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
111a23fd118Syl150051
112a23fd118Syl150051 xge_assert(bar1);
113a23fd118Syl150051 xge_assert(fifo);
114a23fd118Syl150051
115a23fd118Syl150051 /* Initializing the BAR1 address as the start of
116a23fd118Syl150051 * the FIFO queue pointer and as a location of FIFO control
117a23fd118Syl150051 * word. */
118a23fd118Syl150051 fifo->hw_pair =
119a23fd118Syl150051 (xge_hal_fifo_hw_pair_t *) (bar1 +
120a23fd118Syl150051 (fifo->channel.post_qid * XGE_HAL_FIFO_HW_PAIR_OFFSET));
121a23fd118Syl150051 hldev->bar1 = bar1;
122a23fd118Syl150051 }
123a23fd118Syl150051
124a23fd118Syl150051
125a23fd118Syl150051 /**
126a23fd118Syl150051 * xge_hal_device_rev - Get Device revision number.
127a23fd118Syl150051 * @hldev: HAL device handle.
128a23fd118Syl150051 *
129a23fd118Syl150051 * Returns: Device revision number
130a23fd118Syl150051 */
131a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE int
xge_hal_device_rev(xge_hal_device_t * hldev)132a23fd118Syl150051 xge_hal_device_rev(xge_hal_device_t *hldev)
133a23fd118Syl150051 {
134a23fd118Syl150051 return hldev->revision;
135a23fd118Syl150051 }
136a23fd118Syl150051
137a23fd118Syl150051
138a23fd118Syl150051 /**
139a23fd118Syl150051 * xge_hal_device_begin_irq - Begin IRQ processing.
140a23fd118Syl150051 * @hldev: HAL device handle.
141a23fd118Syl150051 * @reason: "Reason" for the interrupt, the value of Xframe's
142a23fd118Syl150051 * general_int_status register.
143a23fd118Syl150051 *
144a23fd118Syl150051 * The function performs two actions, It first checks whether (shared IRQ) the
145a23fd118Syl150051 * interrupt was raised by the device. Next, it masks the device interrupts.
146a23fd118Syl150051 *
147a23fd118Syl150051 * Note:
148a23fd118Syl150051 * xge_hal_device_begin_irq() does not flush MMIO writes through the
149a23fd118Syl150051 * bridge. Therefore, two back-to-back interrupts are potentially possible.
150a23fd118Syl150051 * It is the responsibility of the ULD to make sure that only one
151a23fd118Syl150051 * xge_hal_device_continue_irq() runs at a time.
152a23fd118Syl150051 *
153a23fd118Syl150051 * Returns: 0, if the interrupt is not "ours" (note that in this case the
154a23fd118Syl150051 * device remain enabled).
155a23fd118Syl150051 * Otherwise, xge_hal_device_begin_irq() returns 64bit general adapter
156a23fd118Syl150051 * status.
157a23fd118Syl150051 * See also: xge_hal_device_handle_irq()
158a23fd118Syl150051 */
159a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_begin_irq(xge_hal_device_t * hldev,u64 * reason)160a23fd118Syl150051 xge_hal_device_begin_irq(xge_hal_device_t *hldev, u64 *reason)
161a23fd118Syl150051 {
162a23fd118Syl150051 u64 val64;
163a23fd118Syl150051 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
164a23fd118Syl150051
165a23fd118Syl150051 hldev->stats.sw_dev_info_stats.total_intr_cnt++;
166a23fd118Syl150051
167a23fd118Syl150051 val64 = xge_os_pio_mem_read64(hldev->pdev,
168a23fd118Syl150051 hldev->regh0, &isrbar0->general_int_status);
169a23fd118Syl150051 if (xge_os_unlikely(!val64)) {
170a23fd118Syl150051 /* not Xframe interrupt */
1717eced415Sxw161283 hldev->stats.sw_dev_info_stats.not_xge_intr_cnt++;
172a23fd118Syl150051 *reason = 0;
173a23fd118Syl150051 return XGE_HAL_ERR_WRONG_IRQ;
174a23fd118Syl150051 }
175a23fd118Syl150051
176a23fd118Syl150051 if (xge_os_unlikely(val64 == XGE_HAL_ALL_FOXES)) {
177a23fd118Syl150051 u64 adapter_status =
178a23fd118Syl150051 xge_os_pio_mem_read64(hldev->pdev, hldev->regh0,
179a23fd118Syl150051 &isrbar0->adapter_status);
180a23fd118Syl150051 if (adapter_status == XGE_HAL_ALL_FOXES) {
181a23fd118Syl150051 (void) xge_queue_produce(hldev->queueh,
182a23fd118Syl150051 XGE_HAL_EVENT_SLOT_FREEZE,
183a23fd118Syl150051 hldev,
184a23fd118Syl150051 1, /* critical: slot freeze */
185a23fd118Syl150051 sizeof(u64),
186a23fd118Syl150051 (void*)&adapter_status);
187a23fd118Syl150051 *reason = 0;
188a23fd118Syl150051 return XGE_HAL_ERR_CRITICAL;
189a23fd118Syl150051 }
190a23fd118Syl150051 }
191a23fd118Syl150051
192a23fd118Syl150051 *reason = val64;
193a23fd118Syl150051
194a23fd118Syl150051 /* separate fast path, i.e. no errors */
195a23fd118Syl150051 if (val64 & XGE_HAL_GEN_INTR_RXTRAFFIC) {
196a23fd118Syl150051 hldev->stats.sw_dev_info_stats.rx_traffic_intr_cnt++;
197a23fd118Syl150051 return XGE_HAL_OK;
198a23fd118Syl150051 }
199a23fd118Syl150051 if (val64 & XGE_HAL_GEN_INTR_TXTRAFFIC) {
200a23fd118Syl150051 hldev->stats.sw_dev_info_stats.tx_traffic_intr_cnt++;
201a23fd118Syl150051 return XGE_HAL_OK;
202a23fd118Syl150051 }
203a23fd118Syl150051
2048347601bSyl150051 hldev->stats.sw_dev_info_stats.not_traffic_intr_cnt++;
205a23fd118Syl150051 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXPIC)) {
206a23fd118Syl150051 xge_hal_status_e status;
2078347601bSyl150051 hldev->stats.sw_dev_info_stats.txpic_intr_cnt++;
208a23fd118Syl150051 status = __hal_device_handle_txpic(hldev, val64);
209a23fd118Syl150051 if (status != XGE_HAL_OK) {
210a23fd118Syl150051 return status;
211a23fd118Syl150051 }
212a23fd118Syl150051 }
213a23fd118Syl150051
214a23fd118Syl150051 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXDMA)) {
215a23fd118Syl150051 xge_hal_status_e status;
2168347601bSyl150051 hldev->stats.sw_dev_info_stats.txdma_intr_cnt++;
217a23fd118Syl150051 status = __hal_device_handle_txdma(hldev, val64);
218a23fd118Syl150051 if (status != XGE_HAL_OK) {
219a23fd118Syl150051 return status;
220a23fd118Syl150051 }
221a23fd118Syl150051 }
222a23fd118Syl150051
223a23fd118Syl150051 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXMAC)) {
224a23fd118Syl150051 xge_hal_status_e status;
2258347601bSyl150051 hldev->stats.sw_dev_info_stats.txmac_intr_cnt++;
226a23fd118Syl150051 status = __hal_device_handle_txmac(hldev, val64);
227a23fd118Syl150051 if (status != XGE_HAL_OK) {
228a23fd118Syl150051 return status;
229a23fd118Syl150051 }
230a23fd118Syl150051 }
231a23fd118Syl150051
232a23fd118Syl150051 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXXGXS)) {
233a23fd118Syl150051 xge_hal_status_e status;
2348347601bSyl150051 hldev->stats.sw_dev_info_stats.txxgxs_intr_cnt++;
235a23fd118Syl150051 status = __hal_device_handle_txxgxs(hldev, val64);
236a23fd118Syl150051 if (status != XGE_HAL_OK) {
237a23fd118Syl150051 return status;
238a23fd118Syl150051 }
239a23fd118Syl150051 }
240a23fd118Syl150051
241a23fd118Syl150051 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXPIC)) {
242a23fd118Syl150051 xge_hal_status_e status;
2438347601bSyl150051 hldev->stats.sw_dev_info_stats.rxpic_intr_cnt++;
244a23fd118Syl150051 status = __hal_device_handle_rxpic(hldev, val64);
245a23fd118Syl150051 if (status != XGE_HAL_OK) {
246a23fd118Syl150051 return status;
247a23fd118Syl150051 }
248a23fd118Syl150051 }
249a23fd118Syl150051
250a23fd118Syl150051 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXDMA)) {
251a23fd118Syl150051 xge_hal_status_e status;
2528347601bSyl150051 hldev->stats.sw_dev_info_stats.rxdma_intr_cnt++;
253a23fd118Syl150051 status = __hal_device_handle_rxdma(hldev, val64);
254a23fd118Syl150051 if (status != XGE_HAL_OK) {
255a23fd118Syl150051 return status;
256a23fd118Syl150051 }
257a23fd118Syl150051 }
258a23fd118Syl150051
259a23fd118Syl150051 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXMAC)) {
260a23fd118Syl150051 xge_hal_status_e status;
2618347601bSyl150051 hldev->stats.sw_dev_info_stats.rxmac_intr_cnt++;
262a23fd118Syl150051 status = __hal_device_handle_rxmac(hldev, val64);
263a23fd118Syl150051 if (status != XGE_HAL_OK) {
264a23fd118Syl150051 return status;
265a23fd118Syl150051 }
266a23fd118Syl150051 }
267a23fd118Syl150051
268a23fd118Syl150051 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXXGXS)) {
269a23fd118Syl150051 xge_hal_status_e status;
2708347601bSyl150051 hldev->stats.sw_dev_info_stats.rxxgxs_intr_cnt++;
271a23fd118Syl150051 status = __hal_device_handle_rxxgxs(hldev, val64);
272a23fd118Syl150051 if (status != XGE_HAL_OK) {
273a23fd118Syl150051 return status;
274a23fd118Syl150051 }
275a23fd118Syl150051 }
276a23fd118Syl150051
277a23fd118Syl150051 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_MC)) {
278a23fd118Syl150051 xge_hal_status_e status;
2798347601bSyl150051 hldev->stats.sw_dev_info_stats.mc_intr_cnt++;
280a23fd118Syl150051 status = __hal_device_handle_mc(hldev, val64);
281a23fd118Syl150051 if (status != XGE_HAL_OK) {
282a23fd118Syl150051 return status;
283a23fd118Syl150051 }
284a23fd118Syl150051 }
285a23fd118Syl150051
286a23fd118Syl150051 return XGE_HAL_OK;
287a23fd118Syl150051 }
288a23fd118Syl150051
289a23fd118Syl150051 /**
290a23fd118Syl150051 * xge_hal_device_clear_rx - Acknowledge (that is, clear) the
291a23fd118Syl150051 * condition that has caused the RX interrupt.
292a23fd118Syl150051 * @hldev: HAL device handle.
293a23fd118Syl150051 *
294a23fd118Syl150051 * Acknowledge (that is, clear) the condition that has caused
295a23fd118Syl150051 * the Rx interrupt.
296a23fd118Syl150051 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq(),
297a23fd118Syl150051 * xge_hal_device_clear_tx(), xge_hal_device_mask_rx().
298a23fd118Syl150051 */
299a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_clear_rx(xge_hal_device_t * hldev)300a23fd118Syl150051 xge_hal_device_clear_rx(xge_hal_device_t *hldev)
301a23fd118Syl150051 {
302a23fd118Syl150051 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
303a23fd118Syl150051
304a23fd118Syl150051 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
305a23fd118Syl150051 0xFFFFFFFFFFFFFFFFULL,
306a23fd118Syl150051 &isrbar0->rx_traffic_int);
307a23fd118Syl150051 }
308a23fd118Syl150051
309a23fd118Syl150051 /**
310a23fd118Syl150051 * xge_hal_device_clear_tx - Acknowledge (that is, clear) the
311a23fd118Syl150051 * condition that has caused the TX interrupt.
312a23fd118Syl150051 * @hldev: HAL device handle.
313a23fd118Syl150051 *
314a23fd118Syl150051 * Acknowledge (that is, clear) the condition that has caused
315a23fd118Syl150051 * the Tx interrupt.
316a23fd118Syl150051 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq(),
317a23fd118Syl150051 * xge_hal_device_clear_rx(), xge_hal_device_mask_tx().
318a23fd118Syl150051 */
319a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_clear_tx(xge_hal_device_t * hldev)320a23fd118Syl150051 xge_hal_device_clear_tx(xge_hal_device_t *hldev)
321a23fd118Syl150051 {
322a23fd118Syl150051 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
323a23fd118Syl150051
324a23fd118Syl150051 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
325a23fd118Syl150051 0xFFFFFFFFFFFFFFFFULL,
326a23fd118Syl150051 &isrbar0->tx_traffic_int);
327a23fd118Syl150051 }
328a23fd118Syl150051
329a23fd118Syl150051 /**
3308347601bSyl150051 * xge_hal_device_poll_rx_channel - Poll Rx channel for completed
3318347601bSyl150051 * descriptors and process the same.
3328347601bSyl150051 * @channel: HAL channel.
3337eced415Sxw161283 * @got_rx: Buffer to return the flag set if receive interrupt is occured
3348347601bSyl150051 *
3358347601bSyl150051 * The function polls the Rx channel for the completed descriptors and calls
3368347601bSyl150051 * the upper-layer driver (ULD) via supplied completion callback.
3378347601bSyl150051 *
3388347601bSyl150051 * Returns: XGE_HAL_OK, if the polling is completed successful.
3398347601bSyl150051 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
3408347601bSyl150051 * descriptors available which are yet to be processed.
3418347601bSyl150051 *
3428347601bSyl150051 * See also: xge_hal_device_poll_tx_channel()
3438347601bSyl150051 */
3448347601bSyl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_poll_rx_channel(xge_hal_channel_t * channel,int * got_rx)3458347601bSyl150051 xge_hal_device_poll_rx_channel(xge_hal_channel_t *channel, int *got_rx)
3468347601bSyl150051 {
3478347601bSyl150051 xge_hal_status_e ret = XGE_HAL_OK;
3488347601bSyl150051 xge_hal_dtr_h first_dtrh;
3498347601bSyl150051 xge_hal_device_t *hldev = (xge_hal_device_t *)channel->devh;
3508347601bSyl150051 u8 t_code;
3518347601bSyl150051 int got_bytes;
3528347601bSyl150051
3538347601bSyl150051 /* for each opened rx channel */
3548347601bSyl150051 got_bytes = *got_rx = 0;
3558347601bSyl150051 ((xge_hal_ring_t *)channel)->cmpl_cnt = 0;
3568347601bSyl150051 channel->poll_bytes = 0;
3578347601bSyl150051 if ((ret = xge_hal_ring_dtr_next_completed (channel, &first_dtrh,
3588347601bSyl150051 &t_code)) == XGE_HAL_OK) {
3598347601bSyl150051 if (channel->callback(channel, first_dtrh,
3608347601bSyl150051 t_code, channel->userdata) != XGE_HAL_OK) {
3618347601bSyl150051 (*got_rx) += ((xge_hal_ring_t *)channel)->cmpl_cnt + 1;
3628347601bSyl150051 got_bytes += channel->poll_bytes + 1;
3638347601bSyl150051 ret = XGE_HAL_COMPLETIONS_REMAIN;
3648347601bSyl150051 } else {
3658347601bSyl150051 (*got_rx) += ((xge_hal_ring_t *)channel)->cmpl_cnt + 1;
3668347601bSyl150051 got_bytes += channel->poll_bytes + 1;
3678347601bSyl150051 }
3688347601bSyl150051 }
3698347601bSyl150051
3708347601bSyl150051 if (*got_rx) {
3718347601bSyl150051 hldev->irq_workload_rxd[channel->post_qid] += *got_rx;
3728347601bSyl150051 hldev->irq_workload_rxcnt[channel->post_qid] ++;
3738347601bSyl150051 }
3748347601bSyl150051 hldev->irq_workload_rxlen[channel->post_qid] += got_bytes;
3758347601bSyl150051
3768347601bSyl150051 return ret;
3778347601bSyl150051 }
3788347601bSyl150051
3798347601bSyl150051 /**
3808347601bSyl150051 * xge_hal_device_poll_tx_channel - Poll Tx channel for completed
3818347601bSyl150051 * descriptors and process the same.
3827eced415Sxw161283 * @channel: HAL channel.
3837eced415Sxw161283 * @got_tx: Buffer to return the flag set if transmit interrupt is occured
3848347601bSyl150051 *
3858347601bSyl150051 * The function polls the Tx channel for the completed descriptors and calls
3868347601bSyl150051 * the upper-layer driver (ULD) via supplied completion callback.
3878347601bSyl150051 *
3888347601bSyl150051 * Returns: XGE_HAL_OK, if the polling is completed successful.
3898347601bSyl150051 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
3908347601bSyl150051 * descriptors available which are yet to be processed.
3918347601bSyl150051 *
3928347601bSyl150051 * See also: xge_hal_device_poll_rx_channel().
3938347601bSyl150051 */
3948347601bSyl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_poll_tx_channel(xge_hal_channel_t * channel,int * got_tx)3958347601bSyl150051 xge_hal_device_poll_tx_channel(xge_hal_channel_t *channel, int *got_tx)
3968347601bSyl150051 {
3978347601bSyl150051 xge_hal_dtr_h first_dtrh;
3988347601bSyl150051 xge_hal_device_t *hldev = (xge_hal_device_t *)channel->devh;
3998347601bSyl150051 u8 t_code;
4008347601bSyl150051 int got_bytes;
4018347601bSyl150051
4028347601bSyl150051 /* for each opened tx channel */
4038347601bSyl150051 got_bytes = *got_tx = 0;
4048347601bSyl150051 channel->poll_bytes = 0;
4058347601bSyl150051 if (xge_hal_fifo_dtr_next_completed (channel, &first_dtrh,
4068347601bSyl150051 &t_code) == XGE_HAL_OK) {
4078347601bSyl150051 if (channel->callback(channel, first_dtrh,
4088347601bSyl150051 t_code, channel->userdata) != XGE_HAL_OK) {
4098347601bSyl150051 (*got_tx)++;
4108347601bSyl150051 got_bytes += channel->poll_bytes + 1;
4118347601bSyl150051 return XGE_HAL_COMPLETIONS_REMAIN;
4128347601bSyl150051 }
4138347601bSyl150051 (*got_tx)++;
4148347601bSyl150051 got_bytes += channel->poll_bytes + 1;
4158347601bSyl150051 }
4168347601bSyl150051
4178347601bSyl150051 if (*got_tx) {
4188347601bSyl150051 hldev->irq_workload_txd[channel->post_qid] += *got_tx;
4198347601bSyl150051 hldev->irq_workload_txcnt[channel->post_qid] ++;
4208347601bSyl150051 }
4218347601bSyl150051 hldev->irq_workload_txlen[channel->post_qid] += got_bytes;
4228347601bSyl150051
4238347601bSyl150051 return XGE_HAL_OK;
4248347601bSyl150051 }
4258347601bSyl150051
4268347601bSyl150051 /**
427a23fd118Syl150051 * xge_hal_device_poll_rx_channels - Poll Rx channels for completed
428a23fd118Syl150051 * descriptors and process the same.
429a23fd118Syl150051 * @hldev: HAL device handle.
4307eced415Sxw161283 * @got_rx: Buffer to return flag set if receive is ready
431a23fd118Syl150051 *
432a23fd118Syl150051 * The function polls the Rx channels for the completed descriptors and calls
433a23fd118Syl150051 * the upper-layer driver (ULD) via supplied completion callback.
434a23fd118Syl150051 *
435a23fd118Syl150051 * Returns: XGE_HAL_OK, if the polling is completed successful.
436a23fd118Syl150051 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
437a23fd118Syl150051 * descriptors available which are yet to be processed.
438a23fd118Syl150051 *
439a23fd118Syl150051 * See also: xge_hal_device_poll_tx_channels(), xge_hal_device_continue_irq().
440a23fd118Syl150051 */
441a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_poll_rx_channels(xge_hal_device_t * hldev,int * got_rx)4428347601bSyl150051 xge_hal_device_poll_rx_channels(xge_hal_device_t *hldev, int *got_rx)
443a23fd118Syl150051 {
444a23fd118Syl150051 xge_list_t *item;
445a23fd118Syl150051 xge_hal_channel_t *channel;
446a23fd118Syl150051
447a23fd118Syl150051 /* for each opened rx channel */
448a23fd118Syl150051 xge_list_for_each(item, &hldev->ring_channels) {
4498347601bSyl150051 if (hldev->terminating)
4508347601bSyl150051 return XGE_HAL_OK;
4518347601bSyl150051 channel = xge_container_of(item, xge_hal_channel_t, item);
452*da14cebeSEric Cheng if (!(channel->flags & XGE_HAL_CHANNEL_FLAG_USE_RX_POLLING)) {
4538347601bSyl150051 (void) xge_hal_device_poll_rx_channel(channel, got_rx);
454a23fd118Syl150051 }
455*da14cebeSEric Cheng }
456a23fd118Syl150051
457a23fd118Syl150051 return XGE_HAL_OK;
458a23fd118Syl150051 }
459a23fd118Syl150051
460a23fd118Syl150051 /**
461a23fd118Syl150051 * xge_hal_device_poll_tx_channels - Poll Tx channels for completed
462a23fd118Syl150051 * descriptors and process the same.
463a23fd118Syl150051 * @hldev: HAL device handle.
4647eced415Sxw161283 * @got_tx: Buffer to return flag set if transmit is ready
465a23fd118Syl150051 *
466a23fd118Syl150051 * The function polls the Tx channels for the completed descriptors and calls
467a23fd118Syl150051 * the upper-layer driver (ULD) via supplied completion callback.
468a23fd118Syl150051 *
469a23fd118Syl150051 * Returns: XGE_HAL_OK, if the polling is completed successful.
470a23fd118Syl150051 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
471a23fd118Syl150051 * descriptors available which are yet to be processed.
472a23fd118Syl150051 *
473a23fd118Syl150051 * See also: xge_hal_device_poll_rx_channels(), xge_hal_device_continue_irq().
474a23fd118Syl150051 */
475a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_poll_tx_channels(xge_hal_device_t * hldev,int * got_tx)4768347601bSyl150051 xge_hal_device_poll_tx_channels(xge_hal_device_t *hldev, int *got_tx)
477a23fd118Syl150051 {
478a23fd118Syl150051 xge_list_t *item;
479a23fd118Syl150051 xge_hal_channel_t *channel;
480a23fd118Syl150051
481a23fd118Syl150051 /* for each opened tx channel */
482a23fd118Syl150051 xge_list_for_each(item, &hldev->fifo_channels) {
4838347601bSyl150051 if (hldev->terminating)
4848347601bSyl150051 return XGE_HAL_OK;
4858347601bSyl150051 channel = xge_container_of(item, xge_hal_channel_t, item);
4868347601bSyl150051 (void) xge_hal_device_poll_tx_channel(channel, got_tx);
487a23fd118Syl150051 }
488a23fd118Syl150051
489a23fd118Syl150051 return XGE_HAL_OK;
490a23fd118Syl150051 }
491a23fd118Syl150051
492a23fd118Syl150051 /**
493*da14cebeSEric Cheng *
494*da14cebeSEric Cheng */
495*da14cebeSEric Cheng __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_rx_channel_enable_polling(xge_hal_channel_t * channel)496*da14cebeSEric Cheng xge_hal_device_rx_channel_enable_polling(xge_hal_channel_t *channel)
497*da14cebeSEric Cheng {
498*da14cebeSEric Cheng channel->flags |= XGE_HAL_CHANNEL_FLAG_USE_RX_POLLING;
499*da14cebeSEric Cheng }
500*da14cebeSEric Cheng
501*da14cebeSEric Cheng __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_rx_channel_disable_polling(xge_hal_channel_t * channel)502*da14cebeSEric Cheng xge_hal_device_rx_channel_disable_polling(xge_hal_channel_t *channel)
503*da14cebeSEric Cheng {
504*da14cebeSEric Cheng channel->flags &= ~XGE_HAL_CHANNEL_FLAG_USE_RX_POLLING;
505*da14cebeSEric Cheng }
506*da14cebeSEric Cheng
507*da14cebeSEric Cheng /**
508a23fd118Syl150051 * xge_hal_device_mask_tx - Mask Tx interrupts.
509a23fd118Syl150051 * @hldev: HAL device handle.
510a23fd118Syl150051 *
511a23fd118Syl150051 * Mask Tx device interrupts.
512a23fd118Syl150051 *
513a23fd118Syl150051 * See also: xge_hal_device_unmask_tx(), xge_hal_device_mask_rx(),
514a23fd118Syl150051 * xge_hal_device_clear_tx().
515a23fd118Syl150051 */
516a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_mask_tx(xge_hal_device_t * hldev)517a23fd118Syl150051 xge_hal_device_mask_tx(xge_hal_device_t *hldev)
518a23fd118Syl150051 {
519a23fd118Syl150051 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
520a23fd118Syl150051
521a23fd118Syl150051 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
522a23fd118Syl150051 0xFFFFFFFFFFFFFFFFULL,
523a23fd118Syl150051 &isrbar0->tx_traffic_mask);
524a23fd118Syl150051 }
525a23fd118Syl150051
526a23fd118Syl150051 /**
527a23fd118Syl150051 * xge_hal_device_mask_rx - Mask Rx interrupts.
528a23fd118Syl150051 * @hldev: HAL device handle.
529a23fd118Syl150051 *
530a23fd118Syl150051 * Mask Rx device interrupts.
531a23fd118Syl150051 *
532a23fd118Syl150051 * See also: xge_hal_device_unmask_rx(), xge_hal_device_mask_tx(),
533a23fd118Syl150051 * xge_hal_device_clear_rx().
534a23fd118Syl150051 */
535a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_mask_rx(xge_hal_device_t * hldev)536a23fd118Syl150051 xge_hal_device_mask_rx(xge_hal_device_t *hldev)
537a23fd118Syl150051 {
538a23fd118Syl150051 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
539a23fd118Syl150051
540a23fd118Syl150051 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
541a23fd118Syl150051 0xFFFFFFFFFFFFFFFFULL,
542a23fd118Syl150051 &isrbar0->rx_traffic_mask);
543a23fd118Syl150051 }
544a23fd118Syl150051
545a23fd118Syl150051 /**
546a23fd118Syl150051 * xge_hal_device_mask_all - Mask all device interrupts.
547a23fd118Syl150051 * @hldev: HAL device handle.
548a23fd118Syl150051 *
549a23fd118Syl150051 * Mask all device interrupts.
550a23fd118Syl150051 *
551a23fd118Syl150051 * See also: xge_hal_device_unmask_all()
552a23fd118Syl150051 */
553a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_mask_all(xge_hal_device_t * hldev)554a23fd118Syl150051 xge_hal_device_mask_all(xge_hal_device_t *hldev)
555a23fd118Syl150051 {
556a23fd118Syl150051 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
557a23fd118Syl150051
558a23fd118Syl150051 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
559a23fd118Syl150051 0xFFFFFFFFFFFFFFFFULL,
560a23fd118Syl150051 &isrbar0->general_int_mask);
561a23fd118Syl150051 }
562a23fd118Syl150051
563a23fd118Syl150051 /**
564a23fd118Syl150051 * xge_hal_device_unmask_tx - Unmask Tx interrupts.
565a23fd118Syl150051 * @hldev: HAL device handle.
566a23fd118Syl150051 *
567a23fd118Syl150051 * Unmask Tx device interrupts.
568a23fd118Syl150051 *
569a23fd118Syl150051 * See also: xge_hal_device_mask_tx(), xge_hal_device_clear_tx().
570a23fd118Syl150051 */
571a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_unmask_tx(xge_hal_device_t * hldev)572a23fd118Syl150051 xge_hal_device_unmask_tx(xge_hal_device_t *hldev)
573a23fd118Syl150051 {
574a23fd118Syl150051 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
575a23fd118Syl150051
576a23fd118Syl150051 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
577a23fd118Syl150051 0x0ULL,
578a23fd118Syl150051 &isrbar0->tx_traffic_mask);
579a23fd118Syl150051 }
580a23fd118Syl150051
581a23fd118Syl150051 /**
582a23fd118Syl150051 * xge_hal_device_unmask_rx - Unmask Rx interrupts.
583a23fd118Syl150051 * @hldev: HAL device handle.
584a23fd118Syl150051 *
585a23fd118Syl150051 * Unmask Rx device interrupts.
586a23fd118Syl150051 *
587a23fd118Syl150051 * See also: xge_hal_device_mask_rx(), xge_hal_device_clear_rx().
588a23fd118Syl150051 */
589a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_unmask_rx(xge_hal_device_t * hldev)590a23fd118Syl150051 xge_hal_device_unmask_rx(xge_hal_device_t *hldev)
591a23fd118Syl150051 {
592a23fd118Syl150051 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
593a23fd118Syl150051
594a23fd118Syl150051 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
595a23fd118Syl150051 0x0ULL,
596a23fd118Syl150051 &isrbar0->rx_traffic_mask);
597a23fd118Syl150051 }
598a23fd118Syl150051
599a23fd118Syl150051 /**
600a23fd118Syl150051 * xge_hal_device_unmask_all - Unmask all device interrupts.
601a23fd118Syl150051 * @hldev: HAL device handle.
602a23fd118Syl150051 *
603a23fd118Syl150051 * Unmask all device interrupts.
604a23fd118Syl150051 *
605a23fd118Syl150051 * See also: xge_hal_device_mask_all()
606a23fd118Syl150051 */
607a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_unmask_all(xge_hal_device_t * hldev)608a23fd118Syl150051 xge_hal_device_unmask_all(xge_hal_device_t *hldev)
609a23fd118Syl150051 {
610a23fd118Syl150051 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
611a23fd118Syl150051
612a23fd118Syl150051 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
613a23fd118Syl150051 0x0ULL,
614a23fd118Syl150051 &isrbar0->general_int_mask);
615a23fd118Syl150051 }
616a23fd118Syl150051
617a23fd118Syl150051
618a23fd118Syl150051 /**
619a23fd118Syl150051 * xge_hal_device_continue_irq - Continue handling IRQ: process all
620a23fd118Syl150051 * completed descriptors.
621a23fd118Syl150051 * @hldev: HAL device handle.
622a23fd118Syl150051 *
623a23fd118Syl150051 * Process completed descriptors and unmask the device interrupts.
624a23fd118Syl150051 *
625a23fd118Syl150051 * The xge_hal_device_continue_irq() walks all open channels
626a23fd118Syl150051 * and calls upper-layer driver (ULD) via supplied completion
627a23fd118Syl150051 * callback. Note that the completion callback is specified at channel open
628a23fd118Syl150051 * time, see xge_hal_channel_open().
629a23fd118Syl150051 *
630a23fd118Syl150051 * Note that the xge_hal_device_continue_irq is part of the _fast_ path.
631a23fd118Syl150051 * To optimize the processing, the function does _not_ check for
632a23fd118Syl150051 * errors and alarms.
633a23fd118Syl150051 *
634a23fd118Syl150051 * The latter is done in a polling fashion, via xge_hal_device_poll().
635a23fd118Syl150051 *
636a23fd118Syl150051 * Returns: XGE_HAL_OK.
637a23fd118Syl150051 *
638a23fd118Syl150051 * See also: xge_hal_device_handle_irq(), xge_hal_device_poll(),
639a23fd118Syl150051 * xge_hal_ring_dtr_next_completed(),
640a23fd118Syl150051 * xge_hal_fifo_dtr_next_completed(), xge_hal_channel_callback_f{}.
641a23fd118Syl150051 */
642a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_continue_irq(xge_hal_device_t * hldev)643a23fd118Syl150051 xge_hal_device_continue_irq(xge_hal_device_t *hldev)
644a23fd118Syl150051 {
6458347601bSyl150051 int got_rx = 1, got_tx = 1;
6468347601bSyl150051 int isr_polling_cnt = hldev->config.isr_polling_cnt;
6478347601bSyl150051 int count = 0;
648a23fd118Syl150051
6498347601bSyl150051 do
6508347601bSyl150051 {
6518347601bSyl150051 if (got_rx)
6528347601bSyl150051 (void) xge_hal_device_poll_rx_channels(hldev, &got_rx);
6538347601bSyl150051 if (got_tx && hldev->tti_enabled)
6548347601bSyl150051 (void) xge_hal_device_poll_tx_channels(hldev, &got_tx);
655a23fd118Syl150051
6568347601bSyl150051 if (!got_rx && !got_tx)
6578347601bSyl150051 break;
658a23fd118Syl150051
6598347601bSyl150051 count += (got_rx + got_tx);
6608347601bSyl150051 }while (isr_polling_cnt--);
661a23fd118Syl150051
6628347601bSyl150051 if (!count)
663a23fd118Syl150051 hldev->stats.sw_dev_info_stats.not_traffic_intr_cnt++;
664a23fd118Syl150051
665a23fd118Syl150051 return XGE_HAL_OK;
666a23fd118Syl150051 }
667a23fd118Syl150051
668a23fd118Syl150051 /**
669a23fd118Syl150051 * xge_hal_device_handle_irq - Handle device IRQ.
670a23fd118Syl150051 * @hldev: HAL device handle.
671a23fd118Syl150051 *
672a23fd118Syl150051 * Perform the complete handling of the line interrupt. The function
673a23fd118Syl150051 * performs two calls.
674a23fd118Syl150051 * First it uses xge_hal_device_begin_irq() to check the reason for
675a23fd118Syl150051 * the interrupt and mask the device interrupts.
676a23fd118Syl150051 * Second, it calls xge_hal_device_continue_irq() to process all
677a23fd118Syl150051 * completed descriptors and re-enable the interrupts.
678a23fd118Syl150051 *
679a23fd118Syl150051 * Returns: XGE_HAL_OK - success;
680a23fd118Syl150051 * XGE_HAL_ERR_WRONG_IRQ - (shared) IRQ produced by other device.
681a23fd118Syl150051 *
682a23fd118Syl150051 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq().
683a23fd118Syl150051 */
684a23fd118Syl150051 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_handle_irq(xge_hal_device_t * hldev)685a23fd118Syl150051 xge_hal_device_handle_irq(xge_hal_device_t *hldev)
686a23fd118Syl150051 {
687a23fd118Syl150051 u64 reason;
688a23fd118Syl150051 xge_hal_status_e status;
689a23fd118Syl150051
690a23fd118Syl150051 xge_hal_device_mask_all(hldev);
691a23fd118Syl150051
692a23fd118Syl150051 status = xge_hal_device_begin_irq(hldev, &reason);
693a23fd118Syl150051 if (status != XGE_HAL_OK) {
694a23fd118Syl150051 xge_hal_device_unmask_all(hldev);
695a23fd118Syl150051 return status;
696a23fd118Syl150051 }
697a23fd118Syl150051
698a23fd118Syl150051 if (reason & XGE_HAL_GEN_INTR_RXTRAFFIC) {
699a23fd118Syl150051 xge_hal_device_clear_rx(hldev);
700a23fd118Syl150051 }
701a23fd118Syl150051
702a23fd118Syl150051 status = xge_hal_device_continue_irq(hldev);
703a23fd118Syl150051
704a23fd118Syl150051 xge_hal_device_clear_tx(hldev);
705a23fd118Syl150051
706a23fd118Syl150051 xge_hal_device_unmask_all(hldev);
707a23fd118Syl150051
708a23fd118Syl150051 return status;
709a23fd118Syl150051 }
710a23fd118Syl150051
711a23fd118Syl150051 #if defined(XGE_HAL_CONFIG_LRO)
712a23fd118Syl150051
7138347601bSyl150051
7148347601bSyl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int
__hal_lro_check_for_session_match(lro_t * lro,tcplro_t * tcp,iplro_t * ip)7158347601bSyl150051 __hal_lro_check_for_session_match(lro_t *lro, tcplro_t *tcp, iplro_t *ip)
7168347601bSyl150051 {
7178347601bSyl150051
7188347601bSyl150051 /* Match Source address field */
7198347601bSyl150051 if ((lro->ip_hdr->saddr != ip->saddr))
7208347601bSyl150051 return XGE_HAL_FAIL;
7218347601bSyl150051
7228347601bSyl150051 /* Match Destination address field */
7238347601bSyl150051 if ((lro->ip_hdr->daddr != ip->daddr))
7248347601bSyl150051 return XGE_HAL_FAIL;
7258347601bSyl150051
7268347601bSyl150051 /* Match Source Port field */
7278347601bSyl150051 if ((lro->tcp_hdr->source != tcp->source))
7288347601bSyl150051 return XGE_HAL_FAIL;
7298347601bSyl150051
7308347601bSyl150051 /* Match Destination Port field */
7318347601bSyl150051 if ((lro->tcp_hdr->dest != tcp->dest))
7328347601bSyl150051 return XGE_HAL_FAIL;
7338347601bSyl150051
7348347601bSyl150051 return XGE_HAL_OK;
7358347601bSyl150051 }
7368347601bSyl150051
737a23fd118Syl150051 /*
738a23fd118Syl150051 * __hal_tcp_seg_len: Find the tcp seg len.
739a23fd118Syl150051 * @ip: ip header.
740a23fd118Syl150051 * @tcp: tcp header.
741a23fd118Syl150051 * returns: Tcp seg length.
742a23fd118Syl150051 */
743a23fd118Syl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL u16
__hal_tcp_seg_len(iplro_t * ip,tcplro_t * tcp)744a23fd118Syl150051 __hal_tcp_seg_len(iplro_t *ip, tcplro_t *tcp)
745a23fd118Syl150051 {
746a23fd118Syl150051 u16 ret;
747a23fd118Syl150051
748a23fd118Syl150051 ret = (xge_os_ntohs(ip->tot_len) -
749a23fd118Syl150051 ((ip->version_ihl & 0x0F)<<2) -
750a23fd118Syl150051 ((tcp->doff_res)>>2));
751a23fd118Syl150051 return (ret);
752a23fd118Syl150051 }
753a23fd118Syl150051
754a23fd118Syl150051 /*
755a23fd118Syl150051 * __hal_ip_lro_capable: Finds whether ip is lro capable.
756a23fd118Syl150051 * @ip: ip header.
757a23fd118Syl150051 * @ext_info: descriptor info.
758a23fd118Syl150051 */
759a23fd118Syl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_ip_lro_capable(iplro_t * ip,xge_hal_dtr_info_t * ext_info)760a23fd118Syl150051 __hal_ip_lro_capable(iplro_t *ip,
761a23fd118Syl150051 xge_hal_dtr_info_t *ext_info)
762a23fd118Syl150051 {
763a23fd118Syl150051
764a23fd118Syl150051 #ifdef XGE_LL_DEBUG_DUMP_PKT
765a23fd118Syl150051 {
766a23fd118Syl150051 u16 i;
767a23fd118Syl150051 u8 ch, *iph = (u8 *)ip;
768a23fd118Syl150051
769a23fd118Syl150051 xge_debug_ring(XGE_TRACE, "Dump Ip:" );
770a23fd118Syl150051 for (i =0; i < 40; i++) {
771a23fd118Syl150051 ch = ntohs(*((u8 *)(iph + i)) );
772a23fd118Syl150051 printf("i:%d %02x, ",i,ch);
773a23fd118Syl150051 }
774a23fd118Syl150051 }
775a23fd118Syl150051 #endif
776a23fd118Syl150051
777a23fd118Syl150051 if (ip->version_ihl != IP_FAST_PATH_HDR_MASK) {
778a23fd118Syl150051 xge_debug_ring(XGE_ERR, "iphdr !=45 :%d",ip->version_ihl);
779a23fd118Syl150051 return XGE_HAL_FAIL;
780a23fd118Syl150051 }
781a23fd118Syl150051
782a23fd118Syl150051 if (ext_info->proto & XGE_HAL_FRAME_PROTO_IP_FRAGMENTED) {
783a23fd118Syl150051 xge_debug_ring(XGE_ERR, "IP fragmented");
784a23fd118Syl150051 return XGE_HAL_FAIL;
785a23fd118Syl150051 }
786a23fd118Syl150051
787a23fd118Syl150051 return XGE_HAL_OK;
788a23fd118Syl150051 }
789a23fd118Syl150051
790a23fd118Syl150051 /*
791a23fd118Syl150051 * __hal_tcp_lro_capable: Finds whether tcp is lro capable.
792a23fd118Syl150051 * @ip: ip header.
793a23fd118Syl150051 * @tcp: tcp header.
794a23fd118Syl150051 */
795a23fd118Syl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_tcp_lro_capable(iplro_t * ip,tcplro_t * tcp,lro_t * lro,int * ts_off)7968347601bSyl150051 __hal_tcp_lro_capable(iplro_t *ip, tcplro_t *tcp, lro_t *lro, int *ts_off)
797a23fd118Syl150051 {
798a23fd118Syl150051 #ifdef XGE_LL_DEBUG_DUMP_PKT
799a23fd118Syl150051 {
800a23fd118Syl150051 u8 ch;
801a23fd118Syl150051 u16 i;
802a23fd118Syl150051
803a23fd118Syl150051 xge_debug_ring(XGE_TRACE, "Dump Tcp:" );
804a23fd118Syl150051 for (i =0; i < 20; i++) {
805a23fd118Syl150051 ch = ntohs(*((u8 *)((u8 *)tcp + i)) );
806a23fd118Syl150051 xge_os_printf("i:%d %02x, ",i,ch);
807a23fd118Syl150051 }
808a23fd118Syl150051 }
809a23fd118Syl150051 #endif
8108347601bSyl150051 if ((TCP_FAST_PATH_HDR_MASK2 != tcp->ctrl) &&
8118347601bSyl150051 (TCP_FAST_PATH_HDR_MASK3 != tcp->ctrl))
8128347601bSyl150051 goto _exit_fail;
8138347601bSyl150051
8148347601bSyl150051 *ts_off = -1;
8158347601bSyl150051 if (TCP_FAST_PATH_HDR_MASK1 != tcp->doff_res) {
8168347601bSyl150051 u16 tcp_hdr_len = tcp->doff_res >> 2; /* TCP header len */
8178347601bSyl150051 u16 off = 20; /* Start of tcp options */
8188347601bSyl150051 int i, diff;
8198347601bSyl150051
8208347601bSyl150051 /* Does Packet can contain time stamp */
8218347601bSyl150051 if (tcp_hdr_len < 32) {
8228347601bSyl150051 /*
8238347601bSyl150051 * If the session is not opened, we can consider
8248347601bSyl150051 * this packet for LRO
8258347601bSyl150051 */
8268347601bSyl150051 if (lro == NULL)
8278347601bSyl150051 return XGE_HAL_OK;
8288347601bSyl150051
8298347601bSyl150051 goto _exit_fail;
8308347601bSyl150051 }
8318347601bSyl150051
8328347601bSyl150051 /* Ignore No-operation 0x1 */
8338347601bSyl150051 while (((u8 *)tcp)[off] == 0x1)
8348347601bSyl150051 off++;
8358347601bSyl150051
8368347601bSyl150051 /* Next option == Timestamp */
8378347601bSyl150051 if (((u8 *)tcp)[off] != 0x8) {
8388347601bSyl150051 /*
8398347601bSyl150051 * If the session ie not opened, we can consider
8408347601bSyl150051 * this packet for LRO
8418347601bSyl150051 */
8428347601bSyl150051 if (lro == NULL)
8438347601bSyl150051 return XGE_HAL_OK;
8448347601bSyl150051
8458347601bSyl150051 goto _exit_fail;
8468347601bSyl150051 }
8478347601bSyl150051
8488347601bSyl150051 *ts_off = off;
8498347601bSyl150051 if (lro == NULL)
8508347601bSyl150051 return XGE_HAL_OK;
8518347601bSyl150051
8528347601bSyl150051 /*
8538347601bSyl150051 * Now the session is opened. If the LRO frame doesn't
8548347601bSyl150051 * have time stamp, we cannot consider current packet for
8558347601bSyl150051 * LRO.
8568347601bSyl150051 */
8578347601bSyl150051 if (lro->ts_off == -1) {
8587eced415Sxw161283 xge_debug_ring(XGE_ERR, "Pkt received with time stamp after session opened with no time stamp : %02x %02x", tcp->doff_res, tcp->ctrl);
859a23fd118Syl150051 return XGE_HAL_FAIL;
860a23fd118Syl150051 }
861a23fd118Syl150051
8628347601bSyl150051 /*
8638347601bSyl150051 * If the difference is greater than three, then there are
8648347601bSyl150051 * more options possible.
8658347601bSyl150051 * else, there are two cases:
8668347601bSyl150051 * case 1: remaining are padding bytes.
8678347601bSyl150051 * case 2: remaining can contain options or padding
8688347601bSyl150051 */
8698347601bSyl150051 off += ((u8 *)tcp)[off+1];
8708347601bSyl150051 diff = tcp_hdr_len - off;
8718347601bSyl150051 if (diff > 3) {
8728347601bSyl150051 /*
8738347601bSyl150051 * Probably contains more options.
8748347601bSyl150051 */
8757eced415Sxw161283 xge_debug_ring(XGE_ERR, "tcphdr not fastpth : pkt received with tcp options in addition to time stamp after the session is opened %02x %02x ", tcp->doff_res, tcp->ctrl);
8768347601bSyl150051 return XGE_HAL_FAIL;
8778347601bSyl150051 }
8788347601bSyl150051
8798347601bSyl150051 for (i = 0; i < diff; i++) {
8808347601bSyl150051 u8 byte = ((u8 *)tcp)[off+i];
8818347601bSyl150051
8828347601bSyl150051 /* Ignore No-operation 0x1 */
8838347601bSyl150051 if ((byte == 0x0) || (byte == 0x1))
8848347601bSyl150051 continue;
8857eced415Sxw161283 xge_debug_ring(XGE_ERR, "tcphdr not fastpth : pkt received with tcp options in addition to time stamp after the session is opened %02x %02x ", tcp->doff_res, tcp->ctrl);
8868347601bSyl150051 return XGE_HAL_FAIL;
8878347601bSyl150051 }
8888347601bSyl150051
8898347601bSyl150051 /*
8908347601bSyl150051 * Update the time stamp of LRO frame.
8918347601bSyl150051 */
8928347601bSyl150051 xge_os_memcpy(((char *)lro->tcp_hdr + lro->ts_off + 2),
8938347601bSyl150051 (char *)((char *)tcp + (*ts_off) + 2), 8);
8948347601bSyl150051 }
8958347601bSyl150051
896a23fd118Syl150051 return XGE_HAL_OK;
8978347601bSyl150051
8988347601bSyl150051 _exit_fail:
8997eced415Sxw161283 xge_debug_ring(XGE_TRACE, "tcphdr not fastpth %02x %02x", tcp->doff_res, tcp->ctrl);
9008347601bSyl150051 return XGE_HAL_FAIL;
9018347601bSyl150051
902a23fd118Syl150051 }
903a23fd118Syl150051
904a23fd118Syl150051 /*
905a23fd118Syl150051 * __hal_lro_capable: Finds whether frame is lro capable.
906a23fd118Syl150051 * @buffer: Ethernet frame.
907a23fd118Syl150051 * @ip: ip frame.
908a23fd118Syl150051 * @tcp: tcp frame.
909a23fd118Syl150051 * @ext_info: Descriptor info.
910a23fd118Syl150051 */
911a23fd118Syl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_lro_capable(u8 * buffer,iplro_t ** ip,tcplro_t ** tcp,xge_hal_dtr_info_t * ext_info)912a23fd118Syl150051 __hal_lro_capable( u8 *buffer,
913a23fd118Syl150051 iplro_t **ip,
914a23fd118Syl150051 tcplro_t **tcp,
9157eced415Sxw161283 xge_hal_dtr_info_t *ext_info)
916a23fd118Syl150051 {
917a23fd118Syl150051 u8 ip_off, ip_length;
918a23fd118Syl150051
919a23fd118Syl150051 if (!(ext_info->proto & XGE_HAL_FRAME_PROTO_TCP)) {
920a23fd118Syl150051 xge_debug_ring(XGE_ERR, "Cant do lro %d", ext_info->proto);
921a23fd118Syl150051 return XGE_HAL_FAIL;
922a23fd118Syl150051 }
9237eced415Sxw161283
9247eced415Sxw161283 if ( !*ip )
9257eced415Sxw161283 {
926a23fd118Syl150051 #ifdef XGE_LL_DEBUG_DUMP_PKT
927a23fd118Syl150051 {
928a23fd118Syl150051 u8 ch;
929a23fd118Syl150051 u16 i;
930a23fd118Syl150051
931a23fd118Syl150051 xge_os_printf("Dump Eth:" );
932a23fd118Syl150051 for (i =0; i < 60; i++) {
933a23fd118Syl150051 ch = ntohs(*((u8 *)(buffer + i)) );
934a23fd118Syl150051 xge_os_printf("i:%d %02x, ",i,ch);
935a23fd118Syl150051 }
936a23fd118Syl150051 }
937a23fd118Syl150051 #endif
938a23fd118Syl150051
939a23fd118Syl150051 switch (ext_info->frame) {
940a23fd118Syl150051 case XGE_HAL_FRAME_TYPE_DIX:
941a23fd118Syl150051 ip_off = XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE;
942a23fd118Syl150051 break;
943a23fd118Syl150051 case XGE_HAL_FRAME_TYPE_LLC:
944a23fd118Syl150051 ip_off = (XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE +
945a23fd118Syl150051 XGE_HAL_HEADER_802_2_SIZE);
946a23fd118Syl150051 break;
947a23fd118Syl150051 case XGE_HAL_FRAME_TYPE_SNAP:
948a23fd118Syl150051 ip_off = (XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE +
949a23fd118Syl150051 XGE_HAL_HEADER_SNAP_SIZE);
950a23fd118Syl150051 break;
951a23fd118Syl150051 default: // XGE_HAL_FRAME_TYPE_IPX, etc.
952a23fd118Syl150051 return XGE_HAL_FAIL;
953a23fd118Syl150051 }
954a23fd118Syl150051
955a23fd118Syl150051
956a23fd118Syl150051 if (ext_info->proto & XGE_HAL_FRAME_PROTO_VLAN_TAGGED) {
957a23fd118Syl150051 ip_off += XGE_HAL_HEADER_VLAN_SIZE;
958a23fd118Syl150051 }
959a23fd118Syl150051
960a23fd118Syl150051 /* Grab ip, tcp headers */
961a23fd118Syl150051 *ip = (iplro_t *)((char*)buffer + ip_off);
9627eced415Sxw161283 } /* !*ip */
963a23fd118Syl150051
964a23fd118Syl150051 ip_length = (u8)((*ip)->version_ihl & 0x0F);
965a23fd118Syl150051 ip_length = ip_length <<2;
9667eced415Sxw161283 *tcp = (tcplro_t *)((char *)*ip + ip_length);
967a23fd118Syl150051
9688347601bSyl150051 xge_debug_ring(XGE_TRACE, "ip_length:%d ip:"XGE_OS_LLXFMT
9698347601bSyl150051 " tcp:"XGE_OS_LLXFMT"", (int)ip_length,
9707eced415Sxw161283 (unsigned long long)(ulong_t)*ip, (unsigned long long)(ulong_t)*tcp);
971a23fd118Syl150051
972a23fd118Syl150051 return XGE_HAL_OK;
973a23fd118Syl150051
974a23fd118Syl150051 }
975a23fd118Syl150051
9768347601bSyl150051
9778347601bSyl150051 /*
9788347601bSyl150051 * __hal_open_lro_session: Open a new LRO session.
9798347601bSyl150051 * @buffer: Ethernet frame.
9808347601bSyl150051 * @ip: ip header.
9818347601bSyl150051 * @tcp: tcp header.
9828347601bSyl150051 * @lro: lro pointer
9838347601bSyl150051 * @ext_info: Descriptor info.
9848347601bSyl150051 * @hldev: Hal context.
9857eced415Sxw161283 * @ring_lro: LRO descriptor per rx ring.
9868347601bSyl150051 * @slot: Bucket no.
9878347601bSyl150051 * @tcp_seg_len: Length of tcp segment.
9888347601bSyl150051 * @ts_off: time stamp offset in the packet.
989a23fd118Syl150051 */
990a23fd118Syl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
__hal_open_lro_session(u8 * buffer,iplro_t * ip,tcplro_t * tcp,lro_t ** lro,xge_hal_device_t * hldev,xge_hal_lro_desc_t * ring_lro,int slot,u32 tcp_seg_len,int ts_off)9918347601bSyl150051 __hal_open_lro_session (u8 *buffer, iplro_t *ip, tcplro_t *tcp, lro_t **lro,
9927eced415Sxw161283 xge_hal_device_t *hldev, xge_hal_lro_desc_t *ring_lro, int slot,
9937eced415Sxw161283 u32 tcp_seg_len, int ts_off)
994a23fd118Syl150051 {
9958347601bSyl150051
9967eced415Sxw161283 lro_t *lro_new = &ring_lro->lro_pool[slot];
9978347601bSyl150051
9988347601bSyl150051 lro_new->in_use = 1;
9998347601bSyl150051 lro_new->ll_hdr = buffer;
10008347601bSyl150051 lro_new->ip_hdr = ip;
10018347601bSyl150051 lro_new->tcp_hdr = tcp;
10028347601bSyl150051 lro_new->tcp_next_seq_num = tcp_seg_len + xge_os_ntohl(
10038347601bSyl150051 tcp->seq);
10048347601bSyl150051 lro_new->tcp_seq_num = tcp->seq;
10058347601bSyl150051 lro_new->tcp_ack_num = tcp->ack_seq;
10068347601bSyl150051 lro_new->sg_num = 1;
10078347601bSyl150051 lro_new->total_length = xge_os_ntohs(ip->tot_len);
10088347601bSyl150051 lro_new->frags_len = 0;
10098347601bSyl150051 lro_new->ts_off = ts_off;
10108347601bSyl150051
10118347601bSyl150051 hldev->stats.sw_dev_info_stats.tot_frms_lroised++;
10128347601bSyl150051 hldev->stats.sw_dev_info_stats.tot_lro_sessions++;
10138347601bSyl150051
10147eced415Sxw161283 *lro = ring_lro->lro_recent = lro_new;
10158347601bSyl150051 return;
1016a23fd118Syl150051 }
1017a23fd118Syl150051 /*
10188347601bSyl150051 * __hal_lro_get_free_slot: Get a free LRO bucket.
10197eced415Sxw161283 * @ring_lro: LRO descriptor per ring.
1020a23fd118Syl150051 */
10218347601bSyl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int
__hal_lro_get_free_slot(xge_hal_lro_desc_t * ring_lro)10227eced415Sxw161283 __hal_lro_get_free_slot (xge_hal_lro_desc_t *ring_lro)
1023a23fd118Syl150051 {
10248347601bSyl150051 int i;
1025a23fd118Syl150051
10268347601bSyl150051 for (i = 0; i < XGE_HAL_LRO_MAX_BUCKETS; i++) {
10277eced415Sxw161283 lro_t *lro_temp = &ring_lro->lro_pool[i];
10288347601bSyl150051
10298347601bSyl150051 if (!lro_temp->in_use)
10308347601bSyl150051 return i;
10318347601bSyl150051 }
10328347601bSyl150051 return -1;
10338347601bSyl150051 }
1034a23fd118Syl150051
1035a23fd118Syl150051 /*
1036a23fd118Syl150051 * __hal_get_lro_session: Gets matching LRO session or creates one.
10377eced415Sxw161283 * @eth_hdr: Ethernet header.
1038a23fd118Syl150051 * @ip: ip header.
1039a23fd118Syl150051 * @tcp: tcp header.
1040a23fd118Syl150051 * @lro: lro pointer
1041a23fd118Syl150051 * @ext_info: Descriptor info.
1042a23fd118Syl150051 * @hldev: Hal context.
10437eced415Sxw161283 * @ring_lro: LRO descriptor per rx ring
1044a23fd118Syl150051 */
1045a23fd118Syl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_get_lro_session(u8 * eth_hdr,iplro_t * ip,tcplro_t * tcp,lro_t ** lro,xge_hal_dtr_info_t * ext_info,xge_hal_device_t * hldev,xge_hal_lro_desc_t * ring_lro,lro_t ** lro_end3)10467eced415Sxw161283 __hal_get_lro_session (u8 *eth_hdr,
1047a23fd118Syl150051 iplro_t *ip,
1048a23fd118Syl150051 tcplro_t *tcp,
1049a23fd118Syl150051 lro_t **lro,
1050a23fd118Syl150051 xge_hal_dtr_info_t *ext_info,
10518347601bSyl150051 xge_hal_device_t *hldev,
10527eced415Sxw161283 xge_hal_lro_desc_t *ring_lro,
10538347601bSyl150051 lro_t **lro_end3 /* Valid only when ret=END_3 */)
1054a23fd118Syl150051 {
10558347601bSyl150051 lro_t *lro_match;
1056a23fd118Syl150051 int i, free_slot = -1;
10578347601bSyl150051 u32 tcp_seg_len;
10588347601bSyl150051 int ts_off = -1;
1059a23fd118Syl150051
10608347601bSyl150051 *lro = lro_match = NULL;
10618347601bSyl150051 /*
10628347601bSyl150051 * Compare the incoming frame with the lro session left from the
10638347601bSyl150051 * previous call. There is a good chance that this incoming frame
10648347601bSyl150051 * matches the lro session.
10658347601bSyl150051 */
10667eced415Sxw161283 if (ring_lro->lro_recent && ring_lro->lro_recent->in_use) {
10677eced415Sxw161283 if (__hal_lro_check_for_session_match(ring_lro->lro_recent,
10688347601bSyl150051 tcp, ip)
10698347601bSyl150051 == XGE_HAL_OK)
10707eced415Sxw161283 lro_match = ring_lro->lro_recent;
10718347601bSyl150051 }
1072a23fd118Syl150051
10738347601bSyl150051 if (!lro_match) {
10748347601bSyl150051 /*
10758347601bSyl150051 * Search in the pool of LROs for the session that matches
10768347601bSyl150051 * the incoming frame.
10778347601bSyl150051 */
10788347601bSyl150051 for (i = 0; i < XGE_HAL_LRO_MAX_BUCKETS; i++) {
10797eced415Sxw161283 lro_t *lro_temp = &ring_lro->lro_pool[i];
10808347601bSyl150051
10818347601bSyl150051 if (!lro_temp->in_use) {
1082a23fd118Syl150051 if (free_slot == -1)
1083a23fd118Syl150051 free_slot = i;
1084a23fd118Syl150051 continue;
1085a23fd118Syl150051 }
1086a23fd118Syl150051
10878347601bSyl150051 if (__hal_lro_check_for_session_match(lro_temp, tcp,
10888347601bSyl150051 ip) == XGE_HAL_OK) {
10898347601bSyl150051 lro_match = lro_temp;
10908347601bSyl150051 break;
10918347601bSyl150051 }
10928347601bSyl150051 }
10938347601bSyl150051 }
1094a23fd118Syl150051
1095a23fd118Syl150051
10968347601bSyl150051 if (lro_match) {
10978347601bSyl150051 /*
10988347601bSyl150051 * Matching LRO Session found
10998347601bSyl150051 */
11008347601bSyl150051 *lro = lro_match;
1101a23fd118Syl150051
11028347601bSyl150051 if (lro_match->tcp_next_seq_num != xge_os_ntohl(tcp->seq)) {
1103a23fd118Syl150051 xge_debug_ring(XGE_ERR, "**retransmit **"
1104a23fd118Syl150051 "found***");
11058347601bSyl150051 hldev->stats.sw_dev_info_stats.lro_out_of_seq_pkt_cnt++;
1106a23fd118Syl150051 return XGE_HAL_INF_LRO_END_2;
1107a23fd118Syl150051 }
1108a23fd118Syl150051
1109a23fd118Syl150051 if (XGE_HAL_OK != __hal_ip_lro_capable(ip, ext_info))
11107eced415Sxw161283 {
1111a23fd118Syl150051 return XGE_HAL_INF_LRO_END_2;
11127eced415Sxw161283 }
1113a23fd118Syl150051
11148347601bSyl150051 if (XGE_HAL_OK != __hal_tcp_lro_capable(ip, tcp, lro_match,
11158347601bSyl150051 &ts_off)) {
11168347601bSyl150051 /*
11178347601bSyl150051 * Close the current session and open a new
11188347601bSyl150051 * LRO session with this packet,
11198347601bSyl150051 * provided it has tcp payload
11208347601bSyl150051 */
11218347601bSyl150051 tcp_seg_len = __hal_tcp_seg_len(ip, tcp);
11228347601bSyl150051 if (tcp_seg_len == 0)
11237eced415Sxw161283 {
1124a23fd118Syl150051 return XGE_HAL_INF_LRO_END_2;
11257eced415Sxw161283 }
1126a23fd118Syl150051
11278347601bSyl150051 /* Get a free bucket */
11287eced415Sxw161283 free_slot = __hal_lro_get_free_slot(ring_lro);
11298347601bSyl150051 if (free_slot == -1)
11307eced415Sxw161283 {
11318347601bSyl150051 return XGE_HAL_INF_LRO_END_2;
11327eced415Sxw161283 }
11338347601bSyl150051
11348347601bSyl150051 /*
11358347601bSyl150051 * Open a new LRO session
11368347601bSyl150051 */
11377eced415Sxw161283 __hal_open_lro_session (eth_hdr, ip, tcp, lro_end3,
11387eced415Sxw161283 hldev, ring_lro, free_slot, tcp_seg_len,
11398347601bSyl150051 ts_off);
11408347601bSyl150051
11418347601bSyl150051 return XGE_HAL_INF_LRO_END_3;
11428347601bSyl150051 }
11438347601bSyl150051
1144a23fd118Syl150051 /*
1145a23fd118Syl150051 * The frame is good, in-sequence, can be LRO-ed;
1146a23fd118Syl150051 * take its (latest) ACK - unless it is a dupack.
1147a23fd118Syl150051 * Note: to be exact need to check window size as well..
1148a23fd118Syl150051 */
11498347601bSyl150051 if (lro_match->tcp_ack_num == tcp->ack_seq &&
11508347601bSyl150051 lro_match->tcp_seq_num == tcp->seq) {
11518347601bSyl150051 hldev->stats.sw_dev_info_stats.lro_dup_pkt_cnt++;
1152a23fd118Syl150051 return XGE_HAL_INF_LRO_END_2;
11538347601bSyl150051 }
1154a23fd118Syl150051
11558347601bSyl150051 lro_match->tcp_seq_num = tcp->seq;
11568347601bSyl150051 lro_match->tcp_ack_num = tcp->ack_seq;
11578347601bSyl150051 lro_match->frags_len += __hal_tcp_seg_len(ip, tcp);
11588347601bSyl150051
11597eced415Sxw161283 ring_lro->lro_recent = lro_match;
1160a23fd118Syl150051
1161a23fd118Syl150051 return XGE_HAL_INF_LRO_CONT;
1162a23fd118Syl150051 }
1163a23fd118Syl150051
11648347601bSyl150051 /* ********** New Session ***************/
1165a23fd118Syl150051 if (free_slot == -1)
1166a23fd118Syl150051 return XGE_HAL_INF_LRO_UNCAPABLE;
1167a23fd118Syl150051
1168a23fd118Syl150051 if (XGE_HAL_FAIL == __hal_ip_lro_capable(ip, ext_info))
1169a23fd118Syl150051 return XGE_HAL_INF_LRO_UNCAPABLE;
1170a23fd118Syl150051
11718347601bSyl150051 if (XGE_HAL_FAIL == __hal_tcp_lro_capable(ip, tcp, NULL, &ts_off))
1172a23fd118Syl150051 return XGE_HAL_INF_LRO_UNCAPABLE;
1173a23fd118Syl150051
1174a23fd118Syl150051 xge_debug_ring(XGE_TRACE, "Creating lro session.");
1175a23fd118Syl150051
11768347601bSyl150051 /*
11778347601bSyl150051 * Open a LRO session, provided the packet contains payload.
11788347601bSyl150051 */
11798347601bSyl150051 tcp_seg_len = __hal_tcp_seg_len(ip, tcp);
11808347601bSyl150051 if (tcp_seg_len == 0)
11818347601bSyl150051 return XGE_HAL_INF_LRO_UNCAPABLE;
11828347601bSyl150051
11837eced415Sxw161283 __hal_open_lro_session (eth_hdr, ip, tcp, lro, hldev, ring_lro, free_slot,
11848347601bSyl150051 tcp_seg_len, ts_off);
1185a23fd118Syl150051
1186a23fd118Syl150051 return XGE_HAL_INF_LRO_BEGIN;
1187a23fd118Syl150051 }
1188a23fd118Syl150051
1189a23fd118Syl150051 /*
1190a23fd118Syl150051 * __hal_lro_under_optimal_thresh: Finds whether combined session is optimal.
1191a23fd118Syl150051 * @ip: ip header.
1192a23fd118Syl150051 * @tcp: tcp header.
1193a23fd118Syl150051 * @lro: lro pointer
1194a23fd118Syl150051 * @hldev: Hal context.
1195a23fd118Syl150051 */
1196a23fd118Syl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_lro_under_optimal_thresh(iplro_t * ip,tcplro_t * tcp,lro_t * lro,xge_hal_device_t * hldev)1197a23fd118Syl150051 __hal_lro_under_optimal_thresh (iplro_t *ip,
1198a23fd118Syl150051 tcplro_t *tcp,
1199a23fd118Syl150051 lro_t *lro,
1200a23fd118Syl150051 xge_hal_device_t *hldev)
1201a23fd118Syl150051 {
1202a23fd118Syl150051 if (!lro) return XGE_HAL_FAIL;
1203a23fd118Syl150051
1204a23fd118Syl150051 if ((lro->total_length + __hal_tcp_seg_len(ip, tcp) ) >
12058347601bSyl150051 hldev->config.lro_frm_len) {
12068347601bSyl150051 xge_debug_ring(XGE_TRACE, "Max LRO frame len exceeded:"
12077eced415Sxw161283 "max length %d ", hldev->config.lro_frm_len);
12088347601bSyl150051 hldev->stats.sw_dev_info_stats.lro_frm_len_exceed_cnt++;
1209a23fd118Syl150051 return XGE_HAL_FAIL;
1210a23fd118Syl150051 }
1211a23fd118Syl150051
12128347601bSyl150051 if (lro->sg_num == hldev->config.lro_sg_size) {
12138347601bSyl150051 xge_debug_ring(XGE_TRACE, "Max sg count exceeded:"
12147eced415Sxw161283 "max sg %d ", hldev->config.lro_sg_size);
12158347601bSyl150051 hldev->stats.sw_dev_info_stats.lro_sg_exceed_cnt++;
1216a23fd118Syl150051 return XGE_HAL_FAIL;
1217a23fd118Syl150051 }
1218a23fd118Syl150051
1219a23fd118Syl150051 return XGE_HAL_OK;
1220a23fd118Syl150051 }
1221a23fd118Syl150051
1222a23fd118Syl150051 /*
1223a23fd118Syl150051 * __hal_collapse_ip_hdr: Collapses ip header.
1224a23fd118Syl150051 * @ip: ip header.
1225a23fd118Syl150051 * @tcp: tcp header.
1226a23fd118Syl150051 * @lro: lro pointer
1227a23fd118Syl150051 * @hldev: Hal context.
1228a23fd118Syl150051 */
1229a23fd118Syl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_collapse_ip_hdr(iplro_t * ip,tcplro_t * tcp,lro_t * lro,xge_hal_device_t * hldev)1230a23fd118Syl150051 __hal_collapse_ip_hdr ( iplro_t *ip,
1231a23fd118Syl150051 tcplro_t *tcp,
1232a23fd118Syl150051 lro_t *lro,
1233a23fd118Syl150051 xge_hal_device_t *hldev)
1234a23fd118Syl150051 {
1235a23fd118Syl150051
1236a23fd118Syl150051 lro->total_length += __hal_tcp_seg_len(ip, tcp);
1237a23fd118Syl150051
1238a23fd118Syl150051 /* May be we have to handle time stamps or more options */
1239a23fd118Syl150051
1240a23fd118Syl150051 return XGE_HAL_OK;
1241a23fd118Syl150051
1242a23fd118Syl150051 }
1243a23fd118Syl150051
1244a23fd118Syl150051 /*
1245a23fd118Syl150051 * __hal_collapse_tcp_hdr: Collapses tcp header.
1246a23fd118Syl150051 * @ip: ip header.
1247a23fd118Syl150051 * @tcp: tcp header.
1248a23fd118Syl150051 * @lro: lro pointer
1249a23fd118Syl150051 * @hldev: Hal context.
1250a23fd118Syl150051 */
1251a23fd118Syl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_collapse_tcp_hdr(iplro_t * ip,tcplro_t * tcp,lro_t * lro,xge_hal_device_t * hldev)1252a23fd118Syl150051 __hal_collapse_tcp_hdr ( iplro_t *ip,
1253a23fd118Syl150051 tcplro_t *tcp,
1254a23fd118Syl150051 lro_t *lro,
1255a23fd118Syl150051 xge_hal_device_t *hldev)
1256a23fd118Syl150051 {
1257a23fd118Syl150051 lro->tcp_next_seq_num += __hal_tcp_seg_len(ip, tcp);
1258a23fd118Syl150051 return XGE_HAL_OK;
1259a23fd118Syl150051
1260a23fd118Syl150051 }
1261a23fd118Syl150051
1262a23fd118Syl150051 /*
1263a23fd118Syl150051 * __hal_append_lro: Appends new frame to existing LRO session.
1264a23fd118Syl150051 * @ip: ip header.
12658347601bSyl150051 * @tcp: IN tcp header, OUT tcp payload.
1266a23fd118Syl150051 * @seg_len: tcp payload length.
1267a23fd118Syl150051 * @lro: lro pointer
1268a23fd118Syl150051 * @hldev: Hal context.
1269a23fd118Syl150051 */
1270a23fd118Syl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_append_lro(iplro_t * ip,tcplro_t ** tcp,u32 * seg_len,lro_t * lro,xge_hal_device_t * hldev)1271a23fd118Syl150051 __hal_append_lro(iplro_t *ip,
1272a23fd118Syl150051 tcplro_t **tcp,
1273a23fd118Syl150051 u32 *seg_len,
1274a23fd118Syl150051 lro_t *lro,
1275a23fd118Syl150051 xge_hal_device_t *hldev)
1276a23fd118Syl150051 {
12778347601bSyl150051 (void) __hal_collapse_ip_hdr(ip, *tcp, lro, hldev);
12788347601bSyl150051 (void) __hal_collapse_tcp_hdr(ip, *tcp, lro, hldev);
1279a23fd118Syl150051 // Update mbuf chain will be done in ll driver.
1280a23fd118Syl150051 // xge_hal_accumulate_large_rx on success of appending new frame to
1281a23fd118Syl150051 // lro will return to ll driver tcpdata pointer, and tcp payload length.
1282a23fd118Syl150051 // along with return code lro frame appended.
1283a23fd118Syl150051
1284a23fd118Syl150051 lro->sg_num++;
1285a23fd118Syl150051 *seg_len = __hal_tcp_seg_len(ip, *tcp);
12867eced415Sxw161283 *tcp = (tcplro_t *)((char *)*tcp + (((*tcp)->doff_res)>>2));
1287a23fd118Syl150051
1288a23fd118Syl150051 return XGE_HAL_OK;
1289a23fd118Syl150051
1290a23fd118Syl150051 }
1291a23fd118Syl150051
1292a23fd118Syl150051 /**
12937eced415Sxw161283 * __xge_hal_accumulate_large_rx: LRO a given frame
1294a23fd118Syl150051 * frames
12957eced415Sxw161283 * @ring: rx ring number
12967eced415Sxw161283 * @eth_hdr: ethernet header.
12977eced415Sxw161283 * @ip_hdr: ip header (optional)
1298a23fd118Syl150051 * @tcp: tcp header.
1299a23fd118Syl150051 * @seglen: packet length.
1300a23fd118Syl150051 * @p_lro: lro pointer.
1301a23fd118Syl150051 * @ext_info: descriptor info, see xge_hal_dtr_info_t{}.
1302a23fd118Syl150051 * @hldev: HAL device.
13037eced415Sxw161283 * @lro_end3: for lro_end3 output
1304a23fd118Syl150051 *
1305a23fd118Syl150051 * LRO the newly received frame, i.e. attach it (if possible) to the
1306a23fd118Syl150051 * already accumulated (i.e., already LRO-ed) received frames (if any),
1307a23fd118Syl150051 * to form one super-sized frame for the subsequent processing
1308a23fd118Syl150051 * by the stack.
1309a23fd118Syl150051 */
1310a23fd118Syl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
xge_hal_lro_process_rx(int ring,u8 * eth_hdr,u8 * ip_hdr,tcplro_t ** tcp,u32 * seglen,lro_t ** p_lro,xge_hal_dtr_info_t * ext_info,xge_hal_device_t * hldev,lro_t ** lro_end3)13117eced415Sxw161283 xge_hal_lro_process_rx(int ring, u8 *eth_hdr, u8 *ip_hdr, tcplro_t **tcp,
13127eced415Sxw161283 u32 *seglen, lro_t **p_lro,
13137eced415Sxw161283 xge_hal_dtr_info_t *ext_info, xge_hal_device_t *hldev,
13148347601bSyl150051 lro_t **lro_end3)
1315a23fd118Syl150051 {
13167eced415Sxw161283 iplro_t *ip = (iplro_t *)ip_hdr;
1317a23fd118Syl150051 xge_hal_status_e ret;
1318a23fd118Syl150051 lro_t *lro;
1319a23fd118Syl150051
1320a23fd118Syl150051 xge_debug_ring(XGE_TRACE, "Entered accumu lro. ");
13217eced415Sxw161283 if (XGE_HAL_OK != __hal_lro_capable(eth_hdr, &ip, (tcplro_t **)tcp,
13227eced415Sxw161283 ext_info))
1323a23fd118Syl150051 return XGE_HAL_INF_LRO_UNCAPABLE;
1324a23fd118Syl150051
1325a23fd118Syl150051 /*
1326a23fd118Syl150051 * This function shall get matching LRO or else
1327a23fd118Syl150051 * create one and return it
1328a23fd118Syl150051 */
13297eced415Sxw161283 ret = __hal_get_lro_session(eth_hdr, ip, (tcplro_t *)*tcp,
13307eced415Sxw161283 p_lro, ext_info, hldev, &hldev->lro_desc[ring],
13317eced415Sxw161283 lro_end3);
1332a23fd118Syl150051 xge_debug_ring(XGE_TRACE, "ret from get_lro:%d ",ret);
1333a23fd118Syl150051 lro = *p_lro;
1334a23fd118Syl150051 if (XGE_HAL_INF_LRO_CONT == ret) {
1335a23fd118Syl150051 if (XGE_HAL_OK == __hal_lro_under_optimal_thresh(ip,
1336a23fd118Syl150051 (tcplro_t *)*tcp, lro, hldev)) {
13378347601bSyl150051 (void) __hal_append_lro(ip,(tcplro_t **) tcp, seglen,
13387eced415Sxw161283 lro, hldev);
1339a23fd118Syl150051 hldev->stats.sw_dev_info_stats.tot_frms_lroised++;
1340a23fd118Syl150051
13418347601bSyl150051 if (lro->sg_num >= hldev->config.lro_sg_size) {
13428347601bSyl150051 hldev->stats.sw_dev_info_stats.lro_sg_exceed_cnt++;
1343a23fd118Syl150051 ret = XGE_HAL_INF_LRO_END_1;
13448347601bSyl150051 }
1345a23fd118Syl150051
1346a23fd118Syl150051 } else ret = XGE_HAL_INF_LRO_END_2;
1347a23fd118Syl150051 }
1348a23fd118Syl150051
1349a23fd118Syl150051 /*
1350a23fd118Syl150051 * Since its time to flush,
1351a23fd118Syl150051 * update ip header so that it can be sent up
1352a23fd118Syl150051 */
1353a23fd118Syl150051 if ((ret == XGE_HAL_INF_LRO_END_1) ||
13548347601bSyl150051 (ret == XGE_HAL_INF_LRO_END_2) ||
13558347601bSyl150051 (ret == XGE_HAL_INF_LRO_END_3)) {
1356a23fd118Syl150051 lro->ip_hdr->tot_len = xge_os_htons((*p_lro)->total_length);
1357a23fd118Syl150051 lro->ip_hdr->check = xge_os_htons(0);
13588347601bSyl150051 lro->ip_hdr->check = XGE_LL_IP_FAST_CSUM(((u8 *)(lro->ip_hdr)),
1359a23fd118Syl150051 (lro->ip_hdr->version_ihl & 0x0F));
1360a23fd118Syl150051 lro->tcp_hdr->ack_seq = lro->tcp_ack_num;
1361a23fd118Syl150051 }
1362a23fd118Syl150051
1363a23fd118Syl150051 return (ret);
1364a23fd118Syl150051 }
1365a23fd118Syl150051
1366a23fd118Syl150051 /**
13677eced415Sxw161283 * xge_hal_accumulate_large_rx: LRO a given frame
13687eced415Sxw161283 * frames
13697eced415Sxw161283 * @buffer: Ethernet frame.
13707eced415Sxw161283 * @tcp: tcp header.
13717eced415Sxw161283 * @seglen: packet length.
13727eced415Sxw161283 * @p_lro: lro pointer.
13737eced415Sxw161283 * @ext_info: descriptor info, see xge_hal_dtr_info_t{}.
13747eced415Sxw161283 * @hldev: HAL device.
13757eced415Sxw161283 * @lro_end3: for lro_end3 output
13767eced415Sxw161283 *
13777eced415Sxw161283 * LRO the newly received frame, i.e. attach it (if possible) to the
13787eced415Sxw161283 * already accumulated (i.e., already LRO-ed) received frames (if any),
13797eced415Sxw161283 * to form one super-sized frame for the subsequent processing
13807eced415Sxw161283 * by the stack.
13817eced415Sxw161283 */
13827eced415Sxw161283 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
xge_hal_accumulate_large_rx(u8 * buffer,tcplro_t ** tcp,u32 * seglen,lro_t ** p_lro,xge_hal_dtr_info_t * ext_info,xge_hal_device_t * hldev,lro_t ** lro_end3)13837eced415Sxw161283 xge_hal_accumulate_large_rx(u8 *buffer, tcplro_t **tcp, u32 *seglen,
13847eced415Sxw161283 lro_t **p_lro, xge_hal_dtr_info_t *ext_info, xge_hal_device_t *hldev,
13857eced415Sxw161283 lro_t **lro_end3)
13867eced415Sxw161283 {
13877eced415Sxw161283 int ring = 0;
13887eced415Sxw161283 return xge_hal_lro_process_rx(ring, buffer, NULL, tcp, seglen, p_lro,
13897eced415Sxw161283 ext_info, hldev, lro_end3);
13907eced415Sxw161283 }
13917eced415Sxw161283
13927eced415Sxw161283 /**
13938347601bSyl150051 * xge_hal_lro_close_session: Close LRO session
13948347601bSyl150051 * @lro: LRO Session.
13958347601bSyl150051 * @hldev: HAL Context.
13968347601bSyl150051 */
13978347601bSyl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
xge_hal_lro_close_session(lro_t * lro)13988347601bSyl150051 xge_hal_lro_close_session (lro_t *lro)
13998347601bSyl150051 {
14008347601bSyl150051 lro->in_use = 0;
14018347601bSyl150051 }
14028347601bSyl150051
14038347601bSyl150051 /**
14047eced415Sxw161283 * xge_hal_lro_next_session: Returns next LRO session in the list or NULL
14058347601bSyl150051 * if none exists.
14067eced415Sxw161283 * @hldev: HAL Context.
14077eced415Sxw161283 * @ring: rx ring number.
1408a23fd118Syl150051 */
1409a23fd118Syl150051 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL lro_t *
xge_hal_lro_next_session(xge_hal_device_t * hldev,int ring)14107eced415Sxw161283 xge_hal_lro_next_session (xge_hal_device_t *hldev, int ring)
1411a23fd118Syl150051 {
14127eced415Sxw161283 xge_hal_lro_desc_t *ring_lro = &hldev->lro_desc[ring];
14138347601bSyl150051 int i;
14147eced415Sxw161283 int start_idx = ring_lro->lro_next_idx;
1415a23fd118Syl150051
14168347601bSyl150051 for(i = start_idx; i < XGE_HAL_LRO_MAX_BUCKETS; i++) {
14177eced415Sxw161283 lro_t *lro = &ring_lro->lro_pool[i];
14188347601bSyl150051
14198347601bSyl150051 if (!lro->in_use)
14208347601bSyl150051 continue;
14218347601bSyl150051
1422a23fd118Syl150051 lro->ip_hdr->tot_len = xge_os_htons(lro->total_length);
1423a23fd118Syl150051 lro->ip_hdr->check = xge_os_htons(0);
1424a23fd118Syl150051 lro->ip_hdr->check = XGE_LL_IP_FAST_CSUM(((u8 *)(lro->ip_hdr)),
1425a23fd118Syl150051 (lro->ip_hdr->version_ihl & 0x0F));
14267eced415Sxw161283 ring_lro->lro_next_idx = i + 1;
14278347601bSyl150051 return lro;
1428a23fd118Syl150051 }
1429a23fd118Syl150051
14307eced415Sxw161283 ring_lro->lro_next_idx = 0;
1431a23fd118Syl150051 return NULL;
14328347601bSyl150051
1433a23fd118Syl150051 }
14347eced415Sxw161283
14357eced415Sxw161283 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL lro_t *
xge_hal_lro_get_next_session(xge_hal_device_t * hldev)14367eced415Sxw161283 xge_hal_lro_get_next_session(xge_hal_device_t *hldev)
14377eced415Sxw161283 {
14387eced415Sxw161283 int ring = 0; /* assume default ring=0 */
14397eced415Sxw161283 return xge_hal_lro_next_session(hldev, ring);
14407eced415Sxw161283 }
1441a23fd118Syl150051 #endif
1442