1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 *
21 * Copyright (c) 2002-2006 Neterion, Inc.
22 */
23
24 /*
25 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
26 * Use is subject to license terms.
27 */
28
29 #ifdef XGE_DEBUG_FP
30 #include "xgehal-device.h"
31 #endif
32
33 #include "xgehal-ring.h"
34 #include "xgehal-fifo.h"
35
36 /**
37 * xge_hal_device_bar0 - Get BAR0 mapped address.
38 * @hldev: HAL device handle.
39 *
40 * Returns: BAR0 address of the specified device.
41 */
42 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char *
xge_hal_device_bar0(xge_hal_device_t * hldev)43 xge_hal_device_bar0(xge_hal_device_t *hldev)
44 {
45 return hldev->bar0;
46 }
47
48 /**
49 * xge_hal_device_isrbar0 - Get BAR0 mapped address.
50 * @hldev: HAL device handle.
51 *
52 * Returns: BAR0 address of the specified device.
53 */
54 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char *
xge_hal_device_isrbar0(xge_hal_device_t * hldev)55 xge_hal_device_isrbar0(xge_hal_device_t *hldev)
56 {
57 return hldev->isrbar0;
58 }
59
60 /**
61 * xge_hal_device_bar1 - Get BAR1 mapped address.
62 * @hldev: HAL device handle.
63 *
64 * Returns: BAR1 address of the specified device.
65 */
66 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char *
xge_hal_device_bar1(xge_hal_device_t * hldev)67 xge_hal_device_bar1(xge_hal_device_t *hldev)
68 {
69 return hldev->bar1;
70 }
71
72 /**
73 * xge_hal_device_bar0_set - Set BAR0 mapped address.
74 * @hldev: HAL device handle.
75 * @bar0: BAR0 mapped address.
76 * * Set BAR0 address in the HAL device object.
77 */
78 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_bar0_set(xge_hal_device_t * hldev,char * bar0)79 xge_hal_device_bar0_set(xge_hal_device_t *hldev, char *bar0)
80 {
81 xge_assert(bar0);
82 hldev->bar0 = bar0;
83 }
84
85 /**
86 * xge_hal_device_isrbar0_set - Set BAR0 mapped address.
87 * @hldev: HAL device handle.
88 * @isrbar0: BAR0 mapped address.
89 * * Set BAR0 address in the HAL device object.
90 */
91 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_isrbar0_set(xge_hal_device_t * hldev,char * isrbar0)92 xge_hal_device_isrbar0_set(xge_hal_device_t *hldev, char *isrbar0)
93 {
94 xge_assert(isrbar0);
95 hldev->isrbar0 = isrbar0;
96 }
97
98 /**
99 * xge_hal_device_bar1_set - Set BAR1 mapped address.
100 * @hldev: HAL device handle.
101 * @channelh: Channel handle.
102 * @bar1: BAR1 mapped address.
103 *
104 * Set BAR1 address for the given channel.
105 */
106 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_bar1_set(xge_hal_device_t * hldev,xge_hal_channel_h channelh,char * bar1)107 xge_hal_device_bar1_set(xge_hal_device_t *hldev, xge_hal_channel_h channelh,
108 char *bar1)
109 {
110 xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
111
112 xge_assert(bar1);
113 xge_assert(fifo);
114
115 /* Initializing the BAR1 address as the start of
116 * the FIFO queue pointer and as a location of FIFO control
117 * word. */
118 fifo->hw_pair =
119 (xge_hal_fifo_hw_pair_t *) (bar1 +
120 (fifo->channel.post_qid * XGE_HAL_FIFO_HW_PAIR_OFFSET));
121 hldev->bar1 = bar1;
122 }
123
124
125 /**
126 * xge_hal_device_rev - Get Device revision number.
127 * @hldev: HAL device handle.
128 *
129 * Returns: Device revision number
130 */
131 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE int
xge_hal_device_rev(xge_hal_device_t * hldev)132 xge_hal_device_rev(xge_hal_device_t *hldev)
133 {
134 return hldev->revision;
135 }
136
137
138 /**
139 * xge_hal_device_begin_irq - Begin IRQ processing.
140 * @hldev: HAL device handle.
141 * @reason: "Reason" for the interrupt, the value of Xframe's
142 * general_int_status register.
143 *
144 * The function performs two actions, It first checks whether (shared IRQ) the
145 * interrupt was raised by the device. Next, it masks the device interrupts.
146 *
147 * Note:
148 * xge_hal_device_begin_irq() does not flush MMIO writes through the
149 * bridge. Therefore, two back-to-back interrupts are potentially possible.
150 * It is the responsibility of the ULD to make sure that only one
151 * xge_hal_device_continue_irq() runs at a time.
152 *
153 * Returns: 0, if the interrupt is not "ours" (note that in this case the
154 * device remain enabled).
155 * Otherwise, xge_hal_device_begin_irq() returns 64bit general adapter
156 * status.
157 * See also: xge_hal_device_handle_irq()
158 */
159 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_begin_irq(xge_hal_device_t * hldev,u64 * reason)160 xge_hal_device_begin_irq(xge_hal_device_t *hldev, u64 *reason)
161 {
162 u64 val64;
163 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
164
165 hldev->stats.sw_dev_info_stats.total_intr_cnt++;
166
167 val64 = xge_os_pio_mem_read64(hldev->pdev,
168 hldev->regh0, &isrbar0->general_int_status);
169 if (xge_os_unlikely(!val64)) {
170 /* not Xframe interrupt */
171 hldev->stats.sw_dev_info_stats.not_xge_intr_cnt++;
172 *reason = 0;
173 return XGE_HAL_ERR_WRONG_IRQ;
174 }
175
176 if (xge_os_unlikely(val64 == XGE_HAL_ALL_FOXES)) {
177 u64 adapter_status =
178 xge_os_pio_mem_read64(hldev->pdev, hldev->regh0,
179 &isrbar0->adapter_status);
180 if (adapter_status == XGE_HAL_ALL_FOXES) {
181 (void) xge_queue_produce(hldev->queueh,
182 XGE_HAL_EVENT_SLOT_FREEZE,
183 hldev,
184 1, /* critical: slot freeze */
185 sizeof(u64),
186 (void*)&adapter_status);
187 *reason = 0;
188 return XGE_HAL_ERR_CRITICAL;
189 }
190 }
191
192 *reason = val64;
193
194 /* separate fast path, i.e. no errors */
195 if (val64 & XGE_HAL_GEN_INTR_RXTRAFFIC) {
196 hldev->stats.sw_dev_info_stats.rx_traffic_intr_cnt++;
197 return XGE_HAL_OK;
198 }
199 if (val64 & XGE_HAL_GEN_INTR_TXTRAFFIC) {
200 hldev->stats.sw_dev_info_stats.tx_traffic_intr_cnt++;
201 return XGE_HAL_OK;
202 }
203
204 hldev->stats.sw_dev_info_stats.not_traffic_intr_cnt++;
205 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXPIC)) {
206 xge_hal_status_e status;
207 hldev->stats.sw_dev_info_stats.txpic_intr_cnt++;
208 status = __hal_device_handle_txpic(hldev, val64);
209 if (status != XGE_HAL_OK) {
210 return status;
211 }
212 }
213
214 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXDMA)) {
215 xge_hal_status_e status;
216 hldev->stats.sw_dev_info_stats.txdma_intr_cnt++;
217 status = __hal_device_handle_txdma(hldev, val64);
218 if (status != XGE_HAL_OK) {
219 return status;
220 }
221 }
222
223 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXMAC)) {
224 xge_hal_status_e status;
225 hldev->stats.sw_dev_info_stats.txmac_intr_cnt++;
226 status = __hal_device_handle_txmac(hldev, val64);
227 if (status != XGE_HAL_OK) {
228 return status;
229 }
230 }
231
232 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXXGXS)) {
233 xge_hal_status_e status;
234 hldev->stats.sw_dev_info_stats.txxgxs_intr_cnt++;
235 status = __hal_device_handle_txxgxs(hldev, val64);
236 if (status != XGE_HAL_OK) {
237 return status;
238 }
239 }
240
241 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXPIC)) {
242 xge_hal_status_e status;
243 hldev->stats.sw_dev_info_stats.rxpic_intr_cnt++;
244 status = __hal_device_handle_rxpic(hldev, val64);
245 if (status != XGE_HAL_OK) {
246 return status;
247 }
248 }
249
250 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXDMA)) {
251 xge_hal_status_e status;
252 hldev->stats.sw_dev_info_stats.rxdma_intr_cnt++;
253 status = __hal_device_handle_rxdma(hldev, val64);
254 if (status != XGE_HAL_OK) {
255 return status;
256 }
257 }
258
259 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXMAC)) {
260 xge_hal_status_e status;
261 hldev->stats.sw_dev_info_stats.rxmac_intr_cnt++;
262 status = __hal_device_handle_rxmac(hldev, val64);
263 if (status != XGE_HAL_OK) {
264 return status;
265 }
266 }
267
268 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXXGXS)) {
269 xge_hal_status_e status;
270 hldev->stats.sw_dev_info_stats.rxxgxs_intr_cnt++;
271 status = __hal_device_handle_rxxgxs(hldev, val64);
272 if (status != XGE_HAL_OK) {
273 return status;
274 }
275 }
276
277 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_MC)) {
278 xge_hal_status_e status;
279 hldev->stats.sw_dev_info_stats.mc_intr_cnt++;
280 status = __hal_device_handle_mc(hldev, val64);
281 if (status != XGE_HAL_OK) {
282 return status;
283 }
284 }
285
286 return XGE_HAL_OK;
287 }
288
289 /**
290 * xge_hal_device_clear_rx - Acknowledge (that is, clear) the
291 * condition that has caused the RX interrupt.
292 * @hldev: HAL device handle.
293 *
294 * Acknowledge (that is, clear) the condition that has caused
295 * the Rx interrupt.
296 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq(),
297 * xge_hal_device_clear_tx(), xge_hal_device_mask_rx().
298 */
299 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_clear_rx(xge_hal_device_t * hldev)300 xge_hal_device_clear_rx(xge_hal_device_t *hldev)
301 {
302 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
303
304 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
305 0xFFFFFFFFFFFFFFFFULL,
306 &isrbar0->rx_traffic_int);
307 }
308
309 /**
310 * xge_hal_device_clear_tx - Acknowledge (that is, clear) the
311 * condition that has caused the TX interrupt.
312 * @hldev: HAL device handle.
313 *
314 * Acknowledge (that is, clear) the condition that has caused
315 * the Tx interrupt.
316 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq(),
317 * xge_hal_device_clear_rx(), xge_hal_device_mask_tx().
318 */
319 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_clear_tx(xge_hal_device_t * hldev)320 xge_hal_device_clear_tx(xge_hal_device_t *hldev)
321 {
322 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
323
324 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
325 0xFFFFFFFFFFFFFFFFULL,
326 &isrbar0->tx_traffic_int);
327 }
328
329 /**
330 * xge_hal_device_poll_rx_channel - Poll Rx channel for completed
331 * descriptors and process the same.
332 * @channel: HAL channel.
333 * @got_rx: Buffer to return the flag set if receive interrupt is occured
334 *
335 * The function polls the Rx channel for the completed descriptors and calls
336 * the upper-layer driver (ULD) via supplied completion callback.
337 *
338 * Returns: XGE_HAL_OK, if the polling is completed successful.
339 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
340 * descriptors available which are yet to be processed.
341 *
342 * See also: xge_hal_device_poll_tx_channel()
343 */
344 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_poll_rx_channel(xge_hal_channel_t * channel,int * got_rx)345 xge_hal_device_poll_rx_channel(xge_hal_channel_t *channel, int *got_rx)
346 {
347 xge_hal_status_e ret = XGE_HAL_OK;
348 xge_hal_dtr_h first_dtrh;
349 xge_hal_device_t *hldev = (xge_hal_device_t *)channel->devh;
350 u8 t_code;
351 int got_bytes;
352
353 /* for each opened rx channel */
354 got_bytes = *got_rx = 0;
355 ((xge_hal_ring_t *)channel)->cmpl_cnt = 0;
356 channel->poll_bytes = 0;
357 if ((ret = xge_hal_ring_dtr_next_completed (channel, &first_dtrh,
358 &t_code)) == XGE_HAL_OK) {
359 if (channel->callback(channel, first_dtrh,
360 t_code, channel->userdata) != XGE_HAL_OK) {
361 (*got_rx) += ((xge_hal_ring_t *)channel)->cmpl_cnt + 1;
362 got_bytes += channel->poll_bytes + 1;
363 ret = XGE_HAL_COMPLETIONS_REMAIN;
364 } else {
365 (*got_rx) += ((xge_hal_ring_t *)channel)->cmpl_cnt + 1;
366 got_bytes += channel->poll_bytes + 1;
367 }
368 }
369
370 if (*got_rx) {
371 hldev->irq_workload_rxd[channel->post_qid] += *got_rx;
372 hldev->irq_workload_rxcnt[channel->post_qid] ++;
373 }
374 hldev->irq_workload_rxlen[channel->post_qid] += got_bytes;
375
376 return ret;
377 }
378
379 /**
380 * xge_hal_device_poll_tx_channel - Poll Tx channel for completed
381 * descriptors and process the same.
382 * @channel: HAL channel.
383 * @got_tx: Buffer to return the flag set if transmit interrupt is occured
384 *
385 * The function polls the Tx channel for the completed descriptors and calls
386 * the upper-layer driver (ULD) via supplied completion callback.
387 *
388 * Returns: XGE_HAL_OK, if the polling is completed successful.
389 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
390 * descriptors available which are yet to be processed.
391 *
392 * See also: xge_hal_device_poll_rx_channel().
393 */
394 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_poll_tx_channel(xge_hal_channel_t * channel,int * got_tx)395 xge_hal_device_poll_tx_channel(xge_hal_channel_t *channel, int *got_tx)
396 {
397 xge_hal_dtr_h first_dtrh;
398 xge_hal_device_t *hldev = (xge_hal_device_t *)channel->devh;
399 u8 t_code;
400 int got_bytes;
401
402 /* for each opened tx channel */
403 got_bytes = *got_tx = 0;
404 channel->poll_bytes = 0;
405 if (xge_hal_fifo_dtr_next_completed (channel, &first_dtrh,
406 &t_code) == XGE_HAL_OK) {
407 if (channel->callback(channel, first_dtrh,
408 t_code, channel->userdata) != XGE_HAL_OK) {
409 (*got_tx)++;
410 got_bytes += channel->poll_bytes + 1;
411 return XGE_HAL_COMPLETIONS_REMAIN;
412 }
413 (*got_tx)++;
414 got_bytes += channel->poll_bytes + 1;
415 }
416
417 if (*got_tx) {
418 hldev->irq_workload_txd[channel->post_qid] += *got_tx;
419 hldev->irq_workload_txcnt[channel->post_qid] ++;
420 }
421 hldev->irq_workload_txlen[channel->post_qid] += got_bytes;
422
423 return XGE_HAL_OK;
424 }
425
426 /**
427 * xge_hal_device_poll_rx_channels - Poll Rx channels for completed
428 * descriptors and process the same.
429 * @hldev: HAL device handle.
430 * @got_rx: Buffer to return flag set if receive is ready
431 *
432 * The function polls the Rx channels for the completed descriptors and calls
433 * the upper-layer driver (ULD) via supplied completion callback.
434 *
435 * Returns: XGE_HAL_OK, if the polling is completed successful.
436 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
437 * descriptors available which are yet to be processed.
438 *
439 * See also: xge_hal_device_poll_tx_channels(), xge_hal_device_continue_irq().
440 */
441 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_poll_rx_channels(xge_hal_device_t * hldev,int * got_rx)442 xge_hal_device_poll_rx_channels(xge_hal_device_t *hldev, int *got_rx)
443 {
444 xge_list_t *item;
445 xge_hal_channel_t *channel;
446
447 /* for each opened rx channel */
448 xge_list_for_each(item, &hldev->ring_channels) {
449 if (hldev->terminating)
450 return XGE_HAL_OK;
451 channel = xge_container_of(item, xge_hal_channel_t, item);
452 if (!(channel->flags & XGE_HAL_CHANNEL_FLAG_USE_RX_POLLING)) {
453 (void) xge_hal_device_poll_rx_channel(channel, got_rx);
454 }
455 }
456
457 return XGE_HAL_OK;
458 }
459
460 /**
461 * xge_hal_device_poll_tx_channels - Poll Tx channels for completed
462 * descriptors and process the same.
463 * @hldev: HAL device handle.
464 * @got_tx: Buffer to return flag set if transmit is ready
465 *
466 * The function polls the Tx channels for the completed descriptors and calls
467 * the upper-layer driver (ULD) via supplied completion callback.
468 *
469 * Returns: XGE_HAL_OK, if the polling is completed successful.
470 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
471 * descriptors available which are yet to be processed.
472 *
473 * See also: xge_hal_device_poll_rx_channels(), xge_hal_device_continue_irq().
474 */
475 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_poll_tx_channels(xge_hal_device_t * hldev,int * got_tx)476 xge_hal_device_poll_tx_channels(xge_hal_device_t *hldev, int *got_tx)
477 {
478 xge_list_t *item;
479 xge_hal_channel_t *channel;
480
481 /* for each opened tx channel */
482 xge_list_for_each(item, &hldev->fifo_channels) {
483 if (hldev->terminating)
484 return XGE_HAL_OK;
485 channel = xge_container_of(item, xge_hal_channel_t, item);
486 (void) xge_hal_device_poll_tx_channel(channel, got_tx);
487 }
488
489 return XGE_HAL_OK;
490 }
491
492 /**
493 *
494 */
495 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_rx_channel_enable_polling(xge_hal_channel_t * channel)496 xge_hal_device_rx_channel_enable_polling(xge_hal_channel_t *channel)
497 {
498 channel->flags |= XGE_HAL_CHANNEL_FLAG_USE_RX_POLLING;
499 }
500
501 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_rx_channel_disable_polling(xge_hal_channel_t * channel)502 xge_hal_device_rx_channel_disable_polling(xge_hal_channel_t *channel)
503 {
504 channel->flags &= ~XGE_HAL_CHANNEL_FLAG_USE_RX_POLLING;
505 }
506
507 /**
508 * xge_hal_device_mask_tx - Mask Tx interrupts.
509 * @hldev: HAL device handle.
510 *
511 * Mask Tx device interrupts.
512 *
513 * See also: xge_hal_device_unmask_tx(), xge_hal_device_mask_rx(),
514 * xge_hal_device_clear_tx().
515 */
516 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_mask_tx(xge_hal_device_t * hldev)517 xge_hal_device_mask_tx(xge_hal_device_t *hldev)
518 {
519 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
520
521 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
522 0xFFFFFFFFFFFFFFFFULL,
523 &isrbar0->tx_traffic_mask);
524 }
525
526 /**
527 * xge_hal_device_mask_rx - Mask Rx interrupts.
528 * @hldev: HAL device handle.
529 *
530 * Mask Rx device interrupts.
531 *
532 * See also: xge_hal_device_unmask_rx(), xge_hal_device_mask_tx(),
533 * xge_hal_device_clear_rx().
534 */
535 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_mask_rx(xge_hal_device_t * hldev)536 xge_hal_device_mask_rx(xge_hal_device_t *hldev)
537 {
538 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
539
540 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
541 0xFFFFFFFFFFFFFFFFULL,
542 &isrbar0->rx_traffic_mask);
543 }
544
545 /**
546 * xge_hal_device_mask_all - Mask all device interrupts.
547 * @hldev: HAL device handle.
548 *
549 * Mask all device interrupts.
550 *
551 * See also: xge_hal_device_unmask_all()
552 */
553 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_mask_all(xge_hal_device_t * hldev)554 xge_hal_device_mask_all(xge_hal_device_t *hldev)
555 {
556 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
557
558 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
559 0xFFFFFFFFFFFFFFFFULL,
560 &isrbar0->general_int_mask);
561 }
562
563 /**
564 * xge_hal_device_unmask_tx - Unmask Tx interrupts.
565 * @hldev: HAL device handle.
566 *
567 * Unmask Tx device interrupts.
568 *
569 * See also: xge_hal_device_mask_tx(), xge_hal_device_clear_tx().
570 */
571 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_unmask_tx(xge_hal_device_t * hldev)572 xge_hal_device_unmask_tx(xge_hal_device_t *hldev)
573 {
574 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
575
576 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
577 0x0ULL,
578 &isrbar0->tx_traffic_mask);
579 }
580
581 /**
582 * xge_hal_device_unmask_rx - Unmask Rx interrupts.
583 * @hldev: HAL device handle.
584 *
585 * Unmask Rx device interrupts.
586 *
587 * See also: xge_hal_device_mask_rx(), xge_hal_device_clear_rx().
588 */
589 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_unmask_rx(xge_hal_device_t * hldev)590 xge_hal_device_unmask_rx(xge_hal_device_t *hldev)
591 {
592 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
593
594 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
595 0x0ULL,
596 &isrbar0->rx_traffic_mask);
597 }
598
599 /**
600 * xge_hal_device_unmask_all - Unmask all device interrupts.
601 * @hldev: HAL device handle.
602 *
603 * Unmask all device interrupts.
604 *
605 * See also: xge_hal_device_mask_all()
606 */
607 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
xge_hal_device_unmask_all(xge_hal_device_t * hldev)608 xge_hal_device_unmask_all(xge_hal_device_t *hldev)
609 {
610 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
611
612 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
613 0x0ULL,
614 &isrbar0->general_int_mask);
615 }
616
617
618 /**
619 * xge_hal_device_continue_irq - Continue handling IRQ: process all
620 * completed descriptors.
621 * @hldev: HAL device handle.
622 *
623 * Process completed descriptors and unmask the device interrupts.
624 *
625 * The xge_hal_device_continue_irq() walks all open channels
626 * and calls upper-layer driver (ULD) via supplied completion
627 * callback. Note that the completion callback is specified at channel open
628 * time, see xge_hal_channel_open().
629 *
630 * Note that the xge_hal_device_continue_irq is part of the _fast_ path.
631 * To optimize the processing, the function does _not_ check for
632 * errors and alarms.
633 *
634 * The latter is done in a polling fashion, via xge_hal_device_poll().
635 *
636 * Returns: XGE_HAL_OK.
637 *
638 * See also: xge_hal_device_handle_irq(), xge_hal_device_poll(),
639 * xge_hal_ring_dtr_next_completed(),
640 * xge_hal_fifo_dtr_next_completed(), xge_hal_channel_callback_f{}.
641 */
642 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_continue_irq(xge_hal_device_t * hldev)643 xge_hal_device_continue_irq(xge_hal_device_t *hldev)
644 {
645 int got_rx = 1, got_tx = 1;
646 int isr_polling_cnt = hldev->config.isr_polling_cnt;
647 int count = 0;
648
649 do
650 {
651 if (got_rx)
652 (void) xge_hal_device_poll_rx_channels(hldev, &got_rx);
653 if (got_tx && hldev->tti_enabled)
654 (void) xge_hal_device_poll_tx_channels(hldev, &got_tx);
655
656 if (!got_rx && !got_tx)
657 break;
658
659 count += (got_rx + got_tx);
660 }while (isr_polling_cnt--);
661
662 if (!count)
663 hldev->stats.sw_dev_info_stats.not_traffic_intr_cnt++;
664
665 return XGE_HAL_OK;
666 }
667
668 /**
669 * xge_hal_device_handle_irq - Handle device IRQ.
670 * @hldev: HAL device handle.
671 *
672 * Perform the complete handling of the line interrupt. The function
673 * performs two calls.
674 * First it uses xge_hal_device_begin_irq() to check the reason for
675 * the interrupt and mask the device interrupts.
676 * Second, it calls xge_hal_device_continue_irq() to process all
677 * completed descriptors and re-enable the interrupts.
678 *
679 * Returns: XGE_HAL_OK - success;
680 * XGE_HAL_ERR_WRONG_IRQ - (shared) IRQ produced by other device.
681 *
682 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq().
683 */
684 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
xge_hal_device_handle_irq(xge_hal_device_t * hldev)685 xge_hal_device_handle_irq(xge_hal_device_t *hldev)
686 {
687 u64 reason;
688 xge_hal_status_e status;
689
690 xge_hal_device_mask_all(hldev);
691
692 status = xge_hal_device_begin_irq(hldev, &reason);
693 if (status != XGE_HAL_OK) {
694 xge_hal_device_unmask_all(hldev);
695 return status;
696 }
697
698 if (reason & XGE_HAL_GEN_INTR_RXTRAFFIC) {
699 xge_hal_device_clear_rx(hldev);
700 }
701
702 status = xge_hal_device_continue_irq(hldev);
703
704 xge_hal_device_clear_tx(hldev);
705
706 xge_hal_device_unmask_all(hldev);
707
708 return status;
709 }
710
711 #if defined(XGE_HAL_CONFIG_LRO)
712
713
714 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int
__hal_lro_check_for_session_match(lro_t * lro,tcplro_t * tcp,iplro_t * ip)715 __hal_lro_check_for_session_match(lro_t *lro, tcplro_t *tcp, iplro_t *ip)
716 {
717
718 /* Match Source address field */
719 if ((lro->ip_hdr->saddr != ip->saddr))
720 return XGE_HAL_FAIL;
721
722 /* Match Destination address field */
723 if ((lro->ip_hdr->daddr != ip->daddr))
724 return XGE_HAL_FAIL;
725
726 /* Match Source Port field */
727 if ((lro->tcp_hdr->source != tcp->source))
728 return XGE_HAL_FAIL;
729
730 /* Match Destination Port field */
731 if ((lro->tcp_hdr->dest != tcp->dest))
732 return XGE_HAL_FAIL;
733
734 return XGE_HAL_OK;
735 }
736
737 /*
738 * __hal_tcp_seg_len: Find the tcp seg len.
739 * @ip: ip header.
740 * @tcp: tcp header.
741 * returns: Tcp seg length.
742 */
743 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL u16
__hal_tcp_seg_len(iplro_t * ip,tcplro_t * tcp)744 __hal_tcp_seg_len(iplro_t *ip, tcplro_t *tcp)
745 {
746 u16 ret;
747
748 ret = (xge_os_ntohs(ip->tot_len) -
749 ((ip->version_ihl & 0x0F)<<2) -
750 ((tcp->doff_res)>>2));
751 return (ret);
752 }
753
754 /*
755 * __hal_ip_lro_capable: Finds whether ip is lro capable.
756 * @ip: ip header.
757 * @ext_info: descriptor info.
758 */
759 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_ip_lro_capable(iplro_t * ip,xge_hal_dtr_info_t * ext_info)760 __hal_ip_lro_capable(iplro_t *ip,
761 xge_hal_dtr_info_t *ext_info)
762 {
763
764 #ifdef XGE_LL_DEBUG_DUMP_PKT
765 {
766 u16 i;
767 u8 ch, *iph = (u8 *)ip;
768
769 xge_debug_ring(XGE_TRACE, "Dump Ip:" );
770 for (i =0; i < 40; i++) {
771 ch = ntohs(*((u8 *)(iph + i)) );
772 printf("i:%d %02x, ",i,ch);
773 }
774 }
775 #endif
776
777 if (ip->version_ihl != IP_FAST_PATH_HDR_MASK) {
778 xge_debug_ring(XGE_ERR, "iphdr !=45 :%d",ip->version_ihl);
779 return XGE_HAL_FAIL;
780 }
781
782 if (ext_info->proto & XGE_HAL_FRAME_PROTO_IP_FRAGMENTED) {
783 xge_debug_ring(XGE_ERR, "IP fragmented");
784 return XGE_HAL_FAIL;
785 }
786
787 return XGE_HAL_OK;
788 }
789
790 /*
791 * __hal_tcp_lro_capable: Finds whether tcp is lro capable.
792 * @ip: ip header.
793 * @tcp: tcp header.
794 */
795 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_tcp_lro_capable(iplro_t * ip,tcplro_t * tcp,lro_t * lro,int * ts_off)796 __hal_tcp_lro_capable(iplro_t *ip, tcplro_t *tcp, lro_t *lro, int *ts_off)
797 {
798 #ifdef XGE_LL_DEBUG_DUMP_PKT
799 {
800 u8 ch;
801 u16 i;
802
803 xge_debug_ring(XGE_TRACE, "Dump Tcp:" );
804 for (i =0; i < 20; i++) {
805 ch = ntohs(*((u8 *)((u8 *)tcp + i)) );
806 xge_os_printf("i:%d %02x, ",i,ch);
807 }
808 }
809 #endif
810 if ((TCP_FAST_PATH_HDR_MASK2 != tcp->ctrl) &&
811 (TCP_FAST_PATH_HDR_MASK3 != tcp->ctrl))
812 goto _exit_fail;
813
814 *ts_off = -1;
815 if (TCP_FAST_PATH_HDR_MASK1 != tcp->doff_res) {
816 u16 tcp_hdr_len = tcp->doff_res >> 2; /* TCP header len */
817 u16 off = 20; /* Start of tcp options */
818 int i, diff;
819
820 /* Does Packet can contain time stamp */
821 if (tcp_hdr_len < 32) {
822 /*
823 * If the session is not opened, we can consider
824 * this packet for LRO
825 */
826 if (lro == NULL)
827 return XGE_HAL_OK;
828
829 goto _exit_fail;
830 }
831
832 /* Ignore No-operation 0x1 */
833 while (((u8 *)tcp)[off] == 0x1)
834 off++;
835
836 /* Next option == Timestamp */
837 if (((u8 *)tcp)[off] != 0x8) {
838 /*
839 * If the session ie not opened, we can consider
840 * this packet for LRO
841 */
842 if (lro == NULL)
843 return XGE_HAL_OK;
844
845 goto _exit_fail;
846 }
847
848 *ts_off = off;
849 if (lro == NULL)
850 return XGE_HAL_OK;
851
852 /*
853 * Now the session is opened. If the LRO frame doesn't
854 * have time stamp, we cannot consider current packet for
855 * LRO.
856 */
857 if (lro->ts_off == -1) {
858 xge_debug_ring(XGE_ERR, "Pkt received with time stamp after session opened with no time stamp : %02x %02x", tcp->doff_res, tcp->ctrl);
859 return XGE_HAL_FAIL;
860 }
861
862 /*
863 * If the difference is greater than three, then there are
864 * more options possible.
865 * else, there are two cases:
866 * case 1: remaining are padding bytes.
867 * case 2: remaining can contain options or padding
868 */
869 off += ((u8 *)tcp)[off+1];
870 diff = tcp_hdr_len - off;
871 if (diff > 3) {
872 /*
873 * Probably contains more options.
874 */
875 xge_debug_ring(XGE_ERR, "tcphdr not fastpth : pkt received with tcp options in addition to time stamp after the session is opened %02x %02x ", tcp->doff_res, tcp->ctrl);
876 return XGE_HAL_FAIL;
877 }
878
879 for (i = 0; i < diff; i++) {
880 u8 byte = ((u8 *)tcp)[off+i];
881
882 /* Ignore No-operation 0x1 */
883 if ((byte == 0x0) || (byte == 0x1))
884 continue;
885 xge_debug_ring(XGE_ERR, "tcphdr not fastpth : pkt received with tcp options in addition to time stamp after the session is opened %02x %02x ", tcp->doff_res, tcp->ctrl);
886 return XGE_HAL_FAIL;
887 }
888
889 /*
890 * Update the time stamp of LRO frame.
891 */
892 xge_os_memcpy(((char *)lro->tcp_hdr + lro->ts_off + 2),
893 (char *)((char *)tcp + (*ts_off) + 2), 8);
894 }
895
896 return XGE_HAL_OK;
897
898 _exit_fail:
899 xge_debug_ring(XGE_TRACE, "tcphdr not fastpth %02x %02x", tcp->doff_res, tcp->ctrl);
900 return XGE_HAL_FAIL;
901
902 }
903
904 /*
905 * __hal_lro_capable: Finds whether frame is lro capable.
906 * @buffer: Ethernet frame.
907 * @ip: ip frame.
908 * @tcp: tcp frame.
909 * @ext_info: Descriptor info.
910 */
911 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_lro_capable(u8 * buffer,iplro_t ** ip,tcplro_t ** tcp,xge_hal_dtr_info_t * ext_info)912 __hal_lro_capable( u8 *buffer,
913 iplro_t **ip,
914 tcplro_t **tcp,
915 xge_hal_dtr_info_t *ext_info)
916 {
917 u8 ip_off, ip_length;
918
919 if (!(ext_info->proto & XGE_HAL_FRAME_PROTO_TCP)) {
920 xge_debug_ring(XGE_ERR, "Cant do lro %d", ext_info->proto);
921 return XGE_HAL_FAIL;
922 }
923
924 if ( !*ip )
925 {
926 #ifdef XGE_LL_DEBUG_DUMP_PKT
927 {
928 u8 ch;
929 u16 i;
930
931 xge_os_printf("Dump Eth:" );
932 for (i =0; i < 60; i++) {
933 ch = ntohs(*((u8 *)(buffer + i)) );
934 xge_os_printf("i:%d %02x, ",i,ch);
935 }
936 }
937 #endif
938
939 switch (ext_info->frame) {
940 case XGE_HAL_FRAME_TYPE_DIX:
941 ip_off = XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE;
942 break;
943 case XGE_HAL_FRAME_TYPE_LLC:
944 ip_off = (XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE +
945 XGE_HAL_HEADER_802_2_SIZE);
946 break;
947 case XGE_HAL_FRAME_TYPE_SNAP:
948 ip_off = (XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE +
949 XGE_HAL_HEADER_SNAP_SIZE);
950 break;
951 default: // XGE_HAL_FRAME_TYPE_IPX, etc.
952 return XGE_HAL_FAIL;
953 }
954
955
956 if (ext_info->proto & XGE_HAL_FRAME_PROTO_VLAN_TAGGED) {
957 ip_off += XGE_HAL_HEADER_VLAN_SIZE;
958 }
959
960 /* Grab ip, tcp headers */
961 *ip = (iplro_t *)((char*)buffer + ip_off);
962 } /* !*ip */
963
964 ip_length = (u8)((*ip)->version_ihl & 0x0F);
965 ip_length = ip_length <<2;
966 *tcp = (tcplro_t *)((char *)*ip + ip_length);
967
968 xge_debug_ring(XGE_TRACE, "ip_length:%d ip:"XGE_OS_LLXFMT
969 " tcp:"XGE_OS_LLXFMT"", (int)ip_length,
970 (unsigned long long)(ulong_t)*ip, (unsigned long long)(ulong_t)*tcp);
971
972 return XGE_HAL_OK;
973
974 }
975
976
977 /*
978 * __hal_open_lro_session: Open a new LRO session.
979 * @buffer: Ethernet frame.
980 * @ip: ip header.
981 * @tcp: tcp header.
982 * @lro: lro pointer
983 * @ext_info: Descriptor info.
984 * @hldev: Hal context.
985 * @ring_lro: LRO descriptor per rx ring.
986 * @slot: Bucket no.
987 * @tcp_seg_len: Length of tcp segment.
988 * @ts_off: time stamp offset in the packet.
989 */
990 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
__hal_open_lro_session(u8 * buffer,iplro_t * ip,tcplro_t * tcp,lro_t ** lro,xge_hal_device_t * hldev,xge_hal_lro_desc_t * ring_lro,int slot,u32 tcp_seg_len,int ts_off)991 __hal_open_lro_session (u8 *buffer, iplro_t *ip, tcplro_t *tcp, lro_t **lro,
992 xge_hal_device_t *hldev, xge_hal_lro_desc_t *ring_lro, int slot,
993 u32 tcp_seg_len, int ts_off)
994 {
995
996 lro_t *lro_new = &ring_lro->lro_pool[slot];
997
998 lro_new->in_use = 1;
999 lro_new->ll_hdr = buffer;
1000 lro_new->ip_hdr = ip;
1001 lro_new->tcp_hdr = tcp;
1002 lro_new->tcp_next_seq_num = tcp_seg_len + xge_os_ntohl(
1003 tcp->seq);
1004 lro_new->tcp_seq_num = tcp->seq;
1005 lro_new->tcp_ack_num = tcp->ack_seq;
1006 lro_new->sg_num = 1;
1007 lro_new->total_length = xge_os_ntohs(ip->tot_len);
1008 lro_new->frags_len = 0;
1009 lro_new->ts_off = ts_off;
1010
1011 hldev->stats.sw_dev_info_stats.tot_frms_lroised++;
1012 hldev->stats.sw_dev_info_stats.tot_lro_sessions++;
1013
1014 *lro = ring_lro->lro_recent = lro_new;
1015 return;
1016 }
1017 /*
1018 * __hal_lro_get_free_slot: Get a free LRO bucket.
1019 * @ring_lro: LRO descriptor per ring.
1020 */
1021 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int
__hal_lro_get_free_slot(xge_hal_lro_desc_t * ring_lro)1022 __hal_lro_get_free_slot (xge_hal_lro_desc_t *ring_lro)
1023 {
1024 int i;
1025
1026 for (i = 0; i < XGE_HAL_LRO_MAX_BUCKETS; i++) {
1027 lro_t *lro_temp = &ring_lro->lro_pool[i];
1028
1029 if (!lro_temp->in_use)
1030 return i;
1031 }
1032 return -1;
1033 }
1034
1035 /*
1036 * __hal_get_lro_session: Gets matching LRO session or creates one.
1037 * @eth_hdr: Ethernet header.
1038 * @ip: ip header.
1039 * @tcp: tcp header.
1040 * @lro: lro pointer
1041 * @ext_info: Descriptor info.
1042 * @hldev: Hal context.
1043 * @ring_lro: LRO descriptor per rx ring
1044 */
1045 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_get_lro_session(u8 * eth_hdr,iplro_t * ip,tcplro_t * tcp,lro_t ** lro,xge_hal_dtr_info_t * ext_info,xge_hal_device_t * hldev,xge_hal_lro_desc_t * ring_lro,lro_t ** lro_end3)1046 __hal_get_lro_session (u8 *eth_hdr,
1047 iplro_t *ip,
1048 tcplro_t *tcp,
1049 lro_t **lro,
1050 xge_hal_dtr_info_t *ext_info,
1051 xge_hal_device_t *hldev,
1052 xge_hal_lro_desc_t *ring_lro,
1053 lro_t **lro_end3 /* Valid only when ret=END_3 */)
1054 {
1055 lro_t *lro_match;
1056 int i, free_slot = -1;
1057 u32 tcp_seg_len;
1058 int ts_off = -1;
1059
1060 *lro = lro_match = NULL;
1061 /*
1062 * Compare the incoming frame with the lro session left from the
1063 * previous call. There is a good chance that this incoming frame
1064 * matches the lro session.
1065 */
1066 if (ring_lro->lro_recent && ring_lro->lro_recent->in_use) {
1067 if (__hal_lro_check_for_session_match(ring_lro->lro_recent,
1068 tcp, ip)
1069 == XGE_HAL_OK)
1070 lro_match = ring_lro->lro_recent;
1071 }
1072
1073 if (!lro_match) {
1074 /*
1075 * Search in the pool of LROs for the session that matches
1076 * the incoming frame.
1077 */
1078 for (i = 0; i < XGE_HAL_LRO_MAX_BUCKETS; i++) {
1079 lro_t *lro_temp = &ring_lro->lro_pool[i];
1080
1081 if (!lro_temp->in_use) {
1082 if (free_slot == -1)
1083 free_slot = i;
1084 continue;
1085 }
1086
1087 if (__hal_lro_check_for_session_match(lro_temp, tcp,
1088 ip) == XGE_HAL_OK) {
1089 lro_match = lro_temp;
1090 break;
1091 }
1092 }
1093 }
1094
1095
1096 if (lro_match) {
1097 /*
1098 * Matching LRO Session found
1099 */
1100 *lro = lro_match;
1101
1102 if (lro_match->tcp_next_seq_num != xge_os_ntohl(tcp->seq)) {
1103 xge_debug_ring(XGE_ERR, "**retransmit **"
1104 "found***");
1105 hldev->stats.sw_dev_info_stats.lro_out_of_seq_pkt_cnt++;
1106 return XGE_HAL_INF_LRO_END_2;
1107 }
1108
1109 if (XGE_HAL_OK != __hal_ip_lro_capable(ip, ext_info))
1110 {
1111 return XGE_HAL_INF_LRO_END_2;
1112 }
1113
1114 if (XGE_HAL_OK != __hal_tcp_lro_capable(ip, tcp, lro_match,
1115 &ts_off)) {
1116 /*
1117 * Close the current session and open a new
1118 * LRO session with this packet,
1119 * provided it has tcp payload
1120 */
1121 tcp_seg_len = __hal_tcp_seg_len(ip, tcp);
1122 if (tcp_seg_len == 0)
1123 {
1124 return XGE_HAL_INF_LRO_END_2;
1125 }
1126
1127 /* Get a free bucket */
1128 free_slot = __hal_lro_get_free_slot(ring_lro);
1129 if (free_slot == -1)
1130 {
1131 return XGE_HAL_INF_LRO_END_2;
1132 }
1133
1134 /*
1135 * Open a new LRO session
1136 */
1137 __hal_open_lro_session (eth_hdr, ip, tcp, lro_end3,
1138 hldev, ring_lro, free_slot, tcp_seg_len,
1139 ts_off);
1140
1141 return XGE_HAL_INF_LRO_END_3;
1142 }
1143
1144 /*
1145 * The frame is good, in-sequence, can be LRO-ed;
1146 * take its (latest) ACK - unless it is a dupack.
1147 * Note: to be exact need to check window size as well..
1148 */
1149 if (lro_match->tcp_ack_num == tcp->ack_seq &&
1150 lro_match->tcp_seq_num == tcp->seq) {
1151 hldev->stats.sw_dev_info_stats.lro_dup_pkt_cnt++;
1152 return XGE_HAL_INF_LRO_END_2;
1153 }
1154
1155 lro_match->tcp_seq_num = tcp->seq;
1156 lro_match->tcp_ack_num = tcp->ack_seq;
1157 lro_match->frags_len += __hal_tcp_seg_len(ip, tcp);
1158
1159 ring_lro->lro_recent = lro_match;
1160
1161 return XGE_HAL_INF_LRO_CONT;
1162 }
1163
1164 /* ********** New Session ***************/
1165 if (free_slot == -1)
1166 return XGE_HAL_INF_LRO_UNCAPABLE;
1167
1168 if (XGE_HAL_FAIL == __hal_ip_lro_capable(ip, ext_info))
1169 return XGE_HAL_INF_LRO_UNCAPABLE;
1170
1171 if (XGE_HAL_FAIL == __hal_tcp_lro_capable(ip, tcp, NULL, &ts_off))
1172 return XGE_HAL_INF_LRO_UNCAPABLE;
1173
1174 xge_debug_ring(XGE_TRACE, "Creating lro session.");
1175
1176 /*
1177 * Open a LRO session, provided the packet contains payload.
1178 */
1179 tcp_seg_len = __hal_tcp_seg_len(ip, tcp);
1180 if (tcp_seg_len == 0)
1181 return XGE_HAL_INF_LRO_UNCAPABLE;
1182
1183 __hal_open_lro_session (eth_hdr, ip, tcp, lro, hldev, ring_lro, free_slot,
1184 tcp_seg_len, ts_off);
1185
1186 return XGE_HAL_INF_LRO_BEGIN;
1187 }
1188
1189 /*
1190 * __hal_lro_under_optimal_thresh: Finds whether combined session is optimal.
1191 * @ip: ip header.
1192 * @tcp: tcp header.
1193 * @lro: lro pointer
1194 * @hldev: Hal context.
1195 */
1196 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_lro_under_optimal_thresh(iplro_t * ip,tcplro_t * tcp,lro_t * lro,xge_hal_device_t * hldev)1197 __hal_lro_under_optimal_thresh (iplro_t *ip,
1198 tcplro_t *tcp,
1199 lro_t *lro,
1200 xge_hal_device_t *hldev)
1201 {
1202 if (!lro) return XGE_HAL_FAIL;
1203
1204 if ((lro->total_length + __hal_tcp_seg_len(ip, tcp) ) >
1205 hldev->config.lro_frm_len) {
1206 xge_debug_ring(XGE_TRACE, "Max LRO frame len exceeded:"
1207 "max length %d ", hldev->config.lro_frm_len);
1208 hldev->stats.sw_dev_info_stats.lro_frm_len_exceed_cnt++;
1209 return XGE_HAL_FAIL;
1210 }
1211
1212 if (lro->sg_num == hldev->config.lro_sg_size) {
1213 xge_debug_ring(XGE_TRACE, "Max sg count exceeded:"
1214 "max sg %d ", hldev->config.lro_sg_size);
1215 hldev->stats.sw_dev_info_stats.lro_sg_exceed_cnt++;
1216 return XGE_HAL_FAIL;
1217 }
1218
1219 return XGE_HAL_OK;
1220 }
1221
1222 /*
1223 * __hal_collapse_ip_hdr: Collapses ip header.
1224 * @ip: ip header.
1225 * @tcp: tcp header.
1226 * @lro: lro pointer
1227 * @hldev: Hal context.
1228 */
1229 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_collapse_ip_hdr(iplro_t * ip,tcplro_t * tcp,lro_t * lro,xge_hal_device_t * hldev)1230 __hal_collapse_ip_hdr ( iplro_t *ip,
1231 tcplro_t *tcp,
1232 lro_t *lro,
1233 xge_hal_device_t *hldev)
1234 {
1235
1236 lro->total_length += __hal_tcp_seg_len(ip, tcp);
1237
1238 /* May be we have to handle time stamps or more options */
1239
1240 return XGE_HAL_OK;
1241
1242 }
1243
1244 /*
1245 * __hal_collapse_tcp_hdr: Collapses tcp header.
1246 * @ip: ip header.
1247 * @tcp: tcp header.
1248 * @lro: lro pointer
1249 * @hldev: Hal context.
1250 */
1251 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_collapse_tcp_hdr(iplro_t * ip,tcplro_t * tcp,lro_t * lro,xge_hal_device_t * hldev)1252 __hal_collapse_tcp_hdr ( iplro_t *ip,
1253 tcplro_t *tcp,
1254 lro_t *lro,
1255 xge_hal_device_t *hldev)
1256 {
1257 lro->tcp_next_seq_num += __hal_tcp_seg_len(ip, tcp);
1258 return XGE_HAL_OK;
1259
1260 }
1261
1262 /*
1263 * __hal_append_lro: Appends new frame to existing LRO session.
1264 * @ip: ip header.
1265 * @tcp: IN tcp header, OUT tcp payload.
1266 * @seg_len: tcp payload length.
1267 * @lro: lro pointer
1268 * @hldev: Hal context.
1269 */
1270 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
__hal_append_lro(iplro_t * ip,tcplro_t ** tcp,u32 * seg_len,lro_t * lro,xge_hal_device_t * hldev)1271 __hal_append_lro(iplro_t *ip,
1272 tcplro_t **tcp,
1273 u32 *seg_len,
1274 lro_t *lro,
1275 xge_hal_device_t *hldev)
1276 {
1277 (void) __hal_collapse_ip_hdr(ip, *tcp, lro, hldev);
1278 (void) __hal_collapse_tcp_hdr(ip, *tcp, lro, hldev);
1279 // Update mbuf chain will be done in ll driver.
1280 // xge_hal_accumulate_large_rx on success of appending new frame to
1281 // lro will return to ll driver tcpdata pointer, and tcp payload length.
1282 // along with return code lro frame appended.
1283
1284 lro->sg_num++;
1285 *seg_len = __hal_tcp_seg_len(ip, *tcp);
1286 *tcp = (tcplro_t *)((char *)*tcp + (((*tcp)->doff_res)>>2));
1287
1288 return XGE_HAL_OK;
1289
1290 }
1291
1292 /**
1293 * __xge_hal_accumulate_large_rx: LRO a given frame
1294 * frames
1295 * @ring: rx ring number
1296 * @eth_hdr: ethernet header.
1297 * @ip_hdr: ip header (optional)
1298 * @tcp: tcp header.
1299 * @seglen: packet length.
1300 * @p_lro: lro pointer.
1301 * @ext_info: descriptor info, see xge_hal_dtr_info_t{}.
1302 * @hldev: HAL device.
1303 * @lro_end3: for lro_end3 output
1304 *
1305 * LRO the newly received frame, i.e. attach it (if possible) to the
1306 * already accumulated (i.e., already LRO-ed) received frames (if any),
1307 * to form one super-sized frame for the subsequent processing
1308 * by the stack.
1309 */
1310 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
xge_hal_lro_process_rx(int ring,u8 * eth_hdr,u8 * ip_hdr,tcplro_t ** tcp,u32 * seglen,lro_t ** p_lro,xge_hal_dtr_info_t * ext_info,xge_hal_device_t * hldev,lro_t ** lro_end3)1311 xge_hal_lro_process_rx(int ring, u8 *eth_hdr, u8 *ip_hdr, tcplro_t **tcp,
1312 u32 *seglen, lro_t **p_lro,
1313 xge_hal_dtr_info_t *ext_info, xge_hal_device_t *hldev,
1314 lro_t **lro_end3)
1315 {
1316 iplro_t *ip = (iplro_t *)ip_hdr;
1317 xge_hal_status_e ret;
1318 lro_t *lro;
1319
1320 xge_debug_ring(XGE_TRACE, "Entered accumu lro. ");
1321 if (XGE_HAL_OK != __hal_lro_capable(eth_hdr, &ip, (tcplro_t **)tcp,
1322 ext_info))
1323 return XGE_HAL_INF_LRO_UNCAPABLE;
1324
1325 /*
1326 * This function shall get matching LRO or else
1327 * create one and return it
1328 */
1329 ret = __hal_get_lro_session(eth_hdr, ip, (tcplro_t *)*tcp,
1330 p_lro, ext_info, hldev, &hldev->lro_desc[ring],
1331 lro_end3);
1332 xge_debug_ring(XGE_TRACE, "ret from get_lro:%d ",ret);
1333 lro = *p_lro;
1334 if (XGE_HAL_INF_LRO_CONT == ret) {
1335 if (XGE_HAL_OK == __hal_lro_under_optimal_thresh(ip,
1336 (tcplro_t *)*tcp, lro, hldev)) {
1337 (void) __hal_append_lro(ip,(tcplro_t **) tcp, seglen,
1338 lro, hldev);
1339 hldev->stats.sw_dev_info_stats.tot_frms_lroised++;
1340
1341 if (lro->sg_num >= hldev->config.lro_sg_size) {
1342 hldev->stats.sw_dev_info_stats.lro_sg_exceed_cnt++;
1343 ret = XGE_HAL_INF_LRO_END_1;
1344 }
1345
1346 } else ret = XGE_HAL_INF_LRO_END_2;
1347 }
1348
1349 /*
1350 * Since its time to flush,
1351 * update ip header so that it can be sent up
1352 */
1353 if ((ret == XGE_HAL_INF_LRO_END_1) ||
1354 (ret == XGE_HAL_INF_LRO_END_2) ||
1355 (ret == XGE_HAL_INF_LRO_END_3)) {
1356 lro->ip_hdr->tot_len = xge_os_htons((*p_lro)->total_length);
1357 lro->ip_hdr->check = xge_os_htons(0);
1358 lro->ip_hdr->check = XGE_LL_IP_FAST_CSUM(((u8 *)(lro->ip_hdr)),
1359 (lro->ip_hdr->version_ihl & 0x0F));
1360 lro->tcp_hdr->ack_seq = lro->tcp_ack_num;
1361 }
1362
1363 return (ret);
1364 }
1365
1366 /**
1367 * xge_hal_accumulate_large_rx: LRO a given frame
1368 * frames
1369 * @buffer: Ethernet frame.
1370 * @tcp: tcp header.
1371 * @seglen: packet length.
1372 * @p_lro: lro pointer.
1373 * @ext_info: descriptor info, see xge_hal_dtr_info_t{}.
1374 * @hldev: HAL device.
1375 * @lro_end3: for lro_end3 output
1376 *
1377 * LRO the newly received frame, i.e. attach it (if possible) to the
1378 * already accumulated (i.e., already LRO-ed) received frames (if any),
1379 * to form one super-sized frame for the subsequent processing
1380 * by the stack.
1381 */
1382 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
xge_hal_accumulate_large_rx(u8 * buffer,tcplro_t ** tcp,u32 * seglen,lro_t ** p_lro,xge_hal_dtr_info_t * ext_info,xge_hal_device_t * hldev,lro_t ** lro_end3)1383 xge_hal_accumulate_large_rx(u8 *buffer, tcplro_t **tcp, u32 *seglen,
1384 lro_t **p_lro, xge_hal_dtr_info_t *ext_info, xge_hal_device_t *hldev,
1385 lro_t **lro_end3)
1386 {
1387 int ring = 0;
1388 return xge_hal_lro_process_rx(ring, buffer, NULL, tcp, seglen, p_lro,
1389 ext_info, hldev, lro_end3);
1390 }
1391
1392 /**
1393 * xge_hal_lro_close_session: Close LRO session
1394 * @lro: LRO Session.
1395 * @hldev: HAL Context.
1396 */
1397 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
xge_hal_lro_close_session(lro_t * lro)1398 xge_hal_lro_close_session (lro_t *lro)
1399 {
1400 lro->in_use = 0;
1401 }
1402
1403 /**
1404 * xge_hal_lro_next_session: Returns next LRO session in the list or NULL
1405 * if none exists.
1406 * @hldev: HAL Context.
1407 * @ring: rx ring number.
1408 */
1409 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL lro_t *
xge_hal_lro_next_session(xge_hal_device_t * hldev,int ring)1410 xge_hal_lro_next_session (xge_hal_device_t *hldev, int ring)
1411 {
1412 xge_hal_lro_desc_t *ring_lro = &hldev->lro_desc[ring];
1413 int i;
1414 int start_idx = ring_lro->lro_next_idx;
1415
1416 for(i = start_idx; i < XGE_HAL_LRO_MAX_BUCKETS; i++) {
1417 lro_t *lro = &ring_lro->lro_pool[i];
1418
1419 if (!lro->in_use)
1420 continue;
1421
1422 lro->ip_hdr->tot_len = xge_os_htons(lro->total_length);
1423 lro->ip_hdr->check = xge_os_htons(0);
1424 lro->ip_hdr->check = XGE_LL_IP_FAST_CSUM(((u8 *)(lro->ip_hdr)),
1425 (lro->ip_hdr->version_ihl & 0x0F));
1426 ring_lro->lro_next_idx = i + 1;
1427 return lro;
1428 }
1429
1430 ring_lro->lro_next_idx = 0;
1431 return NULL;
1432
1433 }
1434
1435 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL lro_t *
xge_hal_lro_get_next_session(xge_hal_device_t * hldev)1436 xge_hal_lro_get_next_session(xge_hal_device_t *hldev)
1437 {
1438 int ring = 0; /* assume default ring=0 */
1439 return xge_hal_lro_next_session(hldev, ring);
1440 }
1441 #endif
1442