ena_netmap.c (2287afd8180cb5f48c6a3ff6235e0b743b13353a) ena_netmap.c (9eb1615f33e9174fa5f1ca46954c35b3f026a98a)
1/*-
2 * BSD LICENSE
3 *
4 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 74 unchanged lines hidden (view full) ---

83static void ena_netmap_fill_ctx(struct netmap_kring *,
84 struct ena_netmap_ctx *, uint16_t);
85
86int
87ena_netmap_attach(struct ena_adapter *adapter)
88{
89 struct netmap_adapter na;
90
1/*-
2 * BSD LICENSE
3 *
4 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 74 unchanged lines hidden (view full) ---

83static void ena_netmap_fill_ctx(struct netmap_kring *,
84 struct ena_netmap_ctx *, uint16_t);
85
86int
87ena_netmap_attach(struct ena_adapter *adapter)
88{
89 struct netmap_adapter na;
90
91 ena_trace(ENA_NETMAP, "netmap attach\n");
91 ena_trace(NULL, ENA_NETMAP, "netmap attach\n");
92
93 bzero(&na, sizeof(na));
94 na.na_flags = NAF_MOREFRAG;
95 na.ifp = adapter->ifp;
96 na.num_tx_desc = adapter->requested_tx_ring_size;
97 na.num_rx_desc = adapter->requested_rx_ring_size;
98 na.num_tx_rings = adapter->num_io_queues;
99 na.num_rx_rings = adapter->num_io_queues;

--- 21 unchanged lines hidden (view full) ---

121 if (unlikely(rx_info->netmap_buf_idx != 0))
122 return (0);
123
124 qid = rx_ring->qid;
125 kring = na->rx_rings[qid];
126 nm_i = kring->nr_hwcur;
127 head = kring->rhead;
128
92
93 bzero(&na, sizeof(na));
94 na.na_flags = NAF_MOREFRAG;
95 na.ifp = adapter->ifp;
96 na.num_tx_desc = adapter->requested_tx_ring_size;
97 na.num_rx_desc = adapter->requested_rx_ring_size;
98 na.num_tx_rings = adapter->num_io_queues;
99 na.num_rx_rings = adapter->num_io_queues;

--- 21 unchanged lines hidden (view full) ---

121 if (unlikely(rx_info->netmap_buf_idx != 0))
122 return (0);
123
124 qid = rx_ring->qid;
125 kring = na->rx_rings[qid];
126 nm_i = kring->nr_hwcur;
127 head = kring->rhead;
128
129 ena_trace(ENA_NETMAP | ENA_DBG, "nr_hwcur: %d, nr_hwtail: %d, "
129 ena_trace(NULL, ENA_NETMAP | ENA_DBG, "nr_hwcur: %d, nr_hwtail: %d, "
130 "rhead: %d, rcur: %d, rtail: %d\n", kring->nr_hwcur,
131 kring->nr_hwtail, kring->rhead, kring->rcur, kring->rtail);
132
133 if ((nm_i == head) && rx_ring->initialized) {
130 "rhead: %d, rcur: %d, rtail: %d\n", kring->nr_hwcur,
131 kring->nr_hwtail, kring->rhead, kring->rcur, kring->rtail);
132
133 if ((nm_i == head) && rx_ring->initialized) {
134 ena_trace(ENA_NETMAP, "No free slots in netmap ring\n");
134 ena_trace(NULL, ENA_NETMAP, "No free slots in netmap ring\n");
135 return (ENOMEM);
136 }
137
138 ring = kring->ring;
139 if (ring == NULL) {
140 device_printf(adapter->pdev, "Rx ring %d is NULL\n", qid);
141 return (EFAULT);
142 }
143 slot = &ring->slot[nm_i];
144
145 addr = PNMB(na, slot, &paddr);
146 if (addr == NETMAP_BUF_BASE(na)) {
147 device_printf(adapter->pdev, "Bad buff in slot\n");
148 return (EFAULT);
149 }
150
151 rc = netmap_load_map(na, adapter->rx_buf_tag, rx_info->map, addr);
152 if (rc != 0) {
135 return (ENOMEM);
136 }
137
138 ring = kring->ring;
139 if (ring == NULL) {
140 device_printf(adapter->pdev, "Rx ring %d is NULL\n", qid);
141 return (EFAULT);
142 }
143 slot = &ring->slot[nm_i];
144
145 addr = PNMB(na, slot, &paddr);
146 if (addr == NETMAP_BUF_BASE(na)) {
147 device_printf(adapter->pdev, "Bad buff in slot\n");
148 return (EFAULT);
149 }
150
151 rc = netmap_load_map(na, adapter->rx_buf_tag, rx_info->map, addr);
152 if (rc != 0) {
153 ena_trace(ENA_WARNING, "DMA mapping error\n");
153 ena_trace(NULL, ENA_WARNING, "DMA mapping error\n");
154 return (rc);
155 }
156 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD);
157
158 rx_info->ena_buf.paddr = paddr;
159 rx_info->ena_buf.len = ring->nr_buf_size;
160 rx_info->mbuf = NULL;
161 rx_info->netmap_buf_idx = slot->buf_idx;

--- 43 unchanged lines hidden (view full) ---

205 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
206 BUS_DMASYNC_POSTREAD);
207 netmap_unload_map(na, adapter->rx_buf_tag, rx_info->map);
208
209 KASSERT(kring->ring == NULL, ("Netmap Rx ring is NULL\n"));
210
211 slot = &kring->ring->slot[nm_i];
212
154 return (rc);
155 }
156 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD);
157
158 rx_info->ena_buf.paddr = paddr;
159 rx_info->ena_buf.len = ring->nr_buf_size;
160 rx_info->mbuf = NULL;
161 rx_info->netmap_buf_idx = slot->buf_idx;

--- 43 unchanged lines hidden (view full) ---

205 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
206 BUS_DMASYNC_POSTREAD);
207 netmap_unload_map(na, adapter->rx_buf_tag, rx_info->map);
208
209 KASSERT(kring->ring == NULL, ("Netmap Rx ring is NULL\n"));
210
211 slot = &kring->ring->slot[nm_i];
212
213 ENA_ASSERT(slot->buf_idx == 0, "Overwrite slot buf\n");
213 ENA_WARN(slot->buf_idx != 0, NULL, "Overwrite slot buf\n");
214 slot->buf_idx = rx_info->netmap_buf_idx;
215 slot->flags = NS_BUF_CHANGED;
216
217 rx_info->netmap_buf_idx = 0;
218 kring->nr_hwcur = nm_i;
219}
220
221static bool

--- 25 unchanged lines hidden (view full) ---

247
248static void
249ena_netmap_reset_ring(struct ena_adapter *adapter, int qid, enum txrx x)
250{
251 if (!ena_ring_in_netmap(adapter, qid, x))
252 return;
253
254 netmap_reset(NA(adapter->ifp), x, qid, 0);
214 slot->buf_idx = rx_info->netmap_buf_idx;
215 slot->flags = NS_BUF_CHANGED;
216
217 rx_info->netmap_buf_idx = 0;
218 kring->nr_hwcur = nm_i;
219}
220
221static bool

--- 25 unchanged lines hidden (view full) ---

247
248static void
249ena_netmap_reset_ring(struct ena_adapter *adapter, int qid, enum txrx x)
250{
251 if (!ena_ring_in_netmap(adapter, qid, x))
252 return;
253
254 netmap_reset(NA(adapter->ifp), x, qid, 0);
255 ena_trace(ENA_NETMAP, "%s ring %d is in netmap mode\n",
255 ena_trace(NULL, ENA_NETMAP, "%s ring %d is in netmap mode\n",
256 (x == NR_TX) ? "Tx" : "Rx", qid);
257}
258
259void
260ena_netmap_reset_rx_ring(struct ena_adapter *adapter, int qid)
261{
262 ena_netmap_reset_ring(adapter, qid, NR_RX);
263}

--- 13 unchanged lines hidden (view full) ---

277 enum txrx t;
278 int rc, i;
279
280 ENA_LOCK_LOCK(adapter);
281 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
282 ena_down(adapter);
283
284 if (onoff) {
256 (x == NR_TX) ? "Tx" : "Rx", qid);
257}
258
259void
260ena_netmap_reset_rx_ring(struct ena_adapter *adapter, int qid)
261{
262 ena_netmap_reset_ring(adapter, qid, NR_RX);
263}

--- 13 unchanged lines hidden (view full) ---

277 enum txrx t;
278 int rc, i;
279
280 ENA_LOCK_LOCK(adapter);
281 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
282 ena_down(adapter);
283
284 if (onoff) {
285 ena_trace(ENA_NETMAP, "netmap on\n");
285 ena_trace(NULL, ENA_NETMAP, "netmap on\n");
286 for_rx_tx(t) {
287 for (i = 0; i <= nma_get_nrings(na, t); i++) {
288 kring = NMR(na, t)[i];
289 if (nm_kring_pending_on(kring)) {
290 kring->nr_mode = NKR_NETMAP_ON;
291 }
292 }
293 }
294 nm_set_native_flags(na);
295 } else {
286 for_rx_tx(t) {
287 for (i = 0; i <= nma_get_nrings(na, t); i++) {
288 kring = NMR(na, t)[i];
289 if (nm_kring_pending_on(kring)) {
290 kring->nr_mode = NKR_NETMAP_ON;
291 }
292 }
293 }
294 nm_set_native_flags(na);
295 } else {
296 ena_trace(ENA_NETMAP, "netmap off\n");
296 ena_trace(NULL, ENA_NETMAP, "netmap off\n");
297 nm_clear_native_flags(na);
298 for_rx_tx(t) {
299 for (i = 0; i <= nma_get_nrings(na, t); i++) {
300 kring = NMR(na, t)[i];
301 if (nm_kring_pending_off(kring)) {
302 kring->nr_mode = NKR_NETMAP_OFF;
303 }
304 }
305 }
306 }
307
308 rc = ena_up(adapter);
309 if (rc != 0) {
297 nm_clear_native_flags(na);
298 for_rx_tx(t) {
299 for (i = 0; i <= nma_get_nrings(na, t); i++) {
300 kring = NMR(na, t)[i];
301 if (nm_kring_pending_off(kring)) {
302 kring->nr_mode = NKR_NETMAP_OFF;
303 }
304 }
305 }
306 }
307
308 rc = ena_up(adapter);
309 if (rc != 0) {
310 ena_trace(ENA_WARNING, "ena_up failed with rc=%d\n", rc);
310 ena_trace(NULL, ENA_WARNING, "ena_up failed with rc=%d\n", rc);
311 adapter->reset_reason = ENA_REGS_RESET_DRIVER_INVALID_STATE;
312 nm_clear_native_flags(na);
313 ena_destroy_device(adapter, false);
314 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
315 rc = ena_restore_device(adapter);
316 }
317 ENA_LOCK_UNLOCK(adapter);
318

--- 77 unchanged lines hidden (view full) ---

396 uint16_t header_len;
397 uint16_t packet_len;
398 int nb_hw_desc;
399 int rc;
400 void *push_hdr;
401
402 adapter = ctx->adapter;
403 if (ena_netmap_count_slots(ctx) > adapter->max_tx_sgl_size) {
311 adapter->reset_reason = ENA_REGS_RESET_DRIVER_INVALID_STATE;
312 nm_clear_native_flags(na);
313 ena_destroy_device(adapter, false);
314 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
315 rc = ena_restore_device(adapter);
316 }
317 ENA_LOCK_UNLOCK(adapter);
318

--- 77 unchanged lines hidden (view full) ---

396 uint16_t header_len;
397 uint16_t packet_len;
398 int nb_hw_desc;
399 int rc;
400 void *push_hdr;
401
402 adapter = ctx->adapter;
403 if (ena_netmap_count_slots(ctx) > adapter->max_tx_sgl_size) {
404 ena_trace(ENA_WARNING, "Too many slots per packet\n");
404 ena_trace(NULL, ENA_WARNING, "Too many slots per packet\n");
405 return (EINVAL);
406 }
407
408 tx_ring = ctx->ring;
409
410 req_id = tx_ring->free_tx_ids[ctx->nt];
411 tx_info = &tx_ring->tx_buffer_info[req_id];
412 tx_info->num_of_bufs = 0;

--- 20 unchanged lines hidden (view full) ---

433 ena_com_write_sq_doorbell(ctx->io_sq);
434 counter_u64_add(tx_ring->tx_stats.doorbells, 1);
435 tx_ring->acum_pkts = 0;
436 }
437
438 rc = ena_com_prepare_tx(ctx->io_sq, &ena_tx_ctx, &nb_hw_desc);
439 if (unlikely(rc != 0)) {
440 if (likely(rc == ENA_COM_NO_MEM)) {
405 return (EINVAL);
406 }
407
408 tx_ring = ctx->ring;
409
410 req_id = tx_ring->free_tx_ids[ctx->nt];
411 tx_info = &tx_ring->tx_buffer_info[req_id];
412 tx_info->num_of_bufs = 0;

--- 20 unchanged lines hidden (view full) ---

433 ena_com_write_sq_doorbell(ctx->io_sq);
434 counter_u64_add(tx_ring->tx_stats.doorbells, 1);
435 tx_ring->acum_pkts = 0;
436 }
437
438 rc = ena_com_prepare_tx(ctx->io_sq, &ena_tx_ctx, &nb_hw_desc);
439 if (unlikely(rc != 0)) {
440 if (likely(rc == ENA_COM_NO_MEM)) {
441 ena_trace(ENA_NETMAP | ENA_DBG | ENA_TXPTH,
441 ena_trace(NULL, ENA_NETMAP | ENA_DBG | ENA_TXPTH,
442 "Tx ring[%d] is out of space\n", tx_ring->que->id);
443 } else {
444 device_printf(adapter->pdev,
445 "Failed to prepare Tx bufs\n");
446 }
447 counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1);
448
449 ena_netmap_unmap_last_socket_chain(ctx, tx_info);

--- 77 unchanged lines hidden (view full) ---

527static int
528ena_netmap_map_single_slot(struct netmap_adapter *na, struct netmap_slot *slot,
529 bus_dma_tag_t dmatag, bus_dmamap_t dmamap, void **vaddr, uint64_t *paddr)
530{
531 int rc;
532
533 *vaddr = PNMB(na, slot, paddr);
534 if (unlikely(vaddr == NULL)) {
442 "Tx ring[%d] is out of space\n", tx_ring->que->id);
443 } else {
444 device_printf(adapter->pdev,
445 "Failed to prepare Tx bufs\n");
446 }
447 counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1);
448
449 ena_netmap_unmap_last_socket_chain(ctx, tx_info);

--- 77 unchanged lines hidden (view full) ---

527static int
528ena_netmap_map_single_slot(struct netmap_adapter *na, struct netmap_slot *slot,
529 bus_dma_tag_t dmatag, bus_dmamap_t dmamap, void **vaddr, uint64_t *paddr)
530{
531 int rc;
532
533 *vaddr = PNMB(na, slot, paddr);
534 if (unlikely(vaddr == NULL)) {
535 ena_trace(ENA_ALERT, "Slot address is NULL\n");
535 ena_trace(NULL, ENA_ALERT, "Slot address is NULL\n");
536 return (EINVAL);
537 }
538
539 rc = netmap_load_map(na, dmatag, dmamap, *vaddr);
540 if (unlikely(rc != 0)) {
536 return (EINVAL);
537 }
538
539 rc = netmap_load_map(na, dmatag, dmamap, *vaddr);
540 if (unlikely(rc != 0)) {
541 ena_trace(ENA_ALERT, "Failed to map slot %d for DMA\n",
541 ena_trace(NULL, ENA_ALERT, "Failed to map slot %d for DMA\n",
542 slot->buf_idx);
543 return (EINVAL);
544 }
545
546 return (0);
547}
548
549static int

--- 71 unchanged lines hidden (view full) ---

621 }
622
623 *push_hdr = tx_ring->push_buf_intermediate_buf;
624 counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1);
625
626 delta = push_len - slot_head_len;
627 }
628
542 slot->buf_idx);
543 return (EINVAL);
544 }
545
546 return (0);
547}
548
549static int

--- 71 unchanged lines hidden (view full) ---

621 }
622
623 *push_hdr = tx_ring->push_buf_intermediate_buf;
624 counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1);
625
626 delta = push_len - slot_head_len;
627 }
628
629 ena_trace(ENA_NETMAP | ENA_DBG | ENA_TXPTH,
629 ena_trace(NULL, ENA_NETMAP | ENA_DBG | ENA_TXPTH,
630 "slot: %d header_buf->vaddr: %p push_len: %d\n",
631 slot->buf_idx, *push_hdr, push_len);
632
633 /*
634 * If header was in linear memory space, map for the dma rest of the data
635 * in the first mbuf of the mbuf chain.
636 */
637 if (slot_head_len > push_len) {

--- 217 unchanged lines hidden (view full) ---

855 netmap_unload_map(ctx->na, ctx->adapter->tx_buf_tag,
856 nm_info->map_seg[n]);
857 }
858 tx_info->num_of_bufs = 0;
859
860 /* Next, retain the sockets back to the userspace */
861 for (n = 0; n < nm_info->sockets_used; n++) {
862 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
630 "slot: %d header_buf->vaddr: %p push_len: %d\n",
631 slot->buf_idx, *push_hdr, push_len);
632
633 /*
634 * If header was in linear memory space, map for the dma rest of the data
635 * in the first mbuf of the mbuf chain.
636 */
637 if (slot_head_len > push_len) {

--- 217 unchanged lines hidden (view full) ---

855 netmap_unload_map(ctx->na, ctx->adapter->tx_buf_tag,
856 nm_info->map_seg[n]);
857 }
858 tx_info->num_of_bufs = 0;
859
860 /* Next, retain the sockets back to the userspace */
861 for (n = 0; n < nm_info->sockets_used; n++) {
862 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
863 ENA_ASSERT(ctx->slots[ctx->nm_i].buf_idx == 0,
863 ENA_WARN(ctx->slots[ctx->nm_i].buf_idx != 0, NULL,
864 "Tx idx is not 0.\n");
865 ctx->slots[ctx->nm_i].buf_idx = nm_info->socket_buf_idx[n];
866 ctx->slots[ctx->nm_i].flags = NS_BUF_CHANGED;
867 nm_info->socket_buf_idx[n] = 0;
868 }
869 nm_info->sockets_used = 0;
870
871 ctx->ring->free_tx_ids[ctx->nt] = req_id;

--- 5 unchanged lines hidden (view full) ---

877static inline int
878validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id)
879{
880 struct ena_adapter *adapter = tx_ring->adapter;
881
882 if (likely(req_id < tx_ring->ring_size))
883 return (0);
884
864 "Tx idx is not 0.\n");
865 ctx->slots[ctx->nm_i].buf_idx = nm_info->socket_buf_idx[n];
866 ctx->slots[ctx->nm_i].flags = NS_BUF_CHANGED;
867 nm_info->socket_buf_idx[n] = 0;
868 }
869 nm_info->sockets_used = 0;
870
871 ctx->ring->free_tx_ids[ctx->nt] = req_id;

--- 5 unchanged lines hidden (view full) ---

877static inline int
878validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id)
879{
880 struct ena_adapter *adapter = tx_ring->adapter;
881
882 if (likely(req_id < tx_ring->ring_size))
883 return (0);
884
885 ena_trace(ENA_WARNING, "Invalid req_id: %hu\n", req_id);
885 ena_trace(NULL, ENA_WARNING, "Invalid req_id: %hu\n", req_id);
886 counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
887
888 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_TX_REQ_ID);
889
890 return (EFAULT);
891}
892
893static int

--- 50 unchanged lines hidden (view full) ---

944
945 return (rc);
946}
947
948static inline int
949ena_netmap_rx_frame(struct ena_netmap_ctx *ctx)
950{
951 struct ena_com_rx_ctx ena_rx_ctx;
886 counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
887
888 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_TX_REQ_ID);
889
890 return (EFAULT);
891}
892
893static int

--- 50 unchanged lines hidden (view full) ---

944
945 return (rc);
946}
947
948static inline int
949ena_netmap_rx_frame(struct ena_netmap_ctx *ctx)
950{
951 struct ena_com_rx_ctx ena_rx_ctx;
952 enum ena_regs_reset_reason_types reset_reason;
952 int rc, len = 0;
953 uint16_t buf, nm;
954
955 ena_rx_ctx.ena_bufs = ctx->ring->ena_bufs;
956 ena_rx_ctx.max_bufs = ctx->adapter->max_rx_sgl_size;
957 bus_dmamap_sync(ctx->io_cq->cdesc_addr.mem_handle.tag,
958 ctx->io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD);
959
960 rc = ena_com_rx_pkt(ctx->io_cq, ctx->io_sq, &ena_rx_ctx);
961 if (unlikely(rc != 0)) {
953 int rc, len = 0;
954 uint16_t buf, nm;
955
956 ena_rx_ctx.ena_bufs = ctx->ring->ena_bufs;
957 ena_rx_ctx.max_bufs = ctx->adapter->max_rx_sgl_size;
958 bus_dmamap_sync(ctx->io_cq->cdesc_addr.mem_handle.tag,
959 ctx->io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD);
960
961 rc = ena_com_rx_pkt(ctx->io_cq, ctx->io_sq, &ena_rx_ctx);
962 if (unlikely(rc != 0)) {
962 ena_trace(ENA_ALERT, "Too many desc from the device.\n");
963 counter_u64_add(ctx->ring->rx_stats.bad_desc_num, 1);
964 ena_trigger_reset(ctx->adapter,
965 ENA_REGS_RESET_TOO_MANY_RX_DESCS);
963 ena_trace(NULL, ENA_ALERT,
964 "Failed to read pkt from the device with error: %d\n", rc);
965 if (rc == ENA_COM_NO_SPACE) {
966 counter_u64_add(ctx->ring->rx_stats.bad_desc_num, 1);
967 reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
968 } else {
969 counter_u64_add(ctx->ring->rx_stats.bad_req_id, 1);
970 reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
971 }
972 ena_trigger_reset(ctx->adapter, reset_reason);
966 return (rc);
967 }
968 if (unlikely(ena_rx_ctx.descs == 0))
969 return (ENA_NETMAP_NO_MORE_FRAMES);
970
973 return (rc);
974 }
975 if (unlikely(ena_rx_ctx.descs == 0))
976 return (ENA_NETMAP_NO_MORE_FRAMES);
977
971 ena_trace(ENA_NETMAP | ENA_DBG, "Rx: q %d got packet from ena. descs #:"
978 ena_trace(NULL, ENA_NETMAP | ENA_DBG, "Rx: q %d got packet from ena. descs #:"
972 " %d l3 proto %d l4 proto %d hash: %x\n", ctx->ring->qid,
973 ena_rx_ctx.descs, ena_rx_ctx.l3_proto, ena_rx_ctx.l4_proto,
974 ena_rx_ctx.hash);
975
976 for (buf = 0; buf < ena_rx_ctx.descs; buf++)
977 if ((rc = ena_netmap_rx_load_desc(ctx, buf, &len)) != 0)
978 break;
979 /*

--- 32 unchanged lines hidden (view full) ---

1012 return (rc);
1013}
1014
1015static inline int
1016ena_netmap_rx_load_desc(struct ena_netmap_ctx *ctx, uint16_t buf, int *len)
1017{
1018 struct ena_rx_buffer *rx_info;
1019 uint16_t req_id;
979 " %d l3 proto %d l4 proto %d hash: %x\n", ctx->ring->qid,
980 ena_rx_ctx.descs, ena_rx_ctx.l3_proto, ena_rx_ctx.l4_proto,
981 ena_rx_ctx.hash);
982
983 for (buf = 0; buf < ena_rx_ctx.descs; buf++)
984 if ((rc = ena_netmap_rx_load_desc(ctx, buf, &len)) != 0)
985 break;
986 /*

--- 32 unchanged lines hidden (view full) ---

1019 return (rc);
1020}
1021
1022static inline int
1023ena_netmap_rx_load_desc(struct ena_netmap_ctx *ctx, uint16_t buf, int *len)
1024{
1025 struct ena_rx_buffer *rx_info;
1026 uint16_t req_id;
1020 int rc;
1021
1022 req_id = ctx->ring->ena_bufs[buf].req_id;
1027
1028 req_id = ctx->ring->ena_bufs[buf].req_id;
1023 rc = validate_rx_req_id(ctx->ring, req_id);
1024 if (unlikely(rc != 0))
1025 return (rc);
1026
1027 rx_info = &ctx->ring->rx_buffer_info[req_id];
1028 bus_dmamap_sync(ctx->adapter->rx_buf_tag, rx_info->map,
1029 BUS_DMASYNC_POSTREAD);
1030 netmap_unload_map(ctx->na, ctx->adapter->rx_buf_tag, rx_info->map);
1031
1029 rx_info = &ctx->ring->rx_buffer_info[req_id];
1030 bus_dmamap_sync(ctx->adapter->rx_buf_tag, rx_info->map,
1031 BUS_DMASYNC_POSTREAD);
1032 netmap_unload_map(ctx->na, ctx->adapter->rx_buf_tag, rx_info->map);
1033
1032 ENA_ASSERT(ctx->slots[ctx->nm_i].buf_idx == 0, "Rx idx is not 0.\n");
1034 ENA_WARN(ctx->slots[ctx->nm_i].buf_idx != 0, NULL,
1035 "Rx idx is not 0.\n");
1033
1034 ctx->slots[ctx->nm_i].buf_idx = rx_info->netmap_buf_idx;
1035 rx_info->netmap_buf_idx = 0;
1036 /*
1037 * Set NS_MOREFRAG to all slots.
1038 * Then ena_netmap_rx_frame clears it from last one.
1039 */
1040 ctx->slots[ctx->nm_i].flags |= NS_MOREFRAG | NS_BUF_CHANGED;
1041 ctx->slots[ctx->nm_i].len = ctx->ring->ena_bufs[buf].len;
1042 *len += ctx->slots[ctx->nm_i].len;
1043 ctx->ring->free_rx_ids[ctx->nt] = req_id;
1036
1037 ctx->slots[ctx->nm_i].buf_idx = rx_info->netmap_buf_idx;
1038 rx_info->netmap_buf_idx = 0;
1039 /*
1040 * Set NS_MOREFRAG to all slots.
1041 * Then ena_netmap_rx_frame clears it from last one.
1042 */
1043 ctx->slots[ctx->nm_i].flags |= NS_MOREFRAG | NS_BUF_CHANGED;
1044 ctx->slots[ctx->nm_i].len = ctx->ring->ena_bufs[buf].len;
1045 *len += ctx->slots[ctx->nm_i].len;
1046 ctx->ring->free_rx_ids[ctx->nt] = req_id;
1044 ena_trace(ENA_DBG, "rx_info %p, buf_idx %d, paddr %jx, nm: %d\n",
1047 ena_trace(NULL, ENA_DBG, "rx_info %p, buf_idx %d, paddr %jx, nm: %d\n",
1045 rx_info, ctx->slots[ctx->nm_i].buf_idx,
1046 (uintmax_t)rx_info->ena_buf.paddr, ctx->nm_i);
1047
1048 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
1049 ctx->nt = ENA_RX_RING_IDX_NEXT(ctx->nt, ctx->ring->ring_size);
1050
1051 return (0);
1052}

--- 40 unchanged lines hidden --- 		1048 rx_info, ctx->slots[ctx->nm_i].buf_idx,
1049 (uintmax_t)rx_info->ena_buf.paddr, ctx->nm_i);
1050
1051 ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
1052 ctx->nt = ENA_RX_RING_IDX_NEXT(ctx->nt, ctx->ring->ring_size);
1053
1054 return (0);
1055}

--- 40 unchanged lines hidden ---