ena.c (fceb93871598b5f6779a21e3ae3e380694289aea) ena.c (43fefd1629ab1100a0c6fedec684b9c2aa7c1c6e)
1/*-
2 * BSD LICENSE
3 *
4 * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 88 unchanged lines hidden (view full) ---

97static int ena_setup_tx_dma_tag(struct ena_adapter *);
98static int ena_free_tx_dma_tag(struct ena_adapter *);
99static int ena_setup_rx_dma_tag(struct ena_adapter *);
100static int ena_free_rx_dma_tag(struct ena_adapter *);
101static int ena_setup_tx_resources(struct ena_adapter *, int);
102static void ena_free_tx_resources(struct ena_adapter *, int);
103static int ena_setup_all_tx_resources(struct ena_adapter *);
104static void ena_free_all_tx_resources(struct ena_adapter *);
1/*-
2 * BSD LICENSE
3 *
4 * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 88 unchanged lines hidden (view full) ---

97static int ena_setup_tx_dma_tag(struct ena_adapter *);
98static int ena_free_tx_dma_tag(struct ena_adapter *);
99static int ena_setup_rx_dma_tag(struct ena_adapter *);
100static int ena_free_rx_dma_tag(struct ena_adapter *);
101static int ena_setup_tx_resources(struct ena_adapter *, int);
102static void ena_free_tx_resources(struct ena_adapter *, int);
103static int ena_setup_all_tx_resources(struct ena_adapter *);
104static void ena_free_all_tx_resources(struct ena_adapter *);
105static inline int validate_rx_req_id(struct ena_ring *, uint16_t);
105static int ena_setup_rx_resources(struct ena_adapter *, unsigned int);
106static void ena_free_rx_resources(struct ena_adapter *, unsigned int);
107static int ena_setup_all_rx_resources(struct ena_adapter *);
108static void ena_free_all_rx_resources(struct ena_adapter *);
109static inline int ena_alloc_rx_mbuf(struct ena_adapter *, struct ena_ring *,
110 struct ena_rx_buffer *);
111static void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *,
112 struct ena_rx_buffer *);

--- 647 unchanged lines hidden (view full) ---

760 int i;
761
762 for (i = 0; i < adapter->num_queues; i++)
763 ena_free_tx_resources(adapter, i);
764
765 return;
766}
767
106static int ena_setup_rx_resources(struct ena_adapter *, unsigned int);
107static void ena_free_rx_resources(struct ena_adapter *, unsigned int);
108static int ena_setup_all_rx_resources(struct ena_adapter *);
109static void ena_free_all_rx_resources(struct ena_adapter *);
110static inline int ena_alloc_rx_mbuf(struct ena_adapter *, struct ena_ring *,
111 struct ena_rx_buffer *);
112static void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *,
113 struct ena_rx_buffer *);

--- 647 unchanged lines hidden (view full) ---

761 int i;
762
763 for (i = 0; i < adapter->num_queues; i++)
764 ena_free_tx_resources(adapter, i);
765
766 return;
767}
768
769static inline int
770validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id)
771{
772 if (likely(req_id < rx_ring->ring_size))
773 return (0);
774
775 device_printf(rx_ring->adapter->pdev, "Invalid rx req_id: %hu\n",
776 req_id);
777 counter_u64_add(rx_ring->rx_stats.bad_req_id, 1);
778
779 /* Trigger device reset */
780 rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
781 rx_ring->adapter->trigger_reset = true;
782
783 return (EFAULT);
784}
785
768/**
769 * ena_setup_rx_resources - allocate Rx resources (Descriptors)
770 * @adapter: network interface device structure
771 * @qid: queue index
772 *
773 * Returns 0 on success, otherwise on failure.
774 **/
775static int

--- 11 unchanged lines hidden (view full) ---

787 /*
788 * Alloc extra element so in rx path
789 * we can always prefetch rx_info + 1
790 */
791 size += sizeof(struct ena_rx_buffer);
792
793 rx_ring->rx_buffer_info = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO);
794
786/**
787 * ena_setup_rx_resources - allocate Rx resources (Descriptors)
788 * @adapter: network interface device structure
789 * @qid: queue index
790 *
791 * Returns 0 on success, otherwise on failure.
792 **/
793static int

--- 11 unchanged lines hidden (view full) ---

805 /*
806 * Alloc extra element so in rx path
807 * we can always prefetch rx_info + 1
808 */
809 size += sizeof(struct ena_rx_buffer);
810
811 rx_ring->rx_buffer_info = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO);
812
813 size = sizeof(uint16_t) * rx_ring->ring_size;
814 rx_ring->free_rx_ids = malloc(size, M_DEVBUF, M_WAITOK);
815
816 for (i = 0; i < rx_ring->ring_size; i++)
817 rx_ring->free_rx_ids[i] = i;
818
795 /* Reset RX statistics. */
796 ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats,
797 sizeof(rx_ring->rx_stats));
798
799 rx_ring->next_to_clean = 0;
800 rx_ring->next_to_use = 0;
801
802 /* ... and create the buffer DMA maps */

--- 23 unchanged lines hidden (view full) ---

826 return (0);
827
828err_rx_dma:
829 while (i--) {
830 bus_dmamap_destroy(adapter->rx_buf_tag,
831 rx_ring->rx_buffer_info[i].map);
832 }
833
819 /* Reset RX statistics. */
820 ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats,
821 sizeof(rx_ring->rx_stats));
822
823 rx_ring->next_to_clean = 0;
824 rx_ring->next_to_use = 0;
825
826 /* ... and create the buffer DMA maps */

--- 23 unchanged lines hidden (view full) ---

850 return (0);
851
852err_rx_dma:
853 while (i--) {
854 bus_dmamap_destroy(adapter->rx_buf_tag,
855 rx_ring->rx_buffer_info[i].map);
856 }
857
858 free(rx_ring->free_rx_ids, M_DEVBUF);
859 rx_ring->free_rx_ids = NULL;
834 free(rx_ring->rx_buffer_info, M_DEVBUF);
835 rx_ring->rx_buffer_info = NULL;
836 ena_trace(ENA_ALERT, "RX resource allocation fail");
837 return (ENOMEM);
838}
839
840/**
841 * ena_free_rx_resources - Free Rx Resources

--- 21 unchanged lines hidden (view full) ---

863
864 /* free LRO resources, */
865 tcp_lro_free(&rx_ring->lro);
866
867 /* free allocated memory */
868 free(rx_ring->rx_buffer_info, M_DEVBUF);
869 rx_ring->rx_buffer_info = NULL;
870
860 free(rx_ring->rx_buffer_info, M_DEVBUF);
861 rx_ring->rx_buffer_info = NULL;
862 ena_trace(ENA_ALERT, "RX resource allocation fail");
863 return (ENOMEM);
864}
865
866/**
867 * ena_free_rx_resources - Free Rx Resources

--- 21 unchanged lines hidden (view full) ---

889
890 /* free LRO resources, */
891 tcp_lro_free(&rx_ring->lro);
892
893 /* free allocated memory */
894 free(rx_ring->rx_buffer_info, M_DEVBUF);
895 rx_ring->rx_buffer_info = NULL;
896
897 free(rx_ring->free_rx_ids, M_DEVBUF);
898 rx_ring->free_rx_ids = NULL;
899
871 return;
872}
873
874/**
875 * ena_setup_all_rx_resources - allocate all queues Rx resources
876 * @adapter: network interface device structure
877 *
878 * Returns 0 on success, otherwise on failure.

--- 113 unchanged lines hidden (view full) ---

992 * @rx_ring: the ring which we want to feed with free descriptors
993 * @num: number of descriptors to refill
994 * Refills the ring with newly allocated DMA-mapped mbufs for receiving
995 **/
996static int
997ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)
998{
999 struct ena_adapter *adapter = rx_ring->adapter;
900 return;
901}
902
903/**
904 * ena_setup_all_rx_resources - allocate all queues Rx resources
905 * @adapter: network interface device structure
906 *
907 * Returns 0 on success, otherwise on failure.

--- 113 unchanged lines hidden (view full) ---

1021 * @rx_ring: the ring which we want to feed with free descriptors
1022 * @num: number of descriptors to refill
1023 * Refills the ring with newly allocated DMA-mapped mbufs for receiving
1024 **/
1025static int
1026ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)
1027{
1028 struct ena_adapter *adapter = rx_ring->adapter;
1000 uint16_t next_to_use;
1029 uint16_t next_to_use, req_id;
1001 uint32_t i;
1002 int rc;
1003
1004 ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC, "refill qid: %d",
1005 rx_ring->qid);
1006
1007 next_to_use = rx_ring->next_to_use;
1008
1009 for (i = 0; i < num; i++) {
1030 uint32_t i;
1031 int rc;
1032
1033 ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC, "refill qid: %d",
1034 rx_ring->qid);
1035
1036 next_to_use = rx_ring->next_to_use;
1037
1038 for (i = 0; i < num; i++) {
1039 struct ena_rx_buffer *rx_info;
1040
1010 ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC,
1011 "RX buffer - next to use: %d", next_to_use);
1012
1041 ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC,
1042 "RX buffer - next to use: %d", next_to_use);
1043
1013 struct ena_rx_buffer *rx_info =
1014 &rx_ring->rx_buffer_info[next_to_use];
1044 req_id = rx_ring->free_rx_ids[next_to_use];
1045 rc = validate_rx_req_id(rx_ring, req_id);
1046 if (unlikely(rc))
1047 break;
1015
1048
1049 rx_info = &rx_ring->rx_buffer_info[req_id];
1050
1016 rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);
1017 if (rc != 0) {
1018 device_printf(adapter->pdev,
1019 "failed to alloc buffer for rx queue\n");
1020 break;
1021 }
1022 rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
1051 rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);
1052 if (rc != 0) {
1053 device_printf(adapter->pdev,
1054 "failed to alloc buffer for rx queue\n");
1055 break;
1056 }
1057 rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
1023 &rx_info->ena_buf, next_to_use);
1058 &rx_info->ena_buf, req_id);
1024 if (unlikely(rc)) {
1025 device_printf(adapter->pdev,
1026 "failed to add buffer for rx queue %d\n",
1027 rx_ring->qid);
1028 break;
1029 }
1030 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,
1031 rx_ring->ring_size);

--- 367 unchanged lines hidden (view full) ---

1399 }
1400}
1401
1402/**
1403 * ena_rx_mbuf - assemble mbuf from descriptors
1404 * @rx_ring: ring for which we want to clean packets
1405 * @ena_bufs: buffer info
1406 * @ena_rx_ctx: metadata for this packet(s)
1059 if (unlikely(rc)) {
1060 device_printf(adapter->pdev,
1061 "failed to add buffer for rx queue %d\n",
1062 rx_ring->qid);
1063 break;
1064 }
1065 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,
1066 rx_ring->ring_size);

--- 367 unchanged lines hidden (view full) ---

1434 }
1435}
1436
1437/**
1438 * ena_rx_mbuf - assemble mbuf from descriptors
1439 * @rx_ring: ring for which we want to clean packets
1440 * @ena_bufs: buffer info
1441 * @ena_rx_ctx: metadata for this packet(s)
1407 * @next_to_clean: ring pointer
1442 * @next_to_clean: ring pointer, will be updated only upon success
1408 *
1409 **/
1410static struct mbuf*
1411ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
1412 struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean)
1413{
1414 struct mbuf *mbuf;
1415 struct ena_rx_buffer *rx_info;
1416 struct ena_adapter *adapter;
1443 *
1444 **/
1445static struct mbuf*
1446ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
1447 struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean)
1448{
1449 struct mbuf *mbuf;
1450 struct ena_rx_buffer *rx_info;
1451 struct ena_adapter *adapter;
1417 unsigned int len, buf = 0;
1418 unsigned int descs = ena_rx_ctx->descs;
1452 unsigned int descs = ena_rx_ctx->descs;
1453 uint16_t ntc, len, req_id, buf = 0;
1419
1454
1455 ntc = *next_to_clean;
1420 adapter = rx_ring->adapter;
1456 adapter = rx_ring->adapter;
1421 rx_info = &rx_ring->rx_buffer_info[*next_to_clean];
1457 rx_info = &rx_ring->rx_buffer_info[ntc];
1422
1458
1423 ENA_ASSERT(rx_info->mbuf, "Invalid alloc frag buffer\n");
1459 if (unlikely(rx_info->mbuf == NULL)) {
1460 device_printf(adapter->pdev, "NULL mbuf in rx_info");
1461 return (NULL);
1462 }
1424
1463
1425 len = ena_bufs[0].len;
1464 len = ena_bufs[buf].len;
1465 req_id = ena_bufs[buf].req_id;
1466 rx_info = &rx_ring->rx_buffer_info[req_id];
1467
1426 ena_trace(ENA_DBG | ENA_RXPTH, "rx_info %p, mbuf %p, paddr %jx",
1427 rx_info, rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr);
1428
1429 mbuf = rx_info->mbuf;
1430 mbuf->m_flags |= M_PKTHDR;
1431 mbuf->m_pkthdr.len = len;
1432 mbuf->m_len = len;
1433 mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp;
1434
1435 /* Fill mbuf with hash key and it's interpretation for optimization */
1436 ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf);
1437
1438 ena_trace(ENA_DBG | ENA_RXPTH, "rx mbuf 0x%p, flags=0x%x, len: %d",
1439 mbuf, mbuf->m_flags, mbuf->m_pkthdr.len);
1440
1441 /* DMA address is not needed anymore, unmap it */
1442 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
1443
1444 rx_info->mbuf = NULL;
1468 ena_trace(ENA_DBG | ENA_RXPTH, "rx_info %p, mbuf %p, paddr %jx",
1469 rx_info, rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr);
1470
1471 mbuf = rx_info->mbuf;
1472 mbuf->m_flags |= M_PKTHDR;
1473 mbuf->m_pkthdr.len = len;
1474 mbuf->m_len = len;
1475 mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp;
1476
1477 /* Fill mbuf with hash key and it's interpretation for optimization */
1478 ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf);
1479
1480 ena_trace(ENA_DBG | ENA_RXPTH, "rx mbuf 0x%p, flags=0x%x, len: %d",
1481 mbuf, mbuf->m_flags, mbuf->m_pkthdr.len);
1482
1483 /* DMA address is not needed anymore, unmap it */
1484 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
1485
1486 rx_info->mbuf = NULL;
1445 *next_to_clean = ENA_RX_RING_IDX_NEXT(*next_to_clean,
1446 rx_ring->ring_size);
1487 rx_ring->free_rx_ids[ntc] = req_id;
1488 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
1447
1448 /*
1449 * While we have more than 1 descriptors for one rcvd packet, append
1450 * other mbufs to the main one
1451 */
1452 while (--descs) {
1489
1490 /*
1491 * While we have more than 1 descriptors for one rcvd packet, append
1492 * other mbufs to the main one
1493 */
1494 while (--descs) {
1453 rx_info = &rx_ring->rx_buffer_info[*next_to_clean];
1454 len = ena_bufs[++buf].len;
1495 ++buf;
1496 len = ena_bufs[buf].len;
1497 req_id = ena_bufs[buf].req_id;
1498 rx_info = &rx_ring->rx_buffer_info[req_id];
1455
1499
1456 if (!m_append(mbuf, len, rx_info->mbuf->m_data)) {
1500 if (unlikely(rx_info->mbuf == NULL)) {
1501 device_printf(adapter->pdev, "NULL mbuf in rx_info");
1502 /*
1503 * If one of the required mbufs was not allocated yet,
1504 * we can break there.
1505 * All earlier used descriptors will be reallocated
1506 * later and not used mbufs can be reused.
1507 * The next_to_clean pointer will not be updated in case
1508 * of an error, so caller should advance it manually
1509 * in error handling routine to keep it up to date
1510 * with hw ring.
1511 */
1512 m_freem(mbuf);
1513 return (NULL);
1514 }
1515
1516 if (m_append(mbuf, len, rx_info->mbuf->m_data) == 0) {
1457 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
1458 ena_trace(ENA_WARNING, "Failed to append Rx mbuf %p",
1459 mbuf);
1460 }
1461 /* Free already appended mbuf, it won't be useful anymore */
1462 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
1463 m_freem(rx_info->mbuf);
1464 rx_info->mbuf = NULL;
1465
1517 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
1518 ena_trace(ENA_WARNING, "Failed to append Rx mbuf %p",
1519 mbuf);
1520 }
1521 /* Free already appended mbuf, it won't be useful anymore */
1522 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
1523 m_freem(rx_info->mbuf);
1524 rx_info->mbuf = NULL;
1525
1466 *next_to_clean = ENA_RX_RING_IDX_NEXT(*next_to_clean,
1467 rx_ring->ring_size);
1526 rx_ring->free_rx_ids[ntc] = req_id;
1527 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
1468 }
1469
1528 }
1529
1530 *next_to_clean = ntc;
1531
1470 return (mbuf);
1471}
1472
1473/**
1474 * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum
1475 **/
1476static inline void
1477ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,

--- 40 unchanged lines hidden (view full) ---

1518 /* struct ena_eth_io_intr_reg intr_reg; */
1519 if_t ifp;
1520 uint16_t ena_qid;
1521 uint16_t next_to_clean;
1522 uint32_t refill_required;
1523 uint32_t refill_threshold;
1524 uint32_t do_if_input = 0;
1525 unsigned int qid;
1532 return (mbuf);
1533}
1534
1535/**
1536 * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum
1537 **/
1538static inline void
1539ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,

--- 40 unchanged lines hidden (view full) ---

1580 /* struct ena_eth_io_intr_reg intr_reg; */
1581 if_t ifp;
1582 uint16_t ena_qid;
1583 uint16_t next_to_clean;
1584 uint32_t refill_required;
1585 uint32_t refill_threshold;
1586 uint32_t do_if_input = 0;
1587 unsigned int qid;
1526 int rc;
1588 int rc, i;
1527 int budget = RX_BUDGET;
1528
1529 adapter = rx_ring->que->adapter;
1530 ifp = adapter->ifp;
1531 qid = rx_ring->que->id;
1532 ena_qid = ENA_IO_RXQ_IDX(qid);
1533 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
1534 io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];

--- 12 unchanged lines hidden (view full) ---

1547 break;
1548
1549 /* Receive mbuf from the ring */
1550 mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs,
1551 &ena_rx_ctx, &next_to_clean);
1552
1553 /* Exit if we failed to retrieve a buffer */
1554 if (unlikely(!mbuf)) {
1589 int budget = RX_BUDGET;
1590
1591 adapter = rx_ring->que->adapter;
1592 ifp = adapter->ifp;
1593 qid = rx_ring->que->id;
1594 ena_qid = ENA_IO_RXQ_IDX(qid);
1595 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
1596 io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];

--- 12 unchanged lines hidden (view full) ---

1609 break;
1610
1611 /* Receive mbuf from the ring */
1612 mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs,
1613 &ena_rx_ctx, &next_to_clean);
1614
1615 /* Exit if we failed to retrieve a buffer */
1616 if (unlikely(!mbuf)) {
1555 next_to_clean = ENA_RX_RING_IDX_ADD(next_to_clean,
1556 ena_rx_ctx.descs, rx_ring->ring_size);
1617 for (i = 0; i < ena_rx_ctx.descs; ++i) {
1618 rx_ring->free_rx_ids[next_to_clean] =
1619 rx_ring->ena_bufs[i].req_id;
1620 next_to_clean =
1621 ENA_RX_RING_IDX_NEXT(next_to_clean,
1622 rx_ring->ring_size);
1623
1624 }
1557 break;
1558 }
1559 ena_trace(ENA_DBG | ENA_RXPTH, "Rx: %d bytes",
1560 mbuf->m_pkthdr.len);
1561
1562 if ((ifp->if_capenable & IFCAP_RXCSUM) ||
1563 (ifp->if_capenable & IFCAP_RXCSUM_IPV6)) {
1564 ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf);

--- 2213 unchanged lines hidden --- 		1625 break;
1626 }
1627 ena_trace(ENA_DBG | ENA_RXPTH, "Rx: %d bytes",
1628 mbuf->m_pkthdr.len);
1629
1630 if ((ifp->if_capenable & IFCAP_RXCSUM) ||
1631 (ifp->if_capenable & IFCAP_RXCSUM_IPV6)) {
1632 ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf);

--- 2213 unchanged lines hidden ---