ena.c (56d41ad5fe9c2fcbc2363bf3ca7e69684a26b4bd) ena.c (9762a033da9b457ce82d26364d7ab8c271d5862b)
1/*-
2 * BSD LICENSE
3 *
4 * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 394 unchanged lines hidden (view full) ---

403 txr = &adapter->tx_ring[i];
404 rxr = &adapter->rx_ring[i];
405
406 /* TX/RX common ring state */
407 ena_init_io_rings_common(adapter, txr, i);
408 ena_init_io_rings_common(adapter, rxr, i);
409
410 /* TX specific ring state */
1/*-
2 * BSD LICENSE
3 *
4 * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 394 unchanged lines hidden (view full) ---

403 txr = &adapter->tx_ring[i];
404 rxr = &adapter->rx_ring[i];
405
406 /* TX/RX common ring state */
407 ena_init_io_rings_common(adapter, txr, i);
408 ena_init_io_rings_common(adapter, rxr, i);
409
410 /* TX specific ring state */
411 txr->ring_size = adapter->tx_ring_size;
412 txr->tx_max_header_size = ena_dev->tx_max_header_size;
413 txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
414
411 txr->tx_max_header_size = ena_dev->tx_max_header_size;
412 txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
413
415 /* RX specific ring state */
416 rxr->ring_size = adapter->rx_ring_size;
417
418 que = &adapter->que[i];
419 que->adapter = adapter;
420 que->id = i;
421 que->tx_ring = txr;
422 que->rx_ring = rxr;
423
424 txr->que = que;
425 rxr->que = que;

--- 762 unchanged lines hidden (view full) ---

1188 uint32_t new_rx_size)
1189{
1190 uint32_t old_tx_size, old_rx_size;
1191 int rc = 0;
1192 bool dev_was_up;
1193
1194 ENA_LOCK_LOCK(adapter);
1195
414 que = &adapter->que[i];
415 que->adapter = adapter;
416 que->id = i;
417 que->tx_ring = txr;
418 que->rx_ring = rxr;
419
420 txr->que = que;
421 rxr->que = que;

--- 762 unchanged lines hidden (view full) ---

1184 uint32_t new_rx_size)
1185{
1186 uint32_t old_tx_size, old_rx_size;
1187 int rc = 0;
1188 bool dev_was_up;
1189
1190 ENA_LOCK_LOCK(adapter);
1191
1196 old_tx_size = adapter->tx_ring_size;
1197 old_rx_size = adapter->rx_ring_size;
1198 adapter->tx_ring_size = new_tx_size;
1199 adapter->rx_ring_size = new_rx_size;
1192 old_tx_size = adapter->requested_tx_ring_size;
1193 old_rx_size = adapter->requested_rx_ring_size;
1194 adapter->requested_tx_ring_size = new_tx_size;
1195 adapter->requested_rx_ring_size = new_rx_size;
1200
1201 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
1202 ena_down(adapter);
1203
1204 /* Configure queues with new size. */
1205 ena_init_io_rings_basic(adapter);
1206 if (dev_was_up) {
1207 rc = ena_up(adapter);
1208 if (unlikely(rc != 0)) {
1209 device_printf(adapter->pdev,
1210 "Failed to configure device with the new sizes - Tx: %u Rx: %u. Reverting old values - Tx: %u Rx: %u\n",
1211 new_tx_size, new_rx_size, old_tx_size, old_rx_size);
1212
1213 /* Revert old size. */
1196
1197 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
1198 ena_down(adapter);
1199
1200 /* Configure queues with new size. */
1201 ena_init_io_rings_basic(adapter);
1202 if (dev_was_up) {
1203 rc = ena_up(adapter);
1204 if (unlikely(rc != 0)) {
1205 device_printf(adapter->pdev,
1206 "Failed to configure device with the new sizes - Tx: %u Rx: %u. Reverting old values - Tx: %u Rx: %u\n",
1207 new_tx_size, new_rx_size, old_tx_size, old_rx_size);
1208
1209 /* Revert old size. */
1214 adapter->tx_ring_size = old_tx_size;
1215 adapter->rx_ring_size = old_rx_size;
1210 adapter->requested_tx_ring_size = old_tx_size;
1211 adapter->requested_rx_ring_size = old_rx_size;
1216 ena_init_io_rings_basic(adapter);
1217
1218 /* And try again. */
1219 rc = ena_up(adapter);
1220 if (unlikely(rc != 0)) {
1221 device_printf(adapter->pdev,
1222 "Failed to revert old queue sizes. Triggering device reset.\n");
1223 /*

--- 226 unchanged lines hidden (view full) ---

1450 int rc, i;
1451
1452 /* Create TX queues */
1453 for (i = 0; i < adapter->num_io_queues; i++) {
1454 msix_vector = ENA_IO_IRQ_IDX(i);
1455 ena_qid = ENA_IO_TXQ_IDX(i);
1456 ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
1457 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
1212 ena_init_io_rings_basic(adapter);
1213
1214 /* And try again. */
1215 rc = ena_up(adapter);
1216 if (unlikely(rc != 0)) {
1217 device_printf(adapter->pdev,
1218 "Failed to revert old queue sizes. Triggering device reset.\n");
1219 /*

--- 226 unchanged lines hidden (view full) ---

1446 int rc, i;
1447
1448 /* Create TX queues */
1449 for (i = 0; i < adapter->num_io_queues; i++) {
1450 msix_vector = ENA_IO_IRQ_IDX(i);
1451 ena_qid = ENA_IO_TXQ_IDX(i);
1452 ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
1453 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
1458 ctx.queue_size = adapter->tx_ring_size;
1454 ctx.queue_size = adapter->requested_tx_ring_size;
1459 ctx.msix_vector = msix_vector;
1460 ctx.qid = ena_qid;
1461 rc = ena_com_create_io_queue(ena_dev, &ctx);
1462 if (rc != 0) {
1463 device_printf(adapter->pdev,
1464 "Failed to create io TX queue #%d rc: %d\n", i, rc);
1465 goto err_tx;
1466 }

--- 11 unchanged lines hidden (view full) ---

1478 }
1479
1480 /* Create RX queues */
1481 for (i = 0; i < adapter->num_io_queues; i++) {
1482 msix_vector = ENA_IO_IRQ_IDX(i);
1483 ena_qid = ENA_IO_RXQ_IDX(i);
1484 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
1485 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
1455 ctx.msix_vector = msix_vector;
1456 ctx.qid = ena_qid;
1457 rc = ena_com_create_io_queue(ena_dev, &ctx);
1458 if (rc != 0) {
1459 device_printf(adapter->pdev,
1460 "Failed to create io TX queue #%d rc: %d\n", i, rc);
1461 goto err_tx;
1462 }

--- 11 unchanged lines hidden (view full) ---

1474 }
1475
1476 /* Create RX queues */
1477 for (i = 0; i < adapter->num_io_queues; i++) {
1478 msix_vector = ENA_IO_IRQ_IDX(i);
1479 ena_qid = ENA_IO_RXQ_IDX(i);
1480 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
1481 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
1486 ctx.queue_size = adapter->rx_ring_size;
1482 ctx.queue_size = adapter->requested_rx_ring_size;
1487 ctx.msix_vector = msix_vector;
1488 ctx.qid = ena_qid;
1489 rc = ena_com_create_io_queue(ena_dev, &ctx);
1490 if (unlikely(rc != 0)) {
1491 device_printf(adapter->pdev,
1492 "Failed to create io RX queue[%d] rc: %d\n", i, rc);
1493 goto err_rx;
1494 }

--- 475 unchanged lines hidden (view full) ---

1970
1971 ena_refill_all_rx_bufs(adapter);
1972 ena_reset_counters((counter_u64_t *)&adapter->hw_stats,
1973 sizeof(adapter->hw_stats));
1974
1975 return (0);
1976}
1977
1483 ctx.msix_vector = msix_vector;
1484 ctx.qid = ena_qid;
1485 rc = ena_com_create_io_queue(ena_dev, &ctx);
1486 if (unlikely(rc != 0)) {
1487 device_printf(adapter->pdev,
1488 "Failed to create io RX queue[%d] rc: %d\n", i, rc);
1489 goto err_rx;
1490 }

--- 475 unchanged lines hidden (view full) ---

1966
1967 ena_refill_all_rx_bufs(adapter);
1968 ena_reset_counters((counter_u64_t *)&adapter->hw_stats,
1969 sizeof(adapter->hw_stats));
1970
1971 return (0);
1972}
1973
1974static void
1975set_io_rings_size(struct ena_adapter *adapter, int new_tx_size,
1976 int new_rx_size)
1977{
1978 int i;
1979
1980 for (i = 0; i < adapter->num_io_queues; i++) {
1981 adapter->tx_ring[i].ring_size = new_tx_size;
1982 adapter->rx_ring[i].ring_size = new_rx_size;
1983 }
1984}
1985
1986static int
1987create_queues_with_size_backoff(struct ena_adapter *adapter)
1988{
1989 int rc;
1990 uint32_t cur_rx_ring_size, cur_tx_ring_size;
1991 uint32_t new_rx_ring_size, new_tx_ring_size;
1992
1993 /*
1994 * Current queue sizes might be set to smaller than the requested
1995 * ones due to past queue allocation failures.
1996 */
1997 set_io_rings_size(adapter, adapter->requested_tx_ring_size,
1998 adapter->requested_rx_ring_size);
1999
2000 while (1) {
2001 /* Allocate transmit descriptors */
2002 rc = ena_setup_all_tx_resources(adapter);
2003 if (unlikely(rc != 0)) {
2004 ena_trace(ENA_ALERT, "err_setup_tx\n");
2005 goto err_setup_tx;
2006 }
2007
2008 /* Allocate receive descriptors */
2009 rc = ena_setup_all_rx_resources(adapter);
2010 if (unlikely(rc != 0)) {
2011 ena_trace(ENA_ALERT, "err_setup_rx\n");
2012 goto err_setup_rx;
2013 }
2014
2015 /* Create IO queues for Rx & Tx */
2016 rc = ena_create_io_queues(adapter);
2017 if (unlikely(rc != 0)) {
2018 ena_trace(ENA_ALERT,
2019 "create IO queues failed\n");
2020 goto err_io_que;
2021 }
2022
2023 return (0);
2024
2025err_io_que:
2026 ena_free_all_rx_resources(adapter);
2027err_setup_rx:
2028 ena_free_all_tx_resources(adapter);
2029err_setup_tx:
2030 /*
2031 * Lower the ring size if ENOMEM. Otherwise, return the
2032 * error straightaway.
2033 */
2034 if (unlikely(rc != ENOMEM)) {
2035 ena_trace(ENA_ALERT,
2036 "Queue creation failed with error code: %d\n", rc);
2037 return (rc);
2038 }
2039
2040 cur_tx_ring_size = adapter->tx_ring[0].ring_size;
2041 cur_rx_ring_size = adapter->rx_ring[0].ring_size;
2042
2043 device_printf(adapter->pdev,
2044 "Not enough memory to create queues with sizes TX=%d, RX=%d\n",
2045 cur_tx_ring_size, cur_rx_ring_size);
2046
2047 new_tx_ring_size = cur_tx_ring_size;
2048 new_rx_ring_size = cur_rx_ring_size;
2049
2050 /*
2051 * Decrease the size of a larger queue, or decrease both if they are
2052 * the same size.
2053 */
2054 if (cur_rx_ring_size <= cur_tx_ring_size)
2055 new_tx_ring_size = cur_tx_ring_size / 2;
2056 if (cur_rx_ring_size >= cur_tx_ring_size)
2057 new_rx_ring_size = cur_rx_ring_size / 2;
2058
2059 if (new_tx_ring_size < ENA_MIN_RING_SIZE ||
2060 new_rx_ring_size < ENA_MIN_RING_SIZE) {
2061 device_printf(adapter->pdev,
2062 "Queue creation failed with the smallest possible queue size"
2063 "of %d for both queues. Not retrying with smaller queues\n",
2064 ENA_MIN_RING_SIZE);
2065 return (rc);
2066 }
2067
2068 set_io_rings_size(adapter, new_tx_ring_size, new_rx_ring_size);
2069 }
2070}
2071
1978int
1979ena_up(struct ena_adapter *adapter)
1980{
1981 int rc = 0;
1982
1983 if (unlikely(device_is_attached(adapter->pdev) == 0)) {
1984 device_printf(adapter->pdev, "device is not attached!\n");
1985 return (ENXIO);

--- 12 unchanged lines hidden (view full) ---

1998 }
1999 rc = ena_request_io_irq(adapter);
2000 if (unlikely(rc != 0)) {
2001 ena_trace(ENA_ALERT, "err_req_irq\n");
2002 goto error;
2003 }
2004
2005 device_printf(adapter->pdev,
2072int
2073ena_up(struct ena_adapter *adapter)
2074{
2075 int rc = 0;
2076
2077 if (unlikely(device_is_attached(adapter->pdev) == 0)) {
2078 device_printf(adapter->pdev, "device is not attached!\n");
2079 return (ENXIO);

--- 12 unchanged lines hidden (view full) ---

2092 }
2093 rc = ena_request_io_irq(adapter);
2094 if (unlikely(rc != 0)) {
2095 ena_trace(ENA_ALERT, "err_req_irq\n");
2096 goto error;
2097 }
2098
2099 device_printf(adapter->pdev,
2006 "Creating %u IO queues. Rx queue size: %d, Tx queue size: %d, LLQ is %s\n",
2100 "Creating %u IO queues. Rx queue size: %d, Tx queue size: %d, "
2101 "LLQ is %s\n",
2007 adapter->num_io_queues,
2102 adapter->num_io_queues,
2008 adapter->rx_ring_size,
2009 adapter->tx_ring_size,
2010 (adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) ?
2011 "ENABLED" : "DISABLED");
2103 adapter->requested_rx_ring_size,
2104 adapter->requested_tx_ring_size,
2105 (adapter->ena_dev->tx_mem_queue_type ==
2106 ENA_ADMIN_PLACEMENT_POLICY_DEV) ? "ENABLED" : "DISABLED");
2012
2107
2013 /* allocate transmit descriptors */
2014 rc = ena_setup_all_tx_resources(adapter);
2108 rc = create_queues_with_size_backoff(adapter);
2015 if (unlikely(rc != 0)) {
2109 if (unlikely(rc != 0)) {
2016 ena_trace(ENA_ALERT, "err_setup_tx\n");
2017 goto err_setup_tx;
2018 }
2019
2020 /* allocate receive descriptors */
2021 rc = ena_setup_all_rx_resources(adapter);
2022 if (unlikely(rc != 0)) {
2023 ena_trace(ENA_ALERT, "err_setup_rx\n");
2024 goto err_setup_rx;
2025 }
2026
2027 /* create IO queues for Rx & Tx */
2028 rc = ena_create_io_queues(adapter);
2029 if (unlikely(rc != 0)) {
2030 ena_trace(ENA_ALERT,
2110 ena_trace(ENA_ALERT,
2031 "create IO queues failed\n");
2032 goto err_io_que;
2111 "error creating queues with size backoff\n");
2112 goto err_create_queues_with_backoff;
2033 }
2034
2035 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter))
2036 if_link_state_change(adapter->ifp, LINK_STATE_UP);
2037
2038 rc = ena_up_complete(adapter);
2039 if (unlikely(rc != 0))
2040 goto err_up_complete;

--- 17 unchanged lines hidden (view full) ---

2058 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter);
2059
2060 ena_unmask_all_io_irqs(adapter);
2061
2062 return (0);
2063
2064err_up_complete:
2065 ena_destroy_all_io_queues(adapter);
2113 }
2114
2115 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter))
2116 if_link_state_change(adapter->ifp, LINK_STATE_UP);
2117
2118 rc = ena_up_complete(adapter);
2119 if (unlikely(rc != 0))
2120 goto err_up_complete;

--- 17 unchanged lines hidden (view full) ---

2138 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter);
2139
2140 ena_unmask_all_io_irqs(adapter);
2141
2142 return (0);
2143
2144err_up_complete:
2145 ena_destroy_all_io_queues(adapter);
2066err_io_que:
2067 ena_free_all_rx_resources(adapter);
2146 ena_free_all_rx_resources(adapter);
2068err_setup_rx:
2069 ena_free_all_tx_resources(adapter);
2147 ena_free_all_tx_resources(adapter);
2070err_setup_tx:
2148err_create_queues_with_backoff:
2071 ena_free_io_irq(adapter);
2072error:
2073 return (rc);
2074}
2075
2076static uint64_t
2077ena_get_counter(if_t ifp, ift_counter cnt)
2078{

--- 245 unchanged lines hidden (view full) ---

2324 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST |
2325 IFF_KNOWSEPOCH);
2326 if_setinitfn(ifp, ena_init);
2327 if_settransmitfn(ifp, ena_mq_start);
2328 if_setqflushfn(ifp, ena_qflush);
2329 if_setioctlfn(ifp, ena_ioctl);
2330 if_setgetcounterfn(ifp, ena_get_counter);
2331
2149 ena_free_io_irq(adapter);
2150error:
2151 return (rc);
2152}
2153
2154static uint64_t
2155ena_get_counter(if_t ifp, ift_counter cnt)
2156{

--- 245 unchanged lines hidden (view full) ---

2402 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST |
2403 IFF_KNOWSEPOCH);
2404 if_setinitfn(ifp, ena_init);
2405 if_settransmitfn(ifp, ena_mq_start);
2406 if_setqflushfn(ifp, ena_qflush);
2407 if_setioctlfn(ifp, ena_ioctl);
2408 if_setgetcounterfn(ifp, ena_get_counter);
2409
2332 if_setsendqlen(ifp, adapter->tx_ring_size);
2410 if_setsendqlen(ifp, adapter->requested_tx_ring_size);
2333 if_setsendqready(ifp);
2334 if_setmtu(ifp, ETHERMTU);
2335 if_setbaudrate(ifp, 0);
2336 /* Zeroize capabilities... */
2337 if_setcapabilities(ifp, 0);
2338 if_setcapenable(ifp, 0);
2339 /* check hardware support */
2340 caps = ena_get_dev_offloads(feat);

--- 1115 unchanged lines hidden (view full) ---

3456 max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev,
3457 &get_feat_ctx);
3458 rc = ena_calc_io_queue_size(&calc_queue_ctx);
3459 if (unlikely((rc != 0) || (max_num_io_queues <= 0))) {
3460 rc = EFAULT;
3461 goto err_com_free;
3462 }
3463
2411 if_setsendqready(ifp);
2412 if_setmtu(ifp, ETHERMTU);
2413 if_setbaudrate(ifp, 0);
2414 /* Zeroize capabilities... */
2415 if_setcapabilities(ifp, 0);
2416 if_setcapenable(ifp, 0);
2417 /* check hardware support */
2418 caps = ena_get_dev_offloads(feat);

--- 1115 unchanged lines hidden (view full) ---

3534 max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev,
3535 &get_feat_ctx);
3536 rc = ena_calc_io_queue_size(&calc_queue_ctx);
3537 if (unlikely((rc != 0) || (max_num_io_queues <= 0))) {
3538 rc = EFAULT;
3539 goto err_com_free;
3540 }
3541
3464 adapter->tx_ring_size = calc_queue_ctx.tx_queue_size;
3465 adapter->rx_ring_size = calc_queue_ctx.rx_queue_size;
3542 adapter->requested_tx_ring_size = calc_queue_ctx.tx_queue_size;
3543 adapter->requested_rx_ring_size = calc_queue_ctx.rx_queue_size;
3466 adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size;
3467 adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size;
3468 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size;
3469 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size;
3470
3471 adapter->max_num_io_queues = max_num_io_queues;
3472
3473 adapter->buf_ring_size = ENA_DEFAULT_BUF_RING_SIZE;

--- 285 unchanged lines hidden --- 		3544 adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size;
3545 adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size;
3546 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size;
3547 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size;
3548
3549 adapter->max_num_io_queues = max_num_io_queues;
3550
3551 adapter->buf_ring_size = ENA_DEFAULT_BUF_RING_SIZE;

--- 285 unchanged lines hidden ---