ena.c (eb4c4f4a2e18659b67a6bf1ea5f473c7ed8c854f) ena.c (78554d0c707c9401dbae53fb8bc65d291a5a09a5)
1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 2087 unchanged lines hidden (view full) ---

2096 return (ENXIO);
2097 }
2098
2099 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
2100 return (0);
2101
2102 ena_log(adapter->pdev, INFO, "device is going UP\n");
2103
1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 2087 unchanged lines hidden (view full) ---

2096 return (ENXIO);
2097 }
2098
2099 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
2100 return (0);
2101
2102 ena_log(adapter->pdev, INFO, "device is going UP\n");
2103
2104 /*
2105 * ena_timer_service can use functions, which write to the admin queue.
2106 * Those calls are not protected by ENA_LOCK, and because of that, the
2107 * timer should be stopped when bringing the device up or down.
2108 */
2109 ENA_TIMER_DRAIN(adapter);
2110
2104 /* setup interrupts for IO queues */
2105 rc = ena_setup_io_intr(adapter);
2106 if (unlikely(rc != 0)) {
2107 ena_log(adapter->pdev, ERR, "error setting up IO interrupt\n");
2108 goto error;
2109 }
2110 rc = ena_request_io_irq(adapter);
2111 if (unlikely(rc != 0)) {

--- 26 unchanged lines hidden (view full) ---

2138
2139 counter_u64_add(adapter->dev_stats.interface_up, 1);
2140
2141 ena_update_hwassist(adapter);
2142
2143 if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING,
2144 IFF_DRV_OACTIVE);
2145
2111 /* setup interrupts for IO queues */
2112 rc = ena_setup_io_intr(adapter);
2113 if (unlikely(rc != 0)) {
2114 ena_log(adapter->pdev, ERR, "error setting up IO interrupt\n");
2115 goto error;
2116 }
2117 rc = ena_request_io_irq(adapter);
2118 if (unlikely(rc != 0)) {

--- 26 unchanged lines hidden (view full) ---

2145
2146 counter_u64_add(adapter->dev_stats.interface_up, 1);
2147
2148 ena_update_hwassist(adapter);
2149
2150 if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING,
2151 IFF_DRV_OACTIVE);
2152
2146 /* Activate timer service only if the device is running.
2147 * If this flag is not set, it means that the driver is being
2148 * reset and timer service will be activated afterwards.
2149 */
2150 if (ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)) {
2151 callout_reset_sbt(&adapter->timer_service, SBT_1S,
2152 SBT_1S, ena_timer_service, (void *)adapter, 0);
2153 }
2154
2155 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter);
2156
2157 ena_unmask_all_io_irqs(adapter);
2158
2153 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter);
2154
2155 ena_unmask_all_io_irqs(adapter);
2156
2157 ENA_TIMER_RESET(adapter);
2158
2159 return (0);
2160
2161err_up_complete:
2162 ena_destroy_all_io_queues(adapter);
2163 ena_free_all_rx_resources(adapter);
2164 ena_free_all_tx_resources(adapter);
2165err_create_queues_with_backoff:
2166 ena_free_io_irq(adapter);
2167error:
2159 return (0);
2160
2161err_up_complete:
2162 ena_destroy_all_io_queues(adapter);
2163 ena_free_all_rx_resources(adapter);
2164 ena_free_all_tx_resources(adapter);
2165err_create_queues_with_backoff:
2166 ena_free_io_irq(adapter);
2167error:
2168 ENA_TIMER_RESET(adapter);
2169
2168 return (rc);
2169}
2170
2171static uint64_t
2172ena_get_counter(if_t ifp, ift_counter cnt)
2173{
2174 struct ena_adapter *adapter;
2175 struct ena_hw_stats *stats;

--- 288 unchanged lines hidden (view full) ---

2464{
2465 int rc;
2466
2467 ENA_LOCK_ASSERT();
2468
2469 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
2470 return;
2471
2170 return (rc);
2171}
2172
2173static uint64_t
2174ena_get_counter(if_t ifp, ift_counter cnt)
2175{
2176 struct ena_adapter *adapter;
2177 struct ena_hw_stats *stats;

--- 288 unchanged lines hidden (view full) ---

2466{
2467 int rc;
2468
2469 ENA_LOCK_ASSERT();
2470
2471 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
2472 return;
2473
2474 /* Drain timer service to avoid admin queue race condition. */
2475 ENA_TIMER_DRAIN(adapter);
2476
2472 ena_log(adapter->pdev, INFO, "device is going DOWN\n");
2473
2477 ena_log(adapter->pdev, INFO, "device is going DOWN\n");
2478
2474 callout_drain(&adapter->timer_service);
2475
2476 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP, adapter);
2477 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE,
2478 IFF_DRV_RUNNING);
2479
2480 ena_free_io_irq(adapter);
2481
2482 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) {
2483 rc = ena_com_dev_reset(adapter->ena_dev,

--- 6 unchanged lines hidden (view full) ---

2490 ena_destroy_all_io_queues(adapter);
2491
2492 ena_free_all_tx_bufs(adapter);
2493 ena_free_all_rx_bufs(adapter);
2494 ena_free_all_tx_resources(adapter);
2495 ena_free_all_rx_resources(adapter);
2496
2497 counter_u64_add(adapter->dev_stats.interface_down, 1);
2479 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP, adapter);
2480 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE,
2481 IFF_DRV_RUNNING);
2482
2483 ena_free_io_irq(adapter);
2484
2485 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) {
2486 rc = ena_com_dev_reset(adapter->ena_dev,

--- 6 unchanged lines hidden (view full) ---

2493 ena_destroy_all_io_queues(adapter);
2494
2495 ena_free_all_tx_bufs(adapter);
2496 ena_free_all_rx_bufs(adapter);
2497 ena_free_all_tx_resources(adapter);
2498 ena_free_all_rx_resources(adapter);
2499
2500 counter_u64_add(adapter->dev_stats.interface_down, 1);
2501
2502 ENA_TIMER_RESET(adapter);
2498}
2499
2500static uint32_t
2501ena_calc_max_io_queue_num(device_t pdev, struct ena_com_dev *ena_dev,
2502 struct ena_com_dev_get_features_ctx *get_feat_ctx)
2503{
2504 uint32_t io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues;
2505

--- 738 unchanged lines hidden (view full) ---

3244 * As timer service is executed every second, it's enough to increment
3245 * appropriate counter each time the timer service is executed.
3246 */
3247 if ((adapter->eni_metrics_sample_interval != 0) &&
3248 (++adapter->eni_metrics_sample_interval_cnt >=
3249 adapter->eni_metrics_sample_interval)) {
3250 /*
3251 * There is no race with other admin queue calls, as:
2503}
2504
2505static uint32_t
2506ena_calc_max_io_queue_num(device_t pdev, struct ena_com_dev *ena_dev,
2507 struct ena_com_dev_get_features_ctx *get_feat_ctx)
2508{
2509 uint32_t io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues;
2510

--- 738 unchanged lines hidden (view full) ---

3249 * As timer service is executed every second, it's enough to increment
3250 * appropriate counter each time the timer service is executed.
3251 */
3252 if ((adapter->eni_metrics_sample_interval != 0) &&
3253 (++adapter->eni_metrics_sample_interval_cnt >=
3254 adapter->eni_metrics_sample_interval)) {
3255 /*
3256 * There is no race with other admin queue calls, as:
3252 * - Timer service runs after interface is up, so all
3257 * - Timer service runs after attach function ends, so all
3253 * configuration calls to the admin queue are finished.
3258 * configuration calls to the admin queue are finished.
3259 * - Timer service is temporarily stopped when bringing
3260 * the interface up or down.
3254 * - After interface is up, the driver doesn't use (at least
3255 * for now) other functions writing to the admin queue.
3256 *
3257 * It may change in the future, so in that situation, the lock
3258 * will be needed. ENA_LOCK_*() cannot be used for that purpose,
3259 * as callout ena_timer_service is protected by them. It could
3260 * lead to the deadlock if callout_drain() would hold the lock
3261 * before ena_copy_eni_metrics() was executed. It's advised to

--- 12 unchanged lines hidden (view full) ---

3274 ena_log(adapter->pdev, WARN, "Trigger reset is on\n");
3275 taskqueue_enqueue(adapter->reset_tq, &adapter->reset_task);
3276 return;
3277 }
3278
3279 /*
3280 * Schedule another timeout one second from now.
3281 */
3261 * - After interface is up, the driver doesn't use (at least
3262 * for now) other functions writing to the admin queue.
3263 *
3264 * It may change in the future, so in that situation, the lock
3265 * will be needed. ENA_LOCK_*() cannot be used for that purpose,
3266 * as callout ena_timer_service is protected by them. It could
3267 * lead to the deadlock if callout_drain() would hold the lock
3268 * before ena_copy_eni_metrics() was executed. It's advised to

--- 12 unchanged lines hidden (view full) ---

3281 ena_log(adapter->pdev, WARN, "Trigger reset is on\n");
3282 taskqueue_enqueue(adapter->reset_tq, &adapter->reset_task);
3283 return;
3284 }
3285
3286 /*
3287 * Schedule another timeout one second from now.
3288 */
3282 callout_schedule_sbt(&adapter->timer_service, SBT_1S, SBT_1S, 0);
3289 ENA_TIMER_RESET(adapter);
3283}
3284
3285void
3286ena_destroy_device(struct ena_adapter *adapter, bool graceful)
3287{
3288 if_t ifp = adapter->ifp;
3289 struct ena_com_dev *ena_dev = adapter->ena_dev;
3290 bool dev_up;
3291
3292 if (!ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter))
3293 return;
3294
3295 if_link_state_change(ifp, LINK_STATE_DOWN);
3296
3290}
3291
3292void
3293ena_destroy_device(struct ena_adapter *adapter, bool graceful)
3294{
3295 if_t ifp = adapter->ifp;
3296 struct ena_com_dev *ena_dev = adapter->ena_dev;
3297 bool dev_up;
3298
3299 if (!ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter))
3300 return;
3301
3302 if_link_state_change(ifp, LINK_STATE_DOWN);
3303
3297 callout_drain(&adapter->timer_service);
3304 ENA_TIMER_DRAIN(adapter);
3298
3299 dev_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
3300 if (dev_up)
3301 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
3302
3303 if (!graceful)
3304 ena_com_set_admin_running_state(ena_dev, false);
3305

--- 114 unchanged lines hidden (view full) ---

3420 ena_log(dev, ERR, "Failed to create I/O queues\n");
3421 goto err_disable_msix;
3422 }
3423 }
3424
3425 /* Indicate that device is running again and ready to work */
3426 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3427
3305
3306 dev_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
3307 if (dev_up)
3308 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
3309
3310 if (!graceful)
3311 ena_com_set_admin_running_state(ena_dev, false);
3312

--- 114 unchanged lines hidden (view full) ---

3427 ena_log(dev, ERR, "Failed to create I/O queues\n");
3428 goto err_disable_msix;
3429 }
3430 }
3431
3432 /* Indicate that device is running again and ready to work */
3433 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3434
3428 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter)) {
3429 /*
3430 * As the AENQ handlers weren't executed during reset because
3431 * the flag ENA_FLAG_DEVICE_RUNNING was turned off, the
3432 * timestamp must be updated again That will prevent next reset
3433 * caused by missing keep alive.
3434 */
3435 adapter->keep_alive_timestamp = getsbinuptime();
3436 callout_reset_sbt(&adapter->timer_service, SBT_1S, SBT_1S,
3437 ena_timer_service, (void *)adapter, 0);
3438 }
3435 /*
3436 * As the AENQ handlers weren't executed during reset because
3437 * the flag ENA_FLAG_DEVICE_RUNNING was turned off, the
3438 * timestamp must be updated again That will prevent next reset
3439 * caused by missing keep alive.
3440 */
3441 adapter->keep_alive_timestamp = getsbinuptime();
3442 ENA_TIMER_RESET(adapter);
3443
3439 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
3440
3441 ena_log(dev, INFO,
3442 "Device reset completed successfully, Driver info: %s\n", ena_version);
3443
3444 return (rc);
3445
3446err_disable_msix:

--- 5 unchanged lines hidden (view full) ---

3452 ena_com_admin_destroy(ena_dev);
3453 ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE);
3454 ena_com_mmio_reg_read_request_destroy(ena_dev);
3455err:
3456 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3457 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter);
3458 ena_log(dev, ERR, "Reset attempt failed. Can not reset the device\n");
3459
3444 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
3445
3446 ena_log(dev, INFO,
3447 "Device reset completed successfully, Driver info: %s\n", ena_version);
3448
3449 return (rc);
3450
3451err_disable_msix:

--- 5 unchanged lines hidden (view full) ---

3457 ena_com_admin_destroy(ena_dev);
3458 ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE);
3459 ena_com_mmio_reg_read_request_destroy(ena_dev);
3460err:
3461 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3462 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter);
3463 ena_log(dev, ERR, "Reset attempt failed. Can not reset the device\n");
3464
3465 ENA_TIMER_RESET(adapter);
3466
3460 return (rc);
3461}
3462
3463static void
3464ena_reset_task(void *arg, int pending)
3465{
3466 struct ena_adapter *adapter = (struct ena_adapter *)arg;
3467

--- 31 unchanged lines hidden (view full) ---

3499 adapter = device_get_softc(pdev);
3500 adapter->pdev = pdev;
3501 adapter->first_bind = -1;
3502
3503 /*
3504 * Set up the timer service - driver is responsible for avoiding
3505 * concurrency, as the callout won't be using any locking inside.
3506 */
3467 return (rc);
3468}
3469
3470static void
3471ena_reset_task(void *arg, int pending)
3472{
3473 struct ena_adapter *adapter = (struct ena_adapter *)arg;
3474

--- 31 unchanged lines hidden (view full) ---

3506 adapter = device_get_softc(pdev);
3507 adapter->pdev = pdev;
3508 adapter->first_bind = -1;
3509
3510 /*
3511 * Set up the timer service - driver is responsible for avoiding
3512 * concurrency, as the callout won't be using any locking inside.
3513 */
3507 callout_init(&adapter->timer_service, true);
3514 ENA_TIMER_INIT(adapter);
3508 adapter->keep_alive_timeout = DEFAULT_KEEP_ALIVE_TO;
3509 adapter->missing_tx_timeout = DEFAULT_TX_CMP_TO;
3510 adapter->missing_tx_max_queues = DEFAULT_TX_MONITORED_QUEUES;
3511 adapter->missing_tx_threshold = DEFAULT_TX_CMP_THRESHOLD;
3512
3513 if (version_printed++ == 0)
3514 ena_log(pdev, INFO, "%s\n", ena_version);
3515

--- 168 unchanged lines hidden (view full) ---

3684 goto err_detach;
3685 }
3686#endif /* DEV_NETMAP */
3687
3688 /* Tell the stack that the interface is not active */
3689 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
3690 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3691
3515 adapter->keep_alive_timeout = DEFAULT_KEEP_ALIVE_TO;
3516 adapter->missing_tx_timeout = DEFAULT_TX_CMP_TO;
3517 adapter->missing_tx_max_queues = DEFAULT_TX_MONITORED_QUEUES;
3518 adapter->missing_tx_threshold = DEFAULT_TX_CMP_THRESHOLD;
3519
3520 if (version_printed++ == 0)
3521 ena_log(pdev, INFO, "%s\n", ena_version);
3522

--- 168 unchanged lines hidden (view full) ---

3691 goto err_detach;
3692 }
3693#endif /* DEV_NETMAP */
3694
3695 /* Tell the stack that the interface is not active */
3696 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
3697 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3698
3699 /* Run the timer service */
3700 ENA_TIMER_RESET(adapter);
3701
3692 return (0);
3693
3694#ifdef DEV_NETMAP
3695err_detach:
3696 ether_ifdetach(adapter->ifp);
3697#endif /* DEV_NETMAP */
3698err_msix_free:
3699 ena_com_dev_reset(adapter->ena_dev, ENA_REGS_RESET_INIT_ERR);

--- 37 unchanged lines hidden (view full) ---

3737 ena_log(adapter->pdev, ERR, "VLAN is in use, detach first\n");
3738 return (EBUSY);
3739 }
3740
3741 ether_ifdetach(adapter->ifp);
3742
3743 /* Stop timer service */
3744 ENA_LOCK_LOCK();
3702 return (0);
3703
3704#ifdef DEV_NETMAP
3705err_detach:
3706 ether_ifdetach(adapter->ifp);
3707#endif /* DEV_NETMAP */
3708err_msix_free:
3709 ena_com_dev_reset(adapter->ena_dev, ENA_REGS_RESET_INIT_ERR);

--- 37 unchanged lines hidden (view full) ---

3747 ena_log(adapter->pdev, ERR, "VLAN is in use, detach first\n");
3748 return (EBUSY);
3749 }
3750
3751 ether_ifdetach(adapter->ifp);
3752
3753 /* Stop timer service */
3754 ENA_LOCK_LOCK();
3745 callout_drain(&adapter->timer_service);
3755 ENA_TIMER_DRAIN(adapter);
3746 ENA_LOCK_UNLOCK();
3747
3748 /* Release reset task */
3749 while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL))
3750 taskqueue_drain(adapter->reset_tq, &adapter->reset_task);
3751 taskqueue_free(adapter->reset_tq);
3752
3753 ENA_LOCK_LOCK();

--- 167 unchanged lines hidden --- 		3756 ENA_LOCK_UNLOCK();
3757
3758 /* Release reset task */
3759 while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL))
3760 taskqueue_drain(adapter->reset_tq, &adapter->reset_task);
3761 taskqueue_free(adapter->reset_tq);
3762
3763 ENA_LOCK_LOCK();

--- 167 unchanged lines hidden ---