1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Atlantic Network Driver 3 * 4 * Copyright (C) 2014-2019 aQuantia Corporation 5 * Copyright (C) 2019-2020 Marvell International Ltd. 6 */ 7 8 /* File aq_vec.c: Definition of common structure for vector of Rx and Tx rings. 9 * Definition of functions for Rx and Tx rings. Friendly module for aq_nic. 10 */ 11 12 #include "aq_vec.h" 13 14 struct aq_vec_s { 15 const struct aq_hw_ops *aq_hw_ops; 16 struct aq_hw_s *aq_hw; 17 struct aq_nic_s *aq_nic; 18 unsigned int tx_rings; 19 unsigned int rx_rings; 20 struct aq_ring_param_s aq_ring_param; 21 struct napi_struct napi; 22 struct aq_ring_s ring[AQ_CFG_TCS_MAX][2]; 23 }; 24 25 #define AQ_VEC_TX_ID 0 26 #define AQ_VEC_RX_ID 1 27 28 static int aq_vec_poll(struct napi_struct *napi, int budget) 29 { 30 struct aq_vec_s *self = container_of(napi, struct aq_vec_s, napi); 31 unsigned int sw_tail_old = 0U; 32 struct aq_ring_s *ring = NULL; 33 bool was_tx_cleaned = true; 34 unsigned int i = 0U; 35 int work_done = 0; 36 int err = 0; 37 38 if (!self) { 39 err = -EINVAL; 40 } else { 41 for (i = 0U; self->tx_rings > i; ++i) { 42 ring = self->ring[i]; 43 u64_stats_update_begin(&ring[AQ_VEC_RX_ID].stats.rx.syncp); 44 ring[AQ_VEC_RX_ID].stats.rx.polls++; 45 u64_stats_update_end(&ring[AQ_VEC_RX_ID].stats.rx.syncp); 46 if (self->aq_hw_ops->hw_ring_tx_head_update) { 47 err = self->aq_hw_ops->hw_ring_tx_head_update( 48 self->aq_hw, 49 &ring[AQ_VEC_TX_ID]); 50 if (err < 0) 51 goto err_exit; 52 } 53 54 if (ring[AQ_VEC_TX_ID].sw_head != 55 ring[AQ_VEC_TX_ID].hw_head) { 56 was_tx_cleaned = aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]); 57 aq_ring_update_queue_state(&ring[AQ_VEC_TX_ID]); 58 } 59 60 err = self->aq_hw_ops->hw_ring_rx_receive(self->aq_hw, 61 &ring[AQ_VEC_RX_ID]); 62 if (err < 0) 63 goto err_exit; 64 65 if (ring[AQ_VEC_RX_ID].sw_head != 66 ring[AQ_VEC_RX_ID].hw_head) { 67 err = aq_ring_rx_clean(&ring[AQ_VEC_RX_ID], 68 napi, 69 &work_done, 70 budget - work_done); 71 if (err < 0) 72 goto err_exit; 73 74 sw_tail_old = ring[AQ_VEC_RX_ID].sw_tail; 75 76 err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]); 77 if (err < 0) 78 goto err_exit; 79 80 err = self->aq_hw_ops->hw_ring_rx_fill( 81 self->aq_hw, 82 &ring[AQ_VEC_RX_ID], sw_tail_old); 83 if (err < 0) 84 goto err_exit; 85 } 86 } 87 88 err_exit: 89 if (!was_tx_cleaned) 90 work_done = budget; 91 92 if (work_done < budget) { 93 napi_complete_done(napi, work_done); 94 self->aq_hw_ops->hw_irq_enable(self->aq_hw, 95 1U << self->aq_ring_param.vec_idx); 96 } 97 } 98 99 return work_done; 100 } 101 102 struct aq_vec_s *aq_vec_alloc(struct aq_nic_s *aq_nic, unsigned int idx, 103 struct aq_nic_cfg_s *aq_nic_cfg) 104 { 105 struct aq_vec_s *self = NULL; 106 107 self = kzalloc(sizeof(*self), GFP_KERNEL); 108 if (!self) 109 goto err_exit; 110 111 self->aq_nic = aq_nic; 112 self->aq_ring_param.vec_idx = idx; 113 self->aq_ring_param.cpu = 114 idx + aq_nic_cfg->aq_rss.base_cpu_number; 115 116 cpumask_set_cpu(self->aq_ring_param.cpu, 117 &self->aq_ring_param.affinity_mask); 118 119 self->tx_rings = 0; 120 self->rx_rings = 0; 121 122 netif_napi_add(aq_nic_get_ndev(aq_nic), &self->napi, aq_vec_poll); 123 124 err_exit: 125 return self; 126 } 127 128 int aq_vec_ring_alloc(struct aq_vec_s *self, struct aq_nic_s *aq_nic, 129 unsigned int idx, struct aq_nic_cfg_s *aq_nic_cfg) 130 { 131 struct aq_ring_s *ring = NULL; 132 unsigned int i = 0U; 133 int err = 0; 134 135 for (i = 0; i < aq_nic_cfg->tcs; ++i) { 136 const unsigned int idx_ring = AQ_NIC_CFG_TCVEC2RING(aq_nic_cfg, 137 i, idx); 138 139 ring = &self->ring[i][AQ_VEC_TX_ID]; 140 err = aq_ring_tx_alloc(ring, aq_nic, idx_ring, aq_nic_cfg); 141 if (err) 142 goto err_exit; 143 144 ++self->tx_rings; 145 146 aq_nic_set_tx_ring(aq_nic, idx_ring, ring); 147 148 ring = &self->ring[i][AQ_VEC_RX_ID]; 149 if (xdp_rxq_info_reg(&ring->xdp_rxq, 150 aq_nic->ndev, idx, 151 self->napi.napi_id) < 0) { 152 err = -ENOMEM; 153 goto err_exit; 154 } 155 if (xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 156 MEM_TYPE_PAGE_SHARED, NULL) < 0) { 157 xdp_rxq_info_unreg(&ring->xdp_rxq); 158 err = -ENOMEM; 159 goto err_exit; 160 } 161 162 err = aq_ring_rx_alloc(ring, aq_nic, idx_ring, aq_nic_cfg); 163 if (err) { 164 xdp_rxq_info_unreg(&ring->xdp_rxq); 165 goto err_exit; 166 } 167 168 ++self->rx_rings; 169 } 170 171 err_exit: 172 if (err < 0) { 173 aq_vec_ring_free(self); 174 self = NULL; 175 } 176 177 return err; 178 } 179 180 int aq_vec_init(struct aq_vec_s *self, const struct aq_hw_ops *aq_hw_ops, 181 struct aq_hw_s *aq_hw) 182 { 183 struct aq_ring_s *ring = NULL; 184 unsigned int i = 0U; 185 int err = 0; 186 187 self->aq_hw_ops = aq_hw_ops; 188 self->aq_hw = aq_hw; 189 190 for (i = 0U; self->tx_rings > i; ++i) { 191 ring = self->ring[i]; 192 err = aq_ring_init(&ring[AQ_VEC_TX_ID], ATL_RING_TX); 193 if (err < 0) 194 goto err_exit; 195 196 err = self->aq_hw_ops->hw_ring_tx_init(self->aq_hw, 197 &ring[AQ_VEC_TX_ID], 198 &self->aq_ring_param); 199 if (err < 0) 200 goto err_exit; 201 202 err = aq_ring_init(&ring[AQ_VEC_RX_ID], ATL_RING_RX); 203 if (err < 0) 204 goto err_exit; 205 206 err = self->aq_hw_ops->hw_ring_rx_init(self->aq_hw, 207 &ring[AQ_VEC_RX_ID], 208 &self->aq_ring_param); 209 if (err < 0) 210 goto err_exit; 211 212 err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]); 213 if (err < 0) 214 goto err_exit; 215 216 err = self->aq_hw_ops->hw_ring_rx_fill(self->aq_hw, 217 &ring[AQ_VEC_RX_ID], 0U); 218 if (err < 0) 219 goto err_exit; 220 } 221 222 err_exit: 223 return err; 224 } 225 226 int aq_vec_start(struct aq_vec_s *self) 227 { 228 struct aq_ring_s *ring = NULL; 229 unsigned int i = 0U; 230 int err = 0; 231 232 for (i = 0U; self->tx_rings > i; ++i) { 233 ring = self->ring[i]; 234 err = self->aq_hw_ops->hw_ring_tx_start(self->aq_hw, 235 &ring[AQ_VEC_TX_ID]); 236 if (err < 0) 237 goto err_exit; 238 239 err = self->aq_hw_ops->hw_ring_rx_start(self->aq_hw, 240 &ring[AQ_VEC_RX_ID]); 241 if (err < 0) 242 goto err_exit; 243 } 244 245 napi_enable(&self->napi); 246 247 err_exit: 248 return err; 249 } 250 251 void aq_vec_stop(struct aq_vec_s *self) 252 { 253 struct aq_ring_s *ring = NULL; 254 unsigned int i = 0U; 255 256 for (i = 0U; self->tx_rings > i; ++i) { 257 ring = self->ring[i]; 258 self->aq_hw_ops->hw_ring_tx_stop(self->aq_hw, 259 &ring[AQ_VEC_TX_ID]); 260 261 self->aq_hw_ops->hw_ring_rx_stop(self->aq_hw, 262 &ring[AQ_VEC_RX_ID]); 263 } 264 265 napi_disable(&self->napi); 266 } 267 268 void aq_vec_deinit(struct aq_vec_s *self) 269 { 270 struct aq_ring_s *ring = NULL; 271 unsigned int i = 0U; 272 273 if (!self) 274 goto err_exit; 275 276 for (i = 0U; self->tx_rings > i; ++i) { 277 ring = self->ring[i]; 278 aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]); 279 aq_ring_rx_deinit(&ring[AQ_VEC_RX_ID]); 280 } 281 282 err_exit:; 283 } 284 285 void aq_vec_free(struct aq_vec_s *self) 286 { 287 if (!self) 288 goto err_exit; 289 290 netif_napi_del(&self->napi); 291 292 kfree(self); 293 294 err_exit:; 295 } 296 297 void aq_vec_ring_free(struct aq_vec_s *self) 298 { 299 struct aq_ring_s *ring = NULL; 300 unsigned int i = 0U; 301 302 if (!self) 303 goto err_exit; 304 305 for (i = 0U; self->tx_rings > i; ++i) { 306 ring = self->ring[i]; 307 aq_ring_free(&ring[AQ_VEC_TX_ID]); 308 if (i < self->rx_rings) { 309 xdp_rxq_info_unreg(&ring[AQ_VEC_RX_ID].xdp_rxq); 310 aq_ring_free(&ring[AQ_VEC_RX_ID]); 311 } 312 } 313 314 self->tx_rings = 0; 315 self->rx_rings = 0; 316 err_exit:; 317 } 318 319 irqreturn_t aq_vec_isr(int irq, void *private) 320 { 321 struct aq_vec_s *self = private; 322 int err = 0; 323 324 if (!self) { 325 err = -EINVAL; 326 goto err_exit; 327 } 328 napi_schedule(&self->napi); 329 330 err_exit: 331 return err >= 0 ? IRQ_HANDLED : IRQ_NONE; 332 } 333 334 irqreturn_t aq_vec_isr_legacy(int irq, void *private) 335 { 336 struct aq_vec_s *self = private; 337 u64 irq_mask = 0U; 338 int err; 339 340 if (!self) 341 return IRQ_NONE; 342 err = self->aq_hw_ops->hw_irq_read(self->aq_hw, &irq_mask); 343 if (err < 0) 344 return IRQ_NONE; 345 346 if (irq_mask) { 347 self->aq_hw_ops->hw_irq_disable(self->aq_hw, 348 1U << self->aq_ring_param.vec_idx); 349 napi_schedule(&self->napi); 350 } else { 351 self->aq_hw_ops->hw_irq_enable(self->aq_hw, 1U); 352 return IRQ_NONE; 353 } 354 355 return IRQ_HANDLED; 356 } 357 358 cpumask_t *aq_vec_get_affinity_mask(struct aq_vec_s *self) 359 { 360 return &self->aq_ring_param.affinity_mask; 361 } 362 363 bool aq_vec_is_valid_tc(struct aq_vec_s *self, const unsigned int tc) 364 { 365 return tc < self->rx_rings && tc < self->tx_rings; 366 } 367 368 unsigned int aq_vec_get_sw_stats(struct aq_vec_s *self, const unsigned int tc, u64 *data) 369 { 370 unsigned int count; 371 372 if (!aq_vec_is_valid_tc(self, tc)) 373 return 0; 374 375 count = aq_ring_fill_stats_data(&self->ring[tc][AQ_VEC_RX_ID], data); 376 count += aq_ring_fill_stats_data(&self->ring[tc][AQ_VEC_TX_ID], data + count); 377 378 return count; 379 } 380