1 /* 2 * aQuantia Corporation Network Driver 3 * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 */ 9 10 /* File aq_ring.c: Definition of functions for Rx/Tx rings. */ 11 12 #include "aq_ring.h" 13 #include "aq_nic.h" 14 #include "aq_hw.h" 15 16 #include <linux/netdevice.h> 17 #include <linux/etherdevice.h> 18 19 static struct aq_ring_s *aq_ring_alloc(struct aq_ring_s *self, 20 struct aq_nic_s *aq_nic) 21 { 22 int err = 0; 23 24 self->buff_ring = 25 kcalloc(self->size, sizeof(struct aq_ring_buff_s), GFP_KERNEL); 26 27 if (!self->buff_ring) { 28 err = -ENOMEM; 29 goto err_exit; 30 } 31 self->dx_ring = dma_alloc_coherent(aq_nic_get_dev(aq_nic), 32 self->size * self->dx_size, 33 &self->dx_ring_pa, GFP_KERNEL); 34 if (!self->dx_ring) { 35 err = -ENOMEM; 36 goto err_exit; 37 } 38 39 err_exit: 40 if (err < 0) { 41 aq_ring_free(self); 42 self = NULL; 43 } 44 return self; 45 } 46 47 struct aq_ring_s *aq_ring_tx_alloc(struct aq_ring_s *self, 48 struct aq_nic_s *aq_nic, 49 unsigned int idx, 50 struct aq_nic_cfg_s *aq_nic_cfg) 51 { 52 int err = 0; 53 54 self->aq_nic = aq_nic; 55 self->idx = idx; 56 self->size = aq_nic_cfg->txds; 57 self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size; 58 59 self = aq_ring_alloc(self, aq_nic); 60 if (!self) { 61 err = -ENOMEM; 62 goto err_exit; 63 } 64 65 err_exit: 66 if (err < 0) { 67 aq_ring_free(self); 68 self = NULL; 69 } 70 return self; 71 } 72 73 struct aq_ring_s *aq_ring_rx_alloc(struct aq_ring_s *self, 74 struct aq_nic_s *aq_nic, 75 unsigned int idx, 76 struct aq_nic_cfg_s *aq_nic_cfg) 77 { 78 int err = 0; 79 80 self->aq_nic = aq_nic; 81 self->idx = idx; 82 self->size = aq_nic_cfg->rxds; 83 self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size; 84 85 self = aq_ring_alloc(self, aq_nic); 86 if (!self) { 87 err = -ENOMEM; 88 goto err_exit; 89 } 90 91 err_exit: 92 if (err < 0) { 93 aq_ring_free(self); 94 self = NULL; 95 } 96 return self; 97 } 98 99 int aq_ring_init(struct aq_ring_s *self) 100 { 101 self->hw_head = 0; 102 self->sw_head = 0; 103 self->sw_tail = 0; 104 return 0; 105 } 106 107 static inline bool aq_ring_dx_in_range(unsigned int h, unsigned int i, 108 unsigned int t) 109 { 110 return (h < t) ? ((h < i) && (i < t)) : ((h < i) || (i < t)); 111 } 112 113 void aq_ring_update_queue_state(struct aq_ring_s *ring) 114 { 115 if (aq_ring_avail_dx(ring) <= AQ_CFG_SKB_FRAGS_MAX) 116 aq_ring_queue_stop(ring); 117 else if (aq_ring_avail_dx(ring) > AQ_CFG_RESTART_DESC_THRES) 118 aq_ring_queue_wake(ring); 119 } 120 121 void aq_ring_queue_wake(struct aq_ring_s *ring) 122 { 123 struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); 124 125 if (__netif_subqueue_stopped(ndev, ring->idx)) { 126 netif_wake_subqueue(ndev, ring->idx); 127 ring->stats.tx.queue_restarts++; 128 } 129 } 130 131 void aq_ring_queue_stop(struct aq_ring_s *ring) 132 { 133 struct net_device *ndev = aq_nic_get_ndev(ring->aq_nic); 134 135 if (!__netif_subqueue_stopped(ndev, ring->idx)) 136 netif_stop_subqueue(ndev, ring->idx); 137 } 138 139 bool aq_ring_tx_clean(struct aq_ring_s *self) 140 { 141 struct device *dev = aq_nic_get_dev(self->aq_nic); 142 unsigned int budget = AQ_CFG_TX_CLEAN_BUDGET; 143 144 for (; self->sw_head != self->hw_head && budget--; 145 self->sw_head = aq_ring_next_dx(self, self->sw_head)) { 146 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 147 148 if (likely(buff->is_mapped)) { 149 if (unlikely(buff->is_sop)) { 150 if (!buff->is_eop && 151 buff->eop_index != 0xffffU && 152 (!aq_ring_dx_in_range(self->sw_head, 153 buff->eop_index, 154 self->hw_head))) 155 break; 156 157 dma_unmap_single(dev, buff->pa, buff->len, 158 DMA_TO_DEVICE); 159 } else { 160 dma_unmap_page(dev, buff->pa, buff->len, 161 DMA_TO_DEVICE); 162 } 163 } 164 165 if (unlikely(buff->is_eop)) 166 dev_kfree_skb_any(buff->skb); 167 168 buff->pa = 0U; 169 buff->eop_index = 0xffffU; 170 } 171 172 return !!budget; 173 } 174 175 static void aq_rx_checksum(struct aq_ring_s *self, 176 struct aq_ring_buff_s *buff, 177 struct sk_buff *skb) 178 { 179 if (!(self->aq_nic->ndev->features & NETIF_F_RXCSUM)) 180 return; 181 182 if (unlikely(buff->is_cso_err)) { 183 ++self->stats.rx.errors; 184 skb->ip_summed = CHECKSUM_NONE; 185 return; 186 } 187 if (buff->is_ip_cso) { 188 __skb_incr_checksum_unnecessary(skb); 189 if (buff->is_udp_cso || buff->is_tcp_cso) 190 __skb_incr_checksum_unnecessary(skb); 191 } else { 192 skb->ip_summed = CHECKSUM_NONE; 193 } 194 } 195 196 #define AQ_SKB_ALIGN SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) 197 int aq_ring_rx_clean(struct aq_ring_s *self, 198 struct napi_struct *napi, 199 int *work_done, 200 int budget) 201 { 202 struct net_device *ndev = aq_nic_get_ndev(self->aq_nic); 203 int err = 0; 204 bool is_rsc_completed = true; 205 206 for (; (self->sw_head != self->hw_head) && budget; 207 self->sw_head = aq_ring_next_dx(self, self->sw_head), 208 --budget, ++(*work_done)) { 209 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 210 struct sk_buff *skb = NULL; 211 unsigned int next_ = 0U; 212 unsigned int i = 0U; 213 struct aq_ring_buff_s *buff_ = NULL; 214 215 if (buff->is_error) { 216 __free_pages(buff->page, 0); 217 continue; 218 } 219 220 if (buff->is_cleaned) 221 continue; 222 223 if (!buff->is_eop) { 224 for (next_ = buff->next, 225 buff_ = &self->buff_ring[next_]; true; 226 next_ = buff_->next, 227 buff_ = &self->buff_ring[next_]) { 228 is_rsc_completed = 229 aq_ring_dx_in_range(self->sw_head, 230 next_, 231 self->hw_head); 232 233 if (unlikely(!is_rsc_completed)) { 234 is_rsc_completed = false; 235 break; 236 } 237 238 if (buff_->is_eop) 239 break; 240 } 241 242 if (!is_rsc_completed) { 243 err = 0; 244 goto err_exit; 245 } 246 } 247 248 /* for single fragment packets use build_skb() */ 249 if (buff->is_eop && 250 buff->len <= AQ_CFG_RX_FRAME_MAX - AQ_SKB_ALIGN) { 251 skb = build_skb(page_address(buff->page), 252 AQ_CFG_RX_FRAME_MAX); 253 if (unlikely(!skb)) { 254 err = -ENOMEM; 255 goto err_exit; 256 } 257 258 skb_put(skb, buff->len); 259 } else { 260 skb = netdev_alloc_skb(ndev, ETH_HLEN); 261 if (unlikely(!skb)) { 262 err = -ENOMEM; 263 goto err_exit; 264 } 265 skb_put(skb, ETH_HLEN); 266 memcpy(skb->data, page_address(buff->page), ETH_HLEN); 267 268 skb_add_rx_frag(skb, 0, buff->page, ETH_HLEN, 269 buff->len - ETH_HLEN, 270 SKB_TRUESIZE(buff->len - ETH_HLEN)); 271 272 if (!buff->is_eop) { 273 for (i = 1U, next_ = buff->next, 274 buff_ = &self->buff_ring[next_]; 275 true; next_ = buff_->next, 276 buff_ = &self->buff_ring[next_], ++i) { 277 skb_add_rx_frag(skb, i, 278 buff_->page, 0, 279 buff_->len, 280 SKB_TRUESIZE(buff->len - 281 ETH_HLEN)); 282 buff_->is_cleaned = 1; 283 284 if (buff_->is_eop) 285 break; 286 } 287 } 288 } 289 290 skb->protocol = eth_type_trans(skb, ndev); 291 292 aq_rx_checksum(self, buff, skb); 293 294 skb_set_hash(skb, buff->rss_hash, 295 buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : 296 PKT_HASH_TYPE_NONE); 297 298 skb_record_rx_queue(skb, self->idx); 299 300 ++self->stats.rx.packets; 301 self->stats.rx.bytes += skb->len; 302 303 napi_gro_receive(napi, skb); 304 } 305 306 err_exit: 307 return err; 308 } 309 310 int aq_ring_rx_fill(struct aq_ring_s *self) 311 { 312 unsigned int pages_order = fls(AQ_CFG_RX_FRAME_MAX / PAGE_SIZE + 313 (AQ_CFG_RX_FRAME_MAX % PAGE_SIZE ? 1 : 0)) - 1; 314 struct aq_ring_buff_s *buff = NULL; 315 int err = 0; 316 int i = 0; 317 318 for (i = aq_ring_avail_dx(self); i--; 319 self->sw_tail = aq_ring_next_dx(self, self->sw_tail)) { 320 buff = &self->buff_ring[self->sw_tail]; 321 322 buff->flags = 0U; 323 buff->len = AQ_CFG_RX_FRAME_MAX; 324 325 buff->page = alloc_pages(GFP_ATOMIC | __GFP_COMP, pages_order); 326 if (!buff->page) { 327 err = -ENOMEM; 328 goto err_exit; 329 } 330 331 buff->pa = dma_map_page(aq_nic_get_dev(self->aq_nic), 332 buff->page, 0, 333 AQ_CFG_RX_FRAME_MAX, DMA_FROM_DEVICE); 334 335 if (dma_mapping_error(aq_nic_get_dev(self->aq_nic), buff->pa)) { 336 err = -ENOMEM; 337 goto err_exit; 338 } 339 340 buff = NULL; 341 } 342 343 err_exit: 344 if (err < 0) { 345 if (buff && buff->page) 346 __free_pages(buff->page, 0); 347 } 348 349 return err; 350 } 351 352 void aq_ring_rx_deinit(struct aq_ring_s *self) 353 { 354 if (!self) 355 goto err_exit; 356 357 for (; self->sw_head != self->sw_tail; 358 self->sw_head = aq_ring_next_dx(self, self->sw_head)) { 359 struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; 360 361 dma_unmap_page(aq_nic_get_dev(self->aq_nic), buff->pa, 362 AQ_CFG_RX_FRAME_MAX, DMA_FROM_DEVICE); 363 364 __free_pages(buff->page, 0); 365 } 366 367 err_exit:; 368 } 369 370 void aq_ring_free(struct aq_ring_s *self) 371 { 372 if (!self) 373 goto err_exit; 374 375 kfree(self->buff_ring); 376 377 if (self->dx_ring) 378 dma_free_coherent(aq_nic_get_dev(self->aq_nic), 379 self->size * self->dx_size, self->dx_ring, 380 self->dx_ring_pa); 381 382 err_exit:; 383 } 384