1 /* 2 * Huawei HiNIC PCI Express Linux driver 3 * Copyright(c) 2017 Huawei Technologies Co., Ltd 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * for more details. 13 * 14 */ 15 16 #include <linux/kernel.h> 17 #include <linux/types.h> 18 #include <linux/errno.h> 19 #include <linux/pci.h> 20 #include <linux/device.h> 21 #include <linux/netdevice.h> 22 #include <linux/etherdevice.h> 23 #include <linux/u64_stats_sync.h> 24 #include <linux/slab.h> 25 #include <linux/interrupt.h> 26 #include <linux/skbuff.h> 27 #include <linux/dma-mapping.h> 28 #include <linux/prefetch.h> 29 #include <linux/cpumask.h> 30 #include <asm/barrier.h> 31 32 #include "hinic_common.h" 33 #include "hinic_hw_if.h" 34 #include "hinic_hw_wqe.h" 35 #include "hinic_hw_wq.h" 36 #include "hinic_hw_qp.h" 37 #include "hinic_hw_dev.h" 38 #include "hinic_rx.h" 39 #include "hinic_dev.h" 40 41 #define RX_IRQ_NO_PENDING 0 42 #define RX_IRQ_NO_COALESC 0 43 #define RX_IRQ_NO_LLI_TIMER 0 44 #define RX_IRQ_NO_CREDIT 0 45 #define RX_IRQ_NO_RESEND_TIMER 0 46 #define HINIC_RX_BUFFER_WRITE 16 47 48 #define HINIC_RX_IPV6_PKT 7 49 #define LRO_PKT_HDR_LEN_IPV4 66 50 #define LRO_PKT_HDR_LEN_IPV6 86 51 #define LRO_REPLENISH_THLD 256 52 53 #define LRO_PKT_HDR_LEN(cqe) \ 54 (HINIC_GET_RX_PKT_TYPE(be32_to_cpu((cqe)->offload_type)) == \ 55 HINIC_RX_IPV6_PKT ? LRO_PKT_HDR_LEN_IPV6 : LRO_PKT_HDR_LEN_IPV4) 56 57 /** 58 * hinic_rxq_clean_stats - Clean the statistics of specific queue 59 * @rxq: Logical Rx Queue 60 **/ 61 void hinic_rxq_clean_stats(struct hinic_rxq *rxq) 62 { 63 struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 64 65 u64_stats_update_begin(&rxq_stats->syncp); 66 rxq_stats->pkts = 0; 67 rxq_stats->bytes = 0; 68 u64_stats_update_end(&rxq_stats->syncp); 69 } 70 71 /** 72 * hinic_rxq_get_stats - get statistics of Rx Queue 73 * @rxq: Logical Rx Queue 74 * @stats: return updated stats here 75 **/ 76 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats) 77 { 78 struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 79 unsigned int start; 80 81 u64_stats_update_begin(&stats->syncp); 82 do { 83 start = u64_stats_fetch_begin(&rxq_stats->syncp); 84 stats->pkts = rxq_stats->pkts; 85 stats->bytes = rxq_stats->bytes; 86 } while (u64_stats_fetch_retry(&rxq_stats->syncp, start)); 87 u64_stats_update_end(&stats->syncp); 88 } 89 90 /** 91 * rxq_stats_init - Initialize the statistics of specific queue 92 * @rxq: Logical Rx Queue 93 **/ 94 static void rxq_stats_init(struct hinic_rxq *rxq) 95 { 96 struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; 97 98 u64_stats_init(&rxq_stats->syncp); 99 hinic_rxq_clean_stats(rxq); 100 } 101 102 static void rx_csum(struct hinic_rxq *rxq, u32 status, 103 struct sk_buff *skb) 104 { 105 struct net_device *netdev = rxq->netdev; 106 u32 csum_err; 107 108 csum_err = HINIC_RQ_CQE_STATUS_GET(status, CSUM_ERR); 109 110 if (!(netdev->features & NETIF_F_RXCSUM)) 111 return; 112 113 if (!csum_err) 114 skb->ip_summed = CHECKSUM_UNNECESSARY; 115 else 116 skb->ip_summed = CHECKSUM_NONE; 117 } 118 /** 119 * rx_alloc_skb - allocate skb and map it to dma address 120 * @rxq: rx queue 121 * @dma_addr: returned dma address for the skb 122 * 123 * Return skb 124 **/ 125 static struct sk_buff *rx_alloc_skb(struct hinic_rxq *rxq, 126 dma_addr_t *dma_addr) 127 { 128 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 129 struct hinic_hwdev *hwdev = nic_dev->hwdev; 130 struct hinic_hwif *hwif = hwdev->hwif; 131 struct pci_dev *pdev = hwif->pdev; 132 struct sk_buff *skb; 133 dma_addr_t addr; 134 int err; 135 136 skb = netdev_alloc_skb_ip_align(rxq->netdev, rxq->rq->buf_sz); 137 if (!skb) { 138 netdev_err(rxq->netdev, "Failed to allocate Rx SKB\n"); 139 return NULL; 140 } 141 142 addr = dma_map_single(&pdev->dev, skb->data, rxq->rq->buf_sz, 143 DMA_FROM_DEVICE); 144 err = dma_mapping_error(&pdev->dev, addr); 145 if (err) { 146 dev_err(&pdev->dev, "Failed to map Rx DMA, err = %d\n", err); 147 goto err_rx_map; 148 } 149 150 *dma_addr = addr; 151 return skb; 152 153 err_rx_map: 154 dev_kfree_skb_any(skb); 155 return NULL; 156 } 157 158 /** 159 * rx_unmap_skb - unmap the dma address of the skb 160 * @rxq: rx queue 161 * @dma_addr: dma address of the skb 162 **/ 163 static void rx_unmap_skb(struct hinic_rxq *rxq, dma_addr_t dma_addr) 164 { 165 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 166 struct hinic_hwdev *hwdev = nic_dev->hwdev; 167 struct hinic_hwif *hwif = hwdev->hwif; 168 struct pci_dev *pdev = hwif->pdev; 169 170 dma_unmap_single(&pdev->dev, dma_addr, rxq->rq->buf_sz, 171 DMA_FROM_DEVICE); 172 } 173 174 /** 175 * rx_free_skb - unmap and free skb 176 * @rxq: rx queue 177 * @skb: skb to free 178 * @dma_addr: dma address of the skb 179 **/ 180 static void rx_free_skb(struct hinic_rxq *rxq, struct sk_buff *skb, 181 dma_addr_t dma_addr) 182 { 183 rx_unmap_skb(rxq, dma_addr); 184 dev_kfree_skb_any(skb); 185 } 186 187 /** 188 * rx_alloc_pkts - allocate pkts in rx queue 189 * @rxq: rx queue 190 * 191 * Return number of skbs allocated 192 **/ 193 static int rx_alloc_pkts(struct hinic_rxq *rxq) 194 { 195 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 196 struct hinic_rq_wqe *rq_wqe; 197 unsigned int free_wqebbs; 198 struct hinic_sge sge; 199 dma_addr_t dma_addr; 200 struct sk_buff *skb; 201 u16 prod_idx; 202 int i; 203 204 free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq); 205 206 /* Limit the allocation chunks */ 207 if (free_wqebbs > nic_dev->rx_weight) 208 free_wqebbs = nic_dev->rx_weight; 209 210 for (i = 0; i < free_wqebbs; i++) { 211 skb = rx_alloc_skb(rxq, &dma_addr); 212 if (!skb) { 213 netdev_err(rxq->netdev, "Failed to alloc Rx skb\n"); 214 goto skb_out; 215 } 216 217 hinic_set_sge(&sge, dma_addr, skb->len); 218 219 rq_wqe = hinic_rq_get_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, 220 &prod_idx); 221 if (!rq_wqe) { 222 rx_free_skb(rxq, skb, dma_addr); 223 goto skb_out; 224 } 225 226 hinic_rq_prepare_wqe(rxq->rq, prod_idx, rq_wqe, &sge); 227 228 hinic_rq_write_wqe(rxq->rq, prod_idx, rq_wqe, skb); 229 } 230 231 skb_out: 232 if (i) { 233 wmb(); /* write all the wqes before update PI */ 234 235 hinic_rq_update(rxq->rq, prod_idx); 236 } 237 238 return i; 239 } 240 241 /** 242 * free_all_rx_skbs - free all skbs in rx queue 243 * @rxq: rx queue 244 **/ 245 static void free_all_rx_skbs(struct hinic_rxq *rxq) 246 { 247 struct hinic_rq *rq = rxq->rq; 248 struct hinic_hw_wqe *hw_wqe; 249 struct hinic_sge sge; 250 u16 ci; 251 252 while ((hw_wqe = hinic_read_wqe(rq->wq, HINIC_RQ_WQE_SIZE, &ci))) { 253 if (IS_ERR(hw_wqe)) 254 break; 255 256 hinic_rq_get_sge(rq, &hw_wqe->rq_wqe, ci, &sge); 257 258 hinic_put_wqe(rq->wq, HINIC_RQ_WQE_SIZE); 259 260 rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge)); 261 } 262 } 263 264 /** 265 * rx_recv_jumbo_pkt - Rx handler for jumbo pkt 266 * @rxq: rx queue 267 * @head_skb: the first skb in the list 268 * @left_pkt_len: left size of the pkt exclude head skb 269 * @ci: consumer index 270 * 271 * Return number of wqes that used for the left of the pkt 272 **/ 273 static int rx_recv_jumbo_pkt(struct hinic_rxq *rxq, struct sk_buff *head_skb, 274 unsigned int left_pkt_len, u16 ci) 275 { 276 struct sk_buff *skb, *curr_skb = head_skb; 277 struct hinic_rq_wqe *rq_wqe; 278 unsigned int curr_len; 279 struct hinic_sge sge; 280 int num_wqes = 0; 281 282 while (left_pkt_len > 0) { 283 rq_wqe = hinic_rq_read_next_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, 284 &skb, &ci); 285 286 num_wqes++; 287 288 hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge); 289 290 rx_unmap_skb(rxq, hinic_sge_to_dma(&sge)); 291 292 prefetch(skb->data); 293 294 curr_len = (left_pkt_len > HINIC_RX_BUF_SZ) ? HINIC_RX_BUF_SZ : 295 left_pkt_len; 296 297 left_pkt_len -= curr_len; 298 299 __skb_put(skb, curr_len); 300 301 if (curr_skb == head_skb) 302 skb_shinfo(head_skb)->frag_list = skb; 303 else 304 curr_skb->next = skb; 305 306 head_skb->len += skb->len; 307 head_skb->data_len += skb->len; 308 head_skb->truesize += skb->truesize; 309 310 curr_skb = skb; 311 } 312 313 return num_wqes; 314 } 315 316 /** 317 * rxq_recv - Rx handler 318 * @rxq: rx queue 319 * @budget: maximum pkts to process 320 * 321 * Return number of pkts received 322 **/ 323 static int rxq_recv(struct hinic_rxq *rxq, int budget) 324 { 325 struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq); 326 u64 pkt_len = 0, rx_bytes = 0; 327 struct hinic_rq *rq = rxq->rq; 328 struct hinic_rq_wqe *rq_wqe; 329 unsigned int free_wqebbs; 330 struct hinic_rq_cqe *cqe; 331 int num_wqes, pkts = 0; 332 struct hinic_sge sge; 333 unsigned int status; 334 struct sk_buff *skb; 335 u16 ci, num_lro; 336 u16 num_wqe = 0; 337 338 while (pkts < budget) { 339 num_wqes = 0; 340 341 rq_wqe = hinic_rq_read_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, &skb, 342 &ci); 343 if (!rq_wqe) 344 break; 345 346 cqe = rq->cqe[ci]; 347 status = be32_to_cpu(cqe->status); 348 hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge); 349 350 rx_unmap_skb(rxq, hinic_sge_to_dma(&sge)); 351 352 rx_csum(rxq, status, skb); 353 354 prefetch(skb->data); 355 356 pkt_len = sge.len; 357 358 if (pkt_len <= HINIC_RX_BUF_SZ) { 359 __skb_put(skb, pkt_len); 360 } else { 361 __skb_put(skb, HINIC_RX_BUF_SZ); 362 num_wqes = rx_recv_jumbo_pkt(rxq, skb, pkt_len - 363 HINIC_RX_BUF_SZ, ci); 364 } 365 366 hinic_rq_put_wqe(rq, ci, 367 (num_wqes + 1) * HINIC_RQ_WQE_SIZE); 368 369 skb_record_rx_queue(skb, qp->q_id); 370 skb->protocol = eth_type_trans(skb, rxq->netdev); 371 372 napi_gro_receive(&rxq->napi, skb); 373 374 pkts++; 375 rx_bytes += pkt_len; 376 377 num_lro = HINIC_GET_RX_NUM_LRO(status); 378 if (num_lro) { 379 rx_bytes += ((num_lro - 1) * 380 LRO_PKT_HDR_LEN(cqe)); 381 382 num_wqe += 383 (u16)(pkt_len >> rxq->rx_buff_shift) + 384 ((pkt_len & (rxq->buf_len - 1)) ? 1 : 0); 385 } 386 387 cqe->status = 0; 388 389 if (num_wqe >= LRO_REPLENISH_THLD) 390 break; 391 } 392 393 free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq); 394 if (free_wqebbs > HINIC_RX_BUFFER_WRITE) 395 rx_alloc_pkts(rxq); 396 397 u64_stats_update_begin(&rxq->rxq_stats.syncp); 398 rxq->rxq_stats.pkts += pkts; 399 rxq->rxq_stats.bytes += rx_bytes; 400 u64_stats_update_end(&rxq->rxq_stats.syncp); 401 402 return pkts; 403 } 404 405 static int rx_poll(struct napi_struct *napi, int budget) 406 { 407 struct hinic_rxq *rxq = container_of(napi, struct hinic_rxq, napi); 408 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 409 struct hinic_rq *rq = rxq->rq; 410 int pkts; 411 412 pkts = rxq_recv(rxq, budget); 413 if (pkts >= budget) 414 return budget; 415 416 napi_complete(napi); 417 hinic_hwdev_set_msix_state(nic_dev->hwdev, 418 rq->msix_entry, 419 HINIC_MSIX_ENABLE); 420 421 return pkts; 422 } 423 424 static void rx_add_napi(struct hinic_rxq *rxq) 425 { 426 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 427 428 netif_napi_add(rxq->netdev, &rxq->napi, rx_poll, nic_dev->rx_weight); 429 napi_enable(&rxq->napi); 430 } 431 432 static void rx_del_napi(struct hinic_rxq *rxq) 433 { 434 napi_disable(&rxq->napi); 435 netif_napi_del(&rxq->napi); 436 } 437 438 static irqreturn_t rx_irq(int irq, void *data) 439 { 440 struct hinic_rxq *rxq = (struct hinic_rxq *)data; 441 struct hinic_rq *rq = rxq->rq; 442 struct hinic_dev *nic_dev; 443 444 /* Disable the interrupt until napi will be completed */ 445 nic_dev = netdev_priv(rxq->netdev); 446 hinic_hwdev_set_msix_state(nic_dev->hwdev, 447 rq->msix_entry, 448 HINIC_MSIX_DISABLE); 449 450 nic_dev = netdev_priv(rxq->netdev); 451 hinic_hwdev_msix_cnt_set(nic_dev->hwdev, rq->msix_entry); 452 453 napi_schedule(&rxq->napi); 454 return IRQ_HANDLED; 455 } 456 457 static int rx_request_irq(struct hinic_rxq *rxq) 458 { 459 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev); 460 struct hinic_hwdev *hwdev = nic_dev->hwdev; 461 struct hinic_rq *rq = rxq->rq; 462 struct hinic_qp *qp; 463 struct cpumask mask; 464 int err; 465 466 rx_add_napi(rxq); 467 468 hinic_hwdev_msix_set(hwdev, rq->msix_entry, 469 RX_IRQ_NO_PENDING, RX_IRQ_NO_COALESC, 470 RX_IRQ_NO_LLI_TIMER, RX_IRQ_NO_CREDIT, 471 RX_IRQ_NO_RESEND_TIMER); 472 473 err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq); 474 if (err) { 475 rx_del_napi(rxq); 476 return err; 477 } 478 479 qp = container_of(rq, struct hinic_qp, rq); 480 cpumask_set_cpu(qp->q_id % num_online_cpus(), &mask); 481 return irq_set_affinity_hint(rq->irq, &mask); 482 } 483 484 static void rx_free_irq(struct hinic_rxq *rxq) 485 { 486 struct hinic_rq *rq = rxq->rq; 487 488 irq_set_affinity_hint(rq->irq, NULL); 489 free_irq(rq->irq, rxq); 490 rx_del_napi(rxq); 491 } 492 493 /** 494 * hinic_init_rxq - Initialize the Rx Queue 495 * @rxq: Logical Rx Queue 496 * @rq: Hardware Rx Queue to connect the Logical queue with 497 * @netdev: network device to connect the Logical queue with 498 * 499 * Return 0 - Success, negative - Failure 500 **/ 501 int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq, 502 struct net_device *netdev) 503 { 504 struct hinic_qp *qp = container_of(rq, struct hinic_qp, rq); 505 int err, pkts; 506 507 rxq->netdev = netdev; 508 rxq->rq = rq; 509 rxq->buf_len = HINIC_RX_BUF_SZ; 510 rxq->rx_buff_shift = ilog2(HINIC_RX_BUF_SZ); 511 512 rxq_stats_init(rxq); 513 514 rxq->irq_name = devm_kasprintf(&netdev->dev, GFP_KERNEL, 515 "hinic_rxq%d", qp->q_id); 516 if (!rxq->irq_name) 517 return -ENOMEM; 518 519 pkts = rx_alloc_pkts(rxq); 520 if (!pkts) { 521 err = -ENOMEM; 522 goto err_rx_pkts; 523 } 524 525 err = rx_request_irq(rxq); 526 if (err) { 527 netdev_err(netdev, "Failed to request Rx irq\n"); 528 goto err_req_rx_irq; 529 } 530 531 return 0; 532 533 err_req_rx_irq: 534 err_rx_pkts: 535 free_all_rx_skbs(rxq); 536 devm_kfree(&netdev->dev, rxq->irq_name); 537 return err; 538 } 539 540 /** 541 * hinic_clean_rxq - Clean the Rx Queue 542 * @rxq: Logical Rx Queue 543 **/ 544 void hinic_clean_rxq(struct hinic_rxq *rxq) 545 { 546 struct net_device *netdev = rxq->netdev; 547 548 rx_free_irq(rxq); 549 550 free_all_rx_skbs(rxq); 551 devm_kfree(&netdev->dev, rxq->irq_name); 552 } 553