1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell Octeon EP (EndPoint) Ethernet Driver 3 * 4 * Copyright (C) 2020 Marvell. 5 * 6 */ 7 8 #include <linux/pci.h> 9 #include <linux/etherdevice.h> 10 #include <linux/vmalloc.h> 11 12 #include "octep_config.h" 13 #include "octep_main.h" 14 15 static void octep_oq_reset_indices(struct octep_oq *oq) 16 { 17 oq->host_read_idx = 0; 18 oq->host_refill_idx = 0; 19 oq->refill_count = 0; 20 oq->last_pkt_count = 0; 21 oq->pkts_pending = 0; 22 } 23 24 /** 25 * octep_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring. 26 * 27 * @oq: Octeon Rx queue data structure. 28 * 29 * Return: 0, if successfully filled receive buffers for all descriptors. 30 * -1, if failed to allocate a buffer or failed to map for DMA. 31 */ 32 static int octep_oq_fill_ring_buffers(struct octep_oq *oq) 33 { 34 struct octep_oq_desc_hw *desc_ring = oq->desc_ring; 35 struct page *page; 36 u32 i; 37 38 for (i = 0; i < oq->max_count; i++) { 39 page = dev_alloc_page(); 40 if (unlikely(!page)) { 41 dev_err(oq->dev, "Rx buffer alloc failed\n"); 42 goto rx_buf_alloc_err; 43 } 44 desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0, 45 PAGE_SIZE, 46 DMA_FROM_DEVICE); 47 if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) { 48 dev_err(oq->dev, 49 "OQ-%d buffer alloc: DMA mapping error!\n", 50 oq->q_no); 51 put_page(page); 52 goto dma_map_err; 53 } 54 oq->buff_info[i].page = page; 55 } 56 57 return 0; 58 59 dma_map_err: 60 rx_buf_alloc_err: 61 while (i) { 62 i--; 63 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE); 64 put_page(oq->buff_info[i].page); 65 oq->buff_info[i].page = NULL; 66 } 67 68 return -1; 69 } 70 71 /** 72 * octep_oq_refill() - refill buffers for used Rx ring descriptors. 73 * 74 * @oct: Octeon device private data structure. 75 * @oq: Octeon Rx queue data structure. 76 * 77 * Return: number of descriptors successfully refilled with receive buffers. 78 */ 79 static int octep_oq_refill(struct octep_device *oct, struct octep_oq *oq) 80 { 81 struct octep_oq_desc_hw *desc_ring = oq->desc_ring; 82 struct page *page; 83 u32 refill_idx, i; 84 85 refill_idx = oq->host_refill_idx; 86 for (i = 0; i < oq->refill_count; i++) { 87 page = dev_alloc_page(); 88 if (unlikely(!page)) { 89 dev_err(oq->dev, "refill: rx buffer alloc failed\n"); 90 oq->stats.alloc_failures++; 91 break; 92 } 93 94 desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0, 95 PAGE_SIZE, DMA_FROM_DEVICE); 96 if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) { 97 dev_err(oq->dev, 98 "OQ-%d buffer refill: DMA mapping error!\n", 99 oq->q_no); 100 put_page(page); 101 oq->stats.alloc_failures++; 102 break; 103 } 104 oq->buff_info[refill_idx].page = page; 105 refill_idx++; 106 if (refill_idx == oq->max_count) 107 refill_idx = 0; 108 } 109 oq->host_refill_idx = refill_idx; 110 oq->refill_count -= i; 111 112 return i; 113 } 114 115 /** 116 * octep_setup_oq() - Setup a Rx queue. 117 * 118 * @oct: Octeon device private data structure. 119 * @q_no: Rx queue number to be setup. 120 * 121 * Allocate resources for a Rx queue. 122 */ 123 static int octep_setup_oq(struct octep_device *oct, int q_no) 124 { 125 struct octep_oq *oq; 126 u32 desc_ring_size; 127 128 oq = vzalloc(sizeof(*oq)); 129 if (!oq) 130 goto create_oq_fail; 131 oct->oq[q_no] = oq; 132 133 oq->octep_dev = oct; 134 oq->netdev = oct->netdev; 135 oq->dev = &oct->pdev->dev; 136 oq->q_no = q_no; 137 oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf); 138 oq->ring_size_mask = oq->max_count - 1; 139 oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf); 140 oq->max_single_buffer_size = oq->buffer_size - OCTEP_OQ_RESP_HW_SIZE; 141 142 /* When the hardware/firmware supports additional capabilities, 143 * additional header is filled-in by Octeon after length field in 144 * Rx packets. this header contains additional packet information. 145 */ 146 if (oct->conf->fw_info.rx_ol_flags) 147 oq->max_single_buffer_size -= OCTEP_OQ_RESP_HW_EXT_SIZE; 148 149 oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf); 150 151 desc_ring_size = oq->max_count * OCTEP_OQ_DESC_SIZE; 152 oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size, 153 &oq->desc_ring_dma, GFP_KERNEL); 154 155 if (unlikely(!oq->desc_ring)) { 156 dev_err(oq->dev, 157 "Failed to allocate DMA memory for OQ-%d !!\n", q_no); 158 goto desc_dma_alloc_err; 159 } 160 161 oq->buff_info = vcalloc(oq->max_count, OCTEP_OQ_RECVBUF_SIZE); 162 if (unlikely(!oq->buff_info)) { 163 dev_err(&oct->pdev->dev, 164 "Failed to allocate buffer info for OQ-%d\n", q_no); 165 goto buf_list_err; 166 } 167 168 if (octep_oq_fill_ring_buffers(oq)) 169 goto oq_fill_buff_err; 170 171 octep_oq_reset_indices(oq); 172 oct->hw_ops.setup_oq_regs(oct, q_no); 173 oct->num_oqs++; 174 175 return 0; 176 177 oq_fill_buff_err: 178 vfree(oq->buff_info); 179 oq->buff_info = NULL; 180 buf_list_err: 181 dma_free_coherent(oq->dev, desc_ring_size, 182 oq->desc_ring, oq->desc_ring_dma); 183 oq->desc_ring = NULL; 184 desc_dma_alloc_err: 185 vfree(oq); 186 oct->oq[q_no] = NULL; 187 create_oq_fail: 188 return -1; 189 } 190 191 /** 192 * octep_oq_free_ring_buffers() - Free ring buffers. 193 * 194 * @oq: Octeon Rx queue data structure. 195 * 196 * Free receive buffers in unused Rx queue descriptors. 197 */ 198 static void octep_oq_free_ring_buffers(struct octep_oq *oq) 199 { 200 struct octep_oq_desc_hw *desc_ring = oq->desc_ring; 201 int i; 202 203 if (!oq->desc_ring || !oq->buff_info) 204 return; 205 206 for (i = 0; i < oq->max_count; i++) { 207 if (oq->buff_info[i].page) { 208 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, 209 PAGE_SIZE, DMA_FROM_DEVICE); 210 put_page(oq->buff_info[i].page); 211 oq->buff_info[i].page = NULL; 212 desc_ring[i].buffer_ptr = 0; 213 } 214 } 215 octep_oq_reset_indices(oq); 216 } 217 218 /** 219 * octep_free_oq() - Free Rx queue resources. 220 * 221 * @oq: Octeon Rx queue data structure. 222 * 223 * Free all resources of a Rx queue. 224 */ 225 static int octep_free_oq(struct octep_oq *oq) 226 { 227 struct octep_device *oct = oq->octep_dev; 228 int q_no = oq->q_no; 229 230 octep_oq_free_ring_buffers(oq); 231 232 vfree(oq->buff_info); 233 234 if (oq->desc_ring) 235 dma_free_coherent(oq->dev, 236 oq->max_count * OCTEP_OQ_DESC_SIZE, 237 oq->desc_ring, oq->desc_ring_dma); 238 239 vfree(oq); 240 oct->oq[q_no] = NULL; 241 oct->num_oqs--; 242 return 0; 243 } 244 245 /** 246 * octep_setup_oqs() - setup resources for all Rx queues. 247 * 248 * @oct: Octeon device private data structure. 249 */ 250 int octep_setup_oqs(struct octep_device *oct) 251 { 252 int i, retval = 0; 253 254 oct->num_oqs = 0; 255 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { 256 retval = octep_setup_oq(oct, i); 257 if (retval) { 258 dev_err(&oct->pdev->dev, 259 "Failed to setup OQ(RxQ)-%d.\n", i); 260 goto oq_setup_err; 261 } 262 dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n", i); 263 } 264 265 return 0; 266 267 oq_setup_err: 268 while (i) { 269 i--; 270 octep_free_oq(oct->oq[i]); 271 } 272 return -1; 273 } 274 275 /** 276 * octep_oq_dbell_init() - Initialize Rx queue doorbell. 277 * 278 * @oct: Octeon device private data structure. 279 * 280 * Write number of descriptors to Rx queue doorbell register. 281 */ 282 void octep_oq_dbell_init(struct octep_device *oct) 283 { 284 int i; 285 286 for (i = 0; i < oct->num_oqs; i++) 287 writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg); 288 } 289 290 /** 291 * octep_free_oqs() - Free resources of all Rx queues. 292 * 293 * @oct: Octeon device private data structure. 294 */ 295 void octep_free_oqs(struct octep_device *oct) 296 { 297 int i; 298 299 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { 300 if (!oct->oq[i]) 301 continue; 302 octep_free_oq(oct->oq[i]); 303 dev_dbg(&oct->pdev->dev, 304 "Successfully freed OQ(RxQ)-%d.\n", i); 305 } 306 } 307 308 /** 309 * octep_oq_check_hw_for_pkts() - Check for new Rx packets. 310 * 311 * @oct: Octeon device private data structure. 312 * @oq: Octeon Rx queue data structure. 313 * 314 * Return: packets received after previous check. 315 */ 316 static int octep_oq_check_hw_for_pkts(struct octep_device *oct, 317 struct octep_oq *oq) 318 { 319 u32 pkt_count, new_pkts; 320 321 pkt_count = readl(oq->pkts_sent_reg); 322 new_pkts = pkt_count - oq->last_pkt_count; 323 324 /* Clear the hardware packets counter register if the rx queue is 325 * being processed continuously with-in a single interrupt and 326 * reached half its max value. 327 * this counter is not cleared every time read, to save write cycles. 328 */ 329 if (unlikely(pkt_count > 0xF0000000U)) { 330 writel(pkt_count, oq->pkts_sent_reg); 331 pkt_count = readl(oq->pkts_sent_reg); 332 new_pkts += pkt_count; 333 } 334 oq->last_pkt_count = pkt_count; 335 oq->pkts_pending += new_pkts; 336 return new_pkts; 337 } 338 339 /** 340 * octep_oq_next_pkt() - Move to the next packet in Rx queue. 341 * 342 * @oq: Octeon Rx queue data structure. 343 * @buff_info: Current packet buffer info. 344 * @read_idx: Current packet index in the ring. 345 * @desc_used: Current packet descriptor number. 346 * 347 * Free the resources associated with a packet. 348 * Increment packet index in the ring and packet descriptor number. 349 */ 350 static void octep_oq_next_pkt(struct octep_oq *oq, 351 struct octep_rx_buffer *buff_info, 352 u32 *read_idx, u32 *desc_used) 353 { 354 dma_unmap_page(oq->dev, oq->desc_ring[*read_idx].buffer_ptr, 355 PAGE_SIZE, DMA_FROM_DEVICE); 356 buff_info->page = NULL; 357 (*read_idx)++; 358 (*desc_used)++; 359 if (*read_idx == oq->max_count) 360 *read_idx = 0; 361 } 362 363 /** 364 * octep_oq_drop_rx() - Free the resources associated with a packet. 365 * 366 * @oq: Octeon Rx queue data structure. 367 * @buff_info: Current packet buffer info. 368 * @read_idx: Current packet index in the ring. 369 * @desc_used: Current packet descriptor number. 370 * 371 */ 372 static void octep_oq_drop_rx(struct octep_oq *oq, 373 struct octep_rx_buffer *buff_info, 374 u32 *read_idx, u32 *desc_used) 375 { 376 int data_len = buff_info->len - oq->max_single_buffer_size; 377 378 while (data_len > 0) { 379 octep_oq_next_pkt(oq, buff_info, read_idx, desc_used); 380 data_len -= oq->buffer_size; 381 }; 382 } 383 384 /** 385 * __octep_oq_process_rx() - Process hardware Rx queue and push to stack. 386 * 387 * @oct: Octeon device private data structure. 388 * @oq: Octeon Rx queue data structure. 389 * @pkts_to_process: number of packets to be processed. 390 * 391 * Process the new packets in Rx queue. 392 * Packets larger than single Rx buffer arrive in consecutive descriptors. 393 * But, count returned by the API only accounts full packets, not fragments. 394 * 395 * Return: number of packets processed and pushed to stack. 396 */ 397 static int __octep_oq_process_rx(struct octep_device *oct, 398 struct octep_oq *oq, u16 pkts_to_process) 399 { 400 struct octep_oq_resp_hw_ext *resp_hw_ext = NULL; 401 netdev_features_t feat = oq->netdev->features; 402 struct octep_rx_buffer *buff_info; 403 struct octep_oq_resp_hw *resp_hw; 404 u32 pkt, rx_bytes, desc_used; 405 struct sk_buff *skb; 406 u16 data_offset; 407 u16 rx_ol_flags; 408 u32 read_idx; 409 410 read_idx = oq->host_read_idx; 411 rx_bytes = 0; 412 desc_used = 0; 413 for (pkt = 0; pkt < pkts_to_process; pkt++) { 414 buff_info = (struct octep_rx_buffer *)&oq->buff_info[read_idx]; 415 resp_hw = page_address(buff_info->page); 416 417 /* Swap the length field that is in Big-Endian to CPU */ 418 buff_info->len = be64_to_cpu(resp_hw->length); 419 if (oct->conf->fw_info.rx_ol_flags) { 420 /* Extended response header is immediately after 421 * response header (resp_hw) 422 */ 423 resp_hw_ext = (struct octep_oq_resp_hw_ext *) 424 (resp_hw + 1); 425 buff_info->len -= OCTEP_OQ_RESP_HW_EXT_SIZE; 426 /* Packet Data is immediately after 427 * extended response header. 428 */ 429 data_offset = OCTEP_OQ_RESP_HW_SIZE + 430 OCTEP_OQ_RESP_HW_EXT_SIZE; 431 rx_ol_flags = resp_hw_ext->rx_ol_flags; 432 } else { 433 /* Data is immediately after 434 * Hardware Rx response header. 435 */ 436 data_offset = OCTEP_OQ_RESP_HW_SIZE; 437 rx_ol_flags = 0; 438 } 439 440 octep_oq_next_pkt(oq, buff_info, &read_idx, &desc_used); 441 442 skb = build_skb((void *)resp_hw, PAGE_SIZE); 443 if (!skb) { 444 octep_oq_drop_rx(oq, buff_info, 445 &read_idx, &desc_used); 446 oq->stats.alloc_failures++; 447 continue; 448 } 449 skb_reserve(skb, data_offset); 450 451 rx_bytes += buff_info->len; 452 453 if (buff_info->len <= oq->max_single_buffer_size) { 454 skb_put(skb, buff_info->len); 455 } else { 456 struct skb_shared_info *shinfo; 457 u16 data_len; 458 459 /* Head fragment includes response header(s); 460 * subsequent fragments contains only data. 461 */ 462 skb_put(skb, oq->max_single_buffer_size); 463 shinfo = skb_shinfo(skb); 464 data_len = buff_info->len - oq->max_single_buffer_size; 465 while (data_len) { 466 buff_info = (struct octep_rx_buffer *) 467 &oq->buff_info[read_idx]; 468 if (data_len < oq->buffer_size) { 469 buff_info->len = data_len; 470 data_len = 0; 471 } else { 472 buff_info->len = oq->buffer_size; 473 data_len -= oq->buffer_size; 474 } 475 476 skb_add_rx_frag(skb, shinfo->nr_frags, 477 buff_info->page, 0, 478 buff_info->len, 479 buff_info->len); 480 481 octep_oq_next_pkt(oq, buff_info, &read_idx, &desc_used); 482 } 483 } 484 485 skb->dev = oq->netdev; 486 skb->protocol = eth_type_trans(skb, skb->dev); 487 if (feat & NETIF_F_RXCSUM && 488 OCTEP_RX_CSUM_VERIFIED(rx_ol_flags)) 489 skb->ip_summed = CHECKSUM_UNNECESSARY; 490 else 491 skb->ip_summed = CHECKSUM_NONE; 492 napi_gro_receive(oq->napi, skb); 493 } 494 495 oq->host_read_idx = read_idx; 496 oq->refill_count += desc_used; 497 oq->stats.packets += pkt; 498 oq->stats.bytes += rx_bytes; 499 500 return pkt; 501 } 502 503 /** 504 * octep_oq_process_rx() - Process Rx queue. 505 * 506 * @oq: Octeon Rx queue data structure. 507 * @budget: max number of packets can be processed in one invocation. 508 * 509 * Check for newly received packets and process them. 510 * Keeps checking for new packets until budget is used or no new packets seen. 511 * 512 * Return: number of packets processed. 513 */ 514 int octep_oq_process_rx(struct octep_oq *oq, int budget) 515 { 516 u32 pkts_available, pkts_processed, total_pkts_processed; 517 struct octep_device *oct = oq->octep_dev; 518 519 pkts_available = 0; 520 pkts_processed = 0; 521 total_pkts_processed = 0; 522 while (total_pkts_processed < budget) { 523 /* update pending count only when current one exhausted */ 524 if (oq->pkts_pending == 0) 525 octep_oq_check_hw_for_pkts(oct, oq); 526 pkts_available = min(budget - total_pkts_processed, 527 oq->pkts_pending); 528 if (!pkts_available) 529 break; 530 531 pkts_processed = __octep_oq_process_rx(oct, oq, 532 pkts_available); 533 oq->pkts_pending -= pkts_processed; 534 total_pkts_processed += pkts_processed; 535 } 536 537 if (oq->refill_count >= oq->refill_threshold) { 538 u32 desc_refilled = octep_oq_refill(oct, oq); 539 540 /* flush pending writes before updating credits */ 541 wmb(); 542 writel(desc_refilled, oq->pkts_credit_reg); 543 } 544 545 return total_pkts_processed; 546 } 547