1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2016 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more details. 17 ***********************************************************************/ 18 #include <linux/pci.h> 19 #include <linux/netdevice.h> 20 #include <linux/vmalloc.h> 21 #include "liquidio_common.h" 22 #include "octeon_droq.h" 23 #include "octeon_iq.h" 24 #include "response_manager.h" 25 #include "octeon_device.h" 26 #include "octeon_main.h" 27 #include "octeon_network.h" 28 #include "cn66xx_regs.h" 29 #include "cn66xx_device.h" 30 #include "cn23xx_pf_device.h" 31 #include "cn23xx_vf_device.h" 32 33 struct __dispatch { 34 struct list_head list; 35 struct octeon_recv_info *rinfo; 36 octeon_dispatch_fn_t disp_fn; 37 }; 38 39 /** Get the argument that the user set when registering dispatch 40 * function for a given opcode/subcode. 41 * @param octeon_dev - the octeon device pointer. 42 * @param opcode - the opcode for which the dispatch argument 43 * is to be checked. 44 * @param subcode - the subcode for which the dispatch argument 45 * is to be checked. 46 * @return Success: void * (argument to the dispatch function) 47 * @return Failure: NULL 48 * 49 */ 50 void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev, 51 u16 opcode, u16 subcode) 52 { 53 int idx; 54 struct list_head *dispatch; 55 void *fn_arg = NULL; 56 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); 57 58 idx = combined_opcode & OCTEON_OPCODE_MASK; 59 60 spin_lock_bh(&octeon_dev->dispatch.lock); 61 62 if (octeon_dev->dispatch.count == 0) { 63 spin_unlock_bh(&octeon_dev->dispatch.lock); 64 return NULL; 65 } 66 67 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { 68 fn_arg = octeon_dev->dispatch.dlist[idx].arg; 69 } else { 70 list_for_each(dispatch, 71 &octeon_dev->dispatch.dlist[idx].list) { 72 if (((struct octeon_dispatch *)dispatch)->opcode == 73 combined_opcode) { 74 fn_arg = ((struct octeon_dispatch *) 75 dispatch)->arg; 76 break; 77 } 78 } 79 } 80 81 spin_unlock_bh(&octeon_dev->dispatch.lock); 82 return fn_arg; 83 } 84 85 /** Check for packets on Droq. This function should be called with lock held. 86 * @param droq - Droq on which count is checked. 87 * @return Returns packet count. 88 */ 89 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq) 90 { 91 u32 pkt_count = 0; 92 u32 last_count; 93 94 pkt_count = readl(droq->pkts_sent_reg); 95 96 last_count = pkt_count - droq->pkt_count; 97 droq->pkt_count = pkt_count; 98 99 /* we shall write to cnts at napi irq enable or end of droq tasklet */ 100 if (last_count) 101 atomic_add(last_count, &droq->pkts_pending); 102 103 return last_count; 104 } 105 EXPORT_SYMBOL_GPL(octeon_droq_check_hw_for_pkts); 106 107 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) 108 { 109 u32 count = 0; 110 111 /* max_empty_descs is the max. no. of descs that can have no buffers. 112 * If the empty desc count goes beyond this value, we cannot safely 113 * read in a 64K packet sent by Octeon 114 * (64K is max pkt size from Octeon) 115 */ 116 droq->max_empty_descs = 0; 117 118 do { 119 droq->max_empty_descs++; 120 count += droq->buffer_size; 121 } while (count < (64 * 1024)); 122 123 droq->max_empty_descs = droq->max_count - droq->max_empty_descs; 124 } 125 126 static void octeon_droq_reset_indices(struct octeon_droq *droq) 127 { 128 droq->read_idx = 0; 129 droq->write_idx = 0; 130 droq->refill_idx = 0; 131 droq->refill_count = 0; 132 atomic_set(&droq->pkts_pending, 0); 133 } 134 135 static void 136 octeon_droq_destroy_ring_buffers(struct octeon_device *oct, 137 struct octeon_droq *droq) 138 { 139 u32 i; 140 struct octeon_skb_page_info *pg_info; 141 142 for (i = 0; i < droq->max_count; i++) { 143 pg_info = &droq->recv_buf_list[i].pg_info; 144 if (!pg_info) 145 continue; 146 147 if (pg_info->dma) 148 lio_unmap_ring(oct->pci_dev, 149 (u64)pg_info->dma); 150 pg_info->dma = 0; 151 152 if (pg_info->page) 153 recv_buffer_destroy(droq->recv_buf_list[i].buffer, 154 pg_info); 155 156 droq->recv_buf_list[i].buffer = NULL; 157 } 158 159 octeon_droq_reset_indices(droq); 160 } 161 162 static int 163 octeon_droq_setup_ring_buffers(struct octeon_device *oct, 164 struct octeon_droq *droq) 165 { 166 u32 i; 167 void *buf; 168 struct octeon_droq_desc *desc_ring = droq->desc_ring; 169 170 for (i = 0; i < droq->max_count; i++) { 171 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info); 172 173 if (!buf) { 174 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n", 175 __func__); 176 droq->stats.rx_alloc_failure++; 177 return -ENOMEM; 178 } 179 180 droq->recv_buf_list[i].buffer = buf; 181 droq->recv_buf_list[i].data = get_rbd(buf); 182 desc_ring[i].info_ptr = 0; 183 desc_ring[i].buffer_ptr = 184 lio_map_ring(droq->recv_buf_list[i].buffer); 185 } 186 187 octeon_droq_reset_indices(droq); 188 189 octeon_droq_compute_max_packet_bufs(droq); 190 191 return 0; 192 } 193 194 int octeon_delete_droq(struct octeon_device *oct, u32 q_no) 195 { 196 struct octeon_droq *droq = oct->droq[q_no]; 197 198 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 199 200 octeon_droq_destroy_ring_buffers(oct, droq); 201 vfree(droq->recv_buf_list); 202 203 if (droq->desc_ring) 204 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), 205 droq->desc_ring, droq->desc_ring_dma); 206 207 memset(droq, 0, OCT_DROQ_SIZE); 208 oct->io_qmask.oq &= ~(1ULL << q_no); 209 vfree(oct->droq[q_no]); 210 oct->droq[q_no] = NULL; 211 oct->num_oqs--; 212 213 return 0; 214 } 215 EXPORT_SYMBOL_GPL(octeon_delete_droq); 216 217 int octeon_init_droq(struct octeon_device *oct, 218 u32 q_no, 219 u32 num_descs, 220 u32 desc_size, 221 void *app_ctx) 222 { 223 struct octeon_droq *droq; 224 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0; 225 u32 c_pkts_per_intr = 0, c_refill_threshold = 0; 226 int numa_node = dev_to_node(&oct->pci_dev->dev); 227 228 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 229 230 droq = oct->droq[q_no]; 231 memset(droq, 0, OCT_DROQ_SIZE); 232 233 droq->oct_dev = oct; 234 droq->q_no = q_no; 235 if (app_ctx) 236 droq->app_ctx = app_ctx; 237 else 238 droq->app_ctx = (void *)(size_t)q_no; 239 240 c_num_descs = num_descs; 241 c_buf_size = desc_size; 242 if (OCTEON_CN6XXX(oct)) { 243 struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx); 244 245 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x); 246 c_refill_threshold = 247 (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x); 248 } else if (OCTEON_CN23XX_PF(oct)) { 249 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf); 250 251 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); 252 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); 253 } else if (OCTEON_CN23XX_VF(oct)) { 254 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf); 255 256 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); 257 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); 258 } else { 259 return 1; 260 } 261 262 droq->max_count = c_num_descs; 263 droq->buffer_size = c_buf_size; 264 265 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; 266 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, 267 (dma_addr_t *)&droq->desc_ring_dma); 268 269 if (!droq->desc_ring) { 270 dev_err(&oct->pci_dev->dev, 271 "Output queue %d ring alloc failed\n", q_no); 272 return 1; 273 } 274 275 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", 276 q_no, droq->desc_ring, droq->desc_ring_dma); 277 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no, 278 droq->max_count); 279 280 droq->recv_buf_list = vzalloc_node(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE), 281 numa_node); 282 if (!droq->recv_buf_list) 283 droq->recv_buf_list = vzalloc(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE)); 284 if (!droq->recv_buf_list) { 285 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n"); 286 goto init_droq_fail; 287 } 288 289 if (octeon_droq_setup_ring_buffers(oct, droq)) 290 goto init_droq_fail; 291 292 droq->pkts_per_intr = c_pkts_per_intr; 293 droq->refill_threshold = c_refill_threshold; 294 295 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n", 296 droq->max_empty_descs); 297 298 INIT_LIST_HEAD(&droq->dispatch_list); 299 300 /* For 56xx Pass1, this function won't be called, so no checks. */ 301 oct->fn_list.setup_oq_regs(oct, q_no); 302 303 oct->io_qmask.oq |= BIT_ULL(q_no); 304 305 return 0; 306 307 init_droq_fail: 308 octeon_delete_droq(oct, q_no); 309 return 1; 310 } 311 312 /* octeon_create_recv_info 313 * Parameters: 314 * octeon_dev - pointer to the octeon device structure 315 * droq - droq in which the packet arrived. 316 * buf_cnt - no. of buffers used by the packet. 317 * idx - index in the descriptor for the first buffer in the packet. 318 * Description: 319 * Allocates a recv_info_t and copies the buffer addresses for packet data 320 * into the recv_pkt space which starts at an 8B offset from recv_info_t. 321 * Flags the descriptors for refill later. If available descriptors go 322 * below the threshold to receive a 64K pkt, new buffers are first allocated 323 * before the recv_pkt_t is created. 324 * This routine will be called in interrupt context. 325 * Returns: 326 * Success: Pointer to recv_info_t 327 * Failure: NULL. 328 */ 329 static inline struct octeon_recv_info *octeon_create_recv_info( 330 struct octeon_device *octeon_dev, 331 struct octeon_droq *droq, 332 u32 buf_cnt, 333 u32 idx) 334 { 335 struct octeon_droq_info *info; 336 struct octeon_recv_pkt *recv_pkt; 337 struct octeon_recv_info *recv_info; 338 u32 i, bytes_left; 339 struct octeon_skb_page_info *pg_info; 340 341 info = (struct octeon_droq_info *)droq->recv_buf_list[idx].data; 342 343 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch)); 344 if (!recv_info) 345 return NULL; 346 347 recv_pkt = recv_info->recv_pkt; 348 recv_pkt->rh = info->rh; 349 recv_pkt->length = (u32)info->length; 350 recv_pkt->buffer_count = (u16)buf_cnt; 351 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id; 352 353 i = 0; 354 bytes_left = (u32)info->length; 355 356 while (buf_cnt) { 357 { 358 pg_info = &droq->recv_buf_list[idx].pg_info; 359 360 lio_unmap_ring(octeon_dev->pci_dev, 361 (u64)pg_info->dma); 362 pg_info->page = NULL; 363 pg_info->dma = 0; 364 } 365 366 recv_pkt->buffer_size[i] = 367 (bytes_left >= 368 droq->buffer_size) ? droq->buffer_size : bytes_left; 369 370 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; 371 droq->recv_buf_list[idx].buffer = NULL; 372 373 idx = incr_index(idx, 1, droq->max_count); 374 bytes_left -= droq->buffer_size; 375 i++; 376 buf_cnt--; 377 } 378 379 return recv_info; 380 } 381 382 /* If we were not able to refill all buffers, try to move around 383 * the buffers that were not dispatched. 384 */ 385 static inline u32 386 octeon_droq_refill_pullup_descs(struct octeon_droq *droq, 387 struct octeon_droq_desc *desc_ring) 388 { 389 u32 desc_refilled = 0; 390 391 u32 refill_index = droq->refill_idx; 392 393 while (refill_index != droq->read_idx) { 394 if (droq->recv_buf_list[refill_index].buffer) { 395 droq->recv_buf_list[droq->refill_idx].buffer = 396 droq->recv_buf_list[refill_index].buffer; 397 droq->recv_buf_list[droq->refill_idx].data = 398 droq->recv_buf_list[refill_index].data; 399 desc_ring[droq->refill_idx].buffer_ptr = 400 desc_ring[refill_index].buffer_ptr; 401 droq->recv_buf_list[refill_index].buffer = NULL; 402 desc_ring[refill_index].buffer_ptr = 0; 403 do { 404 droq->refill_idx = incr_index(droq->refill_idx, 405 1, 406 droq->max_count); 407 desc_refilled++; 408 droq->refill_count--; 409 } while (droq->recv_buf_list[droq->refill_idx].buffer); 410 } 411 refill_index = incr_index(refill_index, 1, droq->max_count); 412 } /* while */ 413 return desc_refilled; 414 } 415 416 /* octeon_droq_refill 417 * Parameters: 418 * droq - droq in which descriptors require new buffers. 419 * Description: 420 * Called during normal DROQ processing in interrupt mode or by the poll 421 * thread to refill the descriptors from which buffers were dispatched 422 * to upper layers. Attempts to allocate new buffers. If that fails, moves 423 * up buffers (that were not dispatched) to form a contiguous ring. 424 * Returns: 425 * No of descriptors refilled. 426 */ 427 static u32 428 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) 429 { 430 struct octeon_droq_desc *desc_ring; 431 void *buf = NULL; 432 u8 *data; 433 u32 desc_refilled = 0; 434 struct octeon_skb_page_info *pg_info; 435 436 desc_ring = droq->desc_ring; 437 438 while (droq->refill_count && (desc_refilled < droq->max_count)) { 439 /* If a valid buffer exists (happens if there is no dispatch), 440 * reuse the buffer, else allocate. 441 */ 442 if (!droq->recv_buf_list[droq->refill_idx].buffer) { 443 pg_info = 444 &droq->recv_buf_list[droq->refill_idx].pg_info; 445 /* Either recycle the existing pages or go for 446 * new page alloc 447 */ 448 if (pg_info->page) 449 buf = recv_buffer_reuse(octeon_dev, pg_info); 450 else 451 buf = recv_buffer_alloc(octeon_dev, pg_info); 452 /* If a buffer could not be allocated, no point in 453 * continuing 454 */ 455 if (!buf) { 456 droq->stats.rx_alloc_failure++; 457 break; 458 } 459 droq->recv_buf_list[droq->refill_idx].buffer = 460 buf; 461 data = get_rbd(buf); 462 } else { 463 data = get_rbd(droq->recv_buf_list 464 [droq->refill_idx].buffer); 465 } 466 467 droq->recv_buf_list[droq->refill_idx].data = data; 468 469 desc_ring[droq->refill_idx].buffer_ptr = 470 lio_map_ring(droq->recv_buf_list[ 471 droq->refill_idx].buffer); 472 473 droq->refill_idx = incr_index(droq->refill_idx, 1, 474 droq->max_count); 475 desc_refilled++; 476 droq->refill_count--; 477 } 478 479 if (droq->refill_count) 480 desc_refilled += 481 octeon_droq_refill_pullup_descs(droq, desc_ring); 482 483 /* if droq->refill_count 484 * The refill count would not change in pass two. We only moved buffers 485 * to close the gap in the ring, but we would still have the same no. of 486 * buffers to refill. 487 */ 488 return desc_refilled; 489 } 490 491 /** check if we can allocate packets to get out of oom. 492 * @param droq - Droq being checked. 493 * @return 1 if fails to refill minimum 494 */ 495 int octeon_retry_droq_refill(struct octeon_droq *droq) 496 { 497 struct octeon_device *oct = droq->oct_dev; 498 int desc_refilled, reschedule = 1; 499 u32 pkts_credit; 500 501 pkts_credit = readl(droq->pkts_credit_reg); 502 desc_refilled = octeon_droq_refill(oct, droq); 503 if (desc_refilled) { 504 /* Flush the droq descriptor data to memory to be sure 505 * that when we update the credits the data in memory 506 * is accurate. 507 */ 508 wmb(); 509 writel(desc_refilled, droq->pkts_credit_reg); 510 511 if (pkts_credit + desc_refilled >= CN23XX_SLI_DEF_BP) 512 reschedule = 0; 513 } 514 515 return reschedule; 516 } 517 518 static inline u32 519 octeon_droq_get_bufcount(u32 buf_size, u32 total_len) 520 { 521 return DIV_ROUND_UP(total_len, buf_size); 522 } 523 524 static int 525 octeon_droq_dispatch_pkt(struct octeon_device *oct, 526 struct octeon_droq *droq, 527 union octeon_rh *rh, 528 struct octeon_droq_info *info) 529 { 530 u32 cnt; 531 octeon_dispatch_fn_t disp_fn; 532 struct octeon_recv_info *rinfo; 533 534 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); 535 536 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode, 537 (u16)rh->r.subcode); 538 if (disp_fn) { 539 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); 540 if (rinfo) { 541 struct __dispatch *rdisp = rinfo->rsvd; 542 543 rdisp->rinfo = rinfo; 544 rdisp->disp_fn = disp_fn; 545 rinfo->recv_pkt->rh = *rh; 546 list_add_tail(&rdisp->list, 547 &droq->dispatch_list); 548 } else { 549 droq->stats.dropped_nomem++; 550 } 551 } else { 552 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n", 553 (unsigned int)rh->r.opcode, 554 (unsigned int)rh->r.subcode); 555 droq->stats.dropped_nodispatch++; 556 } 557 558 return cnt; 559 } 560 561 static inline void octeon_droq_drop_packets(struct octeon_device *oct, 562 struct octeon_droq *droq, 563 u32 cnt) 564 { 565 u32 i = 0, buf_cnt; 566 struct octeon_droq_info *info; 567 568 for (i = 0; i < cnt; i++) { 569 info = (struct octeon_droq_info *) 570 droq->recv_buf_list[droq->read_idx].data; 571 octeon_swap_8B_data((u64 *)info, 2); 572 573 if (info->length) { 574 info->length += OCTNET_FRM_LENGTH_SIZE; 575 droq->stats.bytes_received += info->length; 576 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, 577 (u32)info->length); 578 } else { 579 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n"); 580 buf_cnt = 1; 581 } 582 583 droq->read_idx = incr_index(droq->read_idx, buf_cnt, 584 droq->max_count); 585 droq->refill_count += buf_cnt; 586 } 587 } 588 589 static u32 590 octeon_droq_fast_process_packets(struct octeon_device *oct, 591 struct octeon_droq *droq, 592 u32 pkts_to_process) 593 { 594 u32 pkt, total_len = 0, pkt_count, retval; 595 struct octeon_droq_info *info; 596 union octeon_rh *rh; 597 598 pkt_count = pkts_to_process; 599 600 for (pkt = 0; pkt < pkt_count; pkt++) { 601 u32 pkt_len = 0; 602 struct sk_buff *nicbuf = NULL; 603 struct octeon_skb_page_info *pg_info; 604 void *buf; 605 606 info = (struct octeon_droq_info *) 607 droq->recv_buf_list[droq->read_idx].data; 608 octeon_swap_8B_data((u64 *)info, 2); 609 610 if (!info->length) { 611 dev_err(&oct->pci_dev->dev, 612 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", 613 droq->q_no, droq->read_idx, pkt_count); 614 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS, 615 (u8 *)info, 616 OCT_DROQ_INFO_SIZE); 617 break; 618 } 619 620 /* Len of resp hdr in included in the received data len. */ 621 rh = &info->rh; 622 623 info->length += OCTNET_FRM_LENGTH_SIZE; 624 rh->r_dh.len += (ROUNDUP8(OCT_DROQ_INFO_SIZE) / sizeof(u64)); 625 total_len += (u32)info->length; 626 if (opcode_slow_path(rh)) { 627 u32 buf_cnt; 628 629 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); 630 droq->read_idx = incr_index(droq->read_idx, 631 buf_cnt, droq->max_count); 632 droq->refill_count += buf_cnt; 633 } else { 634 if (info->length <= droq->buffer_size) { 635 pkt_len = (u32)info->length; 636 nicbuf = droq->recv_buf_list[ 637 droq->read_idx].buffer; 638 pg_info = &droq->recv_buf_list[ 639 droq->read_idx].pg_info; 640 if (recv_buffer_recycle(oct, pg_info)) 641 pg_info->page = NULL; 642 droq->recv_buf_list[droq->read_idx].buffer = 643 NULL; 644 645 droq->read_idx = incr_index(droq->read_idx, 1, 646 droq->max_count); 647 droq->refill_count++; 648 } else { 649 nicbuf = octeon_fast_packet_alloc((u32) 650 info->length); 651 pkt_len = 0; 652 /* nicbuf allocation can fail. We'll handle it 653 * inside the loop. 654 */ 655 while (pkt_len < info->length) { 656 int cpy_len, idx = droq->read_idx; 657 658 cpy_len = ((pkt_len + droq->buffer_size) 659 > info->length) ? 660 ((u32)info->length - pkt_len) : 661 droq->buffer_size; 662 663 if (nicbuf) { 664 octeon_fast_packet_next(droq, 665 nicbuf, 666 cpy_len, 667 idx); 668 buf = droq->recv_buf_list[ 669 idx].buffer; 670 recv_buffer_fast_free(buf); 671 droq->recv_buf_list[idx].buffer 672 = NULL; 673 } else { 674 droq->stats.rx_alloc_failure++; 675 } 676 677 pkt_len += cpy_len; 678 droq->read_idx = 679 incr_index(droq->read_idx, 1, 680 droq->max_count); 681 droq->refill_count++; 682 } 683 } 684 685 if (nicbuf) { 686 if (droq->ops.fptr) { 687 droq->ops.fptr(oct->octeon_id, 688 nicbuf, pkt_len, 689 rh, &droq->napi, 690 droq->ops.farg); 691 } else { 692 recv_buffer_free(nicbuf); 693 } 694 } 695 } 696 697 if (droq->refill_count >= droq->refill_threshold) { 698 int desc_refilled = octeon_droq_refill(oct, droq); 699 700 if (desc_refilled) { 701 /* Flush the droq descriptor data to memory to 702 * be sure that when we update the credits the 703 * data in memory is accurate. 704 */ 705 wmb(); 706 writel(desc_refilled, droq->pkts_credit_reg); 707 } 708 } 709 } /* for (each packet)... */ 710 711 /* Increment refill_count by the number of buffers processed. */ 712 droq->stats.pkts_received += pkt; 713 droq->stats.bytes_received += total_len; 714 715 retval = pkt; 716 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { 717 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); 718 719 droq->stats.dropped_toomany += (pkts_to_process - pkt); 720 retval = pkts_to_process; 721 } 722 723 atomic_sub(retval, &droq->pkts_pending); 724 725 if (droq->refill_count >= droq->refill_threshold && 726 readl(droq->pkts_credit_reg) < CN23XX_SLI_DEF_BP) { 727 octeon_droq_check_hw_for_pkts(droq); 728 729 /* Make sure there are no pkts_pending */ 730 if (!atomic_read(&droq->pkts_pending)) 731 octeon_schedule_rxq_oom_work(oct, droq); 732 } 733 734 return retval; 735 } 736 737 int 738 octeon_droq_process_packets(struct octeon_device *oct, 739 struct octeon_droq *droq, 740 u32 budget) 741 { 742 u32 pkt_count = 0; 743 struct list_head *tmp, *tmp2; 744 745 octeon_droq_check_hw_for_pkts(droq); 746 pkt_count = atomic_read(&droq->pkts_pending); 747 748 if (!pkt_count) 749 return 0; 750 751 if (pkt_count > budget) 752 pkt_count = budget; 753 754 octeon_droq_fast_process_packets(oct, droq, pkt_count); 755 756 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 757 struct __dispatch *rdisp = (struct __dispatch *)tmp; 758 759 list_del(tmp); 760 rdisp->disp_fn(rdisp->rinfo, 761 octeon_get_dispatch_arg 762 (oct, 763 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 764 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 765 } 766 767 /* If there are packets pending. schedule tasklet again */ 768 if (atomic_read(&droq->pkts_pending)) 769 return 1; 770 771 return 0; 772 } 773 EXPORT_SYMBOL_GPL(octeon_droq_process_packets); 774 775 /* 776 * Utility function to poll for packets. check_hw_for_packets must be 777 * called before calling this routine. 778 */ 779 780 int 781 octeon_droq_process_poll_pkts(struct octeon_device *oct, 782 struct octeon_droq *droq, u32 budget) 783 { 784 struct list_head *tmp, *tmp2; 785 u32 pkts_available = 0, pkts_processed = 0; 786 u32 total_pkts_processed = 0; 787 788 if (budget > droq->max_count) 789 budget = droq->max_count; 790 791 while (total_pkts_processed < budget) { 792 octeon_droq_check_hw_for_pkts(droq); 793 794 pkts_available = min((budget - total_pkts_processed), 795 (u32)(atomic_read(&droq->pkts_pending))); 796 797 if (pkts_available == 0) 798 break; 799 800 pkts_processed = 801 octeon_droq_fast_process_packets(oct, droq, 802 pkts_available); 803 804 total_pkts_processed += pkts_processed; 805 } 806 807 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 808 struct __dispatch *rdisp = (struct __dispatch *)tmp; 809 810 list_del(tmp); 811 rdisp->disp_fn(rdisp->rinfo, 812 octeon_get_dispatch_arg 813 (oct, 814 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 815 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 816 } 817 818 return total_pkts_processed; 819 } 820 821 /* Enable Pkt Interrupt */ 822 int 823 octeon_enable_irq(struct octeon_device *oct, u32 q_no) 824 { 825 switch (oct->chip_id) { 826 case OCTEON_CN66XX: 827 case OCTEON_CN68XX: { 828 struct octeon_cn6xxx *cn6xxx = 829 (struct octeon_cn6xxx *)oct->chip; 830 unsigned long flags; 831 u32 value; 832 833 spin_lock_irqsave 834 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 835 value = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB); 836 value |= (1 << q_no); 837 octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB, value); 838 value = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB); 839 value |= (1 << q_no); 840 octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB, value); 841 842 /* don't bother flushing the enables */ 843 844 spin_unlock_irqrestore 845 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 846 } 847 break; 848 case OCTEON_CN23XX_PF_VID: 849 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); 850 break; 851 852 case OCTEON_CN23XX_VF_VID: 853 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]); 854 break; 855 default: 856 dev_err(&oct->pci_dev->dev, "%s Unknown Chip\n", __func__); 857 return 1; 858 } 859 860 return 0; 861 } 862 863 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no, 864 struct octeon_droq_ops *ops) 865 { 866 struct octeon_config *oct_cfg = NULL; 867 struct octeon_droq *droq; 868 869 oct_cfg = octeon_get_conf(oct); 870 871 if (!oct_cfg) 872 return -EINVAL; 873 874 if (!(ops)) { 875 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n", 876 __func__); 877 return -EINVAL; 878 } 879 880 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 881 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 882 __func__, q_no, (oct->num_oqs - 1)); 883 return -EINVAL; 884 } 885 886 droq = oct->droq[q_no]; 887 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); 888 889 return 0; 890 } 891 892 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no) 893 { 894 struct octeon_config *oct_cfg = NULL; 895 struct octeon_droq *droq; 896 897 oct_cfg = octeon_get_conf(oct); 898 899 if (!oct_cfg) 900 return -EINVAL; 901 902 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 903 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 904 __func__, q_no, oct->num_oqs - 1); 905 return -EINVAL; 906 } 907 908 droq = oct->droq[q_no]; 909 910 if (!droq) { 911 dev_info(&oct->pci_dev->dev, 912 "Droq id (%d) not available.\n", q_no); 913 return 0; 914 } 915 916 droq->ops.fptr = NULL; 917 droq->ops.farg = NULL; 918 droq->ops.drop_on_max = 0; 919 920 return 0; 921 } 922 EXPORT_SYMBOL_GPL(octeon_unregister_droq_ops); 923 924 int octeon_create_droq(struct octeon_device *oct, 925 u32 q_no, u32 num_descs, 926 u32 desc_size, void *app_ctx) 927 { 928 struct octeon_droq *droq; 929 int numa_node = dev_to_node(&oct->pci_dev->dev); 930 931 if (oct->droq[q_no]) { 932 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n", 933 q_no); 934 return 1; 935 } 936 937 /* Allocate the DS for the new droq. */ 938 droq = vmalloc_node(sizeof(*droq), numa_node); 939 if (!droq) 940 droq = vmalloc(sizeof(*droq)); 941 if (!droq) 942 return -1; 943 944 memset(droq, 0, sizeof(struct octeon_droq)); 945 946 /*Disable the pkt o/p for this Q */ 947 octeon_set_droq_pkt_op(oct, q_no, 0); 948 oct->droq[q_no] = droq; 949 950 /* Initialize the Droq */ 951 if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) { 952 vfree(oct->droq[q_no]); 953 oct->droq[q_no] = NULL; 954 return -1; 955 } 956 957 oct->num_oqs++; 958 959 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__, 960 oct->num_oqs); 961 962 /* Global Droq register settings */ 963 964 /* As of now not required, as setting are done for all 32 Droqs at 965 * the same time. 966 */ 967 return 0; 968 } 969