1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2015 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more 17 * details. 18 * 19 * This file may also be available under a different license from Cavium. 20 * Contact Cavium, Inc. for more information 21 **********************************************************************/ 22 #include <linux/version.h> 23 #include <linux/types.h> 24 #include <linux/list.h> 25 #include <linux/pci.h> 26 #include <linux/kthread.h> 27 #include <linux/netdevice.h> 28 #include <linux/vmalloc.h> 29 #include "octeon_config.h" 30 #include "liquidio_common.h" 31 #include "octeon_droq.h" 32 #include "octeon_iq.h" 33 #include "response_manager.h" 34 #include "octeon_device.h" 35 #include "octeon_nic.h" 36 #include "octeon_main.h" 37 #include "octeon_network.h" 38 #include "cn66xx_regs.h" 39 #include "cn66xx_device.h" 40 #include "cn68xx_regs.h" 41 #include "cn68xx_device.h" 42 #include "liquidio_image.h" 43 #include "octeon_mem_ops.h" 44 45 /* #define CAVIUM_ONLY_PERF_MODE */ 46 47 #define CVM_MIN(d1, d2) (((d1) < (d2)) ? (d1) : (d2)) 48 #define CVM_MAX(d1, d2) (((d1) > (d2)) ? (d1) : (d2)) 49 50 struct niclist { 51 struct list_head list; 52 void *ptr; 53 }; 54 55 struct __dispatch { 56 struct list_head list; 57 struct octeon_recv_info *rinfo; 58 octeon_dispatch_fn_t disp_fn; 59 }; 60 61 /** Get the argument that the user set when registering dispatch 62 * function for a given opcode/subcode. 63 * @param octeon_dev - the octeon device pointer. 64 * @param opcode - the opcode for which the dispatch argument 65 * is to be checked. 66 * @param subcode - the subcode for which the dispatch argument 67 * is to be checked. 68 * @return Success: void * (argument to the dispatch function) 69 * @return Failure: NULL 70 * 71 */ 72 static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev, 73 u16 opcode, u16 subcode) 74 { 75 int idx; 76 struct list_head *dispatch; 77 void *fn_arg = NULL; 78 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); 79 80 idx = combined_opcode & OCTEON_OPCODE_MASK; 81 82 spin_lock_bh(&octeon_dev->dispatch.lock); 83 84 if (octeon_dev->dispatch.count == 0) { 85 spin_unlock_bh(&octeon_dev->dispatch.lock); 86 return NULL; 87 } 88 89 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { 90 fn_arg = octeon_dev->dispatch.dlist[idx].arg; 91 } else { 92 list_for_each(dispatch, 93 &octeon_dev->dispatch.dlist[idx].list) { 94 if (((struct octeon_dispatch *)dispatch)->opcode == 95 combined_opcode) { 96 fn_arg = ((struct octeon_dispatch *) 97 dispatch)->arg; 98 break; 99 } 100 } 101 } 102 103 spin_unlock_bh(&octeon_dev->dispatch.lock); 104 return fn_arg; 105 } 106 107 u32 octeon_droq_check_hw_for_pkts(struct octeon_device *oct, 108 struct octeon_droq *droq) 109 { 110 u32 pkt_count = 0; 111 112 pkt_count = readl(droq->pkts_sent_reg); 113 if (pkt_count) { 114 atomic_add(pkt_count, &droq->pkts_pending); 115 writel(pkt_count, droq->pkts_sent_reg); 116 } 117 118 return pkt_count; 119 } 120 121 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) 122 { 123 u32 count = 0; 124 125 /* max_empty_descs is the max. no. of descs that can have no buffers. 126 * If the empty desc count goes beyond this value, we cannot safely 127 * read in a 64K packet sent by Octeon 128 * (64K is max pkt size from Octeon) 129 */ 130 droq->max_empty_descs = 0; 131 132 do { 133 droq->max_empty_descs++; 134 count += droq->buffer_size; 135 } while (count < (64 * 1024)); 136 137 droq->max_empty_descs = droq->max_count - droq->max_empty_descs; 138 } 139 140 static void octeon_droq_reset_indices(struct octeon_droq *droq) 141 { 142 droq->read_idx = 0; 143 droq->write_idx = 0; 144 droq->refill_idx = 0; 145 droq->refill_count = 0; 146 atomic_set(&droq->pkts_pending, 0); 147 } 148 149 static void 150 octeon_droq_destroy_ring_buffers(struct octeon_device *oct, 151 struct octeon_droq *droq) 152 { 153 u32 i; 154 155 for (i = 0; i < droq->max_count; i++) { 156 if (droq->recv_buf_list[i].buffer) { 157 if (droq->desc_ring) { 158 lio_unmap_ring_info(oct->pci_dev, 159 (u64)droq-> 160 desc_ring[i].info_ptr, 161 OCT_DROQ_INFO_SIZE); 162 lio_unmap_ring(oct->pci_dev, 163 (u64)droq->desc_ring[i]. 164 buffer_ptr, 165 droq->buffer_size); 166 } 167 recv_buffer_free(droq->recv_buf_list[i].buffer); 168 droq->recv_buf_list[i].buffer = NULL; 169 } 170 } 171 172 octeon_droq_reset_indices(droq); 173 } 174 175 static int 176 octeon_droq_setup_ring_buffers(struct octeon_device *oct, 177 struct octeon_droq *droq) 178 { 179 u32 i; 180 void *buf; 181 struct octeon_droq_desc *desc_ring = droq->desc_ring; 182 183 for (i = 0; i < droq->max_count; i++) { 184 buf = recv_buffer_alloc(oct, droq->q_no, droq->buffer_size); 185 186 if (!buf) { 187 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n", 188 __func__); 189 return -ENOMEM; 190 } 191 192 droq->recv_buf_list[i].buffer = buf; 193 droq->recv_buf_list[i].data = get_rbd(buf); 194 195 droq->info_list[i].length = 0; 196 197 /* map ring buffers into memory */ 198 desc_ring[i].info_ptr = lio_map_ring_info(droq, i); 199 desc_ring[i].buffer_ptr = 200 lio_map_ring(oct->pci_dev, 201 droq->recv_buf_list[i].buffer, 202 droq->buffer_size); 203 } 204 205 octeon_droq_reset_indices(droq); 206 207 octeon_droq_compute_max_packet_bufs(droq); 208 209 return 0; 210 } 211 212 int octeon_delete_droq(struct octeon_device *oct, u32 q_no) 213 { 214 struct octeon_droq *droq = oct->droq[q_no]; 215 216 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 217 218 octeon_droq_destroy_ring_buffers(oct, droq); 219 220 if (droq->recv_buf_list) 221 vfree(droq->recv_buf_list); 222 223 if (droq->info_base_addr) 224 cnnic_free_aligned_dma(oct->pci_dev, droq->info_list, 225 droq->info_alloc_size, 226 droq->info_base_addr, 227 droq->info_list_dma); 228 229 if (droq->desc_ring) 230 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), 231 droq->desc_ring, droq->desc_ring_dma); 232 233 memset(droq, 0, OCT_DROQ_SIZE); 234 235 return 0; 236 } 237 238 int octeon_init_droq(struct octeon_device *oct, 239 u32 q_no, 240 u32 num_descs, 241 u32 desc_size, 242 void *app_ctx) 243 { 244 struct octeon_droq *droq; 245 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0; 246 u32 c_pkts_per_intr = 0, c_refill_threshold = 0; 247 248 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no); 249 250 droq = oct->droq[q_no]; 251 memset(droq, 0, OCT_DROQ_SIZE); 252 253 droq->oct_dev = oct; 254 droq->q_no = q_no; 255 if (app_ctx) 256 droq->app_ctx = app_ctx; 257 else 258 droq->app_ctx = (void *)(size_t)q_no; 259 260 c_num_descs = num_descs; 261 c_buf_size = desc_size; 262 if (OCTEON_CN6XXX(oct)) { 263 struct octeon_config *conf6x = CHIP_FIELD(oct, cn6xxx, conf); 264 265 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x); 266 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x); 267 } 268 269 droq->max_count = c_num_descs; 270 droq->buffer_size = c_buf_size; 271 272 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; 273 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, 274 (dma_addr_t *)&droq->desc_ring_dma); 275 276 if (!droq->desc_ring) { 277 dev_err(&oct->pci_dev->dev, 278 "Output queue %d ring alloc failed\n", q_no); 279 return 1; 280 } 281 282 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n", 283 q_no, droq->desc_ring, droq->desc_ring_dma); 284 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no, 285 droq->max_count); 286 287 droq->info_list = 288 cnnic_alloc_aligned_dma(oct->pci_dev, 289 (droq->max_count * OCT_DROQ_INFO_SIZE), 290 &droq->info_alloc_size, 291 &droq->info_base_addr, 292 &droq->info_list_dma); 293 294 if (!droq->info_list) { 295 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n"); 296 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), 297 droq->desc_ring, droq->desc_ring_dma); 298 return 1; 299 } 300 301 droq->recv_buf_list = (struct octeon_recv_buffer *) 302 vmalloc(droq->max_count * 303 OCT_DROQ_RECVBUF_SIZE); 304 if (!droq->recv_buf_list) { 305 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n"); 306 goto init_droq_fail; 307 } 308 309 if (octeon_droq_setup_ring_buffers(oct, droq)) 310 goto init_droq_fail; 311 312 droq->pkts_per_intr = c_pkts_per_intr; 313 droq->refill_threshold = c_refill_threshold; 314 315 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n", 316 droq->max_empty_descs); 317 318 spin_lock_init(&droq->lock); 319 320 INIT_LIST_HEAD(&droq->dispatch_list); 321 322 /* For 56xx Pass1, this function won't be called, so no checks. */ 323 oct->fn_list.setup_oq_regs(oct, q_no); 324 325 oct->io_qmask.oq |= (1 << q_no); 326 327 return 0; 328 329 init_droq_fail: 330 octeon_delete_droq(oct, q_no); 331 return 1; 332 } 333 334 /* octeon_create_recv_info 335 * Parameters: 336 * octeon_dev - pointer to the octeon device structure 337 * droq - droq in which the packet arrived. 338 * buf_cnt - no. of buffers used by the packet. 339 * idx - index in the descriptor for the first buffer in the packet. 340 * Description: 341 * Allocates a recv_info_t and copies the buffer addresses for packet data 342 * into the recv_pkt space which starts at an 8B offset from recv_info_t. 343 * Flags the descriptors for refill later. If available descriptors go 344 * below the threshold to receive a 64K pkt, new buffers are first allocated 345 * before the recv_pkt_t is created. 346 * This routine will be called in interrupt context. 347 * Returns: 348 * Success: Pointer to recv_info_t 349 * Failure: NULL. 350 * Locks: 351 * The droq->lock is held when this routine is called. 352 */ 353 static inline struct octeon_recv_info *octeon_create_recv_info( 354 struct octeon_device *octeon_dev, 355 struct octeon_droq *droq, 356 u32 buf_cnt, 357 u32 idx) 358 { 359 struct octeon_droq_info *info; 360 struct octeon_recv_pkt *recv_pkt; 361 struct octeon_recv_info *recv_info; 362 u32 i, bytes_left; 363 364 info = &droq->info_list[idx]; 365 366 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch)); 367 if (!recv_info) 368 return NULL; 369 370 recv_pkt = recv_info->recv_pkt; 371 recv_pkt->rh = info->rh; 372 recv_pkt->length = (u32)info->length; 373 recv_pkt->buffer_count = (u16)buf_cnt; 374 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id; 375 376 i = 0; 377 bytes_left = (u32)info->length; 378 379 while (buf_cnt) { 380 lio_unmap_ring(octeon_dev->pci_dev, 381 (u64)droq->desc_ring[idx].buffer_ptr, 382 droq->buffer_size); 383 384 recv_pkt->buffer_size[i] = 385 (bytes_left >= 386 droq->buffer_size) ? droq->buffer_size : bytes_left; 387 388 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; 389 droq->recv_buf_list[idx].buffer = NULL; 390 391 INCR_INDEX_BY1(idx, droq->max_count); 392 bytes_left -= droq->buffer_size; 393 i++; 394 buf_cnt--; 395 } 396 397 return recv_info; 398 } 399 400 /* If we were not able to refill all buffers, try to move around 401 * the buffers that were not dispatched. 402 */ 403 static inline u32 404 octeon_droq_refill_pullup_descs(struct octeon_droq *droq, 405 struct octeon_droq_desc *desc_ring) 406 { 407 u32 desc_refilled = 0; 408 409 u32 refill_index = droq->refill_idx; 410 411 while (refill_index != droq->read_idx) { 412 if (droq->recv_buf_list[refill_index].buffer) { 413 droq->recv_buf_list[droq->refill_idx].buffer = 414 droq->recv_buf_list[refill_index].buffer; 415 droq->recv_buf_list[droq->refill_idx].data = 416 droq->recv_buf_list[refill_index].data; 417 desc_ring[droq->refill_idx].buffer_ptr = 418 desc_ring[refill_index].buffer_ptr; 419 droq->recv_buf_list[refill_index].buffer = NULL; 420 desc_ring[refill_index].buffer_ptr = 0; 421 do { 422 INCR_INDEX_BY1(droq->refill_idx, 423 droq->max_count); 424 desc_refilled++; 425 droq->refill_count--; 426 } while (droq->recv_buf_list[droq->refill_idx]. 427 buffer); 428 } 429 INCR_INDEX_BY1(refill_index, droq->max_count); 430 } /* while */ 431 return desc_refilled; 432 } 433 434 /* octeon_droq_refill 435 * Parameters: 436 * droq - droq in which descriptors require new buffers. 437 * Description: 438 * Called during normal DROQ processing in interrupt mode or by the poll 439 * thread to refill the descriptors from which buffers were dispatched 440 * to upper layers. Attempts to allocate new buffers. If that fails, moves 441 * up buffers (that were not dispatched) to form a contiguous ring. 442 * Returns: 443 * No of descriptors refilled. 444 * Locks: 445 * This routine is called with droq->lock held. 446 */ 447 static u32 448 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) 449 { 450 struct octeon_droq_desc *desc_ring; 451 void *buf = NULL; 452 u8 *data; 453 u32 desc_refilled = 0; 454 455 desc_ring = droq->desc_ring; 456 457 while (droq->refill_count && (desc_refilled < droq->max_count)) { 458 /* If a valid buffer exists (happens if there is no dispatch), 459 * reuse 460 * the buffer, else allocate. 461 */ 462 if (!droq->recv_buf_list[droq->refill_idx].buffer) { 463 buf = recv_buffer_alloc(octeon_dev, droq->q_no, 464 droq->buffer_size); 465 /* If a buffer could not be allocated, no point in 466 * continuing 467 */ 468 if (!buf) 469 break; 470 droq->recv_buf_list[droq->refill_idx].buffer = 471 buf; 472 data = get_rbd(buf); 473 } else { 474 data = get_rbd(droq->recv_buf_list 475 [droq->refill_idx].buffer); 476 } 477 478 droq->recv_buf_list[droq->refill_idx].data = data; 479 480 desc_ring[droq->refill_idx].buffer_ptr = 481 lio_map_ring(octeon_dev->pci_dev, 482 droq->recv_buf_list[droq-> 483 refill_idx].buffer, 484 droq->buffer_size); 485 486 /* Reset any previous values in the length field. */ 487 droq->info_list[droq->refill_idx].length = 0; 488 489 INCR_INDEX_BY1(droq->refill_idx, droq->max_count); 490 desc_refilled++; 491 droq->refill_count--; 492 } 493 494 if (droq->refill_count) 495 desc_refilled += 496 octeon_droq_refill_pullup_descs(droq, desc_ring); 497 498 /* if droq->refill_count 499 * The refill count would not change in pass two. We only moved buffers 500 * to close the gap in the ring, but we would still have the same no. of 501 * buffers to refill. 502 */ 503 return desc_refilled; 504 } 505 506 static inline u32 507 octeon_droq_get_bufcount(u32 buf_size, u32 total_len) 508 { 509 u32 buf_cnt = 0; 510 511 while (total_len > (buf_size * buf_cnt)) 512 buf_cnt++; 513 return buf_cnt; 514 } 515 516 static int 517 octeon_droq_dispatch_pkt(struct octeon_device *oct, 518 struct octeon_droq *droq, 519 union octeon_rh *rh, 520 struct octeon_droq_info *info) 521 { 522 u32 cnt; 523 octeon_dispatch_fn_t disp_fn; 524 struct octeon_recv_info *rinfo; 525 526 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length); 527 528 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode, 529 (u16)rh->r.subcode); 530 if (disp_fn) { 531 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx); 532 if (rinfo) { 533 struct __dispatch *rdisp = rinfo->rsvd; 534 535 rdisp->rinfo = rinfo; 536 rdisp->disp_fn = disp_fn; 537 rinfo->recv_pkt->rh = *rh; 538 list_add_tail(&rdisp->list, 539 &droq->dispatch_list); 540 } else { 541 droq->stats.dropped_nomem++; 542 } 543 } else { 544 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function\n"); 545 droq->stats.dropped_nodispatch++; 546 } /* else (dispatch_fn ... */ 547 548 return cnt; 549 } 550 551 static inline void octeon_droq_drop_packets(struct octeon_device *oct, 552 struct octeon_droq *droq, 553 u32 cnt) 554 { 555 u32 i = 0, buf_cnt; 556 struct octeon_droq_info *info; 557 558 for (i = 0; i < cnt; i++) { 559 info = &droq->info_list[droq->read_idx]; 560 octeon_swap_8B_data((u64 *)info, 2); 561 562 if (info->length) { 563 info->length -= OCT_RH_SIZE; 564 droq->stats.bytes_received += info->length; 565 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size, 566 (u32)info->length); 567 } else { 568 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n"); 569 buf_cnt = 1; 570 } 571 572 INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count); 573 droq->refill_count += buf_cnt; 574 } 575 } 576 577 static u32 578 octeon_droq_fast_process_packets(struct octeon_device *oct, 579 struct octeon_droq *droq, 580 u32 pkts_to_process) 581 { 582 struct octeon_droq_info *info; 583 union octeon_rh *rh; 584 u32 pkt, total_len = 0, pkt_count; 585 586 pkt_count = pkts_to_process; 587 588 for (pkt = 0; pkt < pkt_count; pkt++) { 589 u32 pkt_len = 0; 590 struct sk_buff *nicbuf = NULL; 591 592 info = &droq->info_list[droq->read_idx]; 593 octeon_swap_8B_data((u64 *)info, 2); 594 595 if (!info->length) { 596 dev_err(&oct->pci_dev->dev, 597 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n", 598 droq->q_no, droq->read_idx, pkt_count); 599 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS, 600 (u8 *)info, 601 OCT_DROQ_INFO_SIZE); 602 break; 603 } 604 605 /* Len of resp hdr in included in the received data len. */ 606 info->length -= OCT_RH_SIZE; 607 rh = &info->rh; 608 609 total_len += (u32)info->length; 610 611 if (OPCODE_SLOW_PATH(rh)) { 612 u32 buf_cnt; 613 614 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); 615 INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count); 616 droq->refill_count += buf_cnt; 617 } else { 618 if (info->length <= droq->buffer_size) { 619 lio_unmap_ring(oct->pci_dev, 620 (u64)droq->desc_ring[ 621 droq->read_idx].buffer_ptr, 622 droq->buffer_size); 623 pkt_len = (u32)info->length; 624 nicbuf = droq->recv_buf_list[ 625 droq->read_idx].buffer; 626 droq->recv_buf_list[droq->read_idx].buffer = 627 NULL; 628 INCR_INDEX_BY1(droq->read_idx, droq->max_count); 629 skb_put(nicbuf, pkt_len); 630 droq->refill_count++; 631 } else { 632 nicbuf = octeon_fast_packet_alloc(oct, droq, 633 droq->q_no, 634 (u32) 635 info->length); 636 pkt_len = 0; 637 /* nicbuf allocation can fail. We'll handle it 638 * inside the loop. 639 */ 640 while (pkt_len < info->length) { 641 int cpy_len; 642 643 cpy_len = ((pkt_len + 644 droq->buffer_size) > 645 info->length) ? 646 ((u32)info->length - pkt_len) : 647 droq->buffer_size; 648 649 if (nicbuf) { 650 lio_unmap_ring(oct->pci_dev, 651 (u64) 652 droq->desc_ring 653 [droq->read_idx]. 654 buffer_ptr, 655 droq-> 656 buffer_size); 657 octeon_fast_packet_next(droq, 658 nicbuf, 659 cpy_len, 660 droq-> 661 read_idx 662 ); 663 } 664 665 pkt_len += cpy_len; 666 INCR_INDEX_BY1(droq->read_idx, 667 droq->max_count); 668 droq->refill_count++; 669 } 670 } 671 672 if (nicbuf) { 673 if (droq->ops.fptr) 674 droq->ops.fptr(oct->octeon_id, 675 nicbuf, pkt_len, 676 rh, &droq->napi); 677 else 678 recv_buffer_free(nicbuf); 679 } 680 } 681 682 if (droq->refill_count >= droq->refill_threshold) { 683 int desc_refilled = octeon_droq_refill(oct, droq); 684 685 /* Flush the droq descriptor data to memory to be sure 686 * that when we update the credits the data in memory 687 * is accurate. 688 */ 689 wmb(); 690 writel((desc_refilled), droq->pkts_credit_reg); 691 /* make sure mmio write completes */ 692 mmiowb(); 693 } 694 695 } /* for ( each packet )... */ 696 697 /* Increment refill_count by the number of buffers processed. */ 698 droq->stats.pkts_received += pkt; 699 droq->stats.bytes_received += total_len; 700 701 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { 702 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); 703 704 droq->stats.dropped_toomany += (pkts_to_process - pkt); 705 return pkts_to_process; 706 } 707 708 return pkt; 709 } 710 711 int 712 octeon_droq_process_packets(struct octeon_device *oct, 713 struct octeon_droq *droq, 714 u32 budget) 715 { 716 u32 pkt_count = 0, pkts_processed = 0; 717 struct list_head *tmp, *tmp2; 718 719 pkt_count = atomic_read(&droq->pkts_pending); 720 if (!pkt_count) 721 return 0; 722 723 if (pkt_count > budget) 724 pkt_count = budget; 725 726 /* Grab the lock */ 727 spin_lock(&droq->lock); 728 729 pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count); 730 731 atomic_sub(pkts_processed, &droq->pkts_pending); 732 733 /* Release the spin lock */ 734 spin_unlock(&droq->lock); 735 736 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 737 struct __dispatch *rdisp = (struct __dispatch *)tmp; 738 739 list_del(tmp); 740 rdisp->disp_fn(rdisp->rinfo, 741 octeon_get_dispatch_arg 742 (oct, 743 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 744 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 745 } 746 747 /* If there are packets pending. schedule tasklet again */ 748 if (atomic_read(&droq->pkts_pending)) 749 return 1; 750 751 return 0; 752 } 753 754 /** 755 * Utility function to poll for packets. check_hw_for_packets must be 756 * called before calling this routine. 757 */ 758 759 static int 760 octeon_droq_process_poll_pkts(struct octeon_device *oct, 761 struct octeon_droq *droq, u32 budget) 762 { 763 struct list_head *tmp, *tmp2; 764 u32 pkts_available = 0, pkts_processed = 0; 765 u32 total_pkts_processed = 0; 766 767 if (budget > droq->max_count) 768 budget = droq->max_count; 769 770 spin_lock(&droq->lock); 771 772 while (total_pkts_processed < budget) { 773 pkts_available = 774 CVM_MIN((budget - total_pkts_processed), 775 (u32)(atomic_read(&droq->pkts_pending))); 776 777 if (pkts_available == 0) 778 break; 779 780 pkts_processed = 781 octeon_droq_fast_process_packets(oct, droq, 782 pkts_available); 783 784 atomic_sub(pkts_processed, &droq->pkts_pending); 785 786 total_pkts_processed += pkts_processed; 787 788 octeon_droq_check_hw_for_pkts(oct, droq); 789 } 790 791 spin_unlock(&droq->lock); 792 793 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { 794 struct __dispatch *rdisp = (struct __dispatch *)tmp; 795 796 list_del(tmp); 797 rdisp->disp_fn(rdisp->rinfo, 798 octeon_get_dispatch_arg 799 (oct, 800 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, 801 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); 802 } 803 804 return total_pkts_processed; 805 } 806 807 int 808 octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd, 809 u32 arg) 810 { 811 struct octeon_droq *droq; 812 struct octeon_config *oct_cfg = NULL; 813 814 oct_cfg = octeon_get_conf(oct); 815 816 if (!oct_cfg) 817 return -EINVAL; 818 819 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 820 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 821 __func__, q_no, (oct->num_oqs - 1)); 822 return -EINVAL; 823 } 824 825 droq = oct->droq[q_no]; 826 827 if (cmd == POLL_EVENT_PROCESS_PKTS) 828 return octeon_droq_process_poll_pkts(oct, droq, arg); 829 830 if (cmd == POLL_EVENT_PENDING_PKTS) { 831 u32 pkt_cnt = atomic_read(&droq->pkts_pending); 832 833 return octeon_droq_process_packets(oct, droq, pkt_cnt); 834 } 835 836 if (cmd == POLL_EVENT_ENABLE_INTR) { 837 u32 value; 838 unsigned long flags; 839 840 /* Enable Pkt Interrupt */ 841 switch (oct->chip_id) { 842 case OCTEON_CN66XX: 843 case OCTEON_CN68XX: { 844 struct octeon_cn6xxx *cn6xxx = 845 (struct octeon_cn6xxx *)oct->chip; 846 spin_lock_irqsave 847 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 848 value = 849 octeon_read_csr(oct, 850 CN6XXX_SLI_PKT_TIME_INT_ENB); 851 value |= (1 << q_no); 852 octeon_write_csr(oct, 853 CN6XXX_SLI_PKT_TIME_INT_ENB, 854 value); 855 value = 856 octeon_read_csr(oct, 857 CN6XXX_SLI_PKT_CNT_INT_ENB); 858 value |= (1 << q_no); 859 octeon_write_csr(oct, 860 CN6XXX_SLI_PKT_CNT_INT_ENB, 861 value); 862 863 /* don't bother flushing the enables */ 864 865 spin_unlock_irqrestore 866 (&cn6xxx->lock_for_droq_int_enb_reg, flags); 867 return 0; 868 } 869 break; 870 } 871 872 return 0; 873 } 874 875 dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd); 876 return -EINVAL; 877 } 878 879 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no, 880 struct octeon_droq_ops *ops) 881 { 882 struct octeon_droq *droq; 883 unsigned long flags; 884 struct octeon_config *oct_cfg = NULL; 885 886 oct_cfg = octeon_get_conf(oct); 887 888 if (!oct_cfg) 889 return -EINVAL; 890 891 if (!(ops)) { 892 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n", 893 __func__); 894 return -EINVAL; 895 } 896 897 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 898 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 899 __func__, q_no, (oct->num_oqs - 1)); 900 return -EINVAL; 901 } 902 903 droq = oct->droq[q_no]; 904 905 spin_lock_irqsave(&droq->lock, flags); 906 907 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); 908 909 spin_unlock_irqrestore(&droq->lock, flags); 910 911 return 0; 912 } 913 914 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no) 915 { 916 unsigned long flags; 917 struct octeon_droq *droq; 918 struct octeon_config *oct_cfg = NULL; 919 920 oct_cfg = octeon_get_conf(oct); 921 922 if (!oct_cfg) 923 return -EINVAL; 924 925 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { 926 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n", 927 __func__, q_no, oct->num_oqs - 1); 928 return -EINVAL; 929 } 930 931 droq = oct->droq[q_no]; 932 933 if (!droq) { 934 dev_info(&oct->pci_dev->dev, 935 "Droq id (%d) not available.\n", q_no); 936 return 0; 937 } 938 939 spin_lock_irqsave(&droq->lock, flags); 940 941 droq->ops.fptr = NULL; 942 droq->ops.drop_on_max = 0; 943 944 spin_unlock_irqrestore(&droq->lock, flags); 945 946 return 0; 947 } 948 949 int octeon_create_droq(struct octeon_device *oct, 950 u32 q_no, u32 num_descs, 951 u32 desc_size, void *app_ctx) 952 { 953 struct octeon_droq *droq; 954 955 if (oct->droq[q_no]) { 956 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n", 957 q_no); 958 return 1; 959 } 960 961 /* Allocate the DS for the new droq. */ 962 droq = vmalloc(sizeof(*droq)); 963 if (!droq) 964 goto create_droq_fail; 965 memset(droq, 0, sizeof(struct octeon_droq)); 966 967 /*Disable the pkt o/p for this Q */ 968 octeon_set_droq_pkt_op(oct, q_no, 0); 969 oct->droq[q_no] = droq; 970 971 /* Initialize the Droq */ 972 octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx); 973 974 oct->num_oqs++; 975 976 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__, 977 oct->num_oqs); 978 979 /* Global Droq register settings */ 980 981 /* As of now not required, as setting are done for all 32 Droqs at 982 * the same time. 983 */ 984 return 0; 985 986 create_droq_fail: 987 octeon_delete_droq(oct, q_no); 988 return -1; 989 } 990