1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2015 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more 17 * details. 18 * 19 * This file may also be available under a different license from Cavium. 20 * Contact Cavium, Inc. for more information 21 **********************************************************************/ 22 #include <linux/version.h> 23 #include <linux/types.h> 24 #include <linux/list.h> 25 #include <linux/interrupt.h> 26 #include <linux/pci.h> 27 #include <linux/kthread.h> 28 #include <linux/netdevice.h> 29 #include <linux/vmalloc.h> 30 #include "octeon_config.h" 31 #include "liquidio_common.h" 32 #include "octeon_droq.h" 33 #include "octeon_iq.h" 34 #include "response_manager.h" 35 #include "octeon_device.h" 36 #include "octeon_nic.h" 37 #include "octeon_main.h" 38 #include "octeon_network.h" 39 #include "cn66xx_regs.h" 40 #include "cn66xx_device.h" 41 #include "cn68xx_regs.h" 42 #include "cn68xx_device.h" 43 #include "liquidio_image.h" 44 45 #define INCR_INSTRQUEUE_PKT_COUNT(octeon_dev_ptr, iq_no, field, count) \ 46 (octeon_dev_ptr->instr_queue[iq_no]->stats.field += count) 47 48 struct iq_post_status { 49 int status; 50 int index; 51 }; 52 53 static void check_db_timeout(struct work_struct *work); 54 static void __check_db_timeout(struct octeon_device *oct, unsigned long iq_no); 55 56 static void (*reqtype_free_fn[MAX_OCTEON_DEVICES][REQTYPE_LAST + 1]) (void *); 57 58 static inline int IQ_INSTR_MODE_64B(struct octeon_device *oct, int iq_no) 59 { 60 struct octeon_instr_queue *iq = 61 (struct octeon_instr_queue *)oct->instr_queue[iq_no]; 62 return iq->iqcmd_64B; 63 } 64 65 #define IQ_INSTR_MODE_32B(oct, iq_no) (!IQ_INSTR_MODE_64B(oct, iq_no)) 66 67 /* Define this to return the request status comaptible to old code */ 68 /*#define OCTEON_USE_OLD_REQ_STATUS*/ 69 70 /* Return 0 on success, 1 on failure */ 71 int octeon_init_instr_queue(struct octeon_device *oct, 72 u32 iq_no, u32 num_descs) 73 { 74 struct octeon_instr_queue *iq; 75 struct octeon_iq_config *conf = NULL; 76 u32 q_size; 77 struct cavium_wq *db_wq; 78 79 if (OCTEON_CN6XXX(oct)) 80 conf = &(CFG_GET_IQ_CFG(CHIP_FIELD(oct, cn6xxx, conf))); 81 82 if (!conf) { 83 dev_err(&oct->pci_dev->dev, "Unsupported Chip %x\n", 84 oct->chip_id); 85 return 1; 86 } 87 88 if (num_descs & (num_descs - 1)) { 89 dev_err(&oct->pci_dev->dev, 90 "Number of descriptors for instr queue %d not in power of 2.\n", 91 iq_no); 92 return 1; 93 } 94 95 q_size = (u32)conf->instr_type * num_descs; 96 97 iq = oct->instr_queue[iq_no]; 98 99 iq->base_addr = lio_dma_alloc(oct, q_size, 100 (dma_addr_t *)&iq->base_addr_dma); 101 if (!iq->base_addr) { 102 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for instr queue %d\n", 103 iq_no); 104 return 1; 105 } 106 107 iq->max_count = num_descs; 108 109 /* Initialize a list to holds requests that have been posted to Octeon 110 * but has yet to be fetched by octeon 111 */ 112 iq->request_list = vmalloc(sizeof(*iq->request_list) * num_descs); 113 if (!iq->request_list) { 114 lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma); 115 dev_err(&oct->pci_dev->dev, "Alloc failed for IQ[%d] nr free list\n", 116 iq_no); 117 return 1; 118 } 119 120 memset(iq->request_list, 0, sizeof(*iq->request_list) * num_descs); 121 122 dev_dbg(&oct->pci_dev->dev, "IQ[%d]: base: %p basedma: %llx count: %d\n", 123 iq_no, iq->base_addr, iq->base_addr_dma, iq->max_count); 124 125 iq->iq_no = iq_no; 126 iq->fill_threshold = (u32)conf->db_min; 127 iq->fill_cnt = 0; 128 iq->host_write_index = 0; 129 iq->octeon_read_index = 0; 130 iq->flush_index = 0; 131 iq->last_db_time = 0; 132 iq->do_auto_flush = 1; 133 iq->db_timeout = (u32)conf->db_timeout; 134 atomic_set(&iq->instr_pending, 0); 135 136 /* Initialize the spinlock for this instruction queue */ 137 spin_lock_init(&iq->lock); 138 139 oct->io_qmask.iq |= (1 << iq_no); 140 141 /* Set the 32B/64B mode for each input queue */ 142 oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no); 143 iq->iqcmd_64B = (conf->instr_type == 64); 144 145 oct->fn_list.setup_iq_regs(oct, iq_no); 146 147 oct->check_db_wq[iq_no].wq = create_workqueue("check_iq_db"); 148 if (!oct->check_db_wq[iq_no].wq) { 149 lio_dma_free(oct, q_size, iq->base_addr, iq->base_addr_dma); 150 dev_err(&oct->pci_dev->dev, "check db wq create failed for iq %d\n", 151 iq_no); 152 return 1; 153 } 154 155 db_wq = &oct->check_db_wq[iq_no]; 156 157 INIT_DELAYED_WORK(&db_wq->wk.work, check_db_timeout); 158 db_wq->wk.ctxptr = oct; 159 db_wq->wk.ctxul = iq_no; 160 queue_delayed_work(db_wq->wq, &db_wq->wk.work, msecs_to_jiffies(1)); 161 162 return 0; 163 } 164 165 int octeon_delete_instr_queue(struct octeon_device *oct, u32 iq_no) 166 { 167 u64 desc_size = 0, q_size; 168 struct octeon_instr_queue *iq = oct->instr_queue[iq_no]; 169 170 cancel_delayed_work_sync(&oct->check_db_wq[iq_no].wk.work); 171 flush_workqueue(oct->check_db_wq[iq_no].wq); 172 destroy_workqueue(oct->check_db_wq[iq_no].wq); 173 174 if (OCTEON_CN6XXX(oct)) 175 desc_size = 176 CFG_GET_IQ_INSTR_TYPE(CHIP_FIELD(oct, cn6xxx, conf)); 177 178 vfree(iq->request_list); 179 180 if (iq->base_addr) { 181 q_size = iq->max_count * desc_size; 182 lio_dma_free(oct, (u32)q_size, iq->base_addr, 183 iq->base_addr_dma); 184 return 0; 185 } 186 return 1; 187 } 188 189 /* Return 0 on success, 1 on failure */ 190 int octeon_setup_iq(struct octeon_device *oct, 191 u32 iq_no, 192 u32 num_descs, 193 void *app_ctx) 194 { 195 if (oct->instr_queue[iq_no]) { 196 dev_dbg(&oct->pci_dev->dev, "IQ is in use. Cannot create the IQ: %d again\n", 197 iq_no); 198 oct->instr_queue[iq_no]->app_ctx = app_ctx; 199 return 0; 200 } 201 oct->instr_queue[iq_no] = 202 vmalloc(sizeof(struct octeon_instr_queue)); 203 if (!oct->instr_queue[iq_no]) 204 return 1; 205 206 memset(oct->instr_queue[iq_no], 0, 207 sizeof(struct octeon_instr_queue)); 208 209 oct->instr_queue[iq_no]->app_ctx = app_ctx; 210 if (octeon_init_instr_queue(oct, iq_no, num_descs)) { 211 vfree(oct->instr_queue[iq_no]); 212 oct->instr_queue[iq_no] = NULL; 213 return 1; 214 } 215 216 oct->num_iqs++; 217 oct->fn_list.enable_io_queues(oct); 218 return 0; 219 } 220 221 int lio_wait_for_instr_fetch(struct octeon_device *oct) 222 { 223 int i, retry = 1000, pending, instr_cnt = 0; 224 225 do { 226 instr_cnt = 0; 227 228 /*for (i = 0; i < oct->num_iqs; i++) {*/ 229 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES; i++) { 230 if (!(oct->io_qmask.iq & (1UL << i))) 231 continue; 232 pending = 233 atomic_read(&oct-> 234 instr_queue[i]->instr_pending); 235 if (pending) 236 __check_db_timeout(oct, i); 237 instr_cnt += pending; 238 } 239 240 if (instr_cnt == 0) 241 break; 242 243 schedule_timeout_uninterruptible(1); 244 245 } while (retry-- && instr_cnt); 246 247 return instr_cnt; 248 } 249 250 static inline void 251 ring_doorbell(struct octeon_device *oct, struct octeon_instr_queue *iq) 252 { 253 if (atomic_read(&oct->status) == OCT_DEV_RUNNING) { 254 writel(iq->fill_cnt, iq->doorbell_reg); 255 /* make sure doorbell write goes through */ 256 mmiowb(); 257 iq->fill_cnt = 0; 258 iq->last_db_time = jiffies; 259 return; 260 } 261 } 262 263 static inline void __copy_cmd_into_iq(struct octeon_instr_queue *iq, 264 u8 *cmd) 265 { 266 u8 *iqptr, cmdsize; 267 268 cmdsize = ((iq->iqcmd_64B) ? 64 : 32); 269 iqptr = iq->base_addr + (cmdsize * iq->host_write_index); 270 271 memcpy(iqptr, cmd, cmdsize); 272 } 273 274 static inline int 275 __post_command(struct octeon_device *octeon_dev __attribute__((unused)), 276 struct octeon_instr_queue *iq, 277 u32 force_db __attribute__((unused)), u8 *cmd) 278 { 279 u32 index = -1; 280 281 /* This ensures that the read index does not wrap around to the same 282 * position if queue gets full before Octeon could fetch any instr. 283 */ 284 if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 1)) 285 return -1; 286 287 __copy_cmd_into_iq(iq, cmd); 288 289 /* "index" is returned, host_write_index is modified. */ 290 index = iq->host_write_index; 291 INCR_INDEX_BY1(iq->host_write_index, iq->max_count); 292 iq->fill_cnt++; 293 294 /* Flush the command into memory. We need to be sure the data is in 295 * memory before indicating that the instruction is pending. 296 */ 297 wmb(); 298 299 atomic_inc(&iq->instr_pending); 300 301 return index; 302 } 303 304 static inline struct iq_post_status 305 __post_command2(struct octeon_device *octeon_dev __attribute__((unused)), 306 struct octeon_instr_queue *iq, 307 u32 force_db __attribute__((unused)), u8 *cmd) 308 { 309 struct iq_post_status st; 310 311 st.status = IQ_SEND_OK; 312 313 /* This ensures that the read index does not wrap around to the same 314 * position if queue gets full before Octeon could fetch any instr. 315 */ 316 if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 1)) { 317 st.status = IQ_SEND_FAILED; 318 st.index = -1; 319 return st; 320 } 321 322 if (atomic_read(&iq->instr_pending) >= (s32)(iq->max_count - 2)) 323 st.status = IQ_SEND_STOP; 324 325 __copy_cmd_into_iq(iq, cmd); 326 327 /* "index" is returned, host_write_index is modified. */ 328 st.index = iq->host_write_index; 329 INCR_INDEX_BY1(iq->host_write_index, iq->max_count); 330 iq->fill_cnt++; 331 332 /* Flush the command into memory. We need to be sure the data is in 333 * memory before indicating that the instruction is pending. 334 */ 335 wmb(); 336 337 atomic_inc(&iq->instr_pending); 338 339 return st; 340 } 341 342 int 343 octeon_register_reqtype_free_fn(struct octeon_device *oct, int reqtype, 344 void (*fn)(void *)) 345 { 346 if (reqtype > REQTYPE_LAST) { 347 dev_err(&oct->pci_dev->dev, "%s: Invalid reqtype: %d\n", 348 __func__, reqtype); 349 return -EINVAL; 350 } 351 352 reqtype_free_fn[oct->octeon_id][reqtype] = fn; 353 354 return 0; 355 } 356 357 static inline void 358 __add_to_request_list(struct octeon_instr_queue *iq, 359 int idx, void *buf, int reqtype) 360 { 361 iq->request_list[idx].buf = buf; 362 iq->request_list[idx].reqtype = reqtype; 363 } 364 365 int 366 lio_process_iq_request_list(struct octeon_device *oct, 367 struct octeon_instr_queue *iq) 368 { 369 int reqtype; 370 void *buf; 371 u32 old = iq->flush_index; 372 u32 inst_count = 0; 373 unsigned pkts_compl = 0, bytes_compl = 0; 374 struct octeon_soft_command *sc; 375 struct octeon_instr_irh *irh; 376 377 while (old != iq->octeon_read_index) { 378 reqtype = iq->request_list[old].reqtype; 379 buf = iq->request_list[old].buf; 380 381 if (reqtype == REQTYPE_NONE) 382 goto skip_this; 383 384 octeon_update_tx_completion_counters(buf, reqtype, &pkts_compl, 385 &bytes_compl); 386 387 switch (reqtype) { 388 case REQTYPE_NORESP_NET: 389 case REQTYPE_NORESP_NET_SG: 390 case REQTYPE_RESP_NET_SG: 391 reqtype_free_fn[oct->octeon_id][reqtype](buf); 392 break; 393 case REQTYPE_RESP_NET: 394 case REQTYPE_SOFT_COMMAND: 395 sc = buf; 396 397 irh = (struct octeon_instr_irh *)&sc->cmd.irh; 398 if (irh->rflag) { 399 /* We're expecting a response from Octeon. 400 * It's up to lio_process_ordered_list() to 401 * process sc. Add sc to the ordered soft 402 * command response list because we expect 403 * a response from Octeon. 404 */ 405 spin_lock_bh(&oct->response_list 406 [OCTEON_ORDERED_SC_LIST].lock); 407 atomic_inc(&oct->response_list 408 [OCTEON_ORDERED_SC_LIST]. 409 pending_req_count); 410 list_add_tail(&sc->node, &oct->response_list 411 [OCTEON_ORDERED_SC_LIST].head); 412 spin_unlock_bh(&oct->response_list 413 [OCTEON_ORDERED_SC_LIST].lock); 414 } else { 415 if (sc->callback) { 416 sc->callback(oct, OCTEON_REQUEST_DONE, 417 sc->callback_arg); 418 } 419 } 420 break; 421 default: 422 dev_err(&oct->pci_dev->dev, 423 "%s Unknown reqtype: %d buf: %p at idx %d\n", 424 __func__, reqtype, buf, old); 425 } 426 427 iq->request_list[old].buf = NULL; 428 iq->request_list[old].reqtype = 0; 429 430 skip_this: 431 inst_count++; 432 INCR_INDEX_BY1(old, iq->max_count); 433 } 434 if (bytes_compl) 435 octeon_report_tx_completion_to_bql(iq->app_ctx, pkts_compl, 436 bytes_compl); 437 iq->flush_index = old; 438 439 return inst_count; 440 } 441 442 static inline void 443 update_iq_indices(struct octeon_device *oct, struct octeon_instr_queue *iq) 444 { 445 u32 inst_processed = 0; 446 447 /* Calculate how many commands Octeon has read and move the read index 448 * accordingly. 449 */ 450 iq->octeon_read_index = oct->fn_list.update_iq_read_idx(oct, iq); 451 452 /* Move the NORESPONSE requests to the per-device completion list. */ 453 if (iq->flush_index != iq->octeon_read_index) 454 inst_processed = lio_process_iq_request_list(oct, iq); 455 456 if (inst_processed) { 457 atomic_sub(inst_processed, &iq->instr_pending); 458 iq->stats.instr_processed += inst_processed; 459 } 460 } 461 462 static void 463 octeon_flush_iq(struct octeon_device *oct, struct octeon_instr_queue *iq, 464 u32 pending_thresh) 465 { 466 if (atomic_read(&iq->instr_pending) >= (s32)pending_thresh) { 467 spin_lock_bh(&iq->lock); 468 update_iq_indices(oct, iq); 469 spin_unlock_bh(&iq->lock); 470 } 471 } 472 473 static void __check_db_timeout(struct octeon_device *oct, unsigned long iq_no) 474 { 475 struct octeon_instr_queue *iq; 476 u64 next_time; 477 478 if (!oct) 479 return; 480 iq = oct->instr_queue[iq_no]; 481 if (!iq) 482 return; 483 484 /* If jiffies - last_db_time < db_timeout do nothing */ 485 next_time = iq->last_db_time + iq->db_timeout; 486 if (!time_after(jiffies, (unsigned long)next_time)) 487 return; 488 iq->last_db_time = jiffies; 489 490 /* Get the lock and prevent tasklets. This routine gets called from 491 * the poll thread. Instructions can now be posted in tasklet context 492 */ 493 spin_lock_bh(&iq->lock); 494 if (iq->fill_cnt != 0) 495 ring_doorbell(oct, iq); 496 497 spin_unlock_bh(&iq->lock); 498 499 /* Flush the instruction queue */ 500 if (iq->do_auto_flush) 501 octeon_flush_iq(oct, iq, 1); 502 } 503 504 /* Called by the Poll thread at regular intervals to check the instruction 505 * queue for commands to be posted and for commands that were fetched by Octeon. 506 */ 507 static void check_db_timeout(struct work_struct *work) 508 { 509 struct cavium_wk *wk = (struct cavium_wk *)work; 510 struct octeon_device *oct = (struct octeon_device *)wk->ctxptr; 511 unsigned long iq_no = wk->ctxul; 512 struct cavium_wq *db_wq = &oct->check_db_wq[iq_no]; 513 514 __check_db_timeout(oct, iq_no); 515 queue_delayed_work(db_wq->wq, &db_wq->wk.work, msecs_to_jiffies(1)); 516 } 517 518 int 519 octeon_send_command(struct octeon_device *oct, u32 iq_no, 520 u32 force_db, void *cmd, void *buf, 521 u32 datasize, u32 reqtype) 522 { 523 struct iq_post_status st; 524 struct octeon_instr_queue *iq = oct->instr_queue[iq_no]; 525 526 spin_lock_bh(&iq->lock); 527 528 st = __post_command2(oct, iq, force_db, cmd); 529 530 if (st.status != IQ_SEND_FAILED) { 531 octeon_report_sent_bytes_to_bql(buf, reqtype); 532 __add_to_request_list(iq, st.index, buf, reqtype); 533 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize); 534 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1); 535 536 if (iq->fill_cnt >= iq->fill_threshold || force_db) 537 ring_doorbell(oct, iq); 538 } else { 539 INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1); 540 } 541 542 spin_unlock_bh(&iq->lock); 543 544 if (iq->do_auto_flush) 545 octeon_flush_iq(oct, iq, 2); 546 547 return st.status; 548 } 549 550 void 551 octeon_prepare_soft_command(struct octeon_device *oct, 552 struct octeon_soft_command *sc, 553 u8 opcode, 554 u8 subcode, 555 u32 irh_ossp, 556 u64 ossp0, 557 u64 ossp1) 558 { 559 struct octeon_config *oct_cfg; 560 struct octeon_instr_ih *ih; 561 struct octeon_instr_irh *irh; 562 struct octeon_instr_rdp *rdp; 563 564 BUG_ON(opcode > 15); 565 BUG_ON(subcode > 127); 566 567 oct_cfg = octeon_get_conf(oct); 568 569 ih = (struct octeon_instr_ih *)&sc->cmd.ih; 570 ih->tagtype = ATOMIC_TAG; 571 ih->tag = LIO_CONTROL; 572 ih->raw = 1; 573 ih->grp = CFG_GET_CTRL_Q_GRP(oct_cfg); 574 575 if (sc->datasize) { 576 ih->dlengsz = sc->datasize; 577 ih->rs = 1; 578 } 579 580 irh = (struct octeon_instr_irh *)&sc->cmd.irh; 581 irh->opcode = opcode; 582 irh->subcode = subcode; 583 584 /* opcode/subcode specific parameters (ossp) */ 585 irh->ossp = irh_ossp; 586 sc->cmd.ossp[0] = ossp0; 587 sc->cmd.ossp[1] = ossp1; 588 589 if (sc->rdatasize) { 590 rdp = (struct octeon_instr_rdp *)&sc->cmd.rdp; 591 rdp->pcie_port = oct->pcie_port; 592 rdp->rlen = sc->rdatasize; 593 594 irh->rflag = 1; 595 irh->len = 4; 596 ih->fsz = 40; /* irh+ossp[0]+ossp[1]+rdp+rptr = 40 bytes */ 597 } else { 598 irh->rflag = 0; 599 irh->len = 2; 600 ih->fsz = 24; /* irh + ossp[0] + ossp[1] = 24 bytes */ 601 } 602 603 while (!(oct->io_qmask.iq & (1 << sc->iq_no))) 604 sc->iq_no++; 605 } 606 607 int octeon_send_soft_command(struct octeon_device *oct, 608 struct octeon_soft_command *sc) 609 { 610 struct octeon_instr_ih *ih; 611 struct octeon_instr_irh *irh; 612 struct octeon_instr_rdp *rdp; 613 614 ih = (struct octeon_instr_ih *)&sc->cmd.ih; 615 if (ih->dlengsz) { 616 BUG_ON(!sc->dmadptr); 617 sc->cmd.dptr = sc->dmadptr; 618 } 619 620 irh = (struct octeon_instr_irh *)&sc->cmd.irh; 621 if (irh->rflag) { 622 BUG_ON(!sc->dmarptr); 623 BUG_ON(!sc->status_word); 624 *sc->status_word = COMPLETION_WORD_INIT; 625 626 rdp = (struct octeon_instr_rdp *)&sc->cmd.rdp; 627 628 sc->cmd.rptr = sc->dmarptr; 629 } 630 631 if (sc->wait_time) 632 sc->timeout = jiffies + sc->wait_time; 633 634 return octeon_send_command(oct, sc->iq_no, 1, &sc->cmd, sc, 635 (u32)ih->dlengsz, REQTYPE_SOFT_COMMAND); 636 } 637 638 int octeon_setup_sc_buffer_pool(struct octeon_device *oct) 639 { 640 int i; 641 u64 dma_addr; 642 struct octeon_soft_command *sc; 643 644 INIT_LIST_HEAD(&oct->sc_buf_pool.head); 645 spin_lock_init(&oct->sc_buf_pool.lock); 646 atomic_set(&oct->sc_buf_pool.alloc_buf_count, 0); 647 648 for (i = 0; i < MAX_SOFT_COMMAND_BUFFERS; i++) { 649 sc = (struct octeon_soft_command *) 650 lio_dma_alloc(oct, 651 SOFT_COMMAND_BUFFER_SIZE, 652 (dma_addr_t *)&dma_addr); 653 if (!sc) 654 return 1; 655 656 sc->dma_addr = dma_addr; 657 sc->size = SOFT_COMMAND_BUFFER_SIZE; 658 659 list_add_tail(&sc->node, &oct->sc_buf_pool.head); 660 } 661 662 return 0; 663 } 664 665 int octeon_free_sc_buffer_pool(struct octeon_device *oct) 666 { 667 struct list_head *tmp, *tmp2; 668 struct octeon_soft_command *sc; 669 670 spin_lock(&oct->sc_buf_pool.lock); 671 672 list_for_each_safe(tmp, tmp2, &oct->sc_buf_pool.head) { 673 list_del(tmp); 674 675 sc = (struct octeon_soft_command *)tmp; 676 677 lio_dma_free(oct, sc->size, sc, sc->dma_addr); 678 } 679 680 INIT_LIST_HEAD(&oct->sc_buf_pool.head); 681 682 spin_unlock(&oct->sc_buf_pool.lock); 683 684 return 0; 685 } 686 687 struct octeon_soft_command *octeon_alloc_soft_command(struct octeon_device *oct, 688 u32 datasize, 689 u32 rdatasize, 690 u32 ctxsize) 691 { 692 u64 dma_addr; 693 u32 size; 694 u32 offset = sizeof(struct octeon_soft_command); 695 struct octeon_soft_command *sc = NULL; 696 struct list_head *tmp; 697 698 BUG_ON((offset + datasize + rdatasize + ctxsize) > 699 SOFT_COMMAND_BUFFER_SIZE); 700 701 spin_lock(&oct->sc_buf_pool.lock); 702 703 if (list_empty(&oct->sc_buf_pool.head)) { 704 spin_unlock(&oct->sc_buf_pool.lock); 705 return NULL; 706 } 707 708 list_for_each(tmp, &oct->sc_buf_pool.head) 709 break; 710 711 list_del(tmp); 712 713 atomic_inc(&oct->sc_buf_pool.alloc_buf_count); 714 715 spin_unlock(&oct->sc_buf_pool.lock); 716 717 sc = (struct octeon_soft_command *)tmp; 718 719 dma_addr = sc->dma_addr; 720 size = sc->size; 721 722 memset(sc, 0, sc->size); 723 724 sc->dma_addr = dma_addr; 725 sc->size = size; 726 727 if (ctxsize) { 728 sc->ctxptr = (u8 *)sc + offset; 729 sc->ctxsize = ctxsize; 730 } 731 732 /* Start data at 128 byte boundary */ 733 offset = (offset + ctxsize + 127) & 0xffffff80; 734 735 if (datasize) { 736 sc->virtdptr = (u8 *)sc + offset; 737 sc->dmadptr = dma_addr + offset; 738 sc->datasize = datasize; 739 } 740 741 /* Start rdata at 128 byte boundary */ 742 offset = (offset + datasize + 127) & 0xffffff80; 743 744 if (rdatasize) { 745 BUG_ON(rdatasize < 16); 746 sc->virtrptr = (u8 *)sc + offset; 747 sc->dmarptr = dma_addr + offset; 748 sc->rdatasize = rdatasize; 749 sc->status_word = (u64 *)((u8 *)(sc->virtrptr) + rdatasize - 8); 750 } 751 752 return sc; 753 } 754 755 void octeon_free_soft_command(struct octeon_device *oct, 756 struct octeon_soft_command *sc) 757 { 758 spin_lock(&oct->sc_buf_pool.lock); 759 760 list_add_tail(&sc->node, &oct->sc_buf_pool.head); 761 762 atomic_dec(&oct->sc_buf_pool.alloc_buf_count); 763 764 spin_unlock(&oct->sc_buf_pool.lock); 765 } 766