1 /* 2 * BSD LICENSE 3 * 4 * Copyright(c) 2017 Cavium, Inc.. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Cavium, Inc. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "lio_bsd.h" 35 #include "lio_common.h" 36 #include "lio_droq.h" 37 #include "lio_iq.h" 38 #include "lio_response_manager.h" 39 #include "lio_device.h" 40 #include "cn23xx_pf_device.h" 41 #include "lio_main.h" 42 #include "lio_rss.h" 43 44 static int 45 lio_cn23xx_pf_soft_reset(struct octeon_device *oct) 46 { 47 48 lio_write_csr64(oct, LIO_CN23XX_SLI_WIN_WR_MASK_REG, 0xFF); 49 50 lio_dev_dbg(oct, "BIST enabled for CN23XX soft reset\n"); 51 52 lio_write_csr64(oct, LIO_CN23XX_SLI_SCRATCH1, 0x1234ULL); 53 54 /* Initiate chip-wide soft reset */ 55 lio_pci_readq(oct, LIO_CN23XX_RST_SOFT_RST); 56 lio_pci_writeq(oct, 1, LIO_CN23XX_RST_SOFT_RST); 57 58 /* Wait for 100ms as Octeon resets. */ 59 lio_mdelay(100); 60 61 if (lio_read_csr64(oct, LIO_CN23XX_SLI_SCRATCH1)) { 62 lio_dev_err(oct, "Soft reset failed\n"); 63 return (1); 64 } 65 66 lio_dev_dbg(oct, "Reset completed\n"); 67 68 /* restore the reset value */ 69 lio_write_csr64(oct, LIO_CN23XX_SLI_WIN_WR_MASK_REG, 0xFF); 70 71 return (0); 72 } 73 74 static void 75 lio_cn23xx_pf_enable_error_reporting(struct octeon_device *oct) 76 { 77 uint32_t corrtable_err_status, uncorrectable_err_mask, regval; 78 79 regval = lio_read_pci_cfg(oct, LIO_CN23XX_CFG_PCIE_DEVCTL); 80 if (regval & LIO_CN23XX_CFG_PCIE_DEVCTL_MASK) { 81 uncorrectable_err_mask = 0; 82 corrtable_err_status = 0; 83 uncorrectable_err_mask = 84 lio_read_pci_cfg(oct, 85 LIO_CN23XX_CFG_PCIE_UNCORRECT_ERR_MASK); 86 corrtable_err_status = 87 lio_read_pci_cfg(oct, 88 LIO_CN23XX_CFG_PCIE_CORRECT_ERR_STATUS); 89 lio_dev_err(oct, "PCI-E Fatal error detected;\n" 90 "\tdev_ctl_status_reg = 0x%08x\n" 91 "\tuncorrectable_error_mask_reg = 0x%08x\n" 92 "\tcorrectable_error_status_reg = 0x%08x\n", 93 regval, uncorrectable_err_mask, 94 corrtable_err_status); 95 } 96 97 regval |= 0xf; /* Enable Link error reporting */ 98 99 lio_dev_dbg(oct, "Enabling PCI-E error reporting..\n"); 100 lio_write_pci_cfg(oct, LIO_CN23XX_CFG_PCIE_DEVCTL, regval); 101 } 102 103 static uint32_t 104 lio_cn23xx_pf_coprocessor_clock(struct octeon_device *oct) 105 { 106 /* 107 * Bits 29:24 of RST_BOOT[PNR_MUL] holds the ref.clock MULTIPLIER 108 * for SLI. 109 */ 110 111 /* TBD: get the info in Hand-shake */ 112 return (((lio_pci_readq(oct, LIO_CN23XX_RST_BOOT) >> 24) & 0x3f) * 50); 113 } 114 115 uint32_t 116 lio_cn23xx_pf_get_oq_ticks(struct octeon_device *oct, uint32_t time_intr_in_us) 117 { 118 /* This gives the SLI clock per microsec */ 119 uint32_t oqticks_per_us = lio_cn23xx_pf_coprocessor_clock(oct); 120 121 oct->pfvf_hsword.coproc_tics_per_us = oqticks_per_us; 122 123 /* This gives the clock cycles per millisecond */ 124 oqticks_per_us *= 1000; 125 126 /* This gives the oq ticks (1024 core clock cycles) per millisecond */ 127 oqticks_per_us /= 1024; 128 129 /* 130 * time_intr is in microseconds. The next 2 steps gives the oq ticks 131 * corresponding to time_intr. 132 */ 133 oqticks_per_us *= time_intr_in_us; 134 oqticks_per_us /= 1000; 135 136 return (oqticks_per_us); 137 } 138 139 static void 140 lio_cn23xx_pf_setup_global_mac_regs(struct octeon_device *oct) 141 { 142 uint64_t reg_val; 143 uint16_t mac_no = oct->pcie_port; 144 uint16_t pf_num = oct->pf_num; 145 /* programming SRN and TRS for each MAC(0..3) */ 146 147 lio_dev_dbg(oct, "%s: Using pcie port %d\n", __func__, mac_no); 148 /* By default, mapping all 64 IOQs to a single MACs */ 149 150 reg_val = 151 lio_read_csr64(oct, LIO_CN23XX_SLI_PKT_MAC_RINFO64(mac_no, pf_num)); 152 153 /* setting SRN <6:0> */ 154 reg_val = pf_num * LIO_CN23XX_PF_MAX_RINGS; 155 156 /* setting TRS <23:16> */ 157 reg_val = reg_val | 158 (oct->sriov_info.trs << LIO_CN23XX_PKT_MAC_CTL_RINFO_TRS_BIT_POS); 159 160 /* write these settings to MAC register */ 161 lio_write_csr64(oct, LIO_CN23XX_SLI_PKT_MAC_RINFO64(mac_no, pf_num), 162 reg_val); 163 164 lio_dev_dbg(oct, "SLI_PKT_MAC(%d)_PF(%d)_RINFO : 0x%016llx\n", mac_no, 165 pf_num, 166 LIO_CAST64(lio_read_csr64(oct, 167 LIO_CN23XX_SLI_PKT_MAC_RINFO64(mac_no, 168 pf_num)))); 169 } 170 171 static int 172 lio_cn23xx_pf_reset_io_queues(struct octeon_device *oct) 173 { 174 uint64_t d64; 175 uint32_t ern, loop = BUSY_READING_REG_PF_LOOP_COUNT; 176 uint32_t q_no, srn; 177 int ret_val = 0; 178 179 srn = oct->sriov_info.pf_srn; 180 ern = srn + oct->sriov_info.num_pf_rings; 181 182 /* As per HRM reg description, s/w cant write 0 to ENB. */ 183 /* to make the queue off, need to set the RST bit. */ 184 185 /* Reset the Enable bit for all the 64 IQs. */ 186 for (q_no = srn; q_no < ern; q_no++) { 187 /* set RST bit to 1. This bit applies to both IQ and OQ */ 188 d64 = lio_read_csr64(oct, 189 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); 190 d64 = d64 | LIO_CN23XX_PKT_INPUT_CTL_RST; 191 lio_write_csr64(oct, 192 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no), d64); 193 } 194 195 /* wait until the RST bit is clear or the RST and quiet bits are set */ 196 for (q_no = srn; q_no < ern; q_no++) { 197 volatile uint64_t reg_val = 198 lio_read_csr64(oct, 199 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); 200 while ((reg_val & LIO_CN23XX_PKT_INPUT_CTL_RST) && 201 !(reg_val & LIO_CN23XX_PKT_INPUT_CTL_QUIET) && 202 loop) { 203 reg_val = lio_read_csr64(oct, 204 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); 205 loop--; 206 } 207 208 if (!loop) { 209 lio_dev_err(oct, 210 "clearing the reset reg failed or setting the quiet reg failed for qno: %u\n", 211 q_no); 212 return (-1); 213 } 214 215 reg_val &= ~LIO_CN23XX_PKT_INPUT_CTL_RST; 216 lio_write_csr64(oct, LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no), 217 reg_val); 218 219 reg_val = lio_read_csr64(oct, 220 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); 221 if (reg_val & LIO_CN23XX_PKT_INPUT_CTL_RST) { 222 lio_dev_err(oct, "clearing the reset failed for qno: %u\n", 223 q_no); 224 ret_val = -1; 225 } 226 } 227 228 return (ret_val); 229 } 230 231 static int 232 lio_cn23xx_pf_setup_global_input_regs(struct octeon_device *oct) 233 { 234 struct lio_cn23xx_pf *cn23xx = (struct lio_cn23xx_pf *)oct->chip; 235 struct lio_instr_queue *iq; 236 uint64_t intr_threshold; 237 uint64_t pf_num, reg_val; 238 uint32_t q_no, ern, srn; 239 240 pf_num = oct->pf_num; 241 242 srn = oct->sriov_info.pf_srn; 243 ern = srn + oct->sriov_info.num_pf_rings; 244 245 if (lio_cn23xx_pf_reset_io_queues(oct)) 246 return (-1); 247 248 /* 249 * Set the MAC_NUM and PVF_NUM in IQ_PKT_CONTROL reg 250 * for all queues.Only PF can set these bits. 251 * bits 29:30 indicate the MAC num. 252 * bits 32:47 indicate the PVF num. 253 */ 254 for (q_no = 0; q_no < ern; q_no++) { 255 reg_val = oct->pcie_port << 256 LIO_CN23XX_PKT_INPUT_CTL_MAC_NUM_POS; 257 258 reg_val |= pf_num << LIO_CN23XX_PKT_INPUT_CTL_PF_NUM_POS; 259 260 lio_write_csr64(oct, LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no), 261 reg_val); 262 } 263 264 /* 265 * Select ES, RO, NS, RDSIZE,DPTR Fomat#0 for 266 * pf queues 267 */ 268 for (q_no = srn; q_no < ern; q_no++) { 269 uint32_t inst_cnt_reg; 270 271 iq = oct->instr_queue[q_no]; 272 if (iq != NULL) 273 inst_cnt_reg = iq->inst_cnt_reg; 274 else 275 inst_cnt_reg = LIO_CN23XX_SLI_IQ_INSTR_COUNT64(q_no); 276 277 reg_val = 278 lio_read_csr64(oct, LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); 279 280 reg_val |= LIO_CN23XX_PKT_INPUT_CTL_MASK; 281 282 lio_write_csr64(oct, LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no), 283 reg_val); 284 285 /* Set WMARK level for triggering PI_INT */ 286 /* intr_threshold = LIO_CN23XX_DEF_IQ_INTR_THRESHOLD & */ 287 intr_threshold = LIO_GET_IQ_INTR_PKT_CFG(cn23xx->conf) & 288 LIO_CN23XX_PKT_IN_DONE_WMARK_MASK; 289 290 lio_write_csr64(oct, inst_cnt_reg, 291 (lio_read_csr64(oct, inst_cnt_reg) & 292 ~(LIO_CN23XX_PKT_IN_DONE_WMARK_MASK << 293 LIO_CN23XX_PKT_IN_DONE_WMARK_BIT_POS)) | 294 (intr_threshold << 295 LIO_CN23XX_PKT_IN_DONE_WMARK_BIT_POS)); 296 } 297 return (0); 298 } 299 300 static void 301 lio_cn23xx_pf_setup_global_output_regs(struct octeon_device *oct) 302 { 303 struct lio_cn23xx_pf *cn23xx = (struct lio_cn23xx_pf *)oct->chip; 304 uint64_t time_threshold; 305 uint32_t ern, q_no, reg_val, srn; 306 307 srn = oct->sriov_info.pf_srn; 308 ern = srn + oct->sriov_info.num_pf_rings; 309 310 if (LIO_GET_IS_SLI_BP_ON_CFG(cn23xx->conf)) { 311 lio_write_csr64(oct, LIO_CN23XX_SLI_OQ_WMARK, 32); 312 } else { 313 /* Set Output queue watermark to 0 to disable backpressure */ 314 lio_write_csr64(oct, LIO_CN23XX_SLI_OQ_WMARK, 0); 315 } 316 317 for (q_no = srn; q_no < ern; q_no++) { 318 reg_val = lio_read_csr32(oct, 319 LIO_CN23XX_SLI_OQ_PKT_CONTROL(q_no)); 320 321 /* set IPTR & DPTR */ 322 reg_val |= LIO_CN23XX_PKT_OUTPUT_CTL_DPTR; 323 324 /* reset BMODE */ 325 reg_val &= ~(LIO_CN23XX_PKT_OUTPUT_CTL_BMODE); 326 327 /* 328 * No Relaxed Ordering, No Snoop, 64-bit Byte swap for 329 * Output Queue ScatterList reset ROR_P, NSR_P 330 */ 331 reg_val &= ~(LIO_CN23XX_PKT_OUTPUT_CTL_ROR_P); 332 reg_val &= ~(LIO_CN23XX_PKT_OUTPUT_CTL_NSR_P); 333 334 #if BYTE_ORDER == LITTLE_ENDIAN 335 reg_val &= ~(LIO_CN23XX_PKT_OUTPUT_CTL_ES_P); 336 #else /* BYTE_ORDER != LITTLE_ENDIAN */ 337 reg_val |= (LIO_CN23XX_PKT_OUTPUT_CTL_ES_P); 338 #endif /* BYTE_ORDER == LITTLE_ENDIAN */ 339 340 /* 341 * No Relaxed Ordering, No Snoop, 64-bit Byte swap for 342 * Output Queue Data reset ROR, NSR 343 */ 344 reg_val &= ~(LIO_CN23XX_PKT_OUTPUT_CTL_ROR); 345 reg_val &= ~(LIO_CN23XX_PKT_OUTPUT_CTL_NSR); 346 /* set the ES bit */ 347 reg_val |= (LIO_CN23XX_PKT_OUTPUT_CTL_ES); 348 349 /* write all the selected settings */ 350 lio_write_csr32(oct, LIO_CN23XX_SLI_OQ_PKT_CONTROL(q_no), 351 reg_val); 352 353 /* 354 * Enabling these interrupt in oct->fn_list.enable_interrupt() 355 * routine which called after IOQ init. 356 * Set up interrupt packet and time thresholds 357 * for all the OQs 358 */ 359 time_threshold =lio_cn23xx_pf_get_oq_ticks( 360 oct, (uint32_t)LIO_GET_OQ_INTR_TIME_CFG(cn23xx->conf)); 361 362 lio_write_csr64(oct, LIO_CN23XX_SLI_OQ_PKT_INT_LEVELS(q_no), 363 (LIO_GET_OQ_INTR_PKT_CFG(cn23xx->conf) | 364 (time_threshold << 32))); 365 } 366 367 /* Setting the water mark level for pko back pressure * */ 368 lio_write_csr64(oct, LIO_CN23XX_SLI_OQ_WMARK, 0x40); 369 370 /* Enable channel-level backpressure */ 371 if (oct->pf_num) 372 lio_write_csr64(oct, LIO_CN23XX_SLI_OUT_BP_EN2_W1S, 373 0xffffffffffffffffULL); 374 else 375 lio_write_csr64(oct, LIO_CN23XX_SLI_OUT_BP_EN_W1S, 376 0xffffffffffffffffULL); 377 } 378 379 static int 380 lio_cn23xx_pf_setup_device_regs(struct octeon_device *oct) 381 { 382 383 lio_cn23xx_pf_enable_error_reporting(oct); 384 385 /* program the MAC(0..3)_RINFO before setting up input/output regs */ 386 lio_cn23xx_pf_setup_global_mac_regs(oct); 387 388 if (lio_cn23xx_pf_setup_global_input_regs(oct)) 389 return (-1); 390 391 lio_cn23xx_pf_setup_global_output_regs(oct); 392 393 /* 394 * Default error timeout value should be 0x200000 to avoid host hang 395 * when reads invalid register 396 */ 397 lio_write_csr64(oct, LIO_CN23XX_SLI_WINDOW_CTL, 398 LIO_CN23XX_SLI_WINDOW_CTL_DEFAULT); 399 400 /* set SLI_PKT_IN_JABBER to handle large VXLAN packets */ 401 lio_write_csr64(oct, LIO_CN23XX_SLI_PKT_IN_JABBER, 402 LIO_CN23XX_MAX_INPUT_JABBER); 403 return (0); 404 } 405 406 static void 407 lio_cn23xx_pf_setup_iq_regs(struct octeon_device *oct, uint32_t iq_no) 408 { 409 struct lio_instr_queue *iq = oct->instr_queue[iq_no]; 410 uint64_t pkt_in_done; 411 412 iq_no += oct->sriov_info.pf_srn; 413 414 /* Write the start of the input queue's ring and its size */ 415 lio_write_csr64(oct, LIO_CN23XX_SLI_IQ_BASE_ADDR64(iq_no), 416 iq->base_addr_dma); 417 lio_write_csr32(oct, LIO_CN23XX_SLI_IQ_SIZE(iq_no), iq->max_count); 418 419 /* 420 * Remember the doorbell & instruction count register addr 421 * for this queue 422 */ 423 iq->doorbell_reg = LIO_CN23XX_SLI_IQ_DOORBELL(iq_no); 424 iq->inst_cnt_reg = LIO_CN23XX_SLI_IQ_INSTR_COUNT64(iq_no); 425 lio_dev_dbg(oct, "InstQ[%d]:dbell reg @ 0x%x instcnt_reg @ 0x%x\n", 426 iq_no, iq->doorbell_reg, iq->inst_cnt_reg); 427 428 /* 429 * Store the current instruction counter (used in flush_iq 430 * calculation) 431 */ 432 pkt_in_done = lio_read_csr64(oct, iq->inst_cnt_reg); 433 434 if (oct->msix_on) { 435 /* Set CINT_ENB to enable IQ interrupt */ 436 lio_write_csr64(oct, iq->inst_cnt_reg, 437 (pkt_in_done | LIO_CN23XX_INTR_CINT_ENB)); 438 } else { 439 /* 440 * Clear the count by writing back what we read, but don't 441 * enable interrupts 442 */ 443 lio_write_csr64(oct, iq->inst_cnt_reg, pkt_in_done); 444 } 445 446 iq->reset_instr_cnt = 0; 447 } 448 449 static void 450 lio_cn23xx_pf_setup_oq_regs(struct octeon_device *oct, uint32_t oq_no) 451 { 452 struct lio_droq *droq = oct->droq[oq_no]; 453 struct lio_cn23xx_pf *cn23xx = (struct lio_cn23xx_pf *)oct->chip; 454 uint64_t cnt_threshold; 455 uint64_t time_threshold; 456 uint32_t reg_val; 457 458 oq_no += oct->sriov_info.pf_srn; 459 460 lio_write_csr64(oct, LIO_CN23XX_SLI_OQ_BASE_ADDR64(oq_no), 461 droq->desc_ring_dma); 462 lio_write_csr32(oct, LIO_CN23XX_SLI_OQ_SIZE(oq_no), droq->max_count); 463 464 lio_write_csr32(oct, LIO_CN23XX_SLI_OQ_BUFF_INFO_SIZE(oq_no), 465 droq->buffer_size); 466 467 /* pkt_sent and pkts_credit regs */ 468 droq->pkts_sent_reg = LIO_CN23XX_SLI_OQ_PKTS_SENT(oq_no); 469 droq->pkts_credit_reg = LIO_CN23XX_SLI_OQ_PKTS_CREDIT(oq_no); 470 471 if (!oct->msix_on) { 472 /* 473 * Enable this output queue to generate Packet Timer 474 * Interrupt 475 */ 476 reg_val = 477 lio_read_csr32(oct, LIO_CN23XX_SLI_OQ_PKT_CONTROL(oq_no)); 478 reg_val |= LIO_CN23XX_PKT_OUTPUT_CTL_TENB; 479 lio_write_csr32(oct, LIO_CN23XX_SLI_OQ_PKT_CONTROL(oq_no), 480 reg_val); 481 482 /* 483 * Enable this output queue to generate Packet Count 484 * Interrupt 485 */ 486 reg_val = 487 lio_read_csr32(oct, LIO_CN23XX_SLI_OQ_PKT_CONTROL(oq_no)); 488 reg_val |= LIO_CN23XX_PKT_OUTPUT_CTL_CENB; 489 lio_write_csr32(oct, LIO_CN23XX_SLI_OQ_PKT_CONTROL(oq_no), 490 reg_val); 491 } else { 492 time_threshold = lio_cn23xx_pf_get_oq_ticks(oct, 493 (uint32_t)LIO_GET_OQ_INTR_TIME_CFG(cn23xx->conf)); 494 cnt_threshold = (uint32_t)LIO_GET_OQ_INTR_PKT_CFG(cn23xx->conf); 495 496 lio_write_csr64(oct, LIO_CN23XX_SLI_OQ_PKT_INT_LEVELS(oq_no), 497 ((time_threshold << 32 | cnt_threshold))); 498 } 499 } 500 501 502 static int 503 lio_cn23xx_pf_enable_io_queues(struct octeon_device *oct) 504 { 505 uint64_t reg_val; 506 uint32_t ern, loop = BUSY_READING_REG_PF_LOOP_COUNT; 507 uint32_t q_no, srn; 508 509 srn = oct->sriov_info.pf_srn; 510 ern = srn + oct->num_iqs; 511 512 for (q_no = srn; q_no < ern; q_no++) { 513 /* set the corresponding IQ IS_64B bit */ 514 if (oct->io_qmask.iq64B & BIT_ULL(q_no - srn)) { 515 reg_val = lio_read_csr64(oct, 516 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); 517 reg_val = reg_val | LIO_CN23XX_PKT_INPUT_CTL_IS_64B; 518 lio_write_csr64(oct, 519 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no), 520 reg_val); 521 } 522 /* set the corresponding IQ ENB bit */ 523 if (oct->io_qmask.iq & BIT_ULL(q_no - srn)) { 524 /* 525 * IOQs are in reset by default in PEM2 mode, 526 * clearing reset bit 527 */ 528 reg_val = lio_read_csr64(oct, 529 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); 530 531 if (reg_val & LIO_CN23XX_PKT_INPUT_CTL_RST) { 532 while ((reg_val & 533 LIO_CN23XX_PKT_INPUT_CTL_RST) && 534 !(reg_val & 535 LIO_CN23XX_PKT_INPUT_CTL_QUIET) && 536 loop) { 537 reg_val = lio_read_csr64(oct, 538 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); 539 loop--; 540 } 541 if (!loop) { 542 lio_dev_err(oct, "clearing the reset reg failed or setting the quiet reg failed for qno: %u\n", 543 q_no); 544 return (-1); 545 } 546 reg_val = reg_val & 547 ~LIO_CN23XX_PKT_INPUT_CTL_RST; 548 lio_write_csr64(oct, 549 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no), 550 reg_val); 551 552 reg_val = lio_read_csr64(oct, 553 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); 554 if (reg_val & LIO_CN23XX_PKT_INPUT_CTL_RST) { 555 lio_dev_err(oct, "clearing the reset failed for qno: %u\n", 556 q_no); 557 return (-1); 558 } 559 } 560 reg_val = lio_read_csr64(oct, 561 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); 562 reg_val = reg_val | LIO_CN23XX_PKT_INPUT_CTL_RING_ENB; 563 lio_write_csr64(oct, 564 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no), 565 reg_val); 566 } 567 } 568 for (q_no = srn; q_no < ern; q_no++) { 569 uint32_t reg_val; 570 /* set the corresponding OQ ENB bit */ 571 if (oct->io_qmask.oq & BIT_ULL(q_no - srn)) { 572 reg_val = lio_read_csr32(oct, 573 LIO_CN23XX_SLI_OQ_PKT_CONTROL(q_no)); 574 reg_val = reg_val | LIO_CN23XX_PKT_OUTPUT_CTL_RING_ENB; 575 lio_write_csr32(oct, 576 LIO_CN23XX_SLI_OQ_PKT_CONTROL(q_no), 577 reg_val); 578 } 579 } 580 return (0); 581 } 582 583 static void 584 lio_cn23xx_pf_disable_io_queues(struct octeon_device *oct) 585 { 586 volatile uint64_t d64; 587 volatile uint32_t d32; 588 int loop; 589 unsigned int q_no; 590 uint32_t ern, srn; 591 592 srn = oct->sriov_info.pf_srn; 593 ern = srn + oct->num_iqs; 594 595 /* Disable Input Queues. */ 596 for (q_no = srn; q_no < ern; q_no++) { 597 loop = lio_ms_to_ticks(1000); 598 599 /* start the Reset for a particular ring */ 600 d64 = lio_read_csr64(oct, 601 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no)); 602 d64 &= ~LIO_CN23XX_PKT_INPUT_CTL_RING_ENB; 603 d64 |= LIO_CN23XX_PKT_INPUT_CTL_RST; 604 lio_write_csr64(oct, LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no), 605 d64); 606 607 /* 608 * Wait until hardware indicates that the particular IQ 609 * is out of reset. 610 */ 611 d64 = lio_read_csr64(oct, LIO_CN23XX_SLI_PKT_IOQ_RING_RST); 612 while (!(d64 & BIT_ULL(q_no)) && loop--) { 613 d64 = lio_read_csr64(oct, 614 LIO_CN23XX_SLI_PKT_IOQ_RING_RST); 615 lio_sleep_timeout(1); 616 loop--; 617 } 618 619 /* Reset the doorbell register for this Input Queue. */ 620 lio_write_csr32(oct, LIO_CN23XX_SLI_IQ_DOORBELL(q_no), 621 0xFFFFFFFF); 622 while (((lio_read_csr64(oct, 623 LIO_CN23XX_SLI_IQ_DOORBELL(q_no))) != 624 0ULL) && loop--) { 625 lio_sleep_timeout(1); 626 } 627 } 628 629 /* Disable Output Queues. */ 630 for (q_no = srn; q_no < ern; q_no++) { 631 loop = lio_ms_to_ticks(1000); 632 633 /* 634 * Wait until hardware indicates that the particular IQ 635 * is out of reset.It given that SLI_PKT_RING_RST is 636 * common for both IQs and OQs 637 */ 638 d64 = lio_read_csr64(oct, LIO_CN23XX_SLI_PKT_IOQ_RING_RST); 639 while (!(d64 & BIT_ULL(q_no)) && loop--) { 640 d64 = lio_read_csr64(oct, 641 LIO_CN23XX_SLI_PKT_IOQ_RING_RST); 642 lio_sleep_timeout(1); 643 loop--; 644 } 645 646 /* Reset the doorbell register for this Output Queue. */ 647 lio_write_csr32(oct, LIO_CN23XX_SLI_OQ_PKTS_CREDIT(q_no), 648 0xFFFFFFFF); 649 while ((lio_read_csr64(oct, 650 LIO_CN23XX_SLI_OQ_PKTS_CREDIT(q_no)) != 651 0ULL) && loop--) { 652 lio_sleep_timeout(1); 653 } 654 655 /* clear the SLI_PKT(0..63)_CNTS[CNT] reg value */ 656 d32 = lio_read_csr32(oct, LIO_CN23XX_SLI_OQ_PKTS_SENT(q_no)); 657 lio_write_csr32(oct, LIO_CN23XX_SLI_OQ_PKTS_SENT(q_no), d32); 658 } 659 } 660 661 static uint64_t 662 lio_cn23xx_pf_msix_interrupt_handler(void *dev) 663 { 664 struct lio_ioq_vector *ioq_vector = (struct lio_ioq_vector *)dev; 665 struct octeon_device *oct = ioq_vector->oct_dev; 666 struct lio_droq *droq = oct->droq[ioq_vector->droq_index]; 667 uint64_t pkts_sent; 668 uint64_t ret = 0; 669 670 if (droq == NULL) { 671 lio_dev_err(oct, "23XX bringup FIXME: oct pfnum:%d ioq_vector->ioq_num :%d droq is NULL\n", 672 oct->pf_num, ioq_vector->ioq_num); 673 return (0); 674 } 675 pkts_sent = lio_read_csr64(oct, droq->pkts_sent_reg); 676 677 /* 678 * If our device has interrupted, then proceed. Also check 679 * for all f's if interrupt was triggered on an error 680 * and the PCI read fails. 681 */ 682 if (!pkts_sent || (pkts_sent == 0xFFFFFFFFFFFFFFFFULL)) 683 return (ret); 684 685 /* Write count reg in sli_pkt_cnts to clear these int. */ 686 if (pkts_sent & LIO_CN23XX_INTR_PO_INT) 687 ret |= LIO_MSIX_PO_INT; 688 689 if (pkts_sent & LIO_CN23XX_INTR_PI_INT) 690 /* We will clear the count when we update the read_index. */ 691 ret |= LIO_MSIX_PI_INT; 692 693 /* 694 * Never need to handle msix mbox intr for pf. They arrive on the last 695 * msix 696 */ 697 return (ret); 698 } 699 700 static void 701 lio_cn23xx_pf_interrupt_handler(void *dev) 702 { 703 struct octeon_device *oct = (struct octeon_device *)dev; 704 struct lio_cn23xx_pf *cn23xx = (struct lio_cn23xx_pf *)oct->chip; 705 uint64_t intr64; 706 707 lio_dev_dbg(oct, "In %s octeon_dev @ %p\n", __func__, oct); 708 intr64 = lio_read_csr64(oct, cn23xx->intr_sum_reg64); 709 710 oct->int_status = 0; 711 712 if (intr64 & LIO_CN23XX_INTR_ERR) 713 lio_dev_err(oct, "Error Intr: 0x%016llx\n", 714 LIO_CAST64(intr64)); 715 716 if (oct->msix_on != LIO_FLAG_MSIX_ENABLED) { 717 if (intr64 & LIO_CN23XX_INTR_PKT_DATA) 718 oct->int_status |= LIO_DEV_INTR_PKT_DATA; 719 } 720 721 if (intr64 & (LIO_CN23XX_INTR_DMA0_FORCE)) 722 oct->int_status |= LIO_DEV_INTR_DMA0_FORCE; 723 724 if (intr64 & (LIO_CN23XX_INTR_DMA1_FORCE)) 725 oct->int_status |= LIO_DEV_INTR_DMA1_FORCE; 726 727 /* Clear the current interrupts */ 728 lio_write_csr64(oct, cn23xx->intr_sum_reg64, intr64); 729 } 730 731 static void 732 lio_cn23xx_pf_bar1_idx_setup(struct octeon_device *oct, uint64_t core_addr, 733 uint32_t idx, int valid) 734 { 735 volatile uint64_t bar1; 736 uint64_t reg_adr; 737 738 if (!valid) { 739 reg_adr = lio_pci_readq(oct, 740 LIO_CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, 741 idx)); 742 bar1 = reg_adr; 743 lio_pci_writeq(oct, (bar1 & 0xFFFFFFFEULL), 744 LIO_CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, 745 idx)); 746 reg_adr = lio_pci_readq(oct, 747 LIO_CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, 748 idx)); 749 bar1 = reg_adr; 750 return; 751 } 752 /* 753 * The PEM(0..3)_BAR1_INDEX(0..15)[ADDR_IDX]<23:4> stores 754 * bits <41:22> of the Core Addr 755 */ 756 lio_pci_writeq(oct, (((core_addr >> 22) << 4) | LIO_PCI_BAR1_MASK), 757 LIO_CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx)); 758 759 bar1 = lio_pci_readq(oct, LIO_CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, 760 idx)); 761 } 762 763 static void 764 lio_cn23xx_pf_bar1_idx_write(struct octeon_device *oct, uint32_t idx, 765 uint32_t mask) 766 { 767 768 lio_pci_writeq(oct, mask, 769 LIO_CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx)); 770 } 771 772 static uint32_t 773 lio_cn23xx_pf_bar1_idx_read(struct octeon_device *oct, uint32_t idx) 774 { 775 776 return ((uint32_t)lio_pci_readq(oct, 777 LIO_CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, 778 idx))); 779 } 780 781 /* always call with lock held */ 782 static uint32_t 783 lio_cn23xx_pf_update_read_index(struct lio_instr_queue *iq) 784 { 785 struct octeon_device *oct = iq->oct_dev; 786 uint32_t new_idx; 787 uint32_t last_done; 788 uint32_t pkt_in_done = lio_read_csr32(oct, iq->inst_cnt_reg); 789 790 last_done = pkt_in_done - iq->pkt_in_done; 791 iq->pkt_in_done = pkt_in_done; 792 793 /* 794 * Modulo of the new index with the IQ size will give us 795 * the new index. The iq->reset_instr_cnt is always zero for 796 * cn23xx, so no extra adjustments are needed. 797 */ 798 new_idx = (iq->octeon_read_index + 799 ((uint32_t)(last_done & LIO_CN23XX_PKT_IN_DONE_CNT_MASK))) % 800 iq->max_count; 801 802 return (new_idx); 803 } 804 805 static void 806 lio_cn23xx_pf_enable_interrupt(struct octeon_device *oct, uint8_t intr_flag) 807 { 808 struct lio_cn23xx_pf *cn23xx = (struct lio_cn23xx_pf *)oct->chip; 809 uint64_t intr_val = 0; 810 811 /* Divide the single write to multiple writes based on the flag. */ 812 /* Enable Interrupt */ 813 if (intr_flag == OCTEON_ALL_INTR) { 814 lio_write_csr64(oct, cn23xx->intr_enb_reg64, 815 cn23xx->intr_mask64); 816 } else if (intr_flag & OCTEON_OUTPUT_INTR) { 817 intr_val = lio_read_csr64(oct, cn23xx->intr_enb_reg64); 818 intr_val |= LIO_CN23XX_INTR_PKT_DATA; 819 lio_write_csr64(oct, cn23xx->intr_enb_reg64, intr_val); 820 } 821 } 822 823 static void 824 lio_cn23xx_pf_disable_interrupt(struct octeon_device *oct, uint8_t intr_flag) 825 { 826 struct lio_cn23xx_pf *cn23xx = (struct lio_cn23xx_pf *)oct->chip; 827 uint64_t intr_val = 0; 828 829 /* Disable Interrupts */ 830 if (intr_flag == OCTEON_ALL_INTR) { 831 lio_write_csr64(oct, cn23xx->intr_enb_reg64, 0); 832 } else if (intr_flag & OCTEON_OUTPUT_INTR) { 833 intr_val = lio_read_csr64(oct, cn23xx->intr_enb_reg64); 834 intr_val &= ~LIO_CN23XX_INTR_PKT_DATA; 835 lio_write_csr64(oct, cn23xx->intr_enb_reg64, intr_val); 836 } 837 } 838 839 static void 840 lio_cn23xx_pf_get_pcie_qlmport(struct octeon_device *oct) 841 { 842 oct->pcie_port = (lio_read_csr32(oct, 843 LIO_CN23XX_SLI_MAC_NUMBER)) & 0xff; 844 845 lio_dev_dbg(oct, "CN23xx uses PCIE Port %d\n", 846 oct->pcie_port); 847 } 848 849 static void 850 lio_cn23xx_pf_get_pf_num(struct octeon_device *oct) 851 { 852 uint32_t fdl_bit; 853 854 /* Read Function Dependency Link reg to get the function number */ 855 fdl_bit = lio_read_pci_cfg(oct, LIO_CN23XX_PCIE_SRIOV_FDL); 856 oct->pf_num = ((fdl_bit >> LIO_CN23XX_PCIE_SRIOV_FDL_BIT_POS) & 857 LIO_CN23XX_PCIE_SRIOV_FDL_MASK); 858 } 859 860 static void 861 lio_cn23xx_pf_setup_reg_address(struct octeon_device *oct) 862 { 863 struct lio_cn23xx_pf *cn23xx = (struct lio_cn23xx_pf *)oct->chip; 864 865 oct->reg_list.pci_win_wr_addr = LIO_CN23XX_SLI_WIN_WR_ADDR64; 866 867 oct->reg_list.pci_win_rd_addr_hi = LIO_CN23XX_SLI_WIN_RD_ADDR_HI; 868 oct->reg_list.pci_win_rd_addr_lo = LIO_CN23XX_SLI_WIN_RD_ADDR64; 869 oct->reg_list.pci_win_rd_addr = LIO_CN23XX_SLI_WIN_RD_ADDR64; 870 871 oct->reg_list.pci_win_wr_data_hi = LIO_CN23XX_SLI_WIN_WR_DATA_HI; 872 oct->reg_list.pci_win_wr_data_lo = LIO_CN23XX_SLI_WIN_WR_DATA_LO; 873 oct->reg_list.pci_win_wr_data = LIO_CN23XX_SLI_WIN_WR_DATA64; 874 875 oct->reg_list.pci_win_rd_data = LIO_CN23XX_SLI_WIN_RD_DATA64; 876 877 lio_cn23xx_pf_get_pcie_qlmport(oct); 878 879 cn23xx->intr_mask64 = LIO_CN23XX_INTR_MASK; 880 if (!oct->msix_on) 881 cn23xx->intr_mask64 |= LIO_CN23XX_INTR_PKT_TIME; 882 883 cn23xx->intr_sum_reg64 = 884 LIO_CN23XX_SLI_MAC_PF_INT_SUM64(oct->pcie_port, oct->pf_num); 885 cn23xx->intr_enb_reg64 = 886 LIO_CN23XX_SLI_MAC_PF_INT_ENB64(oct->pcie_port, oct->pf_num); 887 } 888 889 static int 890 lio_cn23xx_pf_sriov_config(struct octeon_device *oct) 891 { 892 struct lio_cn23xx_pf *cn23xx = (struct lio_cn23xx_pf *)oct->chip; 893 uint32_t num_pf_rings, total_rings, max_rings; 894 cn23xx->conf = (struct lio_config *)lio_get_config_info(oct, LIO_23XX); 895 896 max_rings = LIO_CN23XX_PF_MAX_RINGS; 897 898 if (oct->sriov_info.num_pf_rings) { 899 num_pf_rings = oct->sriov_info.num_pf_rings; 900 if (num_pf_rings > max_rings) { 901 num_pf_rings = min(mp_ncpus, max_rings); 902 lio_dev_warn(oct, "num_queues_per_pf requested %u is more than available rings (%u). Reducing to %u\n", 903 oct->sriov_info.num_pf_rings, 904 max_rings, num_pf_rings); 905 } 906 } else { 907 #ifdef RSS 908 num_pf_rings = min(rss_getnumbuckets(), mp_ncpus); 909 #else 910 num_pf_rings = min(mp_ncpus, max_rings); 911 #endif 912 913 } 914 915 total_rings = num_pf_rings; 916 oct->sriov_info.trs = total_rings; 917 oct->sriov_info.pf_srn = total_rings - num_pf_rings; 918 oct->sriov_info.num_pf_rings = num_pf_rings; 919 920 lio_dev_dbg(oct, "trs:%d pf_srn:%d num_pf_rings:%d\n", 921 oct->sriov_info.trs, oct->sriov_info.pf_srn, 922 oct->sriov_info.num_pf_rings); 923 924 return (0); 925 } 926 927 int 928 lio_cn23xx_pf_setup_device(struct octeon_device *oct) 929 { 930 uint64_t BAR0, BAR1; 931 uint32_t data32; 932 933 data32 = lio_read_pci_cfg(oct, 0x10); 934 BAR0 = (uint64_t)(data32 & ~0xf); 935 data32 = lio_read_pci_cfg(oct, 0x14); 936 BAR0 |= ((uint64_t)data32 << 32); 937 data32 = lio_read_pci_cfg(oct, 0x18); 938 BAR1 = (uint64_t)(data32 & ~0xf); 939 data32 = lio_read_pci_cfg(oct, 0x1c); 940 BAR1 |= ((uint64_t)data32 << 32); 941 942 if (!BAR0 || !BAR1) { 943 if (!BAR0) 944 lio_dev_err(oct, "Device BAR0 unassigned\n"); 945 946 if (!BAR1) 947 lio_dev_err(oct, "Device BAR1 unassigned\n"); 948 949 return (1); 950 } 951 952 if (lio_map_pci_barx(oct, 0)) 953 return (1); 954 955 if (lio_map_pci_barx(oct, 1)) { 956 lio_dev_err(oct, "%s CN23XX BAR1 map failed\n", __func__); 957 lio_unmap_pci_barx(oct, 0); 958 return (1); 959 } 960 961 lio_cn23xx_pf_get_pf_num(oct); 962 963 if (lio_cn23xx_pf_sriov_config(oct)) { 964 lio_unmap_pci_barx(oct, 0); 965 lio_unmap_pci_barx(oct, 1); 966 return (1); 967 } 968 lio_write_csr64(oct, LIO_CN23XX_SLI_MAC_CREDIT_CNT, 969 0x3F802080802080ULL); 970 971 oct->fn_list.setup_iq_regs = lio_cn23xx_pf_setup_iq_regs; 972 oct->fn_list.setup_oq_regs = lio_cn23xx_pf_setup_oq_regs; 973 oct->fn_list.process_interrupt_regs = lio_cn23xx_pf_interrupt_handler; 974 oct->fn_list.msix_interrupt_handler = 975 lio_cn23xx_pf_msix_interrupt_handler; 976 977 oct->fn_list.soft_reset = lio_cn23xx_pf_soft_reset; 978 oct->fn_list.setup_device_regs = lio_cn23xx_pf_setup_device_regs; 979 oct->fn_list.update_iq_read_idx = lio_cn23xx_pf_update_read_index; 980 981 oct->fn_list.bar1_idx_setup = lio_cn23xx_pf_bar1_idx_setup; 982 oct->fn_list.bar1_idx_write = lio_cn23xx_pf_bar1_idx_write; 983 oct->fn_list.bar1_idx_read = lio_cn23xx_pf_bar1_idx_read; 984 985 oct->fn_list.enable_interrupt = lio_cn23xx_pf_enable_interrupt; 986 oct->fn_list.disable_interrupt = lio_cn23xx_pf_disable_interrupt; 987 988 oct->fn_list.enable_io_queues = lio_cn23xx_pf_enable_io_queues; 989 oct->fn_list.disable_io_queues = lio_cn23xx_pf_disable_io_queues; 990 991 lio_cn23xx_pf_setup_reg_address(oct); 992 993 oct->coproc_clock_rate = 1000000ULL * 994 lio_cn23xx_pf_coprocessor_clock(oct); 995 996 return (0); 997 } 998 999 int 1000 lio_cn23xx_pf_fw_loaded(struct octeon_device *oct) 1001 { 1002 uint64_t val; 1003 1004 val = lio_read_csr64(oct, LIO_CN23XX_SLI_SCRATCH2); 1005 return ((val >> SCR2_BIT_FW_LOADED) & 1ULL); 1006 } 1007 1008