1 /*- 2 * Copyright (c) 2017-2019 Alexander Motin <mav@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * The Non-Transparent Bridge (NTB) is a device that allows you to connect 29 * two or more systems using a PCI-e links, providing remote memory access. 30 * 31 * This module contains a driver for NTBs in PLX/Avago/Broadcom PCIe bridges. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/kernel.h> 39 #include <sys/systm.h> 40 #include <sys/bus.h> 41 #include <sys/interrupt.h> 42 #include <sys/module.h> 43 #include <sys/rman.h> 44 #include <sys/sysctl.h> 45 #include <sys/taskqueue.h> 46 #include <sys/tree.h> 47 #include <vm/vm.h> 48 #include <vm/pmap.h> 49 #include <machine/bus.h> 50 #include <machine/intr_machdep.h> 51 #include <machine/resource.h> 52 #include <dev/pci/pcireg.h> 53 #include <dev/pci/pcivar.h> 54 #include <dev/iommu/iommu.h> 55 56 #include "../ntb.h" 57 58 #define PLX_MAX_BARS 4 /* There are at most 4 data BARs. */ 59 #define PLX_NUM_SPAD 8 /* There are 8 scratchpads. */ 60 #define PLX_NUM_SPAD_PATT 4 /* Use test pattern as 4 more. */ 61 #define PLX_NUM_DB 16 /* There are 16 doorbells. */ 62 #define PLX_MAX_SPLIT 128 /* Allow are at most 128 splits. */ 63 64 struct ntb_plx_mw_info { 65 int mw_bar; 66 int mw_64bit; 67 int mw_rid; 68 struct resource *mw_res; 69 vm_paddr_t mw_pbase; 70 caddr_t mw_vbase; 71 vm_size_t mw_size; 72 struct { 73 vm_memattr_t mw_map_mode; 74 bus_addr_t mw_xlat_addr; 75 bus_size_t mw_xlat_size; 76 } splits[PLX_MAX_SPLIT]; 77 }; 78 79 struct ntb_plx_softc { 80 /* ntb.c context. Do not move! Must go first! */ 81 void *ntb_store; 82 83 device_t dev; 84 struct resource *conf_res; 85 int conf_rid; 86 u_int ntx; /* NTx number within chip. */ 87 u_int link; /* Link v/s Virtual side. */ 88 u_int port; /* Port number within chip. */ 89 u_int alut; /* A-LUT is enabled for NTx */ 90 u_int split; /* split BAR2 into 2^x parts */ 91 92 int int_rid; 93 struct resource *int_res; 94 void *int_tag; 95 96 struct ntb_plx_mw_info mw_info[PLX_MAX_BARS]; 97 int mw_count; /* Number of memory windows. */ 98 99 int spad_count1; /* Number of standard spads. */ 100 int spad_count2; /* Number of extra spads. */ 101 uint32_t spad_off1; /* Offset of our spads. */ 102 uint32_t spad_off2; /* Offset of our extra spads. */ 103 uint32_t spad_offp1; /* Offset of peer spads. */ 104 uint32_t spad_offp2; /* Offset of peer extra spads. */ 105 106 /* Parameters of window shared with peer config access in B2B mode. */ 107 int b2b_mw; /* Shared window number. */ 108 uint64_t b2b_off; /* Offset in shared window. */ 109 }; 110 111 #define PLX_NT0_BASE 0x3E000 112 #define PLX_NT1_BASE 0x3C000 113 #define PLX_NTX_BASE(sc) ((sc)->ntx ? PLX_NT1_BASE : PLX_NT0_BASE) 114 #define PLX_NTX_LINK_OFFSET 0x01000 115 116 /* Bases of NTx our/peer interface registers */ 117 #define PLX_NTX_OUR_BASE(sc) \ 118 (PLX_NTX_BASE(sc) + ((sc)->link ? PLX_NTX_LINK_OFFSET : 0)) 119 #define PLX_NTX_PEER_BASE(sc) \ 120 (PLX_NTX_BASE(sc) + ((sc)->link ? 0 : PLX_NTX_LINK_OFFSET)) 121 122 /* Read/write NTx our interface registers */ 123 #define NTX_READ(sc, reg) \ 124 bus_read_4((sc)->conf_res, PLX_NTX_OUR_BASE(sc) + (reg)) 125 #define NTX_WRITE(sc, reg, val) \ 126 bus_write_4((sc)->conf_res, PLX_NTX_OUR_BASE(sc) + (reg), (val)) 127 128 /* Read/write NTx peer interface registers */ 129 #define PNTX_READ(sc, reg) \ 130 bus_read_4((sc)->conf_res, PLX_NTX_PEER_BASE(sc) + (reg)) 131 #define PNTX_WRITE(sc, reg, val) \ 132 bus_write_4((sc)->conf_res, PLX_NTX_PEER_BASE(sc) + (reg), (val)) 133 134 /* Read/write B2B NTx registers */ 135 #define BNTX_READ(sc, reg) \ 136 bus_read_4((sc)->mw_info[(sc)->b2b_mw].mw_res, \ 137 PLX_NTX_BASE(sc) + (reg)) 138 #define BNTX_WRITE(sc, reg, val) \ 139 bus_write_4((sc)->mw_info[(sc)->b2b_mw].mw_res, \ 140 PLX_NTX_BASE(sc) + (reg), (val)) 141 142 #define PLX_PORT_BASE(p) ((p) << 12) 143 #define PLX_STATION_PORT_BASE(sc) PLX_PORT_BASE((sc)->port & ~7) 144 145 #define PLX_PORT_CONTROL(sc) (PLX_STATION_PORT_BASE(sc) + 0x208) 146 147 static int ntb_plx_init(device_t dev); 148 static int ntb_plx_detach(device_t dev); 149 static int ntb_plx_mw_set_trans_internal(device_t dev, unsigned mw_idx); 150 151 static int 152 ntb_plx_probe(device_t dev) 153 { 154 155 switch (pci_get_devid(dev)) { 156 case 0x87a010b5: 157 device_set_desc(dev, "PLX Non-Transparent Bridge NT0 Link"); 158 return (BUS_PROBE_DEFAULT); 159 case 0x87a110b5: 160 device_set_desc(dev, "PLX Non-Transparent Bridge NT1 Link"); 161 return (BUS_PROBE_DEFAULT); 162 case 0x87b010b5: 163 device_set_desc(dev, "PLX Non-Transparent Bridge NT0 Virtual"); 164 return (BUS_PROBE_DEFAULT); 165 case 0x87b110b5: 166 device_set_desc(dev, "PLX Non-Transparent Bridge NT1 Virtual"); 167 return (BUS_PROBE_DEFAULT); 168 } 169 return (ENXIO); 170 } 171 172 static int 173 ntb_plx_init(device_t dev) 174 { 175 struct ntb_plx_softc *sc = device_get_softc(dev); 176 struct ntb_plx_mw_info *mw; 177 uint64_t val64; 178 int i; 179 uint32_t val; 180 181 if (sc->b2b_mw >= 0) { 182 /* Set peer BAR0/1 size and address for B2B NTx access. */ 183 mw = &sc->mw_info[sc->b2b_mw]; 184 if (mw->mw_64bit) { 185 PNTX_WRITE(sc, 0xe4, 0x3); /* 64-bit */ 186 val64 = 0x2000000000000000 * mw->mw_bar | 0x4; 187 PNTX_WRITE(sc, PCIR_BAR(0), val64); 188 PNTX_WRITE(sc, PCIR_BAR(0) + 4, val64 >> 32); 189 } else { 190 PNTX_WRITE(sc, 0xe4, 0x2); /* 32-bit */ 191 val = 0x20000000 * mw->mw_bar; 192 PNTX_WRITE(sc, PCIR_BAR(0), val); 193 } 194 195 /* Set Virtual to Link address translation for B2B. */ 196 for (i = 0; i < sc->mw_count; i++) { 197 mw = &sc->mw_info[i]; 198 if (mw->mw_64bit) { 199 val64 = 0x2000000000000000 * mw->mw_bar; 200 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, val64); 201 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4 + 4, val64 >> 32); 202 } else { 203 val = 0x20000000 * mw->mw_bar; 204 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, val); 205 } 206 } 207 208 /* Make sure Virtual to Link A-LUT is disabled. */ 209 if (sc->alut) 210 PNTX_WRITE(sc, 0xc94, 0); 211 212 /* Enable all Link Interface LUT entries for peer. */ 213 for (i = 0; i < 32; i += 2) { 214 PNTX_WRITE(sc, 0xdb4 + i * 2, 215 0x00010001 | ((i + 1) << 19) | (i << 3)); 216 } 217 } 218 219 /* 220 * Enable Virtual Interface LUT entry 0 for 0:0.*. 221 * entry 1 for our Requester ID reported by the chip, 222 * entries 2-5 for 0/64/128/192:4.* of I/OAT DMA engines. 223 * XXX: Its a hack, we can't know all DMA engines, but this covers all 224 * I/OAT of Xeon E5/E7 at least from Sandy Bridge till Skylake I saw. 225 */ 226 val = (NTX_READ(sc, 0xc90) << 16) | 0x00010001; 227 NTX_WRITE(sc, sc->link ? 0xdb4 : 0xd94, val); 228 NTX_WRITE(sc, sc->link ? 0xdb8 : 0xd98, 0x40210021); 229 NTX_WRITE(sc, sc->link ? 0xdbc : 0xd9c, 0xc0218021); 230 231 /* Set Link to Virtual address translation. */ 232 for (i = 0; i < sc->mw_count; i++) 233 ntb_plx_mw_set_trans_internal(dev, i); 234 235 pci_enable_busmaster(dev); 236 if (sc->b2b_mw >= 0) 237 PNTX_WRITE(sc, PCIR_COMMAND, PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 238 239 return (0); 240 } 241 242 static void 243 ntb_plx_isr(void *arg) 244 { 245 device_t dev = arg; 246 struct ntb_plx_softc *sc = device_get_softc(dev); 247 uint32_t val; 248 249 ntb_db_event((device_t)arg, 0); 250 251 if (sc->link) /* Link Interface has no Link Error registers. */ 252 return; 253 254 val = NTX_READ(sc, 0xfe0); 255 if (val == 0) 256 return; 257 NTX_WRITE(sc, 0xfe0, val); 258 if (val & 1) 259 device_printf(dev, "Correctable Error\n"); 260 if (val & 2) 261 device_printf(dev, "Uncorrectable Error\n"); 262 if (val & 4) { 263 /* DL_Down resets link side registers, have to reinit. */ 264 ntb_plx_init(dev); 265 ntb_link_event(dev); 266 } 267 if (val & 8) 268 device_printf(dev, "Uncorrectable Error Message Drop\n"); 269 } 270 271 static int 272 ntb_plx_setup_intr(device_t dev) 273 { 274 struct ntb_plx_softc *sc = device_get_softc(dev); 275 int error; 276 277 /* 278 * XXX: This hardware supports MSI, but I found it unusable. 279 * It generates new MSI only when doorbell register goes from 280 * zero, but does not generate it when another bit is set or on 281 * partial clear. It makes operation very racy and unreliable. 282 * The data book mentions some mask juggling magic to workaround 283 * that, but I failed to make it work. 284 */ 285 sc->int_rid = 0; 286 sc->int_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 287 &sc->int_rid, RF_SHAREABLE|RF_ACTIVE); 288 if (sc->int_res == NULL) { 289 device_printf(dev, "bus_alloc_resource failed\n"); 290 return (ENOMEM); 291 } 292 error = bus_setup_intr(dev, sc->int_res, INTR_MPSAFE | INTR_TYPE_MISC, 293 NULL, ntb_plx_isr, dev, &sc->int_tag); 294 if (error != 0) { 295 device_printf(dev, "bus_setup_intr failed: %d\n", error); 296 return (error); 297 } 298 299 if (!sc->link) { /* Link Interface has no Link Error registers. */ 300 NTX_WRITE(sc, 0xfe0, 0xf); /* Clear link interrupts. */ 301 NTX_WRITE(sc, 0xfe4, 0x0); /* Unmask link interrupts. */ 302 } 303 return (0); 304 } 305 306 static void 307 ntb_plx_teardown_intr(device_t dev) 308 { 309 struct ntb_plx_softc *sc = device_get_softc(dev); 310 311 if (!sc->link) /* Link Interface has no Link Error registers. */ 312 NTX_WRITE(sc, 0xfe4, 0xf); /* Mask link interrupts. */ 313 314 if (sc->int_res) { 315 bus_teardown_intr(dev, sc->int_res, sc->int_tag); 316 bus_release_resource(dev, SYS_RES_IRQ, sc->int_rid, 317 sc->int_res); 318 } 319 } 320 321 static int 322 ntb_plx_attach(device_t dev) 323 { 324 struct ntb_plx_softc *sc = device_get_softc(dev); 325 struct ntb_plx_mw_info *mw; 326 int error = 0, i, j; 327 uint32_t val; 328 char buf[32]; 329 330 /* Identify what we are (what side of what NTx). */ 331 sc->dev = dev; 332 val = pci_read_config(dev, 0xc8c, 4); 333 sc->ntx = (val & 1) != 0; 334 sc->link = (val & 0x80000000) != 0; 335 336 /* Get access to whole 256KB of chip configuration space via BAR0/1. */ 337 sc->conf_rid = PCIR_BAR(0); 338 sc->conf_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 339 &sc->conf_rid, RF_ACTIVE); 340 if (sc->conf_res == NULL) { 341 device_printf(dev, "Can't allocate configuration BAR.\n"); 342 return (ENXIO); 343 } 344 345 /* 346 * The device occupies whole bus. In translated TLP slot field 347 * keeps LUT index (original bus/slot), function is passed through. 348 */ 349 bus_dma_iommu_set_buswide(dev); 350 351 /* Identify chip port we are connected to. */ 352 val = bus_read_4(sc->conf_res, 0x360); 353 sc->port = (val >> ((sc->ntx == 0) ? 8 : 16)) & 0x1f; 354 355 /* Detect A-LUT enable and size. */ 356 val >>= 30; 357 sc->alut = (val == 0x3) ? 1 : ((val & (1 << sc->ntx)) ? 2 : 0); 358 if (sc->alut) 359 device_printf(dev, "%u A-LUT entries\n", 128 * sc->alut); 360 361 /* Find configured memory windows at BAR2-5. */ 362 sc->mw_count = 0; 363 for (i = 2; i <= 5; i++) { 364 mw = &sc->mw_info[sc->mw_count]; 365 mw->mw_bar = i; 366 mw->mw_rid = PCIR_BAR(mw->mw_bar); 367 mw->mw_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 368 &mw->mw_rid, RF_ACTIVE); 369 if (mw->mw_res == NULL) 370 continue; 371 mw->mw_pbase = rman_get_start(mw->mw_res); 372 mw->mw_size = rman_get_size(mw->mw_res); 373 mw->mw_vbase = rman_get_virtual(mw->mw_res); 374 for (j = 0; j < PLX_MAX_SPLIT; j++) 375 mw->splits[j].mw_map_mode = VM_MEMATTR_UNCACHEABLE; 376 sc->mw_count++; 377 378 /* Skip over adjacent BAR for 64-bit BARs. */ 379 val = pci_read_config(dev, PCIR_BAR(mw->mw_bar), 4); 380 if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64) { 381 mw->mw_64bit = 1; 382 i++; 383 } 384 } 385 386 /* Try to identify B2B mode. */ 387 i = 1; 388 snprintf(buf, sizeof(buf), "hint.%s.%d.b2b", device_get_name(dev), 389 device_get_unit(dev)); 390 TUNABLE_INT_FETCH(buf, &i); 391 if (sc->link) { 392 device_printf(dev, "NTB-to-Root Port mode (Link Interface)\n"); 393 sc->b2b_mw = -1; 394 } else if (i == 0) { 395 device_printf(dev, "NTB-to-Root Port mode (Virtual Interface)\n"); 396 sc->b2b_mw = -1; 397 } else { 398 device_printf(dev, "NTB-to-NTB (back-to-back) mode\n"); 399 400 /* We need at least one memory window for B2B peer access. */ 401 if (sc->mw_count == 0) { 402 device_printf(dev, "No memory window BARs enabled.\n"); 403 error = ENXIO; 404 goto out; 405 } 406 sc->b2b_mw = sc->mw_count - 1; 407 408 /* Use half of the window for B2B, but no less then 1MB. */ 409 mw = &sc->mw_info[sc->b2b_mw]; 410 if (mw->mw_size >= 2 * 1024 * 1024) 411 sc->b2b_off = mw->mw_size / 2; 412 else 413 sc->b2b_off = 0; 414 } 415 416 snprintf(buf, sizeof(buf), "hint.%s.%d.split", device_get_name(dev), 417 device_get_unit(dev)); 418 TUNABLE_INT_FETCH(buf, &sc->split); 419 if (sc->split > 7) { 420 device_printf(dev, "Split value is too high (%u)\n", sc->split); 421 sc->split = 0; 422 } else if (sc->split > 0 && sc->alut == 0) { 423 device_printf(dev, "Can't split with disabled A-LUT\n"); 424 sc->split = 0; 425 } else if (sc->split > 0 && (sc->mw_count == 0 || sc->mw_info[0].mw_bar != 2)) { 426 device_printf(dev, "Can't split disabled BAR2\n"); 427 sc->split = 0; 428 } else if (sc->split > 0 && (sc->b2b_mw == 0 && sc->b2b_off == 0)) { 429 device_printf(dev, "Can't split BAR2 consumed by B2B\n"); 430 sc->split = 0; 431 } else if (sc->split > 0) { 432 device_printf(dev, "Splitting BAR2 into %d memory windows\n", 433 1 << sc->split); 434 } 435 436 /* 437 * Use Physical Layer User Test Pattern as additional scratchpad. 438 * Make sure they are present and enabled by writing to them. 439 * XXX: Its a hack, but standard 8 registers are not enough. 440 */ 441 sc->spad_offp1 = sc->spad_off1 = PLX_NTX_OUR_BASE(sc) + 0xc6c; 442 sc->spad_offp2 = sc->spad_off2 = PLX_PORT_BASE(sc->ntx * 8) + 0x20c; 443 if (sc->b2b_mw >= 0) { 444 /* In NTB-to-NTB mode each side has own scratchpads. */ 445 sc->spad_count1 = PLX_NUM_SPAD; 446 bus_write_4(sc->conf_res, sc->spad_off2, 0x12345678); 447 if (bus_read_4(sc->conf_res, sc->spad_off2) == 0x12345678) 448 sc->spad_count2 = PLX_NUM_SPAD_PATT; 449 } else { 450 /* Otherwise we have share scratchpads with the peer. */ 451 if (sc->link) { 452 sc->spad_off1 += PLX_NUM_SPAD / 2 * 4; 453 sc->spad_off2 += PLX_NUM_SPAD_PATT / 2 * 4; 454 } else { 455 sc->spad_offp1 += PLX_NUM_SPAD / 2 * 4; 456 sc->spad_offp2 += PLX_NUM_SPAD_PATT / 2 * 4; 457 } 458 sc->spad_count1 = PLX_NUM_SPAD / 2; 459 bus_write_4(sc->conf_res, sc->spad_off2, 0x12345678); 460 if (bus_read_4(sc->conf_res, sc->spad_off2) == 0x12345678) 461 sc->spad_count2 = PLX_NUM_SPAD_PATT / 2; 462 } 463 464 /* Apply static part of NTB configuration. */ 465 ntb_plx_init(dev); 466 467 /* Allocate and setup interrupts. */ 468 error = ntb_plx_setup_intr(dev); 469 if (error) 470 goto out; 471 472 /* Attach children to this controller */ 473 error = ntb_register_device(dev); 474 475 out: 476 if (error != 0) 477 ntb_plx_detach(dev); 478 return (error); 479 } 480 481 static int 482 ntb_plx_detach(device_t dev) 483 { 484 struct ntb_plx_softc *sc = device_get_softc(dev); 485 struct ntb_plx_mw_info *mw; 486 int i; 487 488 /* Detach & delete all children */ 489 ntb_unregister_device(dev); 490 491 /* Disable and free interrupts. */ 492 ntb_plx_teardown_intr(dev); 493 494 /* Free memory resources. */ 495 for (i = 0; i < sc->mw_count; i++) { 496 mw = &sc->mw_info[i]; 497 bus_release_resource(dev, SYS_RES_MEMORY, mw->mw_rid, 498 mw->mw_res); 499 } 500 bus_release_resource(dev, SYS_RES_MEMORY, sc->conf_rid, sc->conf_res); 501 return (0); 502 } 503 504 static int 505 ntb_plx_port_number(device_t dev) 506 { 507 struct ntb_plx_softc *sc = device_get_softc(dev); 508 509 return (sc->link ? 1 : 0); 510 } 511 512 static int 513 ntb_plx_peer_port_count(device_t dev) 514 { 515 516 return (1); 517 } 518 519 static int 520 ntb_plx_peer_port_number(device_t dev, int pidx) 521 { 522 struct ntb_plx_softc *sc = device_get_softc(dev); 523 524 if (pidx != 0) 525 return (-EINVAL); 526 527 return (sc->link ? 0 : 1); 528 } 529 530 static int 531 ntb_plx_peer_port_idx(device_t dev, int port) 532 { 533 int peer_port; 534 535 peer_port = ntb_plx_peer_port_number(dev, 0); 536 if (peer_port == -EINVAL || port != peer_port) 537 return (-EINVAL); 538 539 return (0); 540 } 541 542 static bool 543 ntb_plx_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) 544 { 545 uint16_t link; 546 547 link = pcie_read_config(dev, PCIER_LINK_STA, 2); 548 if (speed != NULL) 549 *speed = (link & PCIEM_LINK_STA_SPEED); 550 if (width != NULL) 551 *width = (link & PCIEM_LINK_STA_WIDTH) >> 4; 552 return ((link & PCIEM_LINK_STA_WIDTH) != 0); 553 } 554 555 static int 556 ntb_plx_link_enable(device_t dev, enum ntb_speed speed __unused, 557 enum ntb_width width __unused) 558 { 559 struct ntb_plx_softc *sc = device_get_softc(dev); 560 uint32_t reg, val; 561 562 /* The fact that we see the Link Interface means link is enabled. */ 563 if (sc->link) { 564 ntb_link_event(dev); 565 return (0); 566 } 567 568 reg = PLX_PORT_CONTROL(sc); 569 val = bus_read_4(sc->conf_res, reg); 570 if ((val & (1 << (sc->port & 7))) == 0) { 571 /* If already enabled, generate fake link event and exit. */ 572 ntb_link_event(dev); 573 return (0); 574 } 575 val &= ~(1 << (sc->port & 7)); 576 bus_write_4(sc->conf_res, reg, val); 577 return (0); 578 } 579 580 static int 581 ntb_plx_link_disable(device_t dev) 582 { 583 struct ntb_plx_softc *sc = device_get_softc(dev); 584 uint32_t reg, val; 585 586 /* Link disable for Link Interface would be suicidal. */ 587 if (sc->link) 588 return (0); 589 590 reg = PLX_PORT_CONTROL(sc); 591 val = bus_read_4(sc->conf_res, reg); 592 val |= (1 << (sc->port & 7)); 593 bus_write_4(sc->conf_res, reg, val); 594 return (0); 595 } 596 597 static bool 598 ntb_plx_link_enabled(device_t dev) 599 { 600 struct ntb_plx_softc *sc = device_get_softc(dev); 601 uint32_t reg, val; 602 603 /* The fact that we see the Link Interface means link is enabled. */ 604 if (sc->link) 605 return (TRUE); 606 607 reg = PLX_PORT_CONTROL(sc); 608 val = bus_read_4(sc->conf_res, reg); 609 return ((val & (1 << (sc->port & 7))) == 0); 610 } 611 612 static uint8_t 613 ntb_plx_mw_count(device_t dev) 614 { 615 struct ntb_plx_softc *sc = device_get_softc(dev); 616 uint8_t res; 617 618 res = sc->mw_count; 619 res += (1 << sc->split) - 1; 620 if (sc->b2b_mw >= 0 && sc->b2b_off == 0) 621 res--; /* B2B consumed whole window. */ 622 return (res); 623 } 624 625 static unsigned 626 ntb_plx_user_mw_to_idx(struct ntb_plx_softc *sc, unsigned uidx, unsigned *sp) 627 { 628 unsigned t; 629 630 t = 1 << sc->split; 631 if (uidx < t) { 632 *sp = uidx; 633 return (0); 634 } 635 *sp = 0; 636 return (uidx - (t - 1)); 637 } 638 639 static int 640 ntb_plx_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, 641 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 642 bus_addr_t *plimit) 643 { 644 struct ntb_plx_softc *sc = device_get_softc(dev); 645 struct ntb_plx_mw_info *mw; 646 size_t off, ss; 647 unsigned sp, split; 648 649 mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp); 650 if (mw_idx >= sc->mw_count) 651 return (EINVAL); 652 off = 0; 653 if (mw_idx == sc->b2b_mw) { 654 KASSERT(sc->b2b_off != 0, 655 ("user shouldn't get non-shared b2b mw")); 656 off = sc->b2b_off; 657 } 658 mw = &sc->mw_info[mw_idx]; 659 split = (mw->mw_bar == 2) ? sc->split : 0; 660 ss = (mw->mw_size - off) >> split; 661 662 /* Local to remote memory window parameters. */ 663 if (base != NULL) 664 *base = mw->mw_pbase + off + ss * sp; 665 if (vbase != NULL) 666 *vbase = mw->mw_vbase + off + ss * sp; 667 if (size != NULL) 668 *size = ss; 669 670 /* 671 * Remote to local memory window translation address alignment. 672 * Translation address has to be aligned to the BAR size, but A-LUT 673 * entries re-map addresses can be aligned to 1/128 or 1/256 of it. 674 * XXX: In B2B mode we can change BAR size (and so alignmet) live, 675 * but there is no way to report it here, so report safe value. 676 */ 677 if (align != NULL) { 678 if (sc->alut && mw->mw_bar == 2) 679 *align = (mw->mw_size - off) / 128 / sc->alut; 680 else 681 *align = mw->mw_size - off; 682 } 683 684 /* 685 * Remote to local memory window size alignment. 686 * The chip has no limit registers, but A-LUT, when available, allows 687 * access control with granularity of 1/128 or 1/256 of the BAR size. 688 * XXX: In B2B case we can change BAR size live, but there is no way 689 * to report it, so report half of the BAR size, that should be safe. 690 * In non-B2B case there is no control at all, so report the BAR size. 691 */ 692 if (align_size != NULL) { 693 if (sc->alut && mw->mw_bar == 2) 694 *align_size = (mw->mw_size - off) / 128 / sc->alut; 695 else if (sc->b2b_mw >= 0) 696 *align_size = (mw->mw_size - off) / 2; 697 else 698 *align_size = mw->mw_size - off; 699 } 700 701 /* Remote to local memory window translation address upper limit. */ 702 if (plimit != NULL) 703 *plimit = mw->mw_64bit ? BUS_SPACE_MAXADDR : 704 BUS_SPACE_MAXADDR_32BIT; 705 return (0); 706 } 707 708 static int 709 ntb_plx_mw_set_trans_internal(device_t dev, unsigned mw_idx) 710 { 711 struct ntb_plx_softc *sc = device_get_softc(dev); 712 struct ntb_plx_mw_info *mw; 713 uint64_t addr, eaddr, off, size, bsize, esize, val64; 714 uint32_t val; 715 unsigned i, sp, split; 716 717 mw = &sc->mw_info[mw_idx]; 718 off = (mw_idx == sc->b2b_mw) ? sc->b2b_off : 0; 719 split = (mw->mw_bar == 2) ? sc->split : 0; 720 721 /* Get BAR size. In case of split or B2RP we can't change it. */ 722 if (split || sc->b2b_mw < 0) { 723 bsize = mw->mw_size - off; 724 } else { 725 bsize = mw->splits[0].mw_xlat_size; 726 if (!powerof2(bsize)) 727 bsize = 1LL << flsll(bsize); 728 if (bsize > 0 && bsize < 1024 * 1024) 729 bsize = 1024 * 1024; 730 } 731 732 /* 733 * While for B2B we can set any BAR size on a link side, for shared 734 * window we can't go above preconfigured size due to BAR address 735 * alignment requirements. 736 */ 737 if ((off & (bsize - 1)) != 0) 738 return (EINVAL); 739 740 /* In B2B mode set Link Interface BAR size/address. */ 741 if (sc->b2b_mw >= 0 && mw->mw_64bit) { 742 val64 = 0; 743 if (bsize > 0) 744 val64 = (~(bsize - 1) & ~0xfffff); 745 val64 |= 0xc; 746 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4, val64); 747 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4 + 4, val64 >> 32); 748 749 val64 = 0x2000000000000000 * mw->mw_bar + off; 750 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar), val64); 751 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar) + 4, val64 >> 32); 752 } else if (sc->b2b_mw >= 0) { 753 val = 0; 754 if (bsize > 0) 755 val = (~(bsize - 1) & ~0xfffff); 756 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4, val); 757 758 val64 = 0x20000000 * mw->mw_bar + off; 759 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar), val64); 760 } 761 762 /* Set BARs address translation */ 763 addr = split ? UINT64_MAX : mw->splits[0].mw_xlat_addr; 764 if (mw->mw_64bit) { 765 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, addr); 766 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4 + 4, addr >> 32); 767 } else { 768 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, addr); 769 } 770 771 /* Configure and enable A-LUT if we need it. */ 772 size = split ? 0 : mw->splits[0].mw_xlat_size; 773 if (sc->alut && mw->mw_bar == 2 && (sc->split > 0 || 774 ((addr & (bsize - 1)) != 0 || size != bsize))) { 775 esize = bsize / (128 * sc->alut); 776 for (i = sp = 0; i < 128 * sc->alut; i++) { 777 if (i % (128 * sc->alut >> sc->split) == 0) { 778 eaddr = addr = mw->splits[sp].mw_xlat_addr; 779 size = mw->splits[sp++].mw_xlat_size; 780 } 781 val = sc->link ? 0 : 1; 782 if (sc->alut == 1) 783 val += 2 * sc->ntx; 784 val *= 0x1000 * sc->alut; 785 val += 0x38000 + i * 4 + (i >= 128 ? 0x0e00 : 0); 786 bus_write_4(sc->conf_res, val, eaddr); 787 bus_write_4(sc->conf_res, val + 0x400, eaddr >> 32); 788 bus_write_4(sc->conf_res, val + 0x800, 789 (eaddr < addr + size) ? 0x3 : 0); 790 eaddr += esize; 791 } 792 NTX_WRITE(sc, 0xc94, 0x10000000); 793 } else if (sc->alut && mw->mw_bar == 2) 794 NTX_WRITE(sc, 0xc94, 0); 795 796 return (0); 797 } 798 799 static int 800 ntb_plx_mw_set_trans(device_t dev, unsigned mw_idx, bus_addr_t addr, size_t size) 801 { 802 struct ntb_plx_softc *sc = device_get_softc(dev); 803 struct ntb_plx_mw_info *mw; 804 unsigned sp; 805 806 mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp); 807 if (mw_idx >= sc->mw_count) 808 return (EINVAL); 809 mw = &sc->mw_info[mw_idx]; 810 if (!mw->mw_64bit && 811 ((addr & UINT32_MAX) != addr || 812 ((addr + size) & UINT32_MAX) != (addr + size))) 813 return (ERANGE); 814 mw->splits[sp].mw_xlat_addr = addr; 815 mw->splits[sp].mw_xlat_size = size; 816 return (ntb_plx_mw_set_trans_internal(dev, mw_idx)); 817 } 818 819 static int 820 ntb_plx_mw_clear_trans(device_t dev, unsigned mw_idx) 821 { 822 823 return (ntb_plx_mw_set_trans(dev, mw_idx, 0, 0)); 824 } 825 826 static int 827 ntb_plx_mw_get_wc(device_t dev, unsigned mw_idx, vm_memattr_t *mode) 828 { 829 struct ntb_plx_softc *sc = device_get_softc(dev); 830 struct ntb_plx_mw_info *mw; 831 unsigned sp; 832 833 mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp); 834 if (mw_idx >= sc->mw_count) 835 return (EINVAL); 836 mw = &sc->mw_info[mw_idx]; 837 *mode = mw->splits[sp].mw_map_mode; 838 return (0); 839 } 840 841 static int 842 ntb_plx_mw_set_wc(device_t dev, unsigned mw_idx, vm_memattr_t mode) 843 { 844 struct ntb_plx_softc *sc = device_get_softc(dev); 845 struct ntb_plx_mw_info *mw; 846 uint64_t off, ss; 847 int rc; 848 unsigned sp, split; 849 850 mw_idx = ntb_plx_user_mw_to_idx(sc, mw_idx, &sp); 851 if (mw_idx >= sc->mw_count) 852 return (EINVAL); 853 mw = &sc->mw_info[mw_idx]; 854 if (mw->splits[sp].mw_map_mode == mode) 855 return (0); 856 857 off = 0; 858 if (mw_idx == sc->b2b_mw) { 859 KASSERT(sc->b2b_off != 0, 860 ("user shouldn't get non-shared b2b mw")); 861 off = sc->b2b_off; 862 } 863 864 split = (mw->mw_bar == 2) ? sc->split : 0; 865 ss = (mw->mw_size - off) >> split; 866 rc = pmap_change_attr((vm_offset_t)mw->mw_vbase + off + ss * sp, 867 ss, mode); 868 if (rc == 0) 869 mw->splits[sp].mw_map_mode = mode; 870 return (rc); 871 } 872 873 static uint8_t 874 ntb_plx_spad_count(device_t dev) 875 { 876 struct ntb_plx_softc *sc = device_get_softc(dev); 877 878 return (sc->spad_count1 + sc->spad_count2); 879 } 880 881 static int 882 ntb_plx_spad_write(device_t dev, unsigned int idx, uint32_t val) 883 { 884 struct ntb_plx_softc *sc = device_get_softc(dev); 885 u_int off; 886 887 if (idx >= sc->spad_count1 + sc->spad_count2) 888 return (EINVAL); 889 890 if (idx < sc->spad_count1) 891 off = sc->spad_off1 + idx * 4; 892 else 893 off = sc->spad_off2 + (idx - sc->spad_count1) * 4; 894 bus_write_4(sc->conf_res, off, val); 895 return (0); 896 } 897 898 static void 899 ntb_plx_spad_clear(device_t dev) 900 { 901 struct ntb_plx_softc *sc = device_get_softc(dev); 902 int i; 903 904 for (i = 0; i < sc->spad_count1 + sc->spad_count2; i++) 905 ntb_plx_spad_write(dev, i, 0); 906 } 907 908 static int 909 ntb_plx_spad_read(device_t dev, unsigned int idx, uint32_t *val) 910 { 911 struct ntb_plx_softc *sc = device_get_softc(dev); 912 u_int off; 913 914 if (idx >= sc->spad_count1 + sc->spad_count2) 915 return (EINVAL); 916 917 if (idx < sc->spad_count1) 918 off = sc->spad_off1 + idx * 4; 919 else 920 off = sc->spad_off2 + (idx - sc->spad_count1) * 4; 921 *val = bus_read_4(sc->conf_res, off); 922 return (0); 923 } 924 925 static int 926 ntb_plx_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) 927 { 928 struct ntb_plx_softc *sc = device_get_softc(dev); 929 u_int off; 930 931 if (idx >= sc->spad_count1 + sc->spad_count2) 932 return (EINVAL); 933 934 if (idx < sc->spad_count1) 935 off = sc->spad_offp1 + idx * 4; 936 else 937 off = sc->spad_offp2 + (idx - sc->spad_count1) * 4; 938 if (sc->b2b_mw >= 0) 939 bus_write_4(sc->mw_info[sc->b2b_mw].mw_res, off, val); 940 else 941 bus_write_4(sc->conf_res, off, val); 942 return (0); 943 } 944 945 static int 946 ntb_plx_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) 947 { 948 struct ntb_plx_softc *sc = device_get_softc(dev); 949 u_int off; 950 951 if (idx >= sc->spad_count1 + sc->spad_count2) 952 return (EINVAL); 953 954 if (idx < sc->spad_count1) 955 off = sc->spad_offp1 + idx * 4; 956 else 957 off = sc->spad_offp2 + (idx - sc->spad_count1) * 4; 958 if (sc->b2b_mw >= 0) 959 *val = bus_read_4(sc->mw_info[sc->b2b_mw].mw_res, off); 960 else 961 *val = bus_read_4(sc->conf_res, off); 962 return (0); 963 } 964 965 static uint64_t 966 ntb_plx_db_valid_mask(device_t dev) 967 { 968 969 return ((1LL << PLX_NUM_DB) - 1); 970 } 971 972 static int 973 ntb_plx_db_vector_count(device_t dev) 974 { 975 976 return (1); 977 } 978 979 static uint64_t 980 ntb_plx_db_vector_mask(device_t dev, uint32_t vector) 981 { 982 983 if (vector > 0) 984 return (0); 985 return ((1LL << PLX_NUM_DB) - 1); 986 } 987 988 static void 989 ntb_plx_db_clear(device_t dev, uint64_t bits) 990 { 991 struct ntb_plx_softc *sc = device_get_softc(dev); 992 993 NTX_WRITE(sc, sc->link ? 0xc60 : 0xc50, bits); 994 } 995 996 static void 997 ntb_plx_db_clear_mask(device_t dev, uint64_t bits) 998 { 999 struct ntb_plx_softc *sc = device_get_softc(dev); 1000 1001 NTX_WRITE(sc, sc->link ? 0xc68 : 0xc58, bits); 1002 } 1003 1004 static uint64_t 1005 ntb_plx_db_read(device_t dev) 1006 { 1007 struct ntb_plx_softc *sc = device_get_softc(dev); 1008 1009 return (NTX_READ(sc, sc->link ? 0xc5c : 0xc4c)); 1010 } 1011 1012 static void 1013 ntb_plx_db_set_mask(device_t dev, uint64_t bits) 1014 { 1015 struct ntb_plx_softc *sc = device_get_softc(dev); 1016 1017 NTX_WRITE(sc, sc->link ? 0xc64 : 0xc54, bits); 1018 } 1019 1020 static int 1021 ntb_plx_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) 1022 { 1023 struct ntb_plx_softc *sc = device_get_softc(dev); 1024 struct ntb_plx_mw_info *mw; 1025 1026 KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL")); 1027 1028 if (sc->b2b_mw >= 0) { 1029 mw = &sc->mw_info[sc->b2b_mw]; 1030 *db_addr = (uint64_t)mw->mw_pbase + PLX_NTX_BASE(sc) + 0xc4c; 1031 } else { 1032 *db_addr = rman_get_start(sc->conf_res) + PLX_NTX_BASE(sc); 1033 *db_addr += sc->link ? 0xc4c : 0xc5c; 1034 } 1035 *db_size = 4; 1036 return (0); 1037 } 1038 1039 static void 1040 ntb_plx_peer_db_set(device_t dev, uint64_t bit) 1041 { 1042 struct ntb_plx_softc *sc = device_get_softc(dev); 1043 1044 if (sc->b2b_mw >= 0) 1045 BNTX_WRITE(sc, 0xc4c, bit); 1046 else 1047 NTX_WRITE(sc, sc->link ? 0xc4c : 0xc5c, bit); 1048 } 1049 1050 static device_method_t ntb_plx_methods[] = { 1051 /* Device interface */ 1052 DEVMETHOD(device_probe, ntb_plx_probe), 1053 DEVMETHOD(device_attach, ntb_plx_attach), 1054 DEVMETHOD(device_detach, ntb_plx_detach), 1055 /* Bus interface */ 1056 DEVMETHOD(bus_child_location, ntb_child_location), 1057 DEVMETHOD(bus_print_child, ntb_print_child), 1058 DEVMETHOD(bus_get_dma_tag, ntb_get_dma_tag), 1059 /* NTB interface */ 1060 DEVMETHOD(ntb_port_number, ntb_plx_port_number), 1061 DEVMETHOD(ntb_peer_port_count, ntb_plx_peer_port_count), 1062 DEVMETHOD(ntb_peer_port_number, ntb_plx_peer_port_number), 1063 DEVMETHOD(ntb_peer_port_idx, ntb_plx_peer_port_idx), 1064 DEVMETHOD(ntb_link_is_up, ntb_plx_link_is_up), 1065 DEVMETHOD(ntb_link_enable, ntb_plx_link_enable), 1066 DEVMETHOD(ntb_link_disable, ntb_plx_link_disable), 1067 DEVMETHOD(ntb_link_enabled, ntb_plx_link_enabled), 1068 DEVMETHOD(ntb_mw_count, ntb_plx_mw_count), 1069 DEVMETHOD(ntb_mw_get_range, ntb_plx_mw_get_range), 1070 DEVMETHOD(ntb_mw_set_trans, ntb_plx_mw_set_trans), 1071 DEVMETHOD(ntb_mw_clear_trans, ntb_plx_mw_clear_trans), 1072 DEVMETHOD(ntb_mw_get_wc, ntb_plx_mw_get_wc), 1073 DEVMETHOD(ntb_mw_set_wc, ntb_plx_mw_set_wc), 1074 DEVMETHOD(ntb_spad_count, ntb_plx_spad_count), 1075 DEVMETHOD(ntb_spad_clear, ntb_plx_spad_clear), 1076 DEVMETHOD(ntb_spad_write, ntb_plx_spad_write), 1077 DEVMETHOD(ntb_spad_read, ntb_plx_spad_read), 1078 DEVMETHOD(ntb_peer_spad_write, ntb_plx_peer_spad_write), 1079 DEVMETHOD(ntb_peer_spad_read, ntb_plx_peer_spad_read), 1080 DEVMETHOD(ntb_db_valid_mask, ntb_plx_db_valid_mask), 1081 DEVMETHOD(ntb_db_vector_count, ntb_plx_db_vector_count), 1082 DEVMETHOD(ntb_db_vector_mask, ntb_plx_db_vector_mask), 1083 DEVMETHOD(ntb_db_clear, ntb_plx_db_clear), 1084 DEVMETHOD(ntb_db_clear_mask, ntb_plx_db_clear_mask), 1085 DEVMETHOD(ntb_db_read, ntb_plx_db_read), 1086 DEVMETHOD(ntb_db_set_mask, ntb_plx_db_set_mask), 1087 DEVMETHOD(ntb_peer_db_addr, ntb_plx_peer_db_addr), 1088 DEVMETHOD(ntb_peer_db_set, ntb_plx_peer_db_set), 1089 DEVMETHOD_END 1090 }; 1091 1092 static DEFINE_CLASS_0(ntb_hw, ntb_plx_driver, ntb_plx_methods, 1093 sizeof(struct ntb_plx_softc)); 1094 DRIVER_MODULE(ntb_hw_plx, pci, ntb_plx_driver, ntb_hw_devclass, NULL, NULL); 1095 MODULE_DEPEND(ntb_hw_plx, ntb, 1, 1, 1); 1096 MODULE_VERSION(ntb_hw_plx, 1); 1097