1 /*- 2 * Copyright (c) 2016-2017 Alexander Motin <mav@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/param.h> 28 #include <sys/kernel.h> 29 #include <sys/systm.h> 30 #include <sys/bus.h> 31 #include <sys/rmlock.h> 32 #include <sys/malloc.h> 33 #include <sys/module.h> 34 #include <sys/sbuf.h> 35 #include <sys/sysctl.h> 36 37 #include "ntb.h" 38 39 SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 40 "NTB sysctls"); 41 42 struct ntb_child { 43 device_t dev; 44 int function; 45 int enabled; 46 int mwoff; 47 int mwcnt; 48 int spadoff; 49 int spadcnt; 50 int dboff; 51 int dbcnt; 52 uint64_t dbmask; 53 void *ctx; 54 const struct ntb_ctx_ops *ctx_ops; 55 struct rmlock ctx_lock; 56 struct ntb_child *next; 57 }; 58 59 int 60 ntb_register_device(device_t dev) 61 { 62 struct ntb_child **cpp = device_get_softc(dev); 63 struct ntb_child *nc; 64 int i, mw, mwu, mwt, spad, spadu, spadt, db, dbu, dbt; 65 char cfg[128] = ""; 66 char buf[32]; 67 char *n, *np, *c, *p, *name; 68 69 mwu = 0; 70 mwt = NTB_MW_COUNT(dev); 71 spadu = 0; 72 spadt = NTB_SPAD_COUNT(dev); 73 dbu = 0; 74 dbt = flsll(NTB_DB_VALID_MASK(dev)); 75 76 device_printf(dev, "%d memory windows, %d scratchpads, " 77 "%d doorbells\n", mwt, spadt, dbt); 78 79 snprintf(buf, sizeof(buf), "hint.%s.%d.config", device_get_name(dev), 80 device_get_unit(dev)); 81 TUNABLE_STR_FETCH(buf, cfg, sizeof(cfg)); 82 n = cfg; 83 i = 0; 84 while ((c = strsep(&n, ",")) != NULL) { 85 np = c; 86 name = strsep(&np, ":"); 87 if (name != NULL && name[0] == 0) 88 name = NULL; 89 p = strsep(&np, ":"); 90 mw = (p && p[0] != 0) ? strtol(p, NULL, 10) : mwt - mwu; 91 p = strsep(&np, ":"); 92 spad = (p && p[0] != 0) ? strtol(p, NULL, 10) : spadt - spadu; 93 db = (np && np[0] != 0) ? strtol(np, NULL, 10) : dbt - dbu; 94 95 if (mw > mwt - mwu || spad > spadt - spadu || db > dbt - dbu) { 96 device_printf(dev, "Not enough resources for config\n"); 97 break; 98 } 99 100 nc = malloc(sizeof(*nc), M_DEVBUF, M_WAITOK | M_ZERO); 101 nc->function = i; 102 nc->mwoff = mwu; 103 nc->mwcnt = mw; 104 nc->spadoff = spadu; 105 nc->spadcnt = spad; 106 nc->dboff = dbu; 107 nc->dbcnt = db; 108 nc->dbmask = (db == 0) ? 0 : (0xffffffffffffffff >> (64 - db)); 109 rm_init(&nc->ctx_lock, "ntb ctx"); 110 nc->dev = device_add_child(dev, name, -1); 111 if (nc->dev == NULL) { 112 ntb_unregister_device(dev); 113 return (ENOMEM); 114 } 115 device_set_ivars(nc->dev, nc); 116 *cpp = nc; 117 cpp = &nc->next; 118 119 if (bootverbose) { 120 device_printf(dev, "%d \"%s\":", i, name); 121 if (mw > 0) { 122 printf(" memory windows %d", mwu); 123 if (mw > 1) 124 printf("-%d", mwu + mw - 1); 125 } 126 if (spad > 0) { 127 printf(" scratchpads %d", spadu); 128 if (spad > 1) 129 printf("-%d", spadu + spad - 1); 130 } 131 if (db > 0) { 132 printf(" doorbells %d", dbu); 133 if (db > 1) 134 printf("-%d", dbu + db - 1); 135 } 136 printf("\n"); 137 } 138 139 mwu += mw; 140 spadu += spad; 141 dbu += db; 142 i++; 143 } 144 145 bus_generic_attach(dev); 146 return (0); 147 } 148 149 int 150 ntb_unregister_device(device_t dev) 151 { 152 struct ntb_child **cpp = device_get_softc(dev); 153 struct ntb_child *nc; 154 int error = 0; 155 156 while ((nc = *cpp) != NULL) { 157 *cpp = (*cpp)->next; 158 error = device_delete_child(dev, nc->dev); 159 if (error) 160 break; 161 rm_destroy(&nc->ctx_lock); 162 free(nc, M_DEVBUF); 163 } 164 return (error); 165 } 166 167 int 168 ntb_child_location(device_t dev, device_t child, struct sbuf *sb) 169 { 170 struct ntb_child *nc = device_get_ivars(child); 171 172 sbuf_printf(sb, "function=%d", nc->function); 173 return (0); 174 } 175 176 int 177 ntb_print_child(device_t dev, device_t child) 178 { 179 struct ntb_child *nc = device_get_ivars(child); 180 int retval; 181 182 retval = bus_print_child_header(dev, child); 183 if (nc->mwcnt > 0) { 184 printf(" mw %d", nc->mwoff); 185 if (nc->mwcnt > 1) 186 printf("-%d", nc->mwoff + nc->mwcnt - 1); 187 } 188 if (nc->spadcnt > 0) { 189 printf(" spad %d", nc->spadoff); 190 if (nc->spadcnt > 1) 191 printf("-%d", nc->spadoff + nc->spadcnt - 1); 192 } 193 if (nc->dbcnt > 0) { 194 printf(" db %d", nc->dboff); 195 if (nc->dbcnt > 1) 196 printf("-%d", nc->dboff + nc->dbcnt - 1); 197 } 198 retval += printf(" at function %d", nc->function); 199 retval += bus_print_child_domain(dev, child); 200 retval += bus_print_child_footer(dev, child); 201 202 return (retval); 203 } 204 205 bus_dma_tag_t 206 ntb_get_dma_tag(device_t bus, device_t child) 207 { 208 209 return (bus_get_dma_tag(bus)); 210 } 211 212 void 213 ntb_link_event(device_t dev) 214 { 215 struct ntb_child **cpp = device_get_softc(dev); 216 struct ntb_child *nc; 217 struct rm_priotracker ctx_tracker; 218 enum ntb_speed speed; 219 enum ntb_width width; 220 221 if (NTB_LINK_IS_UP(dev, &speed, &width)) { 222 device_printf(dev, "Link is up (PCIe %d.x / x%d)\n", 223 (int)speed, (int)width); 224 } else { 225 device_printf(dev, "Link is down\n"); 226 } 227 for (nc = *cpp; nc != NULL; nc = nc->next) { 228 rm_rlock(&nc->ctx_lock, &ctx_tracker); 229 if (nc->ctx_ops != NULL && nc->ctx_ops->link_event != NULL) 230 nc->ctx_ops->link_event(nc->ctx); 231 rm_runlock(&nc->ctx_lock, &ctx_tracker); 232 } 233 } 234 235 void 236 ntb_db_event(device_t dev, uint32_t vec) 237 { 238 struct ntb_child **cpp = device_get_softc(dev); 239 struct ntb_child *nc; 240 struct rm_priotracker ctx_tracker; 241 242 for (nc = *cpp; nc != NULL; nc = nc->next) { 243 rm_rlock(&nc->ctx_lock, &ctx_tracker); 244 if (nc->ctx_ops != NULL && nc->ctx_ops->db_event != NULL) 245 nc->ctx_ops->db_event(nc->ctx, vec); 246 rm_runlock(&nc->ctx_lock, &ctx_tracker); 247 } 248 } 249 250 int 251 ntb_port_number(device_t ntb) 252 { 253 return (NTB_PORT_NUMBER(device_get_parent(ntb))); 254 } 255 256 int 257 ntb_peer_port_count(device_t ntb) 258 { 259 return (NTB_PEER_PORT_COUNT(device_get_parent(ntb))); 260 } 261 262 int 263 ntb_peer_port_number(device_t ntb, int pidx) 264 { 265 return (NTB_PEER_PORT_NUMBER(device_get_parent(ntb), pidx)); 266 } 267 268 int 269 ntb_peer_port_idx(device_t ntb, int port) 270 { 271 return (NTB_PEER_PORT_IDX(device_get_parent(ntb), port)); 272 } 273 274 bool 275 ntb_link_is_up(device_t ntb, enum ntb_speed *speed, enum ntb_width *width) 276 { 277 278 return (NTB_LINK_IS_UP(device_get_parent(ntb), speed, width)); 279 } 280 281 int 282 ntb_link_enable(device_t ntb, enum ntb_speed speed, enum ntb_width width) 283 { 284 struct ntb_child *nc = device_get_ivars(ntb); 285 struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev)); 286 struct ntb_child *nc1; 287 288 for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) { 289 if (nc1->enabled) { 290 nc->enabled = 1; 291 return (0); 292 } 293 } 294 nc->enabled = 1; 295 return (NTB_LINK_ENABLE(device_get_parent(ntb), speed, width)); 296 } 297 298 int 299 ntb_link_disable(device_t ntb) 300 { 301 struct ntb_child *nc = device_get_ivars(ntb); 302 struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev)); 303 struct ntb_child *nc1; 304 305 if (!nc->enabled) 306 return (0); 307 nc->enabled = 0; 308 for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) { 309 if (nc1->enabled) 310 return (0); 311 } 312 return (NTB_LINK_DISABLE(device_get_parent(ntb))); 313 } 314 315 bool 316 ntb_link_enabled(device_t ntb) 317 { 318 struct ntb_child *nc = device_get_ivars(ntb); 319 320 return (nc->enabled && NTB_LINK_ENABLED(device_get_parent(ntb))); 321 } 322 323 int 324 ntb_set_ctx(device_t ntb, void *ctx, const struct ntb_ctx_ops *ctx_ops) 325 { 326 struct ntb_child *nc = device_get_ivars(ntb); 327 328 if (ctx == NULL || ctx_ops == NULL) 329 return (EINVAL); 330 331 rm_wlock(&nc->ctx_lock); 332 if (nc->ctx_ops != NULL) { 333 rm_wunlock(&nc->ctx_lock); 334 return (EINVAL); 335 } 336 nc->ctx = ctx; 337 nc->ctx_ops = ctx_ops; 338 339 /* 340 * If applicaiton driver asks for link events, generate fake one now 341 * to let it update link state without races while we hold the lock. 342 */ 343 if (ctx_ops->link_event != NULL) 344 ctx_ops->link_event(ctx); 345 rm_wunlock(&nc->ctx_lock); 346 347 return (0); 348 } 349 350 void * 351 ntb_get_ctx(device_t ntb, const struct ntb_ctx_ops **ctx_ops) 352 { 353 struct ntb_child *nc = device_get_ivars(ntb); 354 355 KASSERT(nc->ctx != NULL && nc->ctx_ops != NULL, ("bogus")); 356 if (ctx_ops != NULL) 357 *ctx_ops = nc->ctx_ops; 358 return (nc->ctx); 359 } 360 361 void 362 ntb_clear_ctx(device_t ntb) 363 { 364 struct ntb_child *nc = device_get_ivars(ntb); 365 366 rm_wlock(&nc->ctx_lock); 367 nc->ctx = NULL; 368 nc->ctx_ops = NULL; 369 rm_wunlock(&nc->ctx_lock); 370 } 371 372 uint8_t 373 ntb_mw_count(device_t ntb) 374 { 375 struct ntb_child *nc = device_get_ivars(ntb); 376 377 return (nc->mwcnt); 378 } 379 380 int 381 ntb_mw_get_range(device_t ntb, unsigned mw_idx, vm_paddr_t *base, 382 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 383 bus_addr_t *plimit) 384 { 385 struct ntb_child *nc = device_get_ivars(ntb); 386 387 return (NTB_MW_GET_RANGE(device_get_parent(ntb), mw_idx + nc->mwoff, 388 base, vbase, size, align, align_size, plimit)); 389 } 390 391 int 392 ntb_mw_set_trans(device_t ntb, unsigned mw_idx, bus_addr_t addr, size_t size) 393 { 394 struct ntb_child *nc = device_get_ivars(ntb); 395 396 return (NTB_MW_SET_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff, 397 addr, size)); 398 } 399 400 int 401 ntb_mw_clear_trans(device_t ntb, unsigned mw_idx) 402 { 403 struct ntb_child *nc = device_get_ivars(ntb); 404 405 return (NTB_MW_CLEAR_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff)); 406 } 407 408 int 409 ntb_mw_get_wc(device_t ntb, unsigned mw_idx, vm_memattr_t *mode) 410 { 411 struct ntb_child *nc = device_get_ivars(ntb); 412 413 return (NTB_MW_GET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode)); 414 } 415 416 int 417 ntb_mw_set_wc(device_t ntb, unsigned mw_idx, vm_memattr_t mode) 418 { 419 struct ntb_child *nc = device_get_ivars(ntb); 420 421 return (NTB_MW_SET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode)); 422 } 423 424 uint8_t 425 ntb_spad_count(device_t ntb) 426 { 427 struct ntb_child *nc = device_get_ivars(ntb); 428 429 return (nc->spadcnt); 430 } 431 432 void 433 ntb_spad_clear(device_t ntb) 434 { 435 struct ntb_child *nc = device_get_ivars(ntb); 436 unsigned i; 437 438 for (i = 0; i < nc->spadcnt; i++) 439 NTB_SPAD_WRITE(device_get_parent(ntb), i + nc->spadoff, 0); 440 } 441 442 int 443 ntb_spad_write(device_t ntb, unsigned int idx, uint32_t val) 444 { 445 struct ntb_child *nc = device_get_ivars(ntb); 446 447 return (NTB_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, val)); 448 } 449 450 int 451 ntb_spad_read(device_t ntb, unsigned int idx, uint32_t *val) 452 { 453 struct ntb_child *nc = device_get_ivars(ntb); 454 455 return (NTB_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, val)); 456 } 457 458 int 459 ntb_peer_spad_write(device_t ntb, unsigned int idx, uint32_t val) 460 { 461 struct ntb_child *nc = device_get_ivars(ntb); 462 463 return (NTB_PEER_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, 464 val)); 465 } 466 467 int 468 ntb_peer_spad_read(device_t ntb, unsigned int idx, uint32_t *val) 469 { 470 struct ntb_child *nc = device_get_ivars(ntb); 471 472 return (NTB_PEER_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, 473 val)); 474 } 475 476 uint64_t 477 ntb_db_valid_mask(device_t ntb) 478 { 479 struct ntb_child *nc = device_get_ivars(ntb); 480 481 return (nc->dbmask); 482 } 483 484 int 485 ntb_db_vector_count(device_t ntb) 486 { 487 488 return (NTB_DB_VECTOR_COUNT(device_get_parent(ntb))); 489 } 490 491 uint64_t 492 ntb_db_vector_mask(device_t ntb, uint32_t vector) 493 { 494 struct ntb_child *nc = device_get_ivars(ntb); 495 496 return ((NTB_DB_VECTOR_MASK(device_get_parent(ntb), vector) 497 >> nc->dboff) & nc->dbmask); 498 } 499 500 int 501 ntb_peer_db_addr(device_t ntb, bus_addr_t *db_addr, vm_size_t *db_size) 502 { 503 504 return (NTB_PEER_DB_ADDR(device_get_parent(ntb), db_addr, db_size)); 505 } 506 507 void 508 ntb_db_clear(device_t ntb, uint64_t bits) 509 { 510 struct ntb_child *nc = device_get_ivars(ntb); 511 512 return (NTB_DB_CLEAR(device_get_parent(ntb), bits << nc->dboff)); 513 } 514 515 void 516 ntb_db_clear_mask(device_t ntb, uint64_t bits) 517 { 518 struct ntb_child *nc = device_get_ivars(ntb); 519 520 return (NTB_DB_CLEAR_MASK(device_get_parent(ntb), bits << nc->dboff)); 521 } 522 523 uint64_t 524 ntb_db_read(device_t ntb) 525 { 526 struct ntb_child *nc = device_get_ivars(ntb); 527 528 return ((NTB_DB_READ(device_get_parent(ntb)) >> nc->dboff) 529 & nc->dbmask); 530 } 531 532 void 533 ntb_db_set_mask(device_t ntb, uint64_t bits) 534 { 535 struct ntb_child *nc = device_get_ivars(ntb); 536 537 return (NTB_DB_SET_MASK(device_get_parent(ntb), bits << nc->dboff)); 538 } 539 540 void 541 ntb_peer_db_set(device_t ntb, uint64_t bits) 542 { 543 struct ntb_child *nc = device_get_ivars(ntb); 544 545 return (NTB_PEER_DB_SET(device_get_parent(ntb), bits << nc->dboff)); 546 } 547 548 MODULE_VERSION(ntb, 1); 549