1 /*- 2 * Copyright (c) 2016-2017 Alexander Motin <mav@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/kernel.h> 32 #include <sys/systm.h> 33 #include <sys/bus.h> 34 #include <sys/rmlock.h> 35 #include <sys/malloc.h> 36 #include <sys/module.h> 37 #include <sys/sysctl.h> 38 39 #include "ntb.h" 40 41 devclass_t ntb_hw_devclass; 42 SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 43 44 struct ntb_child { 45 device_t dev; 46 int function; 47 int enabled; 48 int mwoff; 49 int mwcnt; 50 int spadoff; 51 int spadcnt; 52 int dboff; 53 int dbcnt; 54 uint64_t dbmask; 55 void *ctx; 56 const struct ntb_ctx_ops *ctx_ops; 57 struct rmlock ctx_lock; 58 struct ntb_child *next; 59 }; 60 61 int 62 ntb_register_device(device_t dev) 63 { 64 struct ntb_child **cpp = device_get_softc(dev); 65 struct ntb_child *nc; 66 int i, mw, mwu, mwt, spad, spadu, spadt, db, dbu, dbt; 67 char cfg[128] = ""; 68 char buf[32]; 69 char *n, *np, *c, *p, *name; 70 71 mwu = 0; 72 mwt = NTB_MW_COUNT(dev); 73 spadu = 0; 74 spadt = NTB_SPAD_COUNT(dev); 75 dbu = 0; 76 dbt = flsll(NTB_DB_VALID_MASK(dev)); 77 78 device_printf(dev, "%d memory windows, %d scratchpads, " 79 "%d doorbells\n", mwt, spadt, dbt); 80 81 snprintf(buf, sizeof(buf), "hint.%s.%d.config", device_get_name(dev), 82 device_get_unit(dev)); 83 TUNABLE_STR_FETCH(buf, cfg, sizeof(cfg)); 84 n = cfg; 85 i = 0; 86 while ((c = strsep(&n, ",")) != NULL) { 87 np = c; 88 name = strsep(&np, ":"); 89 if (name != NULL && name[0] == 0) 90 name = NULL; 91 p = strsep(&np, ":"); 92 mw = (p && p[0] != 0) ? strtol(p, NULL, 10) : mwt - mwu; 93 p = strsep(&np, ":"); 94 spad = (p && p[0] != 0) ? strtol(p, NULL, 10) : spadt - spadu; 95 db = (np && np[0] != 0) ? strtol(np, NULL, 10) : dbt - dbu; 96 97 if (mw > mwt - mwu || spad > spadt - spadu || db > dbt - dbu) { 98 device_printf(dev, "Not enough resources for config\n"); 99 break; 100 } 101 102 nc = malloc(sizeof(*nc), M_DEVBUF, M_WAITOK | M_ZERO); 103 nc->function = i; 104 nc->mwoff = mwu; 105 nc->mwcnt = mw; 106 nc->spadoff = spadu; 107 nc->spadcnt = spad; 108 nc->dboff = dbu; 109 nc->dbcnt = db; 110 nc->dbmask = (db == 0) ? 0 : (0xffffffffffffffff >> (64 - db)); 111 rm_init(&nc->ctx_lock, "ntb ctx"); 112 nc->dev = device_add_child(dev, name, -1); 113 if (nc->dev == NULL) { 114 ntb_unregister_device(dev); 115 return (ENOMEM); 116 } 117 device_set_ivars(nc->dev, nc); 118 *cpp = nc; 119 cpp = &nc->next; 120 121 if (bootverbose) { 122 device_printf(dev, "%d \"%s\":", i, name); 123 if (mw > 0) { 124 printf(" memory windows %d", mwu); 125 if (mw > 1) 126 printf("-%d", mwu + mw - 1); 127 } 128 if (spad > 0) { 129 printf(" scratchpads %d", spadu); 130 if (spad > 1) 131 printf("-%d", spadu + spad - 1); 132 } 133 if (db > 0) { 134 printf(" doorbells %d", dbu); 135 if (db > 1) 136 printf("-%d", dbu + db - 1); 137 } 138 printf("\n"); 139 } 140 141 mwu += mw; 142 spadu += spad; 143 dbu += db; 144 i++; 145 } 146 147 bus_generic_attach(dev); 148 return (0); 149 } 150 151 int 152 ntb_unregister_device(device_t dev) 153 { 154 struct ntb_child **cpp = device_get_softc(dev); 155 struct ntb_child *nc; 156 int error = 0; 157 158 while ((nc = *cpp) != NULL) { 159 *cpp = (*cpp)->next; 160 error = device_delete_child(dev, nc->dev); 161 if (error) 162 break; 163 rm_destroy(&nc->ctx_lock); 164 free(nc, M_DEVBUF); 165 } 166 return (error); 167 } 168 169 int 170 ntb_child_location_str(device_t dev, device_t child, char *buf, 171 size_t buflen) 172 { 173 struct ntb_child *nc = device_get_ivars(child); 174 175 snprintf(buf, buflen, "function=%d", nc->function); 176 return (0); 177 } 178 179 int 180 ntb_print_child(device_t dev, device_t child) 181 { 182 struct ntb_child *nc = device_get_ivars(child); 183 int retval; 184 185 retval = bus_print_child_header(dev, child); 186 if (nc->mwcnt > 0) { 187 printf(" mw %d", nc->mwoff); 188 if (nc->mwcnt > 1) 189 printf("-%d", nc->mwoff + nc->mwcnt - 1); 190 } 191 if (nc->spadcnt > 0) { 192 printf(" spad %d", nc->spadoff); 193 if (nc->spadcnt > 1) 194 printf("-%d", nc->spadoff + nc->spadcnt - 1); 195 } 196 if (nc->dbcnt > 0) { 197 printf(" db %d", nc->dboff); 198 if (nc->dbcnt > 1) 199 printf("-%d", nc->dboff + nc->dbcnt - 1); 200 } 201 retval += printf(" at function %d", nc->function); 202 retval += bus_print_child_domain(dev, child); 203 retval += bus_print_child_footer(dev, child); 204 205 return (retval); 206 } 207 208 void 209 ntb_link_event(device_t dev) 210 { 211 struct ntb_child **cpp = device_get_softc(dev); 212 struct ntb_child *nc; 213 struct rm_priotracker ctx_tracker; 214 enum ntb_speed speed; 215 enum ntb_width width; 216 217 if (NTB_LINK_IS_UP(dev, &speed, &width)) { 218 device_printf(dev, "Link is up (PCIe %d.x / x%d)\n", 219 (int)speed, (int)width); 220 } else { 221 device_printf(dev, "Link is down\n"); 222 } 223 for (nc = *cpp; nc != NULL; nc = nc->next) { 224 rm_rlock(&nc->ctx_lock, &ctx_tracker); 225 if (nc->ctx_ops != NULL && nc->ctx_ops->link_event != NULL) 226 nc->ctx_ops->link_event(nc->ctx); 227 rm_runlock(&nc->ctx_lock, &ctx_tracker); 228 } 229 } 230 231 void 232 ntb_db_event(device_t dev, uint32_t vec) 233 { 234 struct ntb_child **cpp = device_get_softc(dev); 235 struct ntb_child *nc; 236 struct rm_priotracker ctx_tracker; 237 238 for (nc = *cpp; nc != NULL; nc = nc->next) { 239 rm_rlock(&nc->ctx_lock, &ctx_tracker); 240 if (nc->ctx_ops != NULL && nc->ctx_ops->db_event != NULL) 241 nc->ctx_ops->db_event(nc->ctx, vec); 242 rm_runlock(&nc->ctx_lock, &ctx_tracker); 243 } 244 } 245 246 int 247 ntb_port_number(device_t ntb) 248 { 249 return (NTB_PORT_NUMBER(device_get_parent(ntb))); 250 } 251 252 int 253 ntb_peer_port_count(device_t ntb) 254 { 255 return (NTB_PEER_PORT_COUNT(device_get_parent(ntb))); 256 } 257 258 int 259 ntb_peer_port_number(device_t ntb, int pidx) 260 { 261 return (NTB_PEER_PORT_NUMBER(device_get_parent(ntb), pidx)); 262 } 263 264 int 265 ntb_peer_port_idx(device_t ntb, int port) 266 { 267 return (NTB_PEER_PORT_IDX(device_get_parent(ntb), port)); 268 } 269 270 bool 271 ntb_link_is_up(device_t ntb, enum ntb_speed *speed, enum ntb_width *width) 272 { 273 274 return (NTB_LINK_IS_UP(device_get_parent(ntb), speed, width)); 275 } 276 277 int 278 ntb_link_enable(device_t ntb, enum ntb_speed speed, enum ntb_width width) 279 { 280 struct ntb_child *nc = device_get_ivars(ntb); 281 struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev)); 282 struct ntb_child *nc1; 283 284 for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) { 285 if (nc1->enabled) { 286 nc->enabled = 1; 287 return (0); 288 } 289 } 290 nc->enabled = 1; 291 return (NTB_LINK_ENABLE(device_get_parent(ntb), speed, width)); 292 } 293 294 int 295 ntb_link_disable(device_t ntb) 296 { 297 struct ntb_child *nc = device_get_ivars(ntb); 298 struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev)); 299 struct ntb_child *nc1; 300 301 if (!nc->enabled) 302 return (0); 303 nc->enabled = 0; 304 for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) { 305 if (nc1->enabled) 306 return (0); 307 } 308 return (NTB_LINK_DISABLE(device_get_parent(ntb))); 309 } 310 311 bool 312 ntb_link_enabled(device_t ntb) 313 { 314 struct ntb_child *nc = device_get_ivars(ntb); 315 316 return (nc->enabled && NTB_LINK_ENABLED(device_get_parent(ntb))); 317 } 318 319 int 320 ntb_set_ctx(device_t ntb, void *ctx, const struct ntb_ctx_ops *ctx_ops) 321 { 322 struct ntb_child *nc = device_get_ivars(ntb); 323 324 if (ctx == NULL || ctx_ops == NULL) 325 return (EINVAL); 326 327 rm_wlock(&nc->ctx_lock); 328 if (nc->ctx_ops != NULL) { 329 rm_wunlock(&nc->ctx_lock); 330 return (EINVAL); 331 } 332 nc->ctx = ctx; 333 nc->ctx_ops = ctx_ops; 334 335 /* 336 * If applicaiton driver asks for link events, generate fake one now 337 * to let it update link state without races while we hold the lock. 338 */ 339 if (ctx_ops->link_event != NULL) 340 ctx_ops->link_event(ctx); 341 rm_wunlock(&nc->ctx_lock); 342 343 return (0); 344 } 345 346 void * 347 ntb_get_ctx(device_t ntb, const struct ntb_ctx_ops **ctx_ops) 348 { 349 struct ntb_child *nc = device_get_ivars(ntb); 350 351 KASSERT(nc->ctx != NULL && nc->ctx_ops != NULL, ("bogus")); 352 if (ctx_ops != NULL) 353 *ctx_ops = nc->ctx_ops; 354 return (nc->ctx); 355 } 356 357 void 358 ntb_clear_ctx(device_t ntb) 359 { 360 struct ntb_child *nc = device_get_ivars(ntb); 361 362 rm_wlock(&nc->ctx_lock); 363 nc->ctx = NULL; 364 nc->ctx_ops = NULL; 365 rm_wunlock(&nc->ctx_lock); 366 } 367 368 uint8_t 369 ntb_mw_count(device_t ntb) 370 { 371 struct ntb_child *nc = device_get_ivars(ntb); 372 373 return (nc->mwcnt); 374 } 375 376 int 377 ntb_mw_get_range(device_t ntb, unsigned mw_idx, vm_paddr_t *base, 378 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 379 bus_addr_t *plimit) 380 { 381 struct ntb_child *nc = device_get_ivars(ntb); 382 383 return (NTB_MW_GET_RANGE(device_get_parent(ntb), mw_idx + nc->mwoff, 384 base, vbase, size, align, align_size, plimit)); 385 } 386 387 int 388 ntb_mw_set_trans(device_t ntb, unsigned mw_idx, bus_addr_t addr, size_t size) 389 { 390 struct ntb_child *nc = device_get_ivars(ntb); 391 392 return (NTB_MW_SET_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff, 393 addr, size)); 394 } 395 396 int 397 ntb_mw_clear_trans(device_t ntb, unsigned mw_idx) 398 { 399 struct ntb_child *nc = device_get_ivars(ntb); 400 401 return (NTB_MW_CLEAR_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff)); 402 } 403 404 int 405 ntb_mw_get_wc(device_t ntb, unsigned mw_idx, vm_memattr_t *mode) 406 { 407 struct ntb_child *nc = device_get_ivars(ntb); 408 409 return (NTB_MW_GET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode)); 410 } 411 412 int 413 ntb_mw_set_wc(device_t ntb, unsigned mw_idx, vm_memattr_t mode) 414 { 415 struct ntb_child *nc = device_get_ivars(ntb); 416 417 return (NTB_MW_SET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode)); 418 } 419 420 uint8_t 421 ntb_spad_count(device_t ntb) 422 { 423 struct ntb_child *nc = device_get_ivars(ntb); 424 425 return (nc->spadcnt); 426 } 427 428 void 429 ntb_spad_clear(device_t ntb) 430 { 431 struct ntb_child *nc = device_get_ivars(ntb); 432 unsigned i; 433 434 for (i = 0; i < nc->spadcnt; i++) 435 NTB_SPAD_WRITE(device_get_parent(ntb), i + nc->spadoff, 0); 436 } 437 438 int 439 ntb_spad_write(device_t ntb, unsigned int idx, uint32_t val) 440 { 441 struct ntb_child *nc = device_get_ivars(ntb); 442 443 return (NTB_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, val)); 444 } 445 446 int 447 ntb_spad_read(device_t ntb, unsigned int idx, uint32_t *val) 448 { 449 struct ntb_child *nc = device_get_ivars(ntb); 450 451 return (NTB_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, val)); 452 } 453 454 int 455 ntb_peer_spad_write(device_t ntb, unsigned int idx, uint32_t val) 456 { 457 struct ntb_child *nc = device_get_ivars(ntb); 458 459 return (NTB_PEER_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, 460 val)); 461 } 462 463 int 464 ntb_peer_spad_read(device_t ntb, unsigned int idx, uint32_t *val) 465 { 466 struct ntb_child *nc = device_get_ivars(ntb); 467 468 return (NTB_PEER_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, 469 val)); 470 } 471 472 uint64_t 473 ntb_db_valid_mask(device_t ntb) 474 { 475 struct ntb_child *nc = device_get_ivars(ntb); 476 477 return (nc->dbmask); 478 } 479 480 int 481 ntb_db_vector_count(device_t ntb) 482 { 483 484 return (NTB_DB_VECTOR_COUNT(device_get_parent(ntb))); 485 } 486 487 uint64_t 488 ntb_db_vector_mask(device_t ntb, uint32_t vector) 489 { 490 struct ntb_child *nc = device_get_ivars(ntb); 491 492 return ((NTB_DB_VECTOR_MASK(device_get_parent(ntb), vector) 493 >> nc->dboff) & nc->dbmask); 494 } 495 496 int 497 ntb_peer_db_addr(device_t ntb, bus_addr_t *db_addr, vm_size_t *db_size) 498 { 499 500 return (NTB_PEER_DB_ADDR(device_get_parent(ntb), db_addr, db_size)); 501 } 502 503 void 504 ntb_db_clear(device_t ntb, uint64_t bits) 505 { 506 struct ntb_child *nc = device_get_ivars(ntb); 507 508 return (NTB_DB_CLEAR(device_get_parent(ntb), bits << nc->dboff)); 509 } 510 511 void 512 ntb_db_clear_mask(device_t ntb, uint64_t bits) 513 { 514 struct ntb_child *nc = device_get_ivars(ntb); 515 516 return (NTB_DB_CLEAR_MASK(device_get_parent(ntb), bits << nc->dboff)); 517 } 518 519 uint64_t 520 ntb_db_read(device_t ntb) 521 { 522 struct ntb_child *nc = device_get_ivars(ntb); 523 524 return ((NTB_DB_READ(device_get_parent(ntb)) >> nc->dboff) 525 & nc->dbmask); 526 } 527 528 void 529 ntb_db_set_mask(device_t ntb, uint64_t bits) 530 { 531 struct ntb_child *nc = device_get_ivars(ntb); 532 533 return (NTB_DB_SET_MASK(device_get_parent(ntb), bits << nc->dboff)); 534 } 535 536 void 537 ntb_peer_db_set(device_t ntb, uint64_t bits) 538 { 539 struct ntb_child *nc = device_get_ivars(ntb); 540 541 return (NTB_PEER_DB_SET(device_get_parent(ntb), bits << nc->dboff)); 542 } 543 544 MODULE_VERSION(ntb, 1); 545