1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 #include <sys/cdefs.h> 35 #include "opt_inet.h" 36 37 #include <sys/ktr.h> 38 39 #include <linux/module.h> 40 #include <linux/moduleparam.h> 41 42 #include <rdma/ib_verbs.h> 43 44 #ifdef TCP_OFFLOAD 45 #include "iw_cxgbe.h" 46 47 void 48 c4iw_release_dev_ucontext(struct c4iw_rdev *rdev, 49 struct c4iw_dev_ucontext *uctx) 50 { 51 struct list_head *pos, *nxt; 52 struct c4iw_qid_list *entry; 53 54 mutex_lock(&uctx->lock); 55 list_for_each_safe(pos, nxt, &uctx->qpids) { 56 entry = list_entry(pos, struct c4iw_qid_list, entry); 57 list_del_init(&entry->entry); 58 if (!(entry->qid & rdev->qpmask)) { 59 c4iw_put_resource(&rdev->resource.qid_table, 60 entry->qid); 61 mutex_lock(&rdev->stats.lock); 62 rdev->stats.qid.cur -= rdev->qpmask + 1; 63 mutex_unlock(&rdev->stats.lock); 64 } 65 kfree(entry); 66 } 67 68 list_for_each_safe(pos, nxt, &uctx->cqids) { 69 entry = list_entry(pos, struct c4iw_qid_list, entry); 70 list_del_init(&entry->entry); 71 kfree(entry); 72 } 73 mutex_unlock(&uctx->lock); 74 } 75 76 void 77 c4iw_init_dev_ucontext(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx) 78 { 79 80 INIT_LIST_HEAD(&uctx->qpids); 81 INIT_LIST_HEAD(&uctx->cqids); 82 mutex_init(&uctx->lock); 83 } 84 85 static int 86 c4iw_rdev_open(struct c4iw_rdev *rdev) 87 { 88 struct adapter *sc = rdev->adap; 89 struct sge_params *sp = &sc->params.sge; 90 int rc; 91 unsigned short ucq_density = 1 << sp->iq_s_qpp; /* # of user CQs/page */ 92 unsigned short udb_density = 1 << sp->eq_s_qpp; /* # of user DB/page */ 93 94 95 c4iw_init_dev_ucontext(rdev, &rdev->uctx); 96 97 /* 98 * This implementation assumes udb_density == ucq_density! Eventually 99 * we might need to support this but for now fail the open. Also the 100 * cqid and qpid range must match for now. 101 */ 102 if (udb_density != ucq_density) { 103 device_printf(sc->dev, "unsupported udb/ucq densities %u/%u\n", 104 udb_density, ucq_density); 105 rc = -EINVAL; 106 goto err1; 107 } 108 if (sc->vres.qp.start != sc->vres.cq.start || 109 sc->vres.qp.size != sc->vres.cq.size) { 110 device_printf(sc->dev, "%s: unsupported qp and cq id ranges " 111 "qp start %u size %u cq start %u size %u\n", __func__, 112 sc->vres.qp.start, sc->vres.qp.size, sc->vres.cq.start, 113 sc->vres.cq.size); 114 rc = -EINVAL; 115 goto err1; 116 } 117 118 rdev->qpshift = PAGE_SHIFT - sp->eq_s_qpp; 119 rdev->qpmask = udb_density - 1; 120 rdev->cqshift = PAGE_SHIFT - sp->iq_s_qpp; 121 rdev->cqmask = ucq_density - 1; 122 123 if (c4iw_num_stags(rdev) == 0) { 124 rc = -EINVAL; 125 goto err1; 126 } 127 128 rdev->stats.pd.total = T4_MAX_NUM_PD; 129 rdev->stats.stag.total = sc->vres.stag.size; 130 rdev->stats.pbl.total = sc->vres.pbl.size; 131 rdev->stats.rqt.total = sc->vres.rq.size; 132 rdev->stats.qid.total = sc->vres.qp.size; 133 134 rc = c4iw_init_resource(rdev, T4_MAX_NUM_PD); 135 if (rc) { 136 device_printf(sc->dev, "error %d initializing resources\n", rc); 137 goto err1; 138 } 139 rc = c4iw_rqtpool_create(rdev); 140 if (rc) { 141 device_printf(sc->dev, "error %d initializing rqt pool\n", rc); 142 goto err2; 143 } 144 rdev->status_page = (struct t4_dev_status_page *) 145 __get_free_page(GFP_KERNEL); 146 if (!rdev->status_page) { 147 rc = -ENOMEM; 148 goto err3; 149 } 150 rdev->status_page->qp_start = sc->vres.qp.start; 151 rdev->status_page->qp_size = sc->vres.qp.size; 152 rdev->status_page->cq_start = sc->vres.cq.start; 153 rdev->status_page->cq_size = sc->vres.cq.size; 154 rdev->status_page->fid_base = sc->tids.ftid_base; 155 156 /* T5 and above devices don't need Doorbell recovery logic, 157 * so db_off is always set to '0'. 158 */ 159 rdev->status_page->db_off = 0; 160 161 rdev->status_page->wc_supported = rdev->adap->iwt.wc_en; 162 rdev->status_page->write_cmpl_supported = 163 rdev->adap->params.write_cmpl_support; 164 165 rdev->free_workq = create_singlethread_workqueue("iw_cxgb4_free"); 166 if (!rdev->free_workq) { 167 rc = -ENOMEM; 168 goto err4; 169 } 170 return (0); 171 err4: 172 free_page((unsigned long)rdev->status_page); 173 err3: 174 c4iw_rqtpool_destroy(rdev); 175 err2: 176 c4iw_destroy_resource(&rdev->resource); 177 err1: 178 return (rc); 179 } 180 181 static void c4iw_rdev_close(struct c4iw_rdev *rdev) 182 { 183 free_page((unsigned long)rdev->status_page); 184 c4iw_rqtpool_destroy(rdev); 185 c4iw_destroy_resource(&rdev->resource); 186 } 187 188 static void 189 c4iw_dealloc(struct c4iw_dev *iwsc) 190 { 191 192 c4iw_rdev_close(&iwsc->rdev); 193 WARN_ON(!xa_empty(&iwsc->cqs)); 194 WARN_ON(!xa_empty(&iwsc->qps)); 195 WARN_ON(!xa_empty(&iwsc->mrs)); 196 ib_dealloc_device(&iwsc->ibdev); 197 } 198 199 static struct c4iw_dev * 200 c4iw_alloc(struct adapter *sc) 201 { 202 struct c4iw_dev *iwsc; 203 int rc; 204 205 iwsc = (struct c4iw_dev *)ib_alloc_device(sizeof(*iwsc)); 206 if (iwsc == NULL) { 207 device_printf(sc->dev, "Cannot allocate ib device.\n"); 208 return (ERR_PTR(-ENOMEM)); 209 } 210 iwsc->rdev.adap = sc; 211 212 /* init various hw-queue params based on lld info */ 213 iwsc->rdev.hw_queue.t4_eq_status_entries = 214 sc->params.sge.spg_len / EQ_ESIZE; 215 iwsc->rdev.hw_queue.t4_max_eq_size = 65520; 216 iwsc->rdev.hw_queue.t4_max_iq_size = 65520; 217 iwsc->rdev.hw_queue.t4_max_rq_size = 8192 - 218 iwsc->rdev.hw_queue.t4_eq_status_entries - 1; 219 iwsc->rdev.hw_queue.t4_max_sq_size = 220 iwsc->rdev.hw_queue.t4_max_eq_size - 221 iwsc->rdev.hw_queue.t4_eq_status_entries - 1; 222 iwsc->rdev.hw_queue.t4_max_qp_depth = 223 iwsc->rdev.hw_queue.t4_max_rq_size; 224 iwsc->rdev.hw_queue.t4_max_cq_depth = 225 iwsc->rdev.hw_queue.t4_max_iq_size - 2; 226 iwsc->rdev.hw_queue.t4_stat_len = iwsc->rdev.adap->params.sge.spg_len; 227 228 /* As T5 and above devices support BAR2 kernel doorbells & WC, we map 229 * all of BAR2, for both User and Kernel Doorbells-GTS. 230 */ 231 iwsc->rdev.bar2_kva = (void __iomem *)((u64)iwsc->rdev.adap->udbs_base); 232 iwsc->rdev.bar2_pa = vtophys(iwsc->rdev.adap->udbs_base); 233 iwsc->rdev.bar2_len = rman_get_size(iwsc->rdev.adap->udbs_res); 234 235 rc = c4iw_rdev_open(&iwsc->rdev); 236 if (rc != 0) { 237 device_printf(sc->dev, "Unable to open CXIO rdev (%d)\n", rc); 238 ib_dealloc_device(&iwsc->ibdev); 239 return (ERR_PTR(rc)); 240 } 241 242 xa_init_flags(&iwsc->cqs, XA_FLAGS_LOCK_IRQ); 243 xa_init_flags(&iwsc->qps, XA_FLAGS_LOCK_IRQ); 244 xa_init_flags(&iwsc->mrs, XA_FLAGS_LOCK_IRQ); 245 spin_lock_init(&iwsc->lock); 246 mutex_init(&iwsc->rdev.stats.lock); 247 iwsc->avail_ird = iwsc->rdev.adap->params.max_ird_adapter; 248 249 return (iwsc); 250 } 251 252 static int c4iw_mod_load(void); 253 static int c4iw_mod_unload(void); 254 static int c4iw_activate(struct adapter *); 255 static int c4iw_deactivate(struct adapter *); 256 static int c4iw_stop(struct adapter *); 257 static int c4iw_restart(struct adapter *); 258 259 static struct uld_info c4iw_uld_info = { 260 .uld_activate = c4iw_activate, 261 .uld_deactivate = c4iw_deactivate, 262 .uld_stop = c4iw_stop, 263 .uld_restart = c4iw_restart, 264 }; 265 266 void _c4iw_free_wr_wait(struct kref *kref) 267 { 268 struct c4iw_wr_wait *wr_waitp; 269 270 wr_waitp = container_of(kref, struct c4iw_wr_wait, kref); 271 CTR(KTR_IW_CXGBE, "Free wr_wait %p", wr_waitp); 272 kfree(wr_waitp); 273 } 274 275 struct c4iw_wr_wait *c4iw_alloc_wr_wait(gfp_t gfp) 276 { 277 struct c4iw_wr_wait *wr_waitp; 278 279 wr_waitp = kzalloc(sizeof(*wr_waitp), gfp); 280 if (wr_waitp) { 281 kref_init(&wr_waitp->kref); 282 CTR(KTR_IW_CXGBE, "wr_wait %p", wr_waitp); 283 } 284 return wr_waitp; 285 } 286 287 static int 288 c4iw_activate(struct adapter *sc) 289 { 290 struct c4iw_dev *iwsc; 291 int rc; 292 293 ASSERT_SYNCHRONIZED_OP(sc); 294 295 if (is_t4(sc)) { 296 device_printf(sc->dev, "No iWARP support for T4 devices, " 297 "please install T5 or above devices.\n"); 298 return (ENOSYS); 299 } 300 301 if (uld_active(sc, ULD_IWARP)) { 302 KASSERT(0, ("%s: RDMA already enabled on sc %p", __func__, sc)); 303 return (0); 304 } 305 306 if (sc->rdmacaps == 0) { 307 device_printf(sc->dev, 308 "RDMA not supported or RDMA cap is not enabled.\n"); 309 return (ENOSYS); 310 } 311 312 iwsc = c4iw_alloc(sc); 313 if (IS_ERR(iwsc)) { 314 rc = -PTR_ERR(iwsc); 315 device_printf(sc->dev, "initialization failed: %d\n", rc); 316 return (rc); 317 } 318 319 sc->iwarp_softc = iwsc; 320 321 rc = -c4iw_register_device(iwsc); 322 if (rc) { 323 device_printf(sc->dev, "RDMA registration failed: %d\n", rc); 324 c4iw_dealloc(iwsc); 325 sc->iwarp_softc = NULL; 326 } 327 328 return (rc); 329 } 330 331 static int 332 c4iw_deactivate(struct adapter *sc) 333 { 334 struct c4iw_dev *iwsc = sc->iwarp_softc; 335 336 ASSERT_SYNCHRONIZED_OP(sc); 337 338 c4iw_unregister_device(iwsc); 339 c4iw_dealloc(iwsc); 340 sc->iwarp_softc = NULL; 341 342 return (0); 343 } 344 345 static int 346 c4iw_stop(struct adapter *sc) 347 { 348 struct c4iw_dev *iwsc = sc->iwarp_softc; 349 350 if (iwsc) { 351 struct ib_event event = {0}; 352 353 device_printf(sc->dev, "iWARP driver stopped.\n"); 354 iwsc->rdev.flags |= T4_IW_STOPPED; 355 event.event = IB_EVENT_DEVICE_FATAL; 356 event.device = &iwsc->ibdev; 357 ib_dispatch_event(&event); 358 } 359 360 return (0); 361 } 362 363 static int 364 c4iw_restart(struct adapter *sc) 365 { 366 struct c4iw_dev *iwsc = sc->iwarp_softc; 367 368 if (iwsc) { 369 device_printf(sc->dev, "iWARP driver restarted.\n"); 370 iwsc->rdev.flags &= ~T4_IW_STOPPED; 371 } 372 return (0); 373 } 374 375 static void 376 c4iw_activate_all(struct adapter *sc, void *arg __unused) 377 { 378 379 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4iwact") != 0) 380 return; 381 382 /* Activate iWARP if any port on this adapter has IFCAP_TOE enabled. */ 383 if (sc->offload_map && !uld_active(sc, ULD_IWARP)) 384 (void) t4_activate_uld(sc, ULD_IWARP); 385 386 end_synchronized_op(sc, 0); 387 } 388 389 static void 390 c4iw_deactivate_all(struct adapter *sc, void *arg __unused) 391 { 392 393 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4iwdea") != 0) 394 return; 395 396 if (uld_active(sc, ULD_IWARP)) 397 (void) t4_deactivate_uld(sc, ULD_IWARP); 398 399 end_synchronized_op(sc, 0); 400 } 401 402 static int 403 c4iw_mod_load(void) 404 { 405 int rc; 406 407 rc = -c4iw_cm_init(); 408 if (rc != 0) 409 return (rc); 410 411 rc = t4_register_uld(&c4iw_uld_info, ULD_IWARP); 412 if (rc != 0) { 413 c4iw_cm_term(); 414 return (rc); 415 } 416 417 t4_iterate(c4iw_activate_all, NULL); 418 419 return (rc); 420 } 421 422 static int 423 c4iw_mod_unload(void) 424 { 425 426 t4_iterate(c4iw_deactivate_all, NULL); 427 428 c4iw_cm_term(); 429 430 if (t4_unregister_uld(&c4iw_uld_info, ULD_IWARP) == EBUSY) 431 return (EBUSY); 432 433 return (0); 434 } 435 436 #endif 437 438 /* 439 * t4_tom won't load on kernels without TCP_OFFLOAD and this module's dependency 440 * on t4_tom ensures that it won't either. So we don't directly check for 441 * TCP_OFFLOAD here. 442 */ 443 static int 444 c4iw_modevent(module_t mod, int cmd, void *arg) 445 { 446 int rc = 0; 447 448 #ifdef TCP_OFFLOAD 449 switch (cmd) { 450 case MOD_LOAD: 451 rc = c4iw_mod_load(); 452 if (rc == 0) 453 printf("iw_cxgbe: Chelsio T5/T6 RDMA driver loaded.\n"); 454 break; 455 456 case MOD_UNLOAD: 457 rc = c4iw_mod_unload(); 458 break; 459 460 default: 461 rc = EINVAL; 462 } 463 #else 464 printf("t4_tom: compiled without TCP_OFFLOAD support.\n"); 465 rc = EOPNOTSUPP; 466 #endif 467 return (rc); 468 } 469 470 static moduledata_t c4iw_mod_data = { 471 "iw_cxgbe", 472 c4iw_modevent, 473 0 474 }; 475 476 MODULE_VERSION(iw_cxgbe, 1); 477 MODULE_DEPEND(iw_cxgbe, t4nex, 1, 1, 1); 478 MODULE_DEPEND(iw_cxgbe, t4_tom, 1, 1, 1); 479 MODULE_DEPEND(iw_cxgbe, ibcore, 1, 1, 1); 480 MODULE_DEPEND(iw_cxgbe, linuxkpi, 1, 1, 1); 481 DECLARE_MODULE(iw_cxgbe, c4iw_mod_data, SI_SUB_EXEC, SI_ORDER_ANY); 482