1 /* 2 * Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2005 Intel Corporation. All rights reserved. 4 * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. 5 * Copyright (c) 2009 HNR Consulting. All rights reserved. 6 * Copyright (c) 2014,2018 Intel Corporation. All rights reserved. 7 * 8 * This software is available to you under a choice of one of two 9 * licenses. You may choose to be licensed under the terms of the GNU 10 * General Public License (GPL) Version 2, available from the file 11 * COPYING in the main directory of this source tree, or the 12 * OpenIB.org BSD license below: 13 * 14 * Redistribution and use in source and binary forms, with or 15 * without modification, are permitted provided that the following 16 * conditions are met: 17 * 18 * - Redistributions of source code must retain the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer. 21 * 22 * - Redistributions in binary form must reproduce the above 23 * copyright notice, this list of conditions and the following 24 * disclaimer in the documentation and/or other materials 25 * provided with the distribution. 26 * 27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 34 * SOFTWARE. 35 * 36 */ 37 38 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 39 40 #include <linux/dma-mapping.h> 41 #include <linux/slab.h> 42 #include <linux/module.h> 43 #include <linux/security.h> 44 #include <linux/xarray.h> 45 #include <rdma/ib_cache.h> 46 47 #include "mad_priv.h" 48 #include "core_priv.h" 49 #include "mad_rmpp.h" 50 #include "smi.h" 51 #include "opa_smi.h" 52 #include "agent.h" 53 54 #define CREATE_TRACE_POINTS 55 #include <trace/events/ib_mad.h> 56 57 #ifdef CONFIG_TRACEPOINTS 58 static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr, 59 struct ib_mad_qp_info *qp_info, 60 struct trace_event_raw_ib_mad_send_template *entry) 61 { 62 struct ib_ud_wr *wr = &mad_send_wr->send_wr; 63 struct rdma_ah_attr attr = {}; 64 65 rdma_query_ah(wr->ah, &attr); 66 67 /* These are common */ 68 entry->sl = attr.sl; 69 entry->rqpn = wr->remote_qpn; 70 entry->rqkey = wr->remote_qkey; 71 entry->dlid = rdma_ah_get_dlid(&attr); 72 } 73 #endif 74 75 static int mad_sendq_size = IB_MAD_QP_SEND_SIZE; 76 static int mad_recvq_size = IB_MAD_QP_RECV_SIZE; 77 78 module_param_named(send_queue_size, mad_sendq_size, int, 0444); 79 MODULE_PARM_DESC(send_queue_size, "Size of send queue in number of work requests"); 80 module_param_named(recv_queue_size, mad_recvq_size, int, 0444); 81 MODULE_PARM_DESC(recv_queue_size, "Size of receive queue in number of work requests"); 82 83 static DEFINE_XARRAY_ALLOC1(ib_mad_clients); 84 static u32 ib_mad_client_next; 85 static struct list_head ib_mad_port_list; 86 87 /* Port list lock */ 88 static DEFINE_SPINLOCK(ib_mad_port_list_lock); 89 90 /* Forward declarations */ 91 static int method_in_use(struct ib_mad_mgmt_method_table **method, 92 struct ib_mad_reg_req *mad_reg_req); 93 static void remove_mad_reg_req(struct ib_mad_agent_private *priv); 94 static struct ib_mad_agent_private *find_mad_agent( 95 struct ib_mad_port_private *port_priv, 96 const struct ib_mad_hdr *mad); 97 static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info, 98 struct ib_mad_private *mad); 99 static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv); 100 static void timeout_sends(struct work_struct *work); 101 static void local_completions(struct work_struct *work); 102 static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req, 103 struct ib_mad_agent_private *agent_priv, 104 u8 mgmt_class); 105 static int add_oui_reg_req(struct ib_mad_reg_req *mad_reg_req, 106 struct ib_mad_agent_private *agent_priv); 107 static bool ib_mad_send_error(struct ib_mad_port_private *port_priv, 108 struct ib_wc *wc); 109 static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc); 110 111 /* 112 * Returns a ib_mad_port_private structure or NULL for a device/port 113 * Assumes ib_mad_port_list_lock is being held 114 */ 115 static inline struct ib_mad_port_private * 116 __ib_get_mad_port(struct ib_device *device, u32 port_num) 117 { 118 struct ib_mad_port_private *entry; 119 120 list_for_each_entry(entry, &ib_mad_port_list, port_list) { 121 if (entry->device == device && entry->port_num == port_num) 122 return entry; 123 } 124 return NULL; 125 } 126 127 /* 128 * Wrapper function to return a ib_mad_port_private structure or NULL 129 * for a device/port 130 */ 131 static inline struct ib_mad_port_private * 132 ib_get_mad_port(struct ib_device *device, u32 port_num) 133 { 134 struct ib_mad_port_private *entry; 135 unsigned long flags; 136 137 spin_lock_irqsave(&ib_mad_port_list_lock, flags); 138 entry = __ib_get_mad_port(device, port_num); 139 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags); 140 141 return entry; 142 } 143 144 static inline u8 convert_mgmt_class(u8 mgmt_class) 145 { 146 /* Alias IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE to 0 */ 147 return mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE ? 148 0 : mgmt_class; 149 } 150 151 static int get_spl_qp_index(enum ib_qp_type qp_type) 152 { 153 switch (qp_type) { 154 case IB_QPT_SMI: 155 return 0; 156 case IB_QPT_GSI: 157 return 1; 158 default: 159 return -1; 160 } 161 } 162 163 static int vendor_class_index(u8 mgmt_class) 164 { 165 return mgmt_class - IB_MGMT_CLASS_VENDOR_RANGE2_START; 166 } 167 168 static int is_vendor_class(u8 mgmt_class) 169 { 170 if ((mgmt_class < IB_MGMT_CLASS_VENDOR_RANGE2_START) || 171 (mgmt_class > IB_MGMT_CLASS_VENDOR_RANGE2_END)) 172 return 0; 173 return 1; 174 } 175 176 static int is_vendor_oui(char *oui) 177 { 178 if (oui[0] || oui[1] || oui[2]) 179 return 1; 180 return 0; 181 } 182 183 static int is_vendor_method_in_use( 184 struct ib_mad_mgmt_vendor_class *vendor_class, 185 struct ib_mad_reg_req *mad_reg_req) 186 { 187 struct ib_mad_mgmt_method_table *method; 188 int i; 189 190 for (i = 0; i < MAX_MGMT_OUI; i++) { 191 if (!memcmp(vendor_class->oui[i], mad_reg_req->oui, 3)) { 192 method = vendor_class->method_table[i]; 193 if (method) { 194 if (method_in_use(&method, mad_reg_req)) 195 return 1; 196 else 197 break; 198 } 199 } 200 } 201 return 0; 202 } 203 204 int ib_response_mad(const struct ib_mad_hdr *hdr) 205 { 206 return ((hdr->method & IB_MGMT_METHOD_RESP) || 207 (hdr->method == IB_MGMT_METHOD_TRAP_REPRESS) || 208 ((hdr->mgmt_class == IB_MGMT_CLASS_BM) && 209 (hdr->attr_mod & IB_BM_ATTR_MOD_RESP))); 210 } 211 EXPORT_SYMBOL(ib_response_mad); 212 213 #define SOL_FC_MAX_DEFAULT_FRAC 4 214 #define SOL_FC_MAX_SA_FRAC 32 215 216 static int get_sol_fc_max_outstanding(struct ib_mad_reg_req *mad_reg_req) 217 { 218 if (!mad_reg_req) 219 /* Send only agent */ 220 return mad_recvq_size / SOL_FC_MAX_DEFAULT_FRAC; 221 222 switch (mad_reg_req->mgmt_class) { 223 case IB_MGMT_CLASS_CM: 224 return mad_recvq_size / SOL_FC_MAX_DEFAULT_FRAC; 225 case IB_MGMT_CLASS_SUBN_ADM: 226 return mad_recvq_size / SOL_FC_MAX_SA_FRAC; 227 case IB_MGMT_CLASS_SUBN_LID_ROUTED: 228 case IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE: 229 return min(mad_recvq_size, IB_MAD_QP_RECV_SIZE) / 230 SOL_FC_MAX_DEFAULT_FRAC; 231 default: 232 return 0; 233 } 234 } 235 236 /* 237 * ib_register_mad_agent - Register to send/receive MADs 238 * 239 * Context: Process context. 240 */ 241 struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device, 242 u32 port_num, 243 enum ib_qp_type qp_type, 244 struct ib_mad_reg_req *mad_reg_req, 245 u8 rmpp_version, 246 ib_mad_send_handler send_handler, 247 ib_mad_recv_handler recv_handler, 248 void *context, 249 u32 registration_flags) 250 { 251 struct ib_mad_port_private *port_priv; 252 struct ib_mad_agent *ret = ERR_PTR(-EINVAL); 253 struct ib_mad_agent_private *mad_agent_priv; 254 struct ib_mad_reg_req *reg_req = NULL; 255 struct ib_mad_mgmt_class_table *class; 256 struct ib_mad_mgmt_vendor_class_table *vendor; 257 struct ib_mad_mgmt_vendor_class *vendor_class; 258 struct ib_mad_mgmt_method_table *method; 259 int ret2, qpn; 260 u8 mgmt_class, vclass; 261 262 if ((qp_type == IB_QPT_SMI && !rdma_cap_ib_smi(device, port_num)) || 263 (qp_type == IB_QPT_GSI && !rdma_cap_ib_cm(device, port_num))) 264 return ERR_PTR(-EPROTONOSUPPORT); 265 266 /* Validate parameters */ 267 qpn = get_spl_qp_index(qp_type); 268 if (qpn == -1) { 269 dev_dbg_ratelimited(&device->dev, "%s: invalid QP Type %d\n", 270 __func__, qp_type); 271 goto error1; 272 } 273 274 if (rmpp_version && rmpp_version != IB_MGMT_RMPP_VERSION) { 275 dev_dbg_ratelimited(&device->dev, 276 "%s: invalid RMPP Version %u\n", 277 __func__, rmpp_version); 278 goto error1; 279 } 280 281 /* Validate MAD registration request if supplied */ 282 if (mad_reg_req) { 283 if (mad_reg_req->mgmt_class_version >= MAX_MGMT_VERSION) { 284 dev_dbg_ratelimited(&device->dev, 285 "%s: invalid Class Version %u\n", 286 __func__, 287 mad_reg_req->mgmt_class_version); 288 goto error1; 289 } 290 if (!recv_handler) { 291 dev_dbg_ratelimited(&device->dev, 292 "%s: no recv_handler\n", __func__); 293 goto error1; 294 } 295 if (mad_reg_req->mgmt_class >= MAX_MGMT_CLASS) { 296 /* 297 * IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE is the only 298 * one in this range currently allowed 299 */ 300 if (mad_reg_req->mgmt_class != 301 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) { 302 dev_dbg_ratelimited(&device->dev, 303 "%s: Invalid Mgmt Class 0x%x\n", 304 __func__, mad_reg_req->mgmt_class); 305 goto error1; 306 } 307 } else if (mad_reg_req->mgmt_class == 0) { 308 /* 309 * Class 0 is reserved in IBA and is used for 310 * aliasing of IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE 311 */ 312 dev_dbg_ratelimited(&device->dev, 313 "%s: Invalid Mgmt Class 0\n", 314 __func__); 315 goto error1; 316 } else if (is_vendor_class(mad_reg_req->mgmt_class)) { 317 /* 318 * If class is in "new" vendor range, 319 * ensure supplied OUI is not zero 320 */ 321 if (!is_vendor_oui(mad_reg_req->oui)) { 322 dev_dbg_ratelimited(&device->dev, 323 "%s: No OUI specified for class 0x%x\n", 324 __func__, 325 mad_reg_req->mgmt_class); 326 goto error1; 327 } 328 } 329 /* Make sure class supplied is consistent with RMPP */ 330 if (!ib_is_mad_class_rmpp(mad_reg_req->mgmt_class)) { 331 if (rmpp_version) { 332 dev_dbg_ratelimited(&device->dev, 333 "%s: RMPP version for non-RMPP class 0x%x\n", 334 __func__, mad_reg_req->mgmt_class); 335 goto error1; 336 } 337 } 338 339 /* Make sure class supplied is consistent with QP type */ 340 if (qp_type == IB_QPT_SMI) { 341 if ((mad_reg_req->mgmt_class != 342 IB_MGMT_CLASS_SUBN_LID_ROUTED) && 343 (mad_reg_req->mgmt_class != 344 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) { 345 dev_dbg_ratelimited(&device->dev, 346 "%s: Invalid SM QP type: class 0x%x\n", 347 __func__, mad_reg_req->mgmt_class); 348 goto error1; 349 } 350 } else { 351 if ((mad_reg_req->mgmt_class == 352 IB_MGMT_CLASS_SUBN_LID_ROUTED) || 353 (mad_reg_req->mgmt_class == 354 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) { 355 dev_dbg_ratelimited(&device->dev, 356 "%s: Invalid GS QP type: class 0x%x\n", 357 __func__, mad_reg_req->mgmt_class); 358 goto error1; 359 } 360 } 361 } else { 362 /* No registration request supplied */ 363 if (!send_handler) 364 goto error1; 365 if (registration_flags & IB_MAD_USER_RMPP) 366 goto error1; 367 } 368 369 /* Validate device and port */ 370 port_priv = ib_get_mad_port(device, port_num); 371 if (!port_priv) { 372 dev_dbg_ratelimited(&device->dev, "%s: Invalid port %u\n", 373 __func__, port_num); 374 ret = ERR_PTR(-ENODEV); 375 goto error1; 376 } 377 378 /* Verify the QP requested is supported. For example, Ethernet devices 379 * will not have QP0. 380 */ 381 if (!port_priv->qp_info[qpn].qp) { 382 dev_dbg_ratelimited(&device->dev, "%s: QP %d not supported\n", 383 __func__, qpn); 384 ret = ERR_PTR(-EPROTONOSUPPORT); 385 goto error1; 386 } 387 388 /* Allocate structures */ 389 mad_agent_priv = kzalloc_obj(*mad_agent_priv); 390 if (!mad_agent_priv) { 391 ret = ERR_PTR(-ENOMEM); 392 goto error1; 393 } 394 395 if (mad_reg_req) { 396 reg_req = kmemdup(mad_reg_req, sizeof *reg_req, GFP_KERNEL); 397 if (!reg_req) { 398 ret = ERR_PTR(-ENOMEM); 399 goto error3; 400 } 401 } 402 403 /* Now, fill in the various structures */ 404 mad_agent_priv->qp_info = &port_priv->qp_info[qpn]; 405 mad_agent_priv->reg_req = reg_req; 406 mad_agent_priv->agent.rmpp_version = rmpp_version; 407 mad_agent_priv->agent.device = device; 408 mad_agent_priv->agent.recv_handler = recv_handler; 409 mad_agent_priv->agent.send_handler = send_handler; 410 mad_agent_priv->agent.context = context; 411 mad_agent_priv->agent.qp = port_priv->qp_info[qpn].qp; 412 mad_agent_priv->agent.port_num = port_num; 413 mad_agent_priv->agent.flags = registration_flags; 414 spin_lock_init(&mad_agent_priv->lock); 415 INIT_LIST_HEAD(&mad_agent_priv->send_list); 416 INIT_LIST_HEAD(&mad_agent_priv->wait_list); 417 INIT_LIST_HEAD(&mad_agent_priv->rmpp_list); 418 INIT_LIST_HEAD(&mad_agent_priv->backlog_list); 419 INIT_DELAYED_WORK(&mad_agent_priv->timed_work, timeout_sends); 420 INIT_LIST_HEAD(&mad_agent_priv->local_list); 421 INIT_WORK(&mad_agent_priv->local_work, local_completions); 422 refcount_set(&mad_agent_priv->refcount, 1); 423 init_completion(&mad_agent_priv->comp); 424 mad_agent_priv->sol_fc_send_count = 0; 425 mad_agent_priv->sol_fc_wait_count = 0; 426 mad_agent_priv->sol_fc_max = 427 recv_handler ? get_sol_fc_max_outstanding(mad_reg_req) : 0; 428 429 ret2 = ib_mad_agent_security_setup(&mad_agent_priv->agent, qp_type); 430 if (ret2) { 431 ret = ERR_PTR(ret2); 432 goto error4; 433 } 434 435 /* 436 * The mlx4 driver uses the top byte to distinguish which virtual 437 * function generated the MAD, so we must avoid using it. 438 */ 439 ret2 = xa_alloc_cyclic(&ib_mad_clients, &mad_agent_priv->agent.hi_tid, 440 mad_agent_priv, XA_LIMIT(0, (1 << 24) - 1), 441 &ib_mad_client_next, GFP_KERNEL); 442 if (ret2 < 0) { 443 ret = ERR_PTR(ret2); 444 goto error5; 445 } 446 447 /* 448 * Make sure MAD registration (if supplied) 449 * is non overlapping with any existing ones 450 */ 451 spin_lock_irq(&port_priv->reg_lock); 452 if (mad_reg_req) { 453 mgmt_class = convert_mgmt_class(mad_reg_req->mgmt_class); 454 if (!is_vendor_class(mgmt_class)) { 455 class = port_priv->version[mad_reg_req-> 456 mgmt_class_version].class; 457 if (class) { 458 method = class->method_table[mgmt_class]; 459 if (method) { 460 if (method_in_use(&method, 461 mad_reg_req)) 462 goto error6; 463 } 464 } 465 ret2 = add_nonoui_reg_req(mad_reg_req, mad_agent_priv, 466 mgmt_class); 467 } else { 468 /* "New" vendor class range */ 469 vendor = port_priv->version[mad_reg_req-> 470 mgmt_class_version].vendor; 471 if (vendor) { 472 vclass = vendor_class_index(mgmt_class); 473 vendor_class = vendor->vendor_class[vclass]; 474 if (vendor_class) { 475 if (is_vendor_method_in_use( 476 vendor_class, 477 mad_reg_req)) 478 goto error6; 479 } 480 } 481 ret2 = add_oui_reg_req(mad_reg_req, mad_agent_priv); 482 } 483 if (ret2) { 484 ret = ERR_PTR(ret2); 485 goto error6; 486 } 487 } 488 spin_unlock_irq(&port_priv->reg_lock); 489 490 trace_ib_mad_create_agent(mad_agent_priv); 491 return &mad_agent_priv->agent; 492 error6: 493 spin_unlock_irq(&port_priv->reg_lock); 494 xa_erase(&ib_mad_clients, mad_agent_priv->agent.hi_tid); 495 error5: 496 ib_mad_agent_security_cleanup(&mad_agent_priv->agent); 497 error4: 498 kfree(reg_req); 499 error3: 500 kfree(mad_agent_priv); 501 error1: 502 return ret; 503 } 504 EXPORT_SYMBOL(ib_register_mad_agent); 505 506 static inline void deref_mad_agent(struct ib_mad_agent_private *mad_agent_priv) 507 { 508 if (refcount_dec_and_test(&mad_agent_priv->refcount)) 509 complete(&mad_agent_priv->comp); 510 } 511 512 static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv) 513 { 514 struct ib_mad_port_private *port_priv; 515 516 /* Note that we could still be handling received MADs */ 517 trace_ib_mad_unregister_agent(mad_agent_priv); 518 519 /* 520 * Canceling all sends results in dropping received response 521 * MADs, preventing us from queuing additional work 522 */ 523 cancel_mads(mad_agent_priv); 524 port_priv = mad_agent_priv->qp_info->port_priv; 525 cancel_delayed_work(&mad_agent_priv->timed_work); 526 527 spin_lock_irq(&port_priv->reg_lock); 528 remove_mad_reg_req(mad_agent_priv); 529 spin_unlock_irq(&port_priv->reg_lock); 530 xa_erase(&ib_mad_clients, mad_agent_priv->agent.hi_tid); 531 532 flush_workqueue(port_priv->wq); 533 534 deref_mad_agent(mad_agent_priv); 535 wait_for_completion(&mad_agent_priv->comp); 536 ib_cancel_rmpp_recvs(mad_agent_priv); 537 538 ib_mad_agent_security_cleanup(&mad_agent_priv->agent); 539 540 kfree(mad_agent_priv->reg_req); 541 kfree_rcu(mad_agent_priv, rcu); 542 } 543 544 /* 545 * ib_unregister_mad_agent - Unregisters a client from using MAD services 546 * 547 * Context: Process context. 548 */ 549 void ib_unregister_mad_agent(struct ib_mad_agent *mad_agent) 550 { 551 struct ib_mad_agent_private *mad_agent_priv; 552 553 mad_agent_priv = container_of(mad_agent, 554 struct ib_mad_agent_private, 555 agent); 556 unregister_mad_agent(mad_agent_priv); 557 } 558 EXPORT_SYMBOL(ib_unregister_mad_agent); 559 560 static void dequeue_mad(struct ib_mad_list_head *mad_list) 561 { 562 struct ib_mad_queue *mad_queue; 563 unsigned long flags; 564 565 mad_queue = mad_list->mad_queue; 566 spin_lock_irqsave(&mad_queue->lock, flags); 567 list_del(&mad_list->list); 568 mad_queue->count--; 569 spin_unlock_irqrestore(&mad_queue->lock, flags); 570 } 571 572 static void build_smp_wc(struct ib_qp *qp, struct ib_cqe *cqe, u16 slid, 573 u16 pkey_index, u32 port_num, struct ib_wc *wc) 574 { 575 memset(wc, 0, sizeof *wc); 576 wc->wr_cqe = cqe; 577 wc->status = IB_WC_SUCCESS; 578 wc->opcode = IB_WC_RECV; 579 wc->pkey_index = pkey_index; 580 wc->byte_len = sizeof(struct ib_mad) + sizeof(struct ib_grh); 581 wc->src_qp = IB_QP0; 582 wc->qp = qp; 583 wc->slid = slid; 584 wc->sl = 0; 585 wc->dlid_path_bits = 0; 586 wc->port_num = port_num; 587 } 588 589 static size_t mad_priv_size(const struct ib_mad_private *mp) 590 { 591 return sizeof(struct ib_mad_private) + mp->mad_size; 592 } 593 594 static struct ib_mad_private *alloc_mad_private(size_t mad_size, gfp_t flags) 595 { 596 size_t size = sizeof(struct ib_mad_private) + mad_size; 597 struct ib_mad_private *ret = kzalloc(size, flags); 598 599 if (ret) 600 ret->mad_size = mad_size; 601 602 return ret; 603 } 604 605 static size_t port_mad_size(const struct ib_mad_port_private *port_priv) 606 { 607 return rdma_max_mad_size(port_priv->device, port_priv->port_num); 608 } 609 610 static size_t mad_priv_dma_size(const struct ib_mad_private *mp) 611 { 612 return sizeof(struct ib_grh) + mp->mad_size; 613 } 614 615 /* 616 * Return 0 if SMP is to be sent 617 * Return 1 if SMP was consumed locally (whether or not solicited) 618 * Return < 0 if error 619 */ 620 static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv, 621 struct ib_mad_send_wr_private *mad_send_wr) 622 { 623 int ret = 0; 624 struct ib_smp *smp = mad_send_wr->send_buf.mad; 625 struct opa_smp *opa_smp = (struct opa_smp *)smp; 626 unsigned long flags; 627 struct ib_mad_local_private *local; 628 struct ib_mad_private *mad_priv; 629 struct ib_mad_port_private *port_priv; 630 struct ib_mad_agent_private *recv_mad_agent = NULL; 631 struct ib_device *device = mad_agent_priv->agent.device; 632 u32 port_num; 633 struct ib_wc mad_wc; 634 struct ib_ud_wr *send_wr = &mad_send_wr->send_wr; 635 size_t mad_size = port_mad_size(mad_agent_priv->qp_info->port_priv); 636 u16 out_mad_pkey_index = 0; 637 u16 drslid; 638 bool opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device, 639 mad_agent_priv->qp_info->port_priv->port_num); 640 641 if (rdma_cap_ib_switch(device) && 642 smp->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) 643 port_num = send_wr->port_num; 644 else 645 port_num = mad_agent_priv->agent.port_num; 646 647 /* 648 * Directed route handling starts if the initial LID routed part of 649 * a request or the ending LID routed part of a response is empty. 650 * If we are at the start of the LID routed part, don't update the 651 * hop_ptr or hop_cnt. See section 14.2.2, Vol 1 IB spec. 652 */ 653 if (opa && smp->class_version == OPA_SM_CLASS_VERSION) { 654 u32 opa_drslid; 655 656 trace_ib_mad_handle_out_opa_smi(opa_smp); 657 658 if ((opa_get_smp_direction(opa_smp) 659 ? opa_smp->route.dr.dr_dlid : opa_smp->route.dr.dr_slid) == 660 OPA_LID_PERMISSIVE && 661 opa_smi_handle_dr_smp_send(opa_smp, 662 rdma_cap_ib_switch(device), 663 port_num) == IB_SMI_DISCARD) { 664 ret = -EINVAL; 665 dev_err(&device->dev, "OPA Invalid directed route\n"); 666 goto out; 667 } 668 opa_drslid = be32_to_cpu(opa_smp->route.dr.dr_slid); 669 if (opa_drslid != be32_to_cpu(OPA_LID_PERMISSIVE) && 670 opa_drslid & 0xffff0000) { 671 ret = -EINVAL; 672 dev_err(&device->dev, "OPA Invalid dr_slid 0x%x\n", 673 opa_drslid); 674 goto out; 675 } 676 drslid = (u16)(opa_drslid & 0x0000ffff); 677 678 /* Check to post send on QP or process locally */ 679 if (opa_smi_check_local_smp(opa_smp, device) == IB_SMI_DISCARD && 680 opa_smi_check_local_returning_smp(opa_smp, device) == IB_SMI_DISCARD) 681 goto out; 682 } else { 683 trace_ib_mad_handle_out_ib_smi(smp); 684 685 if ((ib_get_smp_direction(smp) ? smp->dr_dlid : smp->dr_slid) == 686 IB_LID_PERMISSIVE && 687 smi_handle_dr_smp_send(smp, rdma_cap_ib_switch(device), port_num) == 688 IB_SMI_DISCARD) { 689 ret = -EINVAL; 690 dev_err(&device->dev, "Invalid directed route\n"); 691 goto out; 692 } 693 drslid = be16_to_cpu(smp->dr_slid); 694 695 /* Check to post send on QP or process locally */ 696 if (smi_check_local_smp(smp, device) == IB_SMI_DISCARD && 697 smi_check_local_returning_smp(smp, device) == IB_SMI_DISCARD) 698 goto out; 699 } 700 701 local = kmalloc_obj(*local, GFP_ATOMIC); 702 if (!local) { 703 ret = -ENOMEM; 704 goto out; 705 } 706 local->mad_priv = NULL; 707 local->recv_mad_agent = NULL; 708 mad_priv = alloc_mad_private(mad_size, GFP_ATOMIC); 709 if (!mad_priv) { 710 ret = -ENOMEM; 711 kfree(local); 712 goto out; 713 } 714 715 build_smp_wc(mad_agent_priv->agent.qp, 716 send_wr->wr.wr_cqe, drslid, 717 send_wr->pkey_index, 718 send_wr->port_num, &mad_wc); 719 720 if (opa && smp->base_version == OPA_MGMT_BASE_VERSION) { 721 mad_wc.byte_len = mad_send_wr->send_buf.hdr_len 722 + mad_send_wr->send_buf.data_len 723 + sizeof(struct ib_grh); 724 } 725 726 /* No GRH for DR SMP */ 727 ret = device->ops.process_mad(device, 0, port_num, &mad_wc, NULL, 728 (const struct ib_mad *)smp, 729 (struct ib_mad *)mad_priv->mad, &mad_size, 730 &out_mad_pkey_index); 731 switch (ret) { 732 case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY: 733 if (ib_response_mad((const struct ib_mad_hdr *)mad_priv->mad) && 734 mad_agent_priv->agent.recv_handler) { 735 local->mad_priv = mad_priv; 736 local->recv_mad_agent = mad_agent_priv; 737 /* 738 * Reference MAD agent until receive 739 * side of local completion handled 740 */ 741 refcount_inc(&mad_agent_priv->refcount); 742 } else 743 kfree(mad_priv); 744 break; 745 case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED: 746 kfree(mad_priv); 747 break; 748 case IB_MAD_RESULT_SUCCESS: 749 /* Treat like an incoming receive MAD */ 750 port_priv = ib_get_mad_port(mad_agent_priv->agent.device, 751 mad_agent_priv->agent.port_num); 752 if (port_priv) { 753 memcpy(mad_priv->mad, smp, mad_priv->mad_size); 754 recv_mad_agent = find_mad_agent(port_priv, 755 (const struct ib_mad_hdr *)mad_priv->mad); 756 } 757 if (!port_priv || !recv_mad_agent) { 758 /* 759 * No receiving agent so drop packet and 760 * generate send completion. 761 */ 762 kfree(mad_priv); 763 break; 764 } 765 local->mad_priv = mad_priv; 766 local->recv_mad_agent = recv_mad_agent; 767 break; 768 default: 769 kfree(mad_priv); 770 kfree(local); 771 ret = -EINVAL; 772 goto out; 773 } 774 775 local->mad_send_wr = mad_send_wr; 776 if (opa) { 777 local->mad_send_wr->send_wr.pkey_index = out_mad_pkey_index; 778 local->return_wc_byte_len = mad_size; 779 } 780 /* Reference MAD agent until send side of local completion handled */ 781 refcount_inc(&mad_agent_priv->refcount); 782 /* Queue local completion to local list */ 783 spin_lock_irqsave(&mad_agent_priv->lock, flags); 784 list_add_tail(&local->completion_list, &mad_agent_priv->local_list); 785 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 786 queue_work(mad_agent_priv->qp_info->port_priv->wq, 787 &mad_agent_priv->local_work); 788 ret = 1; 789 out: 790 return ret; 791 } 792 793 static int get_pad_size(int hdr_len, int data_len, size_t mad_size) 794 { 795 int seg_size, pad; 796 797 seg_size = mad_size - hdr_len; 798 if (data_len && seg_size) { 799 pad = seg_size - data_len % seg_size; 800 return pad == seg_size ? 0 : pad; 801 } else 802 return seg_size; 803 } 804 805 static void free_send_rmpp_list(struct ib_mad_send_wr_private *mad_send_wr) 806 { 807 struct ib_rmpp_segment *s, *t; 808 809 list_for_each_entry_safe(s, t, &mad_send_wr->rmpp_list, list) { 810 list_del(&s->list); 811 kfree(s); 812 } 813 } 814 815 static int alloc_send_rmpp_list(struct ib_mad_send_wr_private *send_wr, 816 size_t mad_size, gfp_t gfp_mask) 817 { 818 struct ib_mad_send_buf *send_buf = &send_wr->send_buf; 819 struct ib_rmpp_mad *rmpp_mad = send_buf->mad; 820 struct ib_rmpp_segment *seg = NULL; 821 int left, seg_size, pad; 822 823 send_buf->seg_size = mad_size - send_buf->hdr_len; 824 send_buf->seg_rmpp_size = mad_size - IB_MGMT_RMPP_HDR; 825 seg_size = send_buf->seg_size; 826 pad = send_wr->pad; 827 828 /* Allocate data segments. */ 829 for (left = send_buf->data_len + pad; left > 0; left -= seg_size) { 830 seg = kmalloc(sizeof(*seg) + seg_size, gfp_mask); 831 if (!seg) { 832 free_send_rmpp_list(send_wr); 833 return -ENOMEM; 834 } 835 seg->num = ++send_buf->seg_count; 836 list_add_tail(&seg->list, &send_wr->rmpp_list); 837 } 838 839 /* Zero any padding */ 840 if (pad) 841 memset(seg->data + seg_size - pad, 0, pad); 842 843 rmpp_mad->rmpp_hdr.rmpp_version = send_wr->mad_agent_priv-> 844 agent.rmpp_version; 845 rmpp_mad->rmpp_hdr.rmpp_type = IB_MGMT_RMPP_TYPE_DATA; 846 ib_set_rmpp_flags(&rmpp_mad->rmpp_hdr, IB_MGMT_RMPP_FLAG_ACTIVE); 847 848 send_wr->cur_seg = container_of(send_wr->rmpp_list.next, 849 struct ib_rmpp_segment, list); 850 send_wr->last_ack_seg = send_wr->cur_seg; 851 return 0; 852 } 853 854 int ib_mad_kernel_rmpp_agent(const struct ib_mad_agent *agent) 855 { 856 return agent->rmpp_version && !(agent->flags & IB_MAD_USER_RMPP); 857 } 858 EXPORT_SYMBOL(ib_mad_kernel_rmpp_agent); 859 860 struct ib_mad_send_buf *ib_create_send_mad(struct ib_mad_agent *mad_agent, 861 u32 remote_qpn, u16 pkey_index, 862 int rmpp_active, int hdr_len, 863 int data_len, gfp_t gfp_mask, 864 u8 base_version) 865 { 866 struct ib_mad_agent_private *mad_agent_priv; 867 struct ib_mad_send_wr_private *mad_send_wr; 868 int pad, message_size, ret, size; 869 void *buf; 870 size_t mad_size; 871 bool opa; 872 873 mad_agent_priv = container_of(mad_agent, struct ib_mad_agent_private, 874 agent); 875 876 opa = rdma_cap_opa_mad(mad_agent->device, mad_agent->port_num); 877 878 if (opa && base_version == OPA_MGMT_BASE_VERSION) 879 mad_size = sizeof(struct opa_mad); 880 else 881 mad_size = sizeof(struct ib_mad); 882 883 pad = get_pad_size(hdr_len, data_len, mad_size); 884 message_size = hdr_len + data_len + pad; 885 886 if (ib_mad_kernel_rmpp_agent(mad_agent)) { 887 if (!rmpp_active && message_size > mad_size) 888 return ERR_PTR(-EINVAL); 889 } else 890 if (rmpp_active || message_size > mad_size) 891 return ERR_PTR(-EINVAL); 892 893 size = rmpp_active ? hdr_len : mad_size; 894 buf = kzalloc(sizeof *mad_send_wr + size, gfp_mask); 895 if (!buf) 896 return ERR_PTR(-ENOMEM); 897 898 mad_send_wr = buf + size; 899 INIT_LIST_HEAD(&mad_send_wr->rmpp_list); 900 mad_send_wr->send_buf.mad = buf; 901 mad_send_wr->send_buf.hdr_len = hdr_len; 902 mad_send_wr->send_buf.data_len = data_len; 903 mad_send_wr->pad = pad; 904 905 mad_send_wr->mad_agent_priv = mad_agent_priv; 906 mad_send_wr->sg_list[0].length = hdr_len; 907 mad_send_wr->sg_list[0].lkey = mad_agent->qp->pd->local_dma_lkey; 908 909 /* OPA MADs don't have to be the full 2048 bytes */ 910 if (opa && base_version == OPA_MGMT_BASE_VERSION && 911 data_len < mad_size - hdr_len) 912 mad_send_wr->sg_list[1].length = data_len; 913 else 914 mad_send_wr->sg_list[1].length = mad_size - hdr_len; 915 916 mad_send_wr->sg_list[1].lkey = mad_agent->qp->pd->local_dma_lkey; 917 918 mad_send_wr->mad_list.cqe.done = ib_mad_send_done; 919 920 mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe; 921 mad_send_wr->send_wr.wr.sg_list = mad_send_wr->sg_list; 922 mad_send_wr->send_wr.wr.num_sge = 2; 923 mad_send_wr->send_wr.wr.opcode = IB_WR_SEND; 924 mad_send_wr->send_wr.wr.send_flags = IB_SEND_SIGNALED; 925 mad_send_wr->send_wr.remote_qpn = remote_qpn; 926 mad_send_wr->send_wr.remote_qkey = IB_QP_SET_QKEY; 927 mad_send_wr->send_wr.pkey_index = pkey_index; 928 929 if (rmpp_active) { 930 ret = alloc_send_rmpp_list(mad_send_wr, mad_size, gfp_mask); 931 if (ret) { 932 kfree(buf); 933 return ERR_PTR(ret); 934 } 935 } 936 937 mad_send_wr->send_buf.mad_agent = mad_agent; 938 refcount_inc(&mad_agent_priv->refcount); 939 return &mad_send_wr->send_buf; 940 } 941 EXPORT_SYMBOL(ib_create_send_mad); 942 943 int ib_get_mad_data_offset(u8 mgmt_class) 944 { 945 if (mgmt_class == IB_MGMT_CLASS_SUBN_ADM) 946 return IB_MGMT_SA_HDR; 947 else if ((mgmt_class == IB_MGMT_CLASS_DEVICE_MGMT) || 948 (mgmt_class == IB_MGMT_CLASS_DEVICE_ADM) || 949 (mgmt_class == IB_MGMT_CLASS_BIS)) 950 return IB_MGMT_DEVICE_HDR; 951 else if ((mgmt_class >= IB_MGMT_CLASS_VENDOR_RANGE2_START) && 952 (mgmt_class <= IB_MGMT_CLASS_VENDOR_RANGE2_END)) 953 return IB_MGMT_VENDOR_HDR; 954 else 955 return IB_MGMT_MAD_HDR; 956 } 957 EXPORT_SYMBOL(ib_get_mad_data_offset); 958 959 int ib_is_mad_class_rmpp(u8 mgmt_class) 960 { 961 if ((mgmt_class == IB_MGMT_CLASS_SUBN_ADM) || 962 (mgmt_class == IB_MGMT_CLASS_DEVICE_MGMT) || 963 (mgmt_class == IB_MGMT_CLASS_DEVICE_ADM) || 964 (mgmt_class == IB_MGMT_CLASS_BIS) || 965 ((mgmt_class >= IB_MGMT_CLASS_VENDOR_RANGE2_START) && 966 (mgmt_class <= IB_MGMT_CLASS_VENDOR_RANGE2_END))) 967 return 1; 968 return 0; 969 } 970 EXPORT_SYMBOL(ib_is_mad_class_rmpp); 971 972 void *ib_get_rmpp_segment(struct ib_mad_send_buf *send_buf, int seg_num) 973 { 974 struct ib_mad_send_wr_private *mad_send_wr; 975 struct list_head *list; 976 977 mad_send_wr = container_of(send_buf, struct ib_mad_send_wr_private, 978 send_buf); 979 list = &mad_send_wr->cur_seg->list; 980 981 if (mad_send_wr->cur_seg->num < seg_num) { 982 list_for_each_entry(mad_send_wr->cur_seg, list, list) 983 if (mad_send_wr->cur_seg->num == seg_num) 984 break; 985 } else if (mad_send_wr->cur_seg->num > seg_num) { 986 list_for_each_entry_reverse(mad_send_wr->cur_seg, list, list) 987 if (mad_send_wr->cur_seg->num == seg_num) 988 break; 989 } 990 return mad_send_wr->cur_seg->data; 991 } 992 EXPORT_SYMBOL(ib_get_rmpp_segment); 993 994 static inline void *ib_get_payload(struct ib_mad_send_wr_private *mad_send_wr) 995 { 996 if (mad_send_wr->send_buf.seg_count) 997 return ib_get_rmpp_segment(&mad_send_wr->send_buf, 998 mad_send_wr->seg_num); 999 else 1000 return mad_send_wr->send_buf.mad + 1001 mad_send_wr->send_buf.hdr_len; 1002 } 1003 1004 void ib_free_send_mad(struct ib_mad_send_buf *send_buf) 1005 { 1006 struct ib_mad_agent_private *mad_agent_priv; 1007 struct ib_mad_send_wr_private *mad_send_wr; 1008 1009 mad_agent_priv = container_of(send_buf->mad_agent, 1010 struct ib_mad_agent_private, agent); 1011 mad_send_wr = container_of(send_buf, struct ib_mad_send_wr_private, 1012 send_buf); 1013 1014 free_send_rmpp_list(mad_send_wr); 1015 kfree(send_buf->mad); 1016 deref_mad_agent(mad_agent_priv); 1017 } 1018 EXPORT_SYMBOL(ib_free_send_mad); 1019 1020 int ib_send_mad(struct ib_mad_send_wr_private *mad_send_wr) 1021 { 1022 struct ib_mad_qp_info *qp_info; 1023 struct list_head *list; 1024 struct ib_mad_agent *mad_agent; 1025 struct ib_sge *sge; 1026 unsigned long flags; 1027 int ret; 1028 1029 /* Set WR ID to find mad_send_wr upon completion */ 1030 qp_info = mad_send_wr->mad_agent_priv->qp_info; 1031 mad_send_wr->mad_list.mad_queue = &qp_info->send_queue; 1032 mad_send_wr->mad_list.cqe.done = ib_mad_send_done; 1033 mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe; 1034 1035 mad_agent = mad_send_wr->send_buf.mad_agent; 1036 sge = mad_send_wr->sg_list; 1037 sge[0].addr = ib_dma_map_single(mad_agent->device, 1038 mad_send_wr->send_buf.mad, 1039 sge[0].length, 1040 DMA_TO_DEVICE); 1041 if (unlikely(ib_dma_mapping_error(mad_agent->device, sge[0].addr))) 1042 return -ENOMEM; 1043 1044 mad_send_wr->header_mapping = sge[0].addr; 1045 1046 sge[1].addr = ib_dma_map_single(mad_agent->device, 1047 ib_get_payload(mad_send_wr), 1048 sge[1].length, 1049 DMA_TO_DEVICE); 1050 if (unlikely(ib_dma_mapping_error(mad_agent->device, sge[1].addr))) { 1051 ib_dma_unmap_single(mad_agent->device, 1052 mad_send_wr->header_mapping, 1053 sge[0].length, DMA_TO_DEVICE); 1054 return -ENOMEM; 1055 } 1056 mad_send_wr->payload_mapping = sge[1].addr; 1057 1058 spin_lock_irqsave(&qp_info->send_queue.lock, flags); 1059 if (qp_info->send_queue.count < qp_info->send_queue.max_active) { 1060 trace_ib_mad_ib_send_mad(mad_send_wr, qp_info); 1061 ret = ib_post_send(mad_agent->qp, &mad_send_wr->send_wr.wr, 1062 NULL); 1063 list = &qp_info->send_queue.list; 1064 } else { 1065 ret = 0; 1066 list = &qp_info->overflow_list; 1067 } 1068 1069 if (!ret) { 1070 qp_info->send_queue.count++; 1071 list_add_tail(&mad_send_wr->mad_list.list, list); 1072 } 1073 spin_unlock_irqrestore(&qp_info->send_queue.lock, flags); 1074 if (ret) { 1075 ib_dma_unmap_single(mad_agent->device, 1076 mad_send_wr->header_mapping, 1077 sge[0].length, DMA_TO_DEVICE); 1078 ib_dma_unmap_single(mad_agent->device, 1079 mad_send_wr->payload_mapping, 1080 sge[1].length, DMA_TO_DEVICE); 1081 } 1082 return ret; 1083 } 1084 1085 static void handle_queued_state(struct ib_mad_send_wr_private *mad_send_wr, 1086 struct ib_mad_agent_private *mad_agent_priv) 1087 { 1088 if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP) { 1089 mad_agent_priv->sol_fc_wait_count--; 1090 list_move_tail(&mad_send_wr->agent_list, 1091 &mad_agent_priv->backlog_list); 1092 } else { 1093 expect_mad_state(mad_send_wr, IB_MAD_STATE_INIT); 1094 list_add_tail(&mad_send_wr->agent_list, 1095 &mad_agent_priv->backlog_list); 1096 } 1097 } 1098 1099 static void handle_send_state(struct ib_mad_send_wr_private *mad_send_wr, 1100 struct ib_mad_agent_private *mad_agent_priv) 1101 { 1102 if (mad_send_wr->state == IB_MAD_STATE_INIT) { 1103 list_add_tail(&mad_send_wr->agent_list, 1104 &mad_agent_priv->send_list); 1105 } else { 1106 expect_mad_state2(mad_send_wr, IB_MAD_STATE_WAIT_RESP, 1107 IB_MAD_STATE_QUEUED); 1108 list_move_tail(&mad_send_wr->agent_list, 1109 &mad_agent_priv->send_list); 1110 } 1111 1112 if (mad_send_wr->is_solicited_fc) { 1113 if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP) 1114 mad_agent_priv->sol_fc_wait_count--; 1115 mad_agent_priv->sol_fc_send_count++; 1116 } 1117 } 1118 1119 static void handle_wait_state(struct ib_mad_send_wr_private *mad_send_wr, 1120 struct ib_mad_agent_private *mad_agent_priv) 1121 { 1122 struct ib_mad_send_wr_private *temp_mad_send_wr; 1123 struct list_head *list_item; 1124 unsigned long delay; 1125 1126 expect_mad_state3(mad_send_wr, IB_MAD_STATE_SEND_START, 1127 IB_MAD_STATE_WAIT_RESP, IB_MAD_STATE_CANCELED); 1128 if (mad_send_wr->state == IB_MAD_STATE_SEND_START && 1129 mad_send_wr->is_solicited_fc) { 1130 mad_agent_priv->sol_fc_send_count--; 1131 mad_agent_priv->sol_fc_wait_count++; 1132 } 1133 1134 list_del_init(&mad_send_wr->agent_list); 1135 delay = mad_send_wr->timeout; 1136 mad_send_wr->timeout += jiffies; 1137 1138 if (delay) { 1139 list_for_each_prev(list_item, 1140 &mad_agent_priv->wait_list) { 1141 temp_mad_send_wr = list_entry( 1142 list_item, 1143 struct ib_mad_send_wr_private, 1144 agent_list); 1145 if (time_after(mad_send_wr->timeout, 1146 temp_mad_send_wr->timeout)) 1147 break; 1148 } 1149 } else { 1150 list_item = &mad_agent_priv->wait_list; 1151 } 1152 1153 list_add(&mad_send_wr->agent_list, list_item); 1154 } 1155 1156 static void handle_early_resp_state(struct ib_mad_send_wr_private *mad_send_wr, 1157 struct ib_mad_agent_private *mad_agent_priv) 1158 { 1159 expect_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START); 1160 mad_agent_priv->sol_fc_send_count -= mad_send_wr->is_solicited_fc; 1161 } 1162 1163 static void handle_canceled_state(struct ib_mad_send_wr_private *mad_send_wr, 1164 struct ib_mad_agent_private *mad_agent_priv) 1165 { 1166 not_expect_mad_state(mad_send_wr, IB_MAD_STATE_DONE); 1167 if (mad_send_wr->is_solicited_fc) { 1168 if (mad_send_wr->state == IB_MAD_STATE_SEND_START) 1169 mad_agent_priv->sol_fc_send_count--; 1170 else if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP) 1171 mad_agent_priv->sol_fc_wait_count--; 1172 } 1173 } 1174 1175 static void handle_done_state(struct ib_mad_send_wr_private *mad_send_wr, 1176 struct ib_mad_agent_private *mad_agent_priv) 1177 { 1178 if (mad_send_wr->is_solicited_fc) { 1179 if (mad_send_wr->state == IB_MAD_STATE_SEND_START) 1180 mad_agent_priv->sol_fc_send_count--; 1181 else if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP) 1182 mad_agent_priv->sol_fc_wait_count--; 1183 } 1184 1185 list_del_init(&mad_send_wr->agent_list); 1186 } 1187 1188 void change_mad_state(struct ib_mad_send_wr_private *mad_send_wr, 1189 enum ib_mad_state new_state) 1190 { 1191 struct ib_mad_agent_private *mad_agent_priv = 1192 mad_send_wr->mad_agent_priv; 1193 1194 switch (new_state) { 1195 case IB_MAD_STATE_INIT: 1196 break; 1197 case IB_MAD_STATE_QUEUED: 1198 handle_queued_state(mad_send_wr, mad_agent_priv); 1199 break; 1200 case IB_MAD_STATE_SEND_START: 1201 handle_send_state(mad_send_wr, mad_agent_priv); 1202 break; 1203 case IB_MAD_STATE_WAIT_RESP: 1204 handle_wait_state(mad_send_wr, mad_agent_priv); 1205 if (mad_send_wr->state == IB_MAD_STATE_CANCELED) 1206 return; 1207 break; 1208 case IB_MAD_STATE_EARLY_RESP: 1209 handle_early_resp_state(mad_send_wr, mad_agent_priv); 1210 break; 1211 case IB_MAD_STATE_CANCELED: 1212 handle_canceled_state(mad_send_wr, mad_agent_priv); 1213 break; 1214 case IB_MAD_STATE_DONE: 1215 handle_done_state(mad_send_wr, mad_agent_priv); 1216 break; 1217 } 1218 1219 mad_send_wr->state = new_state; 1220 } 1221 1222 static bool is_solicited_fc_mad(struct ib_mad_send_wr_private *mad_send_wr) 1223 { 1224 struct ib_rmpp_mad *rmpp_mad; 1225 u8 mgmt_class; 1226 1227 if (!mad_send_wr->timeout) 1228 return 0; 1229 1230 rmpp_mad = mad_send_wr->send_buf.mad; 1231 if (mad_send_wr->mad_agent_priv->agent.rmpp_version && 1232 (ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) & IB_MGMT_RMPP_FLAG_ACTIVE)) 1233 return 0; 1234 1235 mgmt_class = 1236 ((struct ib_mad_hdr *)mad_send_wr->send_buf.mad)->mgmt_class; 1237 return mgmt_class == IB_MGMT_CLASS_CM || 1238 mgmt_class == IB_MGMT_CLASS_SUBN_ADM || 1239 mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || 1240 mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE; 1241 } 1242 1243 static bool mad_is_for_backlog(struct ib_mad_send_wr_private *mad_send_wr) 1244 { 1245 struct ib_mad_agent_private *mad_agent_priv = 1246 mad_send_wr->mad_agent_priv; 1247 1248 if (!mad_send_wr->is_solicited_fc || !mad_agent_priv->sol_fc_max) 1249 return false; 1250 1251 if (!list_empty(&mad_agent_priv->backlog_list)) 1252 return true; 1253 1254 return mad_agent_priv->sol_fc_send_count + 1255 mad_agent_priv->sol_fc_wait_count >= 1256 mad_agent_priv->sol_fc_max; 1257 } 1258 1259 /* 1260 * ib_post_send_mad - Posts MAD(s) to the send queue of the QP associated 1261 * with the registered client 1262 */ 1263 int ib_post_send_mad(struct ib_mad_send_buf *send_buf, 1264 struct ib_mad_send_buf **bad_send_buf) 1265 { 1266 struct ib_mad_agent_private *mad_agent_priv; 1267 struct ib_mad_send_buf *next_send_buf; 1268 struct ib_mad_send_wr_private *mad_send_wr; 1269 unsigned long flags; 1270 int ret = -EINVAL; 1271 1272 /* Walk list of send WRs and post each on send list */ 1273 for (; send_buf; send_buf = next_send_buf) { 1274 mad_send_wr = container_of(send_buf, 1275 struct ib_mad_send_wr_private, 1276 send_buf); 1277 mad_agent_priv = mad_send_wr->mad_agent_priv; 1278 1279 ret = ib_mad_enforce_security(mad_agent_priv, 1280 mad_send_wr->send_wr.pkey_index); 1281 if (ret) 1282 goto error; 1283 1284 if (!send_buf->mad_agent->send_handler) { 1285 ret = -EINVAL; 1286 goto error; 1287 } 1288 1289 if (!ib_is_mad_class_rmpp(((struct ib_mad_hdr *) send_buf->mad)->mgmt_class)) { 1290 if (mad_agent_priv->agent.rmpp_version) { 1291 ret = -EINVAL; 1292 goto error; 1293 } 1294 } 1295 1296 /* 1297 * Save pointer to next work request to post in case the 1298 * current one completes, and the user modifies the work 1299 * request associated with the completion 1300 */ 1301 next_send_buf = send_buf->next; 1302 mad_send_wr->send_wr.ah = send_buf->ah; 1303 1304 if (((struct ib_mad_hdr *) send_buf->mad)->mgmt_class == 1305 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) { 1306 ret = handle_outgoing_dr_smp(mad_agent_priv, 1307 mad_send_wr); 1308 if (ret < 0) /* error */ 1309 goto error; 1310 else if (ret == 1) /* locally consumed */ 1311 continue; 1312 } 1313 1314 mad_send_wr->tid = ((struct ib_mad_hdr *) send_buf->mad)->tid; 1315 /* Timeout will be updated after send completes */ 1316 mad_send_wr->timeout = msecs_to_jiffies(send_buf->timeout_ms); 1317 mad_send_wr->max_retries = send_buf->retries; 1318 mad_send_wr->retries_left = send_buf->retries; 1319 send_buf->retries = 0; 1320 change_mad_state(mad_send_wr, IB_MAD_STATE_INIT); 1321 1322 /* Reference MAD agent until send completes */ 1323 refcount_inc(&mad_agent_priv->refcount); 1324 spin_lock_irqsave(&mad_agent_priv->lock, flags); 1325 mad_send_wr->is_solicited_fc = is_solicited_fc_mad(mad_send_wr); 1326 if (mad_is_for_backlog(mad_send_wr)) { 1327 change_mad_state(mad_send_wr, IB_MAD_STATE_QUEUED); 1328 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 1329 return 0; 1330 } 1331 1332 change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START); 1333 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 1334 1335 if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) { 1336 ret = ib_send_rmpp_mad(mad_send_wr); 1337 if (ret >= 0 && ret != IB_RMPP_RESULT_CONSUMED) 1338 ret = ib_send_mad(mad_send_wr); 1339 } else 1340 ret = ib_send_mad(mad_send_wr); 1341 if (ret < 0) { 1342 /* Fail send request */ 1343 spin_lock_irqsave(&mad_agent_priv->lock, flags); 1344 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE); 1345 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 1346 deref_mad_agent(mad_agent_priv); 1347 goto error; 1348 } 1349 } 1350 return 0; 1351 error: 1352 if (bad_send_buf) 1353 *bad_send_buf = send_buf; 1354 return ret; 1355 } 1356 EXPORT_SYMBOL(ib_post_send_mad); 1357 1358 /* 1359 * ib_free_recv_mad - Returns data buffers used to receive 1360 * a MAD to the access layer 1361 */ 1362 void ib_free_recv_mad(struct ib_mad_recv_wc *mad_recv_wc) 1363 { 1364 struct ib_mad_recv_buf *mad_recv_buf, *temp_recv_buf; 1365 struct ib_mad_private_header *mad_priv_hdr; 1366 struct ib_mad_private *priv; 1367 struct list_head free_list; 1368 1369 INIT_LIST_HEAD(&free_list); 1370 list_splice_init(&mad_recv_wc->rmpp_list, &free_list); 1371 1372 list_for_each_entry_safe(mad_recv_buf, temp_recv_buf, 1373 &free_list, list) { 1374 mad_recv_wc = container_of(mad_recv_buf, struct ib_mad_recv_wc, 1375 recv_buf); 1376 mad_priv_hdr = container_of(mad_recv_wc, 1377 struct ib_mad_private_header, 1378 recv_wc); 1379 priv = container_of(mad_priv_hdr, struct ib_mad_private, 1380 header); 1381 kfree(priv); 1382 } 1383 } 1384 EXPORT_SYMBOL(ib_free_recv_mad); 1385 1386 static int method_in_use(struct ib_mad_mgmt_method_table **method, 1387 struct ib_mad_reg_req *mad_reg_req) 1388 { 1389 int i; 1390 1391 for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS) { 1392 if ((*method)->agent[i]) { 1393 pr_err("Method %d already in use\n", i); 1394 return -EINVAL; 1395 } 1396 } 1397 return 0; 1398 } 1399 1400 static int allocate_method_table(struct ib_mad_mgmt_method_table **method) 1401 { 1402 /* Allocate management method table */ 1403 *method = kzalloc_obj(**method, GFP_ATOMIC); 1404 return (*method) ? 0 : (-ENOMEM); 1405 } 1406 1407 /* 1408 * Check to see if there are any methods still in use 1409 */ 1410 static int check_method_table(struct ib_mad_mgmt_method_table *method) 1411 { 1412 int i; 1413 1414 for (i = 0; i < IB_MGMT_MAX_METHODS; i++) 1415 if (method->agent[i]) 1416 return 1; 1417 return 0; 1418 } 1419 1420 /* 1421 * Check to see if there are any method tables for this class still in use 1422 */ 1423 static int check_class_table(struct ib_mad_mgmt_class_table *class) 1424 { 1425 int i; 1426 1427 for (i = 0; i < MAX_MGMT_CLASS; i++) 1428 if (class->method_table[i]) 1429 return 1; 1430 return 0; 1431 } 1432 1433 static int check_vendor_class(struct ib_mad_mgmt_vendor_class *vendor_class) 1434 { 1435 int i; 1436 1437 for (i = 0; i < MAX_MGMT_OUI; i++) 1438 if (vendor_class->method_table[i]) 1439 return 1; 1440 return 0; 1441 } 1442 1443 static int find_vendor_oui(struct ib_mad_mgmt_vendor_class *vendor_class, 1444 const char *oui) 1445 { 1446 int i; 1447 1448 for (i = 0; i < MAX_MGMT_OUI; i++) 1449 /* Is there matching OUI for this vendor class ? */ 1450 if (!memcmp(vendor_class->oui[i], oui, 3)) 1451 return i; 1452 1453 return -1; 1454 } 1455 1456 static int check_vendor_table(struct ib_mad_mgmt_vendor_class_table *vendor) 1457 { 1458 int i; 1459 1460 for (i = 0; i < MAX_MGMT_VENDOR_RANGE2; i++) 1461 if (vendor->vendor_class[i]) 1462 return 1; 1463 1464 return 0; 1465 } 1466 1467 static void remove_methods_mad_agent(struct ib_mad_mgmt_method_table *method, 1468 struct ib_mad_agent_private *agent) 1469 { 1470 int i; 1471 1472 /* Remove any methods for this mad agent */ 1473 for (i = 0; i < IB_MGMT_MAX_METHODS; i++) 1474 if (method->agent[i] == agent) 1475 method->agent[i] = NULL; 1476 } 1477 1478 static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req, 1479 struct ib_mad_agent_private *agent_priv, 1480 u8 mgmt_class) 1481 { 1482 struct ib_mad_port_private *port_priv; 1483 struct ib_mad_mgmt_class_table **class; 1484 struct ib_mad_mgmt_method_table **method; 1485 int i, ret; 1486 1487 port_priv = agent_priv->qp_info->port_priv; 1488 class = &port_priv->version[mad_reg_req->mgmt_class_version].class; 1489 if (!*class) { 1490 /* Allocate management class table for "new" class version */ 1491 *class = kzalloc_obj(**class, GFP_ATOMIC); 1492 if (!*class) { 1493 ret = -ENOMEM; 1494 goto error1; 1495 } 1496 1497 /* Allocate method table for this management class */ 1498 method = &(*class)->method_table[mgmt_class]; 1499 if ((ret = allocate_method_table(method))) 1500 goto error2; 1501 } else { 1502 method = &(*class)->method_table[mgmt_class]; 1503 if (!*method) { 1504 /* Allocate method table for this management class */ 1505 if ((ret = allocate_method_table(method))) 1506 goto error1; 1507 } 1508 } 1509 1510 /* Now, make sure methods are not already in use */ 1511 if (method_in_use(method, mad_reg_req)) 1512 goto error3; 1513 1514 /* Finally, add in methods being registered */ 1515 for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS) 1516 (*method)->agent[i] = agent_priv; 1517 1518 return 0; 1519 1520 error3: 1521 /* Remove any methods for this mad agent */ 1522 remove_methods_mad_agent(*method, agent_priv); 1523 /* Now, check to see if there are any methods in use */ 1524 if (!check_method_table(*method)) { 1525 /* If not, release management method table */ 1526 kfree(*method); 1527 *method = NULL; 1528 } 1529 ret = -EINVAL; 1530 goto error1; 1531 error2: 1532 kfree(*class); 1533 *class = NULL; 1534 error1: 1535 return ret; 1536 } 1537 1538 static int add_oui_reg_req(struct ib_mad_reg_req *mad_reg_req, 1539 struct ib_mad_agent_private *agent_priv) 1540 { 1541 struct ib_mad_port_private *port_priv; 1542 struct ib_mad_mgmt_vendor_class_table **vendor_table; 1543 struct ib_mad_mgmt_vendor_class_table *vendor = NULL; 1544 struct ib_mad_mgmt_vendor_class *vendor_class = NULL; 1545 struct ib_mad_mgmt_method_table **method; 1546 int i, ret = -ENOMEM; 1547 u8 vclass; 1548 1549 /* "New" vendor (with OUI) class */ 1550 vclass = vendor_class_index(mad_reg_req->mgmt_class); 1551 port_priv = agent_priv->qp_info->port_priv; 1552 vendor_table = &port_priv->version[ 1553 mad_reg_req->mgmt_class_version].vendor; 1554 if (!*vendor_table) { 1555 /* Allocate mgmt vendor class table for "new" class version */ 1556 vendor = kzalloc_obj(*vendor, GFP_ATOMIC); 1557 if (!vendor) 1558 goto error1; 1559 1560 *vendor_table = vendor; 1561 } 1562 if (!(*vendor_table)->vendor_class[vclass]) { 1563 /* Allocate table for this management vendor class */ 1564 vendor_class = kzalloc_obj(*vendor_class, GFP_ATOMIC); 1565 if (!vendor_class) 1566 goto error2; 1567 1568 (*vendor_table)->vendor_class[vclass] = vendor_class; 1569 } 1570 for (i = 0; i < MAX_MGMT_OUI; i++) { 1571 /* Is there matching OUI for this vendor class ? */ 1572 if (!memcmp((*vendor_table)->vendor_class[vclass]->oui[i], 1573 mad_reg_req->oui, 3)) { 1574 method = &(*vendor_table)->vendor_class[ 1575 vclass]->method_table[i]; 1576 if (!*method) 1577 goto error3; 1578 goto check_in_use; 1579 } 1580 } 1581 for (i = 0; i < MAX_MGMT_OUI; i++) { 1582 /* OUI slot available ? */ 1583 if (!is_vendor_oui((*vendor_table)->vendor_class[ 1584 vclass]->oui[i])) { 1585 method = &(*vendor_table)->vendor_class[ 1586 vclass]->method_table[i]; 1587 /* Allocate method table for this OUI */ 1588 if (!*method) { 1589 ret = allocate_method_table(method); 1590 if (ret) 1591 goto error3; 1592 } 1593 memcpy((*vendor_table)->vendor_class[vclass]->oui[i], 1594 mad_reg_req->oui, 3); 1595 goto check_in_use; 1596 } 1597 } 1598 dev_err(&agent_priv->agent.device->dev, "All OUI slots in use\n"); 1599 goto error3; 1600 1601 check_in_use: 1602 /* Now, make sure methods are not already in use */ 1603 if (method_in_use(method, mad_reg_req)) 1604 goto error4; 1605 1606 /* Finally, add in methods being registered */ 1607 for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS) 1608 (*method)->agent[i] = agent_priv; 1609 1610 return 0; 1611 1612 error4: 1613 /* Remove any methods for this mad agent */ 1614 remove_methods_mad_agent(*method, agent_priv); 1615 /* Now, check to see if there are any methods in use */ 1616 if (!check_method_table(*method)) { 1617 /* If not, release management method table */ 1618 kfree(*method); 1619 *method = NULL; 1620 } 1621 ret = -EINVAL; 1622 error3: 1623 if (vendor_class) { 1624 (*vendor_table)->vendor_class[vclass] = NULL; 1625 kfree(vendor_class); 1626 } 1627 error2: 1628 if (vendor) { 1629 *vendor_table = NULL; 1630 kfree(vendor); 1631 } 1632 error1: 1633 return ret; 1634 } 1635 1636 static void remove_mad_reg_req(struct ib_mad_agent_private *agent_priv) 1637 { 1638 struct ib_mad_port_private *port_priv; 1639 struct ib_mad_mgmt_class_table *class; 1640 struct ib_mad_mgmt_method_table *method; 1641 struct ib_mad_mgmt_vendor_class_table *vendor; 1642 struct ib_mad_mgmt_vendor_class *vendor_class; 1643 int index; 1644 u8 mgmt_class; 1645 1646 /* 1647 * Was MAD registration request supplied 1648 * with original registration ? 1649 */ 1650 if (!agent_priv->reg_req) 1651 goto out; 1652 1653 port_priv = agent_priv->qp_info->port_priv; 1654 mgmt_class = convert_mgmt_class(agent_priv->reg_req->mgmt_class); 1655 class = port_priv->version[ 1656 agent_priv->reg_req->mgmt_class_version].class; 1657 if (!class) 1658 goto vendor_check; 1659 1660 method = class->method_table[mgmt_class]; 1661 if (method) { 1662 /* Remove any methods for this mad agent */ 1663 remove_methods_mad_agent(method, agent_priv); 1664 /* Now, check to see if there are any methods still in use */ 1665 if (!check_method_table(method)) { 1666 /* If not, release management method table */ 1667 kfree(method); 1668 class->method_table[mgmt_class] = NULL; 1669 /* Any management classes left ? */ 1670 if (!check_class_table(class)) { 1671 /* If not, release management class table */ 1672 kfree(class); 1673 port_priv->version[ 1674 agent_priv->reg_req-> 1675 mgmt_class_version].class = NULL; 1676 } 1677 } 1678 } 1679 1680 vendor_check: 1681 if (!is_vendor_class(mgmt_class)) 1682 goto out; 1683 1684 /* normalize mgmt_class to vendor range 2 */ 1685 mgmt_class = vendor_class_index(agent_priv->reg_req->mgmt_class); 1686 vendor = port_priv->version[ 1687 agent_priv->reg_req->mgmt_class_version].vendor; 1688 1689 if (!vendor) 1690 goto out; 1691 1692 vendor_class = vendor->vendor_class[mgmt_class]; 1693 if (vendor_class) { 1694 index = find_vendor_oui(vendor_class, agent_priv->reg_req->oui); 1695 if (index < 0) 1696 goto out; 1697 method = vendor_class->method_table[index]; 1698 if (method) { 1699 /* Remove any methods for this mad agent */ 1700 remove_methods_mad_agent(method, agent_priv); 1701 /* 1702 * Now, check to see if there are 1703 * any methods still in use 1704 */ 1705 if (!check_method_table(method)) { 1706 /* If not, release management method table */ 1707 kfree(method); 1708 vendor_class->method_table[index] = NULL; 1709 memset(vendor_class->oui[index], 0, 3); 1710 /* Any OUIs left ? */ 1711 if (!check_vendor_class(vendor_class)) { 1712 /* If not, release vendor class table */ 1713 kfree(vendor_class); 1714 vendor->vendor_class[mgmt_class] = NULL; 1715 /* Any other vendor classes left ? */ 1716 if (!check_vendor_table(vendor)) { 1717 kfree(vendor); 1718 port_priv->version[ 1719 agent_priv->reg_req-> 1720 mgmt_class_version]. 1721 vendor = NULL; 1722 } 1723 } 1724 } 1725 } 1726 } 1727 1728 out: 1729 return; 1730 } 1731 1732 static struct ib_mad_agent_private * 1733 find_mad_agent(struct ib_mad_port_private *port_priv, 1734 const struct ib_mad_hdr *mad_hdr) 1735 { 1736 struct ib_mad_agent_private *mad_agent = NULL; 1737 unsigned long flags; 1738 1739 if (ib_response_mad(mad_hdr)) { 1740 u32 hi_tid; 1741 1742 /* 1743 * Routing is based on high 32 bits of transaction ID 1744 * of MAD. 1745 */ 1746 hi_tid = be64_to_cpu(mad_hdr->tid) >> 32; 1747 rcu_read_lock(); 1748 mad_agent = xa_load(&ib_mad_clients, hi_tid); 1749 if (mad_agent && !refcount_inc_not_zero(&mad_agent->refcount)) 1750 mad_agent = NULL; 1751 rcu_read_unlock(); 1752 } else { 1753 struct ib_mad_mgmt_class_table *class; 1754 struct ib_mad_mgmt_method_table *method; 1755 struct ib_mad_mgmt_vendor_class_table *vendor; 1756 struct ib_mad_mgmt_vendor_class *vendor_class; 1757 const struct ib_vendor_mad *vendor_mad; 1758 int index; 1759 1760 spin_lock_irqsave(&port_priv->reg_lock, flags); 1761 /* 1762 * Routing is based on version, class, and method 1763 * For "newer" vendor MADs, also based on OUI 1764 */ 1765 if (mad_hdr->class_version >= MAX_MGMT_VERSION) 1766 goto out; 1767 if (!is_vendor_class(mad_hdr->mgmt_class)) { 1768 class = port_priv->version[ 1769 mad_hdr->class_version].class; 1770 if (!class) 1771 goto out; 1772 if (convert_mgmt_class(mad_hdr->mgmt_class) >= 1773 ARRAY_SIZE(class->method_table)) 1774 goto out; 1775 method = class->method_table[convert_mgmt_class( 1776 mad_hdr->mgmt_class)]; 1777 if (method) 1778 mad_agent = method->agent[mad_hdr->method & 1779 ~IB_MGMT_METHOD_RESP]; 1780 } else { 1781 vendor = port_priv->version[ 1782 mad_hdr->class_version].vendor; 1783 if (!vendor) 1784 goto out; 1785 vendor_class = vendor->vendor_class[vendor_class_index( 1786 mad_hdr->mgmt_class)]; 1787 if (!vendor_class) 1788 goto out; 1789 /* Find matching OUI */ 1790 vendor_mad = (const struct ib_vendor_mad *)mad_hdr; 1791 index = find_vendor_oui(vendor_class, vendor_mad->oui); 1792 if (index == -1) 1793 goto out; 1794 method = vendor_class->method_table[index]; 1795 if (method) { 1796 mad_agent = method->agent[mad_hdr->method & 1797 ~IB_MGMT_METHOD_RESP]; 1798 } 1799 } 1800 if (mad_agent) 1801 refcount_inc(&mad_agent->refcount); 1802 out: 1803 spin_unlock_irqrestore(&port_priv->reg_lock, flags); 1804 } 1805 1806 if (mad_agent && !mad_agent->agent.recv_handler) { 1807 dev_notice(&port_priv->device->dev, 1808 "No receive handler for client %p on port %u\n", 1809 &mad_agent->agent, port_priv->port_num); 1810 deref_mad_agent(mad_agent); 1811 mad_agent = NULL; 1812 } 1813 1814 return mad_agent; 1815 } 1816 1817 static int validate_mad(const struct ib_mad_hdr *mad_hdr, 1818 const struct ib_mad_qp_info *qp_info, 1819 bool opa) 1820 { 1821 int valid = 0; 1822 u32 qp_num = qp_info->qp->qp_num; 1823 1824 /* Make sure MAD base version is understood */ 1825 if (mad_hdr->base_version != IB_MGMT_BASE_VERSION && 1826 (!opa || mad_hdr->base_version != OPA_MGMT_BASE_VERSION)) { 1827 pr_err("MAD received with unsupported base version %u %s\n", 1828 mad_hdr->base_version, opa ? "(opa)" : ""); 1829 goto out; 1830 } 1831 1832 /* Filter SMI packets sent to other than QP0 */ 1833 if ((mad_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED) || 1834 (mad_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) { 1835 if (qp_num == 0) 1836 valid = 1; 1837 } else { 1838 /* CM attributes other than ClassPortInfo only use Send method */ 1839 if ((mad_hdr->mgmt_class == IB_MGMT_CLASS_CM) && 1840 (mad_hdr->attr_id != IB_MGMT_CLASSPORTINFO_ATTR_ID) && 1841 (mad_hdr->method != IB_MGMT_METHOD_SEND)) 1842 goto out; 1843 /* Filter GSI packets sent to QP0 */ 1844 if (qp_num != 0) 1845 valid = 1; 1846 } 1847 1848 out: 1849 return valid; 1850 } 1851 1852 static int is_rmpp_data_mad(const struct ib_mad_agent_private *mad_agent_priv, 1853 const struct ib_mad_hdr *mad_hdr) 1854 { 1855 struct ib_rmpp_mad *rmpp_mad; 1856 1857 rmpp_mad = (struct ib_rmpp_mad *)mad_hdr; 1858 return !mad_agent_priv->agent.rmpp_version || 1859 !ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent) || 1860 !(ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) & 1861 IB_MGMT_RMPP_FLAG_ACTIVE) || 1862 (rmpp_mad->rmpp_hdr.rmpp_type == IB_MGMT_RMPP_TYPE_DATA); 1863 } 1864 1865 static inline int rcv_has_same_class(const struct ib_mad_send_wr_private *wr, 1866 const struct ib_mad_recv_wc *rwc) 1867 { 1868 return ((struct ib_mad_hdr *)(wr->send_buf.mad))->mgmt_class == 1869 rwc->recv_buf.mad->mad_hdr.mgmt_class; 1870 } 1871 1872 static inline int 1873 rcv_has_same_gid(const struct ib_mad_agent_private *mad_agent_priv, 1874 const struct ib_mad_send_wr_private *wr, 1875 const struct ib_mad_recv_wc *rwc) 1876 { 1877 struct rdma_ah_attr attr; 1878 u8 send_resp, rcv_resp; 1879 union ib_gid sgid; 1880 struct ib_device *device = mad_agent_priv->agent.device; 1881 u32 port_num = mad_agent_priv->agent.port_num; 1882 u8 lmc; 1883 bool has_grh; 1884 1885 send_resp = ib_response_mad((struct ib_mad_hdr *)wr->send_buf.mad); 1886 rcv_resp = ib_response_mad(&rwc->recv_buf.mad->mad_hdr); 1887 1888 if (send_resp == rcv_resp) 1889 /* both requests, or both responses. GIDs different */ 1890 return 0; 1891 1892 if (rdma_query_ah(wr->send_buf.ah, &attr)) 1893 /* Assume not equal, to avoid false positives. */ 1894 return 0; 1895 1896 has_grh = !!(rdma_ah_get_ah_flags(&attr) & IB_AH_GRH); 1897 if (has_grh != !!(rwc->wc->wc_flags & IB_WC_GRH)) 1898 /* one has GID, other does not. Assume different */ 1899 return 0; 1900 1901 if (!send_resp && rcv_resp) { 1902 /* is request/response. */ 1903 if (!has_grh) { 1904 if (ib_get_cached_lmc(device, port_num, &lmc)) 1905 return 0; 1906 return (!lmc || !((rdma_ah_get_path_bits(&attr) ^ 1907 rwc->wc->dlid_path_bits) & 1908 ((1 << lmc) - 1))); 1909 } else { 1910 const struct ib_global_route *grh = 1911 rdma_ah_read_grh(&attr); 1912 1913 if (rdma_query_gid(device, port_num, 1914 grh->sgid_index, &sgid)) 1915 return 0; 1916 return !memcmp(sgid.raw, rwc->recv_buf.grh->dgid.raw, 1917 16); 1918 } 1919 } 1920 1921 if (!has_grh) 1922 return rdma_ah_get_dlid(&attr) == rwc->wc->slid; 1923 else 1924 return !memcmp(rdma_ah_read_grh(&attr)->dgid.raw, 1925 rwc->recv_buf.grh->sgid.raw, 1926 16); 1927 } 1928 1929 static inline int is_direct(u8 class) 1930 { 1931 return (class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE); 1932 } 1933 1934 struct ib_mad_send_wr_private* 1935 ib_find_send_mad(const struct ib_mad_agent_private *mad_agent_priv, 1936 const struct ib_mad_recv_wc *wc) 1937 { 1938 struct ib_mad_send_wr_private *wr; 1939 const struct ib_mad_hdr *mad_hdr; 1940 1941 mad_hdr = &wc->recv_buf.mad->mad_hdr; 1942 1943 list_for_each_entry(wr, &mad_agent_priv->wait_list, agent_list) { 1944 if ((wr->tid == mad_hdr->tid) && 1945 rcv_has_same_class(wr, wc) && 1946 /* 1947 * Don't check GID for direct routed MADs. 1948 * These might have permissive LIDs. 1949 */ 1950 (is_direct(mad_hdr->mgmt_class) || 1951 rcv_has_same_gid(mad_agent_priv, wr, wc))) 1952 return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL; 1953 } 1954 1955 list_for_each_entry(wr, &mad_agent_priv->backlog_list, agent_list) { 1956 if ((wr->tid == mad_hdr->tid) && 1957 rcv_has_same_class(wr, wc) && 1958 /* 1959 * Don't check GID for direct routed MADs. 1960 * These might have permissive LIDs. 1961 */ 1962 (is_direct(mad_hdr->mgmt_class) || 1963 rcv_has_same_gid(mad_agent_priv, wr, wc))) 1964 return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL; 1965 } 1966 1967 /* 1968 * It's possible to receive the response before we've 1969 * been notified that the send has completed 1970 */ 1971 list_for_each_entry(wr, &mad_agent_priv->send_list, agent_list) { 1972 if (is_rmpp_data_mad(mad_agent_priv, wr->send_buf.mad) && 1973 wr->tid == mad_hdr->tid && 1974 wr->timeout && 1975 rcv_has_same_class(wr, wc) && 1976 /* 1977 * Don't check GID for direct routed MADs. 1978 * These might have permissive LIDs. 1979 */ 1980 (is_direct(mad_hdr->mgmt_class) || 1981 rcv_has_same_gid(mad_agent_priv, wr, wc))) 1982 /* Verify request has not been canceled */ 1983 return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL; 1984 } 1985 return NULL; 1986 } 1987 1988 static void 1989 process_backlog_mads(struct ib_mad_agent_private *mad_agent_priv) 1990 { 1991 struct ib_mad_send_wr_private *mad_send_wr; 1992 struct ib_mad_send_wc mad_send_wc = {}; 1993 unsigned long flags; 1994 int ret; 1995 1996 spin_lock_irqsave(&mad_agent_priv->lock, flags); 1997 while (!list_empty(&mad_agent_priv->backlog_list) && 1998 (mad_agent_priv->sol_fc_send_count + 1999 mad_agent_priv->sol_fc_wait_count < 2000 mad_agent_priv->sol_fc_max)) { 2001 mad_send_wr = list_entry(mad_agent_priv->backlog_list.next, 2002 struct ib_mad_send_wr_private, 2003 agent_list); 2004 change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START); 2005 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2006 ret = ib_send_mad(mad_send_wr); 2007 if (ret) { 2008 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2009 deref_mad_agent(mad_agent_priv); 2010 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE); 2011 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2012 mad_send_wc.send_buf = &mad_send_wr->send_buf; 2013 mad_send_wc.status = IB_WC_LOC_QP_OP_ERR; 2014 mad_agent_priv->agent.send_handler( 2015 &mad_agent_priv->agent, &mad_send_wc); 2016 } 2017 2018 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2019 } 2020 2021 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2022 } 2023 2024 void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr) 2025 { 2026 mad_send_wr->timeout = 0; 2027 if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP || 2028 mad_send_wr->state == IB_MAD_STATE_QUEUED) 2029 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE); 2030 else 2031 change_mad_state(mad_send_wr, IB_MAD_STATE_EARLY_RESP); 2032 } 2033 2034 static bool is_kernel_rmpp_data_response(struct ib_mad_agent_private *agent, 2035 struct ib_mad_recv_wc *mad_recv_wc) 2036 { 2037 const struct ib_mad_hdr *mad_hdr = &mad_recv_wc->recv_buf.mad->mad_hdr; 2038 struct ib_rmpp_mad *rmpp_mad; 2039 2040 if (!ib_mad_kernel_rmpp_agent(&agent->agent) || 2041 !ib_response_mad(mad_hdr) || 2042 !ib_is_mad_class_rmpp(mad_hdr->mgmt_class)) 2043 return false; 2044 2045 rmpp_mad = (struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad; 2046 2047 return (ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) & 2048 IB_MGMT_RMPP_FLAG_ACTIVE) && 2049 rmpp_mad->rmpp_hdr.rmpp_type == IB_MGMT_RMPP_TYPE_DATA; 2050 } 2051 2052 static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv, 2053 struct ib_mad_recv_wc *mad_recv_wc) 2054 { 2055 struct ib_mad_send_wr_private *mad_send_wr; 2056 struct ib_mad_send_wc mad_send_wc; 2057 unsigned long flags; 2058 bool is_mad_done; 2059 int ret; 2060 2061 INIT_LIST_HEAD(&mad_recv_wc->rmpp_list); 2062 list_add(&mad_recv_wc->recv_buf.list, &mad_recv_wc->rmpp_list); 2063 2064 ret = ib_mad_enforce_security(mad_agent_priv, 2065 mad_recv_wc->wc->pkey_index); 2066 if (ret) { 2067 ib_free_recv_mad(mad_recv_wc); 2068 deref_mad_agent(mad_agent_priv); 2069 return; 2070 } 2071 2072 if (is_kernel_rmpp_data_response(mad_agent_priv, mad_recv_wc)) { 2073 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2074 mad_send_wr = ib_find_send_mad(mad_agent_priv, mad_recv_wc); 2075 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2076 2077 if (!mad_send_wr) { 2078 ib_free_recv_mad(mad_recv_wc); 2079 deref_mad_agent(mad_agent_priv); 2080 return; 2081 } 2082 } 2083 2084 if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) { 2085 mad_recv_wc = ib_process_rmpp_recv_wc(mad_agent_priv, 2086 mad_recv_wc); 2087 if (!mad_recv_wc) { 2088 deref_mad_agent(mad_agent_priv); 2089 return; 2090 } 2091 } 2092 2093 /* Complete corresponding request */ 2094 if (ib_response_mad(&mad_recv_wc->recv_buf.mad->mad_hdr)) { 2095 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2096 mad_send_wr = ib_find_send_mad(mad_agent_priv, mad_recv_wc); 2097 if (!mad_send_wr) { 2098 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2099 if (!ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent) 2100 && ib_is_mad_class_rmpp(mad_recv_wc->recv_buf.mad->mad_hdr.mgmt_class) 2101 && (ib_get_rmpp_flags(&((struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad)->rmpp_hdr) 2102 & IB_MGMT_RMPP_FLAG_ACTIVE)) { 2103 /* user rmpp is in effect 2104 * and this is an active RMPP MAD 2105 */ 2106 mad_agent_priv->agent.recv_handler( 2107 &mad_agent_priv->agent, NULL, 2108 mad_recv_wc); 2109 deref_mad_agent(mad_agent_priv); 2110 } else { 2111 /* not user rmpp, revert to normal behavior and 2112 * drop the mad 2113 */ 2114 ib_free_recv_mad(mad_recv_wc); 2115 deref_mad_agent(mad_agent_priv); 2116 return; 2117 } 2118 } else { 2119 ib_mark_mad_done(mad_send_wr); 2120 is_mad_done = (mad_send_wr->state == IB_MAD_STATE_DONE); 2121 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2122 2123 /* Defined behavior is to complete response before request */ 2124 mad_agent_priv->agent.recv_handler( 2125 &mad_agent_priv->agent, 2126 &mad_send_wr->send_buf, 2127 mad_recv_wc); 2128 deref_mad_agent(mad_agent_priv); 2129 2130 if (is_mad_done) { 2131 mad_send_wc.status = IB_WC_SUCCESS; 2132 mad_send_wc.vendor_err = 0; 2133 mad_send_wc.send_buf = &mad_send_wr->send_buf; 2134 ib_mad_complete_send_wr(mad_send_wr, 2135 &mad_send_wc); 2136 } 2137 } 2138 } else { 2139 mad_agent_priv->agent.recv_handler(&mad_agent_priv->agent, NULL, 2140 mad_recv_wc); 2141 deref_mad_agent(mad_agent_priv); 2142 } 2143 } 2144 2145 static enum smi_action handle_ib_smi(const struct ib_mad_port_private *port_priv, 2146 const struct ib_mad_qp_info *qp_info, 2147 const struct ib_wc *wc, 2148 u32 port_num, 2149 struct ib_mad_private *recv, 2150 struct ib_mad_private *response) 2151 { 2152 enum smi_forward_action retsmi; 2153 struct ib_smp *smp = (struct ib_smp *)recv->mad; 2154 2155 trace_ib_mad_handle_ib_smi(smp); 2156 2157 if (smi_handle_dr_smp_recv(smp, 2158 rdma_cap_ib_switch(port_priv->device), 2159 port_num, 2160 port_priv->device->phys_port_cnt) == 2161 IB_SMI_DISCARD) 2162 return IB_SMI_DISCARD; 2163 2164 retsmi = smi_check_forward_dr_smp(smp); 2165 if (retsmi == IB_SMI_LOCAL) 2166 return IB_SMI_HANDLE; 2167 2168 if (retsmi == IB_SMI_SEND) { /* don't forward */ 2169 if (smi_handle_dr_smp_send(smp, 2170 rdma_cap_ib_switch(port_priv->device), 2171 port_num) == IB_SMI_DISCARD) 2172 return IB_SMI_DISCARD; 2173 2174 if (smi_check_local_smp(smp, port_priv->device) == IB_SMI_DISCARD) 2175 return IB_SMI_DISCARD; 2176 } else if (rdma_cap_ib_switch(port_priv->device)) { 2177 /* forward case for switches */ 2178 memcpy(response, recv, mad_priv_size(response)); 2179 response->header.recv_wc.wc = &response->header.wc; 2180 response->header.recv_wc.recv_buf.mad = (struct ib_mad *)response->mad; 2181 response->header.recv_wc.recv_buf.grh = &response->grh; 2182 2183 agent_send_response((const struct ib_mad_hdr *)response->mad, 2184 &response->grh, wc, 2185 port_priv->device, 2186 smi_get_fwd_port(smp), 2187 qp_info->qp->qp_num, 2188 response->mad_size, 2189 false); 2190 2191 return IB_SMI_DISCARD; 2192 } 2193 return IB_SMI_HANDLE; 2194 } 2195 2196 static bool generate_unmatched_resp(const struct ib_mad_private *recv, 2197 struct ib_mad_private *response, 2198 size_t *resp_len, bool opa) 2199 { 2200 const struct ib_mad_hdr *recv_hdr = (const struct ib_mad_hdr *)recv->mad; 2201 struct ib_mad_hdr *resp_hdr = (struct ib_mad_hdr *)response->mad; 2202 2203 if (recv_hdr->method == IB_MGMT_METHOD_GET || 2204 recv_hdr->method == IB_MGMT_METHOD_SET) { 2205 memcpy(response, recv, mad_priv_size(response)); 2206 response->header.recv_wc.wc = &response->header.wc; 2207 response->header.recv_wc.recv_buf.mad = (struct ib_mad *)response->mad; 2208 response->header.recv_wc.recv_buf.grh = &response->grh; 2209 resp_hdr->method = IB_MGMT_METHOD_GET_RESP; 2210 resp_hdr->status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB); 2211 if (recv_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) 2212 resp_hdr->status |= IB_SMP_DIRECTION; 2213 2214 if (opa && recv_hdr->base_version == OPA_MGMT_BASE_VERSION) { 2215 if (recv_hdr->mgmt_class == 2216 IB_MGMT_CLASS_SUBN_LID_ROUTED || 2217 recv_hdr->mgmt_class == 2218 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) 2219 *resp_len = opa_get_smp_header_size( 2220 (struct opa_smp *)recv->mad); 2221 else 2222 *resp_len = sizeof(struct ib_mad_hdr); 2223 } 2224 2225 return true; 2226 } else { 2227 return false; 2228 } 2229 } 2230 2231 static enum smi_action 2232 handle_opa_smi(struct ib_mad_port_private *port_priv, 2233 struct ib_mad_qp_info *qp_info, 2234 struct ib_wc *wc, 2235 u32 port_num, 2236 struct ib_mad_private *recv, 2237 struct ib_mad_private *response) 2238 { 2239 enum smi_forward_action retsmi; 2240 struct opa_smp *smp = (struct opa_smp *)recv->mad; 2241 2242 trace_ib_mad_handle_opa_smi(smp); 2243 2244 if (opa_smi_handle_dr_smp_recv(smp, 2245 rdma_cap_ib_switch(port_priv->device), 2246 port_num, 2247 port_priv->device->phys_port_cnt) == 2248 IB_SMI_DISCARD) 2249 return IB_SMI_DISCARD; 2250 2251 retsmi = opa_smi_check_forward_dr_smp(smp); 2252 if (retsmi == IB_SMI_LOCAL) 2253 return IB_SMI_HANDLE; 2254 2255 if (retsmi == IB_SMI_SEND) { /* don't forward */ 2256 if (opa_smi_handle_dr_smp_send(smp, 2257 rdma_cap_ib_switch(port_priv->device), 2258 port_num) == IB_SMI_DISCARD) 2259 return IB_SMI_DISCARD; 2260 2261 if (opa_smi_check_local_smp(smp, port_priv->device) == 2262 IB_SMI_DISCARD) 2263 return IB_SMI_DISCARD; 2264 2265 } else if (rdma_cap_ib_switch(port_priv->device)) { 2266 /* forward case for switches */ 2267 memcpy(response, recv, mad_priv_size(response)); 2268 response->header.recv_wc.wc = &response->header.wc; 2269 response->header.recv_wc.recv_buf.opa_mad = 2270 (struct opa_mad *)response->mad; 2271 response->header.recv_wc.recv_buf.grh = &response->grh; 2272 2273 agent_send_response((const struct ib_mad_hdr *)response->mad, 2274 &response->grh, wc, 2275 port_priv->device, 2276 opa_smi_get_fwd_port(smp), 2277 qp_info->qp->qp_num, 2278 recv->header.wc.byte_len, 2279 true); 2280 2281 return IB_SMI_DISCARD; 2282 } 2283 2284 return IB_SMI_HANDLE; 2285 } 2286 2287 static enum smi_action 2288 handle_smi(struct ib_mad_port_private *port_priv, 2289 struct ib_mad_qp_info *qp_info, 2290 struct ib_wc *wc, 2291 u32 port_num, 2292 struct ib_mad_private *recv, 2293 struct ib_mad_private *response, 2294 bool opa) 2295 { 2296 struct ib_mad_hdr *mad_hdr = (struct ib_mad_hdr *)recv->mad; 2297 2298 if (opa && mad_hdr->base_version == OPA_MGMT_BASE_VERSION && 2299 mad_hdr->class_version == OPA_SM_CLASS_VERSION) 2300 return handle_opa_smi(port_priv, qp_info, wc, port_num, recv, 2301 response); 2302 2303 return handle_ib_smi(port_priv, qp_info, wc, port_num, recv, response); 2304 } 2305 2306 static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc) 2307 { 2308 struct ib_mad_port_private *port_priv = cq->cq_context; 2309 struct ib_mad_list_head *mad_list = 2310 container_of(wc->wr_cqe, struct ib_mad_list_head, cqe); 2311 struct ib_mad_qp_info *qp_info; 2312 struct ib_mad_private_header *mad_priv_hdr; 2313 struct ib_mad_private *recv, *response = NULL; 2314 struct ib_mad_agent_private *mad_agent; 2315 u32 port_num; 2316 int ret = IB_MAD_RESULT_SUCCESS; 2317 size_t mad_size; 2318 u16 resp_mad_pkey_index = 0; 2319 bool opa; 2320 2321 if (list_empty_careful(&port_priv->port_list)) 2322 return; 2323 2324 if (wc->status != IB_WC_SUCCESS) { 2325 /* 2326 * Receive errors indicate that the QP has entered the error 2327 * state - error handling/shutdown code will cleanup 2328 */ 2329 return; 2330 } 2331 2332 qp_info = mad_list->mad_queue->qp_info; 2333 dequeue_mad(mad_list); 2334 2335 opa = rdma_cap_opa_mad(qp_info->port_priv->device, 2336 qp_info->port_priv->port_num); 2337 2338 mad_priv_hdr = container_of(mad_list, struct ib_mad_private_header, 2339 mad_list); 2340 recv = container_of(mad_priv_hdr, struct ib_mad_private, header); 2341 ib_dma_unmap_single(port_priv->device, 2342 recv->header.mapping, 2343 mad_priv_dma_size(recv), 2344 DMA_FROM_DEVICE); 2345 2346 /* Setup MAD receive work completion from "normal" work completion */ 2347 recv->header.wc = *wc; 2348 recv->header.recv_wc.wc = &recv->header.wc; 2349 2350 if (opa && ((struct ib_mad_hdr *)(recv->mad))->base_version == OPA_MGMT_BASE_VERSION) { 2351 recv->header.recv_wc.mad_len = wc->byte_len - sizeof(struct ib_grh); 2352 recv->header.recv_wc.mad_seg_size = sizeof(struct opa_mad); 2353 } else { 2354 recv->header.recv_wc.mad_len = sizeof(struct ib_mad); 2355 recv->header.recv_wc.mad_seg_size = sizeof(struct ib_mad); 2356 } 2357 2358 recv->header.recv_wc.recv_buf.mad = (struct ib_mad *)recv->mad; 2359 recv->header.recv_wc.recv_buf.grh = &recv->grh; 2360 2361 /* Validate MAD */ 2362 if (!validate_mad((const struct ib_mad_hdr *)recv->mad, qp_info, opa)) 2363 goto out; 2364 2365 trace_ib_mad_recv_done_handler(qp_info, wc, 2366 (struct ib_mad_hdr *)recv->mad); 2367 2368 mad_size = recv->mad_size; 2369 response = alloc_mad_private(mad_size, GFP_KERNEL); 2370 if (!response) 2371 goto out; 2372 2373 if (rdma_cap_ib_switch(port_priv->device)) 2374 port_num = wc->port_num; 2375 else 2376 port_num = port_priv->port_num; 2377 2378 if (((struct ib_mad_hdr *)recv->mad)->mgmt_class == 2379 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) { 2380 if (handle_smi(port_priv, qp_info, wc, port_num, recv, 2381 response, opa) 2382 == IB_SMI_DISCARD) 2383 goto out; 2384 } 2385 2386 /* Give driver "right of first refusal" on incoming MAD */ 2387 if (port_priv->device->ops.process_mad) { 2388 ret = port_priv->device->ops.process_mad( 2389 port_priv->device, 0, port_priv->port_num, wc, 2390 &recv->grh, (const struct ib_mad *)recv->mad, 2391 (struct ib_mad *)response->mad, &mad_size, 2392 &resp_mad_pkey_index); 2393 2394 if (opa) 2395 wc->pkey_index = resp_mad_pkey_index; 2396 2397 if (ret & IB_MAD_RESULT_SUCCESS) { 2398 if (ret & IB_MAD_RESULT_CONSUMED) 2399 goto out; 2400 if (ret & IB_MAD_RESULT_REPLY) { 2401 agent_send_response((const struct ib_mad_hdr *)response->mad, 2402 &recv->grh, wc, 2403 port_priv->device, 2404 port_num, 2405 qp_info->qp->qp_num, 2406 mad_size, opa); 2407 goto out; 2408 } 2409 } 2410 } 2411 2412 mad_agent = find_mad_agent(port_priv, (const struct ib_mad_hdr *)recv->mad); 2413 if (mad_agent) { 2414 trace_ib_mad_recv_done_agent(mad_agent); 2415 ib_mad_complete_recv(mad_agent, &recv->header.recv_wc); 2416 /* 2417 * recv is freed up in error cases in ib_mad_complete_recv 2418 * or via recv_handler in ib_mad_complete_recv() 2419 */ 2420 recv = NULL; 2421 } else if ((ret & IB_MAD_RESULT_SUCCESS) && 2422 generate_unmatched_resp(recv, response, &mad_size, opa)) { 2423 agent_send_response((const struct ib_mad_hdr *)response->mad, &recv->grh, wc, 2424 port_priv->device, port_num, 2425 qp_info->qp->qp_num, mad_size, opa); 2426 } 2427 2428 out: 2429 /* Post another receive request for this QP */ 2430 if (response) { 2431 ib_mad_post_receive_mads(qp_info, response); 2432 kfree(recv); 2433 } else 2434 ib_mad_post_receive_mads(qp_info, recv); 2435 } 2436 2437 static void adjust_timeout(struct ib_mad_agent_private *mad_agent_priv) 2438 { 2439 struct ib_mad_send_wr_private *mad_send_wr; 2440 unsigned long delay; 2441 2442 if (list_empty(&mad_agent_priv->wait_list)) { 2443 cancel_delayed_work(&mad_agent_priv->timed_work); 2444 } else { 2445 mad_send_wr = list_entry(mad_agent_priv->wait_list.next, 2446 struct ib_mad_send_wr_private, 2447 agent_list); 2448 2449 if (time_after(mad_agent_priv->timeout, 2450 mad_send_wr->timeout)) { 2451 mad_agent_priv->timeout = mad_send_wr->timeout; 2452 delay = mad_send_wr->timeout - jiffies; 2453 if ((long)delay <= 0) 2454 delay = 1; 2455 mod_delayed_work(mad_agent_priv->qp_info->port_priv->wq, 2456 &mad_agent_priv->timed_work, delay); 2457 } 2458 } 2459 } 2460 2461 static void wait_for_response(struct ib_mad_send_wr_private *mad_send_wr) 2462 { 2463 struct ib_mad_agent_private *mad_agent_priv; 2464 unsigned long delay; 2465 2466 mad_agent_priv = mad_send_wr->mad_agent_priv; 2467 delay = mad_send_wr->timeout; 2468 change_mad_state(mad_send_wr, IB_MAD_STATE_WAIT_RESP); 2469 2470 /* Reschedule a work item if we have a shorter timeout */ 2471 if (mad_agent_priv->wait_list.next == &mad_send_wr->agent_list) 2472 mod_delayed_work(mad_agent_priv->qp_info->port_priv->wq, 2473 &mad_agent_priv->timed_work, delay); 2474 } 2475 2476 void ib_reset_mad_timeout(struct ib_mad_send_wr_private *mad_send_wr, 2477 unsigned long timeout_ms) 2478 { 2479 mad_send_wr->timeout = msecs_to_jiffies(timeout_ms); 2480 wait_for_response(mad_send_wr); 2481 } 2482 2483 /* 2484 * Process a send work completion 2485 */ 2486 void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr, 2487 struct ib_mad_send_wc *mad_send_wc) 2488 { 2489 struct ib_mad_agent_private *mad_agent_priv; 2490 unsigned long flags; 2491 int ret; 2492 2493 mad_agent_priv = mad_send_wr->mad_agent_priv; 2494 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2495 if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) { 2496 ret = ib_process_rmpp_send_wc(mad_send_wr, mad_send_wc); 2497 if (ret == IB_RMPP_RESULT_CONSUMED) 2498 goto done; 2499 } else 2500 ret = IB_RMPP_RESULT_UNHANDLED; 2501 2502 if (mad_send_wr->state == IB_MAD_STATE_CANCELED) 2503 mad_send_wc->status = IB_WC_WR_FLUSH_ERR; 2504 else if (mad_send_wr->state == IB_MAD_STATE_SEND_START && 2505 mad_send_wr->timeout) { 2506 wait_for_response(mad_send_wr); 2507 goto done; 2508 } 2509 2510 /* Remove send from MAD agent and notify client of completion */ 2511 if (mad_send_wr->state != IB_MAD_STATE_DONE) 2512 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE); 2513 adjust_timeout(mad_agent_priv); 2514 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2515 2516 if (ret == IB_RMPP_RESULT_INTERNAL) { 2517 ib_rmpp_send_handler(mad_send_wc); 2518 } else { 2519 if (mad_send_wr->is_solicited_fc) 2520 process_backlog_mads(mad_agent_priv); 2521 mad_agent_priv->agent.send_handler(&mad_agent_priv->agent, 2522 mad_send_wc); 2523 } 2524 2525 /* Release reference on agent taken when sending */ 2526 deref_mad_agent(mad_agent_priv); 2527 return; 2528 done: 2529 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2530 } 2531 2532 static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc) 2533 { 2534 struct ib_mad_port_private *port_priv = cq->cq_context; 2535 struct ib_mad_list_head *mad_list = 2536 container_of(wc->wr_cqe, struct ib_mad_list_head, cqe); 2537 struct ib_mad_send_wr_private *mad_send_wr, *queued_send_wr; 2538 struct ib_mad_qp_info *qp_info; 2539 struct ib_mad_queue *send_queue; 2540 struct ib_mad_send_wc mad_send_wc; 2541 unsigned long flags; 2542 int ret; 2543 2544 if (list_empty_careful(&port_priv->port_list)) 2545 return; 2546 2547 if (wc->status != IB_WC_SUCCESS) { 2548 if (!ib_mad_send_error(port_priv, wc)) 2549 return; 2550 } 2551 2552 mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private, 2553 mad_list); 2554 send_queue = mad_list->mad_queue; 2555 qp_info = send_queue->qp_info; 2556 2557 trace_ib_mad_send_done_agent(mad_send_wr->mad_agent_priv); 2558 trace_ib_mad_send_done_handler(mad_send_wr, wc); 2559 2560 retry: 2561 ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device, 2562 mad_send_wr->header_mapping, 2563 mad_send_wr->sg_list[0].length, DMA_TO_DEVICE); 2564 ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device, 2565 mad_send_wr->payload_mapping, 2566 mad_send_wr->sg_list[1].length, DMA_TO_DEVICE); 2567 queued_send_wr = NULL; 2568 spin_lock_irqsave(&send_queue->lock, flags); 2569 list_del(&mad_list->list); 2570 2571 /* Move queued send to the send queue */ 2572 if (send_queue->count-- > send_queue->max_active) { 2573 mad_list = container_of(qp_info->overflow_list.next, 2574 struct ib_mad_list_head, list); 2575 queued_send_wr = container_of(mad_list, 2576 struct ib_mad_send_wr_private, 2577 mad_list); 2578 list_move_tail(&mad_list->list, &send_queue->list); 2579 } 2580 spin_unlock_irqrestore(&send_queue->lock, flags); 2581 2582 mad_send_wc.send_buf = &mad_send_wr->send_buf; 2583 mad_send_wc.status = wc->status; 2584 mad_send_wc.vendor_err = wc->vendor_err; 2585 ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc); 2586 2587 if (queued_send_wr) { 2588 trace_ib_mad_send_done_resend(queued_send_wr, qp_info); 2589 ret = ib_post_send(qp_info->qp, &queued_send_wr->send_wr.wr, 2590 NULL); 2591 if (ret) { 2592 dev_err(&port_priv->device->dev, 2593 "ib_post_send failed: %d\n", ret); 2594 mad_send_wr = queued_send_wr; 2595 wc->status = IB_WC_LOC_QP_OP_ERR; 2596 goto retry; 2597 } 2598 } 2599 } 2600 2601 static void mark_sends_for_retry(struct ib_mad_qp_info *qp_info) 2602 { 2603 struct ib_mad_send_wr_private *mad_send_wr; 2604 struct ib_mad_list_head *mad_list; 2605 unsigned long flags; 2606 2607 spin_lock_irqsave(&qp_info->send_queue.lock, flags); 2608 list_for_each_entry(mad_list, &qp_info->send_queue.list, list) { 2609 mad_send_wr = container_of(mad_list, 2610 struct ib_mad_send_wr_private, 2611 mad_list); 2612 mad_send_wr->retry = 1; 2613 } 2614 spin_unlock_irqrestore(&qp_info->send_queue.lock, flags); 2615 } 2616 2617 static bool ib_mad_send_error(struct ib_mad_port_private *port_priv, 2618 struct ib_wc *wc) 2619 { 2620 struct ib_mad_list_head *mad_list = 2621 container_of(wc->wr_cqe, struct ib_mad_list_head, cqe); 2622 struct ib_mad_qp_info *qp_info = mad_list->mad_queue->qp_info; 2623 struct ib_mad_send_wr_private *mad_send_wr; 2624 int ret; 2625 2626 /* 2627 * Send errors will transition the QP to SQE - move 2628 * QP to RTS and repost flushed work requests 2629 */ 2630 mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private, 2631 mad_list); 2632 if (wc->status == IB_WC_WR_FLUSH_ERR) { 2633 if (mad_send_wr->retry) { 2634 /* Repost send */ 2635 mad_send_wr->retry = 0; 2636 trace_ib_mad_error_handler(mad_send_wr, qp_info); 2637 ret = ib_post_send(qp_info->qp, &mad_send_wr->send_wr.wr, 2638 NULL); 2639 if (!ret) 2640 return false; 2641 } 2642 } else { 2643 struct ib_qp_attr *attr; 2644 2645 /* Transition QP to RTS and fail offending send */ 2646 attr = kmalloc_obj(*attr); 2647 if (attr) { 2648 attr->qp_state = IB_QPS_RTS; 2649 attr->cur_qp_state = IB_QPS_SQE; 2650 ret = ib_modify_qp(qp_info->qp, attr, 2651 IB_QP_STATE | IB_QP_CUR_STATE); 2652 kfree(attr); 2653 if (ret) 2654 dev_err(&port_priv->device->dev, 2655 "%s - ib_modify_qp to RTS: %d\n", 2656 __func__, ret); 2657 else 2658 mark_sends_for_retry(qp_info); 2659 } 2660 } 2661 2662 return true; 2663 } 2664 2665 static void clear_mad_error_list(struct list_head *list, 2666 enum ib_wc_status wc_status, 2667 struct ib_mad_agent_private *mad_agent_priv) 2668 { 2669 struct ib_mad_send_wr_private *mad_send_wr, *n; 2670 struct ib_mad_send_wc mad_send_wc; 2671 2672 mad_send_wc.status = wc_status; 2673 mad_send_wc.vendor_err = 0; 2674 2675 list_for_each_entry_safe(mad_send_wr, n, list, agent_list) { 2676 mad_send_wc.send_buf = &mad_send_wr->send_buf; 2677 mad_agent_priv->agent.send_handler(&mad_agent_priv->agent, 2678 &mad_send_wc); 2679 deref_mad_agent(mad_agent_priv); 2680 } 2681 } 2682 2683 static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv) 2684 { 2685 unsigned long flags; 2686 struct ib_mad_send_wr_private *mad_send_wr, *temp_mad_send_wr; 2687 struct list_head cancel_list; 2688 2689 INIT_LIST_HEAD(&cancel_list); 2690 2691 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2692 list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr, 2693 &mad_agent_priv->send_list, agent_list) 2694 change_mad_state(mad_send_wr, IB_MAD_STATE_CANCELED); 2695 2696 /* Empty wait & backlog list to prevent receives from finding request */ 2697 list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr, 2698 &mad_agent_priv->wait_list, agent_list) { 2699 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE); 2700 list_add_tail(&mad_send_wr->agent_list, &cancel_list); 2701 } 2702 2703 list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr, 2704 &mad_agent_priv->backlog_list, agent_list) { 2705 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE); 2706 list_add_tail(&mad_send_wr->agent_list, &cancel_list); 2707 } 2708 2709 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2710 /* Report all cancelled requests */ 2711 clear_mad_error_list(&cancel_list, IB_WC_WR_FLUSH_ERR, mad_agent_priv); 2712 } 2713 2714 static struct ib_mad_send_wr_private* 2715 find_send_wr(struct ib_mad_agent_private *mad_agent_priv, 2716 struct ib_mad_send_buf *send_buf) 2717 { 2718 struct ib_mad_send_wr_private *mad_send_wr; 2719 2720 list_for_each_entry(mad_send_wr, &mad_agent_priv->wait_list, 2721 agent_list) { 2722 if (&mad_send_wr->send_buf == send_buf) 2723 return mad_send_wr; 2724 } 2725 2726 list_for_each_entry(mad_send_wr, &mad_agent_priv->send_list, 2727 agent_list) { 2728 if (is_rmpp_data_mad(mad_agent_priv, 2729 mad_send_wr->send_buf.mad) && 2730 &mad_send_wr->send_buf == send_buf) 2731 return mad_send_wr; 2732 } 2733 2734 list_for_each_entry(mad_send_wr, &mad_agent_priv->backlog_list, 2735 agent_list) { 2736 if (&mad_send_wr->send_buf == send_buf) 2737 return mad_send_wr; 2738 } 2739 2740 return NULL; 2741 } 2742 2743 int ib_modify_mad(struct ib_mad_send_buf *send_buf, u32 timeout_ms) 2744 { 2745 struct ib_mad_agent_private *mad_agent_priv; 2746 struct ib_mad_send_wr_private *mad_send_wr; 2747 unsigned long flags; 2748 int active; 2749 2750 if (!send_buf) 2751 return -EINVAL; 2752 2753 mad_agent_priv = container_of(send_buf->mad_agent, 2754 struct ib_mad_agent_private, agent); 2755 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2756 mad_send_wr = find_send_wr(mad_agent_priv, send_buf); 2757 if (!mad_send_wr || mad_send_wr->state == IB_MAD_STATE_CANCELED) { 2758 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2759 return -EINVAL; 2760 } 2761 2762 active = ((mad_send_wr->state == IB_MAD_STATE_SEND_START) || 2763 (mad_send_wr->state == IB_MAD_STATE_EARLY_RESP) || 2764 (mad_send_wr->state == IB_MAD_STATE_QUEUED && timeout_ms)); 2765 if (!timeout_ms) 2766 change_mad_state(mad_send_wr, IB_MAD_STATE_CANCELED); 2767 2768 mad_send_wr->send_buf.timeout_ms = timeout_ms; 2769 if (active) 2770 mad_send_wr->timeout = msecs_to_jiffies(timeout_ms); 2771 else 2772 ib_reset_mad_timeout(mad_send_wr, timeout_ms); 2773 2774 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2775 return 0; 2776 } 2777 EXPORT_SYMBOL(ib_modify_mad); 2778 2779 static void local_completions(struct work_struct *work) 2780 { 2781 struct ib_mad_agent_private *mad_agent_priv; 2782 struct ib_mad_local_private *local; 2783 struct ib_mad_agent_private *recv_mad_agent; 2784 unsigned long flags; 2785 int free_mad; 2786 struct ib_wc wc; 2787 struct ib_mad_send_wc mad_send_wc; 2788 bool opa; 2789 2790 mad_agent_priv = 2791 container_of(work, struct ib_mad_agent_private, local_work); 2792 2793 opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device, 2794 mad_agent_priv->qp_info->port_priv->port_num); 2795 2796 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2797 while (!list_empty(&mad_agent_priv->local_list)) { 2798 local = list_entry(mad_agent_priv->local_list.next, 2799 struct ib_mad_local_private, 2800 completion_list); 2801 list_del(&local->completion_list); 2802 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2803 free_mad = 0; 2804 if (local->mad_priv) { 2805 u8 base_version; 2806 recv_mad_agent = local->recv_mad_agent; 2807 if (!recv_mad_agent) { 2808 dev_err(&mad_agent_priv->agent.device->dev, 2809 "No receive MAD agent for local completion\n"); 2810 free_mad = 1; 2811 goto local_send_completion; 2812 } 2813 2814 /* 2815 * Defined behavior is to complete response 2816 * before request 2817 */ 2818 build_smp_wc(recv_mad_agent->agent.qp, 2819 local->mad_send_wr->send_wr.wr.wr_cqe, 2820 be16_to_cpu(IB_LID_PERMISSIVE), 2821 local->mad_send_wr->send_wr.pkey_index, 2822 recv_mad_agent->agent.port_num, &wc); 2823 2824 local->mad_priv->header.recv_wc.wc = &wc; 2825 2826 base_version = ((struct ib_mad_hdr *)(local->mad_priv->mad))->base_version; 2827 if (opa && base_version == OPA_MGMT_BASE_VERSION) { 2828 local->mad_priv->header.recv_wc.mad_len = local->return_wc_byte_len; 2829 local->mad_priv->header.recv_wc.mad_seg_size = sizeof(struct opa_mad); 2830 } else { 2831 local->mad_priv->header.recv_wc.mad_len = sizeof(struct ib_mad); 2832 local->mad_priv->header.recv_wc.mad_seg_size = sizeof(struct ib_mad); 2833 } 2834 2835 INIT_LIST_HEAD(&local->mad_priv->header.recv_wc.rmpp_list); 2836 list_add(&local->mad_priv->header.recv_wc.recv_buf.list, 2837 &local->mad_priv->header.recv_wc.rmpp_list); 2838 local->mad_priv->header.recv_wc.recv_buf.grh = NULL; 2839 local->mad_priv->header.recv_wc.recv_buf.mad = 2840 (struct ib_mad *)local->mad_priv->mad; 2841 recv_mad_agent->agent.recv_handler( 2842 &recv_mad_agent->agent, 2843 &local->mad_send_wr->send_buf, 2844 &local->mad_priv->header.recv_wc); 2845 spin_lock_irqsave(&recv_mad_agent->lock, flags); 2846 deref_mad_agent(recv_mad_agent); 2847 spin_unlock_irqrestore(&recv_mad_agent->lock, flags); 2848 } 2849 2850 local_send_completion: 2851 /* Complete send */ 2852 mad_send_wc.status = IB_WC_SUCCESS; 2853 mad_send_wc.vendor_err = 0; 2854 mad_send_wc.send_buf = &local->mad_send_wr->send_buf; 2855 mad_agent_priv->agent.send_handler(&mad_agent_priv->agent, 2856 &mad_send_wc); 2857 2858 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2859 deref_mad_agent(mad_agent_priv); 2860 if (free_mad) 2861 kfree(local->mad_priv); 2862 kfree(local); 2863 } 2864 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2865 } 2866 2867 static int retry_send(struct ib_mad_send_wr_private *mad_send_wr) 2868 { 2869 int ret; 2870 2871 if (!mad_send_wr->retries_left) 2872 return -ETIMEDOUT; 2873 2874 mad_send_wr->retries_left--; 2875 mad_send_wr->send_buf.retries++; 2876 2877 mad_send_wr->timeout = msecs_to_jiffies(mad_send_wr->send_buf.timeout_ms); 2878 if (mad_send_wr->is_solicited_fc && 2879 !list_empty(&mad_send_wr->mad_agent_priv->backlog_list)) { 2880 change_mad_state(mad_send_wr, IB_MAD_STATE_QUEUED); 2881 return 0; 2882 } 2883 2884 if (ib_mad_kernel_rmpp_agent(&mad_send_wr->mad_agent_priv->agent)) { 2885 ret = ib_retry_rmpp(mad_send_wr); 2886 switch (ret) { 2887 case IB_RMPP_RESULT_UNHANDLED: 2888 ret = ib_send_mad(mad_send_wr); 2889 break; 2890 case IB_RMPP_RESULT_CONSUMED: 2891 ret = 0; 2892 break; 2893 default: 2894 ret = -ECOMM; 2895 break; 2896 } 2897 } else 2898 ret = ib_send_mad(mad_send_wr); 2899 2900 if (!ret) 2901 change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START); 2902 2903 return ret; 2904 } 2905 2906 static void timeout_sends(struct work_struct *work) 2907 { 2908 struct ib_mad_send_wr_private *mad_send_wr; 2909 struct ib_mad_agent_private *mad_agent_priv; 2910 struct list_head timeout_list; 2911 struct list_head cancel_list; 2912 struct list_head *list_item; 2913 unsigned long flags, delay; 2914 2915 mad_agent_priv = container_of(work, struct ib_mad_agent_private, 2916 timed_work.work); 2917 INIT_LIST_HEAD(&timeout_list); 2918 INIT_LIST_HEAD(&cancel_list); 2919 2920 spin_lock_irqsave(&mad_agent_priv->lock, flags); 2921 while (!list_empty(&mad_agent_priv->wait_list)) { 2922 mad_send_wr = list_entry(mad_agent_priv->wait_list.next, 2923 struct ib_mad_send_wr_private, 2924 agent_list); 2925 2926 if (time_after(mad_send_wr->timeout, jiffies)) { 2927 delay = mad_send_wr->timeout - jiffies; 2928 if ((long)delay <= 0) 2929 delay = 1; 2930 queue_delayed_work(mad_agent_priv->qp_info-> 2931 port_priv->wq, 2932 &mad_agent_priv->timed_work, delay); 2933 break; 2934 } 2935 2936 if (mad_send_wr->state == IB_MAD_STATE_CANCELED) 2937 list_item = &cancel_list; 2938 else if (retry_send(mad_send_wr)) 2939 list_item = &timeout_list; 2940 else 2941 continue; 2942 2943 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE); 2944 list_add_tail(&mad_send_wr->agent_list, list_item); 2945 } 2946 2947 spin_unlock_irqrestore(&mad_agent_priv->lock, flags); 2948 process_backlog_mads(mad_agent_priv); 2949 clear_mad_error_list(&timeout_list, IB_WC_RESP_TIMEOUT_ERR, 2950 mad_agent_priv); 2951 clear_mad_error_list(&cancel_list, IB_WC_WR_FLUSH_ERR, mad_agent_priv); 2952 } 2953 2954 /* 2955 * Allocate receive MADs and post receive WRs for them 2956 */ 2957 static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info, 2958 struct ib_mad_private *mad) 2959 { 2960 unsigned long flags; 2961 struct ib_mad_private *mad_priv; 2962 struct ib_sge sg_list; 2963 struct ib_recv_wr recv_wr; 2964 struct ib_mad_queue *recv_queue = &qp_info->recv_queue; 2965 int ret = 0; 2966 2967 /* Initialize common scatter list fields */ 2968 sg_list.lkey = qp_info->port_priv->pd->local_dma_lkey; 2969 2970 /* Initialize common receive WR fields */ 2971 recv_wr.next = NULL; 2972 recv_wr.sg_list = &sg_list; 2973 recv_wr.num_sge = 1; 2974 2975 while (true) { 2976 /* Allocate and map receive buffer */ 2977 if (mad) { 2978 mad_priv = mad; 2979 mad = NULL; 2980 } else { 2981 mad_priv = alloc_mad_private(port_mad_size(qp_info->port_priv), 2982 GFP_ATOMIC); 2983 if (!mad_priv) 2984 return -ENOMEM; 2985 } 2986 sg_list.length = mad_priv_dma_size(mad_priv); 2987 sg_list.addr = ib_dma_map_single(qp_info->port_priv->device, 2988 &mad_priv->grh, 2989 mad_priv_dma_size(mad_priv), 2990 DMA_FROM_DEVICE); 2991 if (unlikely(ib_dma_mapping_error(qp_info->port_priv->device, 2992 sg_list.addr))) { 2993 ret = -ENOMEM; 2994 goto free_mad_priv; 2995 } 2996 mad_priv->header.mapping = sg_list.addr; 2997 mad_priv->header.mad_list.mad_queue = recv_queue; 2998 mad_priv->header.mad_list.cqe.done = ib_mad_recv_done; 2999 recv_wr.wr_cqe = &mad_priv->header.mad_list.cqe; 3000 spin_lock_irqsave(&recv_queue->lock, flags); 3001 if (recv_queue->count >= recv_queue->max_active) { 3002 /* Fully populated the receive queue */ 3003 spin_unlock_irqrestore(&recv_queue->lock, flags); 3004 break; 3005 } 3006 recv_queue->count++; 3007 list_add_tail(&mad_priv->header.mad_list.list, 3008 &recv_queue->list); 3009 spin_unlock_irqrestore(&recv_queue->lock, flags); 3010 3011 ret = ib_post_recv(qp_info->qp, &recv_wr, NULL); 3012 if (ret) { 3013 spin_lock_irqsave(&recv_queue->lock, flags); 3014 list_del(&mad_priv->header.mad_list.list); 3015 recv_queue->count--; 3016 spin_unlock_irqrestore(&recv_queue->lock, flags); 3017 dev_err(&qp_info->port_priv->device->dev, 3018 "ib_post_recv failed: %d\n", ret); 3019 break; 3020 } 3021 } 3022 3023 ib_dma_unmap_single(qp_info->port_priv->device, 3024 mad_priv->header.mapping, 3025 mad_priv_dma_size(mad_priv), DMA_FROM_DEVICE); 3026 free_mad_priv: 3027 kfree(mad_priv); 3028 return ret; 3029 } 3030 3031 /* 3032 * Return all the posted receive MADs 3033 */ 3034 static void cleanup_recv_queue(struct ib_mad_qp_info *qp_info) 3035 { 3036 struct ib_mad_private_header *mad_priv_hdr; 3037 struct ib_mad_private *recv; 3038 struct ib_mad_list_head *mad_list; 3039 3040 if (!qp_info->qp) 3041 return; 3042 3043 while (!list_empty(&qp_info->recv_queue.list)) { 3044 3045 mad_list = list_entry(qp_info->recv_queue.list.next, 3046 struct ib_mad_list_head, list); 3047 mad_priv_hdr = container_of(mad_list, 3048 struct ib_mad_private_header, 3049 mad_list); 3050 recv = container_of(mad_priv_hdr, struct ib_mad_private, 3051 header); 3052 3053 /* Remove from posted receive MAD list */ 3054 list_del(&mad_list->list); 3055 3056 ib_dma_unmap_single(qp_info->port_priv->device, 3057 recv->header.mapping, 3058 mad_priv_dma_size(recv), 3059 DMA_FROM_DEVICE); 3060 kfree(recv); 3061 } 3062 3063 qp_info->recv_queue.count = 0; 3064 } 3065 3066 /* 3067 * Start the port 3068 */ 3069 static int ib_mad_port_start(struct ib_mad_port_private *port_priv) 3070 { 3071 int ret, i; 3072 struct ib_qp_attr *attr; 3073 struct ib_qp *qp; 3074 u16 pkey_index; 3075 3076 attr = kmalloc_obj(*attr); 3077 if (!attr) 3078 return -ENOMEM; 3079 3080 ret = ib_find_pkey(port_priv->device, port_priv->port_num, 3081 IB_DEFAULT_PKEY_FULL, &pkey_index); 3082 if (ret) 3083 pkey_index = 0; 3084 3085 for (i = 0; i < IB_MAD_QPS_CORE; i++) { 3086 qp = port_priv->qp_info[i].qp; 3087 if (!qp) 3088 continue; 3089 3090 /* 3091 * PKey index for QP1 is irrelevant but 3092 * one is needed for the Reset to Init transition 3093 */ 3094 attr->qp_state = IB_QPS_INIT; 3095 attr->pkey_index = pkey_index; 3096 attr->qkey = (qp->qp_num == 0) ? 0 : IB_QP1_QKEY; 3097 ret = ib_modify_qp(qp, attr, IB_QP_STATE | 3098 IB_QP_PKEY_INDEX | IB_QP_QKEY); 3099 if (ret) { 3100 dev_err(&port_priv->device->dev, 3101 "Couldn't change QP%d state to INIT: %d\n", 3102 i, ret); 3103 goto out; 3104 } 3105 3106 attr->qp_state = IB_QPS_RTR; 3107 ret = ib_modify_qp(qp, attr, IB_QP_STATE); 3108 if (ret) { 3109 dev_err(&port_priv->device->dev, 3110 "Couldn't change QP%d state to RTR: %d\n", 3111 i, ret); 3112 goto out; 3113 } 3114 3115 attr->qp_state = IB_QPS_RTS; 3116 attr->sq_psn = IB_MAD_SEND_Q_PSN; 3117 ret = ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_SQ_PSN); 3118 if (ret) { 3119 dev_err(&port_priv->device->dev, 3120 "Couldn't change QP%d state to RTS: %d\n", 3121 i, ret); 3122 goto out; 3123 } 3124 } 3125 3126 ret = ib_req_notify_cq(port_priv->cq, IB_CQ_NEXT_COMP); 3127 if (ret) { 3128 dev_err(&port_priv->device->dev, 3129 "Failed to request completion notification: %d\n", 3130 ret); 3131 goto out; 3132 } 3133 3134 for (i = 0; i < IB_MAD_QPS_CORE; i++) { 3135 if (!port_priv->qp_info[i].qp) 3136 continue; 3137 3138 ret = ib_mad_post_receive_mads(&port_priv->qp_info[i], NULL); 3139 if (ret) { 3140 dev_err(&port_priv->device->dev, 3141 "Couldn't post receive WRs\n"); 3142 goto out; 3143 } 3144 } 3145 out: 3146 kfree(attr); 3147 return ret; 3148 } 3149 3150 static void qp_event_handler(struct ib_event *event, void *qp_context) 3151 { 3152 struct ib_mad_qp_info *qp_info = qp_context; 3153 3154 /* It's worse than that! He's dead, Jim! */ 3155 dev_err(&qp_info->port_priv->device->dev, 3156 "Fatal error (%d) on MAD QP (%u)\n", 3157 event->event, qp_info->qp->qp_num); 3158 } 3159 3160 static void init_mad_queue(struct ib_mad_qp_info *qp_info, 3161 struct ib_mad_queue *mad_queue) 3162 { 3163 mad_queue->qp_info = qp_info; 3164 mad_queue->count = 0; 3165 spin_lock_init(&mad_queue->lock); 3166 INIT_LIST_HEAD(&mad_queue->list); 3167 } 3168 3169 static void init_mad_qp(struct ib_mad_port_private *port_priv, 3170 struct ib_mad_qp_info *qp_info) 3171 { 3172 qp_info->port_priv = port_priv; 3173 init_mad_queue(qp_info, &qp_info->send_queue); 3174 init_mad_queue(qp_info, &qp_info->recv_queue); 3175 INIT_LIST_HEAD(&qp_info->overflow_list); 3176 } 3177 3178 static int create_mad_qp(struct ib_mad_qp_info *qp_info, 3179 enum ib_qp_type qp_type) 3180 { 3181 struct ib_qp_init_attr qp_init_attr; 3182 int ret; 3183 3184 memset(&qp_init_attr, 0, sizeof qp_init_attr); 3185 qp_init_attr.send_cq = qp_info->port_priv->cq; 3186 qp_init_attr.recv_cq = qp_info->port_priv->cq; 3187 qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR; 3188 qp_init_attr.cap.max_send_wr = mad_sendq_size; 3189 qp_init_attr.cap.max_recv_wr = mad_recvq_size; 3190 qp_init_attr.cap.max_send_sge = IB_MAD_SEND_REQ_MAX_SG; 3191 qp_init_attr.cap.max_recv_sge = IB_MAD_RECV_REQ_MAX_SG; 3192 qp_init_attr.qp_type = qp_type; 3193 qp_init_attr.port_num = qp_info->port_priv->port_num; 3194 qp_init_attr.qp_context = qp_info; 3195 qp_init_attr.event_handler = qp_event_handler; 3196 qp_info->qp = ib_create_qp(qp_info->port_priv->pd, &qp_init_attr); 3197 if (IS_ERR(qp_info->qp)) { 3198 dev_err(&qp_info->port_priv->device->dev, 3199 "Couldn't create ib_mad QP%d\n", 3200 get_spl_qp_index(qp_type)); 3201 ret = PTR_ERR(qp_info->qp); 3202 goto error; 3203 } 3204 /* Use minimum queue sizes unless the CQ is resized */ 3205 qp_info->send_queue.max_active = mad_sendq_size; 3206 qp_info->recv_queue.max_active = mad_recvq_size; 3207 return 0; 3208 3209 error: 3210 return ret; 3211 } 3212 3213 static void destroy_mad_qp(struct ib_mad_qp_info *qp_info) 3214 { 3215 if (!qp_info->qp) 3216 return; 3217 3218 ib_destroy_qp(qp_info->qp); 3219 } 3220 3221 /* 3222 * Open the port 3223 * Create the QP, PD, MR, and CQ if needed 3224 */ 3225 static int ib_mad_port_open(struct ib_device *device, 3226 u32 port_num) 3227 { 3228 int ret, cq_size; 3229 struct ib_mad_port_private *port_priv; 3230 unsigned long flags; 3231 int has_smi; 3232 3233 if (WARN_ON(rdma_max_mad_size(device, port_num) < IB_MGMT_MAD_SIZE)) 3234 return -EFAULT; 3235 3236 if (WARN_ON(rdma_cap_opa_mad(device, port_num) && 3237 rdma_max_mad_size(device, port_num) < OPA_MGMT_MAD_SIZE)) 3238 return -EFAULT; 3239 3240 /* Create new device info */ 3241 port_priv = kzalloc_obj(*port_priv); 3242 if (!port_priv) 3243 return -ENOMEM; 3244 3245 port_priv->device = device; 3246 port_priv->port_num = port_num; 3247 spin_lock_init(&port_priv->reg_lock); 3248 init_mad_qp(port_priv, &port_priv->qp_info[0]); 3249 init_mad_qp(port_priv, &port_priv->qp_info[1]); 3250 3251 cq_size = mad_sendq_size + mad_recvq_size; 3252 has_smi = rdma_cap_ib_smi(device, port_num); 3253 if (has_smi) 3254 cq_size *= 2; 3255 3256 port_priv->pd = ib_alloc_pd(device, 0); 3257 if (IS_ERR(port_priv->pd)) { 3258 dev_err(&device->dev, "Couldn't create ib_mad PD\n"); 3259 ret = PTR_ERR(port_priv->pd); 3260 goto error3; 3261 } 3262 3263 port_priv->cq = ib_alloc_cq(port_priv->device, port_priv, cq_size, 0, 3264 IB_POLL_UNBOUND_WORKQUEUE); 3265 if (IS_ERR(port_priv->cq)) { 3266 dev_err(&device->dev, "Couldn't create ib_mad CQ\n"); 3267 ret = PTR_ERR(port_priv->cq); 3268 goto error4; 3269 } 3270 3271 if (has_smi) { 3272 ret = create_mad_qp(&port_priv->qp_info[0], IB_QPT_SMI); 3273 if (ret) 3274 goto error6; 3275 } 3276 3277 if (rdma_cap_ib_cm(device, port_num)) { 3278 ret = create_mad_qp(&port_priv->qp_info[1], IB_QPT_GSI); 3279 if (ret) 3280 goto error7; 3281 } 3282 3283 port_priv->wq = alloc_ordered_workqueue("ib_mad%u", WQ_MEM_RECLAIM, 3284 port_num); 3285 if (!port_priv->wq) { 3286 ret = -ENOMEM; 3287 goto error8; 3288 } 3289 3290 spin_lock_irqsave(&ib_mad_port_list_lock, flags); 3291 list_add_tail(&port_priv->port_list, &ib_mad_port_list); 3292 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags); 3293 3294 ret = ib_mad_port_start(port_priv); 3295 if (ret) { 3296 dev_err(&device->dev, "Couldn't start port\n"); 3297 goto error9; 3298 } 3299 3300 return 0; 3301 3302 error9: 3303 spin_lock_irqsave(&ib_mad_port_list_lock, flags); 3304 list_del_init(&port_priv->port_list); 3305 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags); 3306 3307 destroy_workqueue(port_priv->wq); 3308 error8: 3309 destroy_mad_qp(&port_priv->qp_info[1]); 3310 error7: 3311 destroy_mad_qp(&port_priv->qp_info[0]); 3312 error6: 3313 ib_free_cq(port_priv->cq); 3314 cleanup_recv_queue(&port_priv->qp_info[1]); 3315 cleanup_recv_queue(&port_priv->qp_info[0]); 3316 error4: 3317 ib_dealloc_pd(port_priv->pd); 3318 error3: 3319 kfree(port_priv); 3320 3321 return ret; 3322 } 3323 3324 /* 3325 * Close the port 3326 * If there are no classes using the port, free the port 3327 * resources (CQ, MR, PD, QP) and remove the port's info structure 3328 */ 3329 static int ib_mad_port_close(struct ib_device *device, u32 port_num) 3330 { 3331 struct ib_mad_port_private *port_priv; 3332 unsigned long flags; 3333 3334 spin_lock_irqsave(&ib_mad_port_list_lock, flags); 3335 port_priv = __ib_get_mad_port(device, port_num); 3336 if (port_priv == NULL) { 3337 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags); 3338 dev_err(&device->dev, "Port %u not found\n", port_num); 3339 return -ENODEV; 3340 } 3341 list_del_init(&port_priv->port_list); 3342 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags); 3343 3344 destroy_workqueue(port_priv->wq); 3345 destroy_mad_qp(&port_priv->qp_info[1]); 3346 destroy_mad_qp(&port_priv->qp_info[0]); 3347 ib_free_cq(port_priv->cq); 3348 ib_dealloc_pd(port_priv->pd); 3349 cleanup_recv_queue(&port_priv->qp_info[1]); 3350 cleanup_recv_queue(&port_priv->qp_info[0]); 3351 /* XXX: Handle deallocation of MAD registration tables */ 3352 3353 kfree(port_priv); 3354 3355 return 0; 3356 } 3357 3358 static int ib_mad_init_device(struct ib_device *device) 3359 { 3360 int start, i; 3361 unsigned int count = 0; 3362 int ret; 3363 3364 start = rdma_start_port(device); 3365 3366 for (i = start; i <= rdma_end_port(device); i++) { 3367 if (!rdma_cap_ib_mad(device, i)) 3368 continue; 3369 3370 ret = ib_mad_port_open(device, i); 3371 if (ret) { 3372 dev_err(&device->dev, "Couldn't open port %d\n", i); 3373 goto error; 3374 } 3375 ret = ib_agent_port_open(device, i); 3376 if (ret) { 3377 dev_err(&device->dev, 3378 "Couldn't open port %d for agents\n", i); 3379 goto error_agent; 3380 } 3381 count++; 3382 } 3383 if (!count) 3384 return -EOPNOTSUPP; 3385 3386 return 0; 3387 3388 error_agent: 3389 if (ib_mad_port_close(device, i)) 3390 dev_err(&device->dev, "Couldn't close port %d\n", i); 3391 3392 error: 3393 while (--i >= start) { 3394 if (!rdma_cap_ib_mad(device, i)) 3395 continue; 3396 3397 if (ib_agent_port_close(device, i)) 3398 dev_err(&device->dev, 3399 "Couldn't close port %d for agents\n", i); 3400 if (ib_mad_port_close(device, i)) 3401 dev_err(&device->dev, "Couldn't close port %d\n", i); 3402 } 3403 return ret; 3404 } 3405 3406 static void ib_mad_remove_device(struct ib_device *device, void *client_data) 3407 { 3408 unsigned int i; 3409 3410 rdma_for_each_port (device, i) { 3411 if (!rdma_cap_ib_mad(device, i)) 3412 continue; 3413 3414 if (ib_agent_port_close(device, i)) 3415 dev_err(&device->dev, 3416 "Couldn't close port %u for agents\n", i); 3417 if (ib_mad_port_close(device, i)) 3418 dev_err(&device->dev, "Couldn't close port %u\n", i); 3419 } 3420 } 3421 3422 static struct ib_client mad_client = { 3423 .name = "mad", 3424 .add = ib_mad_init_device, 3425 .remove = ib_mad_remove_device 3426 }; 3427 3428 int ib_mad_init(void) 3429 { 3430 mad_recvq_size = min(mad_recvq_size, IB_MAD_QP_MAX_SIZE); 3431 mad_recvq_size = max(mad_recvq_size, IB_MAD_QP_MIN_SIZE); 3432 3433 mad_sendq_size = min(mad_sendq_size, IB_MAD_QP_MAX_SIZE); 3434 mad_sendq_size = max(mad_sendq_size, IB_MAD_QP_MIN_SIZE); 3435 3436 INIT_LIST_HEAD(&ib_mad_port_list); 3437 3438 if (ib_register_client(&mad_client)) { 3439 pr_err("Couldn't register ib_mad client\n"); 3440 return -EINVAL; 3441 } 3442 3443 return 0; 3444 } 3445 3446 void ib_mad_cleanup(void) 3447 { 3448 ib_unregister_client(&mad_client); 3449 } 3450