1 /* 2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions are met: 6 * 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. Neither the names of the copyright holders nor the names of its 13 * contributors may be used to endorse or promote products derived from 14 * this software without specific prior written permission. 15 * 16 * Alternatively, this software may be distributed under the terms of the 17 * GNU General Public License ("GPL") version 2 as published by the Free 18 * Software Foundation. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <linux/module.h> 34 #include <linux/pid.h> 35 #include <linux/pid_namespace.h> 36 #include <linux/mutex.h> 37 #include <net/netlink.h> 38 #include <rdma/rdma_cm.h> 39 #include <rdma/rdma_netlink.h> 40 #include <rdma/frmr_pools.h> 41 42 #include "core_priv.h" 43 #include "cma_priv.h" 44 #include "restrack.h" 45 #include "uverbs.h" 46 #include "frmr_pools.h" 47 48 /* 49 * This determines whether a non-privileged user is allowed to specify a 50 * controlled QKEY or not, when true non-privileged user is allowed to specify 51 * a controlled QKEY. 52 */ 53 static bool privileged_qkey; 54 static DEFINE_MUTEX(nldev_dellink_mutex); 55 56 typedef int (*res_fill_func_t)(struct sk_buff*, bool, 57 struct rdma_restrack_entry*, uint32_t); 58 59 /* 60 * Sort array elements by the netlink attribute name 61 */ 62 static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { 63 [RDMA_NLDEV_ATTR_CHARDEV] = { .type = NLA_U64 }, 64 [RDMA_NLDEV_ATTR_CHARDEV_ABI] = { .type = NLA_U64 }, 65 [RDMA_NLDEV_ATTR_CHARDEV_NAME] = { .type = NLA_NUL_STRING, 66 .len = RDMA_NLDEV_ATTR_EMPTY_STRING }, 67 [RDMA_NLDEV_ATTR_CHARDEV_TYPE] = { .type = NLA_NUL_STRING, 68 .len = RDMA_NLDEV_ATTR_CHARDEV_TYPE_SIZE }, 69 [RDMA_NLDEV_ATTR_DEV_DIM] = { .type = NLA_U8 }, 70 [RDMA_NLDEV_ATTR_DEV_INDEX] = { .type = NLA_U32 }, 71 [RDMA_NLDEV_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING, 72 .len = IB_DEVICE_NAME_MAX }, 73 [RDMA_NLDEV_ATTR_DEV_NODE_TYPE] = { .type = NLA_U8 }, 74 [RDMA_NLDEV_ATTR_DEV_PROTOCOL] = { .type = NLA_NUL_STRING, 75 .len = RDMA_NLDEV_ATTR_EMPTY_STRING }, 76 [RDMA_NLDEV_ATTR_DRIVER] = { .type = NLA_NESTED }, 77 [RDMA_NLDEV_ATTR_DRIVER_ENTRY] = { .type = NLA_NESTED }, 78 [RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE] = { .type = NLA_U8 }, 79 [RDMA_NLDEV_ATTR_DRIVER_STRING] = { .type = NLA_NUL_STRING, 80 .len = RDMA_NLDEV_ATTR_EMPTY_STRING }, 81 [RDMA_NLDEV_ATTR_DRIVER_S32] = { .type = NLA_S32 }, 82 [RDMA_NLDEV_ATTR_DRIVER_S64] = { .type = NLA_S64 }, 83 [RDMA_NLDEV_ATTR_DRIVER_U32] = { .type = NLA_U32 }, 84 [RDMA_NLDEV_ATTR_DRIVER_U64] = { .type = NLA_U64 }, 85 [RDMA_NLDEV_ATTR_FW_VERSION] = { .type = NLA_NUL_STRING, 86 .len = RDMA_NLDEV_ATTR_EMPTY_STRING }, 87 [RDMA_NLDEV_ATTR_LID] = { .type = NLA_U32 }, 88 [RDMA_NLDEV_ATTR_LINK_TYPE] = { .type = NLA_NUL_STRING, 89 .len = IFNAMSIZ }, 90 [RDMA_NLDEV_ATTR_LMC] = { .type = NLA_U8 }, 91 [RDMA_NLDEV_ATTR_NDEV_INDEX] = { .type = NLA_U32 }, 92 [RDMA_NLDEV_ATTR_NDEV_NAME] = { .type = NLA_NUL_STRING, 93 .len = IFNAMSIZ }, 94 [RDMA_NLDEV_ATTR_NODE_GUID] = { .type = NLA_U64 }, 95 [RDMA_NLDEV_ATTR_PORT_INDEX] = { .type = NLA_U32 }, 96 [RDMA_NLDEV_ATTR_PORT_PHYS_STATE] = { .type = NLA_U8 }, 97 [RDMA_NLDEV_ATTR_PORT_STATE] = { .type = NLA_U8 }, 98 [RDMA_NLDEV_ATTR_RES_CM_ID] = { .type = NLA_NESTED }, 99 [RDMA_NLDEV_ATTR_RES_CM_IDN] = { .type = NLA_U32 }, 100 [RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY] = { .type = NLA_NESTED }, 101 [RDMA_NLDEV_ATTR_RES_CQ] = { .type = NLA_NESTED }, 102 [RDMA_NLDEV_ATTR_RES_CQE] = { .type = NLA_U32 }, 103 [RDMA_NLDEV_ATTR_RES_CQN] = { .type = NLA_U32 }, 104 [RDMA_NLDEV_ATTR_RES_CQ_ENTRY] = { .type = NLA_NESTED }, 105 [RDMA_NLDEV_ATTR_RES_CTX] = { .type = NLA_NESTED }, 106 [RDMA_NLDEV_ATTR_RES_CTXN] = { .type = NLA_U32 }, 107 [RDMA_NLDEV_ATTR_RES_CTX_ENTRY] = { .type = NLA_NESTED }, 108 [RDMA_NLDEV_ATTR_RES_DST_ADDR] = { 109 .len = sizeof(struct __kernel_sockaddr_storage) }, 110 [RDMA_NLDEV_ATTR_RES_IOVA] = { .type = NLA_U64 }, 111 [RDMA_NLDEV_ATTR_RES_KERN_NAME] = { .type = NLA_NUL_STRING, 112 .len = RDMA_NLDEV_ATTR_EMPTY_STRING }, 113 [RDMA_NLDEV_ATTR_RES_LKEY] = { .type = NLA_U32 }, 114 [RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY] = { .type = NLA_U32 }, 115 [RDMA_NLDEV_ATTR_RES_LQPN] = { .type = NLA_U32 }, 116 [RDMA_NLDEV_ATTR_RES_MR] = { .type = NLA_NESTED }, 117 [RDMA_NLDEV_ATTR_RES_MRLEN] = { .type = NLA_U64 }, 118 [RDMA_NLDEV_ATTR_RES_MRN] = { .type = NLA_U32 }, 119 [RDMA_NLDEV_ATTR_RES_MR_ENTRY] = { .type = NLA_NESTED }, 120 [RDMA_NLDEV_ATTR_RES_PATH_MIG_STATE] = { .type = NLA_U8 }, 121 [RDMA_NLDEV_ATTR_RES_PD] = { .type = NLA_NESTED }, 122 [RDMA_NLDEV_ATTR_RES_PDN] = { .type = NLA_U32 }, 123 [RDMA_NLDEV_ATTR_RES_PD_ENTRY] = { .type = NLA_NESTED }, 124 [RDMA_NLDEV_ATTR_RES_PID] = { .type = NLA_U32 }, 125 [RDMA_NLDEV_ATTR_RES_POLL_CTX] = { .type = NLA_U8 }, 126 [RDMA_NLDEV_ATTR_RES_PS] = { .type = NLA_U32 }, 127 [RDMA_NLDEV_ATTR_RES_QP] = { .type = NLA_NESTED }, 128 [RDMA_NLDEV_ATTR_RES_QP_ENTRY] = { .type = NLA_NESTED }, 129 [RDMA_NLDEV_ATTR_RES_RAW] = { .type = NLA_BINARY }, 130 [RDMA_NLDEV_ATTR_RES_RKEY] = { .type = NLA_U32 }, 131 [RDMA_NLDEV_ATTR_RES_RQPN] = { .type = NLA_U32 }, 132 [RDMA_NLDEV_ATTR_RES_RQ_PSN] = { .type = NLA_U32 }, 133 [RDMA_NLDEV_ATTR_RES_SQ_PSN] = { .type = NLA_U32 }, 134 [RDMA_NLDEV_ATTR_RES_SRC_ADDR] = { 135 .len = sizeof(struct __kernel_sockaddr_storage) }, 136 [RDMA_NLDEV_ATTR_RES_STATE] = { .type = NLA_U8 }, 137 [RDMA_NLDEV_ATTR_RES_SUMMARY] = { .type = NLA_NESTED }, 138 [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY] = { .type = NLA_NESTED }, 139 [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR]= { .type = NLA_U64 }, 140 [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME]= { .type = NLA_NUL_STRING, 141 .len = RDMA_NLDEV_ATTR_EMPTY_STRING }, 142 [RDMA_NLDEV_ATTR_RES_TYPE] = { .type = NLA_U8 }, 143 [RDMA_NLDEV_ATTR_RES_SUBTYPE] = { .type = NLA_NUL_STRING, 144 .len = RDMA_NLDEV_ATTR_EMPTY_STRING }, 145 [RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY]= { .type = NLA_U32 }, 146 [RDMA_NLDEV_ATTR_RES_USECNT] = { .type = NLA_U64 }, 147 [RDMA_NLDEV_ATTR_RES_SRQ] = { .type = NLA_NESTED }, 148 [RDMA_NLDEV_ATTR_RES_SRQN] = { .type = NLA_U32 }, 149 [RDMA_NLDEV_ATTR_RES_SRQ_ENTRY] = { .type = NLA_NESTED }, 150 [RDMA_NLDEV_ATTR_MIN_RANGE] = { .type = NLA_U32 }, 151 [RDMA_NLDEV_ATTR_MAX_RANGE] = { .type = NLA_U32 }, 152 [RDMA_NLDEV_ATTR_SM_LID] = { .type = NLA_U32 }, 153 [RDMA_NLDEV_ATTR_SUBNET_PREFIX] = { .type = NLA_U64 }, 154 [RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK] = { .type = NLA_U32 }, 155 [RDMA_NLDEV_ATTR_STAT_MODE] = { .type = NLA_U32 }, 156 [RDMA_NLDEV_ATTR_STAT_RES] = { .type = NLA_U32 }, 157 [RDMA_NLDEV_ATTR_STAT_COUNTER] = { .type = NLA_NESTED }, 158 [RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY] = { .type = NLA_NESTED }, 159 [RDMA_NLDEV_ATTR_STAT_COUNTER_ID] = { .type = NLA_U32 }, 160 [RDMA_NLDEV_ATTR_STAT_HWCOUNTERS] = { .type = NLA_NESTED }, 161 [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY] = { .type = NLA_NESTED }, 162 [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME] = { .type = NLA_NUL_STRING }, 163 [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE] = { .type = NLA_U64 }, 164 [RDMA_NLDEV_ATTR_SYS_IMAGE_GUID] = { .type = NLA_U64 }, 165 [RDMA_NLDEV_ATTR_UVERBS_DRIVER_ID] = { .type = NLA_U32 }, 166 [RDMA_NLDEV_NET_NS_FD] = { .type = NLA_U32 }, 167 [RDMA_NLDEV_SYS_ATTR_NETNS_MODE] = { .type = NLA_U8 }, 168 [RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK] = { .type = NLA_U8 }, 169 [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX] = { .type = NLA_U32 }, 170 [RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC] = { .type = NLA_U8 }, 171 [RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE] = { .type = NLA_U8 }, 172 [RDMA_NLDEV_ATTR_DRIVER_DETAILS] = { .type = NLA_U8 }, 173 [RDMA_NLDEV_ATTR_DEV_TYPE] = { .type = NLA_U8 }, 174 [RDMA_NLDEV_ATTR_PARENT_NAME] = { .type = NLA_NUL_STRING }, 175 [RDMA_NLDEV_ATTR_NAME_ASSIGN_TYPE] = { .type = NLA_U8 }, 176 [RDMA_NLDEV_ATTR_EVENT_TYPE] = { .type = NLA_U8 }, 177 [RDMA_NLDEV_ATTR_STAT_OPCOUNTER_ENABLED] = { .type = NLA_U8 }, 178 [RDMA_NLDEV_ATTR_FRMR_POOLS] = { .type = NLA_NESTED }, 179 [RDMA_NLDEV_ATTR_FRMR_POOL_ENTRY] = { .type = NLA_NESTED }, 180 [RDMA_NLDEV_ATTR_FRMR_POOL_KEY] = { .type = NLA_NESTED }, 181 [RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ATS] = { .type = NLA_U8 }, 182 [RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ACCESS_FLAGS] = { .type = NLA_U32 }, 183 [RDMA_NLDEV_ATTR_FRMR_POOL_KEY_VENDOR_KEY] = { .type = NLA_U64 }, 184 [RDMA_NLDEV_ATTR_FRMR_POOL_KEY_NUM_DMA_BLOCKS] = { .type = NLA_U64 }, 185 [RDMA_NLDEV_ATTR_FRMR_POOL_QUEUE_HANDLES] = { .type = NLA_U32 }, 186 [RDMA_NLDEV_ATTR_FRMR_POOL_MAX_IN_USE] = { .type = NLA_U64 }, 187 [RDMA_NLDEV_ATTR_FRMR_POOL_IN_USE] = { .type = NLA_U64 }, 188 [RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD] = { .type = NLA_U32 }, 189 [RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES] = { .type = NLA_U32 }, 190 [RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY] = { .type = NLA_U64 }, 191 [RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX] = { .type = NLA_U64 }, 192 }; 193 194 static int put_driver_name_print_type(struct sk_buff *msg, const char *name, 195 enum rdma_nldev_print_type print_type) 196 { 197 if (nla_put_string(msg, RDMA_NLDEV_ATTR_DRIVER_STRING, name)) 198 return -EMSGSIZE; 199 if (print_type != RDMA_NLDEV_PRINT_TYPE_UNSPEC && 200 nla_put_u8(msg, RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE, print_type)) 201 return -EMSGSIZE; 202 203 return 0; 204 } 205 206 static int _rdma_nl_put_driver_u32(struct sk_buff *msg, const char *name, 207 enum rdma_nldev_print_type print_type, 208 u32 value) 209 { 210 if (put_driver_name_print_type(msg, name, print_type)) 211 return -EMSGSIZE; 212 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DRIVER_U32, value)) 213 return -EMSGSIZE; 214 215 return 0; 216 } 217 218 static int _rdma_nl_put_driver_u64(struct sk_buff *msg, const char *name, 219 enum rdma_nldev_print_type print_type, 220 u64 value) 221 { 222 if (put_driver_name_print_type(msg, name, print_type)) 223 return -EMSGSIZE; 224 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_DRIVER_U64, value, 225 RDMA_NLDEV_ATTR_PAD)) 226 return -EMSGSIZE; 227 228 return 0; 229 } 230 231 int rdma_nl_put_driver_string(struct sk_buff *msg, const char *name, 232 const char *str) 233 { 234 if (put_driver_name_print_type(msg, name, 235 RDMA_NLDEV_PRINT_TYPE_UNSPEC)) 236 return -EMSGSIZE; 237 if (nla_put_string(msg, RDMA_NLDEV_ATTR_DRIVER_STRING, str)) 238 return -EMSGSIZE; 239 240 return 0; 241 } 242 EXPORT_SYMBOL(rdma_nl_put_driver_string); 243 244 int rdma_nl_put_driver_u32(struct sk_buff *msg, const char *name, u32 value) 245 { 246 return _rdma_nl_put_driver_u32(msg, name, RDMA_NLDEV_PRINT_TYPE_UNSPEC, 247 value); 248 } 249 EXPORT_SYMBOL(rdma_nl_put_driver_u32); 250 251 int rdma_nl_put_driver_u32_hex(struct sk_buff *msg, const char *name, 252 u32 value) 253 { 254 return _rdma_nl_put_driver_u32(msg, name, RDMA_NLDEV_PRINT_TYPE_HEX, 255 value); 256 } 257 EXPORT_SYMBOL(rdma_nl_put_driver_u32_hex); 258 259 int rdma_nl_put_driver_u64(struct sk_buff *msg, const char *name, u64 value) 260 { 261 return _rdma_nl_put_driver_u64(msg, name, RDMA_NLDEV_PRINT_TYPE_UNSPEC, 262 value); 263 } 264 EXPORT_SYMBOL(rdma_nl_put_driver_u64); 265 266 int rdma_nl_put_driver_u64_hex(struct sk_buff *msg, const char *name, u64 value) 267 { 268 return _rdma_nl_put_driver_u64(msg, name, RDMA_NLDEV_PRINT_TYPE_HEX, 269 value); 270 } 271 EXPORT_SYMBOL(rdma_nl_put_driver_u64_hex); 272 273 bool rdma_nl_get_privileged_qkey(void) 274 { 275 return privileged_qkey; 276 } 277 EXPORT_SYMBOL(rdma_nl_get_privileged_qkey); 278 279 static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device) 280 { 281 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DEV_INDEX, device->index)) 282 return -EMSGSIZE; 283 if (nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_NAME, 284 dev_name(&device->dev))) 285 return -EMSGSIZE; 286 287 return 0; 288 } 289 290 static int fill_dev_info(struct sk_buff *msg, struct ib_device *device) 291 { 292 char fw[IB_FW_VERSION_NAME_MAX]; 293 int ret = 0; 294 u32 port; 295 296 if (fill_nldev_handle(msg, device)) 297 return -EMSGSIZE; 298 299 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, rdma_end_port(device))) 300 return -EMSGSIZE; 301 302 BUILD_BUG_ON(sizeof(device->attrs.device_cap_flags) != sizeof(u64)); 303 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CAP_FLAGS, 304 device->attrs.device_cap_flags, 305 RDMA_NLDEV_ATTR_PAD)) 306 return -EMSGSIZE; 307 308 ib_get_device_fw_str(device, fw); 309 /* Device without FW has strlen(fw) = 0 */ 310 if (strlen(fw) && nla_put_string(msg, RDMA_NLDEV_ATTR_FW_VERSION, fw)) 311 return -EMSGSIZE; 312 313 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_NODE_GUID, 314 be64_to_cpu(device->node_guid), 315 RDMA_NLDEV_ATTR_PAD)) 316 return -EMSGSIZE; 317 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_SYS_IMAGE_GUID, 318 be64_to_cpu(device->attrs.sys_image_guid), 319 RDMA_NLDEV_ATTR_PAD)) 320 return -EMSGSIZE; 321 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_NODE_TYPE, device->node_type)) 322 return -EMSGSIZE; 323 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_DIM, device->use_cq_dim)) 324 return -EMSGSIZE; 325 326 if (device->type && 327 nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_TYPE, device->type)) 328 return -EMSGSIZE; 329 330 if (device->parent && 331 nla_put_string(msg, RDMA_NLDEV_ATTR_PARENT_NAME, 332 dev_name(&device->parent->dev))) 333 return -EMSGSIZE; 334 335 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_NAME_ASSIGN_TYPE, 336 device->name_assign_type)) 337 return -EMSGSIZE; 338 339 /* 340 * Link type is determined on first port and mlx4 device 341 * which can potentially have two different link type for the same 342 * IB device is considered as better to be avoided in the future, 343 */ 344 port = rdma_start_port(device); 345 if (rdma_cap_opa_mad(device, port)) 346 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "opa"); 347 else if (rdma_protocol_ib(device, port)) 348 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "ib"); 349 else if (rdma_protocol_iwarp(device, port)) 350 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "iw"); 351 else if (rdma_protocol_roce(device, port)) 352 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "roce"); 353 else if (rdma_protocol_usnic(device, port)) 354 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, 355 "usnic"); 356 return ret; 357 } 358 359 static int fill_port_info(struct sk_buff *msg, 360 struct ib_device *device, u32 port, 361 const struct net *net) 362 { 363 struct net_device *netdev = NULL; 364 struct ib_port_attr attr; 365 int ret; 366 u64 cap_flags = 0; 367 368 if (fill_nldev_handle(msg, device)) 369 return -EMSGSIZE; 370 371 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) 372 return -EMSGSIZE; 373 374 ret = ib_query_port(device, port, &attr); 375 if (ret) 376 return ret; 377 378 if (rdma_protocol_ib(device, port)) { 379 BUILD_BUG_ON((sizeof(attr.port_cap_flags) + 380 sizeof(attr.port_cap_flags2)) > sizeof(u64)); 381 cap_flags = attr.port_cap_flags | 382 ((u64)attr.port_cap_flags2 << 32); 383 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CAP_FLAGS, 384 cap_flags, RDMA_NLDEV_ATTR_PAD)) 385 return -EMSGSIZE; 386 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_SUBNET_PREFIX, 387 attr.subnet_prefix, RDMA_NLDEV_ATTR_PAD)) 388 return -EMSGSIZE; 389 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_LID, attr.lid)) 390 return -EMSGSIZE; 391 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_SM_LID, attr.sm_lid)) 392 return -EMSGSIZE; 393 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_LMC, attr.lmc)) 394 return -EMSGSIZE; 395 } 396 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_PORT_STATE, attr.state)) 397 return -EMSGSIZE; 398 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_PORT_PHYS_STATE, attr.phys_state)) 399 return -EMSGSIZE; 400 401 netdev = ib_device_get_netdev(device, port); 402 if (netdev && net_eq(dev_net(netdev), net)) { 403 ret = nla_put_u32(msg, 404 RDMA_NLDEV_ATTR_NDEV_INDEX, netdev->ifindex); 405 if (ret) 406 goto out; 407 ret = nla_put_string(msg, 408 RDMA_NLDEV_ATTR_NDEV_NAME, netdev->name); 409 } 410 411 out: 412 dev_put(netdev); 413 return ret; 414 } 415 416 static int fill_res_info_entry(struct sk_buff *msg, 417 const char *name, u64 curr, u64 max) 418 { 419 struct nlattr *entry_attr; 420 421 entry_attr = nla_nest_start_noflag(msg, 422 RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY); 423 if (!entry_attr) 424 return -EMSGSIZE; 425 426 if (nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME, name)) 427 goto err; 428 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, curr, 429 RDMA_NLDEV_ATTR_PAD)) 430 goto err; 431 if (max && nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_MAX, 432 max, RDMA_NLDEV_ATTR_PAD)) 433 goto err; 434 435 nla_nest_end(msg, entry_attr); 436 return 0; 437 438 err: 439 nla_nest_cancel(msg, entry_attr); 440 return -EMSGSIZE; 441 } 442 443 static int fill_res_info(struct sk_buff *msg, struct ib_device *device, 444 bool show_details) 445 { 446 static const char * const names[RDMA_RESTRACK_MAX] = { 447 [RDMA_RESTRACK_PD] = "pd", 448 [RDMA_RESTRACK_CQ] = "cq", 449 [RDMA_RESTRACK_QP] = "qp", 450 [RDMA_RESTRACK_CM_ID] = "cm_id", 451 [RDMA_RESTRACK_MR] = "mr", 452 [RDMA_RESTRACK_CTX] = "ctx", 453 [RDMA_RESTRACK_SRQ] = "srq", 454 [RDMA_RESTRACK_COMP_CNTR] = "comp_cntr", 455 }; 456 457 struct ib_comp_cntr_caps comp_cntr_caps = {}; 458 struct nlattr *table_attr; 459 u64 curr, max; 460 int ret, i; 461 462 if (fill_nldev_handle(msg, device)) 463 return -EMSGSIZE; 464 465 table_attr = nla_nest_start_noflag(msg, RDMA_NLDEV_ATTR_RES_SUMMARY); 466 if (!table_attr) 467 return -EMSGSIZE; 468 469 if (device->ops.query_comp_cntr_caps) 470 device->ops.query_comp_cntr_caps(device, &comp_cntr_caps, NULL); 471 472 for (i = 0; i < RDMA_RESTRACK_MAX; i++) { 473 if (!names[i]) 474 continue; 475 curr = rdma_restrack_count(device, i, show_details); 476 switch (i) { 477 case RDMA_RESTRACK_QP: 478 max = device->attrs.max_qp; 479 break; 480 case RDMA_RESTRACK_CQ: 481 max = device->attrs.max_cq; 482 break; 483 case RDMA_RESTRACK_MR: 484 max = device->attrs.max_mr; 485 break; 486 case RDMA_RESTRACK_PD: 487 max = device->attrs.max_pd; 488 break; 489 case RDMA_RESTRACK_SRQ: 490 max = device->attrs.max_srq; 491 break; 492 case RDMA_RESTRACK_COMP_CNTR: 493 max = comp_cntr_caps.max_counters; 494 break; 495 default: 496 max = 0; 497 } 498 ret = fill_res_info_entry(msg, names[i], curr, max); 499 if (ret) 500 goto err; 501 } 502 503 nla_nest_end(msg, table_attr); 504 return 0; 505 506 err: 507 nla_nest_cancel(msg, table_attr); 508 return ret; 509 } 510 511 static int fill_res_name_pid(struct sk_buff *msg, 512 struct rdma_restrack_entry *res) 513 { 514 int err = 0; 515 516 /* 517 * For user resources, user is should read /proc/PID/comm to get the 518 * name of the task file. 519 */ 520 if (rdma_is_kernel_res(res)) { 521 err = nla_put_string(msg, RDMA_NLDEV_ATTR_RES_KERN_NAME, 522 res->kern_name); 523 } else { 524 pid_t pid; 525 526 pid = task_pid_vnr(res->task); 527 /* 528 * Task is dead and in zombie state. 529 * There is no need to print PID anymore. 530 */ 531 if (pid) 532 /* 533 * This part is racy, task can be killed and PID will 534 * be zero right here but it is ok, next query won't 535 * return PID. We don't promise real-time reflection 536 * of SW objects. 537 */ 538 err = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PID, pid); 539 } 540 541 return err ? -EMSGSIZE : 0; 542 } 543 544 static int fill_res_qp_entry_query(struct sk_buff *msg, 545 struct rdma_restrack_entry *res, 546 struct ib_device *dev, 547 struct ib_qp *qp) 548 { 549 struct ib_qp_init_attr qp_init_attr; 550 struct ib_qp_attr qp_attr; 551 int ret; 552 553 ret = ib_query_qp(qp, &qp_attr, 0, &qp_init_attr); 554 if (ret) 555 return ret; 556 557 if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC) { 558 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RQPN, 559 qp_attr.dest_qp_num)) 560 goto err; 561 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RQ_PSN, 562 qp_attr.rq_psn)) 563 goto err; 564 } 565 566 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_SQ_PSN, qp_attr.sq_psn)) 567 goto err; 568 569 if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC || 570 qp->qp_type == IB_QPT_XRC_INI || qp->qp_type == IB_QPT_XRC_TGT) { 571 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_PATH_MIG_STATE, 572 qp_attr.path_mig_state)) 573 goto err; 574 } 575 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, qp->qp_type)) 576 goto err; 577 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, qp_attr.qp_state)) 578 goto err; 579 580 if (dev->ops.fill_res_qp_entry) 581 return dev->ops.fill_res_qp_entry(msg, qp); 582 return 0; 583 584 err: return -EMSGSIZE; 585 } 586 587 static int fill_res_qp_entry(struct sk_buff *msg, bool has_cap_net_admin, 588 struct rdma_restrack_entry *res, uint32_t port) 589 { 590 struct ib_qp *qp = container_of(res, struct ib_qp, res); 591 struct ib_device *dev = qp->device; 592 int ret; 593 594 if (port && port != qp->port) 595 return -EAGAIN; 596 597 /* In create_qp() port is not set yet */ 598 if (qp->port && nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp->port)) 599 return -EMSGSIZE; 600 601 ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp->qp_num); 602 if (ret) 603 return -EMSGSIZE; 604 605 if (!rdma_is_kernel_res(res) && 606 nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, qp->pd->res.id)) 607 return -EMSGSIZE; 608 609 ret = fill_res_name_pid(msg, res); 610 if (ret) 611 return -EMSGSIZE; 612 613 return fill_res_qp_entry_query(msg, res, dev, qp); 614 } 615 616 static int fill_res_qp_raw_entry(struct sk_buff *msg, bool has_cap_net_admin, 617 struct rdma_restrack_entry *res, uint32_t port) 618 { 619 struct ib_qp *qp = container_of(res, struct ib_qp, res); 620 struct ib_device *dev = qp->device; 621 622 if (port && port != qp->port) 623 return -EAGAIN; 624 if (!dev->ops.fill_res_qp_entry_raw) 625 return -EINVAL; 626 return dev->ops.fill_res_qp_entry_raw(msg, qp); 627 } 628 629 static int fill_res_cm_id_entry(struct sk_buff *msg, bool has_cap_net_admin, 630 struct rdma_restrack_entry *res, uint32_t port) 631 { 632 struct rdma_id_private *id_priv = 633 container_of(res, struct rdma_id_private, res); 634 struct ib_device *dev = id_priv->id.device; 635 struct rdma_cm_id *cm_id = &id_priv->id; 636 637 if (port && port != cm_id->port_num) 638 return -EAGAIN; 639 640 if (cm_id->port_num && 641 nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, cm_id->port_num)) 642 goto err; 643 644 if (id_priv->qp_num) { 645 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, id_priv->qp_num)) 646 goto err; 647 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, cm_id->qp_type)) 648 goto err; 649 } 650 651 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PS, cm_id->ps)) 652 goto err; 653 654 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, id_priv->state)) 655 goto err; 656 657 if (cm_id->route.addr.src_addr.ss_family && 658 nla_put(msg, RDMA_NLDEV_ATTR_RES_SRC_ADDR, 659 sizeof(cm_id->route.addr.src_addr), 660 &cm_id->route.addr.src_addr)) 661 goto err; 662 if (cm_id->route.addr.dst_addr.ss_family && 663 nla_put(msg, RDMA_NLDEV_ATTR_RES_DST_ADDR, 664 sizeof(cm_id->route.addr.dst_addr), 665 &cm_id->route.addr.dst_addr)) 666 goto err; 667 668 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CM_IDN, res->id)) 669 goto err; 670 671 if (fill_res_name_pid(msg, res)) 672 goto err; 673 674 if (dev->ops.fill_res_cm_id_entry) 675 return dev->ops.fill_res_cm_id_entry(msg, cm_id); 676 return 0; 677 678 err: return -EMSGSIZE; 679 } 680 681 static int fill_res_cq_entry(struct sk_buff *msg, bool has_cap_net_admin, 682 struct rdma_restrack_entry *res, uint32_t port) 683 { 684 struct ib_cq *cq = container_of(res, struct ib_cq, res); 685 struct ib_device *dev = cq->device; 686 687 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQE, cq->cqe)) 688 return -EMSGSIZE; 689 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_USECNT, 690 atomic_read(&cq->usecnt), RDMA_NLDEV_ATTR_PAD)) 691 return -EMSGSIZE; 692 693 /* Poll context is only valid for kernel CQs */ 694 if (rdma_is_kernel_res(res) && 695 nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_POLL_CTX, cq->poll_ctx)) 696 return -EMSGSIZE; 697 698 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_DIM, (cq->dim != NULL))) 699 return -EMSGSIZE; 700 701 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN, res->id)) 702 return -EMSGSIZE; 703 704 if (cq->uobject && cq->uobject->uevent.uobject.context && 705 nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN, 706 cq->uobject->uevent.uobject.context->res.id)) 707 return -EMSGSIZE; 708 709 if (fill_res_name_pid(msg, res)) 710 return -EMSGSIZE; 711 712 return (dev->ops.fill_res_cq_entry) ? 713 dev->ops.fill_res_cq_entry(msg, cq) : 0; 714 } 715 716 static int fill_res_cq_raw_entry(struct sk_buff *msg, bool has_cap_net_admin, 717 struct rdma_restrack_entry *res, uint32_t port) 718 { 719 struct ib_cq *cq = container_of(res, struct ib_cq, res); 720 struct ib_device *dev = cq->device; 721 722 if (!dev->ops.fill_res_cq_entry_raw) 723 return -EINVAL; 724 return dev->ops.fill_res_cq_entry_raw(msg, cq); 725 } 726 727 static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, 728 struct rdma_restrack_entry *res, uint32_t port) 729 { 730 struct ib_mr *mr = container_of(res, struct ib_mr, res); 731 struct ib_device *dev = mr->device; 732 733 if (has_cap_net_admin) { 734 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr->rkey)) 735 return -EMSGSIZE; 736 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LKEY, mr->lkey)) 737 return -EMSGSIZE; 738 } 739 740 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_MRLEN, mr->length, 741 RDMA_NLDEV_ATTR_PAD)) 742 return -EMSGSIZE; 743 744 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id)) 745 return -EMSGSIZE; 746 747 if (!rdma_is_kernel_res(res)) { 748 struct ib_pd *pd = READ_ONCE(mr->pd); 749 750 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, pd->res.id)) 751 return -EMSGSIZE; 752 } 753 754 if (fill_res_name_pid(msg, res)) 755 return -EMSGSIZE; 756 757 return (dev->ops.fill_res_mr_entry) ? 758 dev->ops.fill_res_mr_entry(msg, mr) : 759 0; 760 } 761 762 static int fill_res_mr_raw_entry(struct sk_buff *msg, bool has_cap_net_admin, 763 struct rdma_restrack_entry *res, uint32_t port) 764 { 765 struct ib_mr *mr = container_of(res, struct ib_mr, res); 766 struct ib_device *dev = mr->device; 767 768 if (!dev->ops.fill_res_mr_entry_raw) 769 return -EINVAL; 770 return dev->ops.fill_res_mr_entry_raw(msg, mr); 771 } 772 773 static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin, 774 struct rdma_restrack_entry *res, uint32_t port) 775 { 776 struct ib_pd *pd = container_of(res, struct ib_pd, res); 777 778 if (has_cap_net_admin) { 779 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY, 780 pd->local_dma_lkey)) 781 goto err; 782 if ((pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) && 783 nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY, 784 pd->unsafe_global_rkey)) 785 goto err; 786 } 787 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_RES_USECNT, 788 atomic_read(&pd->usecnt), RDMA_NLDEV_ATTR_PAD)) 789 goto err; 790 791 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, res->id)) 792 goto err; 793 794 if (pd->uobject && pd->uobject->context && 795 nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN, 796 pd->uobject->context->res.id)) 797 goto err; 798 799 return fill_res_name_pid(msg, res); 800 801 err: return -EMSGSIZE; 802 } 803 804 static int fill_res_ctx_entry(struct sk_buff *msg, bool has_cap_net_admin, 805 struct rdma_restrack_entry *res, uint32_t port) 806 { 807 struct ib_ucontext *ctx = container_of(res, struct ib_ucontext, res); 808 809 if (rdma_is_kernel_res(res)) 810 return 0; 811 812 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN, ctx->res.id)) 813 return -EMSGSIZE; 814 815 return fill_res_name_pid(msg, res); 816 } 817 818 static int fill_res_range_qp_entry(struct sk_buff *msg, uint32_t min_range, 819 uint32_t max_range) 820 { 821 struct nlattr *entry_attr; 822 823 if (!min_range) 824 return 0; 825 826 entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP_ENTRY); 827 if (!entry_attr) 828 return -EMSGSIZE; 829 830 if (min_range == max_range) { 831 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, min_range)) 832 goto err; 833 } else { 834 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_MIN_RANGE, min_range)) 835 goto err; 836 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_MAX_RANGE, max_range)) 837 goto err; 838 } 839 nla_nest_end(msg, entry_attr); 840 return 0; 841 842 err: 843 nla_nest_cancel(msg, entry_attr); 844 return -EMSGSIZE; 845 } 846 847 static int fill_res_srq_qps(struct sk_buff *msg, struct ib_srq *srq) 848 { 849 uint32_t min_range = 0, prev = 0; 850 struct rdma_restrack_entry *res; 851 struct rdma_restrack_root *rt; 852 struct nlattr *table_attr; 853 struct ib_qp *qp = NULL; 854 unsigned long id = 0; 855 856 table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP); 857 if (!table_attr) 858 return -EMSGSIZE; 859 860 rt = &srq->device->res[RDMA_RESTRACK_QP]; 861 xa_lock(&rt->xa); 862 xa_for_each(&rt->xa, id, res) { 863 if (!rdma_restrack_get(res)) 864 continue; 865 866 qp = container_of(res, struct ib_qp, res); 867 if (!qp->srq || (qp->srq->res.id != srq->res.id)) { 868 rdma_restrack_put(res); 869 continue; 870 } 871 872 if (qp->qp_num < prev) 873 /* qp_num should be ascending */ 874 goto err_loop; 875 876 if (min_range == 0) { 877 min_range = qp->qp_num; 878 } else if (qp->qp_num > (prev + 1)) { 879 if (fill_res_range_qp_entry(msg, min_range, prev)) 880 goto err_loop; 881 882 min_range = qp->qp_num; 883 } 884 prev = qp->qp_num; 885 rdma_restrack_put(res); 886 } 887 888 xa_unlock(&rt->xa); 889 890 if (fill_res_range_qp_entry(msg, min_range, prev)) 891 goto err; 892 893 nla_nest_end(msg, table_attr); 894 return 0; 895 896 err_loop: 897 rdma_restrack_put(res); 898 xa_unlock(&rt->xa); 899 err: 900 nla_nest_cancel(msg, table_attr); 901 return -EMSGSIZE; 902 } 903 904 static int fill_res_srq_entry(struct sk_buff *msg, bool has_cap_net_admin, 905 struct rdma_restrack_entry *res, uint32_t port) 906 { 907 struct ib_srq *srq = container_of(res, struct ib_srq, res); 908 struct ib_device *dev = srq->device; 909 910 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_SRQN, srq->res.id)) 911 goto err; 912 913 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, srq->srq_type)) 914 goto err; 915 916 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, srq->pd->res.id)) 917 goto err; 918 919 if (ib_srq_has_cq(srq->srq_type)) { 920 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN, 921 srq->ext.cq->res.id)) 922 goto err; 923 } 924 925 if (fill_res_srq_qps(msg, srq)) 926 goto err; 927 928 if (fill_res_name_pid(msg, res)) 929 goto err; 930 931 if (dev->ops.fill_res_srq_entry) 932 return dev->ops.fill_res_srq_entry(msg, srq); 933 934 return 0; 935 936 err: 937 return -EMSGSIZE; 938 } 939 940 static int fill_res_srq_raw_entry(struct sk_buff *msg, bool has_cap_net_admin, 941 struct rdma_restrack_entry *res, uint32_t port) 942 { 943 struct ib_srq *srq = container_of(res, struct ib_srq, res); 944 struct ib_device *dev = srq->device; 945 946 if (!dev->ops.fill_res_srq_entry_raw) 947 return -EINVAL; 948 return dev->ops.fill_res_srq_entry_raw(msg, srq); 949 } 950 951 static int fill_stat_counter_mode(struct sk_buff *msg, 952 struct rdma_counter *counter) 953 { 954 struct rdma_counter_mode *m = &counter->mode; 955 956 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_MODE, m->mode)) 957 return -EMSGSIZE; 958 959 if (m->mode == RDMA_COUNTER_MODE_AUTO) { 960 if ((m->mask & RDMA_COUNTER_MASK_QP_TYPE) && 961 nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, m->param.qp_type)) 962 return -EMSGSIZE; 963 964 if ((m->mask & RDMA_COUNTER_MASK_PID) && 965 fill_res_name_pid(msg, &counter->res)) 966 return -EMSGSIZE; 967 } 968 969 return 0; 970 } 971 972 static int fill_stat_counter_qp_entry(struct sk_buff *msg, u32 qpn) 973 { 974 struct nlattr *entry_attr; 975 976 entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP_ENTRY); 977 if (!entry_attr) 978 return -EMSGSIZE; 979 980 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn)) 981 goto err; 982 983 nla_nest_end(msg, entry_attr); 984 return 0; 985 986 err: 987 nla_nest_cancel(msg, entry_attr); 988 return -EMSGSIZE; 989 } 990 991 static int fill_stat_counter_qps(struct sk_buff *msg, 992 struct rdma_counter *counter) 993 { 994 struct rdma_restrack_entry *res; 995 struct rdma_restrack_root *rt; 996 struct nlattr *table_attr; 997 struct ib_qp *qp = NULL; 998 unsigned long id = 0; 999 int ret = 0; 1000 1001 table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP); 1002 if (!table_attr) 1003 return -EMSGSIZE; 1004 1005 rt = &counter->device->res[RDMA_RESTRACK_QP]; 1006 xa_lock(&rt->xa); 1007 xa_for_each(&rt->xa, id, res) { 1008 qp = container_of(res, struct ib_qp, res); 1009 if (!qp->counter || (qp->counter->id != counter->id)) 1010 continue; 1011 1012 ret = fill_stat_counter_qp_entry(msg, qp->qp_num); 1013 if (ret) 1014 goto err; 1015 } 1016 1017 xa_unlock(&rt->xa); 1018 nla_nest_end(msg, table_attr); 1019 return 0; 1020 1021 err: 1022 xa_unlock(&rt->xa); 1023 nla_nest_cancel(msg, table_attr); 1024 return ret; 1025 } 1026 1027 int rdma_nl_stat_hwcounter_entry(struct sk_buff *msg, const char *name, 1028 u64 value) 1029 { 1030 struct nlattr *entry_attr; 1031 1032 entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY); 1033 if (!entry_attr) 1034 return -EMSGSIZE; 1035 1036 if (nla_put_string(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME, 1037 name)) 1038 goto err; 1039 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE, 1040 value, RDMA_NLDEV_ATTR_PAD)) 1041 goto err; 1042 1043 nla_nest_end(msg, entry_attr); 1044 return 0; 1045 1046 err: 1047 nla_nest_cancel(msg, entry_attr); 1048 return -EMSGSIZE; 1049 } 1050 EXPORT_SYMBOL(rdma_nl_stat_hwcounter_entry); 1051 1052 static int fill_stat_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, 1053 struct rdma_restrack_entry *res, uint32_t port) 1054 { 1055 struct ib_mr *mr = container_of(res, struct ib_mr, res); 1056 struct ib_device *dev = mr->device; 1057 1058 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id)) 1059 goto err; 1060 1061 if (dev->ops.fill_stat_mr_entry) 1062 return dev->ops.fill_stat_mr_entry(msg, mr); 1063 return 0; 1064 1065 err: 1066 return -EMSGSIZE; 1067 } 1068 1069 static int fill_stat_counter_hwcounters(struct sk_buff *msg, 1070 struct rdma_counter *counter) 1071 { 1072 struct rdma_hw_stats *st = counter->stats; 1073 struct nlattr *table_attr; 1074 int i; 1075 1076 table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS); 1077 if (!table_attr) 1078 return -EMSGSIZE; 1079 1080 mutex_lock(&st->lock); 1081 for (i = 0; i < st->num_counters; i++) { 1082 if (test_bit(i, st->is_disabled)) 1083 continue; 1084 if (rdma_nl_stat_hwcounter_entry(msg, st->descs[i].name, 1085 st->value[i])) 1086 goto err; 1087 } 1088 mutex_unlock(&st->lock); 1089 1090 nla_nest_end(msg, table_attr); 1091 return 0; 1092 1093 err: 1094 mutex_unlock(&st->lock); 1095 nla_nest_cancel(msg, table_attr); 1096 return -EMSGSIZE; 1097 } 1098 1099 static int fill_res_counter_entry(struct sk_buff *msg, bool has_cap_net_admin, 1100 struct rdma_restrack_entry *res, 1101 uint32_t port) 1102 { 1103 struct rdma_counter *counter = 1104 container_of(res, struct rdma_counter, res); 1105 1106 if (port && port != counter->port) 1107 return -EAGAIN; 1108 1109 /* Dump it even query failed */ 1110 rdma_counter_query_stats(counter); 1111 1112 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, counter->port) || 1113 nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, counter->id) || 1114 fill_stat_counter_mode(msg, counter) || 1115 fill_stat_counter_qps(msg, counter) || 1116 fill_stat_counter_hwcounters(msg, counter)) 1117 return -EMSGSIZE; 1118 1119 return 0; 1120 } 1121 1122 static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, 1123 struct netlink_ext_ack *extack) 1124 { 1125 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 1126 struct ib_device *device; 1127 struct sk_buff *msg; 1128 u32 index; 1129 int err; 1130 1131 err = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 1132 nldev_policy, NL_VALIDATE_LIBERAL, extack); 1133 if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX]) 1134 return -EINVAL; 1135 1136 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 1137 1138 device = ib_device_get_by_index(sock_net(skb->sk), index); 1139 if (!device) 1140 return -EINVAL; 1141 1142 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 1143 if (!msg) { 1144 err = -ENOMEM; 1145 goto err; 1146 } 1147 1148 nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq, 1149 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET), 1150 0, 0); 1151 if (!nlh) { 1152 err = -EMSGSIZE; 1153 goto err_free; 1154 } 1155 1156 err = fill_dev_info(msg, device); 1157 if (err) 1158 goto err_free; 1159 1160 nlmsg_end(msg, nlh); 1161 1162 ib_device_put(device); 1163 return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid); 1164 1165 err_free: 1166 nlmsg_free(msg); 1167 err: 1168 ib_device_put(device); 1169 return err; 1170 } 1171 1172 static int nldev_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh, 1173 struct netlink_ext_ack *extack) 1174 { 1175 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 1176 struct ib_device *device; 1177 u32 index; 1178 int err; 1179 1180 err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 1181 nldev_policy, extack); 1182 if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX]) 1183 return -EINVAL; 1184 1185 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 1186 device = ib_device_get_by_index(sock_net(skb->sk), index); 1187 if (!device) 1188 return -EINVAL; 1189 1190 if (tb[RDMA_NLDEV_NET_NS_FD]) { 1191 char name[IB_DEVICE_NAME_MAX] = {}; 1192 u32 ns_fd; 1193 1194 if (tb[RDMA_NLDEV_ATTR_DEV_NAME]) { 1195 nla_strscpy(name, tb[RDMA_NLDEV_ATTR_DEV_NAME], 1196 IB_DEVICE_NAME_MAX); 1197 if (strlen(name) == 0) { 1198 err = -EINVAL; 1199 goto done; 1200 } 1201 } 1202 ns_fd = nla_get_u32(tb[RDMA_NLDEV_NET_NS_FD]); 1203 err = ib_device_set_netns_put(skb, device, ns_fd, 1204 name[0] ? name : NULL, extack); 1205 goto put_done; 1206 } 1207 1208 if (tb[RDMA_NLDEV_ATTR_DEV_NAME]) { 1209 char name[IB_DEVICE_NAME_MAX] = {}; 1210 1211 nla_strscpy(name, tb[RDMA_NLDEV_ATTR_DEV_NAME], 1212 IB_DEVICE_NAME_MAX); 1213 if (strlen(name) == 0) { 1214 err = -EINVAL; 1215 goto done; 1216 } 1217 err = ib_device_rename(device, name); 1218 goto done; 1219 } 1220 1221 if (tb[RDMA_NLDEV_ATTR_DEV_DIM]) { 1222 u8 use_dim; 1223 1224 use_dim = nla_get_u8(tb[RDMA_NLDEV_ATTR_DEV_DIM]); 1225 err = ib_device_set_dim(device, use_dim); 1226 goto done; 1227 } 1228 1229 done: 1230 ib_device_put(device); 1231 put_done: 1232 return err; 1233 } 1234 1235 static int _nldev_get_dumpit(struct ib_device *device, 1236 struct sk_buff *skb, 1237 struct netlink_callback *cb, 1238 unsigned int idx) 1239 { 1240 int start = cb->args[0]; 1241 struct nlmsghdr *nlh; 1242 1243 if (idx < start) 1244 return 0; 1245 1246 nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, 1247 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET), 1248 0, NLM_F_MULTI); 1249 1250 if (!nlh || fill_dev_info(skb, device)) { 1251 nlmsg_cancel(skb, nlh); 1252 goto out; 1253 } 1254 1255 nlmsg_end(skb, nlh); 1256 1257 idx++; 1258 1259 out: cb->args[0] = idx; 1260 return skb->len; 1261 } 1262 1263 static int nldev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 1264 { 1265 /* 1266 * There is no need to take lock, because 1267 * we are relying on ib_core's locking. 1268 */ 1269 return ib_enum_all_devs(_nldev_get_dumpit, skb, cb); 1270 } 1271 1272 static int nldev_port_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, 1273 struct netlink_ext_ack *extack) 1274 { 1275 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 1276 struct ib_device *device; 1277 struct sk_buff *msg; 1278 u32 index; 1279 u32 port; 1280 int err; 1281 1282 err = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 1283 nldev_policy, NL_VALIDATE_LIBERAL, extack); 1284 if (err || 1285 !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || 1286 !tb[RDMA_NLDEV_ATTR_PORT_INDEX]) 1287 return -EINVAL; 1288 1289 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 1290 device = ib_device_get_by_index(sock_net(skb->sk), index); 1291 if (!device) 1292 return -EINVAL; 1293 1294 port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]); 1295 if (!rdma_is_port_valid(device, port)) { 1296 err = -EINVAL; 1297 goto err; 1298 } 1299 1300 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 1301 if (!msg) { 1302 err = -ENOMEM; 1303 goto err; 1304 } 1305 1306 nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq, 1307 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET), 1308 0, 0); 1309 if (!nlh) { 1310 err = -EMSGSIZE; 1311 goto err_free; 1312 } 1313 1314 err = fill_port_info(msg, device, port, sock_net(skb->sk)); 1315 if (err) 1316 goto err_free; 1317 1318 nlmsg_end(msg, nlh); 1319 ib_device_put(device); 1320 1321 return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid); 1322 1323 err_free: 1324 nlmsg_free(msg); 1325 err: 1326 ib_device_put(device); 1327 return err; 1328 } 1329 1330 static int nldev_port_get_dumpit(struct sk_buff *skb, 1331 struct netlink_callback *cb) 1332 { 1333 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 1334 struct ib_device *device; 1335 int start = cb->args[0]; 1336 struct nlmsghdr *nlh; 1337 u32 idx = 0; 1338 u32 ifindex; 1339 int err; 1340 unsigned int p; 1341 1342 err = __nlmsg_parse(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 1343 nldev_policy, NL_VALIDATE_LIBERAL, NULL); 1344 if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX]) 1345 return -EINVAL; 1346 1347 ifindex = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 1348 device = ib_device_get_by_index(sock_net(skb->sk), ifindex); 1349 if (!device) 1350 return -EINVAL; 1351 1352 rdma_for_each_port (device, p) { 1353 /* 1354 * The dumpit function returns all information from specific 1355 * index. This specific index is taken from the netlink 1356 * messages request sent by user and it is available 1357 * in cb->args[0]. 1358 * 1359 * Usually, the user doesn't fill this field and it causes 1360 * to return everything. 1361 * 1362 */ 1363 if (idx < start) { 1364 idx++; 1365 continue; 1366 } 1367 1368 nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, 1369 cb->nlh->nlmsg_seq, 1370 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, 1371 RDMA_NLDEV_CMD_PORT_GET), 1372 0, NLM_F_MULTI); 1373 1374 if (!nlh || fill_port_info(skb, device, p, sock_net(skb->sk))) { 1375 nlmsg_cancel(skb, nlh); 1376 goto out; 1377 } 1378 idx++; 1379 nlmsg_end(skb, nlh); 1380 } 1381 1382 out: 1383 ib_device_put(device); 1384 cb->args[0] = idx; 1385 return skb->len; 1386 } 1387 1388 static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, 1389 struct netlink_ext_ack *extack) 1390 { 1391 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 1392 bool show_details = false; 1393 struct ib_device *device; 1394 struct sk_buff *msg; 1395 u32 index; 1396 int ret; 1397 1398 ret = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 1399 nldev_policy, NL_VALIDATE_LIBERAL, extack); 1400 if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX]) 1401 return -EINVAL; 1402 1403 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 1404 device = ib_device_get_by_index(sock_net(skb->sk), index); 1405 if (!device) 1406 return -EINVAL; 1407 1408 if (tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]) 1409 show_details = nla_get_u8(tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]); 1410 1411 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 1412 if (!msg) { 1413 ret = -ENOMEM; 1414 goto err; 1415 } 1416 1417 nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq, 1418 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET), 1419 0, 0); 1420 if (!nlh) { 1421 ret = -EMSGSIZE; 1422 goto err_free; 1423 } 1424 1425 ret = fill_res_info(msg, device, show_details); 1426 if (ret) 1427 goto err_free; 1428 1429 nlmsg_end(msg, nlh); 1430 ib_device_put(device); 1431 return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid); 1432 1433 err_free: 1434 nlmsg_free(msg); 1435 err: 1436 ib_device_put(device); 1437 return ret; 1438 } 1439 1440 static int _nldev_res_get_dumpit(struct ib_device *device, 1441 struct sk_buff *skb, 1442 struct netlink_callback *cb, 1443 unsigned int idx) 1444 { 1445 int start = cb->args[0]; 1446 struct nlmsghdr *nlh; 1447 1448 if (idx < start) 1449 return 0; 1450 1451 nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, 1452 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET), 1453 0, NLM_F_MULTI); 1454 1455 if (!nlh || fill_res_info(skb, device, false)) { 1456 nlmsg_cancel(skb, nlh); 1457 goto out; 1458 } 1459 nlmsg_end(skb, nlh); 1460 1461 idx++; 1462 1463 out: 1464 cb->args[0] = idx; 1465 return skb->len; 1466 } 1467 1468 static int nldev_res_get_dumpit(struct sk_buff *skb, 1469 struct netlink_callback *cb) 1470 { 1471 return ib_enum_all_devs(_nldev_res_get_dumpit, skb, cb); 1472 } 1473 1474 struct nldev_fill_res_entry { 1475 enum rdma_nldev_attr nldev_attr; 1476 u8 flags; 1477 u32 entry; 1478 u32 id; 1479 }; 1480 1481 enum nldev_res_flags { 1482 NLDEV_PER_DEV = 1 << 0, 1483 }; 1484 1485 static const struct nldev_fill_res_entry fill_entries[RDMA_RESTRACK_MAX] = { 1486 [RDMA_RESTRACK_QP] = { 1487 .nldev_attr = RDMA_NLDEV_ATTR_RES_QP, 1488 .entry = RDMA_NLDEV_ATTR_RES_QP_ENTRY, 1489 .id = RDMA_NLDEV_ATTR_RES_LQPN, 1490 }, 1491 [RDMA_RESTRACK_CM_ID] = { 1492 .nldev_attr = RDMA_NLDEV_ATTR_RES_CM_ID, 1493 .entry = RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY, 1494 .id = RDMA_NLDEV_ATTR_RES_CM_IDN, 1495 }, 1496 [RDMA_RESTRACK_CQ] = { 1497 .nldev_attr = RDMA_NLDEV_ATTR_RES_CQ, 1498 .flags = NLDEV_PER_DEV, 1499 .entry = RDMA_NLDEV_ATTR_RES_CQ_ENTRY, 1500 .id = RDMA_NLDEV_ATTR_RES_CQN, 1501 }, 1502 [RDMA_RESTRACK_MR] = { 1503 .nldev_attr = RDMA_NLDEV_ATTR_RES_MR, 1504 .flags = NLDEV_PER_DEV, 1505 .entry = RDMA_NLDEV_ATTR_RES_MR_ENTRY, 1506 .id = RDMA_NLDEV_ATTR_RES_MRN, 1507 }, 1508 [RDMA_RESTRACK_PD] = { 1509 .nldev_attr = RDMA_NLDEV_ATTR_RES_PD, 1510 .flags = NLDEV_PER_DEV, 1511 .entry = RDMA_NLDEV_ATTR_RES_PD_ENTRY, 1512 .id = RDMA_NLDEV_ATTR_RES_PDN, 1513 }, 1514 [RDMA_RESTRACK_COUNTER] = { 1515 .nldev_attr = RDMA_NLDEV_ATTR_STAT_COUNTER, 1516 .entry = RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY, 1517 .id = RDMA_NLDEV_ATTR_STAT_COUNTER_ID, 1518 }, 1519 [RDMA_RESTRACK_CTX] = { 1520 .nldev_attr = RDMA_NLDEV_ATTR_RES_CTX, 1521 .flags = NLDEV_PER_DEV, 1522 .entry = RDMA_NLDEV_ATTR_RES_CTX_ENTRY, 1523 .id = RDMA_NLDEV_ATTR_RES_CTXN, 1524 }, 1525 [RDMA_RESTRACK_SRQ] = { 1526 .nldev_attr = RDMA_NLDEV_ATTR_RES_SRQ, 1527 .flags = NLDEV_PER_DEV, 1528 .entry = RDMA_NLDEV_ATTR_RES_SRQ_ENTRY, 1529 .id = RDMA_NLDEV_ATTR_RES_SRQN, 1530 }, 1531 1532 }; 1533 1534 static noinline_for_stack int 1535 res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh, 1536 struct netlink_ext_ack *extack, 1537 enum rdma_restrack_type res_type, 1538 res_fill_func_t fill_func) 1539 { 1540 const struct nldev_fill_res_entry *fe = &fill_entries[res_type]; 1541 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 1542 struct rdma_restrack_entry *res; 1543 struct ib_device *device; 1544 u32 index, id, port = 0; 1545 bool has_cap_net_admin; 1546 struct sk_buff *msg; 1547 int ret; 1548 1549 ret = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 1550 nldev_policy, NL_VALIDATE_LIBERAL, extack); 1551 if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !fe->id || !tb[fe->id]) 1552 return -EINVAL; 1553 1554 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 1555 device = ib_device_get_by_index(sock_net(skb->sk), index); 1556 if (!device) 1557 return -EINVAL; 1558 1559 if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) { 1560 port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]); 1561 if (!rdma_is_port_valid(device, port)) { 1562 ret = -EINVAL; 1563 goto err; 1564 } 1565 } 1566 1567 if ((port && fe->flags & NLDEV_PER_DEV) || 1568 (!port && ~fe->flags & NLDEV_PER_DEV)) { 1569 ret = -EINVAL; 1570 goto err; 1571 } 1572 1573 id = nla_get_u32(tb[fe->id]); 1574 res = rdma_restrack_get_byid(device, res_type, id); 1575 if (IS_ERR(res)) { 1576 ret = PTR_ERR(res); 1577 goto err; 1578 } 1579 1580 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 1581 if (!msg) { 1582 ret = -ENOMEM; 1583 goto err_get; 1584 } 1585 1586 nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq, 1587 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, 1588 RDMA_NL_GET_OP(nlh->nlmsg_type)), 1589 0, 0); 1590 1591 if (!nlh || fill_nldev_handle(msg, device)) { 1592 ret = -EMSGSIZE; 1593 goto err_free; 1594 } 1595 1596 has_cap_net_admin = netlink_capable(skb, CAP_NET_ADMIN); 1597 1598 ret = fill_func(msg, has_cap_net_admin, res, port); 1599 if (ret) 1600 goto err_free; 1601 1602 rdma_restrack_put(res); 1603 nlmsg_end(msg, nlh); 1604 ib_device_put(device); 1605 return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid); 1606 1607 err_free: 1608 nlmsg_free(msg); 1609 err_get: 1610 rdma_restrack_put(res); 1611 err: 1612 ib_device_put(device); 1613 return ret; 1614 } 1615 1616 static int res_get_common_dumpit(struct sk_buff *skb, 1617 struct netlink_callback *cb, 1618 enum rdma_restrack_type res_type, 1619 res_fill_func_t fill_func) 1620 { 1621 const struct nldev_fill_res_entry *fe = &fill_entries[res_type]; 1622 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 1623 struct rdma_restrack_entry *res; 1624 struct rdma_restrack_root *rt; 1625 int err, ret = 0, idx = 0; 1626 bool show_details = false; 1627 struct nlattr *table_attr; 1628 struct nlattr *entry_attr; 1629 struct ib_device *device; 1630 int start = cb->args[0]; 1631 bool has_cap_net_admin; 1632 struct nlmsghdr *nlh; 1633 unsigned long id; 1634 u32 index, port = 0; 1635 bool filled = false; 1636 1637 err = __nlmsg_parse(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 1638 nldev_policy, NL_VALIDATE_LIBERAL, NULL); 1639 /* 1640 * Right now, we are expecting the device index to get res information, 1641 * but it is possible to extend this code to return all devices in 1642 * one shot by checking the existence of RDMA_NLDEV_ATTR_DEV_INDEX. 1643 * if it doesn't exist, we will iterate over all devices. 1644 * 1645 * But it is not needed for now. 1646 */ 1647 if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX]) 1648 return -EINVAL; 1649 1650 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 1651 device = ib_device_get_by_index(sock_net(skb->sk), index); 1652 if (!device) 1653 return -EINVAL; 1654 1655 if (tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]) 1656 show_details = nla_get_u8(tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]); 1657 1658 /* 1659 * If no PORT_INDEX is supplied, we will return all QPs from that device 1660 */ 1661 if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) { 1662 port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]); 1663 if (!rdma_is_port_valid(device, port)) { 1664 ret = -EINVAL; 1665 goto err_index; 1666 } 1667 } 1668 1669 nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, 1670 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, 1671 RDMA_NL_GET_OP(cb->nlh->nlmsg_type)), 1672 0, NLM_F_MULTI); 1673 1674 if (!nlh || fill_nldev_handle(skb, device)) { 1675 ret = -EMSGSIZE; 1676 goto err; 1677 } 1678 1679 table_attr = nla_nest_start_noflag(skb, fe->nldev_attr); 1680 if (!table_attr) { 1681 ret = -EMSGSIZE; 1682 goto err; 1683 } 1684 1685 has_cap_net_admin = netlink_capable(cb->skb, CAP_NET_ADMIN); 1686 1687 rt = &device->res[res_type]; 1688 xa_lock(&rt->xa); 1689 /* 1690 * FIXME: if the skip ahead is something common this loop should 1691 * use xas_for_each & xas_pause to optimize, we can have a lot of 1692 * objects. 1693 */ 1694 xa_for_each(&rt->xa, id, res) { 1695 if (xa_get_mark(&rt->xa, res->id, RESTRACK_DD) && !show_details) 1696 goto next; 1697 1698 if (idx < start || !rdma_restrack_get(res)) 1699 goto next; 1700 1701 xa_unlock(&rt->xa); 1702 1703 filled = true; 1704 1705 entry_attr = nla_nest_start_noflag(skb, fe->entry); 1706 if (!entry_attr) { 1707 ret = -EMSGSIZE; 1708 rdma_restrack_put(res); 1709 goto msg_full; 1710 } 1711 1712 ret = fill_func(skb, has_cap_net_admin, res, port); 1713 1714 rdma_restrack_put(res); 1715 1716 if (ret) { 1717 nla_nest_cancel(skb, entry_attr); 1718 if (ret == -EMSGSIZE) 1719 goto msg_full; 1720 if (ret == -EAGAIN) 1721 goto again; 1722 goto res_err; 1723 } 1724 nla_nest_end(skb, entry_attr); 1725 again: xa_lock(&rt->xa); 1726 next: idx++; 1727 } 1728 xa_unlock(&rt->xa); 1729 1730 msg_full: 1731 nla_nest_end(skb, table_attr); 1732 nlmsg_end(skb, nlh); 1733 cb->args[0] = idx; 1734 1735 /* 1736 * No more entries to fill, cancel the message and 1737 * return 0 to mark end of dumpit. 1738 */ 1739 if (!filled) 1740 goto err; 1741 1742 ib_device_put(device); 1743 return skb->len; 1744 1745 res_err: 1746 nla_nest_cancel(skb, table_attr); 1747 1748 err: 1749 nlmsg_cancel(skb, nlh); 1750 1751 err_index: 1752 ib_device_put(device); 1753 return ret; 1754 } 1755 1756 #define RES_GET_FUNCS(name, type) \ 1757 static int nldev_res_get_##name##_dumpit(struct sk_buff *skb, \ 1758 struct netlink_callback *cb) \ 1759 { \ 1760 return res_get_common_dumpit(skb, cb, type, \ 1761 fill_res_##name##_entry); \ 1762 } \ 1763 static int nldev_res_get_##name##_doit(struct sk_buff *skb, \ 1764 struct nlmsghdr *nlh, \ 1765 struct netlink_ext_ack *extack) \ 1766 { \ 1767 return res_get_common_doit(skb, nlh, extack, type, \ 1768 fill_res_##name##_entry); \ 1769 } 1770 1771 RES_GET_FUNCS(qp, RDMA_RESTRACK_QP); 1772 RES_GET_FUNCS(qp_raw, RDMA_RESTRACK_QP); 1773 RES_GET_FUNCS(cm_id, RDMA_RESTRACK_CM_ID); 1774 RES_GET_FUNCS(cq, RDMA_RESTRACK_CQ); 1775 RES_GET_FUNCS(cq_raw, RDMA_RESTRACK_CQ); 1776 RES_GET_FUNCS(pd, RDMA_RESTRACK_PD); 1777 RES_GET_FUNCS(mr, RDMA_RESTRACK_MR); 1778 RES_GET_FUNCS(mr_raw, RDMA_RESTRACK_MR); 1779 RES_GET_FUNCS(counter, RDMA_RESTRACK_COUNTER); 1780 RES_GET_FUNCS(ctx, RDMA_RESTRACK_CTX); 1781 RES_GET_FUNCS(srq, RDMA_RESTRACK_SRQ); 1782 RES_GET_FUNCS(srq_raw, RDMA_RESTRACK_SRQ); 1783 1784 static LIST_HEAD(link_ops); 1785 static DECLARE_RWSEM(link_ops_rwsem); 1786 1787 static const struct rdma_link_ops *link_ops_get(const char *type) 1788 { 1789 const struct rdma_link_ops *ops; 1790 1791 list_for_each_entry(ops, &link_ops, list) { 1792 if (!strcmp(ops->type, type)) 1793 goto out; 1794 } 1795 ops = NULL; 1796 out: 1797 return ops; 1798 } 1799 1800 void rdma_link_register(struct rdma_link_ops *ops) 1801 { 1802 down_write(&link_ops_rwsem); 1803 if (WARN_ON_ONCE(link_ops_get(ops->type))) 1804 goto out; 1805 list_add(&ops->list, &link_ops); 1806 out: 1807 up_write(&link_ops_rwsem); 1808 } 1809 EXPORT_SYMBOL(rdma_link_register); 1810 1811 void rdma_link_unregister(struct rdma_link_ops *ops) 1812 { 1813 down_write(&link_ops_rwsem); 1814 list_del(&ops->list); 1815 up_write(&link_ops_rwsem); 1816 } 1817 EXPORT_SYMBOL(rdma_link_unregister); 1818 1819 static int nldev_newlink(struct sk_buff *skb, struct nlmsghdr *nlh, 1820 struct netlink_ext_ack *extack) 1821 { 1822 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 1823 char ibdev_name[IB_DEVICE_NAME_MAX]; 1824 const struct rdma_link_ops *ops; 1825 char ndev_name[IFNAMSIZ]; 1826 struct net_device *ndev; 1827 char type[IFNAMSIZ]; 1828 int err; 1829 1830 err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 1831 nldev_policy, extack); 1832 if (err || !tb[RDMA_NLDEV_ATTR_DEV_NAME] || 1833 !tb[RDMA_NLDEV_ATTR_LINK_TYPE] || !tb[RDMA_NLDEV_ATTR_NDEV_NAME]) 1834 return -EINVAL; 1835 1836 nla_strscpy(ibdev_name, tb[RDMA_NLDEV_ATTR_DEV_NAME], 1837 sizeof(ibdev_name)); 1838 if (strchr(ibdev_name, '%') || strlen(ibdev_name) == 0) 1839 return -EINVAL; 1840 1841 nla_strscpy(type, tb[RDMA_NLDEV_ATTR_LINK_TYPE], sizeof(type)); 1842 nla_strscpy(ndev_name, tb[RDMA_NLDEV_ATTR_NDEV_NAME], 1843 sizeof(ndev_name)); 1844 1845 ndev = dev_get_by_name(sock_net(skb->sk), ndev_name); 1846 if (!ndev) 1847 return -ENODEV; 1848 1849 down_read(&link_ops_rwsem); 1850 ops = link_ops_get(type); 1851 #ifdef CONFIG_MODULES 1852 if (!ops) { 1853 up_read(&link_ops_rwsem); 1854 request_module("rdma-link-%s", type); 1855 down_read(&link_ops_rwsem); 1856 ops = link_ops_get(type); 1857 } 1858 #endif 1859 err = ops ? ops->newlink(ibdev_name, ndev) : -EINVAL; 1860 up_read(&link_ops_rwsem); 1861 dev_put(ndev); 1862 1863 return err; 1864 } 1865 1866 static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh, 1867 struct netlink_ext_ack *extack) 1868 { 1869 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 1870 struct ib_device *device; 1871 u32 index; 1872 int err; 1873 1874 err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 1875 nldev_policy, extack); 1876 if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX]) 1877 return -EINVAL; 1878 1879 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 1880 device = ib_device_get_by_index(sock_net(skb->sk), index); 1881 if (!device) 1882 return -EINVAL; 1883 1884 if (!(device->attrs.kernel_cap_flags & IBK_ALLOW_USER_UNREG)) { 1885 ib_device_put(device); 1886 return -EINVAL; 1887 } 1888 1889 /* 1890 * This path is triggered by the 'rdma link delete' administrative command. 1891 * For Soft-RoCE (RXE), we ensure that transport sockets are closed here. 1892 * Note: iWARP driver does not implement .dellink, so this logic is 1893 * implicitly scoped to the driver supporting dynamic link deletion like RXE. 1894 */ 1895 if (device->link_ops && device->link_ops->dellink) { 1896 mutex_lock(&nldev_dellink_mutex); 1897 err = device->link_ops->dellink(device); 1898 mutex_unlock(&nldev_dellink_mutex); 1899 if (err) { 1900 ib_device_put(device); 1901 return err; 1902 } 1903 } 1904 1905 ib_unregister_device_and_put(device); 1906 return 0; 1907 } 1908 1909 static int nldev_get_chardev(struct sk_buff *skb, struct nlmsghdr *nlh, 1910 struct netlink_ext_ack *extack) 1911 { 1912 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 1913 char client_name[RDMA_NLDEV_ATTR_CHARDEV_TYPE_SIZE]; 1914 struct ib_client_nl_info data = {}; 1915 struct ib_device *ibdev = NULL; 1916 struct sk_buff *msg; 1917 u32 index; 1918 int err; 1919 1920 err = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, nldev_policy, 1921 NL_VALIDATE_LIBERAL, extack); 1922 if (err || !tb[RDMA_NLDEV_ATTR_CHARDEV_TYPE]) 1923 return -EINVAL; 1924 1925 nla_strscpy(client_name, tb[RDMA_NLDEV_ATTR_CHARDEV_TYPE], 1926 sizeof(client_name)); 1927 1928 if (tb[RDMA_NLDEV_ATTR_DEV_INDEX]) { 1929 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 1930 ibdev = ib_device_get_by_index(sock_net(skb->sk), index); 1931 if (!ibdev) 1932 return -EINVAL; 1933 1934 if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) { 1935 data.port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]); 1936 if (!rdma_is_port_valid(ibdev, data.port)) { 1937 err = -EINVAL; 1938 goto out_put; 1939 } 1940 } else { 1941 data.port = -1; 1942 } 1943 } else if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) { 1944 return -EINVAL; 1945 } 1946 1947 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 1948 if (!msg) { 1949 err = -ENOMEM; 1950 goto out_put; 1951 } 1952 nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq, 1953 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, 1954 RDMA_NLDEV_CMD_GET_CHARDEV), 1955 0, 0); 1956 if (!nlh) { 1957 err = -EMSGSIZE; 1958 goto out_nlmsg; 1959 } 1960 1961 data.nl_msg = msg; 1962 err = ib_get_client_nl_info(ibdev, client_name, &data); 1963 if (err) 1964 goto out_nlmsg; 1965 1966 err = nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CHARDEV, 1967 huge_encode_dev(data.cdev->devt), 1968 RDMA_NLDEV_ATTR_PAD); 1969 if (err) 1970 goto out_data; 1971 err = nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CHARDEV_ABI, data.abi, 1972 RDMA_NLDEV_ATTR_PAD); 1973 if (err) 1974 goto out_data; 1975 if (nla_put_string(msg, RDMA_NLDEV_ATTR_CHARDEV_NAME, 1976 dev_name(data.cdev))) { 1977 err = -EMSGSIZE; 1978 goto out_data; 1979 } 1980 1981 nlmsg_end(msg, nlh); 1982 put_device(data.cdev); 1983 if (ibdev) 1984 ib_device_put(ibdev); 1985 return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid); 1986 1987 out_data: 1988 put_device(data.cdev); 1989 out_nlmsg: 1990 nlmsg_free(msg); 1991 out_put: 1992 if (ibdev) 1993 ib_device_put(ibdev); 1994 return err; 1995 } 1996 1997 static int nldev_sys_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, 1998 struct netlink_ext_ack *extack) 1999 { 2000 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 2001 struct sk_buff *msg; 2002 int err; 2003 2004 err = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 2005 nldev_policy, NL_VALIDATE_LIBERAL, extack); 2006 if (err) 2007 return err; 2008 2009 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 2010 if (!msg) 2011 return -ENOMEM; 2012 2013 nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq, 2014 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, 2015 RDMA_NLDEV_CMD_SYS_GET), 2016 0, 0); 2017 if (!nlh) { 2018 nlmsg_free(msg); 2019 return -EMSGSIZE; 2020 } 2021 2022 err = nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_NETNS_MODE, 2023 (u8)ib_devices_shared_netns); 2024 if (err) { 2025 nlmsg_free(msg); 2026 return err; 2027 } 2028 2029 err = nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE, 2030 (u8)privileged_qkey); 2031 if (err) { 2032 nlmsg_free(msg); 2033 return err; 2034 } 2035 2036 err = nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_MONITOR_MODE, 1); 2037 if (err) { 2038 nlmsg_free(msg); 2039 return err; 2040 } 2041 /* 2042 * Copy-on-fork is supported. 2043 * See commits: 2044 * 70e806e4e645 ("mm: Do early cow for pinned pages during fork() for ptes") 2045 * 4eae4efa2c29 ("hugetlb: do early cow when page pinned on src mm") 2046 * for more details. Don't backport this without them. 2047 * 2048 * Return value ignored on purpose, assume copy-on-fork is not 2049 * supported in case of failure. 2050 */ 2051 nla_put_u8(msg, RDMA_NLDEV_SYS_ATTR_COPY_ON_FORK, 1); 2052 2053 nlmsg_end(msg, nlh); 2054 return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid); 2055 } 2056 2057 static int nldev_set_sys_set_netns_doit(struct nlattr *tb[]) 2058 { 2059 u8 enable; 2060 int err; 2061 2062 enable = nla_get_u8(tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE]); 2063 /* Only 0 and 1 are supported */ 2064 if (enable > 1) 2065 return -EINVAL; 2066 2067 err = rdma_compatdev_set(enable); 2068 return err; 2069 } 2070 2071 static int nldev_set_sys_set_pqkey_doit(struct nlattr *tb[]) 2072 { 2073 u8 enable; 2074 2075 enable = nla_get_u8(tb[RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE]); 2076 /* Only 0 and 1 are supported */ 2077 if (enable > 1) 2078 return -EINVAL; 2079 2080 privileged_qkey = enable; 2081 return 0; 2082 } 2083 2084 static int nldev_set_sys_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh, 2085 struct netlink_ext_ack *extack) 2086 { 2087 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 2088 int err; 2089 2090 err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 2091 nldev_policy, extack); 2092 if (err) 2093 return -EINVAL; 2094 2095 if (tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE]) 2096 return nldev_set_sys_set_netns_doit(tb); 2097 2098 if (tb[RDMA_NLDEV_SYS_ATTR_PRIVILEGED_QKEY_MODE]) 2099 return nldev_set_sys_set_pqkey_doit(tb); 2100 2101 return -EINVAL; 2102 } 2103 2104 2105 static int nldev_stat_set_mode_doit(struct sk_buff *msg, 2106 struct netlink_ext_ack *extack, 2107 struct nlattr *tb[], 2108 struct ib_device *device, u32 port) 2109 { 2110 u32 mode, mask = 0, qpn, cntn = 0; 2111 bool opcnt = false; 2112 int ret; 2113 2114 /* Currently only counter for QP is supported */ 2115 if (!tb[RDMA_NLDEV_ATTR_STAT_RES] || 2116 nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES]) != RDMA_NLDEV_ATTR_RES_QP) 2117 return -EINVAL; 2118 2119 if (tb[RDMA_NLDEV_ATTR_STAT_OPCOUNTER_ENABLED]) 2120 opcnt = !!nla_get_u8( 2121 tb[RDMA_NLDEV_ATTR_STAT_OPCOUNTER_ENABLED]); 2122 2123 mode = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_MODE]); 2124 if (mode == RDMA_COUNTER_MODE_AUTO) { 2125 if (tb[RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK]) 2126 mask = nla_get_u32( 2127 tb[RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK]); 2128 return rdma_counter_set_auto_mode(device, port, mask, opcnt, 2129 extack); 2130 } 2131 2132 if (!tb[RDMA_NLDEV_ATTR_RES_LQPN]) 2133 return -EINVAL; 2134 2135 qpn = nla_get_u32(tb[RDMA_NLDEV_ATTR_RES_LQPN]); 2136 if (tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]) { 2137 cntn = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]); 2138 ret = rdma_counter_bind_qpn(device, port, qpn, cntn); 2139 if (ret) 2140 return ret; 2141 } else { 2142 ret = rdma_counter_bind_qpn_alloc(device, port, qpn, &cntn); 2143 if (ret) 2144 return ret; 2145 } 2146 2147 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, cntn) || 2148 nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn)) { 2149 ret = -EMSGSIZE; 2150 goto err_fill; 2151 } 2152 2153 return 0; 2154 2155 err_fill: 2156 rdma_counter_unbind_qpn(device, port, qpn, cntn); 2157 return ret; 2158 } 2159 2160 static int nldev_stat_set_counter_dynamic_doit(struct nlattr *tb[], 2161 struct ib_device *device, 2162 u32 port) 2163 { 2164 struct rdma_hw_stats *stats; 2165 struct nlattr *entry_attr; 2166 unsigned long *target; 2167 int rem, i, ret = 0; 2168 u32 index; 2169 2170 stats = ib_get_hw_stats_port(device, port); 2171 if (!stats) 2172 return -EINVAL; 2173 2174 target = kcalloc(BITS_TO_LONGS(stats->num_counters), 2175 sizeof(*stats->is_disabled), GFP_KERNEL); 2176 if (!target) 2177 return -ENOMEM; 2178 2179 nla_for_each_nested(entry_attr, tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS], 2180 rem) { 2181 if (nla_len(entry_attr) != sizeof(u32)) { 2182 ret = -EINVAL; 2183 goto out; 2184 } 2185 2186 index = nla_get_u32(entry_attr); 2187 if ((index >= stats->num_counters) || 2188 !(stats->descs[index].flags & IB_STAT_FLAG_OPTIONAL)) { 2189 ret = -EINVAL; 2190 goto out; 2191 } 2192 2193 set_bit(index, target); 2194 } 2195 2196 for (i = 0; i < stats->num_counters; i++) { 2197 if (!(stats->descs[i].flags & IB_STAT_FLAG_OPTIONAL)) 2198 continue; 2199 2200 ret = rdma_counter_modify(device, port, i, test_bit(i, target)); 2201 if (ret) 2202 goto out; 2203 } 2204 2205 out: 2206 kfree(target); 2207 return ret; 2208 } 2209 2210 static int nldev_stat_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh, 2211 struct netlink_ext_ack *extack) 2212 { 2213 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 2214 struct ib_device *device; 2215 struct sk_buff *msg; 2216 u32 index, port; 2217 int ret; 2218 2219 ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, nldev_policy, 2220 extack); 2221 if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || 2222 !tb[RDMA_NLDEV_ATTR_PORT_INDEX]) 2223 return -EINVAL; 2224 2225 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 2226 device = ib_device_get_by_index(sock_net(skb->sk), index); 2227 if (!device) 2228 return -EINVAL; 2229 2230 port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]); 2231 if (!rdma_is_port_valid(device, port)) { 2232 ret = -EINVAL; 2233 goto err_put_device; 2234 } 2235 2236 if (!tb[RDMA_NLDEV_ATTR_STAT_MODE] && 2237 !tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS]) { 2238 ret = -EINVAL; 2239 goto err_put_device; 2240 } 2241 2242 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 2243 if (!msg) { 2244 ret = -ENOMEM; 2245 goto err_put_device; 2246 } 2247 nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq, 2248 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, 2249 RDMA_NLDEV_CMD_STAT_SET), 2250 0, 0); 2251 if (!nlh || fill_nldev_handle(msg, device) || 2252 nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) { 2253 ret = -EMSGSIZE; 2254 goto err_free_msg; 2255 } 2256 2257 if (tb[RDMA_NLDEV_ATTR_STAT_MODE]) { 2258 ret = nldev_stat_set_mode_doit(msg, extack, tb, device, port); 2259 if (ret) 2260 goto err_free_msg; 2261 } 2262 2263 if (tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS]) { 2264 ret = nldev_stat_set_counter_dynamic_doit(tb, device, port); 2265 if (ret) 2266 goto err_free_msg; 2267 } 2268 2269 nlmsg_end(msg, nlh); 2270 ib_device_put(device); 2271 return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid); 2272 2273 err_free_msg: 2274 nlmsg_free(msg); 2275 err_put_device: 2276 ib_device_put(device); 2277 return ret; 2278 } 2279 2280 static int nldev_stat_del_doit(struct sk_buff *skb, struct nlmsghdr *nlh, 2281 struct netlink_ext_ack *extack) 2282 { 2283 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 2284 struct ib_device *device; 2285 struct sk_buff *msg; 2286 u32 index, port, qpn, cntn; 2287 int ret; 2288 2289 ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 2290 nldev_policy, extack); 2291 if (ret || !tb[RDMA_NLDEV_ATTR_STAT_RES] || 2292 !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX] || 2293 !tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID] || 2294 !tb[RDMA_NLDEV_ATTR_RES_LQPN]) 2295 return -EINVAL; 2296 2297 if (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES]) != RDMA_NLDEV_ATTR_RES_QP) 2298 return -EINVAL; 2299 2300 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 2301 device = ib_device_get_by_index(sock_net(skb->sk), index); 2302 if (!device) 2303 return -EINVAL; 2304 2305 port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]); 2306 if (!rdma_is_port_valid(device, port)) { 2307 ret = -EINVAL; 2308 goto err; 2309 } 2310 2311 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 2312 if (!msg) { 2313 ret = -ENOMEM; 2314 goto err; 2315 } 2316 nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq, 2317 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, 2318 RDMA_NLDEV_CMD_STAT_SET), 2319 0, 0); 2320 if (!nlh) { 2321 ret = -EMSGSIZE; 2322 goto err_fill; 2323 } 2324 2325 cntn = nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]); 2326 qpn = nla_get_u32(tb[RDMA_NLDEV_ATTR_RES_LQPN]); 2327 if (fill_nldev_handle(msg, device) || 2328 nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port) || 2329 nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_COUNTER_ID, cntn) || 2330 nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qpn)) { 2331 ret = -EMSGSIZE; 2332 goto err_fill; 2333 } 2334 2335 ret = rdma_counter_unbind_qpn(device, port, qpn, cntn); 2336 if (ret) 2337 goto err_fill; 2338 2339 nlmsg_end(msg, nlh); 2340 ib_device_put(device); 2341 return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid); 2342 2343 err_fill: 2344 nlmsg_free(msg); 2345 err: 2346 ib_device_put(device); 2347 return ret; 2348 } 2349 2350 static noinline_for_stack int 2351 stat_get_doit_default_counter(struct sk_buff *skb, struct nlmsghdr *nlh, 2352 struct netlink_ext_ack *extack, 2353 struct nlattr *tb[]) 2354 { 2355 struct rdma_hw_stats *stats; 2356 struct nlattr *table_attr; 2357 struct ib_device *device; 2358 int ret, num_cnts, i; 2359 struct sk_buff *msg; 2360 u32 index, port; 2361 u64 v; 2362 2363 if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX]) 2364 return -EINVAL; 2365 2366 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 2367 device = ib_device_get_by_index(sock_net(skb->sk), index); 2368 if (!device) 2369 return -EINVAL; 2370 2371 if (!device->ops.alloc_hw_port_stats || !device->ops.get_hw_stats) { 2372 ret = -EINVAL; 2373 goto err; 2374 } 2375 2376 port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]); 2377 stats = ib_get_hw_stats_port(device, port); 2378 if (!stats) { 2379 ret = -EINVAL; 2380 goto err; 2381 } 2382 2383 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 2384 if (!msg) { 2385 ret = -ENOMEM; 2386 goto err; 2387 } 2388 2389 nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq, 2390 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, 2391 RDMA_NLDEV_CMD_STAT_GET), 2392 0, 0); 2393 2394 if (!nlh || fill_nldev_handle(msg, device) || 2395 nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) { 2396 ret = -EMSGSIZE; 2397 goto err_msg; 2398 } 2399 2400 mutex_lock(&stats->lock); 2401 2402 num_cnts = device->ops.get_hw_stats(device, stats, port, 0); 2403 if (num_cnts < 0) { 2404 ret = -EINVAL; 2405 goto err_stats; 2406 } 2407 2408 table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS); 2409 if (!table_attr) { 2410 ret = -EMSGSIZE; 2411 goto err_stats; 2412 } 2413 for (i = 0; i < num_cnts; i++) { 2414 if (test_bit(i, stats->is_disabled)) 2415 continue; 2416 2417 v = stats->value[i] + 2418 rdma_counter_get_hwstat_value(device, port, i); 2419 if (rdma_nl_stat_hwcounter_entry(msg, 2420 stats->descs[i].name, v)) { 2421 ret = -EMSGSIZE; 2422 goto err_table; 2423 } 2424 } 2425 nla_nest_end(msg, table_attr); 2426 2427 mutex_unlock(&stats->lock); 2428 nlmsg_end(msg, nlh); 2429 ib_device_put(device); 2430 return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid); 2431 2432 err_table: 2433 nla_nest_cancel(msg, table_attr); 2434 err_stats: 2435 mutex_unlock(&stats->lock); 2436 err_msg: 2437 nlmsg_free(msg); 2438 err: 2439 ib_device_put(device); 2440 return ret; 2441 } 2442 2443 static noinline_for_stack int 2444 stat_get_doit_qp(struct sk_buff *skb, struct nlmsghdr *nlh, 2445 struct netlink_ext_ack *extack, struct nlattr *tb[]) 2446 2447 { 2448 static enum rdma_nl_counter_mode mode; 2449 static enum rdma_nl_counter_mask mask; 2450 struct ib_device *device; 2451 struct sk_buff *msg; 2452 u32 index, port; 2453 bool opcnt; 2454 int ret; 2455 2456 if (tb[RDMA_NLDEV_ATTR_STAT_COUNTER_ID]) 2457 return nldev_res_get_counter_doit(skb, nlh, extack); 2458 2459 if (!tb[RDMA_NLDEV_ATTR_STAT_MODE] || 2460 !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || !tb[RDMA_NLDEV_ATTR_PORT_INDEX]) 2461 return -EINVAL; 2462 2463 index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 2464 device = ib_device_get_by_index(sock_net(skb->sk), index); 2465 if (!device) 2466 return -EINVAL; 2467 2468 port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]); 2469 if (!rdma_is_port_valid(device, port)) { 2470 ret = -EINVAL; 2471 goto err; 2472 } 2473 2474 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 2475 if (!msg) { 2476 ret = -ENOMEM; 2477 goto err; 2478 } 2479 2480 nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq, 2481 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, 2482 RDMA_NLDEV_CMD_STAT_GET), 2483 0, 0); 2484 if (!nlh) { 2485 ret = -EMSGSIZE; 2486 goto err_msg; 2487 } 2488 2489 ret = rdma_counter_get_mode(device, port, &mode, &mask, &opcnt); 2490 if (ret) 2491 goto err_msg; 2492 2493 if (fill_nldev_handle(msg, device) || 2494 nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port) || 2495 nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_MODE, mode)) { 2496 ret = -EMSGSIZE; 2497 goto err_msg; 2498 } 2499 2500 if ((mode == RDMA_COUNTER_MODE_AUTO) && 2501 nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK, mask)) { 2502 ret = -EMSGSIZE; 2503 goto err_msg; 2504 } 2505 2506 if ((mode == RDMA_COUNTER_MODE_AUTO) && 2507 nla_put_u8(msg, RDMA_NLDEV_ATTR_STAT_OPCOUNTER_ENABLED, opcnt)) { 2508 ret = -EMSGSIZE; 2509 goto err_msg; 2510 } 2511 2512 nlmsg_end(msg, nlh); 2513 ib_device_put(device); 2514 return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid); 2515 2516 err_msg: 2517 nlmsg_free(msg); 2518 err: 2519 ib_device_put(device); 2520 return ret; 2521 } 2522 2523 static int nldev_stat_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh, 2524 struct netlink_ext_ack *extack) 2525 { 2526 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 2527 int ret; 2528 2529 ret = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 2530 nldev_policy, NL_VALIDATE_LIBERAL, extack); 2531 if (ret) 2532 return -EINVAL; 2533 2534 if (!tb[RDMA_NLDEV_ATTR_STAT_RES]) 2535 return stat_get_doit_default_counter(skb, nlh, extack, tb); 2536 2537 switch (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES])) { 2538 case RDMA_NLDEV_ATTR_RES_QP: 2539 ret = stat_get_doit_qp(skb, nlh, extack, tb); 2540 break; 2541 case RDMA_NLDEV_ATTR_RES_MR: 2542 ret = res_get_common_doit(skb, nlh, extack, RDMA_RESTRACK_MR, 2543 fill_stat_mr_entry); 2544 break; 2545 default: 2546 ret = -EINVAL; 2547 break; 2548 } 2549 2550 return ret; 2551 } 2552 2553 static int nldev_stat_get_dumpit(struct sk_buff *skb, 2554 struct netlink_callback *cb) 2555 { 2556 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 2557 int ret; 2558 2559 ret = __nlmsg_parse(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 2560 nldev_policy, NL_VALIDATE_LIBERAL, NULL); 2561 if (ret || !tb[RDMA_NLDEV_ATTR_STAT_RES]) 2562 return -EINVAL; 2563 2564 switch (nla_get_u32(tb[RDMA_NLDEV_ATTR_STAT_RES])) { 2565 case RDMA_NLDEV_ATTR_RES_QP: 2566 ret = nldev_res_get_counter_dumpit(skb, cb); 2567 break; 2568 case RDMA_NLDEV_ATTR_RES_MR: 2569 ret = res_get_common_dumpit(skb, cb, RDMA_RESTRACK_MR, 2570 fill_stat_mr_entry); 2571 break; 2572 default: 2573 ret = -EINVAL; 2574 break; 2575 } 2576 2577 return ret; 2578 } 2579 2580 static int nldev_stat_get_counter_status_doit(struct sk_buff *skb, 2581 struct nlmsghdr *nlh, 2582 struct netlink_ext_ack *extack) 2583 { 2584 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX], *table, *entry; 2585 struct rdma_hw_stats *stats; 2586 struct ib_device *device; 2587 struct sk_buff *msg; 2588 u32 devid, port; 2589 int ret, i; 2590 2591 ret = __nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 2592 nldev_policy, NL_VALIDATE_LIBERAL, extack); 2593 if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || 2594 !tb[RDMA_NLDEV_ATTR_PORT_INDEX]) 2595 return -EINVAL; 2596 2597 devid = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 2598 device = ib_device_get_by_index(sock_net(skb->sk), devid); 2599 if (!device) 2600 return -EINVAL; 2601 2602 port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]); 2603 if (!rdma_is_port_valid(device, port)) { 2604 ret = -EINVAL; 2605 goto err; 2606 } 2607 2608 stats = ib_get_hw_stats_port(device, port); 2609 if (!stats) { 2610 ret = -EINVAL; 2611 goto err; 2612 } 2613 2614 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 2615 if (!msg) { 2616 ret = -ENOMEM; 2617 goto err; 2618 } 2619 2620 nlh = nlmsg_put( 2621 msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq, 2622 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_STAT_GET_STATUS), 2623 0, 0); 2624 2625 ret = -EMSGSIZE; 2626 if (!nlh || fill_nldev_handle(msg, device) || 2627 nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port)) 2628 goto err_msg; 2629 2630 table = nla_nest_start(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTERS); 2631 if (!table) 2632 goto err_msg; 2633 2634 mutex_lock(&stats->lock); 2635 for (i = 0; i < stats->num_counters; i++) { 2636 entry = nla_nest_start(msg, 2637 RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY); 2638 if (!entry) 2639 goto err_msg_table; 2640 2641 if (nla_put_string(msg, 2642 RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME, 2643 stats->descs[i].name) || 2644 nla_put_u32(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_INDEX, i)) 2645 goto err_msg_entry; 2646 2647 if ((stats->descs[i].flags & IB_STAT_FLAG_OPTIONAL) && 2648 (nla_put_u8(msg, RDMA_NLDEV_ATTR_STAT_HWCOUNTER_DYNAMIC, 2649 !test_bit(i, stats->is_disabled)))) 2650 goto err_msg_entry; 2651 2652 nla_nest_end(msg, entry); 2653 } 2654 mutex_unlock(&stats->lock); 2655 2656 nla_nest_end(msg, table); 2657 nlmsg_end(msg, nlh); 2658 ib_device_put(device); 2659 return rdma_nl_unicast(sock_net(skb->sk), msg, NETLINK_CB(skb).portid); 2660 2661 err_msg_entry: 2662 nla_nest_cancel(msg, entry); 2663 err_msg_table: 2664 mutex_unlock(&stats->lock); 2665 nla_nest_cancel(msg, table); 2666 err_msg: 2667 nlmsg_free(msg); 2668 err: 2669 ib_device_put(device); 2670 return ret; 2671 } 2672 2673 static int nldev_newdev(struct sk_buff *skb, struct nlmsghdr *nlh, 2674 struct netlink_ext_ack *extack) 2675 { 2676 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 2677 enum rdma_nl_dev_type type; 2678 struct ib_device *parent; 2679 char name[IFNAMSIZ] = {}; 2680 u32 parentid; 2681 int ret; 2682 2683 ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 2684 nldev_policy, extack); 2685 if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX] || 2686 !tb[RDMA_NLDEV_ATTR_DEV_NAME] || !tb[RDMA_NLDEV_ATTR_DEV_TYPE]) 2687 return -EINVAL; 2688 2689 nla_strscpy(name, tb[RDMA_NLDEV_ATTR_DEV_NAME], sizeof(name)); 2690 type = nla_get_u8(tb[RDMA_NLDEV_ATTR_DEV_TYPE]); 2691 parentid = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 2692 parent = ib_device_get_by_index(sock_net(skb->sk), parentid); 2693 if (!parent) 2694 return -EINVAL; 2695 2696 ret = ib_add_sub_device(parent, type, name); 2697 ib_device_put(parent); 2698 2699 return ret; 2700 } 2701 2702 static int nldev_deldev(struct sk_buff *skb, struct nlmsghdr *nlh, 2703 struct netlink_ext_ack *extack) 2704 { 2705 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 2706 struct ib_device *device; 2707 u32 devid; 2708 int ret; 2709 2710 ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 2711 nldev_policy, extack); 2712 if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX]) 2713 return -EINVAL; 2714 2715 devid = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]); 2716 device = ib_device_get_by_index(sock_net(skb->sk), devid); 2717 if (!device) 2718 return -EINVAL; 2719 2720 return ib_del_sub_device_and_put(device); 2721 } 2722 2723 static int fill_frmr_pool_key(struct sk_buff *msg, struct ib_frmr_key *key) 2724 { 2725 struct nlattr *key_attr; 2726 2727 key_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY); 2728 if (!key_attr) 2729 return -EMSGSIZE; 2730 2731 if (nla_put_u8(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ATS, key->ats)) 2732 goto err; 2733 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ACCESS_FLAGS, 2734 key->access_flags)) 2735 goto err; 2736 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY_VENDOR_KEY, 2737 key->vendor_key, RDMA_NLDEV_ATTR_PAD)) 2738 goto err; 2739 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_FRMR_POOL_KEY_NUM_DMA_BLOCKS, 2740 key->num_dma_blocks, RDMA_NLDEV_ATTR_PAD)) 2741 goto err; 2742 2743 if (key->kernel_vendor_key && 2744 nla_put_u64_64bit(msg, 2745 RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY, 2746 key->kernel_vendor_key, RDMA_NLDEV_ATTR_PAD)) 2747 goto err; 2748 2749 nla_nest_end(msg, key_attr); 2750 return 0; 2751 2752 err: 2753 return -EMSGSIZE; 2754 } 2755 2756 static int fill_frmr_pool_entry(struct sk_buff *msg, struct ib_frmr_pool *pool) 2757 { 2758 if (fill_frmr_pool_key(msg, &pool->key)) 2759 return -EMSGSIZE; 2760 2761 spin_lock(&pool->lock); 2762 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_FRMR_POOL_QUEUE_HANDLES, 2763 pool->queue.ci + pool->inactive_queue.ci)) 2764 goto err_unlock; 2765 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_FRMR_POOL_MAX_IN_USE, 2766 pool->max_in_use, RDMA_NLDEV_ATTR_PAD)) 2767 goto err_unlock; 2768 if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_FRMR_POOL_IN_USE, 2769 pool->in_use, RDMA_NLDEV_ATTR_PAD)) 2770 goto err_unlock; 2771 if (nla_put_u32(msg, RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES, 2772 pool->pinned_handles)) 2773 goto err_unlock; 2774 spin_unlock(&pool->lock); 2775 2776 return 0; 2777 2778 err_unlock: 2779 spin_unlock(&pool->lock); 2780 return -EMSGSIZE; 2781 } 2782 2783 static int nldev_frmr_pools_parse_key(struct nlattr *tb[], 2784 struct ib_frmr_key *key, 2785 struct netlink_ext_ack *extack) 2786 { 2787 if (tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ATS]) 2788 key->ats = nla_get_u8(tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ATS]); 2789 2790 if (tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ACCESS_FLAGS]) 2791 key->access_flags = nla_get_u32( 2792 tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_ACCESS_FLAGS]); 2793 2794 if (tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_VENDOR_KEY]) 2795 key->vendor_key = nla_get_u64( 2796 tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_VENDOR_KEY]); 2797 2798 if (tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_NUM_DMA_BLOCKS]) 2799 key->num_dma_blocks = nla_get_u64( 2800 tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_NUM_DMA_BLOCKS]); 2801 2802 if (tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY_KERNEL_VENDOR_KEY]) 2803 return -EINVAL; 2804 2805 return 0; 2806 } 2807 2808 static int nldev_frmr_pools_set_pinned(struct ib_device *device, 2809 struct nlattr *tb[], 2810 struct netlink_ext_ack *extack) 2811 { 2812 struct nlattr *key_tb[RDMA_NLDEV_ATTR_MAX]; 2813 struct ib_frmr_key key = { 0 }; 2814 u32 pinned_handles = 0; 2815 int err = 0; 2816 2817 pinned_handles = 2818 nla_get_u32(tb[RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES]); 2819 2820 if (!tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY]) 2821 return -EINVAL; 2822 2823 err = nla_parse_nested(key_tb, RDMA_NLDEV_ATTR_MAX - 1, 2824 tb[RDMA_NLDEV_ATTR_FRMR_POOL_KEY], nldev_policy, 2825 extack); 2826 if (err) 2827 return err; 2828 2829 err = nldev_frmr_pools_parse_key(key_tb, &key, extack); 2830 if (err) 2831 return err; 2832 2833 err = ib_frmr_pools_set_pinned(device, &key, pinned_handles); 2834 2835 return err; 2836 } 2837 2838 static int nldev_frmr_pools_get_dumpit(struct sk_buff *skb, 2839 struct netlink_callback *cb) 2840 { 2841 struct nlattr *tb[RDMA_NLDEV_ATTR_MAX]; 2842 struct ib_frmr_pools *pools; 2843 int err, ret = 0, idx = 0; 2844 struct ib_frmr_pool *pool; 2845 struct nlattr *table_attr; 2846 struct nlattr *entry_attr; 2847 bool show_details = false; 2848 struct ib_device *device; 2849 int start = cb->args[0]; 2850 struct rb_node *node; 2851 struct nlmsghdr *nlh; 2852 bool filled = false; 2853 2854 err = __nlmsg_parse(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, 2855 nldev_policy, NL_VALIDATE_LIBERAL, NULL); 2856 if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX]) 2857 return -EINVAL; 2858 2859 device = ib_device_get_by_index( 2860 sock_net(skb->sk), nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX])); 2861 if (!device) 2862 return -EINVAL; 2863 2864 if (tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]) 2865 show_details = nla_get_u8(tb[RDMA_NLDEV_ATTR_DRIVER_DETAILS]); 2866 2867 pools = device->frmr_pools; 2868 if (!pools) { 2869 ib_device_put(device); 2870 return 0; 2871 } 2872 2873 nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, 2874 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, 2875 RDMA_NLDEV_CMD_FRMR_POOLS_GET), 2876 0, NLM_F_MULTI); 2877 2878 if (!nlh || fill_nldev_handle(skb, device)) { 2879 ret = -EMSGSIZE; 2880 goto err; 2881 } 2882 2883 table_attr = nla_nest_start_noflag(skb, RDMA_NLDEV_ATTR_FRMR_POOLS); 2884 if (!table_attr) { 2885 ret = -EMSGSIZE; 2886 goto err; 2887 } 2888 2889 read_lock(&pools->rb_lock); 2890 for (node = rb_first(&pools->rb_root); node; node = rb_next(node)) { 2891 pool = rb_entry(node, struct ib_frmr_pool, node); 2892 if (pool->key.kernel_vendor_key && !show_details) 2893 continue; 2894 2895 if (idx < start) { 2896 idx++; 2897 continue; 2898 } 2899 2900 filled = true; 2901 2902 entry_attr = nla_nest_start_noflag( 2903 skb, RDMA_NLDEV_ATTR_FRMR_POOL_ENTRY); 2904 if (!entry_attr) { 2905 ret = -EMSGSIZE; 2906 goto end_msg; 2907 } 2908 2909 if (fill_frmr_pool_entry(skb, pool)) { 2910 nla_nest_cancel(skb, entry_attr); 2911 ret = -EMSGSIZE; 2912 goto end_msg; 2913 } 2914 2915 nla_nest_end(skb, entry_attr); 2916 idx++; 2917 } 2918 end_msg: 2919 read_unlock(&pools->rb_lock); 2920 2921 nla_nest_end(skb, table_attr); 2922 nlmsg_end(skb, nlh); 2923 cb->args[0] = idx; 2924 2925 /* 2926 * No more entries to fill, cancel the message and 2927 * return 0 to mark end of dumpit. 2928 */ 2929 if (!filled) 2930 goto err; 2931 2932 ib_device_put(device); 2933 return skb->len; 2934 2935 err: 2936 nlmsg_cancel(skb, nlh); 2937 ib_device_put(device); 2938 return ret; 2939 } 2940 2941 static int nldev_frmr_pools_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh, 2942 struct netlink_ext_ack *extack) 2943 { 2944 struct ib_device *device; 2945 struct nlattr **tb; 2946 u32 aging_period; 2947 int err; 2948 2949 tb = kzalloc_objs(*tb, RDMA_NLDEV_ATTR_MAX); 2950 if (!tb) 2951 return -ENOMEM; 2952 2953 err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1, nldev_policy, 2954 extack); 2955 if (err) 2956 goto free_tb; 2957 2958 if (!tb[RDMA_NLDEV_ATTR_DEV_INDEX]) { 2959 err = -EINVAL; 2960 goto free_tb; 2961 } 2962 2963 device = ib_device_get_by_index( 2964 sock_net(skb->sk), nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX])); 2965 if (!device) { 2966 err = -EINVAL; 2967 goto free_tb; 2968 } 2969 2970 if (tb[RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD]) { 2971 aging_period = nla_get_u32( 2972 tb[RDMA_NLDEV_ATTR_FRMR_POOLS_AGING_PERIOD]); 2973 err = ib_frmr_pools_set_aging_period(device, aging_period); 2974 goto done; 2975 } 2976 2977 if (tb[RDMA_NLDEV_ATTR_FRMR_POOL_PINNED_HANDLES]) 2978 err = nldev_frmr_pools_set_pinned(device, tb, extack); 2979 2980 done: 2981 ib_device_put(device); 2982 free_tb: 2983 kfree(tb); 2984 return err; 2985 } 2986 2987 static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = { 2988 [RDMA_NLDEV_CMD_GET] = { 2989 .doit = nldev_get_doit, 2990 .dump = nldev_get_dumpit, 2991 }, 2992 [RDMA_NLDEV_CMD_GET_CHARDEV] = { 2993 .doit = nldev_get_chardev, 2994 }, 2995 [RDMA_NLDEV_CMD_SET] = { 2996 .doit = nldev_set_doit, 2997 .flags = RDMA_NL_ADMIN_PERM, 2998 }, 2999 [RDMA_NLDEV_CMD_NEWLINK] = { 3000 .doit = nldev_newlink, 3001 .flags = RDMA_NL_ADMIN_PERM, 3002 }, 3003 [RDMA_NLDEV_CMD_DELLINK] = { 3004 .doit = nldev_dellink, 3005 .flags = RDMA_NL_ADMIN_PERM, 3006 }, 3007 [RDMA_NLDEV_CMD_PORT_GET] = { 3008 .doit = nldev_port_get_doit, 3009 .dump = nldev_port_get_dumpit, 3010 }, 3011 [RDMA_NLDEV_CMD_RES_GET] = { 3012 .doit = nldev_res_get_doit, 3013 .dump = nldev_res_get_dumpit, 3014 }, 3015 [RDMA_NLDEV_CMD_RES_QP_GET] = { 3016 .doit = nldev_res_get_qp_doit, 3017 .dump = nldev_res_get_qp_dumpit, 3018 }, 3019 [RDMA_NLDEV_CMD_RES_CM_ID_GET] = { 3020 .doit = nldev_res_get_cm_id_doit, 3021 .dump = nldev_res_get_cm_id_dumpit, 3022 }, 3023 [RDMA_NLDEV_CMD_RES_CQ_GET] = { 3024 .doit = nldev_res_get_cq_doit, 3025 .dump = nldev_res_get_cq_dumpit, 3026 }, 3027 [RDMA_NLDEV_CMD_RES_MR_GET] = { 3028 .doit = nldev_res_get_mr_doit, 3029 .dump = nldev_res_get_mr_dumpit, 3030 }, 3031 [RDMA_NLDEV_CMD_RES_PD_GET] = { 3032 .doit = nldev_res_get_pd_doit, 3033 .dump = nldev_res_get_pd_dumpit, 3034 }, 3035 [RDMA_NLDEV_CMD_RES_CTX_GET] = { 3036 .doit = nldev_res_get_ctx_doit, 3037 .dump = nldev_res_get_ctx_dumpit, 3038 }, 3039 [RDMA_NLDEV_CMD_RES_SRQ_GET] = { 3040 .doit = nldev_res_get_srq_doit, 3041 .dump = nldev_res_get_srq_dumpit, 3042 }, 3043 [RDMA_NLDEV_CMD_SYS_GET] = { 3044 .doit = nldev_sys_get_doit, 3045 }, 3046 [RDMA_NLDEV_CMD_SYS_SET] = { 3047 .doit = nldev_set_sys_set_doit, 3048 .flags = RDMA_NL_ADMIN_PERM, 3049 }, 3050 [RDMA_NLDEV_CMD_STAT_SET] = { 3051 .doit = nldev_stat_set_doit, 3052 .flags = RDMA_NL_ADMIN_PERM, 3053 }, 3054 [RDMA_NLDEV_CMD_STAT_GET] = { 3055 .doit = nldev_stat_get_doit, 3056 .dump = nldev_stat_get_dumpit, 3057 }, 3058 [RDMA_NLDEV_CMD_STAT_DEL] = { 3059 .doit = nldev_stat_del_doit, 3060 .flags = RDMA_NL_ADMIN_PERM, 3061 }, 3062 [RDMA_NLDEV_CMD_RES_QP_GET_RAW] = { 3063 .doit = nldev_res_get_qp_raw_doit, 3064 .dump = nldev_res_get_qp_raw_dumpit, 3065 .flags = RDMA_NL_ADMIN_PERM, 3066 }, 3067 [RDMA_NLDEV_CMD_RES_CQ_GET_RAW] = { 3068 .doit = nldev_res_get_cq_raw_doit, 3069 .dump = nldev_res_get_cq_raw_dumpit, 3070 .flags = RDMA_NL_ADMIN_PERM, 3071 }, 3072 [RDMA_NLDEV_CMD_RES_MR_GET_RAW] = { 3073 .doit = nldev_res_get_mr_raw_doit, 3074 .dump = nldev_res_get_mr_raw_dumpit, 3075 .flags = RDMA_NL_ADMIN_PERM, 3076 }, 3077 [RDMA_NLDEV_CMD_RES_SRQ_GET_RAW] = { 3078 .doit = nldev_res_get_srq_raw_doit, 3079 .dump = nldev_res_get_srq_raw_dumpit, 3080 .flags = RDMA_NL_ADMIN_PERM, 3081 }, 3082 [RDMA_NLDEV_CMD_STAT_GET_STATUS] = { 3083 .doit = nldev_stat_get_counter_status_doit, 3084 }, 3085 [RDMA_NLDEV_CMD_NEWDEV] = { 3086 .doit = nldev_newdev, 3087 .flags = RDMA_NL_ADMIN_PERM, 3088 }, 3089 [RDMA_NLDEV_CMD_DELDEV] = { 3090 .doit = nldev_deldev, 3091 .flags = RDMA_NL_ADMIN_PERM, 3092 }, 3093 [RDMA_NLDEV_CMD_FRMR_POOLS_GET] = { 3094 .dump = nldev_frmr_pools_get_dumpit, 3095 }, 3096 [RDMA_NLDEV_CMD_FRMR_POOLS_SET] = { 3097 .doit = nldev_frmr_pools_set_doit, 3098 .flags = RDMA_NL_ADMIN_PERM, 3099 }, 3100 }; 3101 3102 static int fill_mon_netdev_rename(struct sk_buff *msg, 3103 struct ib_device *device, u32 port, 3104 const struct net *net) 3105 { 3106 struct net_device *netdev = ib_device_get_netdev(device, port); 3107 int ret = 0; 3108 3109 if (!netdev || !net_eq(dev_net(netdev), net)) 3110 goto out; 3111 3112 ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_NDEV_INDEX, netdev->ifindex); 3113 if (ret) 3114 goto out; 3115 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_NDEV_NAME, netdev->name); 3116 out: 3117 dev_put(netdev); 3118 return ret; 3119 } 3120 3121 static int fill_mon_netdev_association(struct sk_buff *msg, 3122 struct ib_device *device, u32 port, 3123 const struct net *net) 3124 { 3125 struct net_device *netdev = ib_device_get_netdev(device, port); 3126 int ret = 0; 3127 3128 if (netdev && !net_eq(dev_net(netdev), net)) 3129 goto out; 3130 3131 ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_DEV_INDEX, device->index); 3132 if (ret) 3133 goto out; 3134 3135 ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_NAME, 3136 dev_name(&device->dev)); 3137 if (ret) 3138 goto out; 3139 3140 ret = nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port); 3141 if (ret) 3142 goto out; 3143 3144 if (netdev) { 3145 ret = nla_put_u32(msg, 3146 RDMA_NLDEV_ATTR_NDEV_INDEX, netdev->ifindex); 3147 if (ret) 3148 goto out; 3149 3150 ret = nla_put_string(msg, 3151 RDMA_NLDEV_ATTR_NDEV_NAME, netdev->name); 3152 } 3153 3154 out: 3155 dev_put(netdev); 3156 return ret; 3157 } 3158 3159 static void rdma_nl_notify_err_msg(struct ib_device *device, u32 port_num, 3160 enum rdma_nl_notify_event_type type) 3161 { 3162 struct net_device *netdev; 3163 3164 switch (type) { 3165 case RDMA_REGISTER_EVENT: 3166 dev_warn_ratelimited(&device->dev, 3167 "Failed to send RDMA monitor register device event\n"); 3168 break; 3169 case RDMA_UNREGISTER_EVENT: 3170 dev_warn_ratelimited(&device->dev, 3171 "Failed to send RDMA monitor unregister device event\n"); 3172 break; 3173 case RDMA_NETDEV_ATTACH_EVENT: 3174 netdev = ib_device_get_netdev(device, port_num); 3175 dev_warn_ratelimited(&device->dev, 3176 "Failed to send RDMA monitor netdev attach event: port %d netdev %d\n", 3177 port_num, netdev->ifindex); 3178 dev_put(netdev); 3179 break; 3180 case RDMA_NETDEV_DETACH_EVENT: 3181 dev_warn_ratelimited(&device->dev, 3182 "Failed to send RDMA monitor netdev detach event: port %d\n", 3183 port_num); 3184 break; 3185 case RDMA_RENAME_EVENT: 3186 dev_warn_ratelimited(&device->dev, 3187 "Failed to send RDMA monitor rename device event\n"); 3188 break; 3189 3190 case RDMA_NETDEV_RENAME_EVENT: 3191 netdev = ib_device_get_netdev(device, port_num); 3192 dev_warn_ratelimited(&device->dev, 3193 "Failed to send RDMA monitor netdev rename event: port %d netdev %d\n", 3194 port_num, netdev->ifindex); 3195 dev_put(netdev); 3196 break; 3197 default: 3198 break; 3199 } 3200 } 3201 3202 int rdma_nl_notify_event(struct ib_device *device, u32 port_num, 3203 enum rdma_nl_notify_event_type type) 3204 { 3205 struct sk_buff *skb; 3206 int ret = -EMSGSIZE; 3207 struct net *net; 3208 void *nlh; 3209 3210 net = read_pnet(&device->coredev.rdma_net); 3211 if (!net) 3212 return -EINVAL; 3213 3214 skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 3215 if (!skb) 3216 return -ENOMEM; 3217 nlh = nlmsg_put(skb, 0, 0, 3218 RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_MONITOR), 3219 0, 0); 3220 if (!nlh) 3221 goto err_free; 3222 3223 switch (type) { 3224 case RDMA_REGISTER_EVENT: 3225 case RDMA_UNREGISTER_EVENT: 3226 case RDMA_RENAME_EVENT: 3227 ret = fill_nldev_handle(skb, device); 3228 if (ret) 3229 goto err_free; 3230 break; 3231 case RDMA_NETDEV_ATTACH_EVENT: 3232 case RDMA_NETDEV_DETACH_EVENT: 3233 ret = fill_mon_netdev_association(skb, device, port_num, net); 3234 if (ret) 3235 goto err_free; 3236 break; 3237 case RDMA_NETDEV_RENAME_EVENT: 3238 ret = fill_mon_netdev_rename(skb, device, port_num, net); 3239 if (ret) 3240 goto err_free; 3241 break; 3242 default: 3243 break; 3244 } 3245 3246 ret = nla_put_u8(skb, RDMA_NLDEV_ATTR_EVENT_TYPE, type); 3247 if (ret) 3248 goto err_free; 3249 3250 nlmsg_end(skb, nlh); 3251 ret = rdma_nl_multicast(net, skb, RDMA_NL_GROUP_NOTIFY, GFP_KERNEL); 3252 if (ret && ret != -ESRCH) { 3253 skb = NULL; /* skb is freed in the netlink send-op handling */ 3254 goto err_free; 3255 } 3256 return 0; 3257 3258 err_free: 3259 rdma_nl_notify_err_msg(device, port_num, type); 3260 nlmsg_free(skb); 3261 return ret; 3262 } 3263 3264 void __init nldev_init(void) 3265 { 3266 rdma_nl_register(RDMA_NL_NLDEV, nldev_cb_table); 3267 } 3268 3269 void nldev_exit(void) 3270 { 3271 rdma_nl_unregister(RDMA_NL_NLDEV); 3272 } 3273 3274 MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_NLDEV, 5); 3275