1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2020 Mellanox Technologies Ltd. */ 3 4 #include <linux/module.h> 5 #include <linux/vdpa.h> 6 #include <linux/vringh.h> 7 #include <uapi/linux/virtio_net.h> 8 #include <uapi/linux/virtio_ids.h> 9 #include <uapi/linux/vdpa.h> 10 #include <linux/virtio_config.h> 11 #include <linux/auxiliary_bus.h> 12 #include <linux/mlx5/cq.h> 13 #include <linux/mlx5/qp.h> 14 #include <linux/mlx5/device.h> 15 #include <linux/mlx5/driver.h> 16 #include <linux/mlx5/vport.h> 17 #include <linux/mlx5/fs.h> 18 #include <linux/mlx5/mlx5_ifc_vdpa.h> 19 #include <linux/mlx5/mpfs.h> 20 #include "mlx5_vdpa.h" 21 #include "mlx5_vnet.h" 22 23 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 24 MODULE_DESCRIPTION("Mellanox VDPA driver"); 25 MODULE_LICENSE("Dual BSD/GPL"); 26 27 #define VALID_FEATURES_MASK \ 28 (BIT_ULL(VIRTIO_NET_F_CSUM) | BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) | \ 29 BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) | BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_MAC) | \ 30 BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) | BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) | \ 31 BIT_ULL(VIRTIO_NET_F_GUEST_ECN) | BIT_ULL(VIRTIO_NET_F_GUEST_UFO) | BIT_ULL(VIRTIO_NET_F_HOST_TSO4) | \ 32 BIT_ULL(VIRTIO_NET_F_HOST_TSO6) | BIT_ULL(VIRTIO_NET_F_HOST_ECN) | BIT_ULL(VIRTIO_NET_F_HOST_UFO) | \ 33 BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) | BIT_ULL(VIRTIO_NET_F_STATUS) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ) | \ 34 BIT_ULL(VIRTIO_NET_F_CTRL_RX) | BIT_ULL(VIRTIO_NET_F_CTRL_VLAN) | \ 35 BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA) | BIT_ULL(VIRTIO_NET_F_GUEST_ANNOUNCE) | \ 36 BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) | BIT_ULL(VIRTIO_NET_F_HASH_REPORT) | \ 37 BIT_ULL(VIRTIO_NET_F_RSS) | BIT_ULL(VIRTIO_NET_F_RSC_EXT) | BIT_ULL(VIRTIO_NET_F_STANDBY) | \ 38 BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX) | BIT_ULL(VIRTIO_F_NOTIFY_ON_EMPTY) | \ 39 BIT_ULL(VIRTIO_F_ANY_LAYOUT) | BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) | \ 40 BIT_ULL(VIRTIO_F_RING_PACKED) | BIT_ULL(VIRTIO_F_ORDER_PLATFORM) | BIT_ULL(VIRTIO_F_SR_IOV)) 41 42 #define VALID_STATUS_MASK \ 43 (VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK | \ 44 VIRTIO_CONFIG_S_FEATURES_OK | VIRTIO_CONFIG_S_NEEDS_RESET | VIRTIO_CONFIG_S_FAILED) 45 46 #define MLX5_FEATURE(_mvdev, _feature) (!!((_mvdev)->actual_features & BIT_ULL(_feature))) 47 48 #define MLX5V_UNTAGGED 0x1000 49 50 struct mlx5_vdpa_cq_buf { 51 struct mlx5_frag_buf_ctrl fbc; 52 struct mlx5_frag_buf frag_buf; 53 int cqe_size; 54 int nent; 55 }; 56 57 struct mlx5_vdpa_cq { 58 struct mlx5_core_cq mcq; 59 struct mlx5_vdpa_cq_buf buf; 60 struct mlx5_db db; 61 int cqe; 62 }; 63 64 struct mlx5_vdpa_umem { 65 struct mlx5_frag_buf_ctrl fbc; 66 struct mlx5_frag_buf frag_buf; 67 int size; 68 u32 id; 69 }; 70 71 struct mlx5_vdpa_qp { 72 struct mlx5_core_qp mqp; 73 struct mlx5_frag_buf frag_buf; 74 struct mlx5_db db; 75 u16 head; 76 bool fw; 77 }; 78 79 struct mlx5_vq_restore_info { 80 u32 num_ent; 81 u64 desc_addr; 82 u64 device_addr; 83 u64 driver_addr; 84 u16 avail_index; 85 u16 used_index; 86 bool ready; 87 bool restore; 88 }; 89 90 struct mlx5_vdpa_virtqueue { 91 bool ready; 92 u64 desc_addr; 93 u64 device_addr; 94 u64 driver_addr; 95 u32 num_ent; 96 97 /* Resources for implementing the notification channel from the device 98 * to the driver. fwqp is the firmware end of an RC connection; the 99 * other end is vqqp used by the driver. cq is where completions are 100 * reported. 101 */ 102 struct mlx5_vdpa_cq cq; 103 struct mlx5_vdpa_qp fwqp; 104 struct mlx5_vdpa_qp vqqp; 105 106 /* umem resources are required for the virtqueue operation. They're use 107 * is internal and they must be provided by the driver. 108 */ 109 struct mlx5_vdpa_umem umem1; 110 struct mlx5_vdpa_umem umem2; 111 struct mlx5_vdpa_umem umem3; 112 113 u32 counter_set_id; 114 bool initialized; 115 int index; 116 u32 virtq_id; 117 struct mlx5_vdpa_net *ndev; 118 u16 avail_idx; 119 u16 used_idx; 120 int fw_state; 121 122 /* keep last in the struct */ 123 struct mlx5_vq_restore_info ri; 124 }; 125 126 static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx) 127 { 128 if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) { 129 if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 130 return idx < 2; 131 else 132 return idx < 3; 133 } 134 135 return idx <= mvdev->max_idx; 136 } 137 138 static void free_resources(struct mlx5_vdpa_net *ndev); 139 static void init_mvqs(struct mlx5_vdpa_net *ndev); 140 static int setup_driver(struct mlx5_vdpa_dev *mvdev); 141 static void teardown_driver(struct mlx5_vdpa_net *ndev); 142 143 static bool mlx5_vdpa_debug; 144 145 #define MLX5_CVQ_MAX_ENT 16 146 147 #define MLX5_LOG_VIO_FLAG(_feature) \ 148 do { \ 149 if (features & BIT_ULL(_feature)) \ 150 mlx5_vdpa_info(mvdev, "%s\n", #_feature); \ 151 } while (0) 152 153 #define MLX5_LOG_VIO_STAT(_status) \ 154 do { \ 155 if (status & (_status)) \ 156 mlx5_vdpa_info(mvdev, "%s\n", #_status); \ 157 } while (0) 158 159 /* TODO: cross-endian support */ 160 static inline bool mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev *mvdev) 161 { 162 return virtio_legacy_is_little_endian() || 163 (mvdev->actual_features & BIT_ULL(VIRTIO_F_VERSION_1)); 164 } 165 166 static u16 mlx5vdpa16_to_cpu(struct mlx5_vdpa_dev *mvdev, __virtio16 val) 167 { 168 return __virtio16_to_cpu(mlx5_vdpa_is_little_endian(mvdev), val); 169 } 170 171 static __virtio16 cpu_to_mlx5vdpa16(struct mlx5_vdpa_dev *mvdev, u16 val) 172 { 173 return __cpu_to_virtio16(mlx5_vdpa_is_little_endian(mvdev), val); 174 } 175 176 static u16 ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev) 177 { 178 if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) 179 return 2; 180 181 return mvdev->max_vqs; 182 } 183 184 static bool is_ctrl_vq_idx(struct mlx5_vdpa_dev *mvdev, u16 idx) 185 { 186 return idx == ctrl_vq_idx(mvdev); 187 } 188 189 static void print_status(struct mlx5_vdpa_dev *mvdev, u8 status, bool set) 190 { 191 if (status & ~VALID_STATUS_MASK) 192 mlx5_vdpa_warn(mvdev, "Warning: there are invalid status bits 0x%x\n", 193 status & ~VALID_STATUS_MASK); 194 195 if (!mlx5_vdpa_debug) 196 return; 197 198 mlx5_vdpa_info(mvdev, "driver status %s", set ? "set" : "get"); 199 if (set && !status) { 200 mlx5_vdpa_info(mvdev, "driver resets the device\n"); 201 return; 202 } 203 204 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_ACKNOWLEDGE); 205 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER); 206 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_DRIVER_OK); 207 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FEATURES_OK); 208 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_NEEDS_RESET); 209 MLX5_LOG_VIO_STAT(VIRTIO_CONFIG_S_FAILED); 210 } 211 212 static void print_features(struct mlx5_vdpa_dev *mvdev, u64 features, bool set) 213 { 214 if (features & ~VALID_FEATURES_MASK) 215 mlx5_vdpa_warn(mvdev, "There are invalid feature bits 0x%llx\n", 216 features & ~VALID_FEATURES_MASK); 217 218 if (!mlx5_vdpa_debug) 219 return; 220 221 mlx5_vdpa_info(mvdev, "driver %s feature bits:\n", set ? "sets" : "reads"); 222 if (!features) 223 mlx5_vdpa_info(mvdev, "all feature bits are cleared\n"); 224 225 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CSUM); 226 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_CSUM); 227 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); 228 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MTU); 229 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MAC); 230 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO4); 231 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_TSO6); 232 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ECN); 233 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_UFO); 234 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO4); 235 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_TSO6); 236 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_ECN); 237 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HOST_UFO); 238 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MRG_RXBUF); 239 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STATUS); 240 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VQ); 241 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX); 242 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_VLAN); 243 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_RX_EXTRA); 244 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_GUEST_ANNOUNCE); 245 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_MQ); 246 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_CTRL_MAC_ADDR); 247 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_HASH_REPORT); 248 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSS); 249 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_RSC_EXT); 250 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_STANDBY); 251 MLX5_LOG_VIO_FLAG(VIRTIO_NET_F_SPEED_DUPLEX); 252 MLX5_LOG_VIO_FLAG(VIRTIO_F_NOTIFY_ON_EMPTY); 253 MLX5_LOG_VIO_FLAG(VIRTIO_F_ANY_LAYOUT); 254 MLX5_LOG_VIO_FLAG(VIRTIO_F_VERSION_1); 255 MLX5_LOG_VIO_FLAG(VIRTIO_F_ACCESS_PLATFORM); 256 MLX5_LOG_VIO_FLAG(VIRTIO_F_RING_PACKED); 257 MLX5_LOG_VIO_FLAG(VIRTIO_F_ORDER_PLATFORM); 258 MLX5_LOG_VIO_FLAG(VIRTIO_F_SR_IOV); 259 } 260 261 static int create_tis(struct mlx5_vdpa_net *ndev) 262 { 263 struct mlx5_vdpa_dev *mvdev = &ndev->mvdev; 264 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 265 void *tisc; 266 int err; 267 268 tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 269 MLX5_SET(tisc, tisc, transport_domain, ndev->res.tdn); 270 err = mlx5_vdpa_create_tis(mvdev, in, &ndev->res.tisn); 271 if (err) 272 mlx5_vdpa_warn(mvdev, "create TIS (%d)\n", err); 273 274 return err; 275 } 276 277 static void destroy_tis(struct mlx5_vdpa_net *ndev) 278 { 279 mlx5_vdpa_destroy_tis(&ndev->mvdev, ndev->res.tisn); 280 } 281 282 #define MLX5_VDPA_CQE_SIZE 64 283 #define MLX5_VDPA_LOG_CQE_SIZE ilog2(MLX5_VDPA_CQE_SIZE) 284 285 static int cq_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf, int nent) 286 { 287 struct mlx5_frag_buf *frag_buf = &buf->frag_buf; 288 u8 log_wq_stride = MLX5_VDPA_LOG_CQE_SIZE; 289 u8 log_wq_sz = MLX5_VDPA_LOG_CQE_SIZE; 290 int err; 291 292 err = mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, nent * MLX5_VDPA_CQE_SIZE, frag_buf, 293 ndev->mvdev.mdev->priv.numa_node); 294 if (err) 295 return err; 296 297 mlx5_init_fbc(frag_buf->frags, log_wq_stride, log_wq_sz, &buf->fbc); 298 299 buf->cqe_size = MLX5_VDPA_CQE_SIZE; 300 buf->nent = nent; 301 302 return 0; 303 } 304 305 static int umem_frag_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem, int size) 306 { 307 struct mlx5_frag_buf *frag_buf = &umem->frag_buf; 308 309 return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, size, frag_buf, 310 ndev->mvdev.mdev->priv.numa_node); 311 } 312 313 static void cq_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_cq_buf *buf) 314 { 315 mlx5_frag_buf_free(ndev->mvdev.mdev, &buf->frag_buf); 316 } 317 318 static void *get_cqe(struct mlx5_vdpa_cq *vcq, int n) 319 { 320 return mlx5_frag_buf_get_wqe(&vcq->buf.fbc, n); 321 } 322 323 static void cq_frag_buf_init(struct mlx5_vdpa_cq *vcq, struct mlx5_vdpa_cq_buf *buf) 324 { 325 struct mlx5_cqe64 *cqe64; 326 void *cqe; 327 int i; 328 329 for (i = 0; i < buf->nent; i++) { 330 cqe = get_cqe(vcq, i); 331 cqe64 = cqe; 332 cqe64->op_own = MLX5_CQE_INVALID << 4; 333 } 334 } 335 336 static void *get_sw_cqe(struct mlx5_vdpa_cq *cq, int n) 337 { 338 struct mlx5_cqe64 *cqe64 = get_cqe(cq, n & (cq->cqe - 1)); 339 340 if (likely(get_cqe_opcode(cqe64) != MLX5_CQE_INVALID) && 341 !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & cq->cqe))) 342 return cqe64; 343 344 return NULL; 345 } 346 347 static void rx_post(struct mlx5_vdpa_qp *vqp, int n) 348 { 349 vqp->head += n; 350 vqp->db.db[0] = cpu_to_be32(vqp->head); 351 } 352 353 static void qp_prepare(struct mlx5_vdpa_net *ndev, bool fw, void *in, 354 struct mlx5_vdpa_virtqueue *mvq, u32 num_ent) 355 { 356 struct mlx5_vdpa_qp *vqp; 357 __be64 *pas; 358 void *qpc; 359 360 vqp = fw ? &mvq->fwqp : &mvq->vqqp; 361 MLX5_SET(create_qp_in, in, uid, ndev->mvdev.res.uid); 362 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 363 if (vqp->fw) { 364 /* Firmware QP is allocated by the driver for the firmware's 365 * use so we can skip part of the params as they will be chosen by firmware 366 */ 367 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 368 MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ); 369 MLX5_SET(qpc, qpc, no_sq, 1); 370 return; 371 } 372 373 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 374 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 375 MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 376 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 377 MLX5_SET(qpc, qpc, uar_page, ndev->mvdev.res.uar->index); 378 MLX5_SET(qpc, qpc, log_page_size, vqp->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 379 MLX5_SET(qpc, qpc, no_sq, 1); 380 MLX5_SET(qpc, qpc, cqn_rcv, mvq->cq.mcq.cqn); 381 MLX5_SET(qpc, qpc, log_rq_size, ilog2(num_ent)); 382 MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); 383 pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas); 384 mlx5_fill_page_frag_array(&vqp->frag_buf, pas); 385 } 386 387 static int rq_buf_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp, u32 num_ent) 388 { 389 return mlx5_frag_buf_alloc_node(ndev->mvdev.mdev, 390 num_ent * sizeof(struct mlx5_wqe_data_seg), &vqp->frag_buf, 391 ndev->mvdev.mdev->priv.numa_node); 392 } 393 394 static void rq_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 395 { 396 mlx5_frag_buf_free(ndev->mvdev.mdev, &vqp->frag_buf); 397 } 398 399 static int qp_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 400 struct mlx5_vdpa_qp *vqp) 401 { 402 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 403 int inlen = MLX5_ST_SZ_BYTES(create_qp_in); 404 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; 405 void *qpc; 406 void *in; 407 int err; 408 409 if (!vqp->fw) { 410 vqp = &mvq->vqqp; 411 err = rq_buf_alloc(ndev, vqp, mvq->num_ent); 412 if (err) 413 return err; 414 415 err = mlx5_db_alloc(ndev->mvdev.mdev, &vqp->db); 416 if (err) 417 goto err_db; 418 inlen += vqp->frag_buf.npages * sizeof(__be64); 419 } 420 421 in = kzalloc(inlen, GFP_KERNEL); 422 if (!in) { 423 err = -ENOMEM; 424 goto err_kzalloc; 425 } 426 427 qp_prepare(ndev, vqp->fw, in, mvq, mvq->num_ent); 428 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 429 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 430 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 431 MLX5_SET(qpc, qpc, pd, ndev->mvdev.res.pdn); 432 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 433 if (!vqp->fw) 434 MLX5_SET64(qpc, qpc, dbr_addr, vqp->db.dma); 435 MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); 436 err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); 437 kfree(in); 438 if (err) 439 goto err_kzalloc; 440 441 vqp->mqp.uid = ndev->mvdev.res.uid; 442 vqp->mqp.qpn = MLX5_GET(create_qp_out, out, qpn); 443 444 if (!vqp->fw) 445 rx_post(vqp, mvq->num_ent); 446 447 return 0; 448 449 err_kzalloc: 450 if (!vqp->fw) 451 mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 452 err_db: 453 if (!vqp->fw) 454 rq_buf_free(ndev, vqp); 455 456 return err; 457 } 458 459 static void qp_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_qp *vqp) 460 { 461 u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; 462 463 MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); 464 MLX5_SET(destroy_qp_in, in, qpn, vqp->mqp.qpn); 465 MLX5_SET(destroy_qp_in, in, uid, ndev->mvdev.res.uid); 466 if (mlx5_cmd_exec_in(ndev->mvdev.mdev, destroy_qp, in)) 467 mlx5_vdpa_warn(&ndev->mvdev, "destroy qp 0x%x\n", vqp->mqp.qpn); 468 if (!vqp->fw) { 469 mlx5_db_free(ndev->mvdev.mdev, &vqp->db); 470 rq_buf_free(ndev, vqp); 471 } 472 } 473 474 static void *next_cqe_sw(struct mlx5_vdpa_cq *cq) 475 { 476 return get_sw_cqe(cq, cq->mcq.cons_index); 477 } 478 479 static int mlx5_vdpa_poll_one(struct mlx5_vdpa_cq *vcq) 480 { 481 struct mlx5_cqe64 *cqe64; 482 483 cqe64 = next_cqe_sw(vcq); 484 if (!cqe64) 485 return -EAGAIN; 486 487 vcq->mcq.cons_index++; 488 return 0; 489 } 490 491 static void mlx5_vdpa_handle_completions(struct mlx5_vdpa_virtqueue *mvq, int num) 492 { 493 struct mlx5_vdpa_net *ndev = mvq->ndev; 494 struct vdpa_callback *event_cb; 495 496 event_cb = &ndev->event_cbs[mvq->index]; 497 mlx5_cq_set_ci(&mvq->cq.mcq); 498 499 /* make sure CQ cosumer update is visible to the hardware before updating 500 * RX doorbell record. 501 */ 502 dma_wmb(); 503 rx_post(&mvq->vqqp, num); 504 if (event_cb->callback) 505 event_cb->callback(event_cb->private); 506 } 507 508 static void mlx5_vdpa_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe) 509 { 510 struct mlx5_vdpa_virtqueue *mvq = container_of(mcq, struct mlx5_vdpa_virtqueue, cq.mcq); 511 struct mlx5_vdpa_net *ndev = mvq->ndev; 512 void __iomem *uar_page = ndev->mvdev.res.uar->map; 513 int num = 0; 514 515 while (!mlx5_vdpa_poll_one(&mvq->cq)) { 516 num++; 517 if (num > mvq->num_ent / 2) { 518 /* If completions keep coming while we poll, we want to 519 * let the hardware know that we consumed them by 520 * updating the doorbell record. We also let vdpa core 521 * know about this so it passes it on the virtio driver 522 * on the guest. 523 */ 524 mlx5_vdpa_handle_completions(mvq, num); 525 num = 0; 526 } 527 } 528 529 if (num) 530 mlx5_vdpa_handle_completions(mvq, num); 531 532 mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 533 } 534 535 static int cq_create(struct mlx5_vdpa_net *ndev, u16 idx, u32 num_ent) 536 { 537 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 538 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 539 void __iomem *uar_page = ndev->mvdev.res.uar->map; 540 u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 541 struct mlx5_vdpa_cq *vcq = &mvq->cq; 542 __be64 *pas; 543 int inlen; 544 void *cqc; 545 void *in; 546 int err; 547 int eqn; 548 549 err = mlx5_db_alloc(mdev, &vcq->db); 550 if (err) 551 return err; 552 553 vcq->mcq.set_ci_db = vcq->db.db; 554 vcq->mcq.arm_db = vcq->db.db + 1; 555 vcq->mcq.cqe_sz = 64; 556 557 err = cq_frag_buf_alloc(ndev, &vcq->buf, num_ent); 558 if (err) 559 goto err_db; 560 561 cq_frag_buf_init(vcq, &vcq->buf); 562 563 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 564 MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * vcq->buf.frag_buf.npages; 565 in = kzalloc(inlen, GFP_KERNEL); 566 if (!in) { 567 err = -ENOMEM; 568 goto err_vzalloc; 569 } 570 571 MLX5_SET(create_cq_in, in, uid, ndev->mvdev.res.uid); 572 pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); 573 mlx5_fill_page_frag_array(&vcq->buf.frag_buf, pas); 574 575 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 576 MLX5_SET(cqc, cqc, log_page_size, vcq->buf.frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 577 578 /* Use vector 0 by default. Consider adding code to choose least used 579 * vector. 580 */ 581 err = mlx5_vector2eqn(mdev, 0, &eqn); 582 if (err) 583 goto err_vec; 584 585 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 586 MLX5_SET(cqc, cqc, log_cq_size, ilog2(num_ent)); 587 MLX5_SET(cqc, cqc, uar_page, ndev->mvdev.res.uar->index); 588 MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 589 MLX5_SET64(cqc, cqc, dbr_addr, vcq->db.dma); 590 591 err = mlx5_core_create_cq(mdev, &vcq->mcq, in, inlen, out, sizeof(out)); 592 if (err) 593 goto err_vec; 594 595 vcq->mcq.comp = mlx5_vdpa_cq_comp; 596 vcq->cqe = num_ent; 597 vcq->mcq.set_ci_db = vcq->db.db; 598 vcq->mcq.arm_db = vcq->db.db + 1; 599 mlx5_cq_arm(&mvq->cq.mcq, MLX5_CQ_DB_REQ_NOT, uar_page, mvq->cq.mcq.cons_index); 600 kfree(in); 601 return 0; 602 603 err_vec: 604 kfree(in); 605 err_vzalloc: 606 cq_frag_buf_free(ndev, &vcq->buf); 607 err_db: 608 mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 609 return err; 610 } 611 612 static void cq_destroy(struct mlx5_vdpa_net *ndev, u16 idx) 613 { 614 struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[idx]; 615 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 616 struct mlx5_vdpa_cq *vcq = &mvq->cq; 617 618 if (mlx5_core_destroy_cq(mdev, &vcq->mcq)) { 619 mlx5_vdpa_warn(&ndev->mvdev, "destroy CQ 0x%x\n", vcq->mcq.cqn); 620 return; 621 } 622 cq_frag_buf_free(ndev, &vcq->buf); 623 mlx5_db_free(ndev->mvdev.mdev, &vcq->db); 624 } 625 626 static void set_umem_size(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num, 627 struct mlx5_vdpa_umem **umemp) 628 { 629 struct mlx5_core_dev *mdev = ndev->mvdev.mdev; 630 int p_a; 631 int p_b; 632 633 switch (num) { 634 case 1: 635 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_a); 636 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_1_buffer_param_b); 637 *umemp = &mvq->umem1; 638 break; 639 case 2: 640 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_a); 641 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_2_buffer_param_b); 642 *umemp = &mvq->umem2; 643 break; 644 case 3: 645 p_a = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_a); 646 p_b = MLX5_CAP_DEV_VDPA_EMULATION(mdev, umem_3_buffer_param_b); 647 *umemp = &mvq->umem3; 648 break; 649 } 650 (*umemp)->size = p_a * mvq->num_ent + p_b; 651 } 652 653 static void umem_frag_buf_free(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_umem *umem) 654 { 655 mlx5_frag_buf_free(ndev->mvdev.mdev, &umem->frag_buf); 656 } 657 658 static int create_umem(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 659 { 660 int inlen; 661 u32 out[MLX5_ST_SZ_DW(create_umem_out)] = {}; 662 void *um; 663 void *in; 664 int err; 665 __be64 *pas; 666 struct mlx5_vdpa_umem *umem; 667 668 set_umem_size(ndev, mvq, num, &umem); 669 err = umem_frag_buf_alloc(ndev, umem, umem->size); 670 if (err) 671 return err; 672 673 inlen = MLX5_ST_SZ_BYTES(create_umem_in) + MLX5_ST_SZ_BYTES(mtt) * umem->frag_buf.npages; 674 675 in = kzalloc(inlen, GFP_KERNEL); 676 if (!in) { 677 err = -ENOMEM; 678 goto err_in; 679 } 680 681 MLX5_SET(create_umem_in, in, opcode, MLX5_CMD_OP_CREATE_UMEM); 682 MLX5_SET(create_umem_in, in, uid, ndev->mvdev.res.uid); 683 um = MLX5_ADDR_OF(create_umem_in, in, umem); 684 MLX5_SET(umem, um, log_page_size, umem->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 685 MLX5_SET64(umem, um, num_of_mtt, umem->frag_buf.npages); 686 687 pas = (__be64 *)MLX5_ADDR_OF(umem, um, mtt[0]); 688 mlx5_fill_page_frag_array_perm(&umem->frag_buf, pas, MLX5_MTT_PERM_RW); 689 690 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 691 if (err) { 692 mlx5_vdpa_warn(&ndev->mvdev, "create umem(%d)\n", err); 693 goto err_cmd; 694 } 695 696 kfree(in); 697 umem->id = MLX5_GET(create_umem_out, out, umem_id); 698 699 return 0; 700 701 err_cmd: 702 kfree(in); 703 err_in: 704 umem_frag_buf_free(ndev, umem); 705 return err; 706 } 707 708 static void umem_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int num) 709 { 710 u32 in[MLX5_ST_SZ_DW(destroy_umem_in)] = {}; 711 u32 out[MLX5_ST_SZ_DW(destroy_umem_out)] = {}; 712 struct mlx5_vdpa_umem *umem; 713 714 switch (num) { 715 case 1: 716 umem = &mvq->umem1; 717 break; 718 case 2: 719 umem = &mvq->umem2; 720 break; 721 case 3: 722 umem = &mvq->umem3; 723 break; 724 } 725 726 MLX5_SET(destroy_umem_in, in, opcode, MLX5_CMD_OP_DESTROY_UMEM); 727 MLX5_SET(destroy_umem_in, in, umem_id, umem->id); 728 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 729 return; 730 731 umem_frag_buf_free(ndev, umem); 732 } 733 734 static int umems_create(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 735 { 736 int num; 737 int err; 738 739 for (num = 1; num <= 3; num++) { 740 err = create_umem(ndev, mvq, num); 741 if (err) 742 goto err_umem; 743 } 744 return 0; 745 746 err_umem: 747 for (num--; num > 0; num--) 748 umem_destroy(ndev, mvq, num); 749 750 return err; 751 } 752 753 static void umems_destroy(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 754 { 755 int num; 756 757 for (num = 3; num > 0; num--) 758 umem_destroy(ndev, mvq, num); 759 } 760 761 static int get_queue_type(struct mlx5_vdpa_net *ndev) 762 { 763 u32 type_mask; 764 765 type_mask = MLX5_CAP_DEV_VDPA_EMULATION(ndev->mvdev.mdev, virtio_queue_type); 766 767 /* prefer split queue */ 768 if (type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT) 769 return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_SPLIT; 770 771 WARN_ON(!(type_mask & MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_PACKED)); 772 773 return MLX5_VIRTIO_EMULATION_VIRTIO_QUEUE_TYPE_PACKED; 774 } 775 776 static bool vq_is_tx(u16 idx) 777 { 778 return idx % 2; 779 } 780 781 static u16 get_features_12_3(u64 features) 782 { 783 return (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO4)) << 9) | 784 (!!(features & BIT_ULL(VIRTIO_NET_F_HOST_TSO6)) << 8) | 785 (!!(features & BIT_ULL(VIRTIO_NET_F_CSUM)) << 7) | 786 (!!(features & BIT_ULL(VIRTIO_NET_F_GUEST_CSUM)) << 6); 787 } 788 789 static bool counters_supported(const struct mlx5_vdpa_dev *mvdev) 790 { 791 return MLX5_CAP_GEN_64(mvdev->mdev, general_obj_types) & 792 BIT_ULL(MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 793 } 794 795 static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 796 { 797 int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in); 798 u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {}; 799 void *obj_context; 800 void *cmd_hdr; 801 void *vq_ctx; 802 void *in; 803 int err; 804 805 err = umems_create(ndev, mvq); 806 if (err) 807 return err; 808 809 in = kzalloc(inlen, GFP_KERNEL); 810 if (!in) { 811 err = -ENOMEM; 812 goto err_alloc; 813 } 814 815 cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr); 816 817 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 818 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 819 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 820 821 obj_context = MLX5_ADDR_OF(create_virtio_net_q_in, in, obj_context); 822 MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx); 823 MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx); 824 MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3, 825 get_features_12_3(ndev->mvdev.actual_features)); 826 vq_ctx = MLX5_ADDR_OF(virtio_net_q_object, obj_context, virtio_q_context); 827 MLX5_SET(virtio_q, vq_ctx, virtio_q_type, get_queue_type(ndev)); 828 829 if (vq_is_tx(mvq->index)) 830 MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn); 831 832 MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE); 833 MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index); 834 MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn); 835 MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent); 836 MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, 837 !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1))); 838 MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr); 839 MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr); 840 MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr); 841 MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, ndev->mvdev.mr.mkey); 842 MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id); 843 MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size); 844 MLX5_SET(virtio_q, vq_ctx, umem_2_id, mvq->umem2.id); 845 MLX5_SET(virtio_q, vq_ctx, umem_2_size, mvq->umem2.size); 846 MLX5_SET(virtio_q, vq_ctx, umem_3_id, mvq->umem3.id); 847 MLX5_SET(virtio_q, vq_ctx, umem_3_size, mvq->umem3.size); 848 MLX5_SET(virtio_q, vq_ctx, pd, ndev->mvdev.res.pdn); 849 if (counters_supported(&ndev->mvdev)) 850 MLX5_SET(virtio_q, vq_ctx, counter_set_id, mvq->counter_set_id); 851 852 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 853 if (err) 854 goto err_cmd; 855 856 mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT; 857 kfree(in); 858 mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 859 860 return 0; 861 862 err_cmd: 863 kfree(in); 864 err_alloc: 865 umems_destroy(ndev, mvq); 866 return err; 867 } 868 869 static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 870 { 871 u32 in[MLX5_ST_SZ_DW(destroy_virtio_net_q_in)] = {}; 872 u32 out[MLX5_ST_SZ_DW(destroy_virtio_net_q_out)] = {}; 873 874 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.opcode, 875 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 876 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_id, mvq->virtq_id); 877 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.uid, ndev->mvdev.res.uid); 878 MLX5_SET(destroy_virtio_net_q_in, in, general_obj_out_cmd_hdr.obj_type, 879 MLX5_OBJ_TYPE_VIRTIO_NET_Q); 880 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) { 881 mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id); 882 return; 883 } 884 mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE; 885 umems_destroy(ndev, mvq); 886 } 887 888 static u32 get_rqpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 889 { 890 return fw ? mvq->vqqp.mqp.qpn : mvq->fwqp.mqp.qpn; 891 } 892 893 static u32 get_qpn(struct mlx5_vdpa_virtqueue *mvq, bool fw) 894 { 895 return fw ? mvq->fwqp.mqp.qpn : mvq->vqqp.mqp.qpn; 896 } 897 898 static void alloc_inout(struct mlx5_vdpa_net *ndev, int cmd, void **in, int *inlen, void **out, 899 int *outlen, u32 qpn, u32 rqpn) 900 { 901 void *qpc; 902 void *pp; 903 904 switch (cmd) { 905 case MLX5_CMD_OP_2RST_QP: 906 *inlen = MLX5_ST_SZ_BYTES(qp_2rst_in); 907 *outlen = MLX5_ST_SZ_BYTES(qp_2rst_out); 908 *in = kzalloc(*inlen, GFP_KERNEL); 909 *out = kzalloc(*outlen, GFP_KERNEL); 910 if (!*in || !*out) 911 goto outerr; 912 913 MLX5_SET(qp_2rst_in, *in, opcode, cmd); 914 MLX5_SET(qp_2rst_in, *in, uid, ndev->mvdev.res.uid); 915 MLX5_SET(qp_2rst_in, *in, qpn, qpn); 916 break; 917 case MLX5_CMD_OP_RST2INIT_QP: 918 *inlen = MLX5_ST_SZ_BYTES(rst2init_qp_in); 919 *outlen = MLX5_ST_SZ_BYTES(rst2init_qp_out); 920 *in = kzalloc(*inlen, GFP_KERNEL); 921 *out = kzalloc(MLX5_ST_SZ_BYTES(rst2init_qp_out), GFP_KERNEL); 922 if (!*in || !*out) 923 goto outerr; 924 925 MLX5_SET(rst2init_qp_in, *in, opcode, cmd); 926 MLX5_SET(rst2init_qp_in, *in, uid, ndev->mvdev.res.uid); 927 MLX5_SET(rst2init_qp_in, *in, qpn, qpn); 928 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 929 MLX5_SET(qpc, qpc, remote_qpn, rqpn); 930 MLX5_SET(qpc, qpc, rwe, 1); 931 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 932 MLX5_SET(ads, pp, vhca_port_num, 1); 933 break; 934 case MLX5_CMD_OP_INIT2RTR_QP: 935 *inlen = MLX5_ST_SZ_BYTES(init2rtr_qp_in); 936 *outlen = MLX5_ST_SZ_BYTES(init2rtr_qp_out); 937 *in = kzalloc(*inlen, GFP_KERNEL); 938 *out = kzalloc(MLX5_ST_SZ_BYTES(init2rtr_qp_out), GFP_KERNEL); 939 if (!*in || !*out) 940 goto outerr; 941 942 MLX5_SET(init2rtr_qp_in, *in, opcode, cmd); 943 MLX5_SET(init2rtr_qp_in, *in, uid, ndev->mvdev.res.uid); 944 MLX5_SET(init2rtr_qp_in, *in, qpn, qpn); 945 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 946 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_256_BYTES); 947 MLX5_SET(qpc, qpc, log_msg_max, 30); 948 MLX5_SET(qpc, qpc, remote_qpn, rqpn); 949 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 950 MLX5_SET(ads, pp, fl, 1); 951 break; 952 case MLX5_CMD_OP_RTR2RTS_QP: 953 *inlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_in); 954 *outlen = MLX5_ST_SZ_BYTES(rtr2rts_qp_out); 955 *in = kzalloc(*inlen, GFP_KERNEL); 956 *out = kzalloc(MLX5_ST_SZ_BYTES(rtr2rts_qp_out), GFP_KERNEL); 957 if (!*in || !*out) 958 goto outerr; 959 960 MLX5_SET(rtr2rts_qp_in, *in, opcode, cmd); 961 MLX5_SET(rtr2rts_qp_in, *in, uid, ndev->mvdev.res.uid); 962 MLX5_SET(rtr2rts_qp_in, *in, qpn, qpn); 963 qpc = MLX5_ADDR_OF(rst2init_qp_in, *in, qpc); 964 pp = MLX5_ADDR_OF(qpc, qpc, primary_address_path); 965 MLX5_SET(ads, pp, ack_timeout, 14); 966 MLX5_SET(qpc, qpc, retry_count, 7); 967 MLX5_SET(qpc, qpc, rnr_retry, 7); 968 break; 969 default: 970 goto outerr_nullify; 971 } 972 973 return; 974 975 outerr: 976 kfree(*in); 977 kfree(*out); 978 outerr_nullify: 979 *in = NULL; 980 *out = NULL; 981 } 982 983 static void free_inout(void *in, void *out) 984 { 985 kfree(in); 986 kfree(out); 987 } 988 989 /* Two QPs are used by each virtqueue. One is used by the driver and one by 990 * firmware. The fw argument indicates whether the subjected QP is the one used 991 * by firmware. 992 */ 993 static int modify_qp(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, bool fw, int cmd) 994 { 995 int outlen; 996 int inlen; 997 void *out; 998 void *in; 999 int err; 1000 1001 alloc_inout(ndev, cmd, &in, &inlen, &out, &outlen, get_qpn(mvq, fw), get_rqpn(mvq, fw)); 1002 if (!in || !out) 1003 return -ENOMEM; 1004 1005 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, outlen); 1006 free_inout(in, out); 1007 return err; 1008 } 1009 1010 static int connect_qps(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1011 { 1012 int err; 1013 1014 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_2RST_QP); 1015 if (err) 1016 return err; 1017 1018 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_2RST_QP); 1019 if (err) 1020 return err; 1021 1022 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_RST2INIT_QP); 1023 if (err) 1024 return err; 1025 1026 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_RST2INIT_QP); 1027 if (err) 1028 return err; 1029 1030 err = modify_qp(ndev, mvq, true, MLX5_CMD_OP_INIT2RTR_QP); 1031 if (err) 1032 return err; 1033 1034 err = modify_qp(ndev, mvq, false, MLX5_CMD_OP_INIT2RTR_QP); 1035 if (err) 1036 return err; 1037 1038 return modify_qp(ndev, mvq, true, MLX5_CMD_OP_RTR2RTS_QP); 1039 } 1040 1041 struct mlx5_virtq_attr { 1042 u8 state; 1043 u16 available_index; 1044 u16 used_index; 1045 }; 1046 1047 static int query_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 1048 struct mlx5_virtq_attr *attr) 1049 { 1050 int outlen = MLX5_ST_SZ_BYTES(query_virtio_net_q_out); 1051 u32 in[MLX5_ST_SZ_DW(query_virtio_net_q_in)] = {}; 1052 void *out; 1053 void *obj_context; 1054 void *cmd_hdr; 1055 int err; 1056 1057 out = kzalloc(outlen, GFP_KERNEL); 1058 if (!out) 1059 return -ENOMEM; 1060 1061 cmd_hdr = MLX5_ADDR_OF(query_virtio_net_q_in, in, general_obj_in_cmd_hdr); 1062 1063 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 1064 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 1065 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 1066 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 1067 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, outlen); 1068 if (err) 1069 goto err_cmd; 1070 1071 obj_context = MLX5_ADDR_OF(query_virtio_net_q_out, out, obj_context); 1072 memset(attr, 0, sizeof(*attr)); 1073 attr->state = MLX5_GET(virtio_net_q_object, obj_context, state); 1074 attr->available_index = MLX5_GET(virtio_net_q_object, obj_context, hw_available_index); 1075 attr->used_index = MLX5_GET(virtio_net_q_object, obj_context, hw_used_index); 1076 kfree(out); 1077 return 0; 1078 1079 err_cmd: 1080 kfree(out); 1081 return err; 1082 } 1083 1084 static bool is_valid_state_change(int oldstate, int newstate) 1085 { 1086 switch (oldstate) { 1087 case MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT: 1088 return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY; 1089 case MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY: 1090 return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND; 1091 case MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND: 1092 case MLX5_VIRTIO_NET_Q_OBJECT_STATE_ERR: 1093 default: 1094 return false; 1095 } 1096 } 1097 1098 static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state) 1099 { 1100 int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in); 1101 u32 out[MLX5_ST_SZ_DW(modify_virtio_net_q_out)] = {}; 1102 void *obj_context; 1103 void *cmd_hdr; 1104 void *in; 1105 int err; 1106 1107 if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_NONE) 1108 return 0; 1109 1110 if (!is_valid_state_change(mvq->fw_state, state)) 1111 return -EINVAL; 1112 1113 in = kzalloc(inlen, GFP_KERNEL); 1114 if (!in) 1115 return -ENOMEM; 1116 1117 cmd_hdr = MLX5_ADDR_OF(modify_virtio_net_q_in, in, general_obj_in_cmd_hdr); 1118 1119 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); 1120 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q); 1121 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id); 1122 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 1123 1124 obj_context = MLX5_ADDR_OF(modify_virtio_net_q_in, in, obj_context); 1125 MLX5_SET64(virtio_net_q_object, obj_context, modify_field_select, 1126 MLX5_VIRTQ_MODIFY_MASK_STATE); 1127 MLX5_SET(virtio_net_q_object, obj_context, state, state); 1128 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out)); 1129 kfree(in); 1130 if (!err) 1131 mvq->fw_state = state; 1132 1133 return err; 1134 } 1135 1136 static int counter_set_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1137 { 1138 u32 in[MLX5_ST_SZ_DW(create_virtio_q_counters_in)] = {}; 1139 u32 out[MLX5_ST_SZ_DW(create_virtio_q_counters_out)] = {}; 1140 void *cmd_hdr; 1141 int err; 1142 1143 if (!counters_supported(&ndev->mvdev)) 1144 return 0; 1145 1146 cmd_hdr = MLX5_ADDR_OF(create_virtio_q_counters_in, in, hdr); 1147 1148 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 1149 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 1150 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 1151 1152 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)); 1153 if (err) 1154 return err; 1155 1156 mvq->counter_set_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 1157 1158 return 0; 1159 } 1160 1161 static void counter_set_dealloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1162 { 1163 u32 in[MLX5_ST_SZ_DW(destroy_virtio_q_counters_in)] = {}; 1164 u32 out[MLX5_ST_SZ_DW(destroy_virtio_q_counters_out)] = {}; 1165 1166 if (!counters_supported(&ndev->mvdev)) 1167 return; 1168 1169 MLX5_SET(destroy_virtio_q_counters_in, in, hdr.opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 1170 MLX5_SET(destroy_virtio_q_counters_in, in, hdr.obj_id, mvq->counter_set_id); 1171 MLX5_SET(destroy_virtio_q_counters_in, in, hdr.uid, ndev->mvdev.res.uid); 1172 MLX5_SET(destroy_virtio_q_counters_in, in, hdr.obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 1173 if (mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out))) 1174 mlx5_vdpa_warn(&ndev->mvdev, "dealloc counter set 0x%x\n", mvq->counter_set_id); 1175 } 1176 1177 static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1178 { 1179 u16 idx = mvq->index; 1180 int err; 1181 1182 if (!mvq->num_ent) 1183 return 0; 1184 1185 if (mvq->initialized) 1186 return 0; 1187 1188 err = cq_create(ndev, idx, mvq->num_ent); 1189 if (err) 1190 return err; 1191 1192 err = qp_create(ndev, mvq, &mvq->fwqp); 1193 if (err) 1194 goto err_fwqp; 1195 1196 err = qp_create(ndev, mvq, &mvq->vqqp); 1197 if (err) 1198 goto err_vqqp; 1199 1200 err = connect_qps(ndev, mvq); 1201 if (err) 1202 goto err_connect; 1203 1204 err = counter_set_alloc(ndev, mvq); 1205 if (err) 1206 goto err_counter; 1207 1208 err = create_virtqueue(ndev, mvq); 1209 if (err) 1210 goto err_connect; 1211 1212 if (mvq->ready) { 1213 err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); 1214 if (err) { 1215 mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n", 1216 idx, err); 1217 goto err_connect; 1218 } 1219 } 1220 1221 mvq->initialized = true; 1222 return 0; 1223 1224 err_connect: 1225 counter_set_dealloc(ndev, mvq); 1226 err_counter: 1227 qp_destroy(ndev, &mvq->vqqp); 1228 err_vqqp: 1229 qp_destroy(ndev, &mvq->fwqp); 1230 err_fwqp: 1231 cq_destroy(ndev, idx); 1232 return err; 1233 } 1234 1235 static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1236 { 1237 struct mlx5_virtq_attr attr; 1238 1239 if (!mvq->initialized) 1240 return; 1241 1242 if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 1243 return; 1244 1245 if (modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND)) 1246 mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n"); 1247 1248 if (query_virtqueue(ndev, mvq, &attr)) { 1249 mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n"); 1250 return; 1251 } 1252 mvq->avail_idx = attr.available_index; 1253 mvq->used_idx = attr.used_index; 1254 } 1255 1256 static void suspend_vqs(struct mlx5_vdpa_net *ndev) 1257 { 1258 int i; 1259 1260 for (i = 0; i < ndev->mvdev.max_vqs; i++) 1261 suspend_vq(ndev, &ndev->vqs[i]); 1262 } 1263 1264 static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1265 { 1266 if (!mvq->initialized) 1267 return; 1268 1269 suspend_vq(ndev, mvq); 1270 destroy_virtqueue(ndev, mvq); 1271 counter_set_dealloc(ndev, mvq); 1272 qp_destroy(ndev, &mvq->vqqp); 1273 qp_destroy(ndev, &mvq->fwqp); 1274 cq_destroy(ndev, mvq->index); 1275 mvq->initialized = false; 1276 } 1277 1278 static int create_rqt(struct mlx5_vdpa_net *ndev) 1279 { 1280 int rqt_table_size = roundup_pow_of_two(ndev->rqt_size); 1281 int act_sz = roundup_pow_of_two(ndev->cur_num_vqs / 2); 1282 __be32 *list; 1283 void *rqtc; 1284 int inlen; 1285 void *in; 1286 int i, j; 1287 int err; 1288 1289 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + rqt_table_size * MLX5_ST_SZ_BYTES(rq_num); 1290 in = kzalloc(inlen, GFP_KERNEL); 1291 if (!in) 1292 return -ENOMEM; 1293 1294 MLX5_SET(create_rqt_in, in, uid, ndev->mvdev.res.uid); 1295 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 1296 1297 MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 1298 MLX5_SET(rqtc, rqtc, rqt_max_size, rqt_table_size); 1299 list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 1300 for (i = 0, j = 0; i < act_sz; i++, j += 2) 1301 list[i] = cpu_to_be32(ndev->vqs[j % ndev->cur_num_vqs].virtq_id); 1302 1303 MLX5_SET(rqtc, rqtc, rqt_actual_size, act_sz); 1304 err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn); 1305 kfree(in); 1306 if (err) 1307 return err; 1308 1309 return 0; 1310 } 1311 1312 #define MLX5_MODIFY_RQT_NUM_RQS ((u64)1) 1313 1314 static int modify_rqt(struct mlx5_vdpa_net *ndev, int num) 1315 { 1316 int act_sz = roundup_pow_of_two(num / 2); 1317 __be32 *list; 1318 void *rqtc; 1319 int inlen; 1320 void *in; 1321 int i, j; 1322 int err; 1323 1324 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + act_sz * MLX5_ST_SZ_BYTES(rq_num); 1325 in = kzalloc(inlen, GFP_KERNEL); 1326 if (!in) 1327 return -ENOMEM; 1328 1329 MLX5_SET(modify_rqt_in, in, uid, ndev->mvdev.res.uid); 1330 MLX5_SET64(modify_rqt_in, in, bitmask, MLX5_MODIFY_RQT_NUM_RQS); 1331 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx); 1332 MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); 1333 1334 list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); 1335 for (i = 0, j = 0; i < act_sz; i++, j = j + 2) 1336 list[i] = cpu_to_be32(ndev->vqs[j % num].virtq_id); 1337 1338 MLX5_SET(rqtc, rqtc, rqt_actual_size, act_sz); 1339 err = mlx5_vdpa_modify_rqt(&ndev->mvdev, in, inlen, ndev->res.rqtn); 1340 kfree(in); 1341 if (err) 1342 return err; 1343 1344 return 0; 1345 } 1346 1347 static void destroy_rqt(struct mlx5_vdpa_net *ndev) 1348 { 1349 mlx5_vdpa_destroy_rqt(&ndev->mvdev, ndev->res.rqtn); 1350 } 1351 1352 static int create_tir(struct mlx5_vdpa_net *ndev) 1353 { 1354 #define HASH_IP_L4PORTS \ 1355 (MLX5_HASH_FIELD_SEL_SRC_IP | MLX5_HASH_FIELD_SEL_DST_IP | MLX5_HASH_FIELD_SEL_L4_SPORT | \ 1356 MLX5_HASH_FIELD_SEL_L4_DPORT) 1357 static const u8 rx_hash_toeplitz_key[] = { 0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7, 1358 0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94, 1359 0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1, 1360 0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59, 1361 0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a }; 1362 void *rss_key; 1363 void *outer; 1364 void *tirc; 1365 void *in; 1366 int err; 1367 1368 in = kzalloc(MLX5_ST_SZ_BYTES(create_tir_in), GFP_KERNEL); 1369 if (!in) 1370 return -ENOMEM; 1371 1372 MLX5_SET(create_tir_in, in, uid, ndev->mvdev.res.uid); 1373 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 1374 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 1375 1376 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); 1377 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ); 1378 rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); 1379 memcpy(rss_key, rx_hash_toeplitz_key, sizeof(rx_hash_toeplitz_key)); 1380 1381 outer = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 1382 MLX5_SET(rx_hash_field_select, outer, l3_prot_type, MLX5_L3_PROT_TYPE_IPV4); 1383 MLX5_SET(rx_hash_field_select, outer, l4_prot_type, MLX5_L4_PROT_TYPE_TCP); 1384 MLX5_SET(rx_hash_field_select, outer, selected_fields, HASH_IP_L4PORTS); 1385 1386 MLX5_SET(tirc, tirc, indirect_table, ndev->res.rqtn); 1387 MLX5_SET(tirc, tirc, transport_domain, ndev->res.tdn); 1388 1389 err = mlx5_vdpa_create_tir(&ndev->mvdev, in, &ndev->res.tirn); 1390 kfree(in); 1391 if (err) 1392 return err; 1393 1394 mlx5_vdpa_add_tirn(ndev); 1395 return err; 1396 } 1397 1398 static void destroy_tir(struct mlx5_vdpa_net *ndev) 1399 { 1400 mlx5_vdpa_remove_tirn(ndev); 1401 mlx5_vdpa_destroy_tir(&ndev->mvdev, ndev->res.tirn); 1402 } 1403 1404 #define MAX_STEERING_ENT 0x8000 1405 #define MAX_STEERING_GROUPS 2 1406 1407 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1408 #define NUM_DESTS 2 1409 #else 1410 #define NUM_DESTS 1 1411 #endif 1412 1413 static int add_steering_counters(struct mlx5_vdpa_net *ndev, 1414 struct macvlan_node *node, 1415 struct mlx5_flow_act *flow_act, 1416 struct mlx5_flow_destination *dests) 1417 { 1418 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1419 int err; 1420 1421 node->ucast_counter.counter = mlx5_fc_create(ndev->mvdev.mdev, false); 1422 if (IS_ERR(node->ucast_counter.counter)) 1423 return PTR_ERR(node->ucast_counter.counter); 1424 1425 node->mcast_counter.counter = mlx5_fc_create(ndev->mvdev.mdev, false); 1426 if (IS_ERR(node->mcast_counter.counter)) { 1427 err = PTR_ERR(node->mcast_counter.counter); 1428 goto err_mcast_counter; 1429 } 1430 1431 dests[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1432 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 1433 return 0; 1434 1435 err_mcast_counter: 1436 mlx5_fc_destroy(ndev->mvdev.mdev, node->ucast_counter.counter); 1437 return err; 1438 #else 1439 return 0; 1440 #endif 1441 } 1442 1443 static void remove_steering_counters(struct mlx5_vdpa_net *ndev, 1444 struct macvlan_node *node) 1445 { 1446 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1447 mlx5_fc_destroy(ndev->mvdev.mdev, node->mcast_counter.counter); 1448 mlx5_fc_destroy(ndev->mvdev.mdev, node->ucast_counter.counter); 1449 #endif 1450 } 1451 1452 static int mlx5_vdpa_add_mac_vlan_rules(struct mlx5_vdpa_net *ndev, u8 *mac, 1453 struct macvlan_node *node) 1454 { 1455 struct mlx5_flow_destination dests[NUM_DESTS] = {}; 1456 struct mlx5_flow_act flow_act = {}; 1457 struct mlx5_flow_spec *spec; 1458 void *headers_c; 1459 void *headers_v; 1460 u8 *dmac_c; 1461 u8 *dmac_v; 1462 int err; 1463 u16 vid; 1464 1465 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1466 if (!spec) 1467 return -ENOMEM; 1468 1469 vid = key2vid(node->macvlan); 1470 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1471 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers); 1472 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers); 1473 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, outer_headers.dmac_47_16); 1474 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, outer_headers.dmac_47_16); 1475 eth_broadcast_addr(dmac_c); 1476 ether_addr_copy(dmac_v, mac); 1477 if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VLAN)) { 1478 MLX5_SET(fte_match_set_lyr_2_4, headers_c, cvlan_tag, 1); 1479 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, first_vid); 1480 } 1481 if (node->tagged) { 1482 MLX5_SET(fte_match_set_lyr_2_4, headers_v, cvlan_tag, 1); 1483 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, vid); 1484 } 1485 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1486 dests[0].type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1487 dests[0].tir_num = ndev->res.tirn; 1488 err = add_steering_counters(ndev, node, &flow_act, dests); 1489 if (err) 1490 goto out_free; 1491 1492 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1493 dests[1].counter_id = mlx5_fc_id(node->ucast_counter.counter); 1494 #endif 1495 node->ucast_rule = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, dests, NUM_DESTS); 1496 if (IS_ERR(node->ucast_rule)) { 1497 err = PTR_ERR(node->ucast_rule); 1498 goto err_ucast; 1499 } 1500 1501 #if defined(CONFIG_MLX5_VDPA_STEERING_DEBUG) 1502 dests[1].counter_id = mlx5_fc_id(node->mcast_counter.counter); 1503 #endif 1504 1505 memset(dmac_c, 0, ETH_ALEN); 1506 memset(dmac_v, 0, ETH_ALEN); 1507 dmac_c[0] = 1; 1508 dmac_v[0] = 1; 1509 node->mcast_rule = mlx5_add_flow_rules(ndev->rxft, spec, &flow_act, dests, NUM_DESTS); 1510 if (IS_ERR(node->mcast_rule)) { 1511 err = PTR_ERR(node->mcast_rule); 1512 goto err_mcast; 1513 } 1514 kvfree(spec); 1515 mlx5_vdpa_add_rx_counters(ndev, node); 1516 return 0; 1517 1518 err_mcast: 1519 mlx5_del_flow_rules(node->ucast_rule); 1520 err_ucast: 1521 remove_steering_counters(ndev, node); 1522 out_free: 1523 kvfree(spec); 1524 return err; 1525 } 1526 1527 static void mlx5_vdpa_del_mac_vlan_rules(struct mlx5_vdpa_net *ndev, 1528 struct macvlan_node *node) 1529 { 1530 mlx5_vdpa_remove_rx_counters(ndev, node); 1531 mlx5_del_flow_rules(node->ucast_rule); 1532 mlx5_del_flow_rules(node->mcast_rule); 1533 } 1534 1535 static u64 search_val(u8 *mac, u16 vlan, bool tagged) 1536 { 1537 u64 val; 1538 1539 if (!tagged) 1540 vlan = MLX5V_UNTAGGED; 1541 1542 val = (u64)vlan << 48 | 1543 (u64)mac[0] << 40 | 1544 (u64)mac[1] << 32 | 1545 (u64)mac[2] << 24 | 1546 (u64)mac[3] << 16 | 1547 (u64)mac[4] << 8 | 1548 (u64)mac[5]; 1549 1550 return val; 1551 } 1552 1553 static struct macvlan_node *mac_vlan_lookup(struct mlx5_vdpa_net *ndev, u64 value) 1554 { 1555 struct macvlan_node *pos; 1556 u32 idx; 1557 1558 idx = hash_64(value, 8); // tbd 8 1559 hlist_for_each_entry(pos, &ndev->macvlan_hash[idx], hlist) { 1560 if (pos->macvlan == value) 1561 return pos; 1562 } 1563 return NULL; 1564 } 1565 1566 static int mac_vlan_add(struct mlx5_vdpa_net *ndev, u8 *mac, u16 vid, bool tagged) 1567 { 1568 struct macvlan_node *ptr; 1569 u64 val; 1570 u32 idx; 1571 int err; 1572 1573 val = search_val(mac, vid, tagged); 1574 if (mac_vlan_lookup(ndev, val)) 1575 return -EEXIST; 1576 1577 ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); 1578 if (!ptr) 1579 return -ENOMEM; 1580 1581 ptr->tagged = tagged; 1582 ptr->macvlan = val; 1583 ptr->ndev = ndev; 1584 err = mlx5_vdpa_add_mac_vlan_rules(ndev, ndev->config.mac, ptr); 1585 if (err) 1586 goto err_add; 1587 1588 idx = hash_64(val, 8); 1589 hlist_add_head(&ptr->hlist, &ndev->macvlan_hash[idx]); 1590 return 0; 1591 1592 err_add: 1593 kfree(ptr); 1594 return err; 1595 } 1596 1597 static void mac_vlan_del(struct mlx5_vdpa_net *ndev, u8 *mac, u16 vlan, bool tagged) 1598 { 1599 struct macvlan_node *ptr; 1600 1601 ptr = mac_vlan_lookup(ndev, search_val(mac, vlan, tagged)); 1602 if (!ptr) 1603 return; 1604 1605 hlist_del(&ptr->hlist); 1606 mlx5_vdpa_del_mac_vlan_rules(ndev, ptr); 1607 remove_steering_counters(ndev, ptr); 1608 kfree(ptr); 1609 } 1610 1611 static void clear_mac_vlan_table(struct mlx5_vdpa_net *ndev) 1612 { 1613 struct macvlan_node *pos; 1614 struct hlist_node *n; 1615 int i; 1616 1617 for (i = 0; i < MLX5V_MACVLAN_SIZE; i++) { 1618 hlist_for_each_entry_safe(pos, n, &ndev->macvlan_hash[i], hlist) { 1619 hlist_del(&pos->hlist); 1620 mlx5_vdpa_del_mac_vlan_rules(ndev, pos); 1621 remove_steering_counters(ndev, pos); 1622 kfree(pos); 1623 } 1624 } 1625 } 1626 1627 static int setup_steering(struct mlx5_vdpa_net *ndev) 1628 { 1629 struct mlx5_flow_table_attr ft_attr = {}; 1630 struct mlx5_flow_namespace *ns; 1631 int err; 1632 1633 ft_attr.max_fte = MAX_STEERING_ENT; 1634 ft_attr.autogroup.max_num_groups = MAX_STEERING_GROUPS; 1635 1636 ns = mlx5_get_flow_namespace(ndev->mvdev.mdev, MLX5_FLOW_NAMESPACE_BYPASS); 1637 if (!ns) { 1638 mlx5_vdpa_warn(&ndev->mvdev, "failed to get flow namespace\n"); 1639 return -EOPNOTSUPP; 1640 } 1641 1642 ndev->rxft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 1643 if (IS_ERR(ndev->rxft)) { 1644 mlx5_vdpa_warn(&ndev->mvdev, "failed to create flow table\n"); 1645 return PTR_ERR(ndev->rxft); 1646 } 1647 mlx5_vdpa_add_rx_flow_table(ndev); 1648 1649 err = mac_vlan_add(ndev, ndev->config.mac, 0, false); 1650 if (err) 1651 goto err_add; 1652 1653 return 0; 1654 1655 err_add: 1656 mlx5_vdpa_remove_rx_flow_table(ndev); 1657 mlx5_destroy_flow_table(ndev->rxft); 1658 return err; 1659 } 1660 1661 static void teardown_steering(struct mlx5_vdpa_net *ndev) 1662 { 1663 clear_mac_vlan_table(ndev); 1664 mlx5_vdpa_remove_rx_flow_table(ndev); 1665 mlx5_destroy_flow_table(ndev->rxft); 1666 } 1667 1668 static virtio_net_ctrl_ack handle_ctrl_mac(struct mlx5_vdpa_dev *mvdev, u8 cmd) 1669 { 1670 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1671 struct mlx5_control_vq *cvq = &mvdev->cvq; 1672 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1673 struct mlx5_core_dev *pfmdev; 1674 size_t read; 1675 u8 mac[ETH_ALEN], mac_back[ETH_ALEN]; 1676 1677 pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 1678 switch (cmd) { 1679 case VIRTIO_NET_CTRL_MAC_ADDR_SET: 1680 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)mac, ETH_ALEN); 1681 if (read != ETH_ALEN) 1682 break; 1683 1684 if (!memcmp(ndev->config.mac, mac, 6)) { 1685 status = VIRTIO_NET_OK; 1686 break; 1687 } 1688 1689 if (is_zero_ether_addr(mac)) 1690 break; 1691 1692 if (!is_zero_ether_addr(ndev->config.mac)) { 1693 if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { 1694 mlx5_vdpa_warn(mvdev, "failed to delete old MAC %pM from MPFS table\n", 1695 ndev->config.mac); 1696 break; 1697 } 1698 } 1699 1700 if (mlx5_mpfs_add_mac(pfmdev, mac)) { 1701 mlx5_vdpa_warn(mvdev, "failed to insert new MAC %pM into MPFS table\n", 1702 mac); 1703 break; 1704 } 1705 1706 /* backup the original mac address so that if failed to add the forward rules 1707 * we could restore it 1708 */ 1709 memcpy(mac_back, ndev->config.mac, ETH_ALEN); 1710 1711 memcpy(ndev->config.mac, mac, ETH_ALEN); 1712 1713 /* Need recreate the flow table entry, so that the packet could forward back 1714 */ 1715 mac_vlan_del(ndev, mac_back, 0, false); 1716 1717 if (mac_vlan_add(ndev, ndev->config.mac, 0, false)) { 1718 mlx5_vdpa_warn(mvdev, "failed to insert forward rules, try to restore\n"); 1719 1720 /* Although it hardly run here, we still need double check */ 1721 if (is_zero_ether_addr(mac_back)) { 1722 mlx5_vdpa_warn(mvdev, "restore mac failed: Original MAC is zero\n"); 1723 break; 1724 } 1725 1726 /* Try to restore original mac address to MFPS table, and try to restore 1727 * the forward rule entry. 1728 */ 1729 if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) { 1730 mlx5_vdpa_warn(mvdev, "restore mac failed: delete MAC %pM from MPFS table failed\n", 1731 ndev->config.mac); 1732 } 1733 1734 if (mlx5_mpfs_add_mac(pfmdev, mac_back)) { 1735 mlx5_vdpa_warn(mvdev, "restore mac failed: insert old MAC %pM into MPFS table failed\n", 1736 mac_back); 1737 } 1738 1739 memcpy(ndev->config.mac, mac_back, ETH_ALEN); 1740 1741 if (mac_vlan_add(ndev, ndev->config.mac, 0, false)) 1742 mlx5_vdpa_warn(mvdev, "restore forward rules failed: insert forward rules failed\n"); 1743 1744 break; 1745 } 1746 1747 status = VIRTIO_NET_OK; 1748 break; 1749 1750 default: 1751 break; 1752 } 1753 1754 return status; 1755 } 1756 1757 static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps) 1758 { 1759 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1760 int cur_qps = ndev->cur_num_vqs / 2; 1761 int err; 1762 int i; 1763 1764 if (cur_qps > newqps) { 1765 err = modify_rqt(ndev, 2 * newqps); 1766 if (err) 1767 return err; 1768 1769 for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--) 1770 teardown_vq(ndev, &ndev->vqs[i]); 1771 1772 ndev->cur_num_vqs = 2 * newqps; 1773 } else { 1774 ndev->cur_num_vqs = 2 * newqps; 1775 for (i = cur_qps * 2; i < 2 * newqps; i++) { 1776 err = setup_vq(ndev, &ndev->vqs[i]); 1777 if (err) 1778 goto clean_added; 1779 } 1780 err = modify_rqt(ndev, 2 * newqps); 1781 if (err) 1782 goto clean_added; 1783 } 1784 return 0; 1785 1786 clean_added: 1787 for (--i; i >= 2 * cur_qps; --i) 1788 teardown_vq(ndev, &ndev->vqs[i]); 1789 1790 ndev->cur_num_vqs = 2 * cur_qps; 1791 1792 return err; 1793 } 1794 1795 static virtio_net_ctrl_ack handle_ctrl_mq(struct mlx5_vdpa_dev *mvdev, u8 cmd) 1796 { 1797 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1798 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1799 struct mlx5_control_vq *cvq = &mvdev->cvq; 1800 struct virtio_net_ctrl_mq mq; 1801 size_t read; 1802 u16 newqps; 1803 1804 switch (cmd) { 1805 case VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET: 1806 /* This mq feature check aligns with pre-existing userspace 1807 * implementation. 1808 * 1809 * Without it, an untrusted driver could fake a multiqueue config 1810 * request down to a non-mq device that may cause kernel to 1811 * panic due to uninitialized resources for extra vqs. Even with 1812 * a well behaving guest driver, it is not expected to allow 1813 * changing the number of vqs on a non-mq device. 1814 */ 1815 if (!MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) 1816 break; 1817 1818 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, (void *)&mq, sizeof(mq)); 1819 if (read != sizeof(mq)) 1820 break; 1821 1822 newqps = mlx5vdpa16_to_cpu(mvdev, mq.virtqueue_pairs); 1823 if (newqps < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 1824 newqps > ndev->rqt_size) 1825 break; 1826 1827 if (ndev->cur_num_vqs == 2 * newqps) { 1828 status = VIRTIO_NET_OK; 1829 break; 1830 } 1831 1832 if (!change_num_qps(mvdev, newqps)) 1833 status = VIRTIO_NET_OK; 1834 1835 break; 1836 default: 1837 break; 1838 } 1839 1840 return status; 1841 } 1842 1843 static virtio_net_ctrl_ack handle_ctrl_vlan(struct mlx5_vdpa_dev *mvdev, u8 cmd) 1844 { 1845 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1846 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1847 struct mlx5_control_vq *cvq = &mvdev->cvq; 1848 __virtio16 vlan; 1849 size_t read; 1850 u16 id; 1851 1852 if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VLAN))) 1853 return status; 1854 1855 switch (cmd) { 1856 case VIRTIO_NET_CTRL_VLAN_ADD: 1857 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &vlan, sizeof(vlan)); 1858 if (read != sizeof(vlan)) 1859 break; 1860 1861 id = mlx5vdpa16_to_cpu(mvdev, vlan); 1862 if (mac_vlan_add(ndev, ndev->config.mac, id, true)) 1863 break; 1864 1865 status = VIRTIO_NET_OK; 1866 break; 1867 case VIRTIO_NET_CTRL_VLAN_DEL: 1868 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &vlan, sizeof(vlan)); 1869 if (read != sizeof(vlan)) 1870 break; 1871 1872 id = mlx5vdpa16_to_cpu(mvdev, vlan); 1873 mac_vlan_del(ndev, ndev->config.mac, id, true); 1874 status = VIRTIO_NET_OK; 1875 break; 1876 default: 1877 break; 1878 } 1879 1880 return status; 1881 } 1882 1883 static void mlx5_cvq_kick_handler(struct work_struct *work) 1884 { 1885 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1886 struct virtio_net_ctrl_hdr ctrl; 1887 struct mlx5_vdpa_wq_ent *wqent; 1888 struct mlx5_vdpa_dev *mvdev; 1889 struct mlx5_control_vq *cvq; 1890 struct mlx5_vdpa_net *ndev; 1891 size_t read, write; 1892 int err; 1893 1894 wqent = container_of(work, struct mlx5_vdpa_wq_ent, work); 1895 mvdev = wqent->mvdev; 1896 ndev = to_mlx5_vdpa_ndev(mvdev); 1897 cvq = &mvdev->cvq; 1898 1899 down_write(&ndev->reslock); 1900 1901 if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 1902 goto out; 1903 1904 if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 1905 goto out; 1906 1907 if (!cvq->ready) 1908 goto out; 1909 1910 while (true) { 1911 err = vringh_getdesc_iotlb(&cvq->vring, &cvq->riov, &cvq->wiov, &cvq->head, 1912 GFP_ATOMIC); 1913 if (err <= 0) 1914 break; 1915 1916 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->riov, &ctrl, sizeof(ctrl)); 1917 if (read != sizeof(ctrl)) 1918 break; 1919 1920 cvq->received_desc++; 1921 switch (ctrl.class) { 1922 case VIRTIO_NET_CTRL_MAC: 1923 status = handle_ctrl_mac(mvdev, ctrl.cmd); 1924 break; 1925 case VIRTIO_NET_CTRL_MQ: 1926 status = handle_ctrl_mq(mvdev, ctrl.cmd); 1927 break; 1928 case VIRTIO_NET_CTRL_VLAN: 1929 status = handle_ctrl_vlan(mvdev, ctrl.cmd); 1930 break; 1931 default: 1932 break; 1933 } 1934 1935 /* Make sure data is written before advancing index */ 1936 smp_wmb(); 1937 1938 write = vringh_iov_push_iotlb(&cvq->vring, &cvq->wiov, &status, sizeof(status)); 1939 vringh_complete_iotlb(&cvq->vring, cvq->head, write); 1940 vringh_kiov_cleanup(&cvq->riov); 1941 vringh_kiov_cleanup(&cvq->wiov); 1942 1943 if (vringh_need_notify_iotlb(&cvq->vring)) 1944 vringh_notify(&cvq->vring); 1945 1946 cvq->completed_desc++; 1947 queue_work(mvdev->wq, &wqent->work); 1948 break; 1949 } 1950 1951 out: 1952 up_write(&ndev->reslock); 1953 } 1954 1955 static void mlx5_vdpa_kick_vq(struct vdpa_device *vdev, u16 idx) 1956 { 1957 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 1958 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1959 struct mlx5_vdpa_virtqueue *mvq; 1960 1961 if (!is_index_valid(mvdev, idx)) 1962 return; 1963 1964 if (unlikely(is_ctrl_vq_idx(mvdev, idx))) { 1965 if (!mvdev->wq || !mvdev->cvq.ready) 1966 return; 1967 1968 queue_work(mvdev->wq, &ndev->cvq_ent.work); 1969 return; 1970 } 1971 1972 mvq = &ndev->vqs[idx]; 1973 if (unlikely(!mvq->ready)) 1974 return; 1975 1976 iowrite16(idx, ndev->mvdev.res.kick_addr); 1977 } 1978 1979 static int mlx5_vdpa_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, 1980 u64 driver_area, u64 device_area) 1981 { 1982 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 1983 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 1984 struct mlx5_vdpa_virtqueue *mvq; 1985 1986 if (!is_index_valid(mvdev, idx)) 1987 return -EINVAL; 1988 1989 if (is_ctrl_vq_idx(mvdev, idx)) { 1990 mvdev->cvq.desc_addr = desc_area; 1991 mvdev->cvq.device_addr = device_area; 1992 mvdev->cvq.driver_addr = driver_area; 1993 return 0; 1994 } 1995 1996 mvq = &ndev->vqs[idx]; 1997 mvq->desc_addr = desc_area; 1998 mvq->device_addr = device_area; 1999 mvq->driver_addr = driver_area; 2000 return 0; 2001 } 2002 2003 static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) 2004 { 2005 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2006 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2007 struct mlx5_vdpa_virtqueue *mvq; 2008 2009 if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 2010 return; 2011 2012 mvq = &ndev->vqs[idx]; 2013 mvq->num_ent = num; 2014 } 2015 2016 static void mlx5_vdpa_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) 2017 { 2018 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2019 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2020 2021 ndev->event_cbs[idx] = *cb; 2022 if (is_ctrl_vq_idx(mvdev, idx)) 2023 mvdev->cvq.event_cb = *cb; 2024 } 2025 2026 static void mlx5_cvq_notify(struct vringh *vring) 2027 { 2028 struct mlx5_control_vq *cvq = container_of(vring, struct mlx5_control_vq, vring); 2029 2030 if (!cvq->event_cb.callback) 2031 return; 2032 2033 cvq->event_cb.callback(cvq->event_cb.private); 2034 } 2035 2036 static void set_cvq_ready(struct mlx5_vdpa_dev *mvdev, bool ready) 2037 { 2038 struct mlx5_control_vq *cvq = &mvdev->cvq; 2039 2040 cvq->ready = ready; 2041 if (!ready) 2042 return; 2043 2044 cvq->vring.notify = mlx5_cvq_notify; 2045 } 2046 2047 static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) 2048 { 2049 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2050 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2051 struct mlx5_vdpa_virtqueue *mvq; 2052 int err; 2053 2054 if (!mvdev->actual_features) 2055 return; 2056 2057 if (!is_index_valid(mvdev, idx)) 2058 return; 2059 2060 if (is_ctrl_vq_idx(mvdev, idx)) { 2061 set_cvq_ready(mvdev, ready); 2062 return; 2063 } 2064 2065 mvq = &ndev->vqs[idx]; 2066 if (!ready) { 2067 suspend_vq(ndev, mvq); 2068 } else { 2069 err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); 2070 if (err) { 2071 mlx5_vdpa_warn(mvdev, "modify VQ %d to ready failed (%d)\n", idx, err); 2072 ready = false; 2073 } 2074 } 2075 2076 2077 mvq->ready = ready; 2078 } 2079 2080 static bool mlx5_vdpa_get_vq_ready(struct vdpa_device *vdev, u16 idx) 2081 { 2082 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2083 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2084 2085 if (!is_index_valid(mvdev, idx)) 2086 return false; 2087 2088 if (is_ctrl_vq_idx(mvdev, idx)) 2089 return mvdev->cvq.ready; 2090 2091 return ndev->vqs[idx].ready; 2092 } 2093 2094 static int mlx5_vdpa_set_vq_state(struct vdpa_device *vdev, u16 idx, 2095 const struct vdpa_vq_state *state) 2096 { 2097 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2098 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2099 struct mlx5_vdpa_virtqueue *mvq; 2100 2101 if (!is_index_valid(mvdev, idx)) 2102 return -EINVAL; 2103 2104 if (is_ctrl_vq_idx(mvdev, idx)) { 2105 mvdev->cvq.vring.last_avail_idx = state->split.avail_index; 2106 return 0; 2107 } 2108 2109 mvq = &ndev->vqs[idx]; 2110 if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) { 2111 mlx5_vdpa_warn(mvdev, "can't modify available index\n"); 2112 return -EINVAL; 2113 } 2114 2115 mvq->used_idx = state->split.avail_index; 2116 mvq->avail_idx = state->split.avail_index; 2117 return 0; 2118 } 2119 2120 static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) 2121 { 2122 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2123 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2124 struct mlx5_vdpa_virtqueue *mvq; 2125 struct mlx5_virtq_attr attr; 2126 int err; 2127 2128 if (!is_index_valid(mvdev, idx)) 2129 return -EINVAL; 2130 2131 if (is_ctrl_vq_idx(mvdev, idx)) { 2132 state->split.avail_index = mvdev->cvq.vring.last_avail_idx; 2133 return 0; 2134 } 2135 2136 mvq = &ndev->vqs[idx]; 2137 /* If the virtq object was destroyed, use the value saved at 2138 * the last minute of suspend_vq. This caters for userspace 2139 * that cares about emulating the index after vq is stopped. 2140 */ 2141 if (!mvq->initialized) { 2142 /* Firmware returns a wrong value for the available index. 2143 * Since both values should be identical, we take the value of 2144 * used_idx which is reported correctly. 2145 */ 2146 state->split.avail_index = mvq->used_idx; 2147 return 0; 2148 } 2149 2150 err = query_virtqueue(ndev, mvq, &attr); 2151 if (err) { 2152 mlx5_vdpa_warn(mvdev, "failed to query virtqueue\n"); 2153 return err; 2154 } 2155 state->split.avail_index = attr.used_index; 2156 return 0; 2157 } 2158 2159 static u32 mlx5_vdpa_get_vq_align(struct vdpa_device *vdev) 2160 { 2161 return PAGE_SIZE; 2162 } 2163 2164 static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdev, u16 idx) 2165 { 2166 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2167 2168 if (is_ctrl_vq_idx(mvdev, idx)) 2169 return MLX5_VDPA_CVQ_GROUP; 2170 2171 return MLX5_VDPA_DATAVQ_GROUP; 2172 } 2173 2174 enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9, 2175 MLX5_VIRTIO_NET_F_CSUM = 1 << 10, 2176 MLX5_VIRTIO_NET_F_HOST_TSO6 = 1 << 11, 2177 MLX5_VIRTIO_NET_F_HOST_TSO4 = 1 << 12, 2178 }; 2179 2180 static u64 mlx_to_vritio_features(u16 dev_features) 2181 { 2182 u64 result = 0; 2183 2184 if (dev_features & MLX5_VIRTIO_NET_F_GUEST_CSUM) 2185 result |= BIT_ULL(VIRTIO_NET_F_GUEST_CSUM); 2186 if (dev_features & MLX5_VIRTIO_NET_F_CSUM) 2187 result |= BIT_ULL(VIRTIO_NET_F_CSUM); 2188 if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO6) 2189 result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO6); 2190 if (dev_features & MLX5_VIRTIO_NET_F_HOST_TSO4) 2191 result |= BIT_ULL(VIRTIO_NET_F_HOST_TSO4); 2192 2193 return result; 2194 } 2195 2196 static u64 get_supported_features(struct mlx5_core_dev *mdev) 2197 { 2198 u64 mlx_vdpa_features = 0; 2199 u16 dev_features; 2200 2201 dev_features = MLX5_CAP_DEV_VDPA_EMULATION(mdev, device_features_bits_mask); 2202 mlx_vdpa_features |= mlx_to_vritio_features(dev_features); 2203 if (MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_version_1_0)) 2204 mlx_vdpa_features |= BIT_ULL(VIRTIO_F_VERSION_1); 2205 mlx_vdpa_features |= BIT_ULL(VIRTIO_F_ACCESS_PLATFORM); 2206 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VQ); 2207 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR); 2208 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_MQ); 2209 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_STATUS); 2210 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_MTU); 2211 mlx_vdpa_features |= BIT_ULL(VIRTIO_NET_F_CTRL_VLAN); 2212 2213 return mlx_vdpa_features; 2214 } 2215 2216 static u64 mlx5_vdpa_get_device_features(struct vdpa_device *vdev) 2217 { 2218 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2219 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2220 2221 print_features(mvdev, ndev->mvdev.mlx_features, false); 2222 return ndev->mvdev.mlx_features; 2223 } 2224 2225 static int verify_driver_features(struct mlx5_vdpa_dev *mvdev, u64 features) 2226 { 2227 /* Minimum features to expect */ 2228 if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM))) 2229 return -EOPNOTSUPP; 2230 2231 /* Double check features combination sent down by the driver. 2232 * Fail invalid features due to absence of the depended feature. 2233 * 2234 * Per VIRTIO v1.1 specification, section 5.1.3.1 Feature bit 2235 * requirements: "VIRTIO_NET_F_MQ Requires VIRTIO_NET_F_CTRL_VQ". 2236 * By failing the invalid features sent down by untrusted drivers, 2237 * we're assured the assumption made upon is_index_valid() and 2238 * is_ctrl_vq_idx() will not be compromised. 2239 */ 2240 if ((features & (BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) == 2241 BIT_ULL(VIRTIO_NET_F_MQ)) 2242 return -EINVAL; 2243 2244 return 0; 2245 } 2246 2247 static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev) 2248 { 2249 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2250 int err; 2251 int i; 2252 2253 for (i = 0; i < mvdev->max_vqs; i++) { 2254 err = setup_vq(ndev, &ndev->vqs[i]); 2255 if (err) 2256 goto err_vq; 2257 } 2258 2259 return 0; 2260 2261 err_vq: 2262 for (--i; i >= 0; i--) 2263 teardown_vq(ndev, &ndev->vqs[i]); 2264 2265 return err; 2266 } 2267 2268 static void teardown_virtqueues(struct mlx5_vdpa_net *ndev) 2269 { 2270 struct mlx5_vdpa_virtqueue *mvq; 2271 int i; 2272 2273 for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) { 2274 mvq = &ndev->vqs[i]; 2275 if (!mvq->initialized) 2276 continue; 2277 2278 teardown_vq(ndev, mvq); 2279 } 2280 } 2281 2282 static void update_cvq_info(struct mlx5_vdpa_dev *mvdev) 2283 { 2284 if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_CTRL_VQ)) { 2285 if (MLX5_FEATURE(mvdev, VIRTIO_NET_F_MQ)) { 2286 /* MQ supported. CVQ index is right above the last data virtqueue's */ 2287 mvdev->max_idx = mvdev->max_vqs; 2288 } else { 2289 /* Only CVQ supportted. data virtqueues occupy indices 0 and 1. 2290 * CVQ gets index 2 2291 */ 2292 mvdev->max_idx = 2; 2293 } 2294 } else { 2295 /* Two data virtqueues only: one for rx and one for tx */ 2296 mvdev->max_idx = 1; 2297 } 2298 } 2299 2300 static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) 2301 { 2302 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2303 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2304 int err; 2305 2306 print_features(mvdev, features, true); 2307 2308 err = verify_driver_features(mvdev, features); 2309 if (err) 2310 return err; 2311 2312 ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features; 2313 if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ)) 2314 ndev->rqt_size = mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs); 2315 else 2316 ndev->rqt_size = 1; 2317 2318 ndev->cur_num_vqs = 2 * ndev->rqt_size; 2319 2320 update_cvq_info(mvdev); 2321 return err; 2322 } 2323 2324 static void mlx5_vdpa_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) 2325 { 2326 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2327 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2328 2329 ndev->config_cb = *cb; 2330 } 2331 2332 #define MLX5_VDPA_MAX_VQ_ENTRIES 256 2333 static u16 mlx5_vdpa_get_vq_num_max(struct vdpa_device *vdev) 2334 { 2335 return MLX5_VDPA_MAX_VQ_ENTRIES; 2336 } 2337 2338 static u32 mlx5_vdpa_get_device_id(struct vdpa_device *vdev) 2339 { 2340 return VIRTIO_ID_NET; 2341 } 2342 2343 static u32 mlx5_vdpa_get_vendor_id(struct vdpa_device *vdev) 2344 { 2345 return PCI_VENDOR_ID_MELLANOX; 2346 } 2347 2348 static u8 mlx5_vdpa_get_status(struct vdpa_device *vdev) 2349 { 2350 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2351 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2352 2353 print_status(mvdev, ndev->mvdev.status, false); 2354 return ndev->mvdev.status; 2355 } 2356 2357 static int save_channel_info(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 2358 { 2359 struct mlx5_vq_restore_info *ri = &mvq->ri; 2360 struct mlx5_virtq_attr attr = {}; 2361 int err; 2362 2363 if (mvq->initialized) { 2364 err = query_virtqueue(ndev, mvq, &attr); 2365 if (err) 2366 return err; 2367 } 2368 2369 ri->avail_index = attr.available_index; 2370 ri->used_index = attr.used_index; 2371 ri->ready = mvq->ready; 2372 ri->num_ent = mvq->num_ent; 2373 ri->desc_addr = mvq->desc_addr; 2374 ri->device_addr = mvq->device_addr; 2375 ri->driver_addr = mvq->driver_addr; 2376 ri->restore = true; 2377 return 0; 2378 } 2379 2380 static int save_channels_info(struct mlx5_vdpa_net *ndev) 2381 { 2382 int i; 2383 2384 for (i = 0; i < ndev->mvdev.max_vqs; i++) { 2385 memset(&ndev->vqs[i].ri, 0, sizeof(ndev->vqs[i].ri)); 2386 save_channel_info(ndev, &ndev->vqs[i]); 2387 } 2388 return 0; 2389 } 2390 2391 static void mlx5_clear_vqs(struct mlx5_vdpa_net *ndev) 2392 { 2393 int i; 2394 2395 for (i = 0; i < ndev->mvdev.max_vqs; i++) 2396 memset(&ndev->vqs[i], 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 2397 } 2398 2399 static void restore_channels_info(struct mlx5_vdpa_net *ndev) 2400 { 2401 struct mlx5_vdpa_virtqueue *mvq; 2402 struct mlx5_vq_restore_info *ri; 2403 int i; 2404 2405 mlx5_clear_vqs(ndev); 2406 init_mvqs(ndev); 2407 for (i = 0; i < ndev->mvdev.max_vqs; i++) { 2408 mvq = &ndev->vqs[i]; 2409 ri = &mvq->ri; 2410 if (!ri->restore) 2411 continue; 2412 2413 mvq->avail_idx = ri->avail_index; 2414 mvq->used_idx = ri->used_index; 2415 mvq->ready = ri->ready; 2416 mvq->num_ent = ri->num_ent; 2417 mvq->desc_addr = ri->desc_addr; 2418 mvq->device_addr = ri->device_addr; 2419 mvq->driver_addr = ri->driver_addr; 2420 } 2421 } 2422 2423 static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev, 2424 struct vhost_iotlb *iotlb, unsigned int asid) 2425 { 2426 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2427 int err; 2428 2429 suspend_vqs(ndev); 2430 err = save_channels_info(ndev); 2431 if (err) 2432 goto err_mr; 2433 2434 teardown_driver(ndev); 2435 mlx5_vdpa_destroy_mr(mvdev); 2436 err = mlx5_vdpa_create_mr(mvdev, iotlb, asid); 2437 if (err) 2438 goto err_mr; 2439 2440 if (!(mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 2441 goto err_mr; 2442 2443 restore_channels_info(ndev); 2444 err = setup_driver(mvdev); 2445 if (err) 2446 goto err_setup; 2447 2448 return 0; 2449 2450 err_setup: 2451 mlx5_vdpa_destroy_mr(mvdev); 2452 err_mr: 2453 return err; 2454 } 2455 2456 /* reslock must be held for this function */ 2457 static int setup_driver(struct mlx5_vdpa_dev *mvdev) 2458 { 2459 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2460 int err; 2461 2462 WARN_ON(!rwsem_is_locked(&ndev->reslock)); 2463 2464 if (ndev->setup) { 2465 mlx5_vdpa_warn(mvdev, "setup driver called for already setup driver\n"); 2466 err = 0; 2467 goto out; 2468 } 2469 err = setup_virtqueues(mvdev); 2470 if (err) { 2471 mlx5_vdpa_warn(mvdev, "setup_virtqueues\n"); 2472 goto out; 2473 } 2474 2475 err = create_rqt(ndev); 2476 if (err) { 2477 mlx5_vdpa_warn(mvdev, "create_rqt\n"); 2478 goto err_rqt; 2479 } 2480 2481 err = create_tir(ndev); 2482 if (err) { 2483 mlx5_vdpa_warn(mvdev, "create_tir\n"); 2484 goto err_tir; 2485 } 2486 2487 err = setup_steering(ndev); 2488 if (err) { 2489 mlx5_vdpa_warn(mvdev, "setup_steering\n"); 2490 goto err_fwd; 2491 } 2492 ndev->setup = true; 2493 2494 return 0; 2495 2496 err_fwd: 2497 destroy_tir(ndev); 2498 err_tir: 2499 destroy_rqt(ndev); 2500 err_rqt: 2501 teardown_virtqueues(ndev); 2502 out: 2503 return err; 2504 } 2505 2506 /* reslock must be held for this function */ 2507 static void teardown_driver(struct mlx5_vdpa_net *ndev) 2508 { 2509 2510 WARN_ON(!rwsem_is_locked(&ndev->reslock)); 2511 2512 if (!ndev->setup) 2513 return; 2514 2515 teardown_steering(ndev); 2516 destroy_tir(ndev); 2517 destroy_rqt(ndev); 2518 teardown_virtqueues(ndev); 2519 ndev->setup = false; 2520 } 2521 2522 static void clear_vqs_ready(struct mlx5_vdpa_net *ndev) 2523 { 2524 int i; 2525 2526 for (i = 0; i < ndev->mvdev.max_vqs; i++) 2527 ndev->vqs[i].ready = false; 2528 2529 ndev->mvdev.cvq.ready = false; 2530 } 2531 2532 static int setup_cvq_vring(struct mlx5_vdpa_dev *mvdev) 2533 { 2534 struct mlx5_control_vq *cvq = &mvdev->cvq; 2535 int err = 0; 2536 2537 if (mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)) 2538 err = vringh_init_iotlb(&cvq->vring, mvdev->actual_features, 2539 MLX5_CVQ_MAX_ENT, false, 2540 (struct vring_desc *)(uintptr_t)cvq->desc_addr, 2541 (struct vring_avail *)(uintptr_t)cvq->driver_addr, 2542 (struct vring_used *)(uintptr_t)cvq->device_addr); 2543 2544 return err; 2545 } 2546 2547 static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status) 2548 { 2549 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2550 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2551 int err; 2552 2553 print_status(mvdev, status, true); 2554 2555 down_write(&ndev->reslock); 2556 2557 if ((status ^ ndev->mvdev.status) & VIRTIO_CONFIG_S_DRIVER_OK) { 2558 if (status & VIRTIO_CONFIG_S_DRIVER_OK) { 2559 err = setup_cvq_vring(mvdev); 2560 if (err) { 2561 mlx5_vdpa_warn(mvdev, "failed to setup control VQ vring\n"); 2562 goto err_setup; 2563 } 2564 err = setup_driver(mvdev); 2565 if (err) { 2566 mlx5_vdpa_warn(mvdev, "failed to setup driver\n"); 2567 goto err_setup; 2568 } 2569 } else { 2570 mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n"); 2571 goto err_clear; 2572 } 2573 } 2574 2575 ndev->mvdev.status = status; 2576 up_write(&ndev->reslock); 2577 return; 2578 2579 err_setup: 2580 mlx5_vdpa_destroy_mr(&ndev->mvdev); 2581 ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED; 2582 err_clear: 2583 up_write(&ndev->reslock); 2584 } 2585 2586 static void init_group_to_asid_map(struct mlx5_vdpa_dev *mvdev) 2587 { 2588 int i; 2589 2590 /* default mapping all groups are mapped to asid 0 */ 2591 for (i = 0; i < MLX5_VDPA_NUMVQ_GROUPS; i++) 2592 mvdev->group2asid[i] = 0; 2593 } 2594 2595 static int mlx5_vdpa_reset(struct vdpa_device *vdev) 2596 { 2597 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2598 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2599 2600 print_status(mvdev, 0, true); 2601 mlx5_vdpa_info(mvdev, "performing device reset\n"); 2602 2603 down_write(&ndev->reslock); 2604 teardown_driver(ndev); 2605 clear_vqs_ready(ndev); 2606 mlx5_vdpa_destroy_mr(&ndev->mvdev); 2607 ndev->mvdev.status = 0; 2608 ndev->cur_num_vqs = 0; 2609 ndev->mvdev.cvq.received_desc = 0; 2610 ndev->mvdev.cvq.completed_desc = 0; 2611 memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1)); 2612 ndev->mvdev.actual_features = 0; 2613 init_group_to_asid_map(mvdev); 2614 ++mvdev->generation; 2615 2616 if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 2617 if (mlx5_vdpa_create_mr(mvdev, NULL, 0)) 2618 mlx5_vdpa_warn(mvdev, "create MR failed\n"); 2619 } 2620 up_write(&ndev->reslock); 2621 2622 return 0; 2623 } 2624 2625 static size_t mlx5_vdpa_get_config_size(struct vdpa_device *vdev) 2626 { 2627 return sizeof(struct virtio_net_config); 2628 } 2629 2630 static void mlx5_vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, void *buf, 2631 unsigned int len) 2632 { 2633 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2634 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2635 2636 if (offset + len <= sizeof(struct virtio_net_config)) 2637 memcpy(buf, (u8 *)&ndev->config + offset, len); 2638 } 2639 2640 static void mlx5_vdpa_set_config(struct vdpa_device *vdev, unsigned int offset, const void *buf, 2641 unsigned int len) 2642 { 2643 /* not supported */ 2644 } 2645 2646 static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev) 2647 { 2648 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2649 2650 return mvdev->generation; 2651 } 2652 2653 static int set_map_data(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb, 2654 unsigned int asid) 2655 { 2656 bool change_map; 2657 int err; 2658 2659 err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map, asid); 2660 if (err) { 2661 mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err); 2662 return err; 2663 } 2664 2665 if (change_map) 2666 err = mlx5_vdpa_change_map(mvdev, iotlb, asid); 2667 2668 return err; 2669 } 2670 2671 static int mlx5_vdpa_set_map(struct vdpa_device *vdev, unsigned int asid, 2672 struct vhost_iotlb *iotlb) 2673 { 2674 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2675 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2676 int err = -EINVAL; 2677 2678 down_write(&ndev->reslock); 2679 err = set_map_data(mvdev, iotlb, asid); 2680 up_write(&ndev->reslock); 2681 return err; 2682 } 2683 2684 static void mlx5_vdpa_free(struct vdpa_device *vdev) 2685 { 2686 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2687 struct mlx5_core_dev *pfmdev; 2688 struct mlx5_vdpa_net *ndev; 2689 2690 ndev = to_mlx5_vdpa_ndev(mvdev); 2691 2692 free_resources(ndev); 2693 mlx5_vdpa_destroy_mr(mvdev); 2694 if (!is_zero_ether_addr(ndev->config.mac)) { 2695 pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev)); 2696 mlx5_mpfs_del_mac(pfmdev, ndev->config.mac); 2697 } 2698 mlx5_vdpa_free_resources(&ndev->mvdev); 2699 kfree(ndev->event_cbs); 2700 kfree(ndev->vqs); 2701 } 2702 2703 static struct vdpa_notification_area mlx5_get_vq_notification(struct vdpa_device *vdev, u16 idx) 2704 { 2705 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2706 struct vdpa_notification_area ret = {}; 2707 struct mlx5_vdpa_net *ndev; 2708 phys_addr_t addr; 2709 2710 if (!is_index_valid(mvdev, idx) || is_ctrl_vq_idx(mvdev, idx)) 2711 return ret; 2712 2713 /* If SF BAR size is smaller than PAGE_SIZE, do not use direct 2714 * notification to avoid the risk of mapping pages that contain BAR of more 2715 * than one SF 2716 */ 2717 if (MLX5_CAP_GEN(mvdev->mdev, log_min_sf_size) + 12 < PAGE_SHIFT) 2718 return ret; 2719 2720 ndev = to_mlx5_vdpa_ndev(mvdev); 2721 addr = (phys_addr_t)ndev->mvdev.res.phys_kick_addr; 2722 ret.addr = addr; 2723 ret.size = PAGE_SIZE; 2724 return ret; 2725 } 2726 2727 static int mlx5_get_vq_irq(struct vdpa_device *vdv, u16 idx) 2728 { 2729 return -EOPNOTSUPP; 2730 } 2731 2732 static u64 mlx5_vdpa_get_driver_features(struct vdpa_device *vdev) 2733 { 2734 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2735 2736 return mvdev->actual_features; 2737 } 2738 2739 static int counter_set_query(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, 2740 u64 *received_desc, u64 *completed_desc) 2741 { 2742 u32 in[MLX5_ST_SZ_DW(query_virtio_q_counters_in)] = {}; 2743 u32 out[MLX5_ST_SZ_DW(query_virtio_q_counters_out)] = {}; 2744 void *cmd_hdr; 2745 void *ctx; 2746 int err; 2747 2748 if (!counters_supported(&ndev->mvdev)) 2749 return -EOPNOTSUPP; 2750 2751 if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY) 2752 return -EAGAIN; 2753 2754 cmd_hdr = MLX5_ADDR_OF(query_virtio_q_counters_in, in, hdr); 2755 2756 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); 2757 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 2758 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid); 2759 MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->counter_set_id); 2760 2761 err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, sizeof(out)); 2762 if (err) 2763 return err; 2764 2765 ctx = MLX5_ADDR_OF(query_virtio_q_counters_out, out, counters); 2766 *received_desc = MLX5_GET64(virtio_q_counters, ctx, received_desc); 2767 *completed_desc = MLX5_GET64(virtio_q_counters, ctx, completed_desc); 2768 return 0; 2769 } 2770 2771 static int mlx5_vdpa_get_vendor_vq_stats(struct vdpa_device *vdev, u16 idx, 2772 struct sk_buff *msg, 2773 struct netlink_ext_ack *extack) 2774 { 2775 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2776 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2777 struct mlx5_vdpa_virtqueue *mvq; 2778 struct mlx5_control_vq *cvq; 2779 u64 received_desc; 2780 u64 completed_desc; 2781 int err = 0; 2782 2783 down_read(&ndev->reslock); 2784 if (!is_index_valid(mvdev, idx)) { 2785 NL_SET_ERR_MSG_MOD(extack, "virtqueue index is not valid"); 2786 err = -EINVAL; 2787 goto out_err; 2788 } 2789 2790 if (idx == ctrl_vq_idx(mvdev)) { 2791 cvq = &mvdev->cvq; 2792 received_desc = cvq->received_desc; 2793 completed_desc = cvq->completed_desc; 2794 goto out; 2795 } 2796 2797 mvq = &ndev->vqs[idx]; 2798 err = counter_set_query(ndev, mvq, &received_desc, &completed_desc); 2799 if (err) { 2800 NL_SET_ERR_MSG_MOD(extack, "failed to query hardware"); 2801 goto out_err; 2802 } 2803 2804 out: 2805 err = -EMSGSIZE; 2806 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, "received_desc")) 2807 goto out_err; 2808 2809 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, received_desc, 2810 VDPA_ATTR_PAD)) 2811 goto out_err; 2812 2813 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, "completed_desc")) 2814 goto out_err; 2815 2816 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, completed_desc, 2817 VDPA_ATTR_PAD)) 2818 goto out_err; 2819 2820 err = 0; 2821 out_err: 2822 up_read(&ndev->reslock); 2823 return err; 2824 } 2825 2826 static void mlx5_vdpa_cvq_suspend(struct mlx5_vdpa_dev *mvdev) 2827 { 2828 struct mlx5_control_vq *cvq; 2829 2830 if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 2831 return; 2832 2833 cvq = &mvdev->cvq; 2834 cvq->ready = false; 2835 } 2836 2837 static int mlx5_vdpa_suspend(struct vdpa_device *vdev) 2838 { 2839 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2840 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2841 struct mlx5_vdpa_virtqueue *mvq; 2842 int i; 2843 2844 down_write(&ndev->reslock); 2845 ndev->nb_registered = false; 2846 mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); 2847 flush_workqueue(ndev->mvdev.wq); 2848 for (i = 0; i < ndev->cur_num_vqs; i++) { 2849 mvq = &ndev->vqs[i]; 2850 suspend_vq(ndev, mvq); 2851 } 2852 mlx5_vdpa_cvq_suspend(mvdev); 2853 up_write(&ndev->reslock); 2854 return 0; 2855 } 2856 2857 static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group, 2858 unsigned int asid) 2859 { 2860 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2861 2862 if (group >= MLX5_VDPA_NUMVQ_GROUPS) 2863 return -EINVAL; 2864 2865 mvdev->group2asid[group] = asid; 2866 return 0; 2867 } 2868 2869 static const struct vdpa_config_ops mlx5_vdpa_ops = { 2870 .set_vq_address = mlx5_vdpa_set_vq_address, 2871 .set_vq_num = mlx5_vdpa_set_vq_num, 2872 .kick_vq = mlx5_vdpa_kick_vq, 2873 .set_vq_cb = mlx5_vdpa_set_vq_cb, 2874 .set_vq_ready = mlx5_vdpa_set_vq_ready, 2875 .get_vq_ready = mlx5_vdpa_get_vq_ready, 2876 .set_vq_state = mlx5_vdpa_set_vq_state, 2877 .get_vq_state = mlx5_vdpa_get_vq_state, 2878 .get_vendor_vq_stats = mlx5_vdpa_get_vendor_vq_stats, 2879 .get_vq_notification = mlx5_get_vq_notification, 2880 .get_vq_irq = mlx5_get_vq_irq, 2881 .get_vq_align = mlx5_vdpa_get_vq_align, 2882 .get_vq_group = mlx5_vdpa_get_vq_group, 2883 .get_device_features = mlx5_vdpa_get_device_features, 2884 .set_driver_features = mlx5_vdpa_set_driver_features, 2885 .get_driver_features = mlx5_vdpa_get_driver_features, 2886 .set_config_cb = mlx5_vdpa_set_config_cb, 2887 .get_vq_num_max = mlx5_vdpa_get_vq_num_max, 2888 .get_device_id = mlx5_vdpa_get_device_id, 2889 .get_vendor_id = mlx5_vdpa_get_vendor_id, 2890 .get_status = mlx5_vdpa_get_status, 2891 .set_status = mlx5_vdpa_set_status, 2892 .reset = mlx5_vdpa_reset, 2893 .get_config_size = mlx5_vdpa_get_config_size, 2894 .get_config = mlx5_vdpa_get_config, 2895 .set_config = mlx5_vdpa_set_config, 2896 .get_generation = mlx5_vdpa_get_generation, 2897 .set_map = mlx5_vdpa_set_map, 2898 .set_group_asid = mlx5_set_group_asid, 2899 .free = mlx5_vdpa_free, 2900 .suspend = mlx5_vdpa_suspend, 2901 }; 2902 2903 static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu) 2904 { 2905 u16 hw_mtu; 2906 int err; 2907 2908 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 2909 if (err) 2910 return err; 2911 2912 *mtu = hw_mtu - MLX5V_ETH_HARD_MTU; 2913 return 0; 2914 } 2915 2916 static int alloc_resources(struct mlx5_vdpa_net *ndev) 2917 { 2918 struct mlx5_vdpa_net_resources *res = &ndev->res; 2919 int err; 2920 2921 if (res->valid) { 2922 mlx5_vdpa_warn(&ndev->mvdev, "resources already allocated\n"); 2923 return -EEXIST; 2924 } 2925 2926 err = mlx5_vdpa_alloc_transport_domain(&ndev->mvdev, &res->tdn); 2927 if (err) 2928 return err; 2929 2930 err = create_tis(ndev); 2931 if (err) 2932 goto err_tis; 2933 2934 res->valid = true; 2935 2936 return 0; 2937 2938 err_tis: 2939 mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 2940 return err; 2941 } 2942 2943 static void free_resources(struct mlx5_vdpa_net *ndev) 2944 { 2945 struct mlx5_vdpa_net_resources *res = &ndev->res; 2946 2947 if (!res->valid) 2948 return; 2949 2950 destroy_tis(ndev); 2951 mlx5_vdpa_dealloc_transport_domain(&ndev->mvdev, res->tdn); 2952 res->valid = false; 2953 } 2954 2955 static void init_mvqs(struct mlx5_vdpa_net *ndev) 2956 { 2957 struct mlx5_vdpa_virtqueue *mvq; 2958 int i; 2959 2960 for (i = 0; i < ndev->mvdev.max_vqs; ++i) { 2961 mvq = &ndev->vqs[i]; 2962 memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 2963 mvq->index = i; 2964 mvq->ndev = ndev; 2965 mvq->fwqp.fw = true; 2966 mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE; 2967 } 2968 for (; i < ndev->mvdev.max_vqs; i++) { 2969 mvq = &ndev->vqs[i]; 2970 memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri)); 2971 mvq->index = i; 2972 mvq->ndev = ndev; 2973 } 2974 } 2975 2976 struct mlx5_vdpa_mgmtdev { 2977 struct vdpa_mgmt_dev mgtdev; 2978 struct mlx5_adev *madev; 2979 struct mlx5_vdpa_net *ndev; 2980 }; 2981 2982 static u8 query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport) 2983 { 2984 u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {}; 2985 u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {}; 2986 int err; 2987 2988 MLX5_SET(query_vport_state_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_STATE); 2989 MLX5_SET(query_vport_state_in, in, op_mod, opmod); 2990 MLX5_SET(query_vport_state_in, in, vport_number, vport); 2991 if (vport) 2992 MLX5_SET(query_vport_state_in, in, other_vport, 1); 2993 2994 err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out); 2995 if (err) 2996 return 0; 2997 2998 return MLX5_GET(query_vport_state_out, out, state); 2999 } 3000 3001 static bool get_link_state(struct mlx5_vdpa_dev *mvdev) 3002 { 3003 if (query_vport_state(mvdev->mdev, MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 0) == 3004 VPORT_STATE_UP) 3005 return true; 3006 3007 return false; 3008 } 3009 3010 static void update_carrier(struct work_struct *work) 3011 { 3012 struct mlx5_vdpa_wq_ent *wqent; 3013 struct mlx5_vdpa_dev *mvdev; 3014 struct mlx5_vdpa_net *ndev; 3015 3016 wqent = container_of(work, struct mlx5_vdpa_wq_ent, work); 3017 mvdev = wqent->mvdev; 3018 ndev = to_mlx5_vdpa_ndev(mvdev); 3019 if (get_link_state(mvdev)) 3020 ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 3021 else 3022 ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); 3023 3024 if (ndev->nb_registered && ndev->config_cb.callback) 3025 ndev->config_cb.callback(ndev->config_cb.private); 3026 3027 kfree(wqent); 3028 } 3029 3030 static int event_handler(struct notifier_block *nb, unsigned long event, void *param) 3031 { 3032 struct mlx5_vdpa_net *ndev = container_of(nb, struct mlx5_vdpa_net, nb); 3033 struct mlx5_eqe *eqe = param; 3034 int ret = NOTIFY_DONE; 3035 struct mlx5_vdpa_wq_ent *wqent; 3036 3037 if (event == MLX5_EVENT_TYPE_PORT_CHANGE) { 3038 switch (eqe->sub_type) { 3039 case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 3040 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 3041 wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC); 3042 if (!wqent) 3043 return NOTIFY_DONE; 3044 3045 wqent->mvdev = &ndev->mvdev; 3046 INIT_WORK(&wqent->work, update_carrier); 3047 queue_work(ndev->mvdev.wq, &wqent->work); 3048 ret = NOTIFY_OK; 3049 break; 3050 default: 3051 return NOTIFY_DONE; 3052 } 3053 return ret; 3054 } 3055 return ret; 3056 } 3057 3058 static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu) 3059 { 3060 int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in); 3061 void *in; 3062 int err; 3063 3064 in = kvzalloc(inlen, GFP_KERNEL); 3065 if (!in) 3066 return -ENOMEM; 3067 3068 MLX5_SET(modify_nic_vport_context_in, in, field_select.mtu, 1); 3069 MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.mtu, 3070 mtu + MLX5V_ETH_HARD_MTU); 3071 MLX5_SET(modify_nic_vport_context_in, in, opcode, 3072 MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT); 3073 3074 err = mlx5_cmd_exec_in(mdev, modify_nic_vport_context, in); 3075 3076 kvfree(in); 3077 return err; 3078 } 3079 3080 static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, 3081 const struct vdpa_dev_set_config *add_config) 3082 { 3083 struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 3084 struct virtio_net_config *config; 3085 struct mlx5_core_dev *pfmdev; 3086 struct mlx5_vdpa_dev *mvdev; 3087 struct mlx5_vdpa_net *ndev; 3088 struct mlx5_core_dev *mdev; 3089 u32 max_vqs; 3090 u16 mtu; 3091 int err; 3092 3093 if (mgtdev->ndev) 3094 return -ENOSPC; 3095 3096 mdev = mgtdev->madev->mdev; 3097 if (!(MLX5_CAP_DEV_VDPA_EMULATION(mdev, virtio_queue_type) & 3098 MLX5_VIRTIO_EMULATION_CAP_VIRTIO_QUEUE_TYPE_SPLIT)) { 3099 dev_warn(mdev->device, "missing support for split virtqueues\n"); 3100 return -EOPNOTSUPP; 3101 } 3102 3103 max_vqs = min_t(int, MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues), 3104 1 << MLX5_CAP_GEN(mdev, log_max_rqt_size)); 3105 if (max_vqs < 2) { 3106 dev_warn(mdev->device, 3107 "%d virtqueues are supported. At least 2 are required\n", 3108 max_vqs); 3109 return -EAGAIN; 3110 } 3111 3112 if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP)) { 3113 if (add_config->net.max_vq_pairs > max_vqs / 2) 3114 return -EINVAL; 3115 max_vqs = min_t(u32, max_vqs, 2 * add_config->net.max_vq_pairs); 3116 } else { 3117 max_vqs = 2; 3118 } 3119 3120 ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops, 3121 MLX5_VDPA_NUMVQ_GROUPS, MLX5_VDPA_NUM_AS, name, false); 3122 if (IS_ERR(ndev)) 3123 return PTR_ERR(ndev); 3124 3125 ndev->mvdev.mlx_features = mgtdev->mgtdev.supported_features; 3126 ndev->mvdev.max_vqs = max_vqs; 3127 mvdev = &ndev->mvdev; 3128 mvdev->mdev = mdev; 3129 3130 ndev->vqs = kcalloc(max_vqs, sizeof(*ndev->vqs), GFP_KERNEL); 3131 ndev->event_cbs = kcalloc(max_vqs + 1, sizeof(*ndev->event_cbs), GFP_KERNEL); 3132 if (!ndev->vqs || !ndev->event_cbs) { 3133 err = -ENOMEM; 3134 goto err_alloc; 3135 } 3136 3137 init_mvqs(ndev); 3138 init_rwsem(&ndev->reslock); 3139 config = &ndev->config; 3140 3141 if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU)) { 3142 err = config_func_mtu(mdev, add_config->net.mtu); 3143 if (err) 3144 goto err_alloc; 3145 } 3146 3147 err = query_mtu(mdev, &mtu); 3148 if (err) 3149 goto err_alloc; 3150 3151 ndev->config.mtu = cpu_to_mlx5vdpa16(mvdev, mtu); 3152 3153 if (get_link_state(mvdev)) 3154 ndev->config.status |= cpu_to_mlx5vdpa16(mvdev, VIRTIO_NET_S_LINK_UP); 3155 else 3156 ndev->config.status &= cpu_to_mlx5vdpa16(mvdev, ~VIRTIO_NET_S_LINK_UP); 3157 3158 if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR)) { 3159 memcpy(ndev->config.mac, add_config->net.mac, ETH_ALEN); 3160 } else { 3161 err = mlx5_query_nic_vport_mac_address(mdev, 0, 0, config->mac); 3162 if (err) 3163 goto err_alloc; 3164 } 3165 3166 if (!is_zero_ether_addr(config->mac)) { 3167 pfmdev = pci_get_drvdata(pci_physfn(mdev->pdev)); 3168 err = mlx5_mpfs_add_mac(pfmdev, config->mac); 3169 if (err) 3170 goto err_alloc; 3171 3172 ndev->mvdev.mlx_features |= BIT_ULL(VIRTIO_NET_F_MAC); 3173 } 3174 3175 config->max_virtqueue_pairs = cpu_to_mlx5vdpa16(mvdev, max_vqs / 2); 3176 mvdev->vdev.dma_dev = &mdev->pdev->dev; 3177 err = mlx5_vdpa_alloc_resources(&ndev->mvdev); 3178 if (err) 3179 goto err_mpfs; 3180 3181 if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { 3182 err = mlx5_vdpa_create_mr(mvdev, NULL, 0); 3183 if (err) 3184 goto err_res; 3185 } 3186 3187 err = alloc_resources(ndev); 3188 if (err) 3189 goto err_mr; 3190 3191 ndev->cvq_ent.mvdev = mvdev; 3192 INIT_WORK(&ndev->cvq_ent.work, mlx5_cvq_kick_handler); 3193 mvdev->wq = create_singlethread_workqueue("mlx5_vdpa_wq"); 3194 if (!mvdev->wq) { 3195 err = -ENOMEM; 3196 goto err_res2; 3197 } 3198 3199 ndev->nb.notifier_call = event_handler; 3200 mlx5_notifier_register(mdev, &ndev->nb); 3201 ndev->nb_registered = true; 3202 mvdev->vdev.mdev = &mgtdev->mgtdev; 3203 err = _vdpa_register_device(&mvdev->vdev, max_vqs + 1); 3204 if (err) 3205 goto err_reg; 3206 3207 mlx5_vdpa_add_debugfs(ndev); 3208 mgtdev->ndev = ndev; 3209 return 0; 3210 3211 err_reg: 3212 destroy_workqueue(mvdev->wq); 3213 err_res2: 3214 free_resources(ndev); 3215 err_mr: 3216 mlx5_vdpa_destroy_mr(mvdev); 3217 err_res: 3218 mlx5_vdpa_free_resources(&ndev->mvdev); 3219 err_mpfs: 3220 if (!is_zero_ether_addr(config->mac)) 3221 mlx5_mpfs_del_mac(pfmdev, config->mac); 3222 err_alloc: 3223 put_device(&mvdev->vdev.dev); 3224 return err; 3225 } 3226 3227 static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *dev) 3228 { 3229 struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev); 3230 struct mlx5_vdpa_dev *mvdev = to_mvdev(dev); 3231 struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 3232 struct workqueue_struct *wq; 3233 3234 mlx5_vdpa_remove_debugfs(ndev->debugfs); 3235 ndev->debugfs = NULL; 3236 if (ndev->nb_registered) { 3237 ndev->nb_registered = false; 3238 mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); 3239 } 3240 wq = mvdev->wq; 3241 mvdev->wq = NULL; 3242 destroy_workqueue(wq); 3243 _vdpa_unregister_device(dev); 3244 mgtdev->ndev = NULL; 3245 } 3246 3247 static const struct vdpa_mgmtdev_ops mdev_ops = { 3248 .dev_add = mlx5_vdpa_dev_add, 3249 .dev_del = mlx5_vdpa_dev_del, 3250 }; 3251 3252 static struct virtio_device_id id_table[] = { 3253 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 3254 { 0 }, 3255 }; 3256 3257 static int mlx5v_probe(struct auxiliary_device *adev, 3258 const struct auxiliary_device_id *id) 3259 3260 { 3261 struct mlx5_adev *madev = container_of(adev, struct mlx5_adev, adev); 3262 struct mlx5_core_dev *mdev = madev->mdev; 3263 struct mlx5_vdpa_mgmtdev *mgtdev; 3264 int err; 3265 3266 mgtdev = kzalloc(sizeof(*mgtdev), GFP_KERNEL); 3267 if (!mgtdev) 3268 return -ENOMEM; 3269 3270 mgtdev->mgtdev.ops = &mdev_ops; 3271 mgtdev->mgtdev.device = mdev->device; 3272 mgtdev->mgtdev.id_table = id_table; 3273 mgtdev->mgtdev.config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) | 3274 BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP) | 3275 BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU); 3276 mgtdev->mgtdev.max_supported_vqs = 3277 MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues) + 1; 3278 mgtdev->mgtdev.supported_features = get_supported_features(mdev); 3279 mgtdev->madev = madev; 3280 3281 err = vdpa_mgmtdev_register(&mgtdev->mgtdev); 3282 if (err) 3283 goto reg_err; 3284 3285 auxiliary_set_drvdata(adev, mgtdev); 3286 3287 return 0; 3288 3289 reg_err: 3290 kfree(mgtdev); 3291 return err; 3292 } 3293 3294 static void mlx5v_remove(struct auxiliary_device *adev) 3295 { 3296 struct mlx5_vdpa_mgmtdev *mgtdev; 3297 3298 mgtdev = auxiliary_get_drvdata(adev); 3299 vdpa_mgmtdev_unregister(&mgtdev->mgtdev); 3300 kfree(mgtdev); 3301 } 3302 3303 static const struct auxiliary_device_id mlx5v_id_table[] = { 3304 { .name = MLX5_ADEV_NAME ".vnet", }, 3305 {}, 3306 }; 3307 3308 MODULE_DEVICE_TABLE(auxiliary, mlx5v_id_table); 3309 3310 static struct auxiliary_driver mlx5v_driver = { 3311 .name = "vnet", 3312 .probe = mlx5v_probe, 3313 .remove = mlx5v_remove, 3314 .id_table = mlx5v_id_table, 3315 }; 3316 3317 module_auxiliary_driver(mlx5v_driver); 3318