1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ 2 3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */ 4 /* Kai Shen <kaishen@linux.alibaba.com> */ 5 /* Copyright (c) 2020-2022, Alibaba Group. */ 6 7 #ifndef __ERDMA_VERBS_H__ 8 #define __ERDMA_VERBS_H__ 9 10 #include "erdma.h" 11 12 /* RDMA Capability. */ 13 #define ERDMA_MAX_PD (128 * 1024) 14 #define ERDMA_MAX_SEND_WR 8192 15 #define ERDMA_MAX_ORD 128 16 #define ERDMA_MAX_IRD 128 17 #define ERDMA_MAX_SGE_RD 1 18 #define ERDMA_MAX_CONTEXT (128 * 1024) 19 #define ERDMA_MAX_SEND_SGE 6 20 #define ERDMA_MAX_RECV_SGE 1 21 #define ERDMA_MAX_INLINE (sizeof(struct erdma_sge) * (ERDMA_MAX_SEND_SGE)) 22 #define ERDMA_MAX_FRMR_PA 512 23 24 enum { 25 ERDMA_MMAP_IO_NC = 0, /* no cache */ 26 }; 27 28 struct erdma_user_mmap_entry { 29 struct rdma_user_mmap_entry rdma_entry; 30 u64 address; 31 u8 mmap_flag; 32 }; 33 34 struct erdma_ext_db_info { 35 bool enable; 36 u16 sdb_off; 37 u16 rdb_off; 38 u16 cdb_off; 39 }; 40 41 struct erdma_ucontext { 42 struct ib_ucontext ibucontext; 43 44 struct erdma_ext_db_info ext_db; 45 46 u64 sdb; 47 u64 rdb; 48 u64 cdb; 49 50 struct rdma_user_mmap_entry *sq_db_mmap_entry; 51 struct rdma_user_mmap_entry *rq_db_mmap_entry; 52 struct rdma_user_mmap_entry *cq_db_mmap_entry; 53 54 /* doorbell records */ 55 struct list_head dbrecords_page_list; 56 struct mutex dbrecords_page_mutex; 57 }; 58 59 struct erdma_pd { 60 struct ib_pd ibpd; 61 u32 pdn; 62 }; 63 64 /* 65 * MemoryRegion definition. 66 */ 67 #define ERDMA_MAX_INLINE_MTT_ENTRIES 4 68 #define MTT_SIZE(mtt_cnt) ((mtt_cnt) << 3) /* per mtt entry takes 8 Bytes. */ 69 #define ERDMA_MR_MAX_MTT_CNT 524288 70 #define ERDMA_MTT_ENTRY_SIZE 8 71 72 #define ERDMA_MR_TYPE_NORMAL 0 73 #define ERDMA_MR_TYPE_FRMR 1 74 #define ERDMA_MR_TYPE_DMA 2 75 76 #define ERDMA_MR_MTT_0LEVEL 0 77 #define ERDMA_MR_MTT_1LEVEL 1 78 79 #define ERDMA_MR_ACC_RA BIT(0) 80 #define ERDMA_MR_ACC_LR BIT(1) 81 #define ERDMA_MR_ACC_LW BIT(2) 82 #define ERDMA_MR_ACC_RR BIT(3) 83 #define ERDMA_MR_ACC_RW BIT(4) 84 85 static inline u8 to_erdma_access_flags(int access) 86 { 87 return (access & IB_ACCESS_REMOTE_READ ? ERDMA_MR_ACC_RR : 0) | 88 (access & IB_ACCESS_LOCAL_WRITE ? ERDMA_MR_ACC_LW : 0) | 89 (access & IB_ACCESS_REMOTE_WRITE ? ERDMA_MR_ACC_RW : 0) | 90 (access & IB_ACCESS_REMOTE_ATOMIC ? ERDMA_MR_ACC_RA : 0); 91 } 92 93 /* Hierarchical storage structure for MTT entries */ 94 struct erdma_mtt { 95 u64 *buf; 96 size_t size; 97 98 bool continuous; 99 union { 100 dma_addr_t buf_dma; 101 struct { 102 struct scatterlist *sglist; 103 u32 nsg; 104 u32 level; 105 }; 106 }; 107 108 struct erdma_mtt *low_level; 109 }; 110 111 struct erdma_mem { 112 struct ib_umem *umem; 113 struct erdma_mtt *mtt; 114 115 u32 page_size; 116 u32 page_offset; 117 u32 page_cnt; 118 u32 mtt_nents; 119 120 u64 va; 121 u64 len; 122 }; 123 124 struct erdma_mr { 125 struct ib_mr ibmr; 126 struct erdma_mem mem; 127 u8 type; 128 u8 access; 129 u8 valid; 130 }; 131 132 struct erdma_user_dbrecords_page { 133 struct list_head list; 134 struct ib_umem *umem; 135 u64 va; 136 int refcnt; 137 }; 138 139 struct erdma_av { 140 u8 port; 141 u8 hop_limit; 142 u8 traffic_class; 143 u8 sl; 144 u8 sgid_index; 145 u16 udp_sport; 146 u32 flow_label; 147 u8 dmac[ETH_ALEN]; 148 u8 dgid[ERDMA_ROCEV2_GID_SIZE]; 149 enum erdma_network_type ntype; 150 }; 151 152 struct erdma_ah { 153 struct ib_ah ibah; 154 struct erdma_av av; 155 u32 ahn; 156 }; 157 158 struct erdma_uqp { 159 struct erdma_mem sq_mem; 160 struct erdma_mem rq_mem; 161 162 dma_addr_t sq_dbrec_dma; 163 dma_addr_t rq_dbrec_dma; 164 165 struct erdma_user_dbrecords_page *user_dbr_page; 166 167 u32 rq_offset; 168 }; 169 170 struct erdma_kqp { 171 u16 sq_pi; 172 u16 sq_ci; 173 174 u16 rq_pi; 175 u16 rq_ci; 176 177 u64 *swr_tbl; 178 u64 *rwr_tbl; 179 180 void __iomem *hw_sq_db; 181 void __iomem *hw_rq_db; 182 183 void *sq_buf; 184 dma_addr_t sq_buf_dma_addr; 185 186 void *rq_buf; 187 dma_addr_t rq_buf_dma_addr; 188 189 void *sq_dbrec; 190 void *rq_dbrec; 191 192 dma_addr_t sq_dbrec_dma; 193 dma_addr_t rq_dbrec_dma; 194 195 u8 sig_all; 196 }; 197 198 enum erdma_qps_iwarp { 199 ERDMA_QPS_IWARP_IDLE = 0, 200 ERDMA_QPS_IWARP_RTR = 1, 201 ERDMA_QPS_IWARP_RTS = 2, 202 ERDMA_QPS_IWARP_CLOSING = 3, 203 ERDMA_QPS_IWARP_TERMINATE = 4, 204 ERDMA_QPS_IWARP_ERROR = 5, 205 ERDMA_QPS_IWARP_UNDEF = 6, 206 ERDMA_QPS_IWARP_COUNT = 7, 207 }; 208 209 enum erdma_qpa_mask_iwarp { 210 ERDMA_QPA_IWARP_STATE = (1 << 0), 211 ERDMA_QPA_IWARP_LLP_HANDLE = (1 << 2), 212 ERDMA_QPA_IWARP_ORD = (1 << 3), 213 ERDMA_QPA_IWARP_IRD = (1 << 4), 214 ERDMA_QPA_IWARP_SQ_SIZE = (1 << 5), 215 ERDMA_QPA_IWARP_RQ_SIZE = (1 << 6), 216 ERDMA_QPA_IWARP_MPA = (1 << 7), 217 ERDMA_QPA_IWARP_CC = (1 << 8), 218 }; 219 220 enum erdma_qps_rocev2 { 221 ERDMA_QPS_ROCEV2_RESET = 0, 222 ERDMA_QPS_ROCEV2_INIT = 1, 223 ERDMA_QPS_ROCEV2_RTR = 2, 224 ERDMA_QPS_ROCEV2_RTS = 3, 225 ERDMA_QPS_ROCEV2_SQD = 4, 226 ERDMA_QPS_ROCEV2_SQE = 5, 227 ERDMA_QPS_ROCEV2_ERROR = 6, 228 ERDMA_QPS_ROCEV2_COUNT = 7, 229 }; 230 231 enum erdma_qpa_mask_rocev2 { 232 ERDMA_QPA_ROCEV2_STATE = (1 << 0), 233 ERDMA_QPA_ROCEV2_QKEY = (1 << 1), 234 ERDMA_QPA_ROCEV2_AV = (1 << 2), 235 ERDMA_QPA_ROCEV2_SQ_PSN = (1 << 3), 236 ERDMA_QPA_ROCEV2_RQ_PSN = (1 << 4), 237 ERDMA_QPA_ROCEV2_DST_QPN = (1 << 5), 238 }; 239 240 enum erdma_qp_flags { 241 ERDMA_QP_IN_FLUSHING = (1 << 0), 242 }; 243 244 #define ERDMA_QP_ACTIVE 0 245 #define ERDMA_QP_PASSIVE 1 246 247 struct erdma_mod_qp_params_iwarp { 248 enum erdma_qps_iwarp state; 249 enum erdma_cc_alg cc; 250 u8 qp_type; 251 u8 pd_len; 252 u32 irq_size; 253 u32 orq_size; 254 }; 255 256 struct erdma_qp_attrs_iwarp { 257 enum erdma_qps_iwarp state; 258 u32 cookie; 259 }; 260 261 struct erdma_mod_qp_params_rocev2 { 262 enum erdma_qps_rocev2 state; 263 u32 qkey; 264 u32 sq_psn; 265 u32 rq_psn; 266 u32 dst_qpn; 267 struct erdma_av av; 268 }; 269 270 union erdma_mod_qp_params { 271 struct erdma_mod_qp_params_iwarp iwarp; 272 struct erdma_mod_qp_params_rocev2 rocev2; 273 }; 274 275 struct erdma_qp_attrs_rocev2 { 276 enum erdma_qps_rocev2 state; 277 u32 qkey; 278 u32 dst_qpn; 279 struct erdma_av av; 280 }; 281 282 struct erdma_qp_attrs { 283 enum erdma_cc_alg cc; /* Congestion control algorithm */ 284 u32 sq_size; 285 u32 rq_size; 286 u32 orq_size; 287 u32 irq_size; 288 u32 max_send_sge; 289 u32 max_recv_sge; 290 union { 291 struct erdma_qp_attrs_iwarp iwarp; 292 struct erdma_qp_attrs_rocev2 rocev2; 293 }; 294 }; 295 296 struct erdma_qp { 297 struct ib_qp ibqp; 298 struct kref ref; 299 struct completion safe_free; 300 struct erdma_dev *dev; 301 struct erdma_cep *cep; 302 struct rw_semaphore state_lock; 303 304 unsigned long flags; 305 struct delayed_work reflush_dwork; 306 307 union { 308 struct erdma_kqp kern_qp; 309 struct erdma_uqp user_qp; 310 }; 311 312 struct erdma_cq *scq; 313 struct erdma_cq *rcq; 314 315 struct erdma_qp_attrs attrs; 316 spinlock_t lock; 317 }; 318 319 struct erdma_kcq_info { 320 void *qbuf; 321 dma_addr_t qbuf_dma_addr; 322 u32 ci; 323 u32 cmdsn; 324 u32 notify_cnt; 325 326 spinlock_t lock; 327 u8 __iomem *db; 328 u64 *dbrec; 329 dma_addr_t dbrec_dma; 330 }; 331 332 struct erdma_ucq_info { 333 struct erdma_mem qbuf_mem; 334 struct erdma_user_dbrecords_page *user_dbr_page; 335 dma_addr_t dbrec_dma; 336 }; 337 338 struct erdma_cq { 339 struct ib_cq ibcq; 340 u32 cqn; 341 342 u32 depth; 343 u32 assoc_eqn; 344 345 union { 346 struct erdma_kcq_info kern_cq; 347 struct erdma_ucq_info user_cq; 348 }; 349 }; 350 351 #define QP_ID(qp) ((qp)->ibqp.qp_num) 352 353 static inline struct erdma_qp *find_qp_by_qpn(struct erdma_dev *dev, int id) 354 { 355 return (struct erdma_qp *)xa_load(&dev->qp_xa, id); 356 } 357 358 static inline struct erdma_cq *find_cq_by_cqn(struct erdma_dev *dev, int id) 359 { 360 return (struct erdma_cq *)xa_load(&dev->cq_xa, id); 361 } 362 363 void erdma_qp_get(struct erdma_qp *qp); 364 void erdma_qp_put(struct erdma_qp *qp); 365 int erdma_modify_qp_state_iwarp(struct erdma_qp *qp, 366 struct erdma_mod_qp_params_iwarp *params, 367 int mask); 368 int erdma_modify_qp_state_rocev2(struct erdma_qp *qp, 369 struct erdma_mod_qp_params_rocev2 *params, 370 int attr_mask); 371 void erdma_qp_llp_close(struct erdma_qp *qp); 372 void erdma_qp_cm_drop(struct erdma_qp *qp); 373 374 static inline bool erdma_device_iwarp(struct erdma_dev *dev) 375 { 376 return dev->proto == ERDMA_PROTO_IWARP; 377 } 378 379 static inline bool erdma_device_rocev2(struct erdma_dev *dev) 380 { 381 return dev->proto == ERDMA_PROTO_ROCEV2; 382 } 383 384 static inline struct erdma_ucontext *to_ectx(struct ib_ucontext *ibctx) 385 { 386 return container_of(ibctx, struct erdma_ucontext, ibucontext); 387 } 388 389 static inline struct erdma_pd *to_epd(struct ib_pd *pd) 390 { 391 return container_of(pd, struct erdma_pd, ibpd); 392 } 393 394 static inline struct erdma_mr *to_emr(struct ib_mr *ibmr) 395 { 396 return container_of(ibmr, struct erdma_mr, ibmr); 397 } 398 399 static inline struct erdma_qp *to_eqp(struct ib_qp *qp) 400 { 401 return container_of(qp, struct erdma_qp, ibqp); 402 } 403 404 static inline struct erdma_cq *to_ecq(struct ib_cq *ibcq) 405 { 406 return container_of(ibcq, struct erdma_cq, ibcq); 407 } 408 409 static inline struct erdma_ah *to_eah(struct ib_ah *ibah) 410 { 411 return container_of(ibah, struct erdma_ah, ibah); 412 } 413 414 static inline int erdma_check_gid_attr(const struct ib_gid_attr *attr) 415 { 416 u8 ntype = rdma_gid_attr_network_type(attr); 417 418 if (ntype != RDMA_NETWORK_IPV4 && ntype != RDMA_NETWORK_IPV6) 419 return -EINVAL; 420 421 return 0; 422 } 423 424 static inline struct erdma_user_mmap_entry * 425 to_emmap(struct rdma_user_mmap_entry *ibmmap) 426 { 427 return container_of(ibmmap, struct erdma_user_mmap_entry, rdma_entry); 428 } 429 430 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *data); 431 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx); 432 int erdma_query_device(struct ib_device *dev, struct ib_device_attr *attr, 433 struct ib_udata *data); 434 int erdma_get_port_immutable(struct ib_device *dev, u32 port, 435 struct ib_port_immutable *ib_port_immutable); 436 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 437 struct uverbs_attr_bundle *attrs); 438 int erdma_query_port(struct ib_device *dev, u32 port, 439 struct ib_port_attr *attr); 440 int erdma_query_gid(struct ib_device *dev, u32 port, int idx, 441 union ib_gid *gid); 442 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *data); 443 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata); 444 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, 445 struct ib_udata *data); 446 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, 447 struct ib_qp_init_attr *init_attr); 448 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, 449 struct ib_udata *data); 450 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); 451 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); 452 void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext); 453 int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); 454 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 455 u64 virt, int access, struct ib_udata *udata); 456 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int rights); 457 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *data); 458 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma); 459 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry); 460 void erdma_qp_get_ref(struct ib_qp *ibqp); 461 void erdma_qp_put_ref(struct ib_qp *ibqp); 462 struct ib_qp *erdma_get_ibqp(struct ib_device *dev, int id); 463 int erdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *send_wr, 464 const struct ib_send_wr **bad_send_wr); 465 int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr, 466 const struct ib_recv_wr **bad_recv_wr); 467 int erdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); 468 void erdma_remove_cqes_of_qp(struct ib_cq *ibcq, u32 qpn); 469 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 470 u32 max_num_sg); 471 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 472 unsigned int *sg_offset); 473 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason); 474 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu); 475 struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device, 476 u32 port_num); 477 int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, 478 u32 port, int index); 479 enum rdma_link_layer erdma_get_link_layer(struct ib_device *ibdev, 480 u32 port_num); 481 int erdma_add_gid(const struct ib_gid_attr *attr, void **context); 482 int erdma_del_gid(const struct ib_gid_attr *attr, void **context); 483 int erdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey); 484 void erdma_set_av_cfg(struct erdma_av_cfg *av_cfg, struct erdma_av *av); 485 int erdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr, 486 struct ib_udata *udata); 487 int erdma_destroy_ah(struct ib_ah *ibah, u32 flags); 488 int erdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr); 489 490 #endif 491