1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ 2 3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */ 4 /* Kai Shen <kaishen@linux.alibaba.com> */ 5 /* Copyright (c) 2020-2022, Alibaba Group. */ 6 7 #ifndef __ERDMA_H__ 8 #define __ERDMA_H__ 9 10 #include <linux/bitfield.h> 11 #include <linux/netdevice.h> 12 #include <linux/pci.h> 13 #include <linux/xarray.h> 14 #include <rdma/ib_verbs.h> 15 16 #include "erdma_hw.h" 17 18 #define DRV_MODULE_NAME "erdma" 19 #define ERDMA_NODE_DESC "Elastic RDMA Adapter stack" 20 21 struct erdma_eq { 22 void *qbuf; 23 dma_addr_t qbuf_dma_addr; 24 25 spinlock_t lock; 26 27 u32 depth; 28 29 u16 ci; 30 u16 rsvd; 31 32 atomic64_t event_num; 33 atomic64_t notify_num; 34 35 void __iomem *db; 36 u64 *dbrec; 37 dma_addr_t dbrec_dma; 38 }; 39 40 struct erdma_cmdq_sq { 41 void *qbuf; 42 dma_addr_t qbuf_dma_addr; 43 44 spinlock_t lock; 45 46 u32 depth; 47 u16 ci; 48 u16 pi; 49 50 u16 wqebb_cnt; 51 52 u64 *dbrec; 53 dma_addr_t dbrec_dma; 54 }; 55 56 struct erdma_cmdq_cq { 57 void *qbuf; 58 dma_addr_t qbuf_dma_addr; 59 60 spinlock_t lock; 61 62 u32 depth; 63 u32 ci; 64 u32 cmdsn; 65 66 u64 *dbrec; 67 dma_addr_t dbrec_dma; 68 69 atomic64_t armed_num; 70 }; 71 72 enum { 73 ERDMA_CMD_STATUS_INIT, 74 ERDMA_CMD_STATUS_ISSUED, 75 ERDMA_CMD_STATUS_FINISHED, 76 ERDMA_CMD_STATUS_TIMEOUT 77 }; 78 79 struct erdma_comp_wait { 80 struct completion wait_event; 81 u32 cmd_status; 82 u32 ctx_id; 83 u16 sq_pi; 84 u8 comp_status; 85 u8 rsvd; 86 u32 comp_data[4]; 87 }; 88 89 enum { 90 ERDMA_CMDQ_STATE_OK_BIT = 0, 91 ERDMA_CMDQ_STATE_TIMEOUT_BIT = 1, 92 ERDMA_CMDQ_STATE_CTX_ERR_BIT = 2, 93 }; 94 95 #define ERDMA_CMDQ_TIMEOUT_MS 15000 96 #define ERDMA_REG_ACCESS_WAIT_MS 20 97 #define ERDMA_WAIT_DEV_DONE_CNT 500 98 99 struct erdma_cmdq { 100 unsigned long *comp_wait_bitmap; 101 struct erdma_comp_wait *wait_pool; 102 spinlock_t lock; 103 104 struct erdma_cmdq_sq sq; 105 struct erdma_cmdq_cq cq; 106 struct erdma_eq eq; 107 108 unsigned long state; 109 110 struct semaphore credits; 111 u16 max_outstandings; 112 }; 113 114 #define COMPROMISE_CC ERDMA_CC_CUBIC 115 enum erdma_cc_alg { 116 ERDMA_CC_NEWRENO = 0, 117 ERDMA_CC_CUBIC, 118 ERDMA_CC_HPCC_RTT, 119 ERDMA_CC_HPCC_ECN, 120 ERDMA_CC_HPCC_INT, 121 ERDMA_CC_METHODS_NUM 122 }; 123 124 struct erdma_devattr { 125 u32 fw_version; 126 127 unsigned char peer_addr[ETH_ALEN]; 128 unsigned long cap_flags; 129 130 enum erdma_cc_alg cc; 131 u32 irq_num; 132 133 u32 max_qp; 134 u32 max_send_wr; 135 u32 max_recv_wr; 136 u32 max_ord; 137 u32 max_ird; 138 139 u32 max_send_sge; 140 u32 max_recv_sge; 141 u32 max_sge_rd; 142 u32 max_cq; 143 u32 max_cqe; 144 u64 max_mr_size; 145 u32 max_mr; 146 u32 max_pd; 147 u32 max_mw; 148 u32 max_gid; 149 u32 max_ah; 150 u32 local_dma_key; 151 }; 152 153 #define ERDMA_IRQNAME_SIZE 50 154 155 struct erdma_irq { 156 char name[ERDMA_IRQNAME_SIZE]; 157 u32 msix_vector; 158 cpumask_t affinity_hint_mask; 159 }; 160 161 struct erdma_eq_cb { 162 bool ready; 163 void *dev; /* All EQs use this fields to get erdma_dev struct */ 164 struct erdma_irq irq; 165 struct erdma_eq eq; 166 struct tasklet_struct tasklet; 167 }; 168 169 struct erdma_resource_cb { 170 unsigned long *bitmap; 171 spinlock_t lock; 172 u32 next_alloc_idx; 173 u32 max_cap; 174 }; 175 176 enum { 177 ERDMA_RES_TYPE_PD = 0, 178 ERDMA_RES_TYPE_STAG_IDX = 1, 179 ERDMA_RES_TYPE_AH = 2, 180 ERDMA_RES_CNT = 3, 181 }; 182 183 struct erdma_dev { 184 struct ib_device ibdev; 185 struct net_device *netdev; 186 struct pci_dev *pdev; 187 struct notifier_block netdev_nb; 188 struct workqueue_struct *reflush_wq; 189 190 resource_size_t func_bar_addr; 191 resource_size_t func_bar_len; 192 u8 __iomem *func_bar; 193 194 struct erdma_devattr attrs; 195 u32 mtu; 196 197 /* cmdq and aeq use the same msix vector */ 198 struct erdma_irq comm_irq; 199 struct erdma_cmdq cmdq; 200 struct erdma_eq aeq; 201 struct erdma_eq_cb ceqs[ERDMA_NUM_MSIX_VEC - 1]; 202 203 spinlock_t lock; 204 struct erdma_resource_cb res_cb[ERDMA_RES_CNT]; 205 struct xarray qp_xa; 206 struct xarray cq_xa; 207 208 u32 next_alloc_qpn; 209 u32 next_alloc_cqn; 210 211 atomic_t num_ctx; 212 struct list_head cep_list; 213 214 struct dma_pool *db_pool; 215 struct dma_pool *resp_pool; 216 enum erdma_proto_type proto; 217 }; 218 219 static inline void *get_queue_entry(void *qbuf, u32 idx, u32 depth, u32 shift) 220 { 221 idx &= (depth - 1); 222 223 return qbuf + (idx << shift); 224 } 225 226 static inline struct erdma_dev *to_edev(struct ib_device *ibdev) 227 { 228 return container_of(ibdev, struct erdma_dev, ibdev); 229 } 230 231 static inline u32 erdma_reg_read32(struct erdma_dev *dev, u32 reg) 232 { 233 return readl(dev->func_bar + reg); 234 } 235 236 static inline u64 erdma_reg_read64(struct erdma_dev *dev, u32 reg) 237 { 238 return readq(dev->func_bar + reg); 239 } 240 241 static inline void erdma_reg_write32(struct erdma_dev *dev, u32 reg, u32 value) 242 { 243 writel(value, dev->func_bar + reg); 244 } 245 246 static inline void erdma_reg_write64(struct erdma_dev *dev, u32 reg, u64 value) 247 { 248 writeq(value, dev->func_bar + reg); 249 } 250 251 static inline u32 erdma_reg_read32_filed(struct erdma_dev *dev, u32 reg, 252 u32 filed_mask) 253 { 254 u32 val = erdma_reg_read32(dev, reg); 255 256 return FIELD_GET(filed_mask, val); 257 } 258 259 #define ERDMA_GET(val, name) FIELD_GET(ERDMA_CMD_##name##_MASK, val) 260 261 int erdma_cmdq_init(struct erdma_dev *dev); 262 void erdma_finish_cmdq_init(struct erdma_dev *dev); 263 void erdma_cmdq_destroy(struct erdma_dev *dev); 264 265 void erdma_cmdq_build_reqhdr(u64 *hdr, u32 mod, u32 op); 266 int erdma_post_cmd_wait(struct erdma_cmdq *cmdq, void *req, u32 req_size, 267 u64 *resp0, u64 *resp1, bool sleepable); 268 void erdma_cmdq_completion_handler(struct erdma_cmdq *cmdq); 269 270 int erdma_ceqs_init(struct erdma_dev *dev); 271 void erdma_ceqs_uninit(struct erdma_dev *dev); 272 void notify_eq(struct erdma_eq *eq); 273 void *get_next_valid_eqe(struct erdma_eq *eq); 274 275 int erdma_aeq_init(struct erdma_dev *dev); 276 int erdma_eq_common_init(struct erdma_dev *dev, struct erdma_eq *eq, u32 depth); 277 void erdma_eq_destroy(struct erdma_dev *dev, struct erdma_eq *eq); 278 279 void erdma_aeq_event_handler(struct erdma_dev *dev); 280 void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb); 281 282 #endif 283