1 /* 2 * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term 3 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in 13 * the documentation and/or other materials provided with the 14 * distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * Description: RDMA Controller HW interface (header) 29 */ 30 31 #ifndef __BNXT_QPLIB_RCFW_H__ 32 #define __BNXT_QPLIB_RCFW_H__ 33 34 #include <linux/semaphore.h> 35 #include <linux/module.h> 36 #include <linux/netdevice.h> 37 #include <linux/mutex.h> 38 #include <linux/list.h> 39 #include <linux/rculist.h> 40 #include <linux/spinlock.h> 41 #include <linux/pci.h> 42 #include <net/ipv6.h> 43 #include <linux/if_ether.h> 44 #include <linux/debugfs.h> 45 #include <linux/seq_file.h> 46 #include <linux/interrupt.h> 47 #include <linux/vmalloc.h> 48 #include <linux/delay.h> 49 50 #include "qplib_tlv.h" 51 52 #define RCFW_CMDQ_TRIG_VAL 1 53 #define RCFW_COMM_PCI_BAR_REGION 0 54 #define RCFW_COMM_CONS_PCI_BAR_REGION 2 55 #define RCFW_COMM_BASE_OFFSET 0x600 56 #define RCFW_PF_VF_COMM_PROD_OFFSET 0xc 57 #define RCFW_COMM_TRIG_OFFSET 0x100 58 #define RCFW_COMM_SIZE 0x104 59 60 #define RCFW_DBR_PCI_BAR_REGION 2 61 #define RCFW_DBR_BASE_PAGE_SHIFT 12 62 #define RCFW_MAX_LATENCY_SEC_SLAB_INDEX 128 63 #define RCFW_MAX_LATENCY_MSEC_SLAB_INDEX 3000 64 #define RCFW_MAX_STAT_INDEX 0xFFFF 65 #define RCFW_FW_STALL_MAX_TIMEOUT 40 66 67 #define GET_OPCODE_TYPE(x) \ 68 ((x) == 0x1 ? "CREATE_QP": \ 69 ((x) == 0x2 ? "DESTROY_QP": \ 70 ((x) == 0x3 ? "MODIFY_QP": \ 71 ((x) == 0x4 ? "QUERY_QP": \ 72 ((x) == 0x5 ? "CREATE_SRQ": \ 73 ((x) == 0x6 ? "DESTROY_SRQ": \ 74 ((x) == 0x8 ? "QUERY_SRQ": \ 75 ((x) == 0x9 ? "CREATE_CQ": \ 76 ((x) == 0xa ? "DESTROY_CQ": \ 77 ((x) == 0xc ? "RESIZE_CQ": \ 78 ((x) == 0xd ? "ALLOCATE_MRW": \ 79 ((x) == 0xe ? "DEALLOCATE_KEY": \ 80 ((x) == 0xf ? "REGISTER_MR": \ 81 ((x) == 0x10 ? "DEREGISTER_MR": \ 82 ((x) == 0x11 ? "ADD_GID": \ 83 ((x) == 0x12 ? "DELETE_GID": \ 84 ((x) == 0x17 ? "MODIFY_GID": \ 85 ((x) == 0x18 ? "QUERY_GID": \ 86 ((x) == 0x13 ? "CREATE_QP1": \ 87 ((x) == 0x14 ? "DESTROY_QP1": \ 88 ((x) == 0x15 ? "CREATE_AH": \ 89 ((x) == 0x16 ? "DESTROY_AH": \ 90 ((x) == 0x80 ? "INITIALIZE_FW": \ 91 ((x) == 0x81 ? "DEINITIALIZE_FW": \ 92 ((x) == 0x82 ? "STOP_FUNC": \ 93 ((x) == 0x83 ? "QUERY_FUNC": \ 94 ((x) == 0x84 ? "SET_FUNC_RESOURCES": \ 95 ((x) == 0x85 ? "READ_CONTEXT": \ 96 ((x) == 0x86 ? "VF_BACKCHANNEL_REQUEST": \ 97 ((x) == 0x87 ? "READ_VF_MEMORY": \ 98 ((x) == 0x88 ? "COMPLETE_VF_REQUEST": \ 99 ((x) == 0x89 ? "EXTEND_CONTEXT_ARRRAY": \ 100 ((x) == 0x8a ? "MAP_TC_TO_COS": \ 101 ((x) == 0x8b ? "QUERY_VERSION": \ 102 ((x) == 0x8c ? "MODIFY_ROCE_CC": \ 103 ((x) == 0x8d ? "QUERY_ROCE_CC": \ 104 ((x) == 0x8e ? "QUERY_ROCE_STATS": \ 105 ((x) == 0x8f ? "SET_LINK_AGGR_MODE": \ 106 ((x) == 0x90 ? "MODIFY_CQ": \ 107 ((x) == 0x91 ? "QUERY_QP_EXTEND": \ 108 ((x) == 0x92 ? "QUERY_ROCE_STATS_EXT": \ 109 "Unknown OPCODE" \ 110 ))))))))))))))))))))))))))))))))))))))))) 111 112 extern unsigned int cmdq_shadow_qd; 113 /* Cmdq contains a fix number of a 16-Byte slots */ 114 struct bnxt_qplib_cmdqe { 115 u8 data[16]; 116 }; 117 #define BNXT_QPLIB_CMDQE_UNITS sizeof(struct bnxt_qplib_cmdqe) 118 119 static inline void bnxt_qplib_rcfw_cmd_prep(void *r, u8 opcode, u8 cmd_size) 120 { 121 struct cmdq_base *req = r; 122 123 req->opcode = opcode; 124 req->cmd_size = cmd_size; 125 } 126 127 /* Shadow queue depth for non blocking command */ 128 #define RCFW_CMD_NON_BLOCKING_SHADOW_QD 64 129 #define RCFW_CMD_DEV_ERR_CHECK_TIME_MS 1000 /* 1 Second time out*/ 130 #define RCFW_ERR_RETRY_COUNT (RCFW_CMD_WAIT_TIME_MS / RCFW_CMD_DEV_ERR_CHECK_TIME_MS) 131 132 /* CMDQ elements */ 133 #define BNXT_QPLIB_CMDQE_MAX_CNT 8192 134 #define BNXT_QPLIB_CMDQE_BYTES (BNXT_QPLIB_CMDQE_MAX_CNT * \ 135 BNXT_QPLIB_CMDQE_UNITS) 136 #define BNXT_QPLIB_CMDQE_NPAGES ((BNXT_QPLIB_CMDQE_BYTES % \ 137 PAGE_SIZE) ? \ 138 ((BNXT_QPLIB_CMDQE_BYTES / \ 139 PAGE_SIZE) + 1) : \ 140 (BNXT_QPLIB_CMDQE_BYTES / \ 141 PAGE_SIZE)) 142 #define BNXT_QPLIB_CMDQE_PAGE_SIZE (BNXT_QPLIB_CMDQE_NPAGES * \ 143 PAGE_SIZE) 144 145 #define RCFW_MAX_OUTSTANDING_CMD BNXT_QPLIB_CMDQE_MAX_CNT 146 #define RCFW_MAX_COOKIE_VALUE (BNXT_QPLIB_CMDQE_MAX_CNT - 1) 147 #define RCFW_CMD_IS_BLOCKING 0x8000 148 #define RCFW_NO_FW_ACCESS(rcfw) \ 149 (test_bit(ERR_DEVICE_DETACHED, &(rcfw)->cmdq.flags) || \ 150 pci_channel_offline((rcfw)->pdev)) 151 152 /* Crsq buf is 1024-Byte */ 153 struct bnxt_qplib_crsbe { 154 u8 data[1024]; 155 }; 156 157 /* Get the number of command units required for the req. The 158 * function returns correct value only if called before 159 * setting using bnxt_qplib_set_cmd_slots 160 */ 161 static inline u32 bnxt_qplib_get_cmd_slots(struct cmdq_base *req) 162 { 163 u32 cmd_units = 0; 164 165 if (HAS_TLV_HEADER(req)) { 166 struct roce_tlv *tlv_req = (struct roce_tlv *)req; 167 cmd_units = tlv_req->total_size; 168 } else { 169 cmd_units = (req->cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) / 170 BNXT_QPLIB_CMDQE_UNITS; 171 } 172 return cmd_units; 173 } 174 175 /* Set the cmd_size to a factor of CMDQE unit */ 176 static inline u32 bnxt_qplib_set_cmd_slots(struct cmdq_base *req) 177 { 178 u32 cmd_byte = 0; 179 180 if (HAS_TLV_HEADER(req)) { 181 struct roce_tlv *tlv_req = (struct roce_tlv *)req; 182 cmd_byte = tlv_req->total_size * BNXT_QPLIB_CMDQE_UNITS; 183 } else { 184 cmd_byte = req->cmd_size; 185 req->cmd_size = (req->cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) / 186 BNXT_QPLIB_CMDQE_UNITS; 187 } 188 189 return cmd_byte; 190 } 191 192 /* CREQ */ 193 /* Allocate 1 per QP for async error notification for now */ 194 #define BNXT_QPLIB_CREQE_MAX_CNT (64 * 1024) 195 #define BNXT_QPLIB_CREQE_UNITS 16 /* 16-Bytes per prod unit */ 196 197 #define CREQ_CMP_VALID(hdr, pass) \ 198 (!!((hdr)->v & CREQ_BASE_V) == \ 199 !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK)) 200 201 #define CREQ_ENTRY_POLL_BUDGET 8 202 203 typedef int (*aeq_handler_t)(struct bnxt_qplib_rcfw *, void *, void *); 204 205 struct bnxt_qplib_crsqe { 206 struct creq_qp_event *resp; 207 u32 req_size; 208 bool is_waiter_alive; 209 bool is_internal_cmd; 210 bool is_in_used; 211 212 /* Free slots at the time of submission */ 213 u32 free_slots; 214 unsigned long send_timestamp; 215 u8 opcode; 216 u8 requested_qp_state; 217 }; 218 219 struct bnxt_qplib_rcfw_sbuf { 220 void *sb; 221 dma_addr_t dma_addr; 222 u32 size; 223 }; 224 225 #define BNXT_QPLIB_OOS_COUNT_MASK 0xFFFFFFFF 226 227 #define FIRMWARE_INITIALIZED_FLAG (0) 228 #define FIRMWARE_FIRST_FLAG (31) 229 #define FIRMWARE_STALL_DETECTED (3) 230 #define ERR_DEVICE_DETACHED (4) 231 struct bnxt_qplib_cmdq_mbox { 232 struct bnxt_qplib_reg_desc reg; 233 void __iomem *prod; 234 void __iomem *db; 235 }; 236 237 struct bnxt_qplib_cmdq_ctx { 238 struct bnxt_qplib_hwq hwq; 239 struct bnxt_qplib_cmdq_mbox cmdq_mbox; 240 wait_queue_head_t waitq; 241 unsigned long flags; 242 unsigned long last_seen; 243 u32 seq_num; 244 }; 245 246 struct bnxt_qplib_creq_db { 247 struct bnxt_qplib_reg_desc reg; 248 void __iomem *db; 249 struct bnxt_qplib_db_info dbinfo; 250 }; 251 252 struct bnxt_qplib_creq_stat { 253 u64 creq_arm_count; 254 u64 creq_tasklet_schedule_count; 255 u64 creq_qp_event_processed; 256 u64 creq_func_event_processed; 257 }; 258 259 struct bnxt_qplib_creq_ctx { 260 struct bnxt_qplib_hwq hwq; 261 struct bnxt_qplib_creq_db creq_db; 262 struct bnxt_qplib_creq_stat stats; 263 aeq_handler_t aeq_handler; 264 u16 ring_id; 265 int msix_vec; 266 bool requested; 267 char *irq_name; 268 }; 269 270 /* RCFW Communication Channels */ 271 #define BNXT_QPLIB_RCFW_SEND_RETRY_COUNT 4000 272 struct bnxt_qplib_rcfw { 273 struct pci_dev *pdev; 274 struct bnxt_qplib_res *res; 275 struct bnxt_qplib_cmdq_ctx cmdq; 276 struct bnxt_qplib_creq_ctx creq; 277 struct bnxt_qplib_crsqe *crsqe_tbl; 278 u32 rcfw_lat_slab_sec[RCFW_MAX_LATENCY_SEC_SLAB_INDEX]; 279 280 /* Slow path Perf Stats */ 281 bool sp_perf_stats_enabled; 282 u32 *rcfw_lat_slab_msec; 283 u64 *qp_create_stats; 284 u64 *qp_destroy_stats; 285 u64 *qp_modify_stats; 286 u64 *mr_create_stats; 287 u64 *mr_destroy_stats; 288 u32 qp_create_stats_id; 289 u32 qp_destroy_stats_id; 290 u32 qp_modify_stats_id; 291 u32 mr_create_stats_id; 292 u32 mr_destroy_stats_id; 293 bool init_oos_stats; 294 u64 oos_prev; 295 u32 num_irq_stopped; 296 u32 num_irq_started; 297 u32 poll_in_intr_en; 298 u32 poll_in_intr_dis; 299 atomic_t rcfw_intr_enabled; 300 u32 cmdq_full_dbg; 301 struct semaphore rcfw_inflight; 302 unsigned int curr_shadow_qd; 303 atomic_t timeout_send; 304 /* cached from chip cctx for quick reference in slow path */ 305 u16 max_timeout; 306 }; 307 308 struct bnxt_qplib_cmdqmsg { 309 struct cmdq_base *req; 310 struct creq_base *resp; 311 void *sb; 312 u32 req_sz; 313 u32 res_sz; 314 u8 block; 315 u8 qp_state; 316 }; 317 318 static inline void bnxt_qplib_fill_cmdqmsg(struct bnxt_qplib_cmdqmsg *msg, 319 void *req, void *resp, void *sb, 320 u32 req_sz, u32 res_sz, u8 block) 321 { 322 msg->req = req; 323 msg->resp = resp; 324 msg->sb = sb; 325 msg->req_sz = req_sz; 326 msg->res_sz = res_sz; 327 msg->block = block; 328 } 329 330 void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_res *res); 331 int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res); 332 void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill); 333 void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw); 334 int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector, 335 bool need_init); 336 int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw, 337 int msix_vector, 338 int cp_bar_reg_off, 339 aeq_handler_t aeq_handler); 340 341 struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf( 342 struct bnxt_qplib_rcfw *rcfw, 343 u32 size); 344 void bnxt_qplib_rcfw_free_sbuf(struct bnxt_qplib_rcfw *rcfw, 345 struct bnxt_qplib_rcfw_sbuf *sbuf); 346 int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, 347 struct bnxt_qplib_cmdqmsg *msg); 348 349 int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw); 350 int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, int is_virtfn); 351 void bnxt_qplib_mark_qp_error(void *qp_handle); 352 int __check_cmdq_stall(struct bnxt_qplib_rcfw *rcfw, 353 u32 *cur_prod, u32 *cur_cons); 354 #endif 355