1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) 2 /* QLogic qed NIC Driver 3 * Copyright (c) 2015-2017 QLogic Corporation 4 * Copyright (c) 2019-2020 Marvell International Ltd. 5 */ 6 7 #include <linux/types.h> 8 #include <linux/io.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/errno.h> 12 #include <linux/kernel.h> 13 #include <linux/list.h> 14 #include <linux/mutex.h> 15 #include <linux/pci.h> 16 #include <linux/slab.h> 17 #include <linux/spinlock.h> 18 #include <linux/string.h> 19 #include <linux/qed/qed_chain.h> 20 #include "qed.h" 21 #include "qed_hsi.h" 22 #include "qed_hw.h" 23 #include "qed_reg_addr.h" 24 #include "qed_sriov.h" 25 26 #define QED_BAR_ACQUIRE_TIMEOUT_USLEEP_CNT 1000 27 #define QED_BAR_ACQUIRE_TIMEOUT_USLEEP 1000 28 #define QED_BAR_ACQUIRE_TIMEOUT_UDELAY_CNT 100000 29 #define QED_BAR_ACQUIRE_TIMEOUT_UDELAY 10 30 31 /* Invalid values */ 32 #define QED_BAR_INVALID_OFFSET (cpu_to_le32(-1)) 33 34 struct qed_ptt { 35 struct list_head list_entry; 36 unsigned int idx; 37 struct pxp_ptt_entry pxp; 38 u8 hwfn_id; 39 }; 40 41 struct qed_ptt_pool { 42 struct list_head free_list; 43 spinlock_t lock; /* ptt synchronized access */ 44 struct qed_ptt ptts[PXP_EXTERNAL_BAR_PF_WINDOW_NUM]; 45 }; 46 47 int qed_ptt_pool_alloc(struct qed_hwfn *p_hwfn) 48 { 49 struct qed_ptt_pool *p_pool = kmalloc(sizeof(*p_pool), GFP_KERNEL); 50 int i; 51 52 if (!p_pool) 53 return -ENOMEM; 54 55 INIT_LIST_HEAD(&p_pool->free_list); 56 for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) { 57 p_pool->ptts[i].idx = i; 58 p_pool->ptts[i].pxp.offset = QED_BAR_INVALID_OFFSET; 59 p_pool->ptts[i].pxp.pretend.control = 0; 60 p_pool->ptts[i].hwfn_id = p_hwfn->my_id; 61 if (i >= RESERVED_PTT_MAX) 62 list_add(&p_pool->ptts[i].list_entry, 63 &p_pool->free_list); 64 } 65 66 p_hwfn->p_ptt_pool = p_pool; 67 spin_lock_init(&p_pool->lock); 68 69 return 0; 70 } 71 72 void qed_ptt_pool_free(struct qed_hwfn *p_hwfn) 73 { 74 kfree(p_hwfn->p_ptt_pool); 75 p_hwfn->p_ptt_pool = NULL; 76 } 77 78 struct qed_ptt *qed_ptt_acquire(struct qed_hwfn *p_hwfn) 79 { 80 return qed_ptt_acquire_context(p_hwfn, false); 81 } 82 83 struct qed_ptt *qed_ptt_acquire_context(struct qed_hwfn *p_hwfn, bool is_atomic) 84 { 85 struct qed_ptt *p_ptt; 86 unsigned int i, count; 87 88 if (is_atomic) 89 count = QED_BAR_ACQUIRE_TIMEOUT_UDELAY_CNT; 90 else 91 count = QED_BAR_ACQUIRE_TIMEOUT_USLEEP_CNT; 92 93 /* Take the free PTT from the list */ 94 for (i = 0; i < count; i++) { 95 spin_lock_bh(&p_hwfn->p_ptt_pool->lock); 96 97 if (!list_empty(&p_hwfn->p_ptt_pool->free_list)) { 98 p_ptt = list_first_entry(&p_hwfn->p_ptt_pool->free_list, 99 struct qed_ptt, list_entry); 100 list_del(&p_ptt->list_entry); 101 102 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 103 104 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 105 "allocated ptt %d\n", p_ptt->idx); 106 return p_ptt; 107 } 108 109 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 110 111 if (is_atomic) 112 udelay(QED_BAR_ACQUIRE_TIMEOUT_UDELAY); 113 else 114 usleep_range(QED_BAR_ACQUIRE_TIMEOUT_USLEEP, 115 QED_BAR_ACQUIRE_TIMEOUT_USLEEP * 2); 116 } 117 118 DP_NOTICE(p_hwfn, "PTT acquire timeout - failed to allocate PTT\n"); 119 return NULL; 120 } 121 122 void qed_ptt_release(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 123 { 124 spin_lock_bh(&p_hwfn->p_ptt_pool->lock); 125 list_add(&p_ptt->list_entry, &p_hwfn->p_ptt_pool->free_list); 126 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 127 } 128 129 u32 qed_ptt_get_hw_addr(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 130 { 131 /* The HW is using DWORDS and we need to translate it to Bytes */ 132 return le32_to_cpu(p_ptt->pxp.offset) << 2; 133 } 134 135 static u32 qed_ptt_config_addr(struct qed_ptt *p_ptt) 136 { 137 return PXP_PF_WINDOW_ADMIN_PER_PF_START + 138 p_ptt->idx * sizeof(struct pxp_ptt_entry); 139 } 140 141 u32 qed_ptt_get_bar_addr(struct qed_ptt *p_ptt) 142 { 143 return PXP_EXTERNAL_BAR_PF_WINDOW_START + 144 p_ptt->idx * PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE; 145 } 146 147 void qed_ptt_set_win(struct qed_hwfn *p_hwfn, 148 struct qed_ptt *p_ptt, u32 new_hw_addr) 149 { 150 u32 prev_hw_addr; 151 152 prev_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt); 153 154 if (new_hw_addr == prev_hw_addr) 155 return; 156 157 /* Update PTT entery in admin window */ 158 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 159 "Updating PTT entry %d to offset 0x%x\n", 160 p_ptt->idx, new_hw_addr); 161 162 /* The HW is using DWORDS and the address is in Bytes */ 163 p_ptt->pxp.offset = cpu_to_le32(new_hw_addr >> 2); 164 165 REG_WR(p_hwfn, 166 qed_ptt_config_addr(p_ptt) + 167 offsetof(struct pxp_ptt_entry, offset), 168 le32_to_cpu(p_ptt->pxp.offset)); 169 } 170 171 static u32 qed_set_ptt(struct qed_hwfn *p_hwfn, 172 struct qed_ptt *p_ptt, u32 hw_addr) 173 { 174 u32 win_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt); 175 u32 offset; 176 177 offset = hw_addr - win_hw_addr; 178 179 if (p_ptt->hwfn_id != p_hwfn->my_id) 180 DP_NOTICE(p_hwfn, 181 "ptt[%d] of hwfn[%02x] is used by hwfn[%02x]!\n", 182 p_ptt->idx, p_ptt->hwfn_id, p_hwfn->my_id); 183 184 /* Verify the address is within the window */ 185 if (hw_addr < win_hw_addr || 186 offset >= PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE) { 187 qed_ptt_set_win(p_hwfn, p_ptt, hw_addr); 188 offset = 0; 189 } 190 191 return qed_ptt_get_bar_addr(p_ptt) + offset; 192 } 193 194 struct qed_ptt *qed_get_reserved_ptt(struct qed_hwfn *p_hwfn, 195 enum reserved_ptts ptt_idx) 196 { 197 if (ptt_idx >= RESERVED_PTT_MAX) { 198 DP_NOTICE(p_hwfn, 199 "Requested PTT %d is out of range\n", ptt_idx); 200 return NULL; 201 } 202 203 return &p_hwfn->p_ptt_pool->ptts[ptt_idx]; 204 } 205 206 void qed_wr(struct qed_hwfn *p_hwfn, 207 struct qed_ptt *p_ptt, 208 u32 hw_addr, u32 val) 209 { 210 u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr); 211 212 REG_WR(p_hwfn, bar_addr, val); 213 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 214 "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n", 215 bar_addr, hw_addr, val); 216 } 217 218 u32 qed_rd(struct qed_hwfn *p_hwfn, 219 struct qed_ptt *p_ptt, 220 u32 hw_addr) 221 { 222 u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr); 223 u32 val = REG_RD(p_hwfn, bar_addr); 224 225 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 226 "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n", 227 bar_addr, hw_addr, val); 228 229 return val; 230 } 231 232 static void qed_memcpy_hw(struct qed_hwfn *p_hwfn, 233 struct qed_ptt *p_ptt, 234 void *addr, u32 hw_addr, size_t n, bool to_device) 235 { 236 u32 dw_count, *host_addr, hw_offset; 237 size_t quota, done = 0; 238 u32 __iomem *reg_addr; 239 240 while (done < n) { 241 quota = min_t(size_t, n - done, 242 PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE); 243 244 if (IS_PF(p_hwfn->cdev)) { 245 qed_ptt_set_win(p_hwfn, p_ptt, hw_addr + done); 246 hw_offset = qed_ptt_get_bar_addr(p_ptt); 247 } else { 248 hw_offset = hw_addr + done; 249 } 250 251 dw_count = quota / 4; 252 host_addr = (u32 *)((u8 *)addr + done); 253 reg_addr = (u32 __iomem *)REG_ADDR(p_hwfn, hw_offset); 254 if (to_device) 255 while (dw_count--) 256 DIRECT_REG_WR(reg_addr++, *host_addr++); 257 else 258 while (dw_count--) 259 *host_addr++ = DIRECT_REG_RD(reg_addr++); 260 261 done += quota; 262 } 263 } 264 265 void qed_memcpy_from(struct qed_hwfn *p_hwfn, 266 struct qed_ptt *p_ptt, void *dest, u32 hw_addr, size_t n) 267 { 268 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 269 "hw_addr 0x%x, dest %p hw_addr 0x%x, size %lu\n", 270 hw_addr, dest, hw_addr, (unsigned long)n); 271 272 qed_memcpy_hw(p_hwfn, p_ptt, dest, hw_addr, n, false); 273 } 274 275 void qed_memcpy_to(struct qed_hwfn *p_hwfn, 276 struct qed_ptt *p_ptt, u32 hw_addr, void *src, size_t n) 277 { 278 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 279 "hw_addr 0x%x, hw_addr 0x%x, src %p size %lu\n", 280 hw_addr, hw_addr, src, (unsigned long)n); 281 282 qed_memcpy_hw(p_hwfn, p_ptt, src, hw_addr, n, true); 283 } 284 285 void qed_fid_pretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 fid) 286 { 287 u16 control = 0; 288 289 SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1); 290 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1); 291 292 /* Every pretend undos previous pretends, including 293 * previous port pretend. 294 */ 295 SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); 296 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); 297 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 298 299 if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID)) 300 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID); 301 302 p_ptt->pxp.pretend.control = cpu_to_le16(control); 303 p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid); 304 305 REG_WR(p_hwfn, 306 qed_ptt_config_addr(p_ptt) + 307 offsetof(struct pxp_ptt_entry, pretend), 308 *(u32 *)&p_ptt->pxp.pretend); 309 } 310 311 void qed_port_pretend(struct qed_hwfn *p_hwfn, 312 struct qed_ptt *p_ptt, u8 port_id) 313 { 314 u16 control = 0; 315 316 SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id); 317 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1); 318 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 319 320 p_ptt->pxp.pretend.control = cpu_to_le16(control); 321 322 REG_WR(p_hwfn, 323 qed_ptt_config_addr(p_ptt) + 324 offsetof(struct pxp_ptt_entry, pretend), 325 *(u32 *)&p_ptt->pxp.pretend); 326 } 327 328 void qed_port_unpretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 329 { 330 u16 control = 0; 331 332 SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); 333 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); 334 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 335 336 p_ptt->pxp.pretend.control = cpu_to_le16(control); 337 338 REG_WR(p_hwfn, 339 qed_ptt_config_addr(p_ptt) + 340 offsetof(struct pxp_ptt_entry, pretend), 341 *(u32 *)&p_ptt->pxp.pretend); 342 } 343 344 void qed_port_fid_pretend(struct qed_hwfn *p_hwfn, 345 struct qed_ptt *p_ptt, u8 port_id, u16 fid) 346 { 347 u16 control = 0; 348 349 SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id); 350 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1); 351 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 352 SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1); 353 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1); 354 if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID)) 355 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID); 356 p_ptt->pxp.pretend.control = cpu_to_le16(control); 357 p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid); 358 REG_WR(p_hwfn, 359 qed_ptt_config_addr(p_ptt) + 360 offsetof(struct pxp_ptt_entry, pretend), 361 *(u32 *)&p_ptt->pxp.pretend); 362 } 363 364 u32 qed_vfid_to_concrete(struct qed_hwfn *p_hwfn, u8 vfid) 365 { 366 u32 concrete_fid = 0; 367 368 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID, p_hwfn->rel_pf_id); 369 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID, vfid); 370 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID, 1); 371 372 return concrete_fid; 373 } 374 375 /* DMAE */ 376 #define QED_DMAE_FLAGS_IS_SET(params, flag) \ 377 ((params) != NULL && GET_FIELD((params)->flags, QED_DMAE_PARAMS_##flag)) 378 379 static void qed_dmae_opcode(struct qed_hwfn *p_hwfn, 380 const u8 is_src_type_grc, 381 const u8 is_dst_type_grc, 382 struct qed_dmae_params *p_params) 383 { 384 u8 src_pfid, dst_pfid, port_id; 385 u16 opcode_b = 0; 386 u32 opcode = 0; 387 388 /* Whether the source is the PCIe or the GRC. 389 * 0- The source is the PCIe 390 * 1- The source is the GRC. 391 */ 392 SET_FIELD(opcode, DMAE_CMD_SRC, 393 (is_src_type_grc ? dmae_cmd_src_grc : dmae_cmd_src_pcie)); 394 src_pfid = QED_DMAE_FLAGS_IS_SET(p_params, SRC_PF_VALID) ? 395 p_params->src_pfid : p_hwfn->rel_pf_id; 396 SET_FIELD(opcode, DMAE_CMD_SRC_PF_ID, src_pfid); 397 398 /* The destination of the DMA can be: 0-None 1-PCIe 2-GRC 3-None */ 399 SET_FIELD(opcode, DMAE_CMD_DST, 400 (is_dst_type_grc ? dmae_cmd_dst_grc : dmae_cmd_dst_pcie)); 401 dst_pfid = QED_DMAE_FLAGS_IS_SET(p_params, DST_PF_VALID) ? 402 p_params->dst_pfid : p_hwfn->rel_pf_id; 403 SET_FIELD(opcode, DMAE_CMD_DST_PF_ID, dst_pfid); 404 405 406 /* Whether to write a completion word to the completion destination: 407 * 0-Do not write a completion word 408 * 1-Write the completion word 409 */ 410 SET_FIELD(opcode, DMAE_CMD_COMP_WORD_EN, 1); 411 SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1); 412 413 if (QED_DMAE_FLAGS_IS_SET(p_params, COMPLETION_DST)) 414 SET_FIELD(opcode, DMAE_CMD_COMP_FUNC, 1); 415 416 /* swapping mode 3 - big endian */ 417 SET_FIELD(opcode, DMAE_CMD_ENDIANITY_MODE, DMAE_CMD_ENDIANITY); 418 419 port_id = (QED_DMAE_FLAGS_IS_SET(p_params, PORT_VALID)) ? 420 p_params->port_id : p_hwfn->port_id; 421 SET_FIELD(opcode, DMAE_CMD_PORT_ID, port_id); 422 423 /* reset source address in next go */ 424 SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1); 425 426 /* reset dest address in next go */ 427 SET_FIELD(opcode, DMAE_CMD_DST_ADDR_RESET, 1); 428 429 /* SRC/DST VFID: all 1's - pf, otherwise VF id */ 430 if (QED_DMAE_FLAGS_IS_SET(p_params, SRC_VF_VALID)) { 431 SET_FIELD(opcode, DMAE_CMD_SRC_VF_ID_VALID, 1); 432 SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, p_params->src_vfid); 433 } else { 434 SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, 0xFF); 435 } 436 if (QED_DMAE_FLAGS_IS_SET(p_params, DST_VF_VALID)) { 437 SET_FIELD(opcode, DMAE_CMD_DST_VF_ID_VALID, 1); 438 SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, p_params->dst_vfid); 439 } else { 440 SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, 0xFF); 441 } 442 443 p_hwfn->dmae_info.p_dmae_cmd->opcode = cpu_to_le32(opcode); 444 p_hwfn->dmae_info.p_dmae_cmd->opcode_b = cpu_to_le16(opcode_b); 445 } 446 447 u32 qed_dmae_idx_to_go_cmd(u8 idx) 448 { 449 /* All the DMAE 'go' registers form an array in internal memory */ 450 return DMAE_REG_GO_C0 + (idx << 2); 451 } 452 453 static int qed_dmae_post_command(struct qed_hwfn *p_hwfn, 454 struct qed_ptt *p_ptt) 455 { 456 struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd; 457 u8 idx_cmd = p_hwfn->dmae_info.channel, i; 458 int qed_status = 0; 459 460 /* verify address is not NULL */ 461 if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) || 462 ((!p_command->src_addr_lo) && (!p_command->src_addr_hi)))) { 463 DP_NOTICE(p_hwfn, 464 "source or destination address 0 idx_cmd=%d\n" 465 "opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n", 466 idx_cmd, 467 le32_to_cpu(p_command->opcode), 468 le16_to_cpu(p_command->opcode_b), 469 le16_to_cpu(p_command->length_dw), 470 le32_to_cpu(p_command->src_addr_hi), 471 le32_to_cpu(p_command->src_addr_lo), 472 le32_to_cpu(p_command->dst_addr_hi), 473 le32_to_cpu(p_command->dst_addr_lo)); 474 475 return -EINVAL; 476 } 477 478 DP_VERBOSE(p_hwfn, 479 NETIF_MSG_HW, 480 "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n", 481 idx_cmd, 482 le32_to_cpu(p_command->opcode), 483 le16_to_cpu(p_command->opcode_b), 484 le16_to_cpu(p_command->length_dw), 485 le32_to_cpu(p_command->src_addr_hi), 486 le32_to_cpu(p_command->src_addr_lo), 487 le32_to_cpu(p_command->dst_addr_hi), 488 le32_to_cpu(p_command->dst_addr_lo)); 489 490 /* Copy the command to DMAE - need to do it before every call 491 * for source/dest address no reset. 492 * The first 9 DWs are the command registers, the 10 DW is the 493 * GO register, and the rest are result registers 494 * (which are read only by the client). 495 */ 496 for (i = 0; i < DMAE_CMD_SIZE; i++) { 497 u32 data = (i < DMAE_CMD_SIZE_TO_FILL) ? 498 *(((u32 *)p_command) + i) : 0; 499 500 qed_wr(p_hwfn, p_ptt, 501 DMAE_REG_CMD_MEM + 502 (idx_cmd * DMAE_CMD_SIZE * sizeof(u32)) + 503 (i * sizeof(u32)), data); 504 } 505 506 qed_wr(p_hwfn, p_ptt, qed_dmae_idx_to_go_cmd(idx_cmd), DMAE_GO_VALUE); 507 508 return qed_status; 509 } 510 511 int qed_dmae_info_alloc(struct qed_hwfn *p_hwfn) 512 { 513 dma_addr_t *p_addr = &p_hwfn->dmae_info.completion_word_phys_addr; 514 struct dmae_cmd **p_cmd = &p_hwfn->dmae_info.p_dmae_cmd; 515 u32 **p_buff = &p_hwfn->dmae_info.p_intermediate_buffer; 516 u32 **p_comp = &p_hwfn->dmae_info.p_completion_word; 517 518 *p_comp = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 519 sizeof(u32), p_addr, GFP_KERNEL); 520 if (!*p_comp) 521 goto err; 522 523 p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr; 524 *p_cmd = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 525 sizeof(struct dmae_cmd), 526 p_addr, GFP_KERNEL); 527 if (!*p_cmd) 528 goto err; 529 530 p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr; 531 *p_buff = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 532 sizeof(u32) * DMAE_MAX_RW_SIZE, 533 p_addr, GFP_KERNEL); 534 if (!*p_buff) 535 goto err; 536 537 p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id; 538 539 return 0; 540 err: 541 qed_dmae_info_free(p_hwfn); 542 return -ENOMEM; 543 } 544 545 void qed_dmae_info_free(struct qed_hwfn *p_hwfn) 546 { 547 dma_addr_t p_phys; 548 549 /* Just make sure no one is in the middle */ 550 mutex_lock(&p_hwfn->dmae_info.mutex); 551 552 if (p_hwfn->dmae_info.p_completion_word) { 553 p_phys = p_hwfn->dmae_info.completion_word_phys_addr; 554 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 555 sizeof(u32), 556 p_hwfn->dmae_info.p_completion_word, p_phys); 557 p_hwfn->dmae_info.p_completion_word = NULL; 558 } 559 560 if (p_hwfn->dmae_info.p_dmae_cmd) { 561 p_phys = p_hwfn->dmae_info.dmae_cmd_phys_addr; 562 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 563 sizeof(struct dmae_cmd), 564 p_hwfn->dmae_info.p_dmae_cmd, p_phys); 565 p_hwfn->dmae_info.p_dmae_cmd = NULL; 566 } 567 568 if (p_hwfn->dmae_info.p_intermediate_buffer) { 569 p_phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; 570 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 571 sizeof(u32) * DMAE_MAX_RW_SIZE, 572 p_hwfn->dmae_info.p_intermediate_buffer, 573 p_phys); 574 p_hwfn->dmae_info.p_intermediate_buffer = NULL; 575 } 576 577 mutex_unlock(&p_hwfn->dmae_info.mutex); 578 } 579 580 static int qed_dmae_operation_wait(struct qed_hwfn *p_hwfn) 581 { 582 u32 wait_cnt_limit = 10000, wait_cnt = 0; 583 int qed_status = 0; 584 585 barrier(); 586 while (*p_hwfn->dmae_info.p_completion_word != DMAE_COMPLETION_VAL) { 587 udelay(DMAE_MIN_WAIT_TIME); 588 cond_resched(); 589 if (++wait_cnt > wait_cnt_limit) { 590 DP_NOTICE(p_hwfn->cdev, 591 "Timed-out waiting for operation to complete. Completion word is 0x%08x expected 0x%08x.\n", 592 *p_hwfn->dmae_info.p_completion_word, 593 DMAE_COMPLETION_VAL); 594 qed_status = -EBUSY; 595 break; 596 } 597 598 /* to sync the completion_word since we are not 599 * using the volatile keyword for p_completion_word 600 */ 601 barrier(); 602 } 603 604 if (qed_status == 0) 605 *p_hwfn->dmae_info.p_completion_word = 0; 606 607 return qed_status; 608 } 609 610 static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn, 611 struct qed_ptt *p_ptt, 612 u64 src_addr, 613 u64 dst_addr, 614 u8 src_type, 615 u8 dst_type, 616 u32 length_dw) 617 { 618 dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; 619 struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; 620 int qed_status = 0; 621 622 switch (src_type) { 623 case QED_DMAE_ADDRESS_GRC: 624 case QED_DMAE_ADDRESS_HOST_PHYS: 625 cmd->src_addr_hi = cpu_to_le32(upper_32_bits(src_addr)); 626 cmd->src_addr_lo = cpu_to_le32(lower_32_bits(src_addr)); 627 break; 628 /* for virtual source addresses we use the intermediate buffer. */ 629 case QED_DMAE_ADDRESS_HOST_VIRT: 630 cmd->src_addr_hi = cpu_to_le32(upper_32_bits(phys)); 631 cmd->src_addr_lo = cpu_to_le32(lower_32_bits(phys)); 632 memcpy(&p_hwfn->dmae_info.p_intermediate_buffer[0], 633 (void *)(uintptr_t)src_addr, 634 length_dw * sizeof(u32)); 635 break; 636 default: 637 return -EINVAL; 638 } 639 640 switch (dst_type) { 641 case QED_DMAE_ADDRESS_GRC: 642 case QED_DMAE_ADDRESS_HOST_PHYS: 643 cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(dst_addr)); 644 cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(dst_addr)); 645 break; 646 /* for virtual source addresses we use the intermediate buffer. */ 647 case QED_DMAE_ADDRESS_HOST_VIRT: 648 cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(phys)); 649 cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(phys)); 650 break; 651 default: 652 return -EINVAL; 653 } 654 655 cmd->length_dw = cpu_to_le16((u16)length_dw); 656 657 qed_dmae_post_command(p_hwfn, p_ptt); 658 659 qed_status = qed_dmae_operation_wait(p_hwfn); 660 661 if (qed_status) { 662 DP_NOTICE(p_hwfn, 663 "qed_dmae_host2grc: Wait Failed. source_addr 0x%llx, grc_addr 0x%llx, size_in_dwords 0x%x\n", 664 src_addr, dst_addr, length_dw); 665 return qed_status; 666 } 667 668 if (dst_type == QED_DMAE_ADDRESS_HOST_VIRT) 669 memcpy((void *)(uintptr_t)(dst_addr), 670 &p_hwfn->dmae_info.p_intermediate_buffer[0], 671 length_dw * sizeof(u32)); 672 673 return 0; 674 } 675 676 static int qed_dmae_execute_command(struct qed_hwfn *p_hwfn, 677 struct qed_ptt *p_ptt, 678 u64 src_addr, u64 dst_addr, 679 u8 src_type, u8 dst_type, 680 u32 size_in_dwords, 681 struct qed_dmae_params *p_params) 682 { 683 dma_addr_t phys = p_hwfn->dmae_info.completion_word_phys_addr; 684 u16 length_cur = 0, i = 0, cnt_split = 0, length_mod = 0; 685 struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; 686 u64 src_addr_split = 0, dst_addr_split = 0; 687 u16 length_limit = DMAE_MAX_RW_SIZE; 688 int qed_status = 0; 689 u32 offset = 0; 690 691 if (p_hwfn->cdev->recov_in_prog) { 692 DP_VERBOSE(p_hwfn, 693 NETIF_MSG_HW, 694 "Recovery is in progress. Avoid DMAE transaction [{src: addr 0x%llx, type %d}, {dst: addr 0x%llx, type %d}, size %d].\n", 695 src_addr, src_type, dst_addr, dst_type, 696 size_in_dwords); 697 698 /* Let the flow complete w/o any error handling */ 699 return 0; 700 } 701 702 qed_dmae_opcode(p_hwfn, 703 (src_type == QED_DMAE_ADDRESS_GRC), 704 (dst_type == QED_DMAE_ADDRESS_GRC), 705 p_params); 706 707 cmd->comp_addr_lo = cpu_to_le32(lower_32_bits(phys)); 708 cmd->comp_addr_hi = cpu_to_le32(upper_32_bits(phys)); 709 cmd->comp_val = cpu_to_le32(DMAE_COMPLETION_VAL); 710 711 /* Check if the grc_addr is valid like < MAX_GRC_OFFSET */ 712 cnt_split = size_in_dwords / length_limit; 713 length_mod = size_in_dwords % length_limit; 714 715 src_addr_split = src_addr; 716 dst_addr_split = dst_addr; 717 718 for (i = 0; i <= cnt_split; i++) { 719 offset = length_limit * i; 720 721 if (!QED_DMAE_FLAGS_IS_SET(p_params, RW_REPL_SRC)) { 722 if (src_type == QED_DMAE_ADDRESS_GRC) 723 src_addr_split = src_addr + offset; 724 else 725 src_addr_split = src_addr + (offset * 4); 726 } 727 728 if (dst_type == QED_DMAE_ADDRESS_GRC) 729 dst_addr_split = dst_addr + offset; 730 else 731 dst_addr_split = dst_addr + (offset * 4); 732 733 length_cur = (cnt_split == i) ? length_mod : length_limit; 734 735 /* might be zero on last iteration */ 736 if (!length_cur) 737 continue; 738 739 qed_status = qed_dmae_execute_sub_operation(p_hwfn, 740 p_ptt, 741 src_addr_split, 742 dst_addr_split, 743 src_type, 744 dst_type, 745 length_cur); 746 if (qed_status) { 747 qed_hw_err_notify(p_hwfn, p_ptt, QED_HW_ERR_DMAE_FAIL, 748 "qed_dmae_execute_sub_operation Failed with error 0x%x. source_addr 0x%llx, destination addr 0x%llx, size_in_dwords 0x%x\n", 749 qed_status, src_addr, 750 dst_addr, length_cur); 751 break; 752 } 753 } 754 755 return qed_status; 756 } 757 758 int qed_dmae_host2grc(struct qed_hwfn *p_hwfn, 759 struct qed_ptt *p_ptt, 760 u64 source_addr, u32 grc_addr, u32 size_in_dwords, 761 struct qed_dmae_params *p_params) 762 { 763 u32 grc_addr_in_dw = grc_addr / sizeof(u32); 764 int rc; 765 766 767 mutex_lock(&p_hwfn->dmae_info.mutex); 768 769 rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr, 770 grc_addr_in_dw, 771 QED_DMAE_ADDRESS_HOST_VIRT, 772 QED_DMAE_ADDRESS_GRC, 773 size_in_dwords, p_params); 774 775 mutex_unlock(&p_hwfn->dmae_info.mutex); 776 777 return rc; 778 } 779 780 int qed_dmae_grc2host(struct qed_hwfn *p_hwfn, 781 struct qed_ptt *p_ptt, 782 u32 grc_addr, 783 dma_addr_t dest_addr, u32 size_in_dwords, 784 struct qed_dmae_params *p_params) 785 { 786 u32 grc_addr_in_dw = grc_addr / sizeof(u32); 787 int rc; 788 789 790 mutex_lock(&p_hwfn->dmae_info.mutex); 791 792 rc = qed_dmae_execute_command(p_hwfn, p_ptt, grc_addr_in_dw, 793 dest_addr, QED_DMAE_ADDRESS_GRC, 794 QED_DMAE_ADDRESS_HOST_VIRT, 795 size_in_dwords, p_params); 796 797 mutex_unlock(&p_hwfn->dmae_info.mutex); 798 799 return rc; 800 } 801 802 int qed_dmae_host2host(struct qed_hwfn *p_hwfn, 803 struct qed_ptt *p_ptt, 804 dma_addr_t source_addr, 805 dma_addr_t dest_addr, 806 u32 size_in_dwords, struct qed_dmae_params *p_params) 807 { 808 int rc; 809 810 mutex_lock(&(p_hwfn->dmae_info.mutex)); 811 812 rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr, 813 dest_addr, 814 QED_DMAE_ADDRESS_HOST_PHYS, 815 QED_DMAE_ADDRESS_HOST_PHYS, 816 size_in_dwords, p_params); 817 818 mutex_unlock(&(p_hwfn->dmae_info.mutex)); 819 820 return rc; 821 } 822 823 void qed_hw_err_notify(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, 824 enum qed_hw_err_type err_type, const char *fmt, ...) 825 { 826 char buf[QED_HW_ERR_MAX_STR_SIZE]; 827 va_list vl; 828 int len; 829 830 if (fmt) { 831 va_start(vl, fmt); 832 len = vsnprintf(buf, QED_HW_ERR_MAX_STR_SIZE, fmt, vl); 833 va_end(vl); 834 835 if (len > QED_HW_ERR_MAX_STR_SIZE - 1) 836 len = QED_HW_ERR_MAX_STR_SIZE - 1; 837 838 DP_NOTICE(p_hwfn, "%s", buf); 839 } 840 841 /* Fan failure cannot be masked by handling of another HW error */ 842 if (p_hwfn->cdev->recov_in_prog && 843 err_type != QED_HW_ERR_FAN_FAIL) { 844 DP_VERBOSE(p_hwfn, 845 NETIF_MSG_DRV, 846 "Recovery is in progress. Avoid notifying about HW error %d.\n", 847 err_type); 848 return; 849 } 850 851 qed_hw_error_occurred(p_hwfn, err_type); 852 853 if (fmt) 854 qed_mcp_send_raw_debug_data(p_hwfn, p_ptt, buf, len); 855 } 856 857 int qed_dmae_sanity(struct qed_hwfn *p_hwfn, 858 struct qed_ptt *p_ptt, const char *phase) 859 { 860 u32 size = PAGE_SIZE / 2, val; 861 int rc = 0; 862 dma_addr_t p_phys; 863 void *p_virt; 864 u32 *p_tmp; 865 866 p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 867 2 * size, &p_phys, GFP_KERNEL); 868 if (!p_virt) { 869 DP_NOTICE(p_hwfn, 870 "DMAE sanity [%s]: failed to allocate memory\n", 871 phase); 872 return -ENOMEM; 873 } 874 875 /* Fill the bottom half of the allocated memory with a known pattern */ 876 for (p_tmp = (u32 *)p_virt; 877 p_tmp < (u32 *)((u8 *)p_virt + size); p_tmp++) { 878 /* Save the address itself as the value */ 879 val = (u32)(uintptr_t)p_tmp; 880 *p_tmp = val; 881 } 882 883 /* Zero the top half of the allocated memory */ 884 memset((u8 *)p_virt + size, 0, size); 885 886 DP_VERBOSE(p_hwfn, 887 QED_MSG_SP, 888 "DMAE sanity [%s]: src_addr={phys 0x%llx, virt %p}, dst_addr={phys 0x%llx, virt %p}, size 0x%x\n", 889 phase, 890 (u64)p_phys, 891 p_virt, (u64)(p_phys + size), (u8 *)p_virt + size, size); 892 893 rc = qed_dmae_host2host(p_hwfn, p_ptt, p_phys, p_phys + size, 894 size / 4, NULL); 895 if (rc) { 896 DP_NOTICE(p_hwfn, 897 "DMAE sanity [%s]: qed_dmae_host2host() failed. rc = %d.\n", 898 phase, rc); 899 goto out; 900 } 901 902 /* Verify that the top half of the allocated memory has the pattern */ 903 for (p_tmp = (u32 *)((u8 *)p_virt + size); 904 p_tmp < (u32 *)((u8 *)p_virt + (2 * size)); p_tmp++) { 905 /* The corresponding address in the bottom half */ 906 val = (u32)(uintptr_t)p_tmp - size; 907 908 if (*p_tmp != val) { 909 DP_NOTICE(p_hwfn, 910 "DMAE sanity [%s]: addr={phys 0x%llx, virt %p}, read_val 0x%08x, expected_val 0x%08x\n", 911 phase, 912 (u64)p_phys + ((u8 *)p_tmp - (u8 *)p_virt), 913 p_tmp, *p_tmp, val); 914 rc = -EINVAL; 915 goto out; 916 } 917 } 918 919 out: 920 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 2 * size, p_virt, p_phys); 921 return rc; 922 } 923