1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. 3 4 #include "hinic3_cmdq.h" 5 #include "hinic3_hw_comm.h" 6 #include "hinic3_hw_intf.h" 7 #include "hinic3_hwdev.h" 8 #include "hinic3_hwif.h" 9 #include "hinic3_nic_cfg.h" 10 #include "hinic3_nic_dev.h" 11 #include "hinic3_nic_io.h" 12 13 #define HINIC3_DEFAULT_TX_CI_PENDING_LIMIT 1 14 #define HINIC3_DEFAULT_TX_CI_COALESCING_TIME 1 15 #define HINIC3_DEFAULT_DROP_THD_ON (0xFFFF) 16 #define HINIC3_DEFAULT_DROP_THD_OFF 0 17 18 #define HINIC3_CI_Q_ADDR_SIZE (64) 19 20 #define HINIC3_CI_TABLE_SIZE(num_qps) \ 21 (ALIGN((num_qps) * HINIC3_CI_Q_ADDR_SIZE, HINIC3_MIN_PAGE_SIZE)) 22 23 #define HINIC3_CI_VADDR(base_addr, q_id) \ 24 ((u8 *)(base_addr) + (q_id) * HINIC3_CI_Q_ADDR_SIZE) 25 26 #define HINIC3_CI_PADDR(base_paddr, q_id) \ 27 ((base_paddr) + (q_id) * HINIC3_CI_Q_ADDR_SIZE) 28 29 #define SQ_WQ_PREFETCH_MAX 1 30 #define SQ_WQ_PREFETCH_MIN 1 31 #define SQ_WQ_PREFETCH_THRESHOLD 16 32 33 #define RQ_WQ_PREFETCH_MAX 4 34 #define RQ_WQ_PREFETCH_MIN 1 35 #define RQ_WQ_PREFETCH_THRESHOLD 256 36 37 /* (2048 - 8) / 64 */ 38 #define HINIC3_Q_CTXT_MAX 31 39 40 enum hinic3_qp_ctxt_type { 41 HINIC3_QP_CTXT_TYPE_SQ = 0, 42 HINIC3_QP_CTXT_TYPE_RQ = 1, 43 }; 44 45 struct hinic3_qp_ctxt_hdr { 46 __le16 num_queues; 47 __le16 queue_type; 48 __le16 start_qid; 49 __le16 rsvd; 50 }; 51 52 struct hinic3_sq_ctxt { 53 __le32 ci_pi; 54 __le32 drop_mode_sp; 55 __le32 wq_pfn_hi_owner; 56 __le32 wq_pfn_lo; 57 58 __le32 rsvd0; 59 __le32 pkt_drop_thd; 60 __le32 global_sq_id; 61 __le32 vlan_ceq_attr; 62 63 __le32 pref_cache; 64 __le32 pref_ci_owner; 65 __le32 pref_wq_pfn_hi_ci; 66 __le32 pref_wq_pfn_lo; 67 68 __le32 rsvd8; 69 __le32 rsvd9; 70 __le32 wq_block_pfn_hi; 71 __le32 wq_block_pfn_lo; 72 }; 73 74 struct hinic3_rq_ctxt { 75 __le32 ci_pi; 76 __le32 ceq_attr; 77 __le32 wq_pfn_hi_type_owner; 78 __le32 wq_pfn_lo; 79 80 __le32 rsvd[3]; 81 __le32 cqe_sge_len; 82 83 __le32 pref_cache; 84 __le32 pref_ci_owner; 85 __le32 pref_wq_pfn_hi_ci; 86 __le32 pref_wq_pfn_lo; 87 88 __le32 pi_paddr_hi; 89 __le32 pi_paddr_lo; 90 __le32 wq_block_pfn_hi; 91 __le32 wq_block_pfn_lo; 92 }; 93 94 struct hinic3_sq_ctxt_block { 95 struct hinic3_qp_ctxt_hdr cmdq_hdr; 96 struct hinic3_sq_ctxt sq_ctxt[HINIC3_Q_CTXT_MAX]; 97 }; 98 99 struct hinic3_rq_ctxt_block { 100 struct hinic3_qp_ctxt_hdr cmdq_hdr; 101 struct hinic3_rq_ctxt rq_ctxt[HINIC3_Q_CTXT_MAX]; 102 }; 103 104 struct hinic3_clean_queue_ctxt { 105 struct hinic3_qp_ctxt_hdr cmdq_hdr; 106 __le32 rsvd; 107 }; 108 109 #define SQ_CTXT_SIZE(num_sqs) \ 110 (sizeof(struct hinic3_qp_ctxt_hdr) + \ 111 (num_sqs) * sizeof(struct hinic3_sq_ctxt)) 112 113 #define RQ_CTXT_SIZE(num_rqs) \ 114 (sizeof(struct hinic3_qp_ctxt_hdr) + \ 115 (num_rqs) * sizeof(struct hinic3_rq_ctxt)) 116 117 #define SQ_CTXT_PREF_CI_HI_SHIFT 12 118 #define SQ_CTXT_PREF_CI_HI(val) ((val) >> SQ_CTXT_PREF_CI_HI_SHIFT) 119 120 #define SQ_CTXT_PI_IDX_MASK GENMASK(15, 0) 121 #define SQ_CTXT_CI_IDX_MASK GENMASK(31, 16) 122 #define SQ_CTXT_CI_PI_SET(val, member) \ 123 FIELD_PREP(SQ_CTXT_##member##_MASK, val) 124 125 #define SQ_CTXT_MODE_SP_FLAG_MASK BIT(0) 126 #define SQ_CTXT_MODE_PKT_DROP_MASK BIT(1) 127 #define SQ_CTXT_MODE_SET(val, member) \ 128 FIELD_PREP(SQ_CTXT_MODE_##member##_MASK, val) 129 130 #define SQ_CTXT_WQ_PAGE_HI_PFN_MASK GENMASK(19, 0) 131 #define SQ_CTXT_WQ_PAGE_OWNER_MASK BIT(23) 132 #define SQ_CTXT_WQ_PAGE_SET(val, member) \ 133 FIELD_PREP(SQ_CTXT_WQ_PAGE_##member##_MASK, val) 134 135 #define SQ_CTXT_PKT_DROP_THD_ON_MASK GENMASK(15, 0) 136 #define SQ_CTXT_PKT_DROP_THD_OFF_MASK GENMASK(31, 16) 137 #define SQ_CTXT_PKT_DROP_THD_SET(val, member) \ 138 FIELD_PREP(SQ_CTXT_PKT_DROP_##member##_MASK, val) 139 140 #define SQ_CTXT_GLOBAL_SQ_ID_MASK GENMASK(12, 0) 141 #define SQ_CTXT_GLOBAL_QUEUE_ID_SET(val, member) \ 142 FIELD_PREP(SQ_CTXT_##member##_MASK, val) 143 144 #define SQ_CTXT_VLAN_INSERT_MODE_MASK GENMASK(20, 19) 145 #define SQ_CTXT_VLAN_CEQ_EN_MASK BIT(23) 146 #define SQ_CTXT_VLAN_CEQ_SET(val, member) \ 147 FIELD_PREP(SQ_CTXT_VLAN_##member##_MASK, val) 148 149 #define SQ_CTXT_PREF_CACHE_THRESHOLD_MASK GENMASK(13, 0) 150 #define SQ_CTXT_PREF_CACHE_MAX_MASK GENMASK(24, 14) 151 #define SQ_CTXT_PREF_CACHE_MIN_MASK GENMASK(31, 25) 152 153 #define SQ_CTXT_PREF_CI_HI_MASK GENMASK(3, 0) 154 #define SQ_CTXT_PREF_OWNER_MASK BIT(4) 155 156 #define SQ_CTXT_PREF_WQ_PFN_HI_MASK GENMASK(19, 0) 157 #define SQ_CTXT_PREF_CI_LOW_MASK GENMASK(31, 20) 158 #define SQ_CTXT_PREF_SET(val, member) \ 159 FIELD_PREP(SQ_CTXT_PREF_##member##_MASK, val) 160 161 #define SQ_CTXT_WQ_BLOCK_PFN_HI_MASK GENMASK(22, 0) 162 #define SQ_CTXT_WQ_BLOCK_SET(val, member) \ 163 FIELD_PREP(SQ_CTXT_WQ_BLOCK_##member##_MASK, val) 164 165 #define RQ_CTXT_PI_IDX_MASK GENMASK(15, 0) 166 #define RQ_CTXT_CI_IDX_MASK GENMASK(31, 16) 167 #define RQ_CTXT_CI_PI_SET(val, member) \ 168 FIELD_PREP(RQ_CTXT_##member##_MASK, val) 169 170 #define RQ_CTXT_CEQ_ATTR_INTR_MASK GENMASK(30, 21) 171 #define RQ_CTXT_CEQ_ATTR_EN_MASK BIT(31) 172 #define RQ_CTXT_CEQ_ATTR_SET(val, member) \ 173 FIELD_PREP(RQ_CTXT_CEQ_ATTR_##member##_MASK, val) 174 175 #define RQ_CTXT_WQ_PAGE_HI_PFN_MASK GENMASK(19, 0) 176 #define RQ_CTXT_WQ_PAGE_WQE_TYPE_MASK GENMASK(29, 28) 177 #define RQ_CTXT_WQ_PAGE_OWNER_MASK BIT(31) 178 #define RQ_CTXT_WQ_PAGE_SET(val, member) \ 179 FIELD_PREP(RQ_CTXT_WQ_PAGE_##member##_MASK, val) 180 181 #define RQ_CTXT_CQE_LEN_MASK GENMASK(29, 28) 182 #define RQ_CTXT_CQE_LEN_SET(val, member) \ 183 FIELD_PREP(RQ_CTXT_##member##_MASK, val) 184 185 #define RQ_CTXT_PREF_CACHE_THRESHOLD_MASK GENMASK(13, 0) 186 #define RQ_CTXT_PREF_CACHE_MAX_MASK GENMASK(24, 14) 187 #define RQ_CTXT_PREF_CACHE_MIN_MASK GENMASK(31, 25) 188 189 #define RQ_CTXT_PREF_CI_HI_MASK GENMASK(3, 0) 190 #define RQ_CTXT_PREF_OWNER_MASK BIT(4) 191 192 #define RQ_CTXT_PREF_WQ_PFN_HI_MASK GENMASK(19, 0) 193 #define RQ_CTXT_PREF_CI_LOW_MASK GENMASK(31, 20) 194 #define RQ_CTXT_PREF_SET(val, member) \ 195 FIELD_PREP(RQ_CTXT_PREF_##member##_MASK, val) 196 197 #define RQ_CTXT_WQ_BLOCK_PFN_HI_MASK GENMASK(22, 0) 198 #define RQ_CTXT_WQ_BLOCK_SET(val, member) \ 199 FIELD_PREP(RQ_CTXT_WQ_BLOCK_##member##_MASK, val) 200 201 #define WQ_PAGE_PFN_SHIFT 12 202 #define WQ_BLOCK_PFN_SHIFT 9 203 #define WQ_PAGE_PFN(page_addr) ((page_addr) >> WQ_PAGE_PFN_SHIFT) 204 #define WQ_BLOCK_PFN(page_addr) ((page_addr) >> WQ_BLOCK_PFN_SHIFT) 205 206 int hinic3_init_nic_io(struct hinic3_nic_dev *nic_dev) 207 { 208 struct hinic3_hwdev *hwdev = nic_dev->hwdev; 209 struct hinic3_nic_io *nic_io; 210 int err; 211 212 nic_io = kzalloc(sizeof(*nic_io), GFP_KERNEL); 213 if (!nic_io) 214 return -ENOMEM; 215 216 nic_dev->nic_io = nic_io; 217 218 err = hinic3_set_func_svc_used_state(hwdev, COMM_FUNC_SVC_T_NIC, 1); 219 if (err) { 220 dev_err(hwdev->dev, "Failed to set function svc used state\n"); 221 goto err_free_nicio; 222 } 223 224 err = hinic3_init_function_table(nic_dev); 225 if (err) { 226 dev_err(hwdev->dev, "Failed to init function table\n"); 227 goto err_clear_func_svc_used_state; 228 } 229 230 nic_io->rx_buf_len = nic_dev->rx_buf_len; 231 232 err = hinic3_get_nic_feature_from_hw(nic_dev); 233 if (err) { 234 dev_err(hwdev->dev, "Failed to get nic features\n"); 235 goto err_clear_func_svc_used_state; 236 } 237 238 nic_io->feature_cap &= HINIC3_NIC_F_ALL_MASK; 239 nic_io->feature_cap &= HINIC3_NIC_DRV_DEFAULT_FEATURE; 240 dev_dbg(hwdev->dev, "nic features: 0x%llx\n\n", nic_io->feature_cap); 241 242 return 0; 243 244 err_clear_func_svc_used_state: 245 hinic3_set_func_svc_used_state(hwdev, COMM_FUNC_SVC_T_NIC, 0); 246 err_free_nicio: 247 nic_dev->nic_io = NULL; 248 kfree(nic_io); 249 250 return err; 251 } 252 253 void hinic3_free_nic_io(struct hinic3_nic_dev *nic_dev) 254 { 255 struct hinic3_nic_io *nic_io = nic_dev->nic_io; 256 257 hinic3_set_func_svc_used_state(nic_dev->hwdev, COMM_FUNC_SVC_T_NIC, 0); 258 nic_dev->nic_io = NULL; 259 kfree(nic_io); 260 } 261 262 int hinic3_init_nicio_res(struct hinic3_nic_dev *nic_dev) 263 { 264 struct hinic3_nic_io *nic_io = nic_dev->nic_io; 265 struct hinic3_hwdev *hwdev = nic_dev->hwdev; 266 void __iomem *db_base; 267 int err; 268 269 nic_io->max_qps = hinic3_func_max_qnum(hwdev); 270 271 err = hinic3_alloc_db_addr(hwdev, &db_base, NULL); 272 if (err) { 273 dev_err(hwdev->dev, "Failed to allocate doorbell for sqs\n"); 274 return err; 275 } 276 nic_io->sqs_db_addr = db_base; 277 278 err = hinic3_alloc_db_addr(hwdev, &db_base, NULL); 279 if (err) { 280 hinic3_free_db_addr(hwdev, nic_io->sqs_db_addr); 281 dev_err(hwdev->dev, "Failed to allocate doorbell for rqs\n"); 282 return err; 283 } 284 nic_io->rqs_db_addr = db_base; 285 286 nic_io->ci_vaddr_base = 287 dma_alloc_coherent(hwdev->dev, 288 HINIC3_CI_TABLE_SIZE(nic_io->max_qps), 289 &nic_io->ci_dma_base, 290 GFP_KERNEL); 291 if (!nic_io->ci_vaddr_base) { 292 hinic3_free_db_addr(hwdev, nic_io->sqs_db_addr); 293 hinic3_free_db_addr(hwdev, nic_io->rqs_db_addr); 294 return -ENOMEM; 295 } 296 297 return 0; 298 } 299 300 void hinic3_free_nicio_res(struct hinic3_nic_dev *nic_dev) 301 { 302 struct hinic3_nic_io *nic_io = nic_dev->nic_io; 303 struct hinic3_hwdev *hwdev = nic_dev->hwdev; 304 305 dma_free_coherent(hwdev->dev, 306 HINIC3_CI_TABLE_SIZE(nic_io->max_qps), 307 nic_io->ci_vaddr_base, nic_io->ci_dma_base); 308 309 hinic3_free_db_addr(hwdev, nic_io->sqs_db_addr); 310 hinic3_free_db_addr(hwdev, nic_io->rqs_db_addr); 311 } 312 313 static int hinic3_create_sq(struct hinic3_hwdev *hwdev, 314 struct hinic3_io_queue *sq, 315 u16 q_id, u32 sq_depth, u16 sq_msix_idx) 316 { 317 int err; 318 319 /* sq used & hardware request init 1 */ 320 sq->owner = 1; 321 322 sq->q_id = q_id; 323 sq->msix_entry_idx = sq_msix_idx; 324 325 err = hinic3_wq_create(hwdev, &sq->wq, sq_depth, 326 BIT(HINIC3_SQ_WQEBB_SHIFT)); 327 if (err) { 328 dev_err(hwdev->dev, "Failed to create tx queue %u wq\n", 329 q_id); 330 return err; 331 } 332 333 return 0; 334 } 335 336 static int hinic3_create_rq(struct hinic3_hwdev *hwdev, 337 struct hinic3_io_queue *rq, 338 u16 q_id, u32 rq_depth, u16 rq_msix_idx) 339 { 340 int err; 341 342 rq->q_id = q_id; 343 rq->msix_entry_idx = rq_msix_idx; 344 345 err = hinic3_wq_create(hwdev, &rq->wq, rq_depth, 346 BIT(HINIC3_RQ_WQEBB_SHIFT + 347 HINIC3_NORMAL_RQ_WQE)); 348 if (err) { 349 dev_err(hwdev->dev, "Failed to create rx queue %u wq\n", 350 q_id); 351 return err; 352 } 353 354 return 0; 355 } 356 357 static int hinic3_create_qp(struct hinic3_hwdev *hwdev, 358 struct hinic3_io_queue *sq, 359 struct hinic3_io_queue *rq, u16 q_id, u32 sq_depth, 360 u32 rq_depth, u16 qp_msix_idx) 361 { 362 int err; 363 364 err = hinic3_create_sq(hwdev, sq, q_id, sq_depth, qp_msix_idx); 365 if (err) { 366 dev_err(hwdev->dev, "Failed to create sq, qid: %u\n", 367 q_id); 368 return err; 369 } 370 371 err = hinic3_create_rq(hwdev, rq, q_id, rq_depth, qp_msix_idx); 372 if (err) { 373 dev_err(hwdev->dev, "Failed to create rq, qid: %u\n", 374 q_id); 375 goto err_destroy_sq_wq; 376 } 377 378 return 0; 379 380 err_destroy_sq_wq: 381 hinic3_wq_destroy(hwdev, &sq->wq); 382 383 return err; 384 } 385 386 static void hinic3_destroy_qp(struct hinic3_hwdev *hwdev, 387 struct hinic3_io_queue *sq, 388 struct hinic3_io_queue *rq) 389 { 390 hinic3_wq_destroy(hwdev, &sq->wq); 391 hinic3_wq_destroy(hwdev, &rq->wq); 392 } 393 394 int hinic3_alloc_qps(struct hinic3_nic_dev *nic_dev, 395 struct hinic3_dyna_qp_params *qp_params) 396 { 397 struct msix_entry *qps_msix_entries = nic_dev->qps_msix_entries; 398 struct hinic3_nic_io *nic_io = nic_dev->nic_io; 399 struct hinic3_hwdev *hwdev = nic_dev->hwdev; 400 struct hinic3_io_queue *sqs; 401 struct hinic3_io_queue *rqs; 402 u16 q_id; 403 int err; 404 405 if (qp_params->num_qps > nic_io->max_qps || !qp_params->num_qps) 406 return -EINVAL; 407 408 sqs = kcalloc(qp_params->num_qps, sizeof(*sqs), GFP_KERNEL); 409 if (!sqs) { 410 err = -ENOMEM; 411 goto err_out; 412 } 413 414 rqs = kcalloc(qp_params->num_qps, sizeof(*rqs), GFP_KERNEL); 415 if (!rqs) { 416 err = -ENOMEM; 417 goto err_free_sqs; 418 } 419 420 for (q_id = 0; q_id < qp_params->num_qps; q_id++) { 421 err = hinic3_create_qp(hwdev, &sqs[q_id], &rqs[q_id], q_id, 422 qp_params->sq_depth, qp_params->rq_depth, 423 qps_msix_entries[q_id].entry); 424 if (err) { 425 dev_err(hwdev->dev, "Failed to allocate qp %u, err: %d\n", 426 q_id, err); 427 goto err_destroy_qp; 428 } 429 } 430 431 qp_params->sqs = sqs; 432 qp_params->rqs = rqs; 433 434 return 0; 435 436 err_destroy_qp: 437 while (q_id > 0) { 438 q_id--; 439 hinic3_destroy_qp(hwdev, &sqs[q_id], &rqs[q_id]); 440 } 441 kfree(rqs); 442 err_free_sqs: 443 kfree(sqs); 444 err_out: 445 return err; 446 } 447 448 void hinic3_free_qps(struct hinic3_nic_dev *nic_dev, 449 struct hinic3_dyna_qp_params *qp_params) 450 { 451 struct hinic3_hwdev *hwdev = nic_dev->hwdev; 452 u16 q_id; 453 454 for (q_id = 0; q_id < qp_params->num_qps; q_id++) 455 hinic3_destroy_qp(hwdev, &qp_params->sqs[q_id], 456 &qp_params->rqs[q_id]); 457 458 kfree(qp_params->sqs); 459 kfree(qp_params->rqs); 460 } 461 462 void hinic3_init_qps(struct hinic3_nic_dev *nic_dev, 463 struct hinic3_dyna_qp_params *qp_params) 464 { 465 struct hinic3_nic_io *nic_io = nic_dev->nic_io; 466 struct hinic3_io_queue *sqs = qp_params->sqs; 467 struct hinic3_io_queue *rqs = qp_params->rqs; 468 u16 q_id; 469 470 nic_io->num_qps = qp_params->num_qps; 471 nic_io->sq = qp_params->sqs; 472 nic_io->rq = qp_params->rqs; 473 for (q_id = 0; q_id < nic_io->num_qps; q_id++) { 474 sqs[q_id].cons_idx_addr = 475 (u16 *)HINIC3_CI_VADDR(nic_io->ci_vaddr_base, q_id); 476 /* clear ci value */ 477 WRITE_ONCE(*sqs[q_id].cons_idx_addr, 0); 478 479 sqs[q_id].db_addr = nic_io->sqs_db_addr; 480 rqs[q_id].db_addr = nic_io->rqs_db_addr; 481 } 482 } 483 484 void hinic3_uninit_qps(struct hinic3_nic_dev *nic_dev, 485 struct hinic3_dyna_qp_params *qp_params) 486 { 487 struct hinic3_nic_io *nic_io = nic_dev->nic_io; 488 489 qp_params->sqs = nic_io->sq; 490 qp_params->rqs = nic_io->rq; 491 qp_params->num_qps = nic_io->num_qps; 492 } 493 494 static void hinic3_qp_prepare_cmdq_header(struct hinic3_qp_ctxt_hdr *qp_ctxt_hdr, 495 enum hinic3_qp_ctxt_type ctxt_type, 496 u16 num_queues, u16 q_id) 497 { 498 qp_ctxt_hdr->queue_type = cpu_to_le16(ctxt_type); 499 qp_ctxt_hdr->num_queues = cpu_to_le16(num_queues); 500 qp_ctxt_hdr->start_qid = cpu_to_le16(q_id); 501 qp_ctxt_hdr->rsvd = 0; 502 } 503 504 static void hinic3_sq_prepare_ctxt(struct hinic3_io_queue *sq, u16 sq_id, 505 struct hinic3_sq_ctxt *sq_ctxt) 506 { 507 u64 wq_page_addr, wq_page_pfn, wq_block_pfn; 508 u32 wq_block_pfn_hi, wq_block_pfn_lo; 509 u32 wq_page_pfn_hi, wq_page_pfn_lo; 510 u16 pi_start, ci_start; 511 512 ci_start = hinic3_get_sq_local_ci(sq); 513 pi_start = hinic3_get_sq_local_pi(sq); 514 515 wq_page_addr = hinic3_wq_get_first_wqe_page_addr(&sq->wq); 516 517 wq_page_pfn = WQ_PAGE_PFN(wq_page_addr); 518 wq_page_pfn_hi = upper_32_bits(wq_page_pfn); 519 wq_page_pfn_lo = lower_32_bits(wq_page_pfn); 520 521 wq_block_pfn = WQ_BLOCK_PFN(sq->wq.wq_block_paddr); 522 wq_block_pfn_hi = upper_32_bits(wq_block_pfn); 523 wq_block_pfn_lo = lower_32_bits(wq_block_pfn); 524 525 sq_ctxt->ci_pi = 526 cpu_to_le32(SQ_CTXT_CI_PI_SET(ci_start, CI_IDX) | 527 SQ_CTXT_CI_PI_SET(pi_start, PI_IDX)); 528 529 sq_ctxt->drop_mode_sp = 530 cpu_to_le32(SQ_CTXT_MODE_SET(0, SP_FLAG) | 531 SQ_CTXT_MODE_SET(0, PKT_DROP)); 532 533 sq_ctxt->wq_pfn_hi_owner = 534 cpu_to_le32(SQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, HI_PFN) | 535 SQ_CTXT_WQ_PAGE_SET(1, OWNER)); 536 537 sq_ctxt->wq_pfn_lo = cpu_to_le32(wq_page_pfn_lo); 538 539 sq_ctxt->pkt_drop_thd = 540 cpu_to_le32(SQ_CTXT_PKT_DROP_THD_SET(HINIC3_DEFAULT_DROP_THD_ON, THD_ON) | 541 SQ_CTXT_PKT_DROP_THD_SET(HINIC3_DEFAULT_DROP_THD_OFF, THD_OFF)); 542 543 sq_ctxt->global_sq_id = 544 cpu_to_le32(SQ_CTXT_GLOBAL_QUEUE_ID_SET((u32)sq_id, 545 GLOBAL_SQ_ID)); 546 547 /* enable insert c-vlan by default */ 548 sq_ctxt->vlan_ceq_attr = 549 cpu_to_le32(SQ_CTXT_VLAN_CEQ_SET(0, CEQ_EN) | 550 SQ_CTXT_VLAN_CEQ_SET(1, INSERT_MODE)); 551 552 sq_ctxt->rsvd0 = 0; 553 554 sq_ctxt->pref_cache = 555 cpu_to_le32(SQ_CTXT_PREF_SET(SQ_WQ_PREFETCH_MIN, CACHE_MIN) | 556 SQ_CTXT_PREF_SET(SQ_WQ_PREFETCH_MAX, CACHE_MAX) | 557 SQ_CTXT_PREF_SET(SQ_WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD)); 558 559 sq_ctxt->pref_ci_owner = 560 cpu_to_le32(SQ_CTXT_PREF_SET(SQ_CTXT_PREF_CI_HI(ci_start), CI_HI) | 561 SQ_CTXT_PREF_SET(1, OWNER)); 562 563 sq_ctxt->pref_wq_pfn_hi_ci = 564 cpu_to_le32(SQ_CTXT_PREF_SET(ci_start, CI_LOW) | 565 SQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_PFN_HI)); 566 567 sq_ctxt->pref_wq_pfn_lo = cpu_to_le32(wq_page_pfn_lo); 568 569 sq_ctxt->wq_block_pfn_hi = 570 cpu_to_le32(SQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, PFN_HI)); 571 572 sq_ctxt->wq_block_pfn_lo = cpu_to_le32(wq_block_pfn_lo); 573 } 574 575 static void hinic3_rq_prepare_ctxt_get_wq_info(struct hinic3_io_queue *rq, 576 u32 *wq_page_pfn_hi, 577 u32 *wq_page_pfn_lo, 578 u32 *wq_block_pfn_hi, 579 u32 *wq_block_pfn_lo) 580 { 581 u64 wq_page_addr, wq_page_pfn, wq_block_pfn; 582 583 wq_page_addr = hinic3_wq_get_first_wqe_page_addr(&rq->wq); 584 585 wq_page_pfn = WQ_PAGE_PFN(wq_page_addr); 586 *wq_page_pfn_hi = upper_32_bits(wq_page_pfn); 587 *wq_page_pfn_lo = lower_32_bits(wq_page_pfn); 588 589 wq_block_pfn = WQ_BLOCK_PFN(rq->wq.wq_block_paddr); 590 *wq_block_pfn_hi = upper_32_bits(wq_block_pfn); 591 *wq_block_pfn_lo = lower_32_bits(wq_block_pfn); 592 } 593 594 static void hinic3_rq_prepare_ctxt(struct hinic3_io_queue *rq, 595 struct hinic3_rq_ctxt *rq_ctxt) 596 { 597 u32 wq_block_pfn_hi, wq_block_pfn_lo; 598 u32 wq_page_pfn_hi, wq_page_pfn_lo; 599 u16 pi_start, ci_start; 600 601 ci_start = (rq->wq.cons_idx & rq->wq.idx_mask) << HINIC3_NORMAL_RQ_WQE; 602 pi_start = (rq->wq.prod_idx & rq->wq.idx_mask) << HINIC3_NORMAL_RQ_WQE; 603 604 hinic3_rq_prepare_ctxt_get_wq_info(rq, &wq_page_pfn_hi, &wq_page_pfn_lo, 605 &wq_block_pfn_hi, &wq_block_pfn_lo); 606 607 rq_ctxt->ci_pi = 608 cpu_to_le32(RQ_CTXT_CI_PI_SET(ci_start, CI_IDX) | 609 RQ_CTXT_CI_PI_SET(pi_start, PI_IDX)); 610 611 rq_ctxt->ceq_attr = 612 cpu_to_le32(RQ_CTXT_CEQ_ATTR_SET(0, EN) | 613 RQ_CTXT_CEQ_ATTR_SET(rq->msix_entry_idx, INTR)); 614 615 rq_ctxt->wq_pfn_hi_type_owner = 616 cpu_to_le32(RQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, HI_PFN) | 617 RQ_CTXT_WQ_PAGE_SET(1, OWNER)); 618 619 /* use 16Byte WQE */ 620 rq_ctxt->wq_pfn_hi_type_owner |= 621 cpu_to_le32(RQ_CTXT_WQ_PAGE_SET(2, WQE_TYPE)); 622 rq_ctxt->cqe_sge_len = cpu_to_le32(RQ_CTXT_CQE_LEN_SET(1, CQE_LEN)); 623 624 rq_ctxt->wq_pfn_lo = cpu_to_le32(wq_page_pfn_lo); 625 626 rq_ctxt->pref_cache = 627 cpu_to_le32(RQ_CTXT_PREF_SET(RQ_WQ_PREFETCH_MIN, CACHE_MIN) | 628 RQ_CTXT_PREF_SET(RQ_WQ_PREFETCH_MAX, CACHE_MAX) | 629 RQ_CTXT_PREF_SET(RQ_WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD)); 630 631 rq_ctxt->pref_ci_owner = 632 cpu_to_le32(RQ_CTXT_PREF_SET(SQ_CTXT_PREF_CI_HI(ci_start), CI_HI) | 633 RQ_CTXT_PREF_SET(1, OWNER)); 634 635 rq_ctxt->pref_wq_pfn_hi_ci = 636 cpu_to_le32(RQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_PFN_HI) | 637 RQ_CTXT_PREF_SET(ci_start, CI_LOW)); 638 639 rq_ctxt->pref_wq_pfn_lo = cpu_to_le32(wq_page_pfn_lo); 640 641 rq_ctxt->wq_block_pfn_hi = 642 cpu_to_le32(RQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, PFN_HI)); 643 644 rq_ctxt->wq_block_pfn_lo = cpu_to_le32(wq_block_pfn_lo); 645 } 646 647 static int init_sq_ctxts(struct hinic3_nic_dev *nic_dev) 648 { 649 struct hinic3_nic_io *nic_io = nic_dev->nic_io; 650 struct hinic3_hwdev *hwdev = nic_dev->hwdev; 651 struct hinic3_sq_ctxt_block *sq_ctxt_block; 652 u16 q_id, curr_id, max_ctxts, i; 653 struct hinic3_sq_ctxt *sq_ctxt; 654 struct hinic3_cmd_buf *cmd_buf; 655 struct hinic3_io_queue *sq; 656 __le64 out_param; 657 int err = 0; 658 659 cmd_buf = hinic3_alloc_cmd_buf(hwdev); 660 if (!cmd_buf) { 661 dev_err(hwdev->dev, "Failed to allocate cmd buf\n"); 662 return -ENOMEM; 663 } 664 665 q_id = 0; 666 while (q_id < nic_io->num_qps) { 667 sq_ctxt_block = cmd_buf->buf; 668 sq_ctxt = sq_ctxt_block->sq_ctxt; 669 670 max_ctxts = (nic_io->num_qps - q_id) > HINIC3_Q_CTXT_MAX ? 671 HINIC3_Q_CTXT_MAX : (nic_io->num_qps - q_id); 672 673 hinic3_qp_prepare_cmdq_header(&sq_ctxt_block->cmdq_hdr, 674 HINIC3_QP_CTXT_TYPE_SQ, max_ctxts, 675 q_id); 676 677 for (i = 0; i < max_ctxts; i++) { 678 curr_id = q_id + i; 679 sq = &nic_io->sq[curr_id]; 680 hinic3_sq_prepare_ctxt(sq, curr_id, &sq_ctxt[i]); 681 } 682 683 hinic3_cmdq_buf_swab32(sq_ctxt_block, sizeof(*sq_ctxt_block)); 684 685 cmd_buf->size = cpu_to_le16(SQ_CTXT_SIZE(max_ctxts)); 686 err = hinic3_cmdq_direct_resp(hwdev, MGMT_MOD_L2NIC, 687 L2NIC_UCODE_CMD_MODIFY_QUEUE_CTX, 688 cmd_buf, &out_param); 689 if (err || out_param) { 690 dev_err(hwdev->dev, "Failed to set SQ ctxts, err: %d, out_param: 0x%llx\n", 691 err, out_param); 692 err = -EFAULT; 693 break; 694 } 695 696 q_id += max_ctxts; 697 } 698 699 hinic3_free_cmd_buf(hwdev, cmd_buf); 700 701 return err; 702 } 703 704 static int init_rq_ctxts(struct hinic3_nic_dev *nic_dev) 705 { 706 struct hinic3_nic_io *nic_io = nic_dev->nic_io; 707 struct hinic3_hwdev *hwdev = nic_dev->hwdev; 708 struct hinic3_rq_ctxt_block *rq_ctxt_block; 709 u16 q_id, curr_id, max_ctxts, i; 710 struct hinic3_rq_ctxt *rq_ctxt; 711 struct hinic3_cmd_buf *cmd_buf; 712 struct hinic3_io_queue *rq; 713 __le64 out_param; 714 int err = 0; 715 716 cmd_buf = hinic3_alloc_cmd_buf(hwdev); 717 if (!cmd_buf) { 718 dev_err(hwdev->dev, "Failed to allocate cmd buf\n"); 719 return -ENOMEM; 720 } 721 722 q_id = 0; 723 while (q_id < nic_io->num_qps) { 724 rq_ctxt_block = cmd_buf->buf; 725 rq_ctxt = rq_ctxt_block->rq_ctxt; 726 727 max_ctxts = (nic_io->num_qps - q_id) > HINIC3_Q_CTXT_MAX ? 728 HINIC3_Q_CTXT_MAX : (nic_io->num_qps - q_id); 729 730 hinic3_qp_prepare_cmdq_header(&rq_ctxt_block->cmdq_hdr, 731 HINIC3_QP_CTXT_TYPE_RQ, max_ctxts, 732 q_id); 733 734 for (i = 0; i < max_ctxts; i++) { 735 curr_id = q_id + i; 736 rq = &nic_io->rq[curr_id]; 737 hinic3_rq_prepare_ctxt(rq, &rq_ctxt[i]); 738 } 739 740 hinic3_cmdq_buf_swab32(rq_ctxt_block, sizeof(*rq_ctxt_block)); 741 742 cmd_buf->size = cpu_to_le16(RQ_CTXT_SIZE(max_ctxts)); 743 744 err = hinic3_cmdq_direct_resp(hwdev, MGMT_MOD_L2NIC, 745 L2NIC_UCODE_CMD_MODIFY_QUEUE_CTX, 746 cmd_buf, &out_param); 747 if (err || out_param) { 748 dev_err(hwdev->dev, "Failed to set RQ ctxts, err: %d, out_param: 0x%llx\n", 749 err, out_param); 750 err = -EFAULT; 751 break; 752 } 753 754 q_id += max_ctxts; 755 } 756 757 hinic3_free_cmd_buf(hwdev, cmd_buf); 758 759 return err; 760 } 761 762 static int init_qp_ctxts(struct hinic3_nic_dev *nic_dev) 763 { 764 int err; 765 766 err = init_sq_ctxts(nic_dev); 767 if (err) 768 return err; 769 770 err = init_rq_ctxts(nic_dev); 771 if (err) 772 return err; 773 774 return 0; 775 } 776 777 static int clean_queue_offload_ctxt(struct hinic3_nic_dev *nic_dev, 778 enum hinic3_qp_ctxt_type ctxt_type) 779 { 780 struct hinic3_nic_io *nic_io = nic_dev->nic_io; 781 struct hinic3_hwdev *hwdev = nic_dev->hwdev; 782 struct hinic3_clean_queue_ctxt *ctxt_block; 783 struct hinic3_cmd_buf *cmd_buf; 784 __le64 out_param; 785 int err; 786 787 cmd_buf = hinic3_alloc_cmd_buf(hwdev); 788 if (!cmd_buf) { 789 dev_err(hwdev->dev, "Failed to allocate cmd buf\n"); 790 return -ENOMEM; 791 } 792 793 ctxt_block = cmd_buf->buf; 794 ctxt_block->cmdq_hdr.num_queues = cpu_to_le16(nic_io->max_qps); 795 ctxt_block->cmdq_hdr.queue_type = cpu_to_le16(ctxt_type); 796 ctxt_block->cmdq_hdr.start_qid = 0; 797 ctxt_block->cmdq_hdr.rsvd = 0; 798 ctxt_block->rsvd = 0; 799 800 hinic3_cmdq_buf_swab32(ctxt_block, sizeof(*ctxt_block)); 801 802 cmd_buf->size = cpu_to_le16(sizeof(*ctxt_block)); 803 804 err = hinic3_cmdq_direct_resp(hwdev, MGMT_MOD_L2NIC, 805 L2NIC_UCODE_CMD_CLEAN_QUEUE_CTX, 806 cmd_buf, &out_param); 807 if (err || out_param) { 808 dev_err(hwdev->dev, "Failed to clean queue offload ctxts, err: %d,out_param: 0x%llx\n", 809 err, out_param); 810 811 err = -EFAULT; 812 } 813 814 hinic3_free_cmd_buf(hwdev, cmd_buf); 815 816 return err; 817 } 818 819 static int clean_qp_offload_ctxt(struct hinic3_nic_dev *nic_dev) 820 { 821 /* clean LRO/TSO context space */ 822 return clean_queue_offload_ctxt(nic_dev, HINIC3_QP_CTXT_TYPE_SQ) || 823 clean_queue_offload_ctxt(nic_dev, HINIC3_QP_CTXT_TYPE_RQ); 824 } 825 826 /* init qps ctxt and set sq ci attr and arm all sq */ 827 int hinic3_init_qp_ctxts(struct hinic3_nic_dev *nic_dev) 828 { 829 struct hinic3_nic_io *nic_io = nic_dev->nic_io; 830 struct hinic3_hwdev *hwdev = nic_dev->hwdev; 831 struct hinic3_sq_attr sq_attr; 832 u32 rq_depth; 833 u16 q_id; 834 int err; 835 836 err = init_qp_ctxts(nic_dev); 837 if (err) { 838 dev_err(hwdev->dev, "Failed to init QP ctxts\n"); 839 return err; 840 } 841 842 /* clean LRO/TSO context space */ 843 err = clean_qp_offload_ctxt(nic_dev); 844 if (err) { 845 dev_err(hwdev->dev, "Failed to clean qp offload ctxts\n"); 846 return err; 847 } 848 849 rq_depth = nic_io->rq[0].wq.q_depth << HINIC3_NORMAL_RQ_WQE; 850 851 err = hinic3_set_root_ctxt(hwdev, rq_depth, nic_io->sq[0].wq.q_depth, 852 nic_io->rx_buf_len); 853 if (err) { 854 dev_err(hwdev->dev, "Failed to set root context\n"); 855 return err; 856 } 857 858 for (q_id = 0; q_id < nic_io->num_qps; q_id++) { 859 sq_attr.ci_dma_base = 860 HINIC3_CI_PADDR(nic_io->ci_dma_base, q_id) >> 0x2; 861 sq_attr.pending_limit = HINIC3_DEFAULT_TX_CI_PENDING_LIMIT; 862 sq_attr.coalescing_time = HINIC3_DEFAULT_TX_CI_COALESCING_TIME; 863 sq_attr.intr_en = 1; 864 sq_attr.intr_idx = nic_io->sq[q_id].msix_entry_idx; 865 sq_attr.l2nic_sqn = q_id; 866 sq_attr.dma_attr_off = 0; 867 err = hinic3_set_ci_table(hwdev, &sq_attr); 868 if (err) { 869 dev_err(hwdev->dev, "Failed to set ci table\n"); 870 goto err_clean_root_ctxt; 871 } 872 } 873 874 return 0; 875 876 err_clean_root_ctxt: 877 hinic3_clean_root_ctxt(hwdev); 878 879 return err; 880 } 881 882 void hinic3_free_qp_ctxts(struct hinic3_nic_dev *nic_dev) 883 { 884 hinic3_clean_root_ctxt(nic_dev->hwdev); 885 } 886