1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. 3 4 #include <linux/bitfield.h> 5 #include <linux/device.h> 6 #include <linux/io.h> 7 8 #include "hinic3_common.h" 9 #include "hinic3_csr.h" 10 #include "hinic3_hwdev.h" 11 #include "hinic3_hwif.h" 12 13 #define HINIC3_HWIF_READY_TIMEOUT 10000 14 #define HINIC3_DB_AND_OUTBOUND_EN_TIMEOUT 60000 15 #define HINIC3_PCIE_LINK_DOWN 0xFFFFFFFF 16 17 /* config BAR4/5 4MB, DB & DWQE both 2MB */ 18 #define HINIC3_DB_DWQE_SIZE 0x00400000 19 20 /* db/dwqe page size: 4K */ 21 #define HINIC3_DB_PAGE_SIZE 0x00001000 22 #define HINIC3_DWQE_OFFSET 0x00000800 23 #define HINIC3_DB_MAX_AREAS (HINIC3_DB_DWQE_SIZE / HINIC3_DB_PAGE_SIZE) 24 25 #define HINIC3_MAX_MSIX_ENTRY 2048 26 27 #define HINIC3_AF0_FUNC_GLOBAL_IDX_MASK GENMASK(11, 0) 28 #define HINIC3_AF0_P2P_IDX_MASK GENMASK(16, 12) 29 #define HINIC3_AF0_PCI_INTF_IDX_MASK GENMASK(19, 17) 30 #define HINIC3_AF0_FUNC_TYPE_MASK BIT(28) 31 #define HINIC3_AF0_GET(val, member) \ 32 FIELD_GET(HINIC3_AF0_##member##_MASK, val) 33 34 #define HINIC3_AF1_AEQS_PER_FUNC_MASK GENMASK(9, 8) 35 #define HINIC3_AF1_MGMT_INIT_STATUS_MASK BIT(30) 36 #define HINIC3_AF1_GET(val, member) \ 37 FIELD_GET(HINIC3_AF1_##member##_MASK, val) 38 39 #define HINIC3_AF2_CEQS_PER_FUNC_MASK GENMASK(8, 0) 40 #define HINIC3_AF2_IRQS_PER_FUNC_MASK GENMASK(26, 16) 41 #define HINIC3_AF2_GET(val, member) \ 42 FIELD_GET(HINIC3_AF2_##member##_MASK, val) 43 44 #define HINIC3_AF4_DOORBELL_CTRL_MASK BIT(0) 45 #define HINIC3_AF4_GET(val, member) \ 46 FIELD_GET(HINIC3_AF4_##member##_MASK, val) 47 #define HINIC3_AF4_SET(val, member) \ 48 FIELD_PREP(HINIC3_AF4_##member##_MASK, val) 49 50 #define HINIC3_AF5_OUTBOUND_CTRL_MASK BIT(0) 51 #define HINIC3_AF5_GET(val, member) \ 52 FIELD_GET(HINIC3_AF5_##member##_MASK, val) 53 54 #define HINIC3_AF6_PF_STATUS_MASK GENMASK(15, 0) 55 #define HINIC3_AF6_FUNC_MAX_SQ_MASK GENMASK(31, 23) 56 #define HINIC3_AF6_MSIX_FLEX_EN_MASK BIT(22) 57 #define HINIC3_AF6_GET(val, member) \ 58 FIELD_GET(HINIC3_AF6_##member##_MASK, val) 59 60 #define HINIC3_GET_REG_ADDR(reg) ((reg) & (HINIC3_REGS_FLAG_MASK)) 61 62 static void __iomem *hinic3_reg_addr(struct hinic3_hwif *hwif, u32 reg) 63 { 64 return hwif->cfg_regs_base + HINIC3_GET_REG_ADDR(reg); 65 } 66 67 u32 hinic3_hwif_read_reg(struct hinic3_hwif *hwif, u32 reg) 68 { 69 void __iomem *addr = hinic3_reg_addr(hwif, reg); 70 71 return ioread32be(addr); 72 } 73 74 void hinic3_hwif_write_reg(struct hinic3_hwif *hwif, u32 reg, u32 val) 75 { 76 void __iomem *addr = hinic3_reg_addr(hwif, reg); 77 78 iowrite32be(val, addr); 79 } 80 81 static enum hinic3_wait_return check_hwif_ready_handler(void *priv_data) 82 { 83 struct hinic3_hwdev *hwdev = priv_data; 84 u32 attr1; 85 86 attr1 = hinic3_hwif_read_reg(hwdev->hwif, HINIC3_CSR_FUNC_ATTR1_ADDR); 87 88 return HINIC3_AF1_GET(attr1, MGMT_INIT_STATUS) ? 89 HINIC3_WAIT_PROCESS_CPL : HINIC3_WAIT_PROCESS_WAITING; 90 } 91 92 static int wait_hwif_ready(struct hinic3_hwdev *hwdev) 93 { 94 return hinic3_wait_for_timeout(hwdev, check_hwif_ready_handler, 95 HINIC3_HWIF_READY_TIMEOUT, 96 USEC_PER_MSEC); 97 } 98 99 /* Set attr struct from HW attr values. */ 100 static void set_hwif_attr(struct hinic3_func_attr *attr, u32 attr0, u32 attr1, 101 u32 attr2, u32 attr3, u32 attr6) 102 { 103 attr->func_global_idx = HINIC3_AF0_GET(attr0, FUNC_GLOBAL_IDX); 104 attr->port_to_port_idx = HINIC3_AF0_GET(attr0, P2P_IDX); 105 attr->pci_intf_idx = HINIC3_AF0_GET(attr0, PCI_INTF_IDX); 106 attr->func_type = HINIC3_AF0_GET(attr0, FUNC_TYPE); 107 108 attr->num_aeqs = BIT(HINIC3_AF1_GET(attr1, AEQS_PER_FUNC)); 109 attr->num_ceqs = HINIC3_AF2_GET(attr2, CEQS_PER_FUNC); 110 attr->num_irqs = HINIC3_AF2_GET(attr2, IRQS_PER_FUNC); 111 if (attr->num_irqs > HINIC3_MAX_MSIX_ENTRY) 112 attr->num_irqs = HINIC3_MAX_MSIX_ENTRY; 113 114 attr->num_sq = HINIC3_AF6_GET(attr6, FUNC_MAX_SQ); 115 attr->msix_flex_en = HINIC3_AF6_GET(attr6, MSIX_FLEX_EN); 116 } 117 118 /* Read attributes from HW and set attribute struct. */ 119 static int init_hwif_attr(struct hinic3_hwdev *hwdev) 120 { 121 u32 attr0, attr1, attr2, attr3, attr6; 122 struct hinic3_hwif *hwif; 123 124 hwif = hwdev->hwif; 125 attr0 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR0_ADDR); 126 if (attr0 == HINIC3_PCIE_LINK_DOWN) 127 return -EFAULT; 128 129 attr1 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR1_ADDR); 130 if (attr1 == HINIC3_PCIE_LINK_DOWN) 131 return -EFAULT; 132 133 attr2 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR2_ADDR); 134 if (attr2 == HINIC3_PCIE_LINK_DOWN) 135 return -EFAULT; 136 137 attr3 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR3_ADDR); 138 if (attr3 == HINIC3_PCIE_LINK_DOWN) 139 return -EFAULT; 140 141 attr6 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR6_ADDR); 142 if (attr6 == HINIC3_PCIE_LINK_DOWN) 143 return -EFAULT; 144 145 set_hwif_attr(&hwif->attr, attr0, attr1, attr2, attr3, attr6); 146 147 if (!hwif->attr.num_ceqs) { 148 dev_err(hwdev->dev, "Ceq num cfg in fw is zero\n"); 149 return -EFAULT; 150 } 151 152 if (!hwif->attr.num_irqs) { 153 dev_err(hwdev->dev, 154 "Irq num cfg in fw is zero, msix_flex_en %d\n", 155 hwif->attr.msix_flex_en); 156 return -EFAULT; 157 } 158 159 return 0; 160 } 161 162 static enum hinic3_doorbell_ctrl hinic3_get_doorbell_ctrl_status(struct hinic3_hwif *hwif) 163 { 164 u32 attr4 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR4_ADDR); 165 166 return HINIC3_AF4_GET(attr4, DOORBELL_CTRL); 167 } 168 169 static enum hinic3_outbound_ctrl hinic3_get_outbound_ctrl_status(struct hinic3_hwif *hwif) 170 { 171 u32 attr5 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR5_ADDR); 172 173 return HINIC3_AF5_GET(attr5, OUTBOUND_CTRL); 174 } 175 176 void hinic3_toggle_doorbell(struct hinic3_hwif *hwif, 177 enum hinic3_doorbell_ctrl flag) 178 { 179 u32 addr, attr4; 180 181 addr = HINIC3_CSR_FUNC_ATTR4_ADDR; 182 attr4 = hinic3_hwif_read_reg(hwif, addr); 183 184 attr4 &= ~HINIC3_AF4_DOORBELL_CTRL_MASK; 185 attr4 |= HINIC3_AF4_SET(flag, DOORBELL_CTRL); 186 187 hinic3_hwif_write_reg(hwif, addr, attr4); 188 } 189 190 static int db_area_idx_init(struct hinic3_hwif *hwif, u64 db_base_phy, 191 u8 __iomem *db_base, u64 db_dwqe_len) 192 { 193 struct hinic3_db_area *db_area = &hwif->db_area; 194 u32 db_max_areas; 195 196 hwif->db_base_phy = db_base_phy; 197 hwif->db_base = db_base; 198 hwif->db_dwqe_len = db_dwqe_len; 199 200 db_max_areas = db_dwqe_len > HINIC3_DB_DWQE_SIZE ? 201 HINIC3_DB_MAX_AREAS : db_dwqe_len / HINIC3_DB_PAGE_SIZE; 202 db_area->db_bitmap_array = bitmap_zalloc(db_max_areas, GFP_KERNEL); 203 if (!db_area->db_bitmap_array) 204 return -ENOMEM; 205 206 db_area->db_max_areas = db_max_areas; 207 spin_lock_init(&db_area->idx_lock); 208 209 return 0; 210 } 211 212 static void db_area_idx_free(struct hinic3_db_area *db_area) 213 { 214 bitmap_free(db_area->db_bitmap_array); 215 } 216 217 static int get_db_idx(struct hinic3_hwif *hwif, u32 *idx) 218 { 219 struct hinic3_db_area *db_area = &hwif->db_area; 220 u32 pg_idx; 221 222 spin_lock(&db_area->idx_lock); 223 pg_idx = find_first_zero_bit(db_area->db_bitmap_array, 224 db_area->db_max_areas); 225 if (pg_idx == db_area->db_max_areas) { 226 spin_unlock(&db_area->idx_lock); 227 return -ENOMEM; 228 } 229 set_bit(pg_idx, db_area->db_bitmap_array); 230 spin_unlock(&db_area->idx_lock); 231 232 *idx = pg_idx; 233 234 return 0; 235 } 236 237 static void free_db_idx(struct hinic3_hwif *hwif, u32 idx) 238 { 239 struct hinic3_db_area *db_area = &hwif->db_area; 240 241 spin_lock(&db_area->idx_lock); 242 clear_bit(idx, db_area->db_bitmap_array); 243 spin_unlock(&db_area->idx_lock); 244 } 245 246 void hinic3_free_db_addr(struct hinic3_hwdev *hwdev, const u8 __iomem *db_base) 247 { 248 struct hinic3_hwif *hwif; 249 uintptr_t distance; 250 u32 idx; 251 252 hwif = hwdev->hwif; 253 distance = db_base - hwif->db_base; 254 idx = distance / HINIC3_DB_PAGE_SIZE; 255 256 free_db_idx(hwif, idx); 257 } 258 259 int hinic3_alloc_db_addr(struct hinic3_hwdev *hwdev, void __iomem **db_base, 260 void __iomem **dwqe_base) 261 { 262 struct hinic3_hwif *hwif; 263 u8 __iomem *addr; 264 u32 idx; 265 int err; 266 267 hwif = hwdev->hwif; 268 269 err = get_db_idx(hwif, &idx); 270 if (err) 271 return err; 272 273 addr = hwif->db_base + idx * HINIC3_DB_PAGE_SIZE; 274 *db_base = addr; 275 276 if (dwqe_base) 277 *dwqe_base = addr + HINIC3_DWQE_OFFSET; 278 279 return 0; 280 } 281 282 void hinic3_set_msix_state(struct hinic3_hwdev *hwdev, u16 msix_idx, 283 enum hinic3_msix_state flag) 284 { 285 struct hinic3_hwif *hwif; 286 u8 int_msk = 1; 287 u32 mask_bits; 288 u32 addr; 289 290 hwif = hwdev->hwif; 291 292 if (flag) 293 mask_bits = HINIC3_MSI_CLR_INDIR_SET(int_msk, INT_MSK_SET); 294 else 295 mask_bits = HINIC3_MSI_CLR_INDIR_SET(int_msk, INT_MSK_CLR); 296 mask_bits = mask_bits | 297 HINIC3_MSI_CLR_INDIR_SET(msix_idx, SIMPLE_INDIR_IDX); 298 299 addr = HINIC3_CSR_FUNC_MSI_CLR_WR_ADDR; 300 hinic3_hwif_write_reg(hwif, addr, mask_bits); 301 } 302 303 static void disable_all_msix(struct hinic3_hwdev *hwdev) 304 { 305 u16 num_irqs = hwdev->hwif->attr.num_irqs; 306 u16 i; 307 308 for (i = 0; i < num_irqs; i++) 309 hinic3_set_msix_state(hwdev, i, HINIC3_MSIX_DISABLE); 310 } 311 312 void hinic3_msix_intr_clear_resend_bit(struct hinic3_hwdev *hwdev, u16 msix_idx, 313 u8 clear_resend_en) 314 { 315 struct hinic3_hwif *hwif; 316 u32 msix_ctrl, addr; 317 318 hwif = hwdev->hwif; 319 320 msix_ctrl = HINIC3_MSI_CLR_INDIR_SET(msix_idx, SIMPLE_INDIR_IDX) | 321 HINIC3_MSI_CLR_INDIR_SET(clear_resend_en, RESEND_TIMER_CLR); 322 323 addr = HINIC3_CSR_FUNC_MSI_CLR_WR_ADDR; 324 hinic3_hwif_write_reg(hwif, addr, msix_ctrl); 325 } 326 327 void hinic3_set_msix_auto_mask_state(struct hinic3_hwdev *hwdev, u16 msix_idx, 328 enum hinic3_msix_auto_mask flag) 329 { 330 struct hinic3_hwif *hwif; 331 u32 mask_bits; 332 u32 addr; 333 334 hwif = hwdev->hwif; 335 336 if (flag) 337 mask_bits = HINIC3_MSI_CLR_INDIR_SET(1, AUTO_MSK_SET); 338 else 339 mask_bits = HINIC3_MSI_CLR_INDIR_SET(1, AUTO_MSK_CLR); 340 341 mask_bits = mask_bits | 342 HINIC3_MSI_CLR_INDIR_SET(msix_idx, SIMPLE_INDIR_IDX); 343 344 addr = HINIC3_CSR_FUNC_MSI_CLR_WR_ADDR; 345 hinic3_hwif_write_reg(hwif, addr, mask_bits); 346 } 347 348 static enum hinic3_wait_return check_db_outbound_enable_handler(void *priv_data) 349 { 350 enum hinic3_outbound_ctrl outbound_ctrl; 351 struct hinic3_hwif *hwif = priv_data; 352 enum hinic3_doorbell_ctrl db_ctrl; 353 354 db_ctrl = hinic3_get_doorbell_ctrl_status(hwif); 355 outbound_ctrl = hinic3_get_outbound_ctrl_status(hwif); 356 if (outbound_ctrl == ENABLE_OUTBOUND && db_ctrl == ENABLE_DOORBELL) 357 return HINIC3_WAIT_PROCESS_CPL; 358 359 return HINIC3_WAIT_PROCESS_WAITING; 360 } 361 362 static int wait_until_doorbell_and_outbound_enabled(struct hinic3_hwif *hwif) 363 { 364 return hinic3_wait_for_timeout(hwif, check_db_outbound_enable_handler, 365 HINIC3_DB_AND_OUTBOUND_EN_TIMEOUT, 366 USEC_PER_MSEC); 367 } 368 369 int hinic3_init_hwif(struct hinic3_hwdev *hwdev) 370 { 371 struct hinic3_pcidev *pci_adapter = hwdev->adapter; 372 struct hinic3_hwif *hwif; 373 u32 attr1, attr4, attr5; 374 int err; 375 376 hwif = kzalloc(sizeof(*hwif), GFP_KERNEL); 377 if (!hwif) 378 return -ENOMEM; 379 380 hwdev->hwif = hwif; 381 hwif->cfg_regs_base = (u8 __iomem *)pci_adapter->cfg_reg_base + 382 HINIC3_VF_CFG_REG_OFFSET; 383 384 err = db_area_idx_init(hwif, pci_adapter->db_base_phy, 385 pci_adapter->db_base, 386 pci_adapter->db_dwqe_len); 387 if (err) { 388 dev_err(hwdev->dev, "Failed to init db area.\n"); 389 goto err_free_hwif; 390 } 391 392 err = wait_hwif_ready(hwdev); 393 if (err) { 394 attr1 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR1_ADDR); 395 dev_err(hwdev->dev, "Chip status is not ready, attr1:0x%x\n", 396 attr1); 397 goto err_free_db_area_idx; 398 } 399 400 err = init_hwif_attr(hwdev); 401 if (err) { 402 dev_err(hwdev->dev, "Init hwif attr failed\n"); 403 goto err_free_db_area_idx; 404 } 405 406 err = wait_until_doorbell_and_outbound_enabled(hwif); 407 if (err) { 408 attr4 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR4_ADDR); 409 attr5 = hinic3_hwif_read_reg(hwif, HINIC3_CSR_FUNC_ATTR5_ADDR); 410 dev_err(hwdev->dev, "HW doorbell/outbound is disabled, attr4 0x%x attr5 0x%x\n", 411 attr4, attr5); 412 goto err_free_db_area_idx; 413 } 414 415 disable_all_msix(hwdev); 416 417 return 0; 418 419 err_free_db_area_idx: 420 db_area_idx_free(&hwif->db_area); 421 err_free_hwif: 422 kfree(hwif); 423 424 return err; 425 } 426 427 void hinic3_free_hwif(struct hinic3_hwdev *hwdev) 428 { 429 db_area_idx_free(&hwdev->hwif->db_area); 430 kfree(hwdev->hwif); 431 } 432 433 u16 hinic3_global_func_id(struct hinic3_hwdev *hwdev) 434 { 435 return hwdev->hwif->attr.func_global_idx; 436 } 437