1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2020 Marvell. */ 3 4 #include "otx2_cpt_common.h" 5 #include "otx2_cptlf.h" 6 #include "rvu_reg.h" 7 8 #define CPT_TIMER_HOLD 0x03F 9 #define CPT_COUNT_HOLD 32 10 11 static void cptlf_do_set_done_time_wait(struct otx2_cptlf_info *lf, 12 int time_wait) 13 { 14 union otx2_cptx_lf_done_wait done_wait; 15 16 done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 17 OTX2_CPT_LF_DONE_WAIT); 18 done_wait.s.time_wait = time_wait; 19 otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 20 OTX2_CPT_LF_DONE_WAIT, done_wait.u); 21 } 22 23 static void cptlf_do_set_done_num_wait(struct otx2_cptlf_info *lf, int num_wait) 24 { 25 union otx2_cptx_lf_done_wait done_wait; 26 27 done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 28 OTX2_CPT_LF_DONE_WAIT); 29 done_wait.s.num_wait = num_wait; 30 otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 31 OTX2_CPT_LF_DONE_WAIT, done_wait.u); 32 } 33 34 static void cptlf_set_done_time_wait(struct otx2_cptlfs_info *lfs, 35 int time_wait) 36 { 37 int slot; 38 39 for (slot = 0; slot < lfs->lfs_num; slot++) 40 cptlf_do_set_done_time_wait(&lfs->lf[slot], time_wait); 41 } 42 43 static void cptlf_set_done_num_wait(struct otx2_cptlfs_info *lfs, int num_wait) 44 { 45 int slot; 46 47 for (slot = 0; slot < lfs->lfs_num; slot++) 48 cptlf_do_set_done_num_wait(&lfs->lf[slot], num_wait); 49 } 50 51 static int cptlf_set_pri(struct otx2_cptlf_info *lf, int pri) 52 { 53 struct otx2_cptlfs_info *lfs = lf->lfs; 54 union otx2_cptx_af_lf_ctrl lf_ctrl; 55 int ret; 56 57 ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev, 58 CPT_AF_LFX_CTL(lf->slot), 59 &lf_ctrl.u, lfs->blkaddr); 60 if (ret) 61 return ret; 62 63 lf_ctrl.s.pri = pri ? 1 : 0; 64 65 ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev, 66 CPT_AF_LFX_CTL(lf->slot), 67 lf_ctrl.u, lfs->blkaddr); 68 return ret; 69 } 70 71 static int cptlf_set_eng_grps_mask(struct otx2_cptlf_info *lf, 72 int eng_grps_mask) 73 { 74 struct otx2_cptlfs_info *lfs = lf->lfs; 75 union otx2_cptx_af_lf_ctrl lf_ctrl; 76 int ret; 77 78 ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev, 79 CPT_AF_LFX_CTL(lf->slot), 80 &lf_ctrl.u, lfs->blkaddr); 81 if (ret) 82 return ret; 83 84 lf_ctrl.s.grp = eng_grps_mask; 85 86 ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev, 87 CPT_AF_LFX_CTL(lf->slot), 88 lf_ctrl.u, lfs->blkaddr); 89 return ret; 90 } 91 92 static int cptlf_set_grp_and_pri(struct otx2_cptlfs_info *lfs, 93 int eng_grp_mask, int pri) 94 { 95 int slot, ret = 0; 96 97 for (slot = 0; slot < lfs->lfs_num; slot++) { 98 ret = cptlf_set_pri(&lfs->lf[slot], pri); 99 if (ret) 100 return ret; 101 102 ret = cptlf_set_eng_grps_mask(&lfs->lf[slot], eng_grp_mask); 103 if (ret) 104 return ret; 105 } 106 return ret; 107 } 108 109 static void cptlf_hw_init(struct otx2_cptlfs_info *lfs) 110 { 111 /* Disable instruction queues */ 112 otx2_cptlf_disable_iqueues(lfs); 113 114 /* Set instruction queues base addresses */ 115 otx2_cptlf_set_iqueues_base_addr(lfs); 116 117 /* Set instruction queues sizes */ 118 otx2_cptlf_set_iqueues_size(lfs); 119 120 /* Set done interrupts time wait */ 121 cptlf_set_done_time_wait(lfs, CPT_TIMER_HOLD); 122 123 /* Set done interrupts num wait */ 124 cptlf_set_done_num_wait(lfs, CPT_COUNT_HOLD); 125 126 /* Enable instruction queues */ 127 otx2_cptlf_enable_iqueues(lfs); 128 } 129 130 static void cptlf_hw_cleanup(struct otx2_cptlfs_info *lfs) 131 { 132 /* Disable instruction queues */ 133 otx2_cptlf_disable_iqueues(lfs); 134 } 135 136 static void cptlf_set_misc_intrs(struct otx2_cptlfs_info *lfs, u8 enable) 137 { 138 union otx2_cptx_lf_misc_int_ena_w1s irq_misc = { .u = 0x0 }; 139 u64 reg = enable ? OTX2_CPT_LF_MISC_INT_ENA_W1S : 140 OTX2_CPT_LF_MISC_INT_ENA_W1C; 141 int slot; 142 143 irq_misc.s.fault = 0x1; 144 irq_misc.s.hwerr = 0x1; 145 irq_misc.s.irde = 0x1; 146 irq_misc.s.nqerr = 0x1; 147 irq_misc.s.nwrp = 0x1; 148 149 for (slot = 0; slot < lfs->lfs_num; slot++) 150 otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, reg, 151 irq_misc.u); 152 } 153 154 static void cptlf_enable_intrs(struct otx2_cptlfs_info *lfs) 155 { 156 int slot; 157 158 /* Enable done interrupts */ 159 for (slot = 0; slot < lfs->lfs_num; slot++) 160 otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, 161 OTX2_CPT_LF_DONE_INT_ENA_W1S, 0x1); 162 /* Enable Misc interrupts */ 163 cptlf_set_misc_intrs(lfs, true); 164 } 165 166 static void cptlf_disable_intrs(struct otx2_cptlfs_info *lfs) 167 { 168 int slot; 169 170 for (slot = 0; slot < lfs->lfs_num; slot++) 171 otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, 172 OTX2_CPT_LF_DONE_INT_ENA_W1C, 0x1); 173 cptlf_set_misc_intrs(lfs, false); 174 } 175 176 static inline int cptlf_read_done_cnt(struct otx2_cptlf_info *lf) 177 { 178 union otx2_cptx_lf_done irq_cnt; 179 180 irq_cnt.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 181 OTX2_CPT_LF_DONE); 182 return irq_cnt.s.done; 183 } 184 185 static irqreturn_t cptlf_misc_intr_handler(int __always_unused irq, void *arg) 186 { 187 union otx2_cptx_lf_misc_int irq_misc, irq_misc_ack; 188 struct otx2_cptlf_info *lf = arg; 189 struct device *dev; 190 191 dev = &lf->lfs->pdev->dev; 192 irq_misc.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 193 OTX2_CPT_LF_MISC_INT); 194 irq_misc_ack.u = 0x0; 195 196 if (irq_misc.s.fault) { 197 dev_err(dev, "Memory error detected while executing CPT_INST_S, LF %d.\n", 198 lf->slot); 199 irq_misc_ack.s.fault = 0x1; 200 201 } else if (irq_misc.s.hwerr) { 202 dev_err(dev, "HW error from an engine executing CPT_INST_S, LF %d.", 203 lf->slot); 204 irq_misc_ack.s.hwerr = 0x1; 205 206 } else if (irq_misc.s.nwrp) { 207 dev_err(dev, "SMMU fault while writing CPT_RES_S to CPT_INST_S[RES_ADDR], LF %d.\n", 208 lf->slot); 209 irq_misc_ack.s.nwrp = 0x1; 210 211 } else if (irq_misc.s.irde) { 212 dev_err(dev, "Memory error when accessing instruction memory queue CPT_LF_Q_BASE[ADDR].\n"); 213 irq_misc_ack.s.irde = 0x1; 214 215 } else if (irq_misc.s.nqerr) { 216 dev_err(dev, "Error enqueuing an instruction received at CPT_LF_NQ.\n"); 217 irq_misc_ack.s.nqerr = 0x1; 218 219 } else { 220 dev_err(dev, "Unhandled interrupt in CPT LF %d\n", lf->slot); 221 return IRQ_NONE; 222 } 223 224 /* Acknowledge interrupts */ 225 otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 226 OTX2_CPT_LF_MISC_INT, irq_misc_ack.u); 227 228 return IRQ_HANDLED; 229 } 230 231 static irqreturn_t cptlf_done_intr_handler(int irq, void *arg) 232 { 233 union otx2_cptx_lf_done_wait done_wait; 234 struct otx2_cptlf_info *lf = arg; 235 int irq_cnt; 236 237 /* Read the number of completed requests */ 238 irq_cnt = cptlf_read_done_cnt(lf); 239 if (irq_cnt) { 240 done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, 241 lf->slot, OTX2_CPT_LF_DONE_WAIT); 242 /* Acknowledge the number of completed requests */ 243 otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 244 OTX2_CPT_LF_DONE_ACK, irq_cnt); 245 246 otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, 247 OTX2_CPT_LF_DONE_WAIT, done_wait.u); 248 if (unlikely(!lf->wqe)) { 249 dev_err(&lf->lfs->pdev->dev, "No work for LF %d\n", 250 lf->slot); 251 return IRQ_NONE; 252 } 253 254 /* Schedule processing of completed requests */ 255 tasklet_hi_schedule(&lf->wqe->work); 256 } 257 return IRQ_HANDLED; 258 } 259 260 void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs) 261 { 262 int i, offs, vector; 263 264 for (i = 0; i < lfs->lfs_num; i++) { 265 for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) { 266 if (!lfs->lf[i].is_irq_reg[offs]) 267 continue; 268 269 vector = pci_irq_vector(lfs->pdev, 270 lfs->lf[i].msix_offset + offs); 271 free_irq(vector, &lfs->lf[i]); 272 lfs->lf[i].is_irq_reg[offs] = false; 273 } 274 } 275 cptlf_disable_intrs(lfs); 276 } 277 278 static int cptlf_do_register_interrrupts(struct otx2_cptlfs_info *lfs, 279 int lf_num, int irq_offset, 280 irq_handler_t handler) 281 { 282 int ret, vector; 283 284 vector = pci_irq_vector(lfs->pdev, lfs->lf[lf_num].msix_offset + 285 irq_offset); 286 ret = request_irq(vector, handler, 0, 287 lfs->lf[lf_num].irq_name[irq_offset], 288 &lfs->lf[lf_num]); 289 if (ret) 290 return ret; 291 292 lfs->lf[lf_num].is_irq_reg[irq_offset] = true; 293 294 return ret; 295 } 296 297 int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs) 298 { 299 int irq_offs, ret, i; 300 301 for (i = 0; i < lfs->lfs_num; i++) { 302 irq_offs = OTX2_CPT_LF_INT_VEC_E_MISC; 303 snprintf(lfs->lf[i].irq_name[irq_offs], 32, "CPTLF Misc%d", i); 304 ret = cptlf_do_register_interrrupts(lfs, i, irq_offs, 305 cptlf_misc_intr_handler); 306 if (ret) 307 goto free_irq; 308 309 irq_offs = OTX2_CPT_LF_INT_VEC_E_DONE; 310 snprintf(lfs->lf[i].irq_name[irq_offs], 32, "OTX2_CPTLF Done%d", 311 i); 312 ret = cptlf_do_register_interrrupts(lfs, i, irq_offs, 313 cptlf_done_intr_handler); 314 if (ret) 315 goto free_irq; 316 } 317 cptlf_enable_intrs(lfs); 318 return 0; 319 320 free_irq: 321 otx2_cptlf_unregister_interrupts(lfs); 322 return ret; 323 } 324 325 void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs) 326 { 327 int slot, offs; 328 329 for (slot = 0; slot < lfs->lfs_num; slot++) { 330 for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) 331 irq_set_affinity_hint(pci_irq_vector(lfs->pdev, 332 lfs->lf[slot].msix_offset + 333 offs), NULL); 334 free_cpumask_var(lfs->lf[slot].affinity_mask); 335 } 336 } 337 338 int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs) 339 { 340 struct otx2_cptlf_info *lf = lfs->lf; 341 int slot, offs, ret; 342 343 for (slot = 0; slot < lfs->lfs_num; slot++) { 344 if (!zalloc_cpumask_var(&lf[slot].affinity_mask, GFP_KERNEL)) { 345 dev_err(&lfs->pdev->dev, 346 "cpumask allocation failed for LF %d", slot); 347 ret = -ENOMEM; 348 goto free_affinity_mask; 349 } 350 351 cpumask_set_cpu(cpumask_local_spread(slot, 352 dev_to_node(&lfs->pdev->dev)), 353 lf[slot].affinity_mask); 354 355 for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) { 356 ret = irq_set_affinity_hint(pci_irq_vector(lfs->pdev, 357 lf[slot].msix_offset + offs), 358 lf[slot].affinity_mask); 359 if (ret) 360 goto free_affinity_mask; 361 } 362 } 363 return 0; 364 365 free_affinity_mask: 366 otx2_cptlf_free_irqs_affinity(lfs); 367 return ret; 368 } 369 370 int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_mask, int pri, 371 int lfs_num) 372 { 373 int slot, ret; 374 375 if (!lfs->pdev || !lfs->reg_base) 376 return -EINVAL; 377 378 lfs->lfs_num = lfs_num; 379 for (slot = 0; slot < lfs->lfs_num; slot++) { 380 lfs->lf[slot].lfs = lfs; 381 lfs->lf[slot].slot = slot; 382 if (lfs->lmt_base) 383 lfs->lf[slot].lmtline = lfs->lmt_base + 384 (slot * LMTLINE_SIZE); 385 else 386 lfs->lf[slot].lmtline = lfs->reg_base + 387 OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_LMT, slot, 388 OTX2_CPT_LMT_LF_LMTLINEX(0)); 389 390 lfs->lf[slot].ioreg = lfs->reg_base + 391 OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_CPT0, slot, 392 OTX2_CPT_LF_NQX(0)); 393 } 394 /* Send request to attach LFs */ 395 ret = otx2_cpt_attach_rscrs_msg(lfs); 396 if (ret) 397 goto clear_lfs_num; 398 399 ret = otx2_cpt_alloc_instruction_queues(lfs); 400 if (ret) { 401 dev_err(&lfs->pdev->dev, 402 "Allocating instruction queues failed\n"); 403 goto detach_rsrcs; 404 } 405 cptlf_hw_init(lfs); 406 /* 407 * Allow each LF to execute requests destined to any of 8 engine 408 * groups and set queue priority of each LF to high 409 */ 410 ret = cptlf_set_grp_and_pri(lfs, eng_grp_mask, pri); 411 if (ret) 412 goto free_iq; 413 414 return 0; 415 416 free_iq: 417 otx2_cpt_free_instruction_queues(lfs); 418 cptlf_hw_cleanup(lfs); 419 detach_rsrcs: 420 otx2_cpt_detach_rsrcs_msg(lfs); 421 clear_lfs_num: 422 lfs->lfs_num = 0; 423 return ret; 424 } 425 426 void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs) 427 { 428 lfs->lfs_num = 0; 429 /* Cleanup LFs hardware side */ 430 cptlf_hw_cleanup(lfs); 431 /* Send request to detach LFs */ 432 otx2_cpt_detach_rsrcs_msg(lfs); 433 } 434