1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 2 /* Copyright (c) 2015 - 2024 Intel Corporation */ 3 4 #include "main.h" 5 #include <linux/net/intel/iidc_rdma_ice.h> 6 7 static void icrdma_prep_tc_change(struct irdma_device *iwdev) 8 { 9 iwdev->vsi.tc_change_pending = true; 10 irdma_sc_suspend_resume_qps(&iwdev->vsi, IRDMA_OP_SUSPEND); 11 12 /* Wait for all qp's to suspend */ 13 wait_event_timeout(iwdev->suspend_wq, 14 !atomic_read(&iwdev->vsi.qp_suspend_reqs), 15 msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS)); 16 irdma_ws_reset(&iwdev->vsi); 17 } 18 19 static void icrdma_fill_qos_info(struct irdma_l2params *l2params, 20 struct iidc_rdma_qos_params *qos_info) 21 { 22 int i; 23 24 l2params->num_tc = qos_info->num_tc; 25 l2params->vsi_prio_type = qos_info->vport_priority_type; 26 l2params->vsi_rel_bw = qos_info->vport_relative_bw; 27 for (i = 0; i < l2params->num_tc; i++) { 28 l2params->tc_info[i].egress_virt_up = 29 qos_info->tc_info[i].egress_virt_up; 30 l2params->tc_info[i].ingress_virt_up = 31 qos_info->tc_info[i].ingress_virt_up; 32 l2params->tc_info[i].prio_type = qos_info->tc_info[i].prio_type; 33 l2params->tc_info[i].rel_bw = qos_info->tc_info[i].rel_bw; 34 l2params->tc_info[i].tc_ctx = qos_info->tc_info[i].tc_ctx; 35 } 36 for (i = 0; i < IIDC_MAX_USER_PRIORITY; i++) 37 l2params->up2tc[i] = qos_info->up2tc[i]; 38 if (qos_info->pfc_mode == IIDC_DSCP_PFC_MODE) { 39 l2params->dscp_mode = true; 40 memcpy(l2params->dscp_map, qos_info->dscp_map, sizeof(l2params->dscp_map)); 41 } 42 } 43 44 static void icrdma_iidc_event_handler(struct iidc_rdma_core_dev_info *cdev_info, 45 struct iidc_rdma_event *event) 46 { 47 struct irdma_device *iwdev = dev_get_drvdata(&cdev_info->adev->dev); 48 struct irdma_l2params l2params = {}; 49 50 if (*event->type & BIT(IIDC_RDMA_EVENT_AFTER_MTU_CHANGE)) { 51 ibdev_dbg(&iwdev->ibdev, "CLNT: new MTU = %d\n", iwdev->netdev->mtu); 52 if (iwdev->vsi.mtu != iwdev->netdev->mtu) { 53 l2params.mtu = iwdev->netdev->mtu; 54 l2params.mtu_changed = true; 55 irdma_log_invalid_mtu(l2params.mtu, &iwdev->rf->sc_dev); 56 irdma_change_l2params(&iwdev->vsi, &l2params); 57 } 58 } else if (*event->type & BIT(IIDC_RDMA_EVENT_BEFORE_TC_CHANGE)) { 59 if (iwdev->vsi.tc_change_pending) 60 return; 61 62 icrdma_prep_tc_change(iwdev); 63 } else if (*event->type & BIT(IIDC_RDMA_EVENT_AFTER_TC_CHANGE)) { 64 struct iidc_rdma_priv_dev_info *idc_priv = cdev_info->iidc_priv; 65 66 if (!iwdev->vsi.tc_change_pending) 67 return; 68 69 l2params.tc_changed = true; 70 ibdev_dbg(&iwdev->ibdev, "CLNT: TC Change\n"); 71 72 icrdma_fill_qos_info(&l2params, &idc_priv->qos_info); 73 if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) 74 iwdev->dcb_vlan_mode = 75 l2params.num_tc > 1 && !l2params.dscp_mode; 76 irdma_change_l2params(&iwdev->vsi, &l2params); 77 } else if (*event->type & BIT(IIDC_RDMA_EVENT_CRIT_ERR)) { 78 ibdev_warn(&iwdev->ibdev, "ICE OICR event notification: oicr = 0x%08x\n", 79 event->reg); 80 if (event->reg & IRDMAPFINT_OICR_PE_CRITERR_M) { 81 u32 pe_criterr; 82 83 pe_criterr = readl(iwdev->rf->sc_dev.hw_regs[IRDMA_GLPE_CRITERR]); 84 #define IRDMA_Q1_RESOURCE_ERR 0x0001024d 85 if (pe_criterr != IRDMA_Q1_RESOURCE_ERR) { 86 ibdev_err(&iwdev->ibdev, "critical PE Error, GLPE_CRITERR=0x%08x\n", 87 pe_criterr); 88 iwdev->rf->reset = true; 89 } else { 90 ibdev_warn(&iwdev->ibdev, "Q1 Resource Check\n"); 91 } 92 } 93 if (event->reg & IRDMAPFINT_OICR_HMC_ERR_M) { 94 ibdev_err(&iwdev->ibdev, "HMC Error\n"); 95 iwdev->rf->reset = true; 96 } 97 if (event->reg & IRDMAPFINT_OICR_PE_PUSH_M) { 98 ibdev_err(&iwdev->ibdev, "PE Push Error\n"); 99 iwdev->rf->reset = true; 100 } 101 if (iwdev->rf->reset) 102 iwdev->rf->gen_ops.request_reset(iwdev->rf); 103 } 104 } 105 106 /** 107 * icrdma_lan_register_qset - Register qset with LAN driver 108 * @vsi: vsi structure 109 * @tc_node: Traffic class node 110 */ 111 static int icrdma_lan_register_qset(struct irdma_sc_vsi *vsi, 112 struct irdma_ws_node *tc_node) 113 { 114 struct irdma_device *iwdev = vsi->back_vsi; 115 struct iidc_rdma_core_dev_info *cdev_info = iwdev->rf->cdev; 116 struct iidc_rdma_qset_params qset = {}; 117 int ret; 118 119 qset.qs_handle = tc_node->qs_handle; 120 qset.tc = tc_node->traffic_class; 121 qset.vport_id = vsi->vsi_idx; 122 ret = ice_add_rdma_qset(cdev_info, &qset); 123 if (ret) { 124 ibdev_dbg(&iwdev->ibdev, "WS: LAN alloc_res for rdma qset failed.\n"); 125 return ret; 126 } 127 128 tc_node->l2_sched_node_id = qset.teid; 129 vsi->qos[tc_node->user_pri].l2_sched_node_id = qset.teid; 130 131 return 0; 132 } 133 134 /** 135 * icrdma_lan_unregister_qset - Unregister qset with LAN driver 136 * @vsi: vsi structure 137 * @tc_node: Traffic class node 138 */ 139 static void icrdma_lan_unregister_qset(struct irdma_sc_vsi *vsi, 140 struct irdma_ws_node *tc_node) 141 { 142 struct irdma_device *iwdev = vsi->back_vsi; 143 struct iidc_rdma_core_dev_info *cdev_info = iwdev->rf->cdev; 144 struct iidc_rdma_qset_params qset = {}; 145 146 qset.qs_handle = tc_node->qs_handle; 147 qset.tc = tc_node->traffic_class; 148 qset.vport_id = vsi->vsi_idx; 149 qset.teid = tc_node->l2_sched_node_id; 150 151 if (ice_del_rdma_qset(cdev_info, &qset)) 152 ibdev_dbg(&iwdev->ibdev, "WS: LAN free_res for rdma qset failed.\n"); 153 } 154 155 /** 156 * icrdma_request_reset - Request a reset 157 * @rf: RDMA PCI function 158 */ 159 static void icrdma_request_reset(struct irdma_pci_f *rf) 160 { 161 ibdev_warn(&rf->iwdev->ibdev, "Requesting a reset\n"); 162 ice_rdma_request_reset(rf->cdev, IIDC_FUNC_RESET); 163 } 164 165 static int icrdma_init_interrupts(struct irdma_pci_f *rf, struct iidc_rdma_core_dev_info *cdev) 166 { 167 int i; 168 169 rf->msix_count = num_online_cpus() + IRDMA_NUM_AEQ_MSIX; 170 rf->msix_entries = kcalloc(rf->msix_count, sizeof(*rf->msix_entries), 171 GFP_KERNEL); 172 if (!rf->msix_entries) 173 return -ENOMEM; 174 175 for (i = 0; i < rf->msix_count; i++) 176 if (ice_alloc_rdma_qvector(cdev, &rf->msix_entries[i])) 177 break; 178 179 if (i < IRDMA_MIN_MSIX) { 180 while (--i >= 0) 181 ice_free_rdma_qvector(cdev, &rf->msix_entries[i]); 182 183 kfree(rf->msix_entries); 184 return -ENOMEM; 185 } 186 187 rf->msix_count = i; 188 189 return 0; 190 } 191 192 static void icrdma_deinit_interrupts(struct irdma_pci_f *rf, struct iidc_rdma_core_dev_info *cdev) 193 { 194 int i; 195 196 for (i = 0; i < rf->msix_count; i++) 197 ice_free_rdma_qvector(cdev, &rf->msix_entries[i]); 198 199 kfree(rf->msix_entries); 200 } 201 202 static void icrdma_fill_device_info(struct irdma_device *iwdev, 203 struct iidc_rdma_core_dev_info *cdev_info) 204 { 205 struct iidc_rdma_priv_dev_info *idc_priv = cdev_info->iidc_priv; 206 struct irdma_pci_f *rf = iwdev->rf; 207 208 rf->sc_dev.hw = &rf->hw; 209 rf->iwdev = iwdev; 210 rf->cdev = cdev_info; 211 rf->hw.hw_addr = idc_priv->hw_addr; 212 rf->pcidev = cdev_info->pdev; 213 rf->hw.device = &rf->pcidev->dev; 214 rf->pf_id = idc_priv->pf_id; 215 rf->rdma_ver = IRDMA_GEN_2; 216 rf->sc_dev.hw_attrs.uk_attrs.hw_rev = IRDMA_GEN_2; 217 rf->sc_dev.is_pf = true; 218 rf->sc_dev.privileged = true; 219 220 rf->gen_ops.register_qset = icrdma_lan_register_qset; 221 rf->gen_ops.unregister_qset = icrdma_lan_unregister_qset; 222 223 rf->default_vsi.vsi_idx = idc_priv->vport_id; 224 rf->protocol_used = 225 cdev_info->rdma_protocol == IIDC_RDMA_PROTOCOL_ROCEV2 ? 226 IRDMA_ROCE_PROTOCOL_ONLY : IRDMA_IWARP_PROTOCOL_ONLY; 227 rf->rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT; 228 rf->rst_to = IRDMA_RST_TIMEOUT_HZ; 229 rf->gen_ops.request_reset = icrdma_request_reset; 230 rf->limits_sel = 7; 231 mutex_init(&rf->ah_tbl_lock); 232 233 iwdev->netdev = idc_priv->netdev; 234 iwdev->vsi_num = idc_priv->vport_id; 235 iwdev->init_state = INITIAL_STATE; 236 iwdev->roce_cwnd = IRDMA_ROCE_CWND_DEFAULT; 237 iwdev->roce_ackcreds = IRDMA_ROCE_ACKCREDS_DEFAULT; 238 iwdev->rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED; 239 iwdev->rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE; 240 if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) 241 iwdev->roce_mode = true; 242 } 243 244 static int icrdma_probe(struct auxiliary_device *aux_dev, const struct auxiliary_device_id *id) 245 { 246 struct iidc_rdma_core_auxiliary_dev *iidc_adev; 247 struct iidc_rdma_core_dev_info *cdev_info; 248 struct iidc_rdma_priv_dev_info *idc_priv; 249 struct irdma_l2params l2params = {}; 250 struct irdma_device *iwdev; 251 struct irdma_pci_f *rf; 252 int err; 253 254 iidc_adev = container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev); 255 cdev_info = iidc_adev->cdev_info; 256 idc_priv = cdev_info->iidc_priv; 257 258 iwdev = ib_alloc_device(irdma_device, ibdev); 259 if (!iwdev) 260 return -ENOMEM; 261 iwdev->rf = kzalloc(sizeof(*rf), GFP_KERNEL); 262 if (!iwdev->rf) { 263 ib_dealloc_device(&iwdev->ibdev); 264 return -ENOMEM; 265 } 266 267 icrdma_fill_device_info(iwdev, cdev_info); 268 rf = iwdev->rf; 269 270 err = icrdma_init_interrupts(rf, cdev_info); 271 if (err) 272 goto err_init_interrupts; 273 274 err = irdma_ctrl_init_hw(rf); 275 if (err) 276 goto err_ctrl_init; 277 278 l2params.mtu = iwdev->netdev->mtu; 279 icrdma_fill_qos_info(&l2params, &idc_priv->qos_info); 280 if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) 281 iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode; 282 283 err = irdma_rt_init_hw(iwdev, &l2params); 284 if (err) 285 goto err_rt_init; 286 287 err = irdma_ib_register_device(iwdev); 288 if (err) 289 goto err_ibreg; 290 291 ice_rdma_update_vsi_filter(cdev_info, iwdev->vsi_num, true); 292 293 ibdev_dbg(&iwdev->ibdev, "INIT: Gen2 PF[%d] device probe success\n", PCI_FUNC(rf->pcidev->devfn)); 294 auxiliary_set_drvdata(aux_dev, iwdev); 295 296 return 0; 297 298 err_ibreg: 299 irdma_rt_deinit_hw(iwdev); 300 err_rt_init: 301 irdma_ctrl_deinit_hw(rf); 302 err_ctrl_init: 303 icrdma_deinit_interrupts(rf, cdev_info); 304 err_init_interrupts: 305 kfree(iwdev->rf); 306 ib_dealloc_device(&iwdev->ibdev); 307 308 return err; 309 } 310 311 static void icrdma_remove(struct auxiliary_device *aux_dev) 312 { 313 struct iidc_rdma_core_auxiliary_dev *idc_adev = 314 container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev); 315 struct iidc_rdma_core_dev_info *cdev_info = idc_adev->cdev_info; 316 struct irdma_device *iwdev = auxiliary_get_drvdata(aux_dev); 317 u8 rdma_ver = iwdev->rf->rdma_ver; 318 319 ice_rdma_update_vsi_filter(cdev_info, iwdev->vsi_num, false); 320 irdma_ib_unregister_device(iwdev); 321 icrdma_deinit_interrupts(iwdev->rf, cdev_info); 322 323 pr_debug("INIT: Gen[%d] func[%d] device remove success\n", 324 rdma_ver, PCI_FUNC(cdev_info->pdev->devfn)); 325 } 326 327 static const struct auxiliary_device_id icrdma_auxiliary_id_table[] = { 328 {.name = "ice.iwarp", }, 329 {.name = "ice.roce", }, 330 {}, 331 }; 332 333 MODULE_DEVICE_TABLE(auxiliary, icrdma_auxiliary_id_table); 334 335 struct iidc_rdma_core_auxiliary_drv icrdma_core_auxiliary_drv = { 336 .adrv = { 337 .name = "gen_2", 338 .id_table = icrdma_auxiliary_id_table, 339 .probe = icrdma_probe, 340 .remove = icrdma_remove, 341 }, 342 .event_handler = icrdma_iidc_event_handler, 343 }; 344