1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) Advanced Micro Devices, Inc */ 3 4 #include <linux/module.h> 5 #include <linux/auxiliary_bus.h> 6 #include <linux/pci.h> 7 #include <linux/vmalloc.h> 8 #include <linux/bitfield.h> 9 #include <linux/string.h> 10 11 #include <uapi/fwctl/fwctl.h> 12 #include <uapi/fwctl/pds.h> 13 #include <linux/fwctl.h> 14 15 #include <linux/pds/pds_common.h> 16 #include <linux/pds/pds_core_if.h> 17 #include <linux/pds/pds_adminq.h> 18 #include <linux/pds/pds_auxbus.h> 19 20 struct pdsfc_uctx { 21 struct fwctl_uctx uctx; 22 u32 uctx_caps; 23 }; 24 25 struct pdsfc_rpc_endpoint_info { 26 u32 endpoint; 27 dma_addr_t operations_pa; 28 struct pds_fwctl_query_data *operations; 29 struct mutex lock; /* lock for endpoint info management */ 30 }; 31 32 struct pdsfc_dev { 33 struct fwctl_device fwctl; 34 struct pds_auxiliary_dev *padev; 35 u32 caps; 36 struct pds_fwctl_ident ident; 37 dma_addr_t endpoints_pa; 38 struct pds_fwctl_query_data *endpoints; 39 struct pdsfc_rpc_endpoint_info *endpoint_info; 40 }; 41 42 static int pdsfc_open_uctx(struct fwctl_uctx *uctx) 43 { 44 struct pdsfc_dev *pdsfc = container_of(uctx->fwctl, struct pdsfc_dev, fwctl); 45 struct pdsfc_uctx *pdsfc_uctx = container_of(uctx, struct pdsfc_uctx, uctx); 46 47 pdsfc_uctx->uctx_caps = pdsfc->caps; 48 49 return 0; 50 } 51 52 static void pdsfc_close_uctx(struct fwctl_uctx *uctx) 53 { 54 } 55 56 static void *pdsfc_info(struct fwctl_uctx *uctx, size_t *length) 57 { 58 struct pdsfc_uctx *pdsfc_uctx = container_of(uctx, struct pdsfc_uctx, uctx); 59 struct fwctl_info_pds *info; 60 61 info = kzalloc_obj(*info); 62 if (!info) 63 return ERR_PTR(-ENOMEM); 64 65 info->uctx_caps = pdsfc_uctx->uctx_caps; 66 67 return info; 68 } 69 70 static int pdsfc_identify(struct pdsfc_dev *pdsfc) 71 { 72 struct device *dev = &pdsfc->fwctl.dev; 73 union pds_core_adminq_comp comp = {0}; 74 union pds_core_adminq_cmd cmd; 75 struct pds_fwctl_ident *ident; 76 dma_addr_t ident_pa; 77 int err; 78 79 ident = dma_alloc_coherent(dev->parent, sizeof(*ident), &ident_pa, GFP_KERNEL); 80 if (!ident) { 81 dev_err(dev, "Failed to map ident buffer\n"); 82 return -ENOMEM; 83 } 84 85 cmd = (union pds_core_adminq_cmd) { 86 .fwctl_ident = { 87 .opcode = PDS_FWCTL_CMD_IDENT, 88 .version = 0, 89 .len = cpu_to_le32(sizeof(*ident)), 90 .ident_pa = cpu_to_le64(ident_pa), 91 } 92 }; 93 94 err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0); 95 if (err) 96 dev_err(dev, "Failed to send adminq cmd opcode: %u err: %d\n", 97 cmd.fwctl_ident.opcode, err); 98 else 99 pdsfc->ident = *ident; 100 101 dma_free_coherent(dev->parent, sizeof(*ident), ident, ident_pa); 102 103 return err; 104 } 105 106 static void pdsfc_free_endpoints(struct pdsfc_dev *pdsfc) 107 { 108 struct device *dev = &pdsfc->fwctl.dev; 109 u32 num_endpoints; 110 int i; 111 112 if (!pdsfc->endpoints) 113 return; 114 115 num_endpoints = le32_to_cpu(pdsfc->endpoints->num_entries); 116 for (i = 0; pdsfc->endpoint_info && i < num_endpoints; i++) 117 mutex_destroy(&pdsfc->endpoint_info[i].lock); 118 vfree(pdsfc->endpoint_info); 119 pdsfc->endpoint_info = NULL; 120 dma_free_coherent(dev->parent, PAGE_SIZE, 121 pdsfc->endpoints, pdsfc->endpoints_pa); 122 pdsfc->endpoints = NULL; 123 pdsfc->endpoints_pa = DMA_MAPPING_ERROR; 124 } 125 126 static void pdsfc_free_operations(struct pdsfc_dev *pdsfc) 127 { 128 struct device *dev = &pdsfc->fwctl.dev; 129 u32 num_endpoints; 130 int i; 131 132 num_endpoints = le32_to_cpu(pdsfc->endpoints->num_entries); 133 for (i = 0; i < num_endpoints; i++) { 134 struct pdsfc_rpc_endpoint_info *ei = &pdsfc->endpoint_info[i]; 135 136 if (ei->operations) { 137 dma_free_coherent(dev->parent, PAGE_SIZE, 138 ei->operations, ei->operations_pa); 139 ei->operations = NULL; 140 ei->operations_pa = DMA_MAPPING_ERROR; 141 } 142 } 143 } 144 145 static struct pds_fwctl_query_data *pdsfc_get_endpoints(struct pdsfc_dev *pdsfc, 146 dma_addr_t *pa) 147 { 148 struct device *dev = &pdsfc->fwctl.dev; 149 union pds_core_adminq_comp comp = {0}; 150 struct pds_fwctl_query_data *data; 151 union pds_core_adminq_cmd cmd; 152 dma_addr_t data_pa; 153 int err; 154 155 data = dma_alloc_coherent(dev->parent, PAGE_SIZE, &data_pa, GFP_KERNEL); 156 if (!data) { 157 dev_err(dev, "Failed to map endpoint list\n"); 158 return ERR_PTR(-ENOMEM); 159 } 160 161 cmd = (union pds_core_adminq_cmd) { 162 .fwctl_query = { 163 .opcode = PDS_FWCTL_CMD_QUERY, 164 .entity = PDS_FWCTL_RPC_ROOT, 165 .version = 0, 166 .query_data_buf_len = cpu_to_le32(PAGE_SIZE), 167 .query_data_buf_pa = cpu_to_le64(data_pa), 168 } 169 }; 170 171 err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0); 172 if (err) { 173 dev_err(dev, "Failed to send adminq cmd opcode: %u entity: %u err: %d\n", 174 cmd.fwctl_query.opcode, cmd.fwctl_query.entity, err); 175 dma_free_coherent(dev->parent, PAGE_SIZE, data, data_pa); 176 return ERR_PTR(err); 177 } 178 179 *pa = data_pa; 180 181 return data; 182 } 183 184 static int pdsfc_init_endpoints(struct pdsfc_dev *pdsfc) 185 { 186 struct pds_fwctl_query_data_endpoint *ep_entry; 187 u32 num_endpoints; 188 int i; 189 190 pdsfc->endpoints = pdsfc_get_endpoints(pdsfc, &pdsfc->endpoints_pa); 191 if (IS_ERR(pdsfc->endpoints)) 192 return PTR_ERR(pdsfc->endpoints); 193 194 num_endpoints = le32_to_cpu(pdsfc->endpoints->num_entries); 195 pdsfc->endpoint_info = vcalloc(num_endpoints, 196 sizeof(*pdsfc->endpoint_info)); 197 if (!pdsfc->endpoint_info) { 198 pdsfc_free_endpoints(pdsfc); 199 return -ENOMEM; 200 } 201 202 ep_entry = (struct pds_fwctl_query_data_endpoint *)pdsfc->endpoints->entries; 203 for (i = 0; i < num_endpoints; i++) { 204 mutex_init(&pdsfc->endpoint_info[i].lock); 205 pdsfc->endpoint_info[i].endpoint = le32_to_cpu(ep_entry[i].id); 206 } 207 208 return 0; 209 } 210 211 static struct pds_fwctl_query_data *pdsfc_get_operations(struct pdsfc_dev *pdsfc, 212 dma_addr_t *pa, u32 ep) 213 { 214 struct pds_fwctl_query_data_operation *entries; 215 struct device *dev = &pdsfc->fwctl.dev; 216 union pds_core_adminq_comp comp = {0}; 217 struct pds_fwctl_query_data *data; 218 union pds_core_adminq_cmd cmd; 219 dma_addr_t data_pa; 220 u32 num_entries; 221 int err; 222 int i; 223 224 /* Query the operations list for the given endpoint */ 225 data = dma_alloc_coherent(dev->parent, PAGE_SIZE, &data_pa, GFP_KERNEL); 226 if (!data) { 227 dev_err(dev, "Failed to map operations list\n"); 228 return ERR_PTR(-ENOMEM); 229 } 230 231 cmd = (union pds_core_adminq_cmd) { 232 .fwctl_query = { 233 .opcode = PDS_FWCTL_CMD_QUERY, 234 .entity = PDS_FWCTL_RPC_ENDPOINT, 235 .version = 0, 236 .query_data_buf_len = cpu_to_le32(PAGE_SIZE), 237 .query_data_buf_pa = cpu_to_le64(data_pa), 238 .ep = cpu_to_le32(ep), 239 } 240 }; 241 242 err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0); 243 if (err) { 244 dev_err(dev, "Failed to send adminq cmd opcode: %u entity: %u err: %d\n", 245 cmd.fwctl_query.opcode, cmd.fwctl_query.entity, err); 246 dma_free_coherent(dev->parent, PAGE_SIZE, data, data_pa); 247 return ERR_PTR(err); 248 } 249 250 *pa = data_pa; 251 252 entries = (struct pds_fwctl_query_data_operation *)data->entries; 253 num_entries = le32_to_cpu(data->num_entries); 254 dev_dbg(dev, "num_entries %d\n", num_entries); 255 for (i = 0; i < num_entries; i++) { 256 257 /* Translate FW command attribute to fwctl scope */ 258 switch (entries[i].scope) { 259 case PDSFC_FW_CMD_ATTR_READ: 260 case PDSFC_FW_CMD_ATTR_WRITE: 261 case PDSFC_FW_CMD_ATTR_SYNC: 262 entries[i].scope = FWCTL_RPC_CONFIGURATION; 263 break; 264 case PDSFC_FW_CMD_ATTR_DEBUG_READ: 265 entries[i].scope = FWCTL_RPC_DEBUG_READ_ONLY; 266 break; 267 case PDSFC_FW_CMD_ATTR_DEBUG_WRITE: 268 entries[i].scope = FWCTL_RPC_DEBUG_WRITE; 269 break; 270 default: 271 entries[i].scope = FWCTL_RPC_DEBUG_WRITE_FULL; 272 break; 273 } 274 dev_dbg(dev, "endpoint %d operation: id %x scope %d\n", 275 ep, le32_to_cpu(entries[i].id), entries[i].scope); 276 } 277 278 return data; 279 } 280 281 static int pdsfc_validate_rpc(struct pdsfc_dev *pdsfc, 282 struct fwctl_rpc_pds *rpc, 283 enum fwctl_rpc_scope scope) 284 { 285 struct pds_fwctl_query_data_operation *op_entry; 286 struct pdsfc_rpc_endpoint_info *ep_info = NULL; 287 struct device *dev = &pdsfc->fwctl.dev; 288 u32 num_entries; 289 int i; 290 291 /* validate rpc in_len & out_len based 292 * on ident.max_req_sz & max_resp_sz 293 */ 294 if (rpc->in.len > le32_to_cpu(pdsfc->ident.max_req_sz)) { 295 dev_dbg(dev, "Invalid request size %u, max %u\n", 296 rpc->in.len, le32_to_cpu(pdsfc->ident.max_req_sz)); 297 return -EINVAL; 298 } 299 300 if (rpc->out.len > le32_to_cpu(pdsfc->ident.max_resp_sz)) { 301 dev_dbg(dev, "Invalid response size %u, max %u\n", 302 rpc->out.len, le32_to_cpu(pdsfc->ident.max_resp_sz)); 303 return -EINVAL; 304 } 305 306 num_entries = le32_to_cpu(pdsfc->endpoints->num_entries); 307 for (i = 0; i < num_entries; i++) { 308 if (pdsfc->endpoint_info[i].endpoint == rpc->in.ep) { 309 ep_info = &pdsfc->endpoint_info[i]; 310 break; 311 } 312 } 313 if (!ep_info) { 314 dev_dbg(dev, "Invalid endpoint %d\n", rpc->in.ep); 315 return -EINVAL; 316 } 317 318 /* query and cache this endpoint's operations */ 319 mutex_lock(&ep_info->lock); 320 if (!ep_info->operations) { 321 struct pds_fwctl_query_data *operations; 322 323 operations = pdsfc_get_operations(pdsfc, 324 &ep_info->operations_pa, 325 rpc->in.ep); 326 if (IS_ERR(operations)) { 327 mutex_unlock(&ep_info->lock); 328 return -ENOMEM; 329 } 330 ep_info->operations = operations; 331 } 332 mutex_unlock(&ep_info->lock); 333 334 /* reject unsupported and/or out of scope commands */ 335 op_entry = (struct pds_fwctl_query_data_operation *)ep_info->operations->entries; 336 num_entries = le32_to_cpu(ep_info->operations->num_entries); 337 for (i = 0; i < num_entries; i++) { 338 if (PDS_FWCTL_RPC_OPCODE_CMP(rpc->in.op, le32_to_cpu(op_entry[i].id))) { 339 if (scope < op_entry[i].scope) 340 return -EPERM; 341 return 0; 342 } 343 } 344 345 dev_dbg(dev, "Invalid operation %d for endpoint %d\n", rpc->in.op, rpc->in.ep); 346 347 return -EINVAL; 348 } 349 350 static void *pdsfc_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope, 351 void *in, size_t in_len, size_t *out_len) 352 { 353 struct pdsfc_dev *pdsfc = container_of(uctx->fwctl, struct pdsfc_dev, fwctl); 354 struct device *dev = &uctx->fwctl->dev; 355 union pds_core_adminq_comp comp = {0}; 356 dma_addr_t out_payload_dma_addr = 0; 357 dma_addr_t in_payload_dma_addr = 0; 358 struct fwctl_rpc_pds *rpc = in; 359 union pds_core_adminq_cmd cmd; 360 void *out_payload = NULL; 361 void *in_payload = NULL; 362 void *out = NULL; 363 int err; 364 365 if (in_len < sizeof(*rpc)) 366 return ERR_PTR(-EINVAL); 367 368 err = pdsfc_validate_rpc(pdsfc, rpc, scope); 369 if (err) 370 return ERR_PTR(err); 371 372 if (rpc->in.len > 0) { 373 in_payload = memdup_user(u64_to_user_ptr(rpc->in.payload), rpc->in.len); 374 if (IS_ERR(in_payload)) { 375 dev_dbg(dev, "Failed to copy in_payload from user\n"); 376 return in_payload; 377 } 378 379 in_payload_dma_addr = dma_map_single(dev->parent, in_payload, 380 rpc->in.len, DMA_TO_DEVICE); 381 err = dma_mapping_error(dev->parent, in_payload_dma_addr); 382 if (err) { 383 dev_dbg(dev, "Failed to map in_payload\n"); 384 goto err_in_payload; 385 } 386 } 387 388 if (rpc->out.len > 0) { 389 out_payload = kzalloc(rpc->out.len, GFP_KERNEL); 390 if (!out_payload) { 391 dev_dbg(dev, "Failed to allocate out_payload\n"); 392 err = -ENOMEM; 393 goto err_out_payload; 394 } 395 396 out_payload_dma_addr = dma_map_single(dev->parent, out_payload, 397 rpc->out.len, DMA_FROM_DEVICE); 398 err = dma_mapping_error(dev->parent, out_payload_dma_addr); 399 if (err) { 400 dev_dbg(dev, "Failed to map out_payload\n"); 401 goto err_out_payload; 402 } 403 } 404 405 cmd = (union pds_core_adminq_cmd) { 406 .fwctl_rpc = { 407 .opcode = PDS_FWCTL_CMD_RPC, 408 .flags = cpu_to_le16(PDS_FWCTL_RPC_IND_REQ | PDS_FWCTL_RPC_IND_RESP), 409 .ep = cpu_to_le32(rpc->in.ep), 410 .op = cpu_to_le32(rpc->in.op), 411 .req_pa = cpu_to_le64(in_payload_dma_addr), 412 .req_sz = cpu_to_le32(rpc->in.len), 413 .resp_pa = cpu_to_le64(out_payload_dma_addr), 414 .resp_sz = cpu_to_le32(rpc->out.len), 415 } 416 }; 417 418 err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0); 419 if (err) { 420 dev_dbg(dev, "%s: ep %d op %x req_pa %llx req_sz %d req_sg %d resp_pa %llx resp_sz %d resp_sg %d err %d\n", 421 __func__, rpc->in.ep, rpc->in.op, 422 cmd.fwctl_rpc.req_pa, cmd.fwctl_rpc.req_sz, cmd.fwctl_rpc.req_sg_elems, 423 cmd.fwctl_rpc.resp_pa, cmd.fwctl_rpc.resp_sz, cmd.fwctl_rpc.resp_sg_elems, 424 err); 425 goto done; 426 } 427 428 dynamic_hex_dump("out ", DUMP_PREFIX_OFFSET, 16, 1, out_payload, rpc->out.len, true); 429 430 if (copy_to_user(u64_to_user_ptr(rpc->out.payload), out_payload, rpc->out.len)) { 431 dev_dbg(dev, "Failed to copy out_payload to user\n"); 432 out = ERR_PTR(-EFAULT); 433 goto done; 434 } 435 436 rpc->out.retval = le32_to_cpu(comp.fwctl_rpc.err); 437 *out_len = in_len; 438 out = in; 439 440 done: 441 if (out_payload_dma_addr) 442 dma_unmap_single(dev->parent, out_payload_dma_addr, 443 rpc->out.len, DMA_FROM_DEVICE); 444 err_out_payload: 445 kfree(out_payload); 446 447 if (in_payload_dma_addr) 448 dma_unmap_single(dev->parent, in_payload_dma_addr, 449 rpc->in.len, DMA_TO_DEVICE); 450 err_in_payload: 451 kfree(in_payload); 452 if (err) 453 return ERR_PTR(err); 454 455 return out; 456 } 457 458 static const struct fwctl_ops pdsfc_ops = { 459 .device_type = FWCTL_DEVICE_TYPE_PDS, 460 .uctx_size = sizeof(struct pdsfc_uctx), 461 .open_uctx = pdsfc_open_uctx, 462 .close_uctx = pdsfc_close_uctx, 463 .info = pdsfc_info, 464 .fw_rpc = pdsfc_fw_rpc, 465 }; 466 467 static int pdsfc_probe(struct auxiliary_device *adev, 468 const struct auxiliary_device_id *id) 469 { 470 struct pds_auxiliary_dev *padev = 471 container_of(adev, struct pds_auxiliary_dev, aux_dev); 472 struct device *dev = &adev->dev; 473 struct pdsfc_dev *pdsfc; 474 int err; 475 476 pdsfc = fwctl_alloc_device(&padev->vf_pdev->dev, &pdsfc_ops, 477 struct pdsfc_dev, fwctl); 478 if (!pdsfc) 479 return -ENOMEM; 480 pdsfc->padev = padev; 481 482 err = pdsfc_identify(pdsfc); 483 if (err) { 484 fwctl_put(&pdsfc->fwctl); 485 return dev_err_probe(dev, err, "Failed to identify device\n"); 486 } 487 488 err = pdsfc_init_endpoints(pdsfc); 489 if (err) { 490 fwctl_put(&pdsfc->fwctl); 491 return dev_err_probe(dev, err, "Failed to init endpoints\n"); 492 } 493 494 pdsfc->caps = PDS_FWCTL_QUERY_CAP | PDS_FWCTL_SEND_CAP; 495 496 err = fwctl_register(&pdsfc->fwctl); 497 if (err) { 498 pdsfc_free_endpoints(pdsfc); 499 fwctl_put(&pdsfc->fwctl); 500 return dev_err_probe(dev, err, "Failed to register device\n"); 501 } 502 503 auxiliary_set_drvdata(adev, pdsfc); 504 505 return 0; 506 } 507 508 static void pdsfc_remove(struct auxiliary_device *adev) 509 { 510 struct pdsfc_dev *pdsfc = auxiliary_get_drvdata(adev); 511 512 fwctl_unregister(&pdsfc->fwctl); 513 pdsfc_free_operations(pdsfc); 514 pdsfc_free_endpoints(pdsfc); 515 516 fwctl_put(&pdsfc->fwctl); 517 } 518 519 static const struct auxiliary_device_id pdsfc_id_table[] = { 520 {.name = PDS_CORE_DRV_NAME "." PDS_DEV_TYPE_FWCTL_STR }, 521 {} 522 }; 523 MODULE_DEVICE_TABLE(auxiliary, pdsfc_id_table); 524 525 static struct auxiliary_driver pdsfc_driver = { 526 .name = "pds_fwctl", 527 .probe = pdsfc_probe, 528 .remove = pdsfc_remove, 529 .id_table = pdsfc_id_table, 530 }; 531 532 module_auxiliary_driver(pdsfc_driver); 533 534 MODULE_IMPORT_NS("FWCTL"); 535 MODULE_DESCRIPTION("pds fwctl driver"); 536 MODULE_AUTHOR("Shannon Nelson <shannon.nelson@amd.com>"); 537 MODULE_AUTHOR("Brett Creeley <brett.creeley@amd.com>"); 538 MODULE_LICENSE("GPL"); 539