1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) Advanced Micro Devices, Inc */ 3 4 #include <linux/module.h> 5 #include <linux/auxiliary_bus.h> 6 #include <linux/pci.h> 7 #include <linux/vmalloc.h> 8 #include <linux/bitfield.h> 9 #include <linux/string.h> 10 11 #include <uapi/fwctl/fwctl.h> 12 #include <uapi/fwctl/pds.h> 13 #include <linux/fwctl.h> 14 15 #include <linux/pds/pds_common.h> 16 #include <linux/pds/pds_core_if.h> 17 #include <linux/pds/pds_adminq.h> 18 #include <linux/pds/pds_auxbus.h> 19 20 struct pdsfc_uctx { 21 struct fwctl_uctx uctx; 22 u32 uctx_caps; 23 }; 24 25 struct pdsfc_rpc_endpoint_info { 26 u32 endpoint; 27 dma_addr_t operations_pa; 28 struct pds_fwctl_query_data *operations; 29 struct mutex lock; /* lock for endpoint info management */ 30 }; 31 32 struct pdsfc_dev { 33 struct fwctl_device fwctl; 34 struct pds_auxiliary_dev *padev; 35 u32 caps; 36 struct pds_fwctl_ident ident; 37 dma_addr_t endpoints_pa; 38 struct pds_fwctl_query_data *endpoints; 39 struct pdsfc_rpc_endpoint_info *endpoint_info; 40 }; 41 42 static int pdsfc_open_uctx(struct fwctl_uctx *uctx) 43 { 44 struct pdsfc_dev *pdsfc = container_of(uctx->fwctl, struct pdsfc_dev, fwctl); 45 struct pdsfc_uctx *pdsfc_uctx = container_of(uctx, struct pdsfc_uctx, uctx); 46 47 pdsfc_uctx->uctx_caps = pdsfc->caps; 48 49 return 0; 50 } 51 52 static void pdsfc_close_uctx(struct fwctl_uctx *uctx) 53 { 54 } 55 56 static void *pdsfc_info(struct fwctl_uctx *uctx, size_t *length) 57 { 58 struct pdsfc_uctx *pdsfc_uctx = container_of(uctx, struct pdsfc_uctx, uctx); 59 struct fwctl_info_pds *info; 60 61 info = kzalloc(sizeof(*info), GFP_KERNEL); 62 if (!info) 63 return ERR_PTR(-ENOMEM); 64 65 info->uctx_caps = pdsfc_uctx->uctx_caps; 66 67 return info; 68 } 69 70 static int pdsfc_identify(struct pdsfc_dev *pdsfc) 71 { 72 struct device *dev = &pdsfc->fwctl.dev; 73 union pds_core_adminq_comp comp = {0}; 74 union pds_core_adminq_cmd cmd; 75 struct pds_fwctl_ident *ident; 76 dma_addr_t ident_pa; 77 int err; 78 79 ident = dma_alloc_coherent(dev->parent, sizeof(*ident), &ident_pa, GFP_KERNEL); 80 if (!ident) { 81 dev_err(dev, "Failed to map ident buffer\n"); 82 return -ENOMEM; 83 } 84 85 cmd = (union pds_core_adminq_cmd) { 86 .fwctl_ident = { 87 .opcode = PDS_FWCTL_CMD_IDENT, 88 .version = 0, 89 .len = cpu_to_le32(sizeof(*ident)), 90 .ident_pa = cpu_to_le64(ident_pa), 91 } 92 }; 93 94 err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0); 95 if (err) 96 dev_err(dev, "Failed to send adminq cmd opcode: %u err: %d\n", 97 cmd.fwctl_ident.opcode, err); 98 else 99 pdsfc->ident = *ident; 100 101 dma_free_coherent(dev->parent, sizeof(*ident), ident, ident_pa); 102 103 return err; 104 } 105 106 static void pdsfc_free_endpoints(struct pdsfc_dev *pdsfc) 107 { 108 struct device *dev = &pdsfc->fwctl.dev; 109 u32 num_endpoints; 110 int i; 111 112 if (!pdsfc->endpoints) 113 return; 114 115 num_endpoints = le32_to_cpu(pdsfc->endpoints->num_entries); 116 for (i = 0; pdsfc->endpoint_info && i < num_endpoints; i++) 117 mutex_destroy(&pdsfc->endpoint_info[i].lock); 118 vfree(pdsfc->endpoint_info); 119 pdsfc->endpoint_info = NULL; 120 dma_free_coherent(dev->parent, PAGE_SIZE, 121 pdsfc->endpoints, pdsfc->endpoints_pa); 122 pdsfc->endpoints = NULL; 123 pdsfc->endpoints_pa = DMA_MAPPING_ERROR; 124 } 125 126 static void pdsfc_free_operations(struct pdsfc_dev *pdsfc) 127 { 128 struct device *dev = &pdsfc->fwctl.dev; 129 u32 num_endpoints; 130 int i; 131 132 num_endpoints = le32_to_cpu(pdsfc->endpoints->num_entries); 133 for (i = 0; i < num_endpoints; i++) { 134 struct pdsfc_rpc_endpoint_info *ei = &pdsfc->endpoint_info[i]; 135 136 if (ei->operations) { 137 dma_free_coherent(dev->parent, PAGE_SIZE, 138 ei->operations, ei->operations_pa); 139 ei->operations = NULL; 140 ei->operations_pa = DMA_MAPPING_ERROR; 141 } 142 } 143 } 144 145 static struct pds_fwctl_query_data *pdsfc_get_endpoints(struct pdsfc_dev *pdsfc, 146 dma_addr_t *pa) 147 { 148 struct device *dev = &pdsfc->fwctl.dev; 149 union pds_core_adminq_comp comp = {0}; 150 struct pds_fwctl_query_data *data; 151 union pds_core_adminq_cmd cmd; 152 dma_addr_t data_pa; 153 int err; 154 155 data = dma_alloc_coherent(dev->parent, PAGE_SIZE, &data_pa, GFP_KERNEL); 156 if (!data) { 157 dev_err(dev, "Failed to map endpoint list\n"); 158 return ERR_PTR(-ENOMEM); 159 } 160 161 cmd = (union pds_core_adminq_cmd) { 162 .fwctl_query = { 163 .opcode = PDS_FWCTL_CMD_QUERY, 164 .entity = PDS_FWCTL_RPC_ROOT, 165 .version = 0, 166 .query_data_buf_len = cpu_to_le32(PAGE_SIZE), 167 .query_data_buf_pa = cpu_to_le64(data_pa), 168 } 169 }; 170 171 err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0); 172 if (err) { 173 dev_err(dev, "Failed to send adminq cmd opcode: %u entity: %u err: %d\n", 174 cmd.fwctl_query.opcode, cmd.fwctl_query.entity, err); 175 dma_free_coherent(dev->parent, PAGE_SIZE, data, data_pa); 176 return ERR_PTR(err); 177 } 178 179 *pa = data_pa; 180 181 return data; 182 } 183 184 static int pdsfc_init_endpoints(struct pdsfc_dev *pdsfc) 185 { 186 struct pds_fwctl_query_data_endpoint *ep_entry; 187 u32 num_endpoints; 188 int i; 189 190 pdsfc->endpoints = pdsfc_get_endpoints(pdsfc, &pdsfc->endpoints_pa); 191 if (IS_ERR(pdsfc->endpoints)) 192 return PTR_ERR(pdsfc->endpoints); 193 194 num_endpoints = le32_to_cpu(pdsfc->endpoints->num_entries); 195 pdsfc->endpoint_info = vcalloc(num_endpoints, 196 sizeof(*pdsfc->endpoint_info)); 197 if (!pdsfc->endpoint_info) { 198 pdsfc_free_endpoints(pdsfc); 199 return -ENOMEM; 200 } 201 202 ep_entry = (struct pds_fwctl_query_data_endpoint *)pdsfc->endpoints->entries; 203 for (i = 0; i < num_endpoints; i++) { 204 mutex_init(&pdsfc->endpoint_info[i].lock); 205 pdsfc->endpoint_info[i].endpoint = le32_to_cpu(ep_entry[i].id); 206 } 207 208 return 0; 209 } 210 211 static struct pds_fwctl_query_data *pdsfc_get_operations(struct pdsfc_dev *pdsfc, 212 dma_addr_t *pa, u32 ep) 213 { 214 struct pds_fwctl_query_data_operation *entries; 215 struct device *dev = &pdsfc->fwctl.dev; 216 union pds_core_adminq_comp comp = {0}; 217 struct pds_fwctl_query_data *data; 218 union pds_core_adminq_cmd cmd; 219 dma_addr_t data_pa; 220 u32 num_entries; 221 int err; 222 int i; 223 224 /* Query the operations list for the given endpoint */ 225 data = dma_alloc_coherent(dev->parent, PAGE_SIZE, &data_pa, GFP_KERNEL); 226 if (!data) { 227 dev_err(dev, "Failed to map operations list\n"); 228 return ERR_PTR(-ENOMEM); 229 } 230 231 cmd = (union pds_core_adminq_cmd) { 232 .fwctl_query = { 233 .opcode = PDS_FWCTL_CMD_QUERY, 234 .entity = PDS_FWCTL_RPC_ENDPOINT, 235 .version = 0, 236 .query_data_buf_len = cpu_to_le32(PAGE_SIZE), 237 .query_data_buf_pa = cpu_to_le64(data_pa), 238 .ep = cpu_to_le32(ep), 239 } 240 }; 241 242 err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0); 243 if (err) { 244 dev_err(dev, "Failed to send adminq cmd opcode: %u entity: %u err: %d\n", 245 cmd.fwctl_query.opcode, cmd.fwctl_query.entity, err); 246 dma_free_coherent(dev->parent, PAGE_SIZE, data, data_pa); 247 return ERR_PTR(err); 248 } 249 250 *pa = data_pa; 251 252 entries = (struct pds_fwctl_query_data_operation *)data->entries; 253 num_entries = le32_to_cpu(data->num_entries); 254 dev_dbg(dev, "num_entries %d\n", num_entries); 255 for (i = 0; i < num_entries; i++) { 256 257 /* Translate FW command attribute to fwctl scope */ 258 switch (entries[i].scope) { 259 case PDSFC_FW_CMD_ATTR_READ: 260 case PDSFC_FW_CMD_ATTR_WRITE: 261 case PDSFC_FW_CMD_ATTR_SYNC: 262 entries[i].scope = FWCTL_RPC_CONFIGURATION; 263 break; 264 case PDSFC_FW_CMD_ATTR_DEBUG_READ: 265 entries[i].scope = FWCTL_RPC_DEBUG_READ_ONLY; 266 break; 267 case PDSFC_FW_CMD_ATTR_DEBUG_WRITE: 268 entries[i].scope = FWCTL_RPC_DEBUG_WRITE; 269 break; 270 default: 271 entries[i].scope = FWCTL_RPC_DEBUG_WRITE_FULL; 272 break; 273 } 274 dev_dbg(dev, "endpoint %d operation: id %x scope %d\n", 275 ep, le32_to_cpu(entries[i].id), entries[i].scope); 276 } 277 278 return data; 279 } 280 281 static int pdsfc_validate_rpc(struct pdsfc_dev *pdsfc, 282 struct fwctl_rpc_pds *rpc, 283 enum fwctl_rpc_scope scope) 284 { 285 struct pds_fwctl_query_data_operation *op_entry; 286 struct pdsfc_rpc_endpoint_info *ep_info = NULL; 287 struct device *dev = &pdsfc->fwctl.dev; 288 u32 num_entries; 289 int i; 290 291 /* validate rpc in_len & out_len based 292 * on ident.max_req_sz & max_resp_sz 293 */ 294 if (rpc->in.len > le32_to_cpu(pdsfc->ident.max_req_sz)) { 295 dev_dbg(dev, "Invalid request size %u, max %u\n", 296 rpc->in.len, le32_to_cpu(pdsfc->ident.max_req_sz)); 297 return -EINVAL; 298 } 299 300 if (rpc->out.len > le32_to_cpu(pdsfc->ident.max_resp_sz)) { 301 dev_dbg(dev, "Invalid response size %u, max %u\n", 302 rpc->out.len, le32_to_cpu(pdsfc->ident.max_resp_sz)); 303 return -EINVAL; 304 } 305 306 num_entries = le32_to_cpu(pdsfc->endpoints->num_entries); 307 for (i = 0; i < num_entries; i++) { 308 if (pdsfc->endpoint_info[i].endpoint == rpc->in.ep) { 309 ep_info = &pdsfc->endpoint_info[i]; 310 break; 311 } 312 } 313 if (!ep_info) { 314 dev_dbg(dev, "Invalid endpoint %d\n", rpc->in.ep); 315 return -EINVAL; 316 } 317 318 /* query and cache this endpoint's operations */ 319 mutex_lock(&ep_info->lock); 320 if (!ep_info->operations) { 321 struct pds_fwctl_query_data *operations; 322 323 operations = pdsfc_get_operations(pdsfc, 324 &ep_info->operations_pa, 325 rpc->in.ep); 326 if (IS_ERR(operations)) { 327 mutex_unlock(&ep_info->lock); 328 return -ENOMEM; 329 } 330 ep_info->operations = operations; 331 } 332 mutex_unlock(&ep_info->lock); 333 334 /* reject unsupported and/or out of scope commands */ 335 op_entry = (struct pds_fwctl_query_data_operation *)ep_info->operations->entries; 336 num_entries = le32_to_cpu(ep_info->operations->num_entries); 337 for (i = 0; i < num_entries; i++) { 338 if (PDS_FWCTL_RPC_OPCODE_CMP(rpc->in.op, le32_to_cpu(op_entry[i].id))) { 339 if (scope < op_entry[i].scope) 340 return -EPERM; 341 return 0; 342 } 343 } 344 345 dev_dbg(dev, "Invalid operation %d for endpoint %d\n", rpc->in.op, rpc->in.ep); 346 347 return -EINVAL; 348 } 349 350 static void *pdsfc_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope, 351 void *in, size_t in_len, size_t *out_len) 352 { 353 struct pdsfc_dev *pdsfc = container_of(uctx->fwctl, struct pdsfc_dev, fwctl); 354 struct device *dev = &uctx->fwctl->dev; 355 union pds_core_adminq_comp comp = {0}; 356 dma_addr_t out_payload_dma_addr = 0; 357 dma_addr_t in_payload_dma_addr = 0; 358 struct fwctl_rpc_pds *rpc = in; 359 union pds_core_adminq_cmd cmd; 360 void *out_payload = NULL; 361 void *in_payload = NULL; 362 void *out = NULL; 363 int err; 364 365 err = pdsfc_validate_rpc(pdsfc, rpc, scope); 366 if (err) 367 return ERR_PTR(err); 368 369 if (rpc->in.len > 0) { 370 in_payload = memdup_user(u64_to_user_ptr(rpc->in.payload), rpc->in.len); 371 if (IS_ERR(in_payload)) { 372 dev_dbg(dev, "Failed to copy in_payload from user\n"); 373 return in_payload; 374 } 375 376 in_payload_dma_addr = dma_map_single(dev->parent, in_payload, 377 rpc->in.len, DMA_TO_DEVICE); 378 err = dma_mapping_error(dev->parent, in_payload_dma_addr); 379 if (err) { 380 dev_dbg(dev, "Failed to map in_payload\n"); 381 goto err_in_payload; 382 } 383 } 384 385 if (rpc->out.len > 0) { 386 out_payload = kzalloc(rpc->out.len, GFP_KERNEL); 387 if (!out_payload) { 388 dev_dbg(dev, "Failed to allocate out_payload\n"); 389 err = -ENOMEM; 390 goto err_out_payload; 391 } 392 393 out_payload_dma_addr = dma_map_single(dev->parent, out_payload, 394 rpc->out.len, DMA_FROM_DEVICE); 395 err = dma_mapping_error(dev->parent, out_payload_dma_addr); 396 if (err) { 397 dev_dbg(dev, "Failed to map out_payload\n"); 398 goto err_out_payload; 399 } 400 } 401 402 cmd = (union pds_core_adminq_cmd) { 403 .fwctl_rpc = { 404 .opcode = PDS_FWCTL_CMD_RPC, 405 .flags = cpu_to_le16(PDS_FWCTL_RPC_IND_REQ | PDS_FWCTL_RPC_IND_RESP), 406 .ep = cpu_to_le32(rpc->in.ep), 407 .op = cpu_to_le32(rpc->in.op), 408 .req_pa = cpu_to_le64(in_payload_dma_addr), 409 .req_sz = cpu_to_le32(rpc->in.len), 410 .resp_pa = cpu_to_le64(out_payload_dma_addr), 411 .resp_sz = cpu_to_le32(rpc->out.len), 412 } 413 }; 414 415 err = pds_client_adminq_cmd(pdsfc->padev, &cmd, sizeof(cmd), &comp, 0); 416 if (err) { 417 dev_dbg(dev, "%s: ep %d op %x req_pa %llx req_sz %d req_sg %d resp_pa %llx resp_sz %d resp_sg %d err %d\n", 418 __func__, rpc->in.ep, rpc->in.op, 419 cmd.fwctl_rpc.req_pa, cmd.fwctl_rpc.req_sz, cmd.fwctl_rpc.req_sg_elems, 420 cmd.fwctl_rpc.resp_pa, cmd.fwctl_rpc.resp_sz, cmd.fwctl_rpc.resp_sg_elems, 421 err); 422 goto done; 423 } 424 425 dynamic_hex_dump("out ", DUMP_PREFIX_OFFSET, 16, 1, out_payload, rpc->out.len, true); 426 427 if (copy_to_user(u64_to_user_ptr(rpc->out.payload), out_payload, rpc->out.len)) { 428 dev_dbg(dev, "Failed to copy out_payload to user\n"); 429 out = ERR_PTR(-EFAULT); 430 goto done; 431 } 432 433 rpc->out.retval = le32_to_cpu(comp.fwctl_rpc.err); 434 *out_len = in_len; 435 out = in; 436 437 done: 438 if (out_payload_dma_addr) 439 dma_unmap_single(dev->parent, out_payload_dma_addr, 440 rpc->out.len, DMA_FROM_DEVICE); 441 err_out_payload: 442 kfree(out_payload); 443 444 if (in_payload_dma_addr) 445 dma_unmap_single(dev->parent, in_payload_dma_addr, 446 rpc->in.len, DMA_TO_DEVICE); 447 err_in_payload: 448 kfree(in_payload); 449 if (err) 450 return ERR_PTR(err); 451 452 return out; 453 } 454 455 static const struct fwctl_ops pdsfc_ops = { 456 .device_type = FWCTL_DEVICE_TYPE_PDS, 457 .uctx_size = sizeof(struct pdsfc_uctx), 458 .open_uctx = pdsfc_open_uctx, 459 .close_uctx = pdsfc_close_uctx, 460 .info = pdsfc_info, 461 .fw_rpc = pdsfc_fw_rpc, 462 }; 463 464 static int pdsfc_probe(struct auxiliary_device *adev, 465 const struct auxiliary_device_id *id) 466 { 467 struct pds_auxiliary_dev *padev = 468 container_of(adev, struct pds_auxiliary_dev, aux_dev); 469 struct device *dev = &adev->dev; 470 struct pdsfc_dev *pdsfc; 471 int err; 472 473 pdsfc = fwctl_alloc_device(&padev->vf_pdev->dev, &pdsfc_ops, 474 struct pdsfc_dev, fwctl); 475 if (!pdsfc) 476 return -ENOMEM; 477 pdsfc->padev = padev; 478 479 err = pdsfc_identify(pdsfc); 480 if (err) { 481 fwctl_put(&pdsfc->fwctl); 482 return dev_err_probe(dev, err, "Failed to identify device\n"); 483 } 484 485 err = pdsfc_init_endpoints(pdsfc); 486 if (err) { 487 fwctl_put(&pdsfc->fwctl); 488 return dev_err_probe(dev, err, "Failed to init endpoints\n"); 489 } 490 491 pdsfc->caps = PDS_FWCTL_QUERY_CAP | PDS_FWCTL_SEND_CAP; 492 493 err = fwctl_register(&pdsfc->fwctl); 494 if (err) { 495 pdsfc_free_endpoints(pdsfc); 496 fwctl_put(&pdsfc->fwctl); 497 return dev_err_probe(dev, err, "Failed to register device\n"); 498 } 499 500 auxiliary_set_drvdata(adev, pdsfc); 501 502 return 0; 503 } 504 505 static void pdsfc_remove(struct auxiliary_device *adev) 506 { 507 struct pdsfc_dev *pdsfc = auxiliary_get_drvdata(adev); 508 509 fwctl_unregister(&pdsfc->fwctl); 510 pdsfc_free_operations(pdsfc); 511 pdsfc_free_endpoints(pdsfc); 512 513 fwctl_put(&pdsfc->fwctl); 514 } 515 516 static const struct auxiliary_device_id pdsfc_id_table[] = { 517 {.name = PDS_CORE_DRV_NAME "." PDS_DEV_TYPE_FWCTL_STR }, 518 {} 519 }; 520 MODULE_DEVICE_TABLE(auxiliary, pdsfc_id_table); 521 522 static struct auxiliary_driver pdsfc_driver = { 523 .name = "pds_fwctl", 524 .probe = pdsfc_probe, 525 .remove = pdsfc_remove, 526 .id_table = pdsfc_id_table, 527 }; 528 529 module_auxiliary_driver(pdsfc_driver); 530 531 MODULE_IMPORT_NS("FWCTL"); 532 MODULE_DESCRIPTION("pds fwctl driver"); 533 MODULE_AUTHOR("Shannon Nelson <shannon.nelson@amd.com>"); 534 MODULE_AUTHOR("Brett Creeley <brett.creeley@amd.com>"); 535 MODULE_LICENSE("GPL"); 536