1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Char device for device raw access 4 * 5 * Copyright (C) 2005-2007 Kristian Hoegsberg <krh@bitplanet.net> 6 */ 7 8 #include <linux/bug.h> 9 #include <linux/compat.h> 10 #include <linux/delay.h> 11 #include <linux/device.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/err.h> 14 #include <linux/errno.h> 15 #include <linux/firewire.h> 16 #include <linux/firewire-cdev.h> 17 #include <linux/irqflags.h> 18 #include <linux/jiffies.h> 19 #include <linux/kernel.h> 20 #include <linux/kref.h> 21 #include <linux/mm.h> 22 #include <linux/module.h> 23 #include <linux/mutex.h> 24 #include <linux/poll.h> 25 #include <linux/sched.h> /* required for linux/wait.h */ 26 #include <linux/slab.h> 27 #include <linux/spinlock.h> 28 #include <linux/string.h> 29 #include <linux/time.h> 30 #include <linux/uaccess.h> 31 #include <linux/vmalloc.h> 32 #include <linux/wait.h> 33 #include <linux/workqueue.h> 34 35 36 #include "core.h" 37 #include <trace/events/firewire.h> 38 39 #include "packet-header-definitions.h" 40 41 /* 42 * ABI version history is documented in linux/firewire-cdev.h. 43 */ 44 #define FW_CDEV_KERNEL_VERSION 6 45 #define FW_CDEV_VERSION_EVENT_REQUEST2 4 46 #define FW_CDEV_VERSION_ALLOCATE_REGION_END 4 47 #define FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW 5 48 #define FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP 6 49 50 static DEFINE_MUTEX(phy_receiver_list_mutex); 51 static LIST_HEAD(phy_receiver_list); 52 53 struct client { 54 u32 version; 55 struct fw_device *device; 56 57 struct mutex mutex; 58 bool in_shutdown; 59 struct xarray resource_xa; 60 struct list_head event_list; 61 wait_queue_head_t wait; 62 wait_queue_head_t tx_flush_wait; 63 u64 bus_reset_closure; 64 65 struct fw_iso_context *iso_context; 66 struct mutex iso_context_mutex; 67 u64 iso_closure; 68 struct fw_iso_buffer buffer; 69 unsigned long vm_start; 70 71 struct list_head phy_receiver_link; 72 u64 phy_receiver_closure; 73 74 struct list_head link; 75 struct kref kref; 76 }; 77 78 static inline void client_get(struct client *client) 79 { 80 kref_get(&client->kref); 81 } 82 83 static void client_release(struct kref *kref) 84 { 85 struct client *client = container_of(kref, struct client, kref); 86 87 fw_device_put(client->device); 88 kfree(client); 89 } 90 91 static void client_put(struct client *client) 92 { 93 kref_put(&client->kref, client_release); 94 } 95 96 struct client_resource; 97 typedef void (*client_resource_release_fn_t)(struct client *, 98 struct client_resource *); 99 struct client_resource { 100 client_resource_release_fn_t release; 101 int handle; 102 }; 103 104 struct address_handler_resource { 105 struct client_resource resource; 106 struct fw_address_handler handler; 107 __u64 closure; 108 struct client *client; 109 }; 110 111 struct outbound_transaction_resource { 112 struct client_resource resource; 113 struct fw_transaction transaction; 114 }; 115 116 struct inbound_transaction_resource { 117 struct client_resource resource; 118 struct fw_card *card; 119 struct fw_request *request; 120 bool is_fcp; 121 void *data; 122 size_t length; 123 }; 124 125 struct descriptor_resource { 126 struct client_resource resource; 127 struct fw_descriptor descriptor; 128 u32 data[]; 129 }; 130 131 struct iso_resource_params { 132 u64 channels_mask; 133 s32 bandwidth; 134 }; 135 136 enum { 137 ISO_RES_AUTO_ALLOC, 138 ISO_RES_AUTO_REALLOC, 139 ISO_RES_AUTO_DEALLOC, 140 }; 141 142 struct iso_resource_auto { 143 struct client_resource resource; 144 struct client *client; 145 struct delayed_work work; 146 atomic_t todo; // one of ISO_RES_AUTO_XXX. 147 int generation; 148 struct iso_resource_params params; 149 struct iso_resource_event *e_alloc, *e_dealloc; 150 }; 151 152 struct iso_resource_once { 153 struct client *client; 154 struct work_struct work; 155 enum { 156 ISO_RES_ONCE_ALLOC, 157 ISO_RES_ONCE_DEALLOC, 158 } todo; 159 struct iso_resource_params params; 160 struct iso_resource_event *event; 161 }; 162 163 static struct address_handler_resource *to_address_handler_resource(struct client_resource *resource) 164 { 165 return container_of(resource, struct address_handler_resource, resource); 166 } 167 168 static struct inbound_transaction_resource *to_inbound_transaction_resource(struct client_resource *resource) 169 { 170 return container_of(resource, struct inbound_transaction_resource, resource); 171 } 172 173 static struct descriptor_resource *to_descriptor_resource(struct client_resource *resource) 174 { 175 return container_of(resource, struct descriptor_resource, resource); 176 } 177 178 static struct iso_resource_auto *to_iso_resource_auto(struct client_resource *resource) 179 { 180 return container_of(resource, struct iso_resource_auto, resource); 181 } 182 183 static void release_iso_resource_auto(struct client *, struct client_resource *); 184 185 static int is_iso_resource_auto(const struct client_resource *resource) 186 { 187 return resource->release == release_iso_resource_auto; 188 } 189 190 static void release_transaction(struct client *client, 191 struct client_resource *resource); 192 193 static int is_outbound_transaction_resource(const struct client_resource *resource) 194 { 195 return resource->release == release_transaction; 196 } 197 198 static void schedule_iso_resource_auto(struct iso_resource_auto *r, unsigned long delay) 199 { 200 queue_delayed_work(fw_workqueue, &r->work, delay); 201 } 202 203 /* 204 * dequeue_event() just kfree()'s the event, so the event has to be 205 * the first field in a struct XYZ_event. 206 */ 207 struct event { 208 struct { void *data; size_t size; } v[2]; 209 struct list_head link; 210 }; 211 212 struct bus_reset_event { 213 struct event event; 214 struct fw_cdev_event_bus_reset reset; 215 }; 216 217 struct outbound_transaction_event { 218 struct event event; 219 struct client *client; 220 struct outbound_transaction_resource r; 221 union { 222 struct fw_cdev_event_response without_tstamp; 223 struct fw_cdev_event_response2 with_tstamp; 224 } rsp; 225 }; 226 227 struct inbound_transaction_event { 228 struct event event; 229 union { 230 struct fw_cdev_event_request request; 231 struct fw_cdev_event_request2 request2; 232 struct fw_cdev_event_request3 with_tstamp; 233 } req; 234 }; 235 236 struct iso_interrupt_event { 237 struct event event; 238 struct fw_cdev_event_iso_interrupt interrupt; 239 }; 240 241 struct iso_interrupt_mc_event { 242 struct event event; 243 struct fw_cdev_event_iso_interrupt_mc interrupt; 244 }; 245 246 struct iso_resource_event { 247 struct event event; 248 struct fw_cdev_event_iso_resource iso_resource; 249 }; 250 251 struct outbound_phy_packet_event { 252 struct event event; 253 struct client *client; 254 struct fw_packet p; 255 union { 256 struct fw_cdev_event_phy_packet without_tstamp; 257 struct fw_cdev_event_phy_packet2 with_tstamp; 258 } phy_packet; 259 }; 260 261 struct inbound_phy_packet_event { 262 struct event event; 263 union { 264 struct fw_cdev_event_phy_packet without_tstamp; 265 struct fw_cdev_event_phy_packet2 with_tstamp; 266 } phy_packet; 267 }; 268 269 #ifdef CONFIG_COMPAT 270 static void __user *u64_to_uptr(u64 value) 271 { 272 if (in_compat_syscall()) 273 return compat_ptr(value); 274 else 275 return (void __user *)(unsigned long)value; 276 } 277 278 static u64 uptr_to_u64(void __user *ptr) 279 { 280 if (in_compat_syscall()) 281 return ptr_to_compat(ptr); 282 else 283 return (u64)(unsigned long)ptr; 284 } 285 #else 286 static inline void __user *u64_to_uptr(u64 value) 287 { 288 return (void __user *)(unsigned long)value; 289 } 290 291 static inline u64 uptr_to_u64(void __user *ptr) 292 { 293 return (u64)(unsigned long)ptr; 294 } 295 #endif /* CONFIG_COMPAT */ 296 297 static int fw_device_op_open(struct inode *inode, struct file *file) 298 { 299 struct fw_device *device; 300 struct client *client; 301 302 device = fw_device_get_by_devt(inode->i_rdev); 303 if (device == NULL) 304 return -ENODEV; 305 306 if (fw_device_is_shutdown(device)) { 307 fw_device_put(device); 308 return -ENODEV; 309 } 310 311 client = kzalloc_obj(*client); 312 if (client == NULL) { 313 fw_device_put(device); 314 return -ENOMEM; 315 } 316 317 client->device = device; 318 mutex_init(&client->mutex); 319 xa_init_flags(&client->resource_xa, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_BH); 320 INIT_LIST_HEAD(&client->event_list); 321 init_waitqueue_head(&client->wait); 322 init_waitqueue_head(&client->tx_flush_wait); 323 INIT_LIST_HEAD(&client->phy_receiver_link); 324 INIT_LIST_HEAD(&client->link); 325 kref_init(&client->kref); 326 mutex_init(&client->iso_context_mutex); 327 328 file->private_data = client; 329 330 return nonseekable_open(inode, file); 331 } 332 333 static void queue_event(struct client *client, struct event *event, 334 void *data0, size_t size0, void *data1, size_t size1) 335 { 336 event->v[0].data = data0; 337 event->v[0].size = size0; 338 event->v[1].data = data1; 339 event->v[1].size = size1; 340 341 scoped_guard(mutex, &client->mutex) { 342 if (client->in_shutdown) { 343 kfree(event); 344 } else { 345 list_add_tail(&event->link, &client->event_list); 346 347 wake_up_interruptible(&client->wait); 348 } 349 } 350 } 351 352 static ssize_t dequeue_event(struct client *client, char __user *buffer, size_t count) 353 { 354 struct event *event; 355 356 // After the following block, the event pointer above is guaranteed to have a correct value. 357 { 358 mutex_lock(&client->mutex); 359 360 // This could be replaced with wait_var_event_any_lock() if poll_wait() alternative 361 // would be introduced. 362 int ret = ___wait_event(client->wait, 363 !list_empty(&client->event_list) || fw_device_is_shutdown(client->device), 364 TASK_INTERRUPTIBLE, 0, 0, 365 mutex_unlock(&client->mutex); 366 schedule(); 367 mutex_lock(&client->mutex) 368 ); 369 if (ret < 0) { 370 mutex_unlock(&client->mutex); 371 return ret; 372 } 373 374 if (fw_device_is_shutdown(client->device)) { 375 mutex_unlock(&client->mutex); 376 return -ENODEV; 377 } 378 379 event = list_first_entry(&client->event_list, struct event, link); 380 list_del(&event->link); 381 382 mutex_unlock(&client->mutex); 383 } 384 385 ssize_t ret = 0; 386 387 for (int i = 0; i < ARRAY_SIZE(event->v) && ret < count; i++) { 388 size_t size = min(event->v[i].size, count - ret); 389 if (copy_to_user(buffer + ret, event->v[i].data, size)) { 390 ret = -EFAULT; 391 break; 392 } 393 ret += size; 394 } 395 396 kfree(event); 397 398 return ret; 399 } 400 401 static ssize_t fw_device_op_read(struct file *file, char __user *buffer, 402 size_t count, loff_t *offset) 403 { 404 struct client *client = file->private_data; 405 406 return dequeue_event(client, buffer, count); 407 } 408 409 static void fill_bus_reset_event(struct fw_cdev_event_bus_reset *event, 410 struct client *client) 411 __must_hold(&client->device->client_list_mutex) 412 { 413 lockdep_assert_held(&client->device->client_list_mutex); 414 415 // This member is related to the above mutex. In detail, see 93b37905f70 ("firewire: cdev: 416 // prevent race between first get_info ioctl and bus reset event queuing"). 417 event->closure = client->bus_reset_closure; 418 event->type = FW_CDEV_EVENT_BUS_RESET; 419 420 event->generation = client->device->generation; 421 smp_rmb(); 422 event->node_id = client->device->node_id; 423 424 struct fw_card *card = client->device->card; 425 426 scoped_guard(spinlock_irq, &card->lock) { 427 event->local_node_id = card->local_node->node_id; 428 event->bm_node_id = card->bm_node_id; 429 event->irm_node_id = card->irm_node->node_id; 430 event->root_node_id = card->root_node->node_id; 431 } 432 } 433 434 static void for_each_client(struct fw_device *device, 435 void (*callback)(struct client *client)) 436 { 437 struct client *c; 438 439 guard(mutex)(&device->client_list_mutex); 440 441 list_for_each_entry(c, &device->client_list, link) 442 callback(c); 443 } 444 445 static void queue_bus_reset_event(struct client *client) 446 { 447 struct bus_reset_event *e; 448 struct client_resource *resource; 449 unsigned long index; 450 451 e = kzalloc_obj(*e); 452 if (e == NULL) 453 return; 454 455 fill_bus_reset_event(&e->reset, client); 456 457 queue_event(client, &e->event, 458 &e->reset, sizeof(e->reset), NULL, 0); 459 460 scoped_guard(mutex, &client->mutex) { 461 xa_for_each(&client->resource_xa, index, resource) { 462 if (is_iso_resource_auto(resource)) 463 schedule_iso_resource_auto(to_iso_resource_auto(resource), 0); 464 } 465 } 466 } 467 468 void fw_device_cdev_update(struct fw_device *device) 469 { 470 for_each_client(device, queue_bus_reset_event); 471 } 472 473 static void wake_up_client(struct client *client) 474 { 475 wake_up_interruptible(&client->wait); 476 } 477 478 void fw_device_cdev_remove(struct fw_device *device) 479 { 480 for_each_client(device, wake_up_client); 481 } 482 483 union ioctl_arg { 484 struct fw_cdev_get_info get_info; 485 struct fw_cdev_send_request send_request; 486 struct fw_cdev_allocate allocate; 487 struct fw_cdev_deallocate deallocate; 488 struct fw_cdev_send_response send_response; 489 struct fw_cdev_initiate_bus_reset initiate_bus_reset; 490 struct fw_cdev_add_descriptor add_descriptor; 491 struct fw_cdev_remove_descriptor remove_descriptor; 492 struct fw_cdev_create_iso_context create_iso_context; 493 struct fw_cdev_queue_iso queue_iso; 494 struct fw_cdev_start_iso start_iso; 495 struct fw_cdev_stop_iso stop_iso; 496 struct fw_cdev_get_cycle_timer get_cycle_timer; 497 struct fw_cdev_allocate_iso_resource allocate_iso_resource; 498 struct fw_cdev_send_stream_packet send_stream_packet; 499 struct fw_cdev_get_cycle_timer2 get_cycle_timer2; 500 struct fw_cdev_send_phy_packet send_phy_packet; 501 struct fw_cdev_receive_phy_packets receive_phy_packets; 502 struct fw_cdev_set_iso_channels set_iso_channels; 503 struct fw_cdev_flush_iso flush_iso; 504 }; 505 506 static int ioctl_get_info(struct client *client, union ioctl_arg *arg) 507 { 508 struct fw_cdev_get_info *a = &arg->get_info; 509 510 client->version = a->version; 511 a->version = FW_CDEV_KERNEL_VERSION; 512 a->card = client->device->card->index; 513 514 scoped_guard(rwsem_read, &fw_device_rwsem) { 515 if (a->rom != 0) { 516 size_t length = min_t(size_t, a->rom_length, 517 client->device->config_rom_length * 4); 518 519 if (copy_to_user(u64_to_uptr(a->rom), client->device->config_rom, length)) 520 return -EFAULT; 521 } 522 a->rom_length = client->device->config_rom_length * 4; 523 } 524 525 scoped_guard(mutex, &client->device->client_list_mutex) { 526 // Coordinate concurrent access to this member with bus reset event handling, see 527 // 93b37905f70 ("firewire: cdev: prevent race between first get_info ioctl and bus 528 // reset event queuing"). 529 client->bus_reset_closure = a->bus_reset_closure; 530 531 if (a->bus_reset != 0) { 532 struct fw_cdev_event_bus_reset bus_reset; 533 534 memset(&bus_reset, 0, sizeof(bus_reset)); 535 fill_bus_reset_event(&bus_reset, client); 536 537 // This structure has 4 bytes of trailing padding under the System V ABI 538 // on most architectures (due to 8-byte alignment of the long long type), 539 // except for Intel386 (where long long type is aligned to 4 bytes). In 540 // either case, the effective length is 36 bytes. 541 if (copy_to_user(u64_to_uptr(a->bus_reset), &bus_reset, 36)) 542 return -EFAULT; 543 } 544 545 if (list_empty(&client->link)) 546 list_add_tail(&client->link, &client->device->client_list); 547 } 548 549 return 0; 550 } 551 552 static int add_client_resource(struct client *client, struct client_resource *resource, 553 client_resource_release_fn_t release) 554 { 555 scoped_guard(mutex, &client->mutex) { 556 u32 index; 557 int ret; 558 559 if (client->in_shutdown) 560 return -ECANCELED; 561 562 ret = xa_alloc(&client->resource_xa, &index, resource, xa_limit_32b, GFP_KERNEL); 563 if (ret < 0) 564 return ret; 565 566 resource->handle = index; 567 resource->release = release; 568 client_get(client); 569 } 570 571 return 0; 572 } 573 574 static int release_client_resource(struct client *client, u32 handle, 575 client_resource_release_fn_t release, 576 struct client_resource **return_resource) 577 { 578 unsigned long index = handle; 579 struct client_resource *resource; 580 581 scoped_guard(mutex, &client->mutex) { 582 if (client->in_shutdown) 583 return -EINVAL; 584 585 resource = xa_load(&client->resource_xa, index); 586 if (!resource || resource->release != release) 587 return -EINVAL; 588 589 xa_erase(&client->resource_xa, handle); 590 } 591 592 if (return_resource) 593 *return_resource = resource; 594 else 595 resource->release(client, resource); 596 597 client_put(client); 598 599 return 0; 600 } 601 602 static void release_transaction(struct client *client, 603 struct client_resource *resource) 604 { 605 } 606 607 static void complete_transaction(struct fw_card *card, int rcode, u32 request_tstamp, 608 u32 response_tstamp, void *payload, size_t length, void *data) 609 { 610 struct outbound_transaction_event *e = data; 611 struct client *client = e->client; 612 unsigned long index = e->r.resource.handle; 613 614 scoped_guard(mutex, &client->mutex) { 615 xa_erase(&client->resource_xa, index); 616 if (client->in_shutdown) 617 wake_up(&client->tx_flush_wait); 618 } 619 620 switch (e->rsp.without_tstamp.type) { 621 case FW_CDEV_EVENT_RESPONSE: 622 { 623 struct fw_cdev_event_response *rsp = &e->rsp.without_tstamp; 624 625 if (length < rsp->length) 626 rsp->length = length; 627 if (rcode == RCODE_COMPLETE) 628 memcpy(rsp->data, payload, rsp->length); 629 630 rsp->rcode = rcode; 631 632 // In the case that sizeof(*rsp) doesn't align with the position of the 633 // data, and the read is short, preserve an extra copy of the data 634 // to stay compatible with a pre-2.6.27 bug. Since the bug is harmless 635 // for short reads and some apps depended on it, this is both safe 636 // and prudent for compatibility. 637 if (rsp->length <= sizeof(*rsp) - offsetof(typeof(*rsp), data)) 638 queue_event(client, &e->event, rsp, sizeof(*rsp), rsp->data, rsp->length); 639 else 640 queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, NULL, 0); 641 642 break; 643 } 644 case FW_CDEV_EVENT_RESPONSE2: 645 { 646 struct fw_cdev_event_response2 *rsp = &e->rsp.with_tstamp; 647 648 if (length < rsp->length) 649 rsp->length = length; 650 if (rcode == RCODE_COMPLETE) 651 memcpy(rsp->data, payload, rsp->length); 652 653 rsp->rcode = rcode; 654 rsp->request_tstamp = request_tstamp; 655 rsp->response_tstamp = response_tstamp; 656 657 queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, NULL, 0); 658 659 break; 660 } 661 default: 662 WARN_ON(1); 663 break; 664 } 665 666 // Drop the xarray's reference. 667 client_put(client); 668 } 669 670 static int init_request(struct client *client, 671 struct fw_cdev_send_request *request, 672 int destination_id, int speed) 673 { 674 struct outbound_transaction_event *e; 675 void *payload; 676 int ret; 677 678 if (request->tcode != TCODE_STREAM_DATA && 679 (request->length > 4096 || request->length > 512 << speed)) 680 return -EIO; 681 682 if (request->tcode == TCODE_WRITE_QUADLET_REQUEST && 683 request->length < 4) 684 return -EINVAL; 685 686 e = kzalloc_flex(*e, rsp.with_tstamp.data, DIV_ROUND_UP(request->length, sizeof(u32))); 687 if (!e) 688 return -ENOMEM; 689 e->client = client; 690 691 if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) { 692 struct fw_cdev_event_response *rsp = &e->rsp.without_tstamp; 693 694 rsp->type = FW_CDEV_EVENT_RESPONSE; 695 rsp->length = request->length; 696 rsp->closure = request->closure; 697 payload = rsp->data; 698 } else { 699 struct fw_cdev_event_response2 *rsp = &e->rsp.with_tstamp; 700 701 rsp->type = FW_CDEV_EVENT_RESPONSE2; 702 rsp->length = request->length; 703 rsp->closure = request->closure; 704 payload = rsp->data; 705 } 706 707 if (request->data && copy_from_user(payload, u64_to_uptr(request->data), request->length)) { 708 ret = -EFAULT; 709 goto failed; 710 } 711 712 ret = add_client_resource(client, &e->r.resource, release_transaction); 713 if (ret < 0) 714 goto failed; 715 716 fw_send_request_with_tstamp(client->device->card, &e->r.transaction, request->tcode, 717 destination_id, request->generation, speed, request->offset, 718 payload, request->length, complete_transaction, e); 719 return 0; 720 721 failed: 722 kfree(e); 723 724 return ret; 725 } 726 727 static int ioctl_send_request(struct client *client, union ioctl_arg *arg) 728 { 729 switch (arg->send_request.tcode) { 730 case TCODE_WRITE_QUADLET_REQUEST: 731 case TCODE_WRITE_BLOCK_REQUEST: 732 case TCODE_READ_QUADLET_REQUEST: 733 case TCODE_READ_BLOCK_REQUEST: 734 case TCODE_LOCK_MASK_SWAP: 735 case TCODE_LOCK_COMPARE_SWAP: 736 case TCODE_LOCK_FETCH_ADD: 737 case TCODE_LOCK_LITTLE_ADD: 738 case TCODE_LOCK_BOUNDED_ADD: 739 case TCODE_LOCK_WRAP_ADD: 740 case TCODE_LOCK_VENDOR_DEPENDENT: 741 break; 742 default: 743 return -EINVAL; 744 } 745 746 return init_request(client, &arg->send_request, client->device->node_id, 747 client->device->max_speed); 748 } 749 750 static void release_request(struct client *client, 751 struct client_resource *resource) 752 { 753 struct inbound_transaction_resource *r = to_inbound_transaction_resource(resource); 754 755 if (r->is_fcp) 756 fw_request_put(r->request); 757 else 758 fw_send_response(r->card, r->request, RCODE_CONFLICT_ERROR); 759 760 fw_card_put(r->card); 761 kfree(r); 762 } 763 764 static void handle_request(struct fw_card *card, struct fw_request *request, 765 int tcode, int destination, int source, 766 int generation, unsigned long long offset, 767 void *payload, size_t length, void *callback_data) 768 { 769 struct address_handler_resource *handler = callback_data; 770 bool is_fcp = is_in_fcp_region(offset, length); 771 struct inbound_transaction_resource *r; 772 struct inbound_transaction_event *e; 773 size_t event_size0; 774 int ret; 775 776 /* card may be different from handler->client->device->card */ 777 fw_card_get(card); 778 779 // Extend the lifetime of data for request so that its payload is safely accessible in 780 // the process context for the client. 781 if (is_fcp) 782 fw_request_get(request); 783 784 r = kmalloc_obj(*r); 785 e = kmalloc_obj(*e); 786 if (r == NULL || e == NULL) 787 goto failed; 788 789 r->card = card; 790 r->request = request; 791 r->is_fcp = is_fcp; 792 r->data = payload; 793 r->length = length; 794 795 ret = add_client_resource(handler->client, &r->resource, release_request); 796 if (ret < 0) 797 goto failed; 798 799 if (handler->client->version < FW_CDEV_VERSION_EVENT_REQUEST2) { 800 struct fw_cdev_event_request *req = &e->req.request; 801 802 if (tcode & 0x10) 803 tcode = TCODE_LOCK_REQUEST; 804 805 req->type = FW_CDEV_EVENT_REQUEST; 806 req->tcode = tcode; 807 req->offset = offset; 808 req->length = length; 809 req->handle = r->resource.handle; 810 req->closure = handler->closure; 811 event_size0 = sizeof(*req); 812 } else if (handler->client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) { 813 struct fw_cdev_event_request2 *req = &e->req.request2; 814 815 req->type = FW_CDEV_EVENT_REQUEST2; 816 req->tcode = tcode; 817 req->offset = offset; 818 req->source_node_id = source; 819 req->destination_node_id = destination; 820 req->card = card->index; 821 req->generation = generation; 822 req->length = length; 823 req->handle = r->resource.handle; 824 req->closure = handler->closure; 825 event_size0 = sizeof(*req); 826 } else { 827 struct fw_cdev_event_request3 *req = &e->req.with_tstamp; 828 829 req->type = FW_CDEV_EVENT_REQUEST3; 830 req->tcode = tcode; 831 req->offset = offset; 832 req->source_node_id = source; 833 req->destination_node_id = destination; 834 req->card = card->index; 835 req->generation = generation; 836 req->length = length; 837 req->handle = r->resource.handle; 838 req->closure = handler->closure; 839 req->tstamp = fw_request_get_timestamp(request); 840 event_size0 = sizeof(*req); 841 } 842 843 queue_event(handler->client, &e->event, 844 &e->req, event_size0, r->data, length); 845 return; 846 847 failed: 848 kfree(r); 849 kfree(e); 850 851 if (!is_fcp) 852 fw_send_response(card, request, RCODE_CONFLICT_ERROR); 853 else 854 fw_request_put(request); 855 856 fw_card_put(card); 857 } 858 859 static void release_address_handler(struct client *client, 860 struct client_resource *resource) 861 { 862 struct address_handler_resource *r = to_address_handler_resource(resource); 863 864 fw_core_remove_address_handler(&r->handler); 865 kfree(r); 866 } 867 868 static int ioctl_allocate(struct client *client, union ioctl_arg *arg) 869 { 870 struct fw_cdev_allocate *a = &arg->allocate; 871 struct address_handler_resource *r; 872 struct fw_address_region region; 873 int ret; 874 875 r = kmalloc_obj(*r); 876 if (r == NULL) 877 return -ENOMEM; 878 879 region.start = a->offset; 880 if (client->version < FW_CDEV_VERSION_ALLOCATE_REGION_END) 881 region.end = a->offset + a->length; 882 else 883 region.end = a->region_end; 884 885 r->handler.length = a->length; 886 r->handler.address_callback = handle_request; 887 r->handler.callback_data = r; 888 r->closure = a->closure; 889 r->client = client; 890 891 ret = fw_core_add_address_handler(&r->handler, ®ion); 892 if (ret < 0) { 893 kfree(r); 894 return ret; 895 } 896 a->offset = r->handler.offset; 897 898 ret = add_client_resource(client, &r->resource, release_address_handler); 899 if (ret < 0) { 900 release_address_handler(client, &r->resource); 901 return ret; 902 } 903 a->handle = r->resource.handle; 904 905 return 0; 906 } 907 908 static int ioctl_deallocate(struct client *client, union ioctl_arg *arg) 909 { 910 return release_client_resource(client, arg->deallocate.handle, 911 release_address_handler, NULL); 912 } 913 914 static int ioctl_send_response(struct client *client, union ioctl_arg *arg) 915 { 916 struct fw_cdev_send_response *a = &arg->send_response; 917 struct client_resource *resource; 918 struct inbound_transaction_resource *r; 919 int ret = 0; 920 921 if (release_client_resource(client, a->handle, 922 release_request, &resource) < 0) 923 return -EINVAL; 924 925 r = to_inbound_transaction_resource(resource); 926 if (r->is_fcp) { 927 fw_request_put(r->request); 928 goto out; 929 } 930 931 if (a->length != fw_get_response_length(r->request)) { 932 ret = -EINVAL; 933 fw_request_put(r->request); 934 goto out; 935 } 936 if (copy_from_user(r->data, u64_to_uptr(a->data), a->length)) { 937 ret = -EFAULT; 938 fw_request_put(r->request); 939 goto out; 940 } 941 fw_send_response(r->card, r->request, a->rcode); 942 out: 943 fw_card_put(r->card); 944 kfree(r); 945 946 return ret; 947 } 948 949 static int ioctl_initiate_bus_reset(struct client *client, union ioctl_arg *arg) 950 { 951 fw_schedule_bus_reset(client->device->card, true, 952 arg->initiate_bus_reset.type == FW_CDEV_SHORT_RESET); 953 return 0; 954 } 955 956 static void release_descriptor(struct client *client, 957 struct client_resource *resource) 958 { 959 struct descriptor_resource *r = to_descriptor_resource(resource); 960 961 fw_core_remove_descriptor(&r->descriptor); 962 kfree(r); 963 } 964 965 static int ioctl_add_descriptor(struct client *client, union ioctl_arg *arg) 966 { 967 struct fw_cdev_add_descriptor *a = &arg->add_descriptor; 968 struct descriptor_resource *r; 969 int ret; 970 971 /* Access policy: Allow this ioctl only on local nodes' device files. */ 972 if (!client->device->is_local) 973 return -ENOSYS; 974 975 if (a->length > 256) 976 return -EINVAL; 977 978 r = kmalloc_flex(*r, data, a->length); 979 if (r == NULL) 980 return -ENOMEM; 981 982 if (copy_from_user(r->data, u64_to_uptr(a->data), 983 flex_array_size(r, data, a->length))) { 984 ret = -EFAULT; 985 goto failed; 986 } 987 988 r->descriptor.length = a->length; 989 r->descriptor.immediate = a->immediate; 990 r->descriptor.key = a->key; 991 r->descriptor.data = r->data; 992 993 ret = fw_core_add_descriptor(&r->descriptor); 994 if (ret < 0) 995 goto failed; 996 997 ret = add_client_resource(client, &r->resource, release_descriptor); 998 if (ret < 0) { 999 fw_core_remove_descriptor(&r->descriptor); 1000 goto failed; 1001 } 1002 a->handle = r->resource.handle; 1003 1004 return 0; 1005 failed: 1006 kfree(r); 1007 1008 return ret; 1009 } 1010 1011 static int ioctl_remove_descriptor(struct client *client, union ioctl_arg *arg) 1012 { 1013 return release_client_resource(client, arg->remove_descriptor.handle, 1014 release_descriptor, NULL); 1015 } 1016 1017 static void iso_callback(struct fw_iso_context *context, u32 cycle, 1018 size_t header_length, void *header, void *data) 1019 { 1020 struct client *client = data; 1021 struct iso_interrupt_event *e; 1022 1023 e = kzalloc_flex(*e, interrupt.header, header_length); 1024 if (e == NULL) 1025 return; 1026 1027 e->interrupt.type = FW_CDEV_EVENT_ISO_INTERRUPT; 1028 e->interrupt.closure = client->iso_closure; 1029 e->interrupt.cycle = cycle; 1030 e->interrupt.header_length = header_length; 1031 memcpy(e->interrupt.header, header, header_length); 1032 queue_event(client, &e->event, &e->interrupt, 1033 sizeof(e->interrupt) + header_length, NULL, 0); 1034 } 1035 1036 static void iso_mc_callback(struct fw_iso_context *context, 1037 dma_addr_t completed, void *data) 1038 { 1039 struct client *client = data; 1040 struct iso_interrupt_mc_event *e; 1041 1042 e = kmalloc_obj(*e); 1043 if (e == NULL) 1044 return; 1045 1046 e->interrupt.type = FW_CDEV_EVENT_ISO_INTERRUPT_MULTICHANNEL; 1047 e->interrupt.closure = client->iso_closure; 1048 e->interrupt.completed = fw_iso_buffer_lookup(&client->buffer, 1049 completed); 1050 queue_event(client, &e->event, &e->interrupt, 1051 sizeof(e->interrupt), NULL, 0); 1052 } 1053 1054 static enum dma_data_direction iso_dma_direction(struct fw_iso_context *context) 1055 { 1056 if (context->type == FW_ISO_CONTEXT_TRANSMIT) 1057 return DMA_TO_DEVICE; 1058 else 1059 return DMA_FROM_DEVICE; 1060 } 1061 1062 static int ioctl_create_iso_context(struct client *client, union ioctl_arg *arg) 1063 { 1064 struct fw_cdev_create_iso_context *a = &arg->create_iso_context; 1065 struct fw_iso_context *context; 1066 int ret; 1067 1068 BUILD_BUG_ON(FW_CDEV_ISO_CONTEXT_TRANSMIT != FW_ISO_CONTEXT_TRANSMIT || 1069 FW_CDEV_ISO_CONTEXT_RECEIVE != FW_ISO_CONTEXT_RECEIVE || 1070 FW_CDEV_ISO_CONTEXT_RECEIVE_MULTICHANNEL != 1071 FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL); 1072 1073 switch (a->type) { 1074 case FW_ISO_CONTEXT_TRANSMIT: 1075 if (a->speed > SCODE_3200 || a->channel > 63) 1076 return -EINVAL; 1077 break; 1078 1079 case FW_ISO_CONTEXT_RECEIVE: 1080 if (a->header_size < 4 || (a->header_size & 3) || 1081 a->channel > 63) 1082 return -EINVAL; 1083 break; 1084 1085 case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL: 1086 break; 1087 1088 default: 1089 return -EINVAL; 1090 } 1091 1092 if (a->type == FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL) 1093 context = fw_iso_mc_context_create(client->device->card, iso_mc_callback, client); 1094 else 1095 context = fw_iso_context_create(client->device->card, a->type, a->channel, a->speed, 1096 a->header_size, iso_callback, client); 1097 if (IS_ERR(context)) 1098 return PTR_ERR(context); 1099 if (client->version < FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW) 1100 context->flags |= FW_ISO_CONTEXT_FLAG_DROP_OVERFLOW_HEADERS; 1101 1102 // We only support one context at this time. 1103 scoped_guard(mutex, &client->iso_context_mutex) { 1104 if (client->iso_context != NULL) { 1105 fw_iso_context_destroy(context); 1106 1107 return -EBUSY; 1108 } 1109 // The DMA mapping operation is available if the buffer is already allocated by 1110 // mmap(2) system call. If not, it is delegated to the system call. 1111 if (client->buffer.pages && !client->buffer.dma_addrs) { 1112 ret = fw_iso_buffer_map_dma(&client->buffer, client->device->card, 1113 iso_dma_direction(context)); 1114 if (ret < 0) { 1115 fw_iso_context_destroy(context); 1116 1117 return ret; 1118 } 1119 } 1120 client->iso_closure = a->closure; 1121 client->iso_context = context; 1122 } 1123 1124 a->handle = 0; 1125 1126 return 0; 1127 } 1128 1129 static int ioctl_set_iso_channels(struct client *client, union ioctl_arg *arg) 1130 { 1131 struct fw_cdev_set_iso_channels *a = &arg->set_iso_channels; 1132 struct fw_iso_context *ctx = client->iso_context; 1133 1134 if (ctx == NULL || a->handle != 0) 1135 return -EINVAL; 1136 1137 return fw_iso_context_set_channels(ctx, &a->channels); 1138 } 1139 1140 /* Macros for decoding the iso packet control header. */ 1141 #define GET_PAYLOAD_LENGTH(v) ((v) & 0xffff) 1142 #define GET_INTERRUPT(v) (((v) >> 16) & 0x01) 1143 #define GET_SKIP(v) (((v) >> 17) & 0x01) 1144 #define GET_TAG(v) (((v) >> 18) & 0x03) 1145 #define GET_SY(v) (((v) >> 20) & 0x0f) 1146 #define GET_HEADER_LENGTH(v) (((v) >> 24) & 0xff) 1147 1148 static int ioctl_queue_iso(struct client *client, union ioctl_arg *arg) 1149 { 1150 struct fw_cdev_queue_iso *a = &arg->queue_iso; 1151 struct fw_cdev_iso_packet __user *p, *end, *next; 1152 struct fw_iso_context *ctx = client->iso_context; 1153 unsigned long payload, buffer_end, transmit_header_bytes = 0; 1154 u32 control; 1155 int count; 1156 DEFINE_RAW_FLEX(struct fw_iso_packet, u, header, 64); 1157 1158 if (ctx == NULL || a->handle != 0) 1159 return -EINVAL; 1160 1161 /* 1162 * If the user passes a non-NULL data pointer, has mmap()'ed 1163 * the iso buffer, and the pointer points inside the buffer, 1164 * we setup the payload pointers accordingly. Otherwise we 1165 * set them both to 0, which will still let packets with 1166 * payload_length == 0 through. In other words, if no packets 1167 * use the indirect payload, the iso buffer need not be mapped 1168 * and the a->data pointer is ignored. 1169 */ 1170 payload = (unsigned long)a->data - client->vm_start; 1171 buffer_end = client->buffer.page_count << PAGE_SHIFT; 1172 if (a->data == 0 || client->buffer.pages == NULL || 1173 payload >= buffer_end) { 1174 payload = 0; 1175 buffer_end = 0; 1176 } 1177 1178 if (ctx->type == FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL && payload & 3) 1179 return -EINVAL; 1180 1181 p = (struct fw_cdev_iso_packet __user *)u64_to_uptr(a->packets); 1182 1183 end = (void __user *)p + a->size; 1184 count = 0; 1185 while (p < end) { 1186 if (get_user(control, &p->control)) 1187 return -EFAULT; 1188 u->payload_length = GET_PAYLOAD_LENGTH(control); 1189 u->interrupt = GET_INTERRUPT(control); 1190 u->skip = GET_SKIP(control); 1191 u->tag = GET_TAG(control); 1192 u->sy = GET_SY(control); 1193 u->header_length = GET_HEADER_LENGTH(control); 1194 1195 switch (ctx->type) { 1196 case FW_ISO_CONTEXT_TRANSMIT: 1197 if (u->header_length & 3) 1198 return -EINVAL; 1199 transmit_header_bytes = u->header_length; 1200 break; 1201 1202 case FW_ISO_CONTEXT_RECEIVE: 1203 if (u->header_length == 0 || 1204 u->header_length % ctx->header_size != 0) 1205 return -EINVAL; 1206 break; 1207 1208 case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL: 1209 if (u->payload_length == 0 || 1210 u->payload_length & 3) 1211 return -EINVAL; 1212 break; 1213 } 1214 1215 next = (struct fw_cdev_iso_packet __user *) 1216 &p->header[transmit_header_bytes / 4]; 1217 if (next > end) 1218 return -EINVAL; 1219 if (copy_from_user 1220 (u->header, p->header, transmit_header_bytes)) 1221 return -EFAULT; 1222 if (u->skip && ctx->type == FW_ISO_CONTEXT_TRANSMIT && 1223 u->header_length + u->payload_length > 0) 1224 return -EINVAL; 1225 if (payload + u->payload_length > buffer_end) 1226 return -EINVAL; 1227 1228 if (fw_iso_context_queue(ctx, u, &client->buffer, payload)) 1229 break; 1230 1231 p = next; 1232 payload += u->payload_length; 1233 count++; 1234 } 1235 fw_iso_context_queue_flush(ctx); 1236 1237 a->size -= uptr_to_u64(p) - a->packets; 1238 a->packets = uptr_to_u64(p); 1239 a->data = client->vm_start + payload; 1240 1241 return count; 1242 } 1243 1244 static int ioctl_start_iso(struct client *client, union ioctl_arg *arg) 1245 { 1246 struct fw_cdev_start_iso *a = &arg->start_iso; 1247 1248 BUILD_BUG_ON( 1249 FW_CDEV_ISO_CONTEXT_MATCH_TAG0 != FW_ISO_CONTEXT_MATCH_TAG0 || 1250 FW_CDEV_ISO_CONTEXT_MATCH_TAG1 != FW_ISO_CONTEXT_MATCH_TAG1 || 1251 FW_CDEV_ISO_CONTEXT_MATCH_TAG2 != FW_ISO_CONTEXT_MATCH_TAG2 || 1252 FW_CDEV_ISO_CONTEXT_MATCH_TAG3 != FW_ISO_CONTEXT_MATCH_TAG3 || 1253 FW_CDEV_ISO_CONTEXT_MATCH_ALL_TAGS != FW_ISO_CONTEXT_MATCH_ALL_TAGS); 1254 1255 if (client->iso_context == NULL || a->handle != 0) 1256 return -EINVAL; 1257 1258 if (client->iso_context->type == FW_ISO_CONTEXT_RECEIVE && 1259 (a->tags == 0 || a->tags > 15 || a->sync > 15)) 1260 return -EINVAL; 1261 1262 return fw_iso_context_start(client->iso_context, 1263 a->cycle, a->sync, a->tags); 1264 } 1265 1266 static int ioctl_stop_iso(struct client *client, union ioctl_arg *arg) 1267 { 1268 struct fw_cdev_stop_iso *a = &arg->stop_iso; 1269 1270 if (client->iso_context == NULL || a->handle != 0) 1271 return -EINVAL; 1272 1273 return fw_iso_context_stop(client->iso_context); 1274 } 1275 1276 static int ioctl_flush_iso(struct client *client, union ioctl_arg *arg) 1277 { 1278 struct fw_cdev_flush_iso *a = &arg->flush_iso; 1279 1280 if (client->iso_context == NULL || a->handle != 0) 1281 return -EINVAL; 1282 1283 return fw_iso_context_flush_completions(client->iso_context); 1284 } 1285 1286 static int ioctl_get_cycle_timer2(struct client *client, union ioctl_arg *arg) 1287 { 1288 struct fw_cdev_get_cycle_timer2 *a = &arg->get_cycle_timer2; 1289 struct fw_card *card = client->device->card; 1290 struct timespec64 ts = {0, 0}; 1291 u32 cycle_time = 0; 1292 int ret; 1293 1294 guard(irq)(); 1295 1296 ret = fw_card_read_cycle_time(card, &cycle_time); 1297 if (ret < 0) 1298 return ret; 1299 1300 switch (a->clk_id) { 1301 case CLOCK_REALTIME: ktime_get_real_ts64(&ts); break; 1302 case CLOCK_MONOTONIC: ktime_get_ts64(&ts); break; 1303 case CLOCK_MONOTONIC_RAW: ktime_get_raw_ts64(&ts); break; 1304 default: 1305 return -EINVAL; 1306 } 1307 1308 a->tv_sec = ts.tv_sec; 1309 a->tv_nsec = ts.tv_nsec; 1310 a->cycle_timer = cycle_time; 1311 1312 return 0; 1313 } 1314 1315 static int ioctl_get_cycle_timer(struct client *client, union ioctl_arg *arg) 1316 { 1317 struct fw_cdev_get_cycle_timer *a = &arg->get_cycle_timer; 1318 struct fw_cdev_get_cycle_timer2 ct2; 1319 1320 ct2.clk_id = CLOCK_REALTIME; 1321 ioctl_get_cycle_timer2(client, (union ioctl_arg *)&ct2); 1322 1323 a->local_time = ct2.tv_sec * USEC_PER_SEC + ct2.tv_nsec / NSEC_PER_USEC; 1324 a->cycle_timer = ct2.cycle_timer; 1325 1326 return 0; 1327 } 1328 1329 static int fill_iso_resource_params(struct iso_resource_params *params, 1330 struct fw_cdev_allocate_iso_resource *request) 1331 { 1332 if ((request->channels == 0 && request->bandwidth == 0) || 1333 request->bandwidth > BANDWIDTH_AVAILABLE_INITIAL) 1334 return -EINVAL; 1335 1336 params->channels_mask = request->channels; 1337 params->bandwidth = request->bandwidth; 1338 1339 return 0; 1340 } 1341 1342 static void iso_resource_auto_work(struct work_struct *work) 1343 { 1344 struct iso_resource_event *e; 1345 struct iso_resource_auto *r = from_work(r, work, work.work); 1346 struct client *client = r->client; 1347 unsigned long index = r->resource.handle; 1348 int channel, bandwidth; 1349 bool free; 1350 1351 u64 reset_jiffies = client->device->card->reset_jiffies; 1352 int current_generation = client->device->generation; 1353 1354 int resource_generation = xchg(&r->generation, current_generation); // But no need to be atomic. 1355 int todo = atomic_read(&r->todo); 1356 1357 switch (todo) { 1358 case ISO_RES_AUTO_ALLOC: 1359 // Allow 1000ms grace period for other reallocations. 1360 if (time_is_after_jiffies64(reset_jiffies + secs_to_jiffies(1))) { 1361 schedule_iso_resource_auto(r, msecs_to_jiffies(333)); 1362 return; 1363 } 1364 break; 1365 case ISO_RES_AUTO_REALLOC: 1366 // We could be called twice within the same generation. 1367 if (resource_generation == current_generation) 1368 return; 1369 break; 1370 case ISO_RES_AUTO_DEALLOC: 1371 default: 1372 break; 1373 } 1374 1375 bandwidth = r->params.bandwidth; 1376 1377 fw_iso_resource_manage(client->device->card, current_generation, r->params.channels_mask, 1378 &channel, &bandwidth, todo != ISO_RES_AUTO_DEALLOC); 1379 1380 if (todo == ISO_RES_AUTO_DEALLOC) { 1381 free = true; 1382 e = xchg(&r->e_dealloc, NULL); // But no need to be atomic. 1383 } else { 1384 free = false; 1385 1386 // Is this generation outdated already? As long as this resource sticks in the 1387 // xarray, it will be scheduled again for a newer generation or at shutdown. 1388 if (channel == -EAGAIN) 1389 return; 1390 1391 bool success = channel >= 0 || bandwidth > 0; 1392 1393 if (!success) { 1394 // Allocation or reallocation failure? Pull this resource out of the 1395 // xarray and prepare for deletion, unless the client is shutting down. 1396 scoped_guard(mutex, &client->mutex) { 1397 if (!client->in_shutdown && xa_erase(&client->resource_xa, index)) { 1398 // For the incrementation by add_client_resource(). 1399 client_put(client); 1400 free = true; 1401 } 1402 } 1403 } 1404 1405 if (todo == ISO_RES_AUTO_REALLOC) { 1406 if (success) 1407 return; 1408 1409 // Notify the userspace client of the failure through a deallocation event. 1410 e = xchg(&r->e_dealloc, NULL); // But no need to be atomic. 1411 } else { 1412 // Transit from allocation to reallocation. Use compare-and-swap atomic 1413 // operation because the todo member can be set with ISO_RES_AUTO_DEALLOC 1414 // by release_iso_resource_auto() in parallel. 1415 atomic_cmpxchg_relaxed(&r->todo, ISO_RES_AUTO_ALLOC, ISO_RES_AUTO_REALLOC); 1416 1417 if (channel >= 0) 1418 r->params.channels_mask = BIT_ULL(channel); 1419 1420 e = xchg(&r->e_alloc, NULL); // But no need to be atomic. 1421 } 1422 } 1423 1424 e->iso_resource.handle = r->resource.handle; 1425 e->iso_resource.channel = channel; 1426 e->iso_resource.bandwidth = bandwidth; 1427 1428 queue_event(client, &e->event, 1429 &e->iso_resource, sizeof(e->iso_resource), NULL, 0); 1430 1431 if (free) { 1432 cancel_delayed_work(&r->work); 1433 kfree(r->e_alloc); 1434 kfree(r->e_dealloc); 1435 kfree(r); 1436 1437 // For the incrementation by ioctl_allocate_iso_resource(). 1438 client_put(client); 1439 } 1440 } 1441 1442 static void release_iso_resource_auto(struct client *client, struct client_resource *resource) 1443 { 1444 struct iso_resource_auto *r = to_iso_resource_auto(resource); 1445 1446 atomic_set(&r->todo, ISO_RES_AUTO_DEALLOC); 1447 schedule_iso_resource_auto(r, 0); 1448 } 1449 1450 static int ioctl_allocate_iso_resource(struct client *client, union ioctl_arg *arg) 1451 { 1452 struct fw_cdev_allocate_iso_resource *request = &arg->allocate_iso_resource; 1453 struct iso_resource_event *e1 __free(kfree) = kmalloc_obj(*e1); 1454 struct iso_resource_event *e2 __free(kfree) = kmalloc_obj(*e2); 1455 struct iso_resource_auto *r __free(kfree) = kmalloc_obj(*r); 1456 int err; 1457 1458 if (!r || !e1 || !e2) 1459 return -ENOMEM; 1460 1461 err = fill_iso_resource_params(&r->params, request); 1462 if (err < 0) 1463 return err; 1464 1465 INIT_DELAYED_WORK(&r->work, iso_resource_auto_work); 1466 r->client = client; 1467 atomic_set(&r->todo, ISO_RES_AUTO_ALLOC); 1468 r->e_alloc = e1; 1469 r->e_dealloc = e2; 1470 1471 e1->iso_resource.closure = request->closure; 1472 e1->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_ALLOCATED; 1473 e2->iso_resource.closure = request->closure; 1474 e2->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_DEALLOCATED; 1475 1476 err = add_client_resource(client, &r->resource, release_iso_resource_auto); 1477 if (err < 0) 1478 return err; 1479 request->handle = r->resource.handle; 1480 client_get(client); 1481 1482 retain_and_null_ptr(e1); 1483 retain_and_null_ptr(e2); 1484 schedule_iso_resource_auto(no_free_ptr(r), 0); 1485 1486 return 0; 1487 } 1488 1489 static int ioctl_deallocate_iso_resource(struct client *client, 1490 union ioctl_arg *arg) 1491 { 1492 return release_client_resource(client, 1493 arg->deallocate.handle, release_iso_resource_auto, NULL); 1494 } 1495 1496 #define UNAVAILABLE_HANDLE -1 1497 1498 static void iso_resource_once_work(struct work_struct *work) 1499 { 1500 struct iso_resource_once *r = from_work(r, work, work); 1501 struct client *client = r->client; 1502 struct iso_resource_event *e = r->event; 1503 int channel; 1504 1505 int generation = client->device->generation; 1506 int bandwidth = r->params.bandwidth; 1507 1508 fw_iso_resource_manage(client->device->card, generation, r->params.channels_mask, &channel, 1509 &bandwidth, r->todo == ISO_RES_ONCE_ALLOC); 1510 1511 e->iso_resource.handle = UNAVAILABLE_HANDLE; 1512 e->iso_resource.channel = channel; 1513 e->iso_resource.bandwidth = bandwidth; 1514 1515 queue_event(client, &e->event, &e->iso_resource, sizeof(e->iso_resource), NULL, 0); 1516 1517 cancel_work(&r->work); 1518 kfree(r); 1519 1520 client_put(client); 1521 } 1522 1523 static int init_iso_resource_once(struct client *client, 1524 struct fw_cdev_allocate_iso_resource *request, int todo) 1525 { 1526 struct iso_resource_event *e __free(kfree) = kmalloc_obj(*e); 1527 struct iso_resource_once *r __free(kfree) = kmalloc_obj(*r); 1528 int err; 1529 1530 if (!r || !e) 1531 return -ENOMEM; 1532 1533 err = fill_iso_resource_params(&r->params, request); 1534 if (err < 0) 1535 return err; 1536 1537 INIT_WORK(&r->work, iso_resource_once_work); 1538 r->client = client; 1539 r->todo = todo; 1540 1541 if (todo == ISO_RES_ONCE_ALLOC) 1542 e->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_ALLOCATED; 1543 else 1544 e->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_DEALLOCATED; 1545 e->iso_resource.closure = request->closure; 1546 r->event = no_free_ptr(e); 1547 1548 // Keep the client until work item finishing. 1549 client_get(r->client); 1550 1551 queue_work(fw_workqueue, &no_free_ptr(r)->work); 1552 1553 request->handle = UNAVAILABLE_HANDLE; 1554 1555 return 0; 1556 } 1557 1558 static int ioctl_allocate_iso_resource_once(struct client *client, 1559 union ioctl_arg *arg) 1560 { 1561 return init_iso_resource_once(client, &arg->allocate_iso_resource, ISO_RES_ONCE_ALLOC); 1562 } 1563 1564 static int ioctl_deallocate_iso_resource_once(struct client *client, 1565 union ioctl_arg *arg) 1566 { 1567 return init_iso_resource_once(client, &arg->allocate_iso_resource, ISO_RES_ONCE_DEALLOC); 1568 } 1569 1570 /* 1571 * Returns a speed code: Maximum speed to or from this device, 1572 * limited by the device's link speed, the local node's link speed, 1573 * and all PHY port speeds between the two links. 1574 */ 1575 static int ioctl_get_speed(struct client *client, union ioctl_arg *arg) 1576 { 1577 return client->device->max_speed; 1578 } 1579 1580 static int ioctl_send_broadcast_request(struct client *client, 1581 union ioctl_arg *arg) 1582 { 1583 struct fw_cdev_send_request *a = &arg->send_request; 1584 1585 switch (a->tcode) { 1586 case TCODE_WRITE_QUADLET_REQUEST: 1587 case TCODE_WRITE_BLOCK_REQUEST: 1588 break; 1589 default: 1590 return -EINVAL; 1591 } 1592 1593 /* Security policy: Only allow accesses to Units Space. */ 1594 if (a->offset < CSR_REGISTER_BASE + CSR_CONFIG_ROM_END) 1595 return -EACCES; 1596 1597 return init_request(client, a, LOCAL_BUS | 0x3f, SCODE_100); 1598 } 1599 1600 static int ioctl_send_stream_packet(struct client *client, union ioctl_arg *arg) 1601 { 1602 struct fw_cdev_send_stream_packet *a = &arg->send_stream_packet; 1603 struct fw_cdev_send_request request; 1604 int dest; 1605 1606 if (a->speed > client->device->card->link_speed || 1607 a->length > 1024 << a->speed) 1608 return -EIO; 1609 1610 if (a->tag > 3 || a->channel > 63 || a->sy > 15) 1611 return -EINVAL; 1612 1613 dest = fw_stream_packet_destination_id(a->tag, a->channel, a->sy); 1614 request.tcode = TCODE_STREAM_DATA; 1615 request.length = a->length; 1616 request.closure = a->closure; 1617 request.data = a->data; 1618 request.generation = a->generation; 1619 1620 return init_request(client, &request, dest, a->speed); 1621 } 1622 1623 static void outbound_phy_packet_callback(struct fw_packet *packet, 1624 struct fw_card *card, int status) 1625 { 1626 struct outbound_phy_packet_event *e = 1627 container_of(packet, struct outbound_phy_packet_event, p); 1628 struct client *e_client = e->client; 1629 u32 rcode; 1630 1631 trace_async_phy_outbound_complete((uintptr_t)packet, card->index, status, packet->generation, 1632 packet->timestamp); 1633 1634 switch (status) { 1635 // expected: 1636 case ACK_COMPLETE: 1637 rcode = RCODE_COMPLETE; 1638 break; 1639 // should never happen with PHY packets: 1640 case ACK_PENDING: 1641 rcode = RCODE_COMPLETE; 1642 break; 1643 case ACK_BUSY_X: 1644 case ACK_BUSY_A: 1645 case ACK_BUSY_B: 1646 rcode = RCODE_BUSY; 1647 break; 1648 case ACK_DATA_ERROR: 1649 rcode = RCODE_DATA_ERROR; 1650 break; 1651 case ACK_TYPE_ERROR: 1652 rcode = RCODE_TYPE_ERROR; 1653 break; 1654 // stale generation; cancelled; on certain controllers: no ack 1655 default: 1656 rcode = status; 1657 break; 1658 } 1659 1660 switch (e->phy_packet.without_tstamp.type) { 1661 case FW_CDEV_EVENT_PHY_PACKET_SENT: 1662 { 1663 struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp; 1664 1665 pp->rcode = rcode; 1666 pp->data[0] = packet->timestamp; 1667 queue_event(e->client, &e->event, &e->phy_packet, sizeof(*pp) + pp->length, 1668 NULL, 0); 1669 break; 1670 } 1671 case FW_CDEV_EVENT_PHY_PACKET_SENT2: 1672 { 1673 struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp; 1674 1675 pp->rcode = rcode; 1676 pp->tstamp = packet->timestamp; 1677 queue_event(e->client, &e->event, &e->phy_packet, sizeof(*pp) + pp->length, 1678 NULL, 0); 1679 break; 1680 } 1681 default: 1682 WARN_ON(1); 1683 break; 1684 } 1685 1686 client_put(e_client); 1687 } 1688 1689 static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg) 1690 { 1691 struct fw_cdev_send_phy_packet *a = &arg->send_phy_packet; 1692 struct fw_card *card = client->device->card; 1693 struct outbound_phy_packet_event *e; 1694 1695 /* Access policy: Allow this ioctl only on local nodes' device files. */ 1696 if (!client->device->is_local) 1697 return -ENOSYS; 1698 1699 e = kzalloc_flex(*e, phy_packet.with_tstamp.data, ARRAY_SIZE(a->data)); 1700 if (e == NULL) 1701 return -ENOMEM; 1702 1703 client_get(client); 1704 e->client = client; 1705 e->p.speed = SCODE_100; 1706 e->p.generation = a->generation; 1707 async_header_set_tcode(e->p.header, TCODE_LINK_INTERNAL); 1708 e->p.header[1] = a->data[0]; 1709 e->p.header[2] = a->data[1]; 1710 e->p.header_length = 12; 1711 e->p.callback = outbound_phy_packet_callback; 1712 1713 if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) { 1714 struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp; 1715 1716 pp->closure = a->closure; 1717 pp->type = FW_CDEV_EVENT_PHY_PACKET_SENT; 1718 if (is_ping_packet(a->data)) 1719 pp->length = 4; 1720 } else { 1721 struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp; 1722 1723 pp->closure = a->closure; 1724 pp->type = FW_CDEV_EVENT_PHY_PACKET_SENT2; 1725 // Keep the data field so that application can match the response event to the 1726 // request. 1727 pp->length = sizeof(a->data); 1728 memcpy(pp->data, a->data, sizeof(a->data)); 1729 } 1730 1731 trace_async_phy_outbound_initiate((uintptr_t)&e->p, card->index, e->p.generation, 1732 e->p.header[1], e->p.header[2]); 1733 1734 smp_load_acquire(&card->driver)->send_request(card, &e->p); 1735 1736 return 0; 1737 } 1738 1739 static int ioctl_receive_phy_packets(struct client *client, union ioctl_arg *arg) 1740 { 1741 struct fw_cdev_receive_phy_packets *a = &arg->receive_phy_packets; 1742 1743 /* Access policy: Allow this ioctl only on local nodes' device files. */ 1744 if (!client->device->is_local) 1745 return -ENOSYS; 1746 1747 scoped_guard(mutex, &phy_receiver_list_mutex) 1748 list_move_tail(&client->phy_receiver_link, &phy_receiver_list); 1749 1750 client->phy_receiver_closure = a->closure; 1751 1752 return 0; 1753 } 1754 1755 void fw_cdev_handle_phy_packet(struct fw_card *card, struct fw_packet *p) 1756 { 1757 struct client *client; 1758 1759 guard(mutex)(&phy_receiver_list_mutex); 1760 1761 list_for_each_entry(client, &phy_receiver_list, phy_receiver_link) { 1762 if (client->device->card != card) 1763 continue; 1764 1765 struct inbound_phy_packet_event *e = 1766 kzalloc_flex(*e, phy_packet.with_tstamp.data, 2); 1767 if (!e) 1768 break; 1769 1770 if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) { 1771 struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp; 1772 1773 pp->closure = client->phy_receiver_closure; 1774 pp->type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED; 1775 pp->rcode = RCODE_COMPLETE; 1776 pp->length = 8; 1777 pp->data[0] = p->header[1]; 1778 pp->data[1] = p->header[2]; 1779 queue_event(client, &e->event, &e->phy_packet, sizeof(*pp) + 8, NULL, 0); 1780 } else { 1781 struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp; 1782 1783 pp = &e->phy_packet.with_tstamp; 1784 pp->closure = client->phy_receiver_closure; 1785 pp->type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED2; 1786 pp->rcode = RCODE_COMPLETE; 1787 pp->length = 8; 1788 pp->tstamp = p->timestamp; 1789 pp->data[0] = p->header[1]; 1790 pp->data[1] = p->header[2]; 1791 queue_event(client, &e->event, &e->phy_packet, sizeof(*pp) + 8, NULL, 0); 1792 } 1793 } 1794 } 1795 1796 static int (* const ioctl_handlers[])(struct client *, union ioctl_arg *) = { 1797 [0x00] = ioctl_get_info, 1798 [0x01] = ioctl_send_request, 1799 [0x02] = ioctl_allocate, 1800 [0x03] = ioctl_deallocate, 1801 [0x04] = ioctl_send_response, 1802 [0x05] = ioctl_initiate_bus_reset, 1803 [0x06] = ioctl_add_descriptor, 1804 [0x07] = ioctl_remove_descriptor, 1805 [0x08] = ioctl_create_iso_context, 1806 [0x09] = ioctl_queue_iso, 1807 [0x0a] = ioctl_start_iso, 1808 [0x0b] = ioctl_stop_iso, 1809 [0x0c] = ioctl_get_cycle_timer, 1810 [0x0d] = ioctl_allocate_iso_resource, 1811 [0x0e] = ioctl_deallocate_iso_resource, 1812 [0x0f] = ioctl_allocate_iso_resource_once, 1813 [0x10] = ioctl_deallocate_iso_resource_once, 1814 [0x11] = ioctl_get_speed, 1815 [0x12] = ioctl_send_broadcast_request, 1816 [0x13] = ioctl_send_stream_packet, 1817 [0x14] = ioctl_get_cycle_timer2, 1818 [0x15] = ioctl_send_phy_packet, 1819 [0x16] = ioctl_receive_phy_packets, 1820 [0x17] = ioctl_set_iso_channels, 1821 [0x18] = ioctl_flush_iso, 1822 }; 1823 1824 static int dispatch_ioctl(struct client *client, 1825 unsigned int cmd, void __user *arg) 1826 { 1827 union ioctl_arg buffer; 1828 int ret; 1829 1830 if (fw_device_is_shutdown(client->device)) 1831 return -ENODEV; 1832 1833 if (_IOC_TYPE(cmd) != '#' || 1834 _IOC_NR(cmd) >= ARRAY_SIZE(ioctl_handlers) || 1835 _IOC_SIZE(cmd) > sizeof(buffer)) 1836 return -ENOTTY; 1837 1838 memset(&buffer, 0, sizeof(buffer)); 1839 1840 if (_IOC_DIR(cmd) & _IOC_WRITE) 1841 if (copy_from_user(&buffer, arg, _IOC_SIZE(cmd))) 1842 return -EFAULT; 1843 1844 ret = ioctl_handlers[_IOC_NR(cmd)](client, &buffer); 1845 if (ret < 0) 1846 return ret; 1847 1848 if (_IOC_DIR(cmd) & _IOC_READ) 1849 if (copy_to_user(arg, &buffer, _IOC_SIZE(cmd))) 1850 return -EFAULT; 1851 1852 return ret; 1853 } 1854 1855 static long fw_device_op_ioctl(struct file *file, 1856 unsigned int cmd, unsigned long arg) 1857 { 1858 return dispatch_ioctl(file->private_data, cmd, (void __user *)arg); 1859 } 1860 1861 static int fw_device_op_mmap(struct file *file, struct vm_area_struct *vma) 1862 { 1863 struct client *client = file->private_data; 1864 unsigned long size; 1865 int page_count, ret; 1866 1867 if (fw_device_is_shutdown(client->device)) 1868 return -ENODEV; 1869 1870 /* FIXME: We could support multiple buffers, but we don't. */ 1871 if (client->buffer.pages != NULL) 1872 return -EBUSY; 1873 1874 if (!(vma->vm_flags & VM_SHARED)) 1875 return -EINVAL; 1876 1877 if (vma->vm_start & ~PAGE_MASK) 1878 return -EINVAL; 1879 1880 client->vm_start = vma->vm_start; 1881 size = vma->vm_end - vma->vm_start; 1882 page_count = size >> PAGE_SHIFT; 1883 if (size & ~PAGE_MASK) 1884 return -EINVAL; 1885 1886 ret = fw_iso_buffer_alloc(&client->buffer, page_count); 1887 if (ret < 0) 1888 return ret; 1889 1890 scoped_guard(mutex, &client->iso_context_mutex) { 1891 // The direction of DMA can be determined if the isochronous context is already 1892 // allocated. If not, the DMA mapping operation is postponed after the allocation. 1893 if (client->iso_context) { 1894 ret = fw_iso_buffer_map_dma(&client->buffer, client->device->card, 1895 iso_dma_direction(client->iso_context)); 1896 if (ret < 0) 1897 goto fail; 1898 } 1899 } 1900 1901 ret = vm_map_pages_zero(vma, client->buffer.pages, 1902 client->buffer.page_count); 1903 if (ret < 0) 1904 goto fail; 1905 1906 return 0; 1907 fail: 1908 fw_iso_buffer_destroy(&client->buffer, client->device->card); 1909 return ret; 1910 } 1911 1912 static bool has_outbound_transactions(struct client *client) 1913 { 1914 struct client_resource *resource; 1915 unsigned long index; 1916 1917 scoped_guard(mutex, &client->mutex) { 1918 xa_for_each(&client->resource_xa, index, resource) { 1919 if (is_outbound_transaction_resource(resource)) 1920 return true; 1921 } 1922 } 1923 1924 return false; 1925 } 1926 1927 static int fw_device_op_release(struct inode *inode, struct file *file) 1928 { 1929 struct client *client = file->private_data; 1930 struct event *event, *next_event; 1931 struct client_resource *resource; 1932 unsigned long index; 1933 1934 scoped_guard(mutex, &phy_receiver_list_mutex) 1935 list_del(&client->phy_receiver_link); 1936 1937 scoped_guard(mutex, &client->device->client_list_mutex) 1938 list_del(&client->link); 1939 1940 if (client->iso_context) 1941 fw_iso_context_destroy(client->iso_context); 1942 mutex_destroy(&client->iso_context_mutex); 1943 1944 if (client->buffer.pages) 1945 fw_iso_buffer_destroy(&client->buffer, client->device->card); 1946 1947 // Freeze client->resource_xa and client->event_list. 1948 scoped_guard(mutex, &client->mutex) 1949 client->in_shutdown = true; 1950 1951 wait_event(client->tx_flush_wait, !has_outbound_transactions(client)); 1952 1953 xa_for_each(&client->resource_xa, index, resource) { 1954 resource->release(client, resource); 1955 client_put(client); 1956 } 1957 xa_destroy(&client->resource_xa); 1958 1959 list_for_each_entry_safe(event, next_event, &client->event_list, link) 1960 kfree(event); 1961 1962 client_put(client); 1963 1964 return 0; 1965 } 1966 1967 static __poll_t fw_device_op_poll(struct file *file, poll_table * pt) 1968 { 1969 struct client *client = file->private_data; 1970 __poll_t mask = 0; 1971 1972 poll_wait(file, &client->wait, pt); 1973 1974 if (fw_device_is_shutdown(client->device)) 1975 mask |= EPOLLHUP | EPOLLERR; 1976 if (!list_empty(&client->event_list)) 1977 mask |= EPOLLIN | EPOLLRDNORM; 1978 1979 return mask; 1980 } 1981 1982 const struct file_operations fw_device_ops = { 1983 .owner = THIS_MODULE, 1984 .open = fw_device_op_open, 1985 .read = fw_device_op_read, 1986 .unlocked_ioctl = fw_device_op_ioctl, 1987 .mmap = fw_device_op_mmap, 1988 .release = fw_device_op_release, 1989 .poll = fw_device_op_poll, 1990 .compat_ioctl = compat_ptr_ioctl, 1991 }; 1992