xref: /linux/drivers/firewire/core-cdev.c (revision 23b790766adfb140abc09c83fbe625e2c7f35a36)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Char device for device raw access
4  *
5  * Copyright (C) 2005-2007  Kristian Hoegsberg <krh@bitplanet.net>
6  */
7 
8 #include <linux/bug.h>
9 #include <linux/compat.h>
10 #include <linux/delay.h>
11 #include <linux/device.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/err.h>
14 #include <linux/errno.h>
15 #include <linux/firewire.h>
16 #include <linux/firewire-cdev.h>
17 #include <linux/irqflags.h>
18 #include <linux/jiffies.h>
19 #include <linux/kernel.h>
20 #include <linux/kref.h>
21 #include <linux/mm.h>
22 #include <linux/module.h>
23 #include <linux/mutex.h>
24 #include <linux/poll.h>
25 #include <linux/sched.h> /* required for linux/wait.h */
26 #include <linux/slab.h>
27 #include <linux/spinlock.h>
28 #include <linux/string.h>
29 #include <linux/time.h>
30 #include <linux/uaccess.h>
31 #include <linux/vmalloc.h>
32 #include <linux/wait.h>
33 #include <linux/workqueue.h>
34 
35 
36 #include "core.h"
37 #include <trace/events/firewire.h>
38 
39 #include "packet-header-definitions.h"
40 
41 /*
42  * ABI version history is documented in linux/firewire-cdev.h.
43  */
44 #define FW_CDEV_KERNEL_VERSION			6
45 #define FW_CDEV_VERSION_EVENT_REQUEST2		4
46 #define FW_CDEV_VERSION_ALLOCATE_REGION_END	4
47 #define FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW	5
48 #define FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP	6
49 
50 static DEFINE_MUTEX(phy_receiver_list_mutex);
51 static LIST_HEAD(phy_receiver_list);
52 
53 struct client {
54 	u32 version;
55 	struct fw_device *device;
56 
57 	struct mutex mutex;
58 	bool in_shutdown;
59 	struct xarray resource_xa;
60 	struct list_head event_list;
61 	wait_queue_head_t wait;
62 	wait_queue_head_t tx_flush_wait;
63 	u64 bus_reset_closure;
64 
65 	struct fw_iso_context *iso_context;
66 	struct mutex iso_context_mutex;
67 	u64 iso_closure;
68 	struct fw_iso_buffer buffer;
69 	unsigned long vm_start;
70 
71 	struct list_head phy_receiver_link;
72 	u64 phy_receiver_closure;
73 
74 	struct list_head link;
75 	struct kref kref;
76 };
77 
78 static inline void client_get(struct client *client)
79 {
80 	kref_get(&client->kref);
81 }
82 
83 static void client_release(struct kref *kref)
84 {
85 	struct client *client = container_of(kref, struct client, kref);
86 
87 	fw_device_put(client->device);
88 	kfree(client);
89 }
90 
91 static void client_put(struct client *client)
92 {
93 	kref_put(&client->kref, client_release);
94 }
95 
96 struct client_resource;
97 typedef void (*client_resource_release_fn_t)(struct client *,
98 					     struct client_resource *);
99 struct client_resource {
100 	client_resource_release_fn_t release;
101 	int handle;
102 };
103 
104 struct address_handler_resource {
105 	struct client_resource resource;
106 	struct fw_address_handler handler;
107 	__u64 closure;
108 	struct client *client;
109 };
110 
111 struct outbound_transaction_resource {
112 	struct client_resource resource;
113 	struct fw_transaction transaction;
114 };
115 
116 struct inbound_transaction_resource {
117 	struct client_resource resource;
118 	struct fw_card *card;
119 	struct fw_request *request;
120 	bool is_fcp;
121 	void *data;
122 	size_t length;
123 };
124 
125 struct descriptor_resource {
126 	struct client_resource resource;
127 	struct fw_descriptor descriptor;
128 	u32 data[];
129 };
130 
131 struct iso_resource_params {
132 	u64 channels_mask;
133 	s32 bandwidth;
134 };
135 
136 enum {
137 	ISO_RES_AUTO_ALLOC,
138 	ISO_RES_AUTO_REALLOC,
139 	ISO_RES_AUTO_DEALLOC,
140 };
141 
142 struct iso_resource_auto {
143 	struct client_resource resource;
144 	struct client *client;
145 	struct delayed_work work;
146 	atomic_t todo; // one of ISO_RES_AUTO_XXX.
147 	int generation;
148 	struct iso_resource_params params;
149 	struct iso_resource_event *e_alloc, *e_dealloc;
150 };
151 
152 struct iso_resource_once {
153 	struct client *client;
154 	struct work_struct work;
155 	enum {
156 		ISO_RES_ONCE_ALLOC,
157 		ISO_RES_ONCE_DEALLOC,
158 	} todo;
159 	struct iso_resource_params params;
160 	struct iso_resource_event *event;
161 };
162 
163 static struct address_handler_resource *to_address_handler_resource(struct client_resource *resource)
164 {
165 	return container_of(resource, struct address_handler_resource, resource);
166 }
167 
168 static struct inbound_transaction_resource *to_inbound_transaction_resource(struct client_resource *resource)
169 {
170 	return container_of(resource, struct inbound_transaction_resource, resource);
171 }
172 
173 static struct descriptor_resource *to_descriptor_resource(struct client_resource *resource)
174 {
175 	return container_of(resource, struct descriptor_resource, resource);
176 }
177 
178 static struct iso_resource_auto *to_iso_resource_auto(struct client_resource *resource)
179 {
180 	return container_of(resource, struct iso_resource_auto, resource);
181 }
182 
183 static void release_iso_resource_auto(struct client *, struct client_resource *);
184 
185 static int is_iso_resource_auto(const struct client_resource *resource)
186 {
187 	return resource->release == release_iso_resource_auto;
188 }
189 
190 static void release_transaction(struct client *client,
191 				struct client_resource *resource);
192 
193 static int is_outbound_transaction_resource(const struct client_resource *resource)
194 {
195 	return resource->release == release_transaction;
196 }
197 
198 static void schedule_iso_resource_auto(struct iso_resource_auto *r, unsigned long delay)
199 {
200 	queue_delayed_work(fw_workqueue, &r->work, delay);
201 }
202 
203 /*
204  * dequeue_event() just kfree()'s the event, so the event has to be
205  * the first field in a struct XYZ_event.
206  */
207 struct event {
208 	struct { void *data; size_t size; } v[2];
209 	struct list_head link;
210 };
211 
212 struct bus_reset_event {
213 	struct event event;
214 	struct fw_cdev_event_bus_reset reset;
215 };
216 
217 struct outbound_transaction_event {
218 	struct event event;
219 	struct client *client;
220 	struct outbound_transaction_resource r;
221 	union {
222 		struct fw_cdev_event_response without_tstamp;
223 		struct fw_cdev_event_response2 with_tstamp;
224 	} rsp;
225 };
226 
227 struct inbound_transaction_event {
228 	struct event event;
229 	union {
230 		struct fw_cdev_event_request request;
231 		struct fw_cdev_event_request2 request2;
232 		struct fw_cdev_event_request3 with_tstamp;
233 	} req;
234 };
235 
236 struct iso_interrupt_event {
237 	struct event event;
238 	struct fw_cdev_event_iso_interrupt interrupt;
239 };
240 
241 struct iso_interrupt_mc_event {
242 	struct event event;
243 	struct fw_cdev_event_iso_interrupt_mc interrupt;
244 };
245 
246 struct iso_resource_event {
247 	struct event event;
248 	struct fw_cdev_event_iso_resource iso_resource;
249 };
250 
251 struct outbound_phy_packet_event {
252 	struct event event;
253 	struct client *client;
254 	struct fw_packet p;
255 	union {
256 		struct fw_cdev_event_phy_packet without_tstamp;
257 		struct fw_cdev_event_phy_packet2 with_tstamp;
258 	} phy_packet;
259 };
260 
261 struct inbound_phy_packet_event {
262 	struct event event;
263 	union {
264 		struct fw_cdev_event_phy_packet without_tstamp;
265 		struct fw_cdev_event_phy_packet2 with_tstamp;
266 	} phy_packet;
267 };
268 
269 #ifdef CONFIG_COMPAT
270 static void __user *u64_to_uptr(u64 value)
271 {
272 	if (in_compat_syscall())
273 		return compat_ptr(value);
274 	else
275 		return (void __user *)(unsigned long)value;
276 }
277 
278 static u64 uptr_to_u64(void __user *ptr)
279 {
280 	if (in_compat_syscall())
281 		return ptr_to_compat(ptr);
282 	else
283 		return (u64)(unsigned long)ptr;
284 }
285 #else
286 static inline void __user *u64_to_uptr(u64 value)
287 {
288 	return (void __user *)(unsigned long)value;
289 }
290 
291 static inline u64 uptr_to_u64(void __user *ptr)
292 {
293 	return (u64)(unsigned long)ptr;
294 }
295 #endif /* CONFIG_COMPAT */
296 
297 static int fw_device_op_open(struct inode *inode, struct file *file)
298 {
299 	struct fw_device *device;
300 	struct client *client;
301 
302 	device = fw_device_get_by_devt(inode->i_rdev);
303 	if (device == NULL)
304 		return -ENODEV;
305 
306 	if (fw_device_is_shutdown(device)) {
307 		fw_device_put(device);
308 		return -ENODEV;
309 	}
310 
311 	client = kzalloc_obj(*client);
312 	if (client == NULL) {
313 		fw_device_put(device);
314 		return -ENOMEM;
315 	}
316 
317 	client->device = device;
318 	mutex_init(&client->mutex);
319 	xa_init_flags(&client->resource_xa, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_BH);
320 	INIT_LIST_HEAD(&client->event_list);
321 	init_waitqueue_head(&client->wait);
322 	init_waitqueue_head(&client->tx_flush_wait);
323 	INIT_LIST_HEAD(&client->phy_receiver_link);
324 	INIT_LIST_HEAD(&client->link);
325 	kref_init(&client->kref);
326 	mutex_init(&client->iso_context_mutex);
327 
328 	file->private_data = client;
329 
330 	return nonseekable_open(inode, file);
331 }
332 
333 static void queue_event(struct client *client, struct event *event,
334 			void *data0, size_t size0, void *data1, size_t size1)
335 {
336 	event->v[0].data = data0;
337 	event->v[0].size = size0;
338 	event->v[1].data = data1;
339 	event->v[1].size = size1;
340 
341 	scoped_guard(mutex, &client->mutex) {
342 		if (client->in_shutdown) {
343 			kfree(event);
344 		} else {
345 			list_add_tail(&event->link, &client->event_list);
346 
347 			wake_up_interruptible(&client->wait);
348 		}
349 	}
350 }
351 
352 static ssize_t dequeue_event(struct client *client, char __user *buffer, size_t count)
353 {
354 	struct event *event;
355 
356 	// After the following block, the event pointer above is guaranteed to have a correct value.
357 	{
358 		mutex_lock(&client->mutex);
359 
360 		// This could be replaced with wait_var_event_any_lock() if poll_wait() alternative
361 		// would be introduced.
362 		int ret = ___wait_event(client->wait,
363 			!list_empty(&client->event_list) || fw_device_is_shutdown(client->device),
364 			TASK_INTERRUPTIBLE, 0, 0,
365 			mutex_unlock(&client->mutex);
366 			schedule();
367 			mutex_lock(&client->mutex)
368 		);
369 		if (ret < 0) {
370 			mutex_unlock(&client->mutex);
371 			return ret;
372 		}
373 
374 		if (fw_device_is_shutdown(client->device)) {
375 			mutex_unlock(&client->mutex);
376 			return -ENODEV;
377 		}
378 
379 		event = list_first_entry(&client->event_list, struct event, link);
380 		list_del(&event->link);
381 
382 		mutex_unlock(&client->mutex);
383 	}
384 
385 	ssize_t ret = 0;
386 
387 	for (int i = 0; i < ARRAY_SIZE(event->v) && ret < count; i++) {
388 		size_t size = min(event->v[i].size, count - ret);
389 		if (copy_to_user(buffer + ret, event->v[i].data, size)) {
390 			ret = -EFAULT;
391 			break;
392 		}
393 		ret += size;
394 	}
395 
396 	kfree(event);
397 
398 	return ret;
399 }
400 
401 static ssize_t fw_device_op_read(struct file *file, char __user *buffer,
402 				 size_t count, loff_t *offset)
403 {
404 	struct client *client = file->private_data;
405 
406 	return dequeue_event(client, buffer, count);
407 }
408 
409 static void fill_bus_reset_event(struct fw_cdev_event_bus_reset *event,
410 				 struct client *client)
411 __must_hold(&client->device->client_list_mutex)
412 {
413 	lockdep_assert_held(&client->device->client_list_mutex);
414 
415 	// This member is related to the above mutex. In detail, see 93b37905f70 ("firewire: cdev:
416 	// prevent race between first get_info ioctl and bus reset event queuing").
417 	event->closure	     = client->bus_reset_closure;
418 	event->type          = FW_CDEV_EVENT_BUS_RESET;
419 
420 	event->generation    = client->device->generation;
421 	smp_rmb();
422 	event->node_id       = client->device->node_id;
423 
424 	struct fw_card *card = client->device->card;
425 
426 	scoped_guard(spinlock_irq, &card->lock) {
427 		event->local_node_id = card->local_node->node_id;
428 		event->bm_node_id    = card->bm_node_id;
429 		event->irm_node_id   = card->irm_node->node_id;
430 		event->root_node_id  = card->root_node->node_id;
431 	}
432 }
433 
434 static void for_each_client(struct fw_device *device,
435 			    void (*callback)(struct client *client))
436 {
437 	struct client *c;
438 
439 	guard(mutex)(&device->client_list_mutex);
440 
441 	list_for_each_entry(c, &device->client_list, link)
442 		callback(c);
443 }
444 
445 static void queue_bus_reset_event(struct client *client)
446 {
447 	struct bus_reset_event *e;
448 	struct client_resource *resource;
449 	unsigned long index;
450 
451 	e = kzalloc_obj(*e);
452 	if (e == NULL)
453 		return;
454 
455 	fill_bus_reset_event(&e->reset, client);
456 
457 	queue_event(client, &e->event,
458 		    &e->reset, sizeof(e->reset), NULL, 0);
459 
460 	scoped_guard(mutex, &client->mutex) {
461 		xa_for_each(&client->resource_xa, index, resource) {
462 			if (is_iso_resource_auto(resource))
463 				schedule_iso_resource_auto(to_iso_resource_auto(resource), 0);
464 		}
465 	}
466 }
467 
468 void fw_device_cdev_update(struct fw_device *device)
469 {
470 	for_each_client(device, queue_bus_reset_event);
471 }
472 
473 static void wake_up_client(struct client *client)
474 {
475 	wake_up_interruptible(&client->wait);
476 }
477 
478 void fw_device_cdev_remove(struct fw_device *device)
479 {
480 	for_each_client(device, wake_up_client);
481 }
482 
483 union ioctl_arg {
484 	struct fw_cdev_get_info			get_info;
485 	struct fw_cdev_send_request		send_request;
486 	struct fw_cdev_allocate			allocate;
487 	struct fw_cdev_deallocate		deallocate;
488 	struct fw_cdev_send_response		send_response;
489 	struct fw_cdev_initiate_bus_reset	initiate_bus_reset;
490 	struct fw_cdev_add_descriptor		add_descriptor;
491 	struct fw_cdev_remove_descriptor	remove_descriptor;
492 	struct fw_cdev_create_iso_context	create_iso_context;
493 	struct fw_cdev_queue_iso		queue_iso;
494 	struct fw_cdev_start_iso		start_iso;
495 	struct fw_cdev_stop_iso			stop_iso;
496 	struct fw_cdev_get_cycle_timer		get_cycle_timer;
497 	struct fw_cdev_allocate_iso_resource	allocate_iso_resource;
498 	struct fw_cdev_send_stream_packet	send_stream_packet;
499 	struct fw_cdev_get_cycle_timer2		get_cycle_timer2;
500 	struct fw_cdev_send_phy_packet		send_phy_packet;
501 	struct fw_cdev_receive_phy_packets	receive_phy_packets;
502 	struct fw_cdev_set_iso_channels		set_iso_channels;
503 	struct fw_cdev_flush_iso		flush_iso;
504 };
505 
506 static int ioctl_get_info(struct client *client, union ioctl_arg *arg)
507 {
508 	struct fw_cdev_get_info *a = &arg->get_info;
509 
510 	client->version = a->version;
511 	a->version = FW_CDEV_KERNEL_VERSION;
512 	a->card = client->device->card->index;
513 
514 	scoped_guard(rwsem_read, &fw_device_rwsem) {
515 		if (a->rom != 0) {
516 			size_t length = min_t(size_t, a->rom_length,
517 					      client->device->config_rom_length * 4);
518 
519 			if (copy_to_user(u64_to_uptr(a->rom), client->device->config_rom, length))
520 				return -EFAULT;
521 		}
522 		a->rom_length = client->device->config_rom_length * 4;
523 	}
524 
525 	scoped_guard(mutex, &client->device->client_list_mutex) {
526 		// Coordinate concurrent access to this member with bus reset event handling, see
527 		// 93b37905f70 ("firewire: cdev: prevent race between first get_info ioctl and bus
528 		// reset event queuing").
529 		client->bus_reset_closure = a->bus_reset_closure;
530 
531 		if (a->bus_reset != 0) {
532 			struct fw_cdev_event_bus_reset bus_reset;
533 
534 			memset(&bus_reset, 0, sizeof(bus_reset));
535 			fill_bus_reset_event(&bus_reset, client);
536 
537 			// This structure has 4 bytes of trailing padding under the System V ABI
538 			// on most architectures (due to 8-byte alignment of the long long type),
539 			// except for Intel386 (where long long type is aligned to 4 bytes). In
540 			// either case, the effective length is 36 bytes.
541 			if (copy_to_user(u64_to_uptr(a->bus_reset), &bus_reset, 36))
542 				return -EFAULT;
543 		}
544 
545 		if (list_empty(&client->link))
546 			list_add_tail(&client->link, &client->device->client_list);
547 	}
548 
549 	return 0;
550 }
551 
552 static int add_client_resource(struct client *client, struct client_resource *resource,
553 			       client_resource_release_fn_t release)
554 {
555 	scoped_guard(mutex, &client->mutex) {
556 		u32 index;
557 		int ret;
558 
559 		if (client->in_shutdown)
560 			return  -ECANCELED;
561 
562 		ret = xa_alloc(&client->resource_xa, &index, resource, xa_limit_32b, GFP_KERNEL);
563 		if (ret < 0)
564 			return ret;
565 
566 		resource->handle = index;
567 		resource->release = release;
568 		client_get(client);
569 	}
570 
571 	return 0;
572 }
573 
574 static int release_client_resource(struct client *client, u32 handle,
575 				   client_resource_release_fn_t release,
576 				   struct client_resource **return_resource)
577 {
578 	unsigned long index = handle;
579 	struct client_resource *resource;
580 
581 	scoped_guard(mutex, &client->mutex) {
582 		if (client->in_shutdown)
583 			return -EINVAL;
584 
585 		resource = xa_load(&client->resource_xa, index);
586 		if (!resource || resource->release != release)
587 			return -EINVAL;
588 
589 		xa_erase(&client->resource_xa, handle);
590 	}
591 
592 	if (return_resource)
593 		*return_resource = resource;
594 	else
595 		resource->release(client, resource);
596 
597 	client_put(client);
598 
599 	return 0;
600 }
601 
602 static void release_transaction(struct client *client,
603 				struct client_resource *resource)
604 {
605 }
606 
607 static void complete_transaction(struct fw_card *card, int rcode, u32 request_tstamp,
608 				 u32 response_tstamp, void *payload, size_t length, void *data)
609 {
610 	struct outbound_transaction_event *e = data;
611 	struct client *client = e->client;
612 	unsigned long index = e->r.resource.handle;
613 
614 	scoped_guard(mutex, &client->mutex) {
615 		xa_erase(&client->resource_xa, index);
616 		if (client->in_shutdown)
617 			wake_up(&client->tx_flush_wait);
618 	}
619 
620 	switch (e->rsp.without_tstamp.type) {
621 	case FW_CDEV_EVENT_RESPONSE:
622 	{
623 		struct fw_cdev_event_response *rsp = &e->rsp.without_tstamp;
624 
625 		if (length < rsp->length)
626 			rsp->length = length;
627 		if (rcode == RCODE_COMPLETE)
628 			memcpy(rsp->data, payload, rsp->length);
629 
630 		rsp->rcode = rcode;
631 
632 		// In the case that sizeof(*rsp) doesn't align with the position of the
633 		// data, and the read is short, preserve an extra copy of the data
634 		// to stay compatible with a pre-2.6.27 bug.  Since the bug is harmless
635 		// for short reads and some apps depended on it, this is both safe
636 		// and prudent for compatibility.
637 		if (rsp->length <= sizeof(*rsp) - offsetof(typeof(*rsp), data))
638 			queue_event(client, &e->event, rsp, sizeof(*rsp), rsp->data, rsp->length);
639 		else
640 			queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, NULL, 0);
641 
642 		break;
643 	}
644 	case FW_CDEV_EVENT_RESPONSE2:
645 	{
646 		struct fw_cdev_event_response2 *rsp = &e->rsp.with_tstamp;
647 
648 		if (length < rsp->length)
649 			rsp->length = length;
650 		if (rcode == RCODE_COMPLETE)
651 			memcpy(rsp->data, payload, rsp->length);
652 
653 		rsp->rcode = rcode;
654 		rsp->request_tstamp = request_tstamp;
655 		rsp->response_tstamp = response_tstamp;
656 
657 		queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, NULL, 0);
658 
659 		break;
660 	}
661 	default:
662 		WARN_ON(1);
663 		break;
664 	}
665 
666 	// Drop the xarray's reference.
667 	client_put(client);
668 }
669 
670 static int init_request(struct client *client,
671 			struct fw_cdev_send_request *request,
672 			int destination_id, int speed)
673 {
674 	struct outbound_transaction_event *e;
675 	void *payload;
676 	int ret;
677 
678 	if (request->tcode != TCODE_STREAM_DATA &&
679 	    (request->length > 4096 || request->length > 512 << speed))
680 		return -EIO;
681 
682 	if (request->tcode == TCODE_WRITE_QUADLET_REQUEST &&
683 	    request->length < 4)
684 		return -EINVAL;
685 
686 	e = kzalloc_flex(*e, rsp.with_tstamp.data, DIV_ROUND_UP(request->length, sizeof(u32)));
687 	if (!e)
688 		return -ENOMEM;
689 	e->client = client;
690 
691 	if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
692 		struct fw_cdev_event_response *rsp = &e->rsp.without_tstamp;
693 
694 		rsp->type = FW_CDEV_EVENT_RESPONSE;
695 		rsp->length = request->length;
696 		rsp->closure = request->closure;
697 		payload = rsp->data;
698 	} else {
699 		struct fw_cdev_event_response2 *rsp = &e->rsp.with_tstamp;
700 
701 		rsp->type = FW_CDEV_EVENT_RESPONSE2;
702 		rsp->length = request->length;
703 		rsp->closure = request->closure;
704 		payload = rsp->data;
705 	}
706 
707 	if (request->data && copy_from_user(payload, u64_to_uptr(request->data), request->length)) {
708 		ret = -EFAULT;
709 		goto failed;
710 	}
711 
712 	ret = add_client_resource(client, &e->r.resource, release_transaction);
713 	if (ret < 0)
714 		goto failed;
715 
716 	fw_send_request_with_tstamp(client->device->card, &e->r.transaction, request->tcode,
717 				    destination_id, request->generation, speed, request->offset,
718 				    payload, request->length, complete_transaction, e);
719 	return 0;
720 
721  failed:
722 	kfree(e);
723 
724 	return ret;
725 }
726 
727 static int ioctl_send_request(struct client *client, union ioctl_arg *arg)
728 {
729 	switch (arg->send_request.tcode) {
730 	case TCODE_WRITE_QUADLET_REQUEST:
731 	case TCODE_WRITE_BLOCK_REQUEST:
732 	case TCODE_READ_QUADLET_REQUEST:
733 	case TCODE_READ_BLOCK_REQUEST:
734 	case TCODE_LOCK_MASK_SWAP:
735 	case TCODE_LOCK_COMPARE_SWAP:
736 	case TCODE_LOCK_FETCH_ADD:
737 	case TCODE_LOCK_LITTLE_ADD:
738 	case TCODE_LOCK_BOUNDED_ADD:
739 	case TCODE_LOCK_WRAP_ADD:
740 	case TCODE_LOCK_VENDOR_DEPENDENT:
741 		break;
742 	default:
743 		return -EINVAL;
744 	}
745 
746 	return init_request(client, &arg->send_request, client->device->node_id,
747 			    client->device->max_speed);
748 }
749 
750 static void release_request(struct client *client,
751 			    struct client_resource *resource)
752 {
753 	struct inbound_transaction_resource *r = to_inbound_transaction_resource(resource);
754 
755 	if (r->is_fcp)
756 		fw_request_put(r->request);
757 	else
758 		fw_send_response(r->card, r->request, RCODE_CONFLICT_ERROR);
759 
760 	fw_card_put(r->card);
761 	kfree(r);
762 }
763 
764 static void handle_request(struct fw_card *card, struct fw_request *request,
765 			   int tcode, int destination, int source,
766 			   int generation, unsigned long long offset,
767 			   void *payload, size_t length, void *callback_data)
768 {
769 	struct address_handler_resource *handler = callback_data;
770 	bool is_fcp = is_in_fcp_region(offset, length);
771 	struct inbound_transaction_resource *r;
772 	struct inbound_transaction_event *e;
773 	size_t event_size0;
774 	int ret;
775 
776 	/* card may be different from handler->client->device->card */
777 	fw_card_get(card);
778 
779 	// Extend the lifetime of data for request so that its payload is safely accessible in
780 	// the process context for the client.
781 	if (is_fcp)
782 		fw_request_get(request);
783 
784 	r = kmalloc_obj(*r);
785 	e = kmalloc_obj(*e);
786 	if (r == NULL || e == NULL)
787 		goto failed;
788 
789 	r->card    = card;
790 	r->request = request;
791 	r->is_fcp  = is_fcp;
792 	r->data    = payload;
793 	r->length  = length;
794 
795 	ret = add_client_resource(handler->client, &r->resource, release_request);
796 	if (ret < 0)
797 		goto failed;
798 
799 	if (handler->client->version < FW_CDEV_VERSION_EVENT_REQUEST2) {
800 		struct fw_cdev_event_request *req = &e->req.request;
801 
802 		if (tcode & 0x10)
803 			tcode = TCODE_LOCK_REQUEST;
804 
805 		req->type	= FW_CDEV_EVENT_REQUEST;
806 		req->tcode	= tcode;
807 		req->offset	= offset;
808 		req->length	= length;
809 		req->handle	= r->resource.handle;
810 		req->closure	= handler->closure;
811 		event_size0	= sizeof(*req);
812 	} else if (handler->client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
813 		struct fw_cdev_event_request2 *req = &e->req.request2;
814 
815 		req->type	= FW_CDEV_EVENT_REQUEST2;
816 		req->tcode	= tcode;
817 		req->offset	= offset;
818 		req->source_node_id = source;
819 		req->destination_node_id = destination;
820 		req->card	= card->index;
821 		req->generation	= generation;
822 		req->length	= length;
823 		req->handle	= r->resource.handle;
824 		req->closure	= handler->closure;
825 		event_size0	= sizeof(*req);
826 	} else {
827 		struct fw_cdev_event_request3 *req = &e->req.with_tstamp;
828 
829 		req->type	= FW_CDEV_EVENT_REQUEST3;
830 		req->tcode	= tcode;
831 		req->offset	= offset;
832 		req->source_node_id = source;
833 		req->destination_node_id = destination;
834 		req->card	= card->index;
835 		req->generation	= generation;
836 		req->length	= length;
837 		req->handle	= r->resource.handle;
838 		req->closure	= handler->closure;
839 		req->tstamp	= fw_request_get_timestamp(request);
840 		event_size0	= sizeof(*req);
841 	}
842 
843 	queue_event(handler->client, &e->event,
844 		    &e->req, event_size0, r->data, length);
845 	return;
846 
847  failed:
848 	kfree(r);
849 	kfree(e);
850 
851 	if (!is_fcp)
852 		fw_send_response(card, request, RCODE_CONFLICT_ERROR);
853 	else
854 		fw_request_put(request);
855 
856 	fw_card_put(card);
857 }
858 
859 static void release_address_handler(struct client *client,
860 				    struct client_resource *resource)
861 {
862 	struct address_handler_resource *r = to_address_handler_resource(resource);
863 
864 	fw_core_remove_address_handler(&r->handler);
865 	kfree(r);
866 }
867 
868 static int ioctl_allocate(struct client *client, union ioctl_arg *arg)
869 {
870 	struct fw_cdev_allocate *a = &arg->allocate;
871 	struct address_handler_resource *r;
872 	struct fw_address_region region;
873 	int ret;
874 
875 	r = kmalloc_obj(*r);
876 	if (r == NULL)
877 		return -ENOMEM;
878 
879 	region.start = a->offset;
880 	if (client->version < FW_CDEV_VERSION_ALLOCATE_REGION_END)
881 		region.end = a->offset + a->length;
882 	else
883 		region.end = a->region_end;
884 
885 	r->handler.length           = a->length;
886 	r->handler.address_callback = handle_request;
887 	r->handler.callback_data    = r;
888 	r->closure   = a->closure;
889 	r->client    = client;
890 
891 	ret = fw_core_add_address_handler(&r->handler, &region);
892 	if (ret < 0) {
893 		kfree(r);
894 		return ret;
895 	}
896 	a->offset = r->handler.offset;
897 
898 	ret = add_client_resource(client, &r->resource, release_address_handler);
899 	if (ret < 0) {
900 		release_address_handler(client, &r->resource);
901 		return ret;
902 	}
903 	a->handle = r->resource.handle;
904 
905 	return 0;
906 }
907 
908 static int ioctl_deallocate(struct client *client, union ioctl_arg *arg)
909 {
910 	return release_client_resource(client, arg->deallocate.handle,
911 				       release_address_handler, NULL);
912 }
913 
914 static int ioctl_send_response(struct client *client, union ioctl_arg *arg)
915 {
916 	struct fw_cdev_send_response *a = &arg->send_response;
917 	struct client_resource *resource;
918 	struct inbound_transaction_resource *r;
919 	int ret = 0;
920 
921 	if (release_client_resource(client, a->handle,
922 				    release_request, &resource) < 0)
923 		return -EINVAL;
924 
925 	r = to_inbound_transaction_resource(resource);
926 	if (r->is_fcp) {
927 		fw_request_put(r->request);
928 		goto out;
929 	}
930 
931 	if (a->length != fw_get_response_length(r->request)) {
932 		ret = -EINVAL;
933 		fw_request_put(r->request);
934 		goto out;
935 	}
936 	if (copy_from_user(r->data, u64_to_uptr(a->data), a->length)) {
937 		ret = -EFAULT;
938 		fw_request_put(r->request);
939 		goto out;
940 	}
941 	fw_send_response(r->card, r->request, a->rcode);
942  out:
943 	fw_card_put(r->card);
944 	kfree(r);
945 
946 	return ret;
947 }
948 
949 static int ioctl_initiate_bus_reset(struct client *client, union ioctl_arg *arg)
950 {
951 	fw_schedule_bus_reset(client->device->card, true,
952 			arg->initiate_bus_reset.type == FW_CDEV_SHORT_RESET);
953 	return 0;
954 }
955 
956 static void release_descriptor(struct client *client,
957 			       struct client_resource *resource)
958 {
959 	struct descriptor_resource *r = to_descriptor_resource(resource);
960 
961 	fw_core_remove_descriptor(&r->descriptor);
962 	kfree(r);
963 }
964 
965 static int ioctl_add_descriptor(struct client *client, union ioctl_arg *arg)
966 {
967 	struct fw_cdev_add_descriptor *a = &arg->add_descriptor;
968 	struct descriptor_resource *r;
969 	int ret;
970 
971 	/* Access policy: Allow this ioctl only on local nodes' device files. */
972 	if (!client->device->is_local)
973 		return -ENOSYS;
974 
975 	if (a->length > 256)
976 		return -EINVAL;
977 
978 	r = kmalloc_flex(*r, data, a->length);
979 	if (r == NULL)
980 		return -ENOMEM;
981 
982 	if (copy_from_user(r->data, u64_to_uptr(a->data),
983 			   flex_array_size(r, data, a->length))) {
984 		ret = -EFAULT;
985 		goto failed;
986 	}
987 
988 	r->descriptor.length    = a->length;
989 	r->descriptor.immediate = a->immediate;
990 	r->descriptor.key       = a->key;
991 	r->descriptor.data      = r->data;
992 
993 	ret = fw_core_add_descriptor(&r->descriptor);
994 	if (ret < 0)
995 		goto failed;
996 
997 	ret = add_client_resource(client, &r->resource, release_descriptor);
998 	if (ret < 0) {
999 		fw_core_remove_descriptor(&r->descriptor);
1000 		goto failed;
1001 	}
1002 	a->handle = r->resource.handle;
1003 
1004 	return 0;
1005  failed:
1006 	kfree(r);
1007 
1008 	return ret;
1009 }
1010 
1011 static int ioctl_remove_descriptor(struct client *client, union ioctl_arg *arg)
1012 {
1013 	return release_client_resource(client, arg->remove_descriptor.handle,
1014 				       release_descriptor, NULL);
1015 }
1016 
1017 static void iso_callback(struct fw_iso_context *context, u32 cycle,
1018 			 size_t header_length, void *header, void *data)
1019 {
1020 	struct client *client = data;
1021 	struct iso_interrupt_event *e;
1022 
1023 	e = kzalloc_flex(*e, interrupt.header, header_length);
1024 	if (e == NULL)
1025 		return;
1026 
1027 	e->interrupt.type      = FW_CDEV_EVENT_ISO_INTERRUPT;
1028 	e->interrupt.closure   = client->iso_closure;
1029 	e->interrupt.cycle     = cycle;
1030 	e->interrupt.header_length = header_length;
1031 	memcpy(e->interrupt.header, header, header_length);
1032 	queue_event(client, &e->event, &e->interrupt,
1033 		    sizeof(e->interrupt) + header_length, NULL, 0);
1034 }
1035 
1036 static void iso_mc_callback(struct fw_iso_context *context,
1037 			    dma_addr_t completed, void *data)
1038 {
1039 	struct client *client = data;
1040 	struct iso_interrupt_mc_event *e;
1041 
1042 	e = kmalloc_obj(*e);
1043 	if (e == NULL)
1044 		return;
1045 
1046 	e->interrupt.type      = FW_CDEV_EVENT_ISO_INTERRUPT_MULTICHANNEL;
1047 	e->interrupt.closure   = client->iso_closure;
1048 	e->interrupt.completed = fw_iso_buffer_lookup(&client->buffer,
1049 						      completed);
1050 	queue_event(client, &e->event, &e->interrupt,
1051 		    sizeof(e->interrupt), NULL, 0);
1052 }
1053 
1054 static enum dma_data_direction iso_dma_direction(struct fw_iso_context *context)
1055 {
1056 		if (context->type == FW_ISO_CONTEXT_TRANSMIT)
1057 			return DMA_TO_DEVICE;
1058 		else
1059 			return DMA_FROM_DEVICE;
1060 }
1061 
1062 static int ioctl_create_iso_context(struct client *client, union ioctl_arg *arg)
1063 {
1064 	struct fw_cdev_create_iso_context *a = &arg->create_iso_context;
1065 	struct fw_iso_context *context;
1066 	int ret;
1067 
1068 	BUILD_BUG_ON(FW_CDEV_ISO_CONTEXT_TRANSMIT != FW_ISO_CONTEXT_TRANSMIT ||
1069 		     FW_CDEV_ISO_CONTEXT_RECEIVE  != FW_ISO_CONTEXT_RECEIVE  ||
1070 		     FW_CDEV_ISO_CONTEXT_RECEIVE_MULTICHANNEL !=
1071 					FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL);
1072 
1073 	switch (a->type) {
1074 	case FW_ISO_CONTEXT_TRANSMIT:
1075 		if (a->speed > SCODE_3200 || a->channel > 63)
1076 			return -EINVAL;
1077 		break;
1078 
1079 	case FW_ISO_CONTEXT_RECEIVE:
1080 		if (a->header_size < 4 || (a->header_size & 3) ||
1081 		    a->channel > 63)
1082 			return -EINVAL;
1083 		break;
1084 
1085 	case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
1086 		break;
1087 
1088 	default:
1089 		return -EINVAL;
1090 	}
1091 
1092 	if (a->type == FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL)
1093 		context = fw_iso_mc_context_create(client->device->card, iso_mc_callback, client);
1094 	else
1095 		context = fw_iso_context_create(client->device->card, a->type, a->channel, a->speed,
1096 						a->header_size, iso_callback, client);
1097 	if (IS_ERR(context))
1098 		return PTR_ERR(context);
1099 	if (client->version < FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW)
1100 		context->flags |= FW_ISO_CONTEXT_FLAG_DROP_OVERFLOW_HEADERS;
1101 
1102 	// We only support one context at this time.
1103 	scoped_guard(mutex, &client->iso_context_mutex) {
1104 		if (client->iso_context != NULL) {
1105 			fw_iso_context_destroy(context);
1106 
1107 			return -EBUSY;
1108 		}
1109 		// The DMA mapping operation is available if the buffer is already allocated by
1110 		// mmap(2) system call. If not, it is delegated to the system call.
1111 		if (client->buffer.pages && !client->buffer.dma_addrs) {
1112 			ret = fw_iso_buffer_map_dma(&client->buffer, client->device->card,
1113 						    iso_dma_direction(context));
1114 			if (ret < 0) {
1115 				fw_iso_context_destroy(context);
1116 
1117 				return ret;
1118 			}
1119 		}
1120 		client->iso_closure = a->closure;
1121 		client->iso_context = context;
1122 	}
1123 
1124 	a->handle = 0;
1125 
1126 	return 0;
1127 }
1128 
1129 static int ioctl_set_iso_channels(struct client *client, union ioctl_arg *arg)
1130 {
1131 	struct fw_cdev_set_iso_channels *a = &arg->set_iso_channels;
1132 	struct fw_iso_context *ctx = client->iso_context;
1133 
1134 	if (ctx == NULL || a->handle != 0)
1135 		return -EINVAL;
1136 
1137 	return fw_iso_context_set_channels(ctx, &a->channels);
1138 }
1139 
1140 /* Macros for decoding the iso packet control header. */
1141 #define GET_PAYLOAD_LENGTH(v)	((v) & 0xffff)
1142 #define GET_INTERRUPT(v)	(((v) >> 16) & 0x01)
1143 #define GET_SKIP(v)		(((v) >> 17) & 0x01)
1144 #define GET_TAG(v)		(((v) >> 18) & 0x03)
1145 #define GET_SY(v)		(((v) >> 20) & 0x0f)
1146 #define GET_HEADER_LENGTH(v)	(((v) >> 24) & 0xff)
1147 
1148 static int ioctl_queue_iso(struct client *client, union ioctl_arg *arg)
1149 {
1150 	struct fw_cdev_queue_iso *a = &arg->queue_iso;
1151 	struct fw_cdev_iso_packet __user *p, *end, *next;
1152 	struct fw_iso_context *ctx = client->iso_context;
1153 	unsigned long payload, buffer_end, transmit_header_bytes = 0;
1154 	u32 control;
1155 	int count;
1156 	DEFINE_RAW_FLEX(struct fw_iso_packet, u, header, 64);
1157 
1158 	if (ctx == NULL || a->handle != 0)
1159 		return -EINVAL;
1160 
1161 	/*
1162 	 * If the user passes a non-NULL data pointer, has mmap()'ed
1163 	 * the iso buffer, and the pointer points inside the buffer,
1164 	 * we setup the payload pointers accordingly.  Otherwise we
1165 	 * set them both to 0, which will still let packets with
1166 	 * payload_length == 0 through.  In other words, if no packets
1167 	 * use the indirect payload, the iso buffer need not be mapped
1168 	 * and the a->data pointer is ignored.
1169 	 */
1170 	payload = (unsigned long)a->data - client->vm_start;
1171 	buffer_end = client->buffer.page_count << PAGE_SHIFT;
1172 	if (a->data == 0 || client->buffer.pages == NULL ||
1173 	    payload >= buffer_end) {
1174 		payload = 0;
1175 		buffer_end = 0;
1176 	}
1177 
1178 	if (ctx->type == FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL && payload & 3)
1179 		return -EINVAL;
1180 
1181 	p = (struct fw_cdev_iso_packet __user *)u64_to_uptr(a->packets);
1182 
1183 	end = (void __user *)p + a->size;
1184 	count = 0;
1185 	while (p < end) {
1186 		if (get_user(control, &p->control))
1187 			return -EFAULT;
1188 		u->payload_length = GET_PAYLOAD_LENGTH(control);
1189 		u->interrupt = GET_INTERRUPT(control);
1190 		u->skip = GET_SKIP(control);
1191 		u->tag = GET_TAG(control);
1192 		u->sy = GET_SY(control);
1193 		u->header_length = GET_HEADER_LENGTH(control);
1194 
1195 		switch (ctx->type) {
1196 		case FW_ISO_CONTEXT_TRANSMIT:
1197 			if (u->header_length & 3)
1198 				return -EINVAL;
1199 			transmit_header_bytes = u->header_length;
1200 			break;
1201 
1202 		case FW_ISO_CONTEXT_RECEIVE:
1203 			if (u->header_length == 0 ||
1204 			    u->header_length % ctx->header_size != 0)
1205 				return -EINVAL;
1206 			break;
1207 
1208 		case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
1209 			if (u->payload_length == 0 ||
1210 			    u->payload_length & 3)
1211 				return -EINVAL;
1212 			break;
1213 		}
1214 
1215 		next = (struct fw_cdev_iso_packet __user *)
1216 			&p->header[transmit_header_bytes / 4];
1217 		if (next > end)
1218 			return -EINVAL;
1219 		if (copy_from_user
1220 		    (u->header, p->header, transmit_header_bytes))
1221 			return -EFAULT;
1222 		if (u->skip && ctx->type == FW_ISO_CONTEXT_TRANSMIT &&
1223 		    u->header_length + u->payload_length > 0)
1224 			return -EINVAL;
1225 		if (payload + u->payload_length > buffer_end)
1226 			return -EINVAL;
1227 
1228 		if (fw_iso_context_queue(ctx, u, &client->buffer, payload))
1229 			break;
1230 
1231 		p = next;
1232 		payload += u->payload_length;
1233 		count++;
1234 	}
1235 	fw_iso_context_queue_flush(ctx);
1236 
1237 	a->size    -= uptr_to_u64(p) - a->packets;
1238 	a->packets  = uptr_to_u64(p);
1239 	a->data     = client->vm_start + payload;
1240 
1241 	return count;
1242 }
1243 
1244 static int ioctl_start_iso(struct client *client, union ioctl_arg *arg)
1245 {
1246 	struct fw_cdev_start_iso *a = &arg->start_iso;
1247 
1248 	BUILD_BUG_ON(
1249 	    FW_CDEV_ISO_CONTEXT_MATCH_TAG0 != FW_ISO_CONTEXT_MATCH_TAG0 ||
1250 	    FW_CDEV_ISO_CONTEXT_MATCH_TAG1 != FW_ISO_CONTEXT_MATCH_TAG1 ||
1251 	    FW_CDEV_ISO_CONTEXT_MATCH_TAG2 != FW_ISO_CONTEXT_MATCH_TAG2 ||
1252 	    FW_CDEV_ISO_CONTEXT_MATCH_TAG3 != FW_ISO_CONTEXT_MATCH_TAG3 ||
1253 	    FW_CDEV_ISO_CONTEXT_MATCH_ALL_TAGS != FW_ISO_CONTEXT_MATCH_ALL_TAGS);
1254 
1255 	if (client->iso_context == NULL || a->handle != 0)
1256 		return -EINVAL;
1257 
1258 	if (client->iso_context->type == FW_ISO_CONTEXT_RECEIVE &&
1259 	    (a->tags == 0 || a->tags > 15 || a->sync > 15))
1260 		return -EINVAL;
1261 
1262 	return fw_iso_context_start(client->iso_context,
1263 				    a->cycle, a->sync, a->tags);
1264 }
1265 
1266 static int ioctl_stop_iso(struct client *client, union ioctl_arg *arg)
1267 {
1268 	struct fw_cdev_stop_iso *a = &arg->stop_iso;
1269 
1270 	if (client->iso_context == NULL || a->handle != 0)
1271 		return -EINVAL;
1272 
1273 	return fw_iso_context_stop(client->iso_context);
1274 }
1275 
1276 static int ioctl_flush_iso(struct client *client, union ioctl_arg *arg)
1277 {
1278 	struct fw_cdev_flush_iso *a = &arg->flush_iso;
1279 
1280 	if (client->iso_context == NULL || a->handle != 0)
1281 		return -EINVAL;
1282 
1283 	return fw_iso_context_flush_completions(client->iso_context);
1284 }
1285 
1286 static int ioctl_get_cycle_timer2(struct client *client, union ioctl_arg *arg)
1287 {
1288 	struct fw_cdev_get_cycle_timer2 *a = &arg->get_cycle_timer2;
1289 	struct fw_card *card = client->device->card;
1290 	struct timespec64 ts = {0, 0};
1291 	u32 cycle_time = 0;
1292 	int ret;
1293 
1294 	guard(irq)();
1295 
1296 	ret = fw_card_read_cycle_time(card, &cycle_time);
1297 	if (ret < 0)
1298 		return ret;
1299 
1300 	switch (a->clk_id) {
1301 	case CLOCK_REALTIME:      ktime_get_real_ts64(&ts);	break;
1302 	case CLOCK_MONOTONIC:     ktime_get_ts64(&ts);		break;
1303 	case CLOCK_MONOTONIC_RAW: ktime_get_raw_ts64(&ts);	break;
1304 	default:
1305 		return -EINVAL;
1306 	}
1307 
1308 	a->tv_sec      = ts.tv_sec;
1309 	a->tv_nsec     = ts.tv_nsec;
1310 	a->cycle_timer = cycle_time;
1311 
1312 	return 0;
1313 }
1314 
1315 static int ioctl_get_cycle_timer(struct client *client, union ioctl_arg *arg)
1316 {
1317 	struct fw_cdev_get_cycle_timer *a = &arg->get_cycle_timer;
1318 	struct fw_cdev_get_cycle_timer2 ct2;
1319 
1320 	ct2.clk_id = CLOCK_REALTIME;
1321 	ioctl_get_cycle_timer2(client, (union ioctl_arg *)&ct2);
1322 
1323 	a->local_time = ct2.tv_sec * USEC_PER_SEC + ct2.tv_nsec / NSEC_PER_USEC;
1324 	a->cycle_timer = ct2.cycle_timer;
1325 
1326 	return 0;
1327 }
1328 
1329 static int fill_iso_resource_params(struct iso_resource_params *params,
1330 				    struct fw_cdev_allocate_iso_resource *request)
1331 {
1332 	if ((request->channels == 0 && request->bandwidth == 0) ||
1333 	    request->bandwidth > BANDWIDTH_AVAILABLE_INITIAL)
1334 		return -EINVAL;
1335 
1336 	params->channels_mask = request->channels;
1337 	params->bandwidth = request->bandwidth;
1338 
1339 	return 0;
1340 }
1341 
1342 static void iso_resource_auto_work(struct work_struct *work)
1343 {
1344 	struct iso_resource_event *e;
1345 	struct iso_resource_auto *r = from_work(r, work, work.work);
1346 	struct client *client = r->client;
1347 	unsigned long index = r->resource.handle;
1348 	int channel, bandwidth;
1349 	bool free;
1350 
1351 	u64 reset_jiffies = client->device->card->reset_jiffies;
1352 	int current_generation = client->device->generation;
1353 
1354 	int resource_generation = xchg(&r->generation, current_generation); // But no need to be atomic.
1355 	int todo = atomic_read(&r->todo);
1356 
1357 	switch (todo) {
1358 	case ISO_RES_AUTO_ALLOC:
1359 		// Allow 1000ms grace period for other reallocations.
1360 		if (time_is_after_jiffies64(reset_jiffies + secs_to_jiffies(1))) {
1361 			schedule_iso_resource_auto(r, msecs_to_jiffies(333));
1362 			return;
1363 		}
1364 		break;
1365 	case ISO_RES_AUTO_REALLOC:
1366 		// We could be called twice within the same generation.
1367 		if (resource_generation == current_generation)
1368 			return;
1369 		break;
1370 	case ISO_RES_AUTO_DEALLOC:
1371 	default:
1372 		break;
1373 	}
1374 
1375 	bandwidth = r->params.bandwidth;
1376 
1377 	fw_iso_resource_manage(client->device->card, current_generation, r->params.channels_mask,
1378 			       &channel, &bandwidth, todo != ISO_RES_AUTO_DEALLOC);
1379 
1380 	if (todo == ISO_RES_AUTO_DEALLOC) {
1381 		free = true;
1382 		e = xchg(&r->e_dealloc, NULL); // But no need to be atomic.
1383 	} else {
1384 		free = false;
1385 
1386 		// Is this generation outdated already?  As long as this resource sticks in the
1387 		// xarray, it will be scheduled again for a newer generation or at shutdown.
1388 		if (channel == -EAGAIN)
1389 			return;
1390 
1391 		bool success = channel >= 0 || bandwidth > 0;
1392 
1393 		if (!success) {
1394 			// Allocation or reallocation failure?  Pull this resource out of the
1395 			// xarray and prepare for deletion, unless the client is shutting down.
1396 			scoped_guard(mutex,  &client->mutex) {
1397 				if (!client->in_shutdown && xa_erase(&client->resource_xa, index)) {
1398 					// For the incrementation by add_client_resource().
1399 					client_put(client);
1400 					free = true;
1401 				}
1402 			}
1403 		}
1404 
1405 		if (todo == ISO_RES_AUTO_REALLOC) {
1406 			if (success)
1407 				return;
1408 
1409 			// Notify the userspace client of the failure through a deallocation event.
1410 			e = xchg(&r->e_dealloc, NULL); // But no need to be atomic.
1411 		} else {
1412 			// Transit from allocation to reallocation. Use compare-and-swap atomic
1413 			// operation because the todo member can be set with ISO_RES_AUTO_DEALLOC
1414 			// by release_iso_resource_auto() in parallel.
1415 			atomic_cmpxchg_relaxed(&r->todo, ISO_RES_AUTO_ALLOC, ISO_RES_AUTO_REALLOC);
1416 
1417 			if (channel >= 0)
1418 				r->params.channels_mask = BIT_ULL(channel);
1419 
1420 			e = xchg(&r->e_alloc, NULL); // But no need to be atomic.
1421 		}
1422 	}
1423 
1424 	e->iso_resource.handle    = r->resource.handle;
1425 	e->iso_resource.channel   = channel;
1426 	e->iso_resource.bandwidth = bandwidth;
1427 
1428 	queue_event(client, &e->event,
1429 		    &e->iso_resource, sizeof(e->iso_resource), NULL, 0);
1430 
1431 	if (free) {
1432 		cancel_delayed_work(&r->work);
1433 		kfree(r->e_alloc);
1434 		kfree(r->e_dealloc);
1435 		kfree(r);
1436 
1437 		// For the incrementation by ioctl_allocate_iso_resource().
1438 		client_put(client);
1439 	}
1440 }
1441 
1442 static void release_iso_resource_auto(struct client *client, struct client_resource *resource)
1443 {
1444 	struct iso_resource_auto *r = to_iso_resource_auto(resource);
1445 
1446 	atomic_set(&r->todo, ISO_RES_AUTO_DEALLOC);
1447 	schedule_iso_resource_auto(r, 0);
1448 }
1449 
1450 static int ioctl_allocate_iso_resource(struct client *client, union ioctl_arg *arg)
1451 {
1452 	struct fw_cdev_allocate_iso_resource *request = &arg->allocate_iso_resource;
1453 	struct iso_resource_event *e1 __free(kfree) = kmalloc_obj(*e1);
1454 	struct iso_resource_event *e2 __free(kfree) = kmalloc_obj(*e2);
1455 	struct iso_resource_auto *r  __free(kfree) = kmalloc_obj(*r);
1456 	int err;
1457 
1458 	if (!r || !e1 || !e2)
1459 		return -ENOMEM;
1460 
1461 	err = fill_iso_resource_params(&r->params, request);
1462 	if (err < 0)
1463 		return  err;
1464 
1465 	INIT_DELAYED_WORK(&r->work, iso_resource_auto_work);
1466 	r->client	= client;
1467 	atomic_set(&r->todo, ISO_RES_AUTO_ALLOC);
1468 	r->e_alloc	= e1;
1469 	r->e_dealloc	= e2;
1470 
1471 	e1->iso_resource.closure = request->closure;
1472 	e1->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_ALLOCATED;
1473 	e2->iso_resource.closure = request->closure;
1474 	e2->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_DEALLOCATED;
1475 
1476 	err = add_client_resource(client, &r->resource, release_iso_resource_auto);
1477 	if (err < 0)
1478 		return err;
1479 	request->handle = r->resource.handle;
1480 	client_get(client);
1481 
1482 	retain_and_null_ptr(e1);
1483 	retain_and_null_ptr(e2);
1484 	schedule_iso_resource_auto(no_free_ptr(r), 0);
1485 
1486 	return 0;
1487 }
1488 
1489 static int ioctl_deallocate_iso_resource(struct client *client,
1490 					 union ioctl_arg *arg)
1491 {
1492 	return release_client_resource(client,
1493 			arg->deallocate.handle, release_iso_resource_auto, NULL);
1494 }
1495 
1496 #define UNAVAILABLE_HANDLE	-1
1497 
1498 static void iso_resource_once_work(struct work_struct *work)
1499 {
1500 	struct iso_resource_once *r = from_work(r, work, work);
1501 	struct client *client = r->client;
1502 	struct iso_resource_event *e = r->event;
1503 	int channel;
1504 
1505 	int generation = client->device->generation;
1506 	int bandwidth = r->params.bandwidth;
1507 
1508 	fw_iso_resource_manage(client->device->card, generation, r->params.channels_mask, &channel,
1509 			       &bandwidth, r->todo == ISO_RES_ONCE_ALLOC);
1510 
1511 	e->iso_resource.handle = UNAVAILABLE_HANDLE;
1512 	e->iso_resource.channel = channel;
1513 	e->iso_resource.bandwidth = bandwidth;
1514 
1515 	queue_event(client, &e->event, &e->iso_resource, sizeof(e->iso_resource), NULL, 0);
1516 
1517 	cancel_work(&r->work);
1518 	kfree(r);
1519 
1520 	client_put(client);
1521 }
1522 
1523 static int init_iso_resource_once(struct client *client,
1524 				  struct fw_cdev_allocate_iso_resource *request, int todo)
1525 {
1526 	struct iso_resource_event *e __free(kfree) = kmalloc_obj(*e);
1527 	struct iso_resource_once *r __free(kfree) = kmalloc_obj(*r);
1528 	int err;
1529 
1530 	if (!r || !e)
1531 		return -ENOMEM;
1532 
1533 	err = fill_iso_resource_params(&r->params, request);
1534 	if (err < 0)
1535 		return err;
1536 
1537 	INIT_WORK(&r->work, iso_resource_once_work);
1538 	r->client = client;
1539 	r->todo	= todo;
1540 
1541 	if (todo == ISO_RES_ONCE_ALLOC)
1542 		e->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_ALLOCATED;
1543 	else
1544 		e->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_DEALLOCATED;
1545 	e->iso_resource.closure = request->closure;
1546 	r->event = no_free_ptr(e);
1547 
1548 	// Keep the client until work item finishing.
1549 	client_get(r->client);
1550 
1551 	queue_work(fw_workqueue, &no_free_ptr(r)->work);
1552 
1553 	request->handle = UNAVAILABLE_HANDLE;
1554 
1555 	return 0;
1556 }
1557 
1558 static int ioctl_allocate_iso_resource_once(struct client *client,
1559 					    union ioctl_arg *arg)
1560 {
1561 	return init_iso_resource_once(client, &arg->allocate_iso_resource, ISO_RES_ONCE_ALLOC);
1562 }
1563 
1564 static int ioctl_deallocate_iso_resource_once(struct client *client,
1565 					      union ioctl_arg *arg)
1566 {
1567 	return init_iso_resource_once(client, &arg->allocate_iso_resource, ISO_RES_ONCE_DEALLOC);
1568 }
1569 
1570 /*
1571  * Returns a speed code:  Maximum speed to or from this device,
1572  * limited by the device's link speed, the local node's link speed,
1573  * and all PHY port speeds between the two links.
1574  */
1575 static int ioctl_get_speed(struct client *client, union ioctl_arg *arg)
1576 {
1577 	return client->device->max_speed;
1578 }
1579 
1580 static int ioctl_send_broadcast_request(struct client *client,
1581 					union ioctl_arg *arg)
1582 {
1583 	struct fw_cdev_send_request *a = &arg->send_request;
1584 
1585 	switch (a->tcode) {
1586 	case TCODE_WRITE_QUADLET_REQUEST:
1587 	case TCODE_WRITE_BLOCK_REQUEST:
1588 		break;
1589 	default:
1590 		return -EINVAL;
1591 	}
1592 
1593 	/* Security policy: Only allow accesses to Units Space. */
1594 	if (a->offset < CSR_REGISTER_BASE + CSR_CONFIG_ROM_END)
1595 		return -EACCES;
1596 
1597 	return init_request(client, a, LOCAL_BUS | 0x3f, SCODE_100);
1598 }
1599 
1600 static int ioctl_send_stream_packet(struct client *client, union ioctl_arg *arg)
1601 {
1602 	struct fw_cdev_send_stream_packet *a = &arg->send_stream_packet;
1603 	struct fw_cdev_send_request request;
1604 	int dest;
1605 
1606 	if (a->speed > client->device->card->link_speed ||
1607 	    a->length > 1024 << a->speed)
1608 		return -EIO;
1609 
1610 	if (a->tag > 3 || a->channel > 63 || a->sy > 15)
1611 		return -EINVAL;
1612 
1613 	dest = fw_stream_packet_destination_id(a->tag, a->channel, a->sy);
1614 	request.tcode		= TCODE_STREAM_DATA;
1615 	request.length		= a->length;
1616 	request.closure		= a->closure;
1617 	request.data		= a->data;
1618 	request.generation	= a->generation;
1619 
1620 	return init_request(client, &request, dest, a->speed);
1621 }
1622 
1623 static void outbound_phy_packet_callback(struct fw_packet *packet,
1624 					 struct fw_card *card, int status)
1625 {
1626 	struct outbound_phy_packet_event *e =
1627 		container_of(packet, struct outbound_phy_packet_event, p);
1628 	struct client *e_client = e->client;
1629 	u32 rcode;
1630 
1631 	trace_async_phy_outbound_complete((uintptr_t)packet, card->index, status, packet->generation,
1632 					  packet->timestamp);
1633 
1634 	switch (status) {
1635 	// expected:
1636 	case ACK_COMPLETE:
1637 		rcode = RCODE_COMPLETE;
1638 		break;
1639 	// should never happen with PHY packets:
1640 	case ACK_PENDING:
1641 		rcode = RCODE_COMPLETE;
1642 		break;
1643 	case ACK_BUSY_X:
1644 	case ACK_BUSY_A:
1645 	case ACK_BUSY_B:
1646 		rcode = RCODE_BUSY;
1647 		break;
1648 	case ACK_DATA_ERROR:
1649 		rcode = RCODE_DATA_ERROR;
1650 		break;
1651 	case ACK_TYPE_ERROR:
1652 		rcode = RCODE_TYPE_ERROR;
1653 		break;
1654 	// stale generation; cancelled; on certain controllers: no ack
1655 	default:
1656 		rcode = status;
1657 		break;
1658 	}
1659 
1660 	switch (e->phy_packet.without_tstamp.type) {
1661 	case FW_CDEV_EVENT_PHY_PACKET_SENT:
1662 	{
1663 		struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;
1664 
1665 		pp->rcode = rcode;
1666 		pp->data[0] = packet->timestamp;
1667 		queue_event(e->client, &e->event, &e->phy_packet, sizeof(*pp) + pp->length,
1668 			    NULL, 0);
1669 		break;
1670 	}
1671 	case FW_CDEV_EVENT_PHY_PACKET_SENT2:
1672 	{
1673 		struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;
1674 
1675 		pp->rcode = rcode;
1676 		pp->tstamp = packet->timestamp;
1677 		queue_event(e->client, &e->event, &e->phy_packet, sizeof(*pp) + pp->length,
1678 			    NULL, 0);
1679 		break;
1680 	}
1681 	default:
1682 		WARN_ON(1);
1683 		break;
1684 	}
1685 
1686 	client_put(e_client);
1687 }
1688 
1689 static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg)
1690 {
1691 	struct fw_cdev_send_phy_packet *a = &arg->send_phy_packet;
1692 	struct fw_card *card = client->device->card;
1693 	struct outbound_phy_packet_event *e;
1694 
1695 	/* Access policy: Allow this ioctl only on local nodes' device files. */
1696 	if (!client->device->is_local)
1697 		return -ENOSYS;
1698 
1699 	e = kzalloc_flex(*e, phy_packet.with_tstamp.data, ARRAY_SIZE(a->data));
1700 	if (e == NULL)
1701 		return -ENOMEM;
1702 
1703 	client_get(client);
1704 	e->client		= client;
1705 	e->p.speed		= SCODE_100;
1706 	e->p.generation		= a->generation;
1707 	async_header_set_tcode(e->p.header, TCODE_LINK_INTERNAL);
1708 	e->p.header[1]		= a->data[0];
1709 	e->p.header[2]		= a->data[1];
1710 	e->p.header_length	= 12;
1711 	e->p.callback		= outbound_phy_packet_callback;
1712 
1713 	if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
1714 		struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;
1715 
1716 		pp->closure = a->closure;
1717 		pp->type = FW_CDEV_EVENT_PHY_PACKET_SENT;
1718 		if (is_ping_packet(a->data))
1719 			pp->length = 4;
1720 	} else {
1721 		struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;
1722 
1723 		pp->closure = a->closure;
1724 		pp->type = FW_CDEV_EVENT_PHY_PACKET_SENT2;
1725 		// Keep the data field so that application can match the response event to the
1726 		// request.
1727 		pp->length = sizeof(a->data);
1728 		memcpy(pp->data, a->data, sizeof(a->data));
1729 	}
1730 
1731 	trace_async_phy_outbound_initiate((uintptr_t)&e->p, card->index, e->p.generation,
1732 					  e->p.header[1], e->p.header[2]);
1733 
1734 	smp_load_acquire(&card->driver)->send_request(card, &e->p);
1735 
1736 	return 0;
1737 }
1738 
1739 static int ioctl_receive_phy_packets(struct client *client, union ioctl_arg *arg)
1740 {
1741 	struct fw_cdev_receive_phy_packets *a = &arg->receive_phy_packets;
1742 
1743 	/* Access policy: Allow this ioctl only on local nodes' device files. */
1744 	if (!client->device->is_local)
1745 		return -ENOSYS;
1746 
1747 	scoped_guard(mutex, &phy_receiver_list_mutex)
1748 		list_move_tail(&client->phy_receiver_link, &phy_receiver_list);
1749 
1750 	client->phy_receiver_closure = a->closure;
1751 
1752 	return 0;
1753 }
1754 
1755 void fw_cdev_handle_phy_packet(struct fw_card *card, struct fw_packet *p)
1756 {
1757 	struct client *client;
1758 
1759 	guard(mutex)(&phy_receiver_list_mutex);
1760 
1761 	list_for_each_entry(client, &phy_receiver_list, phy_receiver_link) {
1762 		if (client->device->card != card)
1763 			continue;
1764 
1765 		struct inbound_phy_packet_event *e =
1766 			kzalloc_flex(*e, phy_packet.with_tstamp.data, 2);
1767 		if (!e)
1768 			break;
1769 
1770 		if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
1771 			struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;
1772 
1773 			pp->closure = client->phy_receiver_closure;
1774 			pp->type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED;
1775 			pp->rcode = RCODE_COMPLETE;
1776 			pp->length = 8;
1777 			pp->data[0] = p->header[1];
1778 			pp->data[1] = p->header[2];
1779 			queue_event(client, &e->event, &e->phy_packet, sizeof(*pp) + 8, NULL, 0);
1780 		} else {
1781 			struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;
1782 
1783 			pp = &e->phy_packet.with_tstamp;
1784 			pp->closure = client->phy_receiver_closure;
1785 			pp->type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED2;
1786 			pp->rcode = RCODE_COMPLETE;
1787 			pp->length = 8;
1788 			pp->tstamp = p->timestamp;
1789 			pp->data[0] = p->header[1];
1790 			pp->data[1] = p->header[2];
1791 			queue_event(client, &e->event, &e->phy_packet, sizeof(*pp) + 8, NULL, 0);
1792 		}
1793 	}
1794 }
1795 
1796 static int (* const ioctl_handlers[])(struct client *, union ioctl_arg *) = {
1797 	[0x00] = ioctl_get_info,
1798 	[0x01] = ioctl_send_request,
1799 	[0x02] = ioctl_allocate,
1800 	[0x03] = ioctl_deallocate,
1801 	[0x04] = ioctl_send_response,
1802 	[0x05] = ioctl_initiate_bus_reset,
1803 	[0x06] = ioctl_add_descriptor,
1804 	[0x07] = ioctl_remove_descriptor,
1805 	[0x08] = ioctl_create_iso_context,
1806 	[0x09] = ioctl_queue_iso,
1807 	[0x0a] = ioctl_start_iso,
1808 	[0x0b] = ioctl_stop_iso,
1809 	[0x0c] = ioctl_get_cycle_timer,
1810 	[0x0d] = ioctl_allocate_iso_resource,
1811 	[0x0e] = ioctl_deallocate_iso_resource,
1812 	[0x0f] = ioctl_allocate_iso_resource_once,
1813 	[0x10] = ioctl_deallocate_iso_resource_once,
1814 	[0x11] = ioctl_get_speed,
1815 	[0x12] = ioctl_send_broadcast_request,
1816 	[0x13] = ioctl_send_stream_packet,
1817 	[0x14] = ioctl_get_cycle_timer2,
1818 	[0x15] = ioctl_send_phy_packet,
1819 	[0x16] = ioctl_receive_phy_packets,
1820 	[0x17] = ioctl_set_iso_channels,
1821 	[0x18] = ioctl_flush_iso,
1822 };
1823 
1824 static int dispatch_ioctl(struct client *client,
1825 			  unsigned int cmd, void __user *arg)
1826 {
1827 	union ioctl_arg buffer;
1828 	int ret;
1829 
1830 	if (fw_device_is_shutdown(client->device))
1831 		return -ENODEV;
1832 
1833 	if (_IOC_TYPE(cmd) != '#' ||
1834 	    _IOC_NR(cmd) >= ARRAY_SIZE(ioctl_handlers) ||
1835 	    _IOC_SIZE(cmd) > sizeof(buffer))
1836 		return -ENOTTY;
1837 
1838 	memset(&buffer, 0, sizeof(buffer));
1839 
1840 	if (_IOC_DIR(cmd) & _IOC_WRITE)
1841 		if (copy_from_user(&buffer, arg, _IOC_SIZE(cmd)))
1842 			return -EFAULT;
1843 
1844 	ret = ioctl_handlers[_IOC_NR(cmd)](client, &buffer);
1845 	if (ret < 0)
1846 		return ret;
1847 
1848 	if (_IOC_DIR(cmd) & _IOC_READ)
1849 		if (copy_to_user(arg, &buffer, _IOC_SIZE(cmd)))
1850 			return -EFAULT;
1851 
1852 	return ret;
1853 }
1854 
1855 static long fw_device_op_ioctl(struct file *file,
1856 			       unsigned int cmd, unsigned long arg)
1857 {
1858 	return dispatch_ioctl(file->private_data, cmd, (void __user *)arg);
1859 }
1860 
1861 static int fw_device_op_mmap(struct file *file, struct vm_area_struct *vma)
1862 {
1863 	struct client *client = file->private_data;
1864 	unsigned long size;
1865 	int page_count, ret;
1866 
1867 	if (fw_device_is_shutdown(client->device))
1868 		return -ENODEV;
1869 
1870 	/* FIXME: We could support multiple buffers, but we don't. */
1871 	if (client->buffer.pages != NULL)
1872 		return -EBUSY;
1873 
1874 	if (!(vma->vm_flags & VM_SHARED))
1875 		return -EINVAL;
1876 
1877 	if (vma->vm_start & ~PAGE_MASK)
1878 		return -EINVAL;
1879 
1880 	client->vm_start = vma->vm_start;
1881 	size = vma->vm_end - vma->vm_start;
1882 	page_count = size >> PAGE_SHIFT;
1883 	if (size & ~PAGE_MASK)
1884 		return -EINVAL;
1885 
1886 	ret = fw_iso_buffer_alloc(&client->buffer, page_count);
1887 	if (ret < 0)
1888 		return ret;
1889 
1890 	scoped_guard(mutex, &client->iso_context_mutex) {
1891 		// The direction of DMA can be determined if the isochronous context is already
1892 		// allocated. If not, the DMA mapping operation is postponed after the allocation.
1893 		if (client->iso_context) {
1894 			ret = fw_iso_buffer_map_dma(&client->buffer, client->device->card,
1895 						    iso_dma_direction(client->iso_context));
1896 			if (ret < 0)
1897 				goto fail;
1898 		}
1899 	}
1900 
1901 	ret = vm_map_pages_zero(vma, client->buffer.pages,
1902 				client->buffer.page_count);
1903 	if (ret < 0)
1904 		goto fail;
1905 
1906 	return 0;
1907  fail:
1908 	fw_iso_buffer_destroy(&client->buffer, client->device->card);
1909 	return ret;
1910 }
1911 
1912 static bool has_outbound_transactions(struct client *client)
1913 {
1914 	struct client_resource *resource;
1915 	unsigned long index;
1916 
1917 	scoped_guard(mutex, &client->mutex) {
1918 		xa_for_each(&client->resource_xa, index, resource) {
1919 			if (is_outbound_transaction_resource(resource))
1920 				return true;
1921 		}
1922 	}
1923 
1924 	return false;
1925 }
1926 
1927 static int fw_device_op_release(struct inode *inode, struct file *file)
1928 {
1929 	struct client *client = file->private_data;
1930 	struct event *event, *next_event;
1931 	struct client_resource *resource;
1932 	unsigned long index;
1933 
1934 	scoped_guard(mutex, &phy_receiver_list_mutex)
1935 		list_del(&client->phy_receiver_link);
1936 
1937 	scoped_guard(mutex, &client->device->client_list_mutex)
1938 		list_del(&client->link);
1939 
1940 	if (client->iso_context)
1941 		fw_iso_context_destroy(client->iso_context);
1942 	mutex_destroy(&client->iso_context_mutex);
1943 
1944 	if (client->buffer.pages)
1945 		fw_iso_buffer_destroy(&client->buffer, client->device->card);
1946 
1947 	// Freeze client->resource_xa and client->event_list.
1948 	scoped_guard(mutex, &client->mutex)
1949 		client->in_shutdown = true;
1950 
1951 	wait_event(client->tx_flush_wait, !has_outbound_transactions(client));
1952 
1953 	xa_for_each(&client->resource_xa, index, resource) {
1954 		resource->release(client, resource);
1955 		client_put(client);
1956 	}
1957 	xa_destroy(&client->resource_xa);
1958 
1959 	list_for_each_entry_safe(event, next_event, &client->event_list, link)
1960 		kfree(event);
1961 
1962 	client_put(client);
1963 
1964 	return 0;
1965 }
1966 
1967 static __poll_t fw_device_op_poll(struct file *file, poll_table * pt)
1968 {
1969 	struct client *client = file->private_data;
1970 	__poll_t mask = 0;
1971 
1972 	poll_wait(file, &client->wait, pt);
1973 
1974 	if (fw_device_is_shutdown(client->device))
1975 		mask |= EPOLLHUP | EPOLLERR;
1976 	if (!list_empty(&client->event_list))
1977 		mask |= EPOLLIN | EPOLLRDNORM;
1978 
1979 	return mask;
1980 }
1981 
1982 const struct file_operations fw_device_ops = {
1983 	.owner		= THIS_MODULE,
1984 	.open		= fw_device_op_open,
1985 	.read		= fw_device_op_read,
1986 	.unlocked_ioctl	= fw_device_op_ioctl,
1987 	.mmap		= fw_device_op_mmap,
1988 	.release	= fw_device_op_release,
1989 	.poll		= fw_device_op_poll,
1990 	.compat_ioctl	= compat_ptr_ioctl,
1991 };
1992