xref: /linux/sound/firewire/amdtp-stream.c (revision 05a54fa773284d1a7923cdfdd8f0c8dabb98bd26)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Audio and Music Data Transmission Protocol (IEC 61883-6) streams
4  * with Common Isochronous Packet (IEC 61883-1) headers
5  *
6  * Copyright (c) Clemens Ladisch <clemens@ladisch.de>
7  */
8 
9 #include <linux/device.h>
10 #include <linux/err.h>
11 #include <linux/firewire.h>
12 #include <linux/firewire-constants.h>
13 #include <linux/module.h>
14 #include <linux/slab.h>
15 #include <sound/pcm.h>
16 #include <sound/pcm_params.h>
17 #include "amdtp-stream.h"
18 
19 #define TICKS_PER_CYCLE		3072
20 #define CYCLES_PER_SECOND	8000
21 #define TICKS_PER_SECOND	(TICKS_PER_CYCLE * CYCLES_PER_SECOND)
22 
23 #define OHCI_SECOND_MODULUS		8
24 
25 /* Always support Linux tracing subsystem. */
26 #define CREATE_TRACE_POINTS
27 #include "amdtp-stream-trace.h"
28 
29 #define TRANSFER_DELAY_TICKS	0x2e00 /* 479.17 microseconds */
30 
31 /* isochronous header parameters */
32 #define ISO_DATA_LENGTH_SHIFT	16
33 #define TAG_NO_CIP_HEADER	0
34 #define TAG_CIP			1
35 
36 // Common Isochronous Packet (CIP) header parameters. Use two quadlets CIP header when supported.
37 #define CIP_HEADER_QUADLETS	2
38 #define CIP_EOH_SHIFT		31
39 #define CIP_EOH			(1u << CIP_EOH_SHIFT)
40 #define CIP_EOH_MASK		0x80000000
41 #define CIP_SID_SHIFT		24
42 #define CIP_SID_MASK		0x3f000000
43 #define CIP_DBS_MASK		0x00ff0000
44 #define CIP_DBS_SHIFT		16
45 #define CIP_SPH_MASK		0x00000400
46 #define CIP_SPH_SHIFT		10
47 #define CIP_DBC_MASK		0x000000ff
48 #define CIP_FMT_SHIFT		24
49 #define CIP_FMT_MASK		0x3f000000
50 #define CIP_FDF_MASK		0x00ff0000
51 #define CIP_FDF_SHIFT		16
52 #define CIP_FDF_NO_DATA		0xff
53 #define CIP_SYT_MASK		0x0000ffff
54 #define CIP_SYT_NO_INFO		0xffff
55 #define CIP_SYT_CYCLE_MODULUS	16
56 #define CIP_NO_DATA		((CIP_FDF_NO_DATA << CIP_FDF_SHIFT) | CIP_SYT_NO_INFO)
57 
58 #define CIP_HEADER_SIZE		(sizeof(__be32) * CIP_HEADER_QUADLETS)
59 
60 /* Audio and Music transfer protocol specific parameters */
61 #define CIP_FMT_AM		0x10
62 #define AMDTP_FDF_NO_DATA	0xff
63 
64 // For iso header and tstamp.
65 #define IR_CTX_HEADER_DEFAULT_QUADLETS	2
66 // Add nothing.
67 #define IR_CTX_HEADER_SIZE_NO_CIP	(sizeof(__be32) * IR_CTX_HEADER_DEFAULT_QUADLETS)
68 // Add two quadlets CIP header.
69 #define IR_CTX_HEADER_SIZE_CIP		(IR_CTX_HEADER_SIZE_NO_CIP + CIP_HEADER_SIZE)
70 #define HEADER_TSTAMP_MASK	0x0000ffff
71 
72 #define IT_PKT_HEADER_SIZE_CIP		CIP_HEADER_SIZE
73 #define IT_PKT_HEADER_SIZE_NO_CIP	0 // Nothing.
74 
75 // The initial firmware of OXFW970 can postpone transmission of packet during finishing
76 // asynchronous transaction. This module accepts 5 cycles to skip as maximum to avoid buffer
77 // overrun. Actual device can skip more, then this module stops the packet streaming.
78 #define IR_JUMBO_PAYLOAD_MAX_SKIP_CYCLES	5
79 
80 static void pcm_period_work(struct work_struct *work);
81 
82 /**
83  * amdtp_stream_init - initialize an AMDTP stream structure
84  * @s: the AMDTP stream to initialize
85  * @unit: the target of the stream
86  * @dir: the direction of stream
87  * @flags: the details of the streaming protocol consist of cip_flags enumeration-constants.
88  * @fmt: the value of fmt field in CIP header
89  * @process_ctx_payloads: callback handler to process payloads of isoc context
90  * @protocol_size: the size to allocate newly for protocol
91  */
92 int amdtp_stream_init(struct amdtp_stream *s, struct fw_unit *unit,
93 		      enum amdtp_stream_direction dir, unsigned int flags,
94 		      unsigned int fmt,
95 		      amdtp_stream_process_ctx_payloads_t process_ctx_payloads,
96 		      unsigned int protocol_size)
97 {
98 	if (process_ctx_payloads == NULL)
99 		return -EINVAL;
100 
101 	s->protocol = kzalloc(protocol_size, GFP_KERNEL);
102 	if (!s->protocol)
103 		return -ENOMEM;
104 
105 	s->unit = unit;
106 	s->direction = dir;
107 	s->flags = flags;
108 	s->context = ERR_PTR(-1);
109 	mutex_init(&s->mutex);
110 	INIT_WORK(&s->period_work, pcm_period_work);
111 	s->packet_index = 0;
112 
113 	init_waitqueue_head(&s->ready_wait);
114 
115 	s->fmt = fmt;
116 	s->process_ctx_payloads = process_ctx_payloads;
117 
118 	return 0;
119 }
120 EXPORT_SYMBOL(amdtp_stream_init);
121 
122 /**
123  * amdtp_stream_destroy - free stream resources
124  * @s: the AMDTP stream to destroy
125  */
126 void amdtp_stream_destroy(struct amdtp_stream *s)
127 {
128 	/* Not initialized. */
129 	if (s->protocol == NULL)
130 		return;
131 
132 	WARN_ON(amdtp_stream_running(s));
133 	kfree(s->protocol);
134 	mutex_destroy(&s->mutex);
135 }
136 EXPORT_SYMBOL(amdtp_stream_destroy);
137 
138 const unsigned int amdtp_syt_intervals[CIP_SFC_COUNT] = {
139 	[CIP_SFC_32000]  =  8,
140 	[CIP_SFC_44100]  =  8,
141 	[CIP_SFC_48000]  =  8,
142 	[CIP_SFC_88200]  = 16,
143 	[CIP_SFC_96000]  = 16,
144 	[CIP_SFC_176400] = 32,
145 	[CIP_SFC_192000] = 32,
146 };
147 EXPORT_SYMBOL(amdtp_syt_intervals);
148 
149 const unsigned int amdtp_rate_table[CIP_SFC_COUNT] = {
150 	[CIP_SFC_32000]  =  32000,
151 	[CIP_SFC_44100]  =  44100,
152 	[CIP_SFC_48000]  =  48000,
153 	[CIP_SFC_88200]  =  88200,
154 	[CIP_SFC_96000]  =  96000,
155 	[CIP_SFC_176400] = 176400,
156 	[CIP_SFC_192000] = 192000,
157 };
158 EXPORT_SYMBOL(amdtp_rate_table);
159 
160 static int apply_constraint_to_size(struct snd_pcm_hw_params *params,
161 				    struct snd_pcm_hw_rule *rule)
162 {
163 	struct snd_interval *s = hw_param_interval(params, rule->var);
164 	const struct snd_interval *r =
165 		hw_param_interval_c(params, SNDRV_PCM_HW_PARAM_RATE);
166 	struct snd_interval t = {0};
167 	unsigned int step = 0;
168 	int i;
169 
170 	for (i = 0; i < CIP_SFC_COUNT; ++i) {
171 		if (snd_interval_test(r, amdtp_rate_table[i]))
172 			step = max(step, amdtp_syt_intervals[i]);
173 	}
174 
175 	if (step == 0)
176 		return -EINVAL;
177 
178 	t.min = roundup(s->min, step);
179 	t.max = rounddown(s->max, step);
180 	t.integer = 1;
181 
182 	return snd_interval_refine(s, &t);
183 }
184 
185 /**
186  * amdtp_stream_add_pcm_hw_constraints - add hw constraints for PCM substream
187  * @s:		the AMDTP stream, which must be initialized.
188  * @runtime:	the PCM substream runtime
189  */
190 int amdtp_stream_add_pcm_hw_constraints(struct amdtp_stream *s,
191 					struct snd_pcm_runtime *runtime)
192 {
193 	struct snd_pcm_hardware *hw = &runtime->hw;
194 	unsigned int ctx_header_size;
195 	unsigned int maximum_usec_per_period;
196 	int err;
197 
198 	hw->info = SNDRV_PCM_INFO_BLOCK_TRANSFER |
199 		   SNDRV_PCM_INFO_INTERLEAVED |
200 		   SNDRV_PCM_INFO_JOINT_DUPLEX |
201 		   SNDRV_PCM_INFO_MMAP |
202 		   SNDRV_PCM_INFO_MMAP_VALID |
203 		   SNDRV_PCM_INFO_NO_PERIOD_WAKEUP;
204 
205 	hw->periods_min = 2;
206 	hw->periods_max = UINT_MAX;
207 
208 	/* bytes for a frame */
209 	hw->period_bytes_min = 4 * hw->channels_max;
210 
211 	/* Just to prevent from allocating much pages. */
212 	hw->period_bytes_max = hw->period_bytes_min * 2048;
213 	hw->buffer_bytes_max = hw->period_bytes_max * hw->periods_min;
214 
215 	// Linux driver for 1394 OHCI controller voluntarily flushes isoc
216 	// context when total size of accumulated context header reaches
217 	// PAGE_SIZE. This kicks work for the isoc context and brings
218 	// callback in the middle of scheduled interrupts.
219 	// Although AMDTP streams in the same domain use the same events per
220 	// IRQ, use the largest size of context header between IT/IR contexts.
221 	// Here, use the value of context header in IR context is for both
222 	// contexts.
223 	if (!(s->flags & CIP_NO_HEADER))
224 		ctx_header_size = IR_CTX_HEADER_SIZE_CIP;
225 	else
226 		ctx_header_size = IR_CTX_HEADER_SIZE_NO_CIP;
227 	maximum_usec_per_period = USEC_PER_SEC * PAGE_SIZE /
228 				  CYCLES_PER_SECOND / ctx_header_size;
229 
230 	// In IEC 61883-6, one isoc packet can transfer events up to the value
231 	// of syt interval. This comes from the interval of isoc cycle. As 1394
232 	// OHCI controller can generate hardware IRQ per isoc packet, the
233 	// interval is 125 usec.
234 	// However, there are two ways of transmission in IEC 61883-6; blocking
235 	// and non-blocking modes. In blocking mode, the sequence of isoc packet
236 	// includes 'empty' or 'NODATA' packets which include no event. In
237 	// non-blocking mode, the number of events per packet is variable up to
238 	// the syt interval.
239 	// Due to the above protocol design, the minimum PCM frames per
240 	// interrupt should be double of the value of syt interval, thus it is
241 	// 250 usec.
242 	err = snd_pcm_hw_constraint_minmax(runtime,
243 					   SNDRV_PCM_HW_PARAM_PERIOD_TIME,
244 					   250, maximum_usec_per_period);
245 	if (err < 0)
246 		goto end;
247 
248 	/* Non-Blocking stream has no more constraints */
249 	if (!(s->flags & CIP_BLOCKING))
250 		goto end;
251 
252 	/*
253 	 * One AMDTP packet can include some frames. In blocking mode, the
254 	 * number equals to SYT_INTERVAL. So the number is 8, 16 or 32,
255 	 * depending on its sampling rate. For accurate period interrupt, it's
256 	 * preferrable to align period/buffer sizes to current SYT_INTERVAL.
257 	 */
258 	err = snd_pcm_hw_rule_add(runtime, 0, SNDRV_PCM_HW_PARAM_PERIOD_SIZE,
259 				  apply_constraint_to_size, NULL,
260 				  SNDRV_PCM_HW_PARAM_PERIOD_SIZE,
261 				  SNDRV_PCM_HW_PARAM_RATE, -1);
262 	if (err < 0)
263 		goto end;
264 	err = snd_pcm_hw_rule_add(runtime, 0, SNDRV_PCM_HW_PARAM_BUFFER_SIZE,
265 				  apply_constraint_to_size, NULL,
266 				  SNDRV_PCM_HW_PARAM_BUFFER_SIZE,
267 				  SNDRV_PCM_HW_PARAM_RATE, -1);
268 	if (err < 0)
269 		goto end;
270 end:
271 	return err;
272 }
273 EXPORT_SYMBOL(amdtp_stream_add_pcm_hw_constraints);
274 
275 /**
276  * amdtp_stream_set_parameters - set stream parameters
277  * @s: the AMDTP stream to configure
278  * @rate: the sample rate
279  * @data_block_quadlets: the size of a data block in quadlet unit
280  * @pcm_frame_multiplier: the multiplier to compute the number of PCM frames by the number of AMDTP
281  *			  events.
282  *
283  * The parameters must be set before the stream is started, and must not be
284  * changed while the stream is running.
285  */
286 int amdtp_stream_set_parameters(struct amdtp_stream *s, unsigned int rate,
287 				unsigned int data_block_quadlets, unsigned int pcm_frame_multiplier)
288 {
289 	unsigned int sfc;
290 
291 	for (sfc = 0; sfc < ARRAY_SIZE(amdtp_rate_table); ++sfc) {
292 		if (amdtp_rate_table[sfc] == rate)
293 			break;
294 	}
295 	if (sfc == ARRAY_SIZE(amdtp_rate_table))
296 		return -EINVAL;
297 
298 	s->sfc = sfc;
299 	s->data_block_quadlets = data_block_quadlets;
300 	s->syt_interval = amdtp_syt_intervals[sfc];
301 
302 	// default buffering in the device.
303 	s->transfer_delay = TRANSFER_DELAY_TICKS - TICKS_PER_CYCLE;
304 
305 	// additional buffering needed to adjust for no-data packets.
306 	if (s->flags & CIP_BLOCKING)
307 		s->transfer_delay += TICKS_PER_SECOND * s->syt_interval / rate;
308 
309 	s->pcm_frame_multiplier = pcm_frame_multiplier;
310 
311 	return 0;
312 }
313 EXPORT_SYMBOL(amdtp_stream_set_parameters);
314 
315 // The CIP header is processed in context header apart from context payload.
316 static int amdtp_stream_get_max_ctx_payload_size(struct amdtp_stream *s)
317 {
318 	unsigned int multiplier;
319 
320 	if (s->flags & CIP_JUMBO_PAYLOAD)
321 		multiplier = IR_JUMBO_PAYLOAD_MAX_SKIP_CYCLES;
322 	else
323 		multiplier = 1;
324 
325 	return s->syt_interval * s->data_block_quadlets * sizeof(__be32) * multiplier;
326 }
327 
328 /**
329  * amdtp_stream_get_max_payload - get the stream's packet size
330  * @s: the AMDTP stream
331  *
332  * This function must not be called before the stream has been configured
333  * with amdtp_stream_set_parameters().
334  */
335 unsigned int amdtp_stream_get_max_payload(struct amdtp_stream *s)
336 {
337 	unsigned int cip_header_size;
338 
339 	if (!(s->flags & CIP_NO_HEADER))
340 		cip_header_size = CIP_HEADER_SIZE;
341 	else
342 		cip_header_size = 0;
343 
344 	return cip_header_size + amdtp_stream_get_max_ctx_payload_size(s);
345 }
346 EXPORT_SYMBOL(amdtp_stream_get_max_payload);
347 
348 /**
349  * amdtp_stream_pcm_prepare - prepare PCM device for running
350  * @s: the AMDTP stream
351  *
352  * This function should be called from the PCM device's .prepare callback.
353  */
354 void amdtp_stream_pcm_prepare(struct amdtp_stream *s)
355 {
356 	cancel_work_sync(&s->period_work);
357 	s->pcm_buffer_pointer = 0;
358 	s->pcm_period_pointer = 0;
359 }
360 EXPORT_SYMBOL(amdtp_stream_pcm_prepare);
361 
362 #define prev_packet_desc(s, desc) \
363 	list_prev_entry_circular(desc, &s->packet_descs_list, link)
364 
365 static void pool_blocking_data_blocks(struct amdtp_stream *s, struct seq_desc *descs,
366 				      unsigned int size, unsigned int pos, unsigned int count)
367 {
368 	const unsigned int syt_interval = s->syt_interval;
369 	int i;
370 
371 	for (i = 0; i < count; ++i) {
372 		struct seq_desc *desc = descs + pos;
373 
374 		if (desc->syt_offset != CIP_SYT_NO_INFO)
375 			desc->data_blocks = syt_interval;
376 		else
377 			desc->data_blocks = 0;
378 
379 		pos = (pos + 1) % size;
380 	}
381 }
382 
383 static void pool_ideal_nonblocking_data_blocks(struct amdtp_stream *s, struct seq_desc *descs,
384 					       unsigned int size, unsigned int pos,
385 					       unsigned int count)
386 {
387 	const enum cip_sfc sfc = s->sfc;
388 	unsigned int state = s->ctx_data.rx.data_block_state;
389 	int i;
390 
391 	for (i = 0; i < count; ++i) {
392 		struct seq_desc *desc = descs + pos;
393 
394 		if (!cip_sfc_is_base_44100(sfc)) {
395 			// Sample_rate / 8000 is an integer, and precomputed.
396 			desc->data_blocks = state;
397 		} else {
398 			unsigned int phase = state;
399 
400 		/*
401 		 * This calculates the number of data blocks per packet so that
402 		 * 1) the overall rate is correct and exactly synchronized to
403 		 *    the bus clock, and
404 		 * 2) packets with a rounded-up number of blocks occur as early
405 		 *    as possible in the sequence (to prevent underruns of the
406 		 *    device's buffer).
407 		 */
408 			if (sfc == CIP_SFC_44100)
409 				/* 6 6 5 6 5 6 5 ... */
410 				desc->data_blocks = 5 + ((phase & 1) ^ (phase == 0 || phase >= 40));
411 			else
412 				/* 12 11 11 11 11 ... or 23 22 22 22 22 ... */
413 				desc->data_blocks = 11 * (sfc >> 1) + (phase == 0);
414 			if (++phase >= (80 >> (sfc >> 1)))
415 				phase = 0;
416 			state = phase;
417 		}
418 
419 		pos = (pos + 1) % size;
420 	}
421 
422 	s->ctx_data.rx.data_block_state = state;
423 }
424 
425 static unsigned int calculate_syt_offset(unsigned int *last_syt_offset,
426 			unsigned int *syt_offset_state, enum cip_sfc sfc)
427 {
428 	unsigned int syt_offset;
429 
430 	if (*last_syt_offset < TICKS_PER_CYCLE) {
431 		if (!cip_sfc_is_base_44100(sfc))
432 			syt_offset = *last_syt_offset + *syt_offset_state;
433 		else {
434 		/*
435 		 * The time, in ticks, of the n'th SYT_INTERVAL sample is:
436 		 *   n * SYT_INTERVAL * 24576000 / sample_rate
437 		 * Modulo TICKS_PER_CYCLE, the difference between successive
438 		 * elements is about 1386.23.  Rounding the results of this
439 		 * formula to the SYT precision results in a sequence of
440 		 * differences that begins with:
441 		 *   1386 1386 1387 1386 1386 1386 1387 1386 1386 1386 1387 ...
442 		 * This code generates _exactly_ the same sequence.
443 		 */
444 			unsigned int phase = *syt_offset_state;
445 			unsigned int index = phase % 13;
446 
447 			syt_offset = *last_syt_offset;
448 			syt_offset += 1386 + ((index && !(index & 3)) ||
449 					      phase == 146);
450 			if (++phase >= 147)
451 				phase = 0;
452 			*syt_offset_state = phase;
453 		}
454 	} else
455 		syt_offset = *last_syt_offset - TICKS_PER_CYCLE;
456 	*last_syt_offset = syt_offset;
457 
458 	if (syt_offset >= TICKS_PER_CYCLE)
459 		syt_offset = CIP_SYT_NO_INFO;
460 
461 	return syt_offset;
462 }
463 
464 static void pool_ideal_syt_offsets(struct amdtp_stream *s, struct seq_desc *descs,
465 				   unsigned int size, unsigned int pos, unsigned int count)
466 {
467 	const enum cip_sfc sfc = s->sfc;
468 	unsigned int last = s->ctx_data.rx.last_syt_offset;
469 	unsigned int state = s->ctx_data.rx.syt_offset_state;
470 	int i;
471 
472 	for (i = 0; i < count; ++i) {
473 		struct seq_desc *desc = descs + pos;
474 
475 		desc->syt_offset = calculate_syt_offset(&last, &state, sfc);
476 
477 		pos = (pos + 1) % size;
478 	}
479 
480 	s->ctx_data.rx.last_syt_offset = last;
481 	s->ctx_data.rx.syt_offset_state = state;
482 }
483 
484 static unsigned int compute_syt_offset(unsigned int syt, unsigned int cycle,
485 				       unsigned int transfer_delay)
486 {
487 	unsigned int cycle_lo = (cycle % CYCLES_PER_SECOND) & 0x0f;
488 	unsigned int syt_cycle_lo = (syt & 0xf000) >> 12;
489 	unsigned int syt_offset;
490 
491 	// Round up.
492 	if (syt_cycle_lo < cycle_lo)
493 		syt_cycle_lo += CIP_SYT_CYCLE_MODULUS;
494 	syt_cycle_lo -= cycle_lo;
495 
496 	// Subtract transfer delay so that the synchronization offset is not so large
497 	// at transmission.
498 	syt_offset = syt_cycle_lo * TICKS_PER_CYCLE + (syt & 0x0fff);
499 	if (syt_offset < transfer_delay)
500 		syt_offset += CIP_SYT_CYCLE_MODULUS * TICKS_PER_CYCLE;
501 
502 	return syt_offset - transfer_delay;
503 }
504 
505 // Both of the producer and consumer of the queue runs in the same clock of IEEE 1394 bus.
506 // Additionally, the sequence of tx packets is severely checked against any discontinuity
507 // before filling entries in the queue. The calculation is safe even if it looks fragile by
508 // overrun.
509 static unsigned int calculate_cached_cycle_count(struct amdtp_stream *s, unsigned int head)
510 {
511 	const unsigned int cache_size = s->ctx_data.tx.cache.size;
512 	unsigned int cycles = s->ctx_data.tx.cache.pos;
513 
514 	if (cycles < head)
515 		cycles += cache_size;
516 	cycles -= head;
517 
518 	return cycles;
519 }
520 
521 static void cache_seq(struct amdtp_stream *s, const struct pkt_desc *src, unsigned int desc_count)
522 {
523 	const unsigned int transfer_delay = s->transfer_delay;
524 	const unsigned int cache_size = s->ctx_data.tx.cache.size;
525 	struct seq_desc *cache = s->ctx_data.tx.cache.descs;
526 	unsigned int cache_pos = s->ctx_data.tx.cache.pos;
527 	bool aware_syt = !(s->flags & CIP_UNAWARE_SYT);
528 	int i;
529 
530 	for (i = 0; i < desc_count; ++i) {
531 		struct seq_desc *dst = cache + cache_pos;
532 
533 		if (aware_syt && src->syt != CIP_SYT_NO_INFO)
534 			dst->syt_offset = compute_syt_offset(src->syt, src->cycle, transfer_delay);
535 		else
536 			dst->syt_offset = CIP_SYT_NO_INFO;
537 		dst->data_blocks = src->data_blocks;
538 
539 		cache_pos = (cache_pos + 1) % cache_size;
540 		src = amdtp_stream_next_packet_desc(s, src);
541 	}
542 
543 	s->ctx_data.tx.cache.pos = cache_pos;
544 }
545 
546 static void pool_ideal_seq_descs(struct amdtp_stream *s, struct seq_desc *descs, unsigned int size,
547 				 unsigned int pos, unsigned int count)
548 {
549 	pool_ideal_syt_offsets(s, descs, size, pos, count);
550 
551 	if (s->flags & CIP_BLOCKING)
552 		pool_blocking_data_blocks(s, descs, size, pos, count);
553 	else
554 		pool_ideal_nonblocking_data_blocks(s, descs, size, pos, count);
555 }
556 
557 static void pool_replayed_seq(struct amdtp_stream *s, struct seq_desc *descs, unsigned int size,
558 			      unsigned int pos, unsigned int count)
559 {
560 	struct amdtp_stream *target = s->ctx_data.rx.replay_target;
561 	const struct seq_desc *cache = target->ctx_data.tx.cache.descs;
562 	const unsigned int cache_size = target->ctx_data.tx.cache.size;
563 	unsigned int cache_pos = s->ctx_data.rx.cache_pos;
564 	int i;
565 
566 	for (i = 0; i < count; ++i) {
567 		descs[pos] = cache[cache_pos];
568 		cache_pos = (cache_pos + 1) % cache_size;
569 		pos = (pos + 1) % size;
570 	}
571 
572 	s->ctx_data.rx.cache_pos = cache_pos;
573 }
574 
575 static void pool_seq_descs(struct amdtp_stream *s, struct seq_desc *descs, unsigned int size,
576 			   unsigned int pos, unsigned int count)
577 {
578 	struct amdtp_domain *d = s->domain;
579 	void (*pool_seq_descs)(struct amdtp_stream *s, struct seq_desc *descs, unsigned int size,
580 			       unsigned int pos, unsigned int count);
581 
582 	if (!d->replay.enable || !s->ctx_data.rx.replay_target) {
583 		pool_seq_descs = pool_ideal_seq_descs;
584 	} else {
585 		if (!d->replay.on_the_fly) {
586 			pool_seq_descs = pool_replayed_seq;
587 		} else {
588 			struct amdtp_stream *tx = s->ctx_data.rx.replay_target;
589 			const unsigned int cache_size = tx->ctx_data.tx.cache.size;
590 			const unsigned int cache_pos = s->ctx_data.rx.cache_pos;
591 			unsigned int cached_cycles = calculate_cached_cycle_count(tx, cache_pos);
592 
593 			if (cached_cycles > count && cached_cycles > cache_size / 2)
594 				pool_seq_descs = pool_replayed_seq;
595 			else
596 				pool_seq_descs = pool_ideal_seq_descs;
597 		}
598 	}
599 
600 	pool_seq_descs(s, descs, size, pos, count);
601 }
602 
603 static void update_pcm_pointers(struct amdtp_stream *s,
604 				struct snd_pcm_substream *pcm,
605 				unsigned int frames)
606 {
607 	unsigned int ptr;
608 
609 	ptr = s->pcm_buffer_pointer + frames;
610 	if (ptr >= pcm->runtime->buffer_size)
611 		ptr -= pcm->runtime->buffer_size;
612 	WRITE_ONCE(s->pcm_buffer_pointer, ptr);
613 
614 	s->pcm_period_pointer += frames;
615 	if (s->pcm_period_pointer >= pcm->runtime->period_size) {
616 		s->pcm_period_pointer -= pcm->runtime->period_size;
617 
618 		// The program in user process should periodically check the status of intermediate
619 		// buffer associated to PCM substream to process PCM frames in the buffer, instead
620 		// of receiving notification of period elapsed by poll wait.
621 		//
622 		// Use another work item for period elapsed event to prevent the following AB/BA
623 		// deadlock:
624 		//
625 		//             thread 1                            thread 2
626 		// =================================   =================================
627 		//       A.work item (process)                pcm ioctl (process)
628 		//                 v                                   v
629 		//       process_rx_packets()                  B.PCM stream lock
630 		//       process_tx_packets()                          v
631 		//                 v                        callbacks in snd_pcm_ops
632 		//       update_pcm_pointers()                         v
633 		//         snd_pcm_elapsed()           fw_iso_context_flush_completions()
634 		//  snd_pcm_stream_lock_irqsave()             disable_work_sync()
635 		//                 v                                   v
636 		//     wait until release of B                wait until A exits
637 		if (!pcm->runtime->no_period_wakeup)
638 			queue_work(system_highpri_wq, &s->period_work);
639 	}
640 }
641 
642 static void pcm_period_work(struct work_struct *work)
643 {
644 	struct amdtp_stream *s = container_of(work, struct amdtp_stream,
645 					      period_work);
646 	struct snd_pcm_substream *pcm = READ_ONCE(s->pcm);
647 
648 	if (pcm)
649 		snd_pcm_period_elapsed(pcm);
650 }
651 
652 static int queue_packet(struct amdtp_stream *s, struct fw_iso_packet *params,
653 			bool sched_irq)
654 {
655 	int err;
656 
657 	params->interrupt = sched_irq;
658 	params->tag = s->tag;
659 	params->sy = 0;
660 
661 	err = fw_iso_context_queue(s->context, params, &s->buffer.iso_buffer,
662 				   s->buffer.packets[s->packet_index].offset);
663 	if (err < 0) {
664 		dev_err(&s->unit->device, "queueing error: %d\n", err);
665 		goto end;
666 	}
667 
668 	if (++s->packet_index >= s->queue_size)
669 		s->packet_index = 0;
670 end:
671 	return err;
672 }
673 
674 static inline int queue_out_packet(struct amdtp_stream *s,
675 				   struct fw_iso_packet *params, bool sched_irq)
676 {
677 	params->skip =
678 		!!(params->header_length == 0 && params->payload_length == 0);
679 	return queue_packet(s, params, sched_irq);
680 }
681 
682 static inline int queue_in_packet(struct amdtp_stream *s,
683 				  struct fw_iso_packet *params)
684 {
685 	// Queue one packet for IR context.
686 	params->header_length = s->ctx_data.tx.ctx_header_size;
687 	params->payload_length = s->ctx_data.tx.max_ctx_payload_length;
688 	params->skip = false;
689 	return queue_packet(s, params, false);
690 }
691 
692 static void generate_cip_header(struct amdtp_stream *s, __be32 cip_header[2],
693 			unsigned int data_block_counter, unsigned int syt)
694 {
695 	cip_header[0] = cpu_to_be32(READ_ONCE(s->source_node_id_field) |
696 				(s->data_block_quadlets << CIP_DBS_SHIFT) |
697 				((s->sph << CIP_SPH_SHIFT) & CIP_SPH_MASK) |
698 				data_block_counter);
699 	cip_header[1] = cpu_to_be32(CIP_EOH |
700 			((s->fmt << CIP_FMT_SHIFT) & CIP_FMT_MASK) |
701 			((s->ctx_data.rx.fdf << CIP_FDF_SHIFT) & CIP_FDF_MASK) |
702 			(syt & CIP_SYT_MASK));
703 }
704 
705 static void build_it_pkt_header(struct amdtp_stream *s, unsigned int cycle,
706 				struct fw_iso_packet *params, unsigned int header_length,
707 				unsigned int data_blocks,
708 				unsigned int data_block_counter,
709 				unsigned int syt, unsigned int index, u32 curr_cycle_time)
710 {
711 	unsigned int payload_length;
712 	__be32 *cip_header;
713 
714 	payload_length = data_blocks * sizeof(__be32) * s->data_block_quadlets;
715 	params->payload_length = payload_length;
716 
717 	if (header_length > 0) {
718 		cip_header = (__be32 *)params->header;
719 		generate_cip_header(s, cip_header, data_block_counter, syt);
720 		params->header_length = header_length;
721 	} else {
722 		cip_header = NULL;
723 	}
724 
725 	trace_amdtp_packet(s, cycle, cip_header, payload_length + header_length, data_blocks,
726 			   data_block_counter, s->packet_index, index, curr_cycle_time);
727 }
728 
729 static int check_cip_header(struct amdtp_stream *s, const __be32 *buf,
730 			    unsigned int payload_length,
731 			    unsigned int *data_blocks,
732 			    unsigned int *data_block_counter, unsigned int *syt)
733 {
734 	u32 cip_header[2];
735 	unsigned int sph;
736 	unsigned int fmt;
737 	unsigned int fdf;
738 	unsigned int dbc;
739 	bool lost;
740 
741 	cip_header[0] = be32_to_cpu(buf[0]);
742 	cip_header[1] = be32_to_cpu(buf[1]);
743 
744 	/*
745 	 * This module supports 'Two-quadlet CIP header with SYT field'.
746 	 * For convenience, also check FMT field is AM824 or not.
747 	 */
748 	if ((((cip_header[0] & CIP_EOH_MASK) == CIP_EOH) ||
749 	     ((cip_header[1] & CIP_EOH_MASK) != CIP_EOH)) &&
750 	    (!(s->flags & CIP_HEADER_WITHOUT_EOH))) {
751 		dev_info_ratelimited(&s->unit->device,
752 				"Invalid CIP header for AMDTP: %08X:%08X\n",
753 				cip_header[0], cip_header[1]);
754 		return -EAGAIN;
755 	}
756 
757 	/* Check valid protocol or not. */
758 	sph = (cip_header[0] & CIP_SPH_MASK) >> CIP_SPH_SHIFT;
759 	fmt = (cip_header[1] & CIP_FMT_MASK) >> CIP_FMT_SHIFT;
760 	if (sph != s->sph || fmt != s->fmt) {
761 		dev_info_ratelimited(&s->unit->device,
762 				     "Detect unexpected protocol: %08x %08x\n",
763 				     cip_header[0], cip_header[1]);
764 		return -EAGAIN;
765 	}
766 
767 	/* Calculate data blocks */
768 	fdf = (cip_header[1] & CIP_FDF_MASK) >> CIP_FDF_SHIFT;
769 	if (payload_length == 0 || (fmt == CIP_FMT_AM && fdf == AMDTP_FDF_NO_DATA)) {
770 		*data_blocks = 0;
771 	} else {
772 		unsigned int data_block_quadlets =
773 				(cip_header[0] & CIP_DBS_MASK) >> CIP_DBS_SHIFT;
774 		/* avoid division by zero */
775 		if (data_block_quadlets == 0) {
776 			dev_err(&s->unit->device,
777 				"Detect invalid value in dbs field: %08X\n",
778 				cip_header[0]);
779 			return -EPROTO;
780 		}
781 		if (s->flags & CIP_WRONG_DBS)
782 			data_block_quadlets = s->data_block_quadlets;
783 
784 		*data_blocks = payload_length / sizeof(__be32) / data_block_quadlets;
785 	}
786 
787 	/* Check data block counter continuity */
788 	dbc = cip_header[0] & CIP_DBC_MASK;
789 	if (*data_blocks == 0 && (s->flags & CIP_EMPTY_HAS_WRONG_DBC) &&
790 	    *data_block_counter != UINT_MAX)
791 		dbc = *data_block_counter;
792 
793 	if ((dbc == 0x00 && (s->flags & CIP_SKIP_DBC_ZERO_CHECK)) ||
794 	    *data_block_counter == UINT_MAX) {
795 		lost = false;
796 	} else if (!(s->flags & CIP_DBC_IS_END_EVENT)) {
797 		lost = dbc != *data_block_counter;
798 	} else {
799 		unsigned int dbc_interval;
800 
801 		if (!(s->flags & CIP_DBC_IS_PAYLOAD_QUADLETS)) {
802 			if (*data_blocks > 0 && s->ctx_data.tx.dbc_interval > 0)
803 				dbc_interval = s->ctx_data.tx.dbc_interval;
804 			else
805 				dbc_interval = *data_blocks;
806 		} else {
807 			dbc_interval = payload_length / sizeof(__be32);
808 		}
809 
810 		lost = dbc != ((*data_block_counter + dbc_interval) & 0xff);
811 	}
812 
813 	if (lost) {
814 		dev_err(&s->unit->device,
815 			"Detect discontinuity of CIP: %02X %02X\n",
816 			*data_block_counter, dbc);
817 		return -EIO;
818 	}
819 
820 	*data_block_counter = dbc;
821 
822 	if (!(s->flags & CIP_UNAWARE_SYT))
823 		*syt = cip_header[1] & CIP_SYT_MASK;
824 
825 	return 0;
826 }
827 
828 static int parse_ir_ctx_header(struct amdtp_stream *s, unsigned int cycle,
829 			       const __be32 *ctx_header,
830 			       unsigned int *data_blocks,
831 			       unsigned int *data_block_counter,
832 			       unsigned int *syt, unsigned int packet_index, unsigned int index,
833 			       u32 curr_cycle_time)
834 {
835 	unsigned int payload_length;
836 	const __be32 *cip_header;
837 	unsigned int cip_header_size;
838 
839 	payload_length = be32_to_cpu(ctx_header[0]) >> ISO_DATA_LENGTH_SHIFT;
840 
841 	if (!(s->flags & CIP_NO_HEADER))
842 		cip_header_size = CIP_HEADER_SIZE;
843 	else
844 		cip_header_size = 0;
845 
846 	if (payload_length > cip_header_size + s->ctx_data.tx.max_ctx_payload_length) {
847 		dev_err(&s->unit->device,
848 			"Detect jumbo payload: %04x %04x\n",
849 			payload_length, cip_header_size + s->ctx_data.tx.max_ctx_payload_length);
850 		return -EIO;
851 	}
852 
853 	if (cip_header_size > 0) {
854 		if (payload_length >= cip_header_size) {
855 			int err;
856 
857 			cip_header = ctx_header + IR_CTX_HEADER_DEFAULT_QUADLETS;
858 			err = check_cip_header(s, cip_header, payload_length - cip_header_size,
859 					       data_blocks, data_block_counter, syt);
860 			if (err < 0)
861 				return err;
862 		} else {
863 			// Handle the cycle so that empty packet arrives.
864 			cip_header = NULL;
865 			*data_blocks = 0;
866 			*syt = 0;
867 		}
868 	} else {
869 		cip_header = NULL;
870 		*data_blocks = payload_length / sizeof(__be32) / s->data_block_quadlets;
871 		*syt = 0;
872 
873 		if (*data_block_counter == UINT_MAX)
874 			*data_block_counter = 0;
875 	}
876 
877 	trace_amdtp_packet(s, cycle, cip_header, payload_length, *data_blocks,
878 			   *data_block_counter, packet_index, index, curr_cycle_time);
879 
880 	return 0;
881 }
882 
883 // In CYCLE_TIMER register of IEEE 1394, 7 bits are used to represent second. On
884 // the other hand, in DMA descriptors of 1394 OHCI, 3 bits are used to represent
885 // it. Thus, via Linux firewire subsystem, we can get the 3 bits for second.
886 static inline u32 compute_ohci_iso_ctx_cycle_count(u32 tstamp)
887 {
888 	return (((tstamp >> 13) & 0x07) * CYCLES_PER_SECOND) + (tstamp & 0x1fff);
889 }
890 
891 static inline u32 compute_ohci_cycle_count(__be32 ctx_header_tstamp)
892 {
893 	u32 tstamp = be32_to_cpu(ctx_header_tstamp) & HEADER_TSTAMP_MASK;
894 	return compute_ohci_iso_ctx_cycle_count(tstamp);
895 }
896 
897 static inline u32 increment_ohci_cycle_count(u32 cycle, unsigned int addend)
898 {
899 	cycle += addend;
900 	if (cycle >= OHCI_SECOND_MODULUS * CYCLES_PER_SECOND)
901 		cycle -= OHCI_SECOND_MODULUS * CYCLES_PER_SECOND;
902 	return cycle;
903 }
904 
905 static inline u32 decrement_ohci_cycle_count(u32 minuend, u32 subtrahend)
906 {
907 	if (minuend < subtrahend)
908 		minuend += OHCI_SECOND_MODULUS * CYCLES_PER_SECOND;
909 
910 	return minuend - subtrahend;
911 }
912 
913 static int compare_ohci_cycle_count(u32 lval, u32 rval)
914 {
915 	if (lval == rval)
916 		return 0;
917 	else if (lval < rval && rval - lval < OHCI_SECOND_MODULUS * CYCLES_PER_SECOND / 2)
918 		return -1;
919 	else
920 		return 1;
921 }
922 
923 // Align to actual cycle count for the packet which is going to be scheduled.
924 // This module queued the same number of isochronous cycle as the size of queue
925 // to kip isochronous cycle, therefore it's OK to just increment the cycle by
926 // the size of queue for scheduled cycle.
927 static inline u32 compute_ohci_it_cycle(const __be32 ctx_header_tstamp,
928 					unsigned int queue_size)
929 {
930 	u32 cycle = compute_ohci_cycle_count(ctx_header_tstamp);
931 	return increment_ohci_cycle_count(cycle, queue_size);
932 }
933 
934 static int generate_tx_packet_descs(struct amdtp_stream *s, struct pkt_desc *desc,
935 				    const __be32 *ctx_header, unsigned int packet_count,
936 				    unsigned int *desc_count)
937 {
938 	unsigned int next_cycle = s->next_cycle;
939 	unsigned int dbc = s->data_block_counter;
940 	unsigned int packet_index = s->packet_index;
941 	unsigned int queue_size = s->queue_size;
942 	u32 curr_cycle_time = 0;
943 	int i;
944 	int err;
945 
946 	if (trace_amdtp_packet_enabled())
947 		(void)fw_card_read_cycle_time(fw_parent_device(s->unit)->card, &curr_cycle_time);
948 
949 	*desc_count = 0;
950 	for (i = 0; i < packet_count; ++i) {
951 		unsigned int cycle;
952 		bool lost;
953 		unsigned int data_blocks;
954 		unsigned int syt;
955 
956 		cycle = compute_ohci_cycle_count(ctx_header[1]);
957 		lost = (next_cycle != cycle);
958 		if (lost) {
959 			if (s->flags & CIP_NO_HEADER) {
960 				// Fireface skips transmission just for an isoc cycle corresponding
961 				// to empty packet.
962 				unsigned int prev_cycle = next_cycle;
963 
964 				next_cycle = increment_ohci_cycle_count(next_cycle, 1);
965 				lost = (next_cycle != cycle);
966 				if (!lost) {
967 					// Prepare a description for the skipped cycle for
968 					// sequence replay.
969 					desc->cycle = prev_cycle;
970 					desc->syt = 0;
971 					desc->data_blocks = 0;
972 					desc->data_block_counter = dbc;
973 					desc->ctx_payload = NULL;
974 					desc = amdtp_stream_next_packet_desc(s, desc);
975 					++(*desc_count);
976 				}
977 			} else if (s->flags & CIP_JUMBO_PAYLOAD) {
978 				// OXFW970 skips transmission for several isoc cycles during
979 				// asynchronous transaction. The sequence replay is impossible due
980 				// to the reason.
981 				unsigned int safe_cycle = increment_ohci_cycle_count(next_cycle,
982 								IR_JUMBO_PAYLOAD_MAX_SKIP_CYCLES);
983 				lost = (compare_ohci_cycle_count(safe_cycle, cycle) < 0);
984 			}
985 			if (lost) {
986 				dev_err(&s->unit->device, "Detect discontinuity of cycle: %d %d\n",
987 					next_cycle, cycle);
988 				return -EIO;
989 			}
990 		}
991 
992 		err = parse_ir_ctx_header(s, cycle, ctx_header, &data_blocks, &dbc, &syt,
993 					  packet_index, i, curr_cycle_time);
994 		if (err < 0)
995 			return err;
996 
997 		desc->cycle = cycle;
998 		desc->syt = syt;
999 		desc->data_blocks = data_blocks;
1000 		desc->data_block_counter = dbc;
1001 		desc->ctx_payload = s->buffer.packets[packet_index].buffer;
1002 
1003 		if (!(s->flags & CIP_DBC_IS_END_EVENT))
1004 			dbc = (dbc + desc->data_blocks) & 0xff;
1005 
1006 		next_cycle = increment_ohci_cycle_count(next_cycle, 1);
1007 		desc = amdtp_stream_next_packet_desc(s, desc);
1008 		++(*desc_count);
1009 		ctx_header += s->ctx_data.tx.ctx_header_size / sizeof(*ctx_header);
1010 		packet_index = (packet_index + 1) % queue_size;
1011 	}
1012 
1013 	s->next_cycle = next_cycle;
1014 	s->data_block_counter = dbc;
1015 
1016 	return 0;
1017 }
1018 
1019 static unsigned int compute_syt(unsigned int syt_offset, unsigned int cycle,
1020 				unsigned int transfer_delay)
1021 {
1022 	unsigned int syt;
1023 
1024 	syt_offset += transfer_delay;
1025 	syt = ((cycle + syt_offset / TICKS_PER_CYCLE) << 12) |
1026 	      (syt_offset % TICKS_PER_CYCLE);
1027 	return syt & CIP_SYT_MASK;
1028 }
1029 
1030 static void generate_rx_packet_descs(struct amdtp_stream *s, struct pkt_desc *desc,
1031 				     const __be32 *ctx_header, unsigned int packet_count)
1032 {
1033 	struct seq_desc *seq_descs = s->ctx_data.rx.seq.descs;
1034 	unsigned int seq_size = s->ctx_data.rx.seq.size;
1035 	unsigned int seq_pos = s->ctx_data.rx.seq.pos;
1036 	unsigned int dbc = s->data_block_counter;
1037 	bool aware_syt = !(s->flags & CIP_UNAWARE_SYT);
1038 	int i;
1039 
1040 	pool_seq_descs(s, seq_descs, seq_size, seq_pos, packet_count);
1041 
1042 	for (i = 0; i < packet_count; ++i) {
1043 		unsigned int index = (s->packet_index + i) % s->queue_size;
1044 		const struct seq_desc *seq = seq_descs + seq_pos;
1045 
1046 		desc->cycle = compute_ohci_it_cycle(*ctx_header, s->queue_size);
1047 
1048 		if (aware_syt && seq->syt_offset != CIP_SYT_NO_INFO)
1049 			desc->syt = compute_syt(seq->syt_offset, desc->cycle, s->transfer_delay);
1050 		else
1051 			desc->syt = CIP_SYT_NO_INFO;
1052 
1053 		desc->data_blocks = seq->data_blocks;
1054 
1055 		if (s->flags & CIP_DBC_IS_END_EVENT)
1056 			dbc = (dbc + desc->data_blocks) & 0xff;
1057 
1058 		desc->data_block_counter = dbc;
1059 
1060 		if (!(s->flags & CIP_DBC_IS_END_EVENT))
1061 			dbc = (dbc + desc->data_blocks) & 0xff;
1062 
1063 		desc->ctx_payload = s->buffer.packets[index].buffer;
1064 
1065 		seq_pos = (seq_pos + 1) % seq_size;
1066 		desc = amdtp_stream_next_packet_desc(s, desc);
1067 
1068 		++ctx_header;
1069 	}
1070 
1071 	s->data_block_counter = dbc;
1072 	s->ctx_data.rx.seq.pos = seq_pos;
1073 }
1074 
1075 static inline void cancel_stream(struct amdtp_stream *s)
1076 {
1077 	struct work_struct *work = current_work();
1078 
1079 	s->packet_index = -1;
1080 
1081 	// Detect work items for any isochronous context. The work item for pcm_period_work()
1082 	// should be avoided since the call of snd_pcm_period_elapsed() can reach via
1083 	// snd_pcm_ops.pointer() under acquiring PCM stream(group) lock and causes dead lock at
1084 	// snd_pcm_stop_xrun().
1085 	if (work && work != &s->period_work)
1086 		amdtp_stream_pcm_abort(s);
1087 	WRITE_ONCE(s->pcm_buffer_pointer, SNDRV_PCM_POS_XRUN);
1088 }
1089 
1090 static snd_pcm_sframes_t compute_pcm_extra_delay(struct amdtp_stream *s,
1091 						 const struct pkt_desc *desc, unsigned int count)
1092 {
1093 	unsigned int data_block_count = 0;
1094 	u32 latest_cycle;
1095 	u32 cycle_time;
1096 	u32 curr_cycle;
1097 	u32 cycle_gap;
1098 	int i, err;
1099 
1100 	if (count == 0)
1101 		goto end;
1102 
1103 	// Forward to the latest record.
1104 	for (i = 0; i < count - 1; ++i)
1105 		desc = amdtp_stream_next_packet_desc(s, desc);
1106 	latest_cycle = desc->cycle;
1107 
1108 	err = fw_card_read_cycle_time(fw_parent_device(s->unit)->card, &cycle_time);
1109 	if (err < 0)
1110 		goto end;
1111 
1112 	// Compute cycle count with lower 3 bits of second field and cycle field like timestamp
1113 	// format of 1394 OHCI isochronous context.
1114 	curr_cycle = compute_ohci_iso_ctx_cycle_count((cycle_time >> 12) & 0x0000ffff);
1115 
1116 	if (s->direction == AMDTP_IN_STREAM) {
1117 		// NOTE: The AMDTP packet descriptor should be for the past isochronous cycle since
1118 		// it corresponds to arrived isochronous packet.
1119 		if (compare_ohci_cycle_count(latest_cycle, curr_cycle) > 0)
1120 			goto end;
1121 		cycle_gap = decrement_ohci_cycle_count(curr_cycle, latest_cycle);
1122 
1123 		// NOTE: estimate delay by recent history of arrived AMDTP packets. The estimated
1124 		// value expectedly corresponds to a few packets (0-2) since the packet arrived at
1125 		// the most recent isochronous cycle has been already processed.
1126 		for (i = 0; i < cycle_gap; ++i) {
1127 			desc = amdtp_stream_next_packet_desc(s, desc);
1128 			data_block_count += desc->data_blocks;
1129 		}
1130 	} else {
1131 		// NOTE: The AMDTP packet descriptor should be for the future isochronous cycle
1132 		// since it was already scheduled.
1133 		if (compare_ohci_cycle_count(latest_cycle, curr_cycle) < 0)
1134 			goto end;
1135 		cycle_gap = decrement_ohci_cycle_count(latest_cycle, curr_cycle);
1136 
1137 		// NOTE: use history of scheduled packets.
1138 		for (i = 0; i < cycle_gap; ++i) {
1139 			data_block_count += desc->data_blocks;
1140 			desc = prev_packet_desc(s, desc);
1141 		}
1142 	}
1143 end:
1144 	return data_block_count * s->pcm_frame_multiplier;
1145 }
1146 
1147 static void process_ctx_payloads(struct amdtp_stream *s,
1148 				 const struct pkt_desc *desc,
1149 				 unsigned int count)
1150 {
1151 	struct snd_pcm_substream *pcm;
1152 	int i;
1153 
1154 	pcm = READ_ONCE(s->pcm);
1155 	s->process_ctx_payloads(s, desc, count, pcm);
1156 
1157 	if (pcm) {
1158 		unsigned int data_block_count = 0;
1159 
1160 		pcm->runtime->delay = compute_pcm_extra_delay(s, desc, count);
1161 
1162 		for (i = 0; i < count; ++i) {
1163 			data_block_count += desc->data_blocks;
1164 			desc = amdtp_stream_next_packet_desc(s, desc);
1165 		}
1166 
1167 		update_pcm_pointers(s, pcm, data_block_count * s->pcm_frame_multiplier);
1168 	}
1169 }
1170 
1171 static void process_rx_packets(struct fw_iso_context *context, u32 tstamp, size_t header_length,
1172 			       void *header, void *private_data)
1173 {
1174 	struct amdtp_stream *s = private_data;
1175 	const struct amdtp_domain *d = s->domain;
1176 	const __be32 *ctx_header = header;
1177 	const unsigned int events_per_period = d->events_per_period;
1178 	unsigned int event_count = s->ctx_data.rx.event_count;
1179 	struct pkt_desc *desc = s->packet_descs_cursor;
1180 	unsigned int pkt_header_length;
1181 	unsigned int packets;
1182 	u32 curr_cycle_time;
1183 	bool need_hw_irq;
1184 	int i;
1185 
1186 	if (s->packet_index < 0)
1187 		return;
1188 
1189 	// Calculate the number of packets in buffer and check XRUN.
1190 	packets = header_length / sizeof(*ctx_header);
1191 
1192 	generate_rx_packet_descs(s, desc, ctx_header, packets);
1193 
1194 	process_ctx_payloads(s, desc, packets);
1195 
1196 	if (!(s->flags & CIP_NO_HEADER))
1197 		pkt_header_length = IT_PKT_HEADER_SIZE_CIP;
1198 	else
1199 		pkt_header_length = 0;
1200 
1201 	if (s == d->irq_target) {
1202 		// At NO_PERIOD_WAKEUP mode, the packets for all IT/IR contexts are processed by
1203 		// the tasks of user process operating ALSA PCM character device by calling ioctl(2)
1204 		// with some requests, instead of scheduled hardware IRQ of an IT context.
1205 		struct snd_pcm_substream *pcm = READ_ONCE(s->pcm);
1206 		need_hw_irq = !pcm || !pcm->runtime->no_period_wakeup;
1207 	} else {
1208 		need_hw_irq = false;
1209 	}
1210 
1211 	if (trace_amdtp_packet_enabled())
1212 		(void)fw_card_read_cycle_time(fw_parent_device(s->unit)->card, &curr_cycle_time);
1213 
1214 	for (i = 0; i < packets; ++i) {
1215 		DEFINE_RAW_FLEX(struct fw_iso_packet, template, header, CIP_HEADER_QUADLETS);
1216 		bool sched_irq = false;
1217 
1218 		build_it_pkt_header(s, desc->cycle, template, pkt_header_length,
1219 				    desc->data_blocks, desc->data_block_counter,
1220 				    desc->syt, i, curr_cycle_time);
1221 
1222 		if (s == s->domain->irq_target) {
1223 			event_count += desc->data_blocks;
1224 			if (event_count >= events_per_period) {
1225 				event_count -= events_per_period;
1226 				sched_irq = need_hw_irq;
1227 			}
1228 		}
1229 
1230 		if (queue_out_packet(s, template, sched_irq) < 0) {
1231 			cancel_stream(s);
1232 			return;
1233 		}
1234 
1235 		desc = amdtp_stream_next_packet_desc(s, desc);
1236 	}
1237 
1238 	s->ctx_data.rx.event_count = event_count;
1239 	s->packet_descs_cursor = desc;
1240 }
1241 
1242 static void skip_rx_packets(struct fw_iso_context *context, u32 tstamp, size_t header_length,
1243 			    void *header, void *private_data)
1244 {
1245 	struct amdtp_stream *s = private_data;
1246 	struct amdtp_domain *d = s->domain;
1247 	const __be32 *ctx_header = header;
1248 	unsigned int packets;
1249 	unsigned int cycle;
1250 	int i;
1251 
1252 	if (s->packet_index < 0)
1253 		return;
1254 
1255 	packets = header_length / sizeof(*ctx_header);
1256 
1257 	cycle = compute_ohci_it_cycle(ctx_header[packets - 1], s->queue_size);
1258 	s->next_cycle = increment_ohci_cycle_count(cycle, 1);
1259 
1260 	for (i = 0; i < packets; ++i) {
1261 		struct fw_iso_packet params = {
1262 			.header_length = 0,
1263 			.payload_length = 0,
1264 		};
1265 		bool sched_irq = (s == d->irq_target && i == packets - 1);
1266 
1267 		if (queue_out_packet(s, &params, sched_irq) < 0) {
1268 			cancel_stream(s);
1269 			return;
1270 		}
1271 	}
1272 }
1273 
1274 static void irq_target_callback(struct fw_iso_context *context, u32 tstamp, size_t header_length,
1275 				void *header, void *private_data);
1276 
1277 static void process_rx_packets_intermediately(struct fw_iso_context *context, u32 tstamp,
1278 					size_t header_length, void *header, void *private_data)
1279 {
1280 	struct amdtp_stream *s = private_data;
1281 	struct amdtp_domain *d = s->domain;
1282 	__be32 *ctx_header = header;
1283 	const unsigned int queue_size = s->queue_size;
1284 	unsigned int packets;
1285 	unsigned int offset;
1286 
1287 	if (s->packet_index < 0)
1288 		return;
1289 
1290 	packets = header_length / sizeof(*ctx_header);
1291 
1292 	offset = 0;
1293 	while (offset < packets) {
1294 		unsigned int cycle = compute_ohci_it_cycle(ctx_header[offset], queue_size);
1295 
1296 		if (compare_ohci_cycle_count(cycle, d->processing_cycle.rx_start) >= 0)
1297 			break;
1298 
1299 		++offset;
1300 	}
1301 
1302 	if (offset > 0) {
1303 		unsigned int length = sizeof(*ctx_header) * offset;
1304 
1305 		skip_rx_packets(context, tstamp, length, ctx_header, private_data);
1306 		if (amdtp_streaming_error(s))
1307 			return;
1308 
1309 		ctx_header += offset;
1310 		header_length -= length;
1311 	}
1312 
1313 	if (offset < packets) {
1314 		s->ready_processing = true;
1315 		wake_up(&s->ready_wait);
1316 
1317 		if (d->replay.enable)
1318 			s->ctx_data.rx.cache_pos = 0;
1319 
1320 		process_rx_packets(context, tstamp, header_length, ctx_header, private_data);
1321 		if (amdtp_streaming_error(s))
1322 			return;
1323 
1324 		if (s == d->irq_target)
1325 			s->context->callback.sc = irq_target_callback;
1326 		else
1327 			s->context->callback.sc = process_rx_packets;
1328 	}
1329 }
1330 
1331 static void process_tx_packets(struct fw_iso_context *context, u32 tstamp, size_t header_length,
1332 			       void *header, void *private_data)
1333 {
1334 	struct amdtp_stream *s = private_data;
1335 	__be32 *ctx_header = header;
1336 	struct pkt_desc *desc = s->packet_descs_cursor;
1337 	unsigned int packet_count;
1338 	unsigned int desc_count;
1339 	int i;
1340 	int err;
1341 
1342 	if (s->packet_index < 0)
1343 		return;
1344 
1345 	// Calculate the number of packets in buffer and check XRUN.
1346 	packet_count = header_length / s->ctx_data.tx.ctx_header_size;
1347 
1348 	desc_count = 0;
1349 	err = generate_tx_packet_descs(s, desc, ctx_header, packet_count, &desc_count);
1350 	if (err < 0) {
1351 		if (err != -EAGAIN) {
1352 			cancel_stream(s);
1353 			return;
1354 		}
1355 	} else {
1356 		struct amdtp_domain *d = s->domain;
1357 
1358 		process_ctx_payloads(s, desc, desc_count);
1359 
1360 		if (d->replay.enable)
1361 			cache_seq(s, desc, desc_count);
1362 
1363 		for (i = 0; i < desc_count; ++i)
1364 			desc = amdtp_stream_next_packet_desc(s, desc);
1365 		s->packet_descs_cursor = desc;
1366 	}
1367 
1368 	for (i = 0; i < packet_count; ++i) {
1369 		struct fw_iso_packet params = {0};
1370 
1371 		if (queue_in_packet(s, &params) < 0) {
1372 			cancel_stream(s);
1373 			return;
1374 		}
1375 	}
1376 }
1377 
1378 static void drop_tx_packets(struct fw_iso_context *context, u32 tstamp, size_t header_length,
1379 			    void *header, void *private_data)
1380 {
1381 	struct amdtp_stream *s = private_data;
1382 	const __be32 *ctx_header = header;
1383 	unsigned int packets;
1384 	unsigned int cycle;
1385 	int i;
1386 
1387 	if (s->packet_index < 0)
1388 		return;
1389 
1390 	packets = header_length / s->ctx_data.tx.ctx_header_size;
1391 
1392 	ctx_header += (packets - 1) * s->ctx_data.tx.ctx_header_size / sizeof(*ctx_header);
1393 	cycle = compute_ohci_cycle_count(ctx_header[1]);
1394 	s->next_cycle = increment_ohci_cycle_count(cycle, 1);
1395 
1396 	for (i = 0; i < packets; ++i) {
1397 		struct fw_iso_packet params = {0};
1398 
1399 		if (queue_in_packet(s, &params) < 0) {
1400 			cancel_stream(s);
1401 			return;
1402 		}
1403 	}
1404 }
1405 
1406 static void process_tx_packets_intermediately(struct fw_iso_context *context, u32 tstamp,
1407 					size_t header_length, void *header, void *private_data)
1408 {
1409 	struct amdtp_stream *s = private_data;
1410 	struct amdtp_domain *d = s->domain;
1411 	__be32 *ctx_header;
1412 	unsigned int packets;
1413 	unsigned int offset;
1414 
1415 	if (s->packet_index < 0)
1416 		return;
1417 
1418 	packets = header_length / s->ctx_data.tx.ctx_header_size;
1419 
1420 	offset = 0;
1421 	ctx_header = header;
1422 	while (offset < packets) {
1423 		unsigned int cycle = compute_ohci_cycle_count(ctx_header[1]);
1424 
1425 		if (compare_ohci_cycle_count(cycle, d->processing_cycle.tx_start) >= 0)
1426 			break;
1427 
1428 		ctx_header += s->ctx_data.tx.ctx_header_size / sizeof(__be32);
1429 		++offset;
1430 	}
1431 
1432 	ctx_header = header;
1433 
1434 	if (offset > 0) {
1435 		size_t length = s->ctx_data.tx.ctx_header_size * offset;
1436 
1437 		drop_tx_packets(context, tstamp, length, ctx_header, s);
1438 		if (amdtp_streaming_error(s))
1439 			return;
1440 
1441 		ctx_header += length / sizeof(*ctx_header);
1442 		header_length -= length;
1443 	}
1444 
1445 	if (offset < packets) {
1446 		s->ready_processing = true;
1447 		wake_up(&s->ready_wait);
1448 
1449 		process_tx_packets(context, tstamp, header_length, ctx_header, s);
1450 		if (amdtp_streaming_error(s))
1451 			return;
1452 
1453 		context->callback.sc = process_tx_packets;
1454 	}
1455 }
1456 
1457 static void drop_tx_packets_initially(struct fw_iso_context *context, u32 tstamp,
1458 				      size_t header_length, void *header, void *private_data)
1459 {
1460 	struct amdtp_stream *s = private_data;
1461 	struct amdtp_domain *d = s->domain;
1462 	__be32 *ctx_header;
1463 	unsigned int count;
1464 	unsigned int events;
1465 	int i;
1466 
1467 	if (s->packet_index < 0)
1468 		return;
1469 
1470 	count = header_length / s->ctx_data.tx.ctx_header_size;
1471 
1472 	// Attempt to detect any event in the batch of packets.
1473 	events = 0;
1474 	ctx_header = header;
1475 	for (i = 0; i < count; ++i) {
1476 		unsigned int payload_quads =
1477 			(be32_to_cpu(*ctx_header) >> ISO_DATA_LENGTH_SHIFT) / sizeof(__be32);
1478 		unsigned int data_blocks;
1479 
1480 		if (s->flags & CIP_NO_HEADER) {
1481 			data_blocks = payload_quads / s->data_block_quadlets;
1482 		} else {
1483 			__be32 *cip_headers = ctx_header + IR_CTX_HEADER_DEFAULT_QUADLETS;
1484 
1485 			if (payload_quads < CIP_HEADER_QUADLETS) {
1486 				data_blocks = 0;
1487 			} else {
1488 				payload_quads -= CIP_HEADER_QUADLETS;
1489 
1490 				if (s->flags & CIP_UNAWARE_SYT) {
1491 					data_blocks = payload_quads / s->data_block_quadlets;
1492 				} else {
1493 					u32 cip1 = be32_to_cpu(cip_headers[1]);
1494 
1495 					// NODATA packet can includes any data blocks but they are
1496 					// not available as event.
1497 					if ((cip1 & CIP_NO_DATA) == CIP_NO_DATA)
1498 						data_blocks = 0;
1499 					else
1500 						data_blocks = payload_quads / s->data_block_quadlets;
1501 				}
1502 			}
1503 		}
1504 
1505 		events += data_blocks;
1506 
1507 		ctx_header += s->ctx_data.tx.ctx_header_size / sizeof(__be32);
1508 	}
1509 
1510 	drop_tx_packets(context, tstamp, header_length, header, s);
1511 
1512 	if (events > 0)
1513 		s->ctx_data.tx.event_starts = true;
1514 
1515 	// Decide the cycle count to begin processing content of packet in IR contexts.
1516 	{
1517 		unsigned int stream_count = 0;
1518 		unsigned int event_starts_count = 0;
1519 		unsigned int cycle = UINT_MAX;
1520 
1521 		list_for_each_entry(s, &d->streams, list) {
1522 			if (s->direction == AMDTP_IN_STREAM) {
1523 				++stream_count;
1524 				if (s->ctx_data.tx.event_starts)
1525 					++event_starts_count;
1526 			}
1527 		}
1528 
1529 		if (stream_count == event_starts_count) {
1530 			unsigned int next_cycle;
1531 
1532 			list_for_each_entry(s, &d->streams, list) {
1533 				if (s->direction != AMDTP_IN_STREAM)
1534 					continue;
1535 
1536 				next_cycle = increment_ohci_cycle_count(s->next_cycle,
1537 								d->processing_cycle.tx_init_skip);
1538 				if (cycle == UINT_MAX ||
1539 				    compare_ohci_cycle_count(next_cycle, cycle) > 0)
1540 					cycle = next_cycle;
1541 
1542 				s->context->callback.sc = process_tx_packets_intermediately;
1543 			}
1544 
1545 			d->processing_cycle.tx_start = cycle;
1546 		}
1547 	}
1548 }
1549 
1550 static void process_ctxs_in_domain(struct amdtp_domain *d)
1551 {
1552 	struct amdtp_stream *s;
1553 
1554 	list_for_each_entry(s, &d->streams, list) {
1555 		if (s != d->irq_target && amdtp_stream_running(s))
1556 			fw_iso_context_flush_completions(s->context);
1557 
1558 		if (amdtp_streaming_error(s))
1559 			goto error;
1560 	}
1561 
1562 	return;
1563 error:
1564 	if (amdtp_stream_running(d->irq_target))
1565 		cancel_stream(d->irq_target);
1566 
1567 	list_for_each_entry(s, &d->streams, list) {
1568 		if (amdtp_stream_running(s))
1569 			cancel_stream(s);
1570 	}
1571 }
1572 
1573 static void irq_target_callback(struct fw_iso_context *context, u32 tstamp, size_t header_length,
1574 				void *header, void *private_data)
1575 {
1576 	struct amdtp_stream *s = private_data;
1577 	struct amdtp_domain *d = s->domain;
1578 
1579 	process_rx_packets(context, tstamp, header_length, header, private_data);
1580 	process_ctxs_in_domain(d);
1581 }
1582 
1583 static void irq_target_callback_intermediately(struct fw_iso_context *context, u32 tstamp,
1584 					size_t header_length, void *header, void *private_data)
1585 {
1586 	struct amdtp_stream *s = private_data;
1587 	struct amdtp_domain *d = s->domain;
1588 
1589 	process_rx_packets_intermediately(context, tstamp, header_length, header, private_data);
1590 	process_ctxs_in_domain(d);
1591 }
1592 
1593 static void irq_target_callback_skip(struct fw_iso_context *context, u32 tstamp,
1594 				     size_t header_length, void *header, void *private_data)
1595 {
1596 	struct amdtp_stream *s = private_data;
1597 	struct amdtp_domain *d = s->domain;
1598 	bool ready_to_start;
1599 
1600 	skip_rx_packets(context, tstamp, header_length, header, private_data);
1601 	process_ctxs_in_domain(d);
1602 
1603 	if (d->replay.enable && !d->replay.on_the_fly) {
1604 		unsigned int rx_count = 0;
1605 		unsigned int rx_ready_count = 0;
1606 		struct amdtp_stream *rx;
1607 
1608 		list_for_each_entry(rx, &d->streams, list) {
1609 			struct amdtp_stream *tx;
1610 			unsigned int cached_cycles;
1611 
1612 			if (rx->direction != AMDTP_OUT_STREAM)
1613 				continue;
1614 			++rx_count;
1615 
1616 			tx = rx->ctx_data.rx.replay_target;
1617 			cached_cycles = calculate_cached_cycle_count(tx, 0);
1618 			if (cached_cycles > tx->ctx_data.tx.cache.size / 2)
1619 				++rx_ready_count;
1620 		}
1621 
1622 		ready_to_start = (rx_count == rx_ready_count);
1623 	} else {
1624 		ready_to_start = true;
1625 	}
1626 
1627 	// Decide the cycle count to begin processing content of packet in IT contexts. All of IT
1628 	// contexts are expected to start and get callback when reaching here.
1629 	if (ready_to_start) {
1630 		unsigned int cycle = s->next_cycle;
1631 		list_for_each_entry(s, &d->streams, list) {
1632 			if (s->direction != AMDTP_OUT_STREAM)
1633 				continue;
1634 
1635 			if (compare_ohci_cycle_count(s->next_cycle, cycle) > 0)
1636 				cycle = s->next_cycle;
1637 
1638 			if (s == d->irq_target)
1639 				s->context->callback.sc = irq_target_callback_intermediately;
1640 			else
1641 				s->context->callback.sc = process_rx_packets_intermediately;
1642 		}
1643 
1644 		d->processing_cycle.rx_start = cycle;
1645 	}
1646 }
1647 
1648 // This is executed one time. For in-stream, first packet has come. For out-stream, prepared to
1649 // transmit first packet.
1650 static void amdtp_stream_first_callback(struct fw_iso_context *context,
1651 					u32 tstamp, size_t header_length,
1652 					void *header, void *private_data)
1653 {
1654 	struct amdtp_stream *s = private_data;
1655 	struct amdtp_domain *d = s->domain;
1656 
1657 	if (s->direction == AMDTP_IN_STREAM) {
1658 		context->callback.sc = drop_tx_packets_initially;
1659 	} else {
1660 		if (s == d->irq_target)
1661 			context->callback.sc = irq_target_callback_skip;
1662 		else
1663 			context->callback.sc = skip_rx_packets;
1664 	}
1665 
1666 	context->callback.sc(context, tstamp, header_length, header, s);
1667 }
1668 
1669 /**
1670  * amdtp_stream_start - start transferring packets
1671  * @s: the AMDTP stream to start
1672  * @channel: the isochronous channel on the bus
1673  * @speed: firewire speed code
1674  * @queue_size: The number of packets in the queue.
1675  * @idle_irq_interval: the interval to queue packet during initial state.
1676  *
1677  * The stream cannot be started until it has been configured with
1678  * amdtp_stream_set_parameters() and it must be started before any PCM or MIDI
1679  * device can be started.
1680  */
1681 static int amdtp_stream_start(struct amdtp_stream *s, int channel, int speed,
1682 			      unsigned int queue_size, unsigned int idle_irq_interval)
1683 {
1684 	bool is_irq_target = (s == s->domain->irq_target);
1685 	unsigned int ctx_header_size;
1686 	unsigned int max_ctx_payload_size;
1687 	enum dma_data_direction dir;
1688 	struct pkt_desc *descs;
1689 	int i, type, tag, err;
1690 
1691 	guard(mutex)(&s->mutex);
1692 
1693 	if (WARN_ON(amdtp_stream_running(s) ||
1694 		    (s->data_block_quadlets < 1)))
1695 		return -EBADFD;
1696 
1697 	if (s->direction == AMDTP_IN_STREAM) {
1698 		// NOTE: IT context should be used for constant IRQ.
1699 		if (is_irq_target)
1700 			return -EINVAL;
1701 
1702 		s->data_block_counter = UINT_MAX;
1703 	} else {
1704 		s->data_block_counter = 0;
1705 	}
1706 
1707 	// initialize packet buffer.
1708 	if (s->direction == AMDTP_IN_STREAM) {
1709 		dir = DMA_FROM_DEVICE;
1710 		type = FW_ISO_CONTEXT_RECEIVE;
1711 		if (!(s->flags & CIP_NO_HEADER))
1712 			ctx_header_size = IR_CTX_HEADER_SIZE_CIP;
1713 		else
1714 			ctx_header_size = IR_CTX_HEADER_SIZE_NO_CIP;
1715 	} else {
1716 		dir = DMA_TO_DEVICE;
1717 		type = FW_ISO_CONTEXT_TRANSMIT;
1718 		ctx_header_size = 0;	// No effect for IT context.
1719 	}
1720 	max_ctx_payload_size = amdtp_stream_get_max_ctx_payload_size(s);
1721 
1722 	err = iso_packets_buffer_init(&s->buffer, s->unit, queue_size, max_ctx_payload_size, dir);
1723 	if (err < 0)
1724 		return err;
1725 	s->queue_size = queue_size;
1726 
1727 	s->context = fw_iso_context_create(fw_parent_device(s->unit)->card,
1728 					  type, channel, speed, ctx_header_size,
1729 					  amdtp_stream_first_callback, s);
1730 	if (IS_ERR(s->context)) {
1731 		err = PTR_ERR(s->context);
1732 		if (err == -EBUSY)
1733 			dev_err(&s->unit->device,
1734 				"no free stream on this controller\n");
1735 		goto err_buffer;
1736 	}
1737 
1738 	amdtp_stream_update(s);
1739 
1740 	if (s->direction == AMDTP_IN_STREAM) {
1741 		s->ctx_data.tx.max_ctx_payload_length = max_ctx_payload_size;
1742 		s->ctx_data.tx.ctx_header_size = ctx_header_size;
1743 		s->ctx_data.tx.event_starts = false;
1744 
1745 		if (s->domain->replay.enable) {
1746 			// struct fw_iso_context.drop_overflow_headers is false therefore it's
1747 			// possible to cache much unexpectedly.
1748 			s->ctx_data.tx.cache.size = max_t(unsigned int, s->syt_interval * 2,
1749 							  queue_size * 3 / 2);
1750 			s->ctx_data.tx.cache.pos = 0;
1751 			s->ctx_data.tx.cache.descs = kcalloc(s->ctx_data.tx.cache.size,
1752 						sizeof(*s->ctx_data.tx.cache.descs), GFP_KERNEL);
1753 			if (!s->ctx_data.tx.cache.descs) {
1754 				err = -ENOMEM;
1755 				goto err_context;
1756 			}
1757 		}
1758 	} else {
1759 		static const struct {
1760 			unsigned int data_block;
1761 			unsigned int syt_offset;
1762 		} *entry, initial_state[] = {
1763 			[CIP_SFC_32000]  = {  4, 3072 },
1764 			[CIP_SFC_48000]  = {  6, 1024 },
1765 			[CIP_SFC_96000]  = { 12, 1024 },
1766 			[CIP_SFC_192000] = { 24, 1024 },
1767 			[CIP_SFC_44100]  = {  0,   67 },
1768 			[CIP_SFC_88200]  = {  0,   67 },
1769 			[CIP_SFC_176400] = {  0,   67 },
1770 		};
1771 
1772 		s->ctx_data.rx.seq.descs = kcalloc(queue_size, sizeof(*s->ctx_data.rx.seq.descs), GFP_KERNEL);
1773 		if (!s->ctx_data.rx.seq.descs) {
1774 			err = -ENOMEM;
1775 			goto err_context;
1776 		}
1777 		s->ctx_data.rx.seq.size = queue_size;
1778 		s->ctx_data.rx.seq.pos = 0;
1779 
1780 		entry = &initial_state[s->sfc];
1781 		s->ctx_data.rx.data_block_state = entry->data_block;
1782 		s->ctx_data.rx.syt_offset_state = entry->syt_offset;
1783 		s->ctx_data.rx.last_syt_offset = TICKS_PER_CYCLE;
1784 
1785 		s->ctx_data.rx.event_count = 0;
1786 	}
1787 
1788 	if (s->flags & CIP_NO_HEADER)
1789 		s->tag = TAG_NO_CIP_HEADER;
1790 	else
1791 		s->tag = TAG_CIP;
1792 
1793 	// NOTE: When operating without hardIRQ/softIRQ, applications tends to call ioctl request
1794 	// for runtime of PCM substream in the interval equivalent to the size of PCM buffer. It
1795 	// could take a round over queue of AMDTP packet descriptors and small loss of history. For
1796 	// safe, keep more 8 elements for the queue, equivalent to 1 ms.
1797 	descs = kcalloc(s->queue_size + 8, sizeof(*descs), GFP_KERNEL);
1798 	if (!descs) {
1799 		err = -ENOMEM;
1800 		goto err_context;
1801 	}
1802 	s->packet_descs = descs;
1803 
1804 	INIT_LIST_HEAD(&s->packet_descs_list);
1805 	for (i = 0; i < s->queue_size; ++i) {
1806 		INIT_LIST_HEAD(&descs->link);
1807 		list_add_tail(&descs->link, &s->packet_descs_list);
1808 		++descs;
1809 	}
1810 	s->packet_descs_cursor = list_first_entry(&s->packet_descs_list, struct pkt_desc, link);
1811 
1812 	s->packet_index = 0;
1813 	do {
1814 		struct fw_iso_packet params;
1815 
1816 		if (s->direction == AMDTP_IN_STREAM) {
1817 			err = queue_in_packet(s, &params);
1818 		} else {
1819 			bool sched_irq = false;
1820 
1821 			params.header_length = 0;
1822 			params.payload_length = 0;
1823 
1824 			if (is_irq_target) {
1825 				sched_irq = !((s->packet_index + 1) %
1826 					      idle_irq_interval);
1827 			}
1828 
1829 			err = queue_out_packet(s, &params, sched_irq);
1830 		}
1831 		if (err < 0)
1832 			goto err_pkt_descs;
1833 	} while (s->packet_index > 0);
1834 
1835 	/* NOTE: TAG1 matches CIP. This just affects in stream. */
1836 	tag = FW_ISO_CONTEXT_MATCH_TAG1;
1837 	if ((s->flags & CIP_EMPTY_WITH_TAG0) || (s->flags & CIP_NO_HEADER))
1838 		tag |= FW_ISO_CONTEXT_MATCH_TAG0;
1839 
1840 	s->ready_processing = false;
1841 	err = fw_iso_context_start(s->context, -1, 0, tag);
1842 	if (err < 0)
1843 		goto err_pkt_descs;
1844 
1845 	return 0;
1846 err_pkt_descs:
1847 	kfree(s->packet_descs);
1848 	s->packet_descs = NULL;
1849 err_context:
1850 	if (s->direction == AMDTP_OUT_STREAM) {
1851 		kfree(s->ctx_data.rx.seq.descs);
1852 	} else {
1853 		if (s->domain->replay.enable)
1854 			kfree(s->ctx_data.tx.cache.descs);
1855 	}
1856 	fw_iso_context_destroy(s->context);
1857 	s->context = ERR_PTR(-1);
1858 err_buffer:
1859 	iso_packets_buffer_destroy(&s->buffer, s->unit);
1860 
1861 	return err;
1862 }
1863 
1864 /**
1865  * amdtp_domain_stream_pcm_pointer - get the PCM buffer position
1866  * @d: the AMDTP domain.
1867  * @s: the AMDTP stream that transports the PCM data
1868  *
1869  * Returns the current buffer position, in frames.
1870  */
1871 unsigned long amdtp_domain_stream_pcm_pointer(struct amdtp_domain *d,
1872 					      struct amdtp_stream *s)
1873 {
1874 	struct amdtp_stream *irq_target = d->irq_target;
1875 
1876 	if (irq_target && amdtp_stream_running(irq_target)) {
1877 		// The work item to call snd_pcm_period_elapsed() can reach here by the call of
1878 		// snd_pcm_ops.pointer(), however less packets would be available then. Therefore
1879 		// the following call is just for user process contexts.
1880 		if (current_work() != &s->period_work)
1881 			fw_iso_context_flush_completions(irq_target->context);
1882 	}
1883 
1884 	return READ_ONCE(s->pcm_buffer_pointer);
1885 }
1886 EXPORT_SYMBOL_GPL(amdtp_domain_stream_pcm_pointer);
1887 
1888 /**
1889  * amdtp_domain_stream_pcm_ack - acknowledge queued PCM frames
1890  * @d: the AMDTP domain.
1891  * @s: the AMDTP stream that transfers the PCM frames
1892  *
1893  * Returns zero always.
1894  */
1895 int amdtp_domain_stream_pcm_ack(struct amdtp_domain *d, struct amdtp_stream *s)
1896 {
1897 	struct amdtp_stream *irq_target = d->irq_target;
1898 
1899 	// Process isochronous packets for recent isochronous cycle to handle
1900 	// queued PCM frames.
1901 	if (irq_target && amdtp_stream_running(irq_target))
1902 		fw_iso_context_flush_completions(irq_target->context);
1903 
1904 	return 0;
1905 }
1906 EXPORT_SYMBOL_GPL(amdtp_domain_stream_pcm_ack);
1907 
1908 /**
1909  * amdtp_stream_update - update the stream after a bus reset
1910  * @s: the AMDTP stream
1911  */
1912 void amdtp_stream_update(struct amdtp_stream *s)
1913 {
1914 	/* Precomputing. */
1915 	WRITE_ONCE(s->source_node_id_field,
1916                    (fw_parent_device(s->unit)->card->node_id << CIP_SID_SHIFT) & CIP_SID_MASK);
1917 }
1918 EXPORT_SYMBOL(amdtp_stream_update);
1919 
1920 /**
1921  * amdtp_stream_stop - stop sending packets
1922  * @s: the AMDTP stream to stop
1923  *
1924  * All PCM and MIDI devices of the stream must be stopped before the stream
1925  * itself can be stopped.
1926  */
1927 static void amdtp_stream_stop(struct amdtp_stream *s)
1928 {
1929 	guard(mutex)(&s->mutex);
1930 
1931 	if (!amdtp_stream_running(s))
1932 		return;
1933 
1934 	cancel_work_sync(&s->period_work);
1935 	fw_iso_context_stop(s->context);
1936 	fw_iso_context_destroy(s->context);
1937 	s->context = ERR_PTR(-1);
1938 	iso_packets_buffer_destroy(&s->buffer, s->unit);
1939 	kfree(s->packet_descs);
1940 	s->packet_descs = NULL;
1941 
1942 	if (s->direction == AMDTP_OUT_STREAM) {
1943 		kfree(s->ctx_data.rx.seq.descs);
1944 	} else {
1945 		if (s->domain->replay.enable)
1946 			kfree(s->ctx_data.tx.cache.descs);
1947 	}
1948 }
1949 
1950 /**
1951  * amdtp_stream_pcm_abort - abort the running PCM device
1952  * @s: the AMDTP stream about to be stopped
1953  *
1954  * If the isochronous stream needs to be stopped asynchronously, call this
1955  * function first to stop the PCM device.
1956  */
1957 void amdtp_stream_pcm_abort(struct amdtp_stream *s)
1958 {
1959 	struct snd_pcm_substream *pcm;
1960 
1961 	pcm = READ_ONCE(s->pcm);
1962 	if (pcm)
1963 		snd_pcm_stop_xrun(pcm);
1964 }
1965 EXPORT_SYMBOL(amdtp_stream_pcm_abort);
1966 
1967 /**
1968  * amdtp_domain_init - initialize an AMDTP domain structure
1969  * @d: the AMDTP domain to initialize.
1970  */
1971 int amdtp_domain_init(struct amdtp_domain *d)
1972 {
1973 	INIT_LIST_HEAD(&d->streams);
1974 
1975 	d->events_per_period = 0;
1976 
1977 	return 0;
1978 }
1979 EXPORT_SYMBOL_GPL(amdtp_domain_init);
1980 
1981 /**
1982  * amdtp_domain_destroy - destroy an AMDTP domain structure
1983  * @d: the AMDTP domain to destroy.
1984  */
1985 void amdtp_domain_destroy(struct amdtp_domain *d)
1986 {
1987 	// At present nothing to do.
1988 	return;
1989 }
1990 EXPORT_SYMBOL_GPL(amdtp_domain_destroy);
1991 
1992 /**
1993  * amdtp_domain_add_stream - register isoc context into the domain.
1994  * @d: the AMDTP domain.
1995  * @s: the AMDTP stream.
1996  * @channel: the isochronous channel on the bus.
1997  * @speed: firewire speed code.
1998  */
1999 int amdtp_domain_add_stream(struct amdtp_domain *d, struct amdtp_stream *s,
2000 			    int channel, int speed)
2001 {
2002 	struct amdtp_stream *tmp;
2003 
2004 	list_for_each_entry(tmp, &d->streams, list) {
2005 		if (s == tmp)
2006 			return -EBUSY;
2007 	}
2008 
2009 	list_add(&s->list, &d->streams);
2010 
2011 	s->channel = channel;
2012 	s->speed = speed;
2013 	s->domain = d;
2014 
2015 	return 0;
2016 }
2017 EXPORT_SYMBOL_GPL(amdtp_domain_add_stream);
2018 
2019 // Make the reference from rx stream to tx stream for sequence replay. When the number of tx streams
2020 // is less than the number of rx streams, the first tx stream is selected.
2021 static int make_association(struct amdtp_domain *d)
2022 {
2023 	unsigned int dst_index = 0;
2024 	struct amdtp_stream *rx;
2025 
2026 	// Make association to replay target.
2027 	list_for_each_entry(rx, &d->streams, list) {
2028 		if (rx->direction == AMDTP_OUT_STREAM) {
2029 			unsigned int src_index = 0;
2030 			struct amdtp_stream *tx = NULL;
2031 			struct amdtp_stream *s;
2032 
2033 			list_for_each_entry(s, &d->streams, list) {
2034 				if (s->direction == AMDTP_IN_STREAM) {
2035 					if (dst_index == src_index) {
2036 						tx = s;
2037 						break;
2038 					}
2039 
2040 					++src_index;
2041 				}
2042 			}
2043 			if (!tx) {
2044 				// Select the first entry.
2045 				list_for_each_entry(s, &d->streams, list) {
2046 					if (s->direction == AMDTP_IN_STREAM) {
2047 						tx = s;
2048 						break;
2049 					}
2050 				}
2051 				// No target is available to replay sequence.
2052 				if (!tx)
2053 					return -EINVAL;
2054 			}
2055 
2056 			rx->ctx_data.rx.replay_target = tx;
2057 
2058 			++dst_index;
2059 		}
2060 	}
2061 
2062 	return 0;
2063 }
2064 
2065 /**
2066  * amdtp_domain_start - start sending packets for isoc context in the domain.
2067  * @d: the AMDTP domain.
2068  * @tx_init_skip_cycles: the number of cycles to skip processing packets at initial stage of IR
2069  *			 contexts.
2070  * @replay_seq: whether to replay the sequence of packet in IR context for the sequence of packet in
2071  *		IT context.
2072  * @replay_on_the_fly: transfer rx packets according to nominal frequency, then begin to replay
2073  *		       according to arrival of events in tx packets.
2074  */
2075 int amdtp_domain_start(struct amdtp_domain *d, unsigned int tx_init_skip_cycles, bool replay_seq,
2076 		       bool replay_on_the_fly)
2077 {
2078 	unsigned int events_per_buffer = d->events_per_buffer;
2079 	unsigned int events_per_period = d->events_per_period;
2080 	unsigned int queue_size;
2081 	struct amdtp_stream *s;
2082 	bool found = false;
2083 	int err;
2084 
2085 	if (replay_seq) {
2086 		err = make_association(d);
2087 		if (err < 0)
2088 			return err;
2089 	}
2090 	d->replay.enable = replay_seq;
2091 	d->replay.on_the_fly = replay_on_the_fly;
2092 
2093 	// Select an IT context as IRQ target.
2094 	list_for_each_entry(s, &d->streams, list) {
2095 		if (s->direction == AMDTP_OUT_STREAM) {
2096 			found = true;
2097 			break;
2098 		}
2099 	}
2100 	if (!found)
2101 		return -ENXIO;
2102 	d->irq_target = s;
2103 
2104 	d->processing_cycle.tx_init_skip = tx_init_skip_cycles;
2105 
2106 	// This is a case that AMDTP streams in domain run just for MIDI
2107 	// substream. Use the number of events equivalent to 10 msec as
2108 	// interval of hardware IRQ.
2109 	if (events_per_period == 0)
2110 		events_per_period = amdtp_rate_table[d->irq_target->sfc] / 100;
2111 	if (events_per_buffer == 0)
2112 		events_per_buffer = events_per_period * 3;
2113 
2114 	queue_size = DIV_ROUND_UP(CYCLES_PER_SECOND * events_per_buffer,
2115 				  amdtp_rate_table[d->irq_target->sfc]);
2116 
2117 	list_for_each_entry(s, &d->streams, list) {
2118 		unsigned int idle_irq_interval = 0;
2119 
2120 		if (s->direction == AMDTP_OUT_STREAM && s == d->irq_target) {
2121 			idle_irq_interval = DIV_ROUND_UP(CYCLES_PER_SECOND * events_per_period,
2122 							 amdtp_rate_table[d->irq_target->sfc]);
2123 		}
2124 
2125 		// Starts immediately but actually DMA context starts several hundred cycles later.
2126 		err = amdtp_stream_start(s, s->channel, s->speed, queue_size, idle_irq_interval);
2127 		if (err < 0)
2128 			goto error;
2129 	}
2130 
2131 	return 0;
2132 error:
2133 	list_for_each_entry(s, &d->streams, list)
2134 		amdtp_stream_stop(s);
2135 	return err;
2136 }
2137 EXPORT_SYMBOL_GPL(amdtp_domain_start);
2138 
2139 /**
2140  * amdtp_domain_stop - stop sending packets for isoc context in the same domain.
2141  * @d: the AMDTP domain to which the isoc contexts belong.
2142  */
2143 void amdtp_domain_stop(struct amdtp_domain *d)
2144 {
2145 	struct amdtp_stream *s, *next;
2146 
2147 	if (d->irq_target)
2148 		amdtp_stream_stop(d->irq_target);
2149 
2150 	list_for_each_entry_safe(s, next, &d->streams, list) {
2151 		list_del(&s->list);
2152 
2153 		if (s != d->irq_target)
2154 			amdtp_stream_stop(s);
2155 	}
2156 
2157 	d->events_per_period = 0;
2158 	d->irq_target = NULL;
2159 }
2160 EXPORT_SYMBOL_GPL(amdtp_domain_stop);
2161