113 static int fence_chains_init(struct fence_chains *fc, unsigned int count,  in fence_chains_init()  argument
119 	fc->chains = kvmalloc_array(count, sizeof(*fc->chains),  in fence_chains_init()
121 	if (!fc->chains)  in fence_chains_init()
124 	fc->fences = kvmalloc_array(count, sizeof(*fc->fences),  in fence_chains_init()
126 	if (!fc->fences) {  in fence_chains_init()
131 	fc->tail = NULL;  in fence_chains_init()
133 		fc->fences[i] = mock_fence();  in fence_chains_init()
134 		if (!fc->fences[i]) {  in fence_chains_init()
139 		fc->chains[i] = mock_chain(fc->tail,  in fence_chains_init()
140 					   fc->fences[i],  in fence_chains_init()
142 		if (!fc->chains[i]) {  in fence_chains_init()
147 		fc->tail = fc->chains[i];  in fence_chains_init()
149 		dma_fence_enable_sw_signaling(fc->chains[i]);  in fence_chains_init()
152 	fc->chain_length = i;  in fence_chains_init()
157 		dma_fence_put(fc->fences[i]);  in fence_chains_init()
158 		dma_fence_put(fc->chains[i]);  in fence_chains_init()
160 	kvfree(fc->fences);  in fence_chains_init()
162 	kvfree(fc->chains);  in fence_chains_init()
166 static void fence_chains_fini(struct fence_chains *fc)  in fence_chains_fini()  argument
170 	for (i = 0; i < fc->chain_length; i++) {  in fence_chains_fini()
171 		dma_fence_signal(fc->fences[i]);  in fence_chains_fini()
172 		dma_fence_put(fc->fences[i]);  in fence_chains_fini()
174 	kvfree(fc->fences);  in fence_chains_fini()
176 	for (i = 0; i < fc->chain_length; i++)  in fence_chains_fini()
177 		dma_fence_put(fc->chains[i]);  in fence_chains_fini()
178 	kvfree(fc->chains);  in fence_chains_fini()
183 	struct fence_chains fc;  in find_seqno()  local
188 	err = fence_chains_init(&fc, 64, seqno_inc);  in find_seqno()
192 	fence = dma_fence_get(fc.tail);  in find_seqno()
200 	for (i = 0; i < fc.chain_length; i++) {  in find_seqno()
201 		fence = dma_fence_get(fc.tail);  in find_seqno()
206 			       err, fc.chain_length + 1, i + 1);  in find_seqno()
209 		if (fence != fc.chains[i]) {  in find_seqno()
211 			       fc.chain_length + 1, i + 1);  in find_seqno()
223 		if (fence != fc.chains[i]) {  in find_seqno()
246 		if (i > 0 && fence != fc.chains[i - 1]) {  in find_seqno()
255 	fence_chains_fini(&fc);  in find_seqno()
261 	struct fence_chains fc;  in find_signaled()  local
265 	err = fence_chains_init(&fc, 2, seqno_inc);  in find_signaled()
269 	dma_fence_signal(fc.fences[0]);  in find_signaled()
271 	fence = dma_fence_get(fc.tail);  in find_signaled()
279 	if (fence && fence != fc.chains[0]) {  in find_signaled()
293 	fence_chains_fini(&fc);  in find_signaled()
299 	struct fence_chains fc;  in find_out_of_order()  local
303 	err = fence_chains_init(&fc, 3, seqno_inc);  in find_out_of_order()
307 	dma_fence_signal(fc.fences[1]);  in find_out_of_order()
309 	fence = dma_fence_get(fc.tail);  in find_out_of_order()
324 	if (fence != fc.chains[0]) {  in find_out_of_order()
332 	fence_chains_fini(&fc);  in find_out_of_order()
343 	struct fence_chains fc;  in find_gap()  local
348 	err = fence_chains_init(&fc, 64, seqno_inc2);  in find_gap()
352 	for (i = 0; i < fc.chain_length; i++) {  in find_gap()
353 		fence = dma_fence_get(fc.tail);  in find_gap()
358 			       err, fc.chain_length + 1, 2 * i + 1);  in find_gap()
361 		if (fence != fc.chains[i]) {  in find_gap()
364 			       fc.chain_length + 1,  in find_gap()
377 		if (fence != fc.chains[i]) {  in find_gap()
385 	fence_chains_fini(&fc);  in find_gap()
390 	struct fence_chains fc;  member
400 		struct dma_fence *fence = dma_fence_get(data->fc.tail);  in __find_race()
403 		seqno = get_random_u32_inclusive(1, data->fc.chain_length);  in __find_race()
432 		seqno = get_random_u32_below(data->fc.chain_length - 1);  in __find_race()
433 		dma_fence_signal(data->fc.fences[seqno]);  in __find_race()
451 	err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc);  in find_race()
486 	for (i = 0; i < data.fc.chain_length; i++)  in find_race()
487 		if (dma_fence_is_signaled(data.fc.fences[i]))  in find_race()
492 	fence_chains_fini(&data.fc);  in find_race()
498 	struct fence_chains fc;  in signal_forward()  local
502 	err = fence_chains_init(&fc, 64, seqno_inc);  in signal_forward()
506 	for (i = 0; i < fc.chain_length; i++) {  in signal_forward()
507 		dma_fence_signal(fc.fences[i]);  in signal_forward()
509 		if (!dma_fence_is_signaled(fc.chains[i])) {  in signal_forward()
515 		if (i + 1 < fc.chain_length &&  in signal_forward()
516 		    dma_fence_is_signaled(fc.chains[i + 1])) {  in signal_forward()
524 	fence_chains_fini(&fc);  in signal_forward()
530 	struct fence_chains fc;  in signal_backward()  local
534 	err = fence_chains_init(&fc, 64, seqno_inc);  in signal_backward()
538 	for (i = fc.chain_length; i--; ) {  in signal_backward()
539 		dma_fence_signal(fc.fences[i]);  in signal_backward()
541 		if (i > 0 && dma_fence_is_signaled(fc.chains[i])) {  in signal_backward()
548 	for (i = 0; i < fc.chain_length; i++) {  in signal_backward()
549 		if (!dma_fence_is_signaled(fc.chains[i])) {  in signal_backward()
557 	fence_chains_fini(&fc);  in signal_backward()
563 	struct fence_chains *fc = arg;  in __wait_fence_chains()  local
565 	if (dma_fence_wait(fc->tail, false))  in __wait_fence_chains()
573 	struct fence_chains fc;  in wait_forward()  local
578 	err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);  in wait_forward()
582 	tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");  in wait_forward()
590 	for (i = 0; i < fc.chain_length; i++)  in wait_forward()
591 		dma_fence_signal(fc.fences[i]);  in wait_forward()
596 	fence_chains_fini(&fc);  in wait_forward()
602 	struct fence_chains fc;  in wait_backward()  local
607 	err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);  in wait_backward()
611 	tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");  in wait_backward()
619 	for (i = fc.chain_length; i--; )  in wait_backward()
620 		dma_fence_signal(fc.fences[i]);  in wait_backward()
625 	fence_chains_fini(&fc);  in wait_backward()
629 static void randomise_fences(struct fence_chains *fc)  in randomise_fences()  argument
631 	unsigned int count = fc->chain_length;  in randomise_fences()
641 		swap(fc->fences[count], fc->fences[swp]);  in randomise_fences()
647 	struct fence_chains fc;  in wait_random()  local
652 	err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);  in wait_random()
656 	randomise_fences(&fc);  in wait_random()
658 	tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");  in wait_random()
666 	for (i = 0; i < fc.chain_length; i++)  in wait_random()
667 		dma_fence_signal(fc.fences[i]);  in wait_random()
672 	fence_chains_fini(&fc);  in wait_random()