256 		struct tbstream_frame *sf = &ring->frames[i];  in tbstream_ring_free()  local
258 		if (sf->frame.buffer_phy)  in tbstream_ring_free()
259 			dma_unmap_page(dma_dev, sf->frame.buffer_phy,  in tbstream_ring_free()
261 		sf->frame.buffer_phy = 0;  in tbstream_ring_free()
262 		if (sf->page)  in tbstream_ring_free()
263 			__free_page(sf->page);  in tbstream_ring_free()
264 		sf->page = NULL;  in tbstream_ring_free()
346 	struct tbstream_frame *sf = container_of(frame, typeof(*sf), frame);  in tbstream_dev_rx_callback()  local
347 	struct tbstream_dev *sdev = sf->sdev;  in tbstream_dev_rx_callback()
352 	sf->completed = true;  in tbstream_dev_rx_callback()
355 	if (sf->frame.flags & RING_DESC_CRC_ERROR)  in tbstream_dev_rx_callback()
357 	else if (sf->frame.flags & RING_DESC_BUFFER_OVERRUN)  in tbstream_dev_rx_callback()
367 	struct tbstream_frame *sf;  in tbstream_dev_completed_rx()  local
371 	sf = &sdev->rx_ring.frames[index];  in tbstream_dev_completed_rx()
372 	if (!sf->completed)  in tbstream_dev_completed_rx()
375 	dma_sync_single_for_cpu(dma_dev, sf->frame.buffer_phy,  in tbstream_dev_completed_rx()
376 				tb_ring_frame_size(&sf->frame),  in tbstream_dev_completed_rx()
378 	return sf;  in tbstream_dev_completed_rx()
384 	struct tbstream_frame *sf;  in tbstream_dev_consume_rx()  local
390 	sf = &sdev->rx_ring.frames[index];  in tbstream_dev_consume_rx()
391 	sf->completed = false;  in tbstream_dev_consume_rx()
392 	sf->offset = 0;  in tbstream_dev_consume_rx()
393 	sf->frame.size = 0;  in tbstream_dev_consume_rx()
395 	dma_sync_single_for_device(dma_dev, sf->frame.buffer_phy,  in tbstream_dev_consume_rx()
396 				   tb_ring_frame_size(&sf->frame),  in tbstream_dev_consume_rx()
399 	return tb_ring_rx(sdev->rx_ring.ring, &sf->frame);  in tbstream_dev_consume_rx()
414 		struct tbstream_frame *sf = &sdev->rx_ring.frames[i];  in tbstream_dev_alloc_rx_buffers()  local
417 		sf->page = alloc_page(GFP_KERNEL);  in tbstream_dev_alloc_rx_buffers()
418 		if (!sf->page)  in tbstream_dev_alloc_rx_buffers()
421 		dma_addr = dma_map_page(dma_dev, sf->page, 0, TB_MAX_FRAME_SIZE,  in tbstream_dev_alloc_rx_buffers()
424 			__free_page(sf->page);  in tbstream_dev_alloc_rx_buffers()
425 			sf->page = NULL;  in tbstream_dev_alloc_rx_buffers()
429 		sf->sdev = sdev;  in tbstream_dev_alloc_rx_buffers()
430 		sf->frame.callback = tbstream_dev_rx_callback;  in tbstream_dev_alloc_rx_buffers()
431 		sf->frame.buffer_phy = dma_addr;  in tbstream_dev_alloc_rx_buffers()
433 		tb_ring_rx(sdev->rx_ring.ring, &sf->frame);  in tbstream_dev_alloc_rx_buffers()
445 	struct tbstream_frame *sf = container_of(frame, typeof(*sf), frame);  in tbstream_dev_tx_callback()  local
446 	struct tbstream_dev *sdev = sf->sdev;  in tbstream_dev_tx_callback()
452 	if (sf->frame.eof == TBSTREAM_DATA)  in tbstream_dev_tx_callback()
468 		struct tbstream_frame *sf = &sdev->tx_ring.frames[i];  in tbstream_dev_alloc_tx_buffers()  local
471 		sf->page = alloc_page(GFP_KERNEL);  in tbstream_dev_alloc_tx_buffers()
472 		if (!sf->page)  in tbstream_dev_alloc_tx_buffers()
475 		dma_addr = dma_map_page(dma_dev, sf->page, 0, TB_MAX_FRAME_SIZE,  in tbstream_dev_alloc_tx_buffers()
478 			__free_page(sf->page);  in tbstream_dev_alloc_tx_buffers()
479 			sf->page = NULL;  in tbstream_dev_alloc_tx_buffers()
483 		sf->sdev = sdev;  in tbstream_dev_alloc_tx_buffers()
484 		sf->frame.callback = tbstream_dev_tx_callback;  in tbstream_dev_alloc_tx_buffers()
485 		sf->frame.buffer_phy = dma_addr;  in tbstream_dev_alloc_tx_buffers()
486 		sf->frame.sof = TBSTREAM_FRAME_START;  in tbstream_dev_alloc_tx_buffers()
499 	struct tbstream_frame *sf;  in tbstream_dev_alloc_tx()  local
508 	sf = &sdev->tx_ring.frames[index];  in tbstream_dev_alloc_tx()
509 	sf->frame.size = size < TB_MAX_FRAME_SIZE ? size : 0;  in tbstream_dev_alloc_tx()
510 	sf->frame.eof = pdf;  in tbstream_dev_alloc_tx()
512 	dma_sync_single_for_cpu(dma_dev, sf->frame.buffer_phy, size,  in tbstream_dev_alloc_tx()
515 		if (copy_page_from_iter(sf->page, 0, size, from) != size)  in tbstream_dev_alloc_tx()
518 		memset(page_address(sf->page), 0, size);  in tbstream_dev_alloc_tx()
520 	dma_sync_single_for_device(dma_dev, sf->frame.buffer_phy, size,  in tbstream_dev_alloc_tx()
522 	return sf;  in tbstream_dev_alloc_tx()
529 	struct tbstream_frame *sf;  in tbstream_dev_send_data()  local
531 	sf = tbstream_dev_alloc_tx(sdev, TBSTREAM_DATA, from, size);  in tbstream_dev_send_data()
532 	if (IS_ERR(sf))  in tbstream_dev_send_data()
533 		return PTR_ERR(sf);  in tbstream_dev_send_data()
534 	return tb_ring_tx(sdev->tx_ring.ring, &sf->frame);  in tbstream_dev_send_data()
539 	struct tbstream_frame *sf;  in tbstream_dev_send_close()  local
541 	sf = tbstream_dev_alloc_tx(sdev, TBSTREAM_CLOSE, NULL, SZ_256);  in tbstream_dev_send_close()
542 	if (IS_ERR(sf))  in tbstream_dev_send_close()
543 		return PTR_ERR(sf);  in tbstream_dev_send_close()
544 	return tb_ring_tx(sdev->tx_ring.ring, &sf->frame);  in tbstream_dev_send_close()
675 		struct tbstream_frame *sf;  in tbstream_dev_fops_read_iter()  local
678 		sf = tbstream_dev_completed_rx(sdev);  in tbstream_dev_fops_read_iter()
679 		if (!sf)  in tbstream_dev_fops_read_iter()
687 		if (sf->frame.eof == TBSTREAM_CLOSE) {  in tbstream_dev_fops_read_iter()
695 		sf_size = tb_ring_frame_size(&sf->frame);  in tbstream_dev_fops_read_iter()
698 		if (copy_page_to_iter(sf->page, sf->offset, size, to) != size) {  in tbstream_dev_fops_read_iter()
709 			sf->offset += size;  in tbstream_dev_fops_read_iter()
710 			sf->frame.size = sf_size - size;  in tbstream_dev_fops_read_iter()