Lines Matching +full:burst +full:- +full:wrap
1 // SPDX-License-Identifier: GPL-2.0
5 * Copyright (C) 2014-2015 Intel Corporation.
21 #include <linux/dma-mapping.h>
31 #define msc_dev(x) (&(x)->thdev->dev)
35 * READY -> INUSE -+-> LOCKED -+-> READY -> etc.
36 * \-----------/
42 * All state transitions happen automatically, except for the LOCKED->READY,
57 * struct msc_window - multiblock mode window descriptor
81 * struct msc_iter - iterator for msc buffer
107 * struct msc - MSC device representation
117 * @single_wrap: single mode wrap occurred
122 * @buf_mutex: mutex to serialize access to buffer-related bits
124 * @wrap: wrapping is enabled
126 * @burst_len: write burst length
162 wrap : 1, member
174 * struct msu_buffer_entry - internal MSU buffer bookkeeping
192 if (!strcmp(mbe->mbuf->name, name)) in __msu_buffer_entry_find()
206 if (mbe && !try_module_get(mbe->owner)) in msu_buffer_get()
210 return mbe ? mbe->mbuf : NULL; in msu_buffer_get()
218 mbe = __msu_buffer_entry_find(mbuf->name); in msu_buffer_put()
220 module_put(mbe->owner); in msu_buffer_put()
232 return -ENOMEM; in intel_th_msu_buffer_register()
235 if (__msu_buffer_entry_find(mbuf->name)) { in intel_th_msu_buffer_register()
236 ret = -EEXIST; in intel_th_msu_buffer_register()
241 mbe->mbuf = mbuf; in intel_th_msu_buffer_register()
242 mbe->owner = owner; in intel_th_msu_buffer_register()
243 list_add_tail(&mbe->entry, &msu_buffer_list); in intel_th_msu_buffer_register()
256 mbe = __msu_buffer_entry_find(mbuf->name); in intel_th_msu_buffer_unregister()
258 list_del(&mbe->entry); in intel_th_msu_buffer_unregister()
268 if (!bdesc->valid_dw) in msc_block_is_empty()
280 return win->sgt->sgl; in msc_win_base_sg()
300 * msc_is_last_win() - check if a window is the last one for a given MSC
306 return win->entry.next == &win->msc->win_list; in msc_is_last_win()
310 * msc_next_window() - return next window in the multiblock buffer
318 return list_first_entry(&win->msc->win_list, struct msc_window, in msc_next_window()
330 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) { in msc_win_total_sz()
334 return (size_t)win->nr_blocks << PAGE_SHIFT; in msc_win_total_sz()
345 * msc_find_window() - find a window matching a given sg_table
359 if (list_empty(&msc->win_list)) in msc_find_window()
367 list_for_each_entry(win, &msc->win_list, entry) { in msc_find_window()
368 if (win->sgt == sgt) in msc_find_window()
383 * msc_oldest_window() - locate the window with oldest data
395 if (list_empty(&msc->win_list)) in msc_oldest_window()
398 win = msc_find_window(msc, msc_next_window(msc->cur_win)->sgt, true); in msc_oldest_window()
402 return list_first_entry(&msc->win_list, struct msc_window, entry); in msc_oldest_window()
406 * msc_win_oldest_sg() - locate the oldest block in a given window
425 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) { in msc_win_oldest_sg()
437 return sg_virt(iter->block); in msc_iter_bdesc()
446 return ERR_PTR(-ENOMEM); in msc_iter_install()
448 mutex_lock(&msc->buf_mutex); in msc_iter_install()
456 if (msc->enabled) { in msc_iter_install()
458 iter = ERR_PTR(-EBUSY); in msc_iter_install()
462 iter->msc = msc; in msc_iter_install()
464 list_add_tail(&iter->entry, &msc->iter_list); in msc_iter_install()
466 mutex_unlock(&msc->buf_mutex); in msc_iter_install()
473 mutex_lock(&msc->buf_mutex); in msc_iter_remove()
474 list_del(&iter->entry); in msc_iter_remove()
475 mutex_unlock(&msc->buf_mutex); in msc_iter_remove()
482 if (iter->start_block) in msc_iter_block_start()
485 iter->start_block = msc_win_oldest_sg(iter->win); in msc_iter_block_start()
486 iter->block = iter->start_block; in msc_iter_block_start()
487 iter->wrap_count = 0; in msc_iter_block_start()
494 iter->wrap_count = 2; in msc_iter_block_start()
501 if (iter->start_win) in msc_iter_win_start()
504 iter->start_win = msc_oldest_window(msc); in msc_iter_win_start()
505 if (!iter->start_win) in msc_iter_win_start()
506 return -EINVAL; in msc_iter_win_start()
508 iter->win = iter->start_win; in msc_iter_win_start()
509 iter->start_block = NULL; in msc_iter_win_start()
518 iter->win = msc_next_window(iter->win); in msc_iter_win_advance()
519 iter->start_block = NULL; in msc_iter_win_advance()
521 if (iter->win == iter->start_win) { in msc_iter_win_advance()
522 iter->eof++; in msc_iter_win_advance()
533 iter->block_off = 0; in msc_iter_block_advance()
536 if (iter->wrap_count && iter->block == iter->start_block) { in msc_iter_block_advance()
537 iter->wrap_count--; in msc_iter_block_advance()
538 if (!iter->wrap_count) in msc_iter_block_advance()
544 if (!iter->wrap_count && msc_block_last_written(msc_iter_bdesc(iter))) in msc_iter_block_advance()
549 if (sg_is_last(iter->block)) in msc_iter_block_advance()
550 iter->block = msc_win_base_sg(iter->win); in msc_iter_block_advance()
552 iter->block = sg_next(iter->block); in msc_iter_block_advance()
555 if (!iter->wrap_count && iter->block == iter->start_block) in msc_iter_block_advance()
562 * msc_buffer_iterate() - go through multiblock buffer's data
581 struct msc *msc = iter->msc; in msc_buffer_iterate()
585 if (iter->eof) in msc_buffer_iterate()
612 if (iter->block == iter->start_block && iter->wrap_count == 2) { in msc_buffer_iterate()
613 tocopy = DATA_IN_PAGE - data_bytes; in msc_buffer_iterate()
620 tocopy -= iter->block_off; in msc_buffer_iterate()
621 src += iter->block_off; in msc_buffer_iterate()
633 copied = tocopy - remaining; in msc_buffer_iterate()
634 len -= copied; in msc_buffer_iterate()
635 iter->block_off += copied; in msc_buffer_iterate()
636 iter->offset += copied; in msc_buffer_iterate()
647 return size - len; in msc_buffer_iterate()
651 * msc_buffer_clear_hw_header() - clear hw header for multiblock
659 list_for_each_entry(win, &msc->win_list, entry) { in msc_buffer_clear_hw_header()
662 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) { in msc_buffer_clear_hw_header()
674 if (!msc->do_irq) in intel_th_msu_init()
677 if (!msc->mbuf) in intel_th_msu_init()
680 mintctl = ioread32(msc->msu_base + REG_MSU_MINTCTL); in intel_th_msu_init()
681 mintctl |= msc->index ? M1BLIE : M0BLIE; in intel_th_msu_init()
682 iowrite32(mintctl, msc->msu_base + REG_MSU_MINTCTL); in intel_th_msu_init()
683 if (mintctl != ioread32(msc->msu_base + REG_MSU_MINTCTL)) { in intel_th_msu_init()
685 msc->do_irq = 0; in intel_th_msu_init()
689 msusts = ioread32(msc->msu_base + REG_MSU_MSUSTS); in intel_th_msu_init()
690 iowrite32(msusts, msc->msu_base + REG_MSU_MSUSTS); in intel_th_msu_init()
699 if (!msc->do_irq) in intel_th_msu_deinit()
702 mintctl = ioread32(msc->msu_base + REG_MSU_MINTCTL); in intel_th_msu_deinit()
703 mintctl &= msc->index ? ~M1BLIE : ~M0BLIE; in intel_th_msu_deinit()
704 iowrite32(mintctl, msc->msu_base + REG_MSU_MINTCTL); in intel_th_msu_deinit()
715 if (!win->msc->mbuf) in msc_win_set_lockout()
718 spin_lock_irqsave(&win->lo_lock, flags); in msc_win_set_lockout()
719 old = win->lockout; in msc_win_set_lockout()
722 ret = -EINVAL; in msc_win_set_lockout()
726 win->lockout = new; in msc_win_set_lockout()
729 atomic_inc(&win->msc->user_count); in msc_win_set_lockout()
731 atomic_dec(&win->msc->user_count); in msc_win_set_lockout()
734 spin_unlock_irqrestore(&win->lo_lock, flags); in msc_win_set_lockout()
738 return -EBUSY; in msc_win_set_lockout()
744 dev_warn_ratelimited(msc_dev(win->msc), in msc_win_set_lockout()
752 * msc_configure() - set up MSC hardware
755 * Program storage mode, wrapping, burst length and trace buffer address
765 lockdep_assert_held(&msc->buf_mutex); in msc_configure()
767 if (msc->mode > MSC_MODE_MULTI) in msc_configure()
768 return -EINVAL; in msc_configure()
770 if (msc->mode == MSC_MODE_MULTI) { in msc_configure()
771 if (msc_win_set_lockout(msc->cur_win, WIN_READY, WIN_INUSE)) in msc_configure()
772 return -EBUSY; in msc_configure()
777 msc->orig_addr = ioread32(msc->reg_base + REG_MSU_MSC0BAR); in msc_configure()
778 msc->orig_sz = ioread32(msc->reg_base + REG_MSU_MSC0SIZE); in msc_configure()
780 reg = msc->base_addr >> PAGE_SHIFT; in msc_configure()
781 iowrite32(reg, msc->reg_base + REG_MSU_MSC0BAR); in msc_configure()
783 if (msc->mode == MSC_MODE_SINGLE) { in msc_configure()
784 reg = msc->nr_pages; in msc_configure()
785 iowrite32(reg, msc->reg_base + REG_MSU_MSC0SIZE); in msc_configure()
788 reg = ioread32(msc->reg_base + REG_MSU_MSC0CTL); in msc_configure()
792 reg |= msc->mode << __ffs(MSC_MODE); in msc_configure()
793 reg |= msc->burst_len << __ffs(MSC_LEN); in msc_configure()
795 if (msc->wrap) in msc_configure()
798 iowrite32(reg, msc->reg_base + REG_MSU_MSC0CTL); in msc_configure()
802 msc->thdev->output.multiblock = msc->mode == MSC_MODE_MULTI; in msc_configure()
803 intel_th_trace_enable(msc->thdev); in msc_configure()
804 msc->enabled = 1; in msc_configure()
806 if (msc->mbuf && msc->mbuf->activate) in msc_configure()
807 msc->mbuf->activate(msc->mbuf_priv); in msc_configure()
813 * msc_disable() - disable MSC hardware
821 struct msc_window *win = msc->cur_win; in msc_disable()
824 lockdep_assert_held(&msc->buf_mutex); in msc_disable()
826 if (msc->mode == MSC_MODE_MULTI) in msc_disable()
829 if (msc->mbuf && msc->mbuf->deactivate) in msc_disable()
830 msc->mbuf->deactivate(msc->mbuf_priv); in msc_disable()
832 intel_th_trace_disable(msc->thdev); in msc_disable()
834 if (msc->mode == MSC_MODE_SINGLE) { in msc_disable()
835 reg = ioread32(msc->reg_base + REG_MSU_MSC0STS); in msc_disable()
836 msc->single_wrap = !!(reg & MSCSTS_WRAPSTAT); in msc_disable()
838 reg = ioread32(msc->reg_base + REG_MSU_MSC0MWP); in msc_disable()
839 msc->single_sz = reg & ((msc->nr_pages << PAGE_SHIFT) - 1); in msc_disable()
840 dev_dbg(msc_dev(msc), "MSCnMWP: %08x/%08lx, wrap: %d\n", in msc_disable()
841 reg, msc->single_sz, msc->single_wrap); in msc_disable()
844 reg = ioread32(msc->reg_base + REG_MSU_MSC0CTL); in msc_disable()
846 iowrite32(reg, msc->reg_base + REG_MSU_MSC0CTL); in msc_disable()
848 if (msc->mbuf && msc->mbuf->ready) in msc_disable()
849 msc->mbuf->ready(msc->mbuf_priv, win->sgt, in msc_disable()
852 msc->enabled = 0; in msc_disable()
854 iowrite32(msc->orig_addr, msc->reg_base + REG_MSU_MSC0BAR); in msc_disable()
855 iowrite32(msc->orig_sz, msc->reg_base + REG_MSU_MSC0SIZE); in msc_disable()
858 ioread32(msc->reg_base + REG_MSU_MSC0NWSA)); in msc_disable()
860 reg = ioread32(msc->reg_base + REG_MSU_MSC0STS); in msc_disable()
863 reg = ioread32(msc->reg_base + REG_MSU_MSUSTS); in msc_disable()
864 reg &= msc->index ? MSUSTS_MSC1BLAST : MSUSTS_MSC0BLAST; in msc_disable()
865 iowrite32(reg, msc->reg_base + REG_MSU_MSUSTS); in msc_disable()
870 struct msc *msc = dev_get_drvdata(&thdev->dev); in intel_th_msc_activate()
871 int ret = -EBUSY; in intel_th_msc_activate()
873 if (!atomic_inc_unless_negative(&msc->user_count)) in intel_th_msc_activate()
874 return -ENODEV; in intel_th_msc_activate()
876 mutex_lock(&msc->buf_mutex); in intel_th_msc_activate()
879 if (list_empty(&msc->iter_list)) in intel_th_msc_activate()
882 mutex_unlock(&msc->buf_mutex); in intel_th_msc_activate()
885 atomic_dec(&msc->user_count); in intel_th_msc_activate()
892 struct msc *msc = dev_get_drvdata(&thdev->dev); in intel_th_msc_deactivate()
894 mutex_lock(&msc->buf_mutex); in intel_th_msc_deactivate()
895 if (msc->enabled) { in intel_th_msc_deactivate()
897 atomic_dec(&msc->user_count); in intel_th_msc_deactivate()
899 mutex_unlock(&msc->buf_mutex); in intel_th_msc_deactivate()
903 * msc_buffer_contig_alloc() - allocate a contiguous buffer for SINGLE mode
910 * Return: 0 on success, -errno otherwise.
922 ret = sg_alloc_table(&msc->single_sgt, 1, GFP_KERNEL); in msc_buffer_contig_alloc()
926 ret = -ENOMEM; in msc_buffer_contig_alloc()
932 sg_set_buf(msc->single_sgt.sgl, page_address(page), size); in msc_buffer_contig_alloc()
934 ret = dma_map_sg(msc_dev(msc)->parent->parent, msc->single_sgt.sgl, 1, in msc_buffer_contig_alloc()
939 msc->nr_pages = nr_pages; in msc_buffer_contig_alloc()
940 msc->base = page_address(page); in msc_buffer_contig_alloc()
941 msc->base_addr = sg_dma_address(msc->single_sgt.sgl); in msc_buffer_contig_alloc()
949 sg_free_table(&msc->single_sgt); in msc_buffer_contig_alloc()
956 * msc_buffer_contig_free() - free a contiguous buffer
963 dma_unmap_sg(msc_dev(msc)->parent->parent, msc->single_sgt.sgl, in msc_buffer_contig_free()
965 sg_free_table(&msc->single_sgt); in msc_buffer_contig_free()
967 for (off = 0; off < msc->nr_pages << PAGE_SHIFT; off += PAGE_SIZE) { in msc_buffer_contig_free()
968 struct page *page = virt_to_page(msc->base + off); in msc_buffer_contig_free()
970 page->mapping = NULL; in msc_buffer_contig_free()
974 msc->nr_pages = 0; in msc_buffer_contig_free()
978 * msc_buffer_contig_get_page() - find a page at a given offset
987 if (pgoff >= msc->nr_pages) in msc_buffer_contig_get_page()
990 return virt_to_page(msc->base + (pgoff << PAGE_SHIFT)); in msc_buffer_contig_get_page()
1000 ret = sg_alloc_table(win->sgt, nr_segs, GFP_KERNEL); in __msc_buffer_win_alloc()
1002 return -ENOMEM; in __msc_buffer_win_alloc()
1004 for_each_sg(win->sgt->sgl, sg_ptr, nr_segs, i) { in __msc_buffer_win_alloc()
1005 block = dma_alloc_coherent(msc_dev(win->msc)->parent->parent, in __msc_buffer_win_alloc()
1017 for_each_sg(win->sgt->sgl, sg_ptr, i, ret) in __msc_buffer_win_alloc()
1018 dma_free_coherent(msc_dev(win->msc)->parent->parent, PAGE_SIZE, in __msc_buffer_win_alloc()
1021 sg_free_table(win->sgt); in __msc_buffer_win_alloc()
1023 return -ENOMEM; in __msc_buffer_win_alloc()
1033 if (msc->mode == MSC_MODE_SINGLE) { in msc_buffer_set_uc()
1034 set_memory_uc((unsigned long)msc->base, msc->nr_pages); in msc_buffer_set_uc()
1038 list_for_each_entry(win, &msc->win_list, entry) { in msc_buffer_set_uc()
1039 for_each_sg(win->sgt->sgl, sg_ptr, win->nr_segs, i) { in msc_buffer_set_uc()
1042 PFN_DOWN(sg_ptr->length)); in msc_buffer_set_uc()
1053 if (msc->mode == MSC_MODE_SINGLE) { in msc_buffer_set_wb()
1054 set_memory_wb((unsigned long)msc->base, msc->nr_pages); in msc_buffer_set_wb()
1058 list_for_each_entry(win, &msc->win_list, entry) { in msc_buffer_set_wb()
1059 for_each_sg(win->sgt->sgl, sg_ptr, win->nr_segs, i) { in msc_buffer_set_wb()
1060 /* Reset the page to write-back */ in msc_buffer_set_wb()
1062 PFN_DOWN(sg_ptr->length)); in msc_buffer_set_wb()
1083 * msc_buffer_win_alloc() - alloc a window for a multiblock mode
1090 * Return: 0 on success, -errno otherwise.
1095 int ret = -ENOMEM; in msc_buffer_win_alloc()
1102 return -ENOMEM; in msc_buffer_win_alloc()
1104 win->msc = msc; in msc_buffer_win_alloc()
1105 win->sgt = &win->_sgt; in msc_buffer_win_alloc()
1106 win->lockout = WIN_READY; in msc_buffer_win_alloc()
1107 spin_lock_init(&win->lo_lock); in msc_buffer_win_alloc()
1109 if (!list_empty(&msc->win_list)) { in msc_buffer_win_alloc()
1110 struct msc_window *prev = list_last_entry(&msc->win_list, in msc_buffer_win_alloc()
1114 win->pgoff = prev->pgoff + prev->nr_blocks; in msc_buffer_win_alloc()
1117 if (msc->mbuf && msc->mbuf->alloc_window) in msc_buffer_win_alloc()
1118 ret = msc->mbuf->alloc_window(msc->mbuf_priv, &win->sgt, in msc_buffer_win_alloc()
1126 win->nr_segs = ret; in msc_buffer_win_alloc()
1127 win->nr_blocks = nr_blocks; in msc_buffer_win_alloc()
1129 if (list_empty(&msc->win_list)) { in msc_buffer_win_alloc()
1130 msc->base = msc_win_base(win); in msc_buffer_win_alloc()
1131 msc->base_addr = msc_win_base_dma(win); in msc_buffer_win_alloc()
1132 msc->cur_win = win; in msc_buffer_win_alloc()
1135 list_add_tail(&win->entry, &msc->win_list); in msc_buffer_win_alloc()
1136 msc->nr_pages += nr_blocks; in msc_buffer_win_alloc()
1151 for_each_sg(win->sgt->sgl, sg, win->nr_segs, i) { in __msc_buffer_win_free()
1154 page->mapping = NULL; in __msc_buffer_win_free()
1155 dma_free_coherent(msc_dev(win->msc)->parent->parent, PAGE_SIZE, in __msc_buffer_win_free()
1158 sg_free_table(win->sgt); in __msc_buffer_win_free()
1162 * msc_buffer_win_free() - free a window from MSC's window list
1171 msc->nr_pages -= win->nr_blocks; in msc_buffer_win_free()
1173 list_del(&win->entry); in msc_buffer_win_free()
1174 if (list_empty(&msc->win_list)) { in msc_buffer_win_free()
1175 msc->base = NULL; in msc_buffer_win_free()
1176 msc->base_addr = 0; in msc_buffer_win_free()
1179 if (msc->mbuf && msc->mbuf->free_window) in msc_buffer_win_free()
1180 msc->mbuf->free_window(msc->mbuf_priv, win->sgt); in msc_buffer_win_free()
1188 * msc_buffer_relink() - set up block descriptors for multiblock mode
1199 list_for_each_entry(win, &msc->win_list, entry) { in msc_buffer_relink()
1210 next_win = list_first_entry(&msc->win_list, in msc_buffer_relink()
1216 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) { in msc_buffer_relink()
1221 bdesc->next_win = msc_win_base_pfn(next_win); in msc_buffer_relink()
1227 if (blk == win->nr_segs - 1) { in msc_buffer_relink()
1229 bdesc->next_blk = msc_win_base_pfn(win); in msc_buffer_relink()
1233 bdesc->next_blk = PFN_DOWN(addr); in msc_buffer_relink()
1236 bdesc->sw_tag = sw_tag; in msc_buffer_relink()
1237 bdesc->block_sz = sg->length / 64; in msc_buffer_relink()
1252 list_for_each_entry_safe(win, iter, &msc->win_list, entry) in msc_buffer_multi_free()
1275 * msc_buffer_free() - free buffers for MSC
1287 if (msc->mode == MSC_MODE_SINGLE) in msc_buffer_free()
1289 else if (msc->mode == MSC_MODE_MULTI) in msc_buffer_free()
1294 * msc_buffer_alloc() - allocate a buffer for MSC
1308 * Return: 0 on success, -errno otherwise.
1315 /* -1: buffer not allocated */ in msc_buffer_alloc()
1316 if (atomic_read(&msc->user_count) != -1) in msc_buffer_alloc()
1317 return -EBUSY; in msc_buffer_alloc()
1319 if (msc->mode == MSC_MODE_SINGLE) { in msc_buffer_alloc()
1321 return -EINVAL; in msc_buffer_alloc()
1324 } else if (msc->mode == MSC_MODE_MULTI) { in msc_buffer_alloc()
1327 ret = -EINVAL; in msc_buffer_alloc()
1336 if (WARN_ON_ONCE(atomic_cmpxchg(&msc->user_count, -1, 0) != -1)) in msc_buffer_alloc()
1337 return -EINVAL; in msc_buffer_alloc()
1344 * msc_buffer_unlocked_free_unless_used() - free a buffer unless it's in use
1352 * deallocate, -EBUSY if there are active users.
1358 count = atomic_cmpxchg(&msc->user_count, 0, -1); in msc_buffer_unlocked_free_unless_used()
1362 ret = -EBUSY; in msc_buffer_unlocked_free_unless_used()
1372 * msc_buffer_free_unless_used() - free a buffer unless it's in use
1378 * deallocate, -EBUSY if there are active users.
1384 mutex_lock(&msc->buf_mutex); in msc_buffer_free_unless_used()
1386 mutex_unlock(&msc->buf_mutex); in msc_buffer_free_unless_used()
1392 * msc_buffer_get_page() - get MSC buffer page at a given offset
1407 if (msc->mode == MSC_MODE_SINGLE) in msc_buffer_get_page()
1410 list_for_each_entry(win, &msc->win_list, entry) in msc_buffer_get_page()
1411 if (pgoff >= win->pgoff && pgoff < win->pgoff + win->nr_blocks) in msc_buffer_get_page()
1417 pgoff -= win->pgoff; in msc_buffer_get_page()
1419 for_each_sg(win->sgt->sgl, sg, win->nr_segs, blk) { in msc_buffer_get_page()
1421 size_t pgsz = PFN_DOWN(sg->length); in msc_buffer_get_page()
1426 pgoff -= pgsz; in msc_buffer_get_page()
1433 * struct msc_win_to_user_struct - data for copy_to_user() callback
1443 * msc_win_to_user() - iterator for msc_buffer_iterate() to copy data to user
1448 * Return: >= %0 for success or -errno for error.
1455 ret = copy_to_user(u->buf + u->offset, src, len); in msc_win_to_user()
1456 u->offset += len - ret; in msc_win_to_user()
1468 struct intel_th_device *thdev = file->private_data; in intel_th_msc_open()
1469 struct msc *msc = dev_get_drvdata(&thdev->dev); in intel_th_msc_open()
1473 return -EPERM; in intel_th_msc_open()
1479 file->private_data = iter; in intel_th_msc_open()
1486 struct msc_iter *iter = file->private_data; in intel_th_msc_release()
1487 struct msc *msc = iter->msc; in intel_th_msc_release()
1497 unsigned long size = msc->nr_pages << PAGE_SHIFT, rem = len; in msc_single_to_user()
1500 if (msc->single_wrap) { in msc_single_to_user()
1501 start += msc->single_sz; in msc_single_to_user()
1503 tocopy = min(rem, size - start); in msc_single_to_user()
1504 if (copy_to_user(buf, msc->base + start, tocopy)) in msc_single_to_user()
1505 return -EFAULT; in msc_single_to_user()
1508 rem -= tocopy; in msc_single_to_user()
1512 start &= size - 1; in msc_single_to_user()
1514 tocopy = min(rem, msc->single_sz - start); in msc_single_to_user()
1515 if (copy_to_user(buf, msc->base + start, tocopy)) in msc_single_to_user()
1516 return -EFAULT; in msc_single_to_user()
1518 rem -= tocopy; in msc_single_to_user()
1521 return len - rem; in msc_single_to_user()
1524 if (copy_to_user(buf, msc->base + start, rem)) in msc_single_to_user()
1525 return -EFAULT; in msc_single_to_user()
1533 struct msc_iter *iter = file->private_data; in intel_th_msc_read()
1534 struct msc *msc = iter->msc; in intel_th_msc_read()
1539 if (!atomic_inc_unless_negative(&msc->user_count)) in intel_th_msc_read()
1542 if (msc->mode == MSC_MODE_SINGLE && !msc->single_wrap) in intel_th_msc_read()
1543 size = msc->single_sz; in intel_th_msc_read()
1545 size = msc->nr_pages << PAGE_SHIFT; in intel_th_msc_read()
1554 len = size - off; in intel_th_msc_read()
1556 if (msc->mode == MSC_MODE_SINGLE) { in intel_th_msc_read()
1560 } else if (msc->mode == MSC_MODE_MULTI) { in intel_th_msc_read()
1568 *ppos = iter->offset; in intel_th_msc_read()
1570 ret = -EINVAL; in intel_th_msc_read()
1574 atomic_dec(&msc->user_count); in intel_th_msc_read()
1585 struct msc_iter *iter = vma->vm_file->private_data; in msc_mmap_open()
1586 struct msc *msc = iter->msc; in msc_mmap_open()
1588 atomic_inc(&msc->mmap_count); in msc_mmap_open()
1593 struct msc_iter *iter = vma->vm_file->private_data; in msc_mmap_close()
1594 struct msc *msc = iter->msc; in msc_mmap_close()
1597 if (!atomic_dec_and_mutex_lock(&msc->mmap_count, &msc->buf_mutex)) in msc_mmap_close()
1601 for (pg = 0; pg < msc->nr_pages; pg++) { in msc_mmap_close()
1607 if (page->mapping) in msc_mmap_close()
1608 page->mapping = NULL; in msc_mmap_close()
1611 /* last mapping -- drop user_count */ in msc_mmap_close()
1612 atomic_dec(&msc->user_count); in msc_mmap_close()
1613 mutex_unlock(&msc->buf_mutex); in msc_mmap_close()
1618 struct msc_iter *iter = vmf->vma->vm_file->private_data; in msc_mmap_fault()
1619 struct msc *msc = iter->msc; in msc_mmap_fault()
1621 vmf->page = msc_buffer_get_page(msc, vmf->pgoff); in msc_mmap_fault()
1622 if (!vmf->page) in msc_mmap_fault()
1625 get_page(vmf->page); in msc_mmap_fault()
1626 vmf->page->mapping = vmf->vma->vm_file->f_mapping; in msc_mmap_fault()
1627 vmf->page->index = vmf->pgoff; in msc_mmap_fault()
1640 unsigned long size = vma->vm_end - vma->vm_start; in intel_th_msc_mmap()
1641 struct msc_iter *iter = vma->vm_file->private_data; in intel_th_msc_mmap()
1642 struct msc *msc = iter->msc; in intel_th_msc_mmap()
1643 int ret = -EINVAL; in intel_th_msc_mmap()
1646 return -EINVAL; in intel_th_msc_mmap()
1648 if (vma->vm_pgoff) in intel_th_msc_mmap()
1649 return -EINVAL; in intel_th_msc_mmap()
1652 if (!atomic_inc_unless_negative(&msc->user_count)) in intel_th_msc_mmap()
1653 return -EINVAL; in intel_th_msc_mmap()
1655 if (msc->mode != MSC_MODE_SINGLE && in intel_th_msc_mmap()
1656 msc->mode != MSC_MODE_MULTI) in intel_th_msc_mmap()
1659 if (size >> PAGE_SHIFT != msc->nr_pages) in intel_th_msc_mmap()
1662 atomic_set(&msc->mmap_count, 1); in intel_th_msc_mmap()
1667 atomic_dec(&msc->user_count); in intel_th_msc_mmap()
1669 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); in intel_th_msc_mmap()
1671 vma->vm_ops = &msc_mmap_ops; in intel_th_msc_mmap()
1685 struct msc *msc = dev_get_drvdata(&thdev->dev); in intel_th_msc_wait_empty()
1690 count && !(reg & MSCSTS_PLE); count--) { in intel_th_msc_wait_empty()
1691 reg = __raw_readl(msc->reg_base + REG_MSU_MSC0STS); in intel_th_msc_wait_empty()
1701 atomic_set(&msc->user_count, -1); in intel_th_msc_init()
1703 msc->mode = msc->multi_is_broken ? MSC_MODE_SINGLE : MSC_MODE_MULTI; in intel_th_msc_init()
1704 mutex_init(&msc->buf_mutex); in intel_th_msc_init()
1705 INIT_LIST_HEAD(&msc->win_list); in intel_th_msc_init()
1706 INIT_LIST_HEAD(&msc->iter_list); in intel_th_msc_init()
1708 msc->burst_len = in intel_th_msc_init()
1709 (ioread32(msc->reg_base + REG_MSU_MSC0CTL) & MSC_LEN) >> in intel_th_msc_init()
1719 if (list_empty(&msc->win_list)) in msc_win_switch()
1720 return -EINVAL; in msc_win_switch()
1722 first = list_first_entry(&msc->win_list, struct msc_window, entry); in msc_win_switch()
1724 if (msc_is_last_win(msc->cur_win)) in msc_win_switch()
1725 msc->cur_win = first; in msc_win_switch()
1727 msc->cur_win = list_next_entry(msc->cur_win, entry); in msc_win_switch()
1729 msc->base = msc_win_base(msc->cur_win); in msc_win_switch()
1730 msc->base_addr = msc_win_base_dma(msc->cur_win); in msc_win_switch()
1732 intel_th_trace_switch(msc->thdev); in msc_win_switch()
1738 * intel_th_msc_window_unlock - put the window back in rotation
1755 if (msc->switch_on_unlock == win) { in intel_th_msc_window_unlock()
1756 msc->switch_on_unlock = NULL; in intel_th_msc_window_unlock()
1766 intel_th_msc_deactivate(msc->thdev); in msc_work()
1771 struct msc *msc = dev_get_drvdata(&thdev->dev); in intel_th_msc_interrupt()
1772 u32 msusts = ioread32(msc->msu_base + REG_MSU_MSUSTS); in intel_th_msc_interrupt()
1773 u32 mask = msc->index ? MSUSTS_MSC1BLAST : MSUSTS_MSC0BLAST; in intel_th_msc_interrupt()
1776 if (!msc->do_irq || !msc->mbuf) in intel_th_msc_interrupt()
1782 return msc->enabled ? IRQ_HANDLED : IRQ_NONE; in intel_th_msc_interrupt()
1784 iowrite32(msusts, msc->msu_base + REG_MSU_MSUSTS); in intel_th_msc_interrupt()
1786 if (!msc->enabled) in intel_th_msc_interrupt()
1790 win = msc->cur_win; in intel_th_msc_interrupt()
1799 if (msc->stop_on_full) in intel_th_msc_interrupt()
1800 schedule_work(&msc->work); in intel_th_msc_interrupt()
1802 msc->switch_on_unlock = next_win; in intel_th_msc_interrupt()
1807 /* current window: INUSE -> LOCKED */ in intel_th_msc_interrupt()
1812 if (msc->mbuf && msc->mbuf->ready) in intel_th_msc_interrupt()
1813 msc->mbuf->ready(msc->mbuf_priv, win->sgt, in intel_th_msc_interrupt()
1831 return scnprintf(buf, PAGE_SIZE, "%d\n", msc->wrap); in wrap_show()
1846 msc->wrap = !!val; in wrap_store()
1851 static DEVICE_ATTR_RW(wrap);
1855 lockdep_assert_held(&msc->buf_mutex); in msc_buffer_unassign()
1857 if (!msc->mbuf) in msc_buffer_unassign()
1860 msc->mbuf->unassign(msc->mbuf_priv); in msc_buffer_unassign()
1861 msu_buffer_put(msc->mbuf); in msc_buffer_unassign()
1862 msc->mbuf_priv = NULL; in msc_buffer_unassign()
1863 msc->mbuf = NULL; in msc_buffer_unassign()
1870 const char *mode = msc_mode[msc->mode]; in mode_show()
1873 mutex_lock(&msc->buf_mutex); in mode_show()
1874 if (msc->mbuf) in mode_show()
1875 mode = msc->mbuf->name; in mode_show()
1877 mutex_unlock(&msc->buf_mutex); in mode_show()
1893 return -EPERM; in mode_store()
1897 len = cp - buf; in mode_store()
1901 return -ENOMEM; in mode_store()
1910 if (!msc->do_irq) { in mode_store()
1912 return -EINVAL; in mode_store()
1920 return -EINVAL; in mode_store()
1923 if (i == MSC_MODE_MULTI && msc->multi_is_broken) in mode_store()
1924 return -EOPNOTSUPP; in mode_store()
1926 mutex_lock(&msc->buf_mutex); in mode_store()
1930 if (mbuf && mbuf == msc->mbuf) { in mode_store()
1941 void *mbuf_priv = mbuf->assign(dev, &i); in mode_store()
1944 ret = -ENOMEM; in mode_store()
1949 msc->mbuf_priv = mbuf_priv; in mode_store()
1950 msc->mbuf = mbuf; in mode_store()
1955 msc->mode = i; in mode_store()
1960 mutex_unlock(&msc->buf_mutex); in mode_store()
1974 mutex_lock(&msc->buf_mutex); in nr_pages_show()
1976 if (msc->mode == MSC_MODE_SINGLE) in nr_pages_show()
1977 count = scnprintf(buf, PAGE_SIZE, "%ld\n", msc->nr_pages); in nr_pages_show()
1978 else if (msc->mode == MSC_MODE_MULTI) { in nr_pages_show()
1979 list_for_each_entry(win, &msc->win_list, entry) { in nr_pages_show()
1980 count += scnprintf(buf + count, PAGE_SIZE - count, in nr_pages_show()
1981 "%d%c", win->nr_blocks, in nr_pages_show()
1988 mutex_unlock(&msc->buf_mutex); in nr_pages_show()
2005 return -EPERM; in nr_pages_store()
2011 /* scan the comma-separated list of allocation sizes */ in nr_pages_store()
2014 len = end - buf; in nr_pages_store()
2018 s = kstrndup(p, end ? end - p : len, GFP_KERNEL); in nr_pages_store()
2020 ret = -ENOMEM; in nr_pages_store()
2030 if (nr_wins && msc->mode == MSC_MODE_SINGLE) { in nr_pages_store()
2031 ret = -EINVAL; in nr_pages_store()
2039 return -ENOMEM; in nr_pages_store()
2043 win[nr_wins - 1] = val; in nr_pages_store()
2049 len -= end - p + 1; in nr_pages_store()
2053 mutex_lock(&msc->buf_mutex); in nr_pages_store()
2055 mutex_unlock(&msc->buf_mutex); in nr_pages_store()
2078 return -EINVAL; in win_switch_store()
2080 ret = -EINVAL; in win_switch_store()
2081 mutex_lock(&msc->buf_mutex); in win_switch_store()
2087 if (msc->mode == MSC_MODE_MULTI && !msc->mbuf) in win_switch_store()
2089 mutex_unlock(&msc->buf_mutex); in win_switch_store()
2101 return sprintf(buf, "%d\n", msc->stop_on_full); in stop_on_full_show()
2111 ret = kstrtobool(buf, &msc->stop_on_full); in stop_on_full_store()
2135 struct device *dev = &thdev->dev; in intel_th_msc_probe()
2143 return -ENODEV; in intel_th_msc_probe()
2145 base = devm_ioremap(dev, res->start, resource_size(res)); in intel_th_msc_probe()
2147 return -ENOMEM; in intel_th_msc_probe()
2151 return -ENOMEM; in intel_th_msc_probe()
2155 msc->do_irq = 1; in intel_th_msc_probe()
2158 msc->multi_is_broken = 1; in intel_th_msc_probe()
2160 msc->index = thdev->id; in intel_th_msc_probe()
2162 msc->thdev = thdev; in intel_th_msc_probe()
2163 msc->reg_base = base + msc->index * 0x100; in intel_th_msc_probe()
2164 msc->msu_base = base; in intel_th_msc_probe()
2166 INIT_WORK(&msc->work, msc_work); in intel_th_msc_probe()
2178 struct msc *msc = dev_get_drvdata(&thdev->dev); in intel_th_msc_remove()