Lines Matching +full:out +full:- +full:of +full:- +full:window
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright 2016-17 IBM Corp.
18 #include <asm/ppc-opcode.h>
21 #include "copy-paste.h"
24 #include "vas-trace.h"
27 * Compute the paste address region for the window @window using the
28 * ->paste_base_addr and ->paste_win_id_shift we got from device tree.
30 void vas_win_paste_addr(struct pnv_vas_window *window, u64 *addr, int *len) in vas_win_paste_addr() argument
35 base = window->vinst->paste_base_addr; in vas_win_paste_addr()
36 shift = window->vinst->paste_win_id_shift; in vas_win_paste_addr()
37 winid = window->vas_win.winid; in vas_win_paste_addr()
46 static inline void get_hvwc_mmio_bar(struct pnv_vas_window *window, in get_hvwc_mmio_bar() argument
51 pbaddr = window->vinst->hvwc_bar_start; in get_hvwc_mmio_bar()
52 *start = pbaddr + window->vas_win.winid * VAS_HVWC_SIZE; in get_hvwc_mmio_bar()
56 static inline void get_uwc_mmio_bar(struct pnv_vas_window *window, in get_uwc_mmio_bar() argument
61 pbaddr = window->vinst->uwc_bar_start; in get_uwc_mmio_bar()
62 *start = pbaddr + window->vas_win.winid * VAS_UWC_SIZE; in get_uwc_mmio_bar()
67 * Map the paste bus address of the given send window into kernel address
69 * be mapped cache-able and is only applicable to send windows.
78 name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id, in map_paste_region()
79 txwin->vas_win.winid); in map_paste_region()
83 txwin->paste_addr_name = name; in map_paste_region()
104 return ERR_PTR(-ENOMEM); in map_paste_region()
134 * Unmap the paste address region for a window.
136 static void unmap_paste_region(struct pnv_vas_window *window) in unmap_paste_region() argument
141 if (window->paste_kaddr) { in unmap_paste_region()
142 vas_win_paste_addr(window, &busaddr_start, &len); in unmap_paste_region()
143 unmap_region(window->paste_kaddr, busaddr_start, len); in unmap_paste_region()
144 window->paste_kaddr = NULL; in unmap_paste_region()
145 kfree(window->paste_addr_name); in unmap_paste_region()
146 window->paste_addr_name = NULL; in unmap_paste_region()
151 * Unmap the MMIO regions for a window. Hold the vas_mutex so we don't
152 * unmap when the window's debugfs dir is in use. This serializes close
153 * of a window even on another VAS instance but since its not a critical
155 * a per-instance mutex later if necessary.
157 static void unmap_winctx_mmio_bars(struct pnv_vas_window *window) in unmap_winctx_mmio_bars() argument
166 hvwc_map = window->hvwc_map; in unmap_winctx_mmio_bars()
167 window->hvwc_map = NULL; in unmap_winctx_mmio_bars()
169 uwc_map = window->uwc_map; in unmap_winctx_mmio_bars()
170 window->uwc_map = NULL; in unmap_winctx_mmio_bars()
175 get_hvwc_mmio_bar(window, &busaddr_start, &len); in unmap_winctx_mmio_bars()
180 get_uwc_mmio_bar(window, &busaddr_start, &len); in unmap_winctx_mmio_bars()
186 * Find the Hypervisor Window Context (HVWC) MMIO Base Address Region and the
187 * OS/User Window Context (UWC) MMIO Base Address Region for the given window.
188 * Map these bus addresses and save the mapped kernel addresses in @window.
190 static int map_winctx_mmio_bars(struct pnv_vas_window *window) in map_winctx_mmio_bars() argument
195 get_hvwc_mmio_bar(window, &start, &len); in map_winctx_mmio_bars()
196 window->hvwc_map = map_mmio_region("HVWCM_Window", start, len); in map_winctx_mmio_bars()
198 get_uwc_mmio_bar(window, &start, &len); in map_winctx_mmio_bars()
199 window->uwc_map = map_mmio_region("UWCM_Window", start, len); in map_winctx_mmio_bars()
201 if (!window->hvwc_map || !window->uwc_map) { in map_winctx_mmio_bars()
202 unmap_winctx_mmio_bars(window); in map_winctx_mmio_bars()
203 return -1; in map_winctx_mmio_bars()
210 * Reset all valid registers in the HV and OS/User Window Contexts for
211 * the window identified by @window.
213 * NOTE: We cannot really use a for loop to reset window context. Not all
214 * offsets in a window context are valid registers and the valid
218 static void reset_window_regs(struct pnv_vas_window *window) in reset_window_regs() argument
220 write_hvwc_reg(window, VREG(LPID), 0ULL); in reset_window_regs()
221 write_hvwc_reg(window, VREG(PID), 0ULL); in reset_window_regs()
222 write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL); in reset_window_regs()
223 write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL); in reset_window_regs()
224 write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL); in reset_window_regs()
225 write_hvwc_reg(window, VREG(AMR), 0ULL); in reset_window_regs()
226 write_hvwc_reg(window, VREG(SEIDR), 0ULL); in reset_window_regs()
227 write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL); in reset_window_regs()
228 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL); in reset_window_regs()
229 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL); in reset_window_regs()
230 write_hvwc_reg(window, VREG(PSWID), 0ULL); in reset_window_regs()
231 write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL); in reset_window_regs()
232 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL); in reset_window_regs()
233 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL); in reset_window_regs()
234 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL); in reset_window_regs()
235 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL); in reset_window_regs()
236 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL); in reset_window_regs()
237 write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL); in reset_window_regs()
238 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); in reset_window_regs()
239 write_hvwc_reg(window, VREG(TX_WCRED), 0ULL); in reset_window_regs()
240 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); in reset_window_regs()
241 write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL); in reset_window_regs()
242 write_hvwc_reg(window, VREG(WINCTL), 0ULL); in reset_window_regs()
243 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL); in reset_window_regs()
244 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL); in reset_window_regs()
245 write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL); in reset_window_regs()
246 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL); in reset_window_regs()
247 write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL); in reset_window_regs()
248 write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL); in reset_window_regs()
249 write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL); in reset_window_regs()
250 write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL); in reset_window_regs()
251 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL); in reset_window_regs()
252 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL); in reset_window_regs()
254 /* Skip read-only registers: NX_UTIL and NX_UTIL_SE */ in reset_window_regs()
257 * The send and receive window credit adder registers are also in reset_window_regs()
259 * need to initialize from the OS/User Window Context, so skip in reset_window_regs()
262 * write_uwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); in reset_window_regs()
263 * write_uwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); in reset_window_regs()
268 * Initialize window context registers related to Address Translation.
274 static void init_xlate_regs(struct pnv_vas_window *window, bool user_win) in init_xlate_regs() argument
289 write_hvwc_reg(window, VREG(XLATE_MSR), val); in init_xlate_regs()
294 * NOTE: From Section 5.7.8.1 Segment Lookaside Buffer of the in init_xlate_regs()
297 * NOTE: From Section 1.3.1, Address Translation Context of the in init_xlate_regs()
304 write_hvwc_reg(window, VREG(XLATE_LPCR), val); in init_xlate_regs()
307 * Section 1.3.1 (Address translation Context) of NMMU workbook. in init_xlate_regs()
315 write_hvwc_reg(window, VREG(XLATE_CTL), val); in init_xlate_regs()
322 write_hvwc_reg(window, VREG(AMR), val); in init_xlate_regs()
326 write_hvwc_reg(window, VREG(SEIDR), val); in init_xlate_regs()
330 * Initialize Reserved Send Buffer Count for the send window. It involves
332 * has enough buffers to reserve. See section 1.3.1.2.1 of VAS workbook.
334 * Since we can only make a best-effort attempt to fulfill the request,
347 * Initialize window context registers for a receive window.
348 * Except for caching control and marking window open, the registers
349 * are initialized in the order listed in Section 3.1.4 (Window Context
350 * Cache Register Details) of the VAS workbook although they don't need
357 * write the per-chip RX FIFO addresses to the windows during boot-up
358 * as a one-time task? That could work for NX but what about other
359 * receivers? Let the receivers tell us the rx-fifo buffers for now.
361 static void init_winctx_regs(struct pnv_vas_window *window, in init_winctx_regs() argument
367 reset_window_regs(window); in init_winctx_regs()
370 val = SET_FIELD(VAS_LPID, val, winctx->lpid); in init_winctx_regs()
371 write_hvwc_reg(window, VREG(LPID), val); in init_winctx_regs()
374 val = SET_FIELD(VAS_PID_ID, val, winctx->pidr); in init_winctx_regs()
375 write_hvwc_reg(window, VREG(PID), val); in init_winctx_regs()
377 init_xlate_regs(window, winctx->user_win); in init_winctx_regs()
380 val = SET_FIELD(VAS_FAULT_TX_WIN, val, winctx->fault_win_id); in init_winctx_regs()
381 write_hvwc_reg(window, VREG(FAULT_TX_WIN), val); in init_winctx_regs()
384 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL); in init_winctx_regs()
387 val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port); in init_winctx_regs()
388 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val); in init_winctx_regs()
391 val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid); in init_winctx_regs()
392 write_hvwc_reg(window, VREG(PSWID), val); in init_winctx_regs()
394 write_hvwc_reg(window, VREG(SPARE1), 0ULL); in init_winctx_regs()
395 write_hvwc_reg(window, VREG(SPARE2), 0ULL); in init_winctx_regs()
396 write_hvwc_reg(window, VREG(SPARE3), 0ULL); in init_winctx_regs()
400 * register as is - do NOT shift the address into VAS_LFIFO_BAR in init_winctx_regs()
401 * bit fields! Ok to set the page migration select fields - in init_winctx_regs()
407 val = winctx->rx_fifo; in init_winctx_regs()
409 write_hvwc_reg(window, VREG(LFIFO_BAR), val); in init_winctx_regs()
412 val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp); in init_winctx_regs()
413 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val); in init_winctx_regs()
416 val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type); in init_winctx_regs()
417 val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable); in init_winctx_regs()
418 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val); in init_winctx_regs()
420 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL); in init_winctx_regs()
421 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL); in init_winctx_regs()
422 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL); in init_winctx_regs()
425 val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max); in init_winctx_regs()
426 write_hvwc_reg(window, VREG(LRX_WCRED), val); in init_winctx_regs()
429 val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max); in init_winctx_regs()
430 write_hvwc_reg(window, VREG(TX_WCRED), val); in init_winctx_regs()
432 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL); in init_winctx_regs()
433 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL); in init_winctx_regs()
435 fifo_size = winctx->rx_fifo_size / 1024; in init_winctx_regs()
439 write_hvwc_reg(window, VREG(LFIFO_SIZE), val); in init_winctx_regs()
441 /* Update window control and caching control registers last so in init_winctx_regs()
442 * we mark the window open only after fully initializing it and in init_winctx_regs()
446 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL); in init_winctx_regs()
448 init_rsvd_tx_buf_count(window, winctx); in init_winctx_regs()
450 /* for a send window, point to the matching receive window */ in init_winctx_regs()
452 val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id); in init_winctx_regs()
453 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val); in init_winctx_regs()
455 write_hvwc_reg(window, VREG(SPARE4), 0ULL); in init_winctx_regs()
458 val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable); in init_winctx_regs()
459 val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable); in init_winctx_regs()
460 val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early); in init_winctx_regs()
461 val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg); in init_winctx_regs()
462 write_hvwc_reg(window, VREG(LNOTIFY_CTL), val); in init_winctx_regs()
465 val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid); in init_winctx_regs()
466 write_hvwc_reg(window, VREG(LNOTIFY_PID), val); in init_winctx_regs()
469 val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid); in init_winctx_regs()
470 write_hvwc_reg(window, VREG(LNOTIFY_LPID), val); in init_winctx_regs()
473 val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid); in init_winctx_regs()
474 write_hvwc_reg(window, VREG(LNOTIFY_TID), val); in init_winctx_regs()
477 val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope); in init_winctx_regs()
478 val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope); in init_winctx_regs()
479 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val); in init_winctx_regs()
481 /* Skip read-only registers NX_UTIL and NX_UTIL_SE */ in init_winctx_regs()
483 write_hvwc_reg(window, VREG(SPARE5), 0ULL); in init_winctx_regs()
484 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL); in init_winctx_regs()
485 write_hvwc_reg(window, VREG(SPARE6), 0ULL); in init_winctx_regs()
487 /* Finally, push window context to memory and... */ in init_winctx_regs()
490 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val); in init_winctx_regs()
492 /* ... mark the window open for business */ in init_winctx_regs()
494 val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit); in init_winctx_regs()
495 val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win); in init_winctx_regs()
496 val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode); in init_winctx_regs()
497 val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode); in init_winctx_regs()
498 val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode); in init_winctx_regs()
499 val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode); in init_winctx_regs()
500 val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win); in init_winctx_regs()
501 val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win); in init_winctx_regs()
503 write_hvwc_reg(window, VREG(WINCTL), val); in init_winctx_regs()
513 int winid = ida_alloc_max(ida, VAS_WINDOWS_PER_CHIP - 1, GFP_KERNEL); in vas_assign_window_id()
515 if (winid == -ENOSPC) { in vas_assign_window_id()
517 return -EAGAIN; in vas_assign_window_id()
523 static void vas_window_free(struct pnv_vas_window *window) in vas_window_free() argument
525 struct vas_instance *vinst = window->vinst; in vas_window_free()
526 int winid = window->vas_win.winid; in vas_window_free()
528 unmap_winctx_mmio_bars(window); in vas_window_free()
530 vas_window_free_dbgdir(window); in vas_window_free()
532 kfree(window); in vas_window_free()
534 vas_release_window_id(&vinst->ida, winid); in vas_window_free()
540 struct pnv_vas_window *window; in vas_window_alloc() local
542 winid = vas_assign_window_id(&vinst->ida); in vas_window_alloc()
546 window = kzalloc(sizeof(*window), GFP_KERNEL); in vas_window_alloc()
547 if (!window) in vas_window_alloc()
550 window->vinst = vinst; in vas_window_alloc()
551 window->vas_win.winid = winid; in vas_window_alloc()
553 if (map_winctx_mmio_bars(window)) in vas_window_alloc()
556 vas_window_init_dbgdir(window); in vas_window_alloc()
558 return window; in vas_window_alloc()
561 kfree(window); in vas_window_alloc()
562 vas_release_window_id(&vinst->ida, winid); in vas_window_alloc()
563 return ERR_PTR(-ENOMEM); in vas_window_alloc()
568 /* Better not be a send window! */ in put_rx_win()
569 WARN_ON_ONCE(rxwin->tx_win); in put_rx_win()
571 atomic_dec(&rxwin->num_txwins); in put_rx_win()
575 * Find the user space receive window given the @pswid.
576 * - We must have a valid vasid and it must belong to this instance.
578 * - The window must refer to an OPEN, FTW, RECEIVE window.
580 * NOTE: We access ->windows[] table and assume that vinst->mutex is held.
590 if (vinst->vas_id != vasid) in get_user_rxwin()
591 return ERR_PTR(-EINVAL); in get_user_rxwin()
593 rxwin = vinst->windows[winid]; in get_user_rxwin()
595 if (!rxwin || rxwin->tx_win || rxwin->vas_win.cop != VAS_COP_TYPE_FTW) in get_user_rxwin()
596 return ERR_PTR(-EINVAL); in get_user_rxwin()
602 * Get the VAS receive window associated with NX engine identified
605 * See also function header of set_vinst_win().
612 mutex_lock(&vinst->mutex); in get_vinst_rxwin()
617 rxwin = vinst->rxwin[cop] ?: ERR_PTR(-EINVAL); in get_vinst_rxwin()
620 atomic_inc(&rxwin->num_txwins); in get_vinst_rxwin()
622 mutex_unlock(&vinst->mutex); in get_vinst_rxwin()
628 * We have two tables of windows in a VAS instance. The first one,
629 * ->windows[], contains all the windows in the instance and allows
630 * looking up a window by its id. It is used to look up send windows
634 * The second table, ->rxwin[], contains receive windows that are
636 * entries and is used to look up a receive window by its
639 * Here, we save @window in the ->windows[] table. If it is a receive
640 * window, we also save the window in the ->rxwin[] table.
643 struct pnv_vas_window *window) in set_vinst_win() argument
645 int id = window->vas_win.winid; in set_vinst_win()
647 mutex_lock(&vinst->mutex); in set_vinst_win()
650 * There should only be one receive window for a coprocessor type in set_vinst_win()
651 * unless its a user (FTW) window. in set_vinst_win()
653 if (!window->user_win && !window->tx_win) { in set_vinst_win()
654 WARN_ON_ONCE(vinst->rxwin[window->vas_win.cop]); in set_vinst_win()
655 vinst->rxwin[window->vas_win.cop] = window; in set_vinst_win()
658 WARN_ON_ONCE(vinst->windows[id] != NULL); in set_vinst_win()
659 vinst->windows[id] = window; in set_vinst_win()
661 mutex_unlock(&vinst->mutex); in set_vinst_win()
665 * Clear this window from the table(s) of windows for this VAS instance.
666 * See also function header of set_vinst_win().
668 static void clear_vinst_win(struct pnv_vas_window *window) in clear_vinst_win() argument
670 int id = window->vas_win.winid; in clear_vinst_win()
671 struct vas_instance *vinst = window->vinst; in clear_vinst_win()
673 mutex_lock(&vinst->mutex); in clear_vinst_win()
675 if (!window->user_win && !window->tx_win) { in clear_vinst_win()
676 WARN_ON_ONCE(!vinst->rxwin[window->vas_win.cop]); in clear_vinst_win()
677 vinst->rxwin[window->vas_win.cop] = NULL; in clear_vinst_win()
680 WARN_ON_ONCE(vinst->windows[id] != window); in clear_vinst_win()
681 vinst->windows[id] = NULL; in clear_vinst_win()
683 mutex_unlock(&vinst->mutex); in clear_vinst_win()
691 * We first zero (memset()) all fields and only set non-zero fields. in init_winctx_for_rxwin()
694 * ->notify_os_intr_reg In powerNV, send intrs to HV in init_winctx_for_rxwin()
695 * ->notify_disable False for NX windows in init_winctx_for_rxwin()
696 * ->intr_disable False for Fault Windows in init_winctx_for_rxwin()
697 * ->xtra_write False for NX windows in init_winctx_for_rxwin()
698 * ->notify_early NA for NX windows in init_winctx_for_rxwin()
699 * ->rsvd_txbuf_count NA for Rx windows in init_winctx_for_rxwin()
700 * ->lpid, ->pid, ->tid NA for Rx windows in init_winctx_for_rxwin()
705 winctx->rx_fifo = rxattr->rx_fifo; in init_winctx_for_rxwin()
706 winctx->rx_fifo_size = rxattr->rx_fifo_size; in init_winctx_for_rxwin()
707 winctx->wcreds_max = rxwin->vas_win.wcreds_max; in init_winctx_for_rxwin()
708 winctx->pin_win = rxattr->pin_win; in init_winctx_for_rxwin()
710 winctx->nx_win = rxattr->nx_win; in init_winctx_for_rxwin()
711 winctx->fault_win = rxattr->fault_win; in init_winctx_for_rxwin()
712 winctx->user_win = rxattr->user_win; in init_winctx_for_rxwin()
713 winctx->rej_no_credit = rxattr->rej_no_credit; in init_winctx_for_rxwin()
714 winctx->rx_word_mode = rxattr->rx_win_ord_mode; in init_winctx_for_rxwin()
715 winctx->tx_word_mode = rxattr->tx_win_ord_mode; in init_winctx_for_rxwin()
716 winctx->rx_wcred_mode = rxattr->rx_wcred_mode; in init_winctx_for_rxwin()
717 winctx->tx_wcred_mode = rxattr->tx_wcred_mode; in init_winctx_for_rxwin()
718 winctx->notify_early = rxattr->notify_early; in init_winctx_for_rxwin()
720 if (winctx->nx_win) { in init_winctx_for_rxwin()
721 winctx->data_stamp = true; in init_winctx_for_rxwin()
722 winctx->intr_disable = true; in init_winctx_for_rxwin()
723 winctx->pin_win = true; in init_winctx_for_rxwin()
725 WARN_ON_ONCE(winctx->fault_win); in init_winctx_for_rxwin()
726 WARN_ON_ONCE(!winctx->rx_word_mode); in init_winctx_for_rxwin()
727 WARN_ON_ONCE(!winctx->tx_word_mode); in init_winctx_for_rxwin()
728 WARN_ON_ONCE(winctx->notify_after_count); in init_winctx_for_rxwin()
729 } else if (winctx->fault_win) { in init_winctx_for_rxwin()
730 winctx->notify_disable = true; in init_winctx_for_rxwin()
731 } else if (winctx->user_win) { in init_winctx_for_rxwin()
733 * Section 1.8.1 Low Latency Core-Core Wake up of in init_winctx_for_rxwin()
736 * - disable credit checks ([tr]x_wcred_mode = false) in init_winctx_for_rxwin()
737 * - disable FIFO writes in init_winctx_for_rxwin()
738 * - enable ASB_Notify, disable interrupt in init_winctx_for_rxwin()
740 winctx->fifo_disable = true; in init_winctx_for_rxwin()
741 winctx->intr_disable = true; in init_winctx_for_rxwin()
742 winctx->rx_fifo = 0; in init_winctx_for_rxwin()
745 winctx->lnotify_lpid = rxattr->lnotify_lpid; in init_winctx_for_rxwin()
746 winctx->lnotify_pid = rxattr->lnotify_pid; in init_winctx_for_rxwin()
747 winctx->lnotify_tid = rxattr->lnotify_tid; in init_winctx_for_rxwin()
748 winctx->pswid = rxattr->pswid; in init_winctx_for_rxwin()
749 winctx->dma_type = VAS_DMA_TYPE_INJECT; in init_winctx_for_rxwin()
750 winctx->tc_mode = rxattr->tc_mode; in init_winctx_for_rxwin()
752 winctx->min_scope = VAS_SCOPE_LOCAL; in init_winctx_for_rxwin()
753 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP; in init_winctx_for_rxwin()
754 if (rxwin->vinst->virq) in init_winctx_for_rxwin()
755 winctx->irq_port = rxwin->vinst->irq_port; in init_winctx_for_rxwin()
762 attr->fault_win, attr->notify_disable, in rx_win_args_valid()
763 attr->intr_disable, attr->notify_early, in rx_win_args_valid()
764 attr->rx_fifo_size); in rx_win_args_valid()
770 attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN) in rx_win_args_valid()
773 if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX) in rx_win_args_valid()
776 if (!attr->wcreds_max) in rx_win_args_valid()
779 if (attr->nx_win) { in rx_win_args_valid()
780 /* cannot be fault or user window if it is nx */ in rx_win_args_valid()
781 if (attr->fault_win || attr->user_win) in rx_win_args_valid()
787 if (attr->notify_disable || !attr->intr_disable || in rx_win_args_valid()
788 attr->notify_early) in rx_win_args_valid()
790 } else if (attr->fault_win) { in rx_win_args_valid()
791 /* cannot be both fault and user window */ in rx_win_args_valid()
792 if (attr->user_win) in rx_win_args_valid()
799 if (!attr->notify_disable || attr->intr_disable) in rx_win_args_valid()
802 } else if (attr->user_win) { in rx_win_args_valid()
804 * User receive windows are only for fast-thread-wakeup in rx_win_args_valid()
807 if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable) in rx_win_args_valid()
810 /* Rx window must be one of NX or Fault or User window. */ in rx_win_args_valid()
823 rxattr->pin_win = true; in vas_init_rx_win_attr()
824 rxattr->nx_win = true; in vas_init_rx_win_attr()
825 rxattr->fault_win = false; in vas_init_rx_win_attr()
826 rxattr->intr_disable = true; in vas_init_rx_win_attr()
827 rxattr->rx_wcred_mode = true; in vas_init_rx_win_attr()
828 rxattr->tx_wcred_mode = true; in vas_init_rx_win_attr()
829 rxattr->rx_win_ord_mode = true; in vas_init_rx_win_attr()
830 rxattr->tx_win_ord_mode = true; in vas_init_rx_win_attr()
832 rxattr->pin_win = true; in vas_init_rx_win_attr()
833 rxattr->fault_win = true; in vas_init_rx_win_attr()
834 rxattr->notify_disable = true; in vas_init_rx_win_attr()
835 rxattr->rx_wcred_mode = true; in vas_init_rx_win_attr()
836 rxattr->rx_win_ord_mode = true; in vas_init_rx_win_attr()
837 rxattr->rej_no_credit = true; in vas_init_rx_win_attr()
838 rxattr->tc_mode = VAS_THRESH_DISABLED; in vas_init_rx_win_attr()
840 rxattr->user_win = true; in vas_init_rx_win_attr()
841 rxattr->intr_disable = true; in vas_init_rx_win_attr()
863 return ERR_PTR(-EINVAL); in vas_rx_win_open()
868 return ERR_PTR(-EINVAL); in vas_rx_win_open()
874 pr_devel("Unable to allocate memory for Rx window\n"); in vas_rx_win_open()
878 rxwin->tx_win = false; in vas_rx_win_open()
879 rxwin->nx_win = rxattr->nx_win; in vas_rx_win_open()
880 rxwin->user_win = rxattr->user_win; in vas_rx_win_open()
881 rxwin->vas_win.cop = cop; in vas_rx_win_open()
882 rxwin->vas_win.wcreds_max = rxattr->wcreds_max; in vas_rx_win_open()
889 return &rxwin->vas_win; in vas_rx_win_open()
899 txattr->rej_no_credit = false; in vas_init_tx_win_attr()
900 txattr->rx_wcred_mode = true; in vas_init_tx_win_attr()
901 txattr->tx_wcred_mode = true; in vas_init_tx_win_attr()
902 txattr->rx_win_ord_mode = true; in vas_init_tx_win_attr()
903 txattr->tx_win_ord_mode = true; in vas_init_tx_win_attr()
905 txattr->user_win = true; in vas_init_tx_win_attr()
915 * We first zero all fields and only set non-zero ones. Following in init_winctx_for_txwin()
918 * ->notify_os_intr_reg In powernv, send intrs to HV in init_winctx_for_txwin()
919 * ->rsvd_txbuf_count Not supported yet. in init_winctx_for_txwin()
920 * ->notify_disable False for NX windows in init_winctx_for_txwin()
921 * ->xtra_write False for NX windows in init_winctx_for_txwin()
922 * ->notify_early NA for NX windows in init_winctx_for_txwin()
923 * ->lnotify_lpid NA for Tx windows in init_winctx_for_txwin()
924 * ->lnotify_pid NA for Tx windows in init_winctx_for_txwin()
925 * ->lnotify_tid NA for Tx windows in init_winctx_for_txwin()
926 * ->tx_win_cred_mode Ignore for now for NX windows in init_winctx_for_txwin()
927 * ->rx_win_cred_mode Ignore for now for NX windows in init_winctx_for_txwin()
931 winctx->wcreds_max = txwin->vas_win.wcreds_max; in init_winctx_for_txwin()
933 winctx->user_win = txattr->user_win; in init_winctx_for_txwin()
934 winctx->nx_win = txwin->rxwin->nx_win; in init_winctx_for_txwin()
935 winctx->pin_win = txattr->pin_win; in init_winctx_for_txwin()
936 winctx->rej_no_credit = txattr->rej_no_credit; in init_winctx_for_txwin()
937 winctx->rsvd_txbuf_enable = txattr->rsvd_txbuf_enable; in init_winctx_for_txwin()
939 winctx->rx_wcred_mode = txattr->rx_wcred_mode; in init_winctx_for_txwin()
940 winctx->tx_wcred_mode = txattr->tx_wcred_mode; in init_winctx_for_txwin()
941 winctx->rx_word_mode = txattr->rx_win_ord_mode; in init_winctx_for_txwin()
942 winctx->tx_word_mode = txattr->tx_win_ord_mode; in init_winctx_for_txwin()
943 winctx->rsvd_txbuf_count = txattr->rsvd_txbuf_count; in init_winctx_for_txwin()
945 winctx->intr_disable = true; in init_winctx_for_txwin()
946 if (winctx->nx_win) in init_winctx_for_txwin()
947 winctx->data_stamp = true; in init_winctx_for_txwin()
949 winctx->lpid = txattr->lpid; in init_winctx_for_txwin()
950 winctx->pidr = txattr->pidr; in init_winctx_for_txwin()
951 winctx->rx_win_id = txwin->rxwin->vas_win.winid; in init_winctx_for_txwin()
953 * IRQ and fault window setup is successful. Set fault window in init_winctx_for_txwin()
954 * for the send window so that ready to handle faults. in init_winctx_for_txwin()
956 if (txwin->vinst->virq) in init_winctx_for_txwin()
957 winctx->fault_win_id = txwin->vinst->fault_win->vas_win.winid; in init_winctx_for_txwin()
959 winctx->dma_type = VAS_DMA_TYPE_INJECT; in init_winctx_for_txwin()
960 winctx->tc_mode = txattr->tc_mode; in init_winctx_for_txwin()
961 winctx->min_scope = VAS_SCOPE_LOCAL; in init_winctx_for_txwin()
962 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP; in init_winctx_for_txwin()
963 if (txwin->vinst->virq) in init_winctx_for_txwin()
964 winctx->irq_port = txwin->vinst->irq_port; in init_winctx_for_txwin()
966 winctx->pswid = txattr->pswid ? txattr->pswid : in init_winctx_for_txwin()
967 encode_pswid(txwin->vinst->vas_id, in init_winctx_for_txwin()
968 txwin->vas_win.winid); in init_winctx_for_txwin()
974 if (attr->tc_mode != VAS_THRESH_DISABLED) in tx_win_args_valid()
980 if (attr->wcreds_max > VAS_TX_WCREDS_MAX) in tx_win_args_valid()
983 if (attr->user_win) { in tx_win_args_valid()
984 if (attr->rsvd_txbuf_count) in tx_win_args_valid()
1007 return ERR_PTR(-EINVAL); in vas_tx_win_open()
1010 * If caller did not specify a vasid but specified the PSWID of a in vas_tx_win_open()
1011 * receive window (applicable only to FTW windows), use the vasid in vas_tx_win_open()
1012 * from that receive window. in vas_tx_win_open()
1014 if (vasid == -1 && attr->pswid) in vas_tx_win_open()
1015 decode_pswid(attr->pswid, &vasid, NULL); in vas_tx_win_open()
1020 return ERR_PTR(-EINVAL); in vas_tx_win_open()
1023 rxwin = get_vinst_rxwin(vinst, cop, attr->pswid); in vas_tx_win_open()
1035 txwin->vas_win.cop = cop; in vas_tx_win_open()
1036 txwin->tx_win = 1; in vas_tx_win_open()
1037 txwin->rxwin = rxwin; in vas_tx_win_open()
1038 txwin->nx_win = txwin->rxwin->nx_win; in vas_tx_win_open()
1039 txwin->user_win = attr->user_win; in vas_tx_win_open()
1040 txwin->vas_win.wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT; in vas_tx_win_open()
1047 * If its a kernel send window, map the window address into the in vas_tx_win_open()
1049 * mmap() to map the window into their address space. in vas_tx_win_open()
1054 if (!txwin->user_win) { in vas_tx_win_open()
1055 txwin->paste_kaddr = map_paste_region(txwin); in vas_tx_win_open()
1056 if (IS_ERR(txwin->paste_kaddr)) { in vas_tx_win_open()
1057 rc = PTR_ERR(txwin->paste_kaddr); in vas_tx_win_open()
1062 * Interrupt handler or fault window setup failed. Means in vas_tx_win_open()
1064 * opening for user space tx window. in vas_tx_win_open()
1066 if (!vinst->virq) { in vas_tx_win_open()
1067 rc = -ENODEV; in vas_tx_win_open()
1070 rc = get_vas_user_win_ref(&txwin->vas_win.task_ref); in vas_tx_win_open()
1074 vas_user_win_add_mm_context(&txwin->vas_win.task_ref); in vas_tx_win_open()
1079 return &txwin->vas_win; in vas_tx_win_open()
1110 * report-enable flag is set for NX windows. Ensure software in vas_paste_crb()
1113 WARN_ON_ONCE(txwin->nx_win && !re); in vas_paste_crb()
1115 addr = txwin->paste_kaddr; in vas_paste_crb()
1119 * to 1K offset of the paste address) in vas_paste_crb()
1133 rc = -EINVAL; in vas_paste_crb()
1135 pr_debug("Txwin #%d: Msg count %llu\n", txwin->vas_win.winid, in vas_paste_crb()
1143 * If credit checking is enabled for this window, poll for the return
1144 * of window credits (i.e for NX engines to process any outstanding CRBs).
1145 * Since NX-842 waits for the CRBs to be processed before closing the
1146 * window, we should not have to wait for too long.
1149 * the VAS_LRFIFO_PUSH_OFFSET register to get an estimate of pending
1151 * But that is not really needed until we support NX-GZIP access from
1152 * user space. (NX-842 driver waits for CSB and Fast thread-wakeup
1155 static void poll_window_credits(struct pnv_vas_window *window) in poll_window_credits() argument
1161 val = read_hvwc_reg(window, VREG(WINCTL)); in poll_window_credits()
1162 if (window->tx_win) in poll_window_credits()
1170 if (window->tx_win) { in poll_window_credits()
1171 val = read_hvwc_reg(window, VREG(TX_WCRED)); in poll_window_credits()
1174 val = read_hvwc_reg(window, VREG(LRX_WCRED)); in poll_window_credits()
1181 * TODO: Scan fault FIFO and invalidate CRBs points to this window in poll_window_credits()
1185 if (creds < window->vas_win.wcreds_max) { in poll_window_credits()
1191 * Process can not close send window until all credits are in poll_window_credits()
1195 …pr_warn_ratelimited("VAS: pid %d stuck. Waiting for credits returned for Window(%d). creds %d, Ret… in poll_window_credits()
1196 vas_window_pid(&window->vas_win), in poll_window_credits()
1197 window->vas_win.winid, in poll_window_credits()
1205 * Wait for the window to go to "not-busy" state. It should only take a
1206 * short time to queue a CRB, so window should not be busy for too long.
1209 static void poll_window_busy_state(struct pnv_vas_window *window) in poll_window_busy_state() argument
1216 val = read_hvwc_reg(window, VREG(WIN_STATUS)); in poll_window_busy_state()
1228 pr_warn_ratelimited("VAS: pid %d stuck. Window (ID=%d) is in busy state. Retries %d\n", in poll_window_busy_state()
1229 vas_window_pid(&window->vas_win), in poll_window_busy_state()
1230 window->vas_win.winid, count); in poll_window_busy_state()
1237 * Have the hardware cast a window out of cache and wait for it to
1240 * NOTE: It can take a relatively long time to cast the window context
1241 * out of the cache. It is not strictly necessary to cast out if:
1243 * - we clear the "Pin Window" bit (so hardware is free to evict)
1245 * - we re-initialize the window context when it is reassigned.
1248 * So, ignoring the cast-out for now. We can add it as needed. If
1249 * casting out becomes necessary we should consider offloading the
1250 * job to a worker thread, so the window close can proceed quickly.
1252 static void poll_window_castout(struct pnv_vas_window *window) in poll_window_castout() argument
1258 * Unpin and close a window so no new requests are accepted and the
1259 * hardware can evict this window from cache if necessary.
1261 static void unpin_close_window(struct pnv_vas_window *window) in unpin_close_window() argument
1265 val = read_hvwc_reg(window, VREG(WINCTL)); in unpin_close_window()
1268 write_hvwc_reg(window, VREG(WINCTL), val); in unpin_close_window()
1272 * Close a window.
1274 * See Section 1.12.1 of VAS workbook v1.05 for details on closing window:
1275 * - Disable new paste operations (unmap paste address)
1276 * - Poll for the "Window Busy" bit to be cleared
1277 * - Clear the Open/Enable bit for the Window.
1278 * - Poll for return of window Credits (implies FIFO empty for Rx win?)
1279 * - Unpin and cast window context out of cache
1281 * Besides the hardware, kernel has some bookkeeping of course.
1285 struct pnv_vas_window *window; in vas_win_close() local
1290 window = container_of(vwin, struct pnv_vas_window, vas_win); in vas_win_close()
1292 if (!window->tx_win && atomic_read(&window->num_txwins) != 0) { in vas_win_close()
1293 pr_devel("Attempting to close an active Rx window!\n"); in vas_win_close()
1295 return -EBUSY; in vas_win_close()
1298 unmap_paste_region(window); in vas_win_close()
1300 poll_window_busy_state(window); in vas_win_close()
1302 unpin_close_window(window); in vas_win_close()
1304 poll_window_credits(window); in vas_win_close()
1306 clear_vinst_win(window); in vas_win_close()
1308 poll_window_castout(window); in vas_win_close()
1310 /* if send window, drop reference to matching receive window */ in vas_win_close()
1311 if (window->tx_win) { in vas_win_close()
1312 if (window->user_win) { in vas_win_close()
1313 mm_context_remove_vas_window(vwin->task_ref.mm); in vas_win_close()
1314 put_vas_user_win_ref(&vwin->task_ref); in vas_win_close()
1316 put_rx_win(window->rxwin); in vas_win_close()
1319 vas_window_free(window); in vas_win_close()
1326 * Return credit for the given window.
1327 * Send windows and fault window uses credit mechanism as follows:
1330 * - The default number of credits available for each send window is
1334 * - One credit is taken when NX request is issued.
1335 * - This credit is returned after NX processed that request.
1336 * - If NX encounters translation error, kernel will return the
1337 * credit on the specific send window after processing the fault CRB.
1339 * Fault window:
1340 * - The total number credits available is FIFO_SIZE/CRB_SIZE.
1343 * - A credit is taken when NX pastes CRB in fault FIFO.
1344 * - The kernel with return credit on fault window after reading entry
1347 void vas_return_credit(struct pnv_vas_window *window, bool tx) in vas_return_credit() argument
1352 if (tx) { /* send window */ in vas_return_credit()
1354 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), val); in vas_return_credit()
1357 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), val); in vas_return_credit()
1364 struct pnv_vas_window *window; in vas_pswid_to_window() local
1369 return ERR_PTR(-ESRCH); in vas_pswid_to_window()
1375 return ERR_PTR(-ESRCH); in vas_pswid_to_window()
1378 * If application closes the window before the hardware in vas_pswid_to_window()
1380 * for the pending requests. so the window must be active in vas_pswid_to_window()
1386 window = vinst->windows[winid]; in vas_pswid_to_window()
1388 if (!window) { in vas_pswid_to_window()
1389 pr_err("PSWID decode: Could not find window for winid %d pswid %d vinst 0x%p\n", in vas_pswid_to_window()
1395 * Do some sanity checks on the decoded window. Window should be in vas_pswid_to_window()
1396 * NX GZIP user send window. FTW windows should not incur faults in vas_pswid_to_window()
1400 if (!window->tx_win || !window->user_win || !window->nx_win || in vas_pswid_to_window()
1401 window->vas_win.cop == VAS_COP_TYPE_FAULT || in vas_pswid_to_window()
1402 window->vas_win.cop == VAS_COP_TYPE_FTW) { in vas_pswid_to_window()
1404 winid, window->tx_win, window->user_win, in vas_pswid_to_window()
1405 window->nx_win, window->vas_win.cop); in vas_pswid_to_window()
1409 return window; in vas_pswid_to_window()
1456 * Supporting only nx-gzip coprocessor type now, but this API code