1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Copyright 2016-17 IBM Corp. 4 */ 5 6 #ifndef _VAS_H 7 #define _VAS_H 8 #include <linux/atomic.h> 9 #include <linux/idr.h> 10 #include <asm/vas.h> 11 #include <linux/io.h> 12 #include <linux/dcache.h> 13 #include <linux/mutex.h> 14 #include <linux/stringify.h> 15 16 /* 17 * Overview of Virtual Accelerator Switchboard (VAS). 18 * 19 * VAS is a hardware "switchboard" that allows senders and receivers to 20 * exchange messages with _minimal_ kernel involvment. The receivers are 21 * typically NX coprocessor engines that perform compression or encryption 22 * in hardware, but receivers can also be other software threads. 23 * 24 * Senders are user/kernel threads that submit compression/encryption or 25 * other requests to the receivers. Senders must format their messages as 26 * Coprocessor Request Blocks (CRB)s and submit them using the "copy" and 27 * "paste" instructions which were introduced in Power9. 28 * 29 * A Power node can have (upto?) 8 Power chips. There is one instance of 30 * VAS in each Power9 chip. Each instance of VAS has 64K windows or ports, 31 * Senders and receivers must each connect to a separate window before they 32 * can exchange messages through the switchboard. 33 * 34 * Each window is described by two types of window contexts: 35 * 36 * Hypervisor Window Context (HVWC) of size VAS_HVWC_SIZE bytes 37 * 38 * OS/User Window Context (UWC) of size VAS_UWC_SIZE bytes. 39 * 40 * A window context can be viewed as a set of 64-bit registers. The settings 41 * in these registers configure/control/determine the behavior of the VAS 42 * hardware when messages are sent/received through the window. The registers 43 * in the HVWC are configured by the kernel while the registers in the UWC can 44 * be configured by the kernel or by the user space application that is using 45 * the window. 46 * 47 * The HVWCs for all windows on a specific instance of VAS are in a contiguous 48 * range of hardware addresses or Base address region (BAR) referred to as the 49 * HVWC BAR for the instance. Similarly the UWCs for all windows on an instance 50 * are referred to as the UWC BAR for the instance. 51 * 52 * The two BARs for each instance are defined Power9 MMIO Ranges spreadsheet 53 * and available to the kernel in the VAS node's "reg" property in the device 54 * tree: 55 * 56 * /proc/device-tree/vasm@.../reg 57 * 58 * (see vas_probe() for details on the reg property). 59 * 60 * The kernel maps the HVWC and UWC BAR regions into the kernel address 61 * space (hvwc_map and uwc_map). The kernel can then access the window 62 * contexts of a specific window using: 63 * 64 * hvwc = hvwc_map + winid * VAS_HVWC_SIZE. 65 * uwc = uwc_map + winid * VAS_UWC_SIZE. 66 * 67 * where winid is the window index (0..64K). 68 * 69 * As mentioned, a window context is used to "configure" a window. Besides 70 * this configuration address, each _send_ window also has a unique hardware 71 * "paste" address that is used to submit requests/CRBs (see vas_paste_crb()). 72 * 73 * The hardware paste address for a window is computed using the "paste 74 * base address" and "paste win id shift" reg properties in the VAS device 75 * tree node using: 76 * 77 * paste_addr = paste_base + ((winid << paste_win_id_shift)) 78 * 79 * (again, see vas_probe() for ->paste_base_addr and ->paste_win_id_shift). 80 * 81 * The kernel maps this hardware address into the sender's address space 82 * after which they can use the 'paste' instruction (new in Power9) to 83 * send a message (submit a request aka CRB) to the coprocessor. 84 * 85 * NOTE: In the initial version, senders can only in-kernel drivers/threads. 86 * Support for user space threads will be added in follow-on patches. 87 * 88 * TODO: Do we need to map the UWC into user address space so they can return 89 * credits? Its NA for NX but may be needed for other receive windows. 90 * 91 */ 92 93 #define VAS_WINDOWS_PER_CHIP (64 << 10) 94 95 /* 96 * Hypervisor and OS/USer Window Context sizes 97 */ 98 #define VAS_HVWC_SIZE 512 99 #define VAS_UWC_SIZE PAGE_SIZE 100 101 /* 102 * Initial per-process credits. 103 * Max send window credits: 4K-1 (12-bits in VAS_TX_WCRED) 104 * 105 * TODO: Needs tuning for per-process credits 106 */ 107 #define VAS_TX_WCREDS_MAX ((4 << 10) - 1) 108 #define VAS_WCREDS_DEFAULT (1 << 10) 109 110 /* 111 * VAS Window Context Register Offsets and bitmasks. 112 * See Section 3.1.4 of VAS Work book 113 */ 114 #define VAS_LPID_OFFSET 0x010 115 #define VAS_LPID PPC_BITMASK(0, 11) 116 117 #define VAS_PID_OFFSET 0x018 118 #define VAS_PID_ID PPC_BITMASK(0, 19) 119 120 #define VAS_XLATE_MSR_OFFSET 0x020 121 #define VAS_XLATE_MSR_DR PPC_BIT(0) 122 #define VAS_XLATE_MSR_TA PPC_BIT(1) 123 #define VAS_XLATE_MSR_PR PPC_BIT(2) 124 #define VAS_XLATE_MSR_US PPC_BIT(3) 125 #define VAS_XLATE_MSR_HV PPC_BIT(4) 126 #define VAS_XLATE_MSR_SF PPC_BIT(5) 127 128 #define VAS_XLATE_LPCR_OFFSET 0x028 129 #define VAS_XLATE_LPCR_PAGE_SIZE PPC_BITMASK(0, 2) 130 #define VAS_XLATE_LPCR_ISL PPC_BIT(3) 131 #define VAS_XLATE_LPCR_TC PPC_BIT(4) 132 #define VAS_XLATE_LPCR_SC PPC_BIT(5) 133 134 #define VAS_XLATE_CTL_OFFSET 0x030 135 #define VAS_XLATE_MODE PPC_BITMASK(0, 1) 136 137 #define VAS_AMR_OFFSET 0x040 138 #define VAS_AMR PPC_BITMASK(0, 63) 139 140 #define VAS_SEIDR_OFFSET 0x048 141 #define VAS_SEIDR PPC_BITMASK(0, 63) 142 143 #define VAS_FAULT_TX_WIN_OFFSET 0x050 144 #define VAS_FAULT_TX_WIN PPC_BITMASK(48, 63) 145 146 #define VAS_OSU_INTR_SRC_RA_OFFSET 0x060 147 #define VAS_OSU_INTR_SRC_RA PPC_BITMASK(8, 63) 148 149 #define VAS_HV_INTR_SRC_RA_OFFSET 0x070 150 #define VAS_HV_INTR_SRC_RA PPC_BITMASK(8, 63) 151 152 #define VAS_PSWID_OFFSET 0x078 153 #define VAS_PSWID_EA_HANDLE PPC_BITMASK(0, 31) 154 155 #define VAS_SPARE1_OFFSET 0x080 156 #define VAS_SPARE2_OFFSET 0x088 157 #define VAS_SPARE3_OFFSET 0x090 158 #define VAS_SPARE4_OFFSET 0x130 159 #define VAS_SPARE5_OFFSET 0x160 160 #define VAS_SPARE6_OFFSET 0x188 161 162 #define VAS_LFIFO_BAR_OFFSET 0x0A0 163 #define VAS_LFIFO_BAR PPC_BITMASK(8, 53) 164 #define VAS_PAGE_MIGRATION_SELECT PPC_BITMASK(54, 56) 165 166 #define VAS_LDATA_STAMP_CTL_OFFSET 0x0A8 167 #define VAS_LDATA_STAMP PPC_BITMASK(0, 1) 168 #define VAS_XTRA_WRITE PPC_BIT(2) 169 170 #define VAS_LDMA_CACHE_CTL_OFFSET 0x0B0 171 #define VAS_LDMA_TYPE PPC_BITMASK(0, 1) 172 #define VAS_LDMA_FIFO_DISABLE PPC_BIT(2) 173 174 #define VAS_LRFIFO_PUSH_OFFSET 0x0B8 175 #define VAS_LRFIFO_PUSH PPC_BITMASK(0, 15) 176 177 #define VAS_CURR_MSG_COUNT_OFFSET 0x0C0 178 #define VAS_CURR_MSG_COUNT PPC_BITMASK(0, 7) 179 180 #define VAS_LNOTIFY_AFTER_COUNT_OFFSET 0x0C8 181 #define VAS_LNOTIFY_AFTER_COUNT PPC_BITMASK(0, 7) 182 183 #define VAS_LRX_WCRED_OFFSET 0x0E0 184 #define VAS_LRX_WCRED PPC_BITMASK(0, 15) 185 186 #define VAS_LRX_WCRED_ADDER_OFFSET 0x190 187 #define VAS_LRX_WCRED_ADDER PPC_BITMASK(0, 15) 188 189 #define VAS_TX_WCRED_OFFSET 0x0F0 190 #define VAS_TX_WCRED PPC_BITMASK(4, 15) 191 192 #define VAS_TX_WCRED_ADDER_OFFSET 0x1A0 193 #define VAS_TX_WCRED_ADDER PPC_BITMASK(4, 15) 194 195 #define VAS_LFIFO_SIZE_OFFSET 0x100 196 #define VAS_LFIFO_SIZE PPC_BITMASK(0, 3) 197 198 #define VAS_WINCTL_OFFSET 0x108 199 #define VAS_WINCTL_OPEN PPC_BIT(0) 200 #define VAS_WINCTL_REJ_NO_CREDIT PPC_BIT(1) 201 #define VAS_WINCTL_PIN PPC_BIT(2) 202 #define VAS_WINCTL_TX_WCRED_MODE PPC_BIT(3) 203 #define VAS_WINCTL_RX_WCRED_MODE PPC_BIT(4) 204 #define VAS_WINCTL_TX_WORD_MODE PPC_BIT(5) 205 #define VAS_WINCTL_RX_WORD_MODE PPC_BIT(6) 206 #define VAS_WINCTL_RSVD_TXBUF PPC_BIT(7) 207 #define VAS_WINCTL_THRESH_CTL PPC_BITMASK(8, 9) 208 #define VAS_WINCTL_FAULT_WIN PPC_BIT(10) 209 #define VAS_WINCTL_NX_WIN PPC_BIT(11) 210 211 #define VAS_WIN_STATUS_OFFSET 0x110 212 #define VAS_WIN_BUSY PPC_BIT(1) 213 214 #define VAS_WIN_CTX_CACHING_CTL_OFFSET 0x118 215 #define VAS_CASTOUT_REQ PPC_BIT(0) 216 #define VAS_PUSH_TO_MEM PPC_BIT(1) 217 #define VAS_WIN_CACHE_STATUS PPC_BIT(4) 218 219 #define VAS_TX_RSVD_BUF_COUNT_OFFSET 0x120 220 #define VAS_RXVD_BUF_COUNT PPC_BITMASK(58, 63) 221 222 #define VAS_LRFIFO_WIN_PTR_OFFSET 0x128 223 #define VAS_LRX_WIN_ID PPC_BITMASK(0, 15) 224 225 /* 226 * Local Notification Control Register controls what happens in _response_ 227 * to a paste command and hence applies only to receive windows. 228 */ 229 #define VAS_LNOTIFY_CTL_OFFSET 0x138 230 #define VAS_NOTIFY_DISABLE PPC_BIT(0) 231 #define VAS_INTR_DISABLE PPC_BIT(1) 232 #define VAS_NOTIFY_EARLY PPC_BIT(2) 233 #define VAS_NOTIFY_OSU_INTR PPC_BIT(3) 234 235 #define VAS_LNOTIFY_PID_OFFSET 0x140 236 #define VAS_LNOTIFY_PID PPC_BITMASK(0, 19) 237 238 #define VAS_LNOTIFY_LPID_OFFSET 0x148 239 #define VAS_LNOTIFY_LPID PPC_BITMASK(0, 11) 240 241 #define VAS_LNOTIFY_TID_OFFSET 0x150 242 #define VAS_LNOTIFY_TID PPC_BITMASK(0, 15) 243 244 #define VAS_LNOTIFY_SCOPE_OFFSET 0x158 245 #define VAS_LNOTIFY_MIN_SCOPE PPC_BITMASK(0, 1) 246 #define VAS_LNOTIFY_MAX_SCOPE PPC_BITMASK(2, 3) 247 248 #define VAS_NX_UTIL_OFFSET 0x1B0 249 #define VAS_NX_UTIL PPC_BITMASK(0, 63) 250 251 /* SE: Side effects */ 252 #define VAS_NX_UTIL_SE_OFFSET 0x1B8 253 #define VAS_NX_UTIL_SE PPC_BITMASK(0, 63) 254 255 #define VAS_NX_UTIL_ADDER_OFFSET 0x180 256 #define VAS_NX_UTIL_ADDER PPC_BITMASK(32, 63) 257 258 /* 259 * VREG(x): 260 * Expand a register's short name (eg: LPID) into two parameters: 261 * - the register's short name in string form ("LPID"), and 262 * - the name of the macro (eg: VAS_LPID_OFFSET), defining the 263 * register's offset in the window context 264 */ 265 #define VREG_SFX(n, s) __stringify(n), VAS_##n##s 266 #define VREG(r) VREG_SFX(r, _OFFSET) 267 268 /* 269 * Local Notify Scope Control Register. (Receive windows only). 270 */ 271 enum vas_notify_scope { 272 VAS_SCOPE_LOCAL, 273 VAS_SCOPE_GROUP, 274 VAS_SCOPE_VECTORED_GROUP, 275 VAS_SCOPE_UNUSED, 276 }; 277 278 /* 279 * Local DMA Cache Control Register (Receive windows only). 280 */ 281 enum vas_dma_type { 282 VAS_DMA_TYPE_INJECT, 283 VAS_DMA_TYPE_WRITE, 284 }; 285 286 /* 287 * Local Notify Scope Control Register. (Receive windows only). 288 * Not applicable to NX receive windows. 289 */ 290 enum vas_notify_after_count { 291 VAS_NOTIFY_AFTER_256 = 0, 292 VAS_NOTIFY_NONE, 293 VAS_NOTIFY_AFTER_2 294 }; 295 296 /* 297 * NX can generate an interrupt for multiple faults and expects kernel 298 * to process all of them. So read all valid CRB entries until find the 299 * invalid one. So use pswid which is pasted by NX and ccw[0] (reserved 300 * bit in BE) to check valid CRB. CCW[0] will not be touched by user 301 * space. Application gets CRB formt error if it updates this bit. 302 * 303 * Invalidate FIFO during allocation and process all entries from last 304 * successful read until finds invalid pswid and ccw[0] values. 305 * After reading each CRB entry from fault FIFO, the kernel invalidate 306 * it by updating pswid with FIFO_INVALID_ENTRY and CCW[0] with 307 * CCW0_INVALID. 308 */ 309 #define FIFO_INVALID_ENTRY 0xffffffff 310 #define CCW0_INVALID 1 311 312 /* 313 * One per instance of VAS. Each instance will have a separate set of 314 * receive windows, one per coprocessor type. 315 * 316 * See also function header of set_vinst_win() for details on ->windows[] 317 * and ->rxwin[] tables. 318 */ 319 struct vas_instance { 320 int vas_id; 321 struct ida ida; 322 struct list_head node; 323 struct platform_device *pdev; 324 325 u64 hvwc_bar_start; 326 u64 uwc_bar_start; 327 u64 paste_base_addr; 328 u64 paste_win_id_shift; 329 330 u64 irq_port; 331 int virq; 332 int fault_crbs; 333 int fault_fifo_size; 334 int fifo_in_progress; /* To wake up thread or return IRQ_HANDLED */ 335 spinlock_t fault_lock; /* Protects fifo_in_progress update */ 336 void *fault_fifo; 337 struct pnv_vas_window *fault_win; /* Fault window */ 338 339 struct mutex mutex; 340 struct pnv_vas_window *rxwin[VAS_COP_TYPE_MAX]; 341 struct pnv_vas_window *windows[VAS_WINDOWS_PER_CHIP]; 342 343 char *name; 344 char *dbgname; 345 struct dentry *dbgdir; 346 }; 347 348 /* 349 * In-kernel state a VAS window on PowerNV. One per window. 350 */ 351 struct pnv_vas_window { 352 struct vas_window vas_win; 353 /* Fields common to send and receive windows */ 354 struct vas_instance *vinst; 355 bool tx_win; /* True if send window */ 356 bool nx_win; /* True if NX window */ 357 bool user_win; /* True if user space window */ 358 void *hvwc_map; /* HV window context */ 359 void *uwc_map; /* OS/User window context */ 360 361 /* Fields applicable only to send windows */ 362 void *paste_kaddr; 363 char *paste_addr_name; 364 struct pnv_vas_window *rxwin; 365 366 /* Fields applicable only to receive windows */ 367 atomic_t num_txwins; 368 }; 369 370 /* 371 * Container for the hardware state of a window. One per-window. 372 * 373 * A VAS Window context is a 512-byte area in the hardware that contains 374 * a set of 64-bit registers. Individual bit-fields in these registers 375 * determine the configuration/operation of the hardware. struct vas_winctx 376 * is a container for the register fields in the window context. 377 */ 378 struct vas_winctx { 379 u64 rx_fifo; 380 int rx_fifo_size; 381 int wcreds_max; 382 int rsvd_txbuf_count; 383 384 bool user_win; 385 bool nx_win; 386 bool fault_win; 387 bool rsvd_txbuf_enable; 388 bool pin_win; 389 bool rej_no_credit; 390 bool tx_wcred_mode; 391 bool rx_wcred_mode; 392 bool tx_word_mode; 393 bool rx_word_mode; 394 bool data_stamp; 395 bool xtra_write; 396 bool notify_disable; 397 bool intr_disable; 398 bool fifo_disable; 399 bool notify_early; 400 bool notify_os_intr_reg; 401 402 int lpid; 403 int pidr; /* value from SPRN_PID, not linux pid */ 404 int lnotify_lpid; 405 int lnotify_pid; 406 int lnotify_tid; 407 u32 pswid; 408 int rx_win_id; 409 int fault_win_id; 410 int tc_mode; 411 412 u64 irq_port; 413 414 enum vas_dma_type dma_type; 415 enum vas_notify_scope min_scope; 416 enum vas_notify_scope max_scope; 417 enum vas_notify_after_count notify_after_count; 418 }; 419 420 extern struct mutex vas_mutex; 421 422 extern struct vas_instance *find_vas_instance(int vasid); 423 extern void vas_init_dbgdir(void); 424 extern void vas_instance_init_dbgdir(struct vas_instance *vinst); 425 extern void vas_window_init_dbgdir(struct pnv_vas_window *win); 426 extern void vas_window_free_dbgdir(struct pnv_vas_window *win); 427 extern int vas_setup_fault_window(struct vas_instance *vinst); 428 extern irqreturn_t vas_fault_thread_fn(int irq, void *data); 429 extern irqreturn_t vas_fault_handler(int irq, void *dev_id); 430 extern void vas_return_credit(struct pnv_vas_window *window, bool tx); 431 extern struct pnv_vas_window *vas_pswid_to_window(struct vas_instance *vinst, 432 uint32_t pswid); 433 extern void vas_win_paste_addr(struct pnv_vas_window *window, u64 *addr, 434 int *len); 435 436 static inline int vas_window_pid(struct vas_window *window) 437 { 438 return pid_vnr(window->task_ref.pid); 439 } 440 441 static inline void vas_log_write(struct pnv_vas_window *win, char *name, 442 void *regptr, u64 val) 443 { 444 if (val) 445 pr_debug("%swin #%d: %s reg %p, val 0x%016llx\n", 446 win->tx_win ? "Tx" : "Rx", win->vas_win.winid, 447 name, regptr, val); 448 } 449 450 static inline void write_uwc_reg(struct pnv_vas_window *win, char *name, 451 s32 reg, u64 val) 452 { 453 void *regptr; 454 455 regptr = win->uwc_map + reg; 456 vas_log_write(win, name, regptr, val); 457 458 out_be64(regptr, val); 459 } 460 461 static inline void write_hvwc_reg(struct pnv_vas_window *win, char *name, 462 s32 reg, u64 val) 463 { 464 void *regptr; 465 466 regptr = win->hvwc_map + reg; 467 vas_log_write(win, name, regptr, val); 468 469 out_be64(regptr, val); 470 } 471 472 static inline u64 read_hvwc_reg(struct pnv_vas_window *win, 473 char *name __maybe_unused, s32 reg) 474 { 475 return in_be64(win->hvwc_map+reg); 476 } 477 478 /* 479 * Encode/decode the Partition Send Window ID (PSWID) for a window in 480 * a way that we can uniquely identify any window in the system. i.e. 481 * we should be able to locate the 'struct vas_window' given the PSWID. 482 * 483 * Bits Usage 484 * 0:7 VAS id (8 bits) 485 * 8:15 Unused, 0 (3 bits) 486 * 16:31 Window id (16 bits) 487 */ 488 static inline u32 encode_pswid(int vasid, int winid) 489 { 490 return ((u32)winid | (vasid << (31 - 7))); 491 } 492 493 static inline void decode_pswid(u32 pswid, int *vasid, int *winid) 494 { 495 if (vasid) 496 *vasid = pswid >> (31 - 7) & 0xFF; 497 498 if (winid) 499 *winid = pswid & 0xFFFF; 500 } 501 #endif /* _VAS_H */ 502