1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * VAS Fault handling. 4 * Copyright 2019, IBM Corporation 5 */ 6 7 #define pr_fmt(fmt) "vas: " fmt 8 9 #include <linux/kernel.h> 10 #include <linux/types.h> 11 #include <linux/slab.h> 12 #include <linux/uaccess.h> 13 #include <linux/kthread.h> 14 #include <linux/sched/signal.h> 15 #include <linux/mmu_context.h> 16 #include <asm/icswx.h> 17 18 #include "vas.h" 19 20 /* 21 * The maximum FIFO size for fault window can be 8MB 22 * (VAS_RX_FIFO_SIZE_MAX). Using 4MB FIFO since each VAS 23 * instance will be having fault window. 24 * 8MB FIFO can be used if expects more faults for each VAS 25 * instance. 26 */ 27 #define VAS_FAULT_WIN_FIFO_SIZE (4 << 20) 28 29 static void dump_fifo(struct vas_instance *vinst, void *entry) 30 { 31 unsigned long *end = vinst->fault_fifo + vinst->fault_fifo_size; 32 unsigned long *fifo = entry; 33 int i; 34 35 pr_err("Fault fifo size %d, Max crbs %d\n", vinst->fault_fifo_size, 36 vinst->fault_fifo_size / CRB_SIZE); 37 38 /* Dump 10 CRB entries or until end of FIFO */ 39 pr_err("Fault FIFO Dump:\n"); 40 for (i = 0; i < 10*(CRB_SIZE/8) && fifo < end; i += 4, fifo += 4) { 41 pr_err("[%.3d, %p]: 0x%.16lx 0x%.16lx 0x%.16lx 0x%.16lx\n", 42 i, fifo, *fifo, *(fifo+1), *(fifo+2), *(fifo+3)); 43 } 44 } 45 46 /* 47 * Process valid CRBs in fault FIFO. 48 * NX process user space requests, return credit and update the status 49 * in CRB. If it encounters transalation error when accessing CRB or 50 * request buffers, raises interrupt on the CPU to handle the fault. 51 * It takes credit on fault window, updates nx_fault_stamp in CRB with 52 * the following information and pastes CRB in fault FIFO. 53 * 54 * pswid - window ID of the window on which the request is sent. 55 * fault_storage_addr - fault address 56 * 57 * It can raise a single interrupt for multiple faults. Expects OS to 58 * process all valid faults and return credit for each fault on user 59 * space and fault windows. This fault FIFO control will be done with 60 * credit mechanism. NX can continuously paste CRBs until credits are not 61 * available on fault window. Otherwise, returns with RMA_reject. 62 * 63 * Total credits available on fault window: FIFO_SIZE(4MB)/CRBS_SIZE(128) 64 * 65 */ 66 irqreturn_t vas_fault_thread_fn(int irq, void *data) 67 { 68 struct vas_instance *vinst = data; 69 struct coprocessor_request_block *crb, *entry; 70 struct coprocessor_request_block buf; 71 struct pnv_vas_window *window; 72 unsigned long flags; 73 void *fifo; 74 75 crb = &buf; 76 77 /* 78 * VAS can interrupt with multiple page faults. So process all 79 * valid CRBs within fault FIFO until reaches invalid CRB. 80 * We use CCW[0] and pswid to validate CRBs: 81 * 82 * CCW[0] Reserved bit. When NX pastes CRB, CCW[0]=0 83 * OS sets this bit to 1 after reading CRB. 84 * pswid NX assigns window ID. Set pswid to -1 after 85 * reading CRB from fault FIFO. 86 * 87 * We exit this function if no valid CRBs are available to process. 88 * So acquire fault_lock and reset fifo_in_progress to 0 before 89 * exit. 90 * In case kernel receives another interrupt with different page 91 * fault, interrupt handler returns with IRQ_HANDLED if 92 * fifo_in_progress is set. Means these new faults will be 93 * handled by the current thread. Otherwise set fifo_in_progress 94 * and return IRQ_WAKE_THREAD to wake up thread. 95 */ 96 while (true) { 97 spin_lock_irqsave(&vinst->fault_lock, flags); 98 /* 99 * Advance the fault fifo pointer to next CRB. 100 * Use CRB_SIZE rather than sizeof(*crb) since the latter is 101 * aligned to CRB_ALIGN (256) but the CRB written to by VAS is 102 * only CRB_SIZE in len. 103 */ 104 fifo = vinst->fault_fifo + (vinst->fault_crbs * CRB_SIZE); 105 entry = fifo; 106 107 if ((entry->stamp.nx.pswid == cpu_to_be32(FIFO_INVALID_ENTRY)) 108 || (entry->ccw & cpu_to_be32(CCW0_INVALID))) { 109 vinst->fifo_in_progress = 0; 110 spin_unlock_irqrestore(&vinst->fault_lock, flags); 111 return IRQ_HANDLED; 112 } 113 114 spin_unlock_irqrestore(&vinst->fault_lock, flags); 115 vinst->fault_crbs++; 116 if (vinst->fault_crbs == (vinst->fault_fifo_size / CRB_SIZE)) 117 vinst->fault_crbs = 0; 118 119 memcpy(crb, fifo, CRB_SIZE); 120 entry->stamp.nx.pswid = cpu_to_be32(FIFO_INVALID_ENTRY); 121 entry->ccw |= cpu_to_be32(CCW0_INVALID); 122 /* 123 * Return credit for the fault window. 124 */ 125 vas_return_credit(vinst->fault_win, false); 126 127 pr_devel("VAS[%d] fault_fifo %p, fifo %p, fault_crbs %d\n", 128 vinst->vas_id, vinst->fault_fifo, fifo, 129 vinst->fault_crbs); 130 131 vas_dump_crb(crb); 132 window = vas_pswid_to_window(vinst, 133 be32_to_cpu(crb->stamp.nx.pswid)); 134 135 if (IS_ERR(window)) { 136 /* 137 * We got an interrupt about a specific send 138 * window but we can't find that window and we can't 139 * even clean it up (return credit on user space 140 * window). 141 * But we should not get here. 142 * TODO: Disable IRQ. 143 */ 144 dump_fifo(vinst, (void *)entry); 145 pr_err("VAS[%d] fault_fifo %p, fifo %p, pswid 0x%x, fault_crbs %d bad CRB?\n", 146 vinst->vas_id, vinst->fault_fifo, fifo, 147 be32_to_cpu(crb->stamp.nx.pswid), 148 vinst->fault_crbs); 149 150 WARN_ON_ONCE(1); 151 } else { 152 /* 153 * NX sees faults only with user space windows. 154 */ 155 if (window->user_win) 156 vas_update_csb(crb, &window->vas_win.task_ref); 157 else 158 WARN_ON_ONCE(!window->user_win); 159 160 /* 161 * Return credit for send window after processing 162 * fault CRB. 163 */ 164 vas_return_credit(window, true); 165 } 166 } 167 } 168 169 irqreturn_t vas_fault_handler(int irq, void *dev_id) 170 { 171 struct vas_instance *vinst = dev_id; 172 irqreturn_t ret = IRQ_WAKE_THREAD; 173 unsigned long flags; 174 175 /* 176 * NX can generate an interrupt for multiple faults. So the 177 * fault handler thread process all CRBs until finds invalid 178 * entry. In case if NX sees continuous faults, it is possible 179 * that the thread function entered with the first interrupt 180 * can execute and process all valid CRBs. 181 * So wake up thread only if the fault thread is not in progress. 182 */ 183 spin_lock_irqsave(&vinst->fault_lock, flags); 184 185 if (vinst->fifo_in_progress) 186 ret = IRQ_HANDLED; 187 else 188 vinst->fifo_in_progress = 1; 189 190 spin_unlock_irqrestore(&vinst->fault_lock, flags); 191 192 return ret; 193 } 194 195 /* 196 * Fault window is opened per VAS instance. NX pastes fault CRB in fault 197 * FIFO upon page faults. 198 */ 199 int vas_setup_fault_window(struct vas_instance *vinst) 200 { 201 struct vas_rx_win_attr attr; 202 struct vas_window *win; 203 204 vinst->fault_fifo_size = VAS_FAULT_WIN_FIFO_SIZE; 205 vinst->fault_fifo = kzalloc(vinst->fault_fifo_size, GFP_KERNEL); 206 if (!vinst->fault_fifo) { 207 pr_err("Unable to alloc %d bytes for fault_fifo\n", 208 vinst->fault_fifo_size); 209 return -ENOMEM; 210 } 211 212 /* 213 * Invalidate all CRB entries. NX pastes valid entry for each fault. 214 */ 215 memset(vinst->fault_fifo, FIFO_INVALID_ENTRY, vinst->fault_fifo_size); 216 vas_init_rx_win_attr(&attr, VAS_COP_TYPE_FAULT); 217 218 attr.rx_fifo_size = vinst->fault_fifo_size; 219 attr.rx_fifo = __pa(vinst->fault_fifo); 220 221 /* 222 * Max creds is based on number of CRBs can fit in the FIFO. 223 * (fault_fifo_size/CRB_SIZE). If 8MB FIFO is used, max creds 224 * will be 0xffff since the receive creds field is 16bits wide. 225 */ 226 attr.wcreds_max = vinst->fault_fifo_size / CRB_SIZE; 227 attr.lnotify_lpid = 0; 228 attr.lnotify_pid = mfspr(SPRN_PID); 229 attr.lnotify_tid = mfspr(SPRN_PID); 230 231 win = vas_rx_win_open(vinst->vas_id, VAS_COP_TYPE_FAULT, &attr); 232 if (IS_ERR(win)) { 233 pr_err("VAS: Error %ld opening FaultWin\n", PTR_ERR(win)); 234 kfree(vinst->fault_fifo); 235 return PTR_ERR(win); 236 } 237 238 vinst->fault_win = container_of(win, struct pnv_vas_window, vas_win); 239 240 pr_devel("VAS: Created FaultWin %d, LPID/PID/TID [%d/%d/%d]\n", 241 vinst->fault_win->vas_win.winid, attr.lnotify_lpid, 242 attr.lnotify_pid, attr.lnotify_tid); 243 244 return 0; 245 } 246