1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * Copyright (c) 2014 by Delphix. All rights reserved. 29 */ 30 31 #include <sys/types.h> 32 #include <sys/xpv_support.h> 33 #include <sys/hypervisor.h> 34 #include <sys/machsystm.h> 35 #include <sys/mutex.h> 36 #include <sys/cmn_err.h> 37 #include <sys/dditypes.h> 38 #include <sys/atomic.h> 39 #include <sys/sysmacros.h> 40 #include <sys/cpu.h> 41 #include <sys/psw.h> 42 #include <sys/psm.h> 43 #include <sys/sdt.h> 44 45 extern dev_info_t *xpv_dip; 46 static ddi_intr_handle_t *evtchn_ihp = NULL; 47 static ddi_softint_handle_t evtchn_to_handle[NR_EVENT_CHANNELS]; 48 kmutex_t ec_lock; 49 50 static int evtchn_callback_irq = -1; 51 52 static volatile ulong_t *pending_events; 53 static volatile ulong_t *masked_events; 54 55 /* log2(NBBY * sizeof (ulong)) */ 56 #define EVTCHN_SHIFT 6 57 58 /* Atomically get and clear a ulong from memory. */ 59 #define GET_AND_CLEAR(src, targ) { \ 60 membar_enter(); \ 61 do { \ 62 targ = *src; \ 63 } while (atomic_cas_ulong(src, targ, 0) != targ); \ 64 } 65 66 /* Get the first and last bits set in a bitmap */ 67 #define GET_BOUNDS(bitmap, low, high) { \ 68 int _i; \ 69 low = high = -1; \ 70 for (_i = 0; _i <= sizeof (ulong_t); _i++) \ 71 if (bitmap & (1UL << _i)) { \ 72 if (low == -1) \ 73 low = _i; \ 74 high = _i; \ 75 } \ 76 } 77 78 void 79 ec_bind_evtchn_to_handler(int evtchn, pri_t pri, ec_handler_fcn_t handler, 80 void *arg1) 81 { 82 ddi_softint_handle_t hdl; 83 84 if (evtchn < 0 || evtchn >= NR_EVENT_CHANNELS) { 85 cmn_err(CE_WARN, "Binding invalid event channel: %d", evtchn); 86 return; 87 } 88 89 (void) ddi_intr_add_softint(xpv_dip, &hdl, pri, handler, (caddr_t)arg1); 90 mutex_enter(&ec_lock); 91 ASSERT(evtchn_to_handle[evtchn] == NULL); 92 evtchn_to_handle[evtchn] = hdl; 93 mutex_exit(&ec_lock); 94 95 /* Let the hypervisor know we're prepared to handle this event */ 96 hypervisor_unmask_event(evtchn); 97 } 98 99 void 100 ec_unbind_evtchn(int evtchn) 101 { 102 evtchn_close_t close; 103 ddi_softint_handle_t hdl; 104 105 if (evtchn < 0 || evtchn >= NR_EVENT_CHANNELS) { 106 cmn_err(CE_WARN, "Unbinding invalid event channel: %d", evtchn); 107 return; 108 } 109 110 /* 111 * Let the hypervisor know we're no longer prepared to handle this 112 * event 113 */ 114 hypervisor_mask_event(evtchn); 115 116 /* Cleanup the event handler metadata */ 117 mutex_enter(&ec_lock); 118 hdl = evtchn_to_handle[evtchn]; 119 evtchn_to_handle[evtchn] = NULL; 120 mutex_exit(&ec_lock); 121 122 close.port = evtchn; 123 (void) HYPERVISOR_event_channel_op(EVTCHNOP_close, &close); 124 (void) ddi_intr_remove_softint(hdl); 125 } 126 127 void 128 ec_notify_via_evtchn(unsigned int port) 129 { 130 evtchn_send_t send; 131 132 if ((int)port == -1) 133 return; 134 send.port = port; 135 (void) HYPERVISOR_event_channel_op(EVTCHNOP_send, &send); 136 } 137 138 void 139 hypervisor_unmask_event(unsigned int ev) 140 { 141 int index = ev >> EVTCHN_SHIFT; 142 ulong_t bit = 1UL << (ev & ((1UL << EVTCHN_SHIFT) - 1)); 143 volatile ulong_t *maskp; 144 evtchn_unmask_t unmask; 145 146 /* 147 * index,bit contain the event number as an index into the 148 * masked-events bitmask. Set it to 0. 149 */ 150 maskp = &masked_events[index]; 151 atomic_and_ulong(maskp, ~bit); 152 153 /* Let the hypervisor know the event has been unmasked */ 154 unmask.port = ev; 155 if (HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask) != 0) 156 panic("xen_evtchn_unmask() failed"); 157 } 158 159 /* Set a bit in an evtchan mask word */ 160 void 161 hypervisor_mask_event(uint_t ev) 162 { 163 int index = ev >> EVTCHN_SHIFT; 164 ulong_t bit = 1UL << (ev & ((1UL << EVTCHN_SHIFT) - 1)); 165 volatile ulong_t *maskp; 166 167 maskp = &masked_events[index]; 168 atomic_or_ulong(maskp, bit); 169 } 170 171 void 172 hypervisor_clear_event(uint_t ev) 173 { 174 int index = ev >> EVTCHN_SHIFT; 175 ulong_t bit = 1UL << (ev & ((1UL << EVTCHN_SHIFT) - 1)); 176 volatile ulong_t *maskp; 177 178 maskp = &pending_events[index]; 179 atomic_and_ulong(maskp, ~bit); 180 } 181 182 int 183 xen_alloc_unbound_evtchn(int domid, int *evtchnp) 184 { 185 evtchn_alloc_unbound_t alloc; 186 int err; 187 188 alloc.dom = DOMID_SELF; 189 alloc.remote_dom = (domid_t)domid; 190 191 if ((err = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound, 192 &alloc)) == 0) { 193 *evtchnp = alloc.port; 194 /* ensure evtchn is masked till we're ready to use it */ 195 (void) hypervisor_mask_event(*evtchnp); 196 } else { 197 err = xen_xlate_errcode(err); 198 } 199 200 return (err); 201 } 202 203 int 204 xen_bind_interdomain(int domid, int remote_port, int *port) 205 { 206 evtchn_bind_interdomain_t bind; 207 int err; 208 209 bind.remote_dom = (domid_t)domid; 210 bind.remote_port = remote_port; 211 if ((err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, 212 &bind)) == 0) 213 *port = bind.local_port; 214 else 215 err = xen_xlate_errcode(err); 216 return (err); 217 } 218 219 /*ARGSUSED*/ 220 uint_t 221 evtchn_callback_fcn(caddr_t arg0, caddr_t arg1) 222 { 223 ulong_t pending_word; 224 int i, j, port; 225 volatile struct vcpu_info *vci; 226 uint_t rv = DDI_INTR_UNCLAIMED; 227 ddi_softint_handle_t hdl; 228 int low, high; 229 ulong_t sels; 230 231 /* 232 * Xen hard-codes all notifications to VCPU0, so we bind 233 * ourselves via xpv.conf. Note that this also assumes that all 234 * evtchns are bound to VCPU0, which is true by default. 235 */ 236 ASSERT(CPU->cpu_id == 0); 237 238 vci = &HYPERVISOR_shared_info->vcpu_info[0]; 239 240 again: 241 DTRACE_PROBE2(evtchn__scan__start, int, vci->evtchn_upcall_pending, 242 ulong_t, vci->evtchn_pending_sel); 243 244 atomic_and_8(&vci->evtchn_upcall_pending, 0); 245 246 /* 247 * Find the upper and lower bounds in which we need to search for 248 * pending events. 249 */ 250 GET_AND_CLEAR(&vci->evtchn_pending_sel, sels); 251 252 /* sels == 1 is by far the most common case. Make it fast */ 253 if (sels == 1) 254 low = high = 0; 255 else if (sels == 0) 256 return (rv); 257 else 258 GET_BOUNDS(sels, low, high); 259 260 /* Scan the port list, looking for words with bits set */ 261 for (i = low; i <= high; i++) { 262 ulong_t tmp; 263 264 GET_AND_CLEAR(&pending_events[i], tmp); 265 pending_word = tmp & ~(masked_events[i]); 266 267 /* Scan the bits in the word, looking for pending events */ 268 while (pending_word != 0) { 269 j = lowbit(pending_word) - 1; 270 port = (i << EVTCHN_SHIFT) + j; 271 pending_word = pending_word & ~(1UL << j); 272 273 /* 274 * If there is a handler registered for this event, 275 * schedule a softint of the appropriate priority 276 * to execute it. 277 */ 278 if ((hdl = evtchn_to_handle[port]) != NULL) { 279 (void) ddi_intr_trigger_softint(hdl, NULL); 280 rv = DDI_INTR_CLAIMED; 281 } 282 } 283 } 284 DTRACE_PROBE2(evtchn__scan__end, int, vci->evtchn_upcall_pending, 285 ulong_t, vci->evtchn_pending_sel); 286 287 if ((volatile uint8_t)vci->evtchn_upcall_pending || 288 ((volatile ulong_t)vci->evtchn_pending_sel)) 289 goto again; 290 291 return (rv); 292 } 293 294 static int 295 set_hvm_callback(int irq) 296 { 297 struct xen_hvm_param xhp; 298 299 xhp.domid = DOMID_SELF; 300 xhp.index = HVM_PARAM_CALLBACK_IRQ; 301 xhp.value = irq; 302 return (HYPERVISOR_hvm_op(HVMOP_set_param, &xhp)); 303 } 304 305 void 306 ec_fini() 307 { 308 int i; 309 310 for (i = 0; i < NR_EVENT_CHANNELS; i++) 311 ec_unbind_evtchn(i); 312 313 evtchn_callback_irq = -1; 314 if (evtchn_ihp != NULL) { 315 (void) ddi_intr_disable(*evtchn_ihp); 316 (void) ddi_intr_remove_handler(*evtchn_ihp); 317 (void) ddi_intr_free(*evtchn_ihp); 318 kmem_free(evtchn_ihp, sizeof (ddi_intr_handle_t)); 319 evtchn_ihp = NULL; 320 } 321 } 322 323 int 324 ec_init(void) 325 { 326 int i; 327 int rv, actual; 328 ddi_intr_handle_t *ihp; 329 330 /* 331 * Translate the variable-sized pending and masked event bitmasks 332 * into constant-sized arrays of uint32_t's. 333 */ 334 pending_events = &HYPERVISOR_shared_info->evtchn_pending[0]; 335 masked_events = &HYPERVISOR_shared_info->evtchn_mask[0]; 336 337 /* 338 * Clear our event handler structures and prevent the hypervisor 339 * from triggering any events. 340 */ 341 mutex_init(&ec_lock, NULL, MUTEX_SPIN, (void *)ipltospl(SPL7)); 342 for (i = 0; i < NR_EVENT_CHANNELS; i++) { 343 evtchn_to_handle[i] = NULL; 344 (void) hypervisor_mask_event(i); 345 } 346 347 /* 348 * Allocate and initialize an interrupt handler to process the 349 * hypervisor's "hey you have events pending!" interrupt. 350 */ 351 ihp = kmem_zalloc(sizeof (ddi_intr_handle_t), KM_SLEEP); 352 rv = ddi_intr_alloc(xpv_dip, ihp, DDI_INTR_TYPE_FIXED, 0, 1, &actual, 353 DDI_INTR_ALLOC_NORMAL); 354 if (rv < 0 || actual != 1) { 355 cmn_err(CE_WARN, "Could not allocate evtchn interrupt: %d", 356 rv); 357 return (-1); 358 } 359 360 rv = ddi_intr_add_handler(*ihp, evtchn_callback_fcn, NULL, NULL); 361 if (rv < 0) { 362 (void) ddi_intr_free(*ihp); 363 cmn_err(CE_WARN, "Could not attach evtchn handler"); 364 return (-1); 365 } 366 evtchn_ihp = ihp; 367 368 if (ddi_intr_enable(*ihp) != DDI_SUCCESS) { 369 cmn_err(CE_WARN, "Could not enable evtchn interrupts\n"); 370 return (-1); 371 } 372 373 /* Tell the hypervisor which interrupt we're waiting on. */ 374 evtchn_callback_irq = ((ddi_intr_handle_impl_t *)*ihp)->ih_vector; 375 376 if (set_hvm_callback(evtchn_callback_irq) != 0) { 377 cmn_err(CE_WARN, "Couldn't register evtchn callback"); 378 return (-1); 379 } 380 return (0); 381 } 382 383 void 384 ec_resume(void) 385 { 386 int i; 387 388 /* New event-channel space is not 'live' yet. */ 389 for (i = 0; i < NR_EVENT_CHANNELS; i++) 390 (void) hypervisor_mask_event(i); 391 if (set_hvm_callback(evtchn_callback_irq) != 0) 392 cmn_err(CE_WARN, "Couldn't register evtchn callback"); 393 394 } 395