xref: /freebsd/sys/dev/xen/control/control.c (revision 3a92d97ff0f22d21608e1c19b83104c4937523b6)
1 /*-
2  * Copyright (c) 2010 Justin T. Gibbs, Spectra Logic Corporation
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions, and the following disclaimer,
10  *    without modification.
11  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
12  *    substantially similar to the "NO WARRANTY" disclaimer below
13  *    ("Disclaimer") and any redistribution must be conditioned upon
14  *    including a substantially similar Disclaimer requirement for further
15  *    binary redistribution.
16  *
17  * NO WARRANTY
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
27  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28  * POSSIBILITY OF SUCH DAMAGES.
29  */
30 
31 /*-
32  * PV suspend/resume support:
33  *
34  * Copyright (c) 2004 Christian Limpach.
35  * Copyright (c) 2004-2006,2008 Kip Macy
36  * All rights reserved.
37  *
38  * Redistribution and use in source and binary forms, with or without
39  * modification, are permitted provided that the following conditions
40  * are met:
41  * 1. Redistributions of source code must retain the above copyright
42  *    notice, this list of conditions and the following disclaimer.
43  * 2. Redistributions in binary form must reproduce the above copyright
44  *    notice, this list of conditions and the following disclaimer in the
45  *    documentation and/or other materials provided with the distribution.
46  * 3. All advertising materials mentioning features or use of this software
47  *    must display the following acknowledgement:
48  *      This product includes software developed by Christian Limpach.
49  * 4. The name of the author may not be used to endorse or promote products
50  *    derived from this software without specific prior written permission.
51  *
52  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
53  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
54  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
55  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
56  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
57  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
58  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
59  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
60  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
61  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
62  */
63 
64 /*-
65  * HVM suspend/resume support:
66  *
67  * Copyright (c) 2008 Citrix Systems, Inc.
68  * All rights reserved.
69  *
70  * Redistribution and use in source and binary forms, with or without
71  * modification, are permitted provided that the following conditions
72  * are met:
73  * 1. Redistributions of source code must retain the above copyright
74  *    notice, this list of conditions and the following disclaimer.
75  * 2. Redistributions in binary form must reproduce the above copyright
76  *    notice, this list of conditions and the following disclaimer in the
77  *    documentation and/or other materials provided with the distribution.
78  *
79  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
80  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
81  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
82  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
83  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
84  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
85  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
86  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
87  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
88  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
89  * SUCH DAMAGE.
90  */
91 #include <sys/cdefs.h>
92 __FBSDID("$FreeBSD$");
93 
94 /**
95  * \file control.c
96  *
97  * \brief Device driver to repond to control domain events that impact
98  *        this VM.
99  */
100 
101 #include <sys/param.h>
102 #include <sys/systm.h>
103 #include <sys/kernel.h>
104 #include <sys/malloc.h>
105 
106 #include <sys/bio.h>
107 #include <sys/bus.h>
108 #include <sys/conf.h>
109 #include <sys/disk.h>
110 #include <sys/fcntl.h>
111 #include <sys/filedesc.h>
112 #include <sys/kdb.h>
113 #include <sys/module.h>
114 #include <sys/namei.h>
115 #include <sys/proc.h>
116 #include <sys/reboot.h>
117 #include <sys/rman.h>
118 #include <sys/sched.h>
119 #include <sys/taskqueue.h>
120 #include <sys/types.h>
121 #include <sys/vnode.h>
122 #include <sys/sched.h>
123 #include <sys/smp.h>
124 #include <sys/eventhandler.h>
125 
126 #include <geom/geom.h>
127 
128 #include <machine/_inttypes.h>
129 #include <machine/intr_machdep.h>
130 
131 #include <vm/vm.h>
132 #include <vm/vm_extern.h>
133 #include <vm/vm_kern.h>
134 
135 #include <xen/xen-os.h>
136 #include <xen/blkif.h>
137 #include <xen/evtchn.h>
138 #include <xen/gnttab.h>
139 #include <xen/xen_intr.h>
140 
141 #ifdef XENHVM
142 #include <xen/hvm.h>
143 #endif
144 
145 #include <xen/interface/event_channel.h>
146 #include <xen/interface/grant_table.h>
147 
148 #include <xen/xenbus/xenbusvar.h>
149 
150 #include <machine/xen/xenvar.h>
151 #include <machine/xen/xenfunc.h>
152 
153 /*--------------------------- Forward Declarations --------------------------*/
154 /** Function signature for shutdown event handlers. */
155 typedef	void (xctrl_shutdown_handler_t)(void);
156 
157 static xctrl_shutdown_handler_t xctrl_poweroff;
158 static xctrl_shutdown_handler_t xctrl_reboot;
159 static xctrl_shutdown_handler_t xctrl_suspend;
160 static xctrl_shutdown_handler_t xctrl_crash;
161 
162 /*-------------------------- Private Data Structures -------------------------*/
163 /** Element type for lookup table of event name to handler. */
164 struct xctrl_shutdown_reason {
165 	const char		 *name;
166 	xctrl_shutdown_handler_t *handler;
167 };
168 
169 /** Lookup table for shutdown event name to handler. */
170 static const struct xctrl_shutdown_reason xctrl_shutdown_reasons[] = {
171 	{ "poweroff", xctrl_poweroff },
172 	{ "reboot",   xctrl_reboot   },
173 	{ "suspend",  xctrl_suspend  },
174 	{ "crash",    xctrl_crash    },
175 	{ "halt",     xctrl_poweroff },
176 };
177 
178 struct xctrl_softc {
179 	struct xs_watch    xctrl_watch;
180 };
181 
182 /*------------------------------ Event Handlers ------------------------------*/
183 static void
184 xctrl_poweroff()
185 {
186 	shutdown_nice(RB_POWEROFF|RB_HALT);
187 }
188 
189 static void
190 xctrl_reboot()
191 {
192 	shutdown_nice(0);
193 }
194 
195 #ifndef XENHVM
196 extern void xencons_suspend(void);
197 extern void xencons_resume(void);
198 
199 /* Full PV mode suspension. */
200 static void
201 xctrl_suspend()
202 {
203 	int i, j, k, fpp, suspend_cancelled;
204 	unsigned long max_pfn, start_info_mfn;
205 
206 	EVENTHANDLER_INVOKE(power_suspend);
207 
208 #ifdef SMP
209 	struct thread *td;
210 	cpuset_t map;
211 	u_int cpuid;
212 
213 	/*
214 	 * Bind us to CPU 0 and stop any other VCPUs.
215 	 */
216 	td = curthread;
217 	thread_lock(td);
218 	sched_bind(td, 0);
219 	thread_unlock(td);
220 	cpuid = PCPU_GET(cpuid);
221 	KASSERT(cpuid == 0, ("xen_suspend: not running on cpu 0"));
222 
223 	map = all_cpus;
224 	CPU_CLR(cpuid, &map);
225 	CPU_NAND(&map, &stopped_cpus);
226 	if (!CPU_EMPTY(&map))
227 		stop_cpus(map);
228 #endif
229 
230 	/*
231 	 * Be sure to hold Giant across DEVICE_SUSPEND/RESUME since non-MPSAFE
232 	 * drivers need this.
233 	 */
234 	mtx_lock(&Giant);
235 	if (DEVICE_SUSPEND(root_bus) != 0) {
236 		mtx_unlock(&Giant);
237 		printf("%s: device_suspend failed\n", __func__);
238 #ifdef SMP
239 		if (!CPU_EMPTY(&map))
240 			restart_cpus(map);
241 #endif
242 		return;
243 	}
244 	mtx_unlock(&Giant);
245 
246 	local_irq_disable();
247 
248 	xencons_suspend();
249 	gnttab_suspend();
250 	intr_suspend();
251 
252 	max_pfn = HYPERVISOR_shared_info->arch.max_pfn;
253 
254 	void *shared_info = HYPERVISOR_shared_info;
255 	HYPERVISOR_shared_info = NULL;
256 	pmap_kremove((vm_offset_t) shared_info);
257 	PT_UPDATES_FLUSH();
258 
259 	xen_start_info->store_mfn = MFNTOPFN(xen_start_info->store_mfn);
260 	xen_start_info->console.domU.mfn = MFNTOPFN(xen_start_info->console.domU.mfn);
261 
262 	/*
263 	 * We'll stop somewhere inside this hypercall. When it returns,
264 	 * we'll start resuming after the restore.
265 	 */
266 	start_info_mfn = VTOMFN(xen_start_info);
267 	pmap_suspend();
268 	suspend_cancelled = HYPERVISOR_suspend(start_info_mfn);
269 	pmap_resume();
270 
271 	pmap_kenter_ma((vm_offset_t) shared_info, xen_start_info->shared_info);
272 	HYPERVISOR_shared_info = shared_info;
273 
274 	HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list =
275 		VTOMFN(xen_pfn_to_mfn_frame_list_list);
276 
277 	fpp = PAGE_SIZE/sizeof(unsigned long);
278 	for (i = 0, j = 0, k = -1; i < max_pfn; i += fpp, j++) {
279 		if ((j % fpp) == 0) {
280 			k++;
281 			xen_pfn_to_mfn_frame_list_list[k] =
282 				VTOMFN(xen_pfn_to_mfn_frame_list[k]);
283 			j = 0;
284 		}
285 		xen_pfn_to_mfn_frame_list[k][j] =
286 			VTOMFN(&xen_phys_machine[i]);
287 	}
288 	HYPERVISOR_shared_info->arch.max_pfn = max_pfn;
289 
290 	gnttab_resume();
291 	intr_resume(suspend_cancelled != 0);
292 	local_irq_enable();
293 	xencons_resume();
294 
295 #ifdef CONFIG_SMP
296 	for_each_cpu(i)
297 		vcpu_prepare(i);
298 
299 #endif
300 
301 	/*
302 	 * Only resume xenbus /after/ we've prepared our VCPUs; otherwise
303 	 * the VCPU hotplug callback can race with our vcpu_prepare
304 	 */
305 	mtx_lock(&Giant);
306 	DEVICE_RESUME(root_bus);
307 	mtx_unlock(&Giant);
308 
309 #ifdef SMP
310 	thread_lock(curthread);
311 	sched_unbind(curthread);
312 	thread_unlock(curthread);
313 	if (!CPU_EMPTY(&map))
314 		restart_cpus(map);
315 #endif
316 	EVENTHANDLER_INVOKE(power_resume);
317 }
318 
319 static void
320 xen_pv_shutdown_final(void *arg, int howto)
321 {
322 	/*
323 	 * Inform the hypervisor that shutdown is complete.
324 	 * This is not necessary in HVM domains since Xen
325 	 * emulates ACPI in that mode and FreeBSD's ACPI
326 	 * support will request this transition.
327 	 */
328 	if (howto & (RB_HALT | RB_POWEROFF))
329 		HYPERVISOR_shutdown(SHUTDOWN_poweroff);
330 	else
331 		HYPERVISOR_shutdown(SHUTDOWN_reboot);
332 }
333 
334 #else
335 
336 /* HVM mode suspension. */
337 static void
338 xctrl_suspend()
339 {
340 #ifdef SMP
341 	cpuset_t cpu_suspend_map;
342 #endif
343 	int suspend_cancelled;
344 
345 	EVENTHANDLER_INVOKE(power_suspend);
346 
347 	if (smp_started) {
348 		thread_lock(curthread);
349 		sched_bind(curthread, 0);
350 		thread_unlock(curthread);
351 	}
352 	KASSERT((PCPU_GET(cpuid) == 0), ("Not running on CPU#0"));
353 
354 	/*
355 	 * Clear our XenStore node so the toolstack knows we are
356 	 * responding to the suspend request.
357 	 */
358 	xs_write(XST_NIL, "control", "shutdown", "");
359 
360 	/*
361 	 * Be sure to hold Giant across DEVICE_SUSPEND/RESUME since non-MPSAFE
362 	 * drivers need this.
363 	 */
364 	mtx_lock(&Giant);
365 	if (DEVICE_SUSPEND(root_bus) != 0) {
366 		mtx_unlock(&Giant);
367 		printf("%s: device_suspend failed\n", __func__);
368 		return;
369 	}
370 	mtx_unlock(&Giant);
371 
372 #ifdef SMP
373 	CPU_ZERO(&cpu_suspend_map);	/* silence gcc */
374 	if (smp_started) {
375 		/*
376 		 * Suspend other CPUs. This prevents IPIs while we
377 		 * are resuming, and will allow us to reset per-cpu
378 		 * vcpu_info on resume.
379 		 */
380 		cpu_suspend_map = all_cpus;
381 		CPU_CLR(PCPU_GET(cpuid), &cpu_suspend_map);
382 		if (!CPU_EMPTY(&cpu_suspend_map))
383 			suspend_cpus(cpu_suspend_map);
384 	}
385 #endif
386 
387 	/*
388 	 * Prevent any races with evtchn_interrupt() handler.
389 	 */
390 	disable_intr();
391 	intr_suspend();
392 	xen_hvm_suspend();
393 
394 	suspend_cancelled = HYPERVISOR_suspend(0);
395 
396 	xen_hvm_resume(suspend_cancelled != 0);
397 	intr_resume(suspend_cancelled != 0);
398 	enable_intr();
399 
400 	/*
401 	 * Reset grant table info.
402 	 */
403 	gnttab_resume();
404 
405 #ifdef SMP
406 	if (smp_started && !CPU_EMPTY(&cpu_suspend_map)) {
407 		/*
408 		 * Now that event channels have been initialized,
409 		 * resume CPUs.
410 		 */
411 		resume_cpus(cpu_suspend_map);
412 	}
413 #endif
414 
415 	/*
416 	 * FreeBSD really needs to add DEVICE_SUSPEND_CANCEL or
417 	 * similar.
418 	 */
419 	mtx_lock(&Giant);
420 	DEVICE_RESUME(root_bus);
421 	mtx_unlock(&Giant);
422 
423 	if (smp_started) {
424 		thread_lock(curthread);
425 		sched_unbind(curthread);
426 		thread_unlock(curthread);
427 	}
428 
429 	EVENTHANDLER_INVOKE(power_resume);
430 
431 	if (bootverbose)
432 		printf("System resumed after suspension\n");
433 
434 }
435 #endif
436 
437 static void
438 xctrl_crash()
439 {
440 	panic("Xen directed crash");
441 }
442 
443 /*------------------------------ Event Reception -----------------------------*/
444 static void
445 xctrl_on_watch_event(struct xs_watch *watch, const char **vec, unsigned int len)
446 {
447 	const struct xctrl_shutdown_reason *reason;
448 	const struct xctrl_shutdown_reason *last_reason;
449 	char *result;
450 	int   error;
451 	int   result_len;
452 
453 	error = xs_read(XST_NIL, "control", "shutdown",
454 			&result_len, (void **)&result);
455 	if (error != 0)
456 		return;
457 
458 	reason = xctrl_shutdown_reasons;
459 	last_reason = reason + nitems(xctrl_shutdown_reasons);
460 	while (reason < last_reason) {
461 
462 		if (!strcmp(result, reason->name)) {
463 			reason->handler();
464 			break;
465 		}
466 		reason++;
467 	}
468 
469 	free(result, M_XENSTORE);
470 }
471 
472 /*------------------ Private Device Attachment Functions  --------------------*/
473 /**
474  * \brief Identify instances of this device type in the system.
475  *
476  * \param driver  The driver performing this identify action.
477  * \param parent  The NewBus parent device for any devices this method adds.
478  */
479 static void
480 xctrl_identify(driver_t *driver __unused, device_t parent)
481 {
482 	/*
483 	 * A single device instance for our driver is always present
484 	 * in a system operating under Xen.
485 	 */
486 	BUS_ADD_CHILD(parent, 0, driver->name, 0);
487 }
488 
489 /**
490  * \brief Probe for the existance of the Xen Control device
491  *
492  * \param dev  NewBus device_t for this Xen control instance.
493  *
494  * \return  Always returns 0 indicating success.
495  */
496 static int
497 xctrl_probe(device_t dev)
498 {
499 	device_set_desc(dev, "Xen Control Device");
500 
501 	return (0);
502 }
503 
504 /**
505  * \brief Attach the Xen control device.
506  *
507  * \param dev  NewBus device_t for this Xen control instance.
508  *
509  * \return  On success, 0. Otherwise an errno value indicating the
510  *          type of failure.
511  */
512 static int
513 xctrl_attach(device_t dev)
514 {
515 	struct xctrl_softc *xctrl;
516 
517 	xctrl = device_get_softc(dev);
518 
519 	/* Activate watch */
520 	xctrl->xctrl_watch.node = "control/shutdown";
521 	xctrl->xctrl_watch.callback = xctrl_on_watch_event;
522 	xctrl->xctrl_watch.callback_data = (uintptr_t)xctrl;
523 	xs_register_watch(&xctrl->xctrl_watch);
524 
525 #ifndef XENHVM
526 	EVENTHANDLER_REGISTER(shutdown_final, xen_pv_shutdown_final, NULL,
527 			      SHUTDOWN_PRI_LAST);
528 #endif
529 
530 	return (0);
531 }
532 
533 /**
534  * \brief Detach the Xen control device.
535  *
536  * \param dev  NewBus device_t for this Xen control device instance.
537  *
538  * \return  On success, 0. Otherwise an errno value indicating the
539  *          type of failure.
540  */
541 static int
542 xctrl_detach(device_t dev)
543 {
544 	struct xctrl_softc *xctrl;
545 
546 	xctrl = device_get_softc(dev);
547 
548 	/* Release watch */
549 	xs_unregister_watch(&xctrl->xctrl_watch);
550 
551 	return (0);
552 }
553 
554 /*-------------------- Private Device Attachment Data  -----------------------*/
555 static device_method_t xctrl_methods[] = {
556 	/* Device interface */
557 	DEVMETHOD(device_identify,	xctrl_identify),
558 	DEVMETHOD(device_probe,         xctrl_probe),
559 	DEVMETHOD(device_attach,        xctrl_attach),
560 	DEVMETHOD(device_detach,        xctrl_detach),
561 
562 	DEVMETHOD_END
563 };
564 
565 DEFINE_CLASS_0(xctrl, xctrl_driver, xctrl_methods, sizeof(struct xctrl_softc));
566 devclass_t xctrl_devclass;
567 
568 DRIVER_MODULE(xctrl, xenstore, xctrl_driver, xctrl_devclass, NULL, NULL);
569