1 /* 2 * Xen hypercall batching. 3 * 4 * Xen allows multiple hypercalls to be issued at once, using the 5 * multicall interface. This allows the cost of trapping into the 6 * hypervisor to be amortized over several calls. 7 * 8 * This file implements a simple interface for multicalls. There's a 9 * per-cpu buffer of outstanding multicalls. When you want to queue a 10 * multicall for issuing, you can allocate a multicall slot for the 11 * call and its arguments, along with storage for space which is 12 * pointed to by the arguments (for passing pointers to structures, 13 * etc). When the multicall is actually issued, all the space for the 14 * commands and allocated memory is freed for reuse. 15 * 16 * Multicalls are flushed whenever any of the buffers get full, or 17 * when explicitly requested. There's no way to get per-multicall 18 * return results back. It will BUG if any of the multicalls fail. 19 * 20 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 21 */ 22 #include <linux/percpu.h> 23 #include <linux/hardirq.h> 24 #include <linux/debugfs.h> 25 26 #include <asm/xen/hypercall.h> 27 28 #include "multicalls.h" 29 #include "debugfs.h" 30 31 #define MC_BATCH 32 32 33 #define MC_DEBUG 0 34 35 #define MC_ARGS (MC_BATCH * 16) 36 37 38 struct mc_buffer { 39 unsigned mcidx, argidx, cbidx; 40 struct multicall_entry entries[MC_BATCH]; 41 #if MC_DEBUG 42 struct multicall_entry debug[MC_BATCH]; 43 void *caller[MC_BATCH]; 44 #endif 45 unsigned char args[MC_ARGS]; 46 struct callback { 47 void (*fn)(void *); 48 void *data; 49 } callbacks[MC_BATCH]; 50 }; 51 52 static DEFINE_PER_CPU(struct mc_buffer, mc_buffer); 53 DEFINE_PER_CPU(unsigned long, xen_mc_irq_flags); 54 55 void xen_mc_flush(void) 56 { 57 struct mc_buffer *b = this_cpu_ptr(&mc_buffer); 58 struct multicall_entry *mc; 59 int ret = 0; 60 unsigned long flags; 61 int i; 62 63 BUG_ON(preemptible()); 64 65 /* Disable interrupts in case someone comes in and queues 66 something in the middle */ 67 local_irq_save(flags); 68 69 trace_xen_mc_flush(b->mcidx, b->argidx, b->cbidx); 70 71 switch (b->mcidx) { 72 case 0: 73 /* no-op */ 74 BUG_ON(b->argidx != 0); 75 break; 76 77 case 1: 78 /* Singleton multicall - bypass multicall machinery 79 and just do the call directly. */ 80 mc = &b->entries[0]; 81 82 mc->result = privcmd_call(mc->op, 83 mc->args[0], mc->args[1], mc->args[2], 84 mc->args[3], mc->args[4]); 85 ret = mc->result < 0; 86 break; 87 88 default: 89 #if MC_DEBUG 90 memcpy(b->debug, b->entries, 91 b->mcidx * sizeof(struct multicall_entry)); 92 #endif 93 94 if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0) 95 BUG(); 96 for (i = 0; i < b->mcidx; i++) 97 if (b->entries[i].result < 0) 98 ret++; 99 100 #if MC_DEBUG 101 if (ret) { 102 printk(KERN_ERR "%d multicall(s) failed: cpu %d\n", 103 ret, smp_processor_id()); 104 dump_stack(); 105 for (i = 0; i < b->mcidx; i++) { 106 printk(KERN_DEBUG " call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n", 107 i+1, b->mcidx, 108 b->debug[i].op, 109 b->debug[i].args[0], 110 b->entries[i].result, 111 b->caller[i]); 112 } 113 } 114 #endif 115 } 116 117 b->mcidx = 0; 118 b->argidx = 0; 119 120 for (i = 0; i < b->cbidx; i++) { 121 struct callback *cb = &b->callbacks[i]; 122 123 (*cb->fn)(cb->data); 124 } 125 b->cbidx = 0; 126 127 local_irq_restore(flags); 128 129 WARN_ON(ret); 130 } 131 132 struct multicall_space __xen_mc_entry(size_t args) 133 { 134 struct mc_buffer *b = this_cpu_ptr(&mc_buffer); 135 struct multicall_space ret; 136 unsigned argidx = roundup(b->argidx, sizeof(u64)); 137 138 trace_xen_mc_entry_alloc(args); 139 140 BUG_ON(preemptible()); 141 BUG_ON(b->argidx >= MC_ARGS); 142 143 if (unlikely(b->mcidx == MC_BATCH || 144 (argidx + args) >= MC_ARGS)) { 145 trace_xen_mc_flush_reason((b->mcidx == MC_BATCH) ? 146 XEN_MC_FL_BATCH : XEN_MC_FL_ARGS); 147 xen_mc_flush(); 148 argidx = roundup(b->argidx, sizeof(u64)); 149 } 150 151 ret.mc = &b->entries[b->mcidx]; 152 #if MC_DEBUG 153 b->caller[b->mcidx] = __builtin_return_address(0); 154 #endif 155 b->mcidx++; 156 ret.args = &b->args[argidx]; 157 b->argidx = argidx + args; 158 159 BUG_ON(b->argidx >= MC_ARGS); 160 return ret; 161 } 162 163 struct multicall_space xen_mc_extend_args(unsigned long op, size_t size) 164 { 165 struct mc_buffer *b = this_cpu_ptr(&mc_buffer); 166 struct multicall_space ret = { NULL, NULL }; 167 168 BUG_ON(preemptible()); 169 BUG_ON(b->argidx >= MC_ARGS); 170 171 if (unlikely(b->mcidx == 0 || 172 b->entries[b->mcidx - 1].op != op)) { 173 trace_xen_mc_extend_args(op, size, XEN_MC_XE_BAD_OP); 174 goto out; 175 } 176 177 if (unlikely((b->argidx + size) >= MC_ARGS)) { 178 trace_xen_mc_extend_args(op, size, XEN_MC_XE_NO_SPACE); 179 goto out; 180 } 181 182 ret.mc = &b->entries[b->mcidx - 1]; 183 ret.args = &b->args[b->argidx]; 184 b->argidx += size; 185 186 BUG_ON(b->argidx >= MC_ARGS); 187 188 trace_xen_mc_extend_args(op, size, XEN_MC_XE_OK); 189 out: 190 return ret; 191 } 192 193 void xen_mc_callback(void (*fn)(void *), void *data) 194 { 195 struct mc_buffer *b = this_cpu_ptr(&mc_buffer); 196 struct callback *cb; 197 198 if (b->cbidx == MC_BATCH) { 199 trace_xen_mc_flush_reason(XEN_MC_FL_CALLBACK); 200 xen_mc_flush(); 201 } 202 203 trace_xen_mc_callback(fn, data); 204 205 cb = &b->callbacks[b->cbidx++]; 206 cb->fn = fn; 207 cb->data = data; 208 } 209