1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/smp.h> 3 #include <linux/cpu.h> 4 #include <linux/slab.h> 5 #include <linux/cpumask.h> 6 #include <linux/percpu.h> 7 8 #include <xen/events.h> 9 10 #include <xen/hvc-console.h> 11 #include "xen-ops.h" 12 13 static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 }; 14 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 }; 15 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 }; 16 static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 }; 17 18 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id); 19 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id); 20 21 /* 22 * Reschedule call back. 23 */ 24 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id) 25 { 26 inc_irq_stat(irq_resched_count); 27 scheduler_ipi(); 28 29 return IRQ_HANDLED; 30 } 31 32 void xen_smp_intr_free(unsigned int cpu) 33 { 34 kfree(per_cpu(xen_resched_irq, cpu).name); 35 per_cpu(xen_resched_irq, cpu).name = NULL; 36 if (per_cpu(xen_resched_irq, cpu).irq >= 0) { 37 unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu).irq, NULL); 38 per_cpu(xen_resched_irq, cpu).irq = -1; 39 } 40 kfree(per_cpu(xen_callfunc_irq, cpu).name); 41 per_cpu(xen_callfunc_irq, cpu).name = NULL; 42 if (per_cpu(xen_callfunc_irq, cpu).irq >= 0) { 43 unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu).irq, NULL); 44 per_cpu(xen_callfunc_irq, cpu).irq = -1; 45 } 46 kfree(per_cpu(xen_debug_irq, cpu).name); 47 per_cpu(xen_debug_irq, cpu).name = NULL; 48 if (per_cpu(xen_debug_irq, cpu).irq >= 0) { 49 unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu).irq, NULL); 50 per_cpu(xen_debug_irq, cpu).irq = -1; 51 } 52 kfree(per_cpu(xen_callfuncsingle_irq, cpu).name); 53 per_cpu(xen_callfuncsingle_irq, cpu).name = NULL; 54 if (per_cpu(xen_callfuncsingle_irq, cpu).irq >= 0) { 55 unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu).irq, 56 NULL); 57 per_cpu(xen_callfuncsingle_irq, cpu).irq = -1; 58 } 59 } 60 61 int xen_smp_intr_init(unsigned int cpu) 62 { 63 int rc; 64 char *resched_name, *callfunc_name, *debug_name; 65 66 resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu); 67 if (!resched_name) 68 goto fail_mem; 69 per_cpu(xen_resched_irq, cpu).name = resched_name; 70 rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR, 71 cpu, 72 xen_reschedule_interrupt, 73 IRQF_PERCPU|IRQF_NOBALANCING, 74 resched_name, 75 NULL); 76 if (rc < 0) 77 goto fail; 78 per_cpu(xen_resched_irq, cpu).irq = rc; 79 80 callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu); 81 if (!callfunc_name) 82 goto fail_mem; 83 per_cpu(xen_callfunc_irq, cpu).name = callfunc_name; 84 rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR, 85 cpu, 86 xen_call_function_interrupt, 87 IRQF_PERCPU|IRQF_NOBALANCING, 88 callfunc_name, 89 NULL); 90 if (rc < 0) 91 goto fail; 92 per_cpu(xen_callfunc_irq, cpu).irq = rc; 93 94 if (!xen_fifo_events) { 95 debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu); 96 if (!debug_name) 97 goto fail_mem; 98 99 per_cpu(xen_debug_irq, cpu).name = debug_name; 100 rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, 101 xen_debug_interrupt, 102 IRQF_PERCPU | IRQF_NOBALANCING, 103 debug_name, NULL); 104 if (rc < 0) 105 goto fail; 106 per_cpu(xen_debug_irq, cpu).irq = rc; 107 } 108 109 callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu); 110 if (!callfunc_name) 111 goto fail_mem; 112 113 per_cpu(xen_callfuncsingle_irq, cpu).name = callfunc_name; 114 rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR, 115 cpu, 116 xen_call_function_single_interrupt, 117 IRQF_PERCPU|IRQF_NOBALANCING, 118 callfunc_name, 119 NULL); 120 if (rc < 0) 121 goto fail; 122 per_cpu(xen_callfuncsingle_irq, cpu).irq = rc; 123 124 return 0; 125 126 fail_mem: 127 rc = -ENOMEM; 128 fail: 129 xen_smp_intr_free(cpu); 130 return rc; 131 } 132 133 void __init xen_smp_cpus_done(unsigned int max_cpus) 134 { 135 if (xen_hvm_domain()) 136 native_smp_cpus_done(max_cpus); 137 } 138 139 void xen_smp_send_reschedule(int cpu) 140 { 141 xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR); 142 } 143 144 static void __xen_send_IPI_mask(const struct cpumask *mask, 145 int vector) 146 { 147 unsigned cpu; 148 149 for_each_cpu_and(cpu, mask, cpu_online_mask) 150 xen_send_IPI_one(cpu, vector); 151 } 152 153 void xen_smp_send_call_function_ipi(const struct cpumask *mask) 154 { 155 int cpu; 156 157 __xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR); 158 159 /* Make sure other vcpus get a chance to run if they need to. */ 160 for_each_cpu(cpu, mask) { 161 if (xen_vcpu_stolen(cpu)) { 162 HYPERVISOR_sched_op(SCHEDOP_yield, NULL); 163 break; 164 } 165 } 166 } 167 168 void xen_smp_send_call_function_single_ipi(int cpu) 169 { 170 __xen_send_IPI_mask(cpumask_of(cpu), 171 XEN_CALL_FUNCTION_SINGLE_VECTOR); 172 } 173 174 static inline int xen_map_vector(int vector) 175 { 176 int xen_vector; 177 178 switch (vector) { 179 case RESCHEDULE_VECTOR: 180 xen_vector = XEN_RESCHEDULE_VECTOR; 181 break; 182 case CALL_FUNCTION_VECTOR: 183 xen_vector = XEN_CALL_FUNCTION_VECTOR; 184 break; 185 case CALL_FUNCTION_SINGLE_VECTOR: 186 xen_vector = XEN_CALL_FUNCTION_SINGLE_VECTOR; 187 break; 188 case IRQ_WORK_VECTOR: 189 xen_vector = XEN_IRQ_WORK_VECTOR; 190 break; 191 #ifdef CONFIG_X86_64 192 case NMI_VECTOR: 193 case APIC_DM_NMI: /* Some use that instead of NMI_VECTOR */ 194 xen_vector = XEN_NMI_VECTOR; 195 break; 196 #endif 197 default: 198 xen_vector = -1; 199 printk(KERN_ERR "xen: vector 0x%x is not implemented\n", 200 vector); 201 } 202 203 return xen_vector; 204 } 205 206 void xen_send_IPI_mask(const struct cpumask *mask, 207 int vector) 208 { 209 int xen_vector = xen_map_vector(vector); 210 211 if (xen_vector >= 0) 212 __xen_send_IPI_mask(mask, xen_vector); 213 } 214 215 void xen_send_IPI_all(int vector) 216 { 217 int xen_vector = xen_map_vector(vector); 218 219 if (xen_vector >= 0) 220 __xen_send_IPI_mask(cpu_online_mask, xen_vector); 221 } 222 223 void xen_send_IPI_self(int vector) 224 { 225 int xen_vector = xen_map_vector(vector); 226 227 if (xen_vector >= 0) 228 xen_send_IPI_one(smp_processor_id(), xen_vector); 229 } 230 231 void xen_send_IPI_mask_allbutself(const struct cpumask *mask, 232 int vector) 233 { 234 unsigned cpu; 235 unsigned int this_cpu = smp_processor_id(); 236 int xen_vector = xen_map_vector(vector); 237 238 if (!(num_online_cpus() > 1) || (xen_vector < 0)) 239 return; 240 241 for_each_cpu_and(cpu, mask, cpu_online_mask) { 242 if (this_cpu == cpu) 243 continue; 244 245 xen_send_IPI_one(cpu, xen_vector); 246 } 247 } 248 249 void xen_send_IPI_allbutself(int vector) 250 { 251 xen_send_IPI_mask_allbutself(cpu_online_mask, vector); 252 } 253 254 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id) 255 { 256 generic_smp_call_function_interrupt(); 257 inc_irq_stat(irq_call_count); 258 259 return IRQ_HANDLED; 260 } 261 262 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id) 263 { 264 generic_smp_call_function_single_interrupt(); 265 inc_irq_stat(irq_call_count); 266 267 return IRQ_HANDLED; 268 } 269