1a88b5ba8SSam Ravnborg /* smp.c: Sparc64 SMP support. 2a88b5ba8SSam Ravnborg * 3a88b5ba8SSam Ravnborg * Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net) 4a88b5ba8SSam Ravnborg */ 5a88b5ba8SSam Ravnborg 6066bcacaSPaul Gortmaker #include <linux/export.h> 7a88b5ba8SSam Ravnborg #include <linux/kernel.h> 8a88b5ba8SSam Ravnborg #include <linux/sched.h> 9a88b5ba8SSam Ravnborg #include <linux/mm.h> 10a88b5ba8SSam Ravnborg #include <linux/pagemap.h> 11a88b5ba8SSam Ravnborg #include <linux/threads.h> 12a88b5ba8SSam Ravnborg #include <linux/smp.h> 13a88b5ba8SSam Ravnborg #include <linux/interrupt.h> 14a88b5ba8SSam Ravnborg #include <linux/kernel_stat.h> 15a88b5ba8SSam Ravnborg #include <linux/delay.h> 16a88b5ba8SSam Ravnborg #include <linux/init.h> 17a88b5ba8SSam Ravnborg #include <linux/spinlock.h> 18a88b5ba8SSam Ravnborg #include <linux/fs.h> 19a88b5ba8SSam Ravnborg #include <linux/seq_file.h> 20a88b5ba8SSam Ravnborg #include <linux/cache.h> 21a88b5ba8SSam Ravnborg #include <linux/jiffies.h> 22a88b5ba8SSam Ravnborg #include <linux/profile.h> 2373fffc03SDavid S. Miller #include <linux/bootmem.h> 244fd78a5fSDavid S. Miller #include <linux/vmalloc.h> 259960e9e8SDavid S. Miller #include <linux/ftrace.h> 26a88b5ba8SSam Ravnborg #include <linux/cpu.h> 275a0e3ad6STejun Heo #include <linux/slab.h> 28d3091298SSam Ravnborg #include <linux/kgdb.h> 29a88b5ba8SSam Ravnborg 30a88b5ba8SSam Ravnborg #include <asm/head.h> 31a88b5ba8SSam Ravnborg #include <asm/ptrace.h> 3260063497SArun Sharma #include <linux/atomic.h> 33a88b5ba8SSam Ravnborg #include <asm/tlbflush.h> 34a88b5ba8SSam Ravnborg #include <asm/mmu_context.h> 35a88b5ba8SSam Ravnborg #include <asm/cpudata.h> 36a88b5ba8SSam Ravnborg #include <asm/hvtramp.h> 37a88b5ba8SSam Ravnborg #include <asm/io.h> 38a88b5ba8SSam Ravnborg #include <asm/timer.h> 3959dec13bSSam Ravnborg #include <asm/setup.h> 40a88b5ba8SSam Ravnborg 41a88b5ba8SSam Ravnborg #include <asm/irq.h> 42a88b5ba8SSam Ravnborg #include <asm/irq_regs.h> 43a88b5ba8SSam Ravnborg #include <asm/page.h> 44a88b5ba8SSam Ravnborg #include <asm/pgtable.h> 45a88b5ba8SSam Ravnborg #include <asm/oplib.h> 467c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 47a88b5ba8SSam Ravnborg #include <asm/starfire.h> 48a88b5ba8SSam Ravnborg #include <asm/tlb.h> 49a88b5ba8SSam Ravnborg #include <asm/sections.h> 50a88b5ba8SSam Ravnborg #include <asm/prom.h> 51a88b5ba8SSam Ravnborg #include <asm/mdesc.h> 52a88b5ba8SSam Ravnborg #include <asm/ldc.h> 53a88b5ba8SSam Ravnborg #include <asm/hypervisor.h> 54b62818e5SDavid S. Miller #include <asm/pcr.h> 55a88b5ba8SSam Ravnborg 56280ff974SHong H. Pham #include "cpumap.h" 57a0c54a21SSam Ravnborg #include "kernel.h" 58280ff974SHong H. Pham 59a88b5ba8SSam Ravnborg DEFINE_PER_CPU(cpumask_t, cpu_sibling_map) = CPU_MASK_NONE; 60a88b5ba8SSam Ravnborg cpumask_t cpu_core_map[NR_CPUS] __read_mostly = 61a88b5ba8SSam Ravnborg { [0 ... NR_CPUS-1] = CPU_MASK_NONE }; 62a88b5ba8SSam Ravnborg 63acc455cfSchris hyser cpumask_t cpu_core_sib_map[NR_CPUS] __read_mostly = { 64acc455cfSchris hyser [0 ... NR_CPUS-1] = CPU_MASK_NONE }; 65acc455cfSchris hyser 66d624716bSAtish Patra cpumask_t cpu_core_sib_cache_map[NR_CPUS] __read_mostly = { 67d624716bSAtish Patra [0 ... NR_CPUS - 1] = CPU_MASK_NONE }; 68d624716bSAtish Patra 69a88b5ba8SSam Ravnborg EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); 70a88b5ba8SSam Ravnborg EXPORT_SYMBOL(cpu_core_map); 71acc455cfSchris hyser EXPORT_SYMBOL(cpu_core_sib_map); 72d624716bSAtish Patra EXPORT_SYMBOL(cpu_core_sib_cache_map); 73a88b5ba8SSam Ravnborg 74a88b5ba8SSam Ravnborg static cpumask_t smp_commenced_mask; 75a88b5ba8SSam Ravnborg 76a88b5ba8SSam Ravnborg void smp_info(struct seq_file *m) 77a88b5ba8SSam Ravnborg { 78a88b5ba8SSam Ravnborg int i; 79a88b5ba8SSam Ravnborg 80a88b5ba8SSam Ravnborg seq_printf(m, "State:\n"); 81a88b5ba8SSam Ravnborg for_each_online_cpu(i) 82a88b5ba8SSam Ravnborg seq_printf(m, "CPU%d:\t\tonline\n", i); 83a88b5ba8SSam Ravnborg } 84a88b5ba8SSam Ravnborg 85a88b5ba8SSam Ravnborg void smp_bogo(struct seq_file *m) 86a88b5ba8SSam Ravnborg { 87a88b5ba8SSam Ravnborg int i; 88a88b5ba8SSam Ravnborg 89a88b5ba8SSam Ravnborg for_each_online_cpu(i) 90a88b5ba8SSam Ravnborg seq_printf(m, 91a88b5ba8SSam Ravnborg "Cpu%dClkTck\t: %016lx\n", 92a88b5ba8SSam Ravnborg i, cpu_data(i).clock_tick); 93a88b5ba8SSam Ravnborg } 94a88b5ba8SSam Ravnborg 95a88b5ba8SSam Ravnborg extern void setup_sparc64_timer(void); 96a88b5ba8SSam Ravnborg 97a88b5ba8SSam Ravnborg static volatile unsigned long callin_flag = 0; 98a88b5ba8SSam Ravnborg 992066aaddSPaul Gortmaker void smp_callin(void) 100a88b5ba8SSam Ravnborg { 101a88b5ba8SSam Ravnborg int cpuid = hard_smp_processor_id(); 102a88b5ba8SSam Ravnborg 103a88b5ba8SSam Ravnborg __local_per_cpu_offset = __per_cpu_offset(cpuid); 104a88b5ba8SSam Ravnborg 105a88b5ba8SSam Ravnborg if (tlb_type == hypervisor) 106a88b5ba8SSam Ravnborg sun4v_ktsb_register(); 107a88b5ba8SSam Ravnborg 108a88b5ba8SSam Ravnborg __flush_tlb_all(); 109a88b5ba8SSam Ravnborg 110a88b5ba8SSam Ravnborg setup_sparc64_timer(); 111a88b5ba8SSam Ravnborg 112a88b5ba8SSam Ravnborg if (cheetah_pcache_forced_on) 113a88b5ba8SSam Ravnborg cheetah_enable_pcache(); 114a88b5ba8SSam Ravnborg 115a88b5ba8SSam Ravnborg callin_flag = 1; 116a88b5ba8SSam Ravnborg __asm__ __volatile__("membar #Sync\n\t" 117a88b5ba8SSam Ravnborg "flush %%g6" : : : "memory"); 118a88b5ba8SSam Ravnborg 119a88b5ba8SSam Ravnborg /* Clear this or we will die instantly when we 120a88b5ba8SSam Ravnborg * schedule back to this idler... 121a88b5ba8SSam Ravnborg */ 122a88b5ba8SSam Ravnborg current_thread_info()->new_child = 0; 123a88b5ba8SSam Ravnborg 124a88b5ba8SSam Ravnborg /* Attach to the address space of init_task. */ 125a88b5ba8SSam Ravnborg atomic_inc(&init_mm.mm_count); 126a88b5ba8SSam Ravnborg current->active_mm = &init_mm; 127a88b5ba8SSam Ravnborg 128a88b5ba8SSam Ravnborg /* inform the notifiers about the new cpu */ 129a88b5ba8SSam Ravnborg notify_cpu_starting(cpuid); 130a88b5ba8SSam Ravnborg 131fb1fece5SKOSAKI Motohiro while (!cpumask_test_cpu(cpuid, &smp_commenced_mask)) 132a88b5ba8SSam Ravnborg rmb(); 133a88b5ba8SSam Ravnborg 134fb1fece5SKOSAKI Motohiro set_cpu_online(cpuid, true); 135a88b5ba8SSam Ravnborg 136a88b5ba8SSam Ravnborg /* idle thread is expected to have preempt disabled */ 137a88b5ba8SSam Ravnborg preempt_disable(); 13887fa05aeSSam Ravnborg 139ce2521bfSKirill Tkhai local_irq_enable(); 140ce2521bfSKirill Tkhai 141fc6d73d6SThomas Gleixner cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 142a88b5ba8SSam Ravnborg } 143a88b5ba8SSam Ravnborg 144a88b5ba8SSam Ravnborg void cpu_panic(void) 145a88b5ba8SSam Ravnborg { 146a88b5ba8SSam Ravnborg printk("CPU[%d]: Returns from cpu_idle!\n", smp_processor_id()); 147a88b5ba8SSam Ravnborg panic("SMP bolixed\n"); 148a88b5ba8SSam Ravnborg } 149a88b5ba8SSam Ravnborg 150a88b5ba8SSam Ravnborg /* This tick register synchronization scheme is taken entirely from 151a88b5ba8SSam Ravnborg * the ia64 port, see arch/ia64/kernel/smpboot.c for details and credit. 152a88b5ba8SSam Ravnborg * 153a88b5ba8SSam Ravnborg * The only change I've made is to rework it so that the master 154a88b5ba8SSam Ravnborg * initiates the synchonization instead of the slave. -DaveM 155a88b5ba8SSam Ravnborg */ 156a88b5ba8SSam Ravnborg 157a88b5ba8SSam Ravnborg #define MASTER 0 158a88b5ba8SSam Ravnborg #define SLAVE (SMP_CACHE_BYTES/sizeof(unsigned long)) 159a88b5ba8SSam Ravnborg 160a88b5ba8SSam Ravnborg #define NUM_ROUNDS 64 /* magic value */ 161a88b5ba8SSam Ravnborg #define NUM_ITERS 5 /* likewise */ 162a88b5ba8SSam Ravnborg 16349b6c01fSKirill Tkhai static DEFINE_RAW_SPINLOCK(itc_sync_lock); 164a88b5ba8SSam Ravnborg static unsigned long go[SLAVE + 1]; 165a88b5ba8SSam Ravnborg 166a88b5ba8SSam Ravnborg #define DEBUG_TICK_SYNC 0 167a88b5ba8SSam Ravnborg 168a88b5ba8SSam Ravnborg static inline long get_delta (long *rt, long *master) 169a88b5ba8SSam Ravnborg { 170a88b5ba8SSam Ravnborg unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0; 171a88b5ba8SSam Ravnborg unsigned long tcenter, t0, t1, tm; 172a88b5ba8SSam Ravnborg unsigned long i; 173a88b5ba8SSam Ravnborg 174a88b5ba8SSam Ravnborg for (i = 0; i < NUM_ITERS; i++) { 175a88b5ba8SSam Ravnborg t0 = tick_ops->get_tick(); 176a88b5ba8SSam Ravnborg go[MASTER] = 1; 177a88b5ba8SSam Ravnborg membar_safe("#StoreLoad"); 178a88b5ba8SSam Ravnborg while (!(tm = go[SLAVE])) 179a88b5ba8SSam Ravnborg rmb(); 180a88b5ba8SSam Ravnborg go[SLAVE] = 0; 181a88b5ba8SSam Ravnborg wmb(); 182a88b5ba8SSam Ravnborg t1 = tick_ops->get_tick(); 183a88b5ba8SSam Ravnborg 184a88b5ba8SSam Ravnborg if (t1 - t0 < best_t1 - best_t0) 185a88b5ba8SSam Ravnborg best_t0 = t0, best_t1 = t1, best_tm = tm; 186a88b5ba8SSam Ravnborg } 187a88b5ba8SSam Ravnborg 188a88b5ba8SSam Ravnborg *rt = best_t1 - best_t0; 189a88b5ba8SSam Ravnborg *master = best_tm - best_t0; 190a88b5ba8SSam Ravnborg 191a88b5ba8SSam Ravnborg /* average best_t0 and best_t1 without overflow: */ 192a88b5ba8SSam Ravnborg tcenter = (best_t0/2 + best_t1/2); 193a88b5ba8SSam Ravnborg if (best_t0 % 2 + best_t1 % 2 == 2) 194a88b5ba8SSam Ravnborg tcenter++; 195a88b5ba8SSam Ravnborg return tcenter - best_tm; 196a88b5ba8SSam Ravnborg } 197a88b5ba8SSam Ravnborg 198a88b5ba8SSam Ravnborg void smp_synchronize_tick_client(void) 199a88b5ba8SSam Ravnborg { 200a88b5ba8SSam Ravnborg long i, delta, adj, adjust_latency = 0, done = 0; 201c6fee081SDavid S. Miller unsigned long flags, rt, master_time_stamp; 202a88b5ba8SSam Ravnborg #if DEBUG_TICK_SYNC 203a88b5ba8SSam Ravnborg struct { 204a88b5ba8SSam Ravnborg long rt; /* roundtrip time */ 205a88b5ba8SSam Ravnborg long master; /* master's timestamp */ 206a88b5ba8SSam Ravnborg long diff; /* difference between midpoint and master's timestamp */ 207a88b5ba8SSam Ravnborg long lat; /* estimate of itc adjustment latency */ 208a88b5ba8SSam Ravnborg } t[NUM_ROUNDS]; 209a88b5ba8SSam Ravnborg #endif 210a88b5ba8SSam Ravnborg 211a88b5ba8SSam Ravnborg go[MASTER] = 1; 212a88b5ba8SSam Ravnborg 213a88b5ba8SSam Ravnborg while (go[MASTER]) 214a88b5ba8SSam Ravnborg rmb(); 215a88b5ba8SSam Ravnborg 216a88b5ba8SSam Ravnborg local_irq_save(flags); 217a88b5ba8SSam Ravnborg { 218a88b5ba8SSam Ravnborg for (i = 0; i < NUM_ROUNDS; i++) { 219a88b5ba8SSam Ravnborg delta = get_delta(&rt, &master_time_stamp); 220c6fee081SDavid S. Miller if (delta == 0) 221a88b5ba8SSam Ravnborg done = 1; /* let's lock on to this... */ 222a88b5ba8SSam Ravnborg 223a88b5ba8SSam Ravnborg if (!done) { 224a88b5ba8SSam Ravnborg if (i > 0) { 225a88b5ba8SSam Ravnborg adjust_latency += -delta; 226a88b5ba8SSam Ravnborg adj = -delta + adjust_latency/4; 227a88b5ba8SSam Ravnborg } else 228a88b5ba8SSam Ravnborg adj = -delta; 229a88b5ba8SSam Ravnborg 230a88b5ba8SSam Ravnborg tick_ops->add_tick(adj); 231a88b5ba8SSam Ravnborg } 232a88b5ba8SSam Ravnborg #if DEBUG_TICK_SYNC 233a88b5ba8SSam Ravnborg t[i].rt = rt; 234a88b5ba8SSam Ravnborg t[i].master = master_time_stamp; 235a88b5ba8SSam Ravnborg t[i].diff = delta; 236a88b5ba8SSam Ravnborg t[i].lat = adjust_latency/4; 237a88b5ba8SSam Ravnborg #endif 238a88b5ba8SSam Ravnborg } 239a88b5ba8SSam Ravnborg } 240a88b5ba8SSam Ravnborg local_irq_restore(flags); 241a88b5ba8SSam Ravnborg 242a88b5ba8SSam Ravnborg #if DEBUG_TICK_SYNC 243a88b5ba8SSam Ravnborg for (i = 0; i < NUM_ROUNDS; i++) 244a88b5ba8SSam Ravnborg printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n", 245a88b5ba8SSam Ravnborg t[i].rt, t[i].master, t[i].diff, t[i].lat); 246a88b5ba8SSam Ravnborg #endif 247a88b5ba8SSam Ravnborg 248a88b5ba8SSam Ravnborg printk(KERN_INFO "CPU %d: synchronized TICK with master CPU " 249a88b5ba8SSam Ravnborg "(last diff %ld cycles, maxerr %lu cycles)\n", 250a88b5ba8SSam Ravnborg smp_processor_id(), delta, rt); 251a88b5ba8SSam Ravnborg } 252a88b5ba8SSam Ravnborg 253a88b5ba8SSam Ravnborg static void smp_start_sync_tick_client(int cpu); 254a88b5ba8SSam Ravnborg 255a88b5ba8SSam Ravnborg static void smp_synchronize_one_tick(int cpu) 256a88b5ba8SSam Ravnborg { 257a88b5ba8SSam Ravnborg unsigned long flags, i; 258a88b5ba8SSam Ravnborg 259a88b5ba8SSam Ravnborg go[MASTER] = 0; 260a88b5ba8SSam Ravnborg 261a88b5ba8SSam Ravnborg smp_start_sync_tick_client(cpu); 262a88b5ba8SSam Ravnborg 263a88b5ba8SSam Ravnborg /* wait for client to be ready */ 264a88b5ba8SSam Ravnborg while (!go[MASTER]) 265a88b5ba8SSam Ravnborg rmb(); 266a88b5ba8SSam Ravnborg 267a88b5ba8SSam Ravnborg /* now let the client proceed into his loop */ 268a88b5ba8SSam Ravnborg go[MASTER] = 0; 269a88b5ba8SSam Ravnborg membar_safe("#StoreLoad"); 270a88b5ba8SSam Ravnborg 27149b6c01fSKirill Tkhai raw_spin_lock_irqsave(&itc_sync_lock, flags); 272a88b5ba8SSam Ravnborg { 273a88b5ba8SSam Ravnborg for (i = 0; i < NUM_ROUNDS*NUM_ITERS; i++) { 274a88b5ba8SSam Ravnborg while (!go[MASTER]) 275a88b5ba8SSam Ravnborg rmb(); 276a88b5ba8SSam Ravnborg go[MASTER] = 0; 277a88b5ba8SSam Ravnborg wmb(); 278a88b5ba8SSam Ravnborg go[SLAVE] = tick_ops->get_tick(); 279a88b5ba8SSam Ravnborg membar_safe("#StoreLoad"); 280a88b5ba8SSam Ravnborg } 281a88b5ba8SSam Ravnborg } 28249b6c01fSKirill Tkhai raw_spin_unlock_irqrestore(&itc_sync_lock, flags); 283a88b5ba8SSam Ravnborg } 284a88b5ba8SSam Ravnborg 285a88b5ba8SSam Ravnborg #if defined(CONFIG_SUN_LDOMS) && defined(CONFIG_HOTPLUG_CPU) 2862066aaddSPaul Gortmaker static void ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg, 2872066aaddSPaul Gortmaker void **descrp) 288a88b5ba8SSam Ravnborg { 289a88b5ba8SSam Ravnborg extern unsigned long sparc64_ttable_tl0; 290a88b5ba8SSam Ravnborg extern unsigned long kern_locked_tte_data; 291a88b5ba8SSam Ravnborg struct hvtramp_descr *hdesc; 292a88b5ba8SSam Ravnborg unsigned long trampoline_ra; 293a88b5ba8SSam Ravnborg struct trap_per_cpu *tb; 294a88b5ba8SSam Ravnborg u64 tte_vaddr, tte_data; 295a88b5ba8SSam Ravnborg unsigned long hv_err; 296a88b5ba8SSam Ravnborg int i; 297a88b5ba8SSam Ravnborg 298a88b5ba8SSam Ravnborg hdesc = kzalloc(sizeof(*hdesc) + 299a88b5ba8SSam Ravnborg (sizeof(struct hvtramp_mapping) * 300a88b5ba8SSam Ravnborg num_kernel_image_mappings - 1), 301a88b5ba8SSam Ravnborg GFP_KERNEL); 302a88b5ba8SSam Ravnborg if (!hdesc) { 303a88b5ba8SSam Ravnborg printk(KERN_ERR "ldom_startcpu_cpuid: Cannot allocate " 304a88b5ba8SSam Ravnborg "hvtramp_descr.\n"); 305a88b5ba8SSam Ravnborg return; 306a88b5ba8SSam Ravnborg } 307557fe0e8SDavid S. Miller *descrp = hdesc; 308a88b5ba8SSam Ravnborg 309a88b5ba8SSam Ravnborg hdesc->cpu = cpu; 310a88b5ba8SSam Ravnborg hdesc->num_mappings = num_kernel_image_mappings; 311a88b5ba8SSam Ravnborg 312a88b5ba8SSam Ravnborg tb = &trap_block[cpu]; 313a88b5ba8SSam Ravnborg 314a88b5ba8SSam Ravnborg hdesc->fault_info_va = (unsigned long) &tb->fault_info; 315a88b5ba8SSam Ravnborg hdesc->fault_info_pa = kimage_addr_to_ra(&tb->fault_info); 316a88b5ba8SSam Ravnborg 317a88b5ba8SSam Ravnborg hdesc->thread_reg = thread_reg; 318a88b5ba8SSam Ravnborg 319a88b5ba8SSam Ravnborg tte_vaddr = (unsigned long) KERNBASE; 320a88b5ba8SSam Ravnborg tte_data = kern_locked_tte_data; 321a88b5ba8SSam Ravnborg 322a88b5ba8SSam Ravnborg for (i = 0; i < hdesc->num_mappings; i++) { 323a88b5ba8SSam Ravnborg hdesc->maps[i].vaddr = tte_vaddr; 324a88b5ba8SSam Ravnborg hdesc->maps[i].tte = tte_data; 325a88b5ba8SSam Ravnborg tte_vaddr += 0x400000; 326a88b5ba8SSam Ravnborg tte_data += 0x400000; 327a88b5ba8SSam Ravnborg } 328a88b5ba8SSam Ravnborg 329a88b5ba8SSam Ravnborg trampoline_ra = kimage_addr_to_ra(hv_cpu_startup); 330a88b5ba8SSam Ravnborg 331a88b5ba8SSam Ravnborg hv_err = sun4v_cpu_start(cpu, trampoline_ra, 332a88b5ba8SSam Ravnborg kimage_addr_to_ra(&sparc64_ttable_tl0), 333a88b5ba8SSam Ravnborg __pa(hdesc)); 334a88b5ba8SSam Ravnborg if (hv_err) 335a88b5ba8SSam Ravnborg printk(KERN_ERR "ldom_startcpu_cpuid: sun4v_cpu_start() " 336a88b5ba8SSam Ravnborg "gives error %lu\n", hv_err); 337a88b5ba8SSam Ravnborg } 338a88b5ba8SSam Ravnborg #endif 339a88b5ba8SSam Ravnborg 340a88b5ba8SSam Ravnborg extern unsigned long sparc64_cpu_startup; 341a88b5ba8SSam Ravnborg 342a88b5ba8SSam Ravnborg /* The OBP cpu startup callback truncates the 3rd arg cookie to 343a88b5ba8SSam Ravnborg * 32-bits (I think) so to be safe we have it read the pointer 344a88b5ba8SSam Ravnborg * contained here so we work on >4GB machines. -DaveM 345a88b5ba8SSam Ravnborg */ 346a88b5ba8SSam Ravnborg static struct thread_info *cpu_new_thread = NULL; 347a88b5ba8SSam Ravnborg 3482066aaddSPaul Gortmaker static int smp_boot_one_cpu(unsigned int cpu, struct task_struct *idle) 349a88b5ba8SSam Ravnborg { 350a88b5ba8SSam Ravnborg unsigned long entry = 351a88b5ba8SSam Ravnborg (unsigned long)(&sparc64_cpu_startup); 352a88b5ba8SSam Ravnborg unsigned long cookie = 353a88b5ba8SSam Ravnborg (unsigned long)(&cpu_new_thread); 354557fe0e8SDavid S. Miller void *descr = NULL; 355a88b5ba8SSam Ravnborg int timeout, ret; 356a88b5ba8SSam Ravnborg 357a88b5ba8SSam Ravnborg callin_flag = 0; 358f0a2bc7eSThomas Gleixner cpu_new_thread = task_thread_info(idle); 359a88b5ba8SSam Ravnborg 360a88b5ba8SSam Ravnborg if (tlb_type == hypervisor) { 361a88b5ba8SSam Ravnborg #if defined(CONFIG_SUN_LDOMS) && defined(CONFIG_HOTPLUG_CPU) 362a88b5ba8SSam Ravnborg if (ldom_domaining_enabled) 363a88b5ba8SSam Ravnborg ldom_startcpu_cpuid(cpu, 364557fe0e8SDavid S. Miller (unsigned long) cpu_new_thread, 365557fe0e8SDavid S. Miller &descr); 366a88b5ba8SSam Ravnborg else 367a88b5ba8SSam Ravnborg #endif 368a88b5ba8SSam Ravnborg prom_startcpu_cpuid(cpu, entry, cookie); 369a88b5ba8SSam Ravnborg } else { 370a88b5ba8SSam Ravnborg struct device_node *dp = of_find_node_by_cpuid(cpu); 371a88b5ba8SSam Ravnborg 3726016a363SGrant Likely prom_startcpu(dp->phandle, entry, cookie); 373a88b5ba8SSam Ravnborg } 374a88b5ba8SSam Ravnborg 375a88b5ba8SSam Ravnborg for (timeout = 0; timeout < 50000; timeout++) { 376a88b5ba8SSam Ravnborg if (callin_flag) 377a88b5ba8SSam Ravnborg break; 378a88b5ba8SSam Ravnborg udelay(100); 379a88b5ba8SSam Ravnborg } 380a88b5ba8SSam Ravnborg 381a88b5ba8SSam Ravnborg if (callin_flag) { 382a88b5ba8SSam Ravnborg ret = 0; 383a88b5ba8SSam Ravnborg } else { 384a88b5ba8SSam Ravnborg printk("Processor %d is stuck.\n", cpu); 385a88b5ba8SSam Ravnborg ret = -ENODEV; 386a88b5ba8SSam Ravnborg } 387a88b5ba8SSam Ravnborg cpu_new_thread = NULL; 388a88b5ba8SSam Ravnborg 389557fe0e8SDavid S. Miller kfree(descr); 390a88b5ba8SSam Ravnborg 391a88b5ba8SSam Ravnborg return ret; 392a88b5ba8SSam Ravnborg } 393a88b5ba8SSam Ravnborg 394a88b5ba8SSam Ravnborg static void spitfire_xcall_helper(u64 data0, u64 data1, u64 data2, u64 pstate, unsigned long cpu) 395a88b5ba8SSam Ravnborg { 396a88b5ba8SSam Ravnborg u64 result, target; 397a88b5ba8SSam Ravnborg int stuck, tmp; 398a88b5ba8SSam Ravnborg 399a88b5ba8SSam Ravnborg if (this_is_starfire) { 400a88b5ba8SSam Ravnborg /* map to real upaid */ 401a88b5ba8SSam Ravnborg cpu = (((cpu & 0x3c) << 1) | 402a88b5ba8SSam Ravnborg ((cpu & 0x40) >> 4) | 403a88b5ba8SSam Ravnborg (cpu & 0x3)); 404a88b5ba8SSam Ravnborg } 405a88b5ba8SSam Ravnborg 406a88b5ba8SSam Ravnborg target = (cpu << 14) | 0x70; 407a88b5ba8SSam Ravnborg again: 408a88b5ba8SSam Ravnborg /* Ok, this is the real Spitfire Errata #54. 409a88b5ba8SSam Ravnborg * One must read back from a UDB internal register 410a88b5ba8SSam Ravnborg * after writes to the UDB interrupt dispatch, but 411a88b5ba8SSam Ravnborg * before the membar Sync for that write. 412a88b5ba8SSam Ravnborg * So we use the high UDB control register (ASI 0x7f, 413a88b5ba8SSam Ravnborg * ADDR 0x20) for the dummy read. -DaveM 414a88b5ba8SSam Ravnborg */ 415a88b5ba8SSam Ravnborg tmp = 0x40; 416a88b5ba8SSam Ravnborg __asm__ __volatile__( 417a88b5ba8SSam Ravnborg "wrpr %1, %2, %%pstate\n\t" 418a88b5ba8SSam Ravnborg "stxa %4, [%0] %3\n\t" 419a88b5ba8SSam Ravnborg "stxa %5, [%0+%8] %3\n\t" 420a88b5ba8SSam Ravnborg "add %0, %8, %0\n\t" 421a88b5ba8SSam Ravnborg "stxa %6, [%0+%8] %3\n\t" 422a88b5ba8SSam Ravnborg "membar #Sync\n\t" 423a88b5ba8SSam Ravnborg "stxa %%g0, [%7] %3\n\t" 424a88b5ba8SSam Ravnborg "membar #Sync\n\t" 425a88b5ba8SSam Ravnborg "mov 0x20, %%g1\n\t" 426a88b5ba8SSam Ravnborg "ldxa [%%g1] 0x7f, %%g0\n\t" 427a88b5ba8SSam Ravnborg "membar #Sync" 428a88b5ba8SSam Ravnborg : "=r" (tmp) 429a88b5ba8SSam Ravnborg : "r" (pstate), "i" (PSTATE_IE), "i" (ASI_INTR_W), 430a88b5ba8SSam Ravnborg "r" (data0), "r" (data1), "r" (data2), "r" (target), 431a88b5ba8SSam Ravnborg "r" (0x10), "0" (tmp) 432a88b5ba8SSam Ravnborg : "g1"); 433a88b5ba8SSam Ravnborg 434a88b5ba8SSam Ravnborg /* NOTE: PSTATE_IE is still clear. */ 435a88b5ba8SSam Ravnborg stuck = 100000; 436a88b5ba8SSam Ravnborg do { 437a88b5ba8SSam Ravnborg __asm__ __volatile__("ldxa [%%g0] %1, %0" 438a88b5ba8SSam Ravnborg : "=r" (result) 439a88b5ba8SSam Ravnborg : "i" (ASI_INTR_DISPATCH_STAT)); 440a88b5ba8SSam Ravnborg if (result == 0) { 441a88b5ba8SSam Ravnborg __asm__ __volatile__("wrpr %0, 0x0, %%pstate" 442a88b5ba8SSam Ravnborg : : "r" (pstate)); 443a88b5ba8SSam Ravnborg return; 444a88b5ba8SSam Ravnborg } 445a88b5ba8SSam Ravnborg stuck -= 1; 446a88b5ba8SSam Ravnborg if (stuck == 0) 447a88b5ba8SSam Ravnborg break; 448a88b5ba8SSam Ravnborg } while (result & 0x1); 449a88b5ba8SSam Ravnborg __asm__ __volatile__("wrpr %0, 0x0, %%pstate" 450a88b5ba8SSam Ravnborg : : "r" (pstate)); 451a88b5ba8SSam Ravnborg if (stuck == 0) { 45290181136SSam Ravnborg printk("CPU[%d]: mondo stuckage result[%016llx]\n", 453a88b5ba8SSam Ravnborg smp_processor_id(), result); 454a88b5ba8SSam Ravnborg } else { 455a88b5ba8SSam Ravnborg udelay(2); 456a88b5ba8SSam Ravnborg goto again; 457a88b5ba8SSam Ravnborg } 458a88b5ba8SSam Ravnborg } 459a88b5ba8SSam Ravnborg 460a88b5ba8SSam Ravnborg static void spitfire_xcall_deliver(struct trap_per_cpu *tb, int cnt) 461a88b5ba8SSam Ravnborg { 462a88b5ba8SSam Ravnborg u64 *mondo, data0, data1, data2; 463a88b5ba8SSam Ravnborg u16 *cpu_list; 464a88b5ba8SSam Ravnborg u64 pstate; 465a88b5ba8SSam Ravnborg int i; 466a88b5ba8SSam Ravnborg 467a88b5ba8SSam Ravnborg __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate)); 468a88b5ba8SSam Ravnborg cpu_list = __va(tb->cpu_list_pa); 469a88b5ba8SSam Ravnborg mondo = __va(tb->cpu_mondo_block_pa); 470a88b5ba8SSam Ravnborg data0 = mondo[0]; 471a88b5ba8SSam Ravnborg data1 = mondo[1]; 472a88b5ba8SSam Ravnborg data2 = mondo[2]; 473a88b5ba8SSam Ravnborg for (i = 0; i < cnt; i++) 474a88b5ba8SSam Ravnborg spitfire_xcall_helper(data0, data1, data2, pstate, cpu_list[i]); 475a88b5ba8SSam Ravnborg } 476a88b5ba8SSam Ravnborg 477a88b5ba8SSam Ravnborg /* Cheetah now allows to send the whole 64-bytes of data in the interrupt 478a88b5ba8SSam Ravnborg * packet, but we have no use for that. However we do take advantage of 479a88b5ba8SSam Ravnborg * the new pipelining feature (ie. dispatch to multiple cpus simultaneously). 480a88b5ba8SSam Ravnborg */ 481a88b5ba8SSam Ravnborg static void cheetah_xcall_deliver(struct trap_per_cpu *tb, int cnt) 482a88b5ba8SSam Ravnborg { 483a88b5ba8SSam Ravnborg int nack_busy_id, is_jbus, need_more; 484a88b5ba8SSam Ravnborg u64 *mondo, pstate, ver, busy_mask; 485a88b5ba8SSam Ravnborg u16 *cpu_list; 486a88b5ba8SSam Ravnborg 487a88b5ba8SSam Ravnborg cpu_list = __va(tb->cpu_list_pa); 488a88b5ba8SSam Ravnborg mondo = __va(tb->cpu_mondo_block_pa); 489a88b5ba8SSam Ravnborg 490a88b5ba8SSam Ravnborg /* Unfortunately, someone at Sun had the brilliant idea to make the 491a88b5ba8SSam Ravnborg * busy/nack fields hard-coded by ITID number for this Ultra-III 492a88b5ba8SSam Ravnborg * derivative processor. 493a88b5ba8SSam Ravnborg */ 494a88b5ba8SSam Ravnborg __asm__ ("rdpr %%ver, %0" : "=r" (ver)); 495a88b5ba8SSam Ravnborg is_jbus = ((ver >> 32) == __JALAPENO_ID || 496a88b5ba8SSam Ravnborg (ver >> 32) == __SERRANO_ID); 497a88b5ba8SSam Ravnborg 498a88b5ba8SSam Ravnborg __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate)); 499a88b5ba8SSam Ravnborg 500a88b5ba8SSam Ravnborg retry: 501a88b5ba8SSam Ravnborg need_more = 0; 502a88b5ba8SSam Ravnborg __asm__ __volatile__("wrpr %0, %1, %%pstate\n\t" 503a88b5ba8SSam Ravnborg : : "r" (pstate), "i" (PSTATE_IE)); 504a88b5ba8SSam Ravnborg 505a88b5ba8SSam Ravnborg /* Setup the dispatch data registers. */ 506a88b5ba8SSam Ravnborg __asm__ __volatile__("stxa %0, [%3] %6\n\t" 507a88b5ba8SSam Ravnborg "stxa %1, [%4] %6\n\t" 508a88b5ba8SSam Ravnborg "stxa %2, [%5] %6\n\t" 509a88b5ba8SSam Ravnborg "membar #Sync\n\t" 510a88b5ba8SSam Ravnborg : /* no outputs */ 511a88b5ba8SSam Ravnborg : "r" (mondo[0]), "r" (mondo[1]), "r" (mondo[2]), 512a88b5ba8SSam Ravnborg "r" (0x40), "r" (0x50), "r" (0x60), 513a88b5ba8SSam Ravnborg "i" (ASI_INTR_W)); 514a88b5ba8SSam Ravnborg 515a88b5ba8SSam Ravnborg nack_busy_id = 0; 516a88b5ba8SSam Ravnborg busy_mask = 0; 517a88b5ba8SSam Ravnborg { 518a88b5ba8SSam Ravnborg int i; 519a88b5ba8SSam Ravnborg 520a88b5ba8SSam Ravnborg for (i = 0; i < cnt; i++) { 521a88b5ba8SSam Ravnborg u64 target, nr; 522a88b5ba8SSam Ravnborg 523a88b5ba8SSam Ravnborg nr = cpu_list[i]; 524a88b5ba8SSam Ravnborg if (nr == 0xffff) 525a88b5ba8SSam Ravnborg continue; 526a88b5ba8SSam Ravnborg 527a88b5ba8SSam Ravnborg target = (nr << 14) | 0x70; 528a88b5ba8SSam Ravnborg if (is_jbus) { 529a88b5ba8SSam Ravnborg busy_mask |= (0x1UL << (nr * 2)); 530a88b5ba8SSam Ravnborg } else { 531a88b5ba8SSam Ravnborg target |= (nack_busy_id << 24); 532a88b5ba8SSam Ravnborg busy_mask |= (0x1UL << 533a88b5ba8SSam Ravnborg (nack_busy_id * 2)); 534a88b5ba8SSam Ravnborg } 535a88b5ba8SSam Ravnborg __asm__ __volatile__( 536a88b5ba8SSam Ravnborg "stxa %%g0, [%0] %1\n\t" 537a88b5ba8SSam Ravnborg "membar #Sync\n\t" 538a88b5ba8SSam Ravnborg : /* no outputs */ 539a88b5ba8SSam Ravnborg : "r" (target), "i" (ASI_INTR_W)); 540a88b5ba8SSam Ravnborg nack_busy_id++; 541a88b5ba8SSam Ravnborg if (nack_busy_id == 32) { 542a88b5ba8SSam Ravnborg need_more = 1; 543a88b5ba8SSam Ravnborg break; 544a88b5ba8SSam Ravnborg } 545a88b5ba8SSam Ravnborg } 546a88b5ba8SSam Ravnborg } 547a88b5ba8SSam Ravnborg 548a88b5ba8SSam Ravnborg /* Now, poll for completion. */ 549a88b5ba8SSam Ravnborg { 550a88b5ba8SSam Ravnborg u64 dispatch_stat, nack_mask; 551a88b5ba8SSam Ravnborg long stuck; 552a88b5ba8SSam Ravnborg 553a88b5ba8SSam Ravnborg stuck = 100000 * nack_busy_id; 554a88b5ba8SSam Ravnborg nack_mask = busy_mask << 1; 555a88b5ba8SSam Ravnborg do { 556a88b5ba8SSam Ravnborg __asm__ __volatile__("ldxa [%%g0] %1, %0" 557a88b5ba8SSam Ravnborg : "=r" (dispatch_stat) 558a88b5ba8SSam Ravnborg : "i" (ASI_INTR_DISPATCH_STAT)); 559a88b5ba8SSam Ravnborg if (!(dispatch_stat & (busy_mask | nack_mask))) { 560a88b5ba8SSam Ravnborg __asm__ __volatile__("wrpr %0, 0x0, %%pstate" 561a88b5ba8SSam Ravnborg : : "r" (pstate)); 562a88b5ba8SSam Ravnborg if (unlikely(need_more)) { 563a88b5ba8SSam Ravnborg int i, this_cnt = 0; 564a88b5ba8SSam Ravnborg for (i = 0; i < cnt; i++) { 565a88b5ba8SSam Ravnborg if (cpu_list[i] == 0xffff) 566a88b5ba8SSam Ravnborg continue; 567a88b5ba8SSam Ravnborg cpu_list[i] = 0xffff; 568a88b5ba8SSam Ravnborg this_cnt++; 569a88b5ba8SSam Ravnborg if (this_cnt == 32) 570a88b5ba8SSam Ravnborg break; 571a88b5ba8SSam Ravnborg } 572a88b5ba8SSam Ravnborg goto retry; 573a88b5ba8SSam Ravnborg } 574a88b5ba8SSam Ravnborg return; 575a88b5ba8SSam Ravnborg } 576a88b5ba8SSam Ravnborg if (!--stuck) 577a88b5ba8SSam Ravnborg break; 578a88b5ba8SSam Ravnborg } while (dispatch_stat & busy_mask); 579a88b5ba8SSam Ravnborg 580a88b5ba8SSam Ravnborg __asm__ __volatile__("wrpr %0, 0x0, %%pstate" 581a88b5ba8SSam Ravnborg : : "r" (pstate)); 582a88b5ba8SSam Ravnborg 583a88b5ba8SSam Ravnborg if (dispatch_stat & busy_mask) { 584a88b5ba8SSam Ravnborg /* Busy bits will not clear, continue instead 585a88b5ba8SSam Ravnborg * of freezing up on this cpu. 586a88b5ba8SSam Ravnborg */ 58790181136SSam Ravnborg printk("CPU[%d]: mondo stuckage result[%016llx]\n", 588a88b5ba8SSam Ravnborg smp_processor_id(), dispatch_stat); 589a88b5ba8SSam Ravnborg } else { 590a88b5ba8SSam Ravnborg int i, this_busy_nack = 0; 591a88b5ba8SSam Ravnborg 592a88b5ba8SSam Ravnborg /* Delay some random time with interrupts enabled 593a88b5ba8SSam Ravnborg * to prevent deadlock. 594a88b5ba8SSam Ravnborg */ 595a88b5ba8SSam Ravnborg udelay(2 * nack_busy_id); 596a88b5ba8SSam Ravnborg 597a88b5ba8SSam Ravnborg /* Clear out the mask bits for cpus which did not 598a88b5ba8SSam Ravnborg * NACK us. 599a88b5ba8SSam Ravnborg */ 600a88b5ba8SSam Ravnborg for (i = 0; i < cnt; i++) { 601a88b5ba8SSam Ravnborg u64 check_mask, nr; 602a88b5ba8SSam Ravnborg 603a88b5ba8SSam Ravnborg nr = cpu_list[i]; 604a88b5ba8SSam Ravnborg if (nr == 0xffff) 605a88b5ba8SSam Ravnborg continue; 606a88b5ba8SSam Ravnborg 607a88b5ba8SSam Ravnborg if (is_jbus) 608a88b5ba8SSam Ravnborg check_mask = (0x2UL << (2*nr)); 609a88b5ba8SSam Ravnborg else 610a88b5ba8SSam Ravnborg check_mask = (0x2UL << 611a88b5ba8SSam Ravnborg this_busy_nack); 612a88b5ba8SSam Ravnborg if ((dispatch_stat & check_mask) == 0) 613a88b5ba8SSam Ravnborg cpu_list[i] = 0xffff; 614a88b5ba8SSam Ravnborg this_busy_nack += 2; 615a88b5ba8SSam Ravnborg if (this_busy_nack == 64) 616a88b5ba8SSam Ravnborg break; 617a88b5ba8SSam Ravnborg } 618a88b5ba8SSam Ravnborg 619a88b5ba8SSam Ravnborg goto retry; 620a88b5ba8SSam Ravnborg } 621a88b5ba8SSam Ravnborg } 622a88b5ba8SSam Ravnborg } 623a88b5ba8SSam Ravnborg 624a88b5ba8SSam Ravnborg /* Multi-cpu list version. */ 625a88b5ba8SSam Ravnborg static void hypervisor_xcall_deliver(struct trap_per_cpu *tb, int cnt) 626a88b5ba8SSam Ravnborg { 627a88b5ba8SSam Ravnborg int retries, this_cpu, prev_sent, i, saw_cpu_error; 628a88b5ba8SSam Ravnborg unsigned long status; 629a88b5ba8SSam Ravnborg u16 *cpu_list; 630a88b5ba8SSam Ravnborg 631a88b5ba8SSam Ravnborg this_cpu = smp_processor_id(); 632a88b5ba8SSam Ravnborg 633a88b5ba8SSam Ravnborg cpu_list = __va(tb->cpu_list_pa); 634a88b5ba8SSam Ravnborg 635a88b5ba8SSam Ravnborg saw_cpu_error = 0; 636a88b5ba8SSam Ravnborg retries = 0; 637a88b5ba8SSam Ravnborg prev_sent = 0; 638a88b5ba8SSam Ravnborg do { 639a88b5ba8SSam Ravnborg int forward_progress, n_sent; 640a88b5ba8SSam Ravnborg 641a88b5ba8SSam Ravnborg status = sun4v_cpu_mondo_send(cnt, 642a88b5ba8SSam Ravnborg tb->cpu_list_pa, 643a88b5ba8SSam Ravnborg tb->cpu_mondo_block_pa); 644a88b5ba8SSam Ravnborg 645a88b5ba8SSam Ravnborg /* HV_EOK means all cpus received the xcall, we're done. */ 646a88b5ba8SSam Ravnborg if (likely(status == HV_EOK)) 647a88b5ba8SSam Ravnborg break; 648a88b5ba8SSam Ravnborg 649a88b5ba8SSam Ravnborg /* First, see if we made any forward progress. 650a88b5ba8SSam Ravnborg * 651a88b5ba8SSam Ravnborg * The hypervisor indicates successful sends by setting 652a88b5ba8SSam Ravnborg * cpu list entries to the value 0xffff. 653a88b5ba8SSam Ravnborg */ 654a88b5ba8SSam Ravnborg n_sent = 0; 655a88b5ba8SSam Ravnborg for (i = 0; i < cnt; i++) { 656a88b5ba8SSam Ravnborg if (likely(cpu_list[i] == 0xffff)) 657a88b5ba8SSam Ravnborg n_sent++; 658a88b5ba8SSam Ravnborg } 659a88b5ba8SSam Ravnborg 660a88b5ba8SSam Ravnborg forward_progress = 0; 661a88b5ba8SSam Ravnborg if (n_sent > prev_sent) 662a88b5ba8SSam Ravnborg forward_progress = 1; 663a88b5ba8SSam Ravnborg 664a88b5ba8SSam Ravnborg prev_sent = n_sent; 665a88b5ba8SSam Ravnborg 666a88b5ba8SSam Ravnborg /* If we get a HV_ECPUERROR, then one or more of the cpus 667a88b5ba8SSam Ravnborg * in the list are in error state. Use the cpu_state() 668a88b5ba8SSam Ravnborg * hypervisor call to find out which cpus are in error state. 669a88b5ba8SSam Ravnborg */ 670a88b5ba8SSam Ravnborg if (unlikely(status == HV_ECPUERROR)) { 671a88b5ba8SSam Ravnborg for (i = 0; i < cnt; i++) { 672a88b5ba8SSam Ravnborg long err; 673a88b5ba8SSam Ravnborg u16 cpu; 674a88b5ba8SSam Ravnborg 675a88b5ba8SSam Ravnborg cpu = cpu_list[i]; 676a88b5ba8SSam Ravnborg if (cpu == 0xffff) 677a88b5ba8SSam Ravnborg continue; 678a88b5ba8SSam Ravnborg 679a88b5ba8SSam Ravnborg err = sun4v_cpu_state(cpu); 680a88b5ba8SSam Ravnborg if (err == HV_CPU_STATE_ERROR) { 681a88b5ba8SSam Ravnborg saw_cpu_error = (cpu + 1); 682a88b5ba8SSam Ravnborg cpu_list[i] = 0xffff; 683a88b5ba8SSam Ravnborg } 684a88b5ba8SSam Ravnborg } 685a88b5ba8SSam Ravnborg } else if (unlikely(status != HV_EWOULDBLOCK)) 686a88b5ba8SSam Ravnborg goto fatal_mondo_error; 687a88b5ba8SSam Ravnborg 688a88b5ba8SSam Ravnborg /* Don't bother rewriting the CPU list, just leave the 689a88b5ba8SSam Ravnborg * 0xffff and non-0xffff entries in there and the 690a88b5ba8SSam Ravnborg * hypervisor will do the right thing. 691a88b5ba8SSam Ravnborg * 692a88b5ba8SSam Ravnborg * Only advance timeout state if we didn't make any 693a88b5ba8SSam Ravnborg * forward progress. 694a88b5ba8SSam Ravnborg */ 695a88b5ba8SSam Ravnborg if (unlikely(!forward_progress)) { 696a88b5ba8SSam Ravnborg if (unlikely(++retries > 10000)) 697a88b5ba8SSam Ravnborg goto fatal_mondo_timeout; 698a88b5ba8SSam Ravnborg 699a88b5ba8SSam Ravnborg /* Delay a little bit to let other cpus catch up 700a88b5ba8SSam Ravnborg * on their cpu mondo queue work. 701a88b5ba8SSam Ravnborg */ 702a88b5ba8SSam Ravnborg udelay(2 * cnt); 703a88b5ba8SSam Ravnborg } 704a88b5ba8SSam Ravnborg } while (1); 705a88b5ba8SSam Ravnborg 706a88b5ba8SSam Ravnborg if (unlikely(saw_cpu_error)) 707a88b5ba8SSam Ravnborg goto fatal_mondo_cpu_error; 708a88b5ba8SSam Ravnborg 709a88b5ba8SSam Ravnborg return; 710a88b5ba8SSam Ravnborg 711a88b5ba8SSam Ravnborg fatal_mondo_cpu_error: 712a88b5ba8SSam Ravnborg printk(KERN_CRIT "CPU[%d]: SUN4V mondo cpu error, some target cpus " 713a88b5ba8SSam Ravnborg "(including %d) were in error state\n", 714a88b5ba8SSam Ravnborg this_cpu, saw_cpu_error - 1); 715a88b5ba8SSam Ravnborg return; 716a88b5ba8SSam Ravnborg 717a88b5ba8SSam Ravnborg fatal_mondo_timeout: 718a88b5ba8SSam Ravnborg printk(KERN_CRIT "CPU[%d]: SUN4V mondo timeout, no forward " 719a88b5ba8SSam Ravnborg " progress after %d retries.\n", 720a88b5ba8SSam Ravnborg this_cpu, retries); 721a88b5ba8SSam Ravnborg goto dump_cpu_list_and_out; 722a88b5ba8SSam Ravnborg 723a88b5ba8SSam Ravnborg fatal_mondo_error: 724a88b5ba8SSam Ravnborg printk(KERN_CRIT "CPU[%d]: Unexpected SUN4V mondo error %lu\n", 725a88b5ba8SSam Ravnborg this_cpu, status); 726a88b5ba8SSam Ravnborg printk(KERN_CRIT "CPU[%d]: Args were cnt(%d) cpulist_pa(%lx) " 727a88b5ba8SSam Ravnborg "mondo_block_pa(%lx)\n", 728a88b5ba8SSam Ravnborg this_cpu, cnt, tb->cpu_list_pa, tb->cpu_mondo_block_pa); 729a88b5ba8SSam Ravnborg 730a88b5ba8SSam Ravnborg dump_cpu_list_and_out: 731a88b5ba8SSam Ravnborg printk(KERN_CRIT "CPU[%d]: CPU list [ ", this_cpu); 732a88b5ba8SSam Ravnborg for (i = 0; i < cnt; i++) 733a88b5ba8SSam Ravnborg printk("%u ", cpu_list[i]); 734a88b5ba8SSam Ravnborg printk("]\n"); 735a88b5ba8SSam Ravnborg } 736a88b5ba8SSam Ravnborg 737a88b5ba8SSam Ravnborg static void (*xcall_deliver_impl)(struct trap_per_cpu *, int); 738a88b5ba8SSam Ravnborg 739a88b5ba8SSam Ravnborg static void xcall_deliver(u64 data0, u64 data1, u64 data2, const cpumask_t *mask) 740a88b5ba8SSam Ravnborg { 741a88b5ba8SSam Ravnborg struct trap_per_cpu *tb; 742a88b5ba8SSam Ravnborg int this_cpu, i, cnt; 743a88b5ba8SSam Ravnborg unsigned long flags; 744a88b5ba8SSam Ravnborg u16 *cpu_list; 745a88b5ba8SSam Ravnborg u64 *mondo; 746a88b5ba8SSam Ravnborg 747a88b5ba8SSam Ravnborg /* We have to do this whole thing with interrupts fully disabled. 748a88b5ba8SSam Ravnborg * Otherwise if we send an xcall from interrupt context it will 749a88b5ba8SSam Ravnborg * corrupt both our mondo block and cpu list state. 750a88b5ba8SSam Ravnborg * 751a88b5ba8SSam Ravnborg * One consequence of this is that we cannot use timeout mechanisms 752a88b5ba8SSam Ravnborg * that depend upon interrupts being delivered locally. So, for 753a88b5ba8SSam Ravnborg * example, we cannot sample jiffies and expect it to advance. 754a88b5ba8SSam Ravnborg * 755a88b5ba8SSam Ravnborg * Fortunately, udelay() uses %stick/%tick so we can use that. 756a88b5ba8SSam Ravnborg */ 757a88b5ba8SSam Ravnborg local_irq_save(flags); 758a88b5ba8SSam Ravnborg 759a88b5ba8SSam Ravnborg this_cpu = smp_processor_id(); 760a88b5ba8SSam Ravnborg tb = &trap_block[this_cpu]; 761a88b5ba8SSam Ravnborg 762a88b5ba8SSam Ravnborg mondo = __va(tb->cpu_mondo_block_pa); 763a88b5ba8SSam Ravnborg mondo[0] = data0; 764a88b5ba8SSam Ravnborg mondo[1] = data1; 765a88b5ba8SSam Ravnborg mondo[2] = data2; 766a88b5ba8SSam Ravnborg wmb(); 767a88b5ba8SSam Ravnborg 768a88b5ba8SSam Ravnborg cpu_list = __va(tb->cpu_list_pa); 769a88b5ba8SSam Ravnborg 770a88b5ba8SSam Ravnborg /* Setup the initial cpu list. */ 771a88b5ba8SSam Ravnborg cnt = 0; 7728e757281SRusty Russell for_each_cpu(i, mask) { 773a88b5ba8SSam Ravnborg if (i == this_cpu || !cpu_online(i)) 774a88b5ba8SSam Ravnborg continue; 775a88b5ba8SSam Ravnborg cpu_list[cnt++] = i; 776a88b5ba8SSam Ravnborg } 777a88b5ba8SSam Ravnborg 778a88b5ba8SSam Ravnborg if (cnt) 779a88b5ba8SSam Ravnborg xcall_deliver_impl(tb, cnt); 780a88b5ba8SSam Ravnborg 781a88b5ba8SSam Ravnborg local_irq_restore(flags); 782a88b5ba8SSam Ravnborg } 783a88b5ba8SSam Ravnborg 784a88b5ba8SSam Ravnborg /* Send cross call to all processors mentioned in MASK_P 785a88b5ba8SSam Ravnborg * except self. Really, there are only two cases currently, 786fb1fece5SKOSAKI Motohiro * "cpu_online_mask" and "mm_cpumask(mm)". 787a88b5ba8SSam Ravnborg */ 788a88b5ba8SSam Ravnborg static void smp_cross_call_masked(unsigned long *func, u32 ctx, u64 data1, u64 data2, const cpumask_t *mask) 789a88b5ba8SSam Ravnborg { 790a88b5ba8SSam Ravnborg u64 data0 = (((u64)ctx)<<32 | (((u64)func) & 0xffffffff)); 791a88b5ba8SSam Ravnborg 792a88b5ba8SSam Ravnborg xcall_deliver(data0, data1, data2, mask); 793a88b5ba8SSam Ravnborg } 794a88b5ba8SSam Ravnborg 795a88b5ba8SSam Ravnborg /* Send cross call to all processors except self. */ 796a88b5ba8SSam Ravnborg static void smp_cross_call(unsigned long *func, u32 ctx, u64 data1, u64 data2) 797a88b5ba8SSam Ravnborg { 798fb1fece5SKOSAKI Motohiro smp_cross_call_masked(func, ctx, data1, data2, cpu_online_mask); 799a88b5ba8SSam Ravnborg } 800a88b5ba8SSam Ravnborg 801a88b5ba8SSam Ravnborg extern unsigned long xcall_sync_tick; 802a88b5ba8SSam Ravnborg 803a88b5ba8SSam Ravnborg static void smp_start_sync_tick_client(int cpu) 804a88b5ba8SSam Ravnborg { 805a88b5ba8SSam Ravnborg xcall_deliver((u64) &xcall_sync_tick, 0, 0, 806fb1fece5SKOSAKI Motohiro cpumask_of(cpu)); 807a88b5ba8SSam Ravnborg } 808a88b5ba8SSam Ravnborg 809a88b5ba8SSam Ravnborg extern unsigned long xcall_call_function; 810a88b5ba8SSam Ravnborg 811f46df02aSRusty Russell void arch_send_call_function_ipi_mask(const struct cpumask *mask) 812a88b5ba8SSam Ravnborg { 813f46df02aSRusty Russell xcall_deliver((u64) &xcall_call_function, 0, 0, mask); 814a88b5ba8SSam Ravnborg } 815a88b5ba8SSam Ravnborg 816a88b5ba8SSam Ravnborg extern unsigned long xcall_call_function_single; 817a88b5ba8SSam Ravnborg 818a88b5ba8SSam Ravnborg void arch_send_call_function_single_ipi(int cpu) 819a88b5ba8SSam Ravnborg { 820a88b5ba8SSam Ravnborg xcall_deliver((u64) &xcall_call_function_single, 0, 0, 821fb1fece5SKOSAKI Motohiro cpumask_of(cpu)); 822a88b5ba8SSam Ravnborg } 823a88b5ba8SSam Ravnborg 8249960e9e8SDavid S. Miller void __irq_entry smp_call_function_client(int irq, struct pt_regs *regs) 825a88b5ba8SSam Ravnborg { 826a88b5ba8SSam Ravnborg clear_softint(1 << irq); 827ab5c7809SDavid S. Miller irq_enter(); 828a88b5ba8SSam Ravnborg generic_smp_call_function_interrupt(); 829ab5c7809SDavid S. Miller irq_exit(); 830a88b5ba8SSam Ravnborg } 831a88b5ba8SSam Ravnborg 8329960e9e8SDavid S. Miller void __irq_entry smp_call_function_single_client(int irq, struct pt_regs *regs) 833a88b5ba8SSam Ravnborg { 834a88b5ba8SSam Ravnborg clear_softint(1 << irq); 835ab5c7809SDavid S. Miller irq_enter(); 836a88b5ba8SSam Ravnborg generic_smp_call_function_single_interrupt(); 837ab5c7809SDavid S. Miller irq_exit(); 838a88b5ba8SSam Ravnborg } 839a88b5ba8SSam Ravnborg 840a88b5ba8SSam Ravnborg static void tsb_sync(void *info) 841a88b5ba8SSam Ravnborg { 842a88b5ba8SSam Ravnborg struct trap_per_cpu *tp = &trap_block[raw_smp_processor_id()]; 843a88b5ba8SSam Ravnborg struct mm_struct *mm = info; 844a88b5ba8SSam Ravnborg 84542b2aa86SJustin P. Mattock /* It is not valid to test "current->active_mm == mm" here. 846a88b5ba8SSam Ravnborg * 847a88b5ba8SSam Ravnborg * The value of "current" is not changed atomically with 848a88b5ba8SSam Ravnborg * switch_mm(). But that's OK, we just need to check the 849a88b5ba8SSam Ravnborg * current cpu's trap block PGD physical address. 850a88b5ba8SSam Ravnborg */ 851a88b5ba8SSam Ravnborg if (tp->pgd_paddr == __pa(mm->pgd)) 852a88b5ba8SSam Ravnborg tsb_context_switch(mm); 853a88b5ba8SSam Ravnborg } 854a88b5ba8SSam Ravnborg 855a88b5ba8SSam Ravnborg void smp_tsb_sync(struct mm_struct *mm) 856a88b5ba8SSam Ravnborg { 85781f1adf0SRusty Russell smp_call_function_many(mm_cpumask(mm), tsb_sync, mm, 1); 858a88b5ba8SSam Ravnborg } 859a88b5ba8SSam Ravnborg 860a88b5ba8SSam Ravnborg extern unsigned long xcall_flush_tlb_mm; 861f36391d2SDavid S. Miller extern unsigned long xcall_flush_tlb_page; 862a88b5ba8SSam Ravnborg extern unsigned long xcall_flush_tlb_kernel_range; 863a88b5ba8SSam Ravnborg extern unsigned long xcall_fetch_glob_regs; 864916ca14aSDavid S. Miller extern unsigned long xcall_fetch_glob_pmu; 865916ca14aSDavid S. Miller extern unsigned long xcall_fetch_glob_pmu_n4; 866a88b5ba8SSam Ravnborg extern unsigned long xcall_receive_signal; 867a88b5ba8SSam Ravnborg extern unsigned long xcall_new_mmu_context_version; 868a88b5ba8SSam Ravnborg #ifdef CONFIG_KGDB 869a88b5ba8SSam Ravnborg extern unsigned long xcall_kgdb_capture; 870a88b5ba8SSam Ravnborg #endif 871a88b5ba8SSam Ravnborg 872a88b5ba8SSam Ravnborg #ifdef DCACHE_ALIASING_POSSIBLE 873a88b5ba8SSam Ravnborg extern unsigned long xcall_flush_dcache_page_cheetah; 874a88b5ba8SSam Ravnborg #endif 875a88b5ba8SSam Ravnborg extern unsigned long xcall_flush_dcache_page_spitfire; 876a88b5ba8SSam Ravnborg 877a88b5ba8SSam Ravnborg static inline void __local_flush_dcache_page(struct page *page) 878a88b5ba8SSam Ravnborg { 879a88b5ba8SSam Ravnborg #ifdef DCACHE_ALIASING_POSSIBLE 880a88b5ba8SSam Ravnborg __flush_dcache_page(page_address(page), 881a88b5ba8SSam Ravnborg ((tlb_type == spitfire) && 882a88b5ba8SSam Ravnborg page_mapping(page) != NULL)); 883a88b5ba8SSam Ravnborg #else 884a88b5ba8SSam Ravnborg if (page_mapping(page) != NULL && 885a88b5ba8SSam Ravnborg tlb_type == spitfire) 886a88b5ba8SSam Ravnborg __flush_icache_page(__pa(page_address(page))); 887a88b5ba8SSam Ravnborg #endif 888a88b5ba8SSam Ravnborg } 889a88b5ba8SSam Ravnborg 890a88b5ba8SSam Ravnborg void smp_flush_dcache_page_impl(struct page *page, int cpu) 891a88b5ba8SSam Ravnborg { 892a88b5ba8SSam Ravnborg int this_cpu; 893a88b5ba8SSam Ravnborg 894a88b5ba8SSam Ravnborg if (tlb_type == hypervisor) 895a88b5ba8SSam Ravnborg return; 896a88b5ba8SSam Ravnborg 897a88b5ba8SSam Ravnborg #ifdef CONFIG_DEBUG_DCFLUSH 898a88b5ba8SSam Ravnborg atomic_inc(&dcpage_flushes); 899a88b5ba8SSam Ravnborg #endif 900a88b5ba8SSam Ravnborg 901a88b5ba8SSam Ravnborg this_cpu = get_cpu(); 902a88b5ba8SSam Ravnborg 903a88b5ba8SSam Ravnborg if (cpu == this_cpu) { 904a88b5ba8SSam Ravnborg __local_flush_dcache_page(page); 905a88b5ba8SSam Ravnborg } else if (cpu_online(cpu)) { 906a88b5ba8SSam Ravnborg void *pg_addr = page_address(page); 907a88b5ba8SSam Ravnborg u64 data0 = 0; 908a88b5ba8SSam Ravnborg 909a88b5ba8SSam Ravnborg if (tlb_type == spitfire) { 910a88b5ba8SSam Ravnborg data0 = ((u64)&xcall_flush_dcache_page_spitfire); 911a88b5ba8SSam Ravnborg if (page_mapping(page) != NULL) 912a88b5ba8SSam Ravnborg data0 |= ((u64)1 << 32); 913a88b5ba8SSam Ravnborg } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { 914a88b5ba8SSam Ravnborg #ifdef DCACHE_ALIASING_POSSIBLE 915a88b5ba8SSam Ravnborg data0 = ((u64)&xcall_flush_dcache_page_cheetah); 916a88b5ba8SSam Ravnborg #endif 917a88b5ba8SSam Ravnborg } 918a88b5ba8SSam Ravnborg if (data0) { 919a88b5ba8SSam Ravnborg xcall_deliver(data0, __pa(pg_addr), 920fb1fece5SKOSAKI Motohiro (u64) pg_addr, cpumask_of(cpu)); 921a88b5ba8SSam Ravnborg #ifdef CONFIG_DEBUG_DCFLUSH 922a88b5ba8SSam Ravnborg atomic_inc(&dcpage_flushes_xcall); 923a88b5ba8SSam Ravnborg #endif 924a88b5ba8SSam Ravnborg } 925a88b5ba8SSam Ravnborg } 926a88b5ba8SSam Ravnborg 927a88b5ba8SSam Ravnborg put_cpu(); 928a88b5ba8SSam Ravnborg } 929a88b5ba8SSam Ravnborg 930a88b5ba8SSam Ravnborg void flush_dcache_page_all(struct mm_struct *mm, struct page *page) 931a88b5ba8SSam Ravnborg { 932a88b5ba8SSam Ravnborg void *pg_addr; 933a88b5ba8SSam Ravnborg u64 data0; 934a88b5ba8SSam Ravnborg 935a88b5ba8SSam Ravnborg if (tlb_type == hypervisor) 936a88b5ba8SSam Ravnborg return; 937a88b5ba8SSam Ravnborg 938c6fee081SDavid S. Miller preempt_disable(); 939a88b5ba8SSam Ravnborg 940a88b5ba8SSam Ravnborg #ifdef CONFIG_DEBUG_DCFLUSH 941a88b5ba8SSam Ravnborg atomic_inc(&dcpage_flushes); 942a88b5ba8SSam Ravnborg #endif 943a88b5ba8SSam Ravnborg data0 = 0; 944a88b5ba8SSam Ravnborg pg_addr = page_address(page); 945a88b5ba8SSam Ravnborg if (tlb_type == spitfire) { 946a88b5ba8SSam Ravnborg data0 = ((u64)&xcall_flush_dcache_page_spitfire); 947a88b5ba8SSam Ravnborg if (page_mapping(page) != NULL) 948a88b5ba8SSam Ravnborg data0 |= ((u64)1 << 32); 949a88b5ba8SSam Ravnborg } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { 950a88b5ba8SSam Ravnborg #ifdef DCACHE_ALIASING_POSSIBLE 951a88b5ba8SSam Ravnborg data0 = ((u64)&xcall_flush_dcache_page_cheetah); 952a88b5ba8SSam Ravnborg #endif 953a88b5ba8SSam Ravnborg } 954a88b5ba8SSam Ravnborg if (data0) { 955a88b5ba8SSam Ravnborg xcall_deliver(data0, __pa(pg_addr), 956fb1fece5SKOSAKI Motohiro (u64) pg_addr, cpu_online_mask); 957a88b5ba8SSam Ravnborg #ifdef CONFIG_DEBUG_DCFLUSH 958a88b5ba8SSam Ravnborg atomic_inc(&dcpage_flushes_xcall); 959a88b5ba8SSam Ravnborg #endif 960a88b5ba8SSam Ravnborg } 961a88b5ba8SSam Ravnborg __local_flush_dcache_page(page); 962a88b5ba8SSam Ravnborg 963c6fee081SDavid S. Miller preempt_enable(); 964a88b5ba8SSam Ravnborg } 965a88b5ba8SSam Ravnborg 9669960e9e8SDavid S. Miller void __irq_entry smp_new_mmu_context_version_client(int irq, struct pt_regs *regs) 967a88b5ba8SSam Ravnborg { 968a88b5ba8SSam Ravnborg struct mm_struct *mm; 969a88b5ba8SSam Ravnborg unsigned long flags; 970a88b5ba8SSam Ravnborg 971a88b5ba8SSam Ravnborg clear_softint(1 << irq); 972a88b5ba8SSam Ravnborg 973a88b5ba8SSam Ravnborg /* See if we need to allocate a new TLB context because 974a88b5ba8SSam Ravnborg * the version of the one we are using is now out of date. 975a88b5ba8SSam Ravnborg */ 976a88b5ba8SSam Ravnborg mm = current->active_mm; 977a88b5ba8SSam Ravnborg if (unlikely(!mm || (mm == &init_mm))) 978a88b5ba8SSam Ravnborg return; 979a88b5ba8SSam Ravnborg 980a88b5ba8SSam Ravnborg spin_lock_irqsave(&mm->context.lock, flags); 981a88b5ba8SSam Ravnborg 982a88b5ba8SSam Ravnborg if (unlikely(!CTX_VALID(mm->context))) 983a88b5ba8SSam Ravnborg get_new_mmu_context(mm); 984a88b5ba8SSam Ravnborg 985a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&mm->context.lock, flags); 986a88b5ba8SSam Ravnborg 987a88b5ba8SSam Ravnborg load_secondary_context(mm); 988a88b5ba8SSam Ravnborg __flush_tlb_mm(CTX_HWBITS(mm->context), 989a88b5ba8SSam Ravnborg SECONDARY_CONTEXT); 990a88b5ba8SSam Ravnborg } 991a88b5ba8SSam Ravnborg 992a88b5ba8SSam Ravnborg void smp_new_mmu_context_version(void) 993a88b5ba8SSam Ravnborg { 994a88b5ba8SSam Ravnborg smp_cross_call(&xcall_new_mmu_context_version, 0, 0, 0); 995a88b5ba8SSam Ravnborg } 996a88b5ba8SSam Ravnborg 997a88b5ba8SSam Ravnborg #ifdef CONFIG_KGDB 998a88b5ba8SSam Ravnborg void kgdb_roundup_cpus(unsigned long flags) 999a88b5ba8SSam Ravnborg { 1000a88b5ba8SSam Ravnborg smp_cross_call(&xcall_kgdb_capture, 0, 0, 0); 1001a88b5ba8SSam Ravnborg } 1002a88b5ba8SSam Ravnborg #endif 1003a88b5ba8SSam Ravnborg 1004a88b5ba8SSam Ravnborg void smp_fetch_global_regs(void) 1005a88b5ba8SSam Ravnborg { 1006a88b5ba8SSam Ravnborg smp_cross_call(&xcall_fetch_glob_regs, 0, 0, 0); 1007a88b5ba8SSam Ravnborg } 1008a88b5ba8SSam Ravnborg 1009916ca14aSDavid S. Miller void smp_fetch_global_pmu(void) 1010916ca14aSDavid S. Miller { 1011916ca14aSDavid S. Miller if (tlb_type == hypervisor && 1012916ca14aSDavid S. Miller sun4v_chip_type >= SUN4V_CHIP_NIAGARA4) 1013916ca14aSDavid S. Miller smp_cross_call(&xcall_fetch_glob_pmu_n4, 0, 0, 0); 1014916ca14aSDavid S. Miller else 1015916ca14aSDavid S. Miller smp_cross_call(&xcall_fetch_glob_pmu, 0, 0, 0); 1016916ca14aSDavid S. Miller } 1017916ca14aSDavid S. Miller 1018a88b5ba8SSam Ravnborg /* We know that the window frames of the user have been flushed 1019a88b5ba8SSam Ravnborg * to the stack before we get here because all callers of us 1020a88b5ba8SSam Ravnborg * are flush_tlb_*() routines, and these run after flush_cache_*() 1021a88b5ba8SSam Ravnborg * which performs the flushw. 1022a88b5ba8SSam Ravnborg * 1023a88b5ba8SSam Ravnborg * The SMP TLB coherency scheme we use works as follows: 1024a88b5ba8SSam Ravnborg * 1025a88b5ba8SSam Ravnborg * 1) mm->cpu_vm_mask is a bit mask of which cpus an address 1026a88b5ba8SSam Ravnborg * space has (potentially) executed on, this is the heuristic 1027a88b5ba8SSam Ravnborg * we use to avoid doing cross calls. 1028a88b5ba8SSam Ravnborg * 1029a88b5ba8SSam Ravnborg * Also, for flushing from kswapd and also for clones, we 1030a88b5ba8SSam Ravnborg * use cpu_vm_mask as the list of cpus to make run the TLB. 1031a88b5ba8SSam Ravnborg * 1032a88b5ba8SSam Ravnborg * 2) TLB context numbers are shared globally across all processors 1033a88b5ba8SSam Ravnborg * in the system, this allows us to play several games to avoid 1034a88b5ba8SSam Ravnborg * cross calls. 1035a88b5ba8SSam Ravnborg * 1036a88b5ba8SSam Ravnborg * One invariant is that when a cpu switches to a process, and 1037a88b5ba8SSam Ravnborg * that processes tsk->active_mm->cpu_vm_mask does not have the 1038a88b5ba8SSam Ravnborg * current cpu's bit set, that tlb context is flushed locally. 1039a88b5ba8SSam Ravnborg * 1040a88b5ba8SSam Ravnborg * If the address space is non-shared (ie. mm->count == 1) we avoid 1041a88b5ba8SSam Ravnborg * cross calls when we want to flush the currently running process's 1042a88b5ba8SSam Ravnborg * tlb state. This is done by clearing all cpu bits except the current 1043f9384d41SDavid S. Miller * processor's in current->mm->cpu_vm_mask and performing the 1044a88b5ba8SSam Ravnborg * flush locally only. This will force any subsequent cpus which run 1045a88b5ba8SSam Ravnborg * this task to flush the context from the local tlb if the process 1046a88b5ba8SSam Ravnborg * migrates to another cpu (again). 1047a88b5ba8SSam Ravnborg * 1048a88b5ba8SSam Ravnborg * 3) For shared address spaces (threads) and swapping we bite the 1049a88b5ba8SSam Ravnborg * bullet for most cases and perform the cross call (but only to 1050a88b5ba8SSam Ravnborg * the cpus listed in cpu_vm_mask). 1051a88b5ba8SSam Ravnborg * 1052a88b5ba8SSam Ravnborg * The performance gain from "optimizing" away the cross call for threads is 1053a88b5ba8SSam Ravnborg * questionable (in theory the big win for threads is the massive sharing of 1054a88b5ba8SSam Ravnborg * address space state across processors). 1055a88b5ba8SSam Ravnborg */ 1056a88b5ba8SSam Ravnborg 1057a88b5ba8SSam Ravnborg /* This currently is only used by the hugetlb arch pre-fault 1058a88b5ba8SSam Ravnborg * hook on UltraSPARC-III+ and later when changing the pagesize 1059a88b5ba8SSam Ravnborg * bits of the context register for an address space. 1060a88b5ba8SSam Ravnborg */ 1061a88b5ba8SSam Ravnborg void smp_flush_tlb_mm(struct mm_struct *mm) 1062a88b5ba8SSam Ravnborg { 1063a88b5ba8SSam Ravnborg u32 ctx = CTX_HWBITS(mm->context); 1064a88b5ba8SSam Ravnborg int cpu = get_cpu(); 1065a88b5ba8SSam Ravnborg 1066a88b5ba8SSam Ravnborg if (atomic_read(&mm->mm_users) == 1) { 106781f1adf0SRusty Russell cpumask_copy(mm_cpumask(mm), cpumask_of(cpu)); 1068a88b5ba8SSam Ravnborg goto local_flush_and_out; 1069a88b5ba8SSam Ravnborg } 1070a88b5ba8SSam Ravnborg 1071a88b5ba8SSam Ravnborg smp_cross_call_masked(&xcall_flush_tlb_mm, 1072a88b5ba8SSam Ravnborg ctx, 0, 0, 107381f1adf0SRusty Russell mm_cpumask(mm)); 1074a88b5ba8SSam Ravnborg 1075a88b5ba8SSam Ravnborg local_flush_and_out: 1076a88b5ba8SSam Ravnborg __flush_tlb_mm(ctx, SECONDARY_CONTEXT); 1077a88b5ba8SSam Ravnborg 1078a88b5ba8SSam Ravnborg put_cpu(); 1079a88b5ba8SSam Ravnborg } 1080a88b5ba8SSam Ravnborg 1081f36391d2SDavid S. Miller struct tlb_pending_info { 1082f36391d2SDavid S. Miller unsigned long ctx; 1083f36391d2SDavid S. Miller unsigned long nr; 1084f36391d2SDavid S. Miller unsigned long *vaddrs; 1085f36391d2SDavid S. Miller }; 1086f36391d2SDavid S. Miller 1087f36391d2SDavid S. Miller static void tlb_pending_func(void *info) 1088f36391d2SDavid S. Miller { 1089f36391d2SDavid S. Miller struct tlb_pending_info *t = info; 1090f36391d2SDavid S. Miller 1091f36391d2SDavid S. Miller __flush_tlb_pending(t->ctx, t->nr, t->vaddrs); 1092f36391d2SDavid S. Miller } 1093f36391d2SDavid S. Miller 1094a88b5ba8SSam Ravnborg void smp_flush_tlb_pending(struct mm_struct *mm, unsigned long nr, unsigned long *vaddrs) 1095a88b5ba8SSam Ravnborg { 1096a88b5ba8SSam Ravnborg u32 ctx = CTX_HWBITS(mm->context); 1097f36391d2SDavid S. Miller struct tlb_pending_info info; 1098f36391d2SDavid S. Miller int cpu = get_cpu(); 1099f36391d2SDavid S. Miller 1100f36391d2SDavid S. Miller info.ctx = ctx; 1101f36391d2SDavid S. Miller info.nr = nr; 1102f36391d2SDavid S. Miller info.vaddrs = vaddrs; 1103f36391d2SDavid S. Miller 1104f36391d2SDavid S. Miller if (mm == current->mm && atomic_read(&mm->mm_users) == 1) 1105f36391d2SDavid S. Miller cpumask_copy(mm_cpumask(mm), cpumask_of(cpu)); 1106f36391d2SDavid S. Miller else 1107f36391d2SDavid S. Miller smp_call_function_many(mm_cpumask(mm), tlb_pending_func, 1108f36391d2SDavid S. Miller &info, 1); 1109f36391d2SDavid S. Miller 1110f36391d2SDavid S. Miller __flush_tlb_pending(ctx, nr, vaddrs); 1111f36391d2SDavid S. Miller 1112f36391d2SDavid S. Miller put_cpu(); 1113f36391d2SDavid S. Miller } 1114f36391d2SDavid S. Miller 1115f36391d2SDavid S. Miller void smp_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr) 1116f36391d2SDavid S. Miller { 1117f36391d2SDavid S. Miller unsigned long context = CTX_HWBITS(mm->context); 1118a88b5ba8SSam Ravnborg int cpu = get_cpu(); 1119a88b5ba8SSam Ravnborg 1120f9384d41SDavid S. Miller if (mm == current->mm && atomic_read(&mm->mm_users) == 1) 112181f1adf0SRusty Russell cpumask_copy(mm_cpumask(mm), cpumask_of(cpu)); 1122a88b5ba8SSam Ravnborg else 1123f36391d2SDavid S. Miller smp_cross_call_masked(&xcall_flush_tlb_page, 1124f36391d2SDavid S. Miller context, vaddr, 0, 112581f1adf0SRusty Russell mm_cpumask(mm)); 1126f36391d2SDavid S. Miller __flush_tlb_page(context, vaddr); 1127a88b5ba8SSam Ravnborg 1128a88b5ba8SSam Ravnborg put_cpu(); 1129a88b5ba8SSam Ravnborg } 1130a88b5ba8SSam Ravnborg 1131a88b5ba8SSam Ravnborg void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end) 1132a88b5ba8SSam Ravnborg { 1133a88b5ba8SSam Ravnborg start &= PAGE_MASK; 1134a88b5ba8SSam Ravnborg end = PAGE_ALIGN(end); 1135a88b5ba8SSam Ravnborg if (start != end) { 1136a88b5ba8SSam Ravnborg smp_cross_call(&xcall_flush_tlb_kernel_range, 1137a88b5ba8SSam Ravnborg 0, start, end); 1138a88b5ba8SSam Ravnborg 1139a88b5ba8SSam Ravnborg __flush_tlb_kernel_range(start, end); 1140a88b5ba8SSam Ravnborg } 1141a88b5ba8SSam Ravnborg } 1142a88b5ba8SSam Ravnborg 1143a88b5ba8SSam Ravnborg /* CPU capture. */ 1144a88b5ba8SSam Ravnborg /* #define CAPTURE_DEBUG */ 1145a88b5ba8SSam Ravnborg extern unsigned long xcall_capture; 1146a88b5ba8SSam Ravnborg 1147a88b5ba8SSam Ravnborg static atomic_t smp_capture_depth = ATOMIC_INIT(0); 1148a88b5ba8SSam Ravnborg static atomic_t smp_capture_registry = ATOMIC_INIT(0); 1149a88b5ba8SSam Ravnborg static unsigned long penguins_are_doing_time; 1150a88b5ba8SSam Ravnborg 1151a88b5ba8SSam Ravnborg void smp_capture(void) 1152a88b5ba8SSam Ravnborg { 11534f3316c2SPeter Zijlstra int result = atomic_add_return(1, &smp_capture_depth); 1154a88b5ba8SSam Ravnborg 1155a88b5ba8SSam Ravnborg if (result == 1) { 1156a88b5ba8SSam Ravnborg int ncpus = num_online_cpus(); 1157a88b5ba8SSam Ravnborg 1158a88b5ba8SSam Ravnborg #ifdef CAPTURE_DEBUG 1159a88b5ba8SSam Ravnborg printk("CPU[%d]: Sending penguins to jail...", 1160a88b5ba8SSam Ravnborg smp_processor_id()); 1161a88b5ba8SSam Ravnborg #endif 1162a88b5ba8SSam Ravnborg penguins_are_doing_time = 1; 1163a88b5ba8SSam Ravnborg atomic_inc(&smp_capture_registry); 1164a88b5ba8SSam Ravnborg smp_cross_call(&xcall_capture, 0, 0, 0); 1165a88b5ba8SSam Ravnborg while (atomic_read(&smp_capture_registry) != ncpus) 1166a88b5ba8SSam Ravnborg rmb(); 1167a88b5ba8SSam Ravnborg #ifdef CAPTURE_DEBUG 1168a88b5ba8SSam Ravnborg printk("done\n"); 1169a88b5ba8SSam Ravnborg #endif 1170a88b5ba8SSam Ravnborg } 1171a88b5ba8SSam Ravnborg } 1172a88b5ba8SSam Ravnborg 1173a88b5ba8SSam Ravnborg void smp_release(void) 1174a88b5ba8SSam Ravnborg { 1175a88b5ba8SSam Ravnborg if (atomic_dec_and_test(&smp_capture_depth)) { 1176a88b5ba8SSam Ravnborg #ifdef CAPTURE_DEBUG 1177a88b5ba8SSam Ravnborg printk("CPU[%d]: Giving pardon to " 1178a88b5ba8SSam Ravnborg "imprisoned penguins\n", 1179a88b5ba8SSam Ravnborg smp_processor_id()); 1180a88b5ba8SSam Ravnborg #endif 1181a88b5ba8SSam Ravnborg penguins_are_doing_time = 0; 1182a88b5ba8SSam Ravnborg membar_safe("#StoreLoad"); 1183a88b5ba8SSam Ravnborg atomic_dec(&smp_capture_registry); 1184a88b5ba8SSam Ravnborg } 1185a88b5ba8SSam Ravnborg } 1186a88b5ba8SSam Ravnborg 1187a88b5ba8SSam Ravnborg /* Imprisoned penguins run with %pil == PIL_NORMAL_MAX, but PSTATE_IE 1188a88b5ba8SSam Ravnborg * set, so they can service tlb flush xcalls... 1189a88b5ba8SSam Ravnborg */ 1190a88b5ba8SSam Ravnborg extern void prom_world(int); 1191a88b5ba8SSam Ravnborg 11929960e9e8SDavid S. Miller void __irq_entry smp_penguin_jailcell(int irq, struct pt_regs *regs) 1193a88b5ba8SSam Ravnborg { 1194a88b5ba8SSam Ravnborg clear_softint(1 << irq); 1195a88b5ba8SSam Ravnborg 1196a88b5ba8SSam Ravnborg preempt_disable(); 1197a88b5ba8SSam Ravnborg 1198a88b5ba8SSam Ravnborg __asm__ __volatile__("flushw"); 1199a88b5ba8SSam Ravnborg prom_world(1); 1200a88b5ba8SSam Ravnborg atomic_inc(&smp_capture_registry); 1201a88b5ba8SSam Ravnborg membar_safe("#StoreLoad"); 1202a88b5ba8SSam Ravnborg while (penguins_are_doing_time) 1203a88b5ba8SSam Ravnborg rmb(); 1204a88b5ba8SSam Ravnborg atomic_dec(&smp_capture_registry); 1205a88b5ba8SSam Ravnborg prom_world(0); 1206a88b5ba8SSam Ravnborg 1207a88b5ba8SSam Ravnborg preempt_enable(); 1208a88b5ba8SSam Ravnborg } 1209a88b5ba8SSam Ravnborg 1210a88b5ba8SSam Ravnborg /* /proc/profile writes can call this, don't __init it please. */ 1211a88b5ba8SSam Ravnborg int setup_profiling_timer(unsigned int multiplier) 1212a88b5ba8SSam Ravnborg { 1213a88b5ba8SSam Ravnborg return -EINVAL; 1214a88b5ba8SSam Ravnborg } 1215a88b5ba8SSam Ravnborg 1216a88b5ba8SSam Ravnborg void __init smp_prepare_cpus(unsigned int max_cpus) 1217a88b5ba8SSam Ravnborg { 1218a88b5ba8SSam Ravnborg } 1219a88b5ba8SSam Ravnborg 12207c9503b8SGreg Kroah-Hartman void smp_prepare_boot_cpu(void) 1221a88b5ba8SSam Ravnborg { 1222a88b5ba8SSam Ravnborg } 1223a88b5ba8SSam Ravnborg 1224a88b5ba8SSam Ravnborg void __init smp_setup_processor_id(void) 1225a88b5ba8SSam Ravnborg { 1226a88b5ba8SSam Ravnborg if (tlb_type == spitfire) 1227a88b5ba8SSam Ravnborg xcall_deliver_impl = spitfire_xcall_deliver; 1228a88b5ba8SSam Ravnborg else if (tlb_type == cheetah || tlb_type == cheetah_plus) 1229a88b5ba8SSam Ravnborg xcall_deliver_impl = cheetah_xcall_deliver; 1230a88b5ba8SSam Ravnborg else 1231a88b5ba8SSam Ravnborg xcall_deliver_impl = hypervisor_xcall_deliver; 1232a88b5ba8SSam Ravnborg } 1233a88b5ba8SSam Ravnborg 12349b2f753eSAtish Patra void __init smp_fill_in_cpu_possible_map(void) 12359b2f753eSAtish Patra { 12369b2f753eSAtish Patra int possible_cpus = num_possible_cpus(); 12379b2f753eSAtish Patra int i; 12389b2f753eSAtish Patra 12399b2f753eSAtish Patra if (possible_cpus > nr_cpu_ids) 12409b2f753eSAtish Patra possible_cpus = nr_cpu_ids; 12419b2f753eSAtish Patra 12429b2f753eSAtish Patra for (i = 0; i < possible_cpus; i++) 12439b2f753eSAtish Patra set_cpu_possible(i, true); 12449b2f753eSAtish Patra for (; i < NR_CPUS; i++) 12459b2f753eSAtish Patra set_cpu_possible(i, false); 12469b2f753eSAtish Patra } 12479b2f753eSAtish Patra 12487c9503b8SGreg Kroah-Hartman void smp_fill_in_sib_core_maps(void) 1249a88b5ba8SSam Ravnborg { 1250a88b5ba8SSam Ravnborg unsigned int i; 1251a88b5ba8SSam Ravnborg 1252a88b5ba8SSam Ravnborg for_each_present_cpu(i) { 1253a88b5ba8SSam Ravnborg unsigned int j; 1254a88b5ba8SSam Ravnborg 1255fb1fece5SKOSAKI Motohiro cpumask_clear(&cpu_core_map[i]); 1256a88b5ba8SSam Ravnborg if (cpu_data(i).core_id == 0) { 1257fb1fece5SKOSAKI Motohiro cpumask_set_cpu(i, &cpu_core_map[i]); 1258a88b5ba8SSam Ravnborg continue; 1259a88b5ba8SSam Ravnborg } 1260a88b5ba8SSam Ravnborg 1261a88b5ba8SSam Ravnborg for_each_present_cpu(j) { 1262a88b5ba8SSam Ravnborg if (cpu_data(i).core_id == 1263a88b5ba8SSam Ravnborg cpu_data(j).core_id) 1264fb1fece5SKOSAKI Motohiro cpumask_set_cpu(j, &cpu_core_map[i]); 1265a88b5ba8SSam Ravnborg } 1266a88b5ba8SSam Ravnborg } 1267a88b5ba8SSam Ravnborg 1268a88b5ba8SSam Ravnborg for_each_present_cpu(i) { 1269a88b5ba8SSam Ravnborg unsigned int j; 1270a88b5ba8SSam Ravnborg 1271acc455cfSchris hyser for_each_present_cpu(j) { 1272d624716bSAtish Patra if (cpu_data(i).max_cache_id == 1273d624716bSAtish Patra cpu_data(j).max_cache_id) 1274d624716bSAtish Patra cpumask_set_cpu(j, &cpu_core_sib_cache_map[i]); 1275d624716bSAtish Patra 1276acc455cfSchris hyser if (cpu_data(i).sock_id == cpu_data(j).sock_id) 1277acc455cfSchris hyser cpumask_set_cpu(j, &cpu_core_sib_map[i]); 1278acc455cfSchris hyser } 1279acc455cfSchris hyser } 1280acc455cfSchris hyser 1281acc455cfSchris hyser for_each_present_cpu(i) { 1282acc455cfSchris hyser unsigned int j; 1283acc455cfSchris hyser 1284fb1fece5SKOSAKI Motohiro cpumask_clear(&per_cpu(cpu_sibling_map, i)); 1285a88b5ba8SSam Ravnborg if (cpu_data(i).proc_id == -1) { 1286fb1fece5SKOSAKI Motohiro cpumask_set_cpu(i, &per_cpu(cpu_sibling_map, i)); 1287a88b5ba8SSam Ravnborg continue; 1288a88b5ba8SSam Ravnborg } 1289a88b5ba8SSam Ravnborg 1290a88b5ba8SSam Ravnborg for_each_present_cpu(j) { 1291a88b5ba8SSam Ravnborg if (cpu_data(i).proc_id == 1292a88b5ba8SSam Ravnborg cpu_data(j).proc_id) 1293fb1fece5SKOSAKI Motohiro cpumask_set_cpu(j, &per_cpu(cpu_sibling_map, i)); 1294a88b5ba8SSam Ravnborg } 1295a88b5ba8SSam Ravnborg } 1296a88b5ba8SSam Ravnborg } 1297a88b5ba8SSam Ravnborg 12982066aaddSPaul Gortmaker int __cpu_up(unsigned int cpu, struct task_struct *tidle) 1299a88b5ba8SSam Ravnborg { 1300f0a2bc7eSThomas Gleixner int ret = smp_boot_one_cpu(cpu, tidle); 1301a88b5ba8SSam Ravnborg 1302a88b5ba8SSam Ravnborg if (!ret) { 1303fb1fece5SKOSAKI Motohiro cpumask_set_cpu(cpu, &smp_commenced_mask); 1304fb1fece5SKOSAKI Motohiro while (!cpu_online(cpu)) 1305a88b5ba8SSam Ravnborg mb(); 1306fb1fece5SKOSAKI Motohiro if (!cpu_online(cpu)) { 1307a88b5ba8SSam Ravnborg ret = -ENODEV; 1308a88b5ba8SSam Ravnborg } else { 1309a88b5ba8SSam Ravnborg /* On SUN4V, writes to %tick and %stick are 1310a88b5ba8SSam Ravnborg * not allowed. 1311a88b5ba8SSam Ravnborg */ 1312a88b5ba8SSam Ravnborg if (tlb_type != hypervisor) 1313a88b5ba8SSam Ravnborg smp_synchronize_one_tick(cpu); 1314a88b5ba8SSam Ravnborg } 1315a88b5ba8SSam Ravnborg } 1316a88b5ba8SSam Ravnborg return ret; 1317a88b5ba8SSam Ravnborg } 1318a88b5ba8SSam Ravnborg 1319a88b5ba8SSam Ravnborg #ifdef CONFIG_HOTPLUG_CPU 1320a88b5ba8SSam Ravnborg void cpu_play_dead(void) 1321a88b5ba8SSam Ravnborg { 1322a88b5ba8SSam Ravnborg int cpu = smp_processor_id(); 1323a88b5ba8SSam Ravnborg unsigned long pstate; 1324a88b5ba8SSam Ravnborg 1325a88b5ba8SSam Ravnborg idle_task_exit(); 1326a88b5ba8SSam Ravnborg 1327a88b5ba8SSam Ravnborg if (tlb_type == hypervisor) { 1328a88b5ba8SSam Ravnborg struct trap_per_cpu *tb = &trap_block[cpu]; 1329a88b5ba8SSam Ravnborg 1330a88b5ba8SSam Ravnborg sun4v_cpu_qconf(HV_CPU_QUEUE_CPU_MONDO, 1331a88b5ba8SSam Ravnborg tb->cpu_mondo_pa, 0); 1332a88b5ba8SSam Ravnborg sun4v_cpu_qconf(HV_CPU_QUEUE_DEVICE_MONDO, 1333a88b5ba8SSam Ravnborg tb->dev_mondo_pa, 0); 1334a88b5ba8SSam Ravnborg sun4v_cpu_qconf(HV_CPU_QUEUE_RES_ERROR, 1335a88b5ba8SSam Ravnborg tb->resum_mondo_pa, 0); 1336a88b5ba8SSam Ravnborg sun4v_cpu_qconf(HV_CPU_QUEUE_NONRES_ERROR, 1337a88b5ba8SSam Ravnborg tb->nonresum_mondo_pa, 0); 1338a88b5ba8SSam Ravnborg } 1339a88b5ba8SSam Ravnborg 1340fb1fece5SKOSAKI Motohiro cpumask_clear_cpu(cpu, &smp_commenced_mask); 1341a88b5ba8SSam Ravnborg membar_safe("#Sync"); 1342a88b5ba8SSam Ravnborg 1343a88b5ba8SSam Ravnborg local_irq_disable(); 1344a88b5ba8SSam Ravnborg 1345a88b5ba8SSam Ravnborg __asm__ __volatile__( 1346a88b5ba8SSam Ravnborg "rdpr %%pstate, %0\n\t" 1347a88b5ba8SSam Ravnborg "wrpr %0, %1, %%pstate" 1348a88b5ba8SSam Ravnborg : "=r" (pstate) 1349a88b5ba8SSam Ravnborg : "i" (PSTATE_IE)); 1350a88b5ba8SSam Ravnborg 1351a88b5ba8SSam Ravnborg while (1) 1352a88b5ba8SSam Ravnborg barrier(); 1353a88b5ba8SSam Ravnborg } 1354a88b5ba8SSam Ravnborg 1355a88b5ba8SSam Ravnborg int __cpu_disable(void) 1356a88b5ba8SSam Ravnborg { 1357a88b5ba8SSam Ravnborg int cpu = smp_processor_id(); 1358a88b5ba8SSam Ravnborg cpuinfo_sparc *c; 1359a88b5ba8SSam Ravnborg int i; 1360a88b5ba8SSam Ravnborg 1361fb1fece5SKOSAKI Motohiro for_each_cpu(i, &cpu_core_map[cpu]) 1362fb1fece5SKOSAKI Motohiro cpumask_clear_cpu(cpu, &cpu_core_map[i]); 1363fb1fece5SKOSAKI Motohiro cpumask_clear(&cpu_core_map[cpu]); 1364a88b5ba8SSam Ravnborg 1365fb1fece5SKOSAKI Motohiro for_each_cpu(i, &per_cpu(cpu_sibling_map, cpu)) 1366fb1fece5SKOSAKI Motohiro cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, i)); 1367fb1fece5SKOSAKI Motohiro cpumask_clear(&per_cpu(cpu_sibling_map, cpu)); 1368a88b5ba8SSam Ravnborg 1369a88b5ba8SSam Ravnborg c = &cpu_data(cpu); 1370a88b5ba8SSam Ravnborg 1371a88b5ba8SSam Ravnborg c->core_id = 0; 1372a88b5ba8SSam Ravnborg c->proc_id = -1; 1373a88b5ba8SSam Ravnborg 1374a88b5ba8SSam Ravnborg smp_wmb(); 1375a88b5ba8SSam Ravnborg 1376a88b5ba8SSam Ravnborg /* Make sure no interrupts point to this cpu. */ 1377a88b5ba8SSam Ravnborg fixup_irqs(); 1378a88b5ba8SSam Ravnborg 1379a88b5ba8SSam Ravnborg local_irq_enable(); 1380a88b5ba8SSam Ravnborg mdelay(1); 1381a88b5ba8SSam Ravnborg local_irq_disable(); 1382a88b5ba8SSam Ravnborg 1383fb1fece5SKOSAKI Motohiro set_cpu_online(cpu, false); 1384a88b5ba8SSam Ravnborg 1385280ff974SHong H. Pham cpu_map_rebuild(); 1386280ff974SHong H. Pham 1387a88b5ba8SSam Ravnborg return 0; 1388a88b5ba8SSam Ravnborg } 1389a88b5ba8SSam Ravnborg 1390a88b5ba8SSam Ravnborg void __cpu_die(unsigned int cpu) 1391a88b5ba8SSam Ravnborg { 1392a88b5ba8SSam Ravnborg int i; 1393a88b5ba8SSam Ravnborg 1394a88b5ba8SSam Ravnborg for (i = 0; i < 100; i++) { 1395a88b5ba8SSam Ravnborg smp_rmb(); 1396fb1fece5SKOSAKI Motohiro if (!cpumask_test_cpu(cpu, &smp_commenced_mask)) 1397a88b5ba8SSam Ravnborg break; 1398a88b5ba8SSam Ravnborg msleep(100); 1399a88b5ba8SSam Ravnborg } 1400fb1fece5SKOSAKI Motohiro if (cpumask_test_cpu(cpu, &smp_commenced_mask)) { 1401a88b5ba8SSam Ravnborg printk(KERN_ERR "CPU %u didn't die...\n", cpu); 1402a88b5ba8SSam Ravnborg } else { 1403a88b5ba8SSam Ravnborg #if defined(CONFIG_SUN_LDOMS) 1404a88b5ba8SSam Ravnborg unsigned long hv_err; 1405a88b5ba8SSam Ravnborg int limit = 100; 1406a88b5ba8SSam Ravnborg 1407a88b5ba8SSam Ravnborg do { 1408a88b5ba8SSam Ravnborg hv_err = sun4v_cpu_stop(cpu); 1409a88b5ba8SSam Ravnborg if (hv_err == HV_EOK) { 1410fb1fece5SKOSAKI Motohiro set_cpu_present(cpu, false); 1411a88b5ba8SSam Ravnborg break; 1412a88b5ba8SSam Ravnborg } 1413a88b5ba8SSam Ravnborg } while (--limit > 0); 1414a88b5ba8SSam Ravnborg if (limit <= 0) { 1415a88b5ba8SSam Ravnborg printk(KERN_ERR "sun4v_cpu_stop() fails err=%lu\n", 1416a88b5ba8SSam Ravnborg hv_err); 1417a88b5ba8SSam Ravnborg } 1418a88b5ba8SSam Ravnborg #endif 1419a88b5ba8SSam Ravnborg } 1420a88b5ba8SSam Ravnborg } 1421a88b5ba8SSam Ravnborg #endif 1422a88b5ba8SSam Ravnborg 1423a88b5ba8SSam Ravnborg void __init smp_cpus_done(unsigned int max_cpus) 1424a88b5ba8SSam Ravnborg { 1425a88b5ba8SSam Ravnborg } 1426a88b5ba8SSam Ravnborg 1427a88b5ba8SSam Ravnborg void smp_send_reschedule(int cpu) 1428a88b5ba8SSam Ravnborg { 14291a36265bSKirill Tkhai if (cpu == smp_processor_id()) { 14301a36265bSKirill Tkhai WARN_ON_ONCE(preemptible()); 14311a36265bSKirill Tkhai set_softint(1 << PIL_SMP_RECEIVE_SIGNAL); 14321a36265bSKirill Tkhai } else { 14331a36265bSKirill Tkhai xcall_deliver((u64) &xcall_receive_signal, 14341a36265bSKirill Tkhai 0, 0, cpumask_of(cpu)); 14351a36265bSKirill Tkhai } 1436a88b5ba8SSam Ravnborg } 1437a88b5ba8SSam Ravnborg 14389960e9e8SDavid S. Miller void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs) 1439a88b5ba8SSam Ravnborg { 1440a88b5ba8SSam Ravnborg clear_softint(1 << irq); 1441184748ccSPeter Zijlstra scheduler_ipi(); 1442a88b5ba8SSam Ravnborg } 1443a88b5ba8SSam Ravnborg 144494ab5990SDave Kleikamp static void stop_this_cpu(void *dummy) 144594ab5990SDave Kleikamp { 1446cffb3e76SVijay Kumar set_cpu_online(smp_processor_id(), false); 144794ab5990SDave Kleikamp prom_stopself(); 144894ab5990SDave Kleikamp } 144994ab5990SDave Kleikamp 1450a88b5ba8SSam Ravnborg void smp_send_stop(void) 1451a88b5ba8SSam Ravnborg { 145294ab5990SDave Kleikamp int cpu; 145394ab5990SDave Kleikamp 145494ab5990SDave Kleikamp if (tlb_type == hypervisor) { 1455*7dd4fcf5SVijay Kumar int this_cpu = smp_processor_id(); 1456*7dd4fcf5SVijay Kumar #ifdef CONFIG_SERIAL_SUNHV 1457*7dd4fcf5SVijay Kumar sunhv_migrate_hvcons_irq(this_cpu); 1458*7dd4fcf5SVijay Kumar #endif 145994ab5990SDave Kleikamp for_each_online_cpu(cpu) { 1460*7dd4fcf5SVijay Kumar if (cpu == this_cpu) 146194ab5990SDave Kleikamp continue; 1462cffb3e76SVijay Kumar 1463cffb3e76SVijay Kumar set_cpu_online(cpu, false); 146494ab5990SDave Kleikamp #ifdef CONFIG_SUN_LDOMS 146594ab5990SDave Kleikamp if (ldom_domaining_enabled) { 146694ab5990SDave Kleikamp unsigned long hv_err; 146794ab5990SDave Kleikamp hv_err = sun4v_cpu_stop(cpu); 146894ab5990SDave Kleikamp if (hv_err) 146994ab5990SDave Kleikamp printk(KERN_ERR "sun4v_cpu_stop() " 147094ab5990SDave Kleikamp "failed err=%lu\n", hv_err); 147194ab5990SDave Kleikamp } else 147294ab5990SDave Kleikamp #endif 147394ab5990SDave Kleikamp prom_stopcpu_cpuid(cpu); 147494ab5990SDave Kleikamp } 147594ab5990SDave Kleikamp } else 147694ab5990SDave Kleikamp smp_call_function(stop_this_cpu, NULL, 0); 1477a88b5ba8SSam Ravnborg } 1478a88b5ba8SSam Ravnborg 14794fd78a5fSDavid S. Miller /** 14804fd78a5fSDavid S. Miller * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu 14814fd78a5fSDavid S. Miller * @cpu: cpu to allocate for 14824fd78a5fSDavid S. Miller * @size: size allocation in bytes 14834fd78a5fSDavid S. Miller * @align: alignment 14844fd78a5fSDavid S. Miller * 14854fd78a5fSDavid S. Miller * Allocate @size bytes aligned at @align for cpu @cpu. This wrapper 14864fd78a5fSDavid S. Miller * does the right thing for NUMA regardless of the current 14874fd78a5fSDavid S. Miller * configuration. 14884fd78a5fSDavid S. Miller * 14894fd78a5fSDavid S. Miller * RETURNS: 14904fd78a5fSDavid S. Miller * Pointer to the allocated area on success, NULL on failure. 14914fd78a5fSDavid S. Miller */ 1492bcb2107fSTejun Heo static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size, 1493bcb2107fSTejun Heo size_t align) 14944fd78a5fSDavid S. Miller { 14954fd78a5fSDavid S. Miller const unsigned long goal = __pa(MAX_DMA_ADDRESS); 14964fd78a5fSDavid S. Miller #ifdef CONFIG_NEED_MULTIPLE_NODES 14974fd78a5fSDavid S. Miller int node = cpu_to_node(cpu); 14984fd78a5fSDavid S. Miller void *ptr; 14994fd78a5fSDavid S. Miller 15004fd78a5fSDavid S. Miller if (!node_online(node) || !NODE_DATA(node)) { 15014fd78a5fSDavid S. Miller ptr = __alloc_bootmem(size, align, goal); 15024fd78a5fSDavid S. Miller pr_info("cpu %d has no node %d or node-local memory\n", 15034fd78a5fSDavid S. Miller cpu, node); 15044fd78a5fSDavid S. Miller pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n", 15054fd78a5fSDavid S. Miller cpu, size, __pa(ptr)); 15064fd78a5fSDavid S. Miller } else { 15074fd78a5fSDavid S. Miller ptr = __alloc_bootmem_node(NODE_DATA(node), 15084fd78a5fSDavid S. Miller size, align, goal); 15094fd78a5fSDavid S. Miller pr_debug("per cpu data for cpu%d %lu bytes on node%d at " 15104fd78a5fSDavid S. Miller "%016lx\n", cpu, size, node, __pa(ptr)); 15114fd78a5fSDavid S. Miller } 15124fd78a5fSDavid S. Miller return ptr; 15134fd78a5fSDavid S. Miller #else 15144fd78a5fSDavid S. Miller return __alloc_bootmem(size, align, goal); 15154fd78a5fSDavid S. Miller #endif 15164fd78a5fSDavid S. Miller } 15174fd78a5fSDavid S. Miller 1518bcb2107fSTejun Heo static void __init pcpu_free_bootmem(void *ptr, size_t size) 15194fd78a5fSDavid S. Miller { 1520bcb2107fSTejun Heo free_bootmem(__pa(ptr), size); 15214fd78a5fSDavid S. Miller } 15224fd78a5fSDavid S. Miller 1523a70c6913STejun Heo static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) 1524bcb2107fSTejun Heo { 1525bcb2107fSTejun Heo if (cpu_to_node(from) == cpu_to_node(to)) 1526bcb2107fSTejun Heo return LOCAL_DISTANCE; 1527bcb2107fSTejun Heo else 1528bcb2107fSTejun Heo return REMOTE_DISTANCE; 15294fd78a5fSDavid S. Miller } 15304fd78a5fSDavid S. Miller 1531a70c6913STejun Heo static void __init pcpu_populate_pte(unsigned long addr) 1532a70c6913STejun Heo { 1533a70c6913STejun Heo pgd_t *pgd = pgd_offset_k(addr); 1534a70c6913STejun Heo pud_t *pud; 1535a70c6913STejun Heo pmd_t *pmd; 1536a70c6913STejun Heo 1537ac55c768SDavid S. Miller if (pgd_none(*pgd)) { 1538ac55c768SDavid S. Miller pud_t *new; 1539ac55c768SDavid S. Miller 1540ac55c768SDavid S. Miller new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1541ac55c768SDavid S. Miller pgd_populate(&init_mm, pgd, new); 1542ac55c768SDavid S. Miller } 1543ac55c768SDavid S. Miller 1544a70c6913STejun Heo pud = pud_offset(pgd, addr); 1545a70c6913STejun Heo if (pud_none(*pud)) { 1546a70c6913STejun Heo pmd_t *new; 1547a70c6913STejun Heo 1548a70c6913STejun Heo new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1549a70c6913STejun Heo pud_populate(&init_mm, pud, new); 1550a70c6913STejun Heo } 1551a70c6913STejun Heo 1552a70c6913STejun Heo pmd = pmd_offset(pud, addr); 1553a70c6913STejun Heo if (!pmd_present(*pmd)) { 1554a70c6913STejun Heo pte_t *new; 1555a70c6913STejun Heo 1556a70c6913STejun Heo new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1557a70c6913STejun Heo pmd_populate_kernel(&init_mm, pmd, new); 1558a70c6913STejun Heo } 1559a70c6913STejun Heo } 1560a70c6913STejun Heo 156173fffc03SDavid S. Miller void __init setup_per_cpu_areas(void) 1562a88b5ba8SSam Ravnborg { 1563bcb2107fSTejun Heo unsigned long delta; 1564bcb2107fSTejun Heo unsigned int cpu; 1565a70c6913STejun Heo int rc = -EINVAL; 1566a88b5ba8SSam Ravnborg 1567a70c6913STejun Heo if (pcpu_chosen_fc != PCPU_FC_PAGE) { 1568bcb2107fSTejun Heo rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, 1569bcb2107fSTejun Heo PERCPU_DYNAMIC_RESERVE, 4 << 20, 1570a70c6913STejun Heo pcpu_cpu_distance, 1571a70c6913STejun Heo pcpu_alloc_bootmem, 1572bcb2107fSTejun Heo pcpu_free_bootmem); 1573fb435d52STejun Heo if (rc) 1574a70c6913STejun Heo pr_warning("PERCPU: %s allocator failed (%d), " 1575a70c6913STejun Heo "falling back to page size\n", 1576a70c6913STejun Heo pcpu_fc_names[pcpu_chosen_fc], rc); 1577a70c6913STejun Heo } 1578a70c6913STejun Heo if (rc < 0) 1579a70c6913STejun Heo rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, 1580a70c6913STejun Heo pcpu_alloc_bootmem, 1581a70c6913STejun Heo pcpu_free_bootmem, 1582a70c6913STejun Heo pcpu_populate_pte); 1583a70c6913STejun Heo if (rc < 0) 1584a70c6913STejun Heo panic("cannot initialize percpu area (err=%d)", rc); 15854fd78a5fSDavid S. Miller 15864fd78a5fSDavid S. Miller delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; 1587fb435d52STejun Heo for_each_possible_cpu(cpu) 1588fb435d52STejun Heo __per_cpu_offset(cpu) = delta + pcpu_unit_offsets[cpu]; 1589a88b5ba8SSam Ravnborg 1590a88b5ba8SSam Ravnborg /* Setup %g5 for the boot cpu. */ 1591a88b5ba8SSam Ravnborg __local_per_cpu_offset = __per_cpu_offset(smp_processor_id()); 1592b696fdc2SDavid S. Miller 1593b696fdc2SDavid S. Miller of_fill_in_cpu_data(); 1594b696fdc2SDavid S. Miller if (tlb_type == hypervisor) 15956ac5c610SStephen Rothwell mdesc_fill_in_cpu_data(cpu_all_mask); 1596a88b5ba8SSam Ravnborg } 1597