1 /* 2 * Fast batching percpu counters. 3 */ 4 5 #include <linux/percpu_counter.h> 6 #include <linux/notifier.h> 7 #include <linux/mutex.h> 8 #include <linux/init.h> 9 #include <linux/cpu.h> 10 #include <linux/module.h> 11 12 #ifdef CONFIG_HOTPLUG_CPU 13 static LIST_HEAD(percpu_counters); 14 static DEFINE_MUTEX(percpu_counters_lock); 15 #endif 16 17 void percpu_counter_mod(struct percpu_counter *fbc, s32 amount) 18 { 19 long count; 20 s32 *pcount; 21 int cpu = get_cpu(); 22 23 pcount = per_cpu_ptr(fbc->counters, cpu); 24 count = *pcount + amount; 25 if (count >= FBC_BATCH || count <= -FBC_BATCH) { 26 spin_lock(&fbc->lock); 27 fbc->count += count; 28 *pcount = 0; 29 spin_unlock(&fbc->lock); 30 } else { 31 *pcount = count; 32 } 33 put_cpu(); 34 } 35 EXPORT_SYMBOL(percpu_counter_mod); 36 37 /* 38 * Add up all the per-cpu counts, return the result. This is a more accurate 39 * but much slower version of percpu_counter_read_positive() 40 */ 41 s64 percpu_counter_sum(struct percpu_counter *fbc) 42 { 43 s64 ret; 44 int cpu; 45 46 spin_lock(&fbc->lock); 47 ret = fbc->count; 48 for_each_online_cpu(cpu) { 49 s32 *pcount = per_cpu_ptr(fbc->counters, cpu); 50 ret += *pcount; 51 } 52 spin_unlock(&fbc->lock); 53 return ret < 0 ? 0 : ret; 54 } 55 EXPORT_SYMBOL(percpu_counter_sum); 56 57 void percpu_counter_init(struct percpu_counter *fbc, s64 amount) 58 { 59 spin_lock_init(&fbc->lock); 60 fbc->count = amount; 61 fbc->counters = alloc_percpu(s32); 62 #ifdef CONFIG_HOTPLUG_CPU 63 mutex_lock(&percpu_counters_lock); 64 list_add(&fbc->list, &percpu_counters); 65 mutex_unlock(&percpu_counters_lock); 66 #endif 67 } 68 EXPORT_SYMBOL(percpu_counter_init); 69 70 void percpu_counter_destroy(struct percpu_counter *fbc) 71 { 72 free_percpu(fbc->counters); 73 #ifdef CONFIG_HOTPLUG_CPU 74 mutex_lock(&percpu_counters_lock); 75 list_del(&fbc->list); 76 mutex_unlock(&percpu_counters_lock); 77 #endif 78 } 79 EXPORT_SYMBOL(percpu_counter_destroy); 80 81 #ifdef CONFIG_HOTPLUG_CPU 82 static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb, 83 unsigned long action, void *hcpu) 84 { 85 unsigned int cpu; 86 struct percpu_counter *fbc; 87 88 if (action != CPU_DEAD) 89 return NOTIFY_OK; 90 91 cpu = (unsigned long)hcpu; 92 mutex_lock(&percpu_counters_lock); 93 list_for_each_entry(fbc, &percpu_counters, list) { 94 s32 *pcount; 95 96 spin_lock(&fbc->lock); 97 pcount = per_cpu_ptr(fbc->counters, cpu); 98 fbc->count += *pcount; 99 *pcount = 0; 100 spin_unlock(&fbc->lock); 101 } 102 mutex_unlock(&percpu_counters_lock); 103 return NOTIFY_OK; 104 } 105 106 static int __init percpu_counter_startup(void) 107 { 108 hotcpu_notifier(percpu_counter_hotcpu_callback, 0); 109 return 0; 110 } 111 module_init(percpu_counter_startup); 112 #endif 113