1 /* 2 * Fast batching percpu counters. 3 */ 4 5 #include <linux/percpu_counter.h> 6 #include <linux/notifier.h> 7 #include <linux/mutex.h> 8 #include <linux/init.h> 9 #include <linux/cpu.h> 10 #include <linux/module.h> 11 12 static LIST_HEAD(percpu_counters); 13 static DEFINE_MUTEX(percpu_counters_lock); 14 15 void percpu_counter_set(struct percpu_counter *fbc, s64 amount) 16 { 17 int cpu; 18 19 spin_lock(&fbc->lock); 20 for_each_possible_cpu(cpu) { 21 s32 *pcount = per_cpu_ptr(fbc->counters, cpu); 22 *pcount = 0; 23 } 24 fbc->count = amount; 25 spin_unlock(&fbc->lock); 26 } 27 EXPORT_SYMBOL(percpu_counter_set); 28 29 void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch) 30 { 31 s64 count; 32 s32 *pcount; 33 int cpu = get_cpu(); 34 35 pcount = per_cpu_ptr(fbc->counters, cpu); 36 count = *pcount + amount; 37 if (count >= batch || count <= -batch) { 38 spin_lock(&fbc->lock); 39 fbc->count += count; 40 *pcount = 0; 41 spin_unlock(&fbc->lock); 42 } else { 43 *pcount = count; 44 } 45 put_cpu(); 46 } 47 EXPORT_SYMBOL(__percpu_counter_add); 48 49 /* 50 * Add up all the per-cpu counts, return the result. This is a more accurate 51 * but much slower version of percpu_counter_read_positive() 52 */ 53 s64 __percpu_counter_sum(struct percpu_counter *fbc) 54 { 55 s64 ret; 56 int cpu; 57 58 spin_lock(&fbc->lock); 59 ret = fbc->count; 60 for_each_online_cpu(cpu) { 61 s32 *pcount = per_cpu_ptr(fbc->counters, cpu); 62 ret += *pcount; 63 } 64 spin_unlock(&fbc->lock); 65 return ret; 66 } 67 EXPORT_SYMBOL(__percpu_counter_sum); 68 69 int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, 70 struct lock_class_key *key) 71 { 72 spin_lock_init(&fbc->lock); 73 lockdep_set_class(&fbc->lock, key); 74 fbc->count = amount; 75 fbc->counters = alloc_percpu(s32); 76 if (!fbc->counters) 77 return -ENOMEM; 78 #ifdef CONFIG_HOTPLUG_CPU 79 mutex_lock(&percpu_counters_lock); 80 list_add(&fbc->list, &percpu_counters); 81 mutex_unlock(&percpu_counters_lock); 82 #endif 83 return 0; 84 } 85 EXPORT_SYMBOL(__percpu_counter_init); 86 87 void percpu_counter_destroy(struct percpu_counter *fbc) 88 { 89 if (!fbc->counters) 90 return; 91 92 #ifdef CONFIG_HOTPLUG_CPU 93 mutex_lock(&percpu_counters_lock); 94 list_del(&fbc->list); 95 mutex_unlock(&percpu_counters_lock); 96 #endif 97 free_percpu(fbc->counters); 98 fbc->counters = NULL; 99 } 100 EXPORT_SYMBOL(percpu_counter_destroy); 101 102 int percpu_counter_batch __read_mostly = 32; 103 EXPORT_SYMBOL(percpu_counter_batch); 104 105 static void compute_batch_value(void) 106 { 107 int nr = num_online_cpus(); 108 109 percpu_counter_batch = max(32, nr*2); 110 } 111 112 static int __cpuinit percpu_counter_hotcpu_callback(struct notifier_block *nb, 113 unsigned long action, void *hcpu) 114 { 115 #ifdef CONFIG_HOTPLUG_CPU 116 unsigned int cpu; 117 struct percpu_counter *fbc; 118 119 compute_batch_value(); 120 if (action != CPU_DEAD) 121 return NOTIFY_OK; 122 123 cpu = (unsigned long)hcpu; 124 mutex_lock(&percpu_counters_lock); 125 list_for_each_entry(fbc, &percpu_counters, list) { 126 s32 *pcount; 127 unsigned long flags; 128 129 spin_lock_irqsave(&fbc->lock, flags); 130 pcount = per_cpu_ptr(fbc->counters, cpu); 131 fbc->count += *pcount; 132 *pcount = 0; 133 spin_unlock_irqrestore(&fbc->lock, flags); 134 } 135 mutex_unlock(&percpu_counters_lock); 136 #endif 137 return NOTIFY_OK; 138 } 139 140 static int __init percpu_counter_startup(void) 141 { 142 compute_batch_value(); 143 hotcpu_notifier(percpu_counter_hotcpu_callback, 0); 144 return 0; 145 } 146 module_init(percpu_counter_startup); 147