177 static void __mask_array_destroy(struct mask_array *ma)  in __mask_array_destroy()  argument
179 	free_percpu(ma->masks_usage_stats);  in __mask_array_destroy()
180 	kfree(ma);  in __mask_array_destroy()
185 	struct mask_array *ma = container_of(rcu, struct mask_array, rcu);  in mask_array_rcu_cb()  local
187 	__mask_array_destroy(ma);  in mask_array_rcu_cb()
190 static void tbl_mask_array_reset_counters(struct mask_array *ma)  in tbl_mask_array_reset_counters()  argument
199 	for (i = 0; i < ma->max; i++) {  in tbl_mask_array_reset_counters()
200 		ma->masks_usage_zero_cntr[i] = 0;  in tbl_mask_array_reset_counters()
207 			stats = per_cpu_ptr(ma->masks_usage_stats, cpu);  in tbl_mask_array_reset_counters()
213 			ma->masks_usage_zero_cntr[i] += counter;  in tbl_mask_array_reset_counters()
273 	struct mask_array *ma = ovsl_dereference(tbl->mask_array);  in tbl_mask_array_add_mask()  local
274 	int err, ma_count = READ_ONCE(ma->count);  in tbl_mask_array_add_mask()
276 	if (ma_count >= ma->max) {  in tbl_mask_array_add_mask()
277 		err = tbl_mask_array_realloc(tbl, ma->max +  in tbl_mask_array_add_mask()
282 		ma = ovsl_dereference(tbl->mask_array);  in tbl_mask_array_add_mask()
287 		tbl_mask_array_reset_counters(ma);  in tbl_mask_array_add_mask()
290 	BUG_ON(ovsl_dereference(ma->masks[ma_count]));  in tbl_mask_array_add_mask()
292 	rcu_assign_pointer(ma->masks[ma_count], new);  in tbl_mask_array_add_mask()
293 	WRITE_ONCE(ma->count, ma_count + 1);  in tbl_mask_array_add_mask()
301 	struct mask_array *ma = ovsl_dereference(tbl->mask_array);  in tbl_mask_array_del_mask()  local
302 	int i, ma_count = READ_ONCE(ma->count);  in tbl_mask_array_del_mask()
306 		if (mask == ovsl_dereference(ma->masks[i]))  in tbl_mask_array_del_mask()
314 	WRITE_ONCE(ma->count, ma_count - 1);  in tbl_mask_array_del_mask()
316 	rcu_assign_pointer(ma->masks[i], ma->masks[ma_count - 1]);  in tbl_mask_array_del_mask()
317 	RCU_INIT_POINTER(ma->masks[ma_count - 1], NULL);  in tbl_mask_array_del_mask()
322 	if (ma->max >= (MASK_ARRAY_SIZE_MIN * 2) &&  in tbl_mask_array_del_mask()
323 	    ma_count <= (ma->max / 3))  in tbl_mask_array_del_mask()
324 		tbl_mask_array_realloc(tbl, ma->max / 2);  in tbl_mask_array_del_mask()
326 		tbl_mask_array_reset_counters(ma);  in tbl_mask_array_del_mask()
415 	struct mask_array *ma;  in ovs_flow_tbl_init()  local
421 	ma = tbl_mask_array_alloc(MASK_ARRAY_SIZE_MIN);  in ovs_flow_tbl_init()
422 	if (!ma)  in ovs_flow_tbl_init()
435 	rcu_assign_pointer(table->mask_array, ma);  in ovs_flow_tbl_init()
445 	__mask_array_destroy(ma);  in ovs_flow_tbl_init()
518 	struct mask_array *ma = rcu_dereference_raw(table->mask_array);  in ovs_flow_tbl_destroy()  local
521 	call_rcu(&ma->rcu, mask_array_rcu_cb);  in ovs_flow_tbl_destroy()
729 				   struct mask_array *ma,  in flow_lookup()  argument
735 	struct mask_array_stats *stats = this_cpu_ptr(ma->masks_usage_stats);  in flow_lookup()
740 	if (likely(*index < ma->max)) {  in flow_lookup()
741 		mask = rcu_dereference_ovsl(ma->masks[*index]);  in flow_lookup()
754 	for (i = 0; i < ma->max; i++)  {  in flow_lookup()
759 		mask = rcu_dereference_ovsl(ma->masks[i]);  in flow_lookup()
790 	struct mask_array *ma = rcu_dereference(tbl->mask_array);  in ovs_flow_tbl_lookup_stats()  local
803 		return flow_lookup(tbl, ti, ma, key, n_mask_hit, &cache,  in ovs_flow_tbl_lookup_stats()
824 			flow = flow_lookup(tbl, ti, ma, key, n_mask_hit,  in ovs_flow_tbl_lookup_stats()
838 	flow = flow_lookup(tbl, ti, ma, key, n_mask_hit, n_cache_hit,  in ovs_flow_tbl_lookup_stats()
851 	struct mask_array *ma = rcu_dereference_ovsl(tbl->mask_array);  in ovs_flow_tbl_lookup()  local
862 	flow = flow_lookup(tbl, ti, ma, key, &n_mask_hit, &n_cache_hit, &index);  in ovs_flow_tbl_lookup()
870 	struct mask_array *ma = ovsl_dereference(tbl->mask_array);  in ovs_flow_tbl_lookup_exact()  local
874 	for (i = 0; i < ma->max; i++) {  in ovs_flow_tbl_lookup_exact()
880 		mask = ovsl_dereference(ma->masks[i]);  in ovs_flow_tbl_lookup_exact()
938 	struct mask_array *ma = rcu_dereference_ovsl(table->mask_array);  in ovs_flow_tbl_num_masks()  local
939 	return READ_ONCE(ma->count);  in ovs_flow_tbl_num_masks()
990 	struct mask_array *ma;  in flow_mask_find()  local
993 	ma = ovsl_dereference(tbl->mask_array);  in flow_mask_find()
994 	for (i = 0; i < ma->max; i++) {  in flow_mask_find()
996 		t = ovsl_dereference(ma->masks[i]);  in flow_mask_find()
1107 	struct mask_array *ma = rcu_dereference_ovsl(table->mask_array);  in ovs_flow_masks_rebalance()  local
1114 	masks_and_count = kmalloc_array(ma->max, sizeof(*masks_and_count),  in ovs_flow_masks_rebalance()
1119 	for (i = 0; i < ma->max; i++) {  in ovs_flow_masks_rebalance()
1123 		mask = rcu_dereference_ovsl(ma->masks[i]);  in ovs_flow_masks_rebalance()
1135 			stats = per_cpu_ptr(ma->masks_usage_stats, cpu);  in ovs_flow_masks_rebalance()
1145 		masks_and_count[i].counter -= ma->masks_usage_zero_cntr[i];  in ovs_flow_masks_rebalance()
1150 		ma->masks_usage_zero_cntr[i] += masks_and_count[i].counter;  in ovs_flow_masks_rebalance()
1170 	new = tbl_mask_array_alloc(ma->max);  in ovs_flow_masks_rebalance()
1177 		if (ovsl_dereference(ma->masks[index]))  in ovs_flow_masks_rebalance()
1178 			new->masks[new->count++] = ma->masks[index];  in ovs_flow_masks_rebalance()
1182 	call_rcu(&ma->rcu, mask_array_rcu_cb);  in ovs_flow_masks_rebalance()