175 static void __mask_array_destroy(struct mask_array *ma)  in __mask_array_destroy()  argument
177 	free_percpu(ma->masks_usage_stats);  in __mask_array_destroy()
178 	kfree(ma);  in __mask_array_destroy()
183 	struct mask_array *ma = container_of(rcu, struct mask_array, rcu);  in mask_array_rcu_cb()  local
185 	__mask_array_destroy(ma);  in mask_array_rcu_cb()
188 static void tbl_mask_array_reset_counters(struct mask_array *ma)  in tbl_mask_array_reset_counters()  argument
197 	for (i = 0; i < ma->max; i++) {  in tbl_mask_array_reset_counters()
198 		ma->masks_usage_zero_cntr[i] = 0;  in tbl_mask_array_reset_counters()
205 			stats = per_cpu_ptr(ma->masks_usage_stats, cpu);  in tbl_mask_array_reset_counters()
211 			ma->masks_usage_zero_cntr[i] += counter;  in tbl_mask_array_reset_counters()
271 	struct mask_array *ma = ovsl_dereference(tbl->mask_array);  in tbl_mask_array_add_mask()  local
272 	int err, ma_count = READ_ONCE(ma->count);  in tbl_mask_array_add_mask()
274 	if (ma_count >= ma->max) {  in tbl_mask_array_add_mask()
275 		err = tbl_mask_array_realloc(tbl, ma->max +  in tbl_mask_array_add_mask()
280 		ma = ovsl_dereference(tbl->mask_array);  in tbl_mask_array_add_mask()
285 		tbl_mask_array_reset_counters(ma);  in tbl_mask_array_add_mask()
288 	BUG_ON(ovsl_dereference(ma->masks[ma_count]));  in tbl_mask_array_add_mask()
290 	rcu_assign_pointer(ma->masks[ma_count], new);  in tbl_mask_array_add_mask()
291 	WRITE_ONCE(ma->count, ma_count + 1);  in tbl_mask_array_add_mask()
299 	struct mask_array *ma = ovsl_dereference(tbl->mask_array);  in tbl_mask_array_del_mask()  local
300 	int i, ma_count = READ_ONCE(ma->count);  in tbl_mask_array_del_mask()
304 		if (mask == ovsl_dereference(ma->masks[i]))  in tbl_mask_array_del_mask()
312 	WRITE_ONCE(ma->count, ma_count - 1);  in tbl_mask_array_del_mask()
314 	rcu_assign_pointer(ma->masks[i], ma->masks[ma_count - 1]);  in tbl_mask_array_del_mask()
315 	RCU_INIT_POINTER(ma->masks[ma_count - 1], NULL);  in tbl_mask_array_del_mask()
320 	if (ma->max >= (MASK_ARRAY_SIZE_MIN * 2) &&  in tbl_mask_array_del_mask()
321 	    ma_count <= (ma->max / 3))  in tbl_mask_array_del_mask()
322 		tbl_mask_array_realloc(tbl, ma->max / 2);  in tbl_mask_array_del_mask()
324 		tbl_mask_array_reset_counters(ma);  in tbl_mask_array_del_mask()
413 	struct mask_array *ma;  in ovs_flow_tbl_init()  local
419 	ma = tbl_mask_array_alloc(MASK_ARRAY_SIZE_MIN);  in ovs_flow_tbl_init()
420 	if (!ma)  in ovs_flow_tbl_init()
433 	rcu_assign_pointer(table->mask_array, ma);  in ovs_flow_tbl_init()
443 	__mask_array_destroy(ma);  in ovs_flow_tbl_init()
516 	struct mask_array *ma = rcu_dereference_raw(table->mask_array);  in ovs_flow_tbl_destroy()  local
519 	call_rcu(&ma->rcu, mask_array_rcu_cb);  in ovs_flow_tbl_destroy()
727 				   struct mask_array *ma,  in flow_lookup()  argument
733 	struct mask_array_stats *stats = this_cpu_ptr(ma->masks_usage_stats);  in flow_lookup()
738 	if (likely(*index < ma->max)) {  in flow_lookup()
739 		mask = rcu_dereference_ovsl(ma->masks[*index]);  in flow_lookup()
752 	for (i = 0; i < ma->max; i++)  {  in flow_lookup()
757 		mask = rcu_dereference_ovsl(ma->masks[i]);  in flow_lookup()
788 	struct mask_array *ma = rcu_dereference(tbl->mask_array);  in ovs_flow_tbl_lookup_stats()  local
801 		return flow_lookup(tbl, ti, ma, key, n_mask_hit, &cache,  in ovs_flow_tbl_lookup_stats()
822 			flow = flow_lookup(tbl, ti, ma, key, n_mask_hit,  in ovs_flow_tbl_lookup_stats()
836 	flow = flow_lookup(tbl, ti, ma, key, n_mask_hit, n_cache_hit,  in ovs_flow_tbl_lookup_stats()
849 	struct mask_array *ma = rcu_dereference_ovsl(tbl->mask_array);  in ovs_flow_tbl_lookup()  local
860 	flow = flow_lookup(tbl, ti, ma, key, &n_mask_hit, &n_cache_hit, &index);  in ovs_flow_tbl_lookup()
868 	struct mask_array *ma = ovsl_dereference(tbl->mask_array);  in ovs_flow_tbl_lookup_exact()  local
872 	for (i = 0; i < ma->max; i++) {  in ovs_flow_tbl_lookup_exact()
878 		mask = ovsl_dereference(ma->masks[i]);  in ovs_flow_tbl_lookup_exact()
936 	struct mask_array *ma = rcu_dereference_ovsl(table->mask_array);  in ovs_flow_tbl_num_masks()  local
937 	return READ_ONCE(ma->count);  in ovs_flow_tbl_num_masks()
988 	struct mask_array *ma;  in flow_mask_find()  local
991 	ma = ovsl_dereference(tbl->mask_array);  in flow_mask_find()
992 	for (i = 0; i < ma->max; i++) {  in flow_mask_find()
994 		t = ovsl_dereference(ma->masks[i]);  in flow_mask_find()
1105 	struct mask_array *ma = rcu_dereference_ovsl(table->mask_array);  in ovs_flow_masks_rebalance()  local
1112 	masks_and_count = kmalloc_objs(*masks_and_count, ma->max);  in ovs_flow_masks_rebalance()
1116 	for (i = 0; i < ma->max; i++) {  in ovs_flow_masks_rebalance()
1120 		mask = rcu_dereference_ovsl(ma->masks[i]);  in ovs_flow_masks_rebalance()
1132 			stats = per_cpu_ptr(ma->masks_usage_stats, cpu);  in ovs_flow_masks_rebalance()
1142 		masks_and_count[i].counter -= ma->masks_usage_zero_cntr[i];  in ovs_flow_masks_rebalance()
1147 		ma->masks_usage_zero_cntr[i] += masks_and_count[i].counter;  in ovs_flow_masks_rebalance()
1167 	new = tbl_mask_array_alloc(ma->max);  in ovs_flow_masks_rebalance()
1174 		if (ovsl_dereference(ma->masks[index]))  in ovs_flow_masks_rebalance()
1175 			new->masks[new->count++] = ma->masks[index];  in ovs_flow_masks_rebalance()
1179 	call_rcu(&ma->rcu, mask_array_rcu_cb);  in ovs_flow_masks_rebalance()