814 	struct nfp_fl_ct_zone_entry *zt = m_entry->zt;  in nfp_fl_ct_add_offload()  local
818 	struct nfp_flower_priv *priv = zt->priv;  in nfp_fl_ct_add_offload()
1175 static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt,  in nfp_ct_do_nft_merge()  argument
1217 	nft_m_entry = get_hashentry(&zt->nft_merge_tb,  in nfp_ct_do_nft_merge()
1230 	nft_m_entry->zt = zt;  in nfp_ct_do_nft_merge()
1243 	err = rhashtable_insert_fast(&zt->nft_merge_tb, &nft_m_entry->hash_node,  in nfp_ct_do_nft_merge()
1248 	zt->nft_merge_count++;  in nfp_ct_do_nft_merge()
1261 	nfp_fl_ct_del_offload(zt->priv->app, nft_m_entry->tc_flower_cookie,  in nfp_ct_do_nft_merge()
1270 static int nfp_ct_do_tc_merge(struct nfp_fl_ct_zone_entry *zt,  in nfp_ct_do_tc_merge()  argument
1300 	m_entry = get_hashentry(&zt->tc_merge_tb, &new_cookie,  in nfp_ct_do_tc_merge()
1310 	m_entry->zt = zt;  in nfp_ct_do_tc_merge()
1319 	err = rhashtable_insert_fast(&zt->tc_merge_tb, &m_entry->hash_node,  in nfp_ct_do_tc_merge()
1323 	zt->tc_merge_count++;  in nfp_ct_do_tc_merge()
1326 	list_for_each_entry_safe(nft_entry, nft_tmp, &zt->nft_flows_list,  in nfp_ct_do_tc_merge()
1328 		nfp_ct_do_nft_merge(zt, nft_entry, m_entry);  in nfp_ct_do_tc_merge()
1344 	struct nfp_fl_ct_zone_entry *zt;  in get_nfp_zone_entry()  local
1351 		zt = get_hashentry(&priv->ct_zone_table, &zone,  in get_nfp_zone_entry()
1352 				   nfp_zone_table_params, sizeof(*zt));  in get_nfp_zone_entry()
1355 		if (IS_ERR(zt) || zt->priv)  in get_nfp_zone_entry()
1356 			return zt;  in get_nfp_zone_entry()
1358 		zt = kzalloc(sizeof(*zt), GFP_KERNEL);  in get_nfp_zone_entry()
1359 		if (!zt)  in get_nfp_zone_entry()
1363 	zt->zone = zone;  in get_nfp_zone_entry()
1364 	zt->priv = priv;  in get_nfp_zone_entry()
1365 	zt->nft = NULL;  in get_nfp_zone_entry()
1368 	INIT_LIST_HEAD(&zt->pre_ct_list);  in get_nfp_zone_entry()
1369 	INIT_LIST_HEAD(&zt->post_ct_list);  in get_nfp_zone_entry()
1370 	INIT_LIST_HEAD(&zt->nft_flows_list);  in get_nfp_zone_entry()
1372 	err = rhashtable_init(&zt->tc_merge_tb, &nfp_tc_ct_merge_params);  in get_nfp_zone_entry()
1376 	err = rhashtable_init(&zt->nft_merge_tb, &nfp_nft_ct_merge_params);  in get_nfp_zone_entry()
1381 		priv->ct_zone_wc = zt;  in get_nfp_zone_entry()
1384 					     &zt->hash_node,  in get_nfp_zone_entry()
1390 	return zt;  in get_nfp_zone_entry()
1393 	rhashtable_destroy(&zt->nft_merge_tb);  in get_nfp_zone_entry()
1395 	rhashtable_destroy(&zt->tc_merge_tb);  in get_nfp_zone_entry()
1397 	kfree(zt);  in get_nfp_zone_entry()
1479 nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt,  in nfp_fl_ct_add_flow()  argument
1527 	entry->zt = zt;  in nfp_fl_ct_add_flow()
1573 	map = get_hashentry(&zt->priv->ct_map_table, &flow->cookie,  in nfp_fl_ct_add_flow()
1583 	err = rhashtable_insert_fast(&zt->priv->ct_map_table,  in nfp_fl_ct_add_flow()
1610 	struct nfp_fl_ct_zone_entry *zt;  in cleanup_nft_merge_entry()  local
1613 	zt = m_entry->zt;  in cleanup_nft_merge_entry()
1617 		err = nfp_fl_ct_del_offload(zt->priv->app, m_entry->tc_flower_cookie,  in cleanup_nft_merge_entry()
1623 	WARN_ON_ONCE(rhashtable_remove_fast(&zt->nft_merge_tb,  in cleanup_nft_merge_entry()
1626 	zt->nft_merge_count--;  in cleanup_nft_merge_entry()
1671 	struct nfp_fl_ct_zone_entry *zt;  in nfp_del_tc_merge_entry()  local
1674 	zt = m_ent->zt;  in nfp_del_tc_merge_entry()
1675 	err = rhashtable_remove_fast(&zt->tc_merge_tb,  in nfp_del_tc_merge_entry()
1680 	zt->tc_merge_count--;  in nfp_del_tc_merge_entry()
1787 			 struct nfp_fl_ct_zone_entry *zt)  in nfp_ct_merge_nft_with_tc()  argument
1792 	rhashtable_walk_enter(&zt->tc_merge_tb, &iter);  in nfp_ct_merge_nft_with_tc()
1798 		nfp_ct_do_nft_merge(zt, nft_entry, tc_merge_entry);  in nfp_ct_merge_nft_with_tc()
1813 	struct nfp_fl_ct_zone_entry *zt;  in nfp_fl_ct_handle_pre_ct()  local
1830 	zt = get_nfp_zone_entry(priv, ct_act->ct.zone, false);  in nfp_fl_ct_handle_pre_ct()
1831 	if (IS_ERR(zt)) {  in nfp_fl_ct_handle_pre_ct()
1834 		return PTR_ERR(zt);  in nfp_fl_ct_handle_pre_ct()
1837 	if (!zt->nft) {  in nfp_fl_ct_handle_pre_ct()
1838 		zt->nft = ct_act->ct.flow_table;  in nfp_fl_ct_handle_pre_ct()
1839 		err = nf_flow_table_offload_add_cb(zt->nft, nfp_fl_ct_handle_nft_flow, zt);  in nfp_fl_ct_handle_pre_ct()
1848 	ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);  in nfp_fl_ct_handle_pre_ct()
1868 	list_add(&ct_entry->list_node, &zt->pre_ct_list);  in nfp_fl_ct_handle_pre_ct()
1869 	zt->pre_ct_count++;  in nfp_fl_ct_handle_pre_ct()
1871 	nfp_ct_merge_tc_entries(ct_entry, zt, zt);  in nfp_fl_ct_handle_pre_ct()
1875 		nfp_ct_merge_tc_entries(ct_entry, priv->ct_zone_wc, zt);  in nfp_fl_ct_handle_pre_ct()
1888 	struct nfp_fl_ct_zone_entry *zt;  in nfp_fl_ct_handle_post_ct()  local
1921 	zt = get_nfp_zone_entry(priv, ct.key->ct_zone, wildcarded);  in nfp_fl_ct_handle_post_ct()
1922 	if (IS_ERR(zt)) {  in nfp_fl_ct_handle_post_ct()
1925 		return PTR_ERR(zt);  in nfp_fl_ct_handle_post_ct()
1929 	ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack);  in nfp_fl_ct_handle_post_ct()
1937 	list_add(&ct_entry->list_node, &zt->post_ct_list);  in nfp_fl_ct_handle_post_ct()
1938 	zt->post_ct_count++;  in nfp_fl_ct_handle_post_ct()
1959 		nfp_ct_merge_tc_entries(ct_entry, zt, zt);  in nfp_fl_ct_handle_post_ct()
1980 	err = nfp_fl_ct_handle_pre_ct(pre_ct_entry->zt->priv,  in nfp_fl_create_new_pre_ct()
1992 	struct nfp_flower_priv *priv = nft_merge->zt->priv;  in nfp_fl_ct_sub_stats()
2073 	spin_lock_bh(&ct_entry->zt->priv->stats_lock);  in nfp_fl_ct_stats()
2141 	spin_unlock_bh(&ct_entry->zt->priv->stats_lock);  in nfp_fl_ct_stats()
2168 nfp_fl_ct_offload_nft_flow(struct nfp_fl_ct_zone_entry *zt, struct flow_cls_offload *flow)  in nfp_fl_ct_offload_nft_flow()  argument
2183 		ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,  in nfp_fl_ct_offload_nft_flow()
2186 			ct_entry = nfp_fl_ct_add_flow(zt, NULL, flow, true, extack);  in nfp_fl_ct_offload_nft_flow()
2190 			list_add(&ct_entry->list_node, &zt->nft_flows_list);  in nfp_fl_ct_offload_nft_flow()
2191 			zt->nft_flows_count++;  in nfp_fl_ct_offload_nft_flow()
2192 			nfp_ct_merge_nft_with_tc(ct_entry, zt);  in nfp_fl_ct_offload_nft_flow()
2196 		ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,  in nfp_fl_ct_offload_nft_flow()
2200 		ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie,  in nfp_fl_ct_offload_nft_flow()
2214 	struct nfp_fl_ct_zone_entry *zt = cb_priv;  in nfp_fl_ct_handle_nft_flow()  local
2219 		while (!mutex_trylock(&zt->priv->nfp_fl_lock)) {  in nfp_fl_ct_handle_nft_flow()
2220 			if (!zt->nft) /* avoid deadlock */  in nfp_fl_ct_handle_nft_flow()
2224 		err = nfp_fl_ct_offload_nft_flow(zt, flow);  in nfp_fl_ct_handle_nft_flow()
2225 		mutex_unlock(&zt->priv->nfp_fl_lock);  in nfp_fl_ct_handle_nft_flow()
2234 nfp_fl_ct_clean_nft_entries(struct nfp_fl_ct_zone_entry *zt)  in nfp_fl_ct_clean_nft_entries()  argument
2239 	list_for_each_entry_safe(nft_entry, ct_tmp, &zt->nft_flows_list,  in nfp_fl_ct_clean_nft_entries()
2241 		ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table,  in nfp_fl_ct_clean_nft_entries()
2251 	struct nfp_fl_ct_zone_entry *zt;  in nfp_fl_ct_del_flow()  local
2258 	zt = ct_map_ent->ct_entry->zt;  in nfp_fl_ct_del_flow()
2260 	m_table = &zt->priv->ct_map_table;  in nfp_fl_ct_del_flow()
2264 		zt->pre_ct_count--;  in nfp_fl_ct_del_flow()
2272 		if (!zt->pre_ct_count && zt->nft) {  in nfp_fl_ct_del_flow()
2273 			nft = zt->nft;  in nfp_fl_ct_del_flow()
2274 			zt->nft = NULL; /* avoid deadlock */  in nfp_fl_ct_del_flow()
2277 						     zt);  in nfp_fl_ct_del_flow()
2278 			nfp_fl_ct_clean_nft_entries(zt);  in nfp_fl_ct_del_flow()
2282 		zt->post_ct_count--;  in nfp_fl_ct_del_flow()
2289 		zt->nft_flows_count--;  in nfp_fl_ct_del_flow()