589 nfp_flower_update_merge_with_actions(struct nfp_fl_payload *flow,  in nfp_flower_update_merge_with_actions()  argument
608 	while (act_off < flow->meta.act_len) {  in nfp_flower_update_merge_with_actions()
609 		a = (struct nfp_fl_act_head *)&flow->action_data[act_off];  in nfp_flower_update_merge_with_actions()
705 nfp_flower_populate_merge_match(struct nfp_fl_payload *flow,  in nfp_flower_populate_merge_match()  argument
710 	u8 *mask = flow->mask_data;  in nfp_flower_populate_merge_match()
908 	/* Copy any pre-actions to the start of merge flow action list. */  in nfp_flower_merge_action()
957 /* Flow link code should only be accessed under RTNL. */
971 		if (link->sub_flow.flow == sub_flow) {  in nfp_flower_unlink_flows()
986 	link->merge_flow.flow = merge_flow;  in nfp_flower_link_flows()
988 	link->sub_flow.flow = sub_flow;  in nfp_flower_link_flows()
995  * nfp_flower_merge_offloaded_flows() - Merge 2 existing flows to single flow.
997  * @sub_flow1:	Initial flow matched to produce merge hint
998  * @sub_flow2:	Post recirculation flow matched in merge hint
1000  * Combines 2 flows (if valid) to a single flow, removing the initial from hw
1001  * and offloading the new, merged flow.
1118  * @flow:	Pointer to NFP flow representation of rule
1122  * Verifies the flow as a pre-tunnel rule.
1128 				 struct nfp_fl_payload *flow,  in nfp_flower_validate_pre_tun_rule()  argument
1135 	u8 *ext = flow->unmasked_data;  in nfp_flower_validate_pre_tun_rule()
1137 	u8 *mask = flow->mask_data;  in nfp_flower_validate_pre_tun_rule()
1142 	meta_tci = (struct nfp_flower_meta_tci *)flow->unmasked_data;  in nfp_flower_validate_pre_tun_rule()
1149 			flow->pre_tun_rule.vlan_tci = cpu_to_be16(vlan_tci);  in nfp_flower_validate_pre_tun_rule()
1152 			flow->pre_tun_rule.vlan_tci = cpu_to_be16(0xffff);  in nfp_flower_validate_pre_tun_rule()
1176 		flow->pre_tun_rule.is_ipv6 = true;  in nfp_flower_validate_pre_tun_rule()
1178 		flow->pre_tun_rule.is_ipv6 = false;  in nfp_flower_validate_pre_tun_rule()
1217 	if (memcmp(&mac->mac_dst[0], flow->pre_tun_rule.dev->dev_addr, 6)) {  in nfp_flower_validate_pre_tun_rule()
1224 	memcpy(&flow->pre_tun_rule.loc_mac, &mac->mac_dst[0], ETH_ALEN);  in nfp_flower_validate_pre_tun_rule()
1225 	memcpy(&flow->pre_tun_rule.rem_mac, &mac->mac_src[0], ETH_ALEN);  in nfp_flower_validate_pre_tun_rule()
1264 			flow->pre_tun_rule.vlan_tci = cpu_to_be16(vlan_tci);  in nfp_flower_validate_pre_tun_rule()
1265 			flow->pre_tun_rule.vlan_tpid = cpu_to_be16(vlan_tpid);  in nfp_flower_validate_pre_tun_rule()
1268 			flow->pre_tun_rule.vlan_tci = cpu_to_be16(0xffff);  in nfp_flower_validate_pre_tun_rule()
1269 			flow->pre_tun_rule.vlan_tpid = cpu_to_be16(0xffff);  in nfp_flower_validate_pre_tun_rule()
1275 	act = (struct nfp_fl_act_head *)&flow->action_data[act_offset];  in nfp_flower_validate_pre_tun_rule()
1283 		act = (struct nfp_fl_act_head *)&flow->action_data[act_offset];  in nfp_flower_validate_pre_tun_rule()
1294 	if (act_offset != flow->meta.act_len) {  in nfp_flower_validate_pre_tun_rule()
1302 static bool offload_pre_check(struct flow_cls_offload *flow)  in offload_pre_check()  argument
1304 	struct flow_rule *rule = flow_cls_offload_flow_rule(flow);  in offload_pre_check()
1315 	if (flow->common.chain_index)  in offload_pre_check()
1322  * nfp_flower_add_offload() - Adds a new flow to hardware.
1325  * @flow:	TC flower classifier offload structure.
1327  * Adds a new flow to the repeated hash structure and action payload.
1333 		       struct flow_cls_offload *flow)  in nfp_flower_add_offload()  argument
1335 	struct flow_rule *rule = flow_cls_offload_flow_rule(flow);  in nfp_flower_add_offload()
1344 	extack = flow->common.extack;  in nfp_flower_add_offload()
1348 	if (is_pre_ct_flow(flow))  in nfp_flower_add_offload()
1349 		return nfp_fl_ct_handle_pre_ct(priv, netdev, flow, extack, NULL);  in nfp_flower_add_offload()
1351 	if (is_post_ct_flow(flow))  in nfp_flower_add_offload()
1352 		return nfp_fl_ct_handle_post_ct(priv, netdev, flow, extack);  in nfp_flower_add_offload()
1354 	if (!offload_pre_check(flow))  in nfp_flower_add_offload()
1387 	err = nfp_compile_flow_metadata(app, flow->cookie, flow_pay, netdev, extack);  in nfp_flower_add_offload()
1391 	flow_pay->tc_flower_cookie = flow->cookie;  in nfp_flower_add_offload()
1395 		NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot insert flow into tables for offloads");  in nfp_flower_add_offload()
1431 	/* Deallocate flow payload when flower rule has been destroyed. */  in nfp_flower_add_offload()
1469 	origin = link->sub_flow.flow;  in nfp_flower_remove_merge_flow()
1477 		nfp_flower_cmsg_warn(app, "Metadata fail for merge flow delete.\n");  in nfp_flower_remove_merge_flow()
1485 			nfp_flower_cmsg_warn(app, "Failed to delete merged flow.\n");  in nfp_flower_remove_merge_flow()
1493 			nfp_flower_cmsg_warn(app, "Failed to revert merge flow.\n");  in nfp_flower_remove_merge_flow()
1498 	/* Clean any links connected with the merged flow. */  in nfp_flower_remove_merge_flow()
1501 		u32 ctx_id = be32_to_cpu(link->sub_flow.flow->meta.host_ctx_id);  in nfp_flower_remove_merge_flow()
1532 	/* Remove any merge flow formed from the deleted sub_flow. */  in nfp_flower_del_linked_merge_flows()
1536 					     link->merge_flow.flow);  in nfp_flower_del_linked_merge_flows()
1540  * nfp_flower_del_offload() - Removes a flow from hardware.
1543  * @flow:	TC flower classifier offload structure
1545  * Removes a flow from the repeated hash structure and clears the
1552 		       struct flow_cls_offload *flow)  in nfp_flower_del_offload()  argument
1561 	extack = flow->common.extack;  in nfp_flower_del_offload()
1566 	ct_map_ent = rhashtable_lookup_fast(&priv->ct_map_table, &flow->cookie,  in nfp_flower_del_offload()
1573 	nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, netdev);  in nfp_flower_del_offload()
1575 		NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot remove flow that does not exist");  in nfp_flower_del_offload()
1647 	/* Reset stats for the merge flow. */  in __nfp_flower_update_merge_stats()
1651 	/* The merge flow has received stats updates from firmware.  in __nfp_flower_update_merge_stats()
1656 		sub_flow = link->sub_flow.flow;  in __nfp_flower_update_merge_stats()
1673 		__nfp_flower_update_merge_stats(app, link->merge_flow.flow);  in nfp_flower_update_merge_stats()
1677  * nfp_flower_get_stats() - Populates flow stats obtained from hardware.
1680  * @flow:	TC flower classifier offload structure
1682  * Populates a flow statistics structure which which corresponds to a
1683  * specific flow.
1689 		     struct flow_cls_offload *flow)  in nfp_flower_get_stats()  argument
1698 	ct_map_ent = rhashtable_lookup_fast(&priv->ct_map_table, &flow->cookie,  in nfp_flower_get_stats()
1701 		return nfp_fl_ct_stats(flow, ct_map_ent);  in nfp_flower_get_stats()
1703 	extack = flow->common.extack;  in nfp_flower_get_stats()
1704 	nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, netdev);  in nfp_flower_get_stats()
1706 		NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot dump stats for flow that does not exist");  in nfp_flower_get_stats()
1717 	flow_stats_update(&flow->stats, priv->stats[ctx_id].bytes,  in nfp_flower_get_stats()