1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */ 3 4 #include "tir.h" 5 #include "params.h" 6 #include <linux/mlx5/transobj.h> 7 8 #define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ (64 * 1024) 9 10 /* max() doesn't work inside square brackets. */ 11 #define MLX5E_TIR_CMD_IN_SZ_DW ( \ 12 MLX5_ST_SZ_DW(create_tir_in) > MLX5_ST_SZ_DW(modify_tir_in) ? \ 13 MLX5_ST_SZ_DW(create_tir_in) : MLX5_ST_SZ_DW(modify_tir_in) \ 14 ) 15 16 struct mlx5e_tir_builder { 17 u32 in[MLX5E_TIR_CMD_IN_SZ_DW]; 18 bool modify; 19 }; 20 21 struct mlx5e_tir_builder *mlx5e_tir_builder_alloc(bool modify) 22 { 23 struct mlx5e_tir_builder *builder; 24 25 builder = kvzalloc(sizeof(*builder), GFP_KERNEL); 26 if (!builder) 27 return NULL; 28 29 builder->modify = modify; 30 31 return builder; 32 } 33 34 void mlx5e_tir_builder_free(struct mlx5e_tir_builder *builder) 35 { 36 kvfree(builder); 37 } 38 39 void mlx5e_tir_builder_clear(struct mlx5e_tir_builder *builder) 40 { 41 memset(builder->in, 0, sizeof(builder->in)); 42 } 43 44 static void *mlx5e_tir_builder_get_tirc(struct mlx5e_tir_builder *builder) 45 { 46 if (builder->modify) 47 return MLX5_ADDR_OF(modify_tir_in, builder->in, ctx); 48 return MLX5_ADDR_OF(create_tir_in, builder->in, ctx); 49 } 50 51 void mlx5e_tir_builder_build_inline(struct mlx5e_tir_builder *builder, u32 tdn, u32 rqn) 52 { 53 void *tirc = mlx5e_tir_builder_get_tirc(builder); 54 55 WARN_ON(builder->modify); 56 57 MLX5_SET(tirc, tirc, transport_domain, tdn); 58 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT); 59 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_NONE); 60 MLX5_SET(tirc, tirc, inline_rqn, rqn); 61 } 62 63 void mlx5e_tir_builder_build_rqt(struct mlx5e_tir_builder *builder, u32 tdn, 64 u32 rqtn, bool inner_ft_support) 65 { 66 void *tirc = mlx5e_tir_builder_get_tirc(builder); 67 68 WARN_ON(builder->modify); 69 70 MLX5_SET(tirc, tirc, transport_domain, tdn); 71 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 72 MLX5_SET(tirc, tirc, indirect_table, rqtn); 73 MLX5_SET(tirc, tirc, tunneled_offload_en, inner_ft_support); 74 } 75 76 void mlx5e_tir_builder_build_packet_merge(struct mlx5e_tir_builder *builder, 77 const struct mlx5e_packet_merge_param *pkt_merge_param) 78 { 79 void *tirc = mlx5e_tir_builder_get_tirc(builder); 80 const unsigned int rough_max_l2_l3_hdr_sz = 256; 81 82 if (builder->modify) 83 MLX5_SET(modify_tir_in, builder->in, bitmask.packet_merge, 1); 84 85 switch (pkt_merge_param->type) { 86 case MLX5E_PACKET_MERGE_LRO: 87 MLX5_SET(tirc, tirc, packet_merge_mask, 88 MLX5_TIRC_PACKET_MERGE_MASK_IPV4_LRO | 89 MLX5_TIRC_PACKET_MERGE_MASK_IPV6_LRO); 90 MLX5_SET(tirc, tirc, lro_max_ip_payload_size, 91 (MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ - rough_max_l2_l3_hdr_sz) >> 8); 92 MLX5_SET(tirc, tirc, lro_timeout_period_usecs, pkt_merge_param->timeout); 93 break; 94 default: 95 break; 96 } 97 } 98 99 static int mlx5e_hfunc_to_hw(u8 hfunc) 100 { 101 switch (hfunc) { 102 case ETH_RSS_HASH_TOP: 103 return MLX5_RX_HASH_FN_TOEPLITZ; 104 case ETH_RSS_HASH_XOR: 105 return MLX5_RX_HASH_FN_INVERTED_XOR8; 106 default: 107 return MLX5_RX_HASH_FN_NONE; 108 } 109 } 110 111 void mlx5e_tir_builder_build_rss(struct mlx5e_tir_builder *builder, 112 const struct mlx5e_rss_params_hash *rss_hash, 113 const struct mlx5e_rss_params_traffic_type *rss_tt, 114 bool inner) 115 { 116 void *tirc = mlx5e_tir_builder_get_tirc(builder); 117 void *hfso; 118 119 if (builder->modify) 120 MLX5_SET(modify_tir_in, builder->in, bitmask.hash, 1); 121 122 MLX5_SET(tirc, tirc, rx_hash_fn, mlx5e_hfunc_to_hw(rss_hash->hfunc)); 123 if (rss_hash->hfunc == ETH_RSS_HASH_TOP) { 124 const size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key); 125 void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key); 126 127 MLX5_SET(tirc, tirc, rx_hash_symmetric, rss_hash->symmetric); 128 memcpy(rss_key, rss_hash->toeplitz_hash_key, len); 129 } 130 131 if (inner) 132 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner); 133 else 134 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 135 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, rss_tt->l3_prot_type); 136 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, rss_tt->l4_prot_type); 137 MLX5_SET(rx_hash_field_select, hfso, selected_fields, rss_tt->rx_hash_fields); 138 } 139 140 void mlx5e_tir_builder_build_direct(struct mlx5e_tir_builder *builder) 141 { 142 void *tirc = mlx5e_tir_builder_get_tirc(builder); 143 144 WARN_ON(builder->modify); 145 146 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8); 147 } 148 149 static void mlx5e_tir_context_self_lb_block(void *tirc, bool enable_uc_lb, 150 bool enable_mc_lb) 151 { 152 u8 lb_flags = 0; 153 154 if (enable_uc_lb) 155 lb_flags = MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST; 156 if (enable_mc_lb) 157 lb_flags |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST; 158 159 MLX5_SET(tirc, tirc, self_lb_block, lb_flags); 160 } 161 162 void mlx5e_tir_builder_build_self_lb_block(struct mlx5e_tir_builder *builder, 163 bool enable_uc_lb, 164 bool enable_mc_lb) 165 { 166 void *tirc = mlx5e_tir_builder_get_tirc(builder); 167 168 if (builder->modify) 169 MLX5_SET(modify_tir_in, builder->in, bitmask.self_lb_en, 1); 170 171 mlx5e_tir_context_self_lb_block(tirc, enable_uc_lb, enable_mc_lb); 172 } 173 174 void mlx5e_tir_builder_build_tls(struct mlx5e_tir_builder *builder) 175 { 176 void *tirc = mlx5e_tir_builder_get_tirc(builder); 177 178 WARN_ON(builder->modify); 179 180 MLX5_SET(tirc, tirc, tls_en, 1); 181 mlx5e_tir_context_self_lb_block(tirc, true, true); 182 } 183 184 int mlx5e_tir_init(struct mlx5e_tir *tir, struct mlx5e_tir_builder *builder, 185 struct mlx5_core_dev *mdev, bool reg) 186 { 187 int err; 188 189 tir->mdev = mdev; 190 191 err = mlx5_core_create_tir(tir->mdev, builder->in, &tir->tirn); 192 if (err) 193 return err; 194 195 if (reg) { 196 struct mlx5e_hw_objs *res = &tir->mdev->mlx5e_res.hw_objs; 197 198 mutex_lock(&res->td.list_lock); 199 list_add(&tir->list, &res->td.tirs_list); 200 mutex_unlock(&res->td.list_lock); 201 } else { 202 INIT_LIST_HEAD(&tir->list); 203 } 204 205 return 0; 206 } 207 208 void mlx5e_tir_destroy(struct mlx5e_tir *tir) 209 { 210 struct mlx5e_hw_objs *res = &tir->mdev->mlx5e_res.hw_objs; 211 212 /* Skip mutex if list_del is no-op (the TIR wasn't registered in the 213 * list). list_empty will never return true for an item of tirs_list, 214 * and READ_ONCE/WRITE_ONCE in list_empty/list_del guarantee consistency 215 * of the list->next value. 216 */ 217 if (!list_empty(&tir->list)) { 218 mutex_lock(&res->td.list_lock); 219 list_del(&tir->list); 220 mutex_unlock(&res->td.list_lock); 221 } 222 223 mlx5_core_destroy_tir(tir->mdev, tir->tirn); 224 } 225 226 int mlx5e_tir_modify(struct mlx5e_tir *tir, struct mlx5e_tir_builder *builder) 227 { 228 return mlx5_core_modify_tir(tir->mdev, tir->tirn, builder->in); 229 } 230