1 /*-
2 * Copyright (c) 2013-2018, Mellanox Technologies, Ltd. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include "opt_rss.h"
27 #include "opt_ratelimit.h"
28
29 #include <linux/kernel.h>
30 #include <linux/module.h>
31 #include <dev/mlx5/driver.h>
32 #include <dev/mlx5/mlx5_core/mlx5_core.h>
33
34 static int mlx5_relaxed_ordering_write;
35 SYSCTL_INT(_hw_mlx5, OID_AUTO, relaxed_ordering_write, CTLFLAG_RWTUN,
36 &mlx5_relaxed_ordering_write, 0,
37 "Set to enable relaxed ordering for PCIe writes");
38
mlx5_init_mr_table(struct mlx5_core_dev * dev)39 void mlx5_init_mr_table(struct mlx5_core_dev *dev)
40 {
41 struct mlx5_mr_table *table = &dev->priv.mr_table;
42
43 memset(table, 0, sizeof(*table));
44 spin_lock_init(&table->lock);
45 INIT_RADIX_TREE(&table->tree, GFP_ATOMIC);
46 }
47
mlx5_cleanup_mr_table(struct mlx5_core_dev * dev)48 void mlx5_cleanup_mr_table(struct mlx5_core_dev *dev)
49 {
50 }
51
mlx5_core_create_mkey_cb(struct mlx5_core_dev * dev,struct mlx5_core_mkey * mkey,struct mlx5_async_ctx * async_ctx,u32 * in,int inlen,u32 * out,int outlen,mlx5_async_cbk_t callback,struct mlx5_async_work * context)52 int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,
53 struct mlx5_core_mkey *mkey,
54 struct mlx5_async_ctx *async_ctx, u32 *in,
55 int inlen, u32 *out, int outlen,
56 mlx5_async_cbk_t callback,
57 struct mlx5_async_work *context)
58 {
59 struct mlx5_mr_table *table = &dev->priv.mr_table;
60 u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {0};
61 u32 mkey_index;
62 void *mkc;
63 unsigned long flags;
64 int err;
65 u8 key;
66
67 spin_lock_irq(&dev->priv.mkey_lock);
68 key = dev->priv.mkey_key++;
69 spin_unlock_irq(&dev->priv.mkey_lock);
70 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
71 MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
72 MLX5_SET(mkc, mkc, mkey_7_0, key);
73
74 if (mlx5_relaxed_ordering_write != 0) {
75 if (MLX5_CAP_GEN(dev, relaxed_ordering_write))
76 MLX5_SET(mkc, mkc, relaxed_ordering_write, 1);
77 else
78 return (-EPROTONOSUPPORT);
79 }
80
81 if (callback)
82 return mlx5_cmd_exec_cb(async_ctx, in, inlen, out, outlen,
83 callback, context);
84
85 err = mlx5_cmd_exec(dev, in, inlen, lout, sizeof(lout));
86 if (err) {
87 mlx5_core_dbg(dev, "cmd exec failed %d\n", err);
88 return err;
89 }
90
91 mkey_index = MLX5_GET(create_mkey_out, lout, mkey_index);
92 mkey->iova = MLX5_GET64(mkc, mkc, start_addr);
93 mkey->size = MLX5_GET64(mkc, mkc, len);
94 mkey->key = mlx5_idx_to_mkey(mkey_index) | key;
95 mkey->pd = MLX5_GET(mkc, mkc, pd);
96
97 mlx5_core_dbg(dev, "out 0x%x, key 0x%x, mkey 0x%x\n",
98 mkey_index, key, mkey->key);
99
100 /* connect to MR tree */
101 spin_lock_irqsave(&table->lock, flags);
102 err = radix_tree_insert(&table->tree, mlx5_mkey_to_idx(mkey->key), mkey);
103 spin_unlock_irqrestore(&table->lock, flags);
104 if (err) {
105 mlx5_core_warn(dev, "failed radix tree insert of mr 0x%x, %d\n",
106 mkey->key, err);
107 mlx5_core_destroy_mkey(dev, mkey);
108 }
109
110 return err;
111 }
112 EXPORT_SYMBOL(mlx5_core_create_mkey_cb);
113
mlx5_core_create_mkey(struct mlx5_core_dev * dev,struct mlx5_core_mkey * mkey,u32 * in,int inlen)114 int mlx5_core_create_mkey(struct mlx5_core_dev *dev,
115 struct mlx5_core_mkey *mkey,
116 u32 *in, int inlen)
117 {
118 return mlx5_core_create_mkey_cb(dev, mkey, NULL, in, inlen,
119 NULL, 0, NULL, NULL);
120 }
121 EXPORT_SYMBOL(mlx5_core_create_mkey);
122
mlx5_core_destroy_mkey(struct mlx5_core_dev * dev,struct mlx5_core_mkey * mkey)123 int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *mkey)
124 {
125 struct mlx5_mr_table *table = &dev->priv.mr_table;
126 u32 out[MLX5_ST_SZ_DW(destroy_mkey_out)] = {0};
127 u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {0};
128 struct mlx5_core_mkey *deleted_mr;
129 unsigned long flags;
130
131 spin_lock_irqsave(&table->lock, flags);
132 deleted_mr = radix_tree_delete(&table->tree, mlx5_mkey_to_idx(mkey->key));
133 spin_unlock_irqrestore(&table->lock, flags);
134 if (!deleted_mr) {
135 mlx5_core_warn(dev, "failed radix tree delete of mr 0x%x\n", mkey->key);
136 return -ENOENT;
137 }
138
139 MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY);
140 MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey->key));
141
142 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
143 }
144 EXPORT_SYMBOL(mlx5_core_destroy_mkey);
145
mlx5_core_query_mkey(struct mlx5_core_dev * dev,struct mlx5_core_mkey * mkey,u32 * out,int outlen)146 int mlx5_core_query_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *mkey,
147 u32 *out, int outlen)
148 {
149 u32 in[MLX5_ST_SZ_DW(query_mkey_in)] = {0};
150
151 memset(out, 0, outlen);
152 MLX5_SET(query_mkey_in, in, opcode, MLX5_CMD_OP_QUERY_MKEY);
153 MLX5_SET(query_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey->key));
154
155 return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
156 }
157 EXPORT_SYMBOL(mlx5_core_query_mkey);
158
mlx5_core_dump_fill_mkey(struct mlx5_core_dev * dev,struct mlx5_core_mkey * _mkey,u32 * mkey)159 int mlx5_core_dump_fill_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *_mkey,
160 u32 *mkey)
161 {
162 u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {0};
163 u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {0};
164 int err;
165
166 MLX5_SET(query_special_contexts_in, in, opcode,
167 MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
168 err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
169 if (!err)
170 *mkey = MLX5_GET(query_special_contexts_out, out, dump_fill_mkey);
171
172 return err;
173 }
174 EXPORT_SYMBOL(mlx5_core_dump_fill_mkey);
175
mlx5_get_psv(u32 * out,int psv_index)176 static inline u32 mlx5_get_psv(u32 *out, int psv_index)
177 {
178 switch (psv_index) {
179 case 1: return MLX5_GET(create_psv_out, out, psv1_index);
180 case 2: return MLX5_GET(create_psv_out, out, psv2_index);
181 case 3: return MLX5_GET(create_psv_out, out, psv3_index);
182 default: return MLX5_GET(create_psv_out, out, psv0_index);
183 }
184 }
185
mlx5_core_create_psv(struct mlx5_core_dev * dev,u32 pdn,int npsvs,u32 * sig_index)186 int mlx5_core_create_psv(struct mlx5_core_dev *dev, u32 pdn,
187 int npsvs, u32 *sig_index)
188 {
189 u32 out[MLX5_ST_SZ_DW(create_psv_out)] = {0};
190 u32 in[MLX5_ST_SZ_DW(create_psv_in)] = {0};
191 int i, err;
192
193 if (npsvs > MLX5_MAX_PSVS)
194 return -EINVAL;
195
196 MLX5_SET(create_psv_in, in, opcode, MLX5_CMD_OP_CREATE_PSV);
197 MLX5_SET(create_psv_in, in, pd, pdn);
198 MLX5_SET(create_psv_in, in, num_psv, npsvs);
199 err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
200 if (err) {
201 mlx5_core_err(dev, "create_psv cmd exec failed %d\n", err);
202 return err;
203 }
204
205 for (i = 0; i < npsvs; i++)
206 sig_index[i] = mlx5_get_psv(out, i);
207
208 return err;
209 }
210 EXPORT_SYMBOL(mlx5_core_create_psv);
211
mlx5_core_destroy_psv(struct mlx5_core_dev * dev,int psv_num)212 int mlx5_core_destroy_psv(struct mlx5_core_dev *dev, int psv_num)
213 {
214 u32 out[MLX5_ST_SZ_DW(destroy_psv_out)] = {0};
215 u32 in[MLX5_ST_SZ_DW(destroy_psv_in)] = {0};
216
217 MLX5_SET(destroy_psv_in, in, opcode, MLX5_CMD_OP_DESTROY_PSV);
218 MLX5_SET(destroy_psv_in, in, psvn, psv_num);
219 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
220 }
221 EXPORT_SYMBOL(mlx5_core_destroy_psv);
222