1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015-2024 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 *
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 */
31
32 #include <sys/cdefs.h>
33 #include "opt_rss.h"
34
35 #include "ena_rss.h"
36
37 /*
38 * This function should generate unique key for the whole driver.
39 * If the key was already genereated in the previous call (for example
40 * for another adapter), then it should be returned instead.
41 */
42 void
ena_rss_key_fill(void * key,size_t size)43 ena_rss_key_fill(void *key, size_t size)
44 {
45 static bool key_generated;
46 static uint8_t default_key[ENA_HASH_KEY_SIZE];
47
48 KASSERT(size <= ENA_HASH_KEY_SIZE,
49 ("Requested more bytes than ENA RSS key can hold"));
50
51 if (!key_generated) {
52 arc4random_buf(default_key, ENA_HASH_KEY_SIZE);
53 key_generated = true;
54 }
55
56 memcpy(key, default_key, size);
57 }
58
59 /*
60 * ENA HW expects the key to be in reverse-byte order.
61 */
62 static void
ena_rss_reorder_hash_key(u8 * reordered_key,const u8 * key,size_t key_size)63 ena_rss_reorder_hash_key(u8 *reordered_key, const u8 *key, size_t key_size)
64 {
65 int i;
66
67 key = key + key_size - 1;
68
69 for (i = 0; i < key_size; ++i)
70 *reordered_key++ = *key--;
71 }
72
73 int
ena_rss_set_hash(struct ena_com_dev * ena_dev,const u8 * key)74 ena_rss_set_hash(struct ena_com_dev *ena_dev, const u8 *key)
75 {
76 enum ena_admin_hash_functions ena_func = ENA_ADMIN_TOEPLITZ;
77 u8 hw_key[ENA_HASH_KEY_SIZE];
78
79 ena_rss_reorder_hash_key(hw_key, key, ENA_HASH_KEY_SIZE);
80
81 return (ena_com_fill_hash_function(ena_dev, ena_func, hw_key,
82 ENA_HASH_KEY_SIZE, 0x0));
83 }
84
85 int
ena_rss_get_hash_key(struct ena_com_dev * ena_dev,u8 * key)86 ena_rss_get_hash_key(struct ena_com_dev *ena_dev, u8 *key)
87 {
88 u8 hw_key[ENA_HASH_KEY_SIZE];
89 int rc;
90
91 rc = ena_com_get_hash_key(ena_dev, hw_key);
92 if (rc != 0)
93 return rc;
94
95 ena_rss_reorder_hash_key(key, hw_key, ENA_HASH_KEY_SIZE);
96
97 return (0);
98 }
99
100 static int
ena_rss_init_default(struct ena_adapter * adapter)101 ena_rss_init_default(struct ena_adapter *adapter)
102 {
103 struct ena_com_dev *ena_dev = adapter->ena_dev;
104 device_t dev = adapter->pdev;
105 int qid, rc, i;
106
107 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
108 if (unlikely(rc != 0)) {
109 ena_log(dev, ERR, "Cannot init indirect table\n");
110 return (rc);
111 }
112
113 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
114 #ifdef RSS
115 qid = rss_get_indirection_to_bucket(i) % adapter->num_io_queues;
116 #else
117 qid = i % adapter->num_io_queues;
118 #endif
119 rc = ena_com_indirect_table_fill_entry(ena_dev, i,
120 ENA_IO_RXQ_IDX(qid));
121 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
122 ena_log(dev, ERR, "Cannot fill indirect table\n");
123 goto err_rss_destroy;
124 }
125 }
126
127
128 uint8_t rss_algo = rss_gethashalgo();
129 if (rss_algo == RSS_HASH_TOEPLITZ) {
130 uint8_t hash_key[RSS_KEYSIZE];
131
132 rss_getkey(hash_key);
133 rc = ena_rss_set_hash(ena_dev, hash_key);
134 } else
135 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ,
136 NULL, ENA_HASH_KEY_SIZE, 0x0);
137 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
138 ena_log(dev, ERR, "Cannot fill hash function\n");
139 goto err_rss_destroy;
140 }
141
142 rc = ena_com_set_default_hash_ctrl(ena_dev);
143 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
144 ena_log(dev, ERR, "Cannot fill hash control\n");
145 goto err_rss_destroy;
146 }
147
148 rc = ena_rss_indir_init(adapter);
149
150 return (rc == EOPNOTSUPP ? 0 : rc);
151
152 err_rss_destroy:
153 ena_com_rss_destroy(ena_dev);
154 return (rc);
155 }
156
157 /* Configure the Rx forwarding */
158 int
ena_rss_configure(struct ena_adapter * adapter)159 ena_rss_configure(struct ena_adapter *adapter)
160 {
161 struct ena_com_dev *ena_dev = adapter->ena_dev;
162 int rc;
163
164 /* In case the RSS table was destroyed */
165 if (!ena_dev->rss.tbl_log_size) {
166 rc = ena_rss_init_default(adapter);
167 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
168 ena_log(adapter->pdev, ERR,
169 "WARNING: RSS was not properly re-initialized,"
170 " it will affect bandwidth\n");
171 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter);
172 return (rc);
173 }
174 }
175
176 /* Set indirect table */
177 rc = ena_com_indirect_table_set(ena_dev);
178 if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
179 return (rc);
180
181 /* Configure hash function (if supported) */
182 rc = ena_com_set_hash_function(ena_dev);
183 if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
184 return (rc);
185
186 /* Configure hash inputs (if supported) */
187 rc = ena_com_set_hash_ctrl(ena_dev);
188 if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
189 return (rc);
190
191 return (0);
192 }
193
194 static void
ena_rss_init_default_deferred(void * arg)195 ena_rss_init_default_deferred(void *arg)
196 {
197 struct ena_adapter *adapter;
198 devclass_t dc;
199 int max;
200 int rc;
201
202 dc = devclass_find("ena");
203 if (unlikely(dc == NULL)) {
204 ena_log_raw(ERR, "SYSINIT: %s: No devclass ena\n", __func__);
205 return;
206 }
207
208 max = devclass_get_maxunit(dc);
209 while (max-- >= 0) {
210 adapter = devclass_get_softc(dc, max);
211 if (adapter != NULL) {
212 rc = ena_rss_init_default(adapter);
213 ENA_FLAG_SET_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter);
214 if (unlikely(rc != 0)) {
215 ena_log(adapter->pdev, WARN,
216 "WARNING: RSS was not properly initialized,"
217 " it will affect bandwidth\n");
218 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE,
219 adapter);
220 }
221 }
222 }
223 }
224 SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND,
225 ena_rss_init_default_deferred, NULL);
226
227 int
ena_rss_indir_get(struct ena_adapter * adapter,uint32_t * table)228 ena_rss_indir_get(struct ena_adapter *adapter, uint32_t *table)
229 {
230 int rc, i;
231
232 rc = ena_com_indirect_table_get(adapter->ena_dev, table);
233 if (rc != 0) {
234 if (rc == EOPNOTSUPP)
235 device_printf(adapter->pdev,
236 "Reading from indirection table not supported\n");
237 else
238 device_printf(adapter->pdev,
239 "Unable to get indirection table\n");
240 return (rc);
241 }
242
243 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i)
244 table[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(table[i]);
245
246 return (0);
247 }
248
249 int
ena_rss_indir_set(struct ena_adapter * adapter,uint32_t * table)250 ena_rss_indir_set(struct ena_adapter *adapter, uint32_t *table)
251 {
252 int rc, i;
253
254 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i) {
255 rc = ena_com_indirect_table_fill_entry(adapter->ena_dev, i,
256 ENA_IO_RXQ_IDX(table[i]));
257 if (rc != 0) {
258 device_printf(adapter->pdev,
259 "Cannot fill indirection table entry %d\n", i);
260 return (rc);
261 }
262 }
263
264 rc = ena_com_indirect_table_set(adapter->ena_dev);
265 if (rc == EOPNOTSUPP)
266 device_printf(adapter->pdev,
267 "Writing to indirection table not supported\n");
268 else if (rc != 0)
269 device_printf(adapter->pdev, "Cannot set indirection table\n");
270
271 return (rc);
272 }
273
274 int
ena_rss_indir_init(struct ena_adapter * adapter)275 ena_rss_indir_init(struct ena_adapter *adapter)
276 {
277 struct ena_indir *indir = adapter->rss_indir;
278 int rc;
279
280 if (indir == NULL)
281 adapter->rss_indir = indir = malloc(sizeof(struct ena_indir),
282 M_DEVBUF, M_WAITOK | M_ZERO);
283
284 rc = ena_rss_indir_get(adapter, indir->table);
285 if (rc != 0) {
286 free(adapter->rss_indir, M_DEVBUF);
287 adapter->rss_indir = NULL;
288
289 return (rc);
290 }
291
292 ena_rss_copy_indir_buf(indir->sysctl_buf, indir->table);
293
294 return (0);
295 }
296