xref: /freebsd/sys/dev/ena/ena_rss.c (revision aa339f1d5df9e38f36a34eb522355c4eebcae6c4)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  *
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  *
30  */
31 
32 #include <sys/cdefs.h>
33 #include "opt_rss.h"
34 
35 #include "ena_rss.h"
36 
37 /*
38  * This function should generate unique key for the whole driver.
39  * If the key was already genereated in the previous call (for example
40  * for another adapter), then it should be returned instead.
41  */
42 void
43 ena_rss_key_fill(void *key, size_t size)
44 {
45 	static bool key_generated;
46 	static uint8_t default_key[ENA_HASH_KEY_SIZE];
47 
48 	KASSERT(size <= ENA_HASH_KEY_SIZE,
49 	    ("Requested more bytes than ENA RSS key can hold"));
50 
51 	if (!key_generated) {
52 		arc4random_buf(default_key, ENA_HASH_KEY_SIZE);
53 		key_generated = true;
54 	}
55 
56 	memcpy(key, default_key, size);
57 }
58 
59 /*
60  * ENA HW expects the key to be in reverse-byte order.
61  */
62 static void
63 ena_rss_reorder_hash_key(u8 *reordered_key, const u8 *key, size_t key_size)
64 {
65 	int i;
66 
67 	key = key + key_size - 1;
68 
69 	for (i = 0; i < key_size; ++i)
70 		*reordered_key++ = *key--;
71 }
72 
73 int
74 ena_rss_set_hash(struct ena_com_dev *ena_dev, const u8 *key)
75 {
76 	enum ena_admin_hash_functions ena_func = ENA_ADMIN_TOEPLITZ;
77 	u8 hw_key[ENA_HASH_KEY_SIZE];
78 
79 	ena_rss_reorder_hash_key(hw_key, key, ENA_HASH_KEY_SIZE);
80 
81 	return (ena_com_fill_hash_function(ena_dev, ena_func, hw_key,
82 	    ENA_HASH_KEY_SIZE, 0x0));
83 }
84 
85 int
86 ena_rss_get_hash_key(struct ena_com_dev *ena_dev, u8 *key)
87 {
88 	u8 hw_key[ENA_HASH_KEY_SIZE];
89 	int rc;
90 
91 	rc = ena_com_get_hash_key(ena_dev, hw_key);
92 	if (rc != 0)
93 		return rc;
94 
95 	ena_rss_reorder_hash_key(key, hw_key, ENA_HASH_KEY_SIZE);
96 
97 	return (0);
98 }
99 
100 static int
101 ena_rss_init_default(struct ena_adapter *adapter)
102 {
103 	struct ena_com_dev *ena_dev = adapter->ena_dev;
104 	device_t dev = adapter->pdev;
105 	int qid, rc, i;
106 
107 	rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
108 	if (unlikely(rc != 0)) {
109 		ena_log(dev, ERR, "Cannot init indirect table\n");
110 		return (rc);
111 	}
112 
113 	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
114 #ifdef RSS
115 		qid = rss_get_indirection_to_bucket(i) % adapter->num_io_queues;
116 #else
117 		qid = i % adapter->num_io_queues;
118 #endif
119 		rc = ena_com_indirect_table_fill_entry(ena_dev, i,
120 		    ENA_IO_RXQ_IDX(qid));
121 		if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
122 			ena_log(dev, ERR, "Cannot fill indirect table\n");
123 			goto err_rss_destroy;
124 		}
125 	}
126 
127 
128 #ifdef RSS
129 	uint8_t rss_algo = rss_gethashalgo();
130 	if (rss_algo == RSS_HASH_TOEPLITZ) {
131 		uint8_t hash_key[RSS_KEYSIZE];
132 
133 		rss_getkey(hash_key);
134 		rc = ena_rss_set_hash(ena_dev, hash_key);
135 	} else
136 #endif
137 		rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ,
138 		    NULL, ENA_HASH_KEY_SIZE, 0x0);
139 	if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
140 		ena_log(dev, ERR, "Cannot fill hash function\n");
141 		goto err_rss_destroy;
142 	}
143 
144 	rc = ena_com_set_default_hash_ctrl(ena_dev);
145 	if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
146 		ena_log(dev, ERR, "Cannot fill hash control\n");
147 		goto err_rss_destroy;
148 	}
149 
150 	rc = ena_rss_indir_init(adapter);
151 
152 	return (rc == EOPNOTSUPP ? 0 : rc);
153 
154 err_rss_destroy:
155 	ena_com_rss_destroy(ena_dev);
156 	return (rc);
157 }
158 
159 /* Configure the Rx forwarding */
160 int
161 ena_rss_configure(struct ena_adapter *adapter)
162 {
163 	struct ena_com_dev *ena_dev = adapter->ena_dev;
164 	int rc;
165 
166 	/* In case the RSS table was destroyed */
167 	if (!ena_dev->rss.tbl_log_size) {
168 		rc = ena_rss_init_default(adapter);
169 		if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
170 			ena_log(adapter->pdev, ERR,
171 			    "WARNING: RSS was not properly re-initialized,"
172 			    " it will affect bandwidth\n");
173 			ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter);
174 			return (rc);
175 		}
176 	}
177 
178 	/* Set indirect table */
179 	rc = ena_com_indirect_table_set(ena_dev);
180 	if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
181 		return (rc);
182 
183 	/* Configure hash function (if supported) */
184 	rc = ena_com_set_hash_function(ena_dev);
185 	if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
186 		return (rc);
187 
188 	/* Configure hash inputs (if supported) */
189 	rc = ena_com_set_hash_ctrl(ena_dev);
190 	if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
191 		return (rc);
192 
193 	return (0);
194 }
195 
196 static void
197 ena_rss_init_default_deferred(void *arg)
198 {
199 	struct ena_adapter *adapter;
200 	devclass_t dc;
201 	int max;
202 	int rc;
203 
204 	dc = devclass_find("ena");
205 	if (unlikely(dc == NULL)) {
206 		ena_log_raw(ERR, "SYSINIT: %s: No devclass ena\n", __func__);
207 		return;
208 	}
209 
210 	max = devclass_get_maxunit(dc);
211 	while (max-- >= 0) {
212 		adapter = devclass_get_softc(dc, max);
213 		if (adapter != NULL) {
214 			rc = ena_rss_init_default(adapter);
215 			ENA_FLAG_SET_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter);
216 			if (unlikely(rc != 0)) {
217 				ena_log(adapter->pdev, WARN,
218 				    "WARNING: RSS was not properly initialized,"
219 				    " it will affect bandwidth\n");
220 				ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE,
221 				    adapter);
222 			}
223 		}
224 	}
225 }
226 SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND,
227     ena_rss_init_default_deferred, NULL);
228 
229 int
230 ena_rss_indir_get(struct ena_adapter *adapter, uint32_t *table)
231 {
232 	int rc, i;
233 
234 	rc = ena_com_indirect_table_get(adapter->ena_dev, table);
235 	if (rc != 0) {
236 		if (rc == EOPNOTSUPP)
237 			device_printf(adapter->pdev,
238 			    "Reading from indirection table not supported\n");
239 		else
240 			device_printf(adapter->pdev,
241 			    "Unable to get indirection table\n");
242 		return (rc);
243 	}
244 
245 	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i)
246 		table[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(table[i]);
247 
248 	return (0);
249 }
250 
251 int
252 ena_rss_indir_set(struct ena_adapter *adapter, uint32_t *table)
253 {
254 	int rc, i;
255 
256 	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i) {
257 		rc = ena_com_indirect_table_fill_entry(adapter->ena_dev, i,
258 		    ENA_IO_RXQ_IDX(table[i]));
259 		if (rc != 0) {
260 			device_printf(adapter->pdev,
261 			    "Cannot fill indirection table entry %d\n", i);
262 			return (rc);
263 		}
264 	}
265 
266 	rc = ena_com_indirect_table_set(adapter->ena_dev);
267 	if (rc == EOPNOTSUPP)
268 		device_printf(adapter->pdev,
269 		    "Writing to indirection table not supported\n");
270 	else if (rc != 0)
271 		device_printf(adapter->pdev, "Cannot set indirection table\n");
272 
273 	return (rc);
274 }
275 
276 int
277 ena_rss_indir_init(struct ena_adapter *adapter)
278 {
279 	struct ena_indir *indir = adapter->rss_indir;
280 	int rc;
281 
282 	if (indir == NULL) {
283 		adapter->rss_indir = indir = malloc(sizeof(struct ena_indir),
284 		    M_DEVBUF, M_WAITOK | M_ZERO);
285 		if (indir == NULL)
286 			return (ENOMEM);
287 	}
288 
289 	rc = ena_rss_indir_get(adapter, indir->table);
290 	if (rc != 0) {
291 		free(adapter->rss_indir, M_DEVBUF);
292 		adapter->rss_indir = NULL;
293 
294 		return (rc);
295 	}
296 
297 	ena_rss_copy_indir_buf(indir->sysctl_buf, indir->table);
298 
299 	return (0);
300 }
301