1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2015-2024 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 * 30 */ 31 32 #include <sys/cdefs.h> 33 #include "opt_rss.h" 34 35 #include "ena_rss.h" 36 37 /* 38 * This function should generate unique key for the whole driver. 39 * If the key was already genereated in the previous call (for example 40 * for another adapter), then it should be returned instead. 41 */ 42 void 43 ena_rss_key_fill(void *key, size_t size) 44 { 45 static bool key_generated; 46 static uint8_t default_key[ENA_HASH_KEY_SIZE]; 47 48 KASSERT(size <= ENA_HASH_KEY_SIZE, 49 ("Requested more bytes than ENA RSS key can hold")); 50 51 if (!key_generated) { 52 arc4random_buf(default_key, ENA_HASH_KEY_SIZE); 53 key_generated = true; 54 } 55 56 memcpy(key, default_key, size); 57 } 58 59 /* 60 * ENA HW expects the key to be in reverse-byte order. 61 */ 62 static void 63 ena_rss_reorder_hash_key(u8 *reordered_key, const u8 *key, size_t key_size) 64 { 65 int i; 66 67 key = key + key_size - 1; 68 69 for (i = 0; i < key_size; ++i) 70 *reordered_key++ = *key--; 71 } 72 73 int 74 ena_rss_set_hash(struct ena_com_dev *ena_dev, const u8 *key) 75 { 76 enum ena_admin_hash_functions ena_func = ENA_ADMIN_TOEPLITZ; 77 u8 hw_key[ENA_HASH_KEY_SIZE]; 78 79 ena_rss_reorder_hash_key(hw_key, key, ENA_HASH_KEY_SIZE); 80 81 return (ena_com_fill_hash_function(ena_dev, ena_func, hw_key, 82 ENA_HASH_KEY_SIZE, 0x0)); 83 } 84 85 int 86 ena_rss_get_hash_key(struct ena_com_dev *ena_dev, u8 *key) 87 { 88 u8 hw_key[ENA_HASH_KEY_SIZE]; 89 int rc; 90 91 rc = ena_com_get_hash_key(ena_dev, hw_key); 92 if (rc != 0) 93 return rc; 94 95 ena_rss_reorder_hash_key(key, hw_key, ENA_HASH_KEY_SIZE); 96 97 return (0); 98 } 99 100 static int 101 ena_rss_init_default(struct ena_adapter *adapter) 102 { 103 struct ena_com_dev *ena_dev = adapter->ena_dev; 104 device_t dev = adapter->pdev; 105 int qid, rc, i; 106 107 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 108 if (unlikely(rc != 0)) { 109 ena_log(dev, ERR, "Cannot init indirect table\n"); 110 return (rc); 111 } 112 113 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 114 #ifdef RSS 115 qid = rss_get_indirection_to_bucket(i) % adapter->num_io_queues; 116 #else 117 qid = i % adapter->num_io_queues; 118 #endif 119 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 120 ENA_IO_RXQ_IDX(qid)); 121 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 122 ena_log(dev, ERR, "Cannot fill indirect table\n"); 123 goto err_rss_destroy; 124 } 125 } 126 127 128 uint8_t rss_algo = rss_gethashalgo(); 129 if (rss_algo == RSS_HASH_TOEPLITZ) { 130 uint8_t hash_key[RSS_KEYSIZE]; 131 132 rss_getkey(hash_key); 133 rc = ena_rss_set_hash(ena_dev, hash_key); 134 } else 135 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, 136 NULL, ENA_HASH_KEY_SIZE, 0x0); 137 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 138 ena_log(dev, ERR, "Cannot fill hash function\n"); 139 goto err_rss_destroy; 140 } 141 142 rc = ena_com_set_default_hash_ctrl(ena_dev); 143 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 144 ena_log(dev, ERR, "Cannot fill hash control\n"); 145 goto err_rss_destroy; 146 } 147 148 rc = ena_rss_indir_init(adapter); 149 150 return (rc == EOPNOTSUPP ? 0 : rc); 151 152 err_rss_destroy: 153 ena_com_rss_destroy(ena_dev); 154 return (rc); 155 } 156 157 /* Configure the Rx forwarding */ 158 int 159 ena_rss_configure(struct ena_adapter *adapter) 160 { 161 struct ena_com_dev *ena_dev = adapter->ena_dev; 162 int rc; 163 164 /* In case the RSS table was destroyed */ 165 if (!ena_dev->rss.tbl_log_size) { 166 rc = ena_rss_init_default(adapter); 167 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 168 ena_log(adapter->pdev, ERR, 169 "WARNING: RSS was not properly re-initialized," 170 " it will affect bandwidth\n"); 171 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); 172 return (rc); 173 } 174 } 175 176 /* Set indirect table */ 177 rc = ena_com_indirect_table_set(ena_dev); 178 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 179 return (rc); 180 181 /* Configure hash function (if supported) */ 182 rc = ena_com_set_hash_function(ena_dev); 183 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 184 return (rc); 185 186 /* Configure hash inputs (if supported) */ 187 rc = ena_com_set_hash_ctrl(ena_dev); 188 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 189 return (rc); 190 191 return (0); 192 } 193 194 static void 195 ena_rss_init_default_deferred(void *arg) 196 { 197 struct ena_adapter *adapter; 198 devclass_t dc; 199 int max; 200 int rc; 201 202 dc = devclass_find("ena"); 203 if (unlikely(dc == NULL)) { 204 ena_log_raw(ERR, "SYSINIT: %s: No devclass ena\n", __func__); 205 return; 206 } 207 208 max = devclass_get_maxunit(dc); 209 while (max-- >= 0) { 210 adapter = devclass_get_softc(dc, max); 211 if (adapter != NULL) { 212 rc = ena_rss_init_default(adapter); 213 ENA_FLAG_SET_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); 214 if (unlikely(rc != 0)) { 215 ena_log(adapter->pdev, WARN, 216 "WARNING: RSS was not properly initialized," 217 " it will affect bandwidth\n"); 218 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, 219 adapter); 220 } 221 } 222 } 223 } 224 SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND, 225 ena_rss_init_default_deferred, NULL); 226 227 int 228 ena_rss_indir_get(struct ena_adapter *adapter, uint32_t *table) 229 { 230 int rc, i; 231 232 rc = ena_com_indirect_table_get(adapter->ena_dev, table); 233 if (rc != 0) { 234 if (rc == EOPNOTSUPP) 235 device_printf(adapter->pdev, 236 "Reading from indirection table not supported\n"); 237 else 238 device_printf(adapter->pdev, 239 "Unable to get indirection table\n"); 240 return (rc); 241 } 242 243 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i) 244 table[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(table[i]); 245 246 return (0); 247 } 248 249 int 250 ena_rss_indir_set(struct ena_adapter *adapter, uint32_t *table) 251 { 252 int rc, i; 253 254 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i) { 255 rc = ena_com_indirect_table_fill_entry(adapter->ena_dev, i, 256 ENA_IO_RXQ_IDX(table[i])); 257 if (rc != 0) { 258 device_printf(adapter->pdev, 259 "Cannot fill indirection table entry %d\n", i); 260 return (rc); 261 } 262 } 263 264 rc = ena_com_indirect_table_set(adapter->ena_dev); 265 if (rc == EOPNOTSUPP) 266 device_printf(adapter->pdev, 267 "Writing to indirection table not supported\n"); 268 else if (rc != 0) 269 device_printf(adapter->pdev, "Cannot set indirection table\n"); 270 271 return (rc); 272 } 273 274 int 275 ena_rss_indir_init(struct ena_adapter *adapter) 276 { 277 struct ena_indir *indir = adapter->rss_indir; 278 int rc; 279 280 if (indir == NULL) 281 adapter->rss_indir = indir = malloc(sizeof(struct ena_indir), 282 M_DEVBUF, M_WAITOK | M_ZERO); 283 284 rc = ena_rss_indir_get(adapter, indir->table); 285 if (rc != 0) { 286 free(adapter->rss_indir, M_DEVBUF); 287 adapter->rss_indir = NULL; 288 289 return (rc); 290 } 291 292 ena_rss_copy_indir_buf(indir->sysctl_buf, indir->table); 293 294 return (0); 295 } 296