1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 * 30 */ 31 32 #include <sys/cdefs.h> 33 #include "opt_rss.h" 34 35 #include "ena_rss.h" 36 37 /* 38 * This function should generate unique key for the whole driver. 39 * If the key was already genereated in the previous call (for example 40 * for another adapter), then it should be returned instead. 41 */ 42 void 43 ena_rss_key_fill(void *key, size_t size) 44 { 45 static bool key_generated; 46 static uint8_t default_key[ENA_HASH_KEY_SIZE]; 47 48 KASSERT(size <= ENA_HASH_KEY_SIZE, 49 ("Requested more bytes than ENA RSS key can hold")); 50 51 if (!key_generated) { 52 arc4random_buf(default_key, ENA_HASH_KEY_SIZE); 53 key_generated = true; 54 } 55 56 memcpy(key, default_key, size); 57 } 58 59 /* 60 * ENA HW expects the key to be in reverse-byte order. 61 */ 62 static void 63 ena_rss_reorder_hash_key(u8 *reordered_key, const u8 *key, size_t key_size) 64 { 65 int i; 66 67 key = key + key_size - 1; 68 69 for (i = 0; i < key_size; ++i) 70 *reordered_key++ = *key--; 71 } 72 73 int 74 ena_rss_set_hash(struct ena_com_dev *ena_dev, const u8 *key) 75 { 76 enum ena_admin_hash_functions ena_func = ENA_ADMIN_TOEPLITZ; 77 u8 hw_key[ENA_HASH_KEY_SIZE]; 78 79 ena_rss_reorder_hash_key(hw_key, key, ENA_HASH_KEY_SIZE); 80 81 return (ena_com_fill_hash_function(ena_dev, ena_func, hw_key, 82 ENA_HASH_KEY_SIZE, 0x0)); 83 } 84 85 int 86 ena_rss_get_hash_key(struct ena_com_dev *ena_dev, u8 *key) 87 { 88 u8 hw_key[ENA_HASH_KEY_SIZE]; 89 int rc; 90 91 rc = ena_com_get_hash_key(ena_dev, hw_key); 92 if (rc != 0) 93 return rc; 94 95 ena_rss_reorder_hash_key(key, hw_key, ENA_HASH_KEY_SIZE); 96 97 return (0); 98 } 99 100 static int 101 ena_rss_init_default(struct ena_adapter *adapter) 102 { 103 struct ena_com_dev *ena_dev = adapter->ena_dev; 104 device_t dev = adapter->pdev; 105 int qid, rc, i; 106 107 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 108 if (unlikely(rc != 0)) { 109 ena_log(dev, ERR, "Cannot init indirect table\n"); 110 return (rc); 111 } 112 113 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 114 #ifdef RSS 115 qid = rss_get_indirection_to_bucket(i) % adapter->num_io_queues; 116 #else 117 qid = i % adapter->num_io_queues; 118 #endif 119 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 120 ENA_IO_RXQ_IDX(qid)); 121 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 122 ena_log(dev, ERR, "Cannot fill indirect table\n"); 123 goto err_rss_destroy; 124 } 125 } 126 127 128 #ifdef RSS 129 uint8_t rss_algo = rss_gethashalgo(); 130 if (rss_algo == RSS_HASH_TOEPLITZ) { 131 uint8_t hash_key[RSS_KEYSIZE]; 132 133 rss_getkey(hash_key); 134 rc = ena_rss_set_hash(ena_dev, hash_key); 135 } else 136 #endif 137 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, 138 NULL, ENA_HASH_KEY_SIZE, 0x0); 139 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 140 ena_log(dev, ERR, "Cannot fill hash function\n"); 141 goto err_rss_destroy; 142 } 143 144 rc = ena_com_set_default_hash_ctrl(ena_dev); 145 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 146 ena_log(dev, ERR, "Cannot fill hash control\n"); 147 goto err_rss_destroy; 148 } 149 150 rc = ena_rss_indir_init(adapter); 151 152 return (rc == EOPNOTSUPP ? 0 : rc); 153 154 err_rss_destroy: 155 ena_com_rss_destroy(ena_dev); 156 return (rc); 157 } 158 159 /* Configure the Rx forwarding */ 160 int 161 ena_rss_configure(struct ena_adapter *adapter) 162 { 163 struct ena_com_dev *ena_dev = adapter->ena_dev; 164 int rc; 165 166 /* In case the RSS table was destroyed */ 167 if (!ena_dev->rss.tbl_log_size) { 168 rc = ena_rss_init_default(adapter); 169 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 170 ena_log(adapter->pdev, ERR, 171 "WARNING: RSS was not properly re-initialized," 172 " it will affect bandwidth\n"); 173 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); 174 return (rc); 175 } 176 } 177 178 /* Set indirect table */ 179 rc = ena_com_indirect_table_set(ena_dev); 180 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 181 return (rc); 182 183 /* Configure hash function (if supported) */ 184 rc = ena_com_set_hash_function(ena_dev); 185 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 186 return (rc); 187 188 /* Configure hash inputs (if supported) */ 189 rc = ena_com_set_hash_ctrl(ena_dev); 190 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 191 return (rc); 192 193 return (0); 194 } 195 196 static void 197 ena_rss_init_default_deferred(void *arg) 198 { 199 struct ena_adapter *adapter; 200 devclass_t dc; 201 int max; 202 int rc; 203 204 dc = devclass_find("ena"); 205 if (unlikely(dc == NULL)) { 206 ena_log_raw(ERR, "SYSINIT: %s: No devclass ena\n", __func__); 207 return; 208 } 209 210 max = devclass_get_maxunit(dc); 211 while (max-- >= 0) { 212 adapter = devclass_get_softc(dc, max); 213 if (adapter != NULL) { 214 rc = ena_rss_init_default(adapter); 215 ENA_FLAG_SET_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); 216 if (unlikely(rc != 0)) { 217 ena_log(adapter->pdev, WARN, 218 "WARNING: RSS was not properly initialized," 219 " it will affect bandwidth\n"); 220 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, 221 adapter); 222 } 223 } 224 } 225 } 226 SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND, 227 ena_rss_init_default_deferred, NULL); 228 229 int 230 ena_rss_indir_get(struct ena_adapter *adapter, uint32_t *table) 231 { 232 int rc, i; 233 234 rc = ena_com_indirect_table_get(adapter->ena_dev, table); 235 if (rc != 0) { 236 if (rc == EOPNOTSUPP) 237 device_printf(adapter->pdev, 238 "Reading from indirection table not supported\n"); 239 else 240 device_printf(adapter->pdev, 241 "Unable to get indirection table\n"); 242 return (rc); 243 } 244 245 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i) 246 table[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(table[i]); 247 248 return (0); 249 } 250 251 int 252 ena_rss_indir_set(struct ena_adapter *adapter, uint32_t *table) 253 { 254 int rc, i; 255 256 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i) { 257 rc = ena_com_indirect_table_fill_entry(adapter->ena_dev, i, 258 ENA_IO_RXQ_IDX(table[i])); 259 if (rc != 0) { 260 device_printf(adapter->pdev, 261 "Cannot fill indirection table entry %d\n", i); 262 return (rc); 263 } 264 } 265 266 rc = ena_com_indirect_table_set(adapter->ena_dev); 267 if (rc == EOPNOTSUPP) 268 device_printf(adapter->pdev, 269 "Writing to indirection table not supported\n"); 270 else if (rc != 0) 271 device_printf(adapter->pdev, "Cannot set indirection table\n"); 272 273 return (rc); 274 } 275 276 int 277 ena_rss_indir_init(struct ena_adapter *adapter) 278 { 279 struct ena_indir *indir = adapter->rss_indir; 280 int rc; 281 282 if (indir == NULL) { 283 adapter->rss_indir = indir = malloc(sizeof(struct ena_indir), 284 M_DEVBUF, M_WAITOK | M_ZERO); 285 if (indir == NULL) 286 return (ENOMEM); 287 } 288 289 rc = ena_rss_indir_get(adapter, indir->table); 290 if (rc != 0) { 291 free(adapter->rss_indir, M_DEVBUF); 292 adapter->rss_indir = NULL; 293 294 return (rc); 295 } 296 297 ena_rss_copy_indir_buf(indir->sysctl_buf, indir->table); 298 299 return (0); 300 } 301