1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 * 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_rss.h" 36 37 #include "ena_rss.h" 38 39 /* 40 * This function should generate unique key for the whole driver. 41 * If the key was already genereated in the previous call (for example 42 * for another adapter), then it should be returned instead. 43 */ 44 void 45 ena_rss_key_fill(void *key, size_t size) 46 { 47 static bool key_generated; 48 static uint8_t default_key[ENA_HASH_KEY_SIZE]; 49 50 KASSERT(size <= ENA_HASH_KEY_SIZE, 51 ("Requested more bytes than ENA RSS key can hold")); 52 53 if (!key_generated) { 54 arc4random_buf(default_key, ENA_HASH_KEY_SIZE); 55 key_generated = true; 56 } 57 58 memcpy(key, default_key, size); 59 } 60 61 /* 62 * ENA HW expects the key to be in reverse-byte order. 63 */ 64 static void 65 ena_rss_reorder_hash_key(u8 *reordered_key, const u8 *key, size_t key_size) 66 { 67 int i; 68 69 key = key + key_size - 1; 70 71 for (i = 0; i < key_size; ++i) 72 *reordered_key++ = *key--; 73 } 74 75 int 76 ena_rss_set_hash(struct ena_com_dev *ena_dev, const u8 *key) 77 { 78 enum ena_admin_hash_functions ena_func = ENA_ADMIN_TOEPLITZ; 79 u8 hw_key[ENA_HASH_KEY_SIZE]; 80 81 ena_rss_reorder_hash_key(hw_key, key, ENA_HASH_KEY_SIZE); 82 83 return (ena_com_fill_hash_function(ena_dev, ena_func, hw_key, 84 ENA_HASH_KEY_SIZE, 0x0)); 85 } 86 87 int 88 ena_rss_get_hash_key(struct ena_com_dev *ena_dev, u8 *key) 89 { 90 u8 hw_key[ENA_HASH_KEY_SIZE]; 91 int rc; 92 93 rc = ena_com_get_hash_key(ena_dev, hw_key); 94 if (rc != 0) 95 return rc; 96 97 ena_rss_reorder_hash_key(key, hw_key, ENA_HASH_KEY_SIZE); 98 99 return (0); 100 } 101 102 static int 103 ena_rss_init_default(struct ena_adapter *adapter) 104 { 105 struct ena_com_dev *ena_dev = adapter->ena_dev; 106 device_t dev = adapter->pdev; 107 int qid, rc, i; 108 109 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 110 if (unlikely(rc != 0)) { 111 ena_log(dev, ERR, "Cannot init indirect table\n"); 112 return (rc); 113 } 114 115 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 116 #ifdef RSS 117 qid = rss_get_indirection_to_bucket(i) % adapter->num_io_queues; 118 #else 119 qid = i % adapter->num_io_queues; 120 #endif 121 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 122 ENA_IO_RXQ_IDX(qid)); 123 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 124 ena_log(dev, ERR, "Cannot fill indirect table\n"); 125 goto err_rss_destroy; 126 } 127 } 128 129 130 #ifdef RSS 131 uint8_t rss_algo = rss_gethashalgo(); 132 if (rss_algo == RSS_HASH_TOEPLITZ) { 133 uint8_t hash_key[RSS_KEYSIZE]; 134 135 rss_getkey(hash_key); 136 rc = ena_rss_set_hash(ena_dev, hash_key); 137 } else 138 #endif 139 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, 140 NULL, ENA_HASH_KEY_SIZE, 0x0); 141 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 142 ena_log(dev, ERR, "Cannot fill hash function\n"); 143 goto err_rss_destroy; 144 } 145 146 rc = ena_com_set_default_hash_ctrl(ena_dev); 147 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 148 ena_log(dev, ERR, "Cannot fill hash control\n"); 149 goto err_rss_destroy; 150 } 151 152 rc = ena_rss_indir_init(adapter); 153 154 return (rc == EOPNOTSUPP ? 0 : rc); 155 156 err_rss_destroy: 157 ena_com_rss_destroy(ena_dev); 158 return (rc); 159 } 160 161 /* Configure the Rx forwarding */ 162 int 163 ena_rss_configure(struct ena_adapter *adapter) 164 { 165 struct ena_com_dev *ena_dev = adapter->ena_dev; 166 int rc; 167 168 /* In case the RSS table was destroyed */ 169 if (!ena_dev->rss.tbl_log_size) { 170 rc = ena_rss_init_default(adapter); 171 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 172 ena_log(adapter->pdev, ERR, 173 "WARNING: RSS was not properly re-initialized," 174 " it will affect bandwidth\n"); 175 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); 176 return (rc); 177 } 178 } 179 180 /* Set indirect table */ 181 rc = ena_com_indirect_table_set(ena_dev); 182 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 183 return (rc); 184 185 /* Configure hash function (if supported) */ 186 rc = ena_com_set_hash_function(ena_dev); 187 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 188 return (rc); 189 190 /* Configure hash inputs (if supported) */ 191 rc = ena_com_set_hash_ctrl(ena_dev); 192 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 193 return (rc); 194 195 return (0); 196 } 197 198 static void 199 ena_rss_init_default_deferred(void *arg) 200 { 201 struct ena_adapter *adapter; 202 devclass_t dc; 203 int max; 204 int rc; 205 206 dc = devclass_find("ena"); 207 if (unlikely(dc == NULL)) { 208 ena_log_raw(ERR, "SYSINIT: %s: No devclass ena\n", __func__); 209 return; 210 } 211 212 max = devclass_get_maxunit(dc); 213 while (max-- >= 0) { 214 adapter = devclass_get_softc(dc, max); 215 if (adapter != NULL) { 216 rc = ena_rss_init_default(adapter); 217 ENA_FLAG_SET_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); 218 if (unlikely(rc != 0)) { 219 ena_log(adapter->pdev, WARN, 220 "WARNING: RSS was not properly initialized," 221 " it will affect bandwidth\n"); 222 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, 223 adapter); 224 } 225 } 226 } 227 } 228 SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND, 229 ena_rss_init_default_deferred, NULL); 230 231 int 232 ena_rss_indir_get(struct ena_adapter *adapter, uint32_t *table) 233 { 234 int rc, i; 235 236 rc = ena_com_indirect_table_get(adapter->ena_dev, table); 237 if (rc != 0) { 238 if (rc == EOPNOTSUPP) 239 device_printf(adapter->pdev, 240 "Reading from indirection table not supported\n"); 241 else 242 device_printf(adapter->pdev, 243 "Unable to get indirection table\n"); 244 return (rc); 245 } 246 247 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i) 248 table[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(table[i]); 249 250 return (0); 251 } 252 253 int 254 ena_rss_indir_set(struct ena_adapter *adapter, uint32_t *table) 255 { 256 int rc, i; 257 258 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i) { 259 rc = ena_com_indirect_table_fill_entry(adapter->ena_dev, i, 260 ENA_IO_RXQ_IDX(table[i])); 261 if (rc != 0) { 262 device_printf(adapter->pdev, 263 "Cannot fill indirection table entry %d\n", i); 264 return (rc); 265 } 266 } 267 268 rc = ena_com_indirect_table_set(adapter->ena_dev); 269 if (rc == EOPNOTSUPP) 270 device_printf(adapter->pdev, 271 "Writing to indirection table not supported\n"); 272 else if (rc != 0) 273 device_printf(adapter->pdev, "Cannot set indirection table\n"); 274 275 return (rc); 276 } 277 278 int 279 ena_rss_indir_init(struct ena_adapter *adapter) 280 { 281 struct ena_indir *indir = adapter->rss_indir; 282 int rc; 283 284 if (indir == NULL) { 285 adapter->rss_indir = indir = malloc(sizeof(struct ena_indir), 286 M_DEVBUF, M_WAITOK | M_ZERO); 287 if (indir == NULL) 288 return (ENOMEM); 289 } 290 291 rc = ena_rss_indir_get(adapter, indir->table); 292 if (rc != 0) { 293 free(adapter->rss_indir, M_DEVBUF); 294 adapter->rss_indir = NULL; 295 296 return (rc); 297 } 298 299 ena_rss_copy_indir_buf(indir->sysctl_buf, indir->table); 300 301 return (0); 302 } 303