1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 * 30 * $FreeBSD$ 31 * 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include "opt_rss.h" 38 39 #include "ena_rss.h" 40 41 /* 42 * This function should generate unique key for the whole driver. 43 * If the key was already genereated in the previous call (for example 44 * for another adapter), then it should be returned instead. 45 */ 46 void 47 ena_rss_key_fill(void *key, size_t size) 48 { 49 static bool key_generated; 50 static uint8_t default_key[ENA_HASH_KEY_SIZE]; 51 52 KASSERT(size <= ENA_HASH_KEY_SIZE, 53 ("Requested more bytes than ENA RSS key can hold")); 54 55 if (!key_generated) { 56 arc4random_buf(default_key, ENA_HASH_KEY_SIZE); 57 key_generated = true; 58 } 59 60 memcpy(key, default_key, size); 61 } 62 63 /* 64 * ENA HW expects the key to be in reverse-byte order. 65 */ 66 static void 67 ena_rss_reorder_hash_key(u8 *reordered_key, const u8 *key, size_t key_size) 68 { 69 int i; 70 71 key = key + key_size - 1; 72 73 for (i = 0; i < key_size; ++i) 74 *reordered_key++ = *key--; 75 } 76 77 int 78 ena_rss_set_hash(struct ena_com_dev *ena_dev, const u8 *key) 79 { 80 enum ena_admin_hash_functions ena_func = ENA_ADMIN_TOEPLITZ; 81 u8 hw_key[ENA_HASH_KEY_SIZE]; 82 83 ena_rss_reorder_hash_key(hw_key, key, ENA_HASH_KEY_SIZE); 84 85 return (ena_com_fill_hash_function(ena_dev, ena_func, hw_key, 86 ENA_HASH_KEY_SIZE, 0x0)); 87 } 88 89 int 90 ena_rss_get_hash_key(struct ena_com_dev *ena_dev, u8 *key) 91 { 92 u8 hw_key[ENA_HASH_KEY_SIZE]; 93 int rc; 94 95 rc = ena_com_get_hash_key(ena_dev, hw_key); 96 if (rc != 0) 97 return rc; 98 99 ena_rss_reorder_hash_key(key, hw_key, ENA_HASH_KEY_SIZE); 100 101 return (0); 102 } 103 104 static int 105 ena_rss_init_default(struct ena_adapter *adapter) 106 { 107 struct ena_com_dev *ena_dev = adapter->ena_dev; 108 device_t dev = adapter->pdev; 109 int qid, rc, i; 110 111 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 112 if (unlikely(rc != 0)) { 113 ena_log(dev, ERR, "Cannot init indirect table\n"); 114 return (rc); 115 } 116 117 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 118 #ifdef RSS 119 qid = rss_get_indirection_to_bucket(i) % adapter->num_io_queues; 120 #else 121 qid = i % adapter->num_io_queues; 122 #endif 123 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 124 ENA_IO_RXQ_IDX(qid)); 125 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 126 ena_log(dev, ERR, "Cannot fill indirect table\n"); 127 goto err_rss_destroy; 128 } 129 } 130 131 132 #ifdef RSS 133 uint8_t rss_algo = rss_gethashalgo(); 134 if (rss_algo == RSS_HASH_TOEPLITZ) { 135 uint8_t hash_key[RSS_KEYSIZE]; 136 137 rss_getkey(hash_key); 138 rc = ena_rss_set_hash(ena_dev, hash_key); 139 } else 140 #endif 141 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, 142 NULL, ENA_HASH_KEY_SIZE, 0x0); 143 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 144 ena_log(dev, ERR, "Cannot fill hash function\n"); 145 goto err_rss_destroy; 146 } 147 148 rc = ena_com_set_default_hash_ctrl(ena_dev); 149 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 150 ena_log(dev, ERR, "Cannot fill hash control\n"); 151 goto err_rss_destroy; 152 } 153 154 rc = ena_rss_indir_init(adapter); 155 156 return (rc == EOPNOTSUPP ? 0 : rc); 157 158 err_rss_destroy: 159 ena_com_rss_destroy(ena_dev); 160 return (rc); 161 } 162 163 /* Configure the Rx forwarding */ 164 int 165 ena_rss_configure(struct ena_adapter *adapter) 166 { 167 struct ena_com_dev *ena_dev = adapter->ena_dev; 168 int rc; 169 170 /* In case the RSS table was destroyed */ 171 if (!ena_dev->rss.tbl_log_size) { 172 rc = ena_rss_init_default(adapter); 173 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 174 ena_log(adapter->pdev, ERR, 175 "WARNING: RSS was not properly re-initialized," 176 " it will affect bandwidth\n"); 177 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); 178 return (rc); 179 } 180 } 181 182 /* Set indirect table */ 183 rc = ena_com_indirect_table_set(ena_dev); 184 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 185 return (rc); 186 187 /* Configure hash function (if supported) */ 188 rc = ena_com_set_hash_function(ena_dev); 189 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 190 return (rc); 191 192 /* Configure hash inputs (if supported) */ 193 rc = ena_com_set_hash_ctrl(ena_dev); 194 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 195 return (rc); 196 197 return (0); 198 } 199 200 static void 201 ena_rss_init_default_deferred(void *arg) 202 { 203 struct ena_adapter *adapter; 204 devclass_t dc; 205 int max; 206 int rc; 207 208 dc = devclass_find("ena"); 209 if (unlikely(dc == NULL)) { 210 ena_log_raw(ERR, "SYSINIT: %s: No devclass ena\n", __func__); 211 return; 212 } 213 214 max = devclass_get_maxunit(dc); 215 while (max-- >= 0) { 216 adapter = devclass_get_softc(dc, max); 217 if (adapter != NULL) { 218 rc = ena_rss_init_default(adapter); 219 ENA_FLAG_SET_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); 220 if (unlikely(rc != 0)) { 221 ena_log(adapter->pdev, WARN, 222 "WARNING: RSS was not properly initialized," 223 " it will affect bandwidth\n"); 224 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, 225 adapter); 226 } 227 } 228 } 229 } 230 SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND, 231 ena_rss_init_default_deferred, NULL); 232 233 int 234 ena_rss_indir_get(struct ena_adapter *adapter, uint32_t *table) 235 { 236 int rc, i; 237 238 rc = ena_com_indirect_table_get(adapter->ena_dev, table); 239 if (rc != 0) { 240 if (rc == EOPNOTSUPP) 241 device_printf(adapter->pdev, 242 "Reading from indirection table not supported\n"); 243 else 244 device_printf(adapter->pdev, 245 "Unable to get indirection table\n"); 246 return (rc); 247 } 248 249 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i) 250 table[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(table[i]); 251 252 return (0); 253 } 254 255 int 256 ena_rss_indir_set(struct ena_adapter *adapter, uint32_t *table) 257 { 258 int rc, i; 259 260 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i) { 261 rc = ena_com_indirect_table_fill_entry(adapter->ena_dev, i, 262 ENA_IO_RXQ_IDX(table[i])); 263 if (rc != 0) { 264 device_printf(adapter->pdev, 265 "Cannot fill indirection table entry %d\n", i); 266 return (rc); 267 } 268 } 269 270 rc = ena_com_indirect_table_set(adapter->ena_dev); 271 if (rc == EOPNOTSUPP) 272 device_printf(adapter->pdev, 273 "Writing to indirection table not supported\n"); 274 else if (rc != 0) 275 device_printf(adapter->pdev, "Cannot set indirection table\n"); 276 277 return (rc); 278 } 279 280 int 281 ena_rss_indir_init(struct ena_adapter *adapter) 282 { 283 struct ena_indir *indir = adapter->rss_indir; 284 int rc; 285 286 if (indir == NULL) { 287 adapter->rss_indir = indir = malloc(sizeof(struct ena_indir), 288 M_DEVBUF, M_WAITOK | M_ZERO); 289 if (indir == NULL) 290 return (ENOMEM); 291 } 292 293 rc = ena_rss_indir_get(adapter, indir->table); 294 if (rc != 0) { 295 free(adapter->rss_indir, M_DEVBUF); 296 adapter->rss_indir = NULL; 297 298 return (rc); 299 } 300 301 ena_rss_copy_indir_buf(indir->sysctl_buf, indir->table); 302 303 return (0); 304 } 305