1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/bits.h> 6 #include <linux/cache.h> 7 #include <linux/if_ether.h> 8 #include <linux/iopoll.h> 9 #include <linux/log2.h> 10 #include <linux/sizes.h> 11 12 #include "mpnic.h" 13 #include "mpnic_txrx.h" 14 15 #define MPNIC_MEM_INIT_POLL_US 500 16 #define MPNIC_MEM_INIT_TO_US 5000 17 18 /* BDQ mem init: 19 * bit 1: fifo_wrptr_mem 20 * bit 0: fifo_rdptr_mem 21 */ 22 #define MPNIC_MEM_INIT_BDQ_VAL 0x3 23 24 /* RCM mem init: 25 * bit 3: cq_base_addr 26 * bit 2: cd_fifo_rptr_stats 27 * bit 1: cd_fifo_wptr_stats 28 * bit 0: cq_head_ptr_stats 29 */ 30 #define MPNIC_MEM_INIT_RCM_VAL 0xf 31 32 /* RDE mem init, bits 0-18. Bits 0-12 are the per-queue packet, error and 33 * drop counters, bits 13-18 the two context memories of each of the HPQ, 34 * PPQ and SPQ descriptor prefetchers. 35 */ 36 #define MPNIC_MEM_INIT_RDE_VAL 0x7ffff 37 38 /* RPC mem init, bit 0 covers the whole classifier. */ 39 #define MPNIC_MEM_INIT_RPC_VAL 0x1 40 41 /* TCM mem init: 42 * bit 3: cq_head_ptr_stats 43 * bit 2: cd_fifo_wptr_stats 44 * bit 1: cd_fifo_rptr_stats 45 * bit 0: cq_base_addr 46 */ 47 #define MPNIC_MEM_INIT_TCM_VAL 0xf 48 49 /* TDE mem init: 50 * bit 1: stats mem 51 * bit 0: dma_head_ptr SRAM 52 */ 53 #define MPNIC_MEM_INIT_TDE_VAL 0x3 54 55 /* TDF mem init, bit 0 covers the descriptor fetch SRAM. */ 56 #define MPNIC_MEM_INIT_TDF_VAL 0x1 57 58 /* RXB mem init, bit 0 covers the DMAC TCAM statistics RAM. */ 59 #define MPNIC_MEM_INIT_RXB_VAL 0x1 60 61 /* TQS arbiter SRAM init done, one bit per DWRR level. */ 62 #define MPNIC_TQS_ARB_INIT_VAL 0xf 63 64 /* On-chip SRAM allocated to each queue for descriptor fetch, in units of 65 * descriptors. Valid values are 64, 128, 256, 512 and 1024. 66 * 67 * The partition sizes are chosen for the maximum number of queues the 68 * device supports, so they do not have to be adjusted when the active 69 * queue count changes: 70 * 71 * BDQ: 1 MiB / (8 B/DESC) / (1024 HPQ + 1024 PPQ) = 64 DESC/QUEUE 72 * TWQ: 2 MiB / (8 B/DESC) / (1024 TXQ * 2 TWQ) = 128 DESC/QUEUE 73 */ 74 #define MPNIC_BDQ_SRAM_DESCS 64u 75 #define MPNIC_TDF_SRAM_DESCS 128u 76 77 /* A total of 1 MiB worth of Tx credits is available, in units of 128 B. 78 * The BMC to MAC and host to BMC paths each get a guaranteed share of them 79 * whether or not anything is routed their way, everything else goes to 80 * MAC TC0. 81 */ 82 #define MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL 800 83 #define MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL \ 84 (SZ_1M / 128 - 2 * MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL) 85 86 /* The recommended lower bound for the TXB threshold is 80, based on a 10K 87 * MTU. Round up by 25% to stay on the defensive side. The same reasoning 88 * applies to the arbitration weight, which has to exceed the full packet 89 * size. 90 */ 91 #define MPNIC_TXB_INIT_BMC_THRESH 100 92 #define MPNIC_TXB_INIT_P0_THRESH 100 93 #define MPNIC_TXB_INIT_P0_ARB_WEIGHTS 0x64 94 95 /* RXB host drop threshold in units of 128 B beats. Packets targeting a 96 * queue are dropped when the available credits fall below it. 80 beats is 97 * 10 KB, which is also the largest frame the device is configured for. 98 */ 99 #define MPNIC_RXB_INIT_HOST_DROP_THRESH 0x50 100 101 /* A total of 8 MiB of Rx buffer is available. The recommended per-TC pool 102 * for a 800G configuration is 420 KB with all 8 TCs enabled. Only one TC 103 * is in use, so give it 8 * 420 KB (in units of 128 B) and push the rest 104 * to the common pool. Start drawing from the common pool as soon as the 105 * TC0 credits fall below one max sized frame. The BMC keeps a small 106 * reserve of its own whether or not the host talks to it. 107 */ 108 #define MPNIC_RXB_INIT_POOL_TC_CRDTS_P0 (0xd20 * 8) 109 #define MPNIC_RXB_INIT_BMC_CRDTS 0x20 110 #define MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH \ 111 (SZ_8M / 128 - MPNIC_RXB_INIT_POOL_TC_CRDTS_P0 - \ 112 MPNIC_RXB_INIT_BMC_CRDTS) 113 #define MPNIC_RXB_INIT_COMMON_CRDT_THRSH 0x50 114 115 /* TXB port mode selects the number of active MAC ports for Tx buffer 116 * credit distribution and arbitration. The hardware only supports single, 117 * dual and quad port, encoded as b'001, b'010 and b'100. 118 */ 119 #define MPNIC_TXB_PORT_MODE_SINGLE 1 120 121 /* The MAC traffic classes start at index 8 in the TXB arrays, the BMC 122 * sits above them. 123 */ 124 #define MPNIC_TXB_TC_IDX_MAC_0 8 125 #define MPNIC_TXB_TC_IDX_BMC 16 126 127 /* Largest frame the Tx queue scheduler will pass through. Anything above 128 * it gets truncated. 129 */ 130 #define MPNIC_TQS_MTU_CTL0_MAX 0x2800 131 132 /* The unit of the DWRR quantum is 256 B. It has to be large enough for at 133 * least 11 MTUs to be transmitted in one quantum; use 15 for headroom. 134 */ 135 #define MPNIC_TQS_DWRR_INIT_QUANTUM (15 * MPNIC_TQS_MTU_CTL0_MAX / 256) 136 137 /* Lower bound on the credit available to a queue, group, set or port 138 * before the scheduler stops issuing requests for it. 15 MTUs, matching 139 * the DWRR quantum above. 140 */ 141 #define MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH (15 * MPNIC_TQS_MTU_CTL0_MAX) 142 143 /* The TQS view of the TXB partitioning between the MAC and the BMC, in 144 * units of 1 KiB. It has to match the TXB private credits above. 145 */ 146 #define MPNIC_TQS_GLBL_TXB_CRDT_UNIT SZ_1K 147 #define MPNIC_TQS_GLBL_TXB_CRDT_BMC \ 148 (MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL * 128 / MPNIC_TQS_GLBL_TXB_CRDT_UNIT) 149 #define MPNIC_TQS_GLBL_TXB_CRDT_MAC \ 150 (MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL * 128 / \ 151 MPNIC_TQS_GLBL_TXB_CRDT_UNIT) 152 153 struct mpnic_init_poll { 154 u64 exp_val; 155 u32 addr; 156 }; 157 158 struct mpnic_poll_state { 159 int poll_idx; 160 u64 val; 161 }; 162 163 static void mpnic_tdf_glbl_init(struct mpnic_dev *mpd) 164 { 165 /* Default metadata descriptor, used for frames the driver did not 166 * prepend one to. 167 */ 168 mpnic_wr64(mpd, MPNIC_TWQ_DEF_PRI_TWD, MPNIC_TWD_FLAG_REQ_COMPLETION); 169 } 170 171 static void mpnic_txb_init(struct mpnic_dev *mpd) 172 { 173 int i; 174 175 mpnic_wr64(mpd, MPNIC_TXB_BMC, MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL); 176 177 /* Zero the private credits of every traffic class, then hand the 178 * unreserved ones to MAC TC0. 179 */ 180 for (i = 0; i < MPNIC_TXB_P0_CNT; i++) 181 mpnic_wr64(mpd, MPNIC_TXB_P0(i), 0); 182 mpnic_wr64(mpd, MPNIC_TXB_P0(MPNIC_TXB_TC_IDX_MAC_0), 183 MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL); 184 mpnic_wr64(mpd, MPNIC_TXB_P0(MPNIC_TXB_TC_IDX_BMC), 185 MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL); 186 187 mpnic_wr64(mpd, MPNIC_TXB_P0_THRESH(MPNIC_TXB_TC_IDX_BMC), 188 MPNIC_TXB_INIT_BMC_THRESH); 189 190 mpnic_wr64(mpd, MPNIC_TXB_P0_THRESH(MPNIC_TXB_TC_IDX_MAC_0), 191 MPNIC_TXB_INIT_P0_THRESH); 192 mpnic_wr64(mpd, MPNIC_TXB_P0_ARB_WEIGHTS(MPNIC_TXB_TC_IDX_MAC_0), 193 MPNIC_TXB_INIT_P0_ARB_WEIGHTS); 194 mpnic_wr64(mpd, MPNIC_TXB_PORT_CONFIG, 195 FIELD_PREP(MPNIC_TXB_PORT_CONFIG_PORT_MODE, 196 MPNIC_TXB_PORT_MODE_SINGLE)); 197 } 198 199 static void mpnic_rxb_init(struct mpnic_dev *mpd) 200 { 201 /* Accept all packets that miss dmac tcam, until l2 filtering is 202 * implemented. 203 */ 204 mpnic_wr64(mpd, MPNIC_RXB_PORT_CLASS_CFG(0), 205 FIELD_PREP(MPNIC_RXB_PORT_CLASS_CFG_DEFAULT_L2_ACTION, 206 MPNIC_L2_ACTION_PASS)); 207 mpnic_wr64(mpd, MPNIC_RXB_PORT_CFG(0), 208 FIELD_PREP(MPNIC_RXB_PORT_CFG_FCS_STRIP_MODE, 209 MPNIC_FCS_MODE_STRIP)); 210 211 mpnic_wr64(mpd, MPNIC_RXB_HOST_DROP_THRESH(0), 212 MPNIC_RXB_INIT_HOST_DROP_THRESH); 213 214 mpnic_wr64(mpd, MPNIC_RXB_TC_CRDTS(0), 215 MPNIC_RXB_INIT_POOL_TC_CRDTS_P0); 216 mpnic_wr64(mpd, MPNIC_RXB_COMMON_CRDT_CTRL_TC(0), 217 FIELD_PREP(MPNIC_RXB_COMMON_CRDT_CTRL_TC_THRESH, 218 MPNIC_RXB_INIT_COMMON_CRDT_THRSH) | 219 FIELD_PREP(MPNIC_RXB_COMMON_CRDT_CTRL_TC_MAX_CRDTS, 220 MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH) | 221 MPNIC_RXB_COMMON_CRDT_CTRL_TC_TC_EN); 222 223 /* Only pool 0 is used, it gets all of the common credits */ 224 mpnic_wr64(mpd, MPNIC_RXB_POOL_COMMON_CRDTS(0), 225 MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH); 226 227 mpnic_wr64(mpd, MPNIC_RXB_BMC_CRDTS(0), MPNIC_RXB_INIT_BMC_CRDTS); 228 229 mpnic_wr64(mpd, MPNIC_RXB_MEM_INIT_REQ, MPNIC_MEM_INIT_RXB_VAL); 230 } 231 232 static u64 mpnic_desc_cfg(unsigned int sram_descs, unsigned int q_idx) 233 { 234 /* The hardware encodes the partition size as 64 * 2^n descriptors, 235 * and its start address in units of 64 descriptors. 236 */ 237 return FIELD_PREP(MPNIC_DESC_CFG_NUM_DESCS, __ffs(sram_descs) - 6) | 238 FIELD_PREP(MPNIC_DESC_CFG_START_ADDR, 239 q_idx * (sram_descs >> 6)); 240 } 241 242 static void mpnic_desc_sram_init(struct mpnic_dev *mpd) 243 { 244 int i; 245 246 for (i = 0; i < MPNIC_MAX_TXQS * 2; i++) 247 mpnic_wr64(mpd, MPNIC_TDF_DESC_CFG(i), 248 mpnic_desc_cfg(MPNIC_TDF_SRAM_DESCS, i)); 249 250 for (i = 0; i < MPNIC_MAX_RXQS; i++) { 251 mpnic_wr64(mpd, MPNIC_HPQ_DESC_CFG(i), 252 mpnic_desc_cfg(MPNIC_BDQ_SRAM_DESCS, i)); 253 mpnic_wr64(mpd, MPNIC_PPQ_DESC_CFG(i), 254 mpnic_desc_cfg(MPNIC_BDQ_SRAM_DESCS, 255 MPNIC_MAX_RXQS + i)); 256 } 257 } 258 259 static void mpnic_rxglb_init(struct mpnic_dev *mpd) 260 { 261 /* Descriptor prefetch reads are only issued once 32 descriptors 262 * worth of FIFO space is available, and no more than 64 descriptors 263 * are fetched for one queue at a time so that a single queue cannot 264 * monopolize the bus. Both have to be multiples of 16 to keep the 265 * reads 128 B aligned. 266 */ 267 mpnic_wr64(mpd, MPNIC_BDQ_GLBL_CTL0, 268 FIELD_PREP(MPNIC_BDQ_GLBL_CTL0_PREFETCH_SPACE_THRESH, 32) | 269 FIELD_PREP(MPNIC_BDQ_GLBL_CTL0_MAX_REQ_SIZE, 64)); 270 271 /* Minimum number of descriptors which has to be available before 272 * the descriptor engine considers a queue usable, globally and in 273 * the per-queue prefetch FIFO. 274 */ 275 mpnic_wr64(mpd, MPNIC_RDE_CTL, 276 FIELD_PREP(MPNIC_RDE_CTL_HPQ_DROP_THRESHOLD, 277 MPNIC_RX_HPQ_DROP_THRS - 1) | 278 FIELD_PREP(MPNIC_RDE_CTL_PPQ_DROP_THRESHOLD, 279 MPNIC_RX_PPQ_DROP_THRS - 1) | 280 FIELD_PREP(MPNIC_RDE_CTL_HPQ_LOCAL_DROP_THRESHOLD, 281 MPNIC_RX_HPQ_DROP_THRS - 1) | 282 FIELD_PREP(MPNIC_RDE_CTL_PPQ_LOCAL_DROP_THRESHOLD, 283 MPNIC_RX_PPQ_DROP_THRS - 1)); 284 285 /* Receive side coalescing is not supported yet */ 286 mpnic_wr64(mpd, MPNIC_RSC_GLOBAL_CONF, 287 MPNIC_RSC_GLOBAL_CONF_RSC_DISABLE); 288 289 mpnic_wr64(mpd, MPNIC_BDQ_MEM_INIT_REQ, MPNIC_MEM_INIT_BDQ_VAL); 290 mpnic_wr64(mpd, MPNIC_RCM_MEM_INIT_REQ, MPNIC_MEM_INIT_RCM_VAL); 291 mpnic_wr64(mpd, MPNIC_RDE_MEM_INIT_REQ, MPNIC_MEM_INIT_RDE_VAL); 292 mpnic_wr64(mpd, MPNIC_RPC_MEM_INIT_REQ, MPNIC_MEM_INIT_RPC_VAL); 293 } 294 295 static void mpnic_txglb_init(struct mpnic_dev *mpd) 296 { 297 /* Nothing is redirected to the BMC until the Tx offload TCAM gets 298 * programmed with its addresses. 299 */ 300 mpnic_wr64(mpd, MPNIC_TOF_TCAM_DEST_REMAP, 0); 301 302 mpnic_wr64(mpd, MPNIC_TCM_MEM_INIT_REQ, MPNIC_MEM_INIT_TCM_VAL); 303 mpnic_wr64(mpd, MPNIC_TDE_MEM_INIT_REQ, MPNIC_MEM_INIT_TDE_VAL); 304 mpnic_wr64(mpd, MPNIC_TDF_MEM_INIT_REQ, MPNIC_MEM_INIT_TDF_VAL); 305 } 306 307 /* Fill the DWRR arbiter memories. Setting the INIT bit makes the hardware 308 * write the given credit and quantum into every entry at the queue, group, 309 * set and port level, so nothing has to be programmed per queue. 310 */ 311 static void mpnic_tqs_sram_init(struct mpnic_dev *mpd) 312 { 313 mpnic_wr64(mpd, MPNIC_TQS_GROUP_INIT_CTL, 314 FIELD_PREP(MPNIC_TQS_GROUP_INIT_CTL_QUANTUM, 315 MPNIC_TQS_DWRR_INIT_QUANTUM) | 316 MPNIC_TQS_GROUP_INIT_CTL_INIT); 317 mpnic_wr64(mpd, MPNIC_TQS_SET_INIT_CTL, 318 FIELD_PREP(MPNIC_TQS_SET_INIT_CTL_QUANTUM, 319 MPNIC_TQS_DWRR_INIT_QUANTUM) | 320 MPNIC_TQS_SET_INIT_CTL_INIT); 321 mpnic_wr64(mpd, MPNIC_TQS_PORT_INIT_CTL, 322 FIELD_PREP(MPNIC_TQS_PORT_INIT_CTL_QUANTUM, 323 MPNIC_TQS_DWRR_INIT_QUANTUM) | 324 MPNIC_TQS_PORT_INIT_CTL_INIT); 325 mpnic_wr64(mpd, MPNIC_TQS_SRAM_INIT_CTL, 326 FIELD_PREP(MPNIC_TQS_SRAM_INIT_CTL_QUANTUM, 327 MPNIC_TQS_DWRR_INIT_QUANTUM) | 328 MPNIC_TQS_SRAM_INIT_CTL_INIT); 329 } 330 331 static void mpnic_tqs_init(struct mpnic_dev *mpd) 332 { 333 u64 val; 334 335 /* Initialize to the largest frame we support, the scheduler 336 * truncates anything above it. The BMC gets the same limit. 337 */ 338 mpnic_wr64(mpd, MPNIC_TQS_MTU_CTL0, MPNIC_TQS_MTU_CTL0_MAX); 339 mpnic_wr64(mpd, MPNIC_TQS_MTU_CTL1, MPNIC_TQS_MTU_CTL0_MAX); 340 341 mpnic_wr64(mpd, MPNIC_TQS_GLBL_CTL0, 342 MPNIC_TQS_GLBL_CTL0_TWD_ERROR_CHECK_EN); 343 344 mpnic_tqs_sram_init(mpd); 345 346 /* Only port 0 is used. A single traffic class is in use as well, so 347 * TC0 gets all of the Tx buffer credits not reserved for the BMC. 348 */ 349 mpnic_wr64(mpd, MPNIC_TQS_GLBL_P0_0, 350 FIELD_PREP(MPNIC_TQS_GLBL_P0_0_TXB_MAX_CRDTS_0, 351 MPNIC_TQS_GLBL_TXB_CRDT_MAC)); 352 mpnic_wr64(mpd, MPNIC_TQS_GLBL_P0_1, 0); 353 mpnic_wr64(mpd, MPNIC_TQS_GLBL_BMC, 354 FIELD_PREP(MPNIC_TQS_GLBL_BMC_TXB_MAX_CRDTS, 355 MPNIC_TQS_GLBL_TXB_CRDT_BMC)); 356 357 mpnic_wr64(mpd, MPNIC_TQS_PORT_CTL(0), 0); 358 359 /* Map all sets to port 0 */ 360 mpnic_wr64(mpd, MPNIC_TQS_SET_P0_MAP0(0), ~0ULL); 361 mpnic_wr64(mpd, MPNIC_TQS_SET_P0_MAP1(0), ~0ULL); 362 363 mpnic_wr64(mpd, MPNIC_TQS_CEV_MIN_SCHED_THRESH_0, 364 FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_QUEUE, 365 MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH) | 366 FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_GROUP, 367 MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH)); 368 mpnic_wr64(mpd, MPNIC_TQS_CEV_MIN_SCHED_THRESH_1, 369 FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_SET, 370 MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH) | 371 FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_PORT, 372 MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH)); 373 374 /* The rate limiters are left uninitialized, so shaping has to stay 375 * off or nothing would ever get scheduled. 376 */ 377 mpnic_wr64(mpd, MPNIC_TQS_GLBL_SHAPING, MPNIC_TQS_GLBL_SHAPING_DISABLE); 378 379 /* Enable fairness protection (phantom eligibility). Read modify 380 * write so that the reset default slowdown cycle is preserved. 381 */ 382 val = mpnic_rd64(mpd, MPNIC_TQS_SLOWDOWN_CTL); 383 val |= MPNIC_TQS_SLOWDOWN_CTL_ENABLE; 384 mpnic_wr64(mpd, MPNIC_TQS_SLOWDOWN_CTL, val); 385 386 /* Use immediate credit decrement at every DWRR level so that the 387 * credit reflects a grant in the same cycle. 388 */ 389 val = mpnic_rd64(mpd, MPNIC_TQS_ARB_CTL); 390 val |= MPNIC_TQS_ARB_CTL_SET_CRDT_BUCKET_EN | 391 MPNIC_TQS_ARB_CTL_SET_IMM_DECR_EN | 392 MPNIC_TQS_ARB_CTL_GROUP_CRDT_BUCKET_EN | 393 MPNIC_TQS_ARB_CTL_GROUP_IMM_DECR_EN | 394 MPNIC_TQS_ARB_CTL_QUEUE_CRDT_BUCKET_EN | 395 MPNIC_TQS_ARB_CTL_QUEUE_IMM_DECR_EN; 396 mpnic_wr64(mpd, MPNIC_TQS_ARB_CTL, val); 397 } 398 399 /* The MPS and CLS fields sit at the same bit positions in every block, so 400 * one set of masks covers both the RNI and the TNI registers. 401 */ 402 static void mpnic_mps_init(struct mpnic_dev *mpd, u32 reg, unsigned int mps, 403 unsigned int cls) 404 { 405 u64 val = mpnic_rd64(mpd, reg); 406 407 val &= ~(MPNIC_RNI_RDE_CTL_MPS | MPNIC_RNI_RDE_CTL_CLS); 408 val |= FIELD_PREP(MPNIC_RNI_RDE_CTL_MPS, mps) | 409 FIELD_PREP(MPNIC_RNI_RDE_CTL_CLS, cls); 410 411 mpnic_wr64(mpd, reg, val); 412 } 413 414 /* Likewise for the MRRS and CLS fields, which have their own common 415 * layout. 416 */ 417 static void mpnic_mrrs_init(struct mpnic_dev *mpd, u32 reg, unsigned int mrrs, 418 unsigned int cls) 419 { 420 u64 val = mpnic_rd64(mpd, reg); 421 422 val &= ~(MPNIC_TNI_GLBL_TDF_CTL_MRRS | MPNIC_TNI_GLBL_TDF_CTL_CLS); 423 val |= FIELD_PREP(MPNIC_TNI_GLBL_TDF_CTL_MRRS, mrrs) | 424 FIELD_PREP(MPNIC_TNI_GLBL_TDF_CTL_CLS, cls); 425 426 mpnic_wr64(mpd, reg, val); 427 } 428 429 /** 430 * mpnic_axi_init - Configure AXI bus parameters from host PCIe capabilities 431 * @mpd: Device to configure 432 * 433 * Programs the max read request size, max payload size and cache line size 434 * of the DMA engines. The hardware encodes all three as a power of 2 index, 435 * MRRS and MPS relative to 128 B and CLS relative to 64 B. 436 * 437 * MAX_OT and MAX_OB are left at their hardware defaults. 438 */ 439 static void mpnic_axi_init(struct mpnic_dev *mpd) 440 { 441 int mps, cls, mrrs; 442 443 mps = clamp(ilog2(mpd->mps) - 7, 0, 3); 444 cls = clamp(ilog2(L1_CACHE_BYTES) - 6, 0, 3); 445 446 mpnic_mps_init(mpd, MPNIC_RNI_RDE_CTL, mps, cls); 447 mpnic_mps_init(mpd, MPNIC_RNI_RCM_CTL, mps, cls); 448 mpnic_mps_init(mpd, MPNIC_TNI_GLBL_TCM_CTL, mps, cls); 449 450 mrrs = clamp(ilog2(mpd->readrq) - 7, 0, 3); 451 mpnic_mrrs_init(mpd, MPNIC_RNI_RBP_CTL, mrrs, cls); 452 mpnic_mrrs_init(mpd, MPNIC_TNI_GLBL_TDF_CTL, mrrs, cls); 453 454 /* TDE supports a wider range of MRRS encodings. */ 455 mrrs = clamp(ilog2(mpd->readrq) - 7, 0, 5); 456 mpnic_mrrs_init(mpd, MPNIC_TNI_GLBL_TDE_CTL, mrrs, cls); 457 } 458 459 /** 460 * mpnic_ro_init - Set relaxed ordering on the outbound TLP attributes 461 * @mpd: Device to configure 462 * 463 * Completions must stay ordered so that they are not observed before the 464 * payload DMA they describe has landed, so RCM and TCM are left alone. 465 */ 466 static void mpnic_ro_init(struct mpnic_dev *mpd) 467 { 468 u64 attr = mpd->relaxed_ord ? MPNIC_OB_ATTR_RO : 0; 469 470 mpnic_wr64(mpd, MPNIC_OB_ATTR_TDE_H, attr); 471 mpnic_wr64(mpd, MPNIC_OB_ATTR_TDE_P, attr); 472 mpnic_wr64(mpd, MPNIC_OB_ATTR_TDF, attr); 473 mpnic_wr64(mpd, MPNIC_OB_ATTR_RBP_HPQ, attr); 474 mpnic_wr64(mpd, MPNIC_OB_ATTR_RBP_PPQ, attr); 475 mpnic_wr64(mpd, MPNIC_OB_ATTR_RDE_H, attr); 476 mpnic_wr64(mpd, MPNIC_OB_ATTR_RDE_P, attr); 477 } 478 479 static bool mpnic_init_status_ready(struct mpnic_dev *mpd, 480 const struct mpnic_init_poll *polls, 481 struct mpnic_poll_state *state) 482 { 483 u64 val; 484 int i; 485 486 for (i = state->poll_idx; polls[i].addr; i++) { 487 val = mpnic_rd64(mpd, polls[i].addr); 488 489 if ((val & polls[i].exp_val) != polls[i].exp_val) { 490 state->poll_idx = i; 491 state->val = val; 492 return false; 493 } 494 } 495 496 return true; 497 } 498 499 /** 500 * mpnic_mem_init_poll - Wait for the memory initializations to complete 501 * @mpd: Device to poll 502 * 503 * The blocks initialize their memories in parallel, so walk the status 504 * registers in order and only go back to sleep on the first one which is 505 * not done yet. 506 * 507 * Return: 0 on success, -ETIMEDOUT if not everything completed in time 508 */ 509 static int mpnic_mem_init_poll(struct mpnic_dev *mpd) 510 { 511 static const struct mpnic_init_poll polls[] = { 512 { MPNIC_TQS_ARB_INIT_VAL, MPNIC_TQS_SRAM_STS }, 513 { MPNIC_MEM_INIT_BDQ_VAL, MPNIC_BDQ_MEM_INIT_DONE }, 514 { MPNIC_MEM_INIT_RCM_VAL, MPNIC_RCM_MEM_INIT_DONE }, 515 { MPNIC_MEM_INIT_RDE_VAL, MPNIC_RDE_MEM_INIT_DONE }, 516 { MPNIC_MEM_INIT_RPC_VAL, MPNIC_RPC_MEM_INIT_DONE }, 517 { MPNIC_MEM_INIT_TCM_VAL, MPNIC_TCM_MEM_INIT_DONE }, 518 { MPNIC_MEM_INIT_TDE_VAL, MPNIC_TDE_MEM_INIT_DONE }, 519 { MPNIC_MEM_INIT_TDF_VAL, MPNIC_TDF_MEM_INIT_DONE }, 520 { MPNIC_MEM_INIT_RXB_VAL, MPNIC_RXB_MEM_INIT_DONE }, 521 { 0 }, 522 }; 523 struct mpnic_poll_state state = {}; 524 bool done; 525 int err; 526 527 err = read_poll_timeout(mpnic_init_status_ready, done, done, 528 MPNIC_MEM_INIT_POLL_US, MPNIC_MEM_INIT_TO_US, 529 false, mpd, polls, &state); 530 if (err) 531 dev_err(mpd->dev, "Poll timeout for reg 0x%x: 0x%llx\n", 532 polls[state.poll_idx].addr, state.val); 533 534 return err; 535 } 536 537 int mpnic_dev_init(struct mpnic_dev *mpd) 538 { 539 int err; 540 541 mpnic_tdf_glbl_init(mpd); 542 mpnic_txb_init(mpd); 543 mpnic_rxb_init(mpd); 544 mpnic_desc_sram_init(mpd); 545 mpnic_axi_init(mpd); 546 mpnic_ro_init(mpd); 547 mpnic_rxglb_init(mpd); 548 mpnic_txglb_init(mpd); 549 mpnic_tqs_init(mpd); 550 551 err = mpnic_mem_init_poll(mpd); 552 if (err) { 553 dev_err(mpd->dev, "Device initialization failed: %d\n", err); 554 return err; 555 } 556 557 if (!mpnic_present(mpd)) 558 return -EIO; 559 560 return 0; 561 } 562