1# Chelsio T6 Factory Default configuration file. 2# 3# Copyright (C) 2014-2015 Chelsio Communications. All rights reserved. 4# 5# DO NOT MODIFY THIS FILE UNDER ANY CIRCUMSTANCES. MODIFICATION OF THIS FILE 6# WILL RESULT IN A NON-FUNCTIONAL ADAPTER AND MAY RESULT IN PHYSICAL DAMAGE 7# TO ADAPTERS. 8 9 10# This file provides the default, power-on configuration for 2-port T6-based 11# adapters shipped from the factory. These defaults are designed to address 12# the needs of the vast majority of Terminator customers. The basic idea is to 13# have a default configuration which allows a customer to plug a Terminator 14# adapter in and have it work regardless of OS, driver or application except in 15# the most unusual and/or demanding customer applications. 16# 17# Many of the Terminator resources which are described by this configuration 18# are finite. This requires balancing the configuration/operation needs of 19# device drivers across OSes and a large number of customer application. 20# 21# Some of the more important resources to allocate and their constaints are: 22# 1. Virtual Interfaces: 256. 23# 2. Ingress Queues with Free Lists: 1024. 24# 3. Egress Queues: 128K. 25# 4. MSI-X Vectors: 1088. 26# 5. Multi-Port Support (MPS) TCAM: 336 entries to support MAC destination 27# address matching on Ingress Packets. 28# 29# Some of the important OS/Driver resource needs are: 30# 6. Some OS Drivers will manage all resources through a single Physical 31# Function (currently PF4 but it could be any Physical Function). 32# 7. Some OS Drivers will manage different ports and functions (NIC, 33# storage, etc.) on different Physical Functions. For example, NIC 34# functions for ports 0-1 on PF0-1, FCoE on PF4, iSCSI on PF5, etc. 35# 36# Some of the customer application needs which need to be accommodated: 37# 8. Some customers will want to support large CPU count systems with 38# good scaling. Thus, we'll need to accommodate a number of 39# Ingress Queues and MSI-X Vectors to allow up to some number of CPUs 40# to be involved per port and per application function. For example, 41# in the case where all ports and application functions will be 42# managed via a single Unified PF and we want to accommodate scaling up 43# to 8 CPUs, we would want: 44# 45# 2 ports * 46# 3 application functions (NIC, FCoE, iSCSI) per port * 47# 16 Ingress Queue/MSI-X Vectors per application function 48# 49# for a total of 96 Ingress Queues and MSI-X Vectors on the Unified PF. 50# (Plus a few for Firmware Event Queues, etc.) 51# 52# 9. Some customers will want to use PCI-E SR-IOV Capability to allow Virtual 53# Machines to directly access T6 functionality via SR-IOV Virtual Functions 54# and "PCI Device Passthrough" -- this is especially true for the NIC 55# application functionality. 56# 57 58 59# Global configuration settings. 60# 61[global] 62 rss_glb_config_mode = basicvirtual 63 rss_glb_config_options = tnlmapen,hashtoeplitz,tnlalllkp 64 65 # PL_TIMEOUT register 66 pl_timeout_value = 1000 # the timeout value in units of us 67 68 # The following Scatter Gather Engine (SGE) settings assume a 4KB Host 69 # Page Size and a 64B L1 Cache Line Size. It programs the 70 # EgrStatusPageSize and IngPadBoundary to 64B and the PktShift to 2. 71 # If a Master PF Driver finds itself on a machine with different 72 # parameters, then the Master PF Driver is responsible for initializing 73 # these parameters to appropriate values. 74 # 75 # Notes: 76 # 1. The Free List Buffer Sizes below are raw and the firmware will 77 # round them up to the Ingress Padding Boundary. 78 # 2. The SGE Timer Values below are expressed below in microseconds. 79 # The firmware will convert these values to Core Clock Ticks when 80 # it processes the configuration parameters. 81 # 82 reg[0x1008] = 0x40810/0x21c70 # SGE_CONTROL 83 reg[0x100c] = 0x22222222 # SGE_HOST_PAGE_SIZE 84 reg[0x10a0] = 0x01040810 # SGE_INGRESS_RX_THRESHOLD 85 reg[0x1044] = 4096 # SGE_FL_BUFFER_SIZE0 86 reg[0x1048] = 65536 # SGE_FL_BUFFER_SIZE1 87 reg[0x104c] = 1536 # SGE_FL_BUFFER_SIZE2 88 reg[0x1050] = 9024 # SGE_FL_BUFFER_SIZE3 89 reg[0x1054] = 9216 # SGE_FL_BUFFER_SIZE4 90 reg[0x1058] = 2048 # SGE_FL_BUFFER_SIZE5 91 reg[0x105c] = 128 # SGE_FL_BUFFER_SIZE6 92 reg[0x1060] = 8192 # SGE_FL_BUFFER_SIZE7 93 reg[0x1064] = 16384 # SGE_FL_BUFFER_SIZE8 94 reg[0x10a4] = 0xa000a000/0xf000f000 # SGE_DBFIFO_STATUS 95 reg[0x10a8] = 0x402000/0x402000 # SGE_DOORBELL_CONTROL 96 sge_timer_value = 5, 10, 20, 50, 100, 200 # SGE_TIMER_VALUE* in usecs 97 reg[0x10c4] = 0x20000000/0x20000000 # GK_CONTROL, enable 5th thread 98 99 #DBQ Timer duration = 1 cclk cycle duration * (sge_dbq_timertick+1) * sge_dbq_timer 100 #SGE DBQ tick value. All timers are multiple of this value 101# sge_dbq_timertick = 5 #in usecs 102# sge_dbq_timer = 1, 2, 4, 6, 8, 10, 12, 16 103 # enable TP_OUT_CONFIG.IPIDSPLITMODE 104 reg[0x7d04] = 0x00010000/0x00010000 105 106 reg[0x7dc0] = 0x0e2f8849 # TP_SHIFT_CNT 107 108 #Tick granularities in kbps 109 tsch_ticks = 1000, 100, 10, 1 110 111 # TP_VLAN_PRI_MAP to select filter tuples and enable ServerSram 112 # filter control: compact, fcoemask 113 # server sram : srvrsram 114 # filter tuples : fragmentation, mpshittype, macmatch, ethertype, 115 # protocol, tos, vlan, vnic_id, port, fcoe 116 # valid filterModes are described the Terminator 5 Data Book 117 filterMode = fcoemask, srvrsram, fragmentation, mpshittype, protocol, vlan, port, fcoe 118 119 # filter tuples enforced in LE active region (equal to or subset of filterMode) 120 filterMask = protocol, fcoe 121 122 # Percentage of dynamic memory (in either the EDRAM or external MEM) 123 # to use for TP RX payload 124 tp_pmrx = 30 125 126 # TP RX payload page size 127 tp_pmrx_pagesize = 64K 128 129 # TP number of RX channels 130 tp_nrxch = 0 # 0 (auto) = 1 131 132 # Percentage of dynamic memory (in either the EDRAM or external MEM) 133 # to use for TP TX payload 134 tp_pmtx = 50 135 136 # TP TX payload page size 137 tp_pmtx_pagesize = 64K 138 139 # TP number of TX channels 140 tp_ntxch = 0 # 0 (auto) = equal number of ports 141 142 # TP OFLD MTUs 143 tp_mtus = 88, 256, 512, 576, 808, 1024, 1280, 1488, 1500, 2002, 2048, 4096, 4352, 8192, 9000, 9600 144 145 # enable TP_OUT_CONFIG.IPIDSPLITMODE and CRXPKTENC 146 reg[0x7d04] = 0x00010008/0x00010008 147 148 # TP_GLOBAL_CONFIG 149 reg[0x7d08] = 0x00000800/0x00000800 # set IssFromCplEnable 150 151 # TP_PC_CONFIG 152 reg[0x7d48] = 0x00000000/0x00000400 # clear EnableFLMError 153 154 # TP_PARA_REG0 155 reg[0x7d60] = 0x06000000/0x07000000 # set InitCWND to 6 156 157 # LE_DB_CONFIG 158 reg[0x19c04] = 0x00400000/0x00440000 # LE Server SRAM Enable, 159 # LE IPv4 compression disabled 160 # LE_DB_HASH_CONFIG 161 reg[0x19c28] = 0x00800000/0x01f00000 # LE Hash bucket size 8, 162 163 # ULP_TX_CONFIG 164 reg[0x8dc0] = 0x00000104/0x00000104 # Enable ITT on PI err 165 # Enable more error msg for ... 166 # TPT error. 167 168 # ULP_RX_MISC_FEATURE_ENABLE 169 reg[0x1925c] = 0x01003400/0x01003400 # iscsi tag pi bit 170 # Enable offset decrement after ... 171 # PI extraction and before DDP 172 # ulp insert pi source info in DIF 173 # iscsi_eff_offset_en 174 175 #Enable iscsi completion moderation feature 176 #reg[0x1925c] = 0x000041c0/0x000031c0 # Enable offset decrement after 177 # PI extraction and before DDP. 178 # ulp insert pi source info in 179 # DIF. 180 # Enable iscsi hdr cmd mode. 181 # iscsi force cmd mode. 182 # Enable iscsi cmp mode. 183 184# Some "definitions" to make the rest of this a bit more readable. We support 185# 4 ports, 3 functions (NIC, FCoE and iSCSI), scaling up to 8 "CPU Queue Sets" 186# per function per port ... 187# 188# NMSIX = 1088 # available MSI-X Vectors 189# NVI = 256 # available Virtual Interfaces 190# NMPSTCAM = 336 # MPS TCAM entries 191# 192# NPORTS = 2 # ports 193# NCPUS = 16 # CPUs we want to support scalably 194# NFUNCS = 3 # functions per port (NIC, FCoE, iSCSI) 195 196# Breakdown of Virtual Interface/Queue/Interrupt resources for the "Unified 197# PF" which many OS Drivers will use to manage most or all functions. 198# 199# Each Ingress Queue can use one MSI-X interrupt but some Ingress Queues can 200# use Forwarded Interrupt Ingress Queues. For these latter, an Ingress Queue 201# would be created and the Queue ID of a Forwarded Interrupt Ingress Queue 202# will be specified as the "Ingress Queue Asynchronous Destination Index." 203# Thus, the number of MSI-X Vectors assigned to the Unified PF will be less 204# than or equal to the number of Ingress Queues ... 205# 206# NVI_NIC = 4 # NIC access to NPORTS 207# NFLIQ_NIC = 32 # NIC Ingress Queues with Free Lists 208# NETHCTRL_NIC = 32 # NIC Ethernet Control/TX Queues 209# NEQ_NIC = 64 # NIC Egress Queues (FL, ETHCTRL/TX) 210# NMPSTCAM_NIC = 16 # NIC MPS TCAM Entries (NPORTS*4) 211# NMSIX_NIC = 32 # NIC MSI-X Interrupt Vectors (FLIQ) 212# 213# NVI_OFLD = 0 # Offload uses NIC function to access ports 214# NFLIQ_OFLD = 16 # Offload Ingress Queues with Free Lists 215# NETHCTRL_OFLD = 0 # Offload Ethernet Control/TX Queues 216# NEQ_OFLD = 16 # Offload Egress Queues (FL) 217# NMPSTCAM_OFLD = 0 # Offload MPS TCAM Entries (uses NIC's) 218# NMSIX_OFLD = 16 # Offload MSI-X Interrupt Vectors (FLIQ) 219# 220# NVI_RDMA = 0 # RDMA uses NIC function to access ports 221# NFLIQ_RDMA = 4 # RDMA Ingress Queues with Free Lists 222# NETHCTRL_RDMA = 0 # RDMA Ethernet Control/TX Queues 223# NEQ_RDMA = 4 # RDMA Egress Queues (FL) 224# NMPSTCAM_RDMA = 0 # RDMA MPS TCAM Entries (uses NIC's) 225# NMSIX_RDMA = 4 # RDMA MSI-X Interrupt Vectors (FLIQ) 226# 227# NEQ_WD = 128 # Wire Direct TX Queues and FLs 228# NETHCTRL_WD = 64 # Wire Direct TX Queues 229# NFLIQ_WD = 64 ` # Wire Direct Ingress Queues with Free Lists 230# 231# NVI_ISCSI = 4 # ISCSI access to NPORTS 232# NFLIQ_ISCSI = 4 # ISCSI Ingress Queues with Free Lists 233# NETHCTRL_ISCSI = 0 # ISCSI Ethernet Control/TX Queues 234# NEQ_ISCSI = 4 # ISCSI Egress Queues (FL) 235# NMPSTCAM_ISCSI = 4 # ISCSI MPS TCAM Entries (NPORTS) 236# NMSIX_ISCSI = 4 # ISCSI MSI-X Interrupt Vectors (FLIQ) 237# 238# NVI_FCOE = 4 # FCOE access to NPORTS 239# NFLIQ_FCOE = 34 # FCOE Ingress Queues with Free Lists 240# NETHCTRL_FCOE = 32 # FCOE Ethernet Control/TX Queues 241# NEQ_FCOE = 66 # FCOE Egress Queues (FL) 242# NMPSTCAM_FCOE = 32 # FCOE MPS TCAM Entries (NPORTS) 243# NMSIX_FCOE = 34 # FCOE MSI-X Interrupt Vectors (FLIQ) 244 245# Two extra Ingress Queues per function for Firmware Events and Forwarded 246# Interrupts, and two extra interrupts per function for Firmware Events (or a 247# Forwarded Interrupt Queue) and General Interrupts per function. 248# 249# NFLIQ_EXTRA = 6 # "extra" Ingress Queues 2*NFUNCS (Firmware and 250# # Forwarded Interrupts 251# NMSIX_EXTRA = 6 # extra interrupts 2*NFUNCS (Firmware and 252# # General Interrupts 253 254# Microsoft HyperV resources. The HyperV Virtual Ingress Queues will have 255# their interrupts forwarded to another set of Forwarded Interrupt Queues. 256# 257# NVI_HYPERV = 16 # VMs we want to support 258# NVIIQ_HYPERV = 2 # Virtual Ingress Queues with Free Lists per VM 259# NFLIQ_HYPERV = 40 # VIQs + NCPUS Forwarded Interrupt Queues 260# NEQ_HYPERV = 32 # VIQs Free Lists 261# NMPSTCAM_HYPERV = 16 # MPS TCAM Entries (NVI_HYPERV) 262# NMSIX_HYPERV = 8 # NCPUS Forwarded Interrupt Queues 263 264# Adding all of the above Unified PF resource needs together: (NIC + OFLD + 265# RDMA + ISCSI + FCOE + EXTRA + HYPERV) 266# 267# NVI_UNIFIED = 28 268# NFLIQ_UNIFIED = 106 269# NETHCTRL_UNIFIED = 32 270# NEQ_UNIFIED = 124 271# NMPSTCAM_UNIFIED = 40 272# 273# The sum of all the MSI-X resources above is 74 MSI-X Vectors but we'll round 274# that up to 128 to make sure the Unified PF doesn't run out of resources. 275# 276# NMSIX_UNIFIED = 128 277# 278# The Storage PFs could need up to NPORTS*NCPUS + NMSIX_EXTRA MSI-X Vectors 279# which is 34 but they're probably safe with 32. 280# 281# NMSIX_STORAGE = 32 282 283# Note: The UnifiedPF is PF4 which doesn't have any Virtual Functions 284# associated with it. Thus, the MSI-X Vector allocations we give to the 285# UnifiedPF aren't inherited by any Virtual Functions. As a result we can 286# provision many more Virtual Functions than we can if the UnifiedPF were 287# one of PF0-1. 288# 289 290# All of the below PCI-E parameters are actually stored in various *_init.txt 291# files. We include them below essentially as comments. 292# 293# For PF0-1 we assign 8 vectors each for NIC Ingress Queues of the associated 294# ports 0-1. 295# 296# For PF4, the Unified PF, we give it an MSI-X Table Size as outlined above. 297# 298# For PF5-6 we assign enough MSI-X Vectors to support FCoE and iSCSI 299# storage applications across all four possible ports. 300# 301# Additionally, since the UnifiedPF isn't one of the per-port Physical 302# Functions, we give the UnifiedPF and the PF0-1 Physical Functions 303# different PCI Device IDs which will allow Unified and Per-Port Drivers 304# to directly select the type of Physical Function to which they wish to be 305# attached. 306# 307# Note that the actual values used for the PCI-E Intelectual Property will be 308# 1 less than those below since that's the way it "counts" things. For 309# readability, we use the number we actually mean ... 310# 311# PF0_INT = 8 # NCPUS 312# PF1_INT = 8 # NCPUS 313# PF0_3_INT = 32 # PF0_INT + PF1_INT + PF2_INT + PF3_INT 314# 315# PF4_INT = 128 # NMSIX_UNIFIED 316# PF5_INT = 32 # NMSIX_STORAGE 317# PF6_INT = 32 # NMSIX_STORAGE 318# PF7_INT = 0 # Nothing Assigned 319# PF4_7_INT = 192 # PF4_INT + PF5_INT + PF6_INT + PF7_INT 320# 321# PF0_7_INT = 224 # PF0_3_INT + PF4_7_INT 322# 323# With the above we can get 17 VFs/PF0-3 (limited by 336 MPS TCAM entries) 324# but we'll lower that to 16 to make our total 64 and a nice power of 2 ... 325# 326# NVF = 16 327 328 329# For those OSes which manage different ports on different PFs, we need 330# only enough resources to support a single port's NIC application functions 331# on PF0-3. The below assumes that we're only doing NIC with NCPUS "Queue 332# Sets" for ports 0-3. The FCoE and iSCSI functions for such OSes will be 333# managed on the "storage PFs" (see below). 334# 335 336# Some OS Drivers manage all application functions for all ports via PF4. 337# Thus we need to provide a large number of resources here. For Egress 338# Queues we need to account for both TX Queues as well as Free List Queues 339# (because the host is responsible for producing Free List Buffers for the 340# hardware to consume). 341# 342[function "0"] 343 wx_caps = all # write/execute permissions for all commands 344 r_caps = all # read permissions for all commands 345 nvi = 28 # NVI_UNIFIED 346 niqflint = 170 # NFLIQ_UNIFIED + NLFIQ_WD 347 nethctrl = 96 # NETHCTRL_UNIFIED + NETHCTRL_WD 348 neq = 252 # NEQ_UNIFIED + NEQ_WD 349 nexactf = 40 # NMPSTCAM_UNIFIED 350 nrawf = 2 351 cmask = all # access to all channels 352 pmask = all # access to all four ports ... 353 nethofld = 1024 # number of user mode ethernet flow contexts 354 ncrypto_lookaside = 32 355 nclip = 32 # number of clip region entries 356 nfilter = 48 # number of filter region entries 357 nserver = 48 # number of server region entries 358 nhash = 2048 # number of hash region entries 359 nhpfilter = 0 # number of high priority filter region entries 360 protocol = nic_vm, ofld, rddp, rdmac, iscsi_initiator_pdu, iscsi_target_pdu, iscsi_t10dif, tlskeys, crypto_lookaside 361 tp_l2t = 3072 362 tp_ddp = 2 363 tp_ddp_iscsi = 2 364 tp_tls_key = 3 365 tp_stag = 2 366 tp_pbl = 5 367 tp_rq = 7 368 tp_srq = 128 369 370# We have FCoE and iSCSI storage functions on PF5 and PF6 each of which may 371# need to have Virtual Interfaces on each of the four ports with up to NCPUS 372# "Queue Sets" each. 373# 374[function "1"] 375 wx_caps = all # write/execute permissions for all commands 376 r_caps = all # read permissions for all commands 377 nvi = 4 # NPORTS 378 niqflint = 34 # NPORTS*NCPUS + NMSIX_EXTRA 379 nethctrl = 32 # NPORTS*NCPUS 380 neq = 64 # NPORTS*NCPUS * 2 (FL, ETHCTRL/TX) 381 nexactf = 16 # (NPORTS *(no of snmc grp + 1 hw mac) + 1 anmc grp)) rounded to 16. 382 cmask = all # access to all channels 383 pmask = all # access to all four ports ... 384 nserver = 16 385 nhash = 2048 386 tp_l2t = 1020 387 protocol = iscsi_initiator_fofld 388 tp_ddp_iscsi = 2 389 iscsi_ntask = 2048 390 iscsi_nsess = 2048 391 iscsi_nconn_per_session = 1 392 iscsi_ninitiator_instance = 64 393 394 395# The following function, 1023, is not an actual PCIE function but is used to 396# configure and reserve firmware internal resources that come from the global 397# resource pool. 398# 399[function "1023"] 400 wx_caps = all # write/execute permissions for all commands 401 r_caps = all # read permissions for all commands 402 nvi = 4 # NVI_UNIFIED 403 cmask = all # access to all channels 404 pmask = all # access to all four ports ... 405 nexactf = 8 # NPORTS + DCBX + 406 nfilter = 16 # number of filter region entries 407 #nhpfilter = 0 # number of high priority filter region entries 408 409 410# For Virtual functions, we only allow NIC functionality and we only allow 411# access to one port (1 << PF). Note that because of limitations in the 412# Scatter Gather Engine (SGE) hardware which checks writes to VF KDOORBELL 413# and GTS registers, the number of Ingress and Egress Queues must be a power 414# of 2. 415# 416[function "0/*"] # NVF 417 wx_caps = 0x82 # DMAQ | VF 418 r_caps = 0x86 # DMAQ | VF | PORT 419 nvi = 1 # 1 port 420 niqflint = 4 # 2 "Queue Sets" + NXIQ 421 nethctrl = 2 # 2 "Queue Sets" 422 neq = 4 # 2 "Queue Sets" * 2 423 nexactf = 4 424 cmask = all # access to all channels 425 pmask = 0x1 # access to only one port ... 426 427 428[function "1/*"] # NVF 429 wx_caps = 0x82 # DMAQ | VF 430 r_caps = 0x86 # DMAQ | VF | PORT 431 nvi = 1 # 1 port 432 niqflint = 4 # 2 "Queue Sets" + NXIQ 433 nethctrl = 2 # 2 "Queue Sets" 434 neq = 4 # 2 "Queue Sets" * 2 435 nexactf = 4 436 cmask = all # access to all channels 437 pmask = 0x2 # access to only one port ... 438 439 440# MPS features a 196608 bytes ingress buffer that is used for ingress buffering 441# for packets from the wire as well as the loopback path of the L2 switch. The 442# folling params control how the buffer memory is distributed and the L2 flow 443# control settings: 444# 445# bg_mem: %-age of mem to use for port/buffer group 446# lpbk_mem: %-age of port/bg mem to use for loopback 447# hwm: high watermark; bytes available when starting to send pause 448# frames (in units of 0.1 MTU) 449# lwm: low watermark; bytes remaining when sending 'unpause' frame 450# (in inuits of 0.1 MTU) 451# dwm: minimum delta between high and low watermark (in units of 100 452# Bytes) 453# 454[port "0"] 455 dcb = ppp, dcbx, b2b # configure for DCB PPP and enable DCBX offload 456 hwm = 30 457 lwm = 15 458 dwm = 30 459 dcb_app_tlv[0] = 0x8906, ethertype, 3 460 dcb_app_tlv[1] = 0x8914, ethertype, 3 461 dcb_app_tlv[2] = 3260, socketnum, 5 462 463 464[port "1"] 465 dcb = ppp, dcbx, b2b 466 hwm = 30 467 lwm = 15 468 dwm = 30 469 dcb_app_tlv[0] = 0x8906, ethertype, 3 470 dcb_app_tlv[1] = 0x8914, ethertype, 3 471 dcb_app_tlv[2] = 3260, socketnum, 5 472 473 474[fini] 475 version = 0x1425001d 476 checksum = 0x5001af51 477 478# Total resources used by above allocations: 479# Virtual Interfaces: 104 480# Ingress Queues/w Free Lists and Interrupts: 526 481# Egress Queues: 702 482# MPS TCAM Entries: 336 483# MSI-X Vectors: 736 484# Virtual Functions: 64 485# 486# 487