1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 Intel Corporation */ 3 4 #include "igc.h" 5 #include "igc_hw.h" 6 #include "igc_tsn.h" 7 8 static bool is_any_launchtime(struct igc_adapter *adapter) 9 { 10 int i; 11 12 for (i = 0; i < adapter->num_tx_queues; i++) { 13 struct igc_ring *ring = adapter->tx_ring[i]; 14 15 if (ring->launchtime_enable) 16 return true; 17 } 18 19 return false; 20 } 21 22 static bool is_cbs_enabled(struct igc_adapter *adapter) 23 { 24 int i; 25 26 for (i = 0; i < adapter->num_tx_queues; i++) { 27 struct igc_ring *ring = adapter->tx_ring[i]; 28 29 if (ring->cbs_enable) 30 return true; 31 } 32 33 return false; 34 } 35 36 static unsigned int igc_tsn_new_flags(struct igc_adapter *adapter) 37 { 38 unsigned int new_flags = adapter->flags & ~IGC_FLAG_TSN_ANY_ENABLED; 39 40 if (adapter->taprio_offload_enable) 41 new_flags |= IGC_FLAG_TSN_QBV_ENABLED; 42 43 if (is_any_launchtime(adapter)) 44 new_flags |= IGC_FLAG_TSN_QBV_ENABLED; 45 46 if (is_cbs_enabled(adapter)) 47 new_flags |= IGC_FLAG_TSN_QAV_ENABLED; 48 49 if (adapter->strict_priority_enable) 50 new_flags |= IGC_FLAG_TSN_LEGACY_ENABLED; 51 52 return new_flags; 53 } 54 55 static bool igc_tsn_is_tx_mode_in_tsn(struct igc_adapter *adapter) 56 { 57 struct igc_hw *hw = &adapter->hw; 58 59 return !!(rd32(IGC_TQAVCTRL) & IGC_TQAVCTRL_TRANSMIT_MODE_TSN); 60 } 61 62 void igc_tsn_adjust_txtime_offset(struct igc_adapter *adapter) 63 { 64 struct igc_hw *hw = &adapter->hw; 65 u16 txoffset; 66 67 if (!igc_tsn_is_tx_mode_in_tsn(adapter)) 68 return; 69 70 switch (adapter->link_speed) { 71 case SPEED_10: 72 txoffset = IGC_TXOFFSET_SPEED_10; 73 break; 74 case SPEED_100: 75 txoffset = IGC_TXOFFSET_SPEED_100; 76 break; 77 case SPEED_1000: 78 txoffset = IGC_TXOFFSET_SPEED_1000; 79 break; 80 case SPEED_2500: 81 txoffset = IGC_TXOFFSET_SPEED_2500; 82 break; 83 default: 84 txoffset = 0; 85 break; 86 } 87 88 wr32(IGC_GTXOFFSET, txoffset); 89 } 90 91 static void igc_tsn_restore_retx_default(struct igc_adapter *adapter) 92 { 93 struct igc_hw *hw = &adapter->hw; 94 u32 retxctl; 95 96 retxctl = rd32(IGC_RETX_CTL) & IGC_RETX_CTL_WATERMARK_MASK; 97 wr32(IGC_RETX_CTL, retxctl); 98 } 99 100 bool igc_tsn_is_taprio_activated_by_user(struct igc_adapter *adapter) 101 { 102 struct igc_hw *hw = &adapter->hw; 103 104 return (rd32(IGC_BASET_H) || rd32(IGC_BASET_L)) && 105 adapter->taprio_offload_enable; 106 } 107 108 static void igc_tsn_tx_arb(struct igc_adapter *adapter, u16 *queue_per_tc) 109 { 110 struct igc_hw *hw = &adapter->hw; 111 u32 txarb; 112 113 txarb = rd32(IGC_TXARB); 114 115 txarb &= ~(IGC_TXARB_TXQ_PRIO_0_MASK | 116 IGC_TXARB_TXQ_PRIO_1_MASK | 117 IGC_TXARB_TXQ_PRIO_2_MASK | 118 IGC_TXARB_TXQ_PRIO_3_MASK); 119 120 txarb |= IGC_TXARB_TXQ_PRIO_0(queue_per_tc[3]); 121 txarb |= IGC_TXARB_TXQ_PRIO_1(queue_per_tc[2]); 122 txarb |= IGC_TXARB_TXQ_PRIO_2(queue_per_tc[1]); 123 txarb |= IGC_TXARB_TXQ_PRIO_3(queue_per_tc[0]); 124 125 wr32(IGC_TXARB, txarb); 126 } 127 128 /* Returns the TSN specific registers to their default values after 129 * the adapter is reset. 130 */ 131 static int igc_tsn_disable_offload(struct igc_adapter *adapter) 132 { 133 u16 queue_per_tc[4] = { 3, 2, 1, 0 }; 134 struct igc_hw *hw = &adapter->hw; 135 u32 tqavctrl; 136 int i; 137 138 wr32(IGC_GTXOFFSET, 0); 139 wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); 140 wr32(IGC_DTXMXPKTSZ, IGC_DTXMXPKTSZ_DEFAULT); 141 142 if (igc_is_device_id_i226(hw)) 143 igc_tsn_restore_retx_default(adapter); 144 145 tqavctrl = rd32(IGC_TQAVCTRL); 146 tqavctrl &= ~(IGC_TQAVCTRL_TRANSMIT_MODE_TSN | 147 IGC_TQAVCTRL_ENHANCED_QAV | IGC_TQAVCTRL_FUTSCDDIS); 148 149 wr32(IGC_TQAVCTRL, tqavctrl); 150 151 for (i = 0; i < adapter->num_tx_queues; i++) { 152 wr32(IGC_TXQCTL(i), 0); 153 wr32(IGC_STQT(i), 0); 154 wr32(IGC_ENDQT(i), NSEC_PER_SEC); 155 } 156 157 wr32(IGC_QBVCYCLET_S, 0); 158 wr32(IGC_QBVCYCLET, NSEC_PER_SEC); 159 160 /* Reset mqprio TC configuration. */ 161 netdev_reset_tc(adapter->netdev); 162 163 /* Restore the default Tx arbitration: Priority 0 has the highest 164 * priority and is assigned to queue 0 and so on and so forth. 165 */ 166 igc_tsn_tx_arb(adapter, queue_per_tc); 167 168 adapter->flags &= ~IGC_FLAG_TSN_QBV_ENABLED; 169 adapter->flags &= ~IGC_FLAG_TSN_LEGACY_ENABLED; 170 171 return 0; 172 } 173 174 /* To partially fix i226 HW errata, reduce MAC internal buffering from 192 Bytes 175 * to 88 Bytes by setting RETX_CTL register using the recommendation from: 176 * a) Ethernet Controller I225/I226 Specification Update Rev 2.1 177 * Item 9: TSN: Packet Transmission Might Cross the Qbv Window 178 * b) I225/6 SW User Manual Rev 1.2.4: Section 8.11.5 Retry Buffer Control 179 */ 180 static void igc_tsn_set_retx_qbvfullthreshold(struct igc_adapter *adapter) 181 { 182 struct igc_hw *hw = &adapter->hw; 183 u32 retxctl, watermark; 184 185 retxctl = rd32(IGC_RETX_CTL); 186 watermark = retxctl & IGC_RETX_CTL_WATERMARK_MASK; 187 /* Set QBVFULLTH value using watermark and set QBVFULLEN */ 188 retxctl |= (watermark << IGC_RETX_CTL_QBVFULLTH_SHIFT) | 189 IGC_RETX_CTL_QBVFULLEN; 190 wr32(IGC_RETX_CTL, retxctl); 191 } 192 193 static int igc_tsn_enable_offload(struct igc_adapter *adapter) 194 { 195 struct igc_hw *hw = &adapter->hw; 196 u32 tqavctrl, baset_l, baset_h; 197 u32 sec, nsec, cycle; 198 ktime_t base_time, systim; 199 int i; 200 201 wr32(IGC_TSAUXC, 0); 202 wr32(IGC_DTXMXPKTSZ, IGC_DTXMXPKTSZ_TSN); 203 wr32(IGC_TXPBS, IGC_TXPBSIZE_TSN); 204 205 if (igc_is_device_id_i226(hw)) 206 igc_tsn_set_retx_qbvfullthreshold(adapter); 207 208 if (adapter->strict_priority_enable) { 209 int err; 210 211 err = netdev_set_num_tc(adapter->netdev, adapter->num_tc); 212 if (err) 213 return err; 214 215 for (i = 0; i < adapter->num_tc; i++) { 216 err = netdev_set_tc_queue(adapter->netdev, i, 1, 217 adapter->queue_per_tc[i]); 218 if (err) 219 return err; 220 } 221 222 /* In case the card is configured with less than four queues. */ 223 for (; i < IGC_MAX_TX_QUEUES; i++) 224 adapter->queue_per_tc[i] = i; 225 226 /* Configure queue priorities according to the user provided 227 * mapping. 228 */ 229 igc_tsn_tx_arb(adapter, adapter->queue_per_tc); 230 231 /* Enable legacy TSN mode which will do strict priority without 232 * any other TSN features. 233 */ 234 tqavctrl = rd32(IGC_TQAVCTRL); 235 tqavctrl |= IGC_TQAVCTRL_TRANSMIT_MODE_TSN; 236 tqavctrl &= ~IGC_TQAVCTRL_ENHANCED_QAV; 237 wr32(IGC_TQAVCTRL, tqavctrl); 238 239 return 0; 240 } 241 242 for (i = 0; i < adapter->num_tx_queues; i++) { 243 struct igc_ring *ring = adapter->tx_ring[i]; 244 u32 txqctl = 0; 245 u16 cbs_value; 246 u32 tqavcc; 247 248 wr32(IGC_STQT(i), ring->start_time); 249 wr32(IGC_ENDQT(i), ring->end_time); 250 251 if (adapter->taprio_offload_enable) { 252 /* If taprio_offload_enable is set we are in "taprio" 253 * mode and we need to be strict about the 254 * cycles: only transmit a packet if it can be 255 * completed during that cycle. 256 * 257 * If taprio_offload_enable is NOT true when 258 * enabling TSN offload, the cycle should have 259 * no external effects, but is only used internally 260 * to adapt the base time register after a second 261 * has passed. 262 * 263 * Enabling strict mode in this case would 264 * unnecessarily prevent the transmission of 265 * certain packets (i.e. at the boundary of a 266 * second) and thus interfere with the launchtime 267 * feature that promises transmission at a 268 * certain point in time. 269 */ 270 txqctl |= IGC_TXQCTL_STRICT_CYCLE | 271 IGC_TXQCTL_STRICT_END; 272 } 273 274 if (ring->launchtime_enable) 275 txqctl |= IGC_TXQCTL_QUEUE_MODE_LAUNCHT; 276 277 /* Skip configuring CBS for Q2 and Q3 */ 278 if (i > 1) 279 goto skip_cbs; 280 281 if (ring->cbs_enable) { 282 if (i == 0) 283 txqctl |= IGC_TXQCTL_QAV_SEL_CBS0; 284 else 285 txqctl |= IGC_TXQCTL_QAV_SEL_CBS1; 286 287 /* According to i225 datasheet section 7.5.2.7, we 288 * should set the 'idleSlope' field from TQAVCC 289 * register following the equation: 290 * 291 * value = link-speed 0x7736 * BW * 0.2 292 * ---------- * ----------------- (E1) 293 * 100Mbps 2.5 294 * 295 * Note that 'link-speed' is in Mbps. 296 * 297 * 'BW' is the percentage bandwidth out of full 298 * link speed which can be found with the 299 * following equation. Note that idleSlope here 300 * is the parameter from this function 301 * which is in kbps. 302 * 303 * BW = idleSlope 304 * ----------------- (E2) 305 * link-speed * 1000 306 * 307 * That said, we can come up with a generic 308 * equation to calculate the value we should set 309 * it TQAVCC register by replacing 'BW' in E1 by E2. 310 * The resulting equation is: 311 * 312 * value = link-speed * 0x7736 * idleSlope * 0.2 313 * ------------------------------------- (E3) 314 * 100 * 2.5 * link-speed * 1000 315 * 316 * 'link-speed' is present in both sides of the 317 * fraction so it is canceled out. The final 318 * equation is the following: 319 * 320 * value = idleSlope * 61036 321 * ----------------- (E4) 322 * 2500000 323 * 324 * NOTE: For i225, given the above, we can see 325 * that idleslope is represented in 326 * 40.959433 kbps units by the value at 327 * the TQAVCC register (2.5Gbps / 61036), 328 * which reduces the granularity for 329 * idleslope increments. 330 * 331 * In i225 controller, the sendSlope and loCredit 332 * parameters from CBS are not configurable 333 * by software so we don't do any 334 * 'controller configuration' in respect to 335 * these parameters. 336 */ 337 cbs_value = DIV_ROUND_UP_ULL(ring->idleslope 338 * 61036ULL, 2500000); 339 340 tqavcc = rd32(IGC_TQAVCC(i)); 341 tqavcc &= ~IGC_TQAVCC_IDLESLOPE_MASK; 342 tqavcc |= cbs_value | IGC_TQAVCC_KEEP_CREDITS; 343 wr32(IGC_TQAVCC(i), tqavcc); 344 345 wr32(IGC_TQAVHC(i), 346 0x80000000 + ring->hicredit * 0x7736); 347 } else { 348 /* Disable any CBS for the queue */ 349 txqctl &= ~(IGC_TXQCTL_QAV_SEL_MASK); 350 351 /* Set idleSlope to zero. */ 352 tqavcc = rd32(IGC_TQAVCC(i)); 353 tqavcc &= ~(IGC_TQAVCC_IDLESLOPE_MASK | 354 IGC_TQAVCC_KEEP_CREDITS); 355 wr32(IGC_TQAVCC(i), tqavcc); 356 357 /* Set hiCredit to zero. */ 358 wr32(IGC_TQAVHC(i), 0); 359 } 360 skip_cbs: 361 wr32(IGC_TXQCTL(i), txqctl); 362 } 363 364 tqavctrl = rd32(IGC_TQAVCTRL) & ~IGC_TQAVCTRL_FUTSCDDIS; 365 366 tqavctrl |= IGC_TQAVCTRL_TRANSMIT_MODE_TSN | IGC_TQAVCTRL_ENHANCED_QAV; 367 368 adapter->qbv_count++; 369 370 cycle = adapter->cycle_time; 371 base_time = adapter->base_time; 372 373 nsec = rd32(IGC_SYSTIML); 374 sec = rd32(IGC_SYSTIMH); 375 376 systim = ktime_set(sec, nsec); 377 if (ktime_compare(systim, base_time) > 0) { 378 s64 n = div64_s64(ktime_sub_ns(systim, base_time), cycle); 379 380 base_time = ktime_add_ns(base_time, (n + 1) * cycle); 381 } else { 382 if (igc_is_device_id_i226(hw)) { 383 ktime_t adjust_time, expires_time; 384 385 /* According to datasheet section 7.5.2.9.3.3, FutScdDis bit 386 * has to be configured before the cycle time and base time. 387 * Tx won't hang if a GCL is already running, 388 * so in this case we don't need to set FutScdDis. 389 */ 390 if (!(rd32(IGC_BASET_H) || rd32(IGC_BASET_L))) 391 tqavctrl |= IGC_TQAVCTRL_FUTSCDDIS; 392 393 nsec = rd32(IGC_SYSTIML); 394 sec = rd32(IGC_SYSTIMH); 395 systim = ktime_set(sec, nsec); 396 397 adjust_time = adapter->base_time; 398 expires_time = ktime_sub_ns(adjust_time, systim); 399 hrtimer_start(&adapter->hrtimer, expires_time, HRTIMER_MODE_REL); 400 } 401 } 402 403 wr32(IGC_TQAVCTRL, tqavctrl); 404 405 wr32(IGC_QBVCYCLET_S, cycle); 406 wr32(IGC_QBVCYCLET, cycle); 407 408 baset_h = div_s64_rem(base_time, NSEC_PER_SEC, &baset_l); 409 wr32(IGC_BASET_H, baset_h); 410 411 /* In i226, Future base time is only supported when FutScdDis bit 412 * is enabled and only active for re-configuration. 413 * In this case, initialize the base time with zero to create 414 * "re-configuration" scenario then only set the desired base time. 415 */ 416 if (tqavctrl & IGC_TQAVCTRL_FUTSCDDIS) 417 wr32(IGC_BASET_L, 0); 418 wr32(IGC_BASET_L, baset_l); 419 420 return 0; 421 } 422 423 int igc_tsn_reset(struct igc_adapter *adapter) 424 { 425 unsigned int new_flags; 426 int err = 0; 427 428 new_flags = igc_tsn_new_flags(adapter); 429 430 if (!(new_flags & IGC_FLAG_TSN_ANY_ENABLED)) 431 return igc_tsn_disable_offload(adapter); 432 433 err = igc_tsn_enable_offload(adapter); 434 if (err < 0) 435 return err; 436 437 adapter->flags = new_flags; 438 439 return err; 440 } 441 442 static bool igc_tsn_will_tx_mode_change(struct igc_adapter *adapter) 443 { 444 bool any_tsn_enabled = !!(igc_tsn_new_flags(adapter) & 445 IGC_FLAG_TSN_ANY_ENABLED); 446 447 return (any_tsn_enabled && !igc_tsn_is_tx_mode_in_tsn(adapter)) || 448 (!any_tsn_enabled && igc_tsn_is_tx_mode_in_tsn(adapter)); 449 } 450 451 int igc_tsn_offload_apply(struct igc_adapter *adapter) 452 { 453 /* Per I225/6 HW Design Section 7.5.2.1 guideline, if tx mode change 454 * from legacy->tsn or tsn->legacy, then reset adapter is needed. 455 */ 456 if (netif_running(adapter->netdev) && 457 igc_tsn_will_tx_mode_change(adapter)) { 458 schedule_work(&adapter->reset_task); 459 return 0; 460 } 461 462 igc_tsn_reset(adapter); 463 464 return 0; 465 } 466