1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2015 - 2025 Beijing WangXun Technology Co., Ltd. */
3 /* Copyright (c) 1999 - 2025 Intel Corporation. */
4
5 #include <linux/ptp_classify.h>
6 #include <linux/clocksource.h>
7 #include <linux/pci.h>
8
9 #include "wx_type.h"
10 #include "wx_ptp.h"
11 #include "wx_hw.h"
12
13 #define WX_INCVAL_10GB 0xCCCCCC
14 #define WX_INCVAL_1GB 0x800000
15 #define WX_INCVAL_100 0xA00000
16 #define WX_INCVAL_10 0xC7F380
17 #define WX_INCVAL_EM 0x2000000
18 #define WX_INCVAL_AML 0xA00000
19
20 #define WX_INCVAL_SHIFT_10GB 20
21 #define WX_INCVAL_SHIFT_1GB 18
22 #define WX_INCVAL_SHIFT_100 15
23 #define WX_INCVAL_SHIFT_10 12
24 #define WX_INCVAL_SHIFT_EM 22
25 #define WX_INCVAL_SHIFT_AML 21
26
27 #define WX_OVERFLOW_PERIOD (HZ * 30)
28 #define WX_PTP_TX_TIMEOUT (HZ)
29
30 #define WX_1588_PPS_WIDTH_EM 120
31
32 #define WX_NS_PER_SEC 1000000000ULL
33
wx_ptp_timecounter_cyc2time(struct wx * wx,u64 timestamp)34 static u64 wx_ptp_timecounter_cyc2time(struct wx *wx, u64 timestamp)
35 {
36 unsigned int seq;
37 u64 ns;
38
39 do {
40 seq = read_seqbegin(&wx->hw_tc_lock);
41 ns = timecounter_cyc2time(&wx->hw_tc, timestamp);
42 } while (read_seqretry(&wx->hw_tc_lock, seq));
43
44 return ns;
45 }
46
wx_ptp_readtime(struct wx * wx,struct ptp_system_timestamp * sts)47 static u64 wx_ptp_readtime(struct wx *wx, struct ptp_system_timestamp *sts)
48 {
49 u32 timeh1, timeh2, timel;
50
51 timeh1 = rd32ptp(wx, WX_TSC_1588_SYSTIMH);
52 ptp_read_system_prets(sts);
53 timel = rd32ptp(wx, WX_TSC_1588_SYSTIML);
54 ptp_read_system_postts(sts);
55 timeh2 = rd32ptp(wx, WX_TSC_1588_SYSTIMH);
56
57 if (timeh1 != timeh2) {
58 ptp_read_system_prets(sts);
59 timel = rd32ptp(wx, WX_TSC_1588_SYSTIML);
60 ptp_read_system_prets(sts);
61 }
62 return (u64)timel | (u64)timeh2 << 32;
63 }
64
wx_ptp_adjfine(struct ptp_clock_info * ptp,long ppb)65 static int wx_ptp_adjfine(struct ptp_clock_info *ptp, long ppb)
66 {
67 struct wx *wx = container_of(ptp, struct wx, ptp_caps);
68 u64 incval, mask;
69
70 smp_mb(); /* Force any pending update before accessing. */
71 incval = READ_ONCE(wx->base_incval);
72 incval = adjust_by_scaled_ppm(incval, ppb);
73
74 mask = (wx->mac.type == wx_mac_em) ? 0x7FFFFFF : 0xFFFFFF;
75 incval &= mask;
76 if (wx->mac.type != wx_mac_em)
77 incval |= 2 << 24;
78
79 wr32ptp(wx, WX_TSC_1588_INC, incval);
80
81 return 0;
82 }
83
wx_ptp_adjtime(struct ptp_clock_info * ptp,s64 delta)84 static int wx_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
85 {
86 struct wx *wx = container_of(ptp, struct wx, ptp_caps);
87 unsigned long flags;
88
89 write_seqlock_irqsave(&wx->hw_tc_lock, flags);
90 timecounter_adjtime(&wx->hw_tc, delta);
91 write_sequnlock_irqrestore(&wx->hw_tc_lock, flags);
92
93 if (wx->ptp_setup_sdp)
94 wx->ptp_setup_sdp(wx);
95
96 return 0;
97 }
98
wx_ptp_gettimex64(struct ptp_clock_info * ptp,struct timespec64 * ts,struct ptp_system_timestamp * sts)99 static int wx_ptp_gettimex64(struct ptp_clock_info *ptp,
100 struct timespec64 *ts,
101 struct ptp_system_timestamp *sts)
102 {
103 struct wx *wx = container_of(ptp, struct wx, ptp_caps);
104 u64 ns, stamp;
105
106 stamp = wx_ptp_readtime(wx, sts);
107 ns = wx_ptp_timecounter_cyc2time(wx, stamp);
108 *ts = ns_to_timespec64(ns);
109
110 return 0;
111 }
112
wx_ptp_settime64(struct ptp_clock_info * ptp,const struct timespec64 * ts)113 static int wx_ptp_settime64(struct ptp_clock_info *ptp,
114 const struct timespec64 *ts)
115 {
116 struct wx *wx = container_of(ptp, struct wx, ptp_caps);
117 unsigned long flags;
118 u64 ns;
119
120 ns = timespec64_to_ns(ts);
121 /* reset the timecounter */
122 write_seqlock_irqsave(&wx->hw_tc_lock, flags);
123 timecounter_init(&wx->hw_tc, &wx->hw_cc, ns);
124 write_sequnlock_irqrestore(&wx->hw_tc_lock, flags);
125
126 if (wx->ptp_setup_sdp)
127 wx->ptp_setup_sdp(wx);
128
129 return 0;
130 }
131
132 /**
133 * __wx_ptp_detach_tx_skb - detach the skb tracking the Tx timestamp request
134 * @wx: the private board structure
135 *
136 * Detach the skb of the outstanding request and release the in-progress bit,
137 * so that a new request can be submitted.
138 *
139 * This performs no register access. Callers that need a timestamp the hardware
140 * may have left latched must unlatch it themselves, while the device is known
141 * to be alive. wx_ptp_quiesce() runs during PCIe error recovery, where MMIO is
142 * not reliable, and therefore deliberately skips the unlatch.
143 *
144 * Context: Expects wx->ptp_tx_lock to be held by the caller.
145 * Return: the detached skb, or NULL if no request was outstanding. The caller
146 * owns the returned reference and must release it once the lock is dropped.
147 */
__wx_ptp_detach_tx_skb(struct wx * wx)148 static struct sk_buff *__wx_ptp_detach_tx_skb(struct wx *wx)
149 {
150 struct sk_buff *skb = wx->ptp_tx_skb;
151
152 lockdep_assert_held(&wx->ptp_tx_lock);
153
154 wx->ptp_tx_skb = NULL;
155 clear_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state);
156
157 return skb;
158 }
159
160 /**
161 * wx_ptp_clear_tx_timestamp - utility function to clear Tx timestamp state
162 * @wx: the private board structure
163 *
164 * This function should be called whenever the state related to a Tx timestamp
165 * needs to be cleared. This helps ensure that all related bits are reset for
166 * the next Tx timestamp event.
167 */
wx_ptp_clear_tx_timestamp(struct wx * wx)168 static void wx_ptp_clear_tx_timestamp(struct wx *wx)
169 {
170 struct sk_buff *skb;
171 unsigned long flags;
172
173 spin_lock_irqsave(&wx->ptp_tx_lock, flags);
174 /* Unlatch a timestamp the hardware may have left pending. */
175 rd32ptp(wx, WX_TSC_1588_STMPH);
176 skb = __wx_ptp_detach_tx_skb(wx);
177 spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
178
179 dev_kfree_skb_any(skb);
180 }
181
182 /**
183 * wx_ptp_convert_to_hwtstamp - convert register value to hw timestamp
184 * @wx: private board structure
185 * @hwtstamp: stack timestamp structure
186 * @timestamp: unsigned 64bit system time value
187 *
188 * We need to convert the adapter's RX/TXSTMP registers into a hwtstamp value
189 * which can be used by the stack's ptp functions.
190 *
191 * The lock is used to protect consistency of the cyclecounter and the SYSTIME
192 * registers. However, it does not need to protect against the Rx or Tx
193 * timestamp registers, as there can't be a new timestamp until the old one is
194 * unlatched by reading.
195 *
196 * In addition to the timestamp in hardware, some controllers need a software
197 * overflow cyclecounter, and this function takes this into account as well.
198 **/
wx_ptp_convert_to_hwtstamp(struct wx * wx,struct skb_shared_hwtstamps * hwtstamp,u64 timestamp)199 static void wx_ptp_convert_to_hwtstamp(struct wx *wx,
200 struct skb_shared_hwtstamps *hwtstamp,
201 u64 timestamp)
202 {
203 u64 ns;
204
205 ns = wx_ptp_timecounter_cyc2time(wx, timestamp);
206 hwtstamp->hwtstamp = ns_to_ktime(ns);
207 }
208
209 /**
210 * wx_ptp_tx_hwtstamp_work - check for a pending Tx time stamp
211 * @wx: the private board struct
212 *
213 * If a Tx timestamp request is outstanding and the hardware has latched a
214 * valid value, we convert it into the timecounter ns value, then store that
215 * result into the shhwtstamps structure which is passed up the network stack.
216 *
217 * Return: 0 when there is nothing left to poll for, -1 when the timestamp is
218 * not available yet and the caller should poll again.
219 */
wx_ptp_tx_hwtstamp_work(struct wx * wx)220 static int wx_ptp_tx_hwtstamp_work(struct wx *wx)
221 {
222 struct skb_shared_hwtstamps shhwtstamps;
223 unsigned long flags;
224 struct sk_buff *skb;
225 u32 tsynctxctl;
226 u64 regval = 0;
227
228 spin_lock_irqsave(&wx->ptp_tx_lock, flags);
229
230 /* we have to have a valid skb to poll for a timestamp */
231 if (!wx->ptp_tx_skb) {
232 rd32ptp(wx, WX_TSC_1588_STMPH);
233 __wx_ptp_detach_tx_skb(wx);
234 spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
235 return 0;
236 }
237
238 /* stop polling once we have a valid timestamp */
239 tsynctxctl = rd32ptp(wx, WX_TSC_1588_CTL);
240 if (!(tsynctxctl & WX_TSC_1588_CTL_VALID)) {
241 spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
242 return -1;
243 }
244
245 regval |= (u64)rd32ptp(wx, WX_TSC_1588_STMPL);
246 regval |= (u64)rd32ptp(wx, WX_TSC_1588_STMPH) << 32;
247 skb = wx->ptp_tx_skb;
248 wx->ptp_tx_skb = NULL;
249 clear_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state);
250 spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
251
252 wx_ptp_convert_to_hwtstamp(wx, &shhwtstamps, regval);
253 skb_tstamp_tx(skb, &shhwtstamps);
254 dev_kfree_skb_any(skb);
255 wx->tx_hwtstamp_pkts++;
256
257 return 0;
258 }
259
260 /**
261 * wx_ptp_overflow_check - watchdog task to detect SYSTIME overflow
262 * @wx: pointer to wx struct
263 *
264 * this watchdog task periodically reads the timecounter
265 * in order to prevent missing when the system time registers wrap
266 * around. This needs to be run approximately twice a minute for the fastest
267 * overflowing hardware. We run it for all hardware since it shouldn't have a
268 * large impact.
269 */
wx_ptp_overflow_check(struct wx * wx)270 static void wx_ptp_overflow_check(struct wx *wx)
271 {
272 bool timeout = time_is_before_jiffies(wx->last_overflow_check +
273 WX_OVERFLOW_PERIOD);
274 unsigned long flags;
275
276 if (timeout) {
277 /* Update the timecounter */
278 write_seqlock_irqsave(&wx->hw_tc_lock, flags);
279 timecounter_read(&wx->hw_tc);
280 write_sequnlock_irqrestore(&wx->hw_tc_lock, flags);
281
282 wx->last_overflow_check = jiffies;
283 }
284 }
285
286 /**
287 * wx_ptp_rx_hang - detect error case when Rx timestamp registers latched
288 * @wx: pointer to wx struct
289 *
290 * this watchdog task is scheduled to detect error case where hardware has
291 * dropped an Rx packet that was timestamped when the ring is full. The
292 * particular error is rare but leaves the device in a state unable to
293 * timestamp any future packets.
294 */
wx_ptp_rx_hang(struct wx * wx)295 static void wx_ptp_rx_hang(struct wx *wx)
296 {
297 struct wx_ring *rx_ring;
298 unsigned long rx_event;
299 u32 tsyncrxctl;
300 int n;
301
302 tsyncrxctl = rd32(wx, WX_PSR_1588_CTL);
303
304 /* if we don't have a valid timestamp in the registers, just update the
305 * timeout counter and exit
306 */
307 if (!(tsyncrxctl & WX_PSR_1588_CTL_VALID)) {
308 wx->last_rx_ptp_check = jiffies;
309 return;
310 }
311
312 /* determine the most recent watchdog or rx_timestamp event */
313 rx_event = wx->last_rx_ptp_check;
314 for (n = 0; n < wx->num_rx_queues; n++) {
315 rx_ring = wx->rx_ring[n];
316 if (time_after(rx_ring->last_rx_timestamp, rx_event))
317 rx_event = rx_ring->last_rx_timestamp;
318 }
319
320 /* only need to read the high RXSTMP register to clear the lock */
321 if (time_is_before_jiffies(rx_event + 5 * HZ)) {
322 rd32(wx, WX_PSR_1588_STMPH);
323 wx->last_rx_ptp_check = jiffies;
324
325 wx->rx_hwtstamp_cleared++;
326 dev_warn(&wx->pdev->dev, "clearing RX Timestamp hang");
327 }
328 }
329
330 /**
331 * wx_ptp_tx_hang - detect error case where Tx timestamp never finishes
332 * @wx: private network wx structure
333 */
wx_ptp_tx_hang(struct wx * wx)334 static void wx_ptp_tx_hang(struct wx *wx)
335 {
336 struct sk_buff *skb = NULL;
337 unsigned long flags;
338
339 spin_lock_irqsave(&wx->ptp_tx_lock, flags);
340
341 /* If we haven't received a timestamp within the timeout, it is
342 * reasonable to assume that it will never occur, so we can unlock the
343 * timestamp bit when this occurs.
344 */
345 if (wx->ptp_tx_skb &&
346 test_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state) &&
347 time_is_before_jiffies(wx->ptp_tx_start + WX_PTP_TX_TIMEOUT)) {
348 rd32ptp(wx, WX_TSC_1588_STMPH);
349 skb = __wx_ptp_detach_tx_skb(wx);
350 }
351 spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
352
353 if (!skb)
354 return;
355
356 dev_kfree_skb_any(skb);
357 wx->tx_hwtstamp_timeouts++;
358 dev_warn(&wx->pdev->dev, "clearing Tx timestamp hang\n");
359 }
360
wx_ptp_do_aux_work(struct ptp_clock_info * ptp)361 static long wx_ptp_do_aux_work(struct ptp_clock_info *ptp)
362 {
363 struct wx *wx = container_of(ptp, struct wx, ptp_caps);
364 int ts_done;
365
366 if (!test_bit(WX_STATE_PTP_RUNNING, wx->state))
367 return HZ;
368
369 ts_done = wx_ptp_tx_hwtstamp_work(wx);
370
371 wx_ptp_overflow_check(wx);
372 if (unlikely(test_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER,
373 wx->flags)))
374 wx_ptp_rx_hang(wx);
375 wx_ptp_tx_hang(wx);
376
377 return ts_done ? 1 : HZ;
378 }
379
wx_ptp_trigger_calc(struct wx * wx)380 static u64 wx_ptp_trigger_calc(struct wx *wx)
381 {
382 struct cyclecounter *cc = &wx->hw_cc;
383 unsigned long flags;
384 u64 ns = 0;
385 u32 rem;
386
387 /* Read the current clock time, and save the cycle counter value */
388 write_seqlock_irqsave(&wx->hw_tc_lock, flags);
389 ns = timecounter_read(&wx->hw_tc);
390 wx->pps_edge_start = wx->hw_tc.cycle_last;
391 write_sequnlock_irqrestore(&wx->hw_tc_lock, flags);
392 wx->pps_edge_end = wx->pps_edge_start;
393
394 /* Figure out how far past the next second we are */
395 div_u64_rem(ns, WX_NS_PER_SEC, &rem);
396
397 /* Figure out how many nanoseconds to add to round the clock edge up
398 * to the next full second
399 */
400 rem = (WX_NS_PER_SEC - rem);
401
402 /* Adjust the clock edge to align with the next full second. */
403 wx->pps_edge_start += div_u64(((u64)rem << cc->shift), cc->mult);
404 wx->pps_edge_end += div_u64(((u64)(rem + wx->pps_width) <<
405 cc->shift), cc->mult);
406
407 return (ns + rem);
408 }
409
wx_ptp_setup_sdp(struct wx * wx)410 static int wx_ptp_setup_sdp(struct wx *wx)
411 {
412 struct cyclecounter *cc = &wx->hw_cc;
413 u32 tsauxc;
414 u64 nsec;
415
416 if (wx->pps_width >= WX_NS_PER_SEC) {
417 wx_err(wx, "PTP pps width cannot be longer than 1s!\n");
418 return -EINVAL;
419 }
420
421 /* disable the pin first */
422 wr32ptp(wx, WX_TSC_1588_AUX_CTL, 0);
423 WX_WRITE_FLUSH(wx);
424
425 if (!test_bit(WX_FLAG_PTP_PPS_ENABLED, wx->flags)) {
426 if (wx->pps_enabled) {
427 wx->pps_enabled = false;
428 wx_set_pps(wx, false, 0, 0);
429 }
430 return 0;
431 }
432
433 wx->pps_enabled = true;
434 nsec = wx_ptp_trigger_calc(wx);
435 wx_set_pps(wx, wx->pps_enabled, nsec, wx->pps_edge_start);
436
437 tsauxc = WX_TSC_1588_AUX_CTL_PLSG | WX_TSC_1588_AUX_CTL_EN_TT0 |
438 WX_TSC_1588_AUX_CTL_EN_TT1 | WX_TSC_1588_AUX_CTL_EN_TS0;
439 wr32ptp(wx, WX_TSC_1588_TRGT_L(0), (u32)wx->pps_edge_start);
440 wr32ptp(wx, WX_TSC_1588_TRGT_H(0), (u32)(wx->pps_edge_start >> 32));
441 wr32ptp(wx, WX_TSC_1588_TRGT_L(1), (u32)wx->pps_edge_end);
442 wr32ptp(wx, WX_TSC_1588_TRGT_H(1), (u32)(wx->pps_edge_end >> 32));
443 wr32ptp(wx, WX_TSC_1588_SDP(0),
444 WX_TSC_1588_SDP_FUN_SEL_TT0 | WX_TSC_1588_SDP_OUT_LEVEL_H);
445 wr32ptp(wx, WX_TSC_1588_SDP(1), WX_TSC_1588_SDP_FUN_SEL_TS0);
446 wr32ptp(wx, WX_TSC_1588_AUX_CTL, tsauxc);
447 wr32ptp(wx, WX_TSC_1588_INT_EN, WX_TSC_1588_INT_EN_TT1);
448 WX_WRITE_FLUSH(wx);
449
450 /* Adjust the clock edge to align with the next full second. */
451 wx->sec_to_cc = div_u64(((u64)WX_NS_PER_SEC << cc->shift), cc->mult);
452
453 return 0;
454 }
455
wx_ptp_feature_enable(struct ptp_clock_info * ptp,struct ptp_clock_request * rq,int on)456 static int wx_ptp_feature_enable(struct ptp_clock_info *ptp,
457 struct ptp_clock_request *rq, int on)
458 {
459 struct wx *wx = container_of(ptp, struct wx, ptp_caps);
460
461 /**
462 * When PPS is enabled, unmask the interrupt for the ClockOut
463 * feature, so that the interrupt handler can send the PPS
464 * event when the clock SDP triggers. Clear mask when PPS is
465 * disabled
466 */
467 if (rq->type != PTP_CLK_REQ_PEROUT || !wx->ptp_setup_sdp)
468 return -EOPNOTSUPP;
469
470 /* Reject requests with unsupported flags */
471 if (rq->perout.flags & ~(PTP_PEROUT_DUTY_CYCLE |
472 PTP_PEROUT_PHASE))
473 return -EOPNOTSUPP;
474
475 if (rq->perout.phase.sec || rq->perout.phase.nsec) {
476 wx_err(wx, "Absolute start time not supported.\n");
477 return -EINVAL;
478 }
479
480 if (rq->perout.period.sec != 1 || rq->perout.period.nsec) {
481 wx_err(wx, "Only 1pps is supported.\n");
482 return -EINVAL;
483 }
484
485 if (rq->perout.flags & PTP_PEROUT_DUTY_CYCLE) {
486 struct timespec64 ts_on;
487
488 ts_on.tv_sec = rq->perout.on.sec;
489 ts_on.tv_nsec = rq->perout.on.nsec;
490 wx->pps_width = timespec64_to_ns(&ts_on);
491 } else {
492 wx->pps_width = 120000000;
493 }
494
495 if (on)
496 set_bit(WX_FLAG_PTP_PPS_ENABLED, wx->flags);
497 else
498 clear_bit(WX_FLAG_PTP_PPS_ENABLED, wx->flags);
499
500 return wx->ptp_setup_sdp(wx);
501 }
502
wx_ptp_check_pps_event(struct wx * wx)503 void wx_ptp_check_pps_event(struct wx *wx)
504 {
505 u32 tsauxc, int_status;
506
507 /* this check is necessary in case the interrupt was enabled via some
508 * alternative means (ex. debug_fs). Better to check here than
509 * everywhere that calls this function.
510 */
511 if (!wx->ptp_clock)
512 return;
513
514 int_status = rd32ptp(wx, WX_TSC_1588_INT_ST);
515 if (int_status & WX_TSC_1588_INT_ST_TT1) {
516 /* disable the pin first */
517 wr32ptp(wx, WX_TSC_1588_AUX_CTL, 0);
518 WX_WRITE_FLUSH(wx);
519
520 wx_ptp_trigger_calc(wx);
521
522 tsauxc = WX_TSC_1588_AUX_CTL_PLSG | WX_TSC_1588_AUX_CTL_EN_TT0 |
523 WX_TSC_1588_AUX_CTL_EN_TT1 | WX_TSC_1588_AUX_CTL_EN_TS0;
524 wr32ptp(wx, WX_TSC_1588_TRGT_L(0), (u32)wx->pps_edge_start);
525 wr32ptp(wx, WX_TSC_1588_TRGT_H(0), (u32)(wx->pps_edge_start >> 32));
526 wr32ptp(wx, WX_TSC_1588_TRGT_L(1), (u32)wx->pps_edge_end);
527 wr32ptp(wx, WX_TSC_1588_TRGT_H(1), (u32)(wx->pps_edge_end >> 32));
528 wr32ptp(wx, WX_TSC_1588_AUX_CTL, tsauxc);
529 WX_WRITE_FLUSH(wx);
530 }
531 }
532 EXPORT_SYMBOL(wx_ptp_check_pps_event);
533
wx_ptp_create_clock(struct wx * wx)534 static long wx_ptp_create_clock(struct wx *wx)
535 {
536 struct net_device *netdev = wx->netdev;
537 long err;
538
539 /* do nothing if we already have a clock device */
540 if (!IS_ERR_OR_NULL(wx->ptp_clock))
541 return 0;
542
543 snprintf(wx->ptp_caps.name, sizeof(wx->ptp_caps.name),
544 "%s", netdev->name);
545 wx->ptp_caps.owner = THIS_MODULE;
546 wx->ptp_caps.n_alarm = 0;
547 wx->ptp_caps.n_ext_ts = 0;
548 wx->ptp_caps.pps = 0;
549 wx->ptp_caps.adjfine = wx_ptp_adjfine;
550 wx->ptp_caps.adjtime = wx_ptp_adjtime;
551 wx->ptp_caps.gettimex64 = wx_ptp_gettimex64;
552 wx->ptp_caps.settime64 = wx_ptp_settime64;
553 wx->ptp_caps.do_aux_work = wx_ptp_do_aux_work;
554 switch (wx->mac.type) {
555 case wx_mac_aml:
556 case wx_mac_aml40:
557 wx->ptp_caps.max_adj = 250000000;
558 wx->ptp_caps.n_per_out = 1;
559 wx->ptp_setup_sdp = wx_ptp_setup_sdp;
560 wx->ptp_caps.enable = wx_ptp_feature_enable;
561 break;
562 case wx_mac_sp:
563 wx->ptp_caps.max_adj = 250000000;
564 wx->ptp_caps.n_per_out = 0;
565 wx->ptp_setup_sdp = NULL;
566 break;
567 case wx_mac_em:
568 wx->ptp_caps.max_adj = 500000000;
569 wx->ptp_caps.n_per_out = 1;
570 wx->ptp_setup_sdp = wx_ptp_setup_sdp;
571 wx->ptp_caps.enable = wx_ptp_feature_enable;
572 break;
573 default:
574 return -EOPNOTSUPP;
575 }
576
577 wx->ptp_clock = ptp_clock_register(&wx->ptp_caps, &wx->pdev->dev);
578 if (IS_ERR(wx->ptp_clock)) {
579 err = PTR_ERR(wx->ptp_clock);
580 wx->ptp_clock = NULL;
581 wx_err(wx, "ptp clock register failed\n");
582 return err;
583 } else if (wx->ptp_clock) {
584 dev_info(&wx->pdev->dev, "registered PHC device on %s\n",
585 netdev->name);
586 }
587
588 /* Set the default timestamp mode to disabled here. We do this in
589 * create_clock instead of initialization, because we don't want to
590 * override the previous settings during a suspend/resume cycle.
591 */
592 wx->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
593 wx->tstamp_config.tx_type = HWTSTAMP_TX_OFF;
594
595 return 0;
596 }
597
wx_ptp_set_timestamp_mode(struct wx * wx,struct kernel_hwtstamp_config * config)598 static int wx_ptp_set_timestamp_mode(struct wx *wx,
599 struct kernel_hwtstamp_config *config)
600 {
601 u32 tsync_tx_ctl = WX_TSC_1588_CTL_ENABLED;
602 u32 tsync_rx_ctl = WX_PSR_1588_CTL_ENABLED;
603 u32 tsync_rx_mtrl = PTP_EV_PORT << 16;
604 bool rx_tstamp = false;
605 bool is_l2 = false;
606 u32 regval;
607
608 switch (config->tx_type) {
609 case HWTSTAMP_TX_OFF:
610 tsync_tx_ctl = 0;
611 break;
612 case HWTSTAMP_TX_ON:
613 break;
614 default:
615 return -ERANGE;
616 }
617
618 switch (config->rx_filter) {
619 case HWTSTAMP_FILTER_NONE:
620 tsync_rx_ctl = 0;
621 tsync_rx_mtrl = 0;
622 break;
623 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
624 tsync_rx_ctl |= WX_PSR_1588_CTL_TYPE_L4_V1;
625 tsync_rx_mtrl |= WX_PSR_1588_MSG_V1_SYNC;
626 rx_tstamp = true;
627 break;
628 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
629 tsync_rx_ctl |= WX_PSR_1588_CTL_TYPE_L4_V1;
630 tsync_rx_mtrl |= WX_PSR_1588_MSG_V1_DELAY_REQ;
631 rx_tstamp = true;
632 break;
633 case HWTSTAMP_FILTER_PTP_V2_EVENT:
634 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
635 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
636 case HWTSTAMP_FILTER_PTP_V2_SYNC:
637 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
638 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
639 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
640 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
641 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
642 tsync_rx_ctl |= WX_PSR_1588_CTL_TYPE_EVENT_V2;
643 is_l2 = true;
644 rx_tstamp = true;
645 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
646 break;
647 default:
648 /* register PSR_1588_MSG must be set in order to do V1 packets,
649 * therefore it is not possible to time stamp both V1 Sync and
650 * Delay_Req messages unless hardware supports timestamping all
651 * packets => return error
652 */
653 config->rx_filter = HWTSTAMP_FILTER_NONE;
654 return -ERANGE;
655 }
656
657 /* define ethertype filter for timestamping L2 packets */
658 if (is_l2)
659 wr32(wx, WX_PSR_ETYPE_SWC(WX_PSR_ETYPE_SWC_FILTER_1588),
660 (WX_PSR_ETYPE_SWC_FILTER_EN | /* enable filter */
661 WX_PSR_ETYPE_SWC_1588 | /* enable timestamping */
662 ETH_P_1588)); /* 1588 eth protocol type */
663 else
664 wr32(wx, WX_PSR_ETYPE_SWC(WX_PSR_ETYPE_SWC_FILTER_1588), 0);
665
666 /* enable/disable TX */
667 regval = rd32ptp(wx, WX_TSC_1588_CTL);
668 regval &= ~WX_TSC_1588_CTL_ENABLED;
669 regval |= tsync_tx_ctl;
670 wr32ptp(wx, WX_TSC_1588_CTL, regval);
671
672 /* enable/disable RX */
673 regval = rd32(wx, WX_PSR_1588_CTL);
674 regval &= ~(WX_PSR_1588_CTL_ENABLED | WX_PSR_1588_CTL_TYPE_MASK);
675 regval |= tsync_rx_ctl;
676 wr32(wx, WX_PSR_1588_CTL, regval);
677
678 /* define which PTP packets are time stamped */
679 wr32(wx, WX_PSR_1588_MSG, tsync_rx_mtrl);
680
681 WX_WRITE_FLUSH(wx);
682
683 /* configure adapter flags only when HW is actually configured */
684 assign_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, wx->flags, rx_tstamp);
685 assign_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, wx->flags, rx_tstamp);
686
687 /* clear TX/RX timestamp state, just to be sure */
688 wx_ptp_clear_tx_timestamp(wx);
689 rd32(wx, WX_PSR_1588_STMPH);
690
691 return 0;
692 }
693
wx_ptp_read(struct cyclecounter * hw_cc)694 static u64 wx_ptp_read(struct cyclecounter *hw_cc)
695 {
696 struct wx *wx = container_of(hw_cc, struct wx, hw_cc);
697
698 return wx_ptp_readtime(wx, NULL);
699 }
700
wx_ptp_link_speed_adjust(struct wx * wx,u32 * shift,u32 * incval)701 static void wx_ptp_link_speed_adjust(struct wx *wx, u32 *shift, u32 *incval)
702 {
703 switch (wx->mac.type) {
704 case wx_mac_aml:
705 case wx_mac_aml40:
706 *shift = WX_INCVAL_SHIFT_AML;
707 *incval = WX_INCVAL_AML;
708 return;
709 case wx_mac_em:
710 *shift = WX_INCVAL_SHIFT_EM;
711 *incval = WX_INCVAL_EM;
712 return;
713 default:
714 break;
715 }
716
717 switch (wx->speed) {
718 case SPEED_10:
719 *shift = WX_INCVAL_SHIFT_10;
720 *incval = WX_INCVAL_10;
721 break;
722 case SPEED_100:
723 *shift = WX_INCVAL_SHIFT_100;
724 *incval = WX_INCVAL_100;
725 break;
726 case SPEED_1000:
727 *shift = WX_INCVAL_SHIFT_1GB;
728 *incval = WX_INCVAL_1GB;
729 break;
730 case SPEED_10000:
731 default:
732 *shift = WX_INCVAL_SHIFT_10GB;
733 *incval = WX_INCVAL_10GB;
734 break;
735 }
736 }
737
738 /**
739 * wx_ptp_reset_cyclecounter - create the cycle counter from hw
740 * @wx: pointer to the wx structure
741 *
742 * This function should be called to set the proper values for the TSC_1588_INC
743 * register and tell the cyclecounter structure what the tick rate of SYSTIME
744 * is. It does not directly modify SYSTIME registers or the timecounter
745 * structure. It should be called whenever a new TSC_1588_INC value is
746 * necessary, such as during initialization or when the link speed changes.
747 */
wx_ptp_reset_cyclecounter(struct wx * wx)748 void wx_ptp_reset_cyclecounter(struct wx *wx)
749 {
750 u32 incval = 0, mask = 0;
751 struct cyclecounter cc;
752 unsigned long flags;
753
754 /* For some of the boards below this mask is technically incorrect.
755 * The timestamp mask overflows at approximately 61bits. However the
756 * particular hardware does not overflow on an even bitmask value.
757 * Instead, it overflows due to conversion of upper 32bits billions of
758 * cycles. Timecounters are not really intended for this purpose so
759 * they do not properly function if the overflow point isn't 2^N-1.
760 * However, the actual SYSTIME values in question take ~138 years to
761 * overflow. In practice this means they won't actually overflow. A
762 * proper fix to this problem would require modification of the
763 * timecounter delta calculations.
764 */
765 cc.mask = CLOCKSOURCE_MASK(64);
766 cc.mult = 1;
767 cc.shift = 0;
768
769 cc.read = wx_ptp_read;
770 wx_ptp_link_speed_adjust(wx, &cc.shift, &incval);
771
772 /* update the base incval used to calculate frequency adjustment */
773 WRITE_ONCE(wx->base_incval, incval);
774
775 mask = (wx->mac.type == wx_mac_em) ? 0x7FFFFFF : 0xFFFFFF;
776 incval &= mask;
777 if (wx->mac.type != wx_mac_em)
778 incval |= 2 << 24;
779 wr32ptp(wx, WX_TSC_1588_INC, incval);
780
781 smp_mb(); /* Force the above update. */
782
783 /* need lock to prevent incorrect read while modifying cyclecounter */
784 write_seqlock_irqsave(&wx->hw_tc_lock, flags);
785 memcpy(&wx->hw_cc, &cc, sizeof(wx->hw_cc));
786 write_sequnlock_irqrestore(&wx->hw_tc_lock, flags);
787 }
788 EXPORT_SYMBOL(wx_ptp_reset_cyclecounter);
789
wx_ptp_reset(struct wx * wx)790 void wx_ptp_reset(struct wx *wx)
791 {
792 unsigned long flags;
793
794 /* reset the hardware timestamping mode */
795 wx_ptp_set_timestamp_mode(wx, &wx->tstamp_config);
796 wx_ptp_reset_cyclecounter(wx);
797
798 wr32ptp(wx, WX_TSC_1588_SYSTIML, 0);
799 wr32ptp(wx, WX_TSC_1588_SYSTIMH, 0);
800 WX_WRITE_FLUSH(wx);
801
802 write_seqlock_irqsave(&wx->hw_tc_lock, flags);
803 timecounter_init(&wx->hw_tc, &wx->hw_cc,
804 ktime_to_ns(ktime_get_real()));
805 write_sequnlock_irqrestore(&wx->hw_tc_lock, flags);
806
807 wx->last_overflow_check = jiffies;
808 ptp_schedule_worker(wx->ptp_clock, HZ);
809
810 /* Now that the shift has been calculated and the systime
811 * registers reset, (re-)enable the Clock out feature
812 */
813 if (wx->ptp_setup_sdp)
814 wx->ptp_setup_sdp(wx);
815 }
816 EXPORT_SYMBOL(wx_ptp_reset);
817
wx_ptp_init(struct wx * wx)818 void wx_ptp_init(struct wx *wx)
819 {
820 /* Initialize the seqlock_t first, since the user might call the clock
821 * functions any time after we've initialized the ptp clock device.
822 */
823 seqlock_init(&wx->hw_tc_lock);
824
825 /* obtain a ptp clock device, or re-use an existing device */
826 if (wx_ptp_create_clock(wx))
827 return;
828
829 wx->tx_hwtstamp_pkts = 0;
830 wx->tx_hwtstamp_timeouts = 0;
831 wx->tx_hwtstamp_skipped = 0;
832 wx->tx_hwtstamp_errors = 0;
833 wx->rx_hwtstamp_cleared = 0;
834 /* reset the ptp related hardware bits */
835 wx_ptp_reset(wx);
836
837 /* enter the WX_STATE_PTP_RUNNING state */
838 set_bit(WX_STATE_PTP_RUNNING, wx->state);
839 }
840 EXPORT_SYMBOL(wx_ptp_init);
841
842 /**
843 * wx_ptp_suspend - stop ptp work items
844 * @wx: pointer to wx struct
845 *
846 * This function suspends ptp activity, and prevents more work from being
847 * generated, but does not destroy the clock device.
848 */
wx_ptp_suspend(struct wx * wx)849 void wx_ptp_suspend(struct wx *wx)
850 {
851 /* leave the WX_STATE_PTP_RUNNING STATE */
852 if (!test_and_clear_bit(WX_STATE_PTP_RUNNING, wx->state))
853 return;
854
855 clear_bit(WX_FLAG_PTP_PPS_ENABLED, wx->flags);
856 if (wx->ptp_setup_sdp)
857 wx->ptp_setup_sdp(wx);
858
859 wx_ptp_clear_tx_timestamp(wx);
860 }
861 EXPORT_SYMBOL(wx_ptp_suspend);
862
863 /**
864 * wx_ptp_stop - destroy the ptp_clock device
865 * @wx: pointer to wx struct
866 *
867 * Completely destroy the ptp_clock device, and disable all PTP related
868 * features. Intended to be run when the device is being closed.
869 */
wx_ptp_stop(struct wx * wx)870 void wx_ptp_stop(struct wx *wx)
871 {
872 /* first, suspend ptp activity */
873 wx_ptp_suspend(wx);
874
875 /* now destroy the ptp clock device */
876 if (wx->ptp_clock) {
877 ptp_clock_unregister(wx->ptp_clock);
878 wx->ptp_clock = NULL;
879 dev_info(&wx->pdev->dev, "removed PHC on %s\n", wx->netdev->name);
880 }
881 }
882 EXPORT_SYMBOL(wx_ptp_stop);
883
wx_ptp_quiesce(struct wx * wx)884 void wx_ptp_quiesce(struct wx *wx)
885 {
886 struct sk_buff *skb;
887 unsigned long flags;
888
889 if (!test_and_clear_bit(WX_STATE_PTP_RUNNING, wx->state))
890 return;
891
892 clear_bit(WX_FLAG_PTP_PPS_ENABLED, wx->flags);
893
894 if (wx->ptp_clock)
895 ptp_cancel_worker_sync(wx->ptp_clock);
896
897 /* Drop a pending Tx timestamp request. Do not touch the registers
898 * here: quiesce runs during PCIe error recovery, where the device may
899 * already be gone and MMIO is not reliable.
900 */
901 spin_lock_irqsave(&wx->ptp_tx_lock, flags);
902 skb = __wx_ptp_detach_tx_skb(wx);
903 spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
904 dev_kfree_skb_any(skb);
905
906 if (wx->ptp_clock) {
907 ptp_clock_unregister(wx->ptp_clock);
908 wx->ptp_clock = NULL;
909 dev_info(&wx->pdev->dev, "removed PHC on %s\n", wx->netdev->name);
910 }
911 }
912 EXPORT_SYMBOL(wx_ptp_quiesce);
913
914 /**
915 * wx_ptp_rx_hwtstamp - utility function which checks for RX time stamp
916 * @wx: pointer to wx struct
917 * @skb: particular skb to send timestamp with
918 *
919 * if the timestamp is valid, we convert it into the timecounter ns
920 * value, then store that result into the shhwtstamps structure which
921 * is passed up the network stack
922 */
wx_ptp_rx_hwtstamp(struct wx * wx,struct sk_buff * skb)923 void wx_ptp_rx_hwtstamp(struct wx *wx, struct sk_buff *skb)
924 {
925 u64 regval = 0;
926 u32 tsyncrxctl;
927
928 /* Read the tsyncrxctl register afterwards in order to prevent taking an
929 * I/O hit on every packet.
930 */
931 tsyncrxctl = rd32(wx, WX_PSR_1588_CTL);
932 if (!(tsyncrxctl & WX_PSR_1588_CTL_VALID))
933 return;
934
935 regval |= (u64)rd32(wx, WX_PSR_1588_STMPL);
936 regval |= (u64)rd32(wx, WX_PSR_1588_STMPH) << 32;
937
938 wx_ptp_convert_to_hwtstamp(wx, skb_hwtstamps(skb), regval);
939 }
940
wx_hwtstamp_get(struct net_device * dev,struct kernel_hwtstamp_config * cfg)941 int wx_hwtstamp_get(struct net_device *dev,
942 struct kernel_hwtstamp_config *cfg)
943 {
944 struct wx *wx = netdev_priv(dev);
945
946 if (!netif_running(dev))
947 return -EINVAL;
948
949 *cfg = wx->tstamp_config;
950
951 return 0;
952 }
953 EXPORT_SYMBOL(wx_hwtstamp_get);
954
wx_hwtstamp_set(struct net_device * dev,struct kernel_hwtstamp_config * cfg,struct netlink_ext_ack * extack)955 int wx_hwtstamp_set(struct net_device *dev,
956 struct kernel_hwtstamp_config *cfg,
957 struct netlink_ext_ack *extack)
958 {
959 struct wx *wx = netdev_priv(dev);
960 int err;
961
962 if (!netif_running(dev))
963 return -EINVAL;
964
965 err = wx_ptp_set_timestamp_mode(wx, cfg);
966 if (err)
967 return err;
968
969 /* save these settings for future reference */
970 memcpy(&wx->tstamp_config, cfg, sizeof(wx->tstamp_config));
971
972 return 0;
973 }
974 EXPORT_SYMBOL(wx_hwtstamp_set);
975