1
2
3
4#include <linux/module.h>
5#include <linux/device.h>
6#include <linux/pci.h>
7#include <linux/ptp_classify.h>
8
9#include "igb.h"
10
11#define INCVALUE_MASK 0x7fffffff
12#define ISGN 0x80000000
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64#define IGB_SYSTIM_OVERFLOW_PERIOD (HZ * 60 * 6)
65#define IGB_PTP_TX_TIMEOUT (HZ * 15)
66#define INCPERIOD_82576 BIT(E1000_TIMINCA_16NS_SHIFT)
67#define INCVALUE_82576_MASK GENMASK(E1000_TIMINCA_16NS_SHIFT - 1, 0)
68#define INCVALUE_82576 (16u << IGB_82576_TSYNC_SHIFT)
69#define IGB_NBITS_82580 40
70
71static void igb_ptp_tx_hwtstamp(struct igb_adapter *adapter);
72
73
74static u64 igb_ptp_read_82576(const struct cyclecounter *cc)
75{
76 struct igb_adapter *igb = container_of(cc, struct igb_adapter, cc);
77 struct e1000_hw *hw = &igb->hw;
78 u64 val;
79 u32 lo, hi;
80
81 lo = rd32(E1000_SYSTIML);
82 hi = rd32(E1000_SYSTIMH);
83
84 val = ((u64) hi) << 32;
85 val |= lo;
86
87 return val;
88}
89
90
91static u64 igb_ptp_read_82580(const struct cyclecounter *cc)
92{
93 struct igb_adapter *igb = container_of(cc, struct igb_adapter, cc);
94 struct e1000_hw *hw = &igb->hw;
95 u32 lo, hi;
96 u64 val;
97
98
99
100
101
102 rd32(E1000_SYSTIMR);
103 lo = rd32(E1000_SYSTIML);
104 hi = rd32(E1000_SYSTIMH);
105
106 val = ((u64) hi) << 32;
107 val |= lo;
108
109 return val;
110}
111
112
113static void igb_ptp_read_i210(struct igb_adapter *adapter,
114 struct timespec64 *ts)
115{
116 struct e1000_hw *hw = &adapter->hw;
117 u32 sec, nsec;
118
119
120
121
122
123 rd32(E1000_SYSTIMR);
124 nsec = rd32(E1000_SYSTIML);
125 sec = rd32(E1000_SYSTIMH);
126
127 ts->tv_sec = sec;
128 ts->tv_nsec = nsec;
129}
130
131static void igb_ptp_write_i210(struct igb_adapter *adapter,
132 const struct timespec64 *ts)
133{
134 struct e1000_hw *hw = &adapter->hw;
135
136
137
138
139 wr32(E1000_SYSTIML, ts->tv_nsec);
140 wr32(E1000_SYSTIMH, (u32)ts->tv_sec);
141}
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160static void igb_ptp_systim_to_hwtstamp(struct igb_adapter *adapter,
161 struct skb_shared_hwtstamps *hwtstamps,
162 u64 systim)
163{
164 unsigned long flags;
165 u64 ns;
166
167 switch (adapter->hw.mac.type) {
168 case e1000_82576:
169 case e1000_82580:
170 case e1000_i354:
171 case e1000_i350:
172 spin_lock_irqsave(&adapter->tmreg_lock, flags);
173
174 ns = timecounter_cyc2time(&adapter->tc, systim);
175
176 spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
177
178 memset(hwtstamps, 0, sizeof(*hwtstamps));
179 hwtstamps->hwtstamp = ns_to_ktime(ns);
180 break;
181 case e1000_i210:
182 case e1000_i211:
183 memset(hwtstamps, 0, sizeof(*hwtstamps));
184
185 hwtstamps->hwtstamp = ktime_set(systim >> 32,
186 systim & 0xFFFFFFFF);
187 break;
188 default:
189 break;
190 }
191}
192
193
194static int igb_ptp_adjfreq_82576(struct ptp_clock_info *ptp, s32 ppb)
195{
196 struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
197 ptp_caps);
198 struct e1000_hw *hw = &igb->hw;
199 int neg_adj = 0;
200 u64 rate;
201 u32 incvalue;
202
203 if (ppb < 0) {
204 neg_adj = 1;
205 ppb = -ppb;
206 }
207 rate = ppb;
208 rate <<= 14;
209 rate = div_u64(rate, 1953125);
210
211 incvalue = 16 << IGB_82576_TSYNC_SHIFT;
212
213 if (neg_adj)
214 incvalue -= rate;
215 else
216 incvalue += rate;
217
218 wr32(E1000_TIMINCA, INCPERIOD_82576 | (incvalue & INCVALUE_82576_MASK));
219
220 return 0;
221}
222
223static int igb_ptp_adjfine_82580(struct ptp_clock_info *ptp, long scaled_ppm)
224{
225 struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
226 ptp_caps);
227 struct e1000_hw *hw = &igb->hw;
228 int neg_adj = 0;
229 u64 rate;
230 u32 inca;
231
232 if (scaled_ppm < 0) {
233 neg_adj = 1;
234 scaled_ppm = -scaled_ppm;
235 }
236 rate = scaled_ppm;
237 rate <<= 13;
238 rate = div_u64(rate, 15625);
239
240 inca = rate & INCVALUE_MASK;
241 if (neg_adj)
242 inca |= ISGN;
243
244 wr32(E1000_TIMINCA, inca);
245
246 return 0;
247}
248
249static int igb_ptp_adjtime_82576(struct ptp_clock_info *ptp, s64 delta)
250{
251 struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
252 ptp_caps);
253 unsigned long flags;
254
255 spin_lock_irqsave(&igb->tmreg_lock, flags);
256 timecounter_adjtime(&igb->tc, delta);
257 spin_unlock_irqrestore(&igb->tmreg_lock, flags);
258
259 return 0;
260}
261
262static int igb_ptp_adjtime_i210(struct ptp_clock_info *ptp, s64 delta)
263{
264 struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
265 ptp_caps);
266 unsigned long flags;
267 struct timespec64 now, then = ns_to_timespec64(delta);
268
269 spin_lock_irqsave(&igb->tmreg_lock, flags);
270
271 igb_ptp_read_i210(igb, &now);
272 now = timespec64_add(now, then);
273 igb_ptp_write_i210(igb, (const struct timespec64 *)&now);
274
275 spin_unlock_irqrestore(&igb->tmreg_lock, flags);
276
277 return 0;
278}
279
280static int igb_ptp_gettimex_82576(struct ptp_clock_info *ptp,
281 struct timespec64 *ts,
282 struct ptp_system_timestamp *sts)
283{
284 struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
285 ptp_caps);
286 struct e1000_hw *hw = &igb->hw;
287 unsigned long flags;
288 u32 lo, hi;
289 u64 ns;
290
291 spin_lock_irqsave(&igb->tmreg_lock, flags);
292
293 ptp_read_system_prets(sts);
294 lo = rd32(E1000_SYSTIML);
295 ptp_read_system_postts(sts);
296 hi = rd32(E1000_SYSTIMH);
297
298 ns = timecounter_cyc2time(&igb->tc, ((u64)hi << 32) | lo);
299
300 spin_unlock_irqrestore(&igb->tmreg_lock, flags);
301
302 *ts = ns_to_timespec64(ns);
303
304 return 0;
305}
306
307static int igb_ptp_gettimex_82580(struct ptp_clock_info *ptp,
308 struct timespec64 *ts,
309 struct ptp_system_timestamp *sts)
310{
311 struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
312 ptp_caps);
313 struct e1000_hw *hw = &igb->hw;
314 unsigned long flags;
315 u32 lo, hi;
316 u64 ns;
317
318 spin_lock_irqsave(&igb->tmreg_lock, flags);
319
320 ptp_read_system_prets(sts);
321 rd32(E1000_SYSTIMR);
322 ptp_read_system_postts(sts);
323 lo = rd32(E1000_SYSTIML);
324 hi = rd32(E1000_SYSTIMH);
325
326 ns = timecounter_cyc2time(&igb->tc, ((u64)hi << 32) | lo);
327
328 spin_unlock_irqrestore(&igb->tmreg_lock, flags);
329
330 *ts = ns_to_timespec64(ns);
331
332 return 0;
333}
334
335static int igb_ptp_gettimex_i210(struct ptp_clock_info *ptp,
336 struct timespec64 *ts,
337 struct ptp_system_timestamp *sts)
338{
339 struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
340 ptp_caps);
341 struct e1000_hw *hw = &igb->hw;
342 unsigned long flags;
343
344 spin_lock_irqsave(&igb->tmreg_lock, flags);
345
346 ptp_read_system_prets(sts);
347 rd32(E1000_SYSTIMR);
348 ptp_read_system_postts(sts);
349 ts->tv_nsec = rd32(E1000_SYSTIML);
350 ts->tv_sec = rd32(E1000_SYSTIMH);
351
352 spin_unlock_irqrestore(&igb->tmreg_lock, flags);
353
354 return 0;
355}
356
357static int igb_ptp_settime_82576(struct ptp_clock_info *ptp,
358 const struct timespec64 *ts)
359{
360 struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
361 ptp_caps);
362 unsigned long flags;
363 u64 ns;
364
365 ns = timespec64_to_ns(ts);
366
367 spin_lock_irqsave(&igb->tmreg_lock, flags);
368
369 timecounter_init(&igb->tc, &igb->cc, ns);
370
371 spin_unlock_irqrestore(&igb->tmreg_lock, flags);
372
373 return 0;
374}
375
376static int igb_ptp_settime_i210(struct ptp_clock_info *ptp,
377 const struct timespec64 *ts)
378{
379 struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
380 ptp_caps);
381 unsigned long flags;
382
383 spin_lock_irqsave(&igb->tmreg_lock, flags);
384
385 igb_ptp_write_i210(igb, ts);
386
387 spin_unlock_irqrestore(&igb->tmreg_lock, flags);
388
389 return 0;
390}
391
392static void igb_pin_direction(int pin, int input, u32 *ctrl, u32 *ctrl_ext)
393{
394 u32 *ptr = pin < 2 ? ctrl : ctrl_ext;
395 static const u32 mask[IGB_N_SDP] = {
396 E1000_CTRL_SDP0_DIR,
397 E1000_CTRL_SDP1_DIR,
398 E1000_CTRL_EXT_SDP2_DIR,
399 E1000_CTRL_EXT_SDP3_DIR,
400 };
401
402 if (input)
403 *ptr &= ~mask[pin];
404 else
405 *ptr |= mask[pin];
406}
407
408static void igb_pin_extts(struct igb_adapter *igb, int chan, int pin)
409{
410 static const u32 aux0_sel_sdp[IGB_N_SDP] = {
411 AUX0_SEL_SDP0, AUX0_SEL_SDP1, AUX0_SEL_SDP2, AUX0_SEL_SDP3,
412 };
413 static const u32 aux1_sel_sdp[IGB_N_SDP] = {
414 AUX1_SEL_SDP0, AUX1_SEL_SDP1, AUX1_SEL_SDP2, AUX1_SEL_SDP3,
415 };
416 static const u32 ts_sdp_en[IGB_N_SDP] = {
417 TS_SDP0_EN, TS_SDP1_EN, TS_SDP2_EN, TS_SDP3_EN,
418 };
419 struct e1000_hw *hw = &igb->hw;
420 u32 ctrl, ctrl_ext, tssdp = 0;
421
422 ctrl = rd32(E1000_CTRL);
423 ctrl_ext = rd32(E1000_CTRL_EXT);
424 tssdp = rd32(E1000_TSSDP);
425
426 igb_pin_direction(pin, 1, &ctrl, &ctrl_ext);
427
428
429 tssdp &= ~ts_sdp_en[pin];
430
431 if (chan == 1) {
432 tssdp &= ~AUX1_SEL_SDP3;
433 tssdp |= aux1_sel_sdp[pin] | AUX1_TS_SDP_EN;
434 } else {
435 tssdp &= ~AUX0_SEL_SDP3;
436 tssdp |= aux0_sel_sdp[pin] | AUX0_TS_SDP_EN;
437 }
438
439 wr32(E1000_TSSDP, tssdp);
440 wr32(E1000_CTRL, ctrl);
441 wr32(E1000_CTRL_EXT, ctrl_ext);
442}
443
444static void igb_pin_perout(struct igb_adapter *igb, int chan, int pin, int freq)
445{
446 static const u32 aux0_sel_sdp[IGB_N_SDP] = {
447 AUX0_SEL_SDP0, AUX0_SEL_SDP1, AUX0_SEL_SDP2, AUX0_SEL_SDP3,
448 };
449 static const u32 aux1_sel_sdp[IGB_N_SDP] = {
450 AUX1_SEL_SDP0, AUX1_SEL_SDP1, AUX1_SEL_SDP2, AUX1_SEL_SDP3,
451 };
452 static const u32 ts_sdp_en[IGB_N_SDP] = {
453 TS_SDP0_EN, TS_SDP1_EN, TS_SDP2_EN, TS_SDP3_EN,
454 };
455 static const u32 ts_sdp_sel_tt0[IGB_N_SDP] = {
456 TS_SDP0_SEL_TT0, TS_SDP1_SEL_TT0,
457 TS_SDP2_SEL_TT0, TS_SDP3_SEL_TT0,
458 };
459 static const u32 ts_sdp_sel_tt1[IGB_N_SDP] = {
460 TS_SDP0_SEL_TT1, TS_SDP1_SEL_TT1,
461 TS_SDP2_SEL_TT1, TS_SDP3_SEL_TT1,
462 };
463 static const u32 ts_sdp_sel_fc0[IGB_N_SDP] = {
464 TS_SDP0_SEL_FC0, TS_SDP1_SEL_FC0,
465 TS_SDP2_SEL_FC0, TS_SDP3_SEL_FC0,
466 };
467 static const u32 ts_sdp_sel_fc1[IGB_N_SDP] = {
468 TS_SDP0_SEL_FC1, TS_SDP1_SEL_FC1,
469 TS_SDP2_SEL_FC1, TS_SDP3_SEL_FC1,
470 };
471 static const u32 ts_sdp_sel_clr[IGB_N_SDP] = {
472 TS_SDP0_SEL_FC1, TS_SDP1_SEL_FC1,
473 TS_SDP2_SEL_FC1, TS_SDP3_SEL_FC1,
474 };
475 struct e1000_hw *hw = &igb->hw;
476 u32 ctrl, ctrl_ext, tssdp = 0;
477
478 ctrl = rd32(E1000_CTRL);
479 ctrl_ext = rd32(E1000_CTRL_EXT);
480 tssdp = rd32(E1000_TSSDP);
481
482 igb_pin_direction(pin, 0, &ctrl, &ctrl_ext);
483
484
485 if ((tssdp & AUX0_SEL_SDP3) == aux0_sel_sdp[pin])
486 tssdp &= ~AUX0_TS_SDP_EN;
487
488 if ((tssdp & AUX1_SEL_SDP3) == aux1_sel_sdp[pin])
489 tssdp &= ~AUX1_TS_SDP_EN;
490
491 tssdp &= ~ts_sdp_sel_clr[pin];
492 if (freq) {
493 if (chan == 1)
494 tssdp |= ts_sdp_sel_fc1[pin];
495 else
496 tssdp |= ts_sdp_sel_fc0[pin];
497 } else {
498 if (chan == 1)
499 tssdp |= ts_sdp_sel_tt1[pin];
500 else
501 tssdp |= ts_sdp_sel_tt0[pin];
502 }
503 tssdp |= ts_sdp_en[pin];
504
505 wr32(E1000_TSSDP, tssdp);
506 wr32(E1000_CTRL, ctrl);
507 wr32(E1000_CTRL_EXT, ctrl_ext);
508}
509
510static int igb_ptp_feature_enable_i210(struct ptp_clock_info *ptp,
511 struct ptp_clock_request *rq, int on)
512{
513 struct igb_adapter *igb =
514 container_of(ptp, struct igb_adapter, ptp_caps);
515 struct e1000_hw *hw = &igb->hw;
516 u32 tsauxc, tsim, tsauxc_mask, tsim_mask, trgttiml, trgttimh, freqout;
517 unsigned long flags;
518 struct timespec64 ts;
519 int use_freq = 0, pin = -1;
520 s64 ns;
521
522 switch (rq->type) {
523 case PTP_CLK_REQ_EXTTS:
524
525 if (rq->extts.flags & ~(PTP_ENABLE_FEATURE |
526 PTP_RISING_EDGE |
527 PTP_FALLING_EDGE |
528 PTP_STRICT_FLAGS))
529 return -EOPNOTSUPP;
530
531
532 if ((rq->extts.flags & PTP_STRICT_FLAGS) &&
533 (rq->extts.flags & PTP_ENABLE_FEATURE) &&
534 (rq->extts.flags & PTP_EXTTS_EDGES) != PTP_EXTTS_EDGES)
535 return -EOPNOTSUPP;
536
537 if (on) {
538 pin = ptp_find_pin(igb->ptp_clock, PTP_PF_EXTTS,
539 rq->extts.index);
540 if (pin < 0)
541 return -EBUSY;
542 }
543 if (rq->extts.index == 1) {
544 tsauxc_mask = TSAUXC_EN_TS1;
545 tsim_mask = TSINTR_AUTT1;
546 } else {
547 tsauxc_mask = TSAUXC_EN_TS0;
548 tsim_mask = TSINTR_AUTT0;
549 }
550 spin_lock_irqsave(&igb->tmreg_lock, flags);
551 tsauxc = rd32(E1000_TSAUXC);
552 tsim = rd32(E1000_TSIM);
553 if (on) {
554 igb_pin_extts(igb, rq->extts.index, pin);
555 tsauxc |= tsauxc_mask;
556 tsim |= tsim_mask;
557 } else {
558 tsauxc &= ~tsauxc_mask;
559 tsim &= ~tsim_mask;
560 }
561 wr32(E1000_TSAUXC, tsauxc);
562 wr32(E1000_TSIM, tsim);
563 spin_unlock_irqrestore(&igb->tmreg_lock, flags);
564 return 0;
565
566 case PTP_CLK_REQ_PEROUT:
567
568 if (rq->perout.flags)
569 return -EOPNOTSUPP;
570
571 if (on) {
572 pin = ptp_find_pin(igb->ptp_clock, PTP_PF_PEROUT,
573 rq->perout.index);
574 if (pin < 0)
575 return -EBUSY;
576 }
577 ts.tv_sec = rq->perout.period.sec;
578 ts.tv_nsec = rq->perout.period.nsec;
579 ns = timespec64_to_ns(&ts);
580 ns = ns >> 1;
581 if (on && ((ns <= 70000000LL) || (ns == 125000000LL) ||
582 (ns == 250000000LL) || (ns == 500000000LL))) {
583 if (ns < 8LL)
584 return -EINVAL;
585 use_freq = 1;
586 }
587 ts = ns_to_timespec64(ns);
588 if (rq->perout.index == 1) {
589 if (use_freq) {
590 tsauxc_mask = TSAUXC_EN_CLK1 | TSAUXC_ST1;
591 tsim_mask = 0;
592 } else {
593 tsauxc_mask = TSAUXC_EN_TT1;
594 tsim_mask = TSINTR_TT1;
595 }
596 trgttiml = E1000_TRGTTIML1;
597 trgttimh = E1000_TRGTTIMH1;
598 freqout = E1000_FREQOUT1;
599 } else {
600 if (use_freq) {
601 tsauxc_mask = TSAUXC_EN_CLK0 | TSAUXC_ST0;
602 tsim_mask = 0;
603 } else {
604 tsauxc_mask = TSAUXC_EN_TT0;
605 tsim_mask = TSINTR_TT0;
606 }
607 trgttiml = E1000_TRGTTIML0;
608 trgttimh = E1000_TRGTTIMH0;
609 freqout = E1000_FREQOUT0;
610 }
611 spin_lock_irqsave(&igb->tmreg_lock, flags);
612 tsauxc = rd32(E1000_TSAUXC);
613 tsim = rd32(E1000_TSIM);
614 if (rq->perout.index == 1) {
615 tsauxc &= ~(TSAUXC_EN_TT1 | TSAUXC_EN_CLK1 | TSAUXC_ST1);
616 tsim &= ~TSINTR_TT1;
617 } else {
618 tsauxc &= ~(TSAUXC_EN_TT0 | TSAUXC_EN_CLK0 | TSAUXC_ST0);
619 tsim &= ~TSINTR_TT0;
620 }
621 if (on) {
622 int i = rq->perout.index;
623 igb_pin_perout(igb, i, pin, use_freq);
624 igb->perout[i].start.tv_sec = rq->perout.start.sec;
625 igb->perout[i].start.tv_nsec = rq->perout.start.nsec;
626 igb->perout[i].period.tv_sec = ts.tv_sec;
627 igb->perout[i].period.tv_nsec = ts.tv_nsec;
628 wr32(trgttimh, rq->perout.start.sec);
629 wr32(trgttiml, rq->perout.start.nsec);
630 if (use_freq)
631 wr32(freqout, ns);
632 tsauxc |= tsauxc_mask;
633 tsim |= tsim_mask;
634 }
635 wr32(E1000_TSAUXC, tsauxc);
636 wr32(E1000_TSIM, tsim);
637 spin_unlock_irqrestore(&igb->tmreg_lock, flags);
638 return 0;
639
640 case PTP_CLK_REQ_PPS:
641 spin_lock_irqsave(&igb->tmreg_lock, flags);
642 tsim = rd32(E1000_TSIM);
643 if (on)
644 tsim |= TSINTR_SYS_WRAP;
645 else
646 tsim &= ~TSINTR_SYS_WRAP;
647 igb->pps_sys_wrap_on = !!on;
648 wr32(E1000_TSIM, tsim);
649 spin_unlock_irqrestore(&igb->tmreg_lock, flags);
650 return 0;
651 }
652
653 return -EOPNOTSUPP;
654}
655
656static int igb_ptp_feature_enable(struct ptp_clock_info *ptp,
657 struct ptp_clock_request *rq, int on)
658{
659 return -EOPNOTSUPP;
660}
661
662static int igb_ptp_verify_pin(struct ptp_clock_info *ptp, unsigned int pin,
663 enum ptp_pin_function func, unsigned int chan)
664{
665 switch (func) {
666 case PTP_PF_NONE:
667 case PTP_PF_EXTTS:
668 case PTP_PF_PEROUT:
669 break;
670 case PTP_PF_PHYSYNC:
671 return -1;
672 }
673 return 0;
674}
675
676
677
678
679
680
681
682
683static void igb_ptp_tx_work(struct work_struct *work)
684{
685 struct igb_adapter *adapter = container_of(work, struct igb_adapter,
686 ptp_tx_work);
687 struct e1000_hw *hw = &adapter->hw;
688 u32 tsynctxctl;
689
690 if (!adapter->ptp_tx_skb)
691 return;
692
693 if (time_is_before_jiffies(adapter->ptp_tx_start +
694 IGB_PTP_TX_TIMEOUT)) {
695 dev_kfree_skb_any(adapter->ptp_tx_skb);
696 adapter->ptp_tx_skb = NULL;
697 clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
698 adapter->tx_hwtstamp_timeouts++;
699
700
701
702 rd32(E1000_TXSTMPH);
703 dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang\n");
704 return;
705 }
706
707 tsynctxctl = rd32(E1000_TSYNCTXCTL);
708 if (tsynctxctl & E1000_TSYNCTXCTL_VALID)
709 igb_ptp_tx_hwtstamp(adapter);
710 else
711
712 schedule_work(&adapter->ptp_tx_work);
713}
714
715static void igb_ptp_overflow_check(struct work_struct *work)
716{
717 struct igb_adapter *igb =
718 container_of(work, struct igb_adapter, ptp_overflow_work.work);
719 struct timespec64 ts;
720 u64 ns;
721
722
723 ns = timecounter_read(&igb->tc);
724
725 ts = ns_to_timespec64(ns);
726 pr_debug("igb overflow check at %lld.%09lu\n",
727 (long long) ts.tv_sec, ts.tv_nsec);
728
729 schedule_delayed_work(&igb->ptp_overflow_work,
730 IGB_SYSTIM_OVERFLOW_PERIOD);
731}
732
733
734
735
736
737
738
739
740
741
742void igb_ptp_rx_hang(struct igb_adapter *adapter)
743{
744 struct e1000_hw *hw = &adapter->hw;
745 u32 tsyncrxctl = rd32(E1000_TSYNCRXCTL);
746 unsigned long rx_event;
747
748
749 if (hw->mac.type != e1000_82576)
750 return;
751
752
753
754
755 if (!(tsyncrxctl & E1000_TSYNCRXCTL_VALID)) {
756 adapter->last_rx_ptp_check = jiffies;
757 return;
758 }
759
760
761 rx_event = adapter->last_rx_ptp_check;
762 if (time_after(adapter->last_rx_timestamp, rx_event))
763 rx_event = adapter->last_rx_timestamp;
764
765
766 if (time_is_before_jiffies(rx_event + 5 * HZ)) {
767 rd32(E1000_RXSTMPH);
768 adapter->last_rx_ptp_check = jiffies;
769 adapter->rx_hwtstamp_cleared++;
770 dev_warn(&adapter->pdev->dev, "clearing Rx timestamp hang\n");
771 }
772}
773
774
775
776
777
778void igb_ptp_tx_hang(struct igb_adapter *adapter)
779{
780 struct e1000_hw *hw = &adapter->hw;
781 bool timeout = time_is_before_jiffies(adapter->ptp_tx_start +
782 IGB_PTP_TX_TIMEOUT);
783
784 if (!adapter->ptp_tx_skb)
785 return;
786
787 if (!test_bit(__IGB_PTP_TX_IN_PROGRESS, &adapter->state))
788 return;
789
790
791
792
793
794 if (timeout) {
795 cancel_work_sync(&adapter->ptp_tx_work);
796 dev_kfree_skb_any(adapter->ptp_tx_skb);
797 adapter->ptp_tx_skb = NULL;
798 clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
799 adapter->tx_hwtstamp_timeouts++;
800
801
802
803 rd32(E1000_TXSTMPH);
804 dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang\n");
805 }
806}
807
808
809
810
811
812
813
814
815
816static void igb_ptp_tx_hwtstamp(struct igb_adapter *adapter)
817{
818 struct sk_buff *skb = adapter->ptp_tx_skb;
819 struct e1000_hw *hw = &adapter->hw;
820 struct skb_shared_hwtstamps shhwtstamps;
821 u64 regval;
822 int adjust = 0;
823
824 regval = rd32(E1000_TXSTMPL);
825 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
826
827 igb_ptp_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
828
829 if (adapter->hw.mac.type == e1000_i210) {
830 switch (adapter->link_speed) {
831 case SPEED_10:
832 adjust = IGB_I210_TX_LATENCY_10;
833 break;
834 case SPEED_100:
835 adjust = IGB_I210_TX_LATENCY_100;
836 break;
837 case SPEED_1000:
838 adjust = IGB_I210_TX_LATENCY_1000;
839 break;
840 }
841 }
842
843 shhwtstamps.hwtstamp =
844 ktime_add_ns(shhwtstamps.hwtstamp, adjust);
845
846
847
848
849
850
851 adapter->ptp_tx_skb = NULL;
852 clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
853
854
855 skb_tstamp_tx(skb, &shhwtstamps);
856 dev_kfree_skb_any(skb);
857}
858
859
860
861
862
863
864
865
866
867
868
869
870
871int igb_ptp_rx_pktstamp(struct igb_q_vector *q_vector, void *va,
872 ktime_t *timestamp)
873{
874 struct igb_adapter *adapter = q_vector->adapter;
875 struct skb_shared_hwtstamps ts;
876 __le64 *regval = (__le64 *)va;
877 int adjust = 0;
878
879 if (!(adapter->ptp_flags & IGB_PTP_ENABLED))
880 return 0;
881
882
883
884
885
886
887
888 if (regval[0])
889 return 0;
890
891 igb_ptp_systim_to_hwtstamp(adapter, &ts, le64_to_cpu(regval[1]));
892
893
894 if (adapter->hw.mac.type == e1000_i210) {
895 switch (adapter->link_speed) {
896 case SPEED_10:
897 adjust = IGB_I210_RX_LATENCY_10;
898 break;
899 case SPEED_100:
900 adjust = IGB_I210_RX_LATENCY_100;
901 break;
902 case SPEED_1000:
903 adjust = IGB_I210_RX_LATENCY_1000;
904 break;
905 }
906 }
907
908 *timestamp = ktime_sub_ns(ts.hwtstamp, adjust);
909
910 return IGB_TS_HDR_LEN;
911}
912
913
914
915
916
917
918
919
920
921void igb_ptp_rx_rgtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
922{
923 struct igb_adapter *adapter = q_vector->adapter;
924 struct e1000_hw *hw = &adapter->hw;
925 int adjust = 0;
926 u64 regval;
927
928 if (!(adapter->ptp_flags & IGB_PTP_ENABLED))
929 return;
930
931
932
933
934
935
936
937
938
939
940
941 if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
942 return;
943
944 regval = rd32(E1000_RXSTMPL);
945 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
946
947 igb_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
948
949
950 if (adapter->hw.mac.type == e1000_i210) {
951 switch (adapter->link_speed) {
952 case SPEED_10:
953 adjust = IGB_I210_RX_LATENCY_10;
954 break;
955 case SPEED_100:
956 adjust = IGB_I210_RX_LATENCY_100;
957 break;
958 case SPEED_1000:
959 adjust = IGB_I210_RX_LATENCY_1000;
960 break;
961 }
962 }
963 skb_hwtstamps(skb)->hwtstamp =
964 ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust);
965
966
967
968
969 adapter->last_rx_timestamp = jiffies;
970}
971
972
973
974
975
976
977
978
979
980
981int igb_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr)
982{
983 struct igb_adapter *adapter = netdev_priv(netdev);
984 struct hwtstamp_config *config = &adapter->tstamp_config;
985
986 return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ?
987 -EFAULT : 0;
988}
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007static int igb_ptp_set_timestamp_mode(struct igb_adapter *adapter,
1008 struct hwtstamp_config *config)
1009{
1010 struct e1000_hw *hw = &adapter->hw;
1011 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
1012 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
1013 u32 tsync_rx_cfg = 0;
1014 bool is_l4 = false;
1015 bool is_l2 = false;
1016 u32 regval;
1017
1018
1019 if (config->flags)
1020 return -EINVAL;
1021
1022 switch (config->tx_type) {
1023 case HWTSTAMP_TX_OFF:
1024 tsync_tx_ctl = 0;
1025 break;
1026 case HWTSTAMP_TX_ON:
1027 break;
1028 default:
1029 return -ERANGE;
1030 }
1031
1032 switch (config->rx_filter) {
1033 case HWTSTAMP_FILTER_NONE:
1034 tsync_rx_ctl = 0;
1035 break;
1036 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1037 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
1038 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
1039 is_l4 = true;
1040 break;
1041 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1042 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
1043 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
1044 is_l4 = true;
1045 break;
1046 case HWTSTAMP_FILTER_PTP_V2_EVENT:
1047 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1048 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1049 case HWTSTAMP_FILTER_PTP_V2_SYNC:
1050 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1051 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1052 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1053 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1054 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1055 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
1056 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
1057 is_l2 = true;
1058 is_l4 = true;
1059 break;
1060 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1061 case HWTSTAMP_FILTER_NTP_ALL:
1062 case HWTSTAMP_FILTER_ALL:
1063
1064
1065
1066 if (hw->mac.type != e1000_82576) {
1067 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
1068 config->rx_filter = HWTSTAMP_FILTER_ALL;
1069 break;
1070 }
1071 fallthrough;
1072 default:
1073 config->rx_filter = HWTSTAMP_FILTER_NONE;
1074 return -ERANGE;
1075 }
1076
1077 if (hw->mac.type == e1000_82575) {
1078 if (tsync_rx_ctl | tsync_tx_ctl)
1079 return -EINVAL;
1080 return 0;
1081 }
1082
1083
1084
1085
1086
1087 if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
1088 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
1089 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
1090 config->rx_filter = HWTSTAMP_FILTER_ALL;
1091 is_l2 = true;
1092 is_l4 = true;
1093
1094 if ((hw->mac.type == e1000_i210) ||
1095 (hw->mac.type == e1000_i211)) {
1096 regval = rd32(E1000_RXPBS);
1097 regval |= E1000_RXPBS_CFG_TS_EN;
1098 wr32(E1000_RXPBS, regval);
1099 }
1100 }
1101
1102
1103 regval = rd32(E1000_TSYNCTXCTL);
1104 regval &= ~E1000_TSYNCTXCTL_ENABLED;
1105 regval |= tsync_tx_ctl;
1106 wr32(E1000_TSYNCTXCTL, regval);
1107
1108
1109 regval = rd32(E1000_TSYNCRXCTL);
1110 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
1111 regval |= tsync_rx_ctl;
1112 wr32(E1000_TSYNCRXCTL, regval);
1113
1114
1115 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
1116
1117
1118 if (is_l2)
1119 wr32(E1000_ETQF(IGB_ETQF_FILTER_1588),
1120 (E1000_ETQF_FILTER_ENABLE |
1121 E1000_ETQF_1588 |
1122 ETH_P_1588));
1123 else
1124 wr32(E1000_ETQF(IGB_ETQF_FILTER_1588), 0);
1125
1126
1127 if (is_l4) {
1128 u32 ftqf = (IPPROTO_UDP
1129 | E1000_FTQF_VF_BP
1130 | E1000_FTQF_1588_TIME_STAMP
1131 | E1000_FTQF_MASK);
1132 ftqf &= ~E1000_FTQF_MASK_PROTO_BP;
1133
1134 wr32(E1000_IMIR(3), (__force unsigned int)htons(PTP_EV_PORT));
1135 wr32(E1000_IMIREXT(3),
1136 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
1137 if (hw->mac.type == e1000_82576) {
1138
1139 wr32(E1000_SPQF(3), (__force unsigned int)htons(PTP_EV_PORT));
1140 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
1141 }
1142 wr32(E1000_FTQF(3), ftqf);
1143 } else {
1144 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
1145 }
1146 wrfl();
1147
1148
1149 regval = rd32(E1000_TXSTMPL);
1150 regval = rd32(E1000_TXSTMPH);
1151 regval = rd32(E1000_RXSTMPL);
1152 regval = rd32(E1000_RXSTMPH);
1153
1154 return 0;
1155}
1156
1157
1158
1159
1160
1161
1162
1163int igb_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr)
1164{
1165 struct igb_adapter *adapter = netdev_priv(netdev);
1166 struct hwtstamp_config config;
1167 int err;
1168
1169 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
1170 return -EFAULT;
1171
1172 err = igb_ptp_set_timestamp_mode(adapter, &config);
1173 if (err)
1174 return err;
1175
1176
1177 memcpy(&adapter->tstamp_config, &config,
1178 sizeof(adapter->tstamp_config));
1179
1180 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1181 -EFAULT : 0;
1182}
1183
1184
1185
1186
1187
1188
1189
1190
1191void igb_ptp_init(struct igb_adapter *adapter)
1192{
1193 struct e1000_hw *hw = &adapter->hw;
1194 struct net_device *netdev = adapter->netdev;
1195 int i;
1196
1197 switch (hw->mac.type) {
1198 case e1000_82576:
1199 snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
1200 adapter->ptp_caps.owner = THIS_MODULE;
1201 adapter->ptp_caps.max_adj = 999999881;
1202 adapter->ptp_caps.n_ext_ts = 0;
1203 adapter->ptp_caps.pps = 0;
1204 adapter->ptp_caps.adjfreq = igb_ptp_adjfreq_82576;
1205 adapter->ptp_caps.adjtime = igb_ptp_adjtime_82576;
1206 adapter->ptp_caps.gettimex64 = igb_ptp_gettimex_82576;
1207 adapter->ptp_caps.settime64 = igb_ptp_settime_82576;
1208 adapter->ptp_caps.enable = igb_ptp_feature_enable;
1209 adapter->cc.read = igb_ptp_read_82576;
1210 adapter->cc.mask = CYCLECOUNTER_MASK(64);
1211 adapter->cc.mult = 1;
1212 adapter->cc.shift = IGB_82576_TSYNC_SHIFT;
1213 adapter->ptp_flags |= IGB_PTP_OVERFLOW_CHECK;
1214 break;
1215 case e1000_82580:
1216 case e1000_i354:
1217 case e1000_i350:
1218 snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
1219 adapter->ptp_caps.owner = THIS_MODULE;
1220 adapter->ptp_caps.max_adj = 62499999;
1221 adapter->ptp_caps.n_ext_ts = 0;
1222 adapter->ptp_caps.pps = 0;
1223 adapter->ptp_caps.adjfine = igb_ptp_adjfine_82580;
1224 adapter->ptp_caps.adjtime = igb_ptp_adjtime_82576;
1225 adapter->ptp_caps.gettimex64 = igb_ptp_gettimex_82580;
1226 adapter->ptp_caps.settime64 = igb_ptp_settime_82576;
1227 adapter->ptp_caps.enable = igb_ptp_feature_enable;
1228 adapter->cc.read = igb_ptp_read_82580;
1229 adapter->cc.mask = CYCLECOUNTER_MASK(IGB_NBITS_82580);
1230 adapter->cc.mult = 1;
1231 adapter->cc.shift = 0;
1232 adapter->ptp_flags |= IGB_PTP_OVERFLOW_CHECK;
1233 break;
1234 case e1000_i210:
1235 case e1000_i211:
1236 for (i = 0; i < IGB_N_SDP; i++) {
1237 struct ptp_pin_desc *ppd = &adapter->sdp_config[i];
1238
1239 snprintf(ppd->name, sizeof(ppd->name), "SDP%d", i);
1240 ppd->index = i;
1241 ppd->func = PTP_PF_NONE;
1242 }
1243 snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
1244 adapter->ptp_caps.owner = THIS_MODULE;
1245 adapter->ptp_caps.max_adj = 62499999;
1246 adapter->ptp_caps.n_ext_ts = IGB_N_EXTTS;
1247 adapter->ptp_caps.n_per_out = IGB_N_PEROUT;
1248 adapter->ptp_caps.n_pins = IGB_N_SDP;
1249 adapter->ptp_caps.pps = 1;
1250 adapter->ptp_caps.pin_config = adapter->sdp_config;
1251 adapter->ptp_caps.adjfine = igb_ptp_adjfine_82580;
1252 adapter->ptp_caps.adjtime = igb_ptp_adjtime_i210;
1253 adapter->ptp_caps.gettimex64 = igb_ptp_gettimex_i210;
1254 adapter->ptp_caps.settime64 = igb_ptp_settime_i210;
1255 adapter->ptp_caps.enable = igb_ptp_feature_enable_i210;
1256 adapter->ptp_caps.verify = igb_ptp_verify_pin;
1257 break;
1258 default:
1259 adapter->ptp_clock = NULL;
1260 return;
1261 }
1262
1263 spin_lock_init(&adapter->tmreg_lock);
1264 INIT_WORK(&adapter->ptp_tx_work, igb_ptp_tx_work);
1265
1266 if (adapter->ptp_flags & IGB_PTP_OVERFLOW_CHECK)
1267 INIT_DELAYED_WORK(&adapter->ptp_overflow_work,
1268 igb_ptp_overflow_check);
1269
1270 adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
1271 adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF;
1272
1273 igb_ptp_reset(adapter);
1274
1275 adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps,
1276 &adapter->pdev->dev);
1277 if (IS_ERR(adapter->ptp_clock)) {
1278 adapter->ptp_clock = NULL;
1279 dev_err(&adapter->pdev->dev, "ptp_clock_register failed\n");
1280 } else if (adapter->ptp_clock) {
1281 dev_info(&adapter->pdev->dev, "added PHC on %s\n",
1282 adapter->netdev->name);
1283 adapter->ptp_flags |= IGB_PTP_ENABLED;
1284 }
1285}
1286
1287
1288
1289
1290
1291
1292
1293
1294void igb_ptp_suspend(struct igb_adapter *adapter)
1295{
1296 if (!(adapter->ptp_flags & IGB_PTP_ENABLED))
1297 return;
1298
1299 if (adapter->ptp_flags & IGB_PTP_OVERFLOW_CHECK)
1300 cancel_delayed_work_sync(&adapter->ptp_overflow_work);
1301
1302 cancel_work_sync(&adapter->ptp_tx_work);
1303 if (adapter->ptp_tx_skb) {
1304 dev_kfree_skb_any(adapter->ptp_tx_skb);
1305 adapter->ptp_tx_skb = NULL;
1306 clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
1307 }
1308}
1309
1310
1311
1312
1313
1314
1315
1316void igb_ptp_stop(struct igb_adapter *adapter)
1317{
1318 igb_ptp_suspend(adapter);
1319
1320 if (adapter->ptp_clock) {
1321 ptp_clock_unregister(adapter->ptp_clock);
1322 dev_info(&adapter->pdev->dev, "removed PHC on %s\n",
1323 adapter->netdev->name);
1324 adapter->ptp_flags &= ~IGB_PTP_ENABLED;
1325 }
1326}
1327
1328
1329
1330
1331
1332
1333
1334void igb_ptp_reset(struct igb_adapter *adapter)
1335{
1336 struct e1000_hw *hw = &adapter->hw;
1337 unsigned long flags;
1338
1339
1340 igb_ptp_set_timestamp_mode(adapter, &adapter->tstamp_config);
1341
1342 spin_lock_irqsave(&adapter->tmreg_lock, flags);
1343
1344 switch (adapter->hw.mac.type) {
1345 case e1000_82576:
1346
1347 wr32(E1000_TIMINCA, INCPERIOD_82576 | INCVALUE_82576);
1348 break;
1349 case e1000_82580:
1350 case e1000_i354:
1351 case e1000_i350:
1352 case e1000_i210:
1353 case e1000_i211:
1354 wr32(E1000_TSAUXC, 0x0);
1355 wr32(E1000_TSSDP, 0x0);
1356 wr32(E1000_TSIM,
1357 TSYNC_INTERRUPTS |
1358 (adapter->pps_sys_wrap_on ? TSINTR_SYS_WRAP : 0));
1359 wr32(E1000_IMS, E1000_IMS_TS);
1360 break;
1361 default:
1362
1363 goto out;
1364 }
1365
1366
1367 if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211)) {
1368 struct timespec64 ts = ktime_to_timespec64(ktime_get_real());
1369
1370 igb_ptp_write_i210(adapter, &ts);
1371 } else {
1372 timecounter_init(&adapter->tc, &adapter->cc,
1373 ktime_to_ns(ktime_get_real()));
1374 }
1375out:
1376 spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
1377
1378 wrfl();
1379
1380 if (adapter->ptp_flags & IGB_PTP_OVERFLOW_CHECK)
1381 schedule_delayed_work(&adapter->ptp_overflow_work,
1382 IGB_SYSTIM_OVERFLOW_PERIOD);
1383}
1384