1#ifndef _HFI1_KERNEL_H
2#define _HFI1_KERNEL_H
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51#include <linux/interrupt.h>
52#include <linux/pci.h>
53#include <linux/dma-mapping.h>
54#include <linux/mutex.h>
55#include <linux/list.h>
56#include <linux/scatterlist.h>
57#include <linux/slab.h>
58#include <linux/io.h>
59#include <linux/fs.h>
60#include <linux/completion.h>
61#include <linux/kref.h>
62#include <linux/sched.h>
63#include <linux/cdev.h>
64#include <linux/delay.h>
65#include <linux/kthread.h>
66#include <linux/i2c.h>
67#include <linux/i2c-algo-bit.h>
68#include <linux/xarray.h>
69#include <rdma/ib_hdrs.h>
70#include <rdma/opa_addr.h>
71#include <linux/rhashtable.h>
72#include <linux/netdevice.h>
73#include <rdma/rdma_vt.h>
74
75#include "chip_registers.h"
76#include "common.h"
77#include "opfn.h"
78#include "verbs.h"
79#include "pio.h"
80#include "chip.h"
81#include "mad.h"
82#include "qsfp.h"
83#include "platform.h"
84#include "affinity.h"
85#include "msix.h"
86
87
88#define HFI1_CHIP_VERS_MAJ 3U
89
90
91#define HFI1_CHIP_VERS_MIN 0U
92
93
94#define HFI1_OUI 0x001175
95#define HFI1_OUI_LSB 40
96
97#define DROP_PACKET_OFF 0
98#define DROP_PACKET_ON 1
99
100#define NEIGHBOR_TYPE_HFI 0
101#define NEIGHBOR_TYPE_SWITCH 1
102
103#define HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES 5
104
105extern unsigned long hfi1_cap_mask;
106#define HFI1_CAP_KGET_MASK(mask, cap) ((mask) & HFI1_CAP_##cap)
107#define HFI1_CAP_UGET_MASK(mask, cap) \
108 (((mask) >> HFI1_CAP_USER_SHIFT) & HFI1_CAP_##cap)
109#define HFI1_CAP_KGET(cap) (HFI1_CAP_KGET_MASK(hfi1_cap_mask, cap))
110#define HFI1_CAP_UGET(cap) (HFI1_CAP_UGET_MASK(hfi1_cap_mask, cap))
111#define HFI1_CAP_IS_KSET(cap) (!!HFI1_CAP_KGET(cap))
112#define HFI1_CAP_IS_USET(cap) (!!HFI1_CAP_UGET(cap))
113#define HFI1_MISC_GET() ((hfi1_cap_mask >> HFI1_CAP_MISC_SHIFT) & \
114 HFI1_CAP_MISC_MASK)
115
116#define HFI1_ODR_MASK(rsn) ((rsn) & OPA_PI_MASK_OFFLINE_REASON)
117
118
119
120
121
122#define HFI1_CTRL_CTXT 0
123
124
125
126
127
128#define NUM_CCE_ERR_STATUS_COUNTERS 41
129#define NUM_RCV_ERR_STATUS_COUNTERS 64
130#define NUM_MISC_ERR_STATUS_COUNTERS 13
131#define NUM_SEND_PIO_ERR_STATUS_COUNTERS 36
132#define NUM_SEND_DMA_ERR_STATUS_COUNTERS 4
133#define NUM_SEND_EGRESS_ERR_STATUS_COUNTERS 64
134#define NUM_SEND_ERR_STATUS_COUNTERS 3
135#define NUM_SEND_CTXT_ERR_STATUS_COUNTERS 5
136#define NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS 24
137
138
139
140
141
142
143
144
145
146struct hfi1_ib_stats {
147 __u64 sps_ints;
148 __u64 sps_errints;
149 __u64 sps_txerrs;
150 __u64 sps_rcverrs;
151 __u64 sps_hwerrs;
152 __u64 sps_nopiobufs;
153 __u64 sps_ctxts;
154 __u64 sps_lenerrs;
155 __u64 sps_buffull;
156 __u64 sps_hdrfull;
157};
158
159extern struct hfi1_ib_stats hfi1_stats;
160extern const struct pci_error_handlers hfi1_pci_err_handler;
161
162extern int num_driver_cntrs;
163
164
165
166
167
168
169
170#define HFI1_TRAFFIC_ACTIVE_THRESHOLD (2000)
171
172
173
174
175
176struct hfi1_opcode_stats_perctx;
177
178struct ctxt_eager_bufs {
179 struct eager_buffer {
180 void *addr;
181 dma_addr_t dma;
182 ssize_t len;
183 } *buffers;
184 struct {
185 void *addr;
186 dma_addr_t dma;
187 } *rcvtids;
188 u32 size;
189 u32 rcvtid_size;
190 u16 count;
191 u16 numbufs;
192 u16 alloced;
193 u16 threshold;
194};
195
196struct exp_tid_set {
197 struct list_head list;
198 u32 count;
199};
200
201struct hfi1_ctxtdata;
202typedef int (*intr_handler)(struct hfi1_ctxtdata *rcd, int data);
203typedef void (*rhf_rcv_function_ptr)(struct hfi1_packet *packet);
204
205struct tid_queue {
206 struct list_head queue_head;
207
208 u32 enqueue;
209 u32 dequeue;
210};
211
212struct hfi1_ctxtdata {
213
214 void *rcvhdrq;
215
216 volatile __le64 *rcvhdrtail_kvaddr;
217
218 struct hfi1_pportdata *ppd;
219
220 struct hfi1_devdata *dd;
221
222 struct send_context *sc;
223
224 const rhf_rcv_function_ptr *rhf_rcv_function_map;
225
226
227
228
229
230
231
232 intr_handler do_interrupt;
233
234 intr_handler fast_handler;
235
236 intr_handler slow_handler;
237
238 struct napi_struct *napi;
239
240 struct hfi1_opcode_stats_perctx *opstats;
241
242 u64 imask;
243
244 u32 head;
245
246 u16 rcvhdrq_cnt;
247 u8 ireg;
248
249 u8 seq_cnt;
250
251 u8 rcvhdrqentsize;
252
253 u8 rhf_offset;
254
255 u8 rcvavail_timeout;
256
257 bool is_vnic;
258
259 u8 vnic_q_idx;
260
261 bool aspm_intr_supported;
262
263 bool aspm_enabled;
264
265 bool aspm_intr_enable;
266 struct ctxt_eager_bufs egrbufs;
267
268 struct list_head qp_wait_list;
269
270 struct exp_tid_set tid_group_list;
271 struct exp_tid_set tid_used_list;
272 struct exp_tid_set tid_full_list;
273
274
275 struct timer_list aspm_timer;
276
277 unsigned long flags;
278
279 struct tid_group *groups;
280
281 dma_addr_t rcvhdrq_dma;
282 dma_addr_t rcvhdrqtailaddr_dma;
283
284 ktime_t aspm_ts_last_intr;
285
286 ktime_t aspm_ts_timer_sched;
287
288 spinlock_t aspm_lock;
289
290 struct kref kref;
291
292 int numa_id;
293
294 s16 msix_intr;
295
296 u16 jkey;
297
298 u16 rcv_array_groups;
299
300 u16 eager_base;
301
302 u16 expected_count;
303
304 u16 expected_base;
305
306 u8 ctxt;
307
308
309
310 struct mutex exp_mutex;
311
312 spinlock_t exp_lock;
313
314 struct tid_queue flow_queue;
315
316 struct tid_queue rarr_queue;
317
318 wait_queue_head_t wait;
319
320 u8 uuid[16];
321
322 char comm[TASK_COMM_LEN];
323
324 DECLARE_BITMAP(in_use_ctxts, HFI1_MAX_SHARED_CTXTS);
325
326 unsigned long event_flags;
327
328 void *subctxt_uregbase;
329
330 void *subctxt_rcvegrbuf;
331
332 void *subctxt_rcvhdr_base;
333
334 u32 urgent;
335
336 u32 urgent_poll;
337
338 u16 poll_type;
339
340 u16 subctxt_id;
341
342 u32 userversion;
343
344
345
346
347 u8 subctxt_cnt;
348
349
350 unsigned long flow_mask;
351 struct tid_flow_state flows[RXE_NUM_TID_FLOWS];
352};
353
354
355
356
357
358
359
360
361static inline u32 rcvhdrq_size(struct hfi1_ctxtdata *rcd)
362{
363 return PAGE_ALIGN(rcd->rcvhdrq_cnt *
364 rcd->rcvhdrqentsize * sizeof(u32));
365}
366
367
368
369
370
371
372
373
374struct hfi1_packet {
375 void *ebuf;
376 void *hdr;
377 void *payload;
378 struct hfi1_ctxtdata *rcd;
379 __le32 *rhf_addr;
380 struct rvt_qp *qp;
381 struct ib_other_headers *ohdr;
382 struct ib_grh *grh;
383 struct opa_16b_mgmt *mgmt;
384 u64 rhf;
385 u32 maxcnt;
386 u32 rhqoff;
387 u32 dlid;
388 u32 slid;
389 int numpkt;
390 u16 tlen;
391 s16 etail;
392 u16 pkey;
393 u8 hlen;
394 u8 rsize;
395 u8 updegr;
396 u8 etype;
397 u8 extra_byte;
398 u8 pad;
399 u8 sc;
400 u8 sl;
401 u8 opcode;
402 bool migrated;
403};
404
405
406#define HFI1_PKT_TYPE_9B 0
407#define HFI1_PKT_TYPE_16B 1
408
409
410
411
412#define OPA_16B_L4_MASK 0xFFull
413#define OPA_16B_SC_MASK 0x1F00000ull
414#define OPA_16B_SC_SHIFT 20
415#define OPA_16B_LID_MASK 0xFFFFFull
416#define OPA_16B_DLID_MASK 0xF000ull
417#define OPA_16B_DLID_SHIFT 20
418#define OPA_16B_DLID_HIGH_SHIFT 12
419#define OPA_16B_SLID_MASK 0xF00ull
420#define OPA_16B_SLID_SHIFT 20
421#define OPA_16B_SLID_HIGH_SHIFT 8
422#define OPA_16B_BECN_MASK 0x80000000ull
423#define OPA_16B_BECN_SHIFT 31
424#define OPA_16B_FECN_MASK 0x10000000ull
425#define OPA_16B_FECN_SHIFT 28
426#define OPA_16B_L2_MASK 0x60000000ull
427#define OPA_16B_L2_SHIFT 29
428#define OPA_16B_PKEY_MASK 0xFFFF0000ull
429#define OPA_16B_PKEY_SHIFT 16
430#define OPA_16B_LEN_MASK 0x7FF00000ull
431#define OPA_16B_LEN_SHIFT 20
432#define OPA_16B_RC_MASK 0xE000000ull
433#define OPA_16B_RC_SHIFT 25
434#define OPA_16B_AGE_MASK 0xFF0000ull
435#define OPA_16B_AGE_SHIFT 16
436#define OPA_16B_ENTROPY_MASK 0xFFFFull
437
438
439
440
441#define OPA_16B_L4_9B 0x00
442#define OPA_16B_L2_TYPE 0x02
443#define OPA_16B_L4_FM 0x08
444#define OPA_16B_L4_IB_LOCAL 0x09
445#define OPA_16B_L4_IB_GLOBAL 0x0A
446#define OPA_16B_L4_ETHR OPA_VNIC_L4_ETHR
447
448
449
450
451#define OPA_16B_L4_FM_PAD 3
452#define OPA_16B_L4_FM_HLEN 24
453
454static inline u8 hfi1_16B_get_l4(struct hfi1_16b_header *hdr)
455{
456 return (u8)(hdr->lrh[2] & OPA_16B_L4_MASK);
457}
458
459static inline u8 hfi1_16B_get_sc(struct hfi1_16b_header *hdr)
460{
461 return (u8)((hdr->lrh[1] & OPA_16B_SC_MASK) >> OPA_16B_SC_SHIFT);
462}
463
464static inline u32 hfi1_16B_get_dlid(struct hfi1_16b_header *hdr)
465{
466 return (u32)((hdr->lrh[1] & OPA_16B_LID_MASK) |
467 (((hdr->lrh[2] & OPA_16B_DLID_MASK) >>
468 OPA_16B_DLID_HIGH_SHIFT) << OPA_16B_DLID_SHIFT));
469}
470
471static inline u32 hfi1_16B_get_slid(struct hfi1_16b_header *hdr)
472{
473 return (u32)((hdr->lrh[0] & OPA_16B_LID_MASK) |
474 (((hdr->lrh[2] & OPA_16B_SLID_MASK) >>
475 OPA_16B_SLID_HIGH_SHIFT) << OPA_16B_SLID_SHIFT));
476}
477
478static inline u8 hfi1_16B_get_becn(struct hfi1_16b_header *hdr)
479{
480 return (u8)((hdr->lrh[0] & OPA_16B_BECN_MASK) >> OPA_16B_BECN_SHIFT);
481}
482
483static inline u8 hfi1_16B_get_fecn(struct hfi1_16b_header *hdr)
484{
485 return (u8)((hdr->lrh[1] & OPA_16B_FECN_MASK) >> OPA_16B_FECN_SHIFT);
486}
487
488static inline u8 hfi1_16B_get_l2(struct hfi1_16b_header *hdr)
489{
490 return (u8)((hdr->lrh[1] & OPA_16B_L2_MASK) >> OPA_16B_L2_SHIFT);
491}
492
493static inline u16 hfi1_16B_get_pkey(struct hfi1_16b_header *hdr)
494{
495 return (u16)((hdr->lrh[2] & OPA_16B_PKEY_MASK) >> OPA_16B_PKEY_SHIFT);
496}
497
498static inline u8 hfi1_16B_get_rc(struct hfi1_16b_header *hdr)
499{
500 return (u8)((hdr->lrh[1] & OPA_16B_RC_MASK) >> OPA_16B_RC_SHIFT);
501}
502
503static inline u8 hfi1_16B_get_age(struct hfi1_16b_header *hdr)
504{
505 return (u8)((hdr->lrh[3] & OPA_16B_AGE_MASK) >> OPA_16B_AGE_SHIFT);
506}
507
508static inline u16 hfi1_16B_get_len(struct hfi1_16b_header *hdr)
509{
510 return (u16)((hdr->lrh[0] & OPA_16B_LEN_MASK) >> OPA_16B_LEN_SHIFT);
511}
512
513static inline u16 hfi1_16B_get_entropy(struct hfi1_16b_header *hdr)
514{
515 return (u16)(hdr->lrh[3] & OPA_16B_ENTROPY_MASK);
516}
517
518#define OPA_16B_MAKE_QW(low_dw, high_dw) (((u64)(high_dw) << 32) | (low_dw))
519
520
521
522
523#define OPA_16B_BTH_PAD_MASK 7
524static inline u8 hfi1_16B_bth_get_pad(struct ib_other_headers *ohdr)
525{
526 return (u8)((be32_to_cpu(ohdr->bth[0]) >> IB_BTH_PAD_SHIFT) &
527 OPA_16B_BTH_PAD_MASK);
528}
529
530
531
532
533#define OPA_16B_MGMT_QPN_MASK 0xFFFFFF
534static inline u32 hfi1_16B_get_dest_qpn(struct opa_16b_mgmt *mgmt)
535{
536 return be32_to_cpu(mgmt->dest_qpn) & OPA_16B_MGMT_QPN_MASK;
537}
538
539static inline u32 hfi1_16B_get_src_qpn(struct opa_16b_mgmt *mgmt)
540{
541 return be32_to_cpu(mgmt->src_qpn) & OPA_16B_MGMT_QPN_MASK;
542}
543
544static inline void hfi1_16B_set_qpn(struct opa_16b_mgmt *mgmt,
545 u32 dest_qp, u32 src_qp)
546{
547 mgmt->dest_qpn = cpu_to_be32(dest_qp & OPA_16B_MGMT_QPN_MASK);
548 mgmt->src_qpn = cpu_to_be32(src_qp & OPA_16B_MGMT_QPN_MASK);
549}
550
551
552
553
554
555static inline struct ib_other_headers *
556hfi1_get_rc_ohdr(struct hfi1_opa_header *opah)
557{
558 struct ib_other_headers *ohdr;
559 struct ib_header *hdr = NULL;
560 struct hfi1_16b_header *hdr_16b = NULL;
561
562
563 if (opah->hdr_type == HFI1_PKT_TYPE_9B) {
564 hdr = &opah->ibh;
565 if (ib_get_lnh(hdr) == HFI1_LRH_BTH)
566 ohdr = &hdr->u.oth;
567 else
568 ohdr = &hdr->u.l.oth;
569 } else {
570 u8 l4;
571
572 hdr_16b = &opah->opah;
573 l4 = hfi1_16B_get_l4(hdr_16b);
574 if (l4 == OPA_16B_L4_IB_LOCAL)
575 ohdr = &hdr_16b->u.oth;
576 else
577 ohdr = &hdr_16b->u.l.oth;
578 }
579 return ohdr;
580}
581
582struct rvt_sge_state;
583
584
585
586
587
588
589#define HFI1_IB_CFG_LIDLMC 0
590#define HFI1_IB_CFG_LWID_DG_ENB 1
591#define HFI1_IB_CFG_LWID_ENB 2
592#define HFI1_IB_CFG_LWID 3
593#define HFI1_IB_CFG_SPD_ENB 4
594#define HFI1_IB_CFG_SPD 5
595#define HFI1_IB_CFG_RXPOL_ENB 6
596#define HFI1_IB_CFG_LREV_ENB 7
597#define HFI1_IB_CFG_LINKLATENCY 8
598#define HFI1_IB_CFG_HRTBT 9
599#define HFI1_IB_CFG_OP_VLS 10
600#define HFI1_IB_CFG_VL_HIGH_CAP 11
601#define HFI1_IB_CFG_VL_LOW_CAP 12
602#define HFI1_IB_CFG_OVERRUN_THRESH 13
603#define HFI1_IB_CFG_PHYERR_THRESH 14
604#define HFI1_IB_CFG_LINKDEFAULT 15
605#define HFI1_IB_CFG_PKEYS 16
606#define HFI1_IB_CFG_MTU 17
607#define HFI1_IB_CFG_VL_HIGH_LIMIT 19
608#define HFI1_IB_CFG_PMA_TICKS 20
609#define HFI1_IB_CFG_PORT 21
610
611
612
613
614
615
616
617
618
619#define __HLS_UP_INIT_BP 0
620#define __HLS_UP_ARMED_BP 1
621#define __HLS_UP_ACTIVE_BP 2
622#define __HLS_DN_DOWNDEF_BP 3
623#define __HLS_DN_POLL_BP 4
624#define __HLS_DN_DISABLE_BP 5
625#define __HLS_DN_OFFLINE_BP 6
626#define __HLS_VERIFY_CAP_BP 7
627#define __HLS_GOING_UP_BP 8
628#define __HLS_GOING_OFFLINE_BP 9
629#define __HLS_LINK_COOLDOWN_BP 10
630
631#define HLS_UP_INIT BIT(__HLS_UP_INIT_BP)
632#define HLS_UP_ARMED BIT(__HLS_UP_ARMED_BP)
633#define HLS_UP_ACTIVE BIT(__HLS_UP_ACTIVE_BP)
634#define HLS_DN_DOWNDEF BIT(__HLS_DN_DOWNDEF_BP)
635#define HLS_DN_POLL BIT(__HLS_DN_POLL_BP)
636#define HLS_DN_DISABLE BIT(__HLS_DN_DISABLE_BP)
637#define HLS_DN_OFFLINE BIT(__HLS_DN_OFFLINE_BP)
638#define HLS_VERIFY_CAP BIT(__HLS_VERIFY_CAP_BP)
639#define HLS_GOING_UP BIT(__HLS_GOING_UP_BP)
640#define HLS_GOING_OFFLINE BIT(__HLS_GOING_OFFLINE_BP)
641#define HLS_LINK_COOLDOWN BIT(__HLS_LINK_COOLDOWN_BP)
642
643#define HLS_UP (HLS_UP_INIT | HLS_UP_ARMED | HLS_UP_ACTIVE)
644#define HLS_DOWN ~(HLS_UP)
645
646#define HLS_DEFAULT HLS_DN_POLL
647
648
649#define HFI1_DEFAULT_ACTIVE_MTU 10240
650
651#define HFI1_DEFAULT_MAX_MTU 10240
652
653#define DEFAULT_PKEY 0xffff
654
655
656
657
658#define FM_TBL_VL_HIGH_ARB 1
659#define FM_TBL_VL_LOW_ARB 2
660#define FM_TBL_BUFFER_CONTROL 3
661#define FM_TBL_SC2VLNT 4
662#define FM_TBL_VL_PREEMPT_ELEMS 5
663#define FM_TBL_VL_PREEMPT_MATRIX 6
664
665
666
667
668
669
670#define HFI1_RCVCTRL_TAILUPD_ENB 0x01
671#define HFI1_RCVCTRL_TAILUPD_DIS 0x02
672#define HFI1_RCVCTRL_CTXT_ENB 0x04
673#define HFI1_RCVCTRL_CTXT_DIS 0x08
674#define HFI1_RCVCTRL_INTRAVAIL_ENB 0x10
675#define HFI1_RCVCTRL_INTRAVAIL_DIS 0x20
676#define HFI1_RCVCTRL_PKEY_ENB 0x40
677#define HFI1_RCVCTRL_PKEY_DIS 0x80
678#define HFI1_RCVCTRL_TIDFLOW_ENB 0x0400
679#define HFI1_RCVCTRL_TIDFLOW_DIS 0x0800
680#define HFI1_RCVCTRL_ONE_PKT_EGR_ENB 0x1000
681#define HFI1_RCVCTRL_ONE_PKT_EGR_DIS 0x2000
682#define HFI1_RCVCTRL_NO_RHQ_DROP_ENB 0x4000
683#define HFI1_RCVCTRL_NO_RHQ_DROP_DIS 0x8000
684#define HFI1_RCVCTRL_NO_EGR_DROP_ENB 0x10000
685#define HFI1_RCVCTRL_NO_EGR_DROP_DIS 0x20000
686#define HFI1_RCVCTRL_URGENT_ENB 0x40000
687#define HFI1_RCVCTRL_URGENT_DIS 0x80000
688
689
690#define HFI1_PART_ENFORCE_IN 0x1
691#define HFI1_PART_ENFORCE_OUT 0x2
692
693
694#define SYNTH_CNT_TIME 3
695
696
697#define CNTR_NORMAL 0x0
698#define CNTR_SYNTH 0x1
699#define CNTR_DISABLED 0x2
700#define CNTR_32BIT 0x4
701#define CNTR_VL 0x8
702#define CNTR_SDMA 0x10
703#define CNTR_INVALID_VL -1
704#define CNTR_MODE_W 0x0
705#define CNTR_MODE_R 0x1
706
707
708#define HFI1_MIN_VLS_SUPPORTED 1
709#define HFI1_MAX_VLS_SUPPORTED 8
710
711#define HFI1_GUIDS_PER_PORT 5
712#define HFI1_PORT_GUID_INDEX 0
713
714static inline void incr_cntr64(u64 *cntr)
715{
716 if (*cntr < (u64)-1LL)
717 (*cntr)++;
718}
719
720static inline void incr_cntr32(u32 *cntr)
721{
722 if (*cntr < (u32)-1LL)
723 (*cntr)++;
724}
725
726#define MAX_NAME_SIZE 64
727struct hfi1_msix_entry {
728 enum irq_type type;
729 int irq;
730 void *arg;
731 cpumask_t mask;
732 struct irq_affinity_notify notify;
733};
734
735struct hfi1_msix_info {
736
737 spinlock_t msix_lock;
738 DECLARE_BITMAP(in_use_msix, CCE_NUM_MSIX_VECTORS);
739 struct hfi1_msix_entry *msix_entries;
740 u16 max_requested;
741};
742
743
744struct cca_timer {
745 struct hrtimer hrtimer;
746 struct hfi1_pportdata *ppd;
747 int sl;
748 u16 ccti;
749};
750
751struct link_down_reason {
752
753
754
755
756 u8 sma;
757 u8 latest;
758};
759
760enum {
761 LO_PRIO_TABLE,
762 HI_PRIO_TABLE,
763 MAX_PRIO_TABLE
764};
765
766struct vl_arb_cache {
767
768 spinlock_t lock;
769 struct ib_vl_weight_elem table[VL_ARB_TABLE_SIZE];
770};
771
772
773
774
775
776
777
778struct hfi1_pportdata {
779 struct hfi1_ibport ibport_data;
780
781 struct hfi1_devdata *dd;
782 struct kobject pport_cc_kobj;
783 struct kobject sc2vl_kobj;
784 struct kobject sl2sc_kobj;
785 struct kobject vl2mtu_kobj;
786
787
788 struct qsfp_data qsfp_info;
789
790 u32 port_type;
791 u32 tx_preset_eq;
792 u32 tx_preset_noeq;
793 u32 rx_preset;
794 u8 local_atten;
795 u8 remote_atten;
796 u8 default_atten;
797 u8 max_power_class;
798
799
800 bool config_from_scratch;
801
802
803 u64 guids[HFI1_GUIDS_PER_PORT];
804
805
806 u64 neighbor_guid;
807
808
809 u32 linkup;
810
811
812
813
814
815 u64 *statusp;
816
817
818
819 struct workqueue_struct *hfi1_wq;
820 struct workqueue_struct *link_wq;
821
822
823 struct work_struct link_vc_work;
824 struct work_struct link_up_work;
825 struct work_struct link_down_work;
826 struct work_struct sma_message_work;
827 struct work_struct freeze_work;
828 struct work_struct link_downgrade_work;
829 struct work_struct link_bounce_work;
830 struct delayed_work start_link_work;
831
832 struct mutex hls_lock;
833 u32 host_link_state;
834
835
836
837 u32 ibmtu;
838
839
840
841
842 u32 ibmaxlen;
843 u32 current_egress_rate;
844
845 u32 lid;
846
847 u16 pkeys[MAX_PKEY_VALUES];
848 u16 link_width_supported;
849 u16 link_width_downgrade_supported;
850 u16 link_speed_supported;
851 u16 link_width_enabled;
852 u16 link_width_downgrade_enabled;
853 u16 link_speed_enabled;
854 u16 link_width_active;
855 u16 link_width_downgrade_tx_active;
856 u16 link_width_downgrade_rx_active;
857 u16 link_speed_active;
858 u8 vls_supported;
859 u8 vls_operational;
860 u8 actual_vls_operational;
861
862 u8 lmc;
863
864 u8 rx_pol_inv;
865
866 u8 hw_pidx;
867 u8 port;
868
869 u8 neighbor_type;
870 u8 neighbor_normal;
871 u8 neighbor_fm_security;
872 u8 neighbor_port_number;
873 u8 is_sm_config_started;
874 u8 offline_disabled_reason;
875 u8 is_active_optimize_enabled;
876 u8 driver_link_ready;
877 u8 link_enabled;
878 u8 linkinit_reason;
879 u8 local_tx_rate;
880 u8 qsfp_retry_count;
881
882
883 u8 overrun_threshold;
884 u8 phy_error_threshold;
885 unsigned int is_link_down_queued;
886
887
888
889
890
891
892 unsigned long led_override_vals[2];
893 u8 led_override_phase;
894 atomic_t led_override_timer_active;
895
896 struct timer_list led_override_timer;
897
898 u32 sm_trap_qp;
899 u32 sa_qp;
900
901
902
903
904
905 spinlock_t cca_timer_lock ____cacheline_aligned_in_smp;
906 struct cca_timer cca_timer[OPA_MAX_SLS];
907
908
909 struct ib_cc_table_entry_shadow ccti_entries[CC_TABLE_SHADOW_MAX];
910
911
912 struct opa_congestion_setting_entry_shadow
913 congestion_entries[OPA_MAX_SLS];
914
915
916
917
918
919 spinlock_t cc_state_lock ____cacheline_aligned_in_smp;
920
921 struct cc_state __rcu *cc_state;
922
923
924 u16 total_cct_entry;
925
926
927 u32 cc_sl_control_map;
928
929
930 u8 cc_max_table_entries;
931
932
933
934
935
936 spinlock_t cc_log_lock ____cacheline_aligned_in_smp;
937 u8 threshold_cong_event_map[OPA_MAX_SLS / 8];
938 u16 threshold_event_counter;
939 struct opa_hfi1_cong_log_event_internal cc_events[OPA_CONG_LOG_ELEMS];
940 int cc_log_idx;
941 int cc_mad_idx;
942
943
944 struct vl_arb_cache vl_arb_cache[MAX_PRIO_TABLE];
945
946
947 u64 *cntrs;
948
949 u64 *scntrs;
950
951 u64 port_xmit_discards;
952 u64 port_xmit_discards_vl[C_VL_COUNT];
953 u64 port_xmit_constraint_errors;
954 u64 port_rcv_constraint_errors;
955
956 u64 link_downed;
957
958 u64 link_up;
959
960 u64 unknown_frame_count;
961
962 u16 port_ltp_crc_mode;
963
964 u8 port_crc_mode_enabled;
965
966 u8 mgmt_allowed;
967 u8 part_enforce;
968 struct link_down_reason local_link_down_reason;
969 struct link_down_reason neigh_link_down_reason;
970
971 u8 remote_link_down_reason;
972
973 u32 port_error_action;
974 struct work_struct linkstate_active_work;
975
976 bool cc_prescan;
977
978
979
980
981 u64 port_vl_xmit_wait_last[C_VL_COUNT + 1];
982 u16 prev_link_width;
983 u64 vl_xmit_flit_cnt[C_VL_COUNT + 1];
984};
985
986typedef void (*opcode_handler)(struct hfi1_packet *packet);
987typedef void (*hfi1_make_req)(struct rvt_qp *qp,
988 struct hfi1_pkt_state *ps,
989 struct rvt_swqe *wqe);
990extern const rhf_rcv_function_ptr normal_rhf_rcv_functions[];
991extern const rhf_rcv_function_ptr netdev_rhf_rcv_functions[];
992
993
994#define RHF_RCV_CONTINUE 0
995#define RHF_RCV_DONE 1
996#define RHF_RCV_REPROCESS 2
997
998struct rcv_array_data {
999 u16 ngroups;
1000 u16 nctxt_extra;
1001 u8 group_size;
1002};
1003
1004struct per_vl_data {
1005 u16 mtu;
1006 struct send_context *sc;
1007};
1008
1009
1010#define PER_VL_SEND_CONTEXTS 16
1011
1012struct err_info_rcvport {
1013 u8 status_and_code;
1014 u64 packet_flit1;
1015 u64 packet_flit2;
1016};
1017
1018struct err_info_constraint {
1019 u8 status;
1020 u16 pkey;
1021 u32 slid;
1022};
1023
1024struct hfi1_temp {
1025 unsigned int curr;
1026 unsigned int lo_lim;
1027 unsigned int hi_lim;
1028 unsigned int crit_lim;
1029 u8 triggers;
1030};
1031
1032struct hfi1_i2c_bus {
1033 struct hfi1_devdata *controlling_dd;
1034 struct i2c_adapter adapter;
1035 struct i2c_algo_bit_data algo;
1036 int num;
1037};
1038
1039
1040struct hfi1_asic_data {
1041 struct hfi1_devdata *dds[2];
1042 struct mutex asic_resource_mutex;
1043 struct hfi1_i2c_bus *i2c_bus0;
1044 struct hfi1_i2c_bus *i2c_bus1;
1045};
1046
1047
1048#define NUM_MAP_ENTRIES 256
1049#define NUM_MAP_REGS 32
1050
1051
1052struct hfi1_vnic_data {
1053 struct kmem_cache *txreq_cache;
1054 u8 num_vports;
1055};
1056
1057struct hfi1_vnic_vport_info;
1058
1059
1060
1061
1062struct sdma_engine;
1063struct sdma_vl_map;
1064
1065#define BOARD_VERS_MAX 96
1066#define SERIAL_MAX 16
1067
1068typedef int (*send_routine)(struct rvt_qp *, struct hfi1_pkt_state *, u64);
1069struct hfi1_devdata {
1070 struct hfi1_ibdev verbs_dev;
1071
1072
1073 struct pci_dev *pcidev;
1074 struct cdev user_cdev;
1075 struct cdev diag_cdev;
1076 struct cdev ui_cdev;
1077 struct device *user_device;
1078 struct device *diag_device;
1079 struct device *ui_device;
1080
1081
1082 u8 __iomem *kregbase1;
1083 resource_size_t physaddr;
1084
1085
1086 u8 __iomem *kregbase2;
1087
1088 u32 base2_start;
1089
1090
1091 struct per_vl_data vld[PER_VL_SEND_CONTEXTS];
1092
1093 struct send_context_info *send_contexts;
1094
1095 u8 *hw_to_sw;
1096
1097 spinlock_t sc_lock;
1098
1099 spinlock_t pio_map_lock;
1100
1101 spinlock_t sc_init_lock;
1102
1103 spinlock_t sde_map_lock;
1104
1105 struct send_context **kernel_send_context;
1106
1107 struct pio_vl_map __rcu *pio_map;
1108
1109 u64 default_desc1;
1110
1111
1112
1113 volatile __le64 *sdma_heads_dma;
1114 dma_addr_t sdma_heads_phys;
1115 void *sdma_pad_dma;
1116 dma_addr_t sdma_pad_phys;
1117
1118 size_t sdma_heads_size;
1119
1120 u32 num_sdma;
1121
1122 struct sdma_engine *per_sdma;
1123
1124 struct sdma_vl_map __rcu *sdma_map;
1125
1126 wait_queue_head_t sdma_unfreeze_wq;
1127 atomic_t sdma_unfreeze_count;
1128
1129 u32 lcb_access_count;
1130
1131
1132 struct hfi1_asic_data *asic_data;
1133
1134
1135 void __iomem *piobase;
1136
1137
1138
1139
1140 void __iomem *rcvarray_wc;
1141
1142
1143
1144
1145 struct credit_return_base *cr_base;
1146
1147
1148 struct sc_config_sizes sc_sizes[SC_MAX];
1149
1150 char *boardname;
1151
1152 u64 ctx0_seq_drop;
1153
1154
1155 u64 z_int_counter;
1156 u64 z_rcv_limit;
1157 u64 z_send_schedule;
1158
1159 u64 __percpu *send_schedule;
1160
1161 u16 num_netdev_contexts;
1162
1163 u32 num_rcv_contexts;
1164
1165 u32 num_send_contexts;
1166
1167
1168
1169 u32 freectxts;
1170
1171 u32 num_user_contexts;
1172
1173 u32 rcv_intr_timeout_csr;
1174
1175 spinlock_t sendctrl_lock;
1176 spinlock_t rcvctrl_lock;
1177 spinlock_t uctxt_lock;
1178 struct mutex dc8051_lock;
1179 struct workqueue_struct *update_cntr_wq;
1180 struct work_struct update_cntr_work;
1181
1182 spinlock_t dc8051_memlock;
1183 int dc8051_timed_out;
1184
1185
1186
1187
1188 unsigned long *events;
1189
1190
1191
1192
1193
1194 struct hfi1_status *status;
1195
1196
1197 u64 revision;
1198
1199 u64 base_guid;
1200
1201
1202 u8 link_gen3_capable;
1203 u8 dc_shutdown;
1204
1205 u32 lbus_width;
1206
1207 u32 lbus_speed;
1208 int unit;
1209 int node;
1210
1211
1212 u32 pcibar0;
1213 u32 pcibar1;
1214 u32 pci_rom;
1215 u16 pci_command;
1216 u16 pcie_devctl;
1217 u16 pcie_lnkctl;
1218 u16 pcie_devctl2;
1219 u32 pci_msix0;
1220 u32 pci_tph2;
1221
1222
1223
1224
1225
1226 u8 serial[SERIAL_MAX];
1227
1228 u8 boardversion[BOARD_VERS_MAX];
1229 u8 lbus_info[32];
1230
1231 u8 majrev;
1232
1233 u8 minrev;
1234
1235 u8 hfi1_id;
1236
1237 u8 icode;
1238
1239 u8 vau;
1240
1241 u8 vcu;
1242
1243 u16 link_credits;
1244
1245 u16 vl15_init;
1246
1247
1248
1249
1250
1251
1252
1253 u16 vl15buf_cached;
1254
1255
1256 u8 n_krcv_queues;
1257 u8 qos_shift;
1258
1259 u16 irev;
1260 u32 dc8051_ver;
1261
1262 spinlock_t hfi1_diag_trans_lock;
1263 struct platform_config platform_config;
1264 struct platform_config_cache pcfg_cache;
1265
1266 struct diag_client *diag_client;
1267
1268
1269 u64 gi_mask[CCE_NUM_INT_CSRS];
1270
1271 struct rcv_array_data rcv_entries;
1272
1273
1274 u16 psxmitwait_check_rate;
1275
1276
1277
1278
1279 struct timer_list synth_stats_timer;
1280
1281
1282 struct hfi1_msix_info msix_info;
1283
1284
1285
1286
1287 char *cntrnames;
1288 size_t cntrnameslen;
1289 size_t ndevcntrs;
1290 u64 *cntrs;
1291 u64 *scntrs;
1292
1293
1294
1295
1296 u64 last_tx;
1297 u64 last_rx;
1298
1299
1300
1301
1302 size_t nportcntrs;
1303 char *portcntrnames;
1304 size_t portcntrnameslen;
1305
1306 struct err_info_rcvport err_info_rcvport;
1307 struct err_info_constraint err_info_rcv_constraint;
1308 struct err_info_constraint err_info_xmit_constraint;
1309
1310 atomic_t drop_packet;
1311 bool do_drop;
1312 u8 err_info_uncorrectable;
1313 u8 err_info_fmconfig;
1314
1315
1316
1317
1318
1319 u64 cce_err_status_cnt[NUM_CCE_ERR_STATUS_COUNTERS];
1320 u64 rcv_err_status_cnt[NUM_RCV_ERR_STATUS_COUNTERS];
1321 u64 misc_err_status_cnt[NUM_MISC_ERR_STATUS_COUNTERS];
1322 u64 send_pio_err_status_cnt[NUM_SEND_PIO_ERR_STATUS_COUNTERS];
1323 u64 send_dma_err_status_cnt[NUM_SEND_DMA_ERR_STATUS_COUNTERS];
1324 u64 send_egress_err_status_cnt[NUM_SEND_EGRESS_ERR_STATUS_COUNTERS];
1325 u64 send_err_status_cnt[NUM_SEND_ERR_STATUS_COUNTERS];
1326
1327
1328 u64 sw_ctxt_err_status_cnt[NUM_SEND_CTXT_ERR_STATUS_COUNTERS];
1329
1330 u64 sw_send_dma_eng_err_status_cnt[
1331 NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS];
1332
1333 u64 sw_cce_err_status_aggregate;
1334
1335 u64 sw_rcv_bypass_packet_errors;
1336
1337
1338 u64 lcb_err_en;
1339 struct cpu_mask_set *comp_vect;
1340 int *comp_vect_mappings;
1341 u32 comp_vect_possible_cpus;
1342
1343
1344
1345
1346
1347 send_routine process_pio_send ____cacheline_aligned_in_smp;
1348 send_routine process_dma_send;
1349 void (*pio_inline_send)(struct hfi1_devdata *dd, struct pio_buf *pbuf,
1350 u64 pbc, const void *from, size_t count);
1351 int (*process_vnic_dma_send)(struct hfi1_devdata *dd, u8 q_idx,
1352 struct hfi1_vnic_vport_info *vinfo,
1353 struct sk_buff *skb, u64 pbc, u8 plen);
1354
1355
1356
1357 struct hfi1_pportdata *pport;
1358
1359 struct hfi1_ctxtdata **rcd;
1360 u64 __percpu *int_counter;
1361
1362 struct hfi1_opcode_stats_perctx __percpu *tx_opstats;
1363
1364 u16 flags;
1365
1366 u8 num_pports;
1367
1368 u8 first_dyn_alloc_ctxt;
1369
1370
1371
1372 seqlock_t sc2vl_lock ____cacheline_aligned_in_smp;
1373 u64 sc2vl[4];
1374 u64 __percpu *rcv_limit;
1375
1376
1377
1378 u8 oui1;
1379 u8 oui2;
1380 u8 oui3;
1381
1382
1383 struct timer_list rcverr_timer;
1384
1385 wait_queue_head_t event_queue;
1386
1387
1388 __le64 *rcvhdrtail_dummy_kvaddr;
1389 dma_addr_t rcvhdrtail_dummy_dma;
1390
1391 u32 rcv_ovfl_cnt;
1392
1393 spinlock_t aspm_lock;
1394
1395 atomic_t aspm_disabled_cnt;
1396
1397 atomic_t user_refcount;
1398
1399 struct completion user_comp;
1400
1401 bool eprom_available;
1402 bool aspm_supported;
1403 bool aspm_enabled;
1404 struct rhashtable *sdma_rht;
1405
1406
1407 struct hfi1_vnic_data vnic;
1408
1409 spinlock_t irq_src_lock;
1410 int vnic_num_vports;
1411 struct net_device *dummy_netdev;
1412
1413
1414 atomic_t ipoib_rsm_usr_num;
1415};
1416
1417
1418#define dc8051_ver(a, b, c) ((a) << 16 | (b) << 8 | (c))
1419#define dc8051_ver_maj(a) (((a) & 0xff0000) >> 16)
1420#define dc8051_ver_min(a) (((a) & 0x00ff00) >> 8)
1421#define dc8051_ver_patch(a) ((a) & 0x0000ff)
1422
1423
1424#define PT_EXPECTED 0
1425#define PT_EAGER 1
1426#define PT_INVALID_FLUSH 2
1427#define PT_INVALID 3
1428
1429struct tid_rb_node;
1430struct mmu_rb_node;
1431struct mmu_rb_handler;
1432
1433
1434struct hfi1_filedata {
1435 struct srcu_struct pq_srcu;
1436 struct hfi1_devdata *dd;
1437 struct hfi1_ctxtdata *uctxt;
1438 struct hfi1_user_sdma_comp_q *cq;
1439
1440 spinlock_t pq_rcu_lock;
1441 struct hfi1_user_sdma_pkt_q __rcu *pq;
1442 u16 subctxt;
1443
1444 int rec_cpu_num;
1445 u32 tid_n_pinned;
1446 bool use_mn;
1447 struct tid_rb_node **entry_to_rb;
1448 spinlock_t tid_lock;
1449 u32 tid_limit;
1450 u32 tid_used;
1451 u32 *invalid_tids;
1452 u32 invalid_tid_idx;
1453
1454 spinlock_t invalid_lock;
1455};
1456
1457extern struct xarray hfi1_dev_table;
1458struct hfi1_devdata *hfi1_lookup(int unit);
1459
1460static inline unsigned long uctxt_offset(struct hfi1_ctxtdata *uctxt)
1461{
1462 return (uctxt->ctxt - uctxt->dd->first_dyn_alloc_ctxt) *
1463 HFI1_MAX_SHARED_CTXTS;
1464}
1465
1466int hfi1_init(struct hfi1_devdata *dd, int reinit);
1467int hfi1_count_active_units(void);
1468
1469int hfi1_diag_add(struct hfi1_devdata *dd);
1470void hfi1_diag_remove(struct hfi1_devdata *dd);
1471void handle_linkup_change(struct hfi1_devdata *dd, u32 linkup);
1472
1473void handle_user_interrupt(struct hfi1_ctxtdata *rcd);
1474
1475int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd);
1476int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd);
1477int hfi1_create_kctxts(struct hfi1_devdata *dd);
1478int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
1479 struct hfi1_ctxtdata **rcd);
1480void hfi1_free_ctxt(struct hfi1_ctxtdata *rcd);
1481void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd,
1482 struct hfi1_devdata *dd, u8 hw_pidx, u8 port);
1483void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd);
1484int hfi1_rcd_put(struct hfi1_ctxtdata *rcd);
1485int hfi1_rcd_get(struct hfi1_ctxtdata *rcd);
1486struct hfi1_ctxtdata *hfi1_rcd_get_by_index_safe(struct hfi1_devdata *dd,
1487 u16 ctxt);
1488struct hfi1_ctxtdata *hfi1_rcd_get_by_index(struct hfi1_devdata *dd, u16 ctxt);
1489int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread);
1490int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread);
1491int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread);
1492int handle_receive_interrupt_napi_fp(struct hfi1_ctxtdata *rcd, int budget);
1493int handle_receive_interrupt_napi_sp(struct hfi1_ctxtdata *rcd, int budget);
1494void set_all_slowpath(struct hfi1_devdata *dd);
1495
1496extern const struct pci_device_id hfi1_pci_tbl[];
1497void hfi1_make_ud_req_9B(struct rvt_qp *qp,
1498 struct hfi1_pkt_state *ps,
1499 struct rvt_swqe *wqe);
1500
1501void hfi1_make_ud_req_16B(struct rvt_qp *qp,
1502 struct hfi1_pkt_state *ps,
1503 struct rvt_swqe *wqe);
1504
1505
1506#define RCV_PKT_OK 0x0
1507#define RCV_PKT_LIMIT 0x1
1508#define RCV_PKT_DONE 0x2
1509
1510
1511
1512
1513
1514static inline u32 hfi1_rcd_head(struct hfi1_ctxtdata *rcd)
1515{
1516 return rcd->head;
1517}
1518
1519
1520
1521
1522
1523
1524static inline void hfi1_set_rcd_head(struct hfi1_ctxtdata *rcd, u32 head)
1525{
1526 rcd->head = head;
1527}
1528
1529
1530static inline __le32 *get_rhf_addr(struct hfi1_ctxtdata *rcd)
1531{
1532 return (__le32 *)rcd->rcvhdrq + rcd->head + rcd->rhf_offset;
1533}
1534
1535
1536static inline bool get_dma_rtail_setting(struct hfi1_ctxtdata *rcd)
1537{
1538 return !!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL);
1539}
1540
1541
1542
1543
1544
1545
1546
1547static inline u8 hfi1_seq_incr_wrap(u8 seq)
1548{
1549 if (++seq > RHF_MAX_SEQ)
1550 seq = 1;
1551 return seq;
1552}
1553
1554
1555
1556
1557
1558
1559
1560static inline u8 hfi1_seq_cnt(struct hfi1_ctxtdata *rcd)
1561{
1562 return rcd->seq_cnt;
1563}
1564
1565
1566
1567
1568
1569
1570
1571static inline void hfi1_set_seq_cnt(struct hfi1_ctxtdata *rcd, u8 cnt)
1572{
1573 rcd->seq_cnt = cnt;
1574}
1575
1576
1577
1578
1579
1580
1581
1582
1583static inline bool last_rcv_seq(struct hfi1_ctxtdata *rcd, u32 seq)
1584{
1585 return seq != rcd->seq_cnt;
1586}
1587
1588
1589
1590
1591
1592
1593
1594
1595static inline bool hfi1_seq_incr(struct hfi1_ctxtdata *rcd, u32 seq)
1596{
1597 rcd->seq_cnt = hfi1_seq_incr_wrap(rcd->seq_cnt);
1598 return last_rcv_seq(rcd, seq);
1599}
1600
1601
1602
1603
1604
1605static inline u8 get_hdrqentsize(struct hfi1_ctxtdata *rcd)
1606{
1607 return rcd->rcvhdrqentsize;
1608}
1609
1610
1611
1612
1613
1614static inline u16 get_hdrq_cnt(struct hfi1_ctxtdata *rcd)
1615{
1616 return rcd->rcvhdrq_cnt;
1617}
1618
1619
1620
1621
1622
1623static inline bool hfi1_is_slowpath(struct hfi1_ctxtdata *rcd)
1624{
1625 return rcd->do_interrupt == rcd->slow_handler;
1626}
1627
1628
1629
1630
1631
1632static inline bool hfi1_is_fastpath(struct hfi1_ctxtdata *rcd)
1633{
1634 if (rcd->ctxt == HFI1_CTRL_CTXT)
1635 return false;
1636
1637 return rcd->do_interrupt == rcd->fast_handler;
1638}
1639
1640
1641
1642
1643
1644static inline void hfi1_set_fast(struct hfi1_ctxtdata *rcd)
1645{
1646 if (unlikely(!rcd))
1647 return;
1648 if (unlikely(!hfi1_is_fastpath(rcd)))
1649 rcd->do_interrupt = rcd->fast_handler;
1650}
1651
1652int hfi1_reset_device(int);
1653
1654void receive_interrupt_work(struct work_struct *work);
1655
1656
1657static inline int hfi1_9B_get_sc5(struct ib_header *hdr, u64 rhf)
1658{
1659 return ib_get_sc(hdr) | ((!!(rhf_dc_info(rhf))) << 4);
1660}
1661
1662#define HFI1_JKEY_WIDTH 16
1663#define HFI1_JKEY_MASK (BIT(16) - 1)
1664#define HFI1_ADMIN_JKEY_RANGE 32
1665
1666
1667
1668
1669
1670
1671
1672static inline u16 generate_jkey(kuid_t uid)
1673{
1674 u16 jkey = from_kuid(current_user_ns(), uid) & HFI1_JKEY_MASK;
1675
1676 if (capable(CAP_SYS_ADMIN))
1677 jkey &= HFI1_ADMIN_JKEY_RANGE - 1;
1678 else if (jkey < 64)
1679 jkey |= BIT(HFI1_JKEY_WIDTH - 1);
1680
1681 return jkey;
1682}
1683
1684
1685
1686
1687
1688
1689static inline u32 active_egress_rate(struct hfi1_pportdata *ppd)
1690{
1691 u16 link_speed = ppd->link_speed_active;
1692 u16 link_width = ppd->link_width_active;
1693 u32 egress_rate;
1694
1695 if (link_speed == OPA_LINK_SPEED_25G)
1696 egress_rate = 25000;
1697 else
1698 egress_rate = 12500;
1699
1700 switch (link_width) {
1701 case OPA_LINK_WIDTH_4X:
1702 egress_rate *= 4;
1703 break;
1704 case OPA_LINK_WIDTH_3X:
1705 egress_rate *= 3;
1706 break;
1707 case OPA_LINK_WIDTH_2X:
1708 egress_rate *= 2;
1709 break;
1710 default:
1711
1712 break;
1713 }
1714
1715 return egress_rate;
1716}
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726static inline u32 egress_cycles(u32 len, u32 rate)
1727{
1728 u32 cycles;
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738 cycles = len * 8;
1739 cycles *= 805;
1740 cycles /= rate;
1741
1742 return cycles;
1743}
1744
1745void set_link_ipg(struct hfi1_pportdata *ppd);
1746void process_becn(struct hfi1_pportdata *ppd, u8 sl, u32 rlid, u32 lqpn,
1747 u32 rqpn, u8 svc_type);
1748void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn,
1749 u16 pkey, u32 slid, u32 dlid, u8 sc5,
1750 const struct ib_grh *old_grh);
1751void return_cnp_16B(struct hfi1_ibport *ibp, struct rvt_qp *qp,
1752 u32 remote_qpn, u16 pkey, u32 slid, u32 dlid,
1753 u8 sc5, const struct ib_grh *old_grh);
1754typedef void (*hfi1_handle_cnp)(struct hfi1_ibport *ibp, struct rvt_qp *qp,
1755 u32 remote_qpn, u16 pkey, u32 slid, u32 dlid,
1756 u8 sc5, const struct ib_grh *old_grh);
1757
1758#define PKEY_CHECK_INVALID -1
1759int egress_pkey_check(struct hfi1_pportdata *ppd, u32 slid, u16 pkey,
1760 u8 sc5, int8_t s_pkey_index);
1761
1762#define PACKET_EGRESS_TIMEOUT 350
1763static inline void pause_for_credit_return(struct hfi1_devdata *dd)
1764{
1765
1766 u32 usec = cclock_to_ns(dd, PACKET_EGRESS_TIMEOUT) / 1000;
1767
1768 udelay(usec ? usec : 1);
1769}
1770
1771
1772
1773
1774
1775
1776static inline u8 sc_to_vlt(struct hfi1_devdata *dd, u8 sc5)
1777{
1778 unsigned seq;
1779 u8 rval;
1780
1781 if (sc5 >= OPA_MAX_SCS)
1782 return (u8)(0xff);
1783
1784 do {
1785 seq = read_seqbegin(&dd->sc2vl_lock);
1786 rval = *(((u8 *)dd->sc2vl) + sc5);
1787 } while (read_seqretry(&dd->sc2vl_lock, seq));
1788
1789 return rval;
1790}
1791
1792#define PKEY_MEMBER_MASK 0x8000
1793#define PKEY_LOW_15_MASK 0x7fff
1794
1795
1796
1797
1798
1799
1800
1801static inline int ingress_pkey_matches_entry(u16 pkey, u16 ent)
1802{
1803 u16 mkey = pkey & PKEY_LOW_15_MASK;
1804 u16 ment = ent & PKEY_LOW_15_MASK;
1805
1806 if (mkey == ment) {
1807
1808
1809
1810
1811
1812 if (!(pkey & PKEY_MEMBER_MASK))
1813 return !!(ent & PKEY_MEMBER_MASK);
1814 return 1;
1815 }
1816 return 0;
1817}
1818
1819
1820
1821
1822
1823
1824static int ingress_pkey_table_search(struct hfi1_pportdata *ppd, u16 pkey)
1825{
1826 int i;
1827
1828 for (i = 0; i < MAX_PKEY_VALUES; i++) {
1829 if (ingress_pkey_matches_entry(pkey, ppd->pkeys[i]))
1830 return 0;
1831 }
1832 return 1;
1833}
1834
1835
1836
1837
1838
1839
1840static void ingress_pkey_table_fail(struct hfi1_pportdata *ppd, u16 pkey,
1841 u32 slid)
1842{
1843 struct hfi1_devdata *dd = ppd->dd;
1844
1845 incr_cntr64(&ppd->port_rcv_constraint_errors);
1846 if (!(dd->err_info_rcv_constraint.status & OPA_EI_STATUS_SMASK)) {
1847 dd->err_info_rcv_constraint.status |= OPA_EI_STATUS_SMASK;
1848 dd->err_info_rcv_constraint.slid = slid;
1849 dd->err_info_rcv_constraint.pkey = pkey;
1850 }
1851}
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861static inline int ingress_pkey_check(struct hfi1_pportdata *ppd, u16 pkey,
1862 u8 sc5, u8 idx, u32 slid, bool force)
1863{
1864 if (!(force) && !(ppd->part_enforce & HFI1_PART_ENFORCE_IN))
1865 return 0;
1866
1867
1868 if ((sc5 == 0xf) && ((pkey & PKEY_LOW_15_MASK) != PKEY_LOW_15_MASK))
1869 goto bad;
1870
1871
1872 if ((pkey & PKEY_LOW_15_MASK) == 0)
1873 goto bad;
1874
1875
1876 if (ingress_pkey_matches_entry(pkey, ppd->pkeys[idx]))
1877 return 0;
1878
1879
1880 if (!ingress_pkey_table_search(ppd, pkey))
1881 return 0;
1882
1883bad:
1884 ingress_pkey_table_fail(ppd, pkey, slid);
1885 return 1;
1886}
1887
1888
1889
1890
1891
1892
1893
1894static inline int rcv_pkey_check(struct hfi1_pportdata *ppd, u16 pkey,
1895 u8 sc5, u16 slid)
1896{
1897 if (!(ppd->part_enforce & HFI1_PART_ENFORCE_IN))
1898 return 0;
1899
1900
1901 if ((sc5 == 0xf) && ((pkey & PKEY_LOW_15_MASK) != PKEY_LOW_15_MASK))
1902 goto bad;
1903
1904 return 0;
1905bad:
1906 ingress_pkey_table_fail(ppd, pkey, slid);
1907 return 1;
1908}
1909
1910
1911
1912
1913#define OPA_MTU_0 0
1914#define OPA_MTU_256 1
1915#define OPA_MTU_512 2
1916#define OPA_MTU_1024 3
1917#define OPA_MTU_2048 4
1918#define OPA_MTU_4096 5
1919
1920u32 lrh_max_header_bytes(struct hfi1_devdata *dd);
1921int mtu_to_enum(u32 mtu, int default_if_bad);
1922u16 enum_to_mtu(int mtu);
1923static inline int valid_ib_mtu(unsigned int mtu)
1924{
1925 return mtu == 256 || mtu == 512 ||
1926 mtu == 1024 || mtu == 2048 ||
1927 mtu == 4096;
1928}
1929
1930static inline int valid_opa_max_mtu(unsigned int mtu)
1931{
1932 return mtu >= 2048 &&
1933 (valid_ib_mtu(mtu) || mtu == 8192 || mtu == 10240);
1934}
1935
1936int set_mtu(struct hfi1_pportdata *ppd);
1937
1938int hfi1_set_lid(struct hfi1_pportdata *ppd, u32 lid, u8 lmc);
1939void hfi1_disable_after_error(struct hfi1_devdata *dd);
1940int hfi1_set_uevent_bits(struct hfi1_pportdata *ppd, const int evtbit);
1941int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encode);
1942
1943int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t);
1944int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t);
1945
1946void set_up_vau(struct hfi1_devdata *dd, u8 vau);
1947void set_up_vl15(struct hfi1_devdata *dd, u16 vl15buf);
1948void reset_link_credits(struct hfi1_devdata *dd);
1949void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu);
1950
1951int set_buffer_control(struct hfi1_pportdata *ppd, struct buffer_control *bc);
1952
1953static inline struct hfi1_devdata *dd_from_ppd(struct hfi1_pportdata *ppd)
1954{
1955 return ppd->dd;
1956}
1957
1958static inline struct hfi1_devdata *dd_from_dev(struct hfi1_ibdev *dev)
1959{
1960 return container_of(dev, struct hfi1_devdata, verbs_dev);
1961}
1962
1963static inline struct hfi1_devdata *dd_from_ibdev(struct ib_device *ibdev)
1964{
1965 return dd_from_dev(to_idev(ibdev));
1966}
1967
1968static inline struct hfi1_pportdata *ppd_from_ibp(struct hfi1_ibport *ibp)
1969{
1970 return container_of(ibp, struct hfi1_pportdata, ibport_data);
1971}
1972
1973static inline struct hfi1_ibdev *dev_from_rdi(struct rvt_dev_info *rdi)
1974{
1975 return container_of(rdi, struct hfi1_ibdev, rdi);
1976}
1977
1978static inline struct hfi1_ibport *to_iport(struct ib_device *ibdev, u8 port)
1979{
1980 struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
1981 unsigned pidx = port - 1;
1982
1983 WARN_ON(pidx >= dd->num_pports);
1984 return &dd->pport[pidx].ibport_data;
1985}
1986
1987static inline struct hfi1_ibport *rcd_to_iport(struct hfi1_ctxtdata *rcd)
1988{
1989 return &rcd->ppd->ibport_data;
1990}
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003static inline bool hfi1_may_ecn(struct hfi1_packet *pkt)
2004{
2005 bool fecn, becn;
2006
2007 if (pkt->etype == RHF_RCV_TYPE_BYPASS) {
2008 fecn = hfi1_16B_get_fecn(pkt->hdr);
2009 becn = hfi1_16B_get_becn(pkt->hdr);
2010 } else {
2011 fecn = ib_bth_get_fecn(pkt->ohdr);
2012 becn = ib_bth_get_becn(pkt->ohdr);
2013 }
2014 return fecn || becn;
2015}
2016
2017bool hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
2018 bool prescan);
2019static inline bool process_ecn(struct rvt_qp *qp, struct hfi1_packet *pkt)
2020{
2021 bool do_work;
2022
2023 do_work = hfi1_may_ecn(pkt);
2024 if (unlikely(do_work))
2025 return hfi1_process_ecn_slowpath(qp, pkt, false);
2026 return false;
2027}
2028
2029
2030
2031
2032static inline u16 hfi1_get_pkey(struct hfi1_ibport *ibp, unsigned index)
2033{
2034 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
2035 u16 ret;
2036
2037 if (index >= ARRAY_SIZE(ppd->pkeys))
2038 ret = 0;
2039 else
2040 ret = ppd->pkeys[index];
2041
2042 return ret;
2043}
2044
2045
2046
2047
2048static inline __be64 get_sguid(struct hfi1_ibport *ibp, unsigned int index)
2049{
2050 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
2051
2052 WARN_ON(index >= HFI1_GUIDS_PER_PORT);
2053 return cpu_to_be64(ppd->guids[index]);
2054}
2055
2056
2057
2058
2059static inline struct cc_state *get_cc_state(struct hfi1_pportdata *ppd)
2060{
2061 return rcu_dereference(ppd->cc_state);
2062}
2063
2064
2065
2066
2067static inline
2068struct cc_state *get_cc_state_protected(struct hfi1_pportdata *ppd)
2069{
2070 return rcu_dereference_protected(ppd->cc_state,
2071 lockdep_is_held(&ppd->cc_state_lock));
2072}
2073
2074
2075
2076
2077#define HFI1_INITTED 0x1
2078#define HFI1_PRESENT 0x2
2079#define HFI1_FROZEN 0x4
2080#define HFI1_HAS_SDMA_TIMEOUT 0x8
2081#define HFI1_HAS_SEND_DMA 0x10
2082#define HFI1_FORCED_FREEZE 0x80
2083#define HFI1_SHUTDOWN 0x100
2084
2085
2086#define HFI1_PBC_LENGTH_MASK ((1 << 11) - 1)
2087
2088
2089
2090#define HFI1_CTXT_BASE_UNINIT 1
2091
2092#define HFI1_CTXT_BASE_FAILED 2
2093
2094#define HFI1_CTXT_WAITING_RCV 3
2095
2096#define HFI1_CTXT_WAITING_URG 4
2097
2098
2099int hfi1_init_dd(struct hfi1_devdata *dd);
2100void hfi1_free_devdata(struct hfi1_devdata *dd);
2101
2102
2103void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon,
2104 unsigned int timeoff);
2105void shutdown_led_override(struct hfi1_pportdata *ppd);
2106
2107#define HFI1_CREDIT_RETURN_RATE (100)
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128#define DEFAULT_RCVHDRSIZE 9
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145#define DEFAULT_RCVHDR_ENTSIZE 32
2146
2147bool hfi1_can_pin_pages(struct hfi1_devdata *dd, struct mm_struct *mm,
2148 u32 nlocked, u32 npages);
2149int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr,
2150 size_t npages, bool writable, struct page **pages);
2151void hfi1_release_user_pages(struct mm_struct *mm, struct page **p,
2152 size_t npages, bool dirty);
2153
2154
2155
2156
2157
2158static inline __le64 *hfi1_rcvhdrtail_kvaddr(const struct hfi1_ctxtdata *rcd)
2159{
2160 return (__le64 *)rcd->rcvhdrtail_kvaddr;
2161}
2162
2163static inline void clear_rcvhdrtail(const struct hfi1_ctxtdata *rcd)
2164{
2165 u64 *kv = (u64 *)hfi1_rcvhdrtail_kvaddr(rcd);
2166
2167 if (kv)
2168 *kv = 0ULL;
2169}
2170
2171static inline u32 get_rcvhdrtail(const struct hfi1_ctxtdata *rcd)
2172{
2173
2174
2175
2176
2177 return (u32)le64_to_cpu(*hfi1_rcvhdrtail_kvaddr(rcd));
2178}
2179
2180static inline bool hfi1_packet_present(struct hfi1_ctxtdata *rcd)
2181{
2182 if (likely(!rcd->rcvhdrtail_kvaddr)) {
2183 u32 seq = rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd)));
2184
2185 return !last_rcv_seq(rcd, seq);
2186 }
2187 return hfi1_rcd_head(rcd) != get_rcvhdrtail(rcd);
2188}
2189
2190
2191
2192
2193
2194extern const char ib_hfi1_version[];
2195extern const struct attribute_group ib_hfi1_attr_group;
2196
2197int hfi1_device_create(struct hfi1_devdata *dd);
2198void hfi1_device_remove(struct hfi1_devdata *dd);
2199
2200int hfi1_create_port_files(struct ib_device *ibdev, u8 port_num,
2201 struct kobject *kobj);
2202int hfi1_verbs_register_sysfs(struct hfi1_devdata *dd);
2203void hfi1_verbs_unregister_sysfs(struct hfi1_devdata *dd);
2204
2205int qsfp_dump(struct hfi1_pportdata *ppd, char *buf, int len);
2206
2207int hfi1_pcie_init(struct hfi1_devdata *dd);
2208void hfi1_pcie_cleanup(struct pci_dev *pdev);
2209int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev);
2210void hfi1_pcie_ddcleanup(struct hfi1_devdata *);
2211int pcie_speeds(struct hfi1_devdata *dd);
2212int restore_pci_variables(struct hfi1_devdata *dd);
2213int save_pci_variables(struct hfi1_devdata *dd);
2214int do_pcie_gen3_transition(struct hfi1_devdata *dd);
2215void tune_pcie_caps(struct hfi1_devdata *dd);
2216int parse_platform_config(struct hfi1_devdata *dd);
2217int get_platform_config_field(struct hfi1_devdata *dd,
2218 enum platform_config_table_type_encoding
2219 table_type, int table_index, int field_index,
2220 u32 *data, u32 len);
2221
2222struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi);
2223
2224
2225
2226
2227
2228static inline void flush_wc(void)
2229{
2230 asm volatile("sfence" : : : "memory");
2231}
2232
2233void handle_eflags(struct hfi1_packet *packet);
2234void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd);
2235
2236
2237extern unsigned int hfi1_max_mtu;
2238extern unsigned int hfi1_cu;
2239extern unsigned int user_credit_return_threshold;
2240extern int num_user_contexts;
2241extern unsigned long n_krcvqs;
2242extern uint krcvqs[];
2243extern int krcvqsset;
2244extern uint loopback;
2245extern uint quick_linkup;
2246extern uint rcv_intr_timeout;
2247extern uint rcv_intr_count;
2248extern uint rcv_intr_dynamic;
2249extern ushort link_crc_mask;
2250
2251extern struct mutex hfi1_mutex;
2252
2253
2254#define STATUS_TIMEOUT 60
2255
2256#define DRIVER_NAME "hfi1"
2257#define HFI1_USER_MINOR_BASE 0
2258#define HFI1_TRACE_MINOR 127
2259#define HFI1_NMINORS 255
2260
2261#define PCI_VENDOR_ID_INTEL 0x8086
2262#define PCI_DEVICE_ID_INTEL0 0x24f0
2263#define PCI_DEVICE_ID_INTEL1 0x24f1
2264
2265#define HFI1_PKT_USER_SC_INTEGRITY \
2266 (SEND_CTXT_CHECK_ENABLE_DISALLOW_NON_KDETH_PACKETS_SMASK \
2267 | SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK \
2268 | SEND_CTXT_CHECK_ENABLE_DISALLOW_BYPASS_SMASK \
2269 | SEND_CTXT_CHECK_ENABLE_DISALLOW_GRH_SMASK)
2270
2271#define HFI1_PKT_KERNEL_SC_INTEGRITY \
2272 (SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK)
2273
2274static inline u64 hfi1_pkt_default_send_ctxt_mask(struct hfi1_devdata *dd,
2275 u16 ctxt_type)
2276{
2277 u64 base_sc_integrity;
2278
2279
2280 if (HFI1_CAP_IS_KSET(NO_INTEGRITY))
2281 return 0;
2282
2283 base_sc_integrity =
2284 SEND_CTXT_CHECK_ENABLE_DISALLOW_BYPASS_BAD_PKT_LEN_SMASK
2285 | SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK
2286 | SEND_CTXT_CHECK_ENABLE_DISALLOW_TOO_LONG_BYPASS_PACKETS_SMASK
2287 | SEND_CTXT_CHECK_ENABLE_DISALLOW_TOO_LONG_IB_PACKETS_SMASK
2288 | SEND_CTXT_CHECK_ENABLE_DISALLOW_BAD_PKT_LEN_SMASK
2289#ifndef CONFIG_FAULT_INJECTION
2290 | SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_TEST_SMASK
2291#endif
2292 | SEND_CTXT_CHECK_ENABLE_DISALLOW_TOO_SMALL_BYPASS_PACKETS_SMASK
2293 | SEND_CTXT_CHECK_ENABLE_DISALLOW_TOO_SMALL_IB_PACKETS_SMASK
2294 | SEND_CTXT_CHECK_ENABLE_DISALLOW_RAW_IPV6_SMASK
2295 | SEND_CTXT_CHECK_ENABLE_DISALLOW_RAW_SMASK
2296 | SEND_CTXT_CHECK_ENABLE_CHECK_BYPASS_VL_MAPPING_SMASK
2297 | SEND_CTXT_CHECK_ENABLE_CHECK_VL_MAPPING_SMASK
2298 | SEND_CTXT_CHECK_ENABLE_CHECK_OPCODE_SMASK
2299 | SEND_CTXT_CHECK_ENABLE_CHECK_SLID_SMASK
2300 | SEND_CTXT_CHECK_ENABLE_CHECK_VL_SMASK
2301 | SEND_CTXT_CHECK_ENABLE_CHECK_ENABLE_SMASK;
2302
2303 if (ctxt_type == SC_USER)
2304 base_sc_integrity |=
2305#ifndef CONFIG_FAULT_INJECTION
2306 SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_TEST_SMASK |
2307#endif
2308 HFI1_PKT_USER_SC_INTEGRITY;
2309 else if (ctxt_type != SC_KERNEL)
2310 base_sc_integrity |= HFI1_PKT_KERNEL_SC_INTEGRITY;
2311
2312
2313 if (!is_ax(dd))
2314 base_sc_integrity |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
2315
2316 return base_sc_integrity;
2317}
2318
2319static inline u64 hfi1_pkt_base_sdma_integrity(struct hfi1_devdata *dd)
2320{
2321 u64 base_sdma_integrity;
2322
2323
2324 if (HFI1_CAP_IS_KSET(NO_INTEGRITY))
2325 return 0;
2326
2327 base_sdma_integrity =
2328 SEND_DMA_CHECK_ENABLE_DISALLOW_BYPASS_BAD_PKT_LEN_SMASK
2329 | SEND_DMA_CHECK_ENABLE_DISALLOW_TOO_LONG_BYPASS_PACKETS_SMASK
2330 | SEND_DMA_CHECK_ENABLE_DISALLOW_TOO_LONG_IB_PACKETS_SMASK
2331 | SEND_DMA_CHECK_ENABLE_DISALLOW_BAD_PKT_LEN_SMASK
2332 | SEND_DMA_CHECK_ENABLE_DISALLOW_TOO_SMALL_BYPASS_PACKETS_SMASK
2333 | SEND_DMA_CHECK_ENABLE_DISALLOW_TOO_SMALL_IB_PACKETS_SMASK
2334 | SEND_DMA_CHECK_ENABLE_DISALLOW_RAW_IPV6_SMASK
2335 | SEND_DMA_CHECK_ENABLE_DISALLOW_RAW_SMASK
2336 | SEND_DMA_CHECK_ENABLE_CHECK_BYPASS_VL_MAPPING_SMASK
2337 | SEND_DMA_CHECK_ENABLE_CHECK_VL_MAPPING_SMASK
2338 | SEND_DMA_CHECK_ENABLE_CHECK_OPCODE_SMASK
2339 | SEND_DMA_CHECK_ENABLE_CHECK_SLID_SMASK
2340 | SEND_DMA_CHECK_ENABLE_CHECK_VL_SMASK
2341 | SEND_DMA_CHECK_ENABLE_CHECK_ENABLE_SMASK;
2342
2343 if (!HFI1_CAP_IS_KSET(STATIC_RATE_CTRL))
2344 base_sdma_integrity |=
2345 SEND_DMA_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK;
2346
2347
2348 if (!is_ax(dd))
2349 base_sdma_integrity |=
2350 SEND_DMA_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
2351
2352 return base_sdma_integrity;
2353}
2354
2355#define dd_dev_emerg(dd, fmt, ...) \
2356 dev_emerg(&(dd)->pcidev->dev, "%s: " fmt, \
2357 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
2358
2359#define dd_dev_err(dd, fmt, ...) \
2360 dev_err(&(dd)->pcidev->dev, "%s: " fmt, \
2361 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
2362
2363#define dd_dev_err_ratelimited(dd, fmt, ...) \
2364 dev_err_ratelimited(&(dd)->pcidev->dev, "%s: " fmt, \
2365 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), \
2366 ##__VA_ARGS__)
2367
2368#define dd_dev_warn(dd, fmt, ...) \
2369 dev_warn(&(dd)->pcidev->dev, "%s: " fmt, \
2370 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
2371
2372#define dd_dev_warn_ratelimited(dd, fmt, ...) \
2373 dev_warn_ratelimited(&(dd)->pcidev->dev, "%s: " fmt, \
2374 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), \
2375 ##__VA_ARGS__)
2376
2377#define dd_dev_info(dd, fmt, ...) \
2378 dev_info(&(dd)->pcidev->dev, "%s: " fmt, \
2379 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
2380
2381#define dd_dev_info_ratelimited(dd, fmt, ...) \
2382 dev_info_ratelimited(&(dd)->pcidev->dev, "%s: " fmt, \
2383 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), \
2384 ##__VA_ARGS__)
2385
2386#define dd_dev_dbg(dd, fmt, ...) \
2387 dev_dbg(&(dd)->pcidev->dev, "%s: " fmt, \
2388 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
2389
2390#define hfi1_dev_porterr(dd, port, fmt, ...) \
2391 dev_err(&(dd)->pcidev->dev, "%s: port %u: " fmt, \
2392 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), (port), ##__VA_ARGS__)
2393
2394
2395
2396
2397struct hfi1_hwerror_msgs {
2398 u64 mask;
2399 const char *msg;
2400 size_t sz;
2401};
2402
2403
2404void hfi1_format_hwerrors(u64 hwerrs,
2405 const struct hfi1_hwerror_msgs *hwerrmsgs,
2406 size_t nhwerrmsgs, char *msg, size_t lmsg);
2407
2408#define USER_OPCODE_CHECK_VAL 0xC0
2409#define USER_OPCODE_CHECK_MASK 0xC0
2410#define OPCODE_CHECK_VAL_DISABLED 0x0
2411#define OPCODE_CHECK_MASK_DISABLED 0x0
2412
2413static inline void hfi1_reset_cpu_counters(struct hfi1_devdata *dd)
2414{
2415 struct hfi1_pportdata *ppd;
2416 int i;
2417
2418 dd->z_int_counter = get_all_cpu_total(dd->int_counter);
2419 dd->z_rcv_limit = get_all_cpu_total(dd->rcv_limit);
2420 dd->z_send_schedule = get_all_cpu_total(dd->send_schedule);
2421
2422 ppd = (struct hfi1_pportdata *)(dd + 1);
2423 for (i = 0; i < dd->num_pports; i++, ppd++) {
2424 ppd->ibport_data.rvp.z_rc_acks =
2425 get_all_cpu_total(ppd->ibport_data.rvp.rc_acks);
2426 ppd->ibport_data.rvp.z_rc_qacks =
2427 get_all_cpu_total(ppd->ibport_data.rvp.rc_qacks);
2428 }
2429}
2430
2431
2432static inline void setextled(struct hfi1_devdata *dd, u32 on)
2433{
2434 if (on)
2435 write_csr(dd, DCC_CFG_LED_CNTRL, 0x1F);
2436 else
2437 write_csr(dd, DCC_CFG_LED_CNTRL, 0x10);
2438}
2439
2440
2441static inline u32 i2c_target(u32 target)
2442{
2443 return target ? CR_I2C2 : CR_I2C1;
2444}
2445
2446
2447static inline u32 qsfp_resource(struct hfi1_devdata *dd)
2448{
2449 return i2c_target(dd->hfi1_id);
2450}
2451
2452
2453static inline bool is_integrated(struct hfi1_devdata *dd)
2454{
2455 return dd->pcidev->device == PCI_DEVICE_ID_INTEL1;
2456}
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466static inline bool hfi1_need_drop(struct hfi1_devdata *dd)
2467{
2468 if (unlikely(dd->do_drop &&
2469 atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) ==
2470 DROP_PACKET_ON)) {
2471 dd->do_drop = false;
2472 return true;
2473 }
2474 return false;
2475}
2476
2477int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp);
2478
2479#define DD_DEV_ENTRY(dd) __string(dev, dev_name(&(dd)->pcidev->dev))
2480#define DD_DEV_ASSIGN(dd) __assign_str(dev, dev_name(&(dd)->pcidev->dev))
2481
2482static inline void hfi1_update_ah_attr(struct ib_device *ibdev,
2483 struct rdma_ah_attr *attr)
2484{
2485 struct hfi1_pportdata *ppd;
2486 struct hfi1_ibport *ibp;
2487 u32 dlid = rdma_ah_get_dlid(attr);
2488
2489
2490
2491
2492
2493 ibp = to_iport(ibdev, rdma_ah_get_port_num(attr));
2494 ppd = ppd_from_ibp(ibp);
2495 if ((((dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) ||
2496 (ppd->lid >= be16_to_cpu(IB_MULTICAST_LID_BASE))) &&
2497 (dlid != be32_to_cpu(OPA_LID_PERMISSIVE)) &&
2498 (dlid != be16_to_cpu(IB_LID_PERMISSIVE)) &&
2499 (!(rdma_ah_get_ah_flags(attr) & IB_AH_GRH))) ||
2500 (rdma_ah_get_make_grd(attr))) {
2501 rdma_ah_set_ah_flags(attr, IB_AH_GRH);
2502 rdma_ah_set_interface_id(attr, OPA_MAKE_ID(dlid));
2503 rdma_ah_set_subnet_prefix(attr, ibp->rvp.gid_prefix);
2504 }
2505}
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515static inline bool hfi1_check_mcast(u32 lid)
2516{
2517 return ((lid >= opa_get_mcast_base(OPA_MCAST_NR)) &&
2518 (lid != be32_to_cpu(OPA_LID_PERMISSIVE)));
2519}
2520
2521#define opa_get_lid(lid, format) \
2522 __opa_get_lid(lid, OPA_PORT_PACKET_FORMAT_##format)
2523
2524
2525static inline u32 __opa_get_lid(u32 lid, u8 format)
2526{
2527 bool is_mcast = hfi1_check_mcast(lid);
2528
2529 switch (format) {
2530 case OPA_PORT_PACKET_FORMAT_8B:
2531 case OPA_PORT_PACKET_FORMAT_10B:
2532 if (is_mcast)
2533 return (lid - opa_get_mcast_base(OPA_MCAST_NR) +
2534 0xF0000);
2535 return lid & 0xFFFFF;
2536 case OPA_PORT_PACKET_FORMAT_16B:
2537 if (is_mcast)
2538 return (lid - opa_get_mcast_base(OPA_MCAST_NR) +
2539 0xF00000);
2540 return lid & 0xFFFFFF;
2541 case OPA_PORT_PACKET_FORMAT_9B:
2542 if (is_mcast)
2543 return (lid -
2544 opa_get_mcast_base(OPA_MCAST_NR) +
2545 be16_to_cpu(IB_MULTICAST_LID_BASE));
2546 else
2547 return lid & 0xFFFF;
2548 default:
2549 return lid;
2550 }
2551}
2552
2553
2554static inline bool hfi1_is_16B_mcast(u32 lid)
2555{
2556 return ((lid >=
2557 opa_get_lid(opa_get_mcast_base(OPA_MCAST_NR), 16B)) &&
2558 (lid != opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B)));
2559}
2560
2561static inline void hfi1_make_opa_lid(struct rdma_ah_attr *attr)
2562{
2563 const struct ib_global_route *grh = rdma_ah_read_grh(attr);
2564 u32 dlid = rdma_ah_get_dlid(attr);
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574 if (ib_is_opa_gid(&grh->dgid))
2575 dlid = opa_get_lid_from_gid(&grh->dgid);
2576 else if ((dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
2577 (dlid != be16_to_cpu(IB_LID_PERMISSIVE)) &&
2578 (dlid != be32_to_cpu(OPA_LID_PERMISSIVE)))
2579 dlid = dlid - be16_to_cpu(IB_MULTICAST_LID_BASE) +
2580 opa_get_mcast_base(OPA_MCAST_NR);
2581 else if (dlid == be16_to_cpu(IB_LID_PERMISSIVE))
2582 dlid = be32_to_cpu(OPA_LID_PERMISSIVE);
2583
2584 rdma_ah_set_dlid(attr, dlid);
2585}
2586
2587static inline u8 hfi1_get_packet_type(u32 lid)
2588{
2589
2590 if (lid >= opa_get_mcast_base(OPA_MCAST_NR))
2591 return HFI1_PKT_TYPE_9B;
2592
2593
2594 if (lid >= opa_get_lid(opa_get_mcast_base(OPA_MCAST_NR), 9B))
2595 return HFI1_PKT_TYPE_16B;
2596
2597 return HFI1_PKT_TYPE_9B;
2598}
2599
2600static inline bool hfi1_get_hdr_type(u32 lid, struct rdma_ah_attr *attr)
2601{
2602
2603
2604
2605
2606
2607
2608
2609 if (rdma_ah_get_dlid(attr) == be32_to_cpu(OPA_LID_PERMISSIVE))
2610 return (ib_is_opa_gid(&rdma_ah_read_grh(attr)->dgid)) ?
2611 HFI1_PKT_TYPE_16B : HFI1_PKT_TYPE_9B;
2612
2613
2614
2615
2616
2617 if (hfi1_get_packet_type(rdma_ah_get_dlid(attr)) == HFI1_PKT_TYPE_16B)
2618 return HFI1_PKT_TYPE_16B;
2619
2620 return hfi1_get_packet_type(lid);
2621}
2622
2623static inline void hfi1_make_ext_grh(struct hfi1_packet *packet,
2624 struct ib_grh *grh, u32 slid,
2625 u32 dlid)
2626{
2627 struct hfi1_ibport *ibp = &packet->rcd->ppd->ibport_data;
2628 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
2629
2630 if (!ibp)
2631 return;
2632
2633 grh->hop_limit = 1;
2634 grh->sgid.global.subnet_prefix = ibp->rvp.gid_prefix;
2635 if (slid == opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B))
2636 grh->sgid.global.interface_id =
2637 OPA_MAKE_ID(be32_to_cpu(OPA_LID_PERMISSIVE));
2638 else
2639 grh->sgid.global.interface_id = OPA_MAKE_ID(slid);
2640
2641
2642
2643
2644
2645
2646
2647
2648 grh->dgid.global.subnet_prefix = ibp->rvp.gid_prefix;
2649 grh->dgid.global.interface_id =
2650 cpu_to_be64(ppd->guids[HFI1_PORT_GUID_INDEX]);
2651}
2652
2653static inline int hfi1_get_16b_padding(u32 hdr_size, u32 payload)
2654{
2655 return -(hdr_size + payload + (SIZE_OF_CRC << 2) +
2656 SIZE_OF_LT) & 0x7;
2657}
2658
2659static inline void hfi1_make_ib_hdr(struct ib_header *hdr,
2660 u16 lrh0, u16 len,
2661 u16 dlid, u16 slid)
2662{
2663 hdr->lrh[0] = cpu_to_be16(lrh0);
2664 hdr->lrh[1] = cpu_to_be16(dlid);
2665 hdr->lrh[2] = cpu_to_be16(len);
2666 hdr->lrh[3] = cpu_to_be16(slid);
2667}
2668
2669static inline void hfi1_make_16b_hdr(struct hfi1_16b_header *hdr,
2670 u32 slid, u32 dlid,
2671 u16 len, u16 pkey,
2672 bool becn, bool fecn, u8 l4,
2673 u8 sc)
2674{
2675 u32 lrh0 = 0;
2676 u32 lrh1 = 0x40000000;
2677 u32 lrh2 = 0;
2678 u32 lrh3 = 0;
2679
2680 lrh0 = (lrh0 & ~OPA_16B_BECN_MASK) | (becn << OPA_16B_BECN_SHIFT);
2681 lrh0 = (lrh0 & ~OPA_16B_LEN_MASK) | (len << OPA_16B_LEN_SHIFT);
2682 lrh0 = (lrh0 & ~OPA_16B_LID_MASK) | (slid & OPA_16B_LID_MASK);
2683 lrh1 = (lrh1 & ~OPA_16B_FECN_MASK) | (fecn << OPA_16B_FECN_SHIFT);
2684 lrh1 = (lrh1 & ~OPA_16B_SC_MASK) | (sc << OPA_16B_SC_SHIFT);
2685 lrh1 = (lrh1 & ~OPA_16B_LID_MASK) | (dlid & OPA_16B_LID_MASK);
2686 lrh2 = (lrh2 & ~OPA_16B_SLID_MASK) |
2687 ((slid >> OPA_16B_SLID_SHIFT) << OPA_16B_SLID_HIGH_SHIFT);
2688 lrh2 = (lrh2 & ~OPA_16B_DLID_MASK) |
2689 ((dlid >> OPA_16B_DLID_SHIFT) << OPA_16B_DLID_HIGH_SHIFT);
2690 lrh2 = (lrh2 & ~OPA_16B_PKEY_MASK) | ((u32)pkey << OPA_16B_PKEY_SHIFT);
2691 lrh2 = (lrh2 & ~OPA_16B_L4_MASK) | l4;
2692
2693 hdr->lrh[0] = lrh0;
2694 hdr->lrh[1] = lrh1;
2695 hdr->lrh[2] = lrh2;
2696 hdr->lrh[3] = lrh3;
2697}
2698#endif
2699