1#ifndef _HFI1_KERNEL_H
2#define _HFI1_KERNEL_H
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51#include <linux/interrupt.h>
52#include <linux/pci.h>
53#include <linux/dma-mapping.h>
54#include <linux/mutex.h>
55#include <linux/list.h>
56#include <linux/scatterlist.h>
57#include <linux/slab.h>
58#include <linux/io.h>
59#include <linux/fs.h>
60#include <linux/completion.h>
61#include <linux/kref.h>
62#include <linux/sched.h>
63#include <linux/cdev.h>
64#include <linux/delay.h>
65#include <linux/kthread.h>
66#include <linux/i2c.h>
67#include <linux/i2c-algo-bit.h>
68#include <linux/xarray.h>
69#include <rdma/ib_hdrs.h>
70#include <rdma/opa_addr.h>
71#include <linux/rhashtable.h>
72#include <rdma/rdma_vt.h>
73
74#include "chip_registers.h"
75#include "common.h"
76#include "opfn.h"
77#include "verbs.h"
78#include "pio.h"
79#include "chip.h"
80#include "mad.h"
81#include "qsfp.h"
82#include "platform.h"
83#include "affinity.h"
84#include "msix.h"
85
86
87#define HFI1_CHIP_VERS_MAJ 3U
88
89
90#define HFI1_CHIP_VERS_MIN 0U
91
92
93#define HFI1_OUI 0x001175
94#define HFI1_OUI_LSB 40
95
96#define DROP_PACKET_OFF 0
97#define DROP_PACKET_ON 1
98
99#define NEIGHBOR_TYPE_HFI 0
100#define NEIGHBOR_TYPE_SWITCH 1
101
102#define HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES 5
103
104extern unsigned long hfi1_cap_mask;
105#define HFI1_CAP_KGET_MASK(mask, cap) ((mask) & HFI1_CAP_##cap)
106#define HFI1_CAP_UGET_MASK(mask, cap) \
107 (((mask) >> HFI1_CAP_USER_SHIFT) & HFI1_CAP_##cap)
108#define HFI1_CAP_KGET(cap) (HFI1_CAP_KGET_MASK(hfi1_cap_mask, cap))
109#define HFI1_CAP_UGET(cap) (HFI1_CAP_UGET_MASK(hfi1_cap_mask, cap))
110#define HFI1_CAP_IS_KSET(cap) (!!HFI1_CAP_KGET(cap))
111#define HFI1_CAP_IS_USET(cap) (!!HFI1_CAP_UGET(cap))
112#define HFI1_MISC_GET() ((hfi1_cap_mask >> HFI1_CAP_MISC_SHIFT) & \
113 HFI1_CAP_MISC_MASK)
114
115#define HFI1_ODR_MASK(rsn) ((rsn) & OPA_PI_MASK_OFFLINE_REASON)
116
117
118
119
120
121#define HFI1_CTRL_CTXT 0
122
123
124
125
126
127#define NUM_CCE_ERR_STATUS_COUNTERS 41
128#define NUM_RCV_ERR_STATUS_COUNTERS 64
129#define NUM_MISC_ERR_STATUS_COUNTERS 13
130#define NUM_SEND_PIO_ERR_STATUS_COUNTERS 36
131#define NUM_SEND_DMA_ERR_STATUS_COUNTERS 4
132#define NUM_SEND_EGRESS_ERR_STATUS_COUNTERS 64
133#define NUM_SEND_ERR_STATUS_COUNTERS 3
134#define NUM_SEND_CTXT_ERR_STATUS_COUNTERS 5
135#define NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS 24
136
137
138
139
140
141
142
143
144
145struct hfi1_ib_stats {
146 __u64 sps_ints;
147 __u64 sps_errints;
148 __u64 sps_txerrs;
149 __u64 sps_rcverrs;
150 __u64 sps_hwerrs;
151 __u64 sps_nopiobufs;
152 __u64 sps_ctxts;
153 __u64 sps_lenerrs;
154 __u64 sps_buffull;
155 __u64 sps_hdrfull;
156};
157
158extern struct hfi1_ib_stats hfi1_stats;
159extern const struct pci_error_handlers hfi1_pci_err_handler;
160
161extern int num_driver_cntrs;
162
163
164
165
166
167
168
169#define HFI1_TRAFFIC_ACTIVE_THRESHOLD (2000)
170
171
172
173
174
175struct hfi1_opcode_stats_perctx;
176
177struct ctxt_eager_bufs {
178 struct eager_buffer {
179 void *addr;
180 dma_addr_t dma;
181 ssize_t len;
182 } *buffers;
183 struct {
184 void *addr;
185 dma_addr_t dma;
186 } *rcvtids;
187 u32 size;
188 u32 rcvtid_size;
189 u16 count;
190 u16 numbufs;
191 u16 alloced;
192 u16 threshold;
193};
194
195struct exp_tid_set {
196 struct list_head list;
197 u32 count;
198};
199
200struct hfi1_ctxtdata;
201typedef int (*intr_handler)(struct hfi1_ctxtdata *rcd, int data);
202typedef void (*rhf_rcv_function_ptr)(struct hfi1_packet *packet);
203
204struct tid_queue {
205 struct list_head queue_head;
206
207 u32 enqueue;
208 u32 dequeue;
209};
210
211struct hfi1_ctxtdata {
212
213 void *rcvhdrq;
214
215 volatile __le64 *rcvhdrtail_kvaddr;
216
217 struct hfi1_pportdata *ppd;
218
219 struct hfi1_devdata *dd;
220
221 struct send_context *sc;
222
223 const rhf_rcv_function_ptr *rhf_rcv_function_map;
224
225
226
227
228
229
230
231 intr_handler do_interrupt;
232
233 intr_handler fast_handler;
234
235 intr_handler slow_handler;
236
237 struct napi_struct *napi;
238
239 struct hfi1_opcode_stats_perctx *opstats;
240
241 u64 imask;
242
243 u32 head;
244
245 u16 rcvhdrq_cnt;
246 u8 ireg;
247
248 u8 seq_cnt;
249
250 u8 rcvhdrqentsize;
251
252 u8 rhf_offset;
253
254 u8 rcvavail_timeout;
255
256 bool is_vnic;
257
258 u8 vnic_q_idx;
259
260 bool aspm_intr_supported;
261
262 bool aspm_enabled;
263
264 bool aspm_intr_enable;
265 struct ctxt_eager_bufs egrbufs;
266
267 struct list_head qp_wait_list;
268
269 struct exp_tid_set tid_group_list;
270 struct exp_tid_set tid_used_list;
271 struct exp_tid_set tid_full_list;
272
273
274 struct timer_list aspm_timer;
275
276 unsigned long flags;
277
278 struct tid_group *groups;
279
280 dma_addr_t rcvhdrq_dma;
281 dma_addr_t rcvhdrqtailaddr_dma;
282
283 ktime_t aspm_ts_last_intr;
284
285 ktime_t aspm_ts_timer_sched;
286
287 spinlock_t aspm_lock;
288
289 struct kref kref;
290
291 int numa_id;
292
293 s16 msix_intr;
294
295 u16 jkey;
296
297 u16 rcv_array_groups;
298
299 u16 eager_base;
300
301 u16 expected_count;
302
303 u16 expected_base;
304
305 u8 ctxt;
306
307
308
309 struct mutex exp_mutex;
310
311 spinlock_t exp_lock;
312
313 struct tid_queue flow_queue;
314
315 struct tid_queue rarr_queue;
316
317 wait_queue_head_t wait;
318
319 u8 uuid[16];
320
321 char comm[TASK_COMM_LEN];
322
323 DECLARE_BITMAP(in_use_ctxts, HFI1_MAX_SHARED_CTXTS);
324
325 unsigned long event_flags;
326
327 void *subctxt_uregbase;
328
329 void *subctxt_rcvegrbuf;
330
331 void *subctxt_rcvhdr_base;
332
333 u32 urgent;
334
335 u32 urgent_poll;
336
337 u16 poll_type;
338
339 u16 subctxt_id;
340
341 u32 userversion;
342
343
344
345
346 u8 subctxt_cnt;
347
348
349 unsigned long flow_mask;
350 struct tid_flow_state flows[RXE_NUM_TID_FLOWS];
351};
352
353
354
355
356
357
358
359
360static inline u32 rcvhdrq_size(struct hfi1_ctxtdata *rcd)
361{
362 return PAGE_ALIGN(rcd->rcvhdrq_cnt *
363 rcd->rcvhdrqentsize * sizeof(u32));
364}
365
366
367
368
369
370
371
372
373struct hfi1_packet {
374 void *ebuf;
375 void *hdr;
376 void *payload;
377 struct hfi1_ctxtdata *rcd;
378 __le32 *rhf_addr;
379 struct rvt_qp *qp;
380 struct ib_other_headers *ohdr;
381 struct ib_grh *grh;
382 struct opa_16b_mgmt *mgmt;
383 u64 rhf;
384 u32 maxcnt;
385 u32 rhqoff;
386 u32 dlid;
387 u32 slid;
388 int numpkt;
389 u16 tlen;
390 s16 etail;
391 u16 pkey;
392 u8 hlen;
393 u8 rsize;
394 u8 updegr;
395 u8 etype;
396 u8 extra_byte;
397 u8 pad;
398 u8 sc;
399 u8 sl;
400 u8 opcode;
401 bool migrated;
402};
403
404
405#define HFI1_PKT_TYPE_9B 0
406#define HFI1_PKT_TYPE_16B 1
407
408
409
410
411#define OPA_16B_L4_MASK 0xFFull
412#define OPA_16B_SC_MASK 0x1F00000ull
413#define OPA_16B_SC_SHIFT 20
414#define OPA_16B_LID_MASK 0xFFFFFull
415#define OPA_16B_DLID_MASK 0xF000ull
416#define OPA_16B_DLID_SHIFT 20
417#define OPA_16B_DLID_HIGH_SHIFT 12
418#define OPA_16B_SLID_MASK 0xF00ull
419#define OPA_16B_SLID_SHIFT 20
420#define OPA_16B_SLID_HIGH_SHIFT 8
421#define OPA_16B_BECN_MASK 0x80000000ull
422#define OPA_16B_BECN_SHIFT 31
423#define OPA_16B_FECN_MASK 0x10000000ull
424#define OPA_16B_FECN_SHIFT 28
425#define OPA_16B_L2_MASK 0x60000000ull
426#define OPA_16B_L2_SHIFT 29
427#define OPA_16B_PKEY_MASK 0xFFFF0000ull
428#define OPA_16B_PKEY_SHIFT 16
429#define OPA_16B_LEN_MASK 0x7FF00000ull
430#define OPA_16B_LEN_SHIFT 20
431#define OPA_16B_RC_MASK 0xE000000ull
432#define OPA_16B_RC_SHIFT 25
433#define OPA_16B_AGE_MASK 0xFF0000ull
434#define OPA_16B_AGE_SHIFT 16
435#define OPA_16B_ENTROPY_MASK 0xFFFFull
436
437
438
439
440#define OPA_16B_L4_9B 0x00
441#define OPA_16B_L2_TYPE 0x02
442#define OPA_16B_L4_FM 0x08
443#define OPA_16B_L4_IB_LOCAL 0x09
444#define OPA_16B_L4_IB_GLOBAL 0x0A
445#define OPA_16B_L4_ETHR OPA_VNIC_L4_ETHR
446
447
448
449
450#define OPA_16B_L4_FM_PAD 3
451#define OPA_16B_L4_FM_HLEN 24
452
453static inline u8 hfi1_16B_get_l4(struct hfi1_16b_header *hdr)
454{
455 return (u8)(hdr->lrh[2] & OPA_16B_L4_MASK);
456}
457
458static inline u8 hfi1_16B_get_sc(struct hfi1_16b_header *hdr)
459{
460 return (u8)((hdr->lrh[1] & OPA_16B_SC_MASK) >> OPA_16B_SC_SHIFT);
461}
462
463static inline u32 hfi1_16B_get_dlid(struct hfi1_16b_header *hdr)
464{
465 return (u32)((hdr->lrh[1] & OPA_16B_LID_MASK) |
466 (((hdr->lrh[2] & OPA_16B_DLID_MASK) >>
467 OPA_16B_DLID_HIGH_SHIFT) << OPA_16B_DLID_SHIFT));
468}
469
470static inline u32 hfi1_16B_get_slid(struct hfi1_16b_header *hdr)
471{
472 return (u32)((hdr->lrh[0] & OPA_16B_LID_MASK) |
473 (((hdr->lrh[2] & OPA_16B_SLID_MASK) >>
474 OPA_16B_SLID_HIGH_SHIFT) << OPA_16B_SLID_SHIFT));
475}
476
477static inline u8 hfi1_16B_get_becn(struct hfi1_16b_header *hdr)
478{
479 return (u8)((hdr->lrh[0] & OPA_16B_BECN_MASK) >> OPA_16B_BECN_SHIFT);
480}
481
482static inline u8 hfi1_16B_get_fecn(struct hfi1_16b_header *hdr)
483{
484 return (u8)((hdr->lrh[1] & OPA_16B_FECN_MASK) >> OPA_16B_FECN_SHIFT);
485}
486
487static inline u8 hfi1_16B_get_l2(struct hfi1_16b_header *hdr)
488{
489 return (u8)((hdr->lrh[1] & OPA_16B_L2_MASK) >> OPA_16B_L2_SHIFT);
490}
491
492static inline u16 hfi1_16B_get_pkey(struct hfi1_16b_header *hdr)
493{
494 return (u16)((hdr->lrh[2] & OPA_16B_PKEY_MASK) >> OPA_16B_PKEY_SHIFT);
495}
496
497static inline u8 hfi1_16B_get_rc(struct hfi1_16b_header *hdr)
498{
499 return (u8)((hdr->lrh[1] & OPA_16B_RC_MASK) >> OPA_16B_RC_SHIFT);
500}
501
502static inline u8 hfi1_16B_get_age(struct hfi1_16b_header *hdr)
503{
504 return (u8)((hdr->lrh[3] & OPA_16B_AGE_MASK) >> OPA_16B_AGE_SHIFT);
505}
506
507static inline u16 hfi1_16B_get_len(struct hfi1_16b_header *hdr)
508{
509 return (u16)((hdr->lrh[0] & OPA_16B_LEN_MASK) >> OPA_16B_LEN_SHIFT);
510}
511
512static inline u16 hfi1_16B_get_entropy(struct hfi1_16b_header *hdr)
513{
514 return (u16)(hdr->lrh[3] & OPA_16B_ENTROPY_MASK);
515}
516
517#define OPA_16B_MAKE_QW(low_dw, high_dw) (((u64)(high_dw) << 32) | (low_dw))
518
519
520
521
522#define OPA_16B_BTH_PAD_MASK 7
523static inline u8 hfi1_16B_bth_get_pad(struct ib_other_headers *ohdr)
524{
525 return (u8)((be32_to_cpu(ohdr->bth[0]) >> IB_BTH_PAD_SHIFT) &
526 OPA_16B_BTH_PAD_MASK);
527}
528
529
530
531
532#define OPA_16B_MGMT_QPN_MASK 0xFFFFFF
533static inline u32 hfi1_16B_get_dest_qpn(struct opa_16b_mgmt *mgmt)
534{
535 return be32_to_cpu(mgmt->dest_qpn) & OPA_16B_MGMT_QPN_MASK;
536}
537
538static inline u32 hfi1_16B_get_src_qpn(struct opa_16b_mgmt *mgmt)
539{
540 return be32_to_cpu(mgmt->src_qpn) & OPA_16B_MGMT_QPN_MASK;
541}
542
543static inline void hfi1_16B_set_qpn(struct opa_16b_mgmt *mgmt,
544 u32 dest_qp, u32 src_qp)
545{
546 mgmt->dest_qpn = cpu_to_be32(dest_qp & OPA_16B_MGMT_QPN_MASK);
547 mgmt->src_qpn = cpu_to_be32(src_qp & OPA_16B_MGMT_QPN_MASK);
548}
549
550
551
552
553
554static inline struct ib_other_headers *
555hfi1_get_rc_ohdr(struct hfi1_opa_header *opah)
556{
557 struct ib_other_headers *ohdr;
558 struct ib_header *hdr = NULL;
559 struct hfi1_16b_header *hdr_16b = NULL;
560
561
562 if (opah->hdr_type == HFI1_PKT_TYPE_9B) {
563 hdr = &opah->ibh;
564 if (ib_get_lnh(hdr) == HFI1_LRH_BTH)
565 ohdr = &hdr->u.oth;
566 else
567 ohdr = &hdr->u.l.oth;
568 } else {
569 u8 l4;
570
571 hdr_16b = &opah->opah;
572 l4 = hfi1_16B_get_l4(hdr_16b);
573 if (l4 == OPA_16B_L4_IB_LOCAL)
574 ohdr = &hdr_16b->u.oth;
575 else
576 ohdr = &hdr_16b->u.l.oth;
577 }
578 return ohdr;
579}
580
581struct rvt_sge_state;
582
583
584
585
586
587
588#define HFI1_IB_CFG_LIDLMC 0
589#define HFI1_IB_CFG_LWID_DG_ENB 1
590#define HFI1_IB_CFG_LWID_ENB 2
591#define HFI1_IB_CFG_LWID 3
592#define HFI1_IB_CFG_SPD_ENB 4
593#define HFI1_IB_CFG_SPD 5
594#define HFI1_IB_CFG_RXPOL_ENB 6
595#define HFI1_IB_CFG_LREV_ENB 7
596#define HFI1_IB_CFG_LINKLATENCY 8
597#define HFI1_IB_CFG_HRTBT 9
598#define HFI1_IB_CFG_OP_VLS 10
599#define HFI1_IB_CFG_VL_HIGH_CAP 11
600#define HFI1_IB_CFG_VL_LOW_CAP 12
601#define HFI1_IB_CFG_OVERRUN_THRESH 13
602#define HFI1_IB_CFG_PHYERR_THRESH 14
603#define HFI1_IB_CFG_LINKDEFAULT 15
604#define HFI1_IB_CFG_PKEYS 16
605#define HFI1_IB_CFG_MTU 17
606#define HFI1_IB_CFG_VL_HIGH_LIMIT 19
607#define HFI1_IB_CFG_PMA_TICKS 20
608#define HFI1_IB_CFG_PORT 21
609
610
611
612
613
614
615
616
617
618#define __HLS_UP_INIT_BP 0
619#define __HLS_UP_ARMED_BP 1
620#define __HLS_UP_ACTIVE_BP 2
621#define __HLS_DN_DOWNDEF_BP 3
622#define __HLS_DN_POLL_BP 4
623#define __HLS_DN_DISABLE_BP 5
624#define __HLS_DN_OFFLINE_BP 6
625#define __HLS_VERIFY_CAP_BP 7
626#define __HLS_GOING_UP_BP 8
627#define __HLS_GOING_OFFLINE_BP 9
628#define __HLS_LINK_COOLDOWN_BP 10
629
630#define HLS_UP_INIT BIT(__HLS_UP_INIT_BP)
631#define HLS_UP_ARMED BIT(__HLS_UP_ARMED_BP)
632#define HLS_UP_ACTIVE BIT(__HLS_UP_ACTIVE_BP)
633#define HLS_DN_DOWNDEF BIT(__HLS_DN_DOWNDEF_BP)
634#define HLS_DN_POLL BIT(__HLS_DN_POLL_BP)
635#define HLS_DN_DISABLE BIT(__HLS_DN_DISABLE_BP)
636#define HLS_DN_OFFLINE BIT(__HLS_DN_OFFLINE_BP)
637#define HLS_VERIFY_CAP BIT(__HLS_VERIFY_CAP_BP)
638#define HLS_GOING_UP BIT(__HLS_GOING_UP_BP)
639#define HLS_GOING_OFFLINE BIT(__HLS_GOING_OFFLINE_BP)
640#define HLS_LINK_COOLDOWN BIT(__HLS_LINK_COOLDOWN_BP)
641
642#define HLS_UP (HLS_UP_INIT | HLS_UP_ARMED | HLS_UP_ACTIVE)
643#define HLS_DOWN ~(HLS_UP)
644
645#define HLS_DEFAULT HLS_DN_POLL
646
647
648#define HFI1_DEFAULT_ACTIVE_MTU 10240
649
650#define HFI1_DEFAULT_MAX_MTU 10240
651
652#define DEFAULT_PKEY 0xffff
653
654
655
656
657#define FM_TBL_VL_HIGH_ARB 1
658#define FM_TBL_VL_LOW_ARB 2
659#define FM_TBL_BUFFER_CONTROL 3
660#define FM_TBL_SC2VLNT 4
661#define FM_TBL_VL_PREEMPT_ELEMS 5
662#define FM_TBL_VL_PREEMPT_MATRIX 6
663
664
665
666
667
668
669#define HFI1_RCVCTRL_TAILUPD_ENB 0x01
670#define HFI1_RCVCTRL_TAILUPD_DIS 0x02
671#define HFI1_RCVCTRL_CTXT_ENB 0x04
672#define HFI1_RCVCTRL_CTXT_DIS 0x08
673#define HFI1_RCVCTRL_INTRAVAIL_ENB 0x10
674#define HFI1_RCVCTRL_INTRAVAIL_DIS 0x20
675#define HFI1_RCVCTRL_PKEY_ENB 0x40
676#define HFI1_RCVCTRL_PKEY_DIS 0x80
677#define HFI1_RCVCTRL_TIDFLOW_ENB 0x0400
678#define HFI1_RCVCTRL_TIDFLOW_DIS 0x0800
679#define HFI1_RCVCTRL_ONE_PKT_EGR_ENB 0x1000
680#define HFI1_RCVCTRL_ONE_PKT_EGR_DIS 0x2000
681#define HFI1_RCVCTRL_NO_RHQ_DROP_ENB 0x4000
682#define HFI1_RCVCTRL_NO_RHQ_DROP_DIS 0x8000
683#define HFI1_RCVCTRL_NO_EGR_DROP_ENB 0x10000
684#define HFI1_RCVCTRL_NO_EGR_DROP_DIS 0x20000
685#define HFI1_RCVCTRL_URGENT_ENB 0x40000
686#define HFI1_RCVCTRL_URGENT_DIS 0x80000
687
688
689#define HFI1_PART_ENFORCE_IN 0x1
690#define HFI1_PART_ENFORCE_OUT 0x2
691
692
693#define SYNTH_CNT_TIME 3
694
695
696#define CNTR_NORMAL 0x0
697#define CNTR_SYNTH 0x1
698#define CNTR_DISABLED 0x2
699#define CNTR_32BIT 0x4
700#define CNTR_VL 0x8
701#define CNTR_SDMA 0x10
702#define CNTR_INVALID_VL -1
703#define CNTR_MODE_W 0x0
704#define CNTR_MODE_R 0x1
705
706
707#define HFI1_MIN_VLS_SUPPORTED 1
708#define HFI1_MAX_VLS_SUPPORTED 8
709
710#define HFI1_GUIDS_PER_PORT 5
711#define HFI1_PORT_GUID_INDEX 0
712
713static inline void incr_cntr64(u64 *cntr)
714{
715 if (*cntr < (u64)-1LL)
716 (*cntr)++;
717}
718
719#define MAX_NAME_SIZE 64
720struct hfi1_msix_entry {
721 enum irq_type type;
722 int irq;
723 void *arg;
724 cpumask_t mask;
725 struct irq_affinity_notify notify;
726};
727
728struct hfi1_msix_info {
729
730 spinlock_t msix_lock;
731 DECLARE_BITMAP(in_use_msix, CCE_NUM_MSIX_VECTORS);
732 struct hfi1_msix_entry *msix_entries;
733 u16 max_requested;
734};
735
736
737struct cca_timer {
738 struct hrtimer hrtimer;
739 struct hfi1_pportdata *ppd;
740 int sl;
741 u16 ccti;
742};
743
744struct link_down_reason {
745
746
747
748
749 u8 sma;
750 u8 latest;
751};
752
753enum {
754 LO_PRIO_TABLE,
755 HI_PRIO_TABLE,
756 MAX_PRIO_TABLE
757};
758
759struct vl_arb_cache {
760
761 spinlock_t lock;
762 struct ib_vl_weight_elem table[VL_ARB_TABLE_SIZE];
763};
764
765
766
767
768
769
770
771struct hfi1_pportdata {
772 struct hfi1_ibport ibport_data;
773
774 struct hfi1_devdata *dd;
775 struct kobject pport_cc_kobj;
776 struct kobject sc2vl_kobj;
777 struct kobject sl2sc_kobj;
778 struct kobject vl2mtu_kobj;
779
780
781 struct qsfp_data qsfp_info;
782
783 u32 port_type;
784 u32 tx_preset_eq;
785 u32 tx_preset_noeq;
786 u32 rx_preset;
787 u8 local_atten;
788 u8 remote_atten;
789 u8 default_atten;
790 u8 max_power_class;
791
792
793 bool config_from_scratch;
794
795
796 u64 guids[HFI1_GUIDS_PER_PORT];
797
798
799 u64 neighbor_guid;
800
801
802 u32 linkup;
803
804
805
806
807
808 u64 *statusp;
809
810
811
812 struct workqueue_struct *hfi1_wq;
813 struct workqueue_struct *link_wq;
814
815
816 struct work_struct link_vc_work;
817 struct work_struct link_up_work;
818 struct work_struct link_down_work;
819 struct work_struct sma_message_work;
820 struct work_struct freeze_work;
821 struct work_struct link_downgrade_work;
822 struct work_struct link_bounce_work;
823 struct delayed_work start_link_work;
824
825 struct mutex hls_lock;
826 u32 host_link_state;
827
828
829
830 u32 ibmtu;
831
832
833
834
835 u32 ibmaxlen;
836 u32 current_egress_rate;
837
838 u32 lid;
839
840 u16 pkeys[MAX_PKEY_VALUES];
841 u16 link_width_supported;
842 u16 link_width_downgrade_supported;
843 u16 link_speed_supported;
844 u16 link_width_enabled;
845 u16 link_width_downgrade_enabled;
846 u16 link_speed_enabled;
847 u16 link_width_active;
848 u16 link_width_downgrade_tx_active;
849 u16 link_width_downgrade_rx_active;
850 u16 link_speed_active;
851 u8 vls_supported;
852 u8 vls_operational;
853 u8 actual_vls_operational;
854
855 u8 lmc;
856
857 u8 rx_pol_inv;
858
859 u8 hw_pidx;
860 u32 port;
861
862 u8 neighbor_type;
863 u8 neighbor_normal;
864 u8 neighbor_fm_security;
865 u8 neighbor_port_number;
866 u8 is_sm_config_started;
867 u8 offline_disabled_reason;
868 u8 is_active_optimize_enabled;
869 u8 driver_link_ready;
870 u8 link_enabled;
871 u8 linkinit_reason;
872 u8 local_tx_rate;
873 u8 qsfp_retry_count;
874
875
876 u8 overrun_threshold;
877 u8 phy_error_threshold;
878 unsigned int is_link_down_queued;
879
880
881
882
883
884
885 unsigned long led_override_vals[2];
886 u8 led_override_phase;
887 atomic_t led_override_timer_active;
888
889 struct timer_list led_override_timer;
890
891 u32 sm_trap_qp;
892 u32 sa_qp;
893
894
895
896
897
898 spinlock_t cca_timer_lock ____cacheline_aligned_in_smp;
899 struct cca_timer cca_timer[OPA_MAX_SLS];
900
901
902 struct ib_cc_table_entry_shadow ccti_entries[CC_TABLE_SHADOW_MAX];
903
904
905 struct opa_congestion_setting_entry_shadow
906 congestion_entries[OPA_MAX_SLS];
907
908
909
910
911
912 spinlock_t cc_state_lock ____cacheline_aligned_in_smp;
913
914 struct cc_state __rcu *cc_state;
915
916
917 u16 total_cct_entry;
918
919
920 u32 cc_sl_control_map;
921
922
923 u8 cc_max_table_entries;
924
925
926
927
928
929 spinlock_t cc_log_lock ____cacheline_aligned_in_smp;
930 u8 threshold_cong_event_map[OPA_MAX_SLS / 8];
931 u16 threshold_event_counter;
932 struct opa_hfi1_cong_log_event_internal cc_events[OPA_CONG_LOG_ELEMS];
933 int cc_log_idx;
934 int cc_mad_idx;
935
936
937 struct vl_arb_cache vl_arb_cache[MAX_PRIO_TABLE];
938
939
940 u64 *cntrs;
941
942 u64 *scntrs;
943
944 u64 port_xmit_discards;
945 u64 port_xmit_discards_vl[C_VL_COUNT];
946 u64 port_xmit_constraint_errors;
947 u64 port_rcv_constraint_errors;
948
949 u64 link_downed;
950
951 u64 link_up;
952
953 u64 unknown_frame_count;
954
955 u16 port_ltp_crc_mode;
956
957 u8 port_crc_mode_enabled;
958
959 u8 mgmt_allowed;
960 u8 part_enforce;
961 struct link_down_reason local_link_down_reason;
962 struct link_down_reason neigh_link_down_reason;
963
964 u8 remote_link_down_reason;
965
966 u32 port_error_action;
967 struct work_struct linkstate_active_work;
968
969 bool cc_prescan;
970
971
972
973
974 u64 port_vl_xmit_wait_last[C_VL_COUNT + 1];
975 u16 prev_link_width;
976 u64 vl_xmit_flit_cnt[C_VL_COUNT + 1];
977};
978
979typedef void (*opcode_handler)(struct hfi1_packet *packet);
980typedef void (*hfi1_make_req)(struct rvt_qp *qp,
981 struct hfi1_pkt_state *ps,
982 struct rvt_swqe *wqe);
983extern const rhf_rcv_function_ptr normal_rhf_rcv_functions[];
984extern const rhf_rcv_function_ptr netdev_rhf_rcv_functions[];
985
986
987#define RHF_RCV_CONTINUE 0
988#define RHF_RCV_DONE 1
989#define RHF_RCV_REPROCESS 2
990
991struct rcv_array_data {
992 u16 ngroups;
993 u16 nctxt_extra;
994 u8 group_size;
995};
996
997struct per_vl_data {
998 u16 mtu;
999 struct send_context *sc;
1000};
1001
1002
1003#define PER_VL_SEND_CONTEXTS 16
1004
1005struct err_info_rcvport {
1006 u8 status_and_code;
1007 u64 packet_flit1;
1008 u64 packet_flit2;
1009};
1010
1011struct err_info_constraint {
1012 u8 status;
1013 u16 pkey;
1014 u32 slid;
1015};
1016
1017struct hfi1_temp {
1018 unsigned int curr;
1019 unsigned int lo_lim;
1020 unsigned int hi_lim;
1021 unsigned int crit_lim;
1022 u8 triggers;
1023};
1024
1025struct hfi1_i2c_bus {
1026 struct hfi1_devdata *controlling_dd;
1027 struct i2c_adapter adapter;
1028 struct i2c_algo_bit_data algo;
1029 int num;
1030};
1031
1032
1033struct hfi1_asic_data {
1034 struct hfi1_devdata *dds[2];
1035 struct mutex asic_resource_mutex;
1036 struct hfi1_i2c_bus *i2c_bus0;
1037 struct hfi1_i2c_bus *i2c_bus1;
1038};
1039
1040
1041#define NUM_MAP_ENTRIES 256
1042#define NUM_MAP_REGS 32
1043
1044
1045struct hfi1_vnic_data {
1046 struct kmem_cache *txreq_cache;
1047 u8 num_vports;
1048};
1049
1050struct hfi1_vnic_vport_info;
1051
1052
1053
1054
1055struct sdma_engine;
1056struct sdma_vl_map;
1057
1058#define BOARD_VERS_MAX 96
1059#define SERIAL_MAX 16
1060
1061typedef int (*send_routine)(struct rvt_qp *, struct hfi1_pkt_state *, u64);
1062struct hfi1_netdev_rx;
1063struct hfi1_devdata {
1064 struct hfi1_ibdev verbs_dev;
1065
1066
1067 struct pci_dev *pcidev;
1068 struct cdev user_cdev;
1069 struct cdev diag_cdev;
1070 struct cdev ui_cdev;
1071 struct device *user_device;
1072 struct device *diag_device;
1073 struct device *ui_device;
1074
1075
1076 u8 __iomem *kregbase1;
1077 resource_size_t physaddr;
1078
1079
1080 u8 __iomem *kregbase2;
1081
1082 u32 base2_start;
1083
1084
1085 struct per_vl_data vld[PER_VL_SEND_CONTEXTS];
1086
1087 struct send_context_info *send_contexts;
1088
1089 u8 *hw_to_sw;
1090
1091 spinlock_t sc_lock;
1092
1093 spinlock_t pio_map_lock;
1094
1095 spinlock_t sc_init_lock;
1096
1097 spinlock_t sde_map_lock;
1098
1099 struct send_context **kernel_send_context;
1100
1101 struct pio_vl_map __rcu *pio_map;
1102
1103 u64 default_desc1;
1104
1105
1106
1107 volatile __le64 *sdma_heads_dma;
1108 dma_addr_t sdma_heads_phys;
1109 void *sdma_pad_dma;
1110 dma_addr_t sdma_pad_phys;
1111
1112 size_t sdma_heads_size;
1113
1114 u32 num_sdma;
1115
1116 struct sdma_engine *per_sdma;
1117
1118 struct sdma_vl_map __rcu *sdma_map;
1119
1120 wait_queue_head_t sdma_unfreeze_wq;
1121 atomic_t sdma_unfreeze_count;
1122
1123 u32 lcb_access_count;
1124
1125
1126 struct hfi1_asic_data *asic_data;
1127
1128
1129 void __iomem *piobase;
1130
1131
1132
1133
1134 void __iomem *rcvarray_wc;
1135
1136
1137
1138
1139 struct credit_return_base *cr_base;
1140
1141
1142 struct sc_config_sizes sc_sizes[SC_MAX];
1143
1144 char *boardname;
1145
1146 u64 ctx0_seq_drop;
1147
1148
1149 u64 z_int_counter;
1150 u64 z_rcv_limit;
1151 u64 z_send_schedule;
1152
1153 u64 __percpu *send_schedule;
1154
1155 u16 num_netdev_contexts;
1156
1157 u32 num_rcv_contexts;
1158
1159 u32 num_send_contexts;
1160
1161
1162
1163 u32 freectxts;
1164
1165 u32 num_user_contexts;
1166
1167 u32 rcv_intr_timeout_csr;
1168
1169 spinlock_t sendctrl_lock;
1170 spinlock_t rcvctrl_lock;
1171 spinlock_t uctxt_lock;
1172 struct mutex dc8051_lock;
1173 struct workqueue_struct *update_cntr_wq;
1174 struct work_struct update_cntr_work;
1175
1176 spinlock_t dc8051_memlock;
1177 int dc8051_timed_out;
1178
1179
1180
1181
1182 unsigned long *events;
1183
1184
1185
1186
1187
1188 struct hfi1_status *status;
1189
1190
1191 u64 revision;
1192
1193 u64 base_guid;
1194
1195
1196 u8 link_gen3_capable;
1197 u8 dc_shutdown;
1198
1199 u32 lbus_width;
1200
1201 u32 lbus_speed;
1202 int unit;
1203 int node;
1204
1205
1206 u32 pcibar0;
1207 u32 pcibar1;
1208 u32 pci_rom;
1209 u16 pci_command;
1210 u16 pcie_devctl;
1211 u16 pcie_lnkctl;
1212 u16 pcie_devctl2;
1213 u32 pci_msix0;
1214 u32 pci_tph2;
1215
1216
1217
1218
1219
1220 u8 serial[SERIAL_MAX];
1221
1222 u8 boardversion[BOARD_VERS_MAX];
1223 u8 lbus_info[32];
1224
1225 u8 majrev;
1226
1227 u8 minrev;
1228
1229 u8 hfi1_id;
1230
1231 u8 icode;
1232
1233 u8 vau;
1234
1235 u8 vcu;
1236
1237 u16 link_credits;
1238
1239 u16 vl15_init;
1240
1241
1242
1243
1244
1245
1246
1247 u16 vl15buf_cached;
1248
1249
1250 u8 n_krcv_queues;
1251 u8 qos_shift;
1252
1253 u16 irev;
1254 u32 dc8051_ver;
1255
1256 spinlock_t hfi1_diag_trans_lock;
1257 struct platform_config platform_config;
1258 struct platform_config_cache pcfg_cache;
1259
1260 struct diag_client *diag_client;
1261
1262
1263 u64 gi_mask[CCE_NUM_INT_CSRS];
1264
1265 struct rcv_array_data rcv_entries;
1266
1267
1268 u16 psxmitwait_check_rate;
1269
1270
1271
1272
1273 struct timer_list synth_stats_timer;
1274
1275
1276 struct hfi1_msix_info msix_info;
1277
1278
1279
1280
1281 char *cntrnames;
1282 size_t cntrnameslen;
1283 size_t ndevcntrs;
1284 u64 *cntrs;
1285 u64 *scntrs;
1286
1287
1288
1289
1290 u64 last_tx;
1291 u64 last_rx;
1292
1293
1294
1295
1296 size_t nportcntrs;
1297 char *portcntrnames;
1298 size_t portcntrnameslen;
1299
1300 struct err_info_rcvport err_info_rcvport;
1301 struct err_info_constraint err_info_rcv_constraint;
1302 struct err_info_constraint err_info_xmit_constraint;
1303
1304 atomic_t drop_packet;
1305 bool do_drop;
1306 u8 err_info_uncorrectable;
1307 u8 err_info_fmconfig;
1308
1309
1310
1311
1312
1313 u64 cce_err_status_cnt[NUM_CCE_ERR_STATUS_COUNTERS];
1314 u64 rcv_err_status_cnt[NUM_RCV_ERR_STATUS_COUNTERS];
1315 u64 misc_err_status_cnt[NUM_MISC_ERR_STATUS_COUNTERS];
1316 u64 send_pio_err_status_cnt[NUM_SEND_PIO_ERR_STATUS_COUNTERS];
1317 u64 send_dma_err_status_cnt[NUM_SEND_DMA_ERR_STATUS_COUNTERS];
1318 u64 send_egress_err_status_cnt[NUM_SEND_EGRESS_ERR_STATUS_COUNTERS];
1319 u64 send_err_status_cnt[NUM_SEND_ERR_STATUS_COUNTERS];
1320
1321
1322 u64 sw_ctxt_err_status_cnt[NUM_SEND_CTXT_ERR_STATUS_COUNTERS];
1323
1324 u64 sw_send_dma_eng_err_status_cnt[
1325 NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS];
1326
1327 u64 sw_cce_err_status_aggregate;
1328
1329 u64 sw_rcv_bypass_packet_errors;
1330
1331
1332 u64 lcb_err_en;
1333 struct cpu_mask_set *comp_vect;
1334 int *comp_vect_mappings;
1335 u32 comp_vect_possible_cpus;
1336
1337
1338
1339
1340
1341 send_routine process_pio_send ____cacheline_aligned_in_smp;
1342 send_routine process_dma_send;
1343 void (*pio_inline_send)(struct hfi1_devdata *dd, struct pio_buf *pbuf,
1344 u64 pbc, const void *from, size_t count);
1345 int (*process_vnic_dma_send)(struct hfi1_devdata *dd, u8 q_idx,
1346 struct hfi1_vnic_vport_info *vinfo,
1347 struct sk_buff *skb, u64 pbc, u8 plen);
1348
1349
1350
1351 struct hfi1_pportdata *pport;
1352
1353 struct hfi1_ctxtdata **rcd;
1354 u64 __percpu *int_counter;
1355
1356 struct hfi1_opcode_stats_perctx __percpu *tx_opstats;
1357
1358 u16 flags;
1359
1360 u8 num_pports;
1361
1362 u8 first_dyn_alloc_ctxt;
1363
1364
1365
1366 seqlock_t sc2vl_lock ____cacheline_aligned_in_smp;
1367 u64 sc2vl[4];
1368 u64 __percpu *rcv_limit;
1369
1370
1371
1372 u8 oui1;
1373 u8 oui2;
1374 u8 oui3;
1375
1376
1377 struct timer_list rcverr_timer;
1378
1379 wait_queue_head_t event_queue;
1380
1381
1382 __le64 *rcvhdrtail_dummy_kvaddr;
1383 dma_addr_t rcvhdrtail_dummy_dma;
1384
1385 u32 rcv_ovfl_cnt;
1386
1387 spinlock_t aspm_lock;
1388
1389 atomic_t aspm_disabled_cnt;
1390
1391 atomic_t user_refcount;
1392
1393 struct completion user_comp;
1394
1395 bool eprom_available;
1396 bool aspm_supported;
1397 bool aspm_enabled;
1398 struct rhashtable *sdma_rht;
1399
1400
1401 struct hfi1_vnic_data vnic;
1402
1403 spinlock_t irq_src_lock;
1404 int vnic_num_vports;
1405 struct hfi1_netdev_rx *netdev_rx;
1406 struct hfi1_affinity_node *affinity_entry;
1407
1408
1409 atomic_t ipoib_rsm_usr_num;
1410};
1411
1412
1413#define dc8051_ver(a, b, c) ((a) << 16 | (b) << 8 | (c))
1414#define dc8051_ver_maj(a) (((a) & 0xff0000) >> 16)
1415#define dc8051_ver_min(a) (((a) & 0x00ff00) >> 8)
1416#define dc8051_ver_patch(a) ((a) & 0x0000ff)
1417
1418
1419#define PT_EXPECTED 0
1420#define PT_EAGER 1
1421#define PT_INVALID_FLUSH 2
1422#define PT_INVALID 3
1423
1424struct tid_rb_node;
1425struct mmu_rb_node;
1426struct mmu_rb_handler;
1427
1428
1429struct hfi1_filedata {
1430 struct srcu_struct pq_srcu;
1431 struct hfi1_devdata *dd;
1432 struct hfi1_ctxtdata *uctxt;
1433 struct hfi1_user_sdma_comp_q *cq;
1434
1435 spinlock_t pq_rcu_lock;
1436 struct hfi1_user_sdma_pkt_q __rcu *pq;
1437 u16 subctxt;
1438
1439 int rec_cpu_num;
1440 u32 tid_n_pinned;
1441 bool use_mn;
1442 struct tid_rb_node **entry_to_rb;
1443 spinlock_t tid_lock;
1444 u32 tid_limit;
1445 u32 tid_used;
1446 u32 *invalid_tids;
1447 u32 invalid_tid_idx;
1448
1449 spinlock_t invalid_lock;
1450};
1451
1452extern struct xarray hfi1_dev_table;
1453struct hfi1_devdata *hfi1_lookup(int unit);
1454
1455static inline unsigned long uctxt_offset(struct hfi1_ctxtdata *uctxt)
1456{
1457 return (uctxt->ctxt - uctxt->dd->first_dyn_alloc_ctxt) *
1458 HFI1_MAX_SHARED_CTXTS;
1459}
1460
1461int hfi1_init(struct hfi1_devdata *dd, int reinit);
1462int hfi1_count_active_units(void);
1463
1464int hfi1_diag_add(struct hfi1_devdata *dd);
1465void hfi1_diag_remove(struct hfi1_devdata *dd);
1466void handle_linkup_change(struct hfi1_devdata *dd, u32 linkup);
1467
1468void handle_user_interrupt(struct hfi1_ctxtdata *rcd);
1469
1470int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd);
1471int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd);
1472int hfi1_create_kctxts(struct hfi1_devdata *dd);
1473int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
1474 struct hfi1_ctxtdata **rcd);
1475void hfi1_free_ctxt(struct hfi1_ctxtdata *rcd);
1476void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd,
1477 struct hfi1_devdata *dd, u8 hw_pidx, u32 port);
1478void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd);
1479int hfi1_rcd_put(struct hfi1_ctxtdata *rcd);
1480int hfi1_rcd_get(struct hfi1_ctxtdata *rcd);
1481struct hfi1_ctxtdata *hfi1_rcd_get_by_index_safe(struct hfi1_devdata *dd,
1482 u16 ctxt);
1483struct hfi1_ctxtdata *hfi1_rcd_get_by_index(struct hfi1_devdata *dd, u16 ctxt);
1484int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread);
1485int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread);
1486int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread);
1487int handle_receive_interrupt_napi_fp(struct hfi1_ctxtdata *rcd, int budget);
1488int handle_receive_interrupt_napi_sp(struct hfi1_ctxtdata *rcd, int budget);
1489void set_all_slowpath(struct hfi1_devdata *dd);
1490
1491extern const struct pci_device_id hfi1_pci_tbl[];
1492void hfi1_make_ud_req_9B(struct rvt_qp *qp,
1493 struct hfi1_pkt_state *ps,
1494 struct rvt_swqe *wqe);
1495
1496void hfi1_make_ud_req_16B(struct rvt_qp *qp,
1497 struct hfi1_pkt_state *ps,
1498 struct rvt_swqe *wqe);
1499
1500
1501#define RCV_PKT_OK 0x0
1502#define RCV_PKT_LIMIT 0x1
1503#define RCV_PKT_DONE 0x2
1504
1505
1506
1507
1508
1509static inline u32 hfi1_rcd_head(struct hfi1_ctxtdata *rcd)
1510{
1511 return rcd->head;
1512}
1513
1514
1515
1516
1517
1518
1519static inline void hfi1_set_rcd_head(struct hfi1_ctxtdata *rcd, u32 head)
1520{
1521 rcd->head = head;
1522}
1523
1524
1525static inline __le32 *get_rhf_addr(struct hfi1_ctxtdata *rcd)
1526{
1527 return (__le32 *)rcd->rcvhdrq + rcd->head + rcd->rhf_offset;
1528}
1529
1530
1531static inline bool get_dma_rtail_setting(struct hfi1_ctxtdata *rcd)
1532{
1533 return !!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL);
1534}
1535
1536
1537
1538
1539
1540
1541
1542static inline u8 hfi1_seq_incr_wrap(u8 seq)
1543{
1544 if (++seq > RHF_MAX_SEQ)
1545 seq = 1;
1546 return seq;
1547}
1548
1549
1550
1551
1552
1553
1554
1555static inline u8 hfi1_seq_cnt(struct hfi1_ctxtdata *rcd)
1556{
1557 return rcd->seq_cnt;
1558}
1559
1560
1561
1562
1563
1564
1565
1566static inline void hfi1_set_seq_cnt(struct hfi1_ctxtdata *rcd, u8 cnt)
1567{
1568 rcd->seq_cnt = cnt;
1569}
1570
1571
1572
1573
1574
1575
1576
1577
1578static inline bool last_rcv_seq(struct hfi1_ctxtdata *rcd, u32 seq)
1579{
1580 return seq != rcd->seq_cnt;
1581}
1582
1583
1584
1585
1586
1587
1588
1589
1590static inline bool hfi1_seq_incr(struct hfi1_ctxtdata *rcd, u32 seq)
1591{
1592 rcd->seq_cnt = hfi1_seq_incr_wrap(rcd->seq_cnt);
1593 return last_rcv_seq(rcd, seq);
1594}
1595
1596
1597
1598
1599
1600static inline u8 get_hdrqentsize(struct hfi1_ctxtdata *rcd)
1601{
1602 return rcd->rcvhdrqentsize;
1603}
1604
1605
1606
1607
1608
1609static inline u16 get_hdrq_cnt(struct hfi1_ctxtdata *rcd)
1610{
1611 return rcd->rcvhdrq_cnt;
1612}
1613
1614
1615
1616
1617
1618static inline bool hfi1_is_slowpath(struct hfi1_ctxtdata *rcd)
1619{
1620 return rcd->do_interrupt == rcd->slow_handler;
1621}
1622
1623
1624
1625
1626
1627static inline bool hfi1_is_fastpath(struct hfi1_ctxtdata *rcd)
1628{
1629 if (rcd->ctxt == HFI1_CTRL_CTXT)
1630 return false;
1631
1632 return rcd->do_interrupt == rcd->fast_handler;
1633}
1634
1635
1636
1637
1638
1639static inline void hfi1_set_fast(struct hfi1_ctxtdata *rcd)
1640{
1641 if (unlikely(!rcd))
1642 return;
1643 if (unlikely(!hfi1_is_fastpath(rcd)))
1644 rcd->do_interrupt = rcd->fast_handler;
1645}
1646
1647int hfi1_reset_device(int);
1648
1649void receive_interrupt_work(struct work_struct *work);
1650
1651
1652static inline int hfi1_9B_get_sc5(struct ib_header *hdr, u64 rhf)
1653{
1654 return ib_get_sc(hdr) | ((!!(rhf_dc_info(rhf))) << 4);
1655}
1656
1657#define HFI1_JKEY_WIDTH 16
1658#define HFI1_JKEY_MASK (BIT(16) - 1)
1659#define HFI1_ADMIN_JKEY_RANGE 32
1660
1661
1662
1663
1664
1665
1666
1667static inline u16 generate_jkey(kuid_t uid)
1668{
1669 u16 jkey = from_kuid(current_user_ns(), uid) & HFI1_JKEY_MASK;
1670
1671 if (capable(CAP_SYS_ADMIN))
1672 jkey &= HFI1_ADMIN_JKEY_RANGE - 1;
1673 else if (jkey < 64)
1674 jkey |= BIT(HFI1_JKEY_WIDTH - 1);
1675
1676 return jkey;
1677}
1678
1679
1680
1681
1682
1683
1684static inline u32 active_egress_rate(struct hfi1_pportdata *ppd)
1685{
1686 u16 link_speed = ppd->link_speed_active;
1687 u16 link_width = ppd->link_width_active;
1688 u32 egress_rate;
1689
1690 if (link_speed == OPA_LINK_SPEED_25G)
1691 egress_rate = 25000;
1692 else
1693 egress_rate = 12500;
1694
1695 switch (link_width) {
1696 case OPA_LINK_WIDTH_4X:
1697 egress_rate *= 4;
1698 break;
1699 case OPA_LINK_WIDTH_3X:
1700 egress_rate *= 3;
1701 break;
1702 case OPA_LINK_WIDTH_2X:
1703 egress_rate *= 2;
1704 break;
1705 default:
1706
1707 break;
1708 }
1709
1710 return egress_rate;
1711}
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721static inline u32 egress_cycles(u32 len, u32 rate)
1722{
1723 u32 cycles;
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733 cycles = len * 8;
1734 cycles *= 805;
1735 cycles /= rate;
1736
1737 return cycles;
1738}
1739
1740void set_link_ipg(struct hfi1_pportdata *ppd);
1741void process_becn(struct hfi1_pportdata *ppd, u8 sl, u32 rlid, u32 lqpn,
1742 u32 rqpn, u8 svc_type);
1743void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn,
1744 u16 pkey, u32 slid, u32 dlid, u8 sc5,
1745 const struct ib_grh *old_grh);
1746void return_cnp_16B(struct hfi1_ibport *ibp, struct rvt_qp *qp,
1747 u32 remote_qpn, u16 pkey, u32 slid, u32 dlid,
1748 u8 sc5, const struct ib_grh *old_grh);
1749typedef void (*hfi1_handle_cnp)(struct hfi1_ibport *ibp, struct rvt_qp *qp,
1750 u32 remote_qpn, u16 pkey, u32 slid, u32 dlid,
1751 u8 sc5, const struct ib_grh *old_grh);
1752
1753#define PKEY_CHECK_INVALID -1
1754int egress_pkey_check(struct hfi1_pportdata *ppd, u32 slid, u16 pkey,
1755 u8 sc5, int8_t s_pkey_index);
1756
1757#define PACKET_EGRESS_TIMEOUT 350
1758static inline void pause_for_credit_return(struct hfi1_devdata *dd)
1759{
1760
1761 u32 usec = cclock_to_ns(dd, PACKET_EGRESS_TIMEOUT) / 1000;
1762
1763 udelay(usec ? usec : 1);
1764}
1765
1766
1767
1768
1769
1770
1771static inline u8 sc_to_vlt(struct hfi1_devdata *dd, u8 sc5)
1772{
1773 unsigned seq;
1774 u8 rval;
1775
1776 if (sc5 >= OPA_MAX_SCS)
1777 return (u8)(0xff);
1778
1779 do {
1780 seq = read_seqbegin(&dd->sc2vl_lock);
1781 rval = *(((u8 *)dd->sc2vl) + sc5);
1782 } while (read_seqretry(&dd->sc2vl_lock, seq));
1783
1784 return rval;
1785}
1786
1787#define PKEY_MEMBER_MASK 0x8000
1788#define PKEY_LOW_15_MASK 0x7fff
1789
1790
1791
1792
1793
1794
1795
1796static inline int ingress_pkey_matches_entry(u16 pkey, u16 ent)
1797{
1798 u16 mkey = pkey & PKEY_LOW_15_MASK;
1799 u16 ment = ent & PKEY_LOW_15_MASK;
1800
1801 if (mkey == ment) {
1802
1803
1804
1805
1806
1807 if (!(pkey & PKEY_MEMBER_MASK))
1808 return !!(ent & PKEY_MEMBER_MASK);
1809 return 1;
1810 }
1811 return 0;
1812}
1813
1814
1815
1816
1817
1818
1819static int ingress_pkey_table_search(struct hfi1_pportdata *ppd, u16 pkey)
1820{
1821 int i;
1822
1823 for (i = 0; i < MAX_PKEY_VALUES; i++) {
1824 if (ingress_pkey_matches_entry(pkey, ppd->pkeys[i]))
1825 return 0;
1826 }
1827 return 1;
1828}
1829
1830
1831
1832
1833
1834
1835static void ingress_pkey_table_fail(struct hfi1_pportdata *ppd, u16 pkey,
1836 u32 slid)
1837{
1838 struct hfi1_devdata *dd = ppd->dd;
1839
1840 incr_cntr64(&ppd->port_rcv_constraint_errors);
1841 if (!(dd->err_info_rcv_constraint.status & OPA_EI_STATUS_SMASK)) {
1842 dd->err_info_rcv_constraint.status |= OPA_EI_STATUS_SMASK;
1843 dd->err_info_rcv_constraint.slid = slid;
1844 dd->err_info_rcv_constraint.pkey = pkey;
1845 }
1846}
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856static inline int ingress_pkey_check(struct hfi1_pportdata *ppd, u16 pkey,
1857 u8 sc5, u8 idx, u32 slid, bool force)
1858{
1859 if (!(force) && !(ppd->part_enforce & HFI1_PART_ENFORCE_IN))
1860 return 0;
1861
1862
1863 if ((sc5 == 0xf) && ((pkey & PKEY_LOW_15_MASK) != PKEY_LOW_15_MASK))
1864 goto bad;
1865
1866
1867 if ((pkey & PKEY_LOW_15_MASK) == 0)
1868 goto bad;
1869
1870
1871 if (ingress_pkey_matches_entry(pkey, ppd->pkeys[idx]))
1872 return 0;
1873
1874
1875 if (!ingress_pkey_table_search(ppd, pkey))
1876 return 0;
1877
1878bad:
1879 ingress_pkey_table_fail(ppd, pkey, slid);
1880 return 1;
1881}
1882
1883
1884
1885
1886
1887
1888
1889static inline int rcv_pkey_check(struct hfi1_pportdata *ppd, u16 pkey,
1890 u8 sc5, u16 slid)
1891{
1892 if (!(ppd->part_enforce & HFI1_PART_ENFORCE_IN))
1893 return 0;
1894
1895
1896 if ((sc5 == 0xf) && ((pkey & PKEY_LOW_15_MASK) != PKEY_LOW_15_MASK))
1897 goto bad;
1898
1899 return 0;
1900bad:
1901 ingress_pkey_table_fail(ppd, pkey, slid);
1902 return 1;
1903}
1904
1905
1906
1907
1908#define OPA_MTU_0 0
1909#define OPA_MTU_256 1
1910#define OPA_MTU_512 2
1911#define OPA_MTU_1024 3
1912#define OPA_MTU_2048 4
1913#define OPA_MTU_4096 5
1914
1915u32 lrh_max_header_bytes(struct hfi1_devdata *dd);
1916int mtu_to_enum(u32 mtu, int default_if_bad);
1917u16 enum_to_mtu(int mtu);
1918static inline int valid_ib_mtu(unsigned int mtu)
1919{
1920 return mtu == 256 || mtu == 512 ||
1921 mtu == 1024 || mtu == 2048 ||
1922 mtu == 4096;
1923}
1924
1925static inline int valid_opa_max_mtu(unsigned int mtu)
1926{
1927 return mtu >= 2048 &&
1928 (valid_ib_mtu(mtu) || mtu == 8192 || mtu == 10240);
1929}
1930
1931int set_mtu(struct hfi1_pportdata *ppd);
1932
1933int hfi1_set_lid(struct hfi1_pportdata *ppd, u32 lid, u8 lmc);
1934void hfi1_disable_after_error(struct hfi1_devdata *dd);
1935int hfi1_set_uevent_bits(struct hfi1_pportdata *ppd, const int evtbit);
1936int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encode);
1937
1938int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t);
1939int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t);
1940
1941void set_up_vau(struct hfi1_devdata *dd, u8 vau);
1942void set_up_vl15(struct hfi1_devdata *dd, u16 vl15buf);
1943void reset_link_credits(struct hfi1_devdata *dd);
1944void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu);
1945
1946int set_buffer_control(struct hfi1_pportdata *ppd, struct buffer_control *bc);
1947
1948static inline struct hfi1_devdata *dd_from_ppd(struct hfi1_pportdata *ppd)
1949{
1950 return ppd->dd;
1951}
1952
1953static inline struct hfi1_devdata *dd_from_dev(struct hfi1_ibdev *dev)
1954{
1955 return container_of(dev, struct hfi1_devdata, verbs_dev);
1956}
1957
1958static inline struct hfi1_devdata *dd_from_ibdev(struct ib_device *ibdev)
1959{
1960 return dd_from_dev(to_idev(ibdev));
1961}
1962
1963static inline struct hfi1_pportdata *ppd_from_ibp(struct hfi1_ibport *ibp)
1964{
1965 return container_of(ibp, struct hfi1_pportdata, ibport_data);
1966}
1967
1968static inline struct hfi1_ibdev *dev_from_rdi(struct rvt_dev_info *rdi)
1969{
1970 return container_of(rdi, struct hfi1_ibdev, rdi);
1971}
1972
1973static inline struct hfi1_ibport *to_iport(struct ib_device *ibdev, u32 port)
1974{
1975 struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
1976 u32 pidx = port - 1;
1977
1978 WARN_ON(pidx >= dd->num_pports);
1979 return &dd->pport[pidx].ibport_data;
1980}
1981
1982static inline struct hfi1_ibport *rcd_to_iport(struct hfi1_ctxtdata *rcd)
1983{
1984 return &rcd->ppd->ibport_data;
1985}
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998static inline bool hfi1_may_ecn(struct hfi1_packet *pkt)
1999{
2000 bool fecn, becn;
2001
2002 if (pkt->etype == RHF_RCV_TYPE_BYPASS) {
2003 fecn = hfi1_16B_get_fecn(pkt->hdr);
2004 becn = hfi1_16B_get_becn(pkt->hdr);
2005 } else {
2006 fecn = ib_bth_get_fecn(pkt->ohdr);
2007 becn = ib_bth_get_becn(pkt->ohdr);
2008 }
2009 return fecn || becn;
2010}
2011
2012bool hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
2013 bool prescan);
2014static inline bool process_ecn(struct rvt_qp *qp, struct hfi1_packet *pkt)
2015{
2016 bool do_work;
2017
2018 do_work = hfi1_may_ecn(pkt);
2019 if (unlikely(do_work))
2020 return hfi1_process_ecn_slowpath(qp, pkt, false);
2021 return false;
2022}
2023
2024
2025
2026
2027static inline u16 hfi1_get_pkey(struct hfi1_ibport *ibp, unsigned index)
2028{
2029 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
2030 u16 ret;
2031
2032 if (index >= ARRAY_SIZE(ppd->pkeys))
2033 ret = 0;
2034 else
2035 ret = ppd->pkeys[index];
2036
2037 return ret;
2038}
2039
2040
2041
2042
2043static inline __be64 get_sguid(struct hfi1_ibport *ibp, unsigned int index)
2044{
2045 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
2046
2047 WARN_ON(index >= HFI1_GUIDS_PER_PORT);
2048 return cpu_to_be64(ppd->guids[index]);
2049}
2050
2051
2052
2053
2054static inline struct cc_state *get_cc_state(struct hfi1_pportdata *ppd)
2055{
2056 return rcu_dereference(ppd->cc_state);
2057}
2058
2059
2060
2061
2062static inline
2063struct cc_state *get_cc_state_protected(struct hfi1_pportdata *ppd)
2064{
2065 return rcu_dereference_protected(ppd->cc_state,
2066 lockdep_is_held(&ppd->cc_state_lock));
2067}
2068
2069
2070
2071
2072#define HFI1_INITTED 0x1
2073#define HFI1_PRESENT 0x2
2074#define HFI1_FROZEN 0x4
2075#define HFI1_HAS_SDMA_TIMEOUT 0x8
2076#define HFI1_HAS_SEND_DMA 0x10
2077#define HFI1_FORCED_FREEZE 0x80
2078#define HFI1_SHUTDOWN 0x100
2079
2080
2081#define HFI1_PBC_LENGTH_MASK ((1 << 11) - 1)
2082
2083
2084
2085#define HFI1_CTXT_BASE_UNINIT 1
2086
2087#define HFI1_CTXT_BASE_FAILED 2
2088
2089#define HFI1_CTXT_WAITING_RCV 3
2090
2091#define HFI1_CTXT_WAITING_URG 4
2092
2093
2094int hfi1_init_dd(struct hfi1_devdata *dd);
2095void hfi1_free_devdata(struct hfi1_devdata *dd);
2096
2097
2098void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon,
2099 unsigned int timeoff);
2100void shutdown_led_override(struct hfi1_pportdata *ppd);
2101
2102#define HFI1_CREDIT_RETURN_RATE (100)
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123#define DEFAULT_RCVHDRSIZE 9
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140#define DEFAULT_RCVHDR_ENTSIZE 32
2141
2142bool hfi1_can_pin_pages(struct hfi1_devdata *dd, struct mm_struct *mm,
2143 u32 nlocked, u32 npages);
2144int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr,
2145 size_t npages, bool writable, struct page **pages);
2146void hfi1_release_user_pages(struct mm_struct *mm, struct page **p,
2147 size_t npages, bool dirty);
2148
2149
2150
2151
2152
2153static inline __le64 *hfi1_rcvhdrtail_kvaddr(const struct hfi1_ctxtdata *rcd)
2154{
2155 return (__le64 *)rcd->rcvhdrtail_kvaddr;
2156}
2157
2158static inline void clear_rcvhdrtail(const struct hfi1_ctxtdata *rcd)
2159{
2160 u64 *kv = (u64 *)hfi1_rcvhdrtail_kvaddr(rcd);
2161
2162 if (kv)
2163 *kv = 0ULL;
2164}
2165
2166static inline u32 get_rcvhdrtail(const struct hfi1_ctxtdata *rcd)
2167{
2168
2169
2170
2171
2172 return (u32)le64_to_cpu(*hfi1_rcvhdrtail_kvaddr(rcd));
2173}
2174
2175static inline bool hfi1_packet_present(struct hfi1_ctxtdata *rcd)
2176{
2177 if (likely(!rcd->rcvhdrtail_kvaddr)) {
2178 u32 seq = rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd)));
2179
2180 return !last_rcv_seq(rcd, seq);
2181 }
2182 return hfi1_rcd_head(rcd) != get_rcvhdrtail(rcd);
2183}
2184
2185
2186
2187
2188
2189extern const char ib_hfi1_version[];
2190extern const struct attribute_group ib_hfi1_attr_group;
2191
2192int hfi1_device_create(struct hfi1_devdata *dd);
2193void hfi1_device_remove(struct hfi1_devdata *dd);
2194
2195int hfi1_create_port_files(struct ib_device *ibdev, u32 port_num,
2196 struct kobject *kobj);
2197int hfi1_verbs_register_sysfs(struct hfi1_devdata *dd);
2198void hfi1_verbs_unregister_sysfs(struct hfi1_devdata *dd);
2199
2200int qsfp_dump(struct hfi1_pportdata *ppd, char *buf, int len);
2201
2202int hfi1_pcie_init(struct hfi1_devdata *dd);
2203void hfi1_pcie_cleanup(struct pci_dev *pdev);
2204int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev);
2205void hfi1_pcie_ddcleanup(struct hfi1_devdata *);
2206int pcie_speeds(struct hfi1_devdata *dd);
2207int restore_pci_variables(struct hfi1_devdata *dd);
2208int save_pci_variables(struct hfi1_devdata *dd);
2209int do_pcie_gen3_transition(struct hfi1_devdata *dd);
2210void tune_pcie_caps(struct hfi1_devdata *dd);
2211int parse_platform_config(struct hfi1_devdata *dd);
2212int get_platform_config_field(struct hfi1_devdata *dd,
2213 enum platform_config_table_type_encoding
2214 table_type, int table_index, int field_index,
2215 u32 *data, u32 len);
2216
2217struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi);
2218
2219
2220
2221
2222
2223static inline void flush_wc(void)
2224{
2225 asm volatile("sfence" : : : "memory");
2226}
2227
2228void handle_eflags(struct hfi1_packet *packet);
2229void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd);
2230
2231
2232extern unsigned int hfi1_max_mtu;
2233extern unsigned int hfi1_cu;
2234extern unsigned int user_credit_return_threshold;
2235extern int num_user_contexts;
2236extern unsigned long n_krcvqs;
2237extern uint krcvqs[];
2238extern int krcvqsset;
2239extern uint loopback;
2240extern uint quick_linkup;
2241extern uint rcv_intr_timeout;
2242extern uint rcv_intr_count;
2243extern uint rcv_intr_dynamic;
2244extern ushort link_crc_mask;
2245
2246extern struct mutex hfi1_mutex;
2247
2248
2249#define STATUS_TIMEOUT 60
2250
2251#define DRIVER_NAME "hfi1"
2252#define HFI1_USER_MINOR_BASE 0
2253#define HFI1_TRACE_MINOR 127
2254#define HFI1_NMINORS 255
2255
2256#define PCI_VENDOR_ID_INTEL 0x8086
2257#define PCI_DEVICE_ID_INTEL0 0x24f0
2258#define PCI_DEVICE_ID_INTEL1 0x24f1
2259
2260#define HFI1_PKT_USER_SC_INTEGRITY \
2261 (SEND_CTXT_CHECK_ENABLE_DISALLOW_NON_KDETH_PACKETS_SMASK \
2262 | SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK \
2263 | SEND_CTXT_CHECK_ENABLE_DISALLOW_BYPASS_SMASK \
2264 | SEND_CTXT_CHECK_ENABLE_DISALLOW_GRH_SMASK)
2265
2266#define HFI1_PKT_KERNEL_SC_INTEGRITY \
2267 (SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK)
2268
2269static inline u64 hfi1_pkt_default_send_ctxt_mask(struct hfi1_devdata *dd,
2270 u16 ctxt_type)
2271{
2272 u64 base_sc_integrity;
2273
2274
2275 if (HFI1_CAP_IS_KSET(NO_INTEGRITY))
2276 return 0;
2277
2278 base_sc_integrity =
2279 SEND_CTXT_CHECK_ENABLE_DISALLOW_BYPASS_BAD_PKT_LEN_SMASK
2280 | SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK
2281 | SEND_CTXT_CHECK_ENABLE_DISALLOW_TOO_LONG_BYPASS_PACKETS_SMASK
2282 | SEND_CTXT_CHECK_ENABLE_DISALLOW_TOO_LONG_IB_PACKETS_SMASK
2283 | SEND_CTXT_CHECK_ENABLE_DISALLOW_BAD_PKT_LEN_SMASK
2284#ifndef CONFIG_FAULT_INJECTION
2285 | SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_TEST_SMASK
2286#endif
2287 | SEND_CTXT_CHECK_ENABLE_DISALLOW_TOO_SMALL_BYPASS_PACKETS_SMASK
2288 | SEND_CTXT_CHECK_ENABLE_DISALLOW_TOO_SMALL_IB_PACKETS_SMASK
2289 | SEND_CTXT_CHECK_ENABLE_DISALLOW_RAW_IPV6_SMASK
2290 | SEND_CTXT_CHECK_ENABLE_DISALLOW_RAW_SMASK
2291 | SEND_CTXT_CHECK_ENABLE_CHECK_BYPASS_VL_MAPPING_SMASK
2292 | SEND_CTXT_CHECK_ENABLE_CHECK_VL_MAPPING_SMASK
2293 | SEND_CTXT_CHECK_ENABLE_CHECK_OPCODE_SMASK
2294 | SEND_CTXT_CHECK_ENABLE_CHECK_SLID_SMASK
2295 | SEND_CTXT_CHECK_ENABLE_CHECK_VL_SMASK
2296 | SEND_CTXT_CHECK_ENABLE_CHECK_ENABLE_SMASK;
2297
2298 if (ctxt_type == SC_USER)
2299 base_sc_integrity |=
2300#ifndef CONFIG_FAULT_INJECTION
2301 SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_TEST_SMASK |
2302#endif
2303 HFI1_PKT_USER_SC_INTEGRITY;
2304 else if (ctxt_type != SC_KERNEL)
2305 base_sc_integrity |= HFI1_PKT_KERNEL_SC_INTEGRITY;
2306
2307
2308 if (!is_ax(dd))
2309 base_sc_integrity |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
2310
2311 return base_sc_integrity;
2312}
2313
2314static inline u64 hfi1_pkt_base_sdma_integrity(struct hfi1_devdata *dd)
2315{
2316 u64 base_sdma_integrity;
2317
2318
2319 if (HFI1_CAP_IS_KSET(NO_INTEGRITY))
2320 return 0;
2321
2322 base_sdma_integrity =
2323 SEND_DMA_CHECK_ENABLE_DISALLOW_BYPASS_BAD_PKT_LEN_SMASK
2324 | SEND_DMA_CHECK_ENABLE_DISALLOW_TOO_LONG_BYPASS_PACKETS_SMASK
2325 | SEND_DMA_CHECK_ENABLE_DISALLOW_TOO_LONG_IB_PACKETS_SMASK
2326 | SEND_DMA_CHECK_ENABLE_DISALLOW_BAD_PKT_LEN_SMASK
2327 | SEND_DMA_CHECK_ENABLE_DISALLOW_TOO_SMALL_BYPASS_PACKETS_SMASK
2328 | SEND_DMA_CHECK_ENABLE_DISALLOW_TOO_SMALL_IB_PACKETS_SMASK
2329 | SEND_DMA_CHECK_ENABLE_DISALLOW_RAW_IPV6_SMASK
2330 | SEND_DMA_CHECK_ENABLE_DISALLOW_RAW_SMASK
2331 | SEND_DMA_CHECK_ENABLE_CHECK_BYPASS_VL_MAPPING_SMASK
2332 | SEND_DMA_CHECK_ENABLE_CHECK_VL_MAPPING_SMASK
2333 | SEND_DMA_CHECK_ENABLE_CHECK_OPCODE_SMASK
2334 | SEND_DMA_CHECK_ENABLE_CHECK_SLID_SMASK
2335 | SEND_DMA_CHECK_ENABLE_CHECK_VL_SMASK
2336 | SEND_DMA_CHECK_ENABLE_CHECK_ENABLE_SMASK;
2337
2338 if (!HFI1_CAP_IS_KSET(STATIC_RATE_CTRL))
2339 base_sdma_integrity |=
2340 SEND_DMA_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK;
2341
2342
2343 if (!is_ax(dd))
2344 base_sdma_integrity |=
2345 SEND_DMA_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
2346
2347 return base_sdma_integrity;
2348}
2349
2350#define dd_dev_emerg(dd, fmt, ...) \
2351 dev_emerg(&(dd)->pcidev->dev, "%s: " fmt, \
2352 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
2353
2354#define dd_dev_err(dd, fmt, ...) \
2355 dev_err(&(dd)->pcidev->dev, "%s: " fmt, \
2356 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
2357
2358#define dd_dev_err_ratelimited(dd, fmt, ...) \
2359 dev_err_ratelimited(&(dd)->pcidev->dev, "%s: " fmt, \
2360 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), \
2361 ##__VA_ARGS__)
2362
2363#define dd_dev_warn(dd, fmt, ...) \
2364 dev_warn(&(dd)->pcidev->dev, "%s: " fmt, \
2365 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
2366
2367#define dd_dev_warn_ratelimited(dd, fmt, ...) \
2368 dev_warn_ratelimited(&(dd)->pcidev->dev, "%s: " fmt, \
2369 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), \
2370 ##__VA_ARGS__)
2371
2372#define dd_dev_info(dd, fmt, ...) \
2373 dev_info(&(dd)->pcidev->dev, "%s: " fmt, \
2374 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
2375
2376#define dd_dev_info_ratelimited(dd, fmt, ...) \
2377 dev_info_ratelimited(&(dd)->pcidev->dev, "%s: " fmt, \
2378 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), \
2379 ##__VA_ARGS__)
2380
2381#define dd_dev_dbg(dd, fmt, ...) \
2382 dev_dbg(&(dd)->pcidev->dev, "%s: " fmt, \
2383 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
2384
2385#define hfi1_dev_porterr(dd, port, fmt, ...) \
2386 dev_err(&(dd)->pcidev->dev, "%s: port %u: " fmt, \
2387 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), (port), ##__VA_ARGS__)
2388
2389
2390
2391
2392struct hfi1_hwerror_msgs {
2393 u64 mask;
2394 const char *msg;
2395 size_t sz;
2396};
2397
2398
2399void hfi1_format_hwerrors(u64 hwerrs,
2400 const struct hfi1_hwerror_msgs *hwerrmsgs,
2401 size_t nhwerrmsgs, char *msg, size_t lmsg);
2402
2403#define USER_OPCODE_CHECK_VAL 0xC0
2404#define USER_OPCODE_CHECK_MASK 0xC0
2405#define OPCODE_CHECK_VAL_DISABLED 0x0
2406#define OPCODE_CHECK_MASK_DISABLED 0x0
2407
2408static inline void hfi1_reset_cpu_counters(struct hfi1_devdata *dd)
2409{
2410 struct hfi1_pportdata *ppd;
2411 int i;
2412
2413 dd->z_int_counter = get_all_cpu_total(dd->int_counter);
2414 dd->z_rcv_limit = get_all_cpu_total(dd->rcv_limit);
2415 dd->z_send_schedule = get_all_cpu_total(dd->send_schedule);
2416
2417 ppd = (struct hfi1_pportdata *)(dd + 1);
2418 for (i = 0; i < dd->num_pports; i++, ppd++) {
2419 ppd->ibport_data.rvp.z_rc_acks =
2420 get_all_cpu_total(ppd->ibport_data.rvp.rc_acks);
2421 ppd->ibport_data.rvp.z_rc_qacks =
2422 get_all_cpu_total(ppd->ibport_data.rvp.rc_qacks);
2423 }
2424}
2425
2426
2427static inline void setextled(struct hfi1_devdata *dd, u32 on)
2428{
2429 if (on)
2430 write_csr(dd, DCC_CFG_LED_CNTRL, 0x1F);
2431 else
2432 write_csr(dd, DCC_CFG_LED_CNTRL, 0x10);
2433}
2434
2435
2436static inline u32 i2c_target(u32 target)
2437{
2438 return target ? CR_I2C2 : CR_I2C1;
2439}
2440
2441
2442static inline u32 qsfp_resource(struct hfi1_devdata *dd)
2443{
2444 return i2c_target(dd->hfi1_id);
2445}
2446
2447
2448static inline bool is_integrated(struct hfi1_devdata *dd)
2449{
2450 return dd->pcidev->device == PCI_DEVICE_ID_INTEL1;
2451}
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461static inline bool hfi1_need_drop(struct hfi1_devdata *dd)
2462{
2463 if (unlikely(dd->do_drop &&
2464 atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) ==
2465 DROP_PACKET_ON)) {
2466 dd->do_drop = false;
2467 return true;
2468 }
2469 return false;
2470}
2471
2472int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp);
2473
2474#define DD_DEV_ENTRY(dd) __string(dev, dev_name(&(dd)->pcidev->dev))
2475#define DD_DEV_ASSIGN(dd) __assign_str(dev, dev_name(&(dd)->pcidev->dev))
2476
2477static inline void hfi1_update_ah_attr(struct ib_device *ibdev,
2478 struct rdma_ah_attr *attr)
2479{
2480 struct hfi1_pportdata *ppd;
2481 struct hfi1_ibport *ibp;
2482 u32 dlid = rdma_ah_get_dlid(attr);
2483
2484
2485
2486
2487
2488 ibp = to_iport(ibdev, rdma_ah_get_port_num(attr));
2489 ppd = ppd_from_ibp(ibp);
2490 if ((((dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) ||
2491 (ppd->lid >= be16_to_cpu(IB_MULTICAST_LID_BASE))) &&
2492 (dlid != be32_to_cpu(OPA_LID_PERMISSIVE)) &&
2493 (dlid != be16_to_cpu(IB_LID_PERMISSIVE)) &&
2494 (!(rdma_ah_get_ah_flags(attr) & IB_AH_GRH))) ||
2495 (rdma_ah_get_make_grd(attr))) {
2496 rdma_ah_set_ah_flags(attr, IB_AH_GRH);
2497 rdma_ah_set_interface_id(attr, OPA_MAKE_ID(dlid));
2498 rdma_ah_set_subnet_prefix(attr, ibp->rvp.gid_prefix);
2499 }
2500}
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510static inline bool hfi1_check_mcast(u32 lid)
2511{
2512 return ((lid >= opa_get_mcast_base(OPA_MCAST_NR)) &&
2513 (lid != be32_to_cpu(OPA_LID_PERMISSIVE)));
2514}
2515
2516#define opa_get_lid(lid, format) \
2517 __opa_get_lid(lid, OPA_PORT_PACKET_FORMAT_##format)
2518
2519
2520static inline u32 __opa_get_lid(u32 lid, u8 format)
2521{
2522 bool is_mcast = hfi1_check_mcast(lid);
2523
2524 switch (format) {
2525 case OPA_PORT_PACKET_FORMAT_8B:
2526 case OPA_PORT_PACKET_FORMAT_10B:
2527 if (is_mcast)
2528 return (lid - opa_get_mcast_base(OPA_MCAST_NR) +
2529 0xF0000);
2530 return lid & 0xFFFFF;
2531 case OPA_PORT_PACKET_FORMAT_16B:
2532 if (is_mcast)
2533 return (lid - opa_get_mcast_base(OPA_MCAST_NR) +
2534 0xF00000);
2535 return lid & 0xFFFFFF;
2536 case OPA_PORT_PACKET_FORMAT_9B:
2537 if (is_mcast)
2538 return (lid -
2539 opa_get_mcast_base(OPA_MCAST_NR) +
2540 be16_to_cpu(IB_MULTICAST_LID_BASE));
2541 else
2542 return lid & 0xFFFF;
2543 default:
2544 return lid;
2545 }
2546}
2547
2548
2549static inline bool hfi1_is_16B_mcast(u32 lid)
2550{
2551 return ((lid >=
2552 opa_get_lid(opa_get_mcast_base(OPA_MCAST_NR), 16B)) &&
2553 (lid != opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B)));
2554}
2555
2556static inline void hfi1_make_opa_lid(struct rdma_ah_attr *attr)
2557{
2558 const struct ib_global_route *grh = rdma_ah_read_grh(attr);
2559 u32 dlid = rdma_ah_get_dlid(attr);
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569 if (ib_is_opa_gid(&grh->dgid))
2570 dlid = opa_get_lid_from_gid(&grh->dgid);
2571 else if ((dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
2572 (dlid != be16_to_cpu(IB_LID_PERMISSIVE)) &&
2573 (dlid != be32_to_cpu(OPA_LID_PERMISSIVE)))
2574 dlid = dlid - be16_to_cpu(IB_MULTICAST_LID_BASE) +
2575 opa_get_mcast_base(OPA_MCAST_NR);
2576 else if (dlid == be16_to_cpu(IB_LID_PERMISSIVE))
2577 dlid = be32_to_cpu(OPA_LID_PERMISSIVE);
2578
2579 rdma_ah_set_dlid(attr, dlid);
2580}
2581
2582static inline u8 hfi1_get_packet_type(u32 lid)
2583{
2584
2585 if (lid >= opa_get_mcast_base(OPA_MCAST_NR))
2586 return HFI1_PKT_TYPE_9B;
2587
2588
2589 if (lid >= opa_get_lid(opa_get_mcast_base(OPA_MCAST_NR), 9B))
2590 return HFI1_PKT_TYPE_16B;
2591
2592 return HFI1_PKT_TYPE_9B;
2593}
2594
2595static inline bool hfi1_get_hdr_type(u32 lid, struct rdma_ah_attr *attr)
2596{
2597
2598
2599
2600
2601
2602
2603
2604 if (rdma_ah_get_dlid(attr) == be32_to_cpu(OPA_LID_PERMISSIVE))
2605 return (ib_is_opa_gid(&rdma_ah_read_grh(attr)->dgid)) ?
2606 HFI1_PKT_TYPE_16B : HFI1_PKT_TYPE_9B;
2607
2608
2609
2610
2611
2612 if (hfi1_get_packet_type(rdma_ah_get_dlid(attr)) == HFI1_PKT_TYPE_16B)
2613 return HFI1_PKT_TYPE_16B;
2614
2615 return hfi1_get_packet_type(lid);
2616}
2617
2618static inline void hfi1_make_ext_grh(struct hfi1_packet *packet,
2619 struct ib_grh *grh, u32 slid,
2620 u32 dlid)
2621{
2622 struct hfi1_ibport *ibp = &packet->rcd->ppd->ibport_data;
2623 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
2624
2625 if (!ibp)
2626 return;
2627
2628 grh->hop_limit = 1;
2629 grh->sgid.global.subnet_prefix = ibp->rvp.gid_prefix;
2630 if (slid == opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B))
2631 grh->sgid.global.interface_id =
2632 OPA_MAKE_ID(be32_to_cpu(OPA_LID_PERMISSIVE));
2633 else
2634 grh->sgid.global.interface_id = OPA_MAKE_ID(slid);
2635
2636
2637
2638
2639
2640
2641
2642
2643 grh->dgid.global.subnet_prefix = ibp->rvp.gid_prefix;
2644 grh->dgid.global.interface_id =
2645 cpu_to_be64(ppd->guids[HFI1_PORT_GUID_INDEX]);
2646}
2647
2648static inline int hfi1_get_16b_padding(u32 hdr_size, u32 payload)
2649{
2650 return -(hdr_size + payload + (SIZE_OF_CRC << 2) +
2651 SIZE_OF_LT) & 0x7;
2652}
2653
2654static inline void hfi1_make_ib_hdr(struct ib_header *hdr,
2655 u16 lrh0, u16 len,
2656 u16 dlid, u16 slid)
2657{
2658 hdr->lrh[0] = cpu_to_be16(lrh0);
2659 hdr->lrh[1] = cpu_to_be16(dlid);
2660 hdr->lrh[2] = cpu_to_be16(len);
2661 hdr->lrh[3] = cpu_to_be16(slid);
2662}
2663
2664static inline void hfi1_make_16b_hdr(struct hfi1_16b_header *hdr,
2665 u32 slid, u32 dlid,
2666 u16 len, u16 pkey,
2667 bool becn, bool fecn, u8 l4,
2668 u8 sc)
2669{
2670 u32 lrh0 = 0;
2671 u32 lrh1 = 0x40000000;
2672 u32 lrh2 = 0;
2673 u32 lrh3 = 0;
2674
2675 lrh0 = (lrh0 & ~OPA_16B_BECN_MASK) | (becn << OPA_16B_BECN_SHIFT);
2676 lrh0 = (lrh0 & ~OPA_16B_LEN_MASK) | (len << OPA_16B_LEN_SHIFT);
2677 lrh0 = (lrh0 & ~OPA_16B_LID_MASK) | (slid & OPA_16B_LID_MASK);
2678 lrh1 = (lrh1 & ~OPA_16B_FECN_MASK) | (fecn << OPA_16B_FECN_SHIFT);
2679 lrh1 = (lrh1 & ~OPA_16B_SC_MASK) | (sc << OPA_16B_SC_SHIFT);
2680 lrh1 = (lrh1 & ~OPA_16B_LID_MASK) | (dlid & OPA_16B_LID_MASK);
2681 lrh2 = (lrh2 & ~OPA_16B_SLID_MASK) |
2682 ((slid >> OPA_16B_SLID_SHIFT) << OPA_16B_SLID_HIGH_SHIFT);
2683 lrh2 = (lrh2 & ~OPA_16B_DLID_MASK) |
2684 ((dlid >> OPA_16B_DLID_SHIFT) << OPA_16B_DLID_HIGH_SHIFT);
2685 lrh2 = (lrh2 & ~OPA_16B_PKEY_MASK) | ((u32)pkey << OPA_16B_PKEY_SHIFT);
2686 lrh2 = (lrh2 & ~OPA_16B_L4_MASK) | l4;
2687
2688 hdr->lrh[0] = lrh0;
2689 hdr->lrh[1] = lrh1;
2690 hdr->lrh[2] = lrh2;
2691 hdr->lrh[3] = lrh3;
2692}
2693#endif
2694