1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52#include <linux/pci.h>
53#include <linux/delay.h>
54#include <linux/interrupt.h>
55#include <linux/module.h>
56
57#include "hfi.h"
58#include "trace.h"
59#include "mad.h"
60#include "pio.h"
61#include "sdma.h"
62#include "eprom.h"
63#include "efivar.h"
64#include "platform.h"
65#include "aspm.h"
66#include "affinity.h"
67#include "debugfs.h"
68#include "fault.h"
69
70uint kdeth_qp;
71module_param_named(kdeth_qp, kdeth_qp, uint, S_IRUGO);
72MODULE_PARM_DESC(kdeth_qp, "Set the KDETH queue pair prefix");
73
74uint num_vls = HFI1_MAX_VLS_SUPPORTED;
75module_param(num_vls, uint, S_IRUGO);
76MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)");
77
78
79
80
81
82
83
84
85uint rcv_intr_timeout = (824 + 16);
86module_param(rcv_intr_timeout, uint, S_IRUGO);
87MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns");
88
89uint rcv_intr_count = 16;
90module_param(rcv_intr_count, uint, S_IRUGO);
91MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count");
92
93ushort link_crc_mask = SUPPORTED_CRCS;
94module_param(link_crc_mask, ushort, S_IRUGO);
95MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link");
96
97uint loopback;
98module_param_named(loopback, loopback, uint, S_IRUGO);
99MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable");
100
101
102uint rcv_intr_dynamic = 1;
103static ushort crc_14b_sideband = 1;
104static uint use_flr = 1;
105uint quick_linkup;
106
107struct flag_table {
108 u64 flag;
109 char *str;
110 u16 extra;
111 u16 unused0;
112 u32 unused1;
113};
114
115
116#define FLAG_ENTRY(str, extra, flag) {flag, str, extra}
117#define FLAG_ENTRY0(str, flag) {flag, str, 0}
118
119
120#define SEC_WRITE_DROPPED 0x1
121#define SEC_PACKET_DROPPED 0x2
122#define SEC_SC_HALTED 0x4
123#define SEC_SPC_FREEZE 0x8
124
125#define DEFAULT_KRCVQS 2
126#define MIN_KERNEL_KCTXTS 2
127#define FIRST_KERNEL_KCTXT 1
128
129
130
131
132
133
134
135#define RSM_INS_VERBS 0
136#define RSM_INS_FECN 1
137#define RSM_INS_VNIC 2
138
139
140#define GUID_HFI_INDEX_SHIFT 39
141
142
143#define emulator_rev(dd) ((dd)->irev >> 8)
144
145#define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3)
146#define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4)
147
148
149
150#define IB_PACKET_TYPE 2ull
151#define QW_SHIFT 6ull
152
153#define QPN_WIDTH 7ull
154
155
156#define LRH_BTH_QW 0ull
157#define LRH_BTH_BIT_OFFSET 48ull
158#define LRH_BTH_OFFSET(off) ((LRH_BTH_QW << QW_SHIFT) | (off))
159#define LRH_BTH_MATCH_OFFSET LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET)
160#define LRH_BTH_SELECT
161#define LRH_BTH_MASK 3ull
162#define LRH_BTH_VALUE 2ull
163
164
165#define LRH_SC_QW 0ull
166#define LRH_SC_BIT_OFFSET 56ull
167#define LRH_SC_OFFSET(off) ((LRH_SC_QW << QW_SHIFT) | (off))
168#define LRH_SC_MATCH_OFFSET LRH_SC_OFFSET(LRH_SC_BIT_OFFSET)
169#define LRH_SC_MASK 128ull
170#define LRH_SC_VALUE 0ull
171
172
173#define LRH_SC_SELECT_OFFSET ((LRH_SC_QW << QW_SHIFT) | (60ull))
174
175
176#define QPN_SELECT_OFFSET ((1ull << QW_SHIFT) | (1ull))
177
178
179
180#define L2_TYPE_QW 0ull
181#define L2_TYPE_BIT_OFFSET 61ull
182#define L2_TYPE_OFFSET(off) ((L2_TYPE_QW << QW_SHIFT) | (off))
183#define L2_TYPE_MATCH_OFFSET L2_TYPE_OFFSET(L2_TYPE_BIT_OFFSET)
184#define L2_TYPE_MASK 3ull
185#define L2_16B_VALUE 2ull
186
187
188#define L4_TYPE_QW 1ull
189#define L4_TYPE_BIT_OFFSET 0ull
190#define L4_TYPE_OFFSET(off) ((L4_TYPE_QW << QW_SHIFT) | (off))
191#define L4_TYPE_MATCH_OFFSET L4_TYPE_OFFSET(L4_TYPE_BIT_OFFSET)
192#define L4_16B_TYPE_MASK 0xFFull
193#define L4_16B_ETH_VALUE 0x78ull
194
195
196#define L4_16B_HDR_VESWID_OFFSET ((2 << QW_SHIFT) | (16ull))
197
198#define L2_16B_ENTROPY_OFFSET ((1 << QW_SHIFT) | (32ull))
199
200
201#define SC2VL_VAL( \
202 num, \
203 sc0, sc0val, \
204 sc1, sc1val, \
205 sc2, sc2val, \
206 sc3, sc3val, \
207 sc4, sc4val, \
208 sc5, sc5val, \
209 sc6, sc6val, \
210 sc7, sc7val) \
211( \
212 ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \
213 ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \
214 ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \
215 ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \
216 ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \
217 ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \
218 ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \
219 ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT) \
220)
221
222#define DC_SC_VL_VAL( \
223 range, \
224 e0, e0val, \
225 e1, e1val, \
226 e2, e2val, \
227 e3, e3val, \
228 e4, e4val, \
229 e5, e5val, \
230 e6, e6val, \
231 e7, e7val, \
232 e8, e8val, \
233 e9, e9val, \
234 e10, e10val, \
235 e11, e11val, \
236 e12, e12val, \
237 e13, e13val, \
238 e14, e14val, \
239 e15, e15val) \
240( \
241 ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \
242 ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \
243 ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \
244 ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \
245 ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \
246 ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \
247 ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \
248 ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \
249 ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \
250 ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \
251 ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \
252 ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \
253 ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \
254 ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \
255 ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \
256 ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \
257)
258
259
260#define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \
261 | CCE_STATUS_RXE_FROZE_SMASK \
262 | CCE_STATUS_TXE_FROZE_SMASK \
263 | CCE_STATUS_TXE_PIO_FROZE_SMASK)
264
265#define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \
266 | CCE_STATUS_TXE_PAUSED_SMASK \
267 | CCE_STATUS_SDMA_PAUSED_SMASK)
268
269#define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK
270
271#define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
272#define CNTR_32BIT_MAX 0x00000000FFFFFFFF
273
274
275
276
277static struct flag_table cce_err_status_flags[] = {
278 FLAG_ENTRY0("CceCsrParityErr",
279 CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK),
280 FLAG_ENTRY0("CceCsrReadBadAddrErr",
281 CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK),
282 FLAG_ENTRY0("CceCsrWriteBadAddrErr",
283 CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK),
284 FLAG_ENTRY0("CceTrgtAsyncFifoParityErr",
285 CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK),
286 FLAG_ENTRY0("CceTrgtAccessErr",
287 CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK),
288 FLAG_ENTRY0("CceRspdDataParityErr",
289 CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK),
290 FLAG_ENTRY0("CceCli0AsyncFifoParityErr",
291 CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK),
292 FLAG_ENTRY0("CceCsrCfgBusParityErr",
293 CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK),
294 FLAG_ENTRY0("CceCli2AsyncFifoParityErr",
295 CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK),
296 FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
297 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK),
298 FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
299 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK),
300 FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError",
301 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK),
302 FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError",
303 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK),
304 FLAG_ENTRY0("PcicRetryMemCorErr",
305 CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK),
306 FLAG_ENTRY0("PcicRetryMemCorErr",
307 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK),
308 FLAG_ENTRY0("PcicPostHdQCorErr",
309 CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK),
310 FLAG_ENTRY0("PcicPostHdQCorErr",
311 CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK),
312 FLAG_ENTRY0("PcicPostHdQCorErr",
313 CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK),
314 FLAG_ENTRY0("PcicCplDatQCorErr",
315 CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK),
316 FLAG_ENTRY0("PcicNPostHQParityErr",
317 CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK),
318 FLAG_ENTRY0("PcicNPostDatQParityErr",
319 CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK),
320 FLAG_ENTRY0("PcicRetryMemUncErr",
321 CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK),
322 FLAG_ENTRY0("PcicRetrySotMemUncErr",
323 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK),
324 FLAG_ENTRY0("PcicPostHdQUncErr",
325 CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK),
326 FLAG_ENTRY0("PcicPostDatQUncErr",
327 CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK),
328 FLAG_ENTRY0("PcicCplHdQUncErr",
329 CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK),
330 FLAG_ENTRY0("PcicCplDatQUncErr",
331 CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK),
332 FLAG_ENTRY0("PcicTransmitFrontParityErr",
333 CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK),
334 FLAG_ENTRY0("PcicTransmitBackParityErr",
335 CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK),
336 FLAG_ENTRY0("PcicReceiveParityErr",
337 CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK),
338 FLAG_ENTRY0("CceTrgtCplTimeoutErr",
339 CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK),
340 FLAG_ENTRY0("LATriggered",
341 CCE_ERR_STATUS_LA_TRIGGERED_SMASK),
342 FLAG_ENTRY0("CceSegReadBadAddrErr",
343 CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK),
344 FLAG_ENTRY0("CceSegWriteBadAddrErr",
345 CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK),
346 FLAG_ENTRY0("CceRcplAsyncFifoParityErr",
347 CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK),
348 FLAG_ENTRY0("CceRxdmaConvFifoParityErr",
349 CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK),
350 FLAG_ENTRY0("CceMsixTableCorErr",
351 CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK),
352 FLAG_ENTRY0("CceMsixTableUncErr",
353 CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK),
354 FLAG_ENTRY0("CceIntMapCorErr",
355 CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK),
356 FLAG_ENTRY0("CceIntMapUncErr",
357 CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK),
358 FLAG_ENTRY0("CceMsixCsrParityErr",
359 CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK),
360
361};
362
363
364
365
366#define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK
367static struct flag_table misc_err_status_flags[] = {
368 FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)),
369 FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)),
370 FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)),
371 FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)),
372 FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)),
373 FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)),
374 FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)),
375 FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)),
376 FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)),
377 FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)),
378 FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)),
379 FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)),
380 FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL))
381};
382
383
384
385
386static struct flag_table pio_err_status_flags[] = {
387 FLAG_ENTRY("PioWriteBadCtxt",
388 SEC_WRITE_DROPPED,
389 SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK),
390 FLAG_ENTRY("PioWriteAddrParity",
391 SEC_SPC_FREEZE,
392 SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK),
393 FLAG_ENTRY("PioCsrParity",
394 SEC_SPC_FREEZE,
395 SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK),
396 FLAG_ENTRY("PioSbMemFifo0",
397 SEC_SPC_FREEZE,
398 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK),
399 FLAG_ENTRY("PioSbMemFifo1",
400 SEC_SPC_FREEZE,
401 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK),
402 FLAG_ENTRY("PioPccFifoParity",
403 SEC_SPC_FREEZE,
404 SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK),
405 FLAG_ENTRY("PioPecFifoParity",
406 SEC_SPC_FREEZE,
407 SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK),
408 FLAG_ENTRY("PioSbrdctlCrrelParity",
409 SEC_SPC_FREEZE,
410 SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK),
411 FLAG_ENTRY("PioSbrdctrlCrrelFifoParity",
412 SEC_SPC_FREEZE,
413 SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK),
414 FLAG_ENTRY("PioPktEvictFifoParityErr",
415 SEC_SPC_FREEZE,
416 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK),
417 FLAG_ENTRY("PioSmPktResetParity",
418 SEC_SPC_FREEZE,
419 SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK),
420 FLAG_ENTRY("PioVlLenMemBank0Unc",
421 SEC_SPC_FREEZE,
422 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK),
423 FLAG_ENTRY("PioVlLenMemBank1Unc",
424 SEC_SPC_FREEZE,
425 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK),
426 FLAG_ENTRY("PioVlLenMemBank0Cor",
427 0,
428 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK),
429 FLAG_ENTRY("PioVlLenMemBank1Cor",
430 0,
431 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK),
432 FLAG_ENTRY("PioCreditRetFifoParity",
433 SEC_SPC_FREEZE,
434 SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK),
435 FLAG_ENTRY("PioPpmcPblFifo",
436 SEC_SPC_FREEZE,
437 SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK),
438 FLAG_ENTRY("PioInitSmIn",
439 0,
440 SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK),
441 FLAG_ENTRY("PioPktEvictSmOrArbSm",
442 SEC_SPC_FREEZE,
443 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK),
444 FLAG_ENTRY("PioHostAddrMemUnc",
445 SEC_SPC_FREEZE,
446 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK),
447 FLAG_ENTRY("PioHostAddrMemCor",
448 0,
449 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK),
450 FLAG_ENTRY("PioWriteDataParity",
451 SEC_SPC_FREEZE,
452 SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK),
453 FLAG_ENTRY("PioStateMachine",
454 SEC_SPC_FREEZE,
455 SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK),
456 FLAG_ENTRY("PioWriteQwValidParity",
457 SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
458 SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK),
459 FLAG_ENTRY("PioBlockQwCountParity",
460 SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
461 SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK),
462 FLAG_ENTRY("PioVlfVlLenParity",
463 SEC_SPC_FREEZE,
464 SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK),
465 FLAG_ENTRY("PioVlfSopParity",
466 SEC_SPC_FREEZE,
467 SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK),
468 FLAG_ENTRY("PioVlFifoParity",
469 SEC_SPC_FREEZE,
470 SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK),
471 FLAG_ENTRY("PioPpmcBqcMemParity",
472 SEC_SPC_FREEZE,
473 SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK),
474 FLAG_ENTRY("PioPpmcSopLen",
475 SEC_SPC_FREEZE,
476 SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK),
477
478 FLAG_ENTRY("PioCurrentFreeCntParity",
479 SEC_SPC_FREEZE,
480 SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK),
481 FLAG_ENTRY("PioLastReturnedCntParity",
482 SEC_SPC_FREEZE,
483 SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK),
484 FLAG_ENTRY("PioPccSopHeadParity",
485 SEC_SPC_FREEZE,
486 SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK),
487 FLAG_ENTRY("PioPecSopHeadParityErr",
488 SEC_SPC_FREEZE,
489 SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK),
490
491};
492
493
494#define ALL_PIO_FREEZE_ERR \
495 (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \
496 | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \
497 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \
498 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \
499 | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \
500 | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \
501 | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \
502 | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \
503 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \
504 | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \
505 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \
506 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \
507 | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \
508 | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \
509 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \
510 | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \
511 | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \
512 | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \
513 | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \
514 | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \
515 | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \
516 | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \
517 | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \
518 | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \
519 | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \
520 | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \
521 | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \
522 | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \
523 | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK)
524
525
526
527
528static struct flag_table sdma_err_status_flags[] = {
529 FLAG_ENTRY0("SDmaRpyTagErr",
530 SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK),
531 FLAG_ENTRY0("SDmaCsrParityErr",
532 SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK),
533 FLAG_ENTRY0("SDmaPcieReqTrackingUncErr",
534 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK),
535 FLAG_ENTRY0("SDmaPcieReqTrackingCorErr",
536 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK),
537
538};
539
540
541#define ALL_SDMA_FREEZE_ERR \
542 (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \
543 | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \
544 | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK)
545
546
547#define PORT_DISCARD_EGRESS_ERRS \
548 (SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK \
549 | SEND_EGRESS_ERR_INFO_VL_MAPPING_ERR_SMASK \
550 | SEND_EGRESS_ERR_INFO_VL_ERR_SMASK)
551
552
553
554
555#define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK
556static struct flag_table egress_err_status_flags[] = {
557 FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)),
558 FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)),
559
560 FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr",
561 SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)),
562 FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)),
563 FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)),
564
565 FLAG_ENTRY0("TxPioLaunchIntfParityErr",
566 SEES(TX_PIO_LAUNCH_INTF_PARITY)),
567 FLAG_ENTRY0("TxSdmaLaunchIntfParityErr",
568 SEES(TX_SDMA_LAUNCH_INTF_PARITY)),
569
570 FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr",
571 SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)),
572 FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)),
573 FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)),
574 FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)),
575 FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)),
576 FLAG_ENTRY0("TxSdma0DisallowedPacketErr",
577 SEES(TX_SDMA0_DISALLOWED_PACKET)),
578 FLAG_ENTRY0("TxSdma1DisallowedPacketErr",
579 SEES(TX_SDMA1_DISALLOWED_PACKET)),
580 FLAG_ENTRY0("TxSdma2DisallowedPacketErr",
581 SEES(TX_SDMA2_DISALLOWED_PACKET)),
582 FLAG_ENTRY0("TxSdma3DisallowedPacketErr",
583 SEES(TX_SDMA3_DISALLOWED_PACKET)),
584 FLAG_ENTRY0("TxSdma4DisallowedPacketErr",
585 SEES(TX_SDMA4_DISALLOWED_PACKET)),
586 FLAG_ENTRY0("TxSdma5DisallowedPacketErr",
587 SEES(TX_SDMA5_DISALLOWED_PACKET)),
588 FLAG_ENTRY0("TxSdma6DisallowedPacketErr",
589 SEES(TX_SDMA6_DISALLOWED_PACKET)),
590 FLAG_ENTRY0("TxSdma7DisallowedPacketErr",
591 SEES(TX_SDMA7_DISALLOWED_PACKET)),
592 FLAG_ENTRY0("TxSdma8DisallowedPacketErr",
593 SEES(TX_SDMA8_DISALLOWED_PACKET)),
594 FLAG_ENTRY0("TxSdma9DisallowedPacketErr",
595 SEES(TX_SDMA9_DISALLOWED_PACKET)),
596 FLAG_ENTRY0("TxSdma10DisallowedPacketErr",
597 SEES(TX_SDMA10_DISALLOWED_PACKET)),
598 FLAG_ENTRY0("TxSdma11DisallowedPacketErr",
599 SEES(TX_SDMA11_DISALLOWED_PACKET)),
600 FLAG_ENTRY0("TxSdma12DisallowedPacketErr",
601 SEES(TX_SDMA12_DISALLOWED_PACKET)),
602 FLAG_ENTRY0("TxSdma13DisallowedPacketErr",
603 SEES(TX_SDMA13_DISALLOWED_PACKET)),
604 FLAG_ENTRY0("TxSdma14DisallowedPacketErr",
605 SEES(TX_SDMA14_DISALLOWED_PACKET)),
606 FLAG_ENTRY0("TxSdma15DisallowedPacketErr",
607 SEES(TX_SDMA15_DISALLOWED_PACKET)),
608 FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr",
609 SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)),
610 FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr",
611 SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)),
612 FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr",
613 SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)),
614 FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr",
615 SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)),
616 FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr",
617 SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)),
618 FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr",
619 SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)),
620 FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr",
621 SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)),
622 FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr",
623 SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)),
624 FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr",
625 SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)),
626 FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)),
627 FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)),
628 FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)),
629 FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)),
630 FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)),
631 FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)),
632 FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)),
633 FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)),
634 FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)),
635 FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)),
636 FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)),
637 FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)),
638 FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)),
639 FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)),
640 FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)),
641 FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)),
642 FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)),
643 FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)),
644 FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)),
645 FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)),
646 FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)),
647 FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr",
648 SEES(TX_READ_SDMA_MEMORY_CSR_UNC)),
649 FLAG_ENTRY0("TxReadPioMemoryCsrUncErr",
650 SEES(TX_READ_PIO_MEMORY_CSR_UNC)),
651};
652
653
654
655
656#define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK
657static struct flag_table egress_err_info_flags[] = {
658 FLAG_ENTRY0("Reserved", 0ull),
659 FLAG_ENTRY0("VLErr", SEEI(VL)),
660 FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
661 FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
662 FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)),
663 FLAG_ENTRY0("SLIDErr", SEEI(SLID)),
664 FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)),
665 FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)),
666 FLAG_ENTRY0("RawErr", SEEI(RAW)),
667 FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)),
668 FLAG_ENTRY0("GRHErr", SEEI(GRH)),
669 FLAG_ENTRY0("BypassErr", SEEI(BYPASS)),
670 FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)),
671 FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)),
672 FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)),
673 FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)),
674 FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)),
675 FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)),
676 FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)),
677 FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)),
678 FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)),
679 FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)),
680};
681
682
683#define ALL_TXE_EGRESS_FREEZE_ERR \
684 (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \
685 | SEES(TX_PIO_LAUNCH_INTF_PARITY) \
686 | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \
687 | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \
688 | SEES(TX_LAUNCH_CSR_PARITY) \
689 | SEES(TX_SBRD_CTL_CSR_PARITY) \
690 | SEES(TX_CONFIG_PARITY) \
691 | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \
692 | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \
693 | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \
694 | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \
695 | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \
696 | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \
697 | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \
698 | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \
699 | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \
700 | SEES(TX_CREDIT_RETURN_PARITY))
701
702
703
704
705#define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK
706static struct flag_table send_err_status_flags[] = {
707 FLAG_ENTRY0("SendCsrParityErr", SES(CSR_PARITY)),
708 FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)),
709 FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR))
710};
711
712
713
714
715static struct flag_table sc_err_status_flags[] = {
716 FLAG_ENTRY("InconsistentSop",
717 SEC_PACKET_DROPPED | SEC_SC_HALTED,
718 SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK),
719 FLAG_ENTRY("DisallowedPacket",
720 SEC_PACKET_DROPPED | SEC_SC_HALTED,
721 SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK),
722 FLAG_ENTRY("WriteCrossesBoundary",
723 SEC_WRITE_DROPPED | SEC_SC_HALTED,
724 SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK),
725 FLAG_ENTRY("WriteOverflow",
726 SEC_WRITE_DROPPED | SEC_SC_HALTED,
727 SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK),
728 FLAG_ENTRY("WriteOutOfBounds",
729 SEC_WRITE_DROPPED | SEC_SC_HALTED,
730 SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK),
731
732};
733
734
735
736
737#define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK
738static struct flag_table rxe_err_status_flags[] = {
739 FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)),
740 FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)),
741 FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)),
742 FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)),
743 FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)),
744 FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)),
745 FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)),
746 FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)),
747 FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)),
748 FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)),
749 FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)),
750 FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)),
751 FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)),
752 FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)),
753 FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)),
754 FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)),
755 FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr",
756 RXES(RBUF_LOOKUP_DES_REG_UNC_COR)),
757 FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)),
758 FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)),
759 FLAG_ENTRY0("RxRbufBlockListReadUncErr",
760 RXES(RBUF_BLOCK_LIST_READ_UNC)),
761 FLAG_ENTRY0("RxRbufBlockListReadCorErr",
762 RXES(RBUF_BLOCK_LIST_READ_COR)),
763 FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr",
764 RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)),
765 FLAG_ENTRY0("RxRbufCsrQEntCntParityErr",
766 RXES(RBUF_CSR_QENT_CNT_PARITY)),
767 FLAG_ENTRY0("RxRbufCsrQNextBufParityErr",
768 RXES(RBUF_CSR_QNEXT_BUF_PARITY)),
769 FLAG_ENTRY0("RxRbufCsrQVldBitParityErr",
770 RXES(RBUF_CSR_QVLD_BIT_PARITY)),
771 FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)),
772 FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)),
773 FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr",
774 RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)),
775 FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)),
776 FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)),
777 FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)),
778 FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)),
779 FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)),
780 FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)),
781 FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)),
782 FLAG_ENTRY0("RxRbufFlInitdoneParityErr",
783 RXES(RBUF_FL_INITDONE_PARITY)),
784 FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr",
785 RXES(RBUF_FL_INIT_WR_ADDR_PARITY)),
786 FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)),
787 FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)),
788 FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)),
789 FLAG_ENTRY0("RxLookupDesPart1UncCorErr",
790 RXES(LOOKUP_DES_PART1_UNC_COR)),
791 FLAG_ENTRY0("RxLookupDesPart2ParityErr",
792 RXES(LOOKUP_DES_PART2_PARITY)),
793 FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)),
794 FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)),
795 FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)),
796 FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)),
797 FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)),
798 FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)),
799 FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)),
800 FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)),
801 FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)),
802 FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)),
803 FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)),
804 FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)),
805 FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)),
806 FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)),
807 FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)),
808 FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)),
809 FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)),
810 FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)),
811 FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)),
812 FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)),
813 FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)),
814 FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY))
815};
816
817
818#define ALL_RXE_FREEZE_ERR \
819 (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \
820 | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \
821 | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \
822 | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \
823 | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \
824 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \
825 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \
826 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \
827 | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \
828 | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \
829 | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \
830 | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \
831 | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \
832 | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \
833 | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \
834 | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \
835 | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \
836 | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \
837 | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \
838 | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \
839 | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \
840 | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \
841 | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \
842 | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \
843 | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \
844 | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \
845 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \
846 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \
847 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \
848 | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \
849 | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \
850 | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \
851 | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \
852 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \
853 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \
854 | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \
855 | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \
856 | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \
857 | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \
858 | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \
859 | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \
860 | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \
861 | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \
862 | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK)
863
864#define RXE_FREEZE_ABORT_MASK \
865 (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \
866 RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \
867 RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK)
868
869
870
871
872#define DCCE(name) DCC_ERR_FLG_##name##_SMASK
873static struct flag_table dcc_err_flags[] = {
874 FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)),
875 FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)),
876 FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)),
877 FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)),
878 FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)),
879 FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)),
880 FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)),
881 FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)),
882 FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)),
883 FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)),
884 FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)),
885 FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)),
886 FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)),
887 FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)),
888 FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)),
889 FLAG_ENTRY0("link_err", DCCE(LINK_ERR)),
890 FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)),
891 FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)),
892 FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)),
893 FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)),
894 FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)),
895 FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)),
896 FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)),
897 FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)),
898 FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)),
899 FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)),
900 FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)),
901 FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)),
902 FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)),
903 FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)),
904 FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)),
905 FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)),
906 FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)),
907 FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)),
908 FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)),
909 FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)),
910 FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)),
911 FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)),
912 FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)),
913 FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)),
914 FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)),
915 FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)),
916 FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)),
917 FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)),
918 FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)),
919 FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)),
920};
921
922
923
924
925#define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK
926static struct flag_table lcb_err_flags[] = {
927 FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)),
928 FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)),
929 FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)),
930 FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST",
931 LCBE(ALL_LNS_FAILED_REINIT_TEST)),
932 FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)),
933 FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)),
934 FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)),
935 FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)),
936 FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)),
937 FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)),
938 FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)),
939 FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)),
940 FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)),
941 FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER",
942 LCBE(UNEXPECTED_ROUND_TRIP_MARKER)),
943 FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)),
944 FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)),
945 FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)),
946 FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)),
947 FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)),
948 FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE",
949 LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)),
950 FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)),
951 FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)),
952 FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)),
953 FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)),
954 FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)),
955 FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)),
956 FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP",
957 LCBE(RST_FOR_INCOMPLT_RND_TRIP)),
958 FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)),
959 FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE",
960 LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)),
961 FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR",
962 LCBE(REDUNDANT_FLIT_PARITY_ERR))
963};
964
965
966
967
968#define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK
969static struct flag_table dc8051_err_flags[] = {
970 FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)),
971 FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)),
972 FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)),
973 FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)),
974 FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)),
975 FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)),
976 FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)),
977 FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)),
978 FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES",
979 D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)),
980 FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)),
981};
982
983
984
985
986
987
988static struct flag_table dc8051_info_err_flags[] = {
989 FLAG_ENTRY0("Spico ROM check failed", SPICO_ROM_FAILED),
990 FLAG_ENTRY0("Unknown frame received", UNKNOWN_FRAME),
991 FLAG_ENTRY0("Target BER not met", TARGET_BER_NOT_MET),
992 FLAG_ENTRY0("Serdes internal loopback failure",
993 FAILED_SERDES_INTERNAL_LOOPBACK),
994 FLAG_ENTRY0("Failed SerDes init", FAILED_SERDES_INIT),
995 FLAG_ENTRY0("Failed LNI(Polling)", FAILED_LNI_POLLING),
996 FLAG_ENTRY0("Failed LNI(Debounce)", FAILED_LNI_DEBOUNCE),
997 FLAG_ENTRY0("Failed LNI(EstbComm)", FAILED_LNI_ESTBCOMM),
998 FLAG_ENTRY0("Failed LNI(OptEq)", FAILED_LNI_OPTEQ),
999 FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1),
1000 FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2),
1001 FLAG_ENTRY0("Failed LNI(ConfigLT)", FAILED_LNI_CONFIGLT),
1002 FLAG_ENTRY0("Host Handshake Timeout", HOST_HANDSHAKE_TIMEOUT),
1003 FLAG_ENTRY0("External Device Request Timeout",
1004 EXTERNAL_DEVICE_REQ_TIMEOUT),
1005};
1006
1007
1008
1009
1010
1011
1012static struct flag_table dc8051_info_host_msg_flags[] = {
1013 FLAG_ENTRY0("Host request done", 0x0001),
1014 FLAG_ENTRY0("BC PWR_MGM message", 0x0002),
1015 FLAG_ENTRY0("BC SMA message", 0x0004),
1016 FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008),
1017 FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010),
1018 FLAG_ENTRY0("External device config request", 0x0020),
1019 FLAG_ENTRY0("VerifyCap all frames received", 0x0040),
1020 FLAG_ENTRY0("LinkUp achieved", 0x0080),
1021 FLAG_ENTRY0("Link going down", 0x0100),
1022 FLAG_ENTRY0("Link width downgraded", 0x0200),
1023};
1024
1025static u32 encoded_size(u32 size);
1026static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate);
1027static int set_physical_link_state(struct hfi1_devdata *dd, u64 state);
1028static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
1029 u8 *continuous);
1030static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
1031 u8 *vcu, u16 *vl15buf, u8 *crc_sizes);
1032static void read_vc_remote_link_width(struct hfi1_devdata *dd,
1033 u8 *remote_tx_rate, u16 *link_widths);
1034static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits,
1035 u8 *flag_bits, u16 *link_widths);
1036static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
1037 u8 *device_rev);
1038static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx);
1039static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx,
1040 u8 *tx_polarity_inversion,
1041 u8 *rx_polarity_inversion, u8 *max_rate);
1042static void handle_sdma_eng_err(struct hfi1_devdata *dd,
1043 unsigned int context, u64 err_status);
1044static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg);
1045static void handle_dcc_err(struct hfi1_devdata *dd,
1046 unsigned int context, u64 err_status);
1047static void handle_lcb_err(struct hfi1_devdata *dd,
1048 unsigned int context, u64 err_status);
1049static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg);
1050static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1051static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1052static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1053static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1054static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1055static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1056static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1057static void set_partition_keys(struct hfi1_pportdata *ppd);
1058static const char *link_state_name(u32 state);
1059static const char *link_state_reason_name(struct hfi1_pportdata *ppd,
1060 u32 state);
1061static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
1062 u64 *out_data);
1063static int read_idle_sma(struct hfi1_devdata *dd, u64 *data);
1064static int thermal_init(struct hfi1_devdata *dd);
1065
1066static void update_statusp(struct hfi1_pportdata *ppd, u32 state);
1067static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd,
1068 int msecs);
1069static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
1070 int msecs);
1071static void log_state_transition(struct hfi1_pportdata *ppd, u32 state);
1072static void log_physical_state(struct hfi1_pportdata *ppd, u32 state);
1073static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state,
1074 int msecs);
1075static int wait_phys_link_out_of_offline(struct hfi1_pportdata *ppd,
1076 int msecs);
1077static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc);
1078static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr);
1079static void handle_temp_err(struct hfi1_devdata *dd);
1080static void dc_shutdown(struct hfi1_devdata *dd);
1081static void dc_start(struct hfi1_devdata *dd);
1082static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
1083 unsigned int *np);
1084static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd);
1085static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms);
1086static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index);
1087static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width);
1088
1089
1090
1091
1092
1093
1094
1095struct err_reg_info {
1096 u32 status;
1097 u32 clear;
1098 u32 mask;
1099 void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg);
1100 const char *desc;
1101};
1102
1103#define NUM_MISC_ERRS (IS_GENERAL_ERR_END + 1 - IS_GENERAL_ERR_START)
1104#define NUM_DC_ERRS (IS_DC_END + 1 - IS_DC_START)
1105#define NUM_VARIOUS (IS_VARIOUS_END + 1 - IS_VARIOUS_START)
1106
1107
1108
1109
1110
1111#define EE(reg, handler, desc) \
1112 { reg##_STATUS, reg##_CLEAR, reg##_MASK, \
1113 handler, desc }
1114#define DC_EE1(reg, handler, desc) \
1115 { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc }
1116#define DC_EE2(reg, handler, desc) \
1117 { reg##_FLG, reg##_CLR, reg##_EN, handler, desc }
1118
1119
1120
1121
1122
1123static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = {
1124 EE(CCE_ERR, handle_cce_err, "CceErr"),
1125 EE(RCV_ERR, handle_rxe_err, "RxeErr"),
1126 EE(MISC_ERR, handle_misc_err, "MiscErr"),
1127 { 0, 0, 0, NULL },
1128 EE(SEND_PIO_ERR, handle_pio_err, "PioErr"),
1129 EE(SEND_DMA_ERR, handle_sdma_err, "SDmaErr"),
1130 EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"),
1131 EE(SEND_ERR, handle_txe_err, "TxeErr")
1132
1133};
1134
1135
1136
1137
1138
1139#define TCRIT_INT_SOURCE 4
1140
1141
1142
1143
1144
1145static const struct err_reg_info sdma_eng_err =
1146 EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr");
1147
1148static const struct err_reg_info various_err[NUM_VARIOUS] = {
1149 { 0, 0, 0, NULL },
1150 { 0, 0, 0, NULL },
1151 EE(ASIC_QSFP1, handle_qsfp_int, "QSFP1"),
1152 EE(ASIC_QSFP2, handle_qsfp_int, "QSFP2"),
1153 { 0, 0, 0, NULL },
1154
1155};
1156
1157
1158
1159
1160
1161
1162
1163#define DCC_CFG_PORT_MTU_CAP_10240 7
1164
1165
1166
1167
1168
1169static const struct err_reg_info dc_errs[NUM_DC_ERRS] = {
1170 DC_EE1(DCC_ERR, handle_dcc_err, "DCC Err"),
1171 DC_EE2(DC_LCB_ERR, handle_lcb_err, "LCB Err"),
1172 DC_EE2(DC_DC8051_ERR, handle_8051_interrupt, "DC8051 Interrupt"),
1173
1174
1175};
1176
1177struct cntr_entry {
1178
1179
1180
1181 char *name;
1182
1183
1184
1185
1186 u64 csr;
1187
1188
1189
1190
1191 int offset;
1192
1193
1194
1195
1196 u8 flags;
1197
1198
1199
1200
1201 u64 (*rw_cntr)(const struct cntr_entry *, void *context, int vl,
1202 int mode, u64 data);
1203};
1204
1205#define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0
1206#define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159
1207
1208#define CNTR_ELEM(name, csr, offset, flags, accessor) \
1209{ \
1210 name, \
1211 csr, \
1212 offset, \
1213 flags, \
1214 accessor \
1215}
1216
1217
1218#define RXE32_PORT_CNTR_ELEM(name, counter, flags) \
1219CNTR_ELEM(#name, \
1220 (counter * 8 + RCV_COUNTER_ARRAY32), \
1221 0, flags | CNTR_32BIT, \
1222 port_access_u32_csr)
1223
1224#define RXE32_DEV_CNTR_ELEM(name, counter, flags) \
1225CNTR_ELEM(#name, \
1226 (counter * 8 + RCV_COUNTER_ARRAY32), \
1227 0, flags | CNTR_32BIT, \
1228 dev_access_u32_csr)
1229
1230
1231#define RXE64_PORT_CNTR_ELEM(name, counter, flags) \
1232CNTR_ELEM(#name, \
1233 (counter * 8 + RCV_COUNTER_ARRAY64), \
1234 0, flags, \
1235 port_access_u64_csr)
1236
1237#define RXE64_DEV_CNTR_ELEM(name, counter, flags) \
1238CNTR_ELEM(#name, \
1239 (counter * 8 + RCV_COUNTER_ARRAY64), \
1240 0, flags, \
1241 dev_access_u64_csr)
1242
1243#define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx
1244#define OVR_ELM(ctx) \
1245CNTR_ELEM("RcvHdrOvr" #ctx, \
1246 (RCV_HDR_OVFL_CNT + ctx * 0x100), \
1247 0, CNTR_NORMAL, port_access_u64_csr)
1248
1249
1250#define TXE32_PORT_CNTR_ELEM(name, counter, flags) \
1251CNTR_ELEM(#name, \
1252 (counter * 8 + SEND_COUNTER_ARRAY32), \
1253 0, flags | CNTR_32BIT, \
1254 port_access_u32_csr)
1255
1256
1257#define TXE64_PORT_CNTR_ELEM(name, counter, flags) \
1258CNTR_ELEM(#name, \
1259 (counter * 8 + SEND_COUNTER_ARRAY64), \
1260 0, flags, \
1261 port_access_u64_csr)
1262
1263# define TX64_DEV_CNTR_ELEM(name, counter, flags) \
1264CNTR_ELEM(#name,\
1265 counter * 8 + SEND_COUNTER_ARRAY64, \
1266 0, \
1267 flags, \
1268 dev_access_u64_csr)
1269
1270
1271#define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \
1272CNTR_ELEM(#name, \
1273 (counter * 8 + CCE_COUNTER_ARRAY32), \
1274 0, flags | CNTR_32BIT, \
1275 dev_access_u32_csr)
1276
1277#define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \
1278CNTR_ELEM(#name, \
1279 (counter * 8 + CCE_INT_COUNTER_ARRAY32), \
1280 0, flags | CNTR_32BIT, \
1281 dev_access_u32_csr)
1282
1283
1284#define DC_PERF_CNTR(name, counter, flags) \
1285CNTR_ELEM(#name, \
1286 counter, \
1287 0, \
1288 flags, \
1289 dev_access_u64_csr)
1290
1291#define DC_PERF_CNTR_LCB(name, counter, flags) \
1292CNTR_ELEM(#name, \
1293 counter, \
1294 0, \
1295 flags, \
1296 dc_access_lcb_cntr)
1297
1298
1299#define SW_IBP_CNTR(name, cntr) \
1300CNTR_ELEM(#name, \
1301 0, \
1302 0, \
1303 CNTR_SYNTH, \
1304 access_ibp_##cntr)
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314static inline void __iomem *hfi1_addr_from_offset(
1315 const struct hfi1_devdata *dd,
1316 u32 offset)
1317{
1318 if (offset >= dd->base2_start)
1319 return dd->kregbase2 + (offset - dd->base2_start);
1320 return dd->kregbase1 + offset;
1321}
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331u64 read_csr(const struct hfi1_devdata *dd, u32 offset)
1332{
1333 if (dd->flags & HFI1_PRESENT)
1334 return readq(hfi1_addr_from_offset(dd, offset));
1335 return -1;
1336}
1337
1338
1339
1340
1341
1342
1343
1344void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value)
1345{
1346 if (dd->flags & HFI1_PRESENT) {
1347 void __iomem *base = hfi1_addr_from_offset(dd, offset);
1348
1349
1350 if (WARN_ON(offset >= RCV_ARRAY && offset < dd->base2_start))
1351 return;
1352 writeq(value, base);
1353 }
1354}
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364void __iomem *get_csr_addr(
1365 const struct hfi1_devdata *dd,
1366 u32 offset)
1367{
1368 if (dd->flags & HFI1_PRESENT)
1369 return hfi1_addr_from_offset(dd, offset);
1370 return NULL;
1371}
1372
1373static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr,
1374 int mode, u64 value)
1375{
1376 u64 ret;
1377
1378 if (mode == CNTR_MODE_R) {
1379 ret = read_csr(dd, csr);
1380 } else if (mode == CNTR_MODE_W) {
1381 write_csr(dd, csr, value);
1382 ret = value;
1383 } else {
1384 dd_dev_err(dd, "Invalid cntr register access mode");
1385 return 0;
1386 }
1387
1388 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode);
1389 return ret;
1390}
1391
1392
1393static u64 dev_access_u32_csr(const struct cntr_entry *entry,
1394 void *context, int vl, int mode, u64 data)
1395{
1396 struct hfi1_devdata *dd = context;
1397 u64 csr = entry->csr;
1398
1399 if (entry->flags & CNTR_SDMA) {
1400 if (vl == CNTR_INVALID_VL)
1401 return 0;
1402 csr += 0x100 * vl;
1403 } else {
1404 if (vl != CNTR_INVALID_VL)
1405 return 0;
1406 }
1407 return read_write_csr(dd, csr, mode, data);
1408}
1409
1410static u64 access_sde_err_cnt(const struct cntr_entry *entry,
1411 void *context, int idx, int mode, u64 data)
1412{
1413 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1414
1415 if (dd->per_sdma && idx < dd->num_sdma)
1416 return dd->per_sdma[idx].err_cnt;
1417 return 0;
1418}
1419
1420static u64 access_sde_int_cnt(const struct cntr_entry *entry,
1421 void *context, int idx, int mode, u64 data)
1422{
1423 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1424
1425 if (dd->per_sdma && idx < dd->num_sdma)
1426 return dd->per_sdma[idx].sdma_int_cnt;
1427 return 0;
1428}
1429
1430static u64 access_sde_idle_int_cnt(const struct cntr_entry *entry,
1431 void *context, int idx, int mode, u64 data)
1432{
1433 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1434
1435 if (dd->per_sdma && idx < dd->num_sdma)
1436 return dd->per_sdma[idx].idle_int_cnt;
1437 return 0;
1438}
1439
1440static u64 access_sde_progress_int_cnt(const struct cntr_entry *entry,
1441 void *context, int idx, int mode,
1442 u64 data)
1443{
1444 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1445
1446 if (dd->per_sdma && idx < dd->num_sdma)
1447 return dd->per_sdma[idx].progress_int_cnt;
1448 return 0;
1449}
1450
1451static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context,
1452 int vl, int mode, u64 data)
1453{
1454 struct hfi1_devdata *dd = context;
1455
1456 u64 val = 0;
1457 u64 csr = entry->csr;
1458
1459 if (entry->flags & CNTR_VL) {
1460 if (vl == CNTR_INVALID_VL)
1461 return 0;
1462 csr += 8 * vl;
1463 } else {
1464 if (vl != CNTR_INVALID_VL)
1465 return 0;
1466 }
1467
1468 val = read_write_csr(dd, csr, mode, data);
1469 return val;
1470}
1471
1472static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context,
1473 int vl, int mode, u64 data)
1474{
1475 struct hfi1_devdata *dd = context;
1476 u32 csr = entry->csr;
1477 int ret = 0;
1478
1479 if (vl != CNTR_INVALID_VL)
1480 return 0;
1481 if (mode == CNTR_MODE_R)
1482 ret = read_lcb_csr(dd, csr, &data);
1483 else if (mode == CNTR_MODE_W)
1484 ret = write_lcb_csr(dd, csr, data);
1485
1486 if (ret) {
1487 dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr);
1488 return 0;
1489 }
1490
1491 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode);
1492 return data;
1493}
1494
1495
1496static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context,
1497 int vl, int mode, u64 data)
1498{
1499 struct hfi1_pportdata *ppd = context;
1500
1501 if (vl != CNTR_INVALID_VL)
1502 return 0;
1503 return read_write_csr(ppd->dd, entry->csr, mode, data);
1504}
1505
1506static u64 port_access_u64_csr(const struct cntr_entry *entry,
1507 void *context, int vl, int mode, u64 data)
1508{
1509 struct hfi1_pportdata *ppd = context;
1510 u64 val;
1511 u64 csr = entry->csr;
1512
1513 if (entry->flags & CNTR_VL) {
1514 if (vl == CNTR_INVALID_VL)
1515 return 0;
1516 csr += 8 * vl;
1517 } else {
1518 if (vl != CNTR_INVALID_VL)
1519 return 0;
1520 }
1521 val = read_write_csr(ppd->dd, csr, mode, data);
1522 return val;
1523}
1524
1525
1526static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode,
1527 u64 data)
1528{
1529 u64 ret;
1530
1531 if (mode == CNTR_MODE_R) {
1532 ret = *cntr;
1533 } else if (mode == CNTR_MODE_W) {
1534 *cntr = data;
1535 ret = data;
1536 } else {
1537 dd_dev_err(dd, "Invalid cntr sw access mode");
1538 return 0;
1539 }
1540
1541 hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode);
1542
1543 return ret;
1544}
1545
1546static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context,
1547 int vl, int mode, u64 data)
1548{
1549 struct hfi1_pportdata *ppd = context;
1550
1551 if (vl != CNTR_INVALID_VL)
1552 return 0;
1553 return read_write_sw(ppd->dd, &ppd->link_downed, mode, data);
1554}
1555
1556static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context,
1557 int vl, int mode, u64 data)
1558{
1559 struct hfi1_pportdata *ppd = context;
1560
1561 if (vl != CNTR_INVALID_VL)
1562 return 0;
1563 return read_write_sw(ppd->dd, &ppd->link_up, mode, data);
1564}
1565
1566static u64 access_sw_unknown_frame_cnt(const struct cntr_entry *entry,
1567 void *context, int vl, int mode,
1568 u64 data)
1569{
1570 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1571
1572 if (vl != CNTR_INVALID_VL)
1573 return 0;
1574 return read_write_sw(ppd->dd, &ppd->unknown_frame_count, mode, data);
1575}
1576
1577static u64 access_sw_xmit_discards(const struct cntr_entry *entry,
1578 void *context, int vl, int mode, u64 data)
1579{
1580 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1581 u64 zero = 0;
1582 u64 *counter;
1583
1584 if (vl == CNTR_INVALID_VL)
1585 counter = &ppd->port_xmit_discards;
1586 else if (vl >= 0 && vl < C_VL_COUNT)
1587 counter = &ppd->port_xmit_discards_vl[vl];
1588 else
1589 counter = &zero;
1590
1591 return read_write_sw(ppd->dd, counter, mode, data);
1592}
1593
1594static u64 access_xmit_constraint_errs(const struct cntr_entry *entry,
1595 void *context, int vl, int mode,
1596 u64 data)
1597{
1598 struct hfi1_pportdata *ppd = context;
1599
1600 if (vl != CNTR_INVALID_VL)
1601 return 0;
1602
1603 return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors,
1604 mode, data);
1605}
1606
1607static u64 access_rcv_constraint_errs(const struct cntr_entry *entry,
1608 void *context, int vl, int mode, u64 data)
1609{
1610 struct hfi1_pportdata *ppd = context;
1611
1612 if (vl != CNTR_INVALID_VL)
1613 return 0;
1614
1615 return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors,
1616 mode, data);
1617}
1618
1619u64 get_all_cpu_total(u64 __percpu *cntr)
1620{
1621 int cpu;
1622 u64 counter = 0;
1623
1624 for_each_possible_cpu(cpu)
1625 counter += *per_cpu_ptr(cntr, cpu);
1626 return counter;
1627}
1628
1629static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val,
1630 u64 __percpu *cntr,
1631 int vl, int mode, u64 data)
1632{
1633 u64 ret = 0;
1634
1635 if (vl != CNTR_INVALID_VL)
1636 return 0;
1637
1638 if (mode == CNTR_MODE_R) {
1639 ret = get_all_cpu_total(cntr) - *z_val;
1640 } else if (mode == CNTR_MODE_W) {
1641
1642 if (data == 0)
1643 *z_val = get_all_cpu_total(cntr);
1644 else
1645 dd_dev_err(dd, "Per CPU cntrs can only be zeroed");
1646 } else {
1647 dd_dev_err(dd, "Invalid cntr sw cpu access mode");
1648 return 0;
1649 }
1650
1651 return ret;
1652}
1653
1654static u64 access_sw_cpu_intr(const struct cntr_entry *entry,
1655 void *context, int vl, int mode, u64 data)
1656{
1657 struct hfi1_devdata *dd = context;
1658
1659 return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl,
1660 mode, data);
1661}
1662
1663static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry,
1664 void *context, int vl, int mode, u64 data)
1665{
1666 struct hfi1_devdata *dd = context;
1667
1668 return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl,
1669 mode, data);
1670}
1671
1672static u64 access_sw_pio_wait(const struct cntr_entry *entry,
1673 void *context, int vl, int mode, u64 data)
1674{
1675 struct hfi1_devdata *dd = context;
1676
1677 return dd->verbs_dev.n_piowait;
1678}
1679
1680static u64 access_sw_pio_drain(const struct cntr_entry *entry,
1681 void *context, int vl, int mode, u64 data)
1682{
1683 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1684
1685 return dd->verbs_dev.n_piodrain;
1686}
1687
1688static u64 access_sw_vtx_wait(const struct cntr_entry *entry,
1689 void *context, int vl, int mode, u64 data)
1690{
1691 struct hfi1_devdata *dd = context;
1692
1693 return dd->verbs_dev.n_txwait;
1694}
1695
1696static u64 access_sw_kmem_wait(const struct cntr_entry *entry,
1697 void *context, int vl, int mode, u64 data)
1698{
1699 struct hfi1_devdata *dd = context;
1700
1701 return dd->verbs_dev.n_kmem_wait;
1702}
1703
1704static u64 access_sw_send_schedule(const struct cntr_entry *entry,
1705 void *context, int vl, int mode, u64 data)
1706{
1707 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1708
1709 return read_write_cpu(dd, &dd->z_send_schedule, dd->send_schedule, vl,
1710 mode, data);
1711}
1712
1713
1714static u64 access_misc_pll_lock_fail_err_cnt(const struct cntr_entry *entry,
1715 void *context, int vl, int mode,
1716 u64 data)
1717{
1718 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1719
1720 return dd->misc_err_status_cnt[12];
1721}
1722
1723static u64 access_misc_mbist_fail_err_cnt(const struct cntr_entry *entry,
1724 void *context, int vl, int mode,
1725 u64 data)
1726{
1727 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1728
1729 return dd->misc_err_status_cnt[11];
1730}
1731
1732static u64 access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry *entry,
1733 void *context, int vl, int mode,
1734 u64 data)
1735{
1736 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1737
1738 return dd->misc_err_status_cnt[10];
1739}
1740
1741static u64 access_misc_efuse_done_parity_err_cnt(const struct cntr_entry *entry,
1742 void *context, int vl,
1743 int mode, u64 data)
1744{
1745 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1746
1747 return dd->misc_err_status_cnt[9];
1748}
1749
1750static u64 access_misc_efuse_write_err_cnt(const struct cntr_entry *entry,
1751 void *context, int vl, int mode,
1752 u64 data)
1753{
1754 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1755
1756 return dd->misc_err_status_cnt[8];
1757}
1758
1759static u64 access_misc_efuse_read_bad_addr_err_cnt(
1760 const struct cntr_entry *entry,
1761 void *context, int vl, int mode, u64 data)
1762{
1763 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1764
1765 return dd->misc_err_status_cnt[7];
1766}
1767
1768static u64 access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry *entry,
1769 void *context, int vl,
1770 int mode, u64 data)
1771{
1772 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1773
1774 return dd->misc_err_status_cnt[6];
1775}
1776
1777static u64 access_misc_fw_auth_failed_err_cnt(const struct cntr_entry *entry,
1778 void *context, int vl, int mode,
1779 u64 data)
1780{
1781 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1782
1783 return dd->misc_err_status_cnt[5];
1784}
1785
1786static u64 access_misc_key_mismatch_err_cnt(const struct cntr_entry *entry,
1787 void *context, int vl, int mode,
1788 u64 data)
1789{
1790 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1791
1792 return dd->misc_err_status_cnt[4];
1793}
1794
1795static u64 access_misc_sbus_write_failed_err_cnt(const struct cntr_entry *entry,
1796 void *context, int vl,
1797 int mode, u64 data)
1798{
1799 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1800
1801 return dd->misc_err_status_cnt[3];
1802}
1803
1804static u64 access_misc_csr_write_bad_addr_err_cnt(
1805 const struct cntr_entry *entry,
1806 void *context, int vl, int mode, u64 data)
1807{
1808 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1809
1810 return dd->misc_err_status_cnt[2];
1811}
1812
1813static u64 access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1814 void *context, int vl,
1815 int mode, u64 data)
1816{
1817 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1818
1819 return dd->misc_err_status_cnt[1];
1820}
1821
1822static u64 access_misc_csr_parity_err_cnt(const struct cntr_entry *entry,
1823 void *context, int vl, int mode,
1824 u64 data)
1825{
1826 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1827
1828 return dd->misc_err_status_cnt[0];
1829}
1830
1831
1832
1833
1834
1835static u64 access_sw_cce_err_status_aggregated_cnt(
1836 const struct cntr_entry *entry,
1837 void *context, int vl, int mode, u64 data)
1838{
1839 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1840
1841 return dd->sw_cce_err_status_aggregate;
1842}
1843
1844
1845
1846
1847
1848static u64 access_cce_msix_csr_parity_err_cnt(const struct cntr_entry *entry,
1849 void *context, int vl, int mode,
1850 u64 data)
1851{
1852 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1853
1854 return dd->cce_err_status_cnt[40];
1855}
1856
1857static u64 access_cce_int_map_unc_err_cnt(const struct cntr_entry *entry,
1858 void *context, int vl, int mode,
1859 u64 data)
1860{
1861 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1862
1863 return dd->cce_err_status_cnt[39];
1864}
1865
1866static u64 access_cce_int_map_cor_err_cnt(const struct cntr_entry *entry,
1867 void *context, int vl, int mode,
1868 u64 data)
1869{
1870 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1871
1872 return dd->cce_err_status_cnt[38];
1873}
1874
1875static u64 access_cce_msix_table_unc_err_cnt(const struct cntr_entry *entry,
1876 void *context, int vl, int mode,
1877 u64 data)
1878{
1879 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1880
1881 return dd->cce_err_status_cnt[37];
1882}
1883
1884static u64 access_cce_msix_table_cor_err_cnt(const struct cntr_entry *entry,
1885 void *context, int vl, int mode,
1886 u64 data)
1887{
1888 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1889
1890 return dd->cce_err_status_cnt[36];
1891}
1892
1893static u64 access_cce_rxdma_conv_fifo_parity_err_cnt(
1894 const struct cntr_entry *entry,
1895 void *context, int vl, int mode, u64 data)
1896{
1897 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1898
1899 return dd->cce_err_status_cnt[35];
1900}
1901
1902static u64 access_cce_rcpl_async_fifo_parity_err_cnt(
1903 const struct cntr_entry *entry,
1904 void *context, int vl, int mode, u64 data)
1905{
1906 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1907
1908 return dd->cce_err_status_cnt[34];
1909}
1910
1911static u64 access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry *entry,
1912 void *context, int vl,
1913 int mode, u64 data)
1914{
1915 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1916
1917 return dd->cce_err_status_cnt[33];
1918}
1919
1920static u64 access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1921 void *context, int vl, int mode,
1922 u64 data)
1923{
1924 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1925
1926 return dd->cce_err_status_cnt[32];
1927}
1928
1929static u64 access_la_triggered_cnt(const struct cntr_entry *entry,
1930 void *context, int vl, int mode, u64 data)
1931{
1932 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1933
1934 return dd->cce_err_status_cnt[31];
1935}
1936
1937static u64 access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry *entry,
1938 void *context, int vl, int mode,
1939 u64 data)
1940{
1941 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1942
1943 return dd->cce_err_status_cnt[30];
1944}
1945
1946static u64 access_pcic_receive_parity_err_cnt(const struct cntr_entry *entry,
1947 void *context, int vl, int mode,
1948 u64 data)
1949{
1950 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1951
1952 return dd->cce_err_status_cnt[29];
1953}
1954
1955static u64 access_pcic_transmit_back_parity_err_cnt(
1956 const struct cntr_entry *entry,
1957 void *context, int vl, int mode, u64 data)
1958{
1959 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1960
1961 return dd->cce_err_status_cnt[28];
1962}
1963
1964static u64 access_pcic_transmit_front_parity_err_cnt(
1965 const struct cntr_entry *entry,
1966 void *context, int vl, int mode, u64 data)
1967{
1968 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1969
1970 return dd->cce_err_status_cnt[27];
1971}
1972
1973static u64 access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1974 void *context, int vl, int mode,
1975 u64 data)
1976{
1977 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1978
1979 return dd->cce_err_status_cnt[26];
1980}
1981
1982static u64 access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1983 void *context, int vl, int mode,
1984 u64 data)
1985{
1986 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1987
1988 return dd->cce_err_status_cnt[25];
1989}
1990
1991static u64 access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1992 void *context, int vl, int mode,
1993 u64 data)
1994{
1995 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1996
1997 return dd->cce_err_status_cnt[24];
1998}
1999
2000static u64 access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry *entry,
2001 void *context, int vl, int mode,
2002 u64 data)
2003{
2004 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2005
2006 return dd->cce_err_status_cnt[23];
2007}
2008
2009static u64 access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry *entry,
2010 void *context, int vl,
2011 int mode, u64 data)
2012{
2013 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2014
2015 return dd->cce_err_status_cnt[22];
2016}
2017
2018static u64 access_pcic_retry_mem_unc_err(const struct cntr_entry *entry,
2019 void *context, int vl, int mode,
2020 u64 data)
2021{
2022 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2023
2024 return dd->cce_err_status_cnt[21];
2025}
2026
2027static u64 access_pcic_n_post_dat_q_parity_err_cnt(
2028 const struct cntr_entry *entry,
2029 void *context, int vl, int mode, u64 data)
2030{
2031 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2032
2033 return dd->cce_err_status_cnt[20];
2034}
2035
2036static u64 access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry *entry,
2037 void *context, int vl,
2038 int mode, u64 data)
2039{
2040 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2041
2042 return dd->cce_err_status_cnt[19];
2043}
2044
2045static u64 access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry *entry,
2046 void *context, int vl, int mode,
2047 u64 data)
2048{
2049 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2050
2051 return dd->cce_err_status_cnt[18];
2052}
2053
2054static u64 access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry *entry,
2055 void *context, int vl, int mode,
2056 u64 data)
2057{
2058 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2059
2060 return dd->cce_err_status_cnt[17];
2061}
2062
2063static u64 access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry *entry,
2064 void *context, int vl, int mode,
2065 u64 data)
2066{
2067 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2068
2069 return dd->cce_err_status_cnt[16];
2070}
2071
2072static u64 access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry *entry,
2073 void *context, int vl, int mode,
2074 u64 data)
2075{
2076 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2077
2078 return dd->cce_err_status_cnt[15];
2079}
2080
2081static u64 access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry *entry,
2082 void *context, int vl,
2083 int mode, u64 data)
2084{
2085 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2086
2087 return dd->cce_err_status_cnt[14];
2088}
2089
2090static u64 access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry *entry,
2091 void *context, int vl, int mode,
2092 u64 data)
2093{
2094 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2095
2096 return dd->cce_err_status_cnt[13];
2097}
2098
2099static u64 access_cce_cli1_async_fifo_dbg_parity_err_cnt(
2100 const struct cntr_entry *entry,
2101 void *context, int vl, int mode, u64 data)
2102{
2103 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2104
2105 return dd->cce_err_status_cnt[12];
2106}
2107
2108static u64 access_cce_cli1_async_fifo_rxdma_parity_err_cnt(
2109 const struct cntr_entry *entry,
2110 void *context, int vl, int mode, u64 data)
2111{
2112 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2113
2114 return dd->cce_err_status_cnt[11];
2115}
2116
2117static u64 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt(
2118 const struct cntr_entry *entry,
2119 void *context, int vl, int mode, u64 data)
2120{
2121 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2122
2123 return dd->cce_err_status_cnt[10];
2124}
2125
2126static u64 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt(
2127 const struct cntr_entry *entry,
2128 void *context, int vl, int mode, u64 data)
2129{
2130 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2131
2132 return dd->cce_err_status_cnt[9];
2133}
2134
2135static u64 access_cce_cli2_async_fifo_parity_err_cnt(
2136 const struct cntr_entry *entry,
2137 void *context, int vl, int mode, u64 data)
2138{
2139 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2140
2141 return dd->cce_err_status_cnt[8];
2142}
2143
2144static u64 access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry *entry,
2145 void *context, int vl,
2146 int mode, u64 data)
2147{
2148 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2149
2150 return dd->cce_err_status_cnt[7];
2151}
2152
2153static u64 access_cce_cli0_async_fifo_parity_err_cnt(
2154 const struct cntr_entry *entry,
2155 void *context, int vl, int mode, u64 data)
2156{
2157 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2158
2159 return dd->cce_err_status_cnt[6];
2160}
2161
2162static u64 access_cce_rspd_data_parity_err_cnt(const struct cntr_entry *entry,
2163 void *context, int vl, int mode,
2164 u64 data)
2165{
2166 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2167
2168 return dd->cce_err_status_cnt[5];
2169}
2170
2171static u64 access_cce_trgt_access_err_cnt(const struct cntr_entry *entry,
2172 void *context, int vl, int mode,
2173 u64 data)
2174{
2175 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2176
2177 return dd->cce_err_status_cnt[4];
2178}
2179
2180static u64 access_cce_trgt_async_fifo_parity_err_cnt(
2181 const struct cntr_entry *entry,
2182 void *context, int vl, int mode, u64 data)
2183{
2184 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2185
2186 return dd->cce_err_status_cnt[3];
2187}
2188
2189static u64 access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2190 void *context, int vl,
2191 int mode, u64 data)
2192{
2193 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2194
2195 return dd->cce_err_status_cnt[2];
2196}
2197
2198static u64 access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2199 void *context, int vl,
2200 int mode, u64 data)
2201{
2202 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2203
2204 return dd->cce_err_status_cnt[1];
2205}
2206
2207static u64 access_ccs_csr_parity_err_cnt(const struct cntr_entry *entry,
2208 void *context, int vl, int mode,
2209 u64 data)
2210{
2211 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2212
2213 return dd->cce_err_status_cnt[0];
2214}
2215
2216
2217
2218
2219
2220static u64 access_rx_csr_parity_err_cnt(const struct cntr_entry *entry,
2221 void *context, int vl, int mode,
2222 u64 data)
2223{
2224 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2225
2226 return dd->rcv_err_status_cnt[63];
2227}
2228
2229static u64 access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2230 void *context, int vl,
2231 int mode, u64 data)
2232{
2233 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2234
2235 return dd->rcv_err_status_cnt[62];
2236}
2237
2238static u64 access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2239 void *context, int vl, int mode,
2240 u64 data)
2241{
2242 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2243
2244 return dd->rcv_err_status_cnt[61];
2245}
2246
2247static u64 access_rx_dma_csr_unc_err_cnt(const struct cntr_entry *entry,
2248 void *context, int vl, int mode,
2249 u64 data)
2250{
2251 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2252
2253 return dd->rcv_err_status_cnt[60];
2254}
2255
2256static u64 access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2257 void *context, int vl,
2258 int mode, u64 data)
2259{
2260 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2261
2262 return dd->rcv_err_status_cnt[59];
2263}
2264
2265static u64 access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2266 void *context, int vl,
2267 int mode, u64 data)
2268{
2269 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2270
2271 return dd->rcv_err_status_cnt[58];
2272}
2273
2274static u64 access_rx_dma_csr_parity_err_cnt(const struct cntr_entry *entry,
2275 void *context, int vl, int mode,
2276 u64 data)
2277{
2278 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2279
2280 return dd->rcv_err_status_cnt[57];
2281}
2282
2283static u64 access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry *entry,
2284 void *context, int vl, int mode,
2285 u64 data)
2286{
2287 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2288
2289 return dd->rcv_err_status_cnt[56];
2290}
2291
2292static u64 access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry *entry,
2293 void *context, int vl, int mode,
2294 u64 data)
2295{
2296 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2297
2298 return dd->rcv_err_status_cnt[55];
2299}
2300
2301static u64 access_rx_dma_data_fifo_rd_cor_err_cnt(
2302 const struct cntr_entry *entry,
2303 void *context, int vl, int mode, u64 data)
2304{
2305 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2306
2307 return dd->rcv_err_status_cnt[54];
2308}
2309
2310static u64 access_rx_dma_data_fifo_rd_unc_err_cnt(
2311 const struct cntr_entry *entry,
2312 void *context, int vl, int mode, u64 data)
2313{
2314 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2315
2316 return dd->rcv_err_status_cnt[53];
2317}
2318
2319static u64 access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry *entry,
2320 void *context, int vl,
2321 int mode, u64 data)
2322{
2323 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2324
2325 return dd->rcv_err_status_cnt[52];
2326}
2327
2328static u64 access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry *entry,
2329 void *context, int vl,
2330 int mode, u64 data)
2331{
2332 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2333
2334 return dd->rcv_err_status_cnt[51];
2335}
2336
2337static u64 access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry *entry,
2338 void *context, int vl,
2339 int mode, u64 data)
2340{
2341 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2342
2343 return dd->rcv_err_status_cnt[50];
2344}
2345
2346static u64 access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry *entry,
2347 void *context, int vl,
2348 int mode, u64 data)
2349{
2350 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2351
2352 return dd->rcv_err_status_cnt[49];
2353}
2354
2355static u64 access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry *entry,
2356 void *context, int vl,
2357 int mode, u64 data)
2358{
2359 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2360
2361 return dd->rcv_err_status_cnt[48];
2362}
2363
2364static u64 access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry *entry,
2365 void *context, int vl,
2366 int mode, u64 data)
2367{
2368 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2369
2370 return dd->rcv_err_status_cnt[47];
2371}
2372
2373static u64 access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry *entry,
2374 void *context, int vl, int mode,
2375 u64 data)
2376{
2377 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2378
2379 return dd->rcv_err_status_cnt[46];
2380}
2381
2382static u64 access_rx_hq_intr_csr_parity_err_cnt(
2383 const struct cntr_entry *entry,
2384 void *context, int vl, int mode, u64 data)
2385{
2386 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2387
2388 return dd->rcv_err_status_cnt[45];
2389}
2390
2391static u64 access_rx_lookup_csr_parity_err_cnt(
2392 const struct cntr_entry *entry,
2393 void *context, int vl, int mode, u64 data)
2394{
2395 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2396
2397 return dd->rcv_err_status_cnt[44];
2398}
2399
2400static u64 access_rx_lookup_rcv_array_cor_err_cnt(
2401 const struct cntr_entry *entry,
2402 void *context, int vl, int mode, u64 data)
2403{
2404 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2405
2406 return dd->rcv_err_status_cnt[43];
2407}
2408
2409static u64 access_rx_lookup_rcv_array_unc_err_cnt(
2410 const struct cntr_entry *entry,
2411 void *context, int vl, int mode, u64 data)
2412{
2413 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2414
2415 return dd->rcv_err_status_cnt[42];
2416}
2417
2418static u64 access_rx_lookup_des_part2_parity_err_cnt(
2419 const struct cntr_entry *entry,
2420 void *context, int vl, int mode, u64 data)
2421{
2422 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2423
2424 return dd->rcv_err_status_cnt[41];
2425}
2426
2427static u64 access_rx_lookup_des_part1_unc_cor_err_cnt(
2428 const struct cntr_entry *entry,
2429 void *context, int vl, int mode, u64 data)
2430{
2431 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2432
2433 return dd->rcv_err_status_cnt[40];
2434}
2435
2436static u64 access_rx_lookup_des_part1_unc_err_cnt(
2437 const struct cntr_entry *entry,
2438 void *context, int vl, int mode, u64 data)
2439{
2440 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2441
2442 return dd->rcv_err_status_cnt[39];
2443}
2444
2445static u64 access_rx_rbuf_next_free_buf_cor_err_cnt(
2446 const struct cntr_entry *entry,
2447 void *context, int vl, int mode, u64 data)
2448{
2449 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2450
2451 return dd->rcv_err_status_cnt[38];
2452}
2453
2454static u64 access_rx_rbuf_next_free_buf_unc_err_cnt(
2455 const struct cntr_entry *entry,
2456 void *context, int vl, int mode, u64 data)
2457{
2458 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2459
2460 return dd->rcv_err_status_cnt[37];
2461}
2462
2463static u64 access_rbuf_fl_init_wr_addr_parity_err_cnt(
2464 const struct cntr_entry *entry,
2465 void *context, int vl, int mode, u64 data)
2466{
2467 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2468
2469 return dd->rcv_err_status_cnt[36];
2470}
2471
2472static u64 access_rx_rbuf_fl_initdone_parity_err_cnt(
2473 const struct cntr_entry *entry,
2474 void *context, int vl, int mode, u64 data)
2475{
2476 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2477
2478 return dd->rcv_err_status_cnt[35];
2479}
2480
2481static u64 access_rx_rbuf_fl_write_addr_parity_err_cnt(
2482 const struct cntr_entry *entry,
2483 void *context, int vl, int mode, u64 data)
2484{
2485 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2486
2487 return dd->rcv_err_status_cnt[34];
2488}
2489
2490static u64 access_rx_rbuf_fl_rd_addr_parity_err_cnt(
2491 const struct cntr_entry *entry,
2492 void *context, int vl, int mode, u64 data)
2493{
2494 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2495
2496 return dd->rcv_err_status_cnt[33];
2497}
2498
2499static u64 access_rx_rbuf_empty_err_cnt(const struct cntr_entry *entry,
2500 void *context, int vl, int mode,
2501 u64 data)
2502{
2503 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2504
2505 return dd->rcv_err_status_cnt[32];
2506}
2507
2508static u64 access_rx_rbuf_full_err_cnt(const struct cntr_entry *entry,
2509 void *context, int vl, int mode,
2510 u64 data)
2511{
2512 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2513
2514 return dd->rcv_err_status_cnt[31];
2515}
2516
2517static u64 access_rbuf_bad_lookup_err_cnt(const struct cntr_entry *entry,
2518 void *context, int vl, int mode,
2519 u64 data)
2520{
2521 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2522
2523 return dd->rcv_err_status_cnt[30];
2524}
2525
2526static u64 access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry *entry,
2527 void *context, int vl, int mode,
2528 u64 data)
2529{
2530 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2531
2532 return dd->rcv_err_status_cnt[29];
2533}
2534
2535static u64 access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry *entry,
2536 void *context, int vl,
2537 int mode, u64 data)
2538{
2539 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2540
2541 return dd->rcv_err_status_cnt[28];
2542}
2543
2544static u64 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt(
2545 const struct cntr_entry *entry,
2546 void *context, int vl, int mode, u64 data)
2547{
2548 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2549
2550 return dd->rcv_err_status_cnt[27];
2551}
2552
2553static u64 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt(
2554 const struct cntr_entry *entry,
2555 void *context, int vl, int mode, u64 data)
2556{
2557 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2558
2559 return dd->rcv_err_status_cnt[26];
2560}
2561
2562static u64 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt(
2563 const struct cntr_entry *entry,
2564 void *context, int vl, int mode, u64 data)
2565{
2566 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2567
2568 return dd->rcv_err_status_cnt[25];
2569}
2570
2571static u64 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt(
2572 const struct cntr_entry *entry,
2573 void *context, int vl, int mode, u64 data)
2574{
2575 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2576
2577 return dd->rcv_err_status_cnt[24];
2578}
2579
2580static u64 access_rx_rbuf_csr_q_next_buf_parity_err_cnt(
2581 const struct cntr_entry *entry,
2582 void *context, int vl, int mode, u64 data)
2583{
2584 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2585
2586 return dd->rcv_err_status_cnt[23];
2587}
2588
2589static u64 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt(
2590 const struct cntr_entry *entry,
2591 void *context, int vl, int mode, u64 data)
2592{
2593 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2594
2595 return dd->rcv_err_status_cnt[22];
2596}
2597
2598static u64 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt(
2599 const struct cntr_entry *entry,
2600 void *context, int vl, int mode, u64 data)
2601{
2602 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2603
2604 return dd->rcv_err_status_cnt[21];
2605}
2606
2607static u64 access_rx_rbuf_block_list_read_cor_err_cnt(
2608 const struct cntr_entry *entry,
2609 void *context, int vl, int mode, u64 data)
2610{
2611 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2612
2613 return dd->rcv_err_status_cnt[20];
2614}
2615
2616static u64 access_rx_rbuf_block_list_read_unc_err_cnt(
2617 const struct cntr_entry *entry,
2618 void *context, int vl, int mode, u64 data)
2619{
2620 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2621
2622 return dd->rcv_err_status_cnt[19];
2623}
2624
2625static u64 access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry *entry,
2626 void *context, int vl,
2627 int mode, u64 data)
2628{
2629 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2630
2631 return dd->rcv_err_status_cnt[18];
2632}
2633
2634static u64 access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry *entry,
2635 void *context, int vl,
2636 int mode, u64 data)
2637{
2638 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2639
2640 return dd->rcv_err_status_cnt[17];
2641}
2642
2643static u64 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt(
2644 const struct cntr_entry *entry,
2645 void *context, int vl, int mode, u64 data)
2646{
2647 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2648
2649 return dd->rcv_err_status_cnt[16];
2650}
2651
2652static u64 access_rx_rbuf_lookup_des_reg_unc_err_cnt(
2653 const struct cntr_entry *entry,
2654 void *context, int vl, int mode, u64 data)
2655{
2656 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2657
2658 return dd->rcv_err_status_cnt[15];
2659}
2660
2661static u64 access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry *entry,
2662 void *context, int vl,
2663 int mode, u64 data)
2664{
2665 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2666
2667 return dd->rcv_err_status_cnt[14];
2668}
2669
2670static u64 access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry *entry,
2671 void *context, int vl,
2672 int mode, u64 data)
2673{
2674 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2675
2676 return dd->rcv_err_status_cnt[13];
2677}
2678
2679static u64 access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2680 void *context, int vl, int mode,
2681 u64 data)
2682{
2683 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2684
2685 return dd->rcv_err_status_cnt[12];
2686}
2687
2688static u64 access_rx_dma_flag_cor_err_cnt(const struct cntr_entry *entry,
2689 void *context, int vl, int mode,
2690 u64 data)
2691{
2692 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2693
2694 return dd->rcv_err_status_cnt[11];
2695}
2696
2697static u64 access_rx_dma_flag_unc_err_cnt(const struct cntr_entry *entry,
2698 void *context, int vl, int mode,
2699 u64 data)
2700{
2701 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2702
2703 return dd->rcv_err_status_cnt[10];
2704}
2705
2706static u64 access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry *entry,
2707 void *context, int vl, int mode,
2708 u64 data)
2709{
2710 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2711
2712 return dd->rcv_err_status_cnt[9];
2713}
2714
2715static u64 access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry *entry,
2716 void *context, int vl, int mode,
2717 u64 data)
2718{
2719 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2720
2721 return dd->rcv_err_status_cnt[8];
2722}
2723
2724static u64 access_rx_rcv_qp_map_table_cor_err_cnt(
2725 const struct cntr_entry *entry,
2726 void *context, int vl, int mode, u64 data)
2727{
2728 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2729
2730 return dd->rcv_err_status_cnt[7];
2731}
2732
2733static u64 access_rx_rcv_qp_map_table_unc_err_cnt(
2734 const struct cntr_entry *entry,
2735 void *context, int vl, int mode, u64 data)
2736{
2737 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2738
2739 return dd->rcv_err_status_cnt[6];
2740}
2741
2742static u64 access_rx_rcv_data_cor_err_cnt(const struct cntr_entry *entry,
2743 void *context, int vl, int mode,
2744 u64 data)
2745{
2746 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2747
2748 return dd->rcv_err_status_cnt[5];
2749}
2750
2751static u64 access_rx_rcv_data_unc_err_cnt(const struct cntr_entry *entry,
2752 void *context, int vl, int mode,
2753 u64 data)
2754{
2755 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2756
2757 return dd->rcv_err_status_cnt[4];
2758}
2759
2760static u64 access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry *entry,
2761 void *context, int vl, int mode,
2762 u64 data)
2763{
2764 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2765
2766 return dd->rcv_err_status_cnt[3];
2767}
2768
2769static u64 access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry *entry,
2770 void *context, int vl, int mode,
2771 u64 data)
2772{
2773 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2774
2775 return dd->rcv_err_status_cnt[2];
2776}
2777
2778static u64 access_rx_dc_intf_parity_err_cnt(const struct cntr_entry *entry,
2779 void *context, int vl, int mode,
2780 u64 data)
2781{
2782 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2783
2784 return dd->rcv_err_status_cnt[1];
2785}
2786
2787static u64 access_rx_dma_csr_cor_err_cnt(const struct cntr_entry *entry,
2788 void *context, int vl, int mode,
2789 u64 data)
2790{
2791 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2792
2793 return dd->rcv_err_status_cnt[0];
2794}
2795
2796
2797
2798
2799
2800static u64 access_pio_pec_sop_head_parity_err_cnt(
2801 const struct cntr_entry *entry,
2802 void *context, int vl, int mode, u64 data)
2803{
2804 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2805
2806 return dd->send_pio_err_status_cnt[35];
2807}
2808
2809static u64 access_pio_pcc_sop_head_parity_err_cnt(
2810 const struct cntr_entry *entry,
2811 void *context, int vl, int mode, u64 data)
2812{
2813 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2814
2815 return dd->send_pio_err_status_cnt[34];
2816}
2817
2818static u64 access_pio_last_returned_cnt_parity_err_cnt(
2819 const struct cntr_entry *entry,
2820 void *context, int vl, int mode, u64 data)
2821{
2822 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2823
2824 return dd->send_pio_err_status_cnt[33];
2825}
2826
2827static u64 access_pio_current_free_cnt_parity_err_cnt(
2828 const struct cntr_entry *entry,
2829 void *context, int vl, int mode, u64 data)
2830{
2831 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2832
2833 return dd->send_pio_err_status_cnt[32];
2834}
2835
2836static u64 access_pio_reserved_31_err_cnt(const struct cntr_entry *entry,
2837 void *context, int vl, int mode,
2838 u64 data)
2839{
2840 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2841
2842 return dd->send_pio_err_status_cnt[31];
2843}
2844
2845static u64 access_pio_reserved_30_err_cnt(const struct cntr_entry *entry,
2846 void *context, int vl, int mode,
2847 u64 data)
2848{
2849 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2850
2851 return dd->send_pio_err_status_cnt[30];
2852}
2853
2854static u64 access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry *entry,
2855 void *context, int vl, int mode,
2856 u64 data)
2857{
2858 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2859
2860 return dd->send_pio_err_status_cnt[29];
2861}
2862
2863static u64 access_pio_ppmc_bqc_mem_parity_err_cnt(
2864 const struct cntr_entry *entry,
2865 void *context, int vl, int mode, u64 data)
2866{
2867 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2868
2869 return dd->send_pio_err_status_cnt[28];
2870}
2871
2872static u64 access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry *entry,
2873 void *context, int vl, int mode,
2874 u64 data)
2875{
2876 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2877
2878 return dd->send_pio_err_status_cnt[27];
2879}
2880
2881static u64 access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry *entry,
2882 void *context, int vl, int mode,
2883 u64 data)
2884{
2885 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2886
2887 return dd->send_pio_err_status_cnt[26];
2888}
2889
2890static u64 access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry *entry,
2891 void *context, int vl,
2892 int mode, u64 data)
2893{
2894 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2895
2896 return dd->send_pio_err_status_cnt[25];
2897}
2898
2899static u64 access_pio_block_qw_count_parity_err_cnt(
2900 const struct cntr_entry *entry,
2901 void *context, int vl, int mode, u64 data)
2902{
2903 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2904
2905 return dd->send_pio_err_status_cnt[24];
2906}
2907
2908static u64 access_pio_write_qw_valid_parity_err_cnt(
2909 const struct cntr_entry *entry,
2910 void *context, int vl, int mode, u64 data)
2911{
2912 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2913
2914 return dd->send_pio_err_status_cnt[23];
2915}
2916
2917static u64 access_pio_state_machine_err_cnt(const struct cntr_entry *entry,
2918 void *context, int vl, int mode,
2919 u64 data)
2920{
2921 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2922
2923 return dd->send_pio_err_status_cnt[22];
2924}
2925
2926static u64 access_pio_write_data_parity_err_cnt(const struct cntr_entry *entry,
2927 void *context, int vl,
2928 int mode, u64 data)
2929{
2930 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2931
2932 return dd->send_pio_err_status_cnt[21];
2933}
2934
2935static u64 access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry *entry,
2936 void *context, int vl,
2937 int mode, u64 data)
2938{
2939 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2940
2941 return dd->send_pio_err_status_cnt[20];
2942}
2943
2944static u64 access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry *entry,
2945 void *context, int vl,
2946 int mode, u64 data)
2947{
2948 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2949
2950 return dd->send_pio_err_status_cnt[19];
2951}
2952
2953static u64 access_pio_pkt_evict_sm_or_arb_sm_err_cnt(
2954 const struct cntr_entry *entry,
2955 void *context, int vl, int mode, u64 data)
2956{
2957 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2958
2959 return dd->send_pio_err_status_cnt[18];
2960}
2961
2962static u64 access_pio_init_sm_in_err_cnt(const struct cntr_entry *entry,
2963 void *context, int vl, int mode,
2964 u64 data)
2965{
2966 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2967
2968 return dd->send_pio_err_status_cnt[17];
2969}
2970
2971static u64 access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry *entry,
2972 void *context, int vl, int mode,
2973 u64 data)
2974{
2975 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2976
2977 return dd->send_pio_err_status_cnt[16];
2978}
2979
2980static u64 access_pio_credit_ret_fifo_parity_err_cnt(
2981 const struct cntr_entry *entry,
2982 void *context, int vl, int mode, u64 data)
2983{
2984 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2985
2986 return dd->send_pio_err_status_cnt[15];
2987}
2988
2989static u64 access_pio_v1_len_mem_bank1_cor_err_cnt(
2990 const struct cntr_entry *entry,
2991 void *context, int vl, int mode, u64 data)
2992{
2993 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2994
2995 return dd->send_pio_err_status_cnt[14];
2996}
2997
2998static u64 access_pio_v1_len_mem_bank0_cor_err_cnt(
2999 const struct cntr_entry *entry,
3000 void *context, int vl, int mode, u64 data)
3001{
3002 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3003
3004 return dd->send_pio_err_status_cnt[13];
3005}
3006
3007static u64 access_pio_v1_len_mem_bank1_unc_err_cnt(
3008 const struct cntr_entry *entry,
3009 void *context, int vl, int mode, u64 data)
3010{
3011 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3012
3013 return dd->send_pio_err_status_cnt[12];
3014}
3015
3016static u64 access_pio_v1_len_mem_bank0_unc_err_cnt(
3017 const struct cntr_entry *entry,
3018 void *context, int vl, int mode, u64 data)
3019{
3020 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3021
3022 return dd->send_pio_err_status_cnt[11];
3023}
3024
3025static u64 access_pio_sm_pkt_reset_parity_err_cnt(
3026 const struct cntr_entry *entry,
3027 void *context, int vl, int mode, u64 data)
3028{
3029 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3030
3031 return dd->send_pio_err_status_cnt[10];
3032}
3033
3034static u64 access_pio_pkt_evict_fifo_parity_err_cnt(
3035 const struct cntr_entry *entry,
3036 void *context, int vl, int mode, u64 data)
3037{
3038 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3039
3040 return dd->send_pio_err_status_cnt[9];
3041}
3042
3043static u64 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt(
3044 const struct cntr_entry *entry,
3045 void *context, int vl, int mode, u64 data)
3046{
3047 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3048
3049 return dd->send_pio_err_status_cnt[8];
3050}
3051
3052static u64 access_pio_sbrdctl_crrel_parity_err_cnt(
3053 const struct cntr_entry *entry,
3054 void *context, int vl, int mode, u64 data)
3055{
3056 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3057
3058 return dd->send_pio_err_status_cnt[7];
3059}
3060
3061static u64 access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry *entry,
3062 void *context, int vl, int mode,
3063 u64 data)
3064{
3065 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3066
3067 return dd->send_pio_err_status_cnt[6];
3068}
3069
3070static u64 access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry *entry,
3071 void *context, int vl, int mode,
3072 u64 data)
3073{
3074 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3075
3076 return dd->send_pio_err_status_cnt[5];
3077}
3078
3079static u64 access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry *entry,
3080 void *context, int vl, int mode,
3081 u64 data)
3082{
3083 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3084
3085 return dd->send_pio_err_status_cnt[4];
3086}
3087
3088static u64 access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry *entry,
3089 void *context, int vl, int mode,
3090 u64 data)
3091{
3092 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3093
3094 return dd->send_pio_err_status_cnt[3];
3095}
3096
3097static u64 access_pio_csr_parity_err_cnt(const struct cntr_entry *entry,
3098 void *context, int vl, int mode,
3099 u64 data)
3100{
3101 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3102
3103 return dd->send_pio_err_status_cnt[2];
3104}
3105
3106static u64 access_pio_write_addr_parity_err_cnt(const struct cntr_entry *entry,
3107 void *context, int vl,
3108 int mode, u64 data)
3109{
3110 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3111
3112 return dd->send_pio_err_status_cnt[1];
3113}
3114
3115static u64 access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry *entry,
3116 void *context, int vl, int mode,
3117 u64 data)
3118{
3119 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3120
3121 return dd->send_pio_err_status_cnt[0];
3122}
3123
3124
3125
3126
3127
3128static u64 access_sdma_pcie_req_tracking_cor_err_cnt(
3129 const struct cntr_entry *entry,
3130 void *context, int vl, int mode, u64 data)
3131{
3132 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3133
3134 return dd->send_dma_err_status_cnt[3];
3135}
3136
3137static u64 access_sdma_pcie_req_tracking_unc_err_cnt(
3138 const struct cntr_entry *entry,
3139 void *context, int vl, int mode, u64 data)
3140{
3141 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3142
3143 return dd->send_dma_err_status_cnt[2];
3144}
3145
3146static u64 access_sdma_csr_parity_err_cnt(const struct cntr_entry *entry,
3147 void *context, int vl, int mode,
3148 u64 data)
3149{
3150 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3151
3152 return dd->send_dma_err_status_cnt[1];
3153}
3154
3155static u64 access_sdma_rpy_tag_err_cnt(const struct cntr_entry *entry,
3156 void *context, int vl, int mode,
3157 u64 data)
3158{
3159 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3160
3161 return dd->send_dma_err_status_cnt[0];
3162}
3163
3164
3165
3166
3167
3168static u64 access_tx_read_pio_memory_csr_unc_err_cnt(
3169 const struct cntr_entry *entry,
3170 void *context, int vl, int mode, u64 data)
3171{
3172 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3173
3174 return dd->send_egress_err_status_cnt[63];
3175}
3176
3177static u64 access_tx_read_sdma_memory_csr_err_cnt(
3178 const struct cntr_entry *entry,
3179 void *context, int vl, int mode, u64 data)
3180{
3181 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3182
3183 return dd->send_egress_err_status_cnt[62];
3184}
3185
3186static u64 access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry *entry,
3187 void *context, int vl, int mode,
3188 u64 data)
3189{
3190 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3191
3192 return dd->send_egress_err_status_cnt[61];
3193}
3194
3195static u64 access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry *entry,
3196 void *context, int vl,
3197 int mode, u64 data)
3198{
3199 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3200
3201 return dd->send_egress_err_status_cnt[60];
3202}
3203
3204static u64 access_tx_read_sdma_memory_cor_err_cnt(
3205 const struct cntr_entry *entry,
3206 void *context, int vl, int mode, u64 data)
3207{
3208 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3209
3210 return dd->send_egress_err_status_cnt[59];
3211}
3212
3213static u64 access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry *entry,
3214 void *context, int vl, int mode,
3215 u64 data)
3216{
3217 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3218
3219 return dd->send_egress_err_status_cnt[58];
3220}
3221
3222static u64 access_tx_credit_overrun_err_cnt(const struct cntr_entry *entry,
3223 void *context, int vl, int mode,
3224 u64 data)
3225{
3226 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3227
3228 return dd->send_egress_err_status_cnt[57];
3229}
3230
3231static u64 access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry *entry,
3232 void *context, int vl, int mode,
3233 u64 data)
3234{
3235 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3236
3237 return dd->send_egress_err_status_cnt[56];
3238}
3239
3240static u64 access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry *entry,
3241 void *context, int vl, int mode,
3242 u64 data)
3243{
3244 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3245
3246 return dd->send_egress_err_status_cnt[55];
3247}
3248
3249static u64 access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry *entry,
3250 void *context, int vl, int mode,
3251 u64 data)
3252{
3253 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3254
3255 return dd->send_egress_err_status_cnt[54];
3256}
3257
3258static u64 access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry *entry,
3259 void *context, int vl, int mode,
3260 u64 data)
3261{
3262 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3263
3264 return dd->send_egress_err_status_cnt[53];
3265}
3266
3267static u64 access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry *entry,
3268 void *context, int vl, int mode,
3269 u64 data)
3270{
3271 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3272
3273 return dd->send_egress_err_status_cnt[52];
3274}
3275
3276static u64 access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry *entry,
3277 void *context, int vl, int mode,
3278 u64 data)
3279{
3280 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3281
3282 return dd->send_egress_err_status_cnt[51];
3283}
3284
3285static u64 access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry *entry,
3286 void *context, int vl, int mode,
3287 u64 data)
3288{
3289 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3290
3291 return dd->send_egress_err_status_cnt[50];
3292}
3293
3294static u64 access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry *entry,
3295 void *context, int vl, int mode,
3296 u64 data)
3297{
3298 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3299
3300 return dd->send_egress_err_status_cnt[49];
3301}
3302
3303static u64 access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry *entry,
3304 void *context, int vl, int mode,
3305 u64 data)
3306{
3307 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3308
3309 return dd->send_egress_err_status_cnt[48];
3310}
3311
3312static u64 access_tx_credit_return_vl_err_cnt(const struct cntr_entry *entry,
3313 void *context, int vl, int mode,
3314 u64 data)
3315{
3316 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3317
3318 return dd->send_egress_err_status_cnt[47];
3319}
3320
3321static u64 access_tx_hcrc_insertion_err_cnt(const struct cntr_entry *entry,
3322 void *context, int vl, int mode,
3323 u64 data)
3324{
3325 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3326
3327 return dd->send_egress_err_status_cnt[46];
3328}
3329
3330static u64 access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry *entry,
3331 void *context, int vl, int mode,
3332 u64 data)
3333{
3334 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3335
3336 return dd->send_egress_err_status_cnt[45];
3337}
3338
3339static u64 access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry *entry,
3340 void *context, int vl,
3341 int mode, u64 data)
3342{
3343 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3344
3345 return dd->send_egress_err_status_cnt[44];
3346}
3347
3348static u64 access_tx_read_sdma_memory_unc_err_cnt(
3349 const struct cntr_entry *entry,
3350 void *context, int vl, int mode, u64 data)
3351{
3352 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3353
3354 return dd->send_egress_err_status_cnt[43];
3355}
3356
3357static u64 access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry *entry,
3358 void *context, int vl, int mode,
3359 u64 data)
3360{
3361 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3362
3363 return dd->send_egress_err_status_cnt[42];
3364}
3365
3366static u64 access_tx_credit_return_partiy_err_cnt(
3367 const struct cntr_entry *entry,
3368 void *context, int vl, int mode, u64 data)
3369{
3370 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3371
3372 return dd->send_egress_err_status_cnt[41];
3373}
3374
3375static u64 access_tx_launch_fifo8_unc_or_parity_err_cnt(
3376 const struct cntr_entry *entry,
3377 void *context, int vl, int mode, u64 data)
3378{
3379 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3380
3381 return dd->send_egress_err_status_cnt[40];
3382}
3383
3384static u64 access_tx_launch_fifo7_unc_or_parity_err_cnt(
3385 const struct cntr_entry *entry,
3386 void *context, int vl, int mode, u64 data)
3387{
3388 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3389
3390 return dd->send_egress_err_status_cnt[39];
3391}
3392
3393static u64 access_tx_launch_fifo6_unc_or_parity_err_cnt(
3394 const struct cntr_entry *entry,
3395 void *context, int vl, int mode, u64 data)
3396{
3397 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3398
3399 return dd->send_egress_err_status_cnt[38];
3400}
3401
3402static u64 access_tx_launch_fifo5_unc_or_parity_err_cnt(
3403 const struct cntr_entry *entry,
3404 void *context, int vl, int mode, u64 data)
3405{
3406 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3407
3408 return dd->send_egress_err_status_cnt[37];
3409}
3410
3411static u64 access_tx_launch_fifo4_unc_or_parity_err_cnt(
3412 const struct cntr_entry *entry,
3413 void *context, int vl, int mode, u64 data)
3414{
3415 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3416
3417 return dd->send_egress_err_status_cnt[36];
3418}
3419
3420static u64 access_tx_launch_fifo3_unc_or_parity_err_cnt(
3421 const struct cntr_entry *entry,
3422 void *context, int vl, int mode, u64 data)
3423{
3424 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3425
3426 return dd->send_egress_err_status_cnt[35];
3427}
3428
3429static u64 access_tx_launch_fifo2_unc_or_parity_err_cnt(
3430 const struct cntr_entry *entry,
3431 void *context, int vl, int mode, u64 data)
3432{
3433 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3434
3435 return dd->send_egress_err_status_cnt[34];
3436}
3437
3438static u64 access_tx_launch_fifo1_unc_or_parity_err_cnt(
3439 const struct cntr_entry *entry,
3440 void *context, int vl, int mode, u64 data)
3441{
3442 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3443
3444 return dd->send_egress_err_status_cnt[33];
3445}
3446
3447static u64 access_tx_launch_fifo0_unc_or_parity_err_cnt(
3448 const struct cntr_entry *entry,
3449 void *context, int vl, int mode, u64 data)
3450{
3451 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3452
3453 return dd->send_egress_err_status_cnt[32];
3454}
3455
3456static u64 access_tx_sdma15_disallowed_packet_err_cnt(
3457 const struct cntr_entry *entry,
3458 void *context, int vl, int mode, u64 data)
3459{
3460 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3461
3462 return dd->send_egress_err_status_cnt[31];
3463}
3464
3465static u64 access_tx_sdma14_disallowed_packet_err_cnt(
3466 const struct cntr_entry *entry,
3467 void *context, int vl, int mode, u64 data)
3468{
3469 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3470
3471 return dd->send_egress_err_status_cnt[30];
3472}
3473
3474static u64 access_tx_sdma13_disallowed_packet_err_cnt(
3475 const struct cntr_entry *entry,
3476 void *context, int vl, int mode, u64 data)
3477{
3478 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3479
3480 return dd->send_egress_err_status_cnt[29];
3481}
3482
3483static u64 access_tx_sdma12_disallowed_packet_err_cnt(
3484 const struct cntr_entry *entry,
3485 void *context, int vl, int mode, u64 data)
3486{
3487 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3488
3489 return dd->send_egress_err_status_cnt[28];
3490}
3491
3492static u64 access_tx_sdma11_disallowed_packet_err_cnt(
3493 const struct cntr_entry *entry,
3494 void *context, int vl, int mode, u64 data)
3495{
3496 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3497
3498 return dd->send_egress_err_status_cnt[27];
3499}
3500
3501static u64 access_tx_sdma10_disallowed_packet_err_cnt(
3502 const struct cntr_entry *entry,
3503 void *context, int vl, int mode, u64 data)
3504{
3505 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3506
3507 return dd->send_egress_err_status_cnt[26];
3508}
3509
3510static u64 access_tx_sdma9_disallowed_packet_err_cnt(
3511 const struct cntr_entry *entry,
3512 void *context, int vl, int mode, u64 data)
3513{
3514 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3515
3516 return dd->send_egress_err_status_cnt[25];
3517}
3518
3519static u64 access_tx_sdma8_disallowed_packet_err_cnt(
3520 const struct cntr_entry *entry,
3521 void *context, int vl, int mode, u64 data)
3522{
3523 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3524
3525 return dd->send_egress_err_status_cnt[24];
3526}
3527
3528static u64 access_tx_sdma7_disallowed_packet_err_cnt(
3529 const struct cntr_entry *entry,
3530 void *context, int vl, int mode, u64 data)
3531{
3532 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3533
3534 return dd->send_egress_err_status_cnt[23];
3535}
3536
3537static u64 access_tx_sdma6_disallowed_packet_err_cnt(
3538 const struct cntr_entry *entry,
3539 void *context, int vl, int mode, u64 data)
3540{
3541 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3542
3543 return dd->send_egress_err_status_cnt[22];
3544}
3545
3546static u64 access_tx_sdma5_disallowed_packet_err_cnt(
3547 const struct cntr_entry *entry,
3548 void *context, int vl, int mode, u64 data)
3549{
3550 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3551
3552 return dd->send_egress_err_status_cnt[21];
3553}
3554
3555static u64 access_tx_sdma4_disallowed_packet_err_cnt(
3556 const struct cntr_entry *entry,
3557 void *context, int vl, int mode, u64 data)
3558{
3559 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3560
3561 return dd->send_egress_err_status_cnt[20];
3562}
3563
3564static u64 access_tx_sdma3_disallowed_packet_err_cnt(
3565 const struct cntr_entry *entry,
3566 void *context, int vl, int mode, u64 data)
3567{
3568 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3569
3570 return dd->send_egress_err_status_cnt[19];
3571}
3572
3573static u64 access_tx_sdma2_disallowed_packet_err_cnt(
3574 const struct cntr_entry *entry,
3575 void *context, int vl, int mode, u64 data)
3576{
3577 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3578
3579 return dd->send_egress_err_status_cnt[18];
3580}
3581
3582static u64 access_tx_sdma1_disallowed_packet_err_cnt(
3583 const struct cntr_entry *entry,
3584 void *context, int vl, int mode, u64 data)
3585{
3586 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3587
3588 return dd->send_egress_err_status_cnt[17];
3589}
3590
3591static u64 access_tx_sdma0_disallowed_packet_err_cnt(
3592 const struct cntr_entry *entry,
3593 void *context, int vl, int mode, u64 data)
3594{
3595 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3596
3597 return dd->send_egress_err_status_cnt[16];
3598}
3599
3600static u64 access_tx_config_parity_err_cnt(const struct cntr_entry *entry,
3601 void *context, int vl, int mode,
3602 u64 data)
3603{
3604 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3605
3606 return dd->send_egress_err_status_cnt[15];
3607}
3608
3609static u64 access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry *entry,
3610 void *context, int vl,
3611 int mode, u64 data)
3612{
3613 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3614
3615 return dd->send_egress_err_status_cnt[14];
3616}
3617
3618static u64 access_tx_launch_csr_parity_err_cnt(const struct cntr_entry *entry,
3619 void *context, int vl, int mode,
3620 u64 data)
3621{
3622 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3623
3624 return dd->send_egress_err_status_cnt[13];
3625}
3626
3627static u64 access_tx_illegal_vl_err_cnt(const struct cntr_entry *entry,
3628 void *context, int vl, int mode,
3629 u64 data)
3630{
3631 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3632
3633 return dd->send_egress_err_status_cnt[12];
3634}
3635
3636static u64 access_tx_sbrd_ctl_state_machine_parity_err_cnt(
3637 const struct cntr_entry *entry,
3638 void *context, int vl, int mode, u64 data)
3639{
3640 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3641
3642 return dd->send_egress_err_status_cnt[11];
3643}
3644
3645static u64 access_egress_reserved_10_err_cnt(const struct cntr_entry *entry,
3646 void *context, int vl, int mode,
3647 u64 data)
3648{
3649 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3650
3651 return dd->send_egress_err_status_cnt[10];
3652}
3653
3654static u64 access_egress_reserved_9_err_cnt(const struct cntr_entry *entry,
3655 void *context, int vl, int mode,
3656 u64 data)
3657{
3658 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3659
3660 return dd->send_egress_err_status_cnt[9];
3661}
3662
3663static u64 access_tx_sdma_launch_intf_parity_err_cnt(
3664 const struct cntr_entry *entry,
3665 void *context, int vl, int mode, u64 data)
3666{
3667 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3668
3669 return dd->send_egress_err_status_cnt[8];
3670}
3671
3672static u64 access_tx_pio_launch_intf_parity_err_cnt(
3673 const struct cntr_entry *entry,
3674 void *context, int vl, int mode, u64 data)
3675{
3676 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3677
3678 return dd->send_egress_err_status_cnt[7];
3679}
3680
3681static u64 access_egress_reserved_6_err_cnt(const struct cntr_entry *entry,
3682 void *context, int vl, int mode,
3683 u64 data)
3684{
3685 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3686
3687 return dd->send_egress_err_status_cnt[6];
3688}
3689
3690static u64 access_tx_incorrect_link_state_err_cnt(
3691 const struct cntr_entry *entry,
3692 void *context, int vl, int mode, u64 data)
3693{
3694 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3695
3696 return dd->send_egress_err_status_cnt[5];
3697}
3698
3699static u64 access_tx_linkdown_err_cnt(const struct cntr_entry *entry,
3700 void *context, int vl, int mode,
3701 u64 data)
3702{
3703 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3704
3705 return dd->send_egress_err_status_cnt[4];
3706}
3707
3708static u64 access_tx_egress_fifi_underrun_or_parity_err_cnt(
3709 const struct cntr_entry *entry,
3710 void *context, int vl, int mode, u64 data)
3711{
3712 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3713
3714 return dd->send_egress_err_status_cnt[3];
3715}
3716
3717static u64 access_egress_reserved_2_err_cnt(const struct cntr_entry *entry,
3718 void *context, int vl, int mode,
3719 u64 data)
3720{
3721 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3722
3723 return dd->send_egress_err_status_cnt[2];
3724}
3725
3726static u64 access_tx_pkt_integrity_mem_unc_err_cnt(
3727 const struct cntr_entry *entry,
3728 void *context, int vl, int mode, u64 data)
3729{
3730 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3731
3732 return dd->send_egress_err_status_cnt[1];
3733}
3734
3735static u64 access_tx_pkt_integrity_mem_cor_err_cnt(
3736 const struct cntr_entry *entry,
3737 void *context, int vl, int mode, u64 data)
3738{
3739 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3740
3741 return dd->send_egress_err_status_cnt[0];
3742}
3743
3744
3745
3746
3747
3748static u64 access_send_csr_write_bad_addr_err_cnt(
3749 const struct cntr_entry *entry,
3750 void *context, int vl, int mode, u64 data)
3751{
3752 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3753
3754 return dd->send_err_status_cnt[2];
3755}
3756
3757static u64 access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
3758 void *context, int vl,
3759 int mode, u64 data)
3760{
3761 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3762
3763 return dd->send_err_status_cnt[1];
3764}
3765
3766static u64 access_send_csr_parity_cnt(const struct cntr_entry *entry,
3767 void *context, int vl, int mode,
3768 u64 data)
3769{
3770 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3771
3772 return dd->send_err_status_cnt[0];
3773}
3774
3775
3776
3777
3778
3779static u64 access_pio_write_out_of_bounds_err_cnt(
3780 const struct cntr_entry *entry,
3781 void *context, int vl, int mode, u64 data)
3782{
3783 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3784
3785 return dd->sw_ctxt_err_status_cnt[4];
3786}
3787
3788static u64 access_pio_write_overflow_err_cnt(const struct cntr_entry *entry,
3789 void *context, int vl, int mode,
3790 u64 data)
3791{
3792 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3793
3794 return dd->sw_ctxt_err_status_cnt[3];
3795}
3796
3797static u64 access_pio_write_crosses_boundary_err_cnt(
3798 const struct cntr_entry *entry,
3799 void *context, int vl, int mode, u64 data)
3800{
3801 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3802
3803 return dd->sw_ctxt_err_status_cnt[2];
3804}
3805
3806static u64 access_pio_disallowed_packet_err_cnt(const struct cntr_entry *entry,
3807 void *context, int vl,
3808 int mode, u64 data)
3809{
3810 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3811
3812 return dd->sw_ctxt_err_status_cnt[1];
3813}
3814
3815static u64 access_pio_inconsistent_sop_err_cnt(const struct cntr_entry *entry,
3816 void *context, int vl, int mode,
3817 u64 data)
3818{
3819 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3820
3821 return dd->sw_ctxt_err_status_cnt[0];
3822}
3823
3824
3825
3826
3827
3828static u64 access_sdma_header_request_fifo_cor_err_cnt(
3829 const struct cntr_entry *entry,
3830 void *context, int vl, int mode, u64 data)
3831{
3832 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3833
3834 return dd->sw_send_dma_eng_err_status_cnt[23];
3835}
3836
3837static u64 access_sdma_header_storage_cor_err_cnt(
3838 const struct cntr_entry *entry,
3839 void *context, int vl, int mode, u64 data)
3840{
3841 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3842
3843 return dd->sw_send_dma_eng_err_status_cnt[22];
3844}
3845
3846static u64 access_sdma_packet_tracking_cor_err_cnt(
3847 const struct cntr_entry *entry,
3848 void *context, int vl, int mode, u64 data)
3849{
3850 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3851
3852 return dd->sw_send_dma_eng_err_status_cnt[21];
3853}
3854
3855static u64 access_sdma_assembly_cor_err_cnt(const struct cntr_entry *entry,
3856 void *context, int vl, int mode,
3857 u64 data)
3858{
3859 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3860
3861 return dd->sw_send_dma_eng_err_status_cnt[20];
3862}
3863
3864static u64 access_sdma_desc_table_cor_err_cnt(const struct cntr_entry *entry,
3865 void *context, int vl, int mode,
3866 u64 data)
3867{
3868 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3869
3870 return dd->sw_send_dma_eng_err_status_cnt[19];
3871}
3872
3873static u64 access_sdma_header_request_fifo_unc_err_cnt(
3874 const struct cntr_entry *entry,
3875 void *context, int vl, int mode, u64 data)
3876{
3877 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3878
3879 return dd->sw_send_dma_eng_err_status_cnt[18];
3880}
3881
3882static u64 access_sdma_header_storage_unc_err_cnt(
3883 const struct cntr_entry *entry,
3884 void *context, int vl, int mode, u64 data)
3885{
3886 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3887
3888 return dd->sw_send_dma_eng_err_status_cnt[17];
3889}
3890
3891static u64 access_sdma_packet_tracking_unc_err_cnt(
3892 const struct cntr_entry *entry,
3893 void *context, int vl, int mode, u64 data)
3894{
3895 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3896
3897 return dd->sw_send_dma_eng_err_status_cnt[16];
3898}
3899
3900static u64 access_sdma_assembly_unc_err_cnt(const struct cntr_entry *entry,
3901 void *context, int vl, int mode,
3902 u64 data)
3903{
3904 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3905
3906 return dd->sw_send_dma_eng_err_status_cnt[15];
3907}
3908
3909static u64 access_sdma_desc_table_unc_err_cnt(const struct cntr_entry *entry,
3910 void *context, int vl, int mode,
3911 u64 data)
3912{
3913 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3914
3915 return dd->sw_send_dma_eng_err_status_cnt[14];
3916}
3917
3918static u64 access_sdma_timeout_err_cnt(const struct cntr_entry *entry,
3919 void *context, int vl, int mode,
3920 u64 data)
3921{
3922 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3923
3924 return dd->sw_send_dma_eng_err_status_cnt[13];
3925}
3926
3927static u64 access_sdma_header_length_err_cnt(const struct cntr_entry *entry,
3928 void *context, int vl, int mode,
3929 u64 data)
3930{
3931 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3932
3933 return dd->sw_send_dma_eng_err_status_cnt[12];
3934}
3935
3936static u64 access_sdma_header_address_err_cnt(const struct cntr_entry *entry,
3937 void *context, int vl, int mode,
3938 u64 data)
3939{
3940 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3941
3942 return dd->sw_send_dma_eng_err_status_cnt[11];
3943}
3944
3945static u64 access_sdma_header_select_err_cnt(const struct cntr_entry *entry,
3946 void *context, int vl, int mode,
3947 u64 data)
3948{
3949 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3950
3951 return dd->sw_send_dma_eng_err_status_cnt[10];
3952}
3953
3954static u64 access_sdma_reserved_9_err_cnt(const struct cntr_entry *entry,
3955 void *context, int vl, int mode,
3956 u64 data)
3957{
3958 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3959
3960 return dd->sw_send_dma_eng_err_status_cnt[9];
3961}
3962
3963static u64 access_sdma_packet_desc_overflow_err_cnt(
3964 const struct cntr_entry *entry,
3965 void *context, int vl, int mode, u64 data)
3966{
3967 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3968
3969 return dd->sw_send_dma_eng_err_status_cnt[8];
3970}
3971
3972static u64 access_sdma_length_mismatch_err_cnt(const struct cntr_entry *entry,
3973 void *context, int vl,
3974 int mode, u64 data)
3975{
3976 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3977
3978 return dd->sw_send_dma_eng_err_status_cnt[7];
3979}
3980
3981static u64 access_sdma_halt_err_cnt(const struct cntr_entry *entry,
3982 void *context, int vl, int mode, u64 data)
3983{
3984 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3985
3986 return dd->sw_send_dma_eng_err_status_cnt[6];
3987}
3988
3989static u64 access_sdma_mem_read_err_cnt(const struct cntr_entry *entry,
3990 void *context, int vl, int mode,
3991 u64 data)
3992{
3993 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3994
3995 return dd->sw_send_dma_eng_err_status_cnt[5];
3996}
3997
3998static u64 access_sdma_first_desc_err_cnt(const struct cntr_entry *entry,
3999 void *context, int vl, int mode,
4000 u64 data)
4001{
4002 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4003
4004 return dd->sw_send_dma_eng_err_status_cnt[4];
4005}
4006
4007static u64 access_sdma_tail_out_of_bounds_err_cnt(
4008 const struct cntr_entry *entry,
4009 void *context, int vl, int mode, u64 data)
4010{
4011 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4012
4013 return dd->sw_send_dma_eng_err_status_cnt[3];
4014}
4015
4016static u64 access_sdma_too_long_err_cnt(const struct cntr_entry *entry,
4017 void *context, int vl, int mode,
4018 u64 data)
4019{
4020 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4021
4022 return dd->sw_send_dma_eng_err_status_cnt[2];
4023}
4024
4025static u64 access_sdma_gen_mismatch_err_cnt(const struct cntr_entry *entry,
4026 void *context, int vl, int mode,
4027 u64 data)
4028{
4029 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4030
4031 return dd->sw_send_dma_eng_err_status_cnt[1];
4032}
4033
4034static u64 access_sdma_wrong_dw_err_cnt(const struct cntr_entry *entry,
4035 void *context, int vl, int mode,
4036 u64 data)
4037{
4038 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4039
4040 return dd->sw_send_dma_eng_err_status_cnt[0];
4041}
4042
4043static u64 access_dc_rcv_err_cnt(const struct cntr_entry *entry,
4044 void *context, int vl, int mode,
4045 u64 data)
4046{
4047 struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4048
4049 u64 val = 0;
4050 u64 csr = entry->csr;
4051
4052 val = read_write_csr(dd, csr, mode, data);
4053 if (mode == CNTR_MODE_R) {
4054 val = val > CNTR_MAX - dd->sw_rcv_bypass_packet_errors ?
4055 CNTR_MAX : val + dd->sw_rcv_bypass_packet_errors;
4056 } else if (mode == CNTR_MODE_W) {
4057 dd->sw_rcv_bypass_packet_errors = 0;
4058 } else {
4059 dd_dev_err(dd, "Invalid cntr register access mode");
4060 return 0;
4061 }
4062 return val;
4063}
4064
4065#define def_access_sw_cpu(cntr) \
4066static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry, \
4067 void *context, int vl, int mode, u64 data) \
4068{ \
4069 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \
4070 return read_write_cpu(ppd->dd, &ppd->ibport_data.rvp.z_ ##cntr, \
4071 ppd->ibport_data.rvp.cntr, vl, \
4072 mode, data); \
4073}
4074
4075def_access_sw_cpu(rc_acks);
4076def_access_sw_cpu(rc_qacks);
4077def_access_sw_cpu(rc_delayed_comp);
4078
4079#define def_access_ibp_counter(cntr) \
4080static u64 access_ibp_##cntr(const struct cntr_entry *entry, \
4081 void *context, int vl, int mode, u64 data) \
4082{ \
4083 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \
4084 \
4085 if (vl != CNTR_INVALID_VL) \
4086 return 0; \
4087 \
4088 return read_write_sw(ppd->dd, &ppd->ibport_data.rvp.n_ ##cntr, \
4089 mode, data); \
4090}
4091
4092def_access_ibp_counter(loop_pkts);
4093def_access_ibp_counter(rc_resends);
4094def_access_ibp_counter(rnr_naks);
4095def_access_ibp_counter(other_naks);
4096def_access_ibp_counter(rc_timeouts);
4097def_access_ibp_counter(pkt_drops);
4098def_access_ibp_counter(dmawait);
4099def_access_ibp_counter(rc_seqnak);
4100def_access_ibp_counter(rc_dupreq);
4101def_access_ibp_counter(rdma_seq);
4102def_access_ibp_counter(unaligned);
4103def_access_ibp_counter(seq_naks);
4104
4105static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = {
4106[C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH),
4107[C_RX_LEN_ERR] = RXE32_DEV_CNTR_ELEM(RxLenErr, RCV_LENGTH_ERR_CNT, CNTR_SYNTH),
4108[C_RX_ICRC_ERR] = RXE32_DEV_CNTR_ELEM(RxICrcErr, RCV_ICRC_ERR_CNT, CNTR_SYNTH),
4109[C_RX_EBP] = RXE32_DEV_CNTR_ELEM(RxEbpCnt, RCV_EBP_CNT, CNTR_SYNTH),
4110[C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT,
4111 CNTR_NORMAL),
4112[C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT,
4113 CNTR_NORMAL),
4114[C_RX_TID_FLGMS] = RXE32_DEV_CNTR_ELEM(RxTidFLGMs,
4115 RCV_TID_FLOW_GEN_MISMATCH_CNT,
4116 CNTR_NORMAL),
4117[C_RX_CTX_EGRS] = RXE32_DEV_CNTR_ELEM(RxCtxEgrS, RCV_CONTEXT_EGR_STALL,
4118 CNTR_NORMAL),
4119[C_RCV_TID_FLSMS] = RXE32_DEV_CNTR_ELEM(RxTidFLSMs,
4120 RCV_TID_FLOW_SEQ_MISMATCH_CNT, CNTR_NORMAL),
4121[C_CCE_PCI_CR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciCrSt,
4122 CCE_PCIE_POSTED_CRDT_STALL_CNT, CNTR_NORMAL),
4123[C_CCE_PCI_TR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciTrSt, CCE_PCIE_TRGT_STALL_CNT,
4124 CNTR_NORMAL),
4125[C_CCE_PIO_WR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePioWrSt, CCE_PIO_WR_STALL_CNT,
4126 CNTR_NORMAL),
4127[C_CCE_ERR_INT] = CCE_INT_DEV_CNTR_ELEM(CceErrInt, CCE_ERR_INT_CNT,
4128 CNTR_NORMAL),
4129[C_CCE_SDMA_INT] = CCE_INT_DEV_CNTR_ELEM(CceSdmaInt, CCE_SDMA_INT_CNT,
4130 CNTR_NORMAL),
4131[C_CCE_MISC_INT] = CCE_INT_DEV_CNTR_ELEM(CceMiscInt, CCE_MISC_INT_CNT,
4132 CNTR_NORMAL),
4133[C_CCE_RCV_AV_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvAvInt, CCE_RCV_AVAIL_INT_CNT,
4134 CNTR_NORMAL),
4135[C_CCE_RCV_URG_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvUrgInt,
4136 CCE_RCV_URGENT_INT_CNT, CNTR_NORMAL),
4137[C_CCE_SEND_CR_INT] = CCE_INT_DEV_CNTR_ELEM(CceSndCrInt,
4138 CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL),
4139[C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT,
4140 CNTR_SYNTH),
4141[C_DC_RCV_ERR] = CNTR_ELEM("DcRecvErr", DCC_ERR_PORTRCV_ERR_CNT, 0, CNTR_SYNTH,
4142 access_dc_rcv_err_cnt),
4143[C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT,
4144 CNTR_SYNTH),
4145[C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT,
4146 CNTR_SYNTH),
4147[C_DC_DROPPED_PKT] = DC_PERF_CNTR(DcDroppedPkt, DCC_ERR_DROPPED_PKT_CNT,
4148 CNTR_SYNTH),
4149[C_DC_MC_XMIT_PKTS] = DC_PERF_CNTR(DcMcXmitPkts,
4150 DCC_PRF_PORT_XMIT_MULTICAST_CNT, CNTR_SYNTH),
4151[C_DC_MC_RCV_PKTS] = DC_PERF_CNTR(DcMcRcvPkts,
4152 DCC_PRF_PORT_RCV_MULTICAST_PKT_CNT,
4153 CNTR_SYNTH),
4154[C_DC_XMIT_CERR] = DC_PERF_CNTR(DcXmitCorr,
4155 DCC_PRF_PORT_XMIT_CORRECTABLE_CNT, CNTR_SYNTH),
4156[C_DC_RCV_CERR] = DC_PERF_CNTR(DcRcvCorrCnt, DCC_PRF_PORT_RCV_CORRECTABLE_CNT,
4157 CNTR_SYNTH),
4158[C_DC_RCV_FCC] = DC_PERF_CNTR(DcRxFCntl, DCC_PRF_RX_FLOW_CRTL_CNT,
4159 CNTR_SYNTH),
4160[C_DC_XMIT_FCC] = DC_PERF_CNTR(DcXmitFCntl, DCC_PRF_TX_FLOW_CRTL_CNT,
4161 CNTR_SYNTH),
4162[C_DC_XMIT_FLITS] = DC_PERF_CNTR(DcXmitFlits, DCC_PRF_PORT_XMIT_DATA_CNT,
4163 CNTR_SYNTH),
4164[C_DC_RCV_FLITS] = DC_PERF_CNTR(DcRcvFlits, DCC_PRF_PORT_RCV_DATA_CNT,
4165 CNTR_SYNTH),
4166[C_DC_XMIT_PKTS] = DC_PERF_CNTR(DcXmitPkts, DCC_PRF_PORT_XMIT_PKTS_CNT,
4167 CNTR_SYNTH),
4168[C_DC_RCV_PKTS] = DC_PERF_CNTR(DcRcvPkts, DCC_PRF_PORT_RCV_PKTS_CNT,
4169 CNTR_SYNTH),
4170[C_DC_RX_FLIT_VL] = DC_PERF_CNTR(DcRxFlitVl, DCC_PRF_PORT_VL_RCV_DATA_CNT,
4171 CNTR_SYNTH | CNTR_VL),
4172[C_DC_RX_PKT_VL] = DC_PERF_CNTR(DcRxPktVl, DCC_PRF_PORT_VL_RCV_PKTS_CNT,
4173 CNTR_SYNTH | CNTR_VL),
4174[C_DC_RCV_FCN] = DC_PERF_CNTR(DcRcvFcn, DCC_PRF_PORT_RCV_FECN_CNT, CNTR_SYNTH),
4175[C_DC_RCV_FCN_VL] = DC_PERF_CNTR(DcRcvFcnVl, DCC_PRF_PORT_VL_RCV_FECN_CNT,
4176 CNTR_SYNTH | CNTR_VL),
4177[C_DC_RCV_BCN] = DC_PERF_CNTR(DcRcvBcn, DCC_PRF_PORT_RCV_BECN_CNT, CNTR_SYNTH),
4178[C_DC_RCV_BCN_VL] = DC_PERF_CNTR(DcRcvBcnVl, DCC_PRF_PORT_VL_RCV_BECN_CNT,
4179 CNTR_SYNTH | CNTR_VL),
4180[C_DC_RCV_BBL] = DC_PERF_CNTR(DcRcvBbl, DCC_PRF_PORT_RCV_BUBBLE_CNT,
4181 CNTR_SYNTH),
4182[C_DC_RCV_BBL_VL] = DC_PERF_CNTR(DcRcvBblVl, DCC_PRF_PORT_VL_RCV_BUBBLE_CNT,
4183 CNTR_SYNTH | CNTR_VL),
4184[C_DC_MARK_FECN] = DC_PERF_CNTR(DcMarkFcn, DCC_PRF_PORT_MARK_FECN_CNT,
4185 CNTR_SYNTH),
4186[C_DC_MARK_FECN_VL] = DC_PERF_CNTR(DcMarkFcnVl, DCC_PRF_PORT_VL_MARK_FECN_CNT,
4187 CNTR_SYNTH | CNTR_VL),
4188[C_DC_TOTAL_CRC] =
4189 DC_PERF_CNTR_LCB(DcTotCrc, DC_LCB_ERR_INFO_TOTAL_CRC_ERR,
4190 CNTR_SYNTH),
4191[C_DC_CRC_LN0] = DC_PERF_CNTR_LCB(DcCrcLn0, DC_LCB_ERR_INFO_CRC_ERR_LN0,
4192 CNTR_SYNTH),
4193[C_DC_CRC_LN1] = DC_PERF_CNTR_LCB(DcCrcLn1, DC_LCB_ERR_INFO_CRC_ERR_LN1,
4194 CNTR_SYNTH),
4195[C_DC_CRC_LN2] = DC_PERF_CNTR_LCB(DcCrcLn2, DC_LCB_ERR_INFO_CRC_ERR_LN2,
4196 CNTR_SYNTH),
4197[C_DC_CRC_LN3] = DC_PERF_CNTR_LCB(DcCrcLn3, DC_LCB_ERR_INFO_CRC_ERR_LN3,
4198 CNTR_SYNTH),
4199[C_DC_CRC_MULT_LN] =
4200 DC_PERF_CNTR_LCB(DcMultLn, DC_LCB_ERR_INFO_CRC_ERR_MULTI_LN,
4201 CNTR_SYNTH),
4202[C_DC_TX_REPLAY] = DC_PERF_CNTR_LCB(DcTxReplay, DC_LCB_ERR_INFO_TX_REPLAY_CNT,
4203 CNTR_SYNTH),
4204[C_DC_RX_REPLAY] = DC_PERF_CNTR_LCB(DcRxReplay, DC_LCB_ERR_INFO_RX_REPLAY_CNT,
4205 CNTR_SYNTH),
4206[C_DC_SEQ_CRC_CNT] =
4207 DC_PERF_CNTR_LCB(DcLinkSeqCrc, DC_LCB_ERR_INFO_SEQ_CRC_CNT,
4208 CNTR_SYNTH),
4209[C_DC_ESC0_ONLY_CNT] =
4210 DC_PERF_CNTR_LCB(DcEsc0, DC_LCB_ERR_INFO_ESCAPE_0_ONLY_CNT,
4211 CNTR_SYNTH),
4212[C_DC_ESC0_PLUS1_CNT] =
4213 DC_PERF_CNTR_LCB(DcEsc1, DC_LCB_ERR_INFO_ESCAPE_0_PLUS1_CNT,
4214 CNTR_SYNTH),
4215[C_DC_ESC0_PLUS2_CNT] =
4216 DC_PERF_CNTR_LCB(DcEsc0Plus2, DC_LCB_ERR_INFO_ESCAPE_0_PLUS2_CNT,
4217 CNTR_SYNTH),
4218[C_DC_REINIT_FROM_PEER_CNT] =
4219 DC_PERF_CNTR_LCB(DcReinitPeer, DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT,
4220 CNTR_SYNTH),
4221[C_DC_SBE_CNT] = DC_PERF_CNTR_LCB(DcSbe, DC_LCB_ERR_INFO_SBE_CNT,
4222 CNTR_SYNTH),
4223[C_DC_MISC_FLG_CNT] =
4224 DC_PERF_CNTR_LCB(DcMiscFlg, DC_LCB_ERR_INFO_MISC_FLG_CNT,
4225 CNTR_SYNTH),
4226[C_DC_PRF_GOOD_LTP_CNT] =
4227 DC_PERF_CNTR_LCB(DcGoodLTP, DC_LCB_PRF_GOOD_LTP_CNT, CNTR_SYNTH),
4228[C_DC_PRF_ACCEPTED_LTP_CNT] =
4229 DC_PERF_CNTR_LCB(DcAccLTP, DC_LCB_PRF_ACCEPTED_LTP_CNT,
4230 CNTR_SYNTH),
4231[C_DC_PRF_RX_FLIT_CNT] =
4232 DC_PERF_CNTR_LCB(DcPrfRxFlit, DC_LCB_PRF_RX_FLIT_CNT, CNTR_SYNTH),
4233[C_DC_PRF_TX_FLIT_CNT] =
4234 DC_PERF_CNTR_LCB(DcPrfTxFlit, DC_LCB_PRF_TX_FLIT_CNT, CNTR_SYNTH),
4235[C_DC_PRF_CLK_CNTR] =
4236 DC_PERF_CNTR_LCB(DcPrfClk, DC_LCB_PRF_CLK_CNTR, CNTR_SYNTH),
4237[C_DC_PG_DBG_FLIT_CRDTS_CNT] =
4238 DC_PERF_CNTR_LCB(DcFltCrdts, DC_LCB_PG_DBG_FLIT_CRDTS_CNT, CNTR_SYNTH),
4239[C_DC_PG_STS_PAUSE_COMPLETE_CNT] =
4240 DC_PERF_CNTR_LCB(DcPauseComp, DC_LCB_PG_STS_PAUSE_COMPLETE_CNT,
4241 CNTR_SYNTH),
4242[C_DC_PG_STS_TX_SBE_CNT] =
4243 DC_PERF_CNTR_LCB(DcStsTxSbe, DC_LCB_PG_STS_TX_SBE_CNT, CNTR_SYNTH),
4244[C_DC_PG_STS_TX_MBE_CNT] =
4245 DC_PERF_CNTR_LCB(DcStsTxMbe, DC_LCB_PG_STS_TX_MBE_CNT,
4246 CNTR_SYNTH),
4247[C_SW_CPU_INTR] = CNTR_ELEM("Intr", 0, 0, CNTR_NORMAL,
4248 access_sw_cpu_intr),
4249[C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL,
4250 access_sw_cpu_rcv_limit),
4251[C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL,
4252 access_sw_vtx_wait),
4253[C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL,
4254 access_sw_pio_wait),
4255[C_SW_PIO_DRAIN] = CNTR_ELEM("PioDrain", 0, 0, CNTR_NORMAL,
4256 access_sw_pio_drain),
4257[C_SW_KMEM_WAIT] = CNTR_ELEM("KmemWait", 0, 0, CNTR_NORMAL,
4258 access_sw_kmem_wait),
4259[C_SW_TID_WAIT] = CNTR_ELEM("TidWait", 0, 0, CNTR_NORMAL,
4260 hfi1_access_sw_tid_wait),
4261[C_SW_SEND_SCHED] = CNTR_ELEM("SendSched", 0, 0, CNTR_NORMAL,
4262 access_sw_send_schedule),
4263[C_SDMA_DESC_FETCHED_CNT] = CNTR_ELEM("SDEDscFdCn",
4264 SEND_DMA_DESC_FETCHED_CNT, 0,
4265 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4266 dev_access_u32_csr),
4267[C_SDMA_INT_CNT] = CNTR_ELEM("SDMAInt", 0, 0,
4268 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4269 access_sde_int_cnt),
4270[C_SDMA_ERR_CNT] = CNTR_ELEM("SDMAErrCt", 0, 0,
4271 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4272 access_sde_err_cnt),
4273[C_SDMA_IDLE_INT_CNT] = CNTR_ELEM("SDMAIdInt", 0, 0,
4274 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4275 access_sde_idle_int_cnt),
4276[C_SDMA_PROGRESS_INT_CNT] = CNTR_ELEM("SDMAPrIntCn", 0, 0,
4277 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4278 access_sde_progress_int_cnt),
4279
4280[C_MISC_PLL_LOCK_FAIL_ERR] = CNTR_ELEM("MISC_PLL_LOCK_FAIL_ERR", 0, 0,
4281 CNTR_NORMAL,
4282 access_misc_pll_lock_fail_err_cnt),
4283[C_MISC_MBIST_FAIL_ERR] = CNTR_ELEM("MISC_MBIST_FAIL_ERR", 0, 0,
4284 CNTR_NORMAL,
4285 access_misc_mbist_fail_err_cnt),
4286[C_MISC_INVALID_EEP_CMD_ERR] = CNTR_ELEM("MISC_INVALID_EEP_CMD_ERR", 0, 0,
4287 CNTR_NORMAL,
4288 access_misc_invalid_eep_cmd_err_cnt),
4289[C_MISC_EFUSE_DONE_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_DONE_PARITY_ERR", 0, 0,
4290 CNTR_NORMAL,
4291 access_misc_efuse_done_parity_err_cnt),
4292[C_MISC_EFUSE_WRITE_ERR] = CNTR_ELEM("MISC_EFUSE_WRITE_ERR", 0, 0,
4293 CNTR_NORMAL,
4294 access_misc_efuse_write_err_cnt),
4295[C_MISC_EFUSE_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_EFUSE_READ_BAD_ADDR_ERR", 0,
4296 0, CNTR_NORMAL,
4297 access_misc_efuse_read_bad_addr_err_cnt),
4298[C_MISC_EFUSE_CSR_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_CSR_PARITY_ERR", 0, 0,
4299 CNTR_NORMAL,
4300 access_misc_efuse_csr_parity_err_cnt),
4301[C_MISC_FW_AUTH_FAILED_ERR] = CNTR_ELEM("MISC_FW_AUTH_FAILED_ERR", 0, 0,
4302 CNTR_NORMAL,
4303 access_misc_fw_auth_failed_err_cnt),
4304[C_MISC_KEY_MISMATCH_ERR] = CNTR_ELEM("MISC_KEY_MISMATCH_ERR", 0, 0,
4305 CNTR_NORMAL,
4306 access_misc_key_mismatch_err_cnt),
4307[C_MISC_SBUS_WRITE_FAILED_ERR] = CNTR_ELEM("MISC_SBUS_WRITE_FAILED_ERR", 0, 0,
4308 CNTR_NORMAL,
4309 access_misc_sbus_write_failed_err_cnt),
4310[C_MISC_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_WRITE_BAD_ADDR_ERR", 0, 0,
4311 CNTR_NORMAL,
4312 access_misc_csr_write_bad_addr_err_cnt),
4313[C_MISC_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_READ_BAD_ADDR_ERR", 0, 0,
4314 CNTR_NORMAL,
4315 access_misc_csr_read_bad_addr_err_cnt),
4316[C_MISC_CSR_PARITY_ERR] = CNTR_ELEM("MISC_CSR_PARITY_ERR", 0, 0,
4317 CNTR_NORMAL,
4318 access_misc_csr_parity_err_cnt),
4319
4320[C_CCE_ERR_STATUS_AGGREGATED_CNT] = CNTR_ELEM("CceErrStatusAggregatedCnt", 0, 0,
4321 CNTR_NORMAL,
4322 access_sw_cce_err_status_aggregated_cnt),
4323[C_CCE_MSIX_CSR_PARITY_ERR] = CNTR_ELEM("CceMsixCsrParityErr", 0, 0,
4324 CNTR_NORMAL,
4325 access_cce_msix_csr_parity_err_cnt),
4326[C_CCE_INT_MAP_UNC_ERR] = CNTR_ELEM("CceIntMapUncErr", 0, 0,
4327 CNTR_NORMAL,
4328 access_cce_int_map_unc_err_cnt),
4329[C_CCE_INT_MAP_COR_ERR] = CNTR_ELEM("CceIntMapCorErr", 0, 0,
4330 CNTR_NORMAL,
4331 access_cce_int_map_cor_err_cnt),
4332[C_CCE_MSIX_TABLE_UNC_ERR] = CNTR_ELEM("CceMsixTableUncErr", 0, 0,
4333 CNTR_NORMAL,
4334 access_cce_msix_table_unc_err_cnt),
4335[C_CCE_MSIX_TABLE_COR_ERR] = CNTR_ELEM("CceMsixTableCorErr", 0, 0,
4336 CNTR_NORMAL,
4337 access_cce_msix_table_cor_err_cnt),
4338[C_CCE_RXDMA_CONV_FIFO_PARITY_ERR] = CNTR_ELEM("CceRxdmaConvFifoParityErr", 0,
4339 0, CNTR_NORMAL,
4340 access_cce_rxdma_conv_fifo_parity_err_cnt),
4341[C_CCE_RCPL_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceRcplAsyncFifoParityErr", 0,
4342 0, CNTR_NORMAL,
4343 access_cce_rcpl_async_fifo_parity_err_cnt),
4344[C_CCE_SEG_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceSegWriteBadAddrErr", 0, 0,
4345 CNTR_NORMAL,
4346 access_cce_seg_write_bad_addr_err_cnt),
4347[C_CCE_SEG_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceSegReadBadAddrErr", 0, 0,
4348 CNTR_NORMAL,
4349 access_cce_seg_read_bad_addr_err_cnt),
4350[C_LA_TRIGGERED] = CNTR_ELEM("Cce LATriggered", 0, 0,
4351 CNTR_NORMAL,
4352 access_la_triggered_cnt),
4353[C_CCE_TRGT_CPL_TIMEOUT_ERR] = CNTR_ELEM("CceTrgtCplTimeoutErr", 0, 0,
4354 CNTR_NORMAL,
4355 access_cce_trgt_cpl_timeout_err_cnt),
4356[C_PCIC_RECEIVE_PARITY_ERR] = CNTR_ELEM("PcicReceiveParityErr", 0, 0,
4357 CNTR_NORMAL,
4358 access_pcic_receive_parity_err_cnt),
4359[C_PCIC_TRANSMIT_BACK_PARITY_ERR] = CNTR_ELEM("PcicTransmitBackParityErr", 0, 0,
4360 CNTR_NORMAL,
4361 access_pcic_transmit_back_parity_err_cnt),
4362[C_PCIC_TRANSMIT_FRONT_PARITY_ERR] = CNTR_ELEM("PcicTransmitFrontParityErr", 0,
4363 0, CNTR_NORMAL,
4364 access_pcic_transmit_front_parity_err_cnt),
4365[C_PCIC_CPL_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicCplDatQUncErr", 0, 0,
4366 CNTR_NORMAL,
4367 access_pcic_cpl_dat_q_unc_err_cnt),
4368[C_PCIC_CPL_HD_Q_UNC_ERR] = CNTR_ELEM("PcicCplHdQUncErr", 0, 0,
4369 CNTR_NORMAL,
4370 access_pcic_cpl_hd_q_unc_err_cnt),
4371[C_PCIC_POST_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicPostDatQUncErr", 0, 0,
4372 CNTR_NORMAL,
4373 access_pcic_post_dat_q_unc_err_cnt),
4374[C_PCIC_POST_HD_Q_UNC_ERR] = CNTR_ELEM("PcicPostHdQUncErr", 0, 0,
4375 CNTR_NORMAL,
4376 access_pcic_post_hd_q_unc_err_cnt),
4377[C_PCIC_RETRY_SOT_MEM_UNC_ERR] = CNTR_ELEM("PcicRetrySotMemUncErr", 0, 0,
4378 CNTR_NORMAL,
4379 access_pcic_retry_sot_mem_unc_err_cnt),
4380[C_PCIC_RETRY_MEM_UNC_ERR] = CNTR_ELEM("PcicRetryMemUncErr", 0, 0,
4381 CNTR_NORMAL,
4382 access_pcic_retry_mem_unc_err),
4383[C_PCIC_N_POST_DAT_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostDatQParityErr", 0, 0,
4384 CNTR_NORMAL,
4385 access_pcic_n_post_dat_q_parity_err_cnt),
4386[C_PCIC_N_POST_H_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostHQParityErr", 0, 0,
4387 CNTR_NORMAL,
4388 access_pcic_n_post_h_q_parity_err_cnt),
4389[C_PCIC_CPL_DAT_Q_COR_ERR] = CNTR_ELEM("PcicCplDatQCorErr", 0, 0,
4390 CNTR_NORMAL,
4391 access_pcic_cpl_dat_q_cor_err_cnt),
4392[C_PCIC_CPL_HD_Q_COR_ERR] = CNTR_ELEM("PcicCplHdQCorErr", 0, 0,
4393 CNTR_NORMAL,
4394 access_pcic_cpl_hd_q_cor_err_cnt),
4395[C_PCIC_POST_DAT_Q_COR_ERR] = CNTR_ELEM("PcicPostDatQCorErr", 0, 0,
4396 CNTR_NORMAL,
4397 access_pcic_post_dat_q_cor_err_cnt),
4398[C_PCIC_POST_HD_Q_COR_ERR] = CNTR_ELEM("PcicPostHdQCorErr", 0, 0,
4399 CNTR_NORMAL,
4400 access_pcic_post_hd_q_cor_err_cnt),
4401[C_PCIC_RETRY_SOT_MEM_COR_ERR] = CNTR_ELEM("PcicRetrySotMemCorErr", 0, 0,
4402 CNTR_NORMAL,
4403 access_pcic_retry_sot_mem_cor_err_cnt),
4404[C_PCIC_RETRY_MEM_COR_ERR] = CNTR_ELEM("PcicRetryMemCorErr", 0, 0,
4405 CNTR_NORMAL,
4406 access_pcic_retry_mem_cor_err_cnt),
4407[C_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERR] = CNTR_ELEM(
4408 "CceCli1AsyncFifoDbgParityError", 0, 0,
4409 CNTR_NORMAL,
4410 access_cce_cli1_async_fifo_dbg_parity_err_cnt),
4411[C_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERR] = CNTR_ELEM(
4412 "CceCli1AsyncFifoRxdmaParityError", 0, 0,
4413 CNTR_NORMAL,
4414 access_cce_cli1_async_fifo_rxdma_parity_err_cnt
4415 ),
4416[C_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR] = CNTR_ELEM(
4417 "CceCli1AsyncFifoSdmaHdParityErr", 0, 0,
4418 CNTR_NORMAL,
4419 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt),
4420[C_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR] = CNTR_ELEM(
4421 "CceCli1AsyncFifoPioCrdtParityErr", 0, 0,
4422 CNTR_NORMAL,
4423 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt),
4424[C_CCE_CLI2_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceCli2AsyncFifoParityErr", 0,
4425 0, CNTR_NORMAL,
4426 access_cce_cli2_async_fifo_parity_err_cnt),
4427[C_CCE_CSR_CFG_BUS_PARITY_ERR] = CNTR_ELEM("CceCsrCfgBusParityErr", 0, 0,
4428 CNTR_NORMAL,
4429 access_cce_csr_cfg_bus_parity_err_cnt),
4430[C_CCE_CLI0_ASYNC_FIFO_PARTIY_ERR] = CNTR_ELEM("CceCli0AsyncFifoParityErr", 0,
4431 0, CNTR_NORMAL,
4432 access_cce_cli0_async_fifo_parity_err_cnt),
4433[C_CCE_RSPD_DATA_PARITY_ERR] = CNTR_ELEM("CceRspdDataParityErr", 0, 0,
4434 CNTR_NORMAL,
4435 access_cce_rspd_data_parity_err_cnt),
4436[C_CCE_TRGT_ACCESS_ERR] = CNTR_ELEM("CceTrgtAccessErr", 0, 0,
4437 CNTR_NORMAL,
4438 access_cce_trgt_access_err_cnt),
4439[C_CCE_TRGT_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceTrgtAsyncFifoParityErr", 0,
4440 0, CNTR_NORMAL,
4441 access_cce_trgt_async_fifo_parity_err_cnt),
4442[C_CCE_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrWriteBadAddrErr", 0, 0,
4443 CNTR_NORMAL,
4444 access_cce_csr_write_bad_addr_err_cnt),
4445[C_CCE_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrReadBadAddrErr", 0, 0,
4446 CNTR_NORMAL,
4447 access_cce_csr_read_bad_addr_err_cnt),
4448[C_CCE_CSR_PARITY_ERR] = CNTR_ELEM("CceCsrParityErr", 0, 0,
4449 CNTR_NORMAL,
4450 access_ccs_csr_parity_err_cnt),
4451
4452
4453[C_RX_CSR_PARITY_ERR] = CNTR_ELEM("RxCsrParityErr", 0, 0,
4454 CNTR_NORMAL,
4455 access_rx_csr_parity_err_cnt),
4456[C_RX_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrWriteBadAddrErr", 0, 0,
4457 CNTR_NORMAL,
4458 access_rx_csr_write_bad_addr_err_cnt),
4459[C_RX_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrReadBadAddrErr", 0, 0,
4460 CNTR_NORMAL,
4461 access_rx_csr_read_bad_addr_err_cnt),
4462[C_RX_DMA_CSR_UNC_ERR] = CNTR_ELEM("RxDmaCsrUncErr", 0, 0,
4463 CNTR_NORMAL,
4464 access_rx_dma_csr_unc_err_cnt),
4465[C_RX_DMA_DQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaDqFsmEncodingErr", 0, 0,
4466 CNTR_NORMAL,
4467 access_rx_dma_dq_fsm_encoding_err_cnt),
4468[C_RX_DMA_EQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaEqFsmEncodingErr", 0, 0,
4469 CNTR_NORMAL,
4470 access_rx_dma_eq_fsm_encoding_err_cnt),
4471[C_RX_DMA_CSR_PARITY_ERR] = CNTR_ELEM("RxDmaCsrParityErr", 0, 0,
4472 CNTR_NORMAL,
4473 access_rx_dma_csr_parity_err_cnt),
4474[C_RX_RBUF_DATA_COR_ERR] = CNTR_ELEM("RxRbufDataCorErr", 0, 0,
4475 CNTR_NORMAL,
4476 access_rx_rbuf_data_cor_err_cnt),
4477[C_RX_RBUF_DATA_UNC_ERR] = CNTR_ELEM("RxRbufDataUncErr", 0, 0,
4478 CNTR_NORMAL,
4479 access_rx_rbuf_data_unc_err_cnt),
4480[C_RX_DMA_DATA_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaDataFifoRdCorErr", 0, 0,
4481 CNTR_NORMAL,
4482 access_rx_dma_data_fifo_rd_cor_err_cnt),
4483[C_RX_DMA_DATA_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaDataFifoRdUncErr", 0, 0,
4484 CNTR_NORMAL,
4485 access_rx_dma_data_fifo_rd_unc_err_cnt),
4486[C_RX_DMA_HDR_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaHdrFifoRdCorErr", 0, 0,
4487 CNTR_NORMAL,
4488 access_rx_dma_hdr_fifo_rd_cor_err_cnt),
4489[C_RX_DMA_HDR_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaHdrFifoRdUncErr", 0, 0,
4490 CNTR_NORMAL,
4491 access_rx_dma_hdr_fifo_rd_unc_err_cnt),
4492[C_RX_RBUF_DESC_PART2_COR_ERR] = CNTR_ELEM("RxRbufDescPart2CorErr", 0, 0,
4493 CNTR_NORMAL,
4494 access_rx_rbuf_desc_part2_cor_err_cnt),
4495[C_RX_RBUF_DESC_PART2_UNC_ERR] = CNTR_ELEM("RxRbufDescPart2UncErr", 0, 0,
4496 CNTR_NORMAL,
4497 access_rx_rbuf_desc_part2_unc_err_cnt),
4498[C_RX_RBUF_DESC_PART1_COR_ERR] = CNTR_ELEM("RxRbufDescPart1CorErr", 0, 0,
4499 CNTR_NORMAL,
4500 access_rx_rbuf_desc_part1_cor_err_cnt),
4501[C_RX_RBUF_DESC_PART1_UNC_ERR] = CNTR_ELEM("RxRbufDescPart1UncErr", 0, 0,
4502 CNTR_NORMAL,
4503 access_rx_rbuf_desc_part1_unc_err_cnt),
4504[C_RX_HQ_INTR_FSM_ERR] = CNTR_ELEM("RxHqIntrFsmErr", 0, 0,
4505 CNTR_NORMAL,
4506 access_rx_hq_intr_fsm_err_cnt),
4507[C_RX_HQ_INTR_CSR_PARITY_ERR] = CNTR_ELEM("RxHqIntrCsrParityErr", 0, 0,
4508 CNTR_NORMAL,
4509 access_rx_hq_intr_csr_parity_err_cnt),
4510[C_RX_LOOKUP_CSR_PARITY_ERR] = CNTR_ELEM("RxLookupCsrParityErr", 0, 0,
4511 CNTR_NORMAL,
4512 access_rx_lookup_csr_parity_err_cnt),
4513[C_RX_LOOKUP_RCV_ARRAY_COR_ERR] = CNTR_ELEM("RxLookupRcvArrayCorErr", 0, 0,
4514 CNTR_NORMAL,
4515 access_rx_lookup_rcv_array_cor_err_cnt),
4516[C_RX_LOOKUP_RCV_ARRAY_UNC_ERR] = CNTR_ELEM("RxLookupRcvArrayUncErr", 0, 0,
4517 CNTR_NORMAL,
4518 access_rx_lookup_rcv_array_unc_err_cnt),
4519[C_RX_LOOKUP_DES_PART2_PARITY_ERR] = CNTR_ELEM("RxLookupDesPart2ParityErr", 0,
4520 0, CNTR_NORMAL,
4521 access_rx_lookup_des_part2_parity_err_cnt),
4522[C_RX_LOOKUP_DES_PART1_UNC_COR_ERR] = CNTR_ELEM("RxLookupDesPart1UncCorErr", 0,
4523 0, CNTR_NORMAL,
4524 access_rx_lookup_des_part1_unc_cor_err_cnt),
4525[C_RX_LOOKUP_DES_PART1_UNC_ERR] = CNTR_ELEM("RxLookupDesPart1UncErr", 0, 0,
4526 CNTR_NORMAL,
4527 access_rx_lookup_des_part1_unc_err_cnt),
4528[C_RX_RBUF_NEXT_FREE_BUF_COR_ERR] = CNTR_ELEM("RxRbufNextFreeBufCorErr", 0, 0,
4529 CNTR_NORMAL,
4530 access_rx_rbuf_next_free_buf_cor_err_cnt),
4531[C_RX_RBUF_NEXT_FREE_BUF_UNC_ERR] = CNTR_ELEM("RxRbufNextFreeBufUncErr", 0, 0,
4532 CNTR_NORMAL,
4533 access_rx_rbuf_next_free_buf_unc_err_cnt),
4534[C_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR] = CNTR_ELEM(
4535 "RxRbufFlInitWrAddrParityErr", 0, 0,
4536 CNTR_NORMAL,
4537 access_rbuf_fl_init_wr_addr_parity_err_cnt),
4538[C_RX_RBUF_FL_INITDONE_PARITY_ERR] = CNTR_ELEM("RxRbufFlInitdoneParityErr", 0,
4539 0, CNTR_NORMAL,
4540 access_rx_rbuf_fl_initdone_parity_err_cnt),
4541[C_RX_RBUF_FL_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlWrAddrParityErr", 0,
4542 0, CNTR_NORMAL,
4543 access_rx_rbuf_fl_write_addr_parity_err_cnt),
4544[C_RX_RBUF_FL_RD_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlRdAddrParityErr", 0, 0,
4545 CNTR_NORMAL,
4546 access_rx_rbuf_fl_rd_addr_parity_err_cnt),
4547[C_RX_RBUF_EMPTY_ERR] = CNTR_ELEM("RxRbufEmptyErr", 0, 0,
4548 CNTR_NORMAL,
4549 access_rx_rbuf_empty_err_cnt),
4550[C_RX_RBUF_FULL_ERR] = CNTR_ELEM("RxRbufFullErr", 0, 0,
4551 CNTR_NORMAL,
4552 access_rx_rbuf_full_err_cnt),
4553[C_RX_RBUF_BAD_LOOKUP_ERR] = CNTR_ELEM("RxRBufBadLookupErr", 0, 0,
4554 CNTR_NORMAL,
4555 access_rbuf_bad_lookup_err_cnt),
4556[C_RX_RBUF_CTX_ID_PARITY_ERR] = CNTR_ELEM("RxRbufCtxIdParityErr", 0, 0,
4557 CNTR_NORMAL,
4558 access_rbuf_ctx_id_parity_err_cnt),
4559[C_RX_RBUF_CSR_QEOPDW_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEOPDWParityErr", 0, 0,
4560 CNTR_NORMAL,
4561 access_rbuf_csr_qeopdw_parity_err_cnt),
4562[C_RX_RBUF_CSR_Q_NUM_OF_PKT_PARITY_ERR] = CNTR_ELEM(
4563 "RxRbufCsrQNumOfPktParityErr", 0, 0,
4564 CNTR_NORMAL,
4565 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt),
4566[C_RX_RBUF_CSR_Q_T1_PTR_PARITY_ERR] = CNTR_ELEM(
4567 "RxRbufCsrQTlPtrParityErr", 0, 0,
4568 CNTR_NORMAL,
4569 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt),
4570[C_RX_RBUF_CSR_Q_HD_PTR_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQHdPtrParityErr", 0,
4571 0, CNTR_NORMAL,
4572 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt),
4573[C_RX_RBUF_CSR_Q_VLD_BIT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQVldBitParityErr", 0,
4574 0, CNTR_NORMAL,
4575 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt),
4576[C_RX_RBUF_CSR_Q_NEXT_BUF_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQNextBufParityErr",
4577 0, 0, CNTR_NORMAL,
4578 access_rx_rbuf_csr_q_next_buf_parity_err_cnt),
4579[C_RX_RBUF_CSR_Q_ENT_CNT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEntCntParityErr", 0,
4580 0, CNTR_NORMAL,
4581 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt),
4582[C_RX_RBUF_CSR_Q_HEAD_BUF_NUM_PARITY_ERR] = CNTR_ELEM(
4583 "RxRbufCsrQHeadBufNumParityErr", 0, 0,
4584 CNTR_NORMAL,
4585 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt),
4586[C_RX_RBUF_BLOCK_LIST_READ_COR_ERR] = CNTR_ELEM("RxRbufBlockListReadCorErr", 0,
4587 0, CNTR_NORMAL,
4588 access_rx_rbuf_block_list_read_cor_err_cnt),
4589[C_RX_RBUF_BLOCK_LIST_READ_UNC_ERR] = CNTR_ELEM("RxRbufBlockListReadUncErr", 0,
4590 0, CNTR_NORMAL,
4591 access_rx_rbuf_block_list_read_unc_err_cnt),
4592[C_RX_RBUF_LOOKUP_DES_COR_ERR] = CNTR_ELEM("RxRbufLookupDesCorErr", 0, 0,
4593 CNTR_NORMAL,
4594 access_rx_rbuf_lookup_des_cor_err_cnt),
4595[C_RX_RBUF_LOOKUP_DES_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesUncErr", 0, 0,
4596 CNTR_NORMAL,
4597 access_rx_rbuf_lookup_des_unc_err_cnt),
4598[C_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR] = CNTR_ELEM(
4599 "RxRbufLookupDesRegUncCorErr", 0, 0,
4600 CNTR_NORMAL,
4601 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt),
4602[C_RX_RBUF_LOOKUP_DES_REG_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesRegUncErr", 0, 0,
4603 CNTR_NORMAL,
4604 access_rx_rbuf_lookup_des_reg_unc_err_cnt),
4605[C_RX_RBUF_FREE_LIST_COR_ERR] = CNTR_ELEM("RxRbufFreeListCorErr", 0, 0,
4606 CNTR_NORMAL,
4607 access_rx_rbuf_free_list_cor_err_cnt),
4608[C_RX_RBUF_FREE_LIST_UNC_ERR] = CNTR_ELEM("RxRbufFreeListUncErr", 0, 0,
4609 CNTR_NORMAL,
4610 access_rx_rbuf_free_list_unc_err_cnt),
4611[C_RX_RCV_FSM_ENCODING_ERR] = CNTR_ELEM("RxRcvFsmEncodingErr", 0, 0,
4612 CNTR_NORMAL,
4613 access_rx_rcv_fsm_encoding_err_cnt),
4614[C_RX_DMA_FLAG_COR_ERR] = CNTR_ELEM("RxDmaFlagCorErr", 0, 0,
4615 CNTR_NORMAL,
4616 access_rx_dma_flag_cor_err_cnt),
4617[C_RX_DMA_FLAG_UNC_ERR] = CNTR_ELEM("RxDmaFlagUncErr", 0, 0,
4618 CNTR_NORMAL,
4619 access_rx_dma_flag_unc_err_cnt),
4620[C_RX_DC_SOP_EOP_PARITY_ERR] = CNTR_ELEM("RxDcSopEopParityErr", 0, 0,
4621 CNTR_NORMAL,
4622 access_rx_dc_sop_eop_parity_err_cnt),
4623[C_RX_RCV_CSR_PARITY_ERR] = CNTR_ELEM("RxRcvCsrParityErr", 0, 0,
4624 CNTR_NORMAL,
4625 access_rx_rcv_csr_parity_err_cnt),
4626[C_RX_RCV_QP_MAP_TABLE_COR_ERR] = CNTR_ELEM("RxRcvQpMapTableCorErr", 0, 0,
4627 CNTR_NORMAL,
4628 access_rx_rcv_qp_map_table_cor_err_cnt),
4629[C_RX_RCV_QP_MAP_TABLE_UNC_ERR] = CNTR_ELEM("RxRcvQpMapTableUncErr", 0, 0,
4630 CNTR_NORMAL,
4631 access_rx_rcv_qp_map_table_unc_err_cnt),
4632[C_RX_RCV_DATA_COR_ERR] = CNTR_ELEM("RxRcvDataCorErr", 0, 0,
4633 CNTR_NORMAL,
4634 access_rx_rcv_data_cor_err_cnt),
4635[C_RX_RCV_DATA_UNC_ERR] = CNTR_ELEM("RxRcvDataUncErr", 0, 0,
4636 CNTR_NORMAL,
4637 access_rx_rcv_data_unc_err_cnt),
4638[C_RX_RCV_HDR_COR_ERR] = CNTR_ELEM("RxRcvHdrCorErr", 0, 0,
4639 CNTR_NORMAL,
4640 access_rx_rcv_hdr_cor_err_cnt),
4641[C_RX_RCV_HDR_UNC_ERR] = CNTR_ELEM("RxRcvHdrUncErr", 0, 0,
4642 CNTR_NORMAL,
4643 access_rx_rcv_hdr_unc_err_cnt),
4644[C_RX_DC_INTF_PARITY_ERR] = CNTR_ELEM("RxDcIntfParityErr", 0, 0,
4645 CNTR_NORMAL,
4646 access_rx_dc_intf_parity_err_cnt),
4647[C_RX_DMA_CSR_COR_ERR] = CNTR_ELEM("RxDmaCsrCorErr", 0, 0,
4648 CNTR_NORMAL,
4649 access_rx_dma_csr_cor_err_cnt),
4650
4651[C_PIO_PEC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPecSopHeadParityErr", 0, 0,
4652 CNTR_NORMAL,
4653 access_pio_pec_sop_head_parity_err_cnt),
4654[C_PIO_PCC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPccSopHeadParityErr", 0, 0,
4655 CNTR_NORMAL,
4656 access_pio_pcc_sop_head_parity_err_cnt),
4657[C_PIO_LAST_RETURNED_CNT_PARITY_ERR] = CNTR_ELEM("PioLastReturnedCntParityErr",
4658 0, 0, CNTR_NORMAL,
4659 access_pio_last_returned_cnt_parity_err_cnt),
4660[C_PIO_CURRENT_FREE_CNT_PARITY_ERR] = CNTR_ELEM("PioCurrentFreeCntParityErr", 0,
4661 0, CNTR_NORMAL,
4662 access_pio_current_free_cnt_parity_err_cnt),
4663[C_PIO_RSVD_31_ERR] = CNTR_ELEM("Pio Reserved 31", 0, 0,
4664 CNTR_NORMAL,
4665 access_pio_reserved_31_err_cnt),
4666[C_PIO_RSVD_30_ERR] = CNTR_ELEM("Pio Reserved 30", 0, 0,
4667 CNTR_NORMAL,
4668 access_pio_reserved_30_err_cnt),
4669[C_PIO_PPMC_SOP_LEN_ERR] = CNTR_ELEM("PioPpmcSopLenErr", 0, 0,
4670 CNTR_NORMAL,
4671 access_pio_ppmc_sop_len_err_cnt),
4672[C_PIO_PPMC_BQC_MEM_PARITY_ERR] = CNTR_ELEM("PioPpmcBqcMemParityErr", 0, 0,
4673 CNTR_NORMAL,
4674 access_pio_ppmc_bqc_mem_parity_err_cnt),
4675[C_PIO_VL_FIFO_PARITY_ERR] = CNTR_ELEM("PioVlFifoParityErr", 0, 0,
4676 CNTR_NORMAL,
4677 access_pio_vl_fifo_parity_err_cnt),
4678[C_PIO_VLF_SOP_PARITY_ERR] = CNTR_ELEM("PioVlfSopParityErr", 0, 0,
4679 CNTR_NORMAL,
4680 access_pio_vlf_sop_parity_err_cnt),
4681[C_PIO_VLF_V1_LEN_PARITY_ERR] = CNTR_ELEM("PioVlfVlLenParityErr", 0, 0,
4682 CNTR_NORMAL,
4683 access_pio_vlf_v1_len_parity_err_cnt),
4684[C_PIO_BLOCK_QW_COUNT_PARITY_ERR] = CNTR_ELEM("PioBlockQwCountParityErr", 0, 0,
4685 CNTR_NORMAL,
4686 access_pio_block_qw_count_parity_err_cnt),
4687[C_PIO_WRITE_QW_VALID_PARITY_ERR] = CNTR_ELEM("PioWriteQwValidParityErr", 0, 0,
4688 CNTR_NORMAL,
4689 access_pio_write_qw_valid_parity_err_cnt),
4690[C_PIO_STATE_MACHINE_ERR] = CNTR_ELEM("PioStateMachineErr", 0, 0,
4691 CNTR_NORMAL,
4692 access_pio_state_machine_err_cnt),
4693[C_PIO_WRITE_DATA_PARITY_ERR] = CNTR_ELEM("PioWriteDataParityErr", 0, 0,
4694 CNTR_NORMAL,
4695 access_pio_write_data_parity_err_cnt),
4696[C_PIO_HOST_ADDR_MEM_COR_ERR] = CNTR_ELEM("PioHostAddrMemCorErr", 0, 0,
4697 CNTR_NORMAL,
4698 access_pio_host_addr_mem_cor_err_cnt),
4699[C_PIO_HOST_ADDR_MEM_UNC_ERR] = CNTR_ELEM("PioHostAddrMemUncErr", 0, 0,
4700 CNTR_NORMAL,
4701 access_pio_host_addr_mem_unc_err_cnt),
4702[C_PIO_PKT_EVICT_SM_OR_ARM_SM_ERR] = CNTR_ELEM("PioPktEvictSmOrArbSmErr", 0, 0,
4703 CNTR_NORMAL,
4704 access_pio_pkt_evict_sm_or_arb_sm_err_cnt),
4705[C_PIO_INIT_SM_IN_ERR] = CNTR_ELEM("PioInitSmInErr", 0, 0,
4706 CNTR_NORMAL,
4707 access_pio_init_sm_in_err_cnt),
4708[C_PIO_PPMC_PBL_FIFO_ERR] = CNTR_ELEM("PioPpmcPblFifoErr", 0, 0,
4709 CNTR_NORMAL,
4710 access_pio_ppmc_pbl_fifo_err_cnt),
4711[C_PIO_CREDIT_RET_FIFO_PARITY_ERR] = CNTR_ELEM("PioCreditRetFifoParityErr", 0,
4712 0, CNTR_NORMAL,
4713 access_pio_credit_ret_fifo_parity_err_cnt),
4714[C_PIO_V1_LEN_MEM_BANK1_COR_ERR] = CNTR_ELEM("PioVlLenMemBank1CorErr", 0, 0,
4715 CNTR_NORMAL,
4716 access_pio_v1_len_mem_bank1_cor_err_cnt),
4717[C_PIO_V1_LEN_MEM_BANK0_COR_ERR] = CNTR_ELEM("PioVlLenMemBank0CorErr", 0, 0,
4718 CNTR_NORMAL,
4719 access_pio_v1_len_mem_bank0_cor_err_cnt),
4720[C_PIO_V1_LEN_MEM_BANK1_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank1UncErr", 0, 0,
4721 CNTR_NORMAL,
4722 access_pio_v1_len_mem_bank1_unc_err_cnt),
4723[C_PIO_V1_LEN_MEM_BANK0_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank0UncErr", 0, 0,
4724 CNTR_NORMAL,
4725 access_pio_v1_len_mem_bank0_unc_err_cnt),
4726[C_PIO_SM_PKT_RESET_PARITY_ERR] = CNTR_ELEM("PioSmPktResetParityErr", 0, 0,
4727 CNTR_NORMAL,
4728 access_pio_sm_pkt_reset_parity_err_cnt),
4729[C_PIO_PKT_EVICT_FIFO_PARITY_ERR] = CNTR_ELEM("PioPktEvictFifoParityErr", 0, 0,
4730 CNTR_NORMAL,
4731 access_pio_pkt_evict_fifo_parity_err_cnt),
4732[C_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR] = CNTR_ELEM(
4733 "PioSbrdctrlCrrelFifoParityErr", 0, 0,
4734 CNTR_NORMAL,
4735 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt),
4736[C_PIO_SBRDCTL_CRREL_PARITY_ERR] = CNTR_ELEM("PioSbrdctlCrrelParityErr", 0, 0,
4737 CNTR_NORMAL,
4738 access_pio_sbrdctl_crrel_parity_err_cnt),
4739[C_PIO_PEC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPecFifoParityErr", 0, 0,
4740 CNTR_NORMAL,
4741 access_pio_pec_fifo_parity_err_cnt),
4742[C_PIO_PCC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPccFifoParityErr", 0, 0,
4743 CNTR_NORMAL,
4744 access_pio_pcc_fifo_parity_err_cnt),
4745[C_PIO_SB_MEM_FIFO1_ERR] = CNTR_ELEM("PioSbMemFifo1Err", 0, 0,
4746 CNTR_NORMAL,
4747 access_pio_sb_mem_fifo1_err_cnt),
4748[C_PIO_SB_MEM_FIFO0_ERR] = CNTR_ELEM("PioSbMemFifo0Err", 0, 0,
4749 CNTR_NORMAL,
4750 access_pio_sb_mem_fifo0_err_cnt),
4751[C_PIO_CSR_PARITY_ERR] = CNTR_ELEM("PioCsrParityErr", 0, 0,
4752 CNTR_NORMAL,
4753 access_pio_csr_parity_err_cnt),
4754[C_PIO_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("PioWriteAddrParityErr", 0, 0,
4755 CNTR_NORMAL,
4756 access_pio_write_addr_parity_err_cnt),
4757[C_PIO_WRITE_BAD_CTXT_ERR] = CNTR_ELEM("PioWriteBadCtxtErr", 0, 0,
4758 CNTR_NORMAL,
4759 access_pio_write_bad_ctxt_err_cnt),
4760
4761[C_SDMA_PCIE_REQ_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPcieReqTrackingCorErr", 0,
4762 0, CNTR_NORMAL,
4763 access_sdma_pcie_req_tracking_cor_err_cnt),
4764[C_SDMA_PCIE_REQ_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPcieReqTrackingUncErr", 0,
4765 0, CNTR_NORMAL,
4766 access_sdma_pcie_req_tracking_unc_err_cnt),
4767[C_SDMA_CSR_PARITY_ERR] = CNTR_ELEM("SDmaCsrParityErr", 0, 0,
4768 CNTR_NORMAL,
4769 access_sdma_csr_parity_err_cnt),
4770[C_SDMA_RPY_TAG_ERR] = CNTR_ELEM("SDmaRpyTagErr", 0, 0,
4771 CNTR_NORMAL,
4772 access_sdma_rpy_tag_err_cnt),
4773
4774[C_TX_READ_PIO_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryCsrUncErr", 0, 0,
4775 CNTR_NORMAL,
4776 access_tx_read_pio_memory_csr_unc_err_cnt),
4777[C_TX_READ_SDMA_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryCsrUncErr", 0,
4778 0, CNTR_NORMAL,
4779 access_tx_read_sdma_memory_csr_err_cnt),
4780[C_TX_EGRESS_FIFO_COR_ERR] = CNTR_ELEM("TxEgressFifoCorErr", 0, 0,
4781 CNTR_NORMAL,
4782 access_tx_egress_fifo_cor_err_cnt),
4783[C_TX_READ_PIO_MEMORY_COR_ERR] = CNTR_ELEM("TxReadPioMemoryCorErr", 0, 0,
4784 CNTR_NORMAL,
4785 access_tx_read_pio_memory_cor_err_cnt),
4786[C_TX_READ_SDMA_MEMORY_COR_ERR] = CNTR_ELEM("TxReadSdmaMemoryCorErr", 0, 0,
4787 CNTR_NORMAL,
4788 access_tx_read_sdma_memory_cor_err_cnt),
4789[C_TX_SB_HDR_COR_ERR] = CNTR_ELEM("TxSbHdrCorErr", 0, 0,
4790 CNTR_NORMAL,
4791 access_tx_sb_hdr_cor_err_cnt),
4792[C_TX_CREDIT_OVERRUN_ERR] = CNTR_ELEM("TxCreditOverrunErr", 0, 0,
4793 CNTR_NORMAL,
4794 access_tx_credit_overrun_err_cnt),
4795[C_TX_LAUNCH_FIFO8_COR_ERR] = CNTR_ELEM("TxLaunchFifo8CorErr", 0, 0,
4796 CNTR_NORMAL,
4797 access_tx_launch_fifo8_cor_err_cnt),
4798[C_TX_LAUNCH_FIFO7_COR_ERR] = CNTR_ELEM("TxLaunchFifo7CorErr", 0, 0,
4799 CNTR_NORMAL,
4800 access_tx_launch_fifo7_cor_err_cnt),
4801[C_TX_LAUNCH_FIFO6_COR_ERR] = CNTR_ELEM("TxLaunchFifo6CorErr", 0, 0,
4802 CNTR_NORMAL,
4803 access_tx_launch_fifo6_cor_err_cnt),
4804[C_TX_LAUNCH_FIFO5_COR_ERR] = CNTR_ELEM("TxLaunchFifo5CorErr", 0, 0,
4805 CNTR_NORMAL,
4806 access_tx_launch_fifo5_cor_err_cnt),
4807[C_TX_LAUNCH_FIFO4_COR_ERR] = CNTR_ELEM("TxLaunchFifo4CorErr", 0, 0,
4808 CNTR_NORMAL,
4809 access_tx_launch_fifo4_cor_err_cnt),
4810[C_TX_LAUNCH_FIFO3_COR_ERR] = CNTR_ELEM("TxLaunchFifo3CorErr", 0, 0,
4811 CNTR_NORMAL,
4812 access_tx_launch_fifo3_cor_err_cnt),
4813[C_TX_LAUNCH_FIFO2_COR_ERR] = CNTR_ELEM("TxLaunchFifo2CorErr", 0, 0,
4814 CNTR_NORMAL,
4815 access_tx_launch_fifo2_cor_err_cnt),
4816[C_TX_LAUNCH_FIFO1_COR_ERR] = CNTR_ELEM("TxLaunchFifo1CorErr", 0, 0,
4817 CNTR_NORMAL,
4818 access_tx_launch_fifo1_cor_err_cnt),
4819[C_TX_LAUNCH_FIFO0_COR_ERR] = CNTR_ELEM("TxLaunchFifo0CorErr", 0, 0,
4820 CNTR_NORMAL,
4821 access_tx_launch_fifo0_cor_err_cnt),
4822[C_TX_CREDIT_RETURN_VL_ERR] = CNTR_ELEM("TxCreditReturnVLErr", 0, 0,
4823 CNTR_NORMAL,
4824 access_tx_credit_return_vl_err_cnt),
4825[C_TX_HCRC_INSERTION_ERR] = CNTR_ELEM("TxHcrcInsertionErr", 0, 0,
4826 CNTR_NORMAL,
4827 access_tx_hcrc_insertion_err_cnt),
4828[C_TX_EGRESS_FIFI_UNC_ERR] = CNTR_ELEM("TxEgressFifoUncErr", 0, 0,
4829 CNTR_NORMAL,
4830 access_tx_egress_fifo_unc_err_cnt),
4831[C_TX_READ_PIO_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryUncErr", 0, 0,
4832 CNTR_NORMAL,
4833 access_tx_read_pio_memory_unc_err_cnt),
4834[C_TX_READ_SDMA_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryUncErr", 0, 0,
4835 CNTR_NORMAL,
4836 access_tx_read_sdma_memory_unc_err_cnt),
4837[C_TX_SB_HDR_UNC_ERR] = CNTR_ELEM("TxSbHdrUncErr", 0, 0,
4838 CNTR_NORMAL,
4839 access_tx_sb_hdr_unc_err_cnt),
4840[C_TX_CREDIT_RETURN_PARITY_ERR] = CNTR_ELEM("TxCreditReturnParityErr", 0, 0,
4841 CNTR_NORMAL,
4842 access_tx_credit_return_partiy_err_cnt),
4843[C_TX_LAUNCH_FIFO8_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo8UncOrParityErr",
4844 0, 0, CNTR_NORMAL,
4845 access_tx_launch_fifo8_unc_or_parity_err_cnt),
4846[C_TX_LAUNCH_FIFO7_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo7UncOrParityErr",
4847 0, 0, CNTR_NORMAL,
4848 access_tx_launch_fifo7_unc_or_parity_err_cnt),
4849[C_TX_LAUNCH_FIFO6_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo6UncOrParityErr",
4850 0, 0, CNTR_NORMAL,
4851 access_tx_launch_fifo6_unc_or_parity_err_cnt),
4852[C_TX_LAUNCH_FIFO5_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo5UncOrParityErr",
4853 0, 0, CNTR_NORMAL,
4854 access_tx_launch_fifo5_unc_or_parity_err_cnt),
4855[C_TX_LAUNCH_FIFO4_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo4UncOrParityErr",
4856 0, 0, CNTR_NORMAL,
4857 access_tx_launch_fifo4_unc_or_parity_err_cnt),
4858[C_TX_LAUNCH_FIFO3_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo3UncOrParityErr",
4859 0, 0, CNTR_NORMAL,
4860 access_tx_launch_fifo3_unc_or_parity_err_cnt),
4861[C_TX_LAUNCH_FIFO2_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo2UncOrParityErr",
4862 0, 0, CNTR_NORMAL,
4863 access_tx_launch_fifo2_unc_or_parity_err_cnt),
4864[C_TX_LAUNCH_FIFO1_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo1UncOrParityErr",
4865 0, 0, CNTR_NORMAL,
4866 access_tx_launch_fifo1_unc_or_parity_err_cnt),
4867[C_TX_LAUNCH_FIFO0_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo0UncOrParityErr",
4868 0, 0, CNTR_NORMAL,
4869 access_tx_launch_fifo0_unc_or_parity_err_cnt),
4870[C_TX_SDMA15_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma15DisallowedPacketErr",
4871 0, 0, CNTR_NORMAL,
4872 access_tx_sdma15_disallowed_packet_err_cnt),
4873[C_TX_SDMA14_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma14DisallowedPacketErr",
4874 0, 0, CNTR_NORMAL,
4875 access_tx_sdma14_disallowed_packet_err_cnt),
4876[C_TX_SDMA13_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma13DisallowedPacketErr",
4877 0, 0, CNTR_NORMAL,
4878 access_tx_sdma13_disallowed_packet_err_cnt),
4879[C_TX_SDMA12_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma12DisallowedPacketErr",
4880 0, 0, CNTR_NORMAL,
4881 access_tx_sdma12_disallowed_packet_err_cnt),
4882[C_TX_SDMA11_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma11DisallowedPacketErr",
4883 0, 0, CNTR_NORMAL,
4884 access_tx_sdma11_disallowed_packet_err_cnt),
4885[C_TX_SDMA10_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma10DisallowedPacketErr",
4886 0, 0, CNTR_NORMAL,
4887 access_tx_sdma10_disallowed_packet_err_cnt),
4888[C_TX_SDMA9_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma9DisallowedPacketErr",
4889 0, 0, CNTR_NORMAL,
4890 access_tx_sdma9_disallowed_packet_err_cnt),
4891[C_TX_SDMA8_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma8DisallowedPacketErr",
4892 0, 0, CNTR_NORMAL,
4893 access_tx_sdma8_disallowed_packet_err_cnt),
4894[C_TX_SDMA7_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma7DisallowedPacketErr",
4895 0, 0, CNTR_NORMAL,
4896 access_tx_sdma7_disallowed_packet_err_cnt),
4897[C_TX_SDMA6_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma6DisallowedPacketErr",
4898 0, 0, CNTR_NORMAL,
4899 access_tx_sdma6_disallowed_packet_err_cnt),
4900[C_TX_SDMA5_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma5DisallowedPacketErr",
4901 0, 0, CNTR_NORMAL,
4902 access_tx_sdma5_disallowed_packet_err_cnt),
4903[C_TX_SDMA4_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma4DisallowedPacketErr",
4904 0, 0, CNTR_NORMAL,
4905 access_tx_sdma4_disallowed_packet_err_cnt),
4906[C_TX_SDMA3_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma3DisallowedPacketErr",
4907 0, 0, CNTR_NORMAL,
4908 access_tx_sdma3_disallowed_packet_err_cnt),
4909[C_TX_SDMA2_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma2DisallowedPacketErr",
4910 0, 0, CNTR_NORMAL,
4911 access_tx_sdma2_disallowed_packet_err_cnt),
4912[C_TX_SDMA1_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma1DisallowedPacketErr",
4913 0, 0, CNTR_NORMAL,
4914 access_tx_sdma1_disallowed_packet_err_cnt),
4915[C_TX_SDMA0_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma0DisallowedPacketErr",
4916 0, 0, CNTR_NORMAL,
4917 access_tx_sdma0_disallowed_packet_err_cnt),
4918[C_TX_CONFIG_PARITY_ERR] = CNTR_ELEM("TxConfigParityErr", 0, 0,
4919 CNTR_NORMAL,
4920 access_tx_config_parity_err_cnt),
4921[C_TX_SBRD_CTL_CSR_PARITY_ERR] = CNTR_ELEM("TxSbrdCtlCsrParityErr", 0, 0,
4922 CNTR_NORMAL,
4923 access_tx_sbrd_ctl_csr_parity_err_cnt),
4924[C_TX_LAUNCH_CSR_PARITY_ERR] = CNTR_ELEM("TxLaunchCsrParityErr", 0, 0,
4925 CNTR_NORMAL,
4926 access_tx_launch_csr_parity_err_cnt),
4927[C_TX_ILLEGAL_CL_ERR] = CNTR_ELEM("TxIllegalVLErr", 0, 0,
4928 CNTR_NORMAL,
4929 access_tx_illegal_vl_err_cnt),
4930[C_TX_SBRD_CTL_STATE_MACHINE_PARITY_ERR] = CNTR_ELEM(
4931 "TxSbrdCtlStateMachineParityErr", 0, 0,
4932 CNTR_NORMAL,
4933 access_tx_sbrd_ctl_state_machine_parity_err_cnt),
4934[C_TX_RESERVED_10] = CNTR_ELEM("Tx Egress Reserved 10", 0, 0,
4935 CNTR_NORMAL,
4936 access_egress_reserved_10_err_cnt),
4937[C_TX_RESERVED_9] = CNTR_ELEM("Tx Egress Reserved 9", 0, 0,
4938 CNTR_NORMAL,
4939 access_egress_reserved_9_err_cnt),
4940[C_TX_SDMA_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxSdmaLaunchIntfParityErr",
4941 0, 0, CNTR_NORMAL,
4942 access_tx_sdma_launch_intf_parity_err_cnt),
4943[C_TX_PIO_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxPioLaunchIntfParityErr", 0, 0,
4944 CNTR_NORMAL,
4945 access_tx_pio_launch_intf_parity_err_cnt),
4946[C_TX_RESERVED_6] = CNTR_ELEM("Tx Egress Reserved 6", 0, 0,
4947 CNTR_NORMAL,
4948 access_egress_reserved_6_err_cnt),
4949[C_TX_INCORRECT_LINK_STATE_ERR] = CNTR_ELEM("TxIncorrectLinkStateErr", 0, 0,
4950 CNTR_NORMAL,
4951 access_tx_incorrect_link_state_err_cnt),
4952[C_TX_LINK_DOWN_ERR] = CNTR_ELEM("TxLinkdownErr", 0, 0,
4953 CNTR_NORMAL,
4954 access_tx_linkdown_err_cnt),
4955[C_TX_EGRESS_FIFO_UNDERRUN_OR_PARITY_ERR] = CNTR_ELEM(
4956 "EgressFifoUnderrunOrParityErr", 0, 0,
4957 CNTR_NORMAL,
4958 access_tx_egress_fifi_underrun_or_parity_err_cnt),
4959[C_TX_RESERVED_2] = CNTR_ELEM("Tx Egress Reserved 2", 0, 0,
4960 CNTR_NORMAL,
4961 access_egress_reserved_2_err_cnt),
4962[C_TX_PKT_INTEGRITY_MEM_UNC_ERR] = CNTR_ELEM("TxPktIntegrityMemUncErr", 0, 0,
4963 CNTR_NORMAL,
4964 access_tx_pkt_integrity_mem_unc_err_cnt),
4965[C_TX_PKT_INTEGRITY_MEM_COR_ERR] = CNTR_ELEM("TxPktIntegrityMemCorErr", 0, 0,
4966 CNTR_NORMAL,
4967 access_tx_pkt_integrity_mem_cor_err_cnt),
4968
4969[C_SEND_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("SendCsrWriteBadAddrErr", 0, 0,
4970 CNTR_NORMAL,
4971 access_send_csr_write_bad_addr_err_cnt),
4972[C_SEND_CSR_READ_BAD_ADD_ERR] = CNTR_ELEM("SendCsrReadBadAddrErr", 0, 0,
4973 CNTR_NORMAL,
4974 access_send_csr_read_bad_addr_err_cnt),
4975[C_SEND_CSR_PARITY_ERR] = CNTR_ELEM("SendCsrParityErr", 0, 0,
4976 CNTR_NORMAL,
4977 access_send_csr_parity_cnt),
4978
4979[C_PIO_WRITE_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("PioWriteOutOfBoundsErr", 0, 0,
4980 CNTR_NORMAL,
4981 access_pio_write_out_of_bounds_err_cnt),
4982[C_PIO_WRITE_OVERFLOW_ERR] = CNTR_ELEM("PioWriteOverflowErr", 0, 0,
4983 CNTR_NORMAL,
4984 access_pio_write_overflow_err_cnt),
4985[C_PIO_WRITE_CROSSES_BOUNDARY_ERR] = CNTR_ELEM("PioWriteCrossesBoundaryErr",
4986 0, 0, CNTR_NORMAL,
4987 access_pio_write_crosses_boundary_err_cnt),
4988[C_PIO_DISALLOWED_PACKET_ERR] = CNTR_ELEM("PioDisallowedPacketErr", 0, 0,
4989 CNTR_NORMAL,
4990 access_pio_disallowed_packet_err_cnt),
4991[C_PIO_INCONSISTENT_SOP_ERR] = CNTR_ELEM("PioInconsistentSopErr", 0, 0,
4992 CNTR_NORMAL,
4993 access_pio_inconsistent_sop_err_cnt),
4994
4995[C_SDMA_HEADER_REQUEST_FIFO_COR_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoCorErr",
4996 0, 0, CNTR_NORMAL,
4997 access_sdma_header_request_fifo_cor_err_cnt),
4998[C_SDMA_HEADER_STORAGE_COR_ERR] = CNTR_ELEM("SDmaHeaderStorageCorErr", 0, 0,
4999 CNTR_NORMAL,
5000 access_sdma_header_storage_cor_err_cnt),
5001[C_SDMA_PACKET_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPacketTrackingCorErr", 0, 0,
5002 CNTR_NORMAL,
5003 access_sdma_packet_tracking_cor_err_cnt),
5004[C_SDMA_ASSEMBLY_COR_ERR] = CNTR_ELEM("SDmaAssemblyCorErr", 0, 0,
5005 CNTR_NORMAL,
5006 access_sdma_assembly_cor_err_cnt),
5007[C_SDMA_DESC_TABLE_COR_ERR] = CNTR_ELEM("SDmaDescTableCorErr", 0, 0,
5008 CNTR_NORMAL,
5009 access_sdma_desc_table_cor_err_cnt),
5010[C_SDMA_HEADER_REQUEST_FIFO_UNC_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoUncErr",
5011 0, 0, CNTR_NORMAL,
5012 access_sdma_header_request_fifo_unc_err_cnt),
5013[C_SDMA_HEADER_STORAGE_UNC_ERR] = CNTR_ELEM("SDmaHeaderStorageUncErr", 0, 0,
5014 CNTR_NORMAL,
5015 access_sdma_header_storage_unc_err_cnt),
5016[C_SDMA_PACKET_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPacketTrackingUncErr", 0, 0,
5017 CNTR_NORMAL,
5018 access_sdma_packet_tracking_unc_err_cnt),
5019[C_SDMA_ASSEMBLY_UNC_ERR] = CNTR_ELEM("SDmaAssemblyUncErr", 0, 0,
5020 CNTR_NORMAL,
5021 access_sdma_assembly_unc_err_cnt),
5022[C_SDMA_DESC_TABLE_UNC_ERR] = CNTR_ELEM("SDmaDescTableUncErr", 0, 0,
5023 CNTR_NORMAL,
5024 access_sdma_desc_table_unc_err_cnt),
5025[C_SDMA_TIMEOUT_ERR] = CNTR_ELEM("SDmaTimeoutErr", 0, 0,
5026 CNTR_NORMAL,
5027 access_sdma_timeout_err_cnt),
5028[C_SDMA_HEADER_LENGTH_ERR] = CNTR_ELEM("SDmaHeaderLengthErr", 0, 0,
5029 CNTR_NORMAL,
5030 access_sdma_header_length_err_cnt),
5031[C_SDMA_HEADER_ADDRESS_ERR] = CNTR_ELEM("SDmaHeaderAddressErr", 0, 0,
5032 CNTR_NORMAL,
5033 access_sdma_header_address_err_cnt),
5034[C_SDMA_HEADER_SELECT_ERR] = CNTR_ELEM("SDmaHeaderSelectErr", 0, 0,
5035 CNTR_NORMAL,
5036 access_sdma_header_select_err_cnt),
5037[C_SMDA_RESERVED_9] = CNTR_ELEM("SDma Reserved 9", 0, 0,
5038 CNTR_NORMAL,
5039 access_sdma_reserved_9_err_cnt),
5040[C_SDMA_PACKET_DESC_OVERFLOW_ERR] = CNTR_ELEM("SDmaPacketDescOverflowErr", 0, 0,
5041 CNTR_NORMAL,
5042 access_sdma_packet_desc_overflow_err_cnt),
5043[C_SDMA_LENGTH_MISMATCH_ERR] = CNTR_ELEM("SDmaLengthMismatchErr", 0, 0,
5044 CNTR_NORMAL,
5045 access_sdma_length_mismatch_err_cnt),
5046[C_SDMA_HALT_ERR] = CNTR_ELEM("SDmaHaltErr", 0, 0,
5047 CNTR_NORMAL,
5048 access_sdma_halt_err_cnt),
5049[C_SDMA_MEM_READ_ERR] = CNTR_ELEM("SDmaMemReadErr", 0, 0,
5050 CNTR_NORMAL,
5051 access_sdma_mem_read_err_cnt),
5052[C_SDMA_FIRST_DESC_ERR] = CNTR_ELEM("SDmaFirstDescErr", 0, 0,
5053 CNTR_NORMAL,
5054 access_sdma_first_desc_err_cnt),
5055[C_SDMA_TAIL_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("SDmaTailOutOfBoundsErr", 0, 0,
5056 CNTR_NORMAL,
5057 access_sdma_tail_out_of_bounds_err_cnt),
5058[C_SDMA_TOO_LONG_ERR] = CNTR_ELEM("SDmaTooLongErr", 0, 0,
5059 CNTR_NORMAL,
5060 access_sdma_too_long_err_cnt),
5061[C_SDMA_GEN_MISMATCH_ERR] = CNTR_ELEM("SDmaGenMismatchErr", 0, 0,
5062 CNTR_NORMAL,
5063 access_sdma_gen_mismatch_err_cnt),
5064[C_SDMA_WRONG_DW_ERR] = CNTR_ELEM("SDmaWrongDwErr", 0, 0,
5065 CNTR_NORMAL,
5066 access_sdma_wrong_dw_err_cnt),
5067};
5068
5069static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = {
5070[C_TX_UNSUP_VL] = TXE32_PORT_CNTR_ELEM(TxUnVLErr, SEND_UNSUP_VL_ERR_CNT,
5071 CNTR_NORMAL),
5072[C_TX_INVAL_LEN] = TXE32_PORT_CNTR_ELEM(TxInvalLen, SEND_LEN_ERR_CNT,
5073 CNTR_NORMAL),
5074[C_TX_MM_LEN_ERR] = TXE32_PORT_CNTR_ELEM(TxMMLenErr, SEND_MAX_MIN_LEN_ERR_CNT,
5075 CNTR_NORMAL),
5076[C_TX_UNDERRUN] = TXE32_PORT_CNTR_ELEM(TxUnderrun, SEND_UNDERRUN_CNT,
5077 CNTR_NORMAL),
5078[C_TX_FLOW_STALL] = TXE32_PORT_CNTR_ELEM(TxFlowStall, SEND_FLOW_STALL_CNT,
5079 CNTR_NORMAL),
5080[C_TX_DROPPED] = TXE32_PORT_CNTR_ELEM(TxDropped, SEND_DROPPED_PKT_CNT,
5081 CNTR_NORMAL),
5082[C_TX_HDR_ERR] = TXE32_PORT_CNTR_ELEM(TxHdrErr, SEND_HEADERS_ERR_CNT,
5083 CNTR_NORMAL),
5084[C_TX_PKT] = TXE64_PORT_CNTR_ELEM(TxPkt, SEND_DATA_PKT_CNT, CNTR_NORMAL),
5085[C_TX_WORDS] = TXE64_PORT_CNTR_ELEM(TxWords, SEND_DWORD_CNT, CNTR_NORMAL),
5086[C_TX_WAIT] = TXE64_PORT_CNTR_ELEM(TxWait, SEND_WAIT_CNT, CNTR_SYNTH),
5087[C_TX_FLIT_VL] = TXE64_PORT_CNTR_ELEM(TxFlitVL, SEND_DATA_VL0_CNT,
5088 CNTR_SYNTH | CNTR_VL),
5089[C_TX_PKT_VL] = TXE64_PORT_CNTR_ELEM(TxPktVL, SEND_DATA_PKT_VL0_CNT,
5090 CNTR_SYNTH | CNTR_VL),
5091[C_TX_WAIT_VL] = TXE64_PORT_CNTR_ELEM(TxWaitVL, SEND_WAIT_VL0_CNT,
5092 CNTR_SYNTH | CNTR_VL),
5093[C_RX_PKT] = RXE64_PORT_CNTR_ELEM(RxPkt, RCV_DATA_PKT_CNT, CNTR_NORMAL),
5094[C_RX_WORDS] = RXE64_PORT_CNTR_ELEM(RxWords, RCV_DWORD_CNT, CNTR_NORMAL),
5095[C_SW_LINK_DOWN] = CNTR_ELEM("SwLinkDown", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5096 access_sw_link_dn_cnt),
5097[C_SW_LINK_UP] = CNTR_ELEM("SwLinkUp", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5098 access_sw_link_up_cnt),
5099[C_SW_UNKNOWN_FRAME] = CNTR_ELEM("UnknownFrame", 0, 0, CNTR_NORMAL,
5100 access_sw_unknown_frame_cnt),
5101[C_SW_XMIT_DSCD] = CNTR_ELEM("XmitDscd", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5102 access_sw_xmit_discards),
5103[C_SW_XMIT_DSCD_VL] = CNTR_ELEM("XmitDscdVl", 0, 0,
5104 CNTR_SYNTH | CNTR_32BIT | CNTR_VL,
5105 access_sw_xmit_discards),
5106[C_SW_XMIT_CSTR_ERR] = CNTR_ELEM("XmitCstrErr", 0, 0, CNTR_SYNTH,
5107 access_xmit_constraint_errs),
5108[C_SW_RCV_CSTR_ERR] = CNTR_ELEM("RcvCstrErr", 0, 0, CNTR_SYNTH,
5109 access_rcv_constraint_errs),
5110[C_SW_IBP_LOOP_PKTS] = SW_IBP_CNTR(LoopPkts, loop_pkts),
5111[C_SW_IBP_RC_RESENDS] = SW_IBP_CNTR(RcResend, rc_resends),
5112[C_SW_IBP_RNR_NAKS] = SW_IBP_CNTR(RnrNak, rnr_naks),
5113[C_SW_IBP_OTHER_NAKS] = SW_IBP_CNTR(OtherNak, other_naks),
5114[C_SW_IBP_RC_TIMEOUTS] = SW_IBP_CNTR(RcTimeOut, rc_timeouts),
5115[C_SW_IBP_PKT_DROPS] = SW_IBP_CNTR(PktDrop, pkt_drops),
5116[C_SW_IBP_DMA_WAIT] = SW_IBP_CNTR(DmaWait, dmawait),
5117[C_SW_IBP_RC_SEQNAK] = SW_IBP_CNTR(RcSeqNak, rc_seqnak),
5118[C_SW_IBP_RC_DUPREQ] = SW_IBP_CNTR(RcDupRew, rc_dupreq),
5119[C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq),
5120[C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned),
5121[C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks),
5122[C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL,
5123 access_sw_cpu_rc_acks),
5124[C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL,
5125 access_sw_cpu_rc_qacks),
5126[C_SW_CPU_RC_DELAYED_COMP] = CNTR_ELEM("RcDelayComp", 0, 0, CNTR_NORMAL,
5127 access_sw_cpu_rc_delayed_comp),
5128[OVR_LBL(0)] = OVR_ELM(0), [OVR_LBL(1)] = OVR_ELM(1),
5129[OVR_LBL(2)] = OVR_ELM(2), [OVR_LBL(3)] = OVR_ELM(3),
5130[OVR_LBL(4)] = OVR_ELM(4), [OVR_LBL(5)] = OVR_ELM(5),
5131[OVR_LBL(6)] = OVR_ELM(6), [OVR_LBL(7)] = OVR_ELM(7),
5132[OVR_LBL(8)] = OVR_ELM(8), [OVR_LBL(9)] = OVR_ELM(9),
5133[OVR_LBL(10)] = OVR_ELM(10), [OVR_LBL(11)] = OVR_ELM(11),
5134[OVR_LBL(12)] = OVR_ELM(12), [OVR_LBL(13)] = OVR_ELM(13),
5135[OVR_LBL(14)] = OVR_ELM(14), [OVR_LBL(15)] = OVR_ELM(15),
5136[OVR_LBL(16)] = OVR_ELM(16), [OVR_LBL(17)] = OVR_ELM(17),
5137[OVR_LBL(18)] = OVR_ELM(18), [OVR_LBL(19)] = OVR_ELM(19),
5138[OVR_LBL(20)] = OVR_ELM(20), [OVR_LBL(21)] = OVR_ELM(21),
5139[OVR_LBL(22)] = OVR_ELM(22), [OVR_LBL(23)] = OVR_ELM(23),
5140[OVR_LBL(24)] = OVR_ELM(24), [OVR_LBL(25)] = OVR_ELM(25),
5141[OVR_LBL(26)] = OVR_ELM(26), [OVR_LBL(27)] = OVR_ELM(27),
5142[OVR_LBL(28)] = OVR_ELM(28), [OVR_LBL(29)] = OVR_ELM(29),
5143[OVR_LBL(30)] = OVR_ELM(30), [OVR_LBL(31)] = OVR_ELM(31),
5144[OVR_LBL(32)] = OVR_ELM(32), [OVR_LBL(33)] = OVR_ELM(33),
5145[OVR_LBL(34)] = OVR_ELM(34), [OVR_LBL(35)] = OVR_ELM(35),
5146[OVR_LBL(36)] = OVR_ELM(36), [OVR_LBL(37)] = OVR_ELM(37),
5147[OVR_LBL(38)] = OVR_ELM(38), [OVR_LBL(39)] = OVR_ELM(39),
5148[OVR_LBL(40)] = OVR_ELM(40), [OVR_LBL(41)] = OVR_ELM(41),
5149[OVR_LBL(42)] = OVR_ELM(42), [OVR_LBL(43)] = OVR_ELM(43),
5150[OVR_LBL(44)] = OVR_ELM(44), [OVR_LBL(45)] = OVR_ELM(45),
5151[OVR_LBL(46)] = OVR_ELM(46), [OVR_LBL(47)] = OVR_ELM(47),
5152[OVR_LBL(48)] = OVR_ELM(48), [OVR_LBL(49)] = OVR_ELM(49),
5153[OVR_LBL(50)] = OVR_ELM(50), [OVR_LBL(51)] = OVR_ELM(51),
5154[OVR_LBL(52)] = OVR_ELM(52), [OVR_LBL(53)] = OVR_ELM(53),
5155[OVR_LBL(54)] = OVR_ELM(54), [OVR_LBL(55)] = OVR_ELM(55),
5156[OVR_LBL(56)] = OVR_ELM(56), [OVR_LBL(57)] = OVR_ELM(57),
5157[OVR_LBL(58)] = OVR_ELM(58), [OVR_LBL(59)] = OVR_ELM(59),
5158[OVR_LBL(60)] = OVR_ELM(60), [OVR_LBL(61)] = OVR_ELM(61),
5159[OVR_LBL(62)] = OVR_ELM(62), [OVR_LBL(63)] = OVR_ELM(63),
5160[OVR_LBL(64)] = OVR_ELM(64), [OVR_LBL(65)] = OVR_ELM(65),
5161[OVR_LBL(66)] = OVR_ELM(66), [OVR_LBL(67)] = OVR_ELM(67),
5162[OVR_LBL(68)] = OVR_ELM(68), [OVR_LBL(69)] = OVR_ELM(69),
5163[OVR_LBL(70)] = OVR_ELM(70), [OVR_LBL(71)] = OVR_ELM(71),
5164[OVR_LBL(72)] = OVR_ELM(72), [OVR_LBL(73)] = OVR_ELM(73),
5165[OVR_LBL(74)] = OVR_ELM(74), [OVR_LBL(75)] = OVR_ELM(75),
5166[OVR_LBL(76)] = OVR_ELM(76), [OVR_LBL(77)] = OVR_ELM(77),
5167[OVR_LBL(78)] = OVR_ELM(78), [OVR_LBL(79)] = OVR_ELM(79),
5168[OVR_LBL(80)] = OVR_ELM(80), [OVR_LBL(81)] = OVR_ELM(81),
5169[OVR_LBL(82)] = OVR_ELM(82), [OVR_LBL(83)] = OVR_ELM(83),
5170[OVR_LBL(84)] = OVR_ELM(84), [OVR_LBL(85)] = OVR_ELM(85),
5171[OVR_LBL(86)] = OVR_ELM(86), [OVR_LBL(87)] = OVR_ELM(87),
5172[OVR_LBL(88)] = OVR_ELM(88), [OVR_LBL(89)] = OVR_ELM(89),
5173[OVR_LBL(90)] = OVR_ELM(90), [OVR_LBL(91)] = OVR_ELM(91),
5174[OVR_LBL(92)] = OVR_ELM(92), [OVR_LBL(93)] = OVR_ELM(93),
5175[OVR_LBL(94)] = OVR_ELM(94), [OVR_LBL(95)] = OVR_ELM(95),
5176[OVR_LBL(96)] = OVR_ELM(96), [OVR_LBL(97)] = OVR_ELM(97),
5177[OVR_LBL(98)] = OVR_ELM(98), [OVR_LBL(99)] = OVR_ELM(99),
5178[OVR_LBL(100)] = OVR_ELM(100), [OVR_LBL(101)] = OVR_ELM(101),
5179[OVR_LBL(102)] = OVR_ELM(102), [OVR_LBL(103)] = OVR_ELM(103),
5180[OVR_LBL(104)] = OVR_ELM(104), [OVR_LBL(105)] = OVR_ELM(105),
5181[OVR_LBL(106)] = OVR_ELM(106), [OVR_LBL(107)] = OVR_ELM(107),
5182[OVR_LBL(108)] = OVR_ELM(108), [OVR_LBL(109)] = OVR_ELM(109),
5183[OVR_LBL(110)] = OVR_ELM(110), [OVR_LBL(111)] = OVR_ELM(111),
5184[OVR_LBL(112)] = OVR_ELM(112), [OVR_LBL(113)] = OVR_ELM(113),
5185[OVR_LBL(114)] = OVR_ELM(114), [OVR_LBL(115)] = OVR_ELM(115),
5186[OVR_LBL(116)] = OVR_ELM(116), [OVR_LBL(117)] = OVR_ELM(117),
5187[OVR_LBL(118)] = OVR_ELM(118), [OVR_LBL(119)] = OVR_ELM(119),
5188[OVR_LBL(120)] = OVR_ELM(120), [OVR_LBL(121)] = OVR_ELM(121),
5189[OVR_LBL(122)] = OVR_ELM(122), [OVR_LBL(123)] = OVR_ELM(123),
5190[OVR_LBL(124)] = OVR_ELM(124), [OVR_LBL(125)] = OVR_ELM(125),
5191[OVR_LBL(126)] = OVR_ELM(126), [OVR_LBL(127)] = OVR_ELM(127),
5192[OVR_LBL(128)] = OVR_ELM(128), [OVR_LBL(129)] = OVR_ELM(129),
5193[OVR_LBL(130)] = OVR_ELM(130), [OVR_LBL(131)] = OVR_ELM(131),
5194[OVR_LBL(132)] = OVR_ELM(132), [OVR_LBL(133)] = OVR_ELM(133),
5195[OVR_LBL(134)] = OVR_ELM(134), [OVR_LBL(135)] = OVR_ELM(135),
5196[OVR_LBL(136)] = OVR_ELM(136), [OVR_LBL(137)] = OVR_ELM(137),
5197[OVR_LBL(138)] = OVR_ELM(138), [OVR_LBL(139)] = OVR_ELM(139),
5198[OVR_LBL(140)] = OVR_ELM(140), [OVR_LBL(141)] = OVR_ELM(141),
5199[OVR_LBL(142)] = OVR_ELM(142), [OVR_LBL(143)] = OVR_ELM(143),
5200[OVR_LBL(144)] = OVR_ELM(144), [OVR_LBL(145)] = OVR_ELM(145),
5201[OVR_LBL(146)] = OVR_ELM(146), [OVR_LBL(147)] = OVR_ELM(147),
5202[OVR_LBL(148)] = OVR_ELM(148), [OVR_LBL(149)] = OVR_ELM(149),
5203[OVR_LBL(150)] = OVR_ELM(150), [OVR_LBL(151)] = OVR_ELM(151),
5204[OVR_LBL(152)] = OVR_ELM(152), [OVR_LBL(153)] = OVR_ELM(153),
5205[OVR_LBL(154)] = OVR_ELM(154), [OVR_LBL(155)] = OVR_ELM(155),
5206[OVR_LBL(156)] = OVR_ELM(156), [OVR_LBL(157)] = OVR_ELM(157),
5207[OVR_LBL(158)] = OVR_ELM(158), [OVR_LBL(159)] = OVR_ELM(159),
5208};
5209
5210
5211
5212
5213int is_ax(struct hfi1_devdata *dd)
5214{
5215 u8 chip_rev_minor =
5216 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5217 & CCE_REVISION_CHIP_REV_MINOR_MASK;
5218 return (chip_rev_minor & 0xf0) == 0;
5219}
5220
5221
5222int is_bx(struct hfi1_devdata *dd)
5223{
5224 u8 chip_rev_minor =
5225 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5226 & CCE_REVISION_CHIP_REV_MINOR_MASK;
5227 return (chip_rev_minor & 0xF0) == 0x10;
5228}
5229
5230
5231bool is_urg_masked(struct hfi1_ctxtdata *rcd)
5232{
5233 u64 mask;
5234 u32 is = IS_RCVURGENT_START + rcd->ctxt;
5235 u8 bit = is % 64;
5236
5237 mask = read_csr(rcd->dd, CCE_INT_MASK + (8 * (is / 64)));
5238 return !(mask & BIT_ULL(bit));
5239}
5240
5241
5242
5243
5244
5245
5246
5247static int append_str(char *buf, char **curp, int *lenp, const char *s)
5248{
5249 char *p = *curp;
5250 int len = *lenp;
5251 int result = 0;
5252 char c;
5253
5254
5255 if (p != buf) {
5256 if (len == 0) {
5257 result = 1;
5258 goto done;
5259 }
5260 *p++ = ',';
5261 len--;
5262 }
5263
5264
5265 while ((c = *s++) != 0) {
5266 if (len == 0) {
5267 result = 1;
5268 goto done;
5269 }
5270 *p++ = c;
5271 len--;
5272 }
5273
5274done:
5275
5276 *curp = p;
5277 *lenp = len;
5278
5279 return result;
5280}
5281
5282
5283
5284
5285
5286static char *flag_string(char *buf, int buf_len, u64 flags,
5287 struct flag_table *table, int table_size)
5288{
5289 char extra[32];
5290 char *p = buf;
5291 int len = buf_len;
5292 int no_room = 0;
5293 int i;
5294
5295
5296 if (len < 2)
5297 return "";
5298
5299 len--;
5300 for (i = 0; i < table_size; i++) {
5301 if (flags & table[i].flag) {
5302 no_room = append_str(buf, &p, &len, table[i].str);
5303 if (no_room)
5304 break;
5305 flags &= ~table[i].flag;
5306 }
5307 }
5308
5309
5310 if (!no_room && flags) {
5311 snprintf(extra, sizeof(extra), "bits 0x%llx", flags);
5312 no_room = append_str(buf, &p, &len, extra);
5313 }
5314
5315
5316 if (no_room) {
5317
5318 if (len == 0)
5319 --p;
5320 *p++ = '*';
5321 }
5322
5323
5324 *p = 0;
5325 return buf;
5326}
5327
5328
5329static const char * const cce_misc_names[] = {
5330 "CceErrInt",
5331 "RxeErrInt",
5332 "MiscErrInt",
5333 "Reserved3",
5334 "PioErrInt",
5335 "SDmaErrInt",
5336 "EgressErrInt",
5337 "TxeErrInt"
5338};
5339
5340
5341
5342
5343static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source)
5344{
5345 if (source < ARRAY_SIZE(cce_misc_names))
5346 strncpy(buf, cce_misc_names[source], bsize);
5347 else
5348 snprintf(buf, bsize, "Reserved%u",
5349 source + IS_GENERAL_ERR_START);
5350
5351 return buf;
5352}
5353
5354
5355
5356
5357static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source)
5358{
5359 snprintf(buf, bsize, "SDmaEngErrInt%u", source);
5360 return buf;
5361}
5362
5363
5364
5365
5366static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source)
5367{
5368 snprintf(buf, bsize, "SendCtxtErrInt%u", source);
5369 return buf;
5370}
5371
5372static const char * const various_names[] = {
5373 "PbcInt",
5374 "GpioAssertInt",
5375 "Qsfp1Int",
5376 "Qsfp2Int",
5377 "TCritInt"
5378};
5379
5380
5381
5382
5383static char *is_various_name(char *buf, size_t bsize, unsigned int source)
5384{
5385 if (source < ARRAY_SIZE(various_names))
5386 strncpy(buf, various_names[source], bsize);
5387 else
5388 snprintf(buf, bsize, "Reserved%u", source + IS_VARIOUS_START);
5389 return buf;
5390}
5391
5392
5393
5394
5395static char *is_dc_name(char *buf, size_t bsize, unsigned int source)
5396{
5397 static const char * const dc_int_names[] = {
5398 "common",
5399 "lcb",
5400 "8051",
5401 "lbm"
5402 };
5403
5404 if (source < ARRAY_SIZE(dc_int_names))
5405 snprintf(buf, bsize, "dc_%s_int", dc_int_names[source]);
5406 else
5407 snprintf(buf, bsize, "DCInt%u", source);
5408 return buf;
5409}
5410
5411static const char * const sdma_int_names[] = {
5412 "SDmaInt",
5413 "SdmaIdleInt",
5414 "SdmaProgressInt",
5415};
5416
5417
5418
5419
5420static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source)
5421{
5422
5423 unsigned int what = source / TXE_NUM_SDMA_ENGINES;
5424
5425 unsigned int which = source % TXE_NUM_SDMA_ENGINES;
5426
5427 if (likely(what < 3))
5428 snprintf(buf, bsize, "%s%u", sdma_int_names[what], which);
5429 else
5430 snprintf(buf, bsize, "Invalid SDMA interrupt %u", source);
5431 return buf;
5432}
5433
5434
5435
5436
5437static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source)
5438{
5439 snprintf(buf, bsize, "RcvAvailInt%u", source);
5440 return buf;
5441}
5442
5443
5444
5445
5446static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source)
5447{
5448 snprintf(buf, bsize, "RcvUrgentInt%u", source);
5449 return buf;
5450}
5451
5452
5453
5454
5455static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source)
5456{
5457 snprintf(buf, bsize, "SendCreditInt%u", source);
5458 return buf;
5459}
5460
5461
5462
5463
5464static char *is_reserved_name(char *buf, size_t bsize, unsigned int source)
5465{
5466 snprintf(buf, bsize, "Reserved%u", source + IS_RESERVED_START);
5467 return buf;
5468}
5469
5470static char *cce_err_status_string(char *buf, int buf_len, u64 flags)
5471{
5472 return flag_string(buf, buf_len, flags,
5473 cce_err_status_flags,
5474 ARRAY_SIZE(cce_err_status_flags));
5475}
5476
5477static char *rxe_err_status_string(char *buf, int buf_len, u64 flags)
5478{
5479 return flag_string(buf, buf_len, flags,
5480 rxe_err_status_flags,
5481 ARRAY_SIZE(rxe_err_status_flags));
5482}
5483
5484static char *misc_err_status_string(char *buf, int buf_len, u64 flags)
5485{
5486 return flag_string(buf, buf_len, flags, misc_err_status_flags,
5487 ARRAY_SIZE(misc_err_status_flags));
5488}
5489
5490static char *pio_err_status_string(char *buf, int buf_len, u64 flags)
5491{
5492 return flag_string(buf, buf_len, flags,
5493 pio_err_status_flags,
5494 ARRAY_SIZE(pio_err_status_flags));
5495}
5496
5497static char *sdma_err_status_string(char *buf, int buf_len, u64 flags)
5498{
5499 return flag_string(buf, buf_len, flags,
5500 sdma_err_status_flags,
5501 ARRAY_SIZE(sdma_err_status_flags));
5502}
5503
5504static char *egress_err_status_string(char *buf, int buf_len, u64 flags)
5505{
5506 return flag_string(buf, buf_len, flags,
5507 egress_err_status_flags,
5508 ARRAY_SIZE(egress_err_status_flags));
5509}
5510
5511static char *egress_err_info_string(char *buf, int buf_len, u64 flags)
5512{
5513 return flag_string(buf, buf_len, flags,
5514 egress_err_info_flags,
5515 ARRAY_SIZE(egress_err_info_flags));
5516}
5517
5518static char *send_err_status_string(char *buf, int buf_len, u64 flags)
5519{
5520 return flag_string(buf, buf_len, flags,
5521 send_err_status_flags,
5522 ARRAY_SIZE(send_err_status_flags));
5523}
5524
5525static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5526{
5527 char buf[96];
5528 int i = 0;
5529
5530
5531
5532
5533
5534 dd_dev_info(dd, "CCE Error: %s\n",
5535 cce_err_status_string(buf, sizeof(buf), reg));
5536
5537 if ((reg & CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK) &&
5538 is_ax(dd) && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) {
5539
5540
5541 start_freeze_handling(dd->pport, FREEZE_SELF);
5542 }
5543
5544 for (i = 0; i < NUM_CCE_ERR_STATUS_COUNTERS; i++) {
5545 if (reg & (1ull << i)) {
5546 incr_cntr64(&dd->cce_err_status_cnt[i]);
5547
5548 incr_cntr64(&dd->sw_cce_err_status_aggregate);
5549 }
5550 }
5551}
5552
5553
5554
5555
5556
5557#define RCVERR_CHECK_TIME 10
5558static void update_rcverr_timer(struct timer_list *t)
5559{
5560 struct hfi1_devdata *dd = from_timer(dd, t, rcverr_timer);
5561 struct hfi1_pportdata *ppd = dd->pport;
5562 u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL);
5563
5564 if (dd->rcv_ovfl_cnt < cur_ovfl_cnt &&
5565 ppd->port_error_action & OPA_PI_MASK_EX_BUFFER_OVERRUN) {
5566 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
5567 set_link_down_reason(
5568 ppd, OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0,
5569 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN);
5570 queue_work(ppd->link_wq, &ppd->link_bounce_work);
5571 }
5572 dd->rcv_ovfl_cnt = (u32)cur_ovfl_cnt;
5573
5574 mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5575}
5576
5577static int init_rcverr(struct hfi1_devdata *dd)
5578{
5579 timer_setup(&dd->rcverr_timer, update_rcverr_timer, 0);
5580
5581 dd->rcv_ovfl_cnt = 0;
5582 return mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5583}
5584
5585static void free_rcverr(struct hfi1_devdata *dd)
5586{
5587 if (dd->rcverr_timer.function)
5588 del_timer_sync(&dd->rcverr_timer);
5589}
5590
5591static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5592{
5593 char buf[96];
5594 int i = 0;
5595
5596 dd_dev_info(dd, "Receive Error: %s\n",
5597 rxe_err_status_string(buf, sizeof(buf), reg));
5598
5599 if (reg & ALL_RXE_FREEZE_ERR) {
5600 int flags = 0;
5601
5602
5603
5604
5605
5606 if (is_ax(dd) && (reg & RXE_FREEZE_ABORT_MASK))
5607 flags = FREEZE_ABORT;
5608
5609 start_freeze_handling(dd->pport, flags);
5610 }
5611
5612 for (i = 0; i < NUM_RCV_ERR_STATUS_COUNTERS; i++) {
5613 if (reg & (1ull << i))
5614 incr_cntr64(&dd->rcv_err_status_cnt[i]);
5615 }
5616}
5617
5618static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5619{
5620 char buf[96];
5621 int i = 0;
5622
5623 dd_dev_info(dd, "Misc Error: %s",
5624 misc_err_status_string(buf, sizeof(buf), reg));
5625 for (i = 0; i < NUM_MISC_ERR_STATUS_COUNTERS; i++) {
5626 if (reg & (1ull << i))
5627 incr_cntr64(&dd->misc_err_status_cnt[i]);
5628 }
5629}
5630
5631static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5632{
5633 char buf[96];
5634 int i = 0;
5635
5636 dd_dev_info(dd, "PIO Error: %s\n",
5637 pio_err_status_string(buf, sizeof(buf), reg));
5638
5639 if (reg & ALL_PIO_FREEZE_ERR)
5640 start_freeze_handling(dd->pport, 0);
5641
5642 for (i = 0; i < NUM_SEND_PIO_ERR_STATUS_COUNTERS; i++) {
5643 if (reg & (1ull << i))
5644 incr_cntr64(&dd->send_pio_err_status_cnt[i]);
5645 }
5646}
5647
5648static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5649{
5650 char buf[96];
5651 int i = 0;
5652
5653 dd_dev_info(dd, "SDMA Error: %s\n",
5654 sdma_err_status_string(buf, sizeof(buf), reg));
5655
5656 if (reg & ALL_SDMA_FREEZE_ERR)
5657 start_freeze_handling(dd->pport, 0);
5658
5659 for (i = 0; i < NUM_SEND_DMA_ERR_STATUS_COUNTERS; i++) {
5660 if (reg & (1ull << i))
5661 incr_cntr64(&dd->send_dma_err_status_cnt[i]);
5662 }
5663}
5664
5665static inline void __count_port_discards(struct hfi1_pportdata *ppd)
5666{
5667 incr_cntr64(&ppd->port_xmit_discards);
5668}
5669
5670static void count_port_inactive(struct hfi1_devdata *dd)
5671{
5672 __count_port_discards(dd->pport);
5673}
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684static void handle_send_egress_err_info(struct hfi1_devdata *dd,
5685 int vl)
5686{
5687 struct hfi1_pportdata *ppd = dd->pport;
5688 u64 src = read_csr(dd, SEND_EGRESS_ERR_SOURCE);
5689 u64 info = read_csr(dd, SEND_EGRESS_ERR_INFO);
5690 char buf[96];
5691
5692
5693 write_csr(dd, SEND_EGRESS_ERR_INFO, info);
5694
5695 dd_dev_info(dd,
5696 "Egress Error Info: 0x%llx, %s Egress Error Src 0x%llx\n",
5697 info, egress_err_info_string(buf, sizeof(buf), info), src);
5698
5699
5700 if (info & PORT_DISCARD_EGRESS_ERRS) {
5701 int weight, i;
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725 weight = hweight64(info & PORT_DISCARD_EGRESS_ERRS);
5726 for (i = 0; i < weight; i++) {
5727 __count_port_discards(ppd);
5728 if (vl >= 0 && vl < TXE_NUM_DATA_VL)
5729 incr_cntr64(&ppd->port_xmit_discards_vl[vl]);
5730 else if (vl == 15)
5731 incr_cntr64(&ppd->port_xmit_discards_vl
5732 [C_VL_15]);
5733 }
5734 }
5735}
5736
5737
5738
5739
5740
5741static inline int port_inactive_err(u64 posn)
5742{
5743 return (posn >= SEES(TX_LINKDOWN) &&
5744 posn <= SEES(TX_INCORRECT_LINK_STATE));
5745}
5746
5747
5748
5749
5750
5751static inline int disallowed_pkt_err(int posn)
5752{
5753 return (posn >= SEES(TX_SDMA0_DISALLOWED_PACKET) &&
5754 posn <= SEES(TX_SDMA15_DISALLOWED_PACKET));
5755}
5756
5757
5758
5759
5760
5761
5762static inline int disallowed_pkt_engine(int posn)
5763{
5764 return posn - SEES(TX_SDMA0_DISALLOWED_PACKET);
5765}
5766
5767
5768
5769
5770
5771static int engine_to_vl(struct hfi1_devdata *dd, int engine)
5772{
5773 struct sdma_vl_map *m;
5774 int vl;
5775
5776
5777 if (engine < 0 || engine >= TXE_NUM_SDMA_ENGINES)
5778 return -1;
5779
5780 rcu_read_lock();
5781 m = rcu_dereference(dd->sdma_map);
5782 vl = m->engine_to_vl[engine];
5783 rcu_read_unlock();
5784
5785 return vl;
5786}
5787
5788
5789
5790
5791
5792static int sc_to_vl(struct hfi1_devdata *dd, int sw_index)
5793{
5794 struct send_context_info *sci;
5795 struct send_context *sc;
5796 int i;
5797
5798 sci = &dd->send_contexts[sw_index];
5799
5800
5801 if ((sci->type != SC_KERNEL) && (sci->type != SC_VL15))
5802 return -1;
5803
5804 sc = sci->sc;
5805 if (!sc)
5806 return -1;
5807 if (dd->vld[15].sc == sc)
5808 return 15;
5809 for (i = 0; i < num_vls; i++)
5810 if (dd->vld[i].sc == sc)
5811 return i;
5812
5813 return -1;
5814}
5815
5816static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5817{
5818 u64 reg_copy = reg, handled = 0;
5819 char buf[96];
5820 int i = 0;
5821
5822 if (reg & ALL_TXE_EGRESS_FREEZE_ERR)
5823 start_freeze_handling(dd->pport, 0);
5824 else if (is_ax(dd) &&
5825 (reg & SEND_EGRESS_ERR_STATUS_TX_CREDIT_RETURN_VL_ERR_SMASK) &&
5826 (dd->icode != ICODE_FUNCTIONAL_SIMULATOR))
5827 start_freeze_handling(dd->pport, 0);
5828
5829 while (reg_copy) {
5830 int posn = fls64(reg_copy);
5831
5832 int shift = posn - 1;
5833 u64 mask = 1ULL << shift;
5834
5835 if (port_inactive_err(shift)) {
5836 count_port_inactive(dd);
5837 handled |= mask;
5838 } else if (disallowed_pkt_err(shift)) {
5839 int vl = engine_to_vl(dd, disallowed_pkt_engine(shift));
5840
5841 handle_send_egress_err_info(dd, vl);
5842 handled |= mask;
5843 }
5844 reg_copy &= ~mask;
5845 }
5846
5847 reg &= ~handled;
5848
5849 if (reg)
5850 dd_dev_info(dd, "Egress Error: %s\n",
5851 egress_err_status_string(buf, sizeof(buf), reg));
5852
5853 for (i = 0; i < NUM_SEND_EGRESS_ERR_STATUS_COUNTERS; i++) {
5854 if (reg & (1ull << i))
5855 incr_cntr64(&dd->send_egress_err_status_cnt[i]);
5856 }
5857}
5858
5859static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5860{
5861 char buf[96];
5862 int i = 0;
5863
5864 dd_dev_info(dd, "Send Error: %s\n",
5865 send_err_status_string(buf, sizeof(buf), reg));
5866
5867 for (i = 0; i < NUM_SEND_ERR_STATUS_COUNTERS; i++) {
5868 if (reg & (1ull << i))
5869 incr_cntr64(&dd->send_err_status_cnt[i]);
5870 }
5871}
5872
5873
5874
5875
5876
5877#define MAX_CLEAR_COUNT 20
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890static void interrupt_clear_down(struct hfi1_devdata *dd,
5891 u32 context,
5892 const struct err_reg_info *eri)
5893{
5894 u64 reg;
5895 u32 count;
5896
5897
5898 count = 0;
5899 while (1) {
5900 reg = read_kctxt_csr(dd, context, eri->status);
5901 if (reg == 0)
5902 break;
5903 write_kctxt_csr(dd, context, eri->clear, reg);
5904 if (likely(eri->handler))
5905 eri->handler(dd, context, reg);
5906 count++;
5907 if (count > MAX_CLEAR_COUNT) {
5908 u64 mask;
5909
5910 dd_dev_err(dd, "Repeating %s bits 0x%llx - masking\n",
5911 eri->desc, reg);
5912
5913
5914
5915
5916 mask = read_kctxt_csr(dd, context, eri->mask);
5917 mask &= ~reg;
5918 write_kctxt_csr(dd, context, eri->mask, mask);
5919 break;
5920 }
5921 }
5922}
5923
5924
5925
5926
5927static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source)
5928{
5929 const struct err_reg_info *eri = &misc_errs[source];
5930
5931 if (eri->handler) {
5932 interrupt_clear_down(dd, 0, eri);
5933 } else {
5934 dd_dev_err(dd, "Unexpected misc interrupt (%u) - reserved\n",
5935 source);
5936 }
5937}
5938
5939static char *send_context_err_status_string(char *buf, int buf_len, u64 flags)
5940{
5941 return flag_string(buf, buf_len, flags,
5942 sc_err_status_flags,
5943 ARRAY_SIZE(sc_err_status_flags));
5944}
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955static void is_sendctxt_err_int(struct hfi1_devdata *dd,
5956 unsigned int hw_context)
5957{
5958 struct send_context_info *sci;
5959 struct send_context *sc;
5960 char flags[96];
5961 u64 status;
5962 u32 sw_index;
5963 int i = 0;
5964 unsigned long irq_flags;
5965
5966 sw_index = dd->hw_to_sw[hw_context];
5967 if (sw_index >= dd->num_send_contexts) {
5968 dd_dev_err(dd,
5969 "out of range sw index %u for send context %u\n",
5970 sw_index, hw_context);
5971 return;
5972 }
5973 sci = &dd->send_contexts[sw_index];
5974 spin_lock_irqsave(&dd->sc_lock, irq_flags);
5975 sc = sci->sc;
5976 if (!sc) {
5977 dd_dev_err(dd, "%s: context %u(%u): no sc?\n", __func__,
5978 sw_index, hw_context);
5979 spin_unlock_irqrestore(&dd->sc_lock, irq_flags);
5980 return;
5981 }
5982
5983
5984 sc_stop(sc, SCF_HALTED);
5985
5986 status = read_kctxt_csr(dd, hw_context, SEND_CTXT_ERR_STATUS);
5987
5988 dd_dev_info(dd, "Send Context %u(%u) Error: %s\n", sw_index, hw_context,
5989 send_context_err_status_string(flags, sizeof(flags),
5990 status));
5991
5992 if (status & SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK)
5993 handle_send_egress_err_info(dd, sc_to_vl(dd, sw_index));
5994
5995
5996
5997
5998
5999 if (sc->type != SC_USER)
6000 queue_work(dd->pport->hfi1_wq, &sc->halt_work);
6001 spin_unlock_irqrestore(&dd->sc_lock, irq_flags);
6002
6003
6004
6005
6006
6007
6008 for (i = 0; i < NUM_SEND_CTXT_ERR_STATUS_COUNTERS; i++) {
6009 if (status & (1ull << i))
6010 incr_cntr64(&dd->sw_ctxt_err_status_cnt[i]);
6011 }
6012}
6013
6014static void handle_sdma_eng_err(struct hfi1_devdata *dd,
6015 unsigned int source, u64 status)
6016{
6017 struct sdma_engine *sde;
6018 int i = 0;
6019
6020 sde = &dd->per_sdma[source];
6021#ifdef CONFIG_SDMA_VERBOSITY
6022 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
6023 slashstrip(__FILE__), __LINE__, __func__);
6024 dd_dev_err(sde->dd, "CONFIG SDMA(%u) source: %u status 0x%llx\n",
6025 sde->this_idx, source, (unsigned long long)status);
6026#endif
6027 sde->err_cnt++;
6028 sdma_engine_error(sde, status);
6029
6030
6031
6032
6033
6034
6035 for (i = 0; i < NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS; i++) {
6036 if (status & (1ull << i))
6037 incr_cntr64(&dd->sw_send_dma_eng_err_status_cnt[i]);
6038 }
6039}
6040
6041
6042
6043
6044static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source)
6045{
6046#ifdef CONFIG_SDMA_VERBOSITY
6047 struct sdma_engine *sde = &dd->per_sdma[source];
6048
6049 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
6050 slashstrip(__FILE__), __LINE__, __func__);
6051 dd_dev_err(dd, "CONFIG SDMA(%u) source: %u\n", sde->this_idx,
6052 source);
6053 sdma_dumpstate(sde);
6054#endif
6055 interrupt_clear_down(dd, source, &sdma_eng_err);
6056}
6057
6058
6059
6060
6061static void is_various_int(struct hfi1_devdata *dd, unsigned int source)
6062{
6063 const struct err_reg_info *eri = &various_err[source];
6064
6065
6066
6067
6068
6069
6070 if (source == TCRIT_INT_SOURCE)
6071 handle_temp_err(dd);
6072 else if (eri->handler)
6073 interrupt_clear_down(dd, 0, eri);
6074 else
6075 dd_dev_info(dd,
6076 "%s: Unimplemented/reserved interrupt %d\n",
6077 __func__, source);
6078}
6079
6080static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg)
6081{
6082
6083 struct hfi1_pportdata *ppd = dd->pport;
6084 unsigned long flags;
6085 u64 qsfp_int_mgmt = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
6086
6087 if (reg & QSFP_HFI0_MODPRST_N) {
6088 if (!qsfp_mod_present(ppd)) {
6089 dd_dev_info(dd, "%s: QSFP module removed\n",
6090 __func__);
6091
6092 ppd->driver_link_ready = 0;
6093
6094
6095
6096
6097
6098 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6099
6100
6101
6102
6103 ppd->qsfp_info.cache_valid = 0;
6104 ppd->qsfp_info.reset_needed = 0;
6105 ppd->qsfp_info.limiting_active = 0;
6106 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
6107 flags);
6108
6109 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
6110 ASIC_QSFP1_INVERT, qsfp_int_mgmt);
6111
6112 if ((ppd->offline_disabled_reason >
6113 HFI1_ODR_MASK(
6114 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED)) ||
6115 (ppd->offline_disabled_reason ==
6116 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE)))
6117 ppd->offline_disabled_reason =
6118 HFI1_ODR_MASK(
6119 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED);
6120
6121 if (ppd->host_link_state == HLS_DN_POLL) {
6122
6123
6124
6125
6126
6127
6128 queue_work(ppd->link_wq, &ppd->link_down_work);
6129 }
6130 } else {
6131 dd_dev_info(dd, "%s: QSFP module inserted\n",
6132 __func__);
6133
6134 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6135 ppd->qsfp_info.cache_valid = 0;
6136 ppd->qsfp_info.cache_refresh_required = 1;
6137 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
6138 flags);
6139
6140
6141
6142
6143
6144 qsfp_int_mgmt &= ~(u64)QSFP_HFI0_MODPRST_N;
6145 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
6146 ASIC_QSFP1_INVERT, qsfp_int_mgmt);
6147
6148 ppd->offline_disabled_reason =
6149 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
6150 }
6151 }
6152
6153 if (reg & QSFP_HFI0_INT_N) {
6154 dd_dev_info(dd, "%s: Interrupt received from QSFP module\n",
6155 __func__);
6156 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6157 ppd->qsfp_info.check_interrupt_flags = 1;
6158 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, flags);
6159 }
6160
6161
6162 if (qsfp_mod_present(ppd))
6163 queue_work(ppd->link_wq, &ppd->qsfp_info.qsfp_work);
6164}
6165
6166static int request_host_lcb_access(struct hfi1_devdata *dd)
6167{
6168 int ret;
6169
6170 ret = do_8051_command(dd, HCMD_MISC,
6171 (u64)HCMD_MISC_REQUEST_LCB_ACCESS <<
6172 LOAD_DATA_FIELD_ID_SHIFT, NULL);
6173 if (ret != HCMD_SUCCESS) {
6174 dd_dev_err(dd, "%s: command failed with error %d\n",
6175 __func__, ret);
6176 }
6177 return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6178}
6179
6180static int request_8051_lcb_access(struct hfi1_devdata *dd)
6181{
6182 int ret;
6183
6184 ret = do_8051_command(dd, HCMD_MISC,
6185 (u64)HCMD_MISC_GRANT_LCB_ACCESS <<
6186 LOAD_DATA_FIELD_ID_SHIFT, NULL);
6187 if (ret != HCMD_SUCCESS) {
6188 dd_dev_err(dd, "%s: command failed with error %d\n",
6189 __func__, ret);
6190 }
6191 return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6192}
6193
6194
6195
6196
6197
6198static inline void set_host_lcb_access(struct hfi1_devdata *dd)
6199{
6200 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
6201 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK |
6202 DC_DC8051_CFG_CSR_ACCESS_SEL_LCB_SMASK);
6203}
6204
6205
6206
6207
6208
6209static inline void set_8051_lcb_access(struct hfi1_devdata *dd)
6210{
6211 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
6212 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK);
6213}
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6226{
6227 struct hfi1_pportdata *ppd = dd->pport;
6228 int ret = 0;
6229
6230
6231
6232
6233
6234
6235
6236 if (sleep_ok) {
6237 mutex_lock(&ppd->hls_lock);
6238 } else {
6239 while (!mutex_trylock(&ppd->hls_lock))
6240 udelay(1);
6241 }
6242
6243
6244 if (ppd->host_link_state & HLS_DOWN) {
6245 dd_dev_info(dd, "%s: link state %s not up\n",
6246 __func__, link_state_name(ppd->host_link_state));
6247 ret = -EBUSY;
6248 goto done;
6249 }
6250
6251 if (dd->lcb_access_count == 0) {
6252 ret = request_host_lcb_access(dd);
6253 if (ret) {
6254 dd_dev_err(dd,
6255 "%s: unable to acquire LCB access, err %d\n",
6256 __func__, ret);
6257 goto done;
6258 }
6259 set_host_lcb_access(dd);
6260 }
6261 dd->lcb_access_count++;
6262done:
6263 mutex_unlock(&ppd->hls_lock);
6264 return ret;
6265}
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6276{
6277 int ret = 0;
6278
6279
6280
6281
6282
6283
6284 if (sleep_ok) {
6285 mutex_lock(&dd->pport->hls_lock);
6286 } else {
6287 while (!mutex_trylock(&dd->pport->hls_lock))
6288 udelay(1);
6289 }
6290
6291 if (dd->lcb_access_count == 0) {
6292 dd_dev_err(dd, "%s: LCB access count is zero. Skipping.\n",
6293 __func__);
6294 goto done;
6295 }
6296
6297 if (dd->lcb_access_count == 1) {
6298 set_8051_lcb_access(dd);
6299 ret = request_8051_lcb_access(dd);
6300 if (ret) {
6301 dd_dev_err(dd,
6302 "%s: unable to release LCB access, err %d\n",
6303 __func__, ret);
6304
6305 set_host_lcb_access(dd);
6306 goto done;
6307 }
6308 }
6309 dd->lcb_access_count--;
6310done:
6311 mutex_unlock(&dd->pport->hls_lock);
6312 return ret;
6313}
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324static void init_lcb_access(struct hfi1_devdata *dd)
6325{
6326 dd->lcb_access_count = 0;
6327}
6328
6329
6330
6331
6332static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data)
6333{
6334 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0,
6335 DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK |
6336 (u64)return_code <<
6337 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT |
6338 (u64)rsp_data << DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
6339}
6340
6341
6342
6343
6344static void handle_8051_request(struct hfi1_pportdata *ppd)
6345{
6346 struct hfi1_devdata *dd = ppd->dd;
6347 u64 reg;
6348 u16 data = 0;
6349 u8 type;
6350
6351 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_1);
6352 if ((reg & DC_DC8051_CFG_EXT_DEV_1_REQ_NEW_SMASK) == 0)
6353 return;
6354
6355
6356 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 0);
6357
6358
6359 type = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_SHIFT)
6360 & DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_MASK;
6361 data = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT)
6362 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_MASK;
6363
6364 switch (type) {
6365 case HREQ_LOAD_CONFIG:
6366 case HREQ_SAVE_CONFIG:
6367 case HREQ_READ_CONFIG:
6368 case HREQ_SET_TX_EQ_ABS:
6369 case HREQ_SET_TX_EQ_REL:
6370 case HREQ_ENABLE:
6371 dd_dev_info(dd, "8051 request: request 0x%x not supported\n",
6372 type);
6373 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6374 break;
6375 case HREQ_LCB_RESET:
6376
6377 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_INTO_RESET);
6378
6379 (void)read_csr(dd, DCC_CFG_RESET);
6380
6381 udelay(1);
6382
6383 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_OUT_OF_RESET);
6384 hreq_response(dd, HREQ_SUCCESS, 0);
6385
6386 break;
6387 case HREQ_CONFIG_DONE:
6388 hreq_response(dd, HREQ_SUCCESS, 0);
6389 break;
6390
6391 case HREQ_INTERFACE_TEST:
6392 hreq_response(dd, HREQ_SUCCESS, data);
6393 break;
6394 default:
6395 dd_dev_err(dd, "8051 request: unknown request 0x%x\n", type);
6396 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6397 break;
6398 }
6399}
6400
6401
6402
6403
6404void set_up_vau(struct hfi1_devdata *dd, u8 vau)
6405{
6406 u64 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
6407
6408
6409 reg &= ~SEND_CM_GLOBAL_CREDIT_AU_SMASK;
6410 reg |= (u64)vau << SEND_CM_GLOBAL_CREDIT_AU_SHIFT;
6411 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
6412}
6413
6414
6415
6416
6417
6418
6419void set_up_vl15(struct hfi1_devdata *dd, u16 vl15buf)
6420{
6421 u64 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
6422
6423
6424 reg &= ~(SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK |
6425 SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK);
6426
6427
6428
6429
6430
6431 reg |= (u64)vl15buf << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT;
6432 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
6433
6434 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)vl15buf
6435 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
6436}
6437
6438
6439
6440
6441
6442void reset_link_credits(struct hfi1_devdata *dd)
6443{
6444 int i;
6445
6446
6447 for (i = 0; i < TXE_NUM_DATA_VL; i++)
6448 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
6449 write_csr(dd, SEND_CM_CREDIT_VL15, 0);
6450 write_csr(dd, SEND_CM_GLOBAL_CREDIT, 0);
6451
6452 pio_send_control(dd, PSC_CM_RESET);
6453
6454 dd->vl15buf_cached = 0;
6455}
6456
6457
6458static u32 vcu_to_cu(u8 vcu)
6459{
6460 return 1 << vcu;
6461}
6462
6463
6464static u8 cu_to_vcu(u32 cu)
6465{
6466 return ilog2(cu);
6467}
6468
6469
6470static u32 vau_to_au(u8 vau)
6471{
6472 return 8 * (1 << vau);
6473}
6474
6475static void set_linkup_defaults(struct hfi1_pportdata *ppd)
6476{
6477 ppd->sm_trap_qp = 0x0;
6478 ppd->sa_qp = 0x1;
6479}
6480
6481
6482
6483
6484static void lcb_shutdown(struct hfi1_devdata *dd, int abort)
6485{
6486 u64 reg;
6487
6488
6489 write_csr(dd, DC_LCB_CFG_RUN, 0);
6490
6491 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET,
6492 1ull << DC_LCB_CFG_TX_FIFOS_RESET_VAL_SHIFT);
6493
6494 dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN);
6495 reg = read_csr(dd, DCC_CFG_RESET);
6496 write_csr(dd, DCC_CFG_RESET, reg |
6497 DCC_CFG_RESET_RESET_LCB | DCC_CFG_RESET_RESET_RX_FPE);
6498 (void)read_csr(dd, DCC_CFG_RESET);
6499 if (!abort) {
6500 udelay(1);
6501 write_csr(dd, DCC_CFG_RESET, reg);
6502 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6503 }
6504}
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516static void _dc_shutdown(struct hfi1_devdata *dd)
6517{
6518 lockdep_assert_held(&dd->dc8051_lock);
6519
6520 if (dd->dc_shutdown)
6521 return;
6522
6523 dd->dc_shutdown = 1;
6524
6525 lcb_shutdown(dd, 1);
6526
6527
6528
6529
6530
6531 write_csr(dd, DC_DC8051_CFG_RST, 0x1);
6532}
6533
6534static void dc_shutdown(struct hfi1_devdata *dd)
6535{
6536 mutex_lock(&dd->dc8051_lock);
6537 _dc_shutdown(dd);
6538 mutex_unlock(&dd->dc8051_lock);
6539}
6540
6541
6542
6543
6544
6545
6546
6547static void _dc_start(struct hfi1_devdata *dd)
6548{
6549 lockdep_assert_held(&dd->dc8051_lock);
6550
6551 if (!dd->dc_shutdown)
6552 return;
6553
6554
6555 write_csr(dd, DC_DC8051_CFG_RST, 0ull);
6556
6557 if (wait_fm_ready(dd, TIMEOUT_8051_START))
6558 dd_dev_err(dd, "%s: timeout starting 8051 firmware\n",
6559 __func__);
6560
6561
6562 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_OUT_OF_RESET);
6563
6564 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6565 dd->dc_shutdown = 0;
6566}
6567
6568static void dc_start(struct hfi1_devdata *dd)
6569{
6570 mutex_lock(&dd->dc8051_lock);
6571 _dc_start(dd);
6572 mutex_unlock(&dd->dc8051_lock);
6573}
6574
6575
6576
6577
6578static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd)
6579{
6580 u64 rx_radr, tx_radr;
6581 u32 version;
6582
6583 if (dd->icode != ICODE_FPGA_EMULATION)
6584 return;
6585
6586
6587
6588
6589
6590
6591
6592
6593 if (is_emulator_s(dd))
6594 return;
6595
6596
6597 version = emulator_rev(dd);
6598 if (!is_ax(dd))
6599 version = 0x2d;
6600
6601 if (version <= 0x12) {
6602
6603
6604
6605
6606
6607
6608
6609 rx_radr =
6610 0xaull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6611 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6612 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6613
6614
6615
6616
6617 tx_radr = 6ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6618 } else if (version <= 0x18) {
6619
6620
6621 rx_radr =
6622 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6623 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6624 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6625 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6626 } else if (version == 0x19) {
6627
6628
6629 rx_radr =
6630 0xAull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6631 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6632 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6633 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6634 } else if (version == 0x1a) {
6635
6636
6637 rx_radr =
6638 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6639 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6640 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6641 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6642 write_csr(dd, DC_LCB_CFG_LN_DCLK, 1ull);
6643 } else {
6644
6645
6646 rx_radr =
6647 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6648 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6649 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6650 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6651 }
6652
6653 write_csr(dd, DC_LCB_CFG_RX_FIFOS_RADR, rx_radr);
6654
6655 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK,
6656 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK);
6657 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RADR, tx_radr);
6658}
6659
6660
6661
6662
6663
6664
6665void handle_sma_message(struct work_struct *work)
6666{
6667 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6668 sma_message_work);
6669 struct hfi1_devdata *dd = ppd->dd;
6670 u64 msg;
6671 int ret;
6672
6673
6674
6675
6676
6677 ret = read_idle_sma(dd, &msg);
6678 if (ret)
6679 return;
6680 dd_dev_info(dd, "%s: SMA message 0x%llx\n", __func__, msg);
6681
6682
6683
6684 switch (msg & 0xff) {
6685 case SMA_IDLE_ARM:
6686
6687
6688
6689
6690
6691
6692 if (ppd->host_link_state & (HLS_UP_INIT | HLS_UP_ARMED))
6693 ppd->neighbor_normal = 1;
6694 break;
6695 case SMA_IDLE_ACTIVE:
6696
6697
6698
6699
6700
6701
6702 if (ppd->host_link_state == HLS_UP_ARMED &&
6703 ppd->is_active_optimize_enabled) {
6704 ppd->neighbor_normal = 1;
6705 ret = set_link_state(ppd, HLS_UP_ACTIVE);
6706 if (ret)
6707 dd_dev_err(
6708 dd,
6709 "%s: received Active SMA idle message, couldn't set link to Active\n",
6710 __func__);
6711 }
6712 break;
6713 default:
6714 dd_dev_err(dd,
6715 "%s: received unexpected SMA idle message 0x%llx\n",
6716 __func__, msg);
6717 break;
6718 }
6719}
6720
6721static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear)
6722{
6723 u64 rcvctrl;
6724 unsigned long flags;
6725
6726 spin_lock_irqsave(&dd->rcvctrl_lock, flags);
6727 rcvctrl = read_csr(dd, RCV_CTRL);
6728 rcvctrl |= add;
6729 rcvctrl &= ~clear;
6730 write_csr(dd, RCV_CTRL, rcvctrl);
6731 spin_unlock_irqrestore(&dd->rcvctrl_lock, flags);
6732}
6733
6734static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add)
6735{
6736 adjust_rcvctrl(dd, add, 0);
6737}
6738
6739static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear)
6740{
6741 adjust_rcvctrl(dd, 0, clear);
6742}
6743
6744
6745
6746
6747void start_freeze_handling(struct hfi1_pportdata *ppd, int flags)
6748{
6749 struct hfi1_devdata *dd = ppd->dd;
6750 struct send_context *sc;
6751 int i;
6752 int sc_flags;
6753
6754 if (flags & FREEZE_SELF)
6755 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6756
6757
6758 dd->flags |= HFI1_FROZEN;
6759
6760
6761 sdma_freeze_notify(dd, !!(flags & FREEZE_LINK_DOWN));
6762
6763 sc_flags = SCF_FROZEN | SCF_HALTED | (flags & FREEZE_LINK_DOWN ?
6764 SCF_LINK_DOWN : 0);
6765
6766 for (i = 0; i < dd->num_send_contexts; i++) {
6767 sc = dd->send_contexts[i].sc;
6768 if (sc && (sc->flags & SCF_ENABLED))
6769 sc_stop(sc, sc_flags);
6770 }
6771
6772
6773 hfi1_set_uevent_bits(ppd, _HFI1_EVENT_FROZEN_BIT);
6774
6775 if (flags & FREEZE_ABORT) {
6776 dd_dev_err(dd,
6777 "Aborted freeze recovery. Please REBOOT system\n");
6778 return;
6779 }
6780
6781 queue_work(ppd->hfi1_wq, &ppd->freeze_work);
6782}
6783
6784
6785
6786
6787
6788
6789
6790
6791static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze)
6792{
6793 unsigned long timeout;
6794 u64 reg;
6795
6796 timeout = jiffies + msecs_to_jiffies(FREEZE_STATUS_TIMEOUT);
6797 while (1) {
6798 reg = read_csr(dd, CCE_STATUS);
6799 if (freeze) {
6800
6801 if ((reg & ALL_FROZE) == ALL_FROZE)
6802 return;
6803 } else {
6804
6805 if ((reg & ALL_FROZE) == 0)
6806 return;
6807 }
6808
6809 if (time_after(jiffies, timeout)) {
6810 dd_dev_err(dd,
6811 "Time out waiting for SPC %sfreeze, bits 0x%llx, expecting 0x%llx, continuing",
6812 freeze ? "" : "un", reg & ALL_FROZE,
6813 freeze ? ALL_FROZE : 0ull);
6814 return;
6815 }
6816 usleep_range(80, 120);
6817 }
6818}
6819
6820
6821
6822
6823static void rxe_freeze(struct hfi1_devdata *dd)
6824{
6825 int i;
6826 struct hfi1_ctxtdata *rcd;
6827
6828
6829 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6830
6831
6832 for (i = 0; i < dd->num_rcv_contexts; i++) {
6833 rcd = hfi1_rcd_get_by_index(dd, i);
6834 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, rcd);
6835 hfi1_rcd_put(rcd);
6836 }
6837}
6838
6839
6840
6841
6842
6843
6844
6845static void rxe_kernel_unfreeze(struct hfi1_devdata *dd)
6846{
6847 u32 rcvmask;
6848 u16 i;
6849 struct hfi1_ctxtdata *rcd;
6850
6851
6852 for (i = 0; i < dd->num_rcv_contexts; i++) {
6853 rcd = hfi1_rcd_get_by_index(dd, i);
6854
6855
6856 if (!rcd ||
6857 (i >= dd->first_dyn_alloc_ctxt && !rcd->is_vnic)) {
6858 hfi1_rcd_put(rcd);
6859 continue;
6860 }
6861 rcvmask = HFI1_RCVCTRL_CTXT_ENB;
6862
6863 rcvmask |= rcd->rcvhdrtail_kvaddr ?
6864 HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS;
6865 hfi1_rcvctrl(dd, rcvmask, rcd);
6866 hfi1_rcd_put(rcd);
6867 }
6868
6869
6870 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6871}
6872
6873
6874
6875
6876
6877
6878void handle_freeze(struct work_struct *work)
6879{
6880 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6881 freeze_work);
6882 struct hfi1_devdata *dd = ppd->dd;
6883
6884
6885 wait_for_freeze_status(dd, 1);
6886
6887
6888
6889
6890 pio_freeze(dd);
6891
6892
6893 sdma_freeze(dd);
6894
6895
6896
6897
6898 rxe_freeze(dd);
6899
6900
6901
6902
6903
6904 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6905 wait_for_freeze_status(dd, 0);
6906
6907 if (is_ax(dd)) {
6908 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6909 wait_for_freeze_status(dd, 1);
6910 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6911 wait_for_freeze_status(dd, 0);
6912 }
6913
6914
6915 pio_kernel_unfreeze(dd);
6916
6917
6918 sdma_unfreeze(dd);
6919
6920
6921
6922
6923 rxe_kernel_unfreeze(dd);
6924
6925
6926
6927
6928
6929
6930
6931
6932
6933
6934
6935
6936
6937
6938 dd->flags &= ~HFI1_FROZEN;
6939 wake_up(&dd->event_queue);
6940
6941
6942}
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952
6953static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width)
6954{
6955 int i;
6956 u16 tx_width;
6957 u16 link_speed;
6958
6959 tx_width = tx_link_width(link_width);
6960 link_speed = get_link_speed(ppd->link_speed_active);
6961
6962
6963
6964
6965
6966 for (i = 0; i < C_VL_COUNT + 1; i++)
6967 get_xmit_wait_counters(ppd, tx_width, link_speed, i);
6968}
6969
6970
6971
6972
6973
6974
6975void handle_link_up(struct work_struct *work)
6976{
6977 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6978 link_up_work);
6979 struct hfi1_devdata *dd = ppd->dd;
6980
6981 set_link_state(ppd, HLS_UP_INIT);
6982
6983
6984 read_ltp_rtt(dd);
6985
6986
6987
6988
6989 clear_linkup_counters(dd);
6990
6991
6992
6993 set_linkup_defaults(ppd);
6994
6995
6996
6997
6998
6999
7000
7001 if (!(quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR))
7002 set_up_vl15(dd, dd->vl15buf_cached);
7003
7004
7005 if ((ppd->link_speed_active & ppd->link_speed_enabled) == 0) {
7006
7007 dd_dev_err(dd,
7008 "Link speed active 0x%x is outside enabled 0x%x, downing link\n",
7009 ppd->link_speed_active, ppd->link_speed_enabled);
7010 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0,
7011 OPA_LINKDOWN_REASON_SPEED_POLICY);
7012 set_link_state(ppd, HLS_DN_OFFLINE);
7013 start_link(ppd);
7014 }
7015}
7016
7017
7018
7019
7020
7021static void reset_neighbor_info(struct hfi1_pportdata *ppd)
7022{
7023 ppd->neighbor_guid = 0;
7024 ppd->neighbor_port_number = 0;
7025 ppd->neighbor_type = 0;
7026 ppd->neighbor_fm_security = 0;
7027}
7028
7029static const char * const link_down_reason_strs[] = {
7030 [OPA_LINKDOWN_REASON_NONE] = "None",
7031 [OPA_LINKDOWN_REASON_RCV_ERROR_0] = "Receive error 0",
7032 [OPA_LINKDOWN_REASON_BAD_PKT_LEN] = "Bad packet length",
7033 [OPA_LINKDOWN_REASON_PKT_TOO_LONG] = "Packet too long",
7034 [OPA_LINKDOWN_REASON_PKT_TOO_SHORT] = "Packet too short",
7035 [OPA_LINKDOWN_REASON_BAD_SLID] = "Bad SLID",
7036 [OPA_LINKDOWN_REASON_BAD_DLID] = "Bad DLID",
7037 [OPA_LINKDOWN_REASON_BAD_L2] = "Bad L2",
7038 [OPA_LINKDOWN_REASON_BAD_SC] = "Bad SC",
7039 [OPA_LINKDOWN_REASON_RCV_ERROR_8] = "Receive error 8",
7040 [OPA_LINKDOWN_REASON_BAD_MID_TAIL] = "Bad mid tail",
7041 [OPA_LINKDOWN_REASON_RCV_ERROR_10] = "Receive error 10",
7042 [OPA_LINKDOWN_REASON_PREEMPT_ERROR] = "Preempt error",
7043 [OPA_LINKDOWN_REASON_PREEMPT_VL15] = "Preempt vl15",
7044 [OPA_LINKDOWN_REASON_BAD_VL_MARKER] = "Bad VL marker",
7045 [OPA_LINKDOWN_REASON_RCV_ERROR_14] = "Receive error 14",
7046 [OPA_LINKDOWN_REASON_RCV_ERROR_15] = "Receive error 15",
7047 [OPA_LINKDOWN_REASON_BAD_HEAD_DIST] = "Bad head distance",
7048 [OPA_LINKDOWN_REASON_BAD_TAIL_DIST] = "Bad tail distance",
7049 [OPA_LINKDOWN_REASON_BAD_CTRL_DIST] = "Bad control distance",
7050 [OPA_LINKDOWN_REASON_BAD_CREDIT_ACK] = "Bad credit ack",
7051 [OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER] = "Unsupported VL marker",
7052 [OPA_LINKDOWN_REASON_BAD_PREEMPT] = "Bad preempt",
7053 [OPA_LINKDOWN_REASON_BAD_CONTROL_FLIT] = "Bad control flit",
7054 [OPA_LINKDOWN_REASON_EXCEED_MULTICAST_LIMIT] = "Exceed multicast limit",
7055 [OPA_LINKDOWN_REASON_RCV_ERROR_24] = "Receive error 24",
7056 [OPA_LINKDOWN_REASON_RCV_ERROR_25] = "Receive error 25",
7057 [OPA_LINKDOWN_REASON_RCV_ERROR_26] = "Receive error 26",
7058 [OPA_LINKDOWN_REASON_RCV_ERROR_27] = "Receive error 27",
7059 [OPA_LINKDOWN_REASON_RCV_ERROR_28] = "Receive error 28",
7060 [OPA_LINKDOWN_REASON_RCV_ERROR_29] = "Receive error 29",
7061 [OPA_LINKDOWN_REASON_RCV_ERROR_30] = "Receive error 30",
7062 [OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN] =
7063 "Excessive buffer overrun",
7064 [OPA_LINKDOWN_REASON_UNKNOWN] = "Unknown",
7065 [OPA_LINKDOWN_REASON_REBOOT] = "Reboot",
7066 [OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN] = "Neighbor unknown",
7067 [OPA_LINKDOWN_REASON_FM_BOUNCE] = "FM bounce",
7068 [OPA_LINKDOWN_REASON_SPEED_POLICY] = "Speed policy",
7069 [OPA_LINKDOWN_REASON_WIDTH_POLICY] = "Width policy",
7070 [OPA_LINKDOWN_REASON_DISCONNECTED] = "Disconnected",
7071 [OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED] =
7072 "Local media not installed",
7073 [OPA_LINKDOWN_REASON_NOT_INSTALLED] = "Not installed",
7074 [OPA_LINKDOWN_REASON_CHASSIS_CONFIG] = "Chassis config",
7075 [OPA_LINKDOWN_REASON_END_TO_END_NOT_INSTALLED] =
7076 "End to end not installed",
7077 [OPA_LINKDOWN_REASON_POWER_POLICY] = "Power policy",
7078 [OPA_LINKDOWN_REASON_LINKSPEED_POLICY] = "Link speed policy",
7079 [OPA_LINKDOWN_REASON_LINKWIDTH_POLICY] = "Link width policy",
7080 [OPA_LINKDOWN_REASON_SWITCH_MGMT] = "Switch management",
7081 [OPA_LINKDOWN_REASON_SMA_DISABLED] = "SMA disabled",
7082 [OPA_LINKDOWN_REASON_TRANSIENT] = "Transient"
7083};
7084
7085
7086static const char *link_down_reason_str(u8 reason)
7087{
7088 const char *str = NULL;
7089
7090 if (reason < ARRAY_SIZE(link_down_reason_strs))
7091 str = link_down_reason_strs[reason];
7092 if (!str)
7093 str = "(invalid)";
7094
7095 return str;
7096}
7097
7098
7099
7100
7101
7102
7103void handle_link_down(struct work_struct *work)
7104{
7105 u8 lcl_reason, neigh_reason = 0;
7106 u8 link_down_reason;
7107 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7108 link_down_work);
7109 int was_up;
7110 static const char ldr_str[] = "Link down reason: ";
7111
7112 if ((ppd->host_link_state &
7113 (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) &&
7114 ppd->port_type == PORT_TYPE_FIXED)
7115 ppd->offline_disabled_reason =
7116 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NOT_INSTALLED);
7117
7118
7119 was_up = !!(ppd->host_link_state & HLS_UP);
7120 set_link_state(ppd, HLS_DN_OFFLINE);
7121 xchg(&ppd->is_link_down_queued, 0);
7122
7123 if (was_up) {
7124 lcl_reason = 0;
7125
7126 read_link_down_reason(ppd->dd, &link_down_reason);
7127 switch (link_down_reason) {
7128 case LDR_LINK_TRANSFER_ACTIVE_LOW:
7129
7130 dd_dev_info(ppd->dd, "%sUnexpected link down\n",
7131 ldr_str);
7132 break;
7133 case LDR_RECEIVED_LINKDOWN_IDLE_MSG:
7134
7135
7136
7137
7138 read_planned_down_reason_code(ppd->dd, &neigh_reason);
7139 dd_dev_info(ppd->dd,
7140 "%sNeighbor link down message %d, %s\n",
7141 ldr_str, neigh_reason,
7142 link_down_reason_str(neigh_reason));
7143 break;
7144 case LDR_RECEIVED_HOST_OFFLINE_REQ:
7145 dd_dev_info(ppd->dd,
7146 "%sHost requested link to go offline\n",
7147 ldr_str);
7148 break;
7149 default:
7150 dd_dev_info(ppd->dd, "%sUnknown reason 0x%x\n",
7151 ldr_str, link_down_reason);
7152 break;
7153 }
7154
7155
7156
7157
7158
7159 if (neigh_reason == 0)
7160 lcl_reason = OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN;
7161 } else {
7162
7163 lcl_reason = OPA_LINKDOWN_REASON_TRANSIENT;
7164 }
7165
7166 set_link_down_reason(ppd, lcl_reason, neigh_reason, 0);
7167
7168
7169 if (was_up && ppd->local_link_down_reason.sma == 0 &&
7170 ppd->neigh_link_down_reason.sma == 0) {
7171 ppd->local_link_down_reason.sma =
7172 ppd->local_link_down_reason.latest;
7173 ppd->neigh_link_down_reason.sma =
7174 ppd->neigh_link_down_reason.latest;
7175 }
7176
7177 reset_neighbor_info(ppd);
7178
7179
7180 clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
7181
7182
7183
7184
7185
7186 if (ppd->port_type == PORT_TYPE_QSFP && !qsfp_mod_present(ppd))
7187 dc_shutdown(ppd->dd);
7188 else
7189 start_link(ppd);
7190}
7191
7192void handle_link_bounce(struct work_struct *work)
7193{
7194 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7195 link_bounce_work);
7196
7197
7198
7199
7200 if (ppd->host_link_state & HLS_UP) {
7201 set_link_state(ppd, HLS_DN_OFFLINE);
7202 start_link(ppd);
7203 } else {
7204 dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n",
7205 __func__, link_state_name(ppd->host_link_state));
7206 }
7207}
7208
7209
7210
7211
7212
7213static int cap_to_port_ltp(int cap)
7214{
7215 int port_ltp = PORT_LTP_CRC_MODE_16;
7216
7217 if (cap & CAP_CRC_14B)
7218 port_ltp |= PORT_LTP_CRC_MODE_14;
7219 if (cap & CAP_CRC_48B)
7220 port_ltp |= PORT_LTP_CRC_MODE_48;
7221 if (cap & CAP_CRC_12B_16B_PER_LANE)
7222 port_ltp |= PORT_LTP_CRC_MODE_PER_LANE;
7223
7224 return port_ltp;
7225}
7226
7227
7228
7229
7230int port_ltp_to_cap(int port_ltp)
7231{
7232 int cap_mask = 0;
7233
7234 if (port_ltp & PORT_LTP_CRC_MODE_14)
7235 cap_mask |= CAP_CRC_14B;
7236 if (port_ltp & PORT_LTP_CRC_MODE_48)
7237 cap_mask |= CAP_CRC_48B;
7238 if (port_ltp & PORT_LTP_CRC_MODE_PER_LANE)
7239 cap_mask |= CAP_CRC_12B_16B_PER_LANE;
7240
7241 return cap_mask;
7242}
7243
7244
7245
7246
7247static int lcb_to_port_ltp(int lcb_crc)
7248{
7249 int port_ltp = 0;
7250
7251 if (lcb_crc == LCB_CRC_12B_16B_PER_LANE)
7252 port_ltp = PORT_LTP_CRC_MODE_PER_LANE;
7253 else if (lcb_crc == LCB_CRC_48B)
7254 port_ltp = PORT_LTP_CRC_MODE_48;
7255 else if (lcb_crc == LCB_CRC_14B)
7256 port_ltp = PORT_LTP_CRC_MODE_14;
7257 else
7258 port_ltp = PORT_LTP_CRC_MODE_16;
7259
7260 return port_ltp;
7261}
7262
7263static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd)
7264{
7265 if (ppd->pkeys[2] != 0) {
7266 ppd->pkeys[2] = 0;
7267 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0);
7268 hfi1_event_pkey_change(ppd->dd, ppd->port);
7269 }
7270}
7271
7272
7273
7274
7275static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width)
7276{
7277 switch (width) {
7278 case 0:
7279
7280
7281
7282
7283 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || quick_linkup)
7284 return OPA_LINK_WIDTH_4X;
7285 return 0;
7286 case 1: return OPA_LINK_WIDTH_1X;
7287 case 2: return OPA_LINK_WIDTH_2X;
7288 case 3: return OPA_LINK_WIDTH_3X;
7289 default:
7290 dd_dev_info(dd, "%s: invalid width %d, using 4\n",
7291 __func__, width);
7292
7293 case 4: return OPA_LINK_WIDTH_4X;
7294 }
7295}
7296
7297
7298
7299
7300static const u8 bit_counts[16] = {
7301 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4
7302};
7303
7304static inline u8 nibble_to_count(u8 nibble)
7305{
7306 return bit_counts[nibble & 0xf];
7307}
7308
7309
7310
7311
7312
7313
7314
7315
7316
7317static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width,
7318 u16 *rx_width)
7319{
7320 u16 tx, rx;
7321 u8 enable_lane_rx;
7322 u8 enable_lane_tx;
7323 u8 tx_polarity_inversion;
7324 u8 rx_polarity_inversion;
7325 u8 max_rate;
7326
7327
7328 read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
7329 &rx_polarity_inversion, &max_rate);
7330 read_local_lni(dd, &enable_lane_rx);
7331
7332
7333 tx = nibble_to_count(enable_lane_tx);
7334 rx = nibble_to_count(enable_lane_rx);
7335
7336
7337
7338
7339
7340
7341 if ((dd->icode == ICODE_RTL_SILICON) &&
7342 (dd->dc8051_ver < dc8051_ver(0, 19, 0))) {
7343
7344 switch (max_rate) {
7345 case 0:
7346 dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G;
7347 break;
7348 default:
7349 dd_dev_err(dd,
7350 "%s: unexpected max rate %d, using 25Gb\n",
7351 __func__, (int)max_rate);
7352
7353 case 1:
7354 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
7355 break;
7356 }
7357 }
7358
7359 dd_dev_info(dd,
7360 "Fabric active lanes (width): tx 0x%x (%d), rx 0x%x (%d)\n",
7361 enable_lane_tx, tx, enable_lane_rx, rx);
7362 *tx_width = link_width_to_bits(dd, tx);
7363 *rx_width = link_width_to_bits(dd, rx);
7364}
7365
7366
7367
7368
7369
7370
7371
7372
7373
7374
7375
7376
7377
7378
7379static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width,
7380 u16 *rx_width)
7381{
7382 u16 widths, tx, rx;
7383 u8 misc_bits, local_flags;
7384 u16 active_tx, active_rx;
7385
7386 read_vc_local_link_mode(dd, &misc_bits, &local_flags, &widths);
7387 tx = widths >> 12;
7388 rx = (widths >> 8) & 0xf;
7389
7390 *tx_width = link_width_to_bits(dd, tx);
7391 *rx_width = link_width_to_bits(dd, rx);
7392
7393
7394 get_link_widths(dd, &active_tx, &active_rx);
7395}
7396
7397
7398
7399
7400
7401
7402
7403
7404
7405void get_linkup_link_widths(struct hfi1_pportdata *ppd)
7406{
7407 u16 tx_width, rx_width;
7408
7409
7410 get_linkup_widths(ppd->dd, &tx_width, &rx_width);
7411
7412
7413 ppd->link_width_active = tx_width;
7414
7415 ppd->link_width_downgrade_tx_active = ppd->link_width_active;
7416 ppd->link_width_downgrade_rx_active = ppd->link_width_active;
7417
7418 ppd->link_width_downgrade_enabled = ppd->link_width_downgrade_supported;
7419
7420 ppd->current_egress_rate = active_egress_rate(ppd);
7421}
7422
7423
7424
7425
7426
7427
7428void handle_verify_cap(struct work_struct *work)
7429{
7430 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7431 link_vc_work);
7432 struct hfi1_devdata *dd = ppd->dd;
7433 u64 reg;
7434 u8 power_management;
7435 u8 continuous;
7436 u8 vcu;
7437 u8 vau;
7438 u8 z;
7439 u16 vl15buf;
7440 u16 link_widths;
7441 u16 crc_mask;
7442 u16 crc_val;
7443 u16 device_id;
7444 u16 active_tx, active_rx;
7445 u8 partner_supported_crc;
7446 u8 remote_tx_rate;
7447 u8 device_rev;
7448
7449 set_link_state(ppd, HLS_VERIFY_CAP);
7450
7451 lcb_shutdown(dd, 0);
7452 adjust_lcb_for_fpga_serdes(dd);
7453
7454 read_vc_remote_phy(dd, &power_management, &continuous);
7455 read_vc_remote_fabric(dd, &vau, &z, &vcu, &vl15buf,
7456 &partner_supported_crc);
7457 read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths);
7458 read_remote_device_id(dd, &device_id, &device_rev);
7459
7460
7461 get_link_widths(dd, &active_tx, &active_rx);
7462 dd_dev_info(dd,
7463 "Peer PHY: power management 0x%x, continuous updates 0x%x\n",
7464 (int)power_management, (int)continuous);
7465 dd_dev_info(dd,
7466 "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n",
7467 (int)vau, (int)z, (int)vcu, (int)vl15buf,
7468 (int)partner_supported_crc);
7469 dd_dev_info(dd, "Peer Link Width: tx rate 0x%x, widths 0x%x\n",
7470 (u32)remote_tx_rate, (u32)link_widths);
7471 dd_dev_info(dd, "Peer Device ID: 0x%04x, Revision 0x%02x\n",
7472 (u32)device_id, (u32)device_rev);
7473
7474
7475
7476
7477
7478
7479
7480
7481
7482 if (vau == 0)
7483 vau = 1;
7484 set_up_vau(dd, vau);
7485
7486
7487
7488
7489
7490 set_up_vl15(dd, 0);
7491 dd->vl15buf_cached = vl15buf;
7492
7493
7494 crc_mask = ppd->port_crc_mode_enabled & partner_supported_crc;
7495
7496
7497 if (crc_mask & CAP_CRC_14B)
7498 crc_val = LCB_CRC_14B;
7499 else if (crc_mask & CAP_CRC_48B)
7500 crc_val = LCB_CRC_48B;
7501 else if (crc_mask & CAP_CRC_12B_16B_PER_LANE)
7502 crc_val = LCB_CRC_12B_16B_PER_LANE;
7503 else
7504 crc_val = LCB_CRC_16B;
7505
7506 dd_dev_info(dd, "Final LCB CRC mode: %d\n", (int)crc_val);
7507 write_csr(dd, DC_LCB_CFG_CRC_MODE,
7508 (u64)crc_val << DC_LCB_CFG_CRC_MODE_TX_VAL_SHIFT);
7509
7510
7511 reg = read_csr(dd, SEND_CM_CTRL);
7512 if (crc_val == LCB_CRC_14B && crc_14b_sideband) {
7513 write_csr(dd, SEND_CM_CTRL,
7514 reg | SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
7515 } else {
7516 write_csr(dd, SEND_CM_CTRL,
7517 reg & ~SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
7518 }
7519
7520 ppd->link_speed_active = 0;
7521 if (dd->dc8051_ver < dc8051_ver(0, 20, 0)) {
7522
7523 switch (remote_tx_rate) {
7524 case 0:
7525 ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7526 break;
7527 case 1:
7528 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7529 break;
7530 }
7531 } else {
7532
7533 u8 rate = remote_tx_rate & ppd->local_tx_rate;
7534
7535 if (rate & 2)
7536 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7537 else if (rate & 1)
7538 ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7539 }
7540 if (ppd->link_speed_active == 0) {
7541 dd_dev_err(dd, "%s: unexpected remote tx rate %d, using 25Gb\n",
7542 __func__, (int)remote_tx_rate);
7543 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7544 }
7545
7546
7547
7548
7549
7550
7551
7552
7553 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
7554
7555 ppd->port_ltp_crc_mode |=
7556 cap_to_port_ltp(ppd->port_crc_mode_enabled) << 4;
7557
7558 ppd->port_ltp_crc_mode |= lcb_to_port_ltp(crc_val);
7559
7560
7561
7562 assign_remote_cm_au_table(dd, vcu);
7563
7564
7565
7566
7567
7568
7569
7570
7571
7572
7573 if (is_ax(dd)) {
7574 reg = read_csr(dd, DC_LCB_CFG_LINK_KILL_EN);
7575 reg |= DC_LCB_CFG_LINK_KILL_EN_REPLAY_BUF_MBE_SMASK
7576 | DC_LCB_CFG_LINK_KILL_EN_FLIT_INPUT_BUF_MBE_SMASK;
7577 write_csr(dd, DC_LCB_CFG_LINK_KILL_EN, reg);
7578 }
7579
7580
7581 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
7582
7583
7584 write_csr(dd, DC_LCB_ERR_EN, 0);
7585 set_8051_lcb_access(dd);
7586
7587
7588 set_link_state(ppd, HLS_GOING_UP);
7589}
7590
7591
7592
7593
7594
7595
7596
7597
7598
7599
7600
7601
7602
7603
7604
7605
7606bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd,
7607 bool refresh_widths)
7608{
7609 int do_bounce = 0;
7610 int tries;
7611 u16 lwde;
7612 u16 tx, rx;
7613 bool link_downgraded = refresh_widths;
7614
7615
7616 tries = 0;
7617retry:
7618 mutex_lock(&ppd->hls_lock);
7619
7620 if (ppd->host_link_state & HLS_DOWN) {
7621
7622 if (ppd->host_link_state & HLS_GOING_UP) {
7623 if (++tries < 1000) {
7624 mutex_unlock(&ppd->hls_lock);
7625 usleep_range(100, 120);
7626 goto retry;
7627 }
7628 dd_dev_err(ppd->dd,
7629 "%s: giving up waiting for link state change\n",
7630 __func__);
7631 }
7632 goto done;
7633 }
7634
7635 lwde = ppd->link_width_downgrade_enabled;
7636
7637 if (refresh_widths) {
7638 get_link_widths(ppd->dd, &tx, &rx);
7639 ppd->link_width_downgrade_tx_active = tx;
7640 ppd->link_width_downgrade_rx_active = rx;
7641 }
7642
7643 if (ppd->link_width_downgrade_tx_active == 0 ||
7644 ppd->link_width_downgrade_rx_active == 0) {
7645
7646 dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n");
7647 link_downgraded = false;
7648 } else if (lwde == 0) {
7649
7650
7651
7652 if ((ppd->link_width_active !=
7653 ppd->link_width_downgrade_tx_active) ||
7654 (ppd->link_width_active !=
7655 ppd->link_width_downgrade_rx_active)) {
7656 dd_dev_err(ppd->dd,
7657 "Link downgrade is disabled and link has downgraded, downing link\n");
7658 dd_dev_err(ppd->dd,
7659 " original 0x%x, tx active 0x%x, rx active 0x%x\n",
7660 ppd->link_width_active,
7661 ppd->link_width_downgrade_tx_active,
7662 ppd->link_width_downgrade_rx_active);
7663 do_bounce = 1;
7664 link_downgraded = false;
7665 }
7666 } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 ||
7667 (lwde & ppd->link_width_downgrade_rx_active) == 0) {
7668
7669 dd_dev_err(ppd->dd,
7670 "Link is outside of downgrade allowed, downing link\n");
7671 dd_dev_err(ppd->dd,
7672 " enabled 0x%x, tx active 0x%x, rx active 0x%x\n",
7673 lwde, ppd->link_width_downgrade_tx_active,
7674 ppd->link_width_downgrade_rx_active);
7675 do_bounce = 1;
7676 link_downgraded = false;
7677 }
7678
7679done:
7680 mutex_unlock(&ppd->hls_lock);
7681
7682 if (do_bounce) {
7683 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0,
7684 OPA_LINKDOWN_REASON_WIDTH_POLICY);
7685 set_link_state(ppd, HLS_DN_OFFLINE);
7686 start_link(ppd);
7687 }
7688
7689 return link_downgraded;
7690}
7691
7692
7693
7694
7695
7696
7697void handle_link_downgrade(struct work_struct *work)
7698{
7699 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7700 link_downgrade_work);
7701
7702 dd_dev_info(ppd->dd, "8051: Link width downgrade\n");
7703 if (apply_link_downgrade_policy(ppd, true))
7704 update_xmit_counters(ppd, ppd->link_width_downgrade_tx_active);
7705}
7706
7707static char *dcc_err_string(char *buf, int buf_len, u64 flags)
7708{
7709 return flag_string(buf, buf_len, flags, dcc_err_flags,
7710 ARRAY_SIZE(dcc_err_flags));
7711}
7712
7713static char *lcb_err_string(char *buf, int buf_len, u64 flags)
7714{
7715 return flag_string(buf, buf_len, flags, lcb_err_flags,
7716 ARRAY_SIZE(lcb_err_flags));
7717}
7718
7719static char *dc8051_err_string(char *buf, int buf_len, u64 flags)
7720{
7721 return flag_string(buf, buf_len, flags, dc8051_err_flags,
7722 ARRAY_SIZE(dc8051_err_flags));
7723}
7724
7725static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags)
7726{
7727 return flag_string(buf, buf_len, flags, dc8051_info_err_flags,
7728 ARRAY_SIZE(dc8051_info_err_flags));
7729}
7730
7731static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags)
7732{
7733 return flag_string(buf, buf_len, flags, dc8051_info_host_msg_flags,
7734 ARRAY_SIZE(dc8051_info_host_msg_flags));
7735}
7736
7737static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg)
7738{
7739 struct hfi1_pportdata *ppd = dd->pport;
7740 u64 info, err, host_msg;
7741 int queue_link_down = 0;
7742 char buf[96];
7743
7744
7745 if (reg & DC_DC8051_ERR_FLG_SET_BY_8051_SMASK) {
7746
7747
7748 info = read_csr(dd, DC_DC8051_DBG_ERR_INFO_SET_BY_8051);
7749 err = (info >> DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_SHIFT)
7750 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_MASK;
7751 host_msg = (info >>
7752 DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_SHIFT)
7753 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_MASK;
7754
7755
7756
7757
7758 if (err & FAILED_LNI) {
7759
7760
7761
7762
7763
7764
7765 if (ppd->host_link_state
7766 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
7767 queue_link_down = 1;
7768 dd_dev_info(dd, "Link error: %s\n",
7769 dc8051_info_err_string(buf,
7770 sizeof(buf),
7771 err &
7772 FAILED_LNI));
7773 }
7774 err &= ~(u64)FAILED_LNI;
7775 }
7776
7777 if (err & UNKNOWN_FRAME) {
7778 ppd->unknown_frame_count++;
7779 err &= ~(u64)UNKNOWN_FRAME;
7780 }
7781 if (err) {
7782
7783 dd_dev_err(dd, "8051 info error: %s\n",
7784 dc8051_info_err_string(buf, sizeof(buf),
7785 err));
7786 }
7787
7788
7789
7790
7791 if (host_msg & HOST_REQ_DONE) {
7792
7793
7794
7795
7796
7797
7798
7799
7800
7801 host_msg &= ~(u64)HOST_REQ_DONE;
7802 }
7803 if (host_msg & BC_SMA_MSG) {
7804 queue_work(ppd->link_wq, &ppd->sma_message_work);
7805 host_msg &= ~(u64)BC_SMA_MSG;
7806 }
7807 if (host_msg & LINKUP_ACHIEVED) {
7808 dd_dev_info(dd, "8051: Link up\n");
7809 queue_work(ppd->link_wq, &ppd->link_up_work);
7810 host_msg &= ~(u64)LINKUP_ACHIEVED;
7811 }
7812 if (host_msg & EXT_DEVICE_CFG_REQ) {
7813 handle_8051_request(ppd);
7814 host_msg &= ~(u64)EXT_DEVICE_CFG_REQ;
7815 }
7816 if (host_msg & VERIFY_CAP_FRAME) {
7817 queue_work(ppd->link_wq, &ppd->link_vc_work);
7818 host_msg &= ~(u64)VERIFY_CAP_FRAME;
7819 }
7820 if (host_msg & LINK_GOING_DOWN) {
7821 const char *extra = "";
7822
7823 if (host_msg & LINK_WIDTH_DOWNGRADED) {
7824 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7825 extra = " (ignoring downgrade)";
7826 }
7827 dd_dev_info(dd, "8051: Link down%s\n", extra);
7828 queue_link_down = 1;
7829 host_msg &= ~(u64)LINK_GOING_DOWN;
7830 }
7831 if (host_msg & LINK_WIDTH_DOWNGRADED) {
7832 queue_work(ppd->link_wq, &ppd->link_downgrade_work);
7833 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7834 }
7835 if (host_msg) {
7836
7837 dd_dev_info(dd, "8051 info host message: %s\n",
7838 dc8051_info_host_msg_string(buf,
7839 sizeof(buf),
7840 host_msg));
7841 }
7842
7843 reg &= ~DC_DC8051_ERR_FLG_SET_BY_8051_SMASK;
7844 }
7845 if (reg & DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK) {
7846
7847
7848
7849
7850
7851 dd_dev_err(dd, "Lost 8051 heartbeat\n");
7852 write_csr(dd, DC_DC8051_ERR_EN,
7853 read_csr(dd, DC_DC8051_ERR_EN) &
7854 ~DC_DC8051_ERR_EN_LOST_8051_HEART_BEAT_SMASK);
7855
7856 reg &= ~DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK;
7857 }
7858 if (reg) {
7859
7860 dd_dev_err(dd, "8051 error: %s\n",
7861 dc8051_err_string(buf, sizeof(buf), reg));
7862 }
7863
7864 if (queue_link_down) {
7865
7866
7867
7868
7869
7870 if ((ppd->host_link_state &
7871 (HLS_GOING_OFFLINE | HLS_LINK_COOLDOWN)) ||
7872 ppd->link_enabled == 0) {
7873 dd_dev_info(dd, "%s: not queuing link down. host_link_state %x, link_enabled %x\n",
7874 __func__, ppd->host_link_state,
7875 ppd->link_enabled);
7876 } else {
7877 if (xchg(&ppd->is_link_down_queued, 1) == 1)
7878 dd_dev_info(dd,
7879 "%s: link down request already queued\n",
7880 __func__);
7881 else
7882 queue_work(ppd->link_wq, &ppd->link_down_work);
7883 }
7884 }
7885}
7886
7887static const char * const fm_config_txt[] = {
7888[0] =
7889 "BadHeadDist: Distance violation between two head flits",
7890[1] =
7891 "BadTailDist: Distance violation between two tail flits",
7892[2] =
7893 "BadCtrlDist: Distance violation between two credit control flits",
7894[3] =
7895 "BadCrdAck: Credits return for unsupported VL",
7896[4] =
7897 "UnsupportedVLMarker: Received VL Marker",
7898[5] =
7899 "BadPreempt: Exceeded the preemption nesting level",
7900[6] =
7901 "BadControlFlit: Received unsupported control flit",
7902
7903[8] =
7904 "UnsupportedVLMarker: Received VL Marker for unconfigured or disabled VL",
7905};
7906
7907static const char * const port_rcv_txt[] = {
7908[1] =
7909 "BadPktLen: Illegal PktLen",
7910[2] =
7911 "PktLenTooLong: Packet longer than PktLen",
7912[3] =
7913 "PktLenTooShort: Packet shorter than PktLen",
7914[4] =
7915 "BadSLID: Illegal SLID (0, using multicast as SLID, does not include security validation of SLID)",
7916[5] =
7917 "BadDLID: Illegal DLID (0, doesn't match HFI)",
7918[6] =
7919 "BadL2: Illegal L2 opcode",
7920[7] =
7921 "BadSC: Unsupported SC",
7922[9] =
7923 "BadRC: Illegal RC",
7924[11] =
7925 "PreemptError: Preempting with same VL",
7926[12] =
7927 "PreemptVL15: Preempting a VL15 packet",
7928};
7929
7930#define OPA_LDR_FMCONFIG_OFFSET 16
7931#define OPA_LDR_PORTRCV_OFFSET 0
7932static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
7933{
7934 u64 info, hdr0, hdr1;
7935 const char *extra;
7936 char buf[96];
7937 struct hfi1_pportdata *ppd = dd->pport;
7938 u8 lcl_reason = 0;
7939 int do_bounce = 0;
7940
7941 if (reg & DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK) {
7942 if (!(dd->err_info_uncorrectable & OPA_EI_STATUS_SMASK)) {
7943 info = read_csr(dd, DCC_ERR_INFO_UNCORRECTABLE);
7944 dd->err_info_uncorrectable = info & OPA_EI_CODE_SMASK;
7945
7946 dd->err_info_uncorrectable |= OPA_EI_STATUS_SMASK;
7947 }
7948 reg &= ~DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK;
7949 }
7950
7951 if (reg & DCC_ERR_FLG_LINK_ERR_SMASK) {
7952 struct hfi1_pportdata *ppd = dd->pport;
7953
7954 if (ppd->link_downed < (u32)UINT_MAX)
7955 ppd->link_downed++;
7956 reg &= ~DCC_ERR_FLG_LINK_ERR_SMASK;
7957 }
7958
7959 if (reg & DCC_ERR_FLG_FMCONFIG_ERR_SMASK) {
7960 u8 reason_valid = 1;
7961
7962 info = read_csr(dd, DCC_ERR_INFO_FMCONFIG);
7963 if (!(dd->err_info_fmconfig & OPA_EI_STATUS_SMASK)) {
7964 dd->err_info_fmconfig = info & OPA_EI_CODE_SMASK;
7965
7966 dd->err_info_fmconfig |= OPA_EI_STATUS_SMASK;
7967 }
7968 switch (info) {
7969 case 0:
7970 case 1:
7971 case 2:
7972 case 3:
7973 case 4:
7974 case 5:
7975 case 6:
7976 extra = fm_config_txt[info];
7977 break;
7978 case 8:
7979 extra = fm_config_txt[info];
7980 if (ppd->port_error_action &
7981 OPA_PI_MASK_FM_CFG_UNSUPPORTED_VL_MARKER) {
7982 do_bounce = 1;
7983
7984
7985
7986
7987 lcl_reason =
7988 OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER;
7989 }
7990 break;
7991 default:
7992 reason_valid = 0;
7993 snprintf(buf, sizeof(buf), "reserved%lld", info);
7994 extra = buf;
7995 break;
7996 }
7997
7998 if (reason_valid && !do_bounce) {
7999 do_bounce = ppd->port_error_action &
8000 (1 << (OPA_LDR_FMCONFIG_OFFSET + info));
8001 lcl_reason = info + OPA_LINKDOWN_REASON_BAD_HEAD_DIST;
8002 }
8003
8004
8005 dd_dev_info_ratelimited(dd, "DCC Error: fmconfig error: %s\n",
8006 extra);
8007 reg &= ~DCC_ERR_FLG_FMCONFIG_ERR_SMASK;
8008 }
8009
8010 if (reg & DCC_ERR_FLG_RCVPORT_ERR_SMASK) {
8011 u8 reason_valid = 1;
8012
8013 info = read_csr(dd, DCC_ERR_INFO_PORTRCV);
8014 hdr0 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR0);
8015 hdr1 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR1);
8016 if (!(dd->err_info_rcvport.status_and_code &
8017 OPA_EI_STATUS_SMASK)) {
8018 dd->err_info_rcvport.status_and_code =
8019 info & OPA_EI_CODE_SMASK;
8020
8021 dd->err_info_rcvport.status_and_code |=
8022 OPA_EI_STATUS_SMASK;
8023
8024
8025
8026
8027 dd->err_info_rcvport.packet_flit1 = hdr0;
8028 dd->err_info_rcvport.packet_flit2 = hdr1;
8029 }
8030 switch (info) {
8031 case 1:
8032 case 2:
8033 case 3:
8034 case 4:
8035 case 5:
8036 case 6:
8037 case 7:
8038 case 9:
8039 case 11:
8040 case 12:
8041 extra = port_rcv_txt[info];
8042 break;
8043 default:
8044 reason_valid = 0;
8045 snprintf(buf, sizeof(buf), "reserved%lld", info);
8046 extra = buf;
8047 break;
8048 }
8049
8050 if (reason_valid && !do_bounce) {
8051 do_bounce = ppd->port_error_action &
8052 (1 << (OPA_LDR_PORTRCV_OFFSET + info));
8053 lcl_reason = info + OPA_LINKDOWN_REASON_RCV_ERROR_0;
8054 }
8055
8056
8057 dd_dev_info_ratelimited(dd, "DCC Error: PortRcv error: %s\n"
8058 " hdr0 0x%llx, hdr1 0x%llx\n",
8059 extra, hdr0, hdr1);
8060
8061 reg &= ~DCC_ERR_FLG_RCVPORT_ERR_SMASK;
8062 }
8063
8064 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK) {
8065
8066 dd_dev_info_ratelimited(dd, "8051 access to LCB blocked\n");
8067 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK;
8068 }
8069 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK) {
8070
8071 dd_dev_info_ratelimited(dd, "host access to LCB blocked\n");
8072 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK;
8073 }
8074
8075 if (unlikely(hfi1_dbg_fault_suppress_err(&dd->verbs_dev)))
8076 reg &= ~DCC_ERR_FLG_LATE_EBP_ERR_SMASK;
8077
8078
8079 if (reg)
8080 dd_dev_info_ratelimited(dd, "DCC Error: %s\n",
8081 dcc_err_string(buf, sizeof(buf), reg));
8082
8083 if (lcl_reason == 0)
8084 lcl_reason = OPA_LINKDOWN_REASON_UNKNOWN;
8085
8086 if (do_bounce) {
8087 dd_dev_info_ratelimited(dd, "%s: PortErrorAction bounce\n",
8088 __func__);
8089 set_link_down_reason(ppd, lcl_reason, 0, lcl_reason);
8090 queue_work(ppd->link_wq, &ppd->link_bounce_work);
8091 }
8092}
8093
8094static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
8095{
8096 char buf[96];
8097
8098 dd_dev_info(dd, "LCB Error: %s\n",
8099 lcb_err_string(buf, sizeof(buf), reg));
8100}
8101
8102
8103
8104
8105static void is_dc_int(struct hfi1_devdata *dd, unsigned int source)
8106{
8107 const struct err_reg_info *eri = &dc_errs[source];
8108
8109 if (eri->handler) {
8110 interrupt_clear_down(dd, 0, eri);
8111 } else if (source == 3 ) {
8112
8113
8114
8115
8116
8117
8118
8119
8120
8121 dd_dev_err(dd, "Parity error in DC LBM block\n");
8122 } else {
8123 dd_dev_err(dd, "Invalid DC interrupt %u\n", source);
8124 }
8125}
8126
8127
8128
8129
8130static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source)
8131{
8132 sc_group_release_update(dd, source);
8133}
8134
8135
8136
8137
8138
8139
8140
8141
8142
8143
8144static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source)
8145{
8146
8147 unsigned int what = source / TXE_NUM_SDMA_ENGINES;
8148
8149 unsigned int which = source % TXE_NUM_SDMA_ENGINES;
8150
8151#ifdef CONFIG_SDMA_VERBOSITY
8152 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", which,
8153 slashstrip(__FILE__), __LINE__, __func__);
8154 sdma_dumpstate(&dd->per_sdma[which]);
8155#endif
8156
8157 if (likely(what < 3 && which < dd->num_sdma)) {
8158 sdma_engine_interrupt(&dd->per_sdma[which], 1ull << source);
8159 } else {
8160
8161 dd_dev_err(dd, "Invalid SDMA interrupt 0x%x\n", source);
8162 }
8163}
8164
8165
8166
8167
8168
8169
8170
8171
8172
8173
8174
8175static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source)
8176{
8177 struct hfi1_ctxtdata *rcd;
8178 char *err_detail;
8179
8180 if (likely(source < dd->num_rcv_contexts)) {
8181 rcd = hfi1_rcd_get_by_index(dd, source);
8182 if (rcd) {
8183 handle_user_interrupt(rcd);
8184 hfi1_rcd_put(rcd);
8185 return;
8186 }
8187
8188 err_detail = "dataless";
8189 } else {
8190
8191 err_detail = "out of range";
8192 }
8193 dd_dev_err(dd, "unexpected %s receive available context interrupt %u\n",
8194 err_detail, source);
8195}
8196
8197
8198
8199
8200
8201
8202
8203
8204
8205
8206static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source)
8207{
8208 struct hfi1_ctxtdata *rcd;
8209 char *err_detail;
8210
8211 if (likely(source < dd->num_rcv_contexts)) {
8212 rcd = hfi1_rcd_get_by_index(dd, source);
8213 if (rcd) {
8214 handle_user_interrupt(rcd);
8215 hfi1_rcd_put(rcd);
8216 return;
8217 }
8218
8219 err_detail = "dataless";
8220 } else {
8221
8222 err_detail = "out of range";
8223 }
8224 dd_dev_err(dd, "unexpected %s receive urgent context interrupt %u\n",
8225 err_detail, source);
8226}
8227
8228
8229
8230
8231static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source)
8232{
8233 char name[64];
8234
8235 dd_dev_err(dd, "unexpected %s interrupt\n",
8236 is_reserved_name(name, sizeof(name), source));
8237}
8238
8239static const struct is_table is_table[] = {
8240
8241
8242
8243
8244{ IS_GENERAL_ERR_START, IS_GENERAL_ERR_END,
8245 is_misc_err_name, is_misc_err_int },
8246{ IS_SDMAENG_ERR_START, IS_SDMAENG_ERR_END,
8247 is_sdma_eng_err_name, is_sdma_eng_err_int },
8248{ IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END,
8249 is_sendctxt_err_name, is_sendctxt_err_int },
8250{ IS_SDMA_START, IS_SDMA_IDLE_END,
8251 is_sdma_eng_name, is_sdma_eng_int },
8252{ IS_VARIOUS_START, IS_VARIOUS_END,
8253 is_various_name, is_various_int },
8254{ IS_DC_START, IS_DC_END,
8255 is_dc_name, is_dc_int },
8256{ IS_RCVAVAIL_START, IS_RCVAVAIL_END,
8257 is_rcv_avail_name, is_rcv_avail_int },
8258{ IS_RCVURGENT_START, IS_RCVURGENT_END,
8259 is_rcv_urgent_name, is_rcv_urgent_int },
8260{ IS_SENDCREDIT_START, IS_SENDCREDIT_END,
8261 is_send_credit_name, is_send_credit_int},
8262{ IS_RESERVED_START, IS_RESERVED_END,
8263 is_reserved_name, is_reserved_int},
8264};
8265
8266
8267
8268
8269
8270static void is_interrupt(struct hfi1_devdata *dd, unsigned int source)
8271{
8272 const struct is_table *entry;
8273
8274
8275 for (entry = &is_table[0]; entry->is_name; entry++) {
8276 if (source <= entry->end) {
8277 trace_hfi1_interrupt(dd, entry, source);
8278 entry->is_int(dd, source - entry->start);
8279 return;
8280 }
8281 }
8282
8283 dd_dev_err(dd, "invalid interrupt source %u\n", source);
8284}
8285
8286
8287
8288
8289
8290
8291
8292
8293
8294
8295irqreturn_t general_interrupt(int irq, void *data)
8296{
8297 struct hfi1_devdata *dd = data;
8298 u64 regs[CCE_NUM_INT_CSRS];
8299 u32 bit;
8300 int i;
8301 irqreturn_t handled = IRQ_NONE;
8302
8303 this_cpu_inc(*dd->int_counter);
8304
8305
8306 for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
8307 if (dd->gi_mask[i] == 0) {
8308 regs[i] = 0;
8309 continue;
8310 }
8311 regs[i] = read_csr(dd, CCE_INT_STATUS + (8 * i)) &
8312 dd->gi_mask[i];
8313
8314 if (regs[i])
8315 write_csr(dd, CCE_INT_CLEAR + (8 * i), regs[i]);
8316 }
8317
8318
8319 for_each_set_bit(bit, (unsigned long *)®s[0],
8320 CCE_NUM_INT_CSRS * 64) {
8321 is_interrupt(dd, bit);
8322 handled = IRQ_HANDLED;
8323 }
8324
8325 return handled;
8326}
8327
8328irqreturn_t sdma_interrupt(int irq, void *data)
8329{
8330 struct sdma_engine *sde = data;
8331 struct hfi1_devdata *dd = sde->dd;
8332 u64 status;
8333
8334#ifdef CONFIG_SDMA_VERBOSITY
8335 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
8336 slashstrip(__FILE__), __LINE__, __func__);
8337 sdma_dumpstate(sde);
8338#endif
8339
8340 this_cpu_inc(*dd->int_counter);
8341
8342
8343 status = read_csr(dd,
8344 CCE_INT_STATUS + (8 * (IS_SDMA_START / 64)))
8345 & sde->imask;
8346 if (likely(status)) {
8347
8348 write_csr(dd,
8349 CCE_INT_CLEAR + (8 * (IS_SDMA_START / 64)),
8350 status);
8351
8352
8353 sdma_engine_interrupt(sde, status);
8354 } else {
8355 dd_dev_info_ratelimited(dd, "SDMA engine %u interrupt, but no status bits set\n",
8356 sde->this_idx);
8357 }
8358 return IRQ_HANDLED;
8359}
8360
8361
8362
8363
8364
8365
8366static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd)
8367{
8368 struct hfi1_devdata *dd = rcd->dd;
8369 u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg);
8370
8371 write_csr(dd, addr, rcd->imask);
8372
8373 (void)read_csr(dd, addr);
8374}
8375
8376
8377void force_recv_intr(struct hfi1_ctxtdata *rcd)
8378{
8379 write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask);
8380}
8381
8382
8383
8384
8385
8386
8387
8388
8389
8390
8391
8392static inline int check_packet_present(struct hfi1_ctxtdata *rcd)
8393{
8394 u32 tail;
8395 int present;
8396
8397 if (!rcd->rcvhdrtail_kvaddr)
8398 present = (rcd->seq_cnt ==
8399 rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd))));
8400 else
8401 present = (rcd->head != get_rcvhdrtail(rcd));
8402
8403 if (present)
8404 return 1;
8405
8406
8407 tail = (u32)read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
8408 return rcd->head != tail;
8409}
8410
8411
8412
8413
8414
8415
8416
8417
8418
8419irqreturn_t receive_context_interrupt(int irq, void *data)
8420{
8421 struct hfi1_ctxtdata *rcd = data;
8422 struct hfi1_devdata *dd = rcd->dd;
8423 int disposition;
8424 int present;
8425
8426 trace_hfi1_receive_interrupt(dd, rcd);
8427 this_cpu_inc(*dd->int_counter);
8428 aspm_ctx_disable(rcd);
8429
8430
8431 disposition = rcd->do_interrupt(rcd, 0);
8432
8433
8434
8435
8436
8437
8438 if (disposition == RCV_PKT_LIMIT)
8439 return IRQ_WAKE_THREAD;
8440
8441
8442
8443
8444
8445
8446
8447 clear_recv_intr(rcd);
8448 present = check_packet_present(rcd);
8449 if (present)
8450 force_recv_intr(rcd);
8451
8452 return IRQ_HANDLED;
8453}
8454
8455
8456
8457
8458
8459irqreturn_t receive_context_thread(int irq, void *data)
8460{
8461 struct hfi1_ctxtdata *rcd = data;
8462 int present;
8463
8464
8465 (void)rcd->do_interrupt(rcd, 1);
8466
8467
8468
8469
8470
8471
8472
8473
8474 local_irq_disable();
8475 clear_recv_intr(rcd);
8476 present = check_packet_present(rcd);
8477 if (present)
8478 force_recv_intr(rcd);
8479 local_irq_enable();
8480
8481 return IRQ_HANDLED;
8482}
8483
8484
8485
8486u32 read_physical_state(struct hfi1_devdata *dd)
8487{
8488 u64 reg;
8489
8490 reg = read_csr(dd, DC_DC8051_STS_CUR_STATE);
8491 return (reg >> DC_DC8051_STS_CUR_STATE_PORT_SHIFT)
8492 & DC_DC8051_STS_CUR_STATE_PORT_MASK;
8493}
8494
8495u32 read_logical_state(struct hfi1_devdata *dd)
8496{
8497 u64 reg;
8498
8499 reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8500 return (reg >> DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT)
8501 & DCC_CFG_PORT_CONFIG_LINK_STATE_MASK;
8502}
8503
8504static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate)
8505{
8506 u64 reg;
8507
8508 reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8509
8510 reg &= ~DCC_CFG_PORT_CONFIG_LINK_STATE_SMASK;
8511 reg |= (u64)chip_lstate << DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT;
8512 write_csr(dd, DCC_CFG_PORT_CONFIG, reg);
8513}
8514
8515
8516
8517
8518static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data)
8519{
8520 u32 regno;
8521 int ret;
8522
8523 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
8524 if (acquire_lcb_access(dd, 0) == 0) {
8525 *data = read_csr(dd, addr);
8526 release_lcb_access(dd, 0);
8527 return 0;
8528 }
8529 return -EBUSY;
8530 }
8531
8532
8533 regno = (addr - DC_LCB_CFG_RUN) >> 3;
8534 ret = do_8051_command(dd, HCMD_READ_LCB_CSR, regno, data);
8535 if (ret != HCMD_SUCCESS)
8536 return -EBUSY;
8537 return 0;
8538}
8539
8540
8541
8542
8543
8544
8545struct lcb_datum {
8546 u32 off;
8547 u64 val;
8548};
8549
8550static struct lcb_datum lcb_cache[] = {
8551 { DC_LCB_ERR_INFO_RX_REPLAY_CNT, 0},
8552 { DC_LCB_ERR_INFO_SEQ_CRC_CNT, 0 },
8553 { DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT, 0 },
8554};
8555
8556static void update_lcb_cache(struct hfi1_devdata *dd)
8557{
8558 int i;
8559 int ret;
8560 u64 val;
8561
8562 for (i = 0; i < ARRAY_SIZE(lcb_cache); i++) {
8563 ret = read_lcb_csr(dd, lcb_cache[i].off, &val);
8564
8565
8566 if (likely(ret != -EBUSY))
8567 lcb_cache[i].val = val;
8568 }
8569}
8570
8571static int read_lcb_cache(u32 off, u64 *val)
8572{
8573 int i;
8574
8575 for (i = 0; i < ARRAY_SIZE(lcb_cache); i++) {
8576 if (lcb_cache[i].off == off) {
8577 *val = lcb_cache[i].val;
8578 return 0;
8579 }
8580 }
8581
8582 pr_warn("%s bad offset 0x%x\n", __func__, off);
8583 return -1;
8584}
8585
8586
8587
8588
8589
8590int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data)
8591{
8592 struct hfi1_pportdata *ppd = dd->pport;
8593
8594
8595 if (ppd->host_link_state & HLS_UP)
8596 return read_lcb_via_8051(dd, addr, data);
8597
8598 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE)) {
8599 if (read_lcb_cache(addr, data))
8600 return -EBUSY;
8601 return 0;
8602 }
8603
8604
8605 *data = read_csr(dd, addr);
8606 return 0;
8607}
8608
8609
8610
8611
8612static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data)
8613{
8614 u32 regno;
8615 int ret;
8616
8617 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR ||
8618 (dd->dc8051_ver < dc8051_ver(0, 20, 0))) {
8619 if (acquire_lcb_access(dd, 0) == 0) {
8620 write_csr(dd, addr, data);
8621 release_lcb_access(dd, 0);
8622 return 0;
8623 }
8624 return -EBUSY;
8625 }
8626
8627
8628 regno = (addr - DC_LCB_CFG_RUN) >> 3;
8629 ret = do_8051_command(dd, HCMD_WRITE_LCB_CSR, regno, &data);
8630 if (ret != HCMD_SUCCESS)
8631 return -EBUSY;
8632 return 0;
8633}
8634
8635
8636
8637
8638
8639int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data)
8640{
8641 struct hfi1_pportdata *ppd = dd->pport;
8642
8643
8644 if (ppd->host_link_state & HLS_UP)
8645 return write_lcb_via_8051(dd, addr, data);
8646
8647 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
8648 return -EBUSY;
8649
8650 write_csr(dd, addr, data);
8651 return 0;
8652}
8653
8654
8655
8656
8657
8658
8659static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
8660 u64 *out_data)
8661{
8662 u64 reg, completed;
8663 int return_code;
8664 unsigned long timeout;
8665
8666 hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data);
8667
8668 mutex_lock(&dd->dc8051_lock);
8669
8670
8671 if (dd->dc_shutdown) {
8672 return_code = -ENODEV;
8673 goto fail;
8674 }
8675
8676
8677
8678
8679
8680
8681
8682
8683
8684
8685
8686 if (dd->dc8051_timed_out) {
8687 if (dd->dc8051_timed_out > 1) {
8688 dd_dev_err(dd,
8689 "Previous 8051 host command timed out, skipping command %u\n",
8690 type);
8691 return_code = -ENXIO;
8692 goto fail;
8693 }
8694 _dc_shutdown(dd);
8695 _dc_start(dd);
8696 }
8697
8698
8699
8700
8701
8702
8703
8704
8705
8706
8707
8708
8709
8710
8711
8712
8713
8714 if (type == HCMD_WRITE_LCB_CSR) {
8715 in_data |= ((*out_data) & 0xffffffffffull) << 8;
8716
8717 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_0);
8718 reg &= DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK;
8719 reg |= ((((*out_data) >> 40) & 0xff) <<
8720 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT)
8721 | ((((*out_data) >> 48) & 0xffff) <<
8722 DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
8723 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, reg);
8724 }
8725
8726
8727
8728
8729
8730 reg = ((u64)type & DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_MASK)
8731 << DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_SHIFT
8732 | (in_data & DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_MASK)
8733 << DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_SHIFT;
8734 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8735 reg |= DC_DC8051_CFG_HOST_CMD_0_REQ_NEW_SMASK;
8736 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8737
8738
8739 timeout = jiffies + msecs_to_jiffies(DC8051_COMMAND_TIMEOUT);
8740 while (1) {
8741 reg = read_csr(dd, DC_DC8051_CFG_HOST_CMD_1);
8742 completed = reg & DC_DC8051_CFG_HOST_CMD_1_COMPLETED_SMASK;
8743 if (completed)
8744 break;
8745 if (time_after(jiffies, timeout)) {
8746 dd->dc8051_timed_out++;
8747 dd_dev_err(dd, "8051 host command %u timeout\n", type);
8748 if (out_data)
8749 *out_data = 0;
8750 return_code = -ETIMEDOUT;
8751 goto fail;
8752 }
8753 udelay(2);
8754 }
8755
8756 if (out_data) {
8757 *out_data = (reg >> DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_SHIFT)
8758 & DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_MASK;
8759 if (type == HCMD_READ_LCB_CSR) {
8760
8761 *out_data |= (read_csr(dd, DC_DC8051_CFG_EXT_DEV_1)
8762 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SMASK)
8763 << (48
8764 - DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT);
8765 }
8766 }
8767 return_code = (reg >> DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_SHIFT)
8768 & DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_MASK;
8769 dd->dc8051_timed_out = 0;
8770
8771
8772
8773 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0);
8774
8775fail:
8776 mutex_unlock(&dd->dc8051_lock);
8777 return return_code;
8778}
8779
8780static int set_physical_link_state(struct hfi1_devdata *dd, u64 state)
8781{
8782 return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL);
8783}
8784
8785int load_8051_config(struct hfi1_devdata *dd, u8 field_id,
8786 u8 lane_id, u32 config_data)
8787{
8788 u64 data;
8789 int ret;
8790
8791 data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT
8792 | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT
8793 | (u64)config_data << LOAD_DATA_DATA_SHIFT;
8794 ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL);
8795 if (ret != HCMD_SUCCESS) {
8796 dd_dev_err(dd,
8797 "load 8051 config: field id %d, lane %d, err %d\n",
8798 (int)field_id, (int)lane_id, ret);
8799 }
8800 return ret;
8801}
8802
8803
8804
8805
8806
8807
8808int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id,
8809 u32 *result)
8810{
8811 u64 big_data;
8812 u32 addr;
8813 int ret;
8814
8815
8816 if (lane_id < 4)
8817 addr = (4 * NUM_GENERAL_FIELDS)
8818 + (lane_id * 4 * NUM_LANE_FIELDS);
8819 else
8820 addr = 0;
8821 addr += field_id * 4;
8822
8823
8824 ret = read_8051_data(dd, addr, 8, &big_data);
8825
8826 if (ret == 0) {
8827
8828 if (addr & 0x4)
8829 *result = (u32)(big_data >> 32);
8830 else
8831 *result = (u32)big_data;
8832 } else {
8833 *result = 0;
8834 dd_dev_err(dd, "%s: direct read failed, lane %d, field %d!\n",
8835 __func__, lane_id, field_id);
8836 }
8837
8838 return ret;
8839}
8840
8841static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management,
8842 u8 continuous)
8843{
8844 u32 frame;
8845
8846 frame = continuous << CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT
8847 | power_management << POWER_MANAGEMENT_SHIFT;
8848 return load_8051_config(dd, VERIFY_CAP_LOCAL_PHY,
8849 GENERAL_CONFIG, frame);
8850}
8851
8852static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu,
8853 u16 vl15buf, u8 crc_sizes)
8854{
8855 u32 frame;
8856
8857 frame = (u32)vau << VAU_SHIFT
8858 | (u32)z << Z_SHIFT
8859 | (u32)vcu << VCU_SHIFT
8860 | (u32)vl15buf << VL15BUF_SHIFT
8861 | (u32)crc_sizes << CRC_SIZES_SHIFT;
8862 return load_8051_config(dd, VERIFY_CAP_LOCAL_FABRIC,
8863 GENERAL_CONFIG, frame);
8864}
8865
8866static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits,
8867 u8 *flag_bits, u16 *link_widths)
8868{
8869 u32 frame;
8870
8871 read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_MODE, GENERAL_CONFIG,
8872 &frame);
8873 *misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK;
8874 *flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK;
8875 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
8876}
8877
8878static int write_vc_local_link_mode(struct hfi1_devdata *dd,
8879 u8 misc_bits,
8880 u8 flag_bits,
8881 u16 link_widths)
8882{
8883 u32 frame;
8884
8885 frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT
8886 | (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT
8887 | (u32)link_widths << LINK_WIDTH_SHIFT;
8888 return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_MODE, GENERAL_CONFIG,
8889 frame);
8890}
8891
8892static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id,
8893 u8 device_rev)
8894{
8895 u32 frame;
8896
8897 frame = ((u32)device_id << LOCAL_DEVICE_ID_SHIFT)
8898 | ((u32)device_rev << LOCAL_DEVICE_REV_SHIFT);
8899 return load_8051_config(dd, LOCAL_DEVICE_ID, GENERAL_CONFIG, frame);
8900}
8901
8902static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
8903 u8 *device_rev)
8904{
8905 u32 frame;
8906
8907 read_8051_config(dd, REMOTE_DEVICE_ID, GENERAL_CONFIG, &frame);
8908 *device_id = (frame >> REMOTE_DEVICE_ID_SHIFT) & REMOTE_DEVICE_ID_MASK;
8909 *device_rev = (frame >> REMOTE_DEVICE_REV_SHIFT)
8910 & REMOTE_DEVICE_REV_MASK;
8911}
8912
8913int write_host_interface_version(struct hfi1_devdata *dd, u8 version)
8914{
8915 u32 frame;
8916 u32 mask;
8917
8918 mask = (HOST_INTERFACE_VERSION_MASK << HOST_INTERFACE_VERSION_SHIFT);
8919 read_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, &frame);
8920
8921 frame &= ~mask;
8922 frame |= ((u32)version << HOST_INTERFACE_VERSION_SHIFT);
8923 return load_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG,
8924 frame);
8925}
8926
8927void read_misc_status(struct hfi1_devdata *dd, u8 *ver_major, u8 *ver_minor,
8928 u8 *ver_patch)
8929{
8930 u32 frame;
8931
8932 read_8051_config(dd, MISC_STATUS, GENERAL_CONFIG, &frame);
8933 *ver_major = (frame >> STS_FM_VERSION_MAJOR_SHIFT) &
8934 STS_FM_VERSION_MAJOR_MASK;
8935 *ver_minor = (frame >> STS_FM_VERSION_MINOR_SHIFT) &
8936 STS_FM_VERSION_MINOR_MASK;
8937
8938 read_8051_config(dd, VERSION_PATCH, GENERAL_CONFIG, &frame);
8939 *ver_patch = (frame >> STS_FM_VERSION_PATCH_SHIFT) &
8940 STS_FM_VERSION_PATCH_MASK;
8941}
8942
8943static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
8944 u8 *continuous)
8945{
8946 u32 frame;
8947
8948 read_8051_config(dd, VERIFY_CAP_REMOTE_PHY, GENERAL_CONFIG, &frame);
8949 *power_management = (frame >> POWER_MANAGEMENT_SHIFT)
8950 & POWER_MANAGEMENT_MASK;
8951 *continuous = (frame >> CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT)
8952 & CONTINIOUS_REMOTE_UPDATE_SUPPORT_MASK;
8953}
8954
8955static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
8956 u8 *vcu, u16 *vl15buf, u8 *crc_sizes)
8957{
8958 u32 frame;
8959
8960 read_8051_config(dd, VERIFY_CAP_REMOTE_FABRIC, GENERAL_CONFIG, &frame);
8961 *vau = (frame >> VAU_SHIFT) & VAU_MASK;
8962 *z = (frame >> Z_SHIFT) & Z_MASK;
8963 *vcu = (frame >> VCU_SHIFT) & VCU_MASK;
8964 *vl15buf = (frame >> VL15BUF_SHIFT) & VL15BUF_MASK;
8965 *crc_sizes = (frame >> CRC_SIZES_SHIFT) & CRC_SIZES_MASK;
8966}
8967
8968static void read_vc_remote_link_width(struct hfi1_devdata *dd,
8969 u8 *remote_tx_rate,
8970 u16 *link_widths)
8971{
8972 u32 frame;
8973
8974 read_8051_config(dd, VERIFY_CAP_REMOTE_LINK_WIDTH, GENERAL_CONFIG,
8975 &frame);
8976 *remote_tx_rate = (frame >> REMOTE_TX_RATE_SHIFT)
8977 & REMOTE_TX_RATE_MASK;
8978 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
8979}
8980
8981static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx)
8982{
8983 u32 frame;
8984
8985 read_8051_config(dd, LOCAL_LNI_INFO, GENERAL_CONFIG, &frame);
8986 *enable_lane_rx = (frame >> ENABLE_LANE_RX_SHIFT) & ENABLE_LANE_RX_MASK;
8987}
8988
8989static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls)
8990{
8991 read_8051_config(dd, LAST_LOCAL_STATE_COMPLETE, GENERAL_CONFIG, lls);
8992}
8993
8994static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs)
8995{
8996 read_8051_config(dd, LAST_REMOTE_STATE_COMPLETE, GENERAL_CONFIG, lrs);
8997}
8998
8999void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality)
9000{
9001 u32 frame;
9002 int ret;
9003
9004 *link_quality = 0;
9005 if (dd->pport->host_link_state & HLS_UP) {
9006 ret = read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG,
9007 &frame);
9008 if (ret == 0)
9009 *link_quality = (frame >> LINK_QUALITY_SHIFT)
9010 & LINK_QUALITY_MASK;
9011 }
9012}
9013
9014static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc)
9015{
9016 u32 frame;
9017
9018 read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, &frame);
9019 *pdrrc = (frame >> DOWN_REMOTE_REASON_SHIFT) & DOWN_REMOTE_REASON_MASK;
9020}
9021
9022static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr)
9023{
9024 u32 frame;
9025
9026 read_8051_config(dd, LINK_DOWN_REASON, GENERAL_CONFIG, &frame);
9027 *ldr = (frame & 0xff);
9028}
9029
9030static int read_tx_settings(struct hfi1_devdata *dd,
9031 u8 *enable_lane_tx,
9032 u8 *tx_polarity_inversion,
9033 u8 *rx_polarity_inversion,
9034 u8 *max_rate)
9035{
9036 u32 frame;
9037 int ret;
9038
9039 ret = read_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, &frame);
9040 *enable_lane_tx = (frame >> ENABLE_LANE_TX_SHIFT)
9041 & ENABLE_LANE_TX_MASK;
9042 *tx_polarity_inversion = (frame >> TX_POLARITY_INVERSION_SHIFT)
9043 & TX_POLARITY_INVERSION_MASK;
9044 *rx_polarity_inversion = (frame >> RX_POLARITY_INVERSION_SHIFT)
9045 & RX_POLARITY_INVERSION_MASK;
9046 *max_rate = (frame >> MAX_RATE_SHIFT) & MAX_RATE_MASK;
9047 return ret;
9048}
9049
9050static int write_tx_settings(struct hfi1_devdata *dd,
9051 u8 enable_lane_tx,
9052 u8 tx_polarity_inversion,
9053 u8 rx_polarity_inversion,
9054 u8 max_rate)
9055{
9056 u32 frame;
9057
9058
9059 frame = enable_lane_tx << ENABLE_LANE_TX_SHIFT
9060 | tx_polarity_inversion << TX_POLARITY_INVERSION_SHIFT
9061 | rx_polarity_inversion << RX_POLARITY_INVERSION_SHIFT
9062 | max_rate << MAX_RATE_SHIFT;
9063 return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame);
9064}
9065
9066
9067
9068
9069
9070
9071static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out)
9072{
9073 int ret;
9074
9075 ret = do_8051_command(dd, HCMD_READ_LCB_IDLE_MSG, type, data_out);
9076 if (ret != HCMD_SUCCESS) {
9077 dd_dev_err(dd, "read idle message: type %d, err %d\n",
9078 (u32)type, ret);
9079 return -EINVAL;
9080 }
9081 dd_dev_info(dd, "%s: read idle message 0x%llx\n", __func__, *data_out);
9082
9083 *data_out >>= IDLE_PAYLOAD_SHIFT;
9084 return 0;
9085}
9086
9087
9088
9089
9090
9091
9092
9093static int read_idle_sma(struct hfi1_devdata *dd, u64 *data)
9094{
9095 return read_idle_message(dd, (u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT,
9096 data);
9097}
9098
9099
9100
9101
9102
9103
9104static int send_idle_message(struct hfi1_devdata *dd, u64 data)
9105{
9106 int ret;
9107
9108 dd_dev_info(dd, "%s: sending idle message 0x%llx\n", __func__, data);
9109 ret = do_8051_command(dd, HCMD_SEND_LCB_IDLE_MSG, data, NULL);
9110 if (ret != HCMD_SUCCESS) {
9111 dd_dev_err(dd, "send idle message: data 0x%llx, err %d\n",
9112 data, ret);
9113 return -EINVAL;
9114 }
9115 return 0;
9116}
9117
9118
9119
9120
9121
9122
9123int send_idle_sma(struct hfi1_devdata *dd, u64 message)
9124{
9125 u64 data;
9126
9127 data = ((message & IDLE_PAYLOAD_MASK) << IDLE_PAYLOAD_SHIFT) |
9128 ((u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT);
9129 return send_idle_message(dd, data);
9130}
9131
9132
9133
9134
9135
9136
9137
9138static int do_quick_linkup(struct hfi1_devdata *dd)
9139{
9140 int ret;
9141
9142 lcb_shutdown(dd, 0);
9143
9144 if (loopback) {
9145
9146
9147 write_csr(dd, DC_LCB_CFG_LOOPBACK,
9148 IB_PACKET_TYPE << DC_LCB_CFG_LOOPBACK_VAL_SHIFT);
9149 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0);
9150 }
9151
9152
9153
9154 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
9155
9156
9157 if (loopback && dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
9158
9159 write_csr(dd, DC_LCB_CFG_RUN,
9160 1ull << DC_LCB_CFG_RUN_EN_SHIFT);
9161
9162 ret = wait_link_transfer_active(dd, 10);
9163 if (ret)
9164 return ret;
9165
9166 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP,
9167 1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT);
9168 }
9169
9170 if (!loopback) {
9171
9172
9173
9174
9175
9176
9177
9178 dd_dev_err(dd,
9179 "Pausing for peer to be finished with LCB set up\n");
9180 msleep(5000);
9181 dd_dev_err(dd, "Continuing with quick linkup\n");
9182 }
9183
9184 write_csr(dd, DC_LCB_ERR_EN, 0);
9185 set_8051_lcb_access(dd);
9186
9187
9188
9189
9190
9191
9192 ret = set_physical_link_state(dd, PLS_QUICK_LINKUP);
9193 if (ret != HCMD_SUCCESS) {
9194 dd_dev_err(dd,
9195 "%s: set physical link state to quick LinkUp failed with return %d\n",
9196 __func__, ret);
9197
9198 set_host_lcb_access(dd);
9199 write_csr(dd, DC_LCB_ERR_EN, ~0ull);
9200
9201 if (ret >= 0)
9202 ret = -EINVAL;
9203 return ret;
9204 }
9205
9206 return 0;
9207}
9208
9209
9210
9211
9212static int init_loopback(struct hfi1_devdata *dd)
9213{
9214 dd_dev_info(dd, "Entering loopback mode\n");
9215
9216
9217 write_csr(dd, DC_DC8051_CFG_MODE,
9218 (read_csr(dd, DC_DC8051_CFG_MODE) | DISABLE_SELF_GUID_CHECK));
9219
9220
9221
9222
9223
9224
9225
9226 if ((dd->icode == ICODE_FUNCTIONAL_SIMULATOR) &&
9227 (loopback == LOOPBACK_SERDES || loopback == LOOPBACK_LCB ||
9228 loopback == LOOPBACK_CABLE)) {
9229 loopback = LOOPBACK_LCB;
9230 quick_linkup = 1;
9231 return 0;
9232 }
9233
9234
9235
9236
9237 if (loopback == LOOPBACK_SERDES)
9238 return 0;
9239
9240
9241 if (loopback == LOOPBACK_LCB) {
9242 quick_linkup = 1;
9243
9244
9245 if (dd->icode == ICODE_FPGA_EMULATION) {
9246 dd_dev_err(dd,
9247 "LCB loopback not supported in emulation\n");
9248 return -EINVAL;
9249 }
9250 return 0;
9251 }
9252
9253
9254 if (loopback == LOOPBACK_CABLE)
9255 return 0;
9256
9257 dd_dev_err(dd, "Invalid loopback mode %d\n", loopback);
9258 return -EINVAL;
9259}
9260
9261
9262
9263
9264
9265static u16 opa_to_vc_link_widths(u16 opa_widths)
9266{
9267 int i;
9268 u16 result = 0;
9269
9270 static const struct link_bits {
9271 u16 from;
9272 u16 to;
9273 } opa_link_xlate[] = {
9274 { OPA_LINK_WIDTH_1X, 1 << (1 - 1) },
9275 { OPA_LINK_WIDTH_2X, 1 << (2 - 1) },
9276 { OPA_LINK_WIDTH_3X, 1 << (3 - 1) },
9277 { OPA_LINK_WIDTH_4X, 1 << (4 - 1) },
9278 };
9279
9280 for (i = 0; i < ARRAY_SIZE(opa_link_xlate); i++) {
9281 if (opa_widths & opa_link_xlate[i].from)
9282 result |= opa_link_xlate[i].to;
9283 }
9284 return result;
9285}
9286
9287
9288
9289
9290static int set_local_link_attributes(struct hfi1_pportdata *ppd)
9291{
9292 struct hfi1_devdata *dd = ppd->dd;
9293 u8 enable_lane_tx;
9294 u8 tx_polarity_inversion;
9295 u8 rx_polarity_inversion;
9296 int ret;
9297 u32 misc_bits = 0;
9298
9299 fabric_serdes_reset(dd);
9300
9301
9302 ret = read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
9303 &rx_polarity_inversion, &ppd->local_tx_rate);
9304 if (ret)
9305 goto set_local_link_attributes_fail;
9306
9307 if (dd->dc8051_ver < dc8051_ver(0, 20, 0)) {
9308
9309 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
9310 ppd->local_tx_rate = 1;
9311 else
9312 ppd->local_tx_rate = 0;
9313 } else {
9314
9315 ppd->local_tx_rate = 0;
9316 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
9317 ppd->local_tx_rate |= 2;
9318 if (ppd->link_speed_enabled & OPA_LINK_SPEED_12_5G)
9319 ppd->local_tx_rate |= 1;
9320 }
9321
9322 enable_lane_tx = 0xF;
9323 ret = write_tx_settings(dd, enable_lane_tx, tx_polarity_inversion,
9324 rx_polarity_inversion, ppd->local_tx_rate);
9325 if (ret != HCMD_SUCCESS)
9326 goto set_local_link_attributes_fail;
9327
9328 ret = write_host_interface_version(dd, HOST_INTERFACE_VERSION);
9329 if (ret != HCMD_SUCCESS) {
9330 dd_dev_err(dd,
9331 "Failed to set host interface version, return 0x%x\n",
9332 ret);
9333 goto set_local_link_attributes_fail;
9334 }
9335
9336
9337
9338
9339 ret = write_vc_local_phy(dd,
9340 0 ,
9341 1 );
9342 if (ret != HCMD_SUCCESS)
9343 goto set_local_link_attributes_fail;
9344
9345
9346 ret = write_vc_local_fabric(dd, dd->vau, 1, dd->vcu, dd->vl15_init,
9347 ppd->port_crc_mode_enabled);
9348 if (ret != HCMD_SUCCESS)
9349 goto set_local_link_attributes_fail;
9350
9351
9352
9353
9354
9355 if (loopback == LOOPBACK_SERDES)
9356 misc_bits |= 1 << LOOPBACK_SERDES_CONFIG_BIT_MASK_SHIFT;
9357
9358
9359
9360
9361
9362
9363 if (dd->dc8051_ver >= dc8051_ver(1, 25, 0))
9364 misc_bits |= 1 << EXT_CFG_LCB_RESET_SUPPORTED_SHIFT;
9365
9366 ret = write_vc_local_link_mode(dd, misc_bits, 0,
9367 opa_to_vc_link_widths(
9368 ppd->link_width_enabled));
9369 if (ret != HCMD_SUCCESS)
9370 goto set_local_link_attributes_fail;
9371
9372
9373 ret = write_local_device_id(dd, dd->pcidev->device, dd->minrev);
9374 if (ret == HCMD_SUCCESS)
9375 return 0;
9376
9377set_local_link_attributes_fail:
9378 dd_dev_err(dd,
9379 "Failed to set local link attributes, return 0x%x\n",
9380 ret);
9381 return ret;
9382}
9383
9384
9385
9386
9387
9388
9389int start_link(struct hfi1_pportdata *ppd)
9390{
9391
9392
9393
9394
9395 tune_serdes(ppd);
9396
9397 if (!ppd->driver_link_ready) {
9398 dd_dev_info(ppd->dd,
9399 "%s: stopping link start because driver is not ready\n",
9400 __func__);
9401 return 0;
9402 }
9403
9404
9405
9406
9407
9408
9409 clear_full_mgmt_pkey(ppd);
9410
9411 return set_link_state(ppd, HLS_DN_POLL);
9412}
9413
9414static void wait_for_qsfp_init(struct hfi1_pportdata *ppd)
9415{
9416 struct hfi1_devdata *dd = ppd->dd;
9417 u64 mask;
9418 unsigned long timeout;
9419
9420
9421
9422
9423
9424
9425
9426
9427
9428 msleep(500);
9429
9430
9431
9432
9433 timeout = jiffies + msecs_to_jiffies(2000);
9434 while (1) {
9435 mask = read_csr(dd, dd->hfi1_id ?
9436 ASIC_QSFP2_IN : ASIC_QSFP1_IN);
9437 if (!(mask & QSFP_HFI0_INT_N))
9438 break;
9439 if (time_after(jiffies, timeout)) {
9440 dd_dev_info(dd, "%s: No IntN detected, reset complete\n",
9441 __func__);
9442 break;
9443 }
9444 udelay(2);
9445 }
9446}
9447
9448static void set_qsfp_int_n(struct hfi1_pportdata *ppd, u8 enable)
9449{
9450 struct hfi1_devdata *dd = ppd->dd;
9451 u64 mask;
9452
9453 mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK);
9454 if (enable) {
9455
9456
9457
9458
9459 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
9460 QSFP_HFI0_INT_N);
9461 mask |= (u64)QSFP_HFI0_INT_N;
9462 } else {
9463 mask &= ~(u64)QSFP_HFI0_INT_N;
9464 }
9465 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, mask);
9466}
9467
9468int reset_qsfp(struct hfi1_pportdata *ppd)
9469{
9470 struct hfi1_devdata *dd = ppd->dd;
9471 u64 mask, qsfp_mask;
9472
9473
9474 set_qsfp_int_n(ppd, 0);
9475
9476
9477 mask = (u64)QSFP_HFI0_RESET_N;
9478
9479 qsfp_mask = read_csr(dd,
9480 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT);
9481 qsfp_mask &= ~mask;
9482 write_csr(dd,
9483 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
9484
9485 udelay(10);
9486
9487 qsfp_mask |= mask;
9488 write_csr(dd,
9489 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
9490
9491 wait_for_qsfp_init(ppd);
9492
9493
9494
9495
9496
9497 set_qsfp_int_n(ppd, 1);
9498
9499
9500
9501
9502
9503
9504 return set_qsfp_tx(ppd, 0);
9505}
9506
9507static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd,
9508 u8 *qsfp_interrupt_status)
9509{
9510 struct hfi1_devdata *dd = ppd->dd;
9511
9512 if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) ||
9513 (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING))
9514 dd_dev_err(dd, "%s: QSFP cable temperature too high\n",
9515 __func__);
9516
9517 if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) ||
9518 (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING))
9519 dd_dev_err(dd, "%s: QSFP cable temperature too low\n",
9520 __func__);
9521
9522
9523
9524
9525 if (ppd->host_link_state & HLS_DOWN)
9526 return 0;
9527
9528 if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) ||
9529 (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING))
9530 dd_dev_err(dd, "%s: QSFP supply voltage too high\n",
9531 __func__);
9532
9533 if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) ||
9534 (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING))
9535 dd_dev_err(dd, "%s: QSFP supply voltage too low\n",
9536 __func__);
9537
9538
9539
9540 if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) ||
9541 (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING))
9542 dd_dev_err(dd, "%s: Cable RX channel 1/2 power too high\n",
9543 __func__);
9544
9545 if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) ||
9546 (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING))
9547 dd_dev_err(dd, "%s: Cable RX channel 1/2 power too low\n",
9548 __func__);
9549
9550 if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) ||
9551 (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING))
9552 dd_dev_err(dd, "%s: Cable RX channel 3/4 power too high\n",
9553 __func__);
9554
9555 if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) ||
9556 (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING))
9557 dd_dev_err(dd, "%s: Cable RX channel 3/4 power too low\n",
9558 __func__);
9559
9560 if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) ||
9561 (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING))
9562 dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too high\n",
9563 __func__);
9564
9565 if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) ||
9566 (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING))
9567 dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too low\n",
9568 __func__);
9569
9570 if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) ||
9571 (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING))
9572 dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too high\n",
9573 __func__);
9574
9575 if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) ||
9576 (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING))
9577 dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too low\n",
9578 __func__);
9579
9580 if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) ||
9581 (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING))
9582 dd_dev_err(dd, "%s: Cable TX channel 1/2 power too high\n",
9583 __func__);
9584
9585 if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) ||
9586 (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING))
9587 dd_dev_err(dd, "%s: Cable TX channel 1/2 power too low\n",
9588 __func__);
9589
9590 if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) ||
9591 (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING))
9592 dd_dev_err(dd, "%s: Cable TX channel 3/4 power too high\n",
9593 __func__);
9594
9595 if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) ||
9596 (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING))
9597 dd_dev_err(dd, "%s: Cable TX channel 3/4 power too low\n",
9598 __func__);
9599
9600
9601
9602
9603 return 0;
9604}
9605
9606
9607void qsfp_event(struct work_struct *work)
9608{
9609 struct qsfp_data *qd;
9610 struct hfi1_pportdata *ppd;
9611 struct hfi1_devdata *dd;
9612
9613 qd = container_of(work, struct qsfp_data, qsfp_work);
9614 ppd = qd->ppd;
9615 dd = ppd->dd;
9616
9617
9618 if (!qsfp_mod_present(ppd))
9619 return;
9620
9621 if (ppd->host_link_state == HLS_DN_DISABLE) {
9622 dd_dev_info(ppd->dd,
9623 "%s: stopping link start because link is disabled\n",
9624 __func__);
9625 return;
9626 }
9627
9628
9629
9630
9631
9632 dc_start(dd);
9633
9634 if (qd->cache_refresh_required) {
9635 set_qsfp_int_n(ppd, 0);
9636
9637 wait_for_qsfp_init(ppd);
9638
9639
9640
9641
9642
9643 set_qsfp_int_n(ppd, 1);
9644
9645 start_link(ppd);
9646 }
9647
9648 if (qd->check_interrupt_flags) {
9649 u8 qsfp_interrupt_status[16] = {0,};
9650
9651 if (one_qsfp_read(ppd, dd->hfi1_id, 6,
9652 &qsfp_interrupt_status[0], 16) != 16) {
9653 dd_dev_info(dd,
9654 "%s: Failed to read status of QSFP module\n",
9655 __func__);
9656 } else {
9657 unsigned long flags;
9658
9659 handle_qsfp_error_conditions(
9660 ppd, qsfp_interrupt_status);
9661 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
9662 ppd->qsfp_info.check_interrupt_flags = 0;
9663 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
9664 flags);
9665 }
9666 }
9667}
9668
9669void init_qsfp_int(struct hfi1_devdata *dd)
9670{
9671 struct hfi1_pportdata *ppd = dd->pport;
9672 u64 qsfp_mask;
9673
9674 qsfp_mask = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
9675
9676 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
9677 qsfp_mask);
9678 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK,
9679 qsfp_mask);
9680
9681 set_qsfp_int_n(ppd, 0);
9682
9683
9684 if (qsfp_mod_present(ppd))
9685 qsfp_mask &= ~(u64)QSFP_HFI0_MODPRST_N;
9686 write_csr(dd,
9687 dd->hfi1_id ? ASIC_QSFP2_INVERT : ASIC_QSFP1_INVERT,
9688 qsfp_mask);
9689
9690
9691 if (!dd->hfi1_id)
9692 set_intr_bits(dd, QSFP1_INT, QSFP1_INT, true);
9693 else
9694 set_intr_bits(dd, QSFP2_INT, QSFP2_INT, true);
9695}
9696
9697
9698
9699
9700static void init_lcb(struct hfi1_devdata *dd)
9701{
9702
9703 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
9704 return;
9705
9706
9707
9708
9709 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x01);
9710 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0x00);
9711 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0x00);
9712 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110);
9713 write_csr(dd, DC_LCB_CFG_CLK_CNTR, 0x08);
9714 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x02);
9715 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x00);
9716}
9717
9718
9719
9720
9721
9722static int test_qsfp_read(struct hfi1_pportdata *ppd)
9723{
9724 int ret;
9725 u8 status;
9726
9727
9728
9729
9730
9731 if (ppd->port_type != PORT_TYPE_QSFP || !qsfp_mod_present(ppd))
9732 return 0;
9733
9734
9735 ret = one_qsfp_read(ppd, ppd->dd->hfi1_id, 2, &status, 1);
9736 if (ret < 0)
9737 return ret;
9738 if (ret != 1)
9739 return -EIO;
9740
9741 return 0;
9742}
9743
9744
9745
9746
9747
9748
9749
9750#define MAX_QSFP_RETRIES 20
9751#define QSFP_RETRY_WAIT 500
9752
9753
9754
9755
9756
9757static void try_start_link(struct hfi1_pportdata *ppd)
9758{
9759 if (test_qsfp_read(ppd)) {
9760
9761 if (ppd->qsfp_retry_count >= MAX_QSFP_RETRIES) {
9762 dd_dev_err(ppd->dd, "QSFP not responding, giving up\n");
9763 return;
9764 }
9765 dd_dev_info(ppd->dd,
9766 "QSFP not responding, waiting and retrying %d\n",
9767 (int)ppd->qsfp_retry_count);
9768 ppd->qsfp_retry_count++;
9769 queue_delayed_work(ppd->link_wq, &ppd->start_link_work,
9770 msecs_to_jiffies(QSFP_RETRY_WAIT));
9771 return;
9772 }
9773 ppd->qsfp_retry_count = 0;
9774
9775 start_link(ppd);
9776}
9777
9778
9779
9780
9781void handle_start_link(struct work_struct *work)
9782{
9783 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
9784 start_link_work.work);
9785 try_start_link(ppd);
9786}
9787
9788int bringup_serdes(struct hfi1_pportdata *ppd)
9789{
9790 struct hfi1_devdata *dd = ppd->dd;
9791 u64 guid;
9792 int ret;
9793
9794 if (HFI1_CAP_IS_KSET(EXTENDED_PSN))
9795 add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK);
9796
9797 guid = ppd->guids[HFI1_PORT_GUID_INDEX];
9798 if (!guid) {
9799 if (dd->base_guid)
9800 guid = dd->base_guid + ppd->port - 1;
9801 ppd->guids[HFI1_PORT_GUID_INDEX] = guid;
9802 }
9803
9804
9805 ppd->linkinit_reason = OPA_LINKINIT_REASON_LINKUP;
9806
9807
9808 init_lcb(dd);
9809
9810 if (loopback) {
9811 ret = init_loopback(dd);
9812 if (ret < 0)
9813 return ret;
9814 }
9815
9816 get_port_type(ppd);
9817 if (ppd->port_type == PORT_TYPE_QSFP) {
9818 set_qsfp_int_n(ppd, 0);
9819 wait_for_qsfp_init(ppd);
9820 set_qsfp_int_n(ppd, 1);
9821 }
9822
9823 try_start_link(ppd);
9824 return 0;
9825}
9826
9827void hfi1_quiet_serdes(struct hfi1_pportdata *ppd)
9828{
9829 struct hfi1_devdata *dd = ppd->dd;
9830
9831
9832
9833
9834
9835
9836
9837
9838 ppd->driver_link_ready = 0;
9839 ppd->link_enabled = 0;
9840
9841 ppd->qsfp_retry_count = MAX_QSFP_RETRIES;
9842 flush_delayed_work(&ppd->start_link_work);
9843 cancel_delayed_work_sync(&ppd->start_link_work);
9844
9845 ppd->offline_disabled_reason =
9846 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_REBOOT);
9847 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_REBOOT, 0,
9848 OPA_LINKDOWN_REASON_REBOOT);
9849 set_link_state(ppd, HLS_DN_OFFLINE);
9850
9851
9852 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
9853 cancel_work_sync(&ppd->freeze_work);
9854}
9855
9856static inline int init_cpu_counters(struct hfi1_devdata *dd)
9857{
9858 struct hfi1_pportdata *ppd;
9859 int i;
9860
9861 ppd = (struct hfi1_pportdata *)(dd + 1);
9862 for (i = 0; i < dd->num_pports; i++, ppd++) {
9863 ppd->ibport_data.rvp.rc_acks = NULL;
9864 ppd->ibport_data.rvp.rc_qacks = NULL;
9865 ppd->ibport_data.rvp.rc_acks = alloc_percpu(u64);
9866 ppd->ibport_data.rvp.rc_qacks = alloc_percpu(u64);
9867 ppd->ibport_data.rvp.rc_delayed_comp = alloc_percpu(u64);
9868 if (!ppd->ibport_data.rvp.rc_acks ||
9869 !ppd->ibport_data.rvp.rc_delayed_comp ||
9870 !ppd->ibport_data.rvp.rc_qacks)
9871 return -ENOMEM;
9872 }
9873
9874 return 0;
9875}
9876
9877
9878
9879
9880void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
9881 u32 type, unsigned long pa, u16 order)
9882{
9883 u64 reg;
9884
9885 if (!(dd->flags & HFI1_PRESENT))
9886 goto done;
9887
9888 if (type == PT_INVALID || type == PT_INVALID_FLUSH) {
9889 pa = 0;
9890 order = 0;
9891 } else if (type > PT_INVALID) {
9892 dd_dev_err(dd,
9893 "unexpected receive array type %u for index %u, not handled\n",
9894 type, index);
9895 goto done;
9896 }
9897 trace_hfi1_put_tid(dd, index, type, pa, order);
9898
9899#define RT_ADDR_SHIFT 12
9900 reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK
9901 | (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT
9902 | ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK)
9903 << RCV_ARRAY_RT_ADDR_SHIFT;
9904 trace_hfi1_write_rcvarray(dd->rcvarray_wc + (index * 8), reg);
9905 writeq(reg, dd->rcvarray_wc + (index * 8));
9906
9907 if (type == PT_EAGER || type == PT_INVALID_FLUSH || (index & 3) == 3)
9908
9909
9910
9911
9912
9913 flush_wc();
9914done:
9915 return;
9916}
9917
9918void hfi1_clear_tids(struct hfi1_ctxtdata *rcd)
9919{
9920 struct hfi1_devdata *dd = rcd->dd;
9921 u32 i;
9922
9923
9924 for (i = rcd->eager_base; i < rcd->eager_base +
9925 rcd->egrbufs.alloced; i++)
9926 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9927
9928 for (i = rcd->expected_base;
9929 i < rcd->expected_base + rcd->expected_count; i++)
9930 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9931}
9932
9933static const char * const ib_cfg_name_strings[] = {
9934 "HFI1_IB_CFG_LIDLMC",
9935 "HFI1_IB_CFG_LWID_DG_ENB",
9936 "HFI1_IB_CFG_LWID_ENB",
9937 "HFI1_IB_CFG_LWID",
9938 "HFI1_IB_CFG_SPD_ENB",
9939 "HFI1_IB_CFG_SPD",
9940 "HFI1_IB_CFG_RXPOL_ENB",
9941 "HFI1_IB_CFG_LREV_ENB",
9942 "HFI1_IB_CFG_LINKLATENCY",
9943 "HFI1_IB_CFG_HRTBT",
9944 "HFI1_IB_CFG_OP_VLS",
9945 "HFI1_IB_CFG_VL_HIGH_CAP",
9946 "HFI1_IB_CFG_VL_LOW_CAP",
9947 "HFI1_IB_CFG_OVERRUN_THRESH",
9948 "HFI1_IB_CFG_PHYERR_THRESH",
9949 "HFI1_IB_CFG_LINKDEFAULT",
9950 "HFI1_IB_CFG_PKEYS",
9951 "HFI1_IB_CFG_MTU",
9952 "HFI1_IB_CFG_LSTATE",
9953 "HFI1_IB_CFG_VL_HIGH_LIMIT",
9954 "HFI1_IB_CFG_PMA_TICKS",
9955 "HFI1_IB_CFG_PORT"
9956};
9957
9958static const char *ib_cfg_name(int which)
9959{
9960 if (which < 0 || which >= ARRAY_SIZE(ib_cfg_name_strings))
9961 return "invalid";
9962 return ib_cfg_name_strings[which];
9963}
9964
9965int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which)
9966{
9967 struct hfi1_devdata *dd = ppd->dd;
9968 int val = 0;
9969
9970 switch (which) {
9971 case HFI1_IB_CFG_LWID_ENB:
9972 val = ppd->link_width_enabled;
9973 break;
9974 case HFI1_IB_CFG_LWID:
9975 val = ppd->link_width_active;
9976 break;
9977 case HFI1_IB_CFG_SPD_ENB:
9978 val = ppd->link_speed_enabled;
9979 break;
9980 case HFI1_IB_CFG_SPD:
9981 val = ppd->link_speed_active;
9982 break;
9983
9984 case HFI1_IB_CFG_RXPOL_ENB:
9985 case HFI1_IB_CFG_LREV_ENB:
9986 case HFI1_IB_CFG_LINKLATENCY:
9987 goto unimplemented;
9988
9989 case HFI1_IB_CFG_OP_VLS:
9990 val = ppd->actual_vls_operational;
9991 break;
9992 case HFI1_IB_CFG_VL_HIGH_CAP:
9993 val = VL_ARB_HIGH_PRIO_TABLE_SIZE;
9994 break;
9995 case HFI1_IB_CFG_VL_LOW_CAP:
9996 val = VL_ARB_LOW_PRIO_TABLE_SIZE;
9997 break;
9998 case HFI1_IB_CFG_OVERRUN_THRESH:
9999 val = ppd->overrun_threshold;
10000 break;
10001 case HFI1_IB_CFG_PHYERR_THRESH:
10002 val = ppd->phy_error_threshold;
10003 break;
10004 case HFI1_IB_CFG_LINKDEFAULT:
10005 val = HLS_DEFAULT;
10006 break;
10007
10008 case HFI1_IB_CFG_HRTBT:
10009 case HFI1_IB_CFG_PMA_TICKS:
10010 default:
10011unimplemented:
10012 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
10013 dd_dev_info(
10014 dd,
10015 "%s: which %s: not implemented\n",
10016 __func__,
10017 ib_cfg_name(which));
10018 break;
10019 }
10020
10021 return val;
10022}
10023
10024
10025
10026
10027#define MAX_MAD_PACKET 2048
10028
10029
10030
10031
10032
10033
10034
10035
10036
10037
10038u32 lrh_max_header_bytes(struct hfi1_devdata *dd)
10039{
10040
10041
10042
10043
10044
10045
10046
10047
10048
10049
10050 return (dd->rcd[0]->rcvhdrqentsize - 2 + 1) << 2;
10051}
10052
10053
10054
10055
10056
10057
10058
10059
10060
10061
10062
10063
10064static void set_send_length(struct hfi1_pportdata *ppd)
10065{
10066 struct hfi1_devdata *dd = ppd->dd;
10067 u32 max_hb = lrh_max_header_bytes(dd), dcmtu;
10068 u32 maxvlmtu = dd->vld[15].mtu;
10069 u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2)
10070 & SEND_LEN_CHECK1_LEN_VL15_MASK) <<
10071 SEND_LEN_CHECK1_LEN_VL15_SHIFT;
10072 int i, j;
10073 u32 thres;
10074
10075 for (i = 0; i < ppd->vls_supported; i++) {
10076 if (dd->vld[i].mtu > maxvlmtu)
10077 maxvlmtu = dd->vld[i].mtu;
10078 if (i <= 3)
10079 len1 |= (((dd->vld[i].mtu + max_hb) >> 2)
10080 & SEND_LEN_CHECK0_LEN_VL0_MASK) <<
10081 ((i % 4) * SEND_LEN_CHECK0_LEN_VL1_SHIFT);
10082 else
10083 len2 |= (((dd->vld[i].mtu + max_hb) >> 2)
10084 & SEND_LEN_CHECK1_LEN_VL4_MASK) <<
10085 ((i % 4) * SEND_LEN_CHECK1_LEN_VL5_SHIFT);
10086 }
10087 write_csr(dd, SEND_LEN_CHECK0, len1);
10088 write_csr(dd, SEND_LEN_CHECK1, len2);
10089
10090
10091 for (i = 0; i < ppd->vls_supported; i++) {
10092 thres = min(sc_percent_to_threshold(dd->vld[i].sc, 50),
10093 sc_mtu_to_threshold(dd->vld[i].sc,
10094 dd->vld[i].mtu,
10095 dd->rcd[0]->rcvhdrqentsize));
10096 for (j = 0; j < INIT_SC_PER_VL; j++)
10097 sc_set_cr_threshold(
10098 pio_select_send_context_vl(dd, j, i),
10099 thres);
10100 }
10101 thres = min(sc_percent_to_threshold(dd->vld[15].sc, 50),
10102 sc_mtu_to_threshold(dd->vld[15].sc,
10103 dd->vld[15].mtu,
10104 dd->rcd[0]->rcvhdrqentsize));
10105 sc_set_cr_threshold(dd->vld[15].sc, thres);
10106
10107
10108 dcmtu = maxvlmtu == 10240 ? DCC_CFG_PORT_MTU_CAP_10240 :
10109 (ilog2(maxvlmtu >> 8) + 1);
10110 len1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG);
10111 len1 &= ~DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK;
10112 len1 |= ((u64)dcmtu & DCC_CFG_PORT_CONFIG_MTU_CAP_MASK) <<
10113 DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT;
10114 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG, len1);
10115}
10116
10117static void set_lidlmc(struct hfi1_pportdata *ppd)
10118{
10119 int i;
10120 u64 sreg = 0;
10121 struct hfi1_devdata *dd = ppd->dd;
10122 u32 mask = ~((1U << ppd->lmc) - 1);
10123 u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1);
10124 u32 lid;
10125
10126
10127
10128
10129
10130 lid = (ppd->lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) ? 0 : ppd->lid;
10131 c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK
10132 | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK);
10133 c1 |= ((lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
10134 << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) |
10135 ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK)
10136 << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT);
10137 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG1, c1);
10138
10139
10140
10141
10142 sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) <<
10143 SEND_CTXT_CHECK_SLID_MASK_SHIFT) |
10144 (((lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
10145 SEND_CTXT_CHECK_SLID_VALUE_SHIFT);
10146
10147 for (i = 0; i < chip_send_contexts(dd); i++) {
10148 hfi1_cdbg(LINKVERB, "SendContext[%d].SLID_CHECK = 0x%x",
10149 i, (u32)sreg);
10150 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, sreg);
10151 }
10152
10153
10154 sdma_update_lmc(dd, mask, lid);
10155}
10156
10157static const char *state_completed_string(u32 completed)
10158{
10159 static const char * const state_completed[] = {
10160 "EstablishComm",
10161 "OptimizeEQ",
10162 "VerifyCap"
10163 };
10164
10165 if (completed < ARRAY_SIZE(state_completed))
10166 return state_completed[completed];
10167
10168 return "unknown";
10169}
10170
10171static const char all_lanes_dead_timeout_expired[] =
10172 "All lanes were inactive – was the interconnect media removed?";
10173static const char tx_out_of_policy[] =
10174 "Passing lanes on local port do not meet the local link width policy";
10175static const char no_state_complete[] =
10176 "State timeout occurred before link partner completed the state";
10177static const char * const state_complete_reasons[] = {
10178 [0x00] = "Reason unknown",
10179 [0x01] = "Link was halted by driver, refer to LinkDownReason",
10180 [0x02] = "Link partner reported failure",
10181 [0x10] = "Unable to achieve frame sync on any lane",
10182 [0x11] =
10183 "Unable to find a common bit rate with the link partner",
10184 [0x12] =
10185 "Unable to achieve frame sync on sufficient lanes to meet the local link width policy",
10186 [0x13] =
10187 "Unable to identify preset equalization on sufficient lanes to meet the local link width policy",
10188 [0x14] = no_state_complete,
10189 [0x15] =
10190 "State timeout occurred before link partner identified equalization presets",
10191 [0x16] =
10192 "Link partner completed the EstablishComm state, but the passing lanes do not meet the local link width policy",
10193 [0x17] = tx_out_of_policy,
10194 [0x20] = all_lanes_dead_timeout_expired,
10195 [0x21] =
10196 "Unable to achieve acceptable BER on sufficient lanes to meet the local link width policy",
10197 [0x22] = no_state_complete,
10198 [0x23] =
10199 "Link partner completed the OptimizeEq state, but the passing lanes do not meet the local link width policy",
10200 [0x24] = tx_out_of_policy,
10201 [0x30] = all_lanes_dead_timeout_expired,
10202 [0x31] =
10203 "State timeout occurred waiting for host to process received frames",
10204 [0x32] = no_state_complete,
10205 [0x33] =
10206 "Link partner completed the VerifyCap state, but the passing lanes do not meet the local link width policy",
10207 [0x34] = tx_out_of_policy,
10208 [0x35] = "Negotiated link width is mutually exclusive",
10209 [0x36] =
10210 "Timed out before receiving verifycap frames in VerifyCap.Exchange",
10211 [0x37] = "Unable to resolve secure data exchange",
10212};
10213
10214static const char *state_complete_reason_code_string(struct hfi1_pportdata *ppd,
10215 u32 code)
10216{
10217 const char *str = NULL;
10218
10219 if (code < ARRAY_SIZE(state_complete_reasons))
10220 str = state_complete_reasons[code];
10221
10222 if (str)
10223 return str;
10224 return "Reserved";
10225}
10226
10227
10228static void decode_state_complete(struct hfi1_pportdata *ppd, u32 frame,
10229 const char *prefix)
10230{
10231 struct hfi1_devdata *dd = ppd->dd;
10232 u32 success;
10233 u32 state;
10234 u32 reason;
10235 u32 lanes;
10236
10237
10238
10239
10240
10241
10242
10243
10244
10245 success = frame & 0x1;
10246 state = (frame >> 1) & 0x7;
10247 reason = (frame >> 8) & 0xff;
10248 lanes = (frame >> 16) & 0xffff;
10249
10250 dd_dev_err(dd, "Last %s LNI state complete frame 0x%08x:\n",
10251 prefix, frame);
10252 dd_dev_err(dd, " last reported state state: %s (0x%x)\n",
10253 state_completed_string(state), state);
10254 dd_dev_err(dd, " state successfully completed: %s\n",
10255 success ? "yes" : "no");
10256 dd_dev_err(dd, " fail reason 0x%x: %s\n",
10257 reason, state_complete_reason_code_string(ppd, reason));
10258 dd_dev_err(dd, " passing lane mask: 0x%x", lanes);
10259}
10260
10261
10262
10263
10264
10265
10266static void check_lni_states(struct hfi1_pportdata *ppd)
10267{
10268 u32 last_local_state;
10269 u32 last_remote_state;
10270
10271 read_last_local_state(ppd->dd, &last_local_state);
10272 read_last_remote_state(ppd->dd, &last_remote_state);
10273
10274
10275
10276
10277
10278
10279 if (last_local_state == 0 && last_remote_state == 0)
10280 return;
10281
10282 decode_state_complete(ppd, last_local_state, "transmitted");
10283 decode_state_complete(ppd, last_remote_state, "received");
10284}
10285
10286
10287static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms)
10288{
10289 u64 reg;
10290 unsigned long timeout;
10291
10292
10293 timeout = jiffies + msecs_to_jiffies(wait_ms);
10294 while (1) {
10295 reg = read_csr(dd, DC_LCB_STS_LINK_TRANSFER_ACTIVE);
10296 if (reg)
10297 break;
10298 if (time_after(jiffies, timeout)) {
10299 dd_dev_err(dd,
10300 "timeout waiting for LINK_TRANSFER_ACTIVE\n");
10301 return -ETIMEDOUT;
10302 }
10303 udelay(2);
10304 }
10305 return 0;
10306}
10307
10308
10309static void force_logical_link_state_down(struct hfi1_pportdata *ppd)
10310{
10311 struct hfi1_devdata *dd = ppd->dd;
10312
10313
10314
10315
10316 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1);
10317 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK,
10318 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK);
10319
10320 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0);
10321 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0);
10322 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110);
10323 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x2);
10324
10325 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
10326 (void)read_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET);
10327 udelay(3);
10328 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 1);
10329 write_csr(dd, DC_LCB_CFG_RUN, 1ull << DC_LCB_CFG_RUN_EN_SHIFT);
10330
10331 wait_link_transfer_active(dd, 100);
10332
10333
10334
10335
10336 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1);
10337 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 0);
10338 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 0);
10339
10340 dd_dev_info(ppd->dd, "logical state forced to LINK_DOWN\n");
10341}
10342
10343
10344
10345
10346
10347
10348
10349
10350
10351static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason)
10352{
10353 struct hfi1_devdata *dd = ppd->dd;
10354 u32 previous_state;
10355 int offline_state_ret;
10356 int ret;
10357
10358 update_lcb_cache(dd);
10359
10360 previous_state = ppd->host_link_state;
10361 ppd->host_link_state = HLS_GOING_OFFLINE;
10362
10363
10364 ret = set_physical_link_state(dd, (rem_reason << 8) | PLS_OFFLINE);
10365
10366 if (ret != HCMD_SUCCESS) {
10367 dd_dev_err(dd,
10368 "Failed to transition to Offline link state, return %d\n",
10369 ret);
10370 return -EINVAL;
10371 }
10372 if (ppd->offline_disabled_reason ==
10373 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE))
10374 ppd->offline_disabled_reason =
10375 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
10376
10377 offline_state_ret = wait_phys_link_offline_substates(ppd, 10000);
10378 if (offline_state_ret < 0)
10379 return offline_state_ret;
10380
10381
10382 if (ppd->port_type == PORT_TYPE_QSFP &&
10383 ppd->qsfp_info.limiting_active &&
10384 qsfp_mod_present(ppd)) {
10385 int ret;
10386
10387 ret = acquire_chip_resource(dd, qsfp_resource(dd), QSFP_WAIT);
10388 if (ret == 0) {
10389 set_qsfp_tx(ppd, 0);
10390 release_chip_resource(dd, qsfp_resource(dd));
10391 } else {
10392
10393 dd_dev_err(dd,
10394 "Unable to acquire lock to turn off QSFP TX\n");
10395 }
10396 }
10397
10398
10399
10400
10401
10402 if (offline_state_ret != PLS_OFFLINE_QUIET) {
10403 ret = wait_physical_linkstate(ppd, PLS_OFFLINE, 30000);
10404 if (ret < 0)
10405 return ret;
10406 }
10407
10408
10409
10410
10411
10412 set_host_lcb_access(dd);
10413 write_csr(dd, DC_LCB_ERR_EN, ~0ull);
10414
10415
10416 ret = wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000);
10417 if (ret)
10418 force_logical_link_state_down(ppd);
10419
10420 ppd->host_link_state = HLS_LINK_COOLDOWN;
10421 update_statusp(ppd, IB_PORT_DOWN);
10422
10423
10424
10425
10426
10427
10428
10429
10430
10431
10432 ret = wait_fm_ready(dd, 7000);
10433 if (ret) {
10434 dd_dev_err(dd,
10435 "After going offline, timed out waiting for the 8051 to become ready to accept host requests\n");
10436
10437 ppd->host_link_state = HLS_DN_OFFLINE;
10438 return ret;
10439 }
10440
10441
10442
10443
10444
10445
10446
10447 ppd->host_link_state = HLS_DN_OFFLINE;
10448 if (previous_state & HLS_UP) {
10449
10450 handle_linkup_change(dd, 0);
10451 } else if (previous_state
10452 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
10453
10454 check_lni_states(ppd);
10455
10456
10457 ppd->qsfp_info.reset_needed = 0;
10458 }
10459
10460
10461 ppd->link_width_active = 0;
10462 ppd->link_width_downgrade_tx_active = 0;
10463 ppd->link_width_downgrade_rx_active = 0;
10464 ppd->current_egress_rate = 0;
10465 return 0;
10466}
10467
10468
10469static const char *link_state_name(u32 state)
10470{
10471 const char *name;
10472 int n = ilog2(state);
10473 static const char * const names[] = {
10474 [__HLS_UP_INIT_BP] = "INIT",
10475 [__HLS_UP_ARMED_BP] = "ARMED",
10476 [__HLS_UP_ACTIVE_BP] = "ACTIVE",
10477 [__HLS_DN_DOWNDEF_BP] = "DOWNDEF",
10478 [__HLS_DN_POLL_BP] = "POLL",
10479 [__HLS_DN_DISABLE_BP] = "DISABLE",
10480 [__HLS_DN_OFFLINE_BP] = "OFFLINE",
10481 [__HLS_VERIFY_CAP_BP] = "VERIFY_CAP",
10482 [__HLS_GOING_UP_BP] = "GOING_UP",
10483 [__HLS_GOING_OFFLINE_BP] = "GOING_OFFLINE",
10484 [__HLS_LINK_COOLDOWN_BP] = "LINK_COOLDOWN"
10485 };
10486
10487 name = n < ARRAY_SIZE(names) ? names[n] : NULL;
10488 return name ? name : "unknown";
10489}
10490
10491
10492static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state)
10493{
10494 if (state == HLS_UP_INIT) {
10495 switch (ppd->linkinit_reason) {
10496 case OPA_LINKINIT_REASON_LINKUP:
10497 return "(LINKUP)";
10498 case OPA_LINKINIT_REASON_FLAPPING:
10499 return "(FLAPPING)";
10500 case OPA_LINKINIT_OUTSIDE_POLICY:
10501 return "(OUTSIDE_POLICY)";
10502 case OPA_LINKINIT_QUARANTINED:
10503 return "(QUARANTINED)";
10504 case OPA_LINKINIT_INSUFIC_CAPABILITY:
10505 return "(INSUFIC_CAPABILITY)";
10506 default:
10507 break;
10508 }
10509 }
10510 return "";
10511}
10512
10513
10514
10515
10516
10517
10518u32 driver_pstate(struct hfi1_pportdata *ppd)
10519{
10520 switch (ppd->host_link_state) {
10521 case HLS_UP_INIT:
10522 case HLS_UP_ARMED:
10523 case HLS_UP_ACTIVE:
10524 return IB_PORTPHYSSTATE_LINKUP;
10525 case HLS_DN_POLL:
10526 return IB_PORTPHYSSTATE_POLLING;
10527 case HLS_DN_DISABLE:
10528 return IB_PORTPHYSSTATE_DISABLED;
10529 case HLS_DN_OFFLINE:
10530 return OPA_PORTPHYSSTATE_OFFLINE;
10531 case HLS_VERIFY_CAP:
10532 return IB_PORTPHYSSTATE_TRAINING;
10533 case HLS_GOING_UP:
10534 return IB_PORTPHYSSTATE_TRAINING;
10535 case HLS_GOING_OFFLINE:
10536 return OPA_PORTPHYSSTATE_OFFLINE;
10537 case HLS_LINK_COOLDOWN:
10538 return OPA_PORTPHYSSTATE_OFFLINE;
10539 case HLS_DN_DOWNDEF:
10540 default:
10541 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
10542 ppd->host_link_state);
10543 return -1;
10544 }
10545}
10546
10547
10548
10549
10550
10551
10552u32 driver_lstate(struct hfi1_pportdata *ppd)
10553{
10554 if (ppd->host_link_state && (ppd->host_link_state & HLS_DOWN))
10555 return IB_PORT_DOWN;
10556
10557 switch (ppd->host_link_state & HLS_UP) {
10558 case HLS_UP_INIT:
10559 return IB_PORT_INIT;
10560 case HLS_UP_ARMED:
10561 return IB_PORT_ARMED;
10562 case HLS_UP_ACTIVE:
10563 return IB_PORT_ACTIVE;
10564 default:
10565 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
10566 ppd->host_link_state);
10567 return -1;
10568 }
10569}
10570
10571void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason,
10572 u8 neigh_reason, u8 rem_reason)
10573{
10574 if (ppd->local_link_down_reason.latest == 0 &&
10575 ppd->neigh_link_down_reason.latest == 0) {
10576 ppd->local_link_down_reason.latest = lcl_reason;
10577 ppd->neigh_link_down_reason.latest = neigh_reason;
10578 ppd->remote_link_down_reason = rem_reason;
10579 }
10580}
10581
10582
10583
10584
10585
10586
10587
10588
10589static inline bool data_vls_operational(struct hfi1_pportdata *ppd)
10590{
10591 int i;
10592 u64 reg;
10593
10594 if (!ppd->actual_vls_operational)
10595 return false;
10596
10597 for (i = 0; i < ppd->vls_supported; i++) {
10598 reg = read_csr(ppd->dd, SEND_CM_CREDIT_VL + (8 * i));
10599 if ((reg && !ppd->dd->vld[i].mtu) ||
10600 (!reg && ppd->dd->vld[i].mtu))
10601 return false;
10602 }
10603
10604 return true;
10605}
10606
10607
10608
10609
10610
10611
10612
10613
10614
10615int set_link_state(struct hfi1_pportdata *ppd, u32 state)
10616{
10617 struct hfi1_devdata *dd = ppd->dd;
10618 struct ib_event event = {.device = NULL};
10619 int ret1, ret = 0;
10620 int orig_new_state, poll_bounce;
10621
10622 mutex_lock(&ppd->hls_lock);
10623
10624 orig_new_state = state;
10625 if (state == HLS_DN_DOWNDEF)
10626 state = HLS_DEFAULT;
10627
10628
10629 poll_bounce = ppd->host_link_state == HLS_DN_POLL &&
10630 state == HLS_DN_POLL;
10631
10632 dd_dev_info(dd, "%s: current %s, new %s %s%s\n", __func__,
10633 link_state_name(ppd->host_link_state),
10634 link_state_name(orig_new_state),
10635 poll_bounce ? "(bounce) " : "",
10636 link_state_reason_name(ppd, state));
10637
10638
10639
10640
10641
10642
10643 if (!(state & (HLS_UP_ARMED | HLS_UP_ACTIVE)))
10644 ppd->is_sm_config_started = 0;
10645
10646
10647
10648
10649
10650 if (ppd->host_link_state == state && !poll_bounce)
10651 goto done;
10652
10653 switch (state) {
10654 case HLS_UP_INIT:
10655 if (ppd->host_link_state == HLS_DN_POLL &&
10656 (quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) {
10657
10658
10659
10660
10661
10662
10663
10664
10665 } else if (ppd->host_link_state != HLS_GOING_UP) {
10666 goto unexpected;
10667 }
10668
10669
10670
10671
10672
10673
10674 ret = wait_physical_linkstate(ppd, PLS_LINKUP, 1000);
10675 if (ret) {
10676 dd_dev_err(dd,
10677 "%s: physical state did not change to LINK-UP\n",
10678 __func__);
10679 break;
10680 }
10681
10682 ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000);
10683 if (ret) {
10684 dd_dev_err(dd,
10685 "%s: logical state did not change to INIT\n",
10686 __func__);
10687 break;
10688 }
10689
10690
10691 if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR)
10692 ppd->linkinit_reason =
10693 OPA_LINKINIT_REASON_LINKUP;
10694
10695
10696 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
10697
10698 handle_linkup_change(dd, 1);
10699 pio_kernel_linkup(dd);
10700
10701
10702
10703
10704
10705
10706 update_xmit_counters(ppd, ppd->link_width_active);
10707
10708 ppd->host_link_state = HLS_UP_INIT;
10709 update_statusp(ppd, IB_PORT_INIT);
10710 break;
10711 case HLS_UP_ARMED:
10712 if (ppd->host_link_state != HLS_UP_INIT)
10713 goto unexpected;
10714
10715 if (!data_vls_operational(ppd)) {
10716 dd_dev_err(dd,
10717 "%s: Invalid data VL credits or mtu\n",
10718 __func__);
10719 ret = -EINVAL;
10720 break;
10721 }
10722
10723 set_logical_state(dd, LSTATE_ARMED);
10724 ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000);
10725 if (ret) {
10726 dd_dev_err(dd,
10727 "%s: logical state did not change to ARMED\n",
10728 __func__);
10729 break;
10730 }
10731 ppd->host_link_state = HLS_UP_ARMED;
10732 update_statusp(ppd, IB_PORT_ARMED);
10733
10734
10735
10736
10737
10738 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
10739 ppd->neighbor_normal = 1;
10740 break;
10741 case HLS_UP_ACTIVE:
10742 if (ppd->host_link_state != HLS_UP_ARMED)
10743 goto unexpected;
10744
10745 set_logical_state(dd, LSTATE_ACTIVE);
10746 ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000);
10747 if (ret) {
10748 dd_dev_err(dd,
10749 "%s: logical state did not change to ACTIVE\n",
10750 __func__);
10751 } else {
10752
10753 sdma_all_running(dd);
10754 ppd->host_link_state = HLS_UP_ACTIVE;
10755 update_statusp(ppd, IB_PORT_ACTIVE);
10756
10757
10758 event.device = &dd->verbs_dev.rdi.ibdev;
10759 event.element.port_num = ppd->port;
10760 event.event = IB_EVENT_PORT_ACTIVE;
10761 }
10762 break;
10763 case HLS_DN_POLL:
10764 if ((ppd->host_link_state == HLS_DN_DISABLE ||
10765 ppd->host_link_state == HLS_DN_OFFLINE) &&
10766 dd->dc_shutdown)
10767 dc_start(dd);
10768
10769 write_csr(dd, DCC_CFG_LED_CNTRL, 0);
10770
10771 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10772 u8 tmp = ppd->link_enabled;
10773
10774 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10775 if (ret) {
10776 ppd->link_enabled = tmp;
10777 break;
10778 }
10779 ppd->remote_link_down_reason = 0;
10780
10781 if (ppd->driver_link_ready)
10782 ppd->link_enabled = 1;
10783 }
10784
10785 set_all_slowpath(ppd->dd);
10786 ret = set_local_link_attributes(ppd);
10787 if (ret)
10788 break;
10789
10790 ppd->port_error_action = 0;
10791
10792 if (quick_linkup) {
10793
10794 ret = do_quick_linkup(dd);
10795 } else {
10796 ret1 = set_physical_link_state(dd, PLS_POLLING);
10797 if (!ret1)
10798 ret1 = wait_phys_link_out_of_offline(ppd,
10799 3000);
10800 if (ret1 != HCMD_SUCCESS) {
10801 dd_dev_err(dd,
10802 "Failed to transition to Polling link state, return 0x%x\n",
10803 ret1);
10804 ret = -EINVAL;
10805 }
10806 }
10807
10808
10809
10810
10811
10812
10813
10814 ppd->host_link_state = HLS_DN_POLL;
10815 ppd->offline_disabled_reason =
10816 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE);
10817
10818
10819
10820
10821 if (ret)
10822 goto_offline(ppd, 0);
10823 else
10824 log_physical_state(ppd, PLS_POLLING);
10825 break;
10826 case HLS_DN_DISABLE:
10827
10828 ppd->link_enabled = 0;
10829
10830
10831
10832
10833 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10834 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10835 if (ret)
10836 break;
10837 ppd->remote_link_down_reason = 0;
10838 }
10839
10840 if (!dd->dc_shutdown) {
10841 ret1 = set_physical_link_state(dd, PLS_DISABLED);
10842 if (ret1 != HCMD_SUCCESS) {
10843 dd_dev_err(dd,
10844 "Failed to transition to Disabled link state, return 0x%x\n",
10845 ret1);
10846 ret = -EINVAL;
10847 break;
10848 }
10849 ret = wait_physical_linkstate(ppd, PLS_DISABLED, 10000);
10850 if (ret) {
10851 dd_dev_err(dd,
10852 "%s: physical state did not change to DISABLED\n",
10853 __func__);
10854 break;
10855 }
10856 dc_shutdown(dd);
10857 }
10858 ppd->host_link_state = HLS_DN_DISABLE;
10859 break;
10860 case HLS_DN_OFFLINE:
10861 if (ppd->host_link_state == HLS_DN_DISABLE)
10862 dc_start(dd);
10863
10864
10865 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10866 if (!ret)
10867 ppd->remote_link_down_reason = 0;
10868 break;
10869 case HLS_VERIFY_CAP:
10870 if (ppd->host_link_state != HLS_DN_POLL)
10871 goto unexpected;
10872 ppd->host_link_state = HLS_VERIFY_CAP;
10873 log_physical_state(ppd, PLS_CONFIGPHY_VERIFYCAP);
10874 break;
10875 case HLS_GOING_UP:
10876 if (ppd->host_link_state != HLS_VERIFY_CAP)
10877 goto unexpected;
10878
10879 ret1 = set_physical_link_state(dd, PLS_LINKUP);
10880 if (ret1 != HCMD_SUCCESS) {
10881 dd_dev_err(dd,
10882 "Failed to transition to link up state, return 0x%x\n",
10883 ret1);
10884 ret = -EINVAL;
10885 break;
10886 }
10887 ppd->host_link_state = HLS_GOING_UP;
10888 break;
10889
10890 case HLS_GOING_OFFLINE:
10891 case HLS_LINK_COOLDOWN:
10892 default:
10893 dd_dev_info(dd, "%s: state 0x%x: not supported\n",
10894 __func__, state);
10895 ret = -EINVAL;
10896 break;
10897 }
10898
10899 goto done;
10900
10901unexpected:
10902 dd_dev_err(dd, "%s: unexpected state transition from %s to %s\n",
10903 __func__, link_state_name(ppd->host_link_state),
10904 link_state_name(state));
10905 ret = -EINVAL;
10906
10907done:
10908 mutex_unlock(&ppd->hls_lock);
10909
10910 if (event.device)
10911 ib_dispatch_event(&event);
10912
10913 return ret;
10914}
10915
10916int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val)
10917{
10918 u64 reg;
10919 int ret = 0;
10920
10921 switch (which) {
10922 case HFI1_IB_CFG_LIDLMC:
10923 set_lidlmc(ppd);
10924 break;
10925 case HFI1_IB_CFG_VL_HIGH_LIMIT:
10926
10927
10928
10929
10930 val *= 4096 / 64;
10931 reg = ((u64)val & SEND_HIGH_PRIORITY_LIMIT_LIMIT_MASK)
10932 << SEND_HIGH_PRIORITY_LIMIT_LIMIT_SHIFT;
10933 write_csr(ppd->dd, SEND_HIGH_PRIORITY_LIMIT, reg);
10934 break;
10935 case HFI1_IB_CFG_LINKDEFAULT:
10936
10937 if (val != HLS_DN_POLL)
10938 ret = -EINVAL;
10939 break;
10940 case HFI1_IB_CFG_OP_VLS:
10941 if (ppd->vls_operational != val) {
10942 ppd->vls_operational = val;
10943 if (!ppd->port)
10944 ret = -EINVAL;
10945 }
10946 break;
10947
10948
10949
10950
10951
10952
10953
10954
10955 case HFI1_IB_CFG_LWID_ENB:
10956 ppd->link_width_enabled = val & ppd->link_width_supported;
10957 break;
10958 case HFI1_IB_CFG_LWID_DG_ENB:
10959 ppd->link_width_downgrade_enabled =
10960 val & ppd->link_width_downgrade_supported;
10961 break;
10962 case HFI1_IB_CFG_SPD_ENB:
10963 ppd->link_speed_enabled = val & ppd->link_speed_supported;
10964 break;
10965 case HFI1_IB_CFG_OVERRUN_THRESH:
10966
10967
10968
10969
10970 ppd->overrun_threshold = val;
10971 break;
10972 case HFI1_IB_CFG_PHYERR_THRESH:
10973
10974
10975
10976
10977 ppd->phy_error_threshold = val;
10978 break;
10979
10980 case HFI1_IB_CFG_MTU:
10981 set_send_length(ppd);
10982 break;
10983
10984 case HFI1_IB_CFG_PKEYS:
10985 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
10986 set_partition_keys(ppd);
10987 break;
10988
10989 default:
10990 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
10991 dd_dev_info(ppd->dd,
10992 "%s: which %s, val 0x%x: not implemented\n",
10993 __func__, ib_cfg_name(which), val);
10994 break;
10995 }
10996 return ret;
10997}
10998
10999
11000static void init_vl_arb_caches(struct hfi1_pportdata *ppd)
11001{
11002 int i;
11003
11004 BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
11005 VL_ARB_LOW_PRIO_TABLE_SIZE);
11006 BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
11007 VL_ARB_HIGH_PRIO_TABLE_SIZE);
11008
11009
11010
11011
11012
11013
11014
11015
11016
11017
11018
11019 for (i = 0; i < MAX_PRIO_TABLE; i++)
11020 spin_lock_init(&ppd->vl_arb_cache[i].lock);
11021}
11022
11023
11024
11025
11026
11027
11028
11029static inline struct vl_arb_cache *
11030vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx)
11031{
11032 if (idx != LO_PRIO_TABLE && idx != HI_PRIO_TABLE)
11033 return NULL;
11034 spin_lock(&ppd->vl_arb_cache[idx].lock);
11035 return &ppd->vl_arb_cache[idx];
11036}
11037
11038static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx)
11039{
11040 spin_unlock(&ppd->vl_arb_cache[idx].lock);
11041}
11042
11043static void vl_arb_get_cache(struct vl_arb_cache *cache,
11044 struct ib_vl_weight_elem *vl)
11045{
11046 memcpy(vl, cache->table, VL_ARB_TABLE_SIZE * sizeof(*vl));
11047}
11048
11049static void vl_arb_set_cache(struct vl_arb_cache *cache,
11050 struct ib_vl_weight_elem *vl)
11051{
11052 memcpy(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
11053}
11054
11055static int vl_arb_match_cache(struct vl_arb_cache *cache,
11056 struct ib_vl_weight_elem *vl)
11057{
11058 return !memcmp(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
11059}
11060
11061
11062
11063static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target,
11064 u32 size, struct ib_vl_weight_elem *vl)
11065{
11066 struct hfi1_devdata *dd = ppd->dd;
11067 u64 reg;
11068 unsigned int i, is_up = 0;
11069 int drain, ret = 0;
11070
11071 mutex_lock(&ppd->hls_lock);
11072
11073 if (ppd->host_link_state & HLS_UP)
11074 is_up = 1;
11075
11076 drain = !is_ax(dd) && is_up;
11077
11078 if (drain)
11079
11080
11081
11082
11083
11084
11085 ret = stop_drain_data_vls(dd);
11086
11087 if (ret) {
11088 dd_dev_err(
11089 dd,
11090 "%s: cannot stop/drain VLs - refusing to change VL arbitration weights\n",
11091 __func__);
11092 goto err;
11093 }
11094
11095 for (i = 0; i < size; i++, vl++) {
11096
11097
11098
11099
11100 reg = (((u64)vl->vl & SEND_LOW_PRIORITY_LIST_VL_MASK)
11101 << SEND_LOW_PRIORITY_LIST_VL_SHIFT)
11102 | (((u64)vl->weight
11103 & SEND_LOW_PRIORITY_LIST_WEIGHT_MASK)
11104 << SEND_LOW_PRIORITY_LIST_WEIGHT_SHIFT);
11105 write_csr(dd, target + (i * 8), reg);
11106 }
11107 pio_send_control(dd, PSC_GLOBAL_VLARB_ENABLE);
11108
11109 if (drain)
11110 open_fill_data_vls(dd);
11111
11112err:
11113 mutex_unlock(&ppd->hls_lock);
11114
11115 return ret;
11116}
11117
11118
11119
11120
11121static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr,
11122 struct vl_limit *vll)
11123{
11124 u64 reg = read_csr(dd, csr);
11125
11126 vll->dedicated = cpu_to_be16(
11127 (reg >> SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT)
11128 & SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_MASK);
11129 vll->shared = cpu_to_be16(
11130 (reg >> SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT)
11131 & SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_MASK);
11132}
11133
11134
11135
11136
11137static int get_buffer_control(struct hfi1_devdata *dd,
11138 struct buffer_control *bc, u16 *overall_limit)
11139{
11140 u64 reg;
11141 int i;
11142
11143
11144 memset(bc, 0, sizeof(*bc));
11145
11146
11147 for (i = 0; i < TXE_NUM_DATA_VL; i++)
11148 read_one_cm_vl(dd, SEND_CM_CREDIT_VL + (8 * i), &bc->vl[i]);
11149
11150
11151 read_one_cm_vl(dd, SEND_CM_CREDIT_VL15, &bc->vl[15]);
11152
11153 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
11154 bc->overall_shared_limit = cpu_to_be16(
11155 (reg >> SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT)
11156 & SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_MASK);
11157 if (overall_limit)
11158 *overall_limit = (reg
11159 >> SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT)
11160 & SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_MASK;
11161 return sizeof(struct buffer_control);
11162}
11163
11164static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
11165{
11166 u64 reg;
11167 int i;
11168
11169
11170 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_15_0);
11171 for (i = 0; i < sizeof(u64); i++) {
11172 u8 byte = *(((u8 *)®) + i);
11173
11174 dp->vlnt[2 * i] = byte & 0xf;
11175 dp->vlnt[(2 * i) + 1] = (byte & 0xf0) >> 4;
11176 }
11177
11178 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_31_16);
11179 for (i = 0; i < sizeof(u64); i++) {
11180 u8 byte = *(((u8 *)®) + i);
11181
11182 dp->vlnt[16 + (2 * i)] = byte & 0xf;
11183 dp->vlnt[16 + (2 * i) + 1] = (byte & 0xf0) >> 4;
11184 }
11185 return sizeof(struct sc2vlnt);
11186}
11187
11188static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems,
11189 struct ib_vl_weight_elem *vl)
11190{
11191 unsigned int i;
11192
11193 for (i = 0; i < nelems; i++, vl++) {
11194 vl->vl = 0xf;
11195 vl->weight = 0;
11196 }
11197}
11198
11199static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
11200{
11201 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0,
11202 DC_SC_VL_VAL(15_0,
11203 0, dp->vlnt[0] & 0xf,
11204 1, dp->vlnt[1] & 0xf,
11205 2, dp->vlnt[2] & 0xf,
11206 3, dp->vlnt[3] & 0xf,
11207 4, dp->vlnt[4] & 0xf,
11208 5, dp->vlnt[5] & 0xf,
11209 6, dp->vlnt[6] & 0xf,
11210 7, dp->vlnt[7] & 0xf,
11211 8, dp->vlnt[8] & 0xf,
11212 9, dp->vlnt[9] & 0xf,
11213 10, dp->vlnt[10] & 0xf,
11214 11, dp->vlnt[11] & 0xf,
11215 12, dp->vlnt[12] & 0xf,
11216 13, dp->vlnt[13] & 0xf,
11217 14, dp->vlnt[14] & 0xf,
11218 15, dp->vlnt[15] & 0xf));
11219 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16,
11220 DC_SC_VL_VAL(31_16,
11221 16, dp->vlnt[16] & 0xf,
11222 17, dp->vlnt[17] & 0xf,
11223 18, dp->vlnt[18] & 0xf,
11224 19, dp->vlnt[19] & 0xf,
11225 20, dp->vlnt[20] & 0xf,
11226 21, dp->vlnt[21] & 0xf,
11227 22, dp->vlnt[22] & 0xf,
11228 23, dp->vlnt[23] & 0xf,
11229 24, dp->vlnt[24] & 0xf,
11230 25, dp->vlnt[25] & 0xf,
11231 26, dp->vlnt[26] & 0xf,
11232 27, dp->vlnt[27] & 0xf,
11233 28, dp->vlnt[28] & 0xf,
11234 29, dp->vlnt[29] & 0xf,
11235 30, dp->vlnt[30] & 0xf,
11236 31, dp->vlnt[31] & 0xf));
11237}
11238
11239static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what,
11240 u16 limit)
11241{
11242 if (limit != 0)
11243 dd_dev_info(dd, "Invalid %s limit %d on VL %d, ignoring\n",
11244 what, (int)limit, idx);
11245}
11246
11247
11248static void set_global_shared(struct hfi1_devdata *dd, u16 limit)
11249{
11250 u64 reg;
11251
11252 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
11253 reg &= ~SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK;
11254 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT;
11255 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
11256}
11257
11258
11259static void set_global_limit(struct hfi1_devdata *dd, u16 limit)
11260{
11261 u64 reg;
11262
11263 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
11264 reg &= ~SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK;
11265 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT;
11266 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
11267}
11268
11269
11270static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit)
11271{
11272 u64 reg;
11273 u32 addr;
11274
11275 if (vl < TXE_NUM_DATA_VL)
11276 addr = SEND_CM_CREDIT_VL + (8 * vl);
11277 else
11278 addr = SEND_CM_CREDIT_VL15;
11279
11280 reg = read_csr(dd, addr);
11281 reg &= ~SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SMASK;
11282 reg |= (u64)limit << SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT;
11283 write_csr(dd, addr, reg);
11284}
11285
11286
11287static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit)
11288{
11289 u64 reg;
11290 u32 addr;
11291
11292 if (vl < TXE_NUM_DATA_VL)
11293 addr = SEND_CM_CREDIT_VL + (8 * vl);
11294 else
11295 addr = SEND_CM_CREDIT_VL15;
11296
11297 reg = read_csr(dd, addr);
11298 reg &= ~SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SMASK;
11299 reg |= (u64)limit << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT;
11300 write_csr(dd, addr, reg);
11301}
11302
11303
11304static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask,
11305 const char *which)
11306{
11307 unsigned long timeout;
11308 u64 reg;
11309
11310 timeout = jiffies + msecs_to_jiffies(VL_STATUS_CLEAR_TIMEOUT);
11311 while (1) {
11312 reg = read_csr(dd, SEND_CM_CREDIT_USED_STATUS) & mask;
11313
11314 if (reg == 0)
11315 return;
11316 if (time_after(jiffies, timeout))
11317 break;
11318 udelay(1);
11319 }
11320
11321 dd_dev_err(dd,
11322 "%s credit change status not clearing after %dms, mask 0x%llx, not clear 0x%llx\n",
11323 which, VL_STATUS_CLEAR_TIMEOUT, mask, reg);
11324
11325
11326
11327
11328 dd_dev_err(dd,
11329 "Continuing anyway. A credit loss may occur. Suggest a link bounce\n");
11330}
11331
11332
11333
11334
11335
11336
11337
11338
11339
11340
11341
11342
11343
11344
11345
11346
11347
11348
11349
11350
11351
11352
11353
11354
11355
11356int set_buffer_control(struct hfi1_pportdata *ppd,
11357 struct buffer_control *new_bc)
11358{
11359 struct hfi1_devdata *dd = ppd->dd;
11360 u64 changing_mask, ld_mask, stat_mask;
11361 int change_count;
11362 int i, use_all_mask;
11363 int this_shared_changing;
11364 int vl_count = 0, ret;
11365
11366
11367
11368
11369 int any_shared_limit_changing;
11370 struct buffer_control cur_bc;
11371 u8 changing[OPA_MAX_VLS];
11372 u8 lowering_dedicated[OPA_MAX_VLS];
11373 u16 cur_total;
11374 u32 new_total = 0;
11375 const u64 all_mask =
11376 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK
11377 | SEND_CM_CREDIT_USED_STATUS_VL1_RETURN_CREDIT_STATUS_SMASK
11378 | SEND_CM_CREDIT_USED_STATUS_VL2_RETURN_CREDIT_STATUS_SMASK
11379 | SEND_CM_CREDIT_USED_STATUS_VL3_RETURN_CREDIT_STATUS_SMASK
11380 | SEND_CM_CREDIT_USED_STATUS_VL4_RETURN_CREDIT_STATUS_SMASK
11381 | SEND_CM_CREDIT_USED_STATUS_VL5_RETURN_CREDIT_STATUS_SMASK
11382 | SEND_CM_CREDIT_USED_STATUS_VL6_RETURN_CREDIT_STATUS_SMASK
11383 | SEND_CM_CREDIT_USED_STATUS_VL7_RETURN_CREDIT_STATUS_SMASK
11384 | SEND_CM_CREDIT_USED_STATUS_VL15_RETURN_CREDIT_STATUS_SMASK;
11385
11386#define valid_vl(idx) ((idx) < TXE_NUM_DATA_VL || (idx) == 15)
11387#define NUM_USABLE_VLS 16
11388
11389
11390 for (i = 0; i < OPA_MAX_VLS; i++) {
11391 if (valid_vl(i)) {
11392 new_total += be16_to_cpu(new_bc->vl[i].dedicated);
11393 continue;
11394 }
11395 nonzero_msg(dd, i, "dedicated",
11396 be16_to_cpu(new_bc->vl[i].dedicated));
11397 nonzero_msg(dd, i, "shared",
11398 be16_to_cpu(new_bc->vl[i].shared));
11399 new_bc->vl[i].dedicated = 0;
11400 new_bc->vl[i].shared = 0;
11401 }
11402 new_total += be16_to_cpu(new_bc->overall_shared_limit);
11403
11404
11405 get_buffer_control(dd, &cur_bc, &cur_total);
11406
11407
11408
11409
11410 memset(changing, 0, sizeof(changing));
11411 memset(lowering_dedicated, 0, sizeof(lowering_dedicated));
11412
11413
11414
11415
11416 stat_mask =
11417 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK;
11418 changing_mask = 0;
11419 ld_mask = 0;
11420 change_count = 0;
11421 any_shared_limit_changing = 0;
11422 for (i = 0; i < NUM_USABLE_VLS; i++, stat_mask <<= 1) {
11423 if (!valid_vl(i))
11424 continue;
11425 this_shared_changing = new_bc->vl[i].shared
11426 != cur_bc.vl[i].shared;
11427 if (this_shared_changing)
11428 any_shared_limit_changing = 1;
11429 if (new_bc->vl[i].dedicated != cur_bc.vl[i].dedicated ||
11430 this_shared_changing) {
11431 changing[i] = 1;
11432 changing_mask |= stat_mask;
11433 change_count++;
11434 }
11435 if (be16_to_cpu(new_bc->vl[i].dedicated) <
11436 be16_to_cpu(cur_bc.vl[i].dedicated)) {
11437 lowering_dedicated[i] = 1;
11438 ld_mask |= stat_mask;
11439 }
11440 }
11441
11442
11443 if (new_total > cur_total)
11444 set_global_limit(dd, new_total);
11445
11446
11447
11448
11449 use_all_mask = 0;
11450 if ((be16_to_cpu(new_bc->overall_shared_limit) <
11451 be16_to_cpu(cur_bc.overall_shared_limit)) ||
11452 (is_ax(dd) && any_shared_limit_changing)) {
11453 set_global_shared(dd, 0);
11454 cur_bc.overall_shared_limit = 0;
11455 use_all_mask = 1;
11456 }
11457
11458 for (i = 0; i < NUM_USABLE_VLS; i++) {
11459 if (!valid_vl(i))
11460 continue;
11461
11462 if (changing[i]) {
11463 set_vl_shared(dd, i, 0);
11464 cur_bc.vl[i].shared = 0;
11465 }
11466 }
11467
11468 wait_for_vl_status_clear(dd, use_all_mask ? all_mask : changing_mask,
11469 "shared");
11470
11471 if (change_count > 0) {
11472 for (i = 0; i < NUM_USABLE_VLS; i++) {
11473 if (!valid_vl(i))
11474 continue;
11475
11476 if (lowering_dedicated[i]) {
11477 set_vl_dedicated(dd, i,
11478 be16_to_cpu(new_bc->
11479 vl[i].dedicated));
11480 cur_bc.vl[i].dedicated =
11481 new_bc->vl[i].dedicated;
11482 }
11483 }
11484
11485 wait_for_vl_status_clear(dd, ld_mask, "dedicated");
11486
11487
11488 for (i = 0; i < NUM_USABLE_VLS; i++) {
11489 if (!valid_vl(i))
11490 continue;
11491
11492 if (be16_to_cpu(new_bc->vl[i].dedicated) >
11493 be16_to_cpu(cur_bc.vl[i].dedicated))
11494 set_vl_dedicated(dd, i,
11495 be16_to_cpu(new_bc->
11496 vl[i].dedicated));
11497 }
11498 }
11499
11500
11501 for (i = 0; i < NUM_USABLE_VLS; i++) {
11502 if (!valid_vl(i))
11503 continue;
11504
11505 if (be16_to_cpu(new_bc->vl[i].shared) >
11506 be16_to_cpu(cur_bc.vl[i].shared))
11507 set_vl_shared(dd, i, be16_to_cpu(new_bc->vl[i].shared));
11508 }
11509
11510
11511 if (be16_to_cpu(new_bc->overall_shared_limit) >
11512 be16_to_cpu(cur_bc.overall_shared_limit))
11513 set_global_shared(dd,
11514 be16_to_cpu(new_bc->overall_shared_limit));
11515
11516
11517 if (new_total < cur_total)
11518 set_global_limit(dd, new_total);
11519
11520
11521
11522
11523
11524 if (change_count > 0) {
11525 for (i = 0; i < TXE_NUM_DATA_VL; i++)
11526 if (be16_to_cpu(new_bc->vl[i].dedicated) > 0 ||
11527 be16_to_cpu(new_bc->vl[i].shared) > 0)
11528 vl_count++;
11529 ppd->actual_vls_operational = vl_count;
11530 ret = sdma_map_init(dd, ppd->port - 1, vl_count ?
11531 ppd->actual_vls_operational :
11532 ppd->vls_operational,
11533 NULL);
11534 if (ret == 0)
11535 ret = pio_map_init(dd, ppd->port - 1, vl_count ?
11536 ppd->actual_vls_operational :
11537 ppd->vls_operational, NULL);
11538 if (ret)
11539 return ret;
11540 }
11541 return 0;
11542}
11543
11544
11545
11546
11547
11548
11549int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t)
11550
11551{
11552 int size;
11553 struct vl_arb_cache *vlc;
11554
11555 switch (which) {
11556 case FM_TBL_VL_HIGH_ARB:
11557 size = 256;
11558
11559
11560
11561
11562 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
11563 vl_arb_get_cache(vlc, t);
11564 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11565 break;
11566 case FM_TBL_VL_LOW_ARB:
11567 size = 256;
11568
11569
11570
11571
11572 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
11573 vl_arb_get_cache(vlc, t);
11574 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11575 break;
11576 case FM_TBL_BUFFER_CONTROL:
11577 size = get_buffer_control(ppd->dd, t, NULL);
11578 break;
11579 case FM_TBL_SC2VLNT:
11580 size = get_sc2vlnt(ppd->dd, t);
11581 break;
11582 case FM_TBL_VL_PREEMPT_ELEMS:
11583 size = 256;
11584
11585 get_vlarb_preempt(ppd->dd, OPA_MAX_VLS, t);
11586 break;
11587 case FM_TBL_VL_PREEMPT_MATRIX:
11588 size = 256;
11589
11590
11591
11592
11593 break;
11594 default:
11595 return -EINVAL;
11596 }
11597 return size;
11598}
11599
11600
11601
11602
11603int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t)
11604{
11605 int ret = 0;
11606 struct vl_arb_cache *vlc;
11607
11608 switch (which) {
11609 case FM_TBL_VL_HIGH_ARB:
11610 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
11611 if (vl_arb_match_cache(vlc, t)) {
11612 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11613 break;
11614 }
11615 vl_arb_set_cache(vlc, t);
11616 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11617 ret = set_vl_weights(ppd, SEND_HIGH_PRIORITY_LIST,
11618 VL_ARB_HIGH_PRIO_TABLE_SIZE, t);
11619 break;
11620 case FM_TBL_VL_LOW_ARB:
11621 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
11622 if (vl_arb_match_cache(vlc, t)) {
11623 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11624 break;
11625 }
11626 vl_arb_set_cache(vlc, t);
11627 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11628 ret = set_vl_weights(ppd, SEND_LOW_PRIORITY_LIST,
11629 VL_ARB_LOW_PRIO_TABLE_SIZE, t);
11630 break;
11631 case FM_TBL_BUFFER_CONTROL:
11632 ret = set_buffer_control(ppd, t);
11633 break;
11634 case FM_TBL_SC2VLNT:
11635 set_sc2vlnt(ppd->dd, t);
11636 break;
11637 default:
11638 ret = -EINVAL;
11639 }
11640 return ret;
11641}
11642
11643
11644
11645
11646
11647
11648static int disable_data_vls(struct hfi1_devdata *dd)
11649{
11650 if (is_ax(dd))
11651 return 1;
11652
11653 pio_send_control(dd, PSC_DATA_VL_DISABLE);
11654
11655 return 0;
11656}
11657
11658
11659
11660
11661
11662
11663
11664
11665
11666int open_fill_data_vls(struct hfi1_devdata *dd)
11667{
11668 if (is_ax(dd))
11669 return 1;
11670
11671 pio_send_control(dd, PSC_DATA_VL_ENABLE);
11672
11673 return 0;
11674}
11675
11676
11677
11678
11679
11680
11681static void drain_data_vls(struct hfi1_devdata *dd)
11682{
11683 sc_wait(dd);
11684 sdma_wait(dd);
11685 pause_for_credit_return(dd);
11686}
11687
11688
11689
11690
11691
11692
11693
11694
11695
11696
11697
11698int stop_drain_data_vls(struct hfi1_devdata *dd)
11699{
11700 int ret;
11701
11702 ret = disable_data_vls(dd);
11703 if (ret == 0)
11704 drain_data_vls(dd);
11705
11706 return ret;
11707}
11708
11709
11710
11711
11712
11713u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns)
11714{
11715 u32 cclocks;
11716
11717 if (dd->icode == ICODE_FPGA_EMULATION)
11718 cclocks = (ns * 1000) / FPGA_CCLOCK_PS;
11719 else
11720 cclocks = (ns * 1000) / ASIC_CCLOCK_PS;
11721 if (ns && !cclocks)
11722 cclocks = 1;
11723 return cclocks;
11724}
11725
11726
11727
11728
11729
11730u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks)
11731{
11732 u32 ns;
11733
11734 if (dd->icode == ICODE_FPGA_EMULATION)
11735 ns = (cclocks * FPGA_CCLOCK_PS) / 1000;
11736 else
11737 ns = (cclocks * ASIC_CCLOCK_PS) / 1000;
11738 if (cclocks && !ns)
11739 ns = 1;
11740 return ns;
11741}
11742
11743
11744
11745
11746
11747
11748
11749static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts)
11750{
11751 struct hfi1_devdata *dd = rcd->dd;
11752 u32 timeout = rcd->rcvavail_timeout;
11753
11754
11755
11756
11757
11758
11759
11760
11761
11762
11763 if (npkts < rcv_intr_count) {
11764
11765
11766
11767
11768 if (timeout < 2)
11769 return;
11770 timeout >>= 1;
11771 } else {
11772
11773
11774
11775
11776 if (timeout >= dd->rcv_intr_timeout_csr)
11777 return;
11778 timeout = min(timeout << 1, dd->rcv_intr_timeout_csr);
11779 }
11780
11781 rcd->rcvavail_timeout = timeout;
11782
11783
11784
11785
11786 write_kctxt_csr(dd, rcd->ctxt, RCV_AVAIL_TIME_OUT,
11787 (u64)timeout <<
11788 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
11789}
11790
11791void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd,
11792 u32 intr_adjust, u32 npkts)
11793{
11794 struct hfi1_devdata *dd = rcd->dd;
11795 u64 reg;
11796 u32 ctxt = rcd->ctxt;
11797
11798
11799
11800
11801
11802 if (intr_adjust)
11803 adjust_rcv_timeout(rcd, npkts);
11804 if (updegr) {
11805 reg = (egrhd & RCV_EGR_INDEX_HEAD_HEAD_MASK)
11806 << RCV_EGR_INDEX_HEAD_HEAD_SHIFT;
11807 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, reg);
11808 }
11809 reg = ((u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT) |
11810 (((u64)hd & RCV_HDR_HEAD_HEAD_MASK)
11811 << RCV_HDR_HEAD_HEAD_SHIFT);
11812 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
11813}
11814
11815u32 hdrqempty(struct hfi1_ctxtdata *rcd)
11816{
11817 u32 head, tail;
11818
11819 head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD)
11820 & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT;
11821
11822 if (rcd->rcvhdrtail_kvaddr)
11823 tail = get_rcvhdrtail(rcd);
11824 else
11825 tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
11826
11827 return head == tail;
11828}
11829
11830
11831
11832
11833
11834
11835
11836
11837
11838
11839
11840
11841
11842
11843
11844
11845
11846
11847
11848
11849static u32 encoded_size(u32 size)
11850{
11851 switch (size) {
11852 case 4 * 1024: return 0x1;
11853 case 8 * 1024: return 0x2;
11854 case 16 * 1024: return 0x3;
11855 case 32 * 1024: return 0x4;
11856 case 64 * 1024: return 0x5;
11857 case 128 * 1024: return 0x6;
11858 case 256 * 1024: return 0x7;
11859 case 512 * 1024: return 0x8;
11860 case 1 * 1024 * 1024: return 0x9;
11861 case 2 * 1024 * 1024: return 0xa;
11862 }
11863 return 0x1;
11864}
11865
11866void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op,
11867 struct hfi1_ctxtdata *rcd)
11868{
11869 u64 rcvctrl, reg;
11870 int did_enable = 0;
11871 u16 ctxt;
11872
11873 if (!rcd)
11874 return;
11875
11876 ctxt = rcd->ctxt;
11877
11878 hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op);
11879
11880 rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL);
11881
11882 if ((op & HFI1_RCVCTRL_CTXT_ENB) &&
11883 !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) {
11884
11885 write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR,
11886 rcd->rcvhdrq_dma);
11887 if (rcd->rcvhdrtail_kvaddr)
11888 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11889 rcd->rcvhdrqtailaddr_dma);
11890 rcd->seq_cnt = 1;
11891
11892
11893 rcd->head = 0;
11894
11895
11896
11897
11898
11899
11900
11901 memset(rcd->rcvhdrq, 0, rcvhdrq_size(rcd));
11902
11903
11904 rcd->rcvavail_timeout = dd->rcv_intr_timeout_csr;
11905
11906
11907 rcvctrl |= RCV_CTXT_CTRL_ENABLE_SMASK;
11908
11909
11910 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
11911 rcvctrl |= ((u64)encoded_size(rcd->egrbufs.rcvtid_size)
11912 & RCV_CTXT_CTRL_EGR_BUF_SIZE_MASK)
11913 << RCV_CTXT_CTRL_EGR_BUF_SIZE_SHIFT;
11914
11915
11916 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0);
11917 did_enable = 1;
11918
11919
11920 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, 0);
11921
11922
11923 reg = (((u64)(rcd->egrbufs.alloced >> RCV_SHIFT)
11924 & RCV_EGR_CTRL_EGR_CNT_MASK)
11925 << RCV_EGR_CTRL_EGR_CNT_SHIFT) |
11926 (((rcd->eager_base >> RCV_SHIFT)
11927 & RCV_EGR_CTRL_EGR_BASE_INDEX_MASK)
11928 << RCV_EGR_CTRL_EGR_BASE_INDEX_SHIFT);
11929 write_kctxt_csr(dd, ctxt, RCV_EGR_CTRL, reg);
11930
11931
11932
11933
11934
11935
11936
11937 reg = (((rcd->expected_count >> RCV_SHIFT)
11938 & RCV_TID_CTRL_TID_PAIR_CNT_MASK)
11939 << RCV_TID_CTRL_TID_PAIR_CNT_SHIFT) |
11940 (((rcd->expected_base >> RCV_SHIFT)
11941 & RCV_TID_CTRL_TID_BASE_INDEX_MASK)
11942 << RCV_TID_CTRL_TID_BASE_INDEX_SHIFT);
11943 write_kctxt_csr(dd, ctxt, RCV_TID_CTRL, reg);
11944 if (ctxt == HFI1_CTRL_CTXT)
11945 write_csr(dd, RCV_VL15, HFI1_CTRL_CTXT);
11946 }
11947 if (op & HFI1_RCVCTRL_CTXT_DIS) {
11948 write_csr(dd, RCV_VL15, 0);
11949
11950
11951
11952
11953
11954 if (dd->rcvhdrtail_dummy_dma) {
11955 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11956 dd->rcvhdrtail_dummy_dma);
11957
11958 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11959 }
11960
11961 rcvctrl &= ~RCV_CTXT_CTRL_ENABLE_SMASK;
11962 }
11963 if (op & HFI1_RCVCTRL_INTRAVAIL_ENB) {
11964 set_intr_bits(dd, IS_RCVAVAIL_START + rcd->ctxt,
11965 IS_RCVAVAIL_START + rcd->ctxt, true);
11966 rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
11967 }
11968 if (op & HFI1_RCVCTRL_INTRAVAIL_DIS) {
11969 set_intr_bits(dd, IS_RCVAVAIL_START + rcd->ctxt,
11970 IS_RCVAVAIL_START + rcd->ctxt, false);
11971 rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
11972 }
11973 if ((op & HFI1_RCVCTRL_TAILUPD_ENB) && rcd->rcvhdrtail_kvaddr)
11974 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11975 if (op & HFI1_RCVCTRL_TAILUPD_DIS) {
11976
11977 if (!(op & HFI1_RCVCTRL_CTXT_DIS))
11978 rcvctrl &= ~RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11979 }
11980 if (op & HFI1_RCVCTRL_TIDFLOW_ENB)
11981 rcvctrl |= RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
11982 if (op & HFI1_RCVCTRL_TIDFLOW_DIS)
11983 rcvctrl &= ~RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
11984 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_ENB) {
11985
11986
11987
11988
11989 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
11990 rcvctrl |= RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
11991 }
11992 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_DIS)
11993 rcvctrl &= ~RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
11994 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_ENB)
11995 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
11996 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_DIS)
11997 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
11998 if (op & HFI1_RCVCTRL_NO_EGR_DROP_ENB)
11999 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
12000 if (op & HFI1_RCVCTRL_NO_EGR_DROP_DIS)
12001 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
12002 if (op & HFI1_RCVCTRL_URGENT_ENB)
12003 set_intr_bits(dd, IS_RCVURGENT_START + rcd->ctxt,
12004 IS_RCVURGENT_START + rcd->ctxt, true);
12005 if (op & HFI1_RCVCTRL_URGENT_DIS)
12006 set_intr_bits(dd, IS_RCVURGENT_START + rcd->ctxt,
12007 IS_RCVURGENT_START + rcd->ctxt, false);
12008
12009 hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl);
12010 write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcvctrl);
12011
12012
12013 if (did_enable &&
12014 (rcvctrl & RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK)) {
12015 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
12016 if (reg != 0) {
12017 dd_dev_info(dd, "ctxt %d status %lld (blocked)\n",
12018 ctxt, reg);
12019 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
12020 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x10);
12021 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x00);
12022 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
12023 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
12024 dd_dev_info(dd, "ctxt %d status %lld (%s blocked)\n",
12025 ctxt, reg, reg == 0 ? "not" : "still");
12026 }
12027 }
12028
12029 if (did_enable) {
12030
12031
12032
12033
12034
12035 write_kctxt_csr(dd, ctxt, RCV_AVAIL_TIME_OUT,
12036 (u64)rcd->rcvavail_timeout <<
12037 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
12038
12039
12040 reg = (u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT;
12041 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
12042 }
12043
12044 if (op & (HFI1_RCVCTRL_TAILUPD_DIS | HFI1_RCVCTRL_CTXT_DIS))
12045
12046
12047
12048
12049
12050 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
12051 dd->rcvhdrtail_dummy_dma);
12052}
12053
12054u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp)
12055{
12056 int ret;
12057 u64 val = 0;
12058
12059 if (namep) {
12060 ret = dd->cntrnameslen;
12061 *namep = dd->cntrnames;
12062 } else {
12063 const struct cntr_entry *entry;
12064 int i, j;
12065
12066 ret = (dd->ndevcntrs) * sizeof(u64);
12067
12068
12069 *cntrp = dd->cntrs;
12070
12071
12072
12073
12074 for (i = 0; i < DEV_CNTR_LAST; i++) {
12075 entry = &dev_cntrs[i];
12076 hfi1_cdbg(CNTR, "reading %s", entry->name);
12077 if (entry->flags & CNTR_DISABLED) {
12078
12079 hfi1_cdbg(CNTR, "\tDisabled\n");
12080 } else {
12081 if (entry->flags & CNTR_VL) {
12082 hfi1_cdbg(CNTR, "\tPer VL\n");
12083 for (j = 0; j < C_VL_COUNT; j++) {
12084 val = entry->rw_cntr(entry,
12085 dd, j,
12086 CNTR_MODE_R,
12087 0);
12088 hfi1_cdbg(
12089 CNTR,
12090 "\t\tRead 0x%llx for %d\n",
12091 val, j);
12092 dd->cntrs[entry->offset + j] =
12093 val;
12094 }
12095 } else if (entry->flags & CNTR_SDMA) {
12096 hfi1_cdbg(CNTR,
12097 "\t Per SDMA Engine\n");
12098 for (j = 0; j < chip_sdma_engines(dd);
12099 j++) {
12100 val =
12101 entry->rw_cntr(entry, dd, j,
12102 CNTR_MODE_R, 0);
12103 hfi1_cdbg(CNTR,
12104 "\t\tRead 0x%llx for %d\n",
12105 val, j);
12106 dd->cntrs[entry->offset + j] =
12107 val;
12108 }
12109 } else {
12110 val = entry->rw_cntr(entry, dd,
12111 CNTR_INVALID_VL,
12112 CNTR_MODE_R, 0);
12113 dd->cntrs[entry->offset] = val;
12114 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
12115 }
12116 }
12117 }
12118 }
12119 return ret;
12120}
12121
12122
12123
12124
12125u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp)
12126{
12127 int ret;
12128 u64 val = 0;
12129
12130 if (namep) {
12131 ret = ppd->dd->portcntrnameslen;
12132 *namep = ppd->dd->portcntrnames;
12133 } else {
12134 const struct cntr_entry *entry;
12135 int i, j;
12136
12137 ret = ppd->dd->nportcntrs * sizeof(u64);
12138 *cntrp = ppd->cntrs;
12139
12140 for (i = 0; i < PORT_CNTR_LAST; i++) {
12141 entry = &port_cntrs[i];
12142 hfi1_cdbg(CNTR, "reading %s", entry->name);
12143 if (entry->flags & CNTR_DISABLED) {
12144
12145 hfi1_cdbg(CNTR, "\tDisabled\n");
12146 continue;
12147 }
12148
12149 if (entry->flags & CNTR_VL) {
12150 hfi1_cdbg(CNTR, "\tPer VL");
12151 for (j = 0; j < C_VL_COUNT; j++) {
12152 val = entry->rw_cntr(entry, ppd, j,
12153 CNTR_MODE_R,
12154 0);
12155 hfi1_cdbg(
12156 CNTR,
12157 "\t\tRead 0x%llx for %d",
12158 val, j);
12159 ppd->cntrs[entry->offset + j] = val;
12160 }
12161 } else {
12162 val = entry->rw_cntr(entry, ppd,
12163 CNTR_INVALID_VL,
12164 CNTR_MODE_R,
12165 0);
12166 ppd->cntrs[entry->offset] = val;
12167 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
12168 }
12169 }
12170 }
12171 return ret;
12172}
12173
12174static void free_cntrs(struct hfi1_devdata *dd)
12175{
12176 struct hfi1_pportdata *ppd;
12177 int i;
12178
12179 if (dd->synth_stats_timer.function)
12180 del_timer_sync(&dd->synth_stats_timer);
12181 ppd = (struct hfi1_pportdata *)(dd + 1);
12182 for (i = 0; i < dd->num_pports; i++, ppd++) {
12183 kfree(ppd->cntrs);
12184 kfree(ppd->scntrs);
12185 free_percpu(ppd->ibport_data.rvp.rc_acks);
12186 free_percpu(ppd->ibport_data.rvp.rc_qacks);
12187 free_percpu(ppd->ibport_data.rvp.rc_delayed_comp);
12188 ppd->cntrs = NULL;
12189 ppd->scntrs = NULL;
12190 ppd->ibport_data.rvp.rc_acks = NULL;
12191 ppd->ibport_data.rvp.rc_qacks = NULL;
12192 ppd->ibport_data.rvp.rc_delayed_comp = NULL;
12193 }
12194 kfree(dd->portcntrnames);
12195 dd->portcntrnames = NULL;
12196 kfree(dd->cntrs);
12197 dd->cntrs = NULL;
12198 kfree(dd->scntrs);
12199 dd->scntrs = NULL;
12200 kfree(dd->cntrnames);
12201 dd->cntrnames = NULL;
12202 if (dd->update_cntr_wq) {
12203 destroy_workqueue(dd->update_cntr_wq);
12204 dd->update_cntr_wq = NULL;
12205 }
12206}
12207
12208static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry,
12209 u64 *psval, void *context, int vl)
12210{
12211 u64 val;
12212 u64 sval = *psval;
12213
12214 if (entry->flags & CNTR_DISABLED) {
12215 dd_dev_err(dd, "Counter %s not enabled", entry->name);
12216 return 0;
12217 }
12218
12219 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
12220
12221 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_R, 0);
12222
12223
12224 if (entry->flags & CNTR_SYNTH) {
12225 if (sval == CNTR_MAX) {
12226
12227 return CNTR_MAX;
12228 }
12229
12230 if (entry->flags & CNTR_32BIT) {
12231
12232 u64 upper = sval >> 32;
12233 u64 lower = (sval << 32) >> 32;
12234
12235 if (lower > val) {
12236 if (upper == CNTR_32BIT_MAX)
12237 val = CNTR_MAX;
12238 else
12239 upper++;
12240 }
12241
12242 if (val != CNTR_MAX)
12243 val = (upper << 32) | val;
12244
12245 } else {
12246
12247 if ((val < sval) || (val > CNTR_MAX))
12248 val = CNTR_MAX;
12249 }
12250 }
12251
12252 *psval = val;
12253
12254 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
12255
12256 return val;
12257}
12258
12259static u64 write_dev_port_cntr(struct hfi1_devdata *dd,
12260 struct cntr_entry *entry,
12261 u64 *psval, void *context, int vl, u64 data)
12262{
12263 u64 val;
12264
12265 if (entry->flags & CNTR_DISABLED) {
12266 dd_dev_err(dd, "Counter %s not enabled", entry->name);
12267 return 0;
12268 }
12269
12270 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
12271
12272 if (entry->flags & CNTR_SYNTH) {
12273 *psval = data;
12274 if (entry->flags & CNTR_32BIT) {
12275 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
12276 (data << 32) >> 32);
12277 val = data;
12278 } else {
12279 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
12280 data);
12281 }
12282 } else {
12283 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, data);
12284 }
12285
12286 *psval = val;
12287
12288 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
12289
12290 return val;
12291}
12292
12293u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl)
12294{
12295 struct cntr_entry *entry;
12296 u64 *sval;
12297
12298 entry = &dev_cntrs[index];
12299 sval = dd->scntrs + entry->offset;
12300
12301 if (vl != CNTR_INVALID_VL)
12302 sval += vl;
12303
12304 return read_dev_port_cntr(dd, entry, sval, dd, vl);
12305}
12306
12307u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data)
12308{
12309 struct cntr_entry *entry;
12310 u64 *sval;
12311
12312 entry = &dev_cntrs[index];
12313 sval = dd->scntrs + entry->offset;
12314
12315 if (vl != CNTR_INVALID_VL)
12316 sval += vl;
12317
12318 return write_dev_port_cntr(dd, entry, sval, dd, vl, data);
12319}
12320
12321u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl)
12322{
12323 struct cntr_entry *entry;
12324 u64 *sval;
12325
12326 entry = &port_cntrs[index];
12327 sval = ppd->scntrs + entry->offset;
12328
12329 if (vl != CNTR_INVALID_VL)
12330 sval += vl;
12331
12332 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
12333 (index <= C_RCV_HDR_OVF_LAST)) {
12334
12335 return 0;
12336 }
12337
12338 return read_dev_port_cntr(ppd->dd, entry, sval, ppd, vl);
12339}
12340
12341u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data)
12342{
12343 struct cntr_entry *entry;
12344 u64 *sval;
12345
12346 entry = &port_cntrs[index];
12347 sval = ppd->scntrs + entry->offset;
12348
12349 if (vl != CNTR_INVALID_VL)
12350 sval += vl;
12351
12352 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
12353 (index <= C_RCV_HDR_OVF_LAST)) {
12354
12355 return 0;
12356 }
12357
12358 return write_dev_port_cntr(ppd->dd, entry, sval, ppd, vl, data);
12359}
12360
12361static void do_update_synth_timer(struct work_struct *work)
12362{
12363 u64 cur_tx;
12364 u64 cur_rx;
12365 u64 total_flits;
12366 u8 update = 0;
12367 int i, j, vl;
12368 struct hfi1_pportdata *ppd;
12369 struct cntr_entry *entry;
12370 struct hfi1_devdata *dd = container_of(work, struct hfi1_devdata,
12371 update_cntr_work);
12372
12373
12374
12375
12376
12377
12378
12379 entry = &dev_cntrs[C_DC_RCV_FLITS];
12380 cur_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
12381
12382 entry = &dev_cntrs[C_DC_XMIT_FLITS];
12383 cur_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
12384
12385 hfi1_cdbg(
12386 CNTR,
12387 "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n",
12388 dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx);
12389
12390 if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) {
12391
12392
12393
12394
12395 update = 1;
12396 hfi1_cdbg(CNTR, "[%d] Tripwire counter rolled, updating",
12397 dd->unit);
12398 } else {
12399 total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx);
12400 hfi1_cdbg(CNTR,
12401 "[%d] total flits 0x%llx limit 0x%llx\n", dd->unit,
12402 total_flits, (u64)CNTR_32BIT_MAX);
12403 if (total_flits >= CNTR_32BIT_MAX) {
12404 hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating",
12405 dd->unit);
12406 update = 1;
12407 }
12408 }
12409
12410 if (update) {
12411 hfi1_cdbg(CNTR, "[%d] Updating dd and ppd counters", dd->unit);
12412 for (i = 0; i < DEV_CNTR_LAST; i++) {
12413 entry = &dev_cntrs[i];
12414 if (entry->flags & CNTR_VL) {
12415 for (vl = 0; vl < C_VL_COUNT; vl++)
12416 read_dev_cntr(dd, i, vl);
12417 } else {
12418 read_dev_cntr(dd, i, CNTR_INVALID_VL);
12419 }
12420 }
12421 ppd = (struct hfi1_pportdata *)(dd + 1);
12422 for (i = 0; i < dd->num_pports; i++, ppd++) {
12423 for (j = 0; j < PORT_CNTR_LAST; j++) {
12424 entry = &port_cntrs[j];
12425 if (entry->flags & CNTR_VL) {
12426 for (vl = 0; vl < C_VL_COUNT; vl++)
12427 read_port_cntr(ppd, j, vl);
12428 } else {
12429 read_port_cntr(ppd, j, CNTR_INVALID_VL);
12430 }
12431 }
12432 }
12433
12434
12435
12436
12437
12438
12439
12440 entry = &dev_cntrs[C_DC_XMIT_FLITS];
12441 dd->last_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
12442 CNTR_MODE_R, 0);
12443
12444 entry = &dev_cntrs[C_DC_RCV_FLITS];
12445 dd->last_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
12446 CNTR_MODE_R, 0);
12447
12448 hfi1_cdbg(CNTR, "[%d] setting last tx/rx to 0x%llx 0x%llx",
12449 dd->unit, dd->last_tx, dd->last_rx);
12450
12451 } else {
12452 hfi1_cdbg(CNTR, "[%d] No update necessary", dd->unit);
12453 }
12454}
12455
12456static void update_synth_timer(struct timer_list *t)
12457{
12458 struct hfi1_devdata *dd = from_timer(dd, t, synth_stats_timer);
12459
12460 queue_work(dd->update_cntr_wq, &dd->update_cntr_work);
12461 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
12462}
12463
12464#define C_MAX_NAME 16
12465static int init_cntrs(struct hfi1_devdata *dd)
12466{
12467 int i, rcv_ctxts, j;
12468 size_t sz;
12469 char *p;
12470 char name[C_MAX_NAME];
12471 struct hfi1_pportdata *ppd;
12472 const char *bit_type_32 = ",32";
12473 const int bit_type_32_sz = strlen(bit_type_32);
12474 u32 sdma_engines = chip_sdma_engines(dd);
12475
12476
12477 timer_setup(&dd->synth_stats_timer, update_synth_timer, 0);
12478
12479
12480
12481
12482
12483
12484 dd->ndevcntrs = 0;
12485 sz = 0;
12486
12487 for (i = 0; i < DEV_CNTR_LAST; i++) {
12488 if (dev_cntrs[i].flags & CNTR_DISABLED) {
12489 hfi1_dbg_early("\tSkipping %s\n", dev_cntrs[i].name);
12490 continue;
12491 }
12492
12493 if (dev_cntrs[i].flags & CNTR_VL) {
12494 dev_cntrs[i].offset = dd->ndevcntrs;
12495 for (j = 0; j < C_VL_COUNT; j++) {
12496 snprintf(name, C_MAX_NAME, "%s%d",
12497 dev_cntrs[i].name, vl_from_idx(j));
12498 sz += strlen(name);
12499
12500 if (dev_cntrs[i].flags & CNTR_32BIT)
12501 sz += bit_type_32_sz;
12502 sz++;
12503 dd->ndevcntrs++;
12504 }
12505 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
12506 dev_cntrs[i].offset = dd->ndevcntrs;
12507 for (j = 0; j < sdma_engines; j++) {
12508 snprintf(name, C_MAX_NAME, "%s%d",
12509 dev_cntrs[i].name, j);
12510 sz += strlen(name);
12511
12512 if (dev_cntrs[i].flags & CNTR_32BIT)
12513 sz += bit_type_32_sz;
12514 sz++;
12515 dd->ndevcntrs++;
12516 }
12517 } else {
12518
12519 sz += strlen(dev_cntrs[i].name) + 1;
12520
12521 if (dev_cntrs[i].flags & CNTR_32BIT)
12522 sz += bit_type_32_sz;
12523 dev_cntrs[i].offset = dd->ndevcntrs;
12524 dd->ndevcntrs++;
12525 }
12526 }
12527
12528
12529 dd->cntrs = kcalloc(dd->ndevcntrs + num_driver_cntrs, sizeof(u64),
12530 GFP_KERNEL);
12531 if (!dd->cntrs)
12532 goto bail;
12533
12534 dd->scntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL);
12535 if (!dd->scntrs)
12536 goto bail;
12537
12538
12539 dd->cntrnameslen = sz;
12540 dd->cntrnames = kmalloc(sz, GFP_KERNEL);
12541 if (!dd->cntrnames)
12542 goto bail;
12543
12544
12545 for (p = dd->cntrnames, i = 0; i < DEV_CNTR_LAST; i++) {
12546 if (dev_cntrs[i].flags & CNTR_DISABLED) {
12547
12548 } else if (dev_cntrs[i].flags & CNTR_VL) {
12549 for (j = 0; j < C_VL_COUNT; j++) {
12550 snprintf(name, C_MAX_NAME, "%s%d",
12551 dev_cntrs[i].name,
12552 vl_from_idx(j));
12553 memcpy(p, name, strlen(name));
12554 p += strlen(name);
12555
12556
12557 if (dev_cntrs[i].flags & CNTR_32BIT) {
12558 memcpy(p, bit_type_32, bit_type_32_sz);
12559 p += bit_type_32_sz;
12560 }
12561
12562 *p++ = '\n';
12563 }
12564 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
12565 for (j = 0; j < sdma_engines; j++) {
12566 snprintf(name, C_MAX_NAME, "%s%d",
12567 dev_cntrs[i].name, j);
12568 memcpy(p, name, strlen(name));
12569 p += strlen(name);
12570
12571
12572 if (dev_cntrs[i].flags & CNTR_32BIT) {
12573 memcpy(p, bit_type_32, bit_type_32_sz);
12574 p += bit_type_32_sz;
12575 }
12576
12577 *p++ = '\n';
12578 }
12579 } else {
12580 memcpy(p, dev_cntrs[i].name, strlen(dev_cntrs[i].name));
12581 p += strlen(dev_cntrs[i].name);
12582
12583
12584 if (dev_cntrs[i].flags & CNTR_32BIT) {
12585 memcpy(p, bit_type_32, bit_type_32_sz);
12586 p += bit_type_32_sz;
12587 }
12588
12589 *p++ = '\n';
12590 }
12591 }
12592
12593
12594
12595
12596
12597
12598
12599
12600
12601
12602 rcv_ctxts = dd->num_rcv_contexts;
12603 for (i = C_RCV_HDR_OVF_FIRST + rcv_ctxts;
12604 i <= C_RCV_HDR_OVF_LAST; i++) {
12605 port_cntrs[i].flags |= CNTR_DISABLED;
12606 }
12607
12608
12609 sz = 0;
12610 dd->nportcntrs = 0;
12611 for (i = 0; i < PORT_CNTR_LAST; i++) {
12612 if (port_cntrs[i].flags & CNTR_DISABLED) {
12613 hfi1_dbg_early("\tSkipping %s\n", port_cntrs[i].name);
12614 continue;
12615 }
12616
12617 if (port_cntrs[i].flags & CNTR_VL) {
12618 port_cntrs[i].offset = dd->nportcntrs;
12619 for (j = 0; j < C_VL_COUNT; j++) {
12620 snprintf(name, C_MAX_NAME, "%s%d",
12621 port_cntrs[i].name, vl_from_idx(j));
12622 sz += strlen(name);
12623
12624 if (port_cntrs[i].flags & CNTR_32BIT)
12625 sz += bit_type_32_sz;
12626 sz++;
12627 dd->nportcntrs++;
12628 }
12629 } else {
12630
12631 sz += strlen(port_cntrs[i].name) + 1;
12632
12633 if (port_cntrs[i].flags & CNTR_32BIT)
12634 sz += bit_type_32_sz;
12635 port_cntrs[i].offset = dd->nportcntrs;
12636 dd->nportcntrs++;
12637 }
12638 }
12639
12640
12641 dd->portcntrnameslen = sz;
12642 dd->portcntrnames = kmalloc(sz, GFP_KERNEL);
12643 if (!dd->portcntrnames)
12644 goto bail;
12645
12646
12647 for (p = dd->portcntrnames, i = 0; i < PORT_CNTR_LAST; i++) {
12648 if (port_cntrs[i].flags & CNTR_DISABLED)
12649 continue;
12650
12651 if (port_cntrs[i].flags & CNTR_VL) {
12652 for (j = 0; j < C_VL_COUNT; j++) {
12653 snprintf(name, C_MAX_NAME, "%s%d",
12654 port_cntrs[i].name, vl_from_idx(j));
12655 memcpy(p, name, strlen(name));
12656 p += strlen(name);
12657
12658
12659 if (port_cntrs[i].flags & CNTR_32BIT) {
12660 memcpy(p, bit_type_32, bit_type_32_sz);
12661 p += bit_type_32_sz;
12662 }
12663
12664 *p++ = '\n';
12665 }
12666 } else {
12667 memcpy(p, port_cntrs[i].name,
12668 strlen(port_cntrs[i].name));
12669 p += strlen(port_cntrs[i].name);
12670
12671
12672 if (port_cntrs[i].flags & CNTR_32BIT) {
12673 memcpy(p, bit_type_32, bit_type_32_sz);
12674 p += bit_type_32_sz;
12675 }
12676
12677 *p++ = '\n';
12678 }
12679 }
12680
12681
12682 ppd = (struct hfi1_pportdata *)(dd + 1);
12683 for (i = 0; i < dd->num_pports; i++, ppd++) {
12684 ppd->cntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12685 if (!ppd->cntrs)
12686 goto bail;
12687
12688 ppd->scntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12689 if (!ppd->scntrs)
12690 goto bail;
12691 }
12692
12693
12694 if (init_cpu_counters(dd))
12695 goto bail;
12696
12697 dd->update_cntr_wq = alloc_ordered_workqueue("hfi1_update_cntr_%d",
12698 WQ_MEM_RECLAIM, dd->unit);
12699 if (!dd->update_cntr_wq)
12700 goto bail;
12701
12702 INIT_WORK(&dd->update_cntr_work, do_update_synth_timer);
12703
12704 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
12705 return 0;
12706bail:
12707 free_cntrs(dd);
12708 return -ENOMEM;
12709}
12710
12711static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate)
12712{
12713 switch (chip_lstate) {
12714 default:
12715 dd_dev_err(dd,
12716 "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",
12717 chip_lstate);
12718
12719 case LSTATE_DOWN:
12720 return IB_PORT_DOWN;
12721 case LSTATE_INIT:
12722 return IB_PORT_INIT;
12723 case LSTATE_ARMED:
12724 return IB_PORT_ARMED;
12725 case LSTATE_ACTIVE:
12726 return IB_PORT_ACTIVE;
12727 }
12728}
12729
12730u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate)
12731{
12732
12733 switch (chip_pstate & 0xf0) {
12734 default:
12735 dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",
12736 chip_pstate);
12737
12738 case PLS_DISABLED:
12739 return IB_PORTPHYSSTATE_DISABLED;
12740 case PLS_OFFLINE:
12741 return OPA_PORTPHYSSTATE_OFFLINE;
12742 case PLS_POLLING:
12743 return IB_PORTPHYSSTATE_POLLING;
12744 case PLS_CONFIGPHY:
12745 return IB_PORTPHYSSTATE_TRAINING;
12746 case PLS_LINKUP:
12747 return IB_PORTPHYSSTATE_LINKUP;
12748 case PLS_PHYTEST:
12749 return IB_PORTPHYSSTATE_PHY_TEST;
12750 }
12751}
12752
12753
12754const char *opa_lstate_name(u32 lstate)
12755{
12756 static const char * const port_logical_names[] = {
12757 "PORT_NOP",
12758 "PORT_DOWN",
12759 "PORT_INIT",
12760 "PORT_ARMED",
12761 "PORT_ACTIVE",
12762 "PORT_ACTIVE_DEFER",
12763 };
12764 if (lstate < ARRAY_SIZE(port_logical_names))
12765 return port_logical_names[lstate];
12766 return "unknown";
12767}
12768
12769
12770const char *opa_pstate_name(u32 pstate)
12771{
12772 static const char * const port_physical_names[] = {
12773 "PHYS_NOP",
12774 "reserved1",
12775 "PHYS_POLL",
12776 "PHYS_DISABLED",
12777 "PHYS_TRAINING",
12778 "PHYS_LINKUP",
12779 "PHYS_LINK_ERR_RECOVER",
12780 "PHYS_PHY_TEST",
12781 "reserved8",
12782 "PHYS_OFFLINE",
12783 "PHYS_GANGED",
12784 "PHYS_TEST",
12785 };
12786 if (pstate < ARRAY_SIZE(port_physical_names))
12787 return port_physical_names[pstate];
12788 return "unknown";
12789}
12790
12791
12792
12793
12794
12795
12796
12797
12798
12799
12800
12801
12802static void update_statusp(struct hfi1_pportdata *ppd, u32 state)
12803{
12804
12805
12806
12807
12808
12809
12810
12811
12812 if (ppd->statusp) {
12813 switch (state) {
12814 case IB_PORT_DOWN:
12815 case IB_PORT_INIT:
12816 *ppd->statusp &= ~(HFI1_STATUS_IB_CONF |
12817 HFI1_STATUS_IB_READY);
12818 break;
12819 case IB_PORT_ARMED:
12820 *ppd->statusp |= HFI1_STATUS_IB_CONF;
12821 break;
12822 case IB_PORT_ACTIVE:
12823 *ppd->statusp |= HFI1_STATUS_IB_READY;
12824 break;
12825 }
12826 }
12827 dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n",
12828 opa_lstate_name(state), state);
12829}
12830
12831
12832
12833
12834
12835
12836
12837
12838
12839
12840
12841static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
12842 int msecs)
12843{
12844 unsigned long timeout;
12845 u32 new_state;
12846
12847 timeout = jiffies + msecs_to_jiffies(msecs);
12848 while (1) {
12849 new_state = chip_to_opa_lstate(ppd->dd,
12850 read_logical_state(ppd->dd));
12851 if (new_state == state)
12852 break;
12853 if (time_after(jiffies, timeout)) {
12854 dd_dev_err(ppd->dd,
12855 "timeout waiting for link state 0x%x\n",
12856 state);
12857 return -ETIMEDOUT;
12858 }
12859 msleep(20);
12860 }
12861
12862 return 0;
12863}
12864
12865static void log_state_transition(struct hfi1_pportdata *ppd, u32 state)
12866{
12867 u32 ib_pstate = chip_to_opa_pstate(ppd->dd, state);
12868
12869 dd_dev_info(ppd->dd,
12870 "physical state changed to %s (0x%x), phy 0x%x\n",
12871 opa_pstate_name(ib_pstate), ib_pstate, state);
12872}
12873
12874
12875
12876
12877
12878static void log_physical_state(struct hfi1_pportdata *ppd, u32 state)
12879{
12880 u32 read_state = read_physical_state(ppd->dd);
12881
12882 if (read_state == state) {
12883 log_state_transition(ppd, state);
12884 } else {
12885 dd_dev_err(ppd->dd,
12886 "anticipated phy link state 0x%x, read 0x%x\n",
12887 state, read_state);
12888 }
12889}
12890
12891
12892
12893
12894
12895
12896
12897
12898
12899
12900static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state,
12901 int msecs)
12902{
12903 u32 read_state;
12904 unsigned long timeout;
12905
12906 timeout = jiffies + msecs_to_jiffies(msecs);
12907 while (1) {
12908 read_state = read_physical_state(ppd->dd);
12909 if (read_state == state)
12910 break;
12911 if (time_after(jiffies, timeout)) {
12912 dd_dev_err(ppd->dd,
12913 "timeout waiting for phy link state 0x%x\n",
12914 state);
12915 return -ETIMEDOUT;
12916 }
12917 usleep_range(1950, 2050);
12918 }
12919
12920 log_state_transition(ppd, state);
12921 return 0;
12922}
12923
12924
12925
12926
12927
12928
12929
12930
12931
12932
12933static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd,
12934 int msecs)
12935{
12936 u32 read_state;
12937 unsigned long timeout;
12938
12939 timeout = jiffies + msecs_to_jiffies(msecs);
12940 while (1) {
12941 read_state = read_physical_state(ppd->dd);
12942 if ((read_state & 0xF0) == PLS_OFFLINE)
12943 break;
12944 if (time_after(jiffies, timeout)) {
12945 dd_dev_err(ppd->dd,
12946 "timeout waiting for phy link offline.quiet substates. Read state 0x%x, %dms\n",
12947 read_state, msecs);
12948 return -ETIMEDOUT;
12949 }
12950 usleep_range(1950, 2050);
12951 }
12952
12953 log_state_transition(ppd, read_state);
12954 return read_state;
12955}
12956
12957
12958
12959
12960
12961
12962
12963
12964
12965
12966static int wait_phys_link_out_of_offline(struct hfi1_pportdata *ppd,
12967 int msecs)
12968{
12969 u32 read_state;
12970 unsigned long timeout;
12971
12972 timeout = jiffies + msecs_to_jiffies(msecs);
12973 while (1) {
12974 read_state = read_physical_state(ppd->dd);
12975 if ((read_state & 0xF0) != PLS_OFFLINE)
12976 break;
12977 if (time_after(jiffies, timeout)) {
12978 dd_dev_err(ppd->dd,
12979 "timeout waiting for phy link out of offline. Read state 0x%x, %dms\n",
12980 read_state, msecs);
12981 return -ETIMEDOUT;
12982 }
12983 usleep_range(1950, 2050);
12984 }
12985
12986 log_state_transition(ppd, read_state);
12987 return read_state;
12988}
12989
12990#define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \
12991(r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
12992
12993#define SET_STATIC_RATE_CONTROL_SMASK(r) \
12994(r |= SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
12995
12996void hfi1_init_ctxt(struct send_context *sc)
12997{
12998 if (sc) {
12999 struct hfi1_devdata *dd = sc->dd;
13000 u64 reg;
13001 u8 set = (sc->type == SC_USER ?
13002 HFI1_CAP_IS_USET(STATIC_RATE_CTRL) :
13003 HFI1_CAP_IS_KSET(STATIC_RATE_CTRL));
13004 reg = read_kctxt_csr(dd, sc->hw_context,
13005 SEND_CTXT_CHECK_ENABLE);
13006 if (set)
13007 CLEAR_STATIC_RATE_CONTROL_SMASK(reg);
13008 else
13009 SET_STATIC_RATE_CONTROL_SMASK(reg);
13010 write_kctxt_csr(dd, sc->hw_context,
13011 SEND_CTXT_CHECK_ENABLE, reg);
13012 }
13013}
13014
13015int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp)
13016{
13017 int ret = 0;
13018 u64 reg;
13019
13020 if (dd->icode != ICODE_RTL_SILICON) {
13021 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
13022 dd_dev_info(dd, "%s: tempsense not supported by HW\n",
13023 __func__);
13024 return -EINVAL;
13025 }
13026 reg = read_csr(dd, ASIC_STS_THERM);
13027 temp->curr = ((reg >> ASIC_STS_THERM_CURR_TEMP_SHIFT) &
13028 ASIC_STS_THERM_CURR_TEMP_MASK);
13029 temp->lo_lim = ((reg >> ASIC_STS_THERM_LO_TEMP_SHIFT) &
13030 ASIC_STS_THERM_LO_TEMP_MASK);
13031 temp->hi_lim = ((reg >> ASIC_STS_THERM_HI_TEMP_SHIFT) &
13032 ASIC_STS_THERM_HI_TEMP_MASK);
13033 temp->crit_lim = ((reg >> ASIC_STS_THERM_CRIT_TEMP_SHIFT) &
13034 ASIC_STS_THERM_CRIT_TEMP_MASK);
13035
13036 temp->triggers = (u8)((reg >> ASIC_STS_THERM_LOW_SHIFT) & 0x7);
13037
13038 return ret;
13039}
13040
13041
13042
13043
13044
13045
13046
13047
13048
13049
13050
13051static void read_mod_write(struct hfi1_devdata *dd, u16 src, u64 bits,
13052 bool set)
13053{
13054 u64 reg;
13055 u16 idx = src / BITS_PER_REGISTER;
13056
13057 spin_lock(&dd->irq_src_lock);
13058 reg = read_csr(dd, CCE_INT_MASK + (8 * idx));
13059 if (set)
13060 reg |= bits;
13061 else
13062 reg &= ~bits;
13063 write_csr(dd, CCE_INT_MASK + (8 * idx), reg);
13064 spin_unlock(&dd->irq_src_lock);
13065}
13066
13067
13068
13069
13070
13071
13072
13073
13074
13075
13076int set_intr_bits(struct hfi1_devdata *dd, u16 first, u16 last, bool set)
13077{
13078 u64 bits = 0;
13079 u64 bit;
13080 u16 src;
13081
13082 if (first > NUM_INTERRUPT_SOURCES || last > NUM_INTERRUPT_SOURCES)
13083 return -EINVAL;
13084
13085 if (last < first)
13086 return -ERANGE;
13087
13088 for (src = first; src <= last; src++) {
13089 bit = src % BITS_PER_REGISTER;
13090
13091 if (!bit && bits) {
13092 read_mod_write(dd, src - 1, bits, set);
13093 bits = 0;
13094 }
13095 bits |= BIT_ULL(bit);
13096 }
13097 read_mod_write(dd, last, bits, set);
13098
13099 return 0;
13100}
13101
13102
13103
13104
13105void clear_all_interrupts(struct hfi1_devdata *dd)
13106{
13107 int i;
13108
13109 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
13110 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~(u64)0);
13111
13112 write_csr(dd, CCE_ERR_CLEAR, ~(u64)0);
13113 write_csr(dd, MISC_ERR_CLEAR, ~(u64)0);
13114 write_csr(dd, RCV_ERR_CLEAR, ~(u64)0);
13115 write_csr(dd, SEND_ERR_CLEAR, ~(u64)0);
13116 write_csr(dd, SEND_PIO_ERR_CLEAR, ~(u64)0);
13117 write_csr(dd, SEND_DMA_ERR_CLEAR, ~(u64)0);
13118 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~(u64)0);
13119 for (i = 0; i < chip_send_contexts(dd); i++)
13120 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~(u64)0);
13121 for (i = 0; i < chip_sdma_engines(dd); i++)
13122 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~(u64)0);
13123
13124 write_csr(dd, DCC_ERR_FLG_CLR, ~(u64)0);
13125 write_csr(dd, DC_LCB_ERR_CLR, ~(u64)0);
13126 write_csr(dd, DC_DC8051_ERR_CLR, ~(u64)0);
13127}
13128
13129
13130
13131
13132
13133void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr)
13134{
13135 u64 reg;
13136 int m, n;
13137
13138
13139 m = isrc / 64;
13140 n = isrc % 64;
13141 if (likely(m < CCE_NUM_INT_CSRS)) {
13142 dd->gi_mask[m] &= ~((u64)1 << n);
13143 } else {
13144 dd_dev_err(dd, "remap interrupt err\n");
13145 return;
13146 }
13147
13148
13149 m = isrc / 8;
13150 n = isrc % 8;
13151 reg = read_csr(dd, CCE_INT_MAP + (8 * m));
13152 reg &= ~((u64)0xff << (8 * n));
13153 reg |= ((u64)msix_intr & 0xff) << (8 * n);
13154 write_csr(dd, CCE_INT_MAP + (8 * m), reg);
13155}
13156
13157void remap_sdma_interrupts(struct hfi1_devdata *dd, int engine, int msix_intr)
13158{
13159
13160
13161
13162
13163
13164
13165
13166 remap_intr(dd, IS_SDMA_START + engine, msix_intr);
13167 remap_intr(dd, IS_SDMA_PROGRESS_START + engine, msix_intr);
13168 remap_intr(dd, IS_SDMA_IDLE_START + engine, msix_intr);
13169}
13170
13171
13172
13173
13174
13175void reset_interrupts(struct hfi1_devdata *dd)
13176{
13177 int i;
13178
13179
13180 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
13181 dd->gi_mask[i] = ~(u64)0;
13182
13183
13184 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13185 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
13186}
13187
13188
13189
13190
13191
13192
13193static int set_up_interrupts(struct hfi1_devdata *dd)
13194{
13195 int ret;
13196
13197
13198 set_intr_bits(dd, IS_FIRST_SOURCE, IS_LAST_SOURCE, false);
13199
13200
13201 clear_all_interrupts(dd);
13202
13203
13204 reset_interrupts(dd);
13205
13206
13207 ret = msix_initialize(dd);
13208 if (ret)
13209 return ret;
13210
13211 ret = msix_request_irqs(dd);
13212 if (ret)
13213 msix_clean_up_interrupts(dd);
13214
13215 return ret;
13216}
13217
13218
13219
13220
13221
13222
13223
13224
13225
13226
13227
13228
13229static int set_up_context_variables(struct hfi1_devdata *dd)
13230{
13231 unsigned long num_kernel_contexts;
13232 u16 num_vnic_contexts = HFI1_NUM_VNIC_CTXT;
13233 int total_contexts;
13234 int ret;
13235 unsigned ngroups;
13236 int rmt_count;
13237 int user_rmt_reduced;
13238 u32 n_usr_ctxts;
13239 u32 send_contexts = chip_send_contexts(dd);
13240 u32 rcv_contexts = chip_rcv_contexts(dd);
13241
13242
13243
13244
13245
13246
13247
13248
13249 if (n_krcvqs)
13250
13251
13252
13253
13254
13255 num_kernel_contexts = n_krcvqs + 1;
13256 else
13257 num_kernel_contexts = DEFAULT_KRCVQS + 1;
13258
13259
13260
13261
13262 if (num_kernel_contexts > (send_contexts - num_vls - 1)) {
13263 dd_dev_err(dd,
13264 "Reducing # kernel rcv contexts to: %d, from %lu\n",
13265 send_contexts - num_vls - 1,
13266 num_kernel_contexts);
13267 num_kernel_contexts = send_contexts - num_vls - 1;
13268 }
13269
13270
13271 if ((num_kernel_contexts + num_vnic_contexts) > rcv_contexts) {
13272 dd_dev_err(dd, "No receive contexts available for VNIC\n");
13273 num_vnic_contexts = 0;
13274 }
13275 total_contexts = num_kernel_contexts + num_vnic_contexts;
13276
13277
13278
13279
13280
13281
13282 if (num_user_contexts < 0)
13283 n_usr_ctxts = cpumask_weight(&node_affinity.real_cpu_mask);
13284 else
13285 n_usr_ctxts = num_user_contexts;
13286
13287
13288
13289 if (total_contexts + n_usr_ctxts > rcv_contexts) {
13290 dd_dev_err(dd,
13291 "Reducing # user receive contexts to: %d, from %u\n",
13292 rcv_contexts - total_contexts,
13293 n_usr_ctxts);
13294
13295 n_usr_ctxts = rcv_contexts - total_contexts;
13296 }
13297
13298
13299
13300
13301
13302
13303
13304
13305
13306
13307
13308
13309
13310 rmt_count = qos_rmt_entries(dd, NULL, NULL) + (num_vnic_contexts * 2);
13311 if (HFI1_CAP_IS_KSET(TID_RDMA))
13312 rmt_count += num_kernel_contexts - 1;
13313 if (rmt_count + n_usr_ctxts > NUM_MAP_ENTRIES) {
13314 user_rmt_reduced = NUM_MAP_ENTRIES - rmt_count;
13315 dd_dev_err(dd,
13316 "RMT size is reducing the number of user receive contexts from %u to %d\n",
13317 n_usr_ctxts,
13318 user_rmt_reduced);
13319
13320 n_usr_ctxts = user_rmt_reduced;
13321 }
13322
13323 total_contexts += n_usr_ctxts;
13324
13325
13326 dd->num_rcv_contexts = total_contexts;
13327 dd->n_krcv_queues = num_kernel_contexts;
13328 dd->first_dyn_alloc_ctxt = num_kernel_contexts;
13329 dd->num_vnic_contexts = num_vnic_contexts;
13330 dd->num_user_contexts = n_usr_ctxts;
13331 dd->freectxts = n_usr_ctxts;
13332 dd_dev_info(dd,
13333 "rcv contexts: chip %d, used %d (kernel %d, vnic %u, user %u)\n",
13334 rcv_contexts,
13335 (int)dd->num_rcv_contexts,
13336 (int)dd->n_krcv_queues,
13337 dd->num_vnic_contexts,
13338 dd->num_user_contexts);
13339
13340
13341
13342
13343
13344
13345
13346
13347
13348
13349
13350
13351 dd->rcv_entries.group_size = RCV_INCREMENT;
13352 ngroups = chip_rcv_array_count(dd) / dd->rcv_entries.group_size;
13353 dd->rcv_entries.ngroups = ngroups / dd->num_rcv_contexts;
13354 dd->rcv_entries.nctxt_extra = ngroups -
13355 (dd->num_rcv_contexts * dd->rcv_entries.ngroups);
13356 dd_dev_info(dd, "RcvArray groups %u, ctxts extra %u\n",
13357 dd->rcv_entries.ngroups,
13358 dd->rcv_entries.nctxt_extra);
13359 if (dd->rcv_entries.ngroups * dd->rcv_entries.group_size >
13360 MAX_EAGER_ENTRIES * 2) {
13361 dd->rcv_entries.ngroups = (MAX_EAGER_ENTRIES * 2) /
13362 dd->rcv_entries.group_size;
13363 dd_dev_info(dd,
13364 "RcvArray group count too high, change to %u\n",
13365 dd->rcv_entries.ngroups);
13366 dd->rcv_entries.nctxt_extra = 0;
13367 }
13368
13369
13370
13371 ret = init_sc_pools_and_sizes(dd);
13372 if (ret >= 0) {
13373 dd->num_send_contexts = ret;
13374 dd_dev_info(
13375 dd,
13376 "send contexts: chip %d, used %d (kernel %d, ack %d, user %d, vl15 %d)\n",
13377 send_contexts,
13378 dd->num_send_contexts,
13379 dd->sc_sizes[SC_KERNEL].count,
13380 dd->sc_sizes[SC_ACK].count,
13381 dd->sc_sizes[SC_USER].count,
13382 dd->sc_sizes[SC_VL15].count);
13383 ret = 0;
13384 }
13385
13386 return ret;
13387}
13388
13389
13390
13391
13392
13393
13394static void set_partition_keys(struct hfi1_pportdata *ppd)
13395{
13396 struct hfi1_devdata *dd = ppd->dd;
13397 u64 reg = 0;
13398 int i;
13399
13400 dd_dev_info(dd, "Setting partition keys\n");
13401 for (i = 0; i < hfi1_get_npkeys(dd); i++) {
13402 reg |= (ppd->pkeys[i] &
13403 RCV_PARTITION_KEY_PARTITION_KEY_A_MASK) <<
13404 ((i % 4) *
13405 RCV_PARTITION_KEY_PARTITION_KEY_B_SHIFT);
13406
13407 if ((i % 4) == 3) {
13408 write_csr(dd, RCV_PARTITION_KEY +
13409 ((i - 3) * 2), reg);
13410 reg = 0;
13411 }
13412 }
13413
13414
13415 add_rcvctrl(dd, RCV_CTRL_RCV_PARTITION_KEY_ENABLE_SMASK);
13416}
13417
13418
13419
13420
13421
13422
13423
13424
13425
13426static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd)
13427{
13428 int i, j;
13429
13430
13431 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13432 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
13433
13434
13435 for (i = 0; i < chip_send_contexts(dd); i++)
13436 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
13437
13438
13439
13440
13441
13442
13443
13444
13445
13446
13447
13448 for (i = 0; i < chip_rcv_contexts(dd); i++) {
13449 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
13450 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
13451 for (j = 0; j < RXE_NUM_TID_FLOWS; j++)
13452 write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE + (8 * j), 0);
13453 }
13454
13455
13456 for (i = 0; i < chip_rcv_array_count(dd); i++)
13457 hfi1_put_tid(dd, i, PT_INVALID_FLUSH, 0, 0);
13458
13459
13460 for (i = 0; i < 32; i++)
13461 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
13462}
13463
13464
13465
13466
13467static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits,
13468 u64 ctrl_bits)
13469{
13470 unsigned long timeout;
13471 u64 reg;
13472
13473
13474 reg = read_csr(dd, CCE_STATUS);
13475 if ((reg & status_bits) == 0)
13476 return;
13477
13478
13479 write_csr(dd, CCE_CTRL, ctrl_bits);
13480
13481
13482 timeout = jiffies + msecs_to_jiffies(CCE_STATUS_TIMEOUT);
13483 while (1) {
13484 reg = read_csr(dd, CCE_STATUS);
13485 if ((reg & status_bits) == 0)
13486 return;
13487 if (time_after(jiffies, timeout)) {
13488 dd_dev_err(dd,
13489 "Timeout waiting for CceStatus to clear bits 0x%llx, remaining 0x%llx\n",
13490 status_bits, reg & status_bits);
13491 return;
13492 }
13493 udelay(1);
13494 }
13495}
13496
13497
13498static void reset_cce_csrs(struct hfi1_devdata *dd)
13499{
13500 int i;
13501
13502
13503
13504
13505
13506 clear_cce_status(dd, ALL_FROZE, CCE_CTRL_SPC_UNFREEZE_SMASK);
13507 clear_cce_status(dd, ALL_TXE_PAUSE, CCE_CTRL_TXE_RESUME_SMASK);
13508 clear_cce_status(dd, ALL_RXE_PAUSE, CCE_CTRL_RXE_RESUME_SMASK);
13509 for (i = 0; i < CCE_NUM_SCRATCH; i++)
13510 write_csr(dd, CCE_SCRATCH + (8 * i), 0);
13511
13512 write_csr(dd, CCE_ERR_MASK, 0);
13513 write_csr(dd, CCE_ERR_CLEAR, ~0ull);
13514
13515 for (i = 0; i < CCE_NUM_32_BIT_COUNTERS; i++)
13516 write_csr(dd, CCE_COUNTER_ARRAY32 + (8 * i), 0);
13517 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_RESETCSR);
13518
13519 for (i = 0; i < CCE_NUM_MSIX_VECTORS; i++) {
13520 write_csr(dd, CCE_MSIX_TABLE_LOWER + (8 * i), 0);
13521 write_csr(dd, CCE_MSIX_TABLE_UPPER + (8 * i),
13522 CCE_MSIX_TABLE_UPPER_RESETCSR);
13523 }
13524 for (i = 0; i < CCE_NUM_MSIX_PBAS; i++) {
13525
13526 write_csr(dd, CCE_MSIX_INT_GRANTED, ~0ull);
13527 write_csr(dd, CCE_MSIX_VEC_CLR_WITHOUT_INT, ~0ull);
13528 }
13529 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13530 write_csr(dd, CCE_INT_MAP, 0);
13531 for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
13532
13533 write_csr(dd, CCE_INT_MASK + (8 * i), 0);
13534 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~0ull);
13535
13536
13537 }
13538 for (i = 0; i < CCE_NUM_32_BIT_INT_COUNTERS; i++)
13539 write_csr(dd, CCE_INT_COUNTER_ARRAY32 + (8 * i), 0);
13540}
13541
13542
13543static void reset_misc_csrs(struct hfi1_devdata *dd)
13544{
13545 int i;
13546
13547 for (i = 0; i < 32; i++) {
13548 write_csr(dd, MISC_CFG_RSA_R2 + (8 * i), 0);
13549 write_csr(dd, MISC_CFG_RSA_SIGNATURE + (8 * i), 0);
13550 write_csr(dd, MISC_CFG_RSA_MODULUS + (8 * i), 0);
13551 }
13552
13553
13554
13555
13556
13557 write_csr(dd, MISC_CFG_RSA_CMD, 1);
13558 write_csr(dd, MISC_CFG_RSA_MU, 0);
13559 write_csr(dd, MISC_CFG_FW_CTRL, 0);
13560
13561
13562
13563
13564
13565 write_csr(dd, MISC_ERR_MASK, 0);
13566 write_csr(dd, MISC_ERR_CLEAR, ~0ull);
13567
13568}
13569
13570
13571static void reset_txe_csrs(struct hfi1_devdata *dd)
13572{
13573 int i;
13574
13575
13576
13577
13578 write_csr(dd, SEND_CTRL, 0);
13579 __cm_reset(dd, 0);
13580
13581
13582
13583
13584 write_csr(dd, SEND_HIGH_PRIORITY_LIMIT, 0);
13585 pio_reset_all(dd);
13586
13587 write_csr(dd, SEND_PIO_ERR_MASK, 0);
13588 write_csr(dd, SEND_PIO_ERR_CLEAR, ~0ull);
13589
13590
13591 write_csr(dd, SEND_DMA_ERR_MASK, 0);
13592 write_csr(dd, SEND_DMA_ERR_CLEAR, ~0ull);
13593
13594
13595 write_csr(dd, SEND_EGRESS_ERR_MASK, 0);
13596 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~0ull);
13597
13598 write_csr(dd, SEND_BTH_QP, 0);
13599 write_csr(dd, SEND_STATIC_RATE_CONTROL, 0);
13600 write_csr(dd, SEND_SC2VLT0, 0);
13601 write_csr(dd, SEND_SC2VLT1, 0);
13602 write_csr(dd, SEND_SC2VLT2, 0);
13603 write_csr(dd, SEND_SC2VLT3, 0);
13604 write_csr(dd, SEND_LEN_CHECK0, 0);
13605 write_csr(dd, SEND_LEN_CHECK1, 0);
13606
13607 write_csr(dd, SEND_ERR_MASK, 0);
13608 write_csr(dd, SEND_ERR_CLEAR, ~0ull);
13609
13610 for (i = 0; i < VL_ARB_LOW_PRIO_TABLE_SIZE; i++)
13611 write_csr(dd, SEND_LOW_PRIORITY_LIST + (8 * i), 0);
13612 for (i = 0; i < VL_ARB_HIGH_PRIO_TABLE_SIZE; i++)
13613 write_csr(dd, SEND_HIGH_PRIORITY_LIST + (8 * i), 0);
13614 for (i = 0; i < chip_send_contexts(dd) / NUM_CONTEXTS_PER_SET; i++)
13615 write_csr(dd, SEND_CONTEXT_SET_CTRL + (8 * i), 0);
13616 for (i = 0; i < TXE_NUM_32_BIT_COUNTER; i++)
13617 write_csr(dd, SEND_COUNTER_ARRAY32 + (8 * i), 0);
13618 for (i = 0; i < TXE_NUM_64_BIT_COUNTER; i++)
13619 write_csr(dd, SEND_COUNTER_ARRAY64 + (8 * i), 0);
13620 write_csr(dd, SEND_CM_CTRL, SEND_CM_CTRL_RESETCSR);
13621 write_csr(dd, SEND_CM_GLOBAL_CREDIT, SEND_CM_GLOBAL_CREDIT_RESETCSR);
13622
13623 write_csr(dd, SEND_CM_TIMER_CTRL, 0);
13624 write_csr(dd, SEND_CM_LOCAL_AU_TABLE0_TO3, 0);
13625 write_csr(dd, SEND_CM_LOCAL_AU_TABLE4_TO7, 0);
13626 write_csr(dd, SEND_CM_REMOTE_AU_TABLE0_TO3, 0);
13627 write_csr(dd, SEND_CM_REMOTE_AU_TABLE4_TO7, 0);
13628 for (i = 0; i < TXE_NUM_DATA_VL; i++)
13629 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
13630 write_csr(dd, SEND_CM_CREDIT_VL15, 0);
13631
13632
13633
13634
13635 write_csr(dd, SEND_EGRESS_ERR_INFO, ~0ull);
13636
13637
13638
13639
13640
13641
13642 for (i = 0; i < chip_send_contexts(dd); i++) {
13643 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
13644 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_CTRL, 0);
13645 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
13646 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_FORCE, 0);
13647 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, 0);
13648 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~0ull);
13649 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_ENABLE, 0);
13650 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_VL, 0);
13651 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_JOB_KEY, 0);
13652 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_PARTITION_KEY, 0);
13653 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, 0);
13654 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_OPCODE, 0);
13655 }
13656
13657
13658
13659
13660 for (i = 0; i < chip_sdma_engines(dd); i++) {
13661 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
13662
13663 write_kctxt_csr(dd, i, SEND_DMA_BASE_ADDR, 0);
13664 write_kctxt_csr(dd, i, SEND_DMA_LEN_GEN, 0);
13665 write_kctxt_csr(dd, i, SEND_DMA_TAIL, 0);
13666
13667 write_kctxt_csr(dd, i, SEND_DMA_HEAD_ADDR, 0);
13668 write_kctxt_csr(dd, i, SEND_DMA_PRIORITY_THLD, 0);
13669
13670 write_kctxt_csr(dd, i, SEND_DMA_RELOAD_CNT, 0);
13671 write_kctxt_csr(dd, i, SEND_DMA_DESC_CNT, 0);
13672
13673
13674 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, 0);
13675 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~0ull);
13676
13677 write_kctxt_csr(dd, i, SEND_DMA_CHECK_ENABLE, 0);
13678 write_kctxt_csr(dd, i, SEND_DMA_CHECK_VL, 0);
13679 write_kctxt_csr(dd, i, SEND_DMA_CHECK_JOB_KEY, 0);
13680 write_kctxt_csr(dd, i, SEND_DMA_CHECK_PARTITION_KEY, 0);
13681 write_kctxt_csr(dd, i, SEND_DMA_CHECK_SLID, 0);
13682 write_kctxt_csr(dd, i, SEND_DMA_CHECK_OPCODE, 0);
13683 write_kctxt_csr(dd, i, SEND_DMA_MEMORY, 0);
13684 }
13685}
13686
13687
13688
13689
13690
13691static void init_rbufs(struct hfi1_devdata *dd)
13692{
13693 u64 reg;
13694 int count;
13695
13696
13697
13698
13699
13700 count = 0;
13701 while (1) {
13702 reg = read_csr(dd, RCV_STATUS);
13703 if ((reg & (RCV_STATUS_RX_RBUF_PKT_PENDING_SMASK
13704 | RCV_STATUS_RX_PKT_IN_PROGRESS_SMASK)) == 0)
13705 break;
13706
13707
13708
13709
13710
13711
13712
13713 if (count++ > 500) {
13714 dd_dev_err(dd,
13715 "%s: in-progress DMA not clearing: RcvStatus 0x%llx, continuing\n",
13716 __func__, reg);
13717 break;
13718 }
13719 udelay(2);
13720 }
13721
13722
13723 write_csr(dd, RCV_CTRL, RCV_CTRL_RX_RBUF_INIT_SMASK);
13724
13725
13726
13727
13728
13729
13730
13731 read_csr(dd, RCV_CTRL);
13732
13733
13734 count = 0;
13735 while (1) {
13736
13737 udelay(2);
13738 reg = read_csr(dd, RCV_STATUS);
13739 if (reg & (RCV_STATUS_RX_RBUF_INIT_DONE_SMASK))
13740 break;
13741
13742
13743 if (count++ > 50) {
13744 dd_dev_err(dd,
13745 "%s: RcvStatus.RxRbufInit not set, continuing\n",
13746 __func__);
13747 break;
13748 }
13749 }
13750}
13751
13752
13753static void reset_rxe_csrs(struct hfi1_devdata *dd)
13754{
13755 int i, j;
13756
13757
13758
13759
13760 write_csr(dd, RCV_CTRL, 0);
13761 init_rbufs(dd);
13762
13763
13764
13765
13766 write_csr(dd, RCV_BTH_QP, 0);
13767 write_csr(dd, RCV_MULTICAST, 0);
13768 write_csr(dd, RCV_BYPASS, 0);
13769 write_csr(dd, RCV_VL15, 0);
13770
13771 write_csr(dd, RCV_ERR_INFO,
13772 RCV_ERR_INFO_RCV_EXCESS_BUFFER_OVERRUN_SMASK);
13773
13774 write_csr(dd, RCV_ERR_MASK, 0);
13775 write_csr(dd, RCV_ERR_CLEAR, ~0ull);
13776
13777 for (i = 0; i < 32; i++)
13778 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
13779 for (i = 0; i < 4; i++)
13780 write_csr(dd, RCV_PARTITION_KEY + (8 * i), 0);
13781 for (i = 0; i < RXE_NUM_32_BIT_COUNTERS; i++)
13782 write_csr(dd, RCV_COUNTER_ARRAY32 + (8 * i), 0);
13783 for (i = 0; i < RXE_NUM_64_BIT_COUNTERS; i++)
13784 write_csr(dd, RCV_COUNTER_ARRAY64 + (8 * i), 0);
13785 for (i = 0; i < RXE_NUM_RSM_INSTANCES; i++)
13786 clear_rsm_rule(dd, i);
13787 for (i = 0; i < 32; i++)
13788 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), 0);
13789
13790
13791
13792
13793 for (i = 0; i < chip_rcv_contexts(dd); i++) {
13794
13795 write_kctxt_csr(dd, i, RCV_CTXT_CTRL, 0);
13796
13797 write_kctxt_csr(dd, i, RCV_EGR_CTRL, 0);
13798 write_kctxt_csr(dd, i, RCV_TID_CTRL, 0);
13799 write_kctxt_csr(dd, i, RCV_KEY_CTRL, 0);
13800 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
13801 write_kctxt_csr(dd, i, RCV_HDR_CNT, 0);
13802 write_kctxt_csr(dd, i, RCV_HDR_ENT_SIZE, 0);
13803 write_kctxt_csr(dd, i, RCV_HDR_SIZE, 0);
13804 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
13805 write_kctxt_csr(dd, i, RCV_AVAIL_TIME_OUT, 0);
13806 write_kctxt_csr(dd, i, RCV_HDR_OVFL_CNT, 0);
13807
13808
13809
13810 write_uctxt_csr(dd, i, RCV_HDR_HEAD, 0);
13811
13812 write_uctxt_csr(dd, i, RCV_EGR_INDEX_HEAD, 0);
13813
13814 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) {
13815 write_uctxt_csr(dd, i,
13816 RCV_TID_FLOW_TABLE + (8 * j), 0);
13817 }
13818 }
13819}
13820
13821
13822
13823
13824
13825
13826
13827
13828
13829
13830
13831
13832static void init_sc2vl_tables(struct hfi1_devdata *dd)
13833{
13834 int i;
13835
13836
13837
13838 write_csr(dd, SEND_SC2VLT0, SC2VL_VAL(
13839 0,
13840 0, 0, 1, 1,
13841 2, 2, 3, 3,
13842 4, 4, 5, 5,
13843 6, 6, 7, 7));
13844 write_csr(dd, SEND_SC2VLT1, SC2VL_VAL(
13845 1,
13846 8, 0, 9, 0,
13847 10, 0, 11, 0,
13848 12, 0, 13, 0,
13849 14, 0, 15, 15));
13850 write_csr(dd, SEND_SC2VLT2, SC2VL_VAL(
13851 2,
13852 16, 0, 17, 0,
13853 18, 0, 19, 0,
13854 20, 0, 21, 0,
13855 22, 0, 23, 0));
13856 write_csr(dd, SEND_SC2VLT3, SC2VL_VAL(
13857 3,
13858 24, 0, 25, 0,
13859 26, 0, 27, 0,
13860 28, 0, 29, 0,
13861 30, 0, 31, 0));
13862
13863
13864 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, DC_SC_VL_VAL(
13865 15_0,
13866 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
13867 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 15));
13868 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, DC_SC_VL_VAL(
13869 31_16,
13870 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0,
13871 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0));
13872
13873
13874 for (i = 0; i < 32; i++) {
13875 if (i < 8 || i == 15)
13876 *((u8 *)(dd->sc2vl) + i) = (u8)i;
13877 else
13878 *((u8 *)(dd->sc2vl) + i) = 0;
13879 }
13880}
13881
13882
13883
13884
13885
13886
13887
13888
13889
13890
13891static int init_chip(struct hfi1_devdata *dd)
13892{
13893 int i;
13894 int ret = 0;
13895
13896
13897
13898
13899
13900
13901
13902
13903
13904
13905
13906
13907
13908 write_csr(dd, SEND_CTRL, 0);
13909 for (i = 0; i < chip_send_contexts(dd); i++)
13910 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
13911 for (i = 0; i < chip_sdma_engines(dd); i++)
13912 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
13913
13914 write_csr(dd, RCV_CTRL, 0);
13915 for (i = 0; i < chip_rcv_contexts(dd); i++)
13916 write_csr(dd, RCV_CTXT_CTRL, 0);
13917
13918 for (i = 0; i < CCE_NUM_INT_CSRS; i++)
13919 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull);
13920
13921
13922
13923
13924
13925
13926
13927 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK);
13928 (void)read_csr(dd, CCE_DC_CTRL);
13929
13930 if (use_flr) {
13931
13932
13933
13934
13935
13936 dd_dev_info(dd, "Resetting CSRs with FLR\n");
13937
13938
13939 pcie_flr(dd->pcidev);
13940
13941
13942 ret = restore_pci_variables(dd);
13943 if (ret) {
13944 dd_dev_err(dd, "%s: Could not restore PCI variables\n",
13945 __func__);
13946 return ret;
13947 }
13948
13949 if (is_ax(dd)) {
13950 dd_dev_info(dd, "Resetting CSRs with FLR\n");
13951 pcie_flr(dd->pcidev);
13952 ret = restore_pci_variables(dd);
13953 if (ret) {
13954 dd_dev_err(dd, "%s: Could not restore PCI variables\n",
13955 __func__);
13956 return ret;
13957 }
13958 }
13959 } else {
13960 dd_dev_info(dd, "Resetting CSRs with writes\n");
13961 reset_cce_csrs(dd);
13962 reset_txe_csrs(dd);
13963 reset_rxe_csrs(dd);
13964 reset_misc_csrs(dd);
13965 }
13966
13967 write_csr(dd, CCE_DC_CTRL, 0);
13968
13969
13970 setextled(dd, 0);
13971
13972
13973
13974
13975
13976
13977
13978
13979
13980
13981
13982 write_csr(dd, ASIC_QSFP1_OUT, 0x1f);
13983 write_csr(dd, ASIC_QSFP2_OUT, 0x1f);
13984 init_chip_resources(dd);
13985 return ret;
13986}
13987
13988static void init_early_variables(struct hfi1_devdata *dd)
13989{
13990 int i;
13991
13992
13993 dd->vau = CM_VAU;
13994 dd->link_credits = CM_GLOBAL_CREDITS;
13995 if (is_ax(dd))
13996 dd->link_credits--;
13997 dd->vcu = cu_to_vcu(hfi1_cu);
13998
13999 dd->vl15_init = (8 * (2048 + 128)) / vau_to_au(dd->vau);
14000 if (dd->vl15_init > dd->link_credits)
14001 dd->vl15_init = dd->link_credits;
14002
14003 write_uninitialized_csrs_and_memories(dd);
14004
14005 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
14006 for (i = 0; i < dd->num_pports; i++) {
14007 struct hfi1_pportdata *ppd = &dd->pport[i];
14008
14009 set_partition_keys(ppd);
14010 }
14011 init_sc2vl_tables(dd);
14012}
14013
14014static void init_kdeth_qp(struct hfi1_devdata *dd)
14015{
14016
14017 if (kdeth_qp != 0 && kdeth_qp >= 0xff) {
14018
14019 dd_dev_err(dd, "Invalid KDETH queue pair prefix, ignoring");
14020 kdeth_qp = 0;
14021 }
14022 if (kdeth_qp == 0)
14023 kdeth_qp = DEFAULT_KDETH_QP;
14024
14025 write_csr(dd, SEND_BTH_QP,
14026 (kdeth_qp & SEND_BTH_QP_KDETH_QP_MASK) <<
14027 SEND_BTH_QP_KDETH_QP_SHIFT);
14028
14029 write_csr(dd, RCV_BTH_QP,
14030 (kdeth_qp & RCV_BTH_QP_KDETH_QP_MASK) <<
14031 RCV_BTH_QP_KDETH_QP_SHIFT);
14032}
14033
14034
14035
14036
14037
14038
14039u8 hfi1_get_qp_map(struct hfi1_devdata *dd, u8 idx)
14040{
14041 u64 reg = read_csr(dd, RCV_QP_MAP_TABLE + (idx / 8) * 8);
14042
14043 reg >>= (idx % 8) * 8;
14044 return reg;
14045}
14046
14047
14048
14049
14050
14051
14052
14053
14054
14055
14056
14057
14058
14059
14060
14061
14062
14063
14064static void init_qpmap_table(struct hfi1_devdata *dd,
14065 u32 first_ctxt,
14066 u32 last_ctxt)
14067{
14068 u64 reg = 0;
14069 u64 regno = RCV_QP_MAP_TABLE;
14070 int i;
14071 u64 ctxt = first_ctxt;
14072
14073 for (i = 0; i < 256; i++) {
14074 reg |= ctxt << (8 * (i % 8));
14075 ctxt++;
14076 if (ctxt > last_ctxt)
14077 ctxt = first_ctxt;
14078 if (i % 8 == 7) {
14079 write_csr(dd, regno, reg);
14080 reg = 0;
14081 regno += 8;
14082 }
14083 }
14084
14085 add_rcvctrl(dd, RCV_CTRL_RCV_QP_MAP_ENABLE_SMASK
14086 | RCV_CTRL_RCV_BYPASS_ENABLE_SMASK);
14087}
14088
14089struct rsm_map_table {
14090 u64 map[NUM_MAP_REGS];
14091 unsigned int used;
14092};
14093
14094struct rsm_rule_data {
14095 u8 offset;
14096 u8 pkt_type;
14097 u32 field1_off;
14098 u32 field2_off;
14099 u32 index1_off;
14100 u32 index1_width;
14101 u32 index2_off;
14102 u32 index2_width;
14103 u32 mask1;
14104 u32 value1;
14105 u32 mask2;
14106 u32 value2;
14107};
14108
14109
14110
14111
14112
14113static struct rsm_map_table *alloc_rsm_map_table(struct hfi1_devdata *dd)
14114{
14115 struct rsm_map_table *rmt;
14116 u8 rxcontext = is_ax(dd) ? 0 : 0xff;
14117
14118 rmt = kmalloc(sizeof(*rmt), GFP_KERNEL);
14119 if (rmt) {
14120 memset(rmt->map, rxcontext, sizeof(rmt->map));
14121 rmt->used = 0;
14122 }
14123
14124 return rmt;
14125}
14126
14127
14128
14129
14130
14131static void complete_rsm_map_table(struct hfi1_devdata *dd,
14132 struct rsm_map_table *rmt)
14133{
14134 int i;
14135
14136 if (rmt) {
14137
14138 for (i = 0; i < NUM_MAP_REGS; i++)
14139 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), rmt->map[i]);
14140
14141
14142 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
14143 }
14144}
14145
14146
14147
14148
14149static void add_rsm_rule(struct hfi1_devdata *dd, u8 rule_index,
14150 struct rsm_rule_data *rrd)
14151{
14152 write_csr(dd, RCV_RSM_CFG + (8 * rule_index),
14153 (u64)rrd->offset << RCV_RSM_CFG_OFFSET_SHIFT |
14154 1ull << rule_index |
14155 (u64)rrd->pkt_type << RCV_RSM_CFG_PACKET_TYPE_SHIFT);
14156 write_csr(dd, RCV_RSM_SELECT + (8 * rule_index),
14157 (u64)rrd->field1_off << RCV_RSM_SELECT_FIELD1_OFFSET_SHIFT |
14158 (u64)rrd->field2_off << RCV_RSM_SELECT_FIELD2_OFFSET_SHIFT |
14159 (u64)rrd->index1_off << RCV_RSM_SELECT_INDEX1_OFFSET_SHIFT |
14160 (u64)rrd->index1_width << RCV_RSM_SELECT_INDEX1_WIDTH_SHIFT |
14161 (u64)rrd->index2_off << RCV_RSM_SELECT_INDEX2_OFFSET_SHIFT |
14162 (u64)rrd->index2_width << RCV_RSM_SELECT_INDEX2_WIDTH_SHIFT);
14163 write_csr(dd, RCV_RSM_MATCH + (8 * rule_index),
14164 (u64)rrd->mask1 << RCV_RSM_MATCH_MASK1_SHIFT |
14165 (u64)rrd->value1 << RCV_RSM_MATCH_VALUE1_SHIFT |
14166 (u64)rrd->mask2 << RCV_RSM_MATCH_MASK2_SHIFT |
14167 (u64)rrd->value2 << RCV_RSM_MATCH_VALUE2_SHIFT);
14168}
14169
14170
14171
14172
14173static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index)
14174{
14175 write_csr(dd, RCV_RSM_CFG + (8 * rule_index), 0);
14176 write_csr(dd, RCV_RSM_SELECT + (8 * rule_index), 0);
14177 write_csr(dd, RCV_RSM_MATCH + (8 * rule_index), 0);
14178}
14179
14180
14181static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
14182 unsigned int *np)
14183{
14184 int i;
14185 unsigned int m, n;
14186 u8 max_by_vl = 0;
14187
14188
14189 if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS ||
14190 num_vls == 1 ||
14191 krcvqsset <= 1)
14192 goto no_qos;
14193
14194
14195 for (i = 0; i < min_t(unsigned int, num_vls, krcvqsset); i++)
14196 if (krcvqs[i] > max_by_vl)
14197 max_by_vl = krcvqs[i];
14198 if (max_by_vl > 32)
14199 goto no_qos;
14200 m = ilog2(__roundup_pow_of_two(max_by_vl));
14201
14202
14203 n = ilog2(__roundup_pow_of_two(num_vls));
14204
14205
14206 if ((m + n) > 7)
14207 goto no_qos;
14208
14209 if (mp)
14210 *mp = m;
14211 if (np)
14212 *np = n;
14213
14214 return 1 << (m + n);
14215
14216no_qos:
14217 if (mp)
14218 *mp = 0;
14219 if (np)
14220 *np = 0;
14221 return 0;
14222}
14223
14224
14225
14226
14227
14228
14229
14230
14231
14232
14233
14234
14235
14236
14237
14238static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt)
14239{
14240 struct rsm_rule_data rrd;
14241 unsigned qpns_per_vl, ctxt, i, qpn, n = 1, m;
14242 unsigned int rmt_entries;
14243 u64 reg;
14244
14245 if (!rmt)
14246 goto bail;
14247 rmt_entries = qos_rmt_entries(dd, &m, &n);
14248 if (rmt_entries == 0)
14249 goto bail;
14250 qpns_per_vl = 1 << m;
14251
14252
14253 rmt_entries = 1 << (m + n);
14254 if (rmt->used + rmt_entries >= NUM_MAP_ENTRIES)
14255 goto bail;
14256
14257
14258 for (i = 0, ctxt = FIRST_KERNEL_KCTXT; i < num_vls; i++) {
14259 unsigned tctxt;
14260
14261 for (qpn = 0, tctxt = ctxt;
14262 krcvqs[i] && qpn < qpns_per_vl; qpn++) {
14263 unsigned idx, regoff, regidx;
14264
14265
14266 idx = rmt->used + ((qpn << n) ^ i);
14267 regoff = (idx % 8) * 8;
14268 regidx = idx / 8;
14269
14270 reg = rmt->map[regidx];
14271 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK
14272 << regoff);
14273 reg |= (u64)(tctxt++) << regoff;
14274 rmt->map[regidx] = reg;
14275 if (tctxt == ctxt + krcvqs[i])
14276 tctxt = ctxt;
14277 }
14278 ctxt += krcvqs[i];
14279 }
14280
14281 rrd.offset = rmt->used;
14282 rrd.pkt_type = 2;
14283 rrd.field1_off = LRH_BTH_MATCH_OFFSET;
14284 rrd.field2_off = LRH_SC_MATCH_OFFSET;
14285 rrd.index1_off = LRH_SC_SELECT_OFFSET;
14286 rrd.index1_width = n;
14287 rrd.index2_off = QPN_SELECT_OFFSET;
14288 rrd.index2_width = m + n;
14289 rrd.mask1 = LRH_BTH_MASK;
14290 rrd.value1 = LRH_BTH_VALUE;
14291 rrd.mask2 = LRH_SC_MASK;
14292 rrd.value2 = LRH_SC_VALUE;
14293
14294
14295 add_rsm_rule(dd, RSM_INS_VERBS, &rrd);
14296
14297
14298 rmt->used += rmt_entries;
14299
14300 init_qpmap_table(dd, HFI1_CTRL_CTXT, HFI1_CTRL_CTXT);
14301 dd->qos_shift = n + 1;
14302 return;
14303bail:
14304 dd->qos_shift = 1;
14305 init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1);
14306}
14307
14308static void init_fecn_handling(struct hfi1_devdata *dd,
14309 struct rsm_map_table *rmt)
14310{
14311 struct rsm_rule_data rrd;
14312 u64 reg;
14313 int i, idx, regoff, regidx, start;
14314 u8 offset;
14315 u32 total_cnt;
14316
14317 if (HFI1_CAP_IS_KSET(TID_RDMA))
14318
14319 start = 1;
14320 else
14321 start = dd->first_dyn_alloc_ctxt;
14322
14323 total_cnt = dd->num_rcv_contexts - start;
14324
14325
14326 if (rmt->used + total_cnt >= NUM_MAP_ENTRIES) {
14327 dd_dev_err(dd, "FECN handling disabled - too many contexts allocated\n");
14328 return;
14329 }
14330
14331
14332
14333
14334
14335
14336
14337
14338
14339
14340
14341 offset = (u8)(NUM_MAP_ENTRIES + rmt->used - start);
14342
14343 for (i = start, idx = rmt->used; i < dd->num_rcv_contexts;
14344 i++, idx++) {
14345
14346 regoff = (idx % 8) * 8;
14347 regidx = idx / 8;
14348 reg = rmt->map[regidx];
14349 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK << regoff);
14350 reg |= (u64)i << regoff;
14351 rmt->map[regidx] = reg;
14352 }
14353
14354
14355
14356
14357
14358
14359
14360
14361
14362
14363 rrd.offset = offset;
14364 rrd.pkt_type = 0;
14365 rrd.field1_off = 95;
14366 rrd.field2_off = 133;
14367 rrd.index1_off = 64;
14368 rrd.index1_width = 8;
14369 rrd.index2_off = 0;
14370 rrd.index2_width = 0;
14371 rrd.mask1 = 1;
14372 rrd.value1 = 1;
14373 rrd.mask2 = 1;
14374 rrd.value2 = 1;
14375
14376
14377 add_rsm_rule(dd, RSM_INS_FECN, &rrd);
14378
14379 rmt->used += total_cnt;
14380}
14381
14382
14383void hfi1_init_vnic_rsm(struct hfi1_devdata *dd)
14384{
14385 u8 i, j;
14386 u8 ctx_id = 0;
14387 u64 reg;
14388 u32 regoff;
14389 struct rsm_rule_data rrd;
14390
14391 if (hfi1_vnic_is_rsm_full(dd, NUM_VNIC_MAP_ENTRIES)) {
14392 dd_dev_err(dd, "Vnic RSM disabled, rmt entries used = %d\n",
14393 dd->vnic.rmt_start);
14394 return;
14395 }
14396
14397 dev_dbg(&(dd)->pcidev->dev, "Vnic rsm start = %d, end %d\n",
14398 dd->vnic.rmt_start,
14399 dd->vnic.rmt_start + NUM_VNIC_MAP_ENTRIES);
14400
14401
14402 regoff = RCV_RSM_MAP_TABLE + (dd->vnic.rmt_start / 8) * 8;
14403 reg = read_csr(dd, regoff);
14404 for (i = 0; i < NUM_VNIC_MAP_ENTRIES; i++) {
14405
14406 j = (dd->vnic.rmt_start + i) % 8;
14407 reg &= ~(0xffllu << (j * 8));
14408 reg |= (u64)dd->vnic.ctxt[ctx_id++]->ctxt << (j * 8);
14409
14410 ctx_id %= dd->vnic.num_ctxt;
14411
14412 if (j == 7 || ((i + 1) == NUM_VNIC_MAP_ENTRIES)) {
14413 dev_dbg(&(dd)->pcidev->dev,
14414 "Vnic rsm map reg[%d] =0x%llx\n",
14415 regoff - RCV_RSM_MAP_TABLE, reg);
14416
14417 write_csr(dd, regoff, reg);
14418 regoff += 8;
14419 if (i < (NUM_VNIC_MAP_ENTRIES - 1))
14420 reg = read_csr(dd, regoff);
14421 }
14422 }
14423
14424
14425 rrd.offset = dd->vnic.rmt_start;
14426 rrd.pkt_type = 4;
14427
14428 rrd.field1_off = L2_TYPE_MATCH_OFFSET;
14429 rrd.mask1 = L2_TYPE_MASK;
14430 rrd.value1 = L2_16B_VALUE;
14431
14432 rrd.field2_off = L4_TYPE_MATCH_OFFSET;
14433 rrd.mask2 = L4_16B_TYPE_MASK;
14434 rrd.value2 = L4_16B_ETH_VALUE;
14435
14436 rrd.index1_off = L4_16B_HDR_VESWID_OFFSET;
14437 rrd.index1_width = ilog2(NUM_VNIC_MAP_ENTRIES);
14438 rrd.index2_off = L2_16B_ENTROPY_OFFSET;
14439 rrd.index2_width = ilog2(NUM_VNIC_MAP_ENTRIES);
14440 add_rsm_rule(dd, RSM_INS_VNIC, &rrd);
14441
14442
14443 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
14444}
14445
14446void hfi1_deinit_vnic_rsm(struct hfi1_devdata *dd)
14447{
14448 clear_rsm_rule(dd, RSM_INS_VNIC);
14449
14450
14451 if (dd->vnic.rmt_start == 0)
14452 clear_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
14453}
14454
14455static int init_rxe(struct hfi1_devdata *dd)
14456{
14457 struct rsm_map_table *rmt;
14458 u64 val;
14459
14460
14461 write_csr(dd, RCV_ERR_MASK, ~0ull);
14462
14463 rmt = alloc_rsm_map_table(dd);
14464 if (!rmt)
14465 return -ENOMEM;
14466
14467
14468 init_qos(dd, rmt);
14469 init_fecn_handling(dd, rmt);
14470 complete_rsm_map_table(dd, rmt);
14471
14472 dd->vnic.rmt_start = rmt->used;
14473 kfree(rmt);
14474
14475
14476
14477
14478
14479
14480
14481
14482
14483
14484
14485
14486
14487
14488 val = read_csr(dd, RCV_BYPASS);
14489 val &= ~RCV_BYPASS_HDR_SIZE_SMASK;
14490 val |= ((4ull & RCV_BYPASS_HDR_SIZE_MASK) <<
14491 RCV_BYPASS_HDR_SIZE_SHIFT);
14492 write_csr(dd, RCV_BYPASS, val);
14493 return 0;
14494}
14495
14496static void init_other(struct hfi1_devdata *dd)
14497{
14498
14499 write_csr(dd, CCE_ERR_MASK, ~0ull);
14500
14501 write_csr(dd, MISC_ERR_MASK, DRIVER_MISC_MASK);
14502
14503 write_csr(dd, DCC_ERR_FLG_EN, ~0ull);
14504 write_csr(dd, DC_DC8051_ERR_EN, ~0ull);
14505}
14506
14507
14508
14509
14510
14511
14512
14513
14514
14515static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu,
14516 u32 csr0to3, u32 csr4to7)
14517{
14518 write_csr(dd, csr0to3,
14519 0ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE0_SHIFT |
14520 1ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE1_SHIFT |
14521 2ull * cu <<
14522 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE2_SHIFT |
14523 4ull * cu <<
14524 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE3_SHIFT);
14525 write_csr(dd, csr4to7,
14526 8ull * cu <<
14527 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE4_SHIFT |
14528 16ull * cu <<
14529 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE5_SHIFT |
14530 32ull * cu <<
14531 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE6_SHIFT |
14532 64ull * cu <<
14533 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE7_SHIFT);
14534}
14535
14536static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
14537{
14538 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_LOCAL_AU_TABLE0_TO3,
14539 SEND_CM_LOCAL_AU_TABLE4_TO7);
14540}
14541
14542void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
14543{
14544 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_REMOTE_AU_TABLE0_TO3,
14545 SEND_CM_REMOTE_AU_TABLE4_TO7);
14546}
14547
14548static void init_txe(struct hfi1_devdata *dd)
14549{
14550 int i;
14551
14552
14553 write_csr(dd, SEND_PIO_ERR_MASK, ~0ull);
14554 write_csr(dd, SEND_DMA_ERR_MASK, ~0ull);
14555 write_csr(dd, SEND_ERR_MASK, ~0ull);
14556 write_csr(dd, SEND_EGRESS_ERR_MASK, ~0ull);
14557
14558
14559 for (i = 0; i < chip_send_contexts(dd); i++)
14560 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, ~0ull);
14561 for (i = 0; i < chip_sdma_engines(dd); i++)
14562 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, ~0ull);
14563
14564
14565 assign_local_cm_au_table(dd, dd->vcu);
14566
14567
14568
14569
14570
14571 if (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)
14572 write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE);
14573}
14574
14575int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd,
14576 u16 jkey)
14577{
14578 u8 hw_ctxt;
14579 u64 reg;
14580
14581 if (!rcd || !rcd->sc)
14582 return -EINVAL;
14583
14584 hw_ctxt = rcd->sc->hw_context;
14585 reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK |
14586 ((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) <<
14587 SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT);
14588
14589 if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY))
14590 reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK;
14591 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, reg);
14592
14593
14594
14595 if (!is_ax(dd)) {
14596 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14597 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
14598 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14599 }
14600
14601
14602 reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK |
14603 ((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) <<
14604 RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT);
14605 write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, reg);
14606
14607 return 0;
14608}
14609
14610int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
14611{
14612 u8 hw_ctxt;
14613 u64 reg;
14614
14615 if (!rcd || !rcd->sc)
14616 return -EINVAL;
14617
14618 hw_ctxt = rcd->sc->hw_context;
14619 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, 0);
14620
14621
14622
14623
14624
14625 if (!is_ax(dd)) {
14626 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14627 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
14628 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14629 }
14630
14631 write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, 0);
14632
14633 return 0;
14634}
14635
14636int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd,
14637 u16 pkey)
14638{
14639 u8 hw_ctxt;
14640 u64 reg;
14641
14642 if (!rcd || !rcd->sc)
14643 return -EINVAL;
14644
14645 hw_ctxt = rcd->sc->hw_context;
14646 reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) <<
14647 SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT;
14648 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg);
14649 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14650 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
14651 reg &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK;
14652 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14653
14654 return 0;
14655}
14656
14657int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt)
14658{
14659 u8 hw_ctxt;
14660 u64 reg;
14661
14662 if (!ctxt || !ctxt->sc)
14663 return -EINVAL;
14664
14665 hw_ctxt = ctxt->sc->hw_context;
14666 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14667 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
14668 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14669 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, 0);
14670
14671 return 0;
14672}
14673
14674
14675
14676
14677
14678void hfi1_start_cleanup(struct hfi1_devdata *dd)
14679{
14680 aspm_exit(dd);
14681 free_cntrs(dd);
14682 free_rcverr(dd);
14683 finish_chip_resources(dd);
14684}
14685
14686#define HFI_BASE_GUID(dev) \
14687 ((dev)->base_guid & ~(1ULL << GUID_HFI_INDEX_SHIFT))
14688
14689
14690
14691
14692
14693
14694static int init_asic_data(struct hfi1_devdata *dd)
14695{
14696 unsigned long index;
14697 struct hfi1_devdata *peer;
14698 struct hfi1_asic_data *asic_data;
14699 int ret = 0;
14700
14701
14702 asic_data = kzalloc(sizeof(*dd->asic_data), GFP_KERNEL);
14703 if (!asic_data)
14704 return -ENOMEM;
14705
14706 xa_lock_irq(&hfi1_dev_table);
14707
14708 xa_for_each(&hfi1_dev_table, index, peer) {
14709 if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(peer)) &&
14710 dd->unit != peer->unit)
14711 break;
14712 }
14713
14714 if (peer) {
14715
14716 dd->asic_data = peer->asic_data;
14717 kfree(asic_data);
14718 } else {
14719 dd->asic_data = asic_data;
14720 mutex_init(&dd->asic_data->asic_resource_mutex);
14721 }
14722 dd->asic_data->dds[dd->hfi1_id] = dd;
14723 xa_unlock_irq(&hfi1_dev_table);
14724
14725
14726 if (!peer)
14727 ret = set_up_i2c(dd, dd->asic_data);
14728
14729 return ret;
14730}
14731
14732
14733
14734
14735
14736
14737
14738static int obtain_boardname(struct hfi1_devdata *dd)
14739{
14740
14741 const char generic[] =
14742 "Intel Omni-Path Host Fabric Interface Adapter 100 Series";
14743 unsigned long size;
14744 int ret;
14745
14746 ret = read_hfi1_efi_var(dd, "description", &size,
14747 (void **)&dd->boardname);
14748 if (ret) {
14749 dd_dev_info(dd, "Board description not found\n");
14750
14751 dd->boardname = kstrdup(generic, GFP_KERNEL);
14752 if (!dd->boardname)
14753 return -ENOMEM;
14754 }
14755 return 0;
14756}
14757
14758
14759
14760
14761
14762
14763
14764
14765
14766static int check_int_registers(struct hfi1_devdata *dd)
14767{
14768 u64 reg;
14769 u64 all_bits = ~(u64)0;
14770 u64 mask;
14771
14772
14773 mask = read_csr(dd, CCE_INT_MASK);
14774 write_csr(dd, CCE_INT_MASK, 0ull);
14775 reg = read_csr(dd, CCE_INT_MASK);
14776 if (reg)
14777 goto err_exit;
14778
14779
14780 write_csr(dd, CCE_INT_CLEAR, all_bits);
14781 reg = read_csr(dd, CCE_INT_STATUS);
14782 if (reg)
14783 goto err_exit;
14784
14785
14786 write_csr(dd, CCE_INT_FORCE, all_bits);
14787 reg = read_csr(dd, CCE_INT_STATUS);
14788 if (reg != all_bits)
14789 goto err_exit;
14790
14791
14792 write_csr(dd, CCE_INT_CLEAR, all_bits);
14793 write_csr(dd, CCE_INT_MASK, mask);
14794
14795 return 0;
14796err_exit:
14797 write_csr(dd, CCE_INT_MASK, mask);
14798 dd_dev_err(dd, "Interrupt registers not properly mapped by VMM\n");
14799 return -EINVAL;
14800}
14801
14802
14803
14804
14805
14806
14807
14808
14809
14810int hfi1_init_dd(struct hfi1_devdata *dd)
14811{
14812 struct pci_dev *pdev = dd->pcidev;
14813 struct hfi1_pportdata *ppd;
14814 u64 reg;
14815 int i, ret;
14816 static const char * const inames[] = {
14817 "RTL silicon",
14818 "RTL VCS simulation",
14819 "RTL FPGA emulation",
14820 "Functional simulator"
14821 };
14822 struct pci_dev *parent = pdev->bus->self;
14823 u32 sdma_engines = chip_sdma_engines(dd);
14824
14825 ppd = dd->pport;
14826 for (i = 0; i < dd->num_pports; i++, ppd++) {
14827 int vl;
14828
14829 hfi1_init_pportdata(pdev, ppd, dd, 0, 1);
14830
14831 ppd->link_width_supported =
14832 OPA_LINK_WIDTH_1X | OPA_LINK_WIDTH_2X |
14833 OPA_LINK_WIDTH_3X | OPA_LINK_WIDTH_4X;
14834 ppd->link_width_downgrade_supported =
14835 ppd->link_width_supported;
14836
14837 ppd->link_width_enabled = OPA_LINK_WIDTH_4X;
14838 ppd->link_width_downgrade_enabled =
14839 ppd->link_width_downgrade_supported;
14840
14841
14842
14843 if (num_vls < HFI1_MIN_VLS_SUPPORTED ||
14844 num_vls > HFI1_MAX_VLS_SUPPORTED) {
14845 dd_dev_err(dd, "Invalid num_vls %u, using %u VLs\n",
14846 num_vls, HFI1_MAX_VLS_SUPPORTED);
14847 num_vls = HFI1_MAX_VLS_SUPPORTED;
14848 }
14849 ppd->vls_supported = num_vls;
14850 ppd->vls_operational = ppd->vls_supported;
14851
14852 for (vl = 0; vl < num_vls; vl++)
14853 dd->vld[vl].mtu = hfi1_max_mtu;
14854 dd->vld[15].mtu = MAX_MAD_PACKET;
14855
14856
14857
14858
14859 ppd->overrun_threshold = 0x4;
14860 ppd->phy_error_threshold = 0xf;
14861 ppd->port_crc_mode_enabled = link_crc_mask;
14862
14863 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
14864
14865 ppd->port_ltp_crc_mode |= cap_to_port_ltp(link_crc_mask) << 4;
14866
14867 ppd->host_link_state = HLS_DN_OFFLINE;
14868 init_vl_arb_caches(ppd);
14869 }
14870
14871
14872
14873
14874
14875
14876 ret = hfi1_pcie_ddinit(dd, pdev);
14877 if (ret < 0)
14878 goto bail_free;
14879
14880
14881 ret = save_pci_variables(dd);
14882 if (ret < 0)
14883 goto bail_cleanup;
14884
14885 dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT)
14886 & CCE_REVISION_CHIP_REV_MAJOR_MASK;
14887 dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT)
14888 & CCE_REVISION_CHIP_REV_MINOR_MASK;
14889
14890
14891
14892
14893
14894
14895 if (!parent) {
14896 ret = check_int_registers(dd);
14897 if (ret)
14898 goto bail_cleanup;
14899 }
14900
14901
14902
14903
14904
14905 reg = read_csr(dd, CCE_REVISION2);
14906 dd->hfi1_id = (reg >> CCE_REVISION2_HFI_ID_SHIFT)
14907 & CCE_REVISION2_HFI_ID_MASK;
14908
14909 dd->icode = reg >> CCE_REVISION2_IMPL_CODE_SHIFT;
14910 dd->irev = reg >> CCE_REVISION2_IMPL_REVISION_SHIFT;
14911 dd_dev_info(dd, "Implementation: %s, revision 0x%x\n",
14912 dd->icode < ARRAY_SIZE(inames) ?
14913 inames[dd->icode] : "unknown", (int)dd->irev);
14914
14915
14916 dd->pport->link_speed_supported = OPA_LINK_SPEED_25G;
14917
14918 dd->pport->link_speed_enabled = dd->pport->link_speed_supported;
14919
14920 dd->pport->link_speed_active = OPA_LINK_SPEED_25G;
14921
14922
14923 ppd = dd->pport;
14924 if (dd->icode == ICODE_FPGA_EMULATION && is_emulator_p(dd)) {
14925 ppd->link_width_supported =
14926 ppd->link_width_enabled =
14927 ppd->link_width_downgrade_supported =
14928 ppd->link_width_downgrade_enabled =
14929 OPA_LINK_WIDTH_1X;
14930 }
14931
14932 if (HFI1_CAP_IS_KSET(SDMA) && num_vls > sdma_engines) {
14933 dd_dev_err(dd, "num_vls %u too large, using %u VLs\n",
14934 num_vls, sdma_engines);
14935 num_vls = sdma_engines;
14936 ppd->vls_supported = sdma_engines;
14937 ppd->vls_operational = ppd->vls_supported;
14938 }
14939
14940
14941
14942
14943
14944
14945
14946
14947
14948 dd->rcv_intr_timeout_csr = ns_to_cclock(dd, rcv_intr_timeout) / 64;
14949 if (dd->rcv_intr_timeout_csr >
14950 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK)
14951 dd->rcv_intr_timeout_csr =
14952 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK;
14953 else if (dd->rcv_intr_timeout_csr == 0 && rcv_intr_timeout)
14954 dd->rcv_intr_timeout_csr = 1;
14955
14956
14957 read_guid(dd);
14958
14959
14960 ret = init_asic_data(dd);
14961 if (ret)
14962 goto bail_cleanup;
14963
14964
14965 ret = init_chip(dd);
14966 if (ret)
14967 goto bail_cleanup;
14968
14969
14970 ret = pcie_speeds(dd);
14971 if (ret)
14972 goto bail_cleanup;
14973
14974
14975 ret = eprom_init(dd);
14976 if (ret)
14977 goto bail_free_rcverr;
14978
14979
14980 get_platform_config(dd);
14981
14982
14983 ret = hfi1_firmware_init(dd);
14984 if (ret)
14985 goto bail_cleanup;
14986
14987
14988
14989
14990
14991
14992
14993
14994
14995
14996
14997
14998
14999 ret = do_pcie_gen3_transition(dd);
15000 if (ret)
15001 goto bail_cleanup;
15002
15003
15004
15005
15006
15007 tune_pcie_caps(dd);
15008
15009
15010 init_early_variables(dd);
15011
15012 parse_platform_config(dd);
15013
15014 ret = obtain_boardname(dd);
15015 if (ret)
15016 goto bail_cleanup;
15017
15018 snprintf(dd->boardversion, BOARD_VERS_MAX,
15019 "ChipABI %u.%u, ChipRev %u.%u, SW Compat %llu\n",
15020 HFI1_CHIP_VERS_MAJ, HFI1_CHIP_VERS_MIN,
15021 (u32)dd->majrev,
15022 (u32)dd->minrev,
15023 (dd->revision >> CCE_REVISION_SW_SHIFT)
15024 & CCE_REVISION_SW_MASK);
15025
15026 ret = set_up_context_variables(dd);
15027 if (ret)
15028 goto bail_cleanup;
15029
15030
15031 ret = init_rxe(dd);
15032 if (ret)
15033 goto bail_cleanup;
15034
15035
15036 init_txe(dd);
15037
15038 init_other(dd);
15039
15040 init_kdeth_qp(dd);
15041
15042 ret = hfi1_dev_affinity_init(dd);
15043 if (ret)
15044 goto bail_cleanup;
15045
15046
15047 ret = init_send_contexts(dd);
15048 if (ret)
15049 goto bail_cleanup;
15050
15051 ret = hfi1_create_kctxts(dd);
15052 if (ret)
15053 goto bail_cleanup;
15054
15055
15056
15057
15058
15059 aspm_init(dd);
15060
15061 ret = init_pervl_scs(dd);
15062 if (ret)
15063 goto bail_cleanup;
15064
15065
15066 for (i = 0; i < dd->num_pports; ++i) {
15067 ret = sdma_init(dd, i);
15068 if (ret)
15069 goto bail_cleanup;
15070 }
15071
15072
15073 ret = set_up_interrupts(dd);
15074 if (ret)
15075 goto bail_cleanup;
15076
15077 ret = hfi1_comp_vectors_set_up(dd);
15078 if (ret)
15079 goto bail_clear_intr;
15080
15081
15082 init_lcb_access(dd);
15083
15084
15085
15086
15087
15088
15089 snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n",
15090 (dd->base_guid & 0xFFFFFF) |
15091 ((dd->base_guid >> 11) & 0xF000000));
15092
15093 dd->oui1 = dd->base_guid >> 56 & 0xFF;
15094 dd->oui2 = dd->base_guid >> 48 & 0xFF;
15095 dd->oui3 = dd->base_guid >> 40 & 0xFF;
15096
15097 ret = load_firmware(dd);
15098 if (ret)
15099 goto bail_clear_intr;
15100
15101 thermal_init(dd);
15102
15103 ret = init_cntrs(dd);
15104 if (ret)
15105 goto bail_clear_intr;
15106
15107 ret = init_rcverr(dd);
15108 if (ret)
15109 goto bail_free_cntrs;
15110
15111 init_completion(&dd->user_comp);
15112
15113
15114 atomic_set(&dd->user_refcount, 1);
15115
15116 goto bail;
15117
15118bail_free_rcverr:
15119 free_rcverr(dd);
15120bail_free_cntrs:
15121 free_cntrs(dd);
15122bail_clear_intr:
15123 hfi1_comp_vectors_clean_up(dd);
15124 msix_clean_up_interrupts(dd);
15125bail_cleanup:
15126 hfi1_pcie_ddcleanup(dd);
15127bail_free:
15128 hfi1_free_devdata(dd);
15129bail:
15130 return ret;
15131}
15132
15133static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate,
15134 u32 dw_len)
15135{
15136 u32 delta_cycles;
15137 u32 current_egress_rate = ppd->current_egress_rate;
15138
15139
15140 if (desired_egress_rate == -1)
15141 return 0;
15142
15143 if (desired_egress_rate >= current_egress_rate)
15144 return 0;
15145
15146 delta_cycles = egress_cycles(dw_len * 4, desired_egress_rate) -
15147 egress_cycles(dw_len * 4, current_egress_rate);
15148
15149 return (u16)delta_cycles;
15150}
15151
15152
15153
15154
15155
15156
15157
15158
15159
15160
15161
15162
15163
15164
15165
15166u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl,
15167 u32 dw_len)
15168{
15169 u64 pbc, delay = 0;
15170
15171 if (unlikely(srate_mbs))
15172 delay = delay_cycles(ppd, srate_mbs, dw_len);
15173
15174 pbc = flags
15175 | (delay << PBC_STATIC_RATE_CONTROL_COUNT_SHIFT)
15176 | ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT)
15177 | (vl & PBC_VL_MASK) << PBC_VL_SHIFT
15178 | (dw_len & PBC_LENGTH_DWS_MASK)
15179 << PBC_LENGTH_DWS_SHIFT;
15180
15181 return pbc;
15182}
15183
15184#define SBUS_THERMAL 0x4f
15185#define SBUS_THERM_MONITOR_MODE 0x1
15186
15187#define THERM_FAILURE(dev, ret, reason) \
15188 dd_dev_err((dd), \
15189 "Thermal sensor initialization failed: %s (%d)\n", \
15190 (reason), (ret))
15191
15192
15193
15194
15195
15196
15197
15198
15199
15200
15201
15202static int thermal_init(struct hfi1_devdata *dd)
15203{
15204 int ret = 0;
15205
15206 if (dd->icode != ICODE_RTL_SILICON ||
15207 check_chip_resource(dd, CR_THERM_INIT, NULL))
15208 return ret;
15209
15210 ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);
15211 if (ret) {
15212 THERM_FAILURE(dd, ret, "Acquire SBus");
15213 return ret;
15214 }
15215
15216 dd_dev_info(dd, "Initializing thermal sensor\n");
15217
15218 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0);
15219 msleep(100);
15220
15221
15222 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
15223 RESET_SBUS_RECEIVER, 0);
15224 if (ret) {
15225 THERM_FAILURE(dd, ret, "Bus Reset");
15226 goto done;
15227 }
15228
15229 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
15230 WRITE_SBUS_RECEIVER, 0x1);
15231 if (ret) {
15232 THERM_FAILURE(dd, ret, "Therm Block Reset");
15233 goto done;
15234 }
15235
15236 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x1,
15237 WRITE_SBUS_RECEIVER, 0x32);
15238 if (ret) {
15239 THERM_FAILURE(dd, ret, "Write Clock Div");
15240 goto done;
15241 }
15242
15243 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x3,
15244 WRITE_SBUS_RECEIVER,
15245 SBUS_THERM_MONITOR_MODE);
15246 if (ret) {
15247 THERM_FAILURE(dd, ret, "Write Mode Sel");
15248 goto done;
15249 }
15250
15251 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
15252 WRITE_SBUS_RECEIVER, 0x2);
15253 if (ret) {
15254 THERM_FAILURE(dd, ret, "Write Reset Deassert");
15255 goto done;
15256 }
15257
15258 msleep(22);
15259
15260
15261 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1);
15262
15263
15264 ret = acquire_chip_resource(dd, CR_THERM_INIT, 0);
15265 if (ret)
15266 THERM_FAILURE(dd, ret, "Unable to set thermal init flag");
15267
15268done:
15269 release_chip_resource(dd, CR_SBUS);
15270 return ret;
15271}
15272
15273static void handle_temp_err(struct hfi1_devdata *dd)
15274{
15275 struct hfi1_pportdata *ppd = &dd->pport[0];
15276
15277
15278
15279
15280
15281 dd_dev_emerg(dd,
15282 "Critical temperature reached! Forcing device into freeze mode!\n");
15283 dd->flags |= HFI1_FORCED_FREEZE;
15284 start_freeze_handling(ppd, FREEZE_SELF | FREEZE_ABORT);
15285
15286
15287
15288
15289
15290
15291
15292
15293
15294
15295
15296 ppd->driver_link_ready = 0;
15297 ppd->link_enabled = 0;
15298 set_physical_link_state(dd, (OPA_LINKDOWN_REASON_SMA_DISABLED << 8) |
15299 PLS_OFFLINE);
15300
15301
15302
15303
15304 dc_shutdown(dd);
15305}
15306