1#ifndef _HFI1_USER_SDMA_H
2#define _HFI1_USER_SDMA_H
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49#include <linux/device.h>
50#include <linux/wait.h>
51
52#include "common.h"
53#include "iowait.h"
54#include "user_exp_rcv.h"
55
56
57#define MAX_VECTORS_PER_REQ 8
58
59
60
61
62#define MAX_PKTS_PER_QUEUE 16
63
64#define num_pages(x) (1 + ((((x) - 1) & PAGE_MASK) >> PAGE_SHIFT))
65
66#define req_opcode(x) \
67 (((x) >> HFI1_SDMA_REQ_OPCODE_SHIFT) & HFI1_SDMA_REQ_OPCODE_MASK)
68#define req_version(x) \
69 (((x) >> HFI1_SDMA_REQ_VERSION_SHIFT) & HFI1_SDMA_REQ_OPCODE_MASK)
70#define req_iovcnt(x) \
71 (((x) >> HFI1_SDMA_REQ_IOVCNT_SHIFT) & HFI1_SDMA_REQ_IOVCNT_MASK)
72
73
74#define BTH_SEQ_MASK 0x7ffull
75
76#define AHG_KDETH_INTR_SHIFT 12
77#define AHG_KDETH_SH_SHIFT 13
78#define AHG_KDETH_ARRAY_SIZE 9
79
80#define PBC2LRH(x) ((((x) & 0xfff) << 2) - 4)
81#define LRH2PBC(x) ((((x) >> 2) + 1) & 0xfff)
82
83#define AHG_HEADER_SET(arr, idx, dw, bit, width, value) \
84 do { \
85 if ((idx) < ARRAY_SIZE((arr))) \
86 (arr)[(idx++)] = sdma_build_ahg_descriptor( \
87 (__force u16)(value), (dw), (bit), \
88 (width)); \
89 else \
90 return -ERANGE; \
91 } while (0)
92
93
94#define TXREQ_FLAGS_REQ_ACK BIT(0)
95#define TXREQ_FLAGS_REQ_DISABLE_SH BIT(1)
96
97#define SDMA_PKT_Q_INACTIVE BIT(0)
98#define SDMA_PKT_Q_ACTIVE BIT(1)
99#define SDMA_PKT_Q_DEFERRED BIT(2)
100
101
102
103
104
105#define MAX_DEFER_RETRY_COUNT 1
106
107#define SDMA_IOWAIT_TIMEOUT 1000
108
109#define SDMA_DBG(req, fmt, ...) \
110 hfi1_cdbg(SDMA, "[%u:%u:%u:%u] " fmt, (req)->pq->dd->unit, \
111 (req)->pq->ctxt, (req)->pq->subctxt, (req)->info.comp_idx, \
112 ##__VA_ARGS__)
113
114extern uint extended_psn;
115
116struct hfi1_user_sdma_pkt_q {
117 u16 ctxt;
118 u16 subctxt;
119 u16 n_max_reqs;
120 atomic_t n_reqs;
121 u16 reqidx;
122 struct hfi1_devdata *dd;
123 struct kmem_cache *txreq_cache;
124 struct user_sdma_request *reqs;
125 unsigned long *req_in_use;
126 struct iowait busy;
127 unsigned state;
128 wait_queue_head_t wait;
129 unsigned long unpinned;
130 struct mmu_rb_handler *handler;
131 atomic_t n_locked;
132 struct mm_struct *mm;
133};
134
135struct hfi1_user_sdma_comp_q {
136 u16 nentries;
137 struct hfi1_sdma_comp_entry *comps;
138};
139
140struct sdma_mmu_node {
141 struct mmu_rb_node rb;
142 struct hfi1_user_sdma_pkt_q *pq;
143 atomic_t refcount;
144 struct page **pages;
145 unsigned int npages;
146};
147
148struct user_sdma_iovec {
149 struct list_head list;
150 struct iovec iov;
151
152 unsigned int npages;
153
154 struct page **pages;
155
156
157
158
159 u64 offset;
160 struct sdma_mmu_node *node;
161};
162
163
164struct evict_data {
165 u32 cleared;
166 u32 target;
167};
168
169struct user_sdma_request {
170
171 struct hfi1_pkt_header hdr;
172
173
174 struct hfi1_user_sdma_pkt_q *pq ____cacheline_aligned_in_smp;
175 struct hfi1_user_sdma_comp_q *cq;
176
177
178
179
180
181 struct sdma_engine *sde;
182 struct sdma_req_info info;
183
184 u32 *tids;
185
186 u32 data_len;
187
188 u16 n_tids;
189
190
191
192
193 u8 data_iovs;
194 s8 ahg_idx;
195
196
197 u64 seqcomp ____cacheline_aligned_in_smp;
198 u64 seqsubmitted;
199
200 int status;
201
202
203 struct list_head txps ____cacheline_aligned_in_smp;
204 u64 seqnum;
205
206
207
208
209
210 u32 tidoffset;
211
212
213
214
215
216 u32 koffset;
217 u32 sent;
218
219 u16 tididx;
220
221 u8 iov_idx;
222 u8 done;
223 u8 has_error;
224
225 struct user_sdma_iovec iovs[MAX_VECTORS_PER_REQ];
226} ____cacheline_aligned_in_smp;
227
228
229
230
231
232
233
234struct user_sdma_txreq {
235
236 struct hfi1_pkt_header hdr;
237 struct sdma_txreq txreq;
238 struct list_head list;
239 struct user_sdma_request *req;
240 u16 flags;
241 unsigned int busycount;
242 u64 seqnum;
243};
244
245int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt,
246 struct hfi1_filedata *fd);
247int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd,
248 struct hfi1_ctxtdata *uctxt);
249int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
250 struct iovec *iovec, unsigned long dim,
251 unsigned long *count);
252
253#endif
254