1
2
3
4
5
6
7
8
9#ifndef __XEN_PUBLIC_IO_RING_H__
10#define __XEN_PUBLIC_IO_RING_H__
11
12typedef unsigned int RING_IDX;
13
14
15#define __RD2(_x) (((_x) & 0x00000002) ? 0x2 : ((_x) & 0x1))
16#define __RD4(_x) (((_x) & 0x0000000c) ? __RD2((_x)>>2)<<2 : __RD2(_x))
17#define __RD8(_x) (((_x) & 0x000000f0) ? __RD4((_x)>>4)<<4 : __RD4(_x))
18#define __RD16(_x) (((_x) & 0x0000ff00) ? __RD8((_x)>>8)<<8 : __RD8(_x))
19#define __RD32(_x) (((_x) & 0xffff0000) ? __RD16((_x)>>16)<<16 : __RD16(_x))
20
21
22
23
24
25
26
27#define __CONST_RING_SIZE(_s, _sz) \
28 (__RD32(((_sz) - offsetof(struct _s##_sring, ring)) / \
29 sizeof(((struct _s##_sring *)0)->ring[0])))
30
31
32
33
34#define __RING_SIZE(_s, _sz) \
35 (__RD32(((_sz) - (long)&(_s)->ring + (long)(_s)) / sizeof((_s)->ring[0])))
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71#define DEFINE_RING_TYPES(__name, __req_t, __rsp_t) \
72 \
73 \
74union __name##_sring_entry { \
75 __req_t req; \
76 __rsp_t rsp; \
77}; \
78 \
79 \
80struct __name##_sring { \
81 RING_IDX req_prod, req_event; \
82 RING_IDX rsp_prod, rsp_event; \
83 uint8_t pad[48]; \
84 union __name##_sring_entry ring[1]; \
85}; \
86 \
87 \
88struct __name##_front_ring { \
89 RING_IDX req_prod_pvt; \
90 RING_IDX rsp_cons; \
91 unsigned int nr_ents; \
92 struct __name##_sring *sring; \
93}; \
94 \
95 \
96struct __name##_back_ring { \
97 RING_IDX rsp_prod_pvt; \
98 RING_IDX req_cons; \
99 unsigned int nr_ents; \
100 struct __name##_sring *sring; \
101};
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119#define SHARED_RING_INIT(_s) do { \
120 (_s)->req_prod = (_s)->rsp_prod = 0; \
121 (_s)->req_event = (_s)->rsp_event = 1; \
122 memset((_s)->pad, 0, sizeof((_s)->pad)); \
123} while(0)
124
125#define FRONT_RING_INIT(_r, _s, __size) do { \
126 (_r)->req_prod_pvt = 0; \
127 (_r)->rsp_cons = 0; \
128 (_r)->nr_ents = __RING_SIZE(_s, __size); \
129 (_r)->sring = (_s); \
130} while (0)
131
132#define BACK_RING_INIT(_r, _s, __size) do { \
133 (_r)->rsp_prod_pvt = 0; \
134 (_r)->req_cons = 0; \
135 (_r)->nr_ents = __RING_SIZE(_s, __size); \
136 (_r)->sring = (_s); \
137} while (0)
138
139
140#define FRONT_RING_ATTACH(_r, _s, __size) do { \
141 (_r)->sring = (_s); \
142 (_r)->req_prod_pvt = (_s)->req_prod; \
143 (_r)->rsp_cons = (_s)->rsp_prod; \
144 (_r)->nr_ents = __RING_SIZE(_s, __size); \
145} while (0)
146
147#define BACK_RING_ATTACH(_r, _s, __size) do { \
148 (_r)->sring = (_s); \
149 (_r)->rsp_prod_pvt = (_s)->rsp_prod; \
150 (_r)->req_cons = (_s)->req_prod; \
151 (_r)->nr_ents = __RING_SIZE(_s, __size); \
152} while (0)
153
154
155#define RING_SIZE(_r) \
156 ((_r)->nr_ents)
157
158
159#define RING_FREE_REQUESTS(_r) \
160 (RING_SIZE(_r) - ((_r)->req_prod_pvt - (_r)->rsp_cons))
161
162
163
164
165#define RING_FULL(_r) \
166 (RING_FREE_REQUESTS(_r) == 0)
167
168
169#define RING_HAS_UNCONSUMED_RESPONSES(_r) \
170 ((_r)->sring->rsp_prod - (_r)->rsp_cons)
171
172#define RING_HAS_UNCONSUMED_REQUESTS(_r) \
173 ({ \
174 unsigned int req = (_r)->sring->req_prod - (_r)->req_cons; \
175 unsigned int rsp = RING_SIZE(_r) - \
176 ((_r)->req_cons - (_r)->rsp_prod_pvt); \
177 req < rsp ? req : rsp; \
178 })
179
180
181#define RING_GET_REQUEST(_r, _idx) \
182 (&((_r)->sring->ring[((_idx) & (RING_SIZE(_r) - 1))].req))
183
184
185
186
187
188
189
190
191
192
193#define RING_COPY_REQUEST(_r, _idx, _req) do { \
194 \
195 *(_req) = *(volatile typeof(_req))RING_GET_REQUEST(_r, _idx); \
196} while (0)
197
198#define RING_GET_RESPONSE(_r, _idx) \
199 (&((_r)->sring->ring[((_idx) & (RING_SIZE(_r) - 1))].rsp))
200
201
202#define RING_REQUEST_CONS_OVERFLOW(_r, _cons) \
203 (((_cons) - (_r)->rsp_prod_pvt) >= RING_SIZE(_r))
204
205
206#define RING_REQUEST_PROD_OVERFLOW(_r, _prod) \
207 (((_prod) - (_r)->rsp_prod_pvt) > RING_SIZE(_r))
208
209
210#define RING_PUSH_REQUESTS(_r) do { \
211 virt_wmb(); \
212 (_r)->sring->req_prod = (_r)->req_prod_pvt; \
213} while (0)
214
215#define RING_PUSH_RESPONSES(_r) do { \
216 virt_wmb(); \
217 (_r)->sring->rsp_prod = (_r)->rsp_prod_pvt; \
218} while (0)
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250#define RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(_r, _notify) do { \
251 RING_IDX __old = (_r)->sring->req_prod; \
252 RING_IDX __new = (_r)->req_prod_pvt; \
253 virt_wmb(); \
254 (_r)->sring->req_prod = __new; \
255 virt_mb(); \
256 (_notify) = ((RING_IDX)(__new - (_r)->sring->req_event) < \
257 (RING_IDX)(__new - __old)); \
258} while (0)
259
260#define RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(_r, _notify) do { \
261 RING_IDX __old = (_r)->sring->rsp_prod; \
262 RING_IDX __new = (_r)->rsp_prod_pvt; \
263 virt_wmb(); \
264 (_r)->sring->rsp_prod = __new; \
265 virt_mb(); \
266 (_notify) = ((RING_IDX)(__new - (_r)->sring->rsp_event) < \
267 (RING_IDX)(__new - __old)); \
268} while (0)
269
270#define RING_FINAL_CHECK_FOR_REQUESTS(_r, _work_to_do) do { \
271 (_work_to_do) = RING_HAS_UNCONSUMED_REQUESTS(_r); \
272 if (_work_to_do) break; \
273 (_r)->sring->req_event = (_r)->req_cons + 1; \
274 virt_mb(); \
275 (_work_to_do) = RING_HAS_UNCONSUMED_REQUESTS(_r); \
276} while (0)
277
278#define RING_FINAL_CHECK_FOR_RESPONSES(_r, _work_to_do) do { \
279 (_work_to_do) = RING_HAS_UNCONSUMED_RESPONSES(_r); \
280 if (_work_to_do) break; \
281 (_r)->sring->rsp_event = (_r)->rsp_cons + 1; \
282 virt_mb(); \
283 (_work_to_do) = RING_HAS_UNCONSUMED_RESPONSES(_r); \
284} while (0)
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328#ifndef XEN_PAGE_SHIFT
329
330
331
332
333#define XEN_PAGE_SHIFT 12
334#endif
335#define XEN_FLEX_RING_SIZE(order) \
336 (1UL << ((order) + XEN_PAGE_SHIFT - 1))
337
338#define DEFINE_XEN_FLEX_RING(name) \
339static inline RING_IDX name##_mask(RING_IDX idx, RING_IDX ring_size) \
340{ \
341 return idx & (ring_size - 1); \
342} \
343 \
344static inline unsigned char *name##_get_ring_ptr(unsigned char *buf, \
345 RING_IDX idx, \
346 RING_IDX ring_size) \
347{ \
348 return buf + name##_mask(idx, ring_size); \
349} \
350 \
351static inline void name##_read_packet(void *opaque, \
352 const unsigned char *buf, \
353 size_t size, \
354 RING_IDX masked_prod, \
355 RING_IDX *masked_cons, \
356 RING_IDX ring_size) \
357{ \
358 if (*masked_cons < masked_prod || \
359 size <= ring_size - *masked_cons) { \
360 memcpy(opaque, buf + *masked_cons, size); \
361 } else { \
362 memcpy(opaque, buf + *masked_cons, ring_size - *masked_cons); \
363 memcpy((unsigned char *)opaque + ring_size - *masked_cons, buf, \
364 size - (ring_size - *masked_cons)); \
365 } \
366 *masked_cons = name##_mask(*masked_cons + size, ring_size); \
367} \
368 \
369static inline void name##_write_packet(unsigned char *buf, \
370 const void *opaque, \
371 size_t size, \
372 RING_IDX *masked_prod, \
373 RING_IDX masked_cons, \
374 RING_IDX ring_size) \
375{ \
376 if (*masked_prod < masked_cons || \
377 size <= ring_size - *masked_prod) { \
378 memcpy(buf + *masked_prod, opaque, size); \
379 } else { \
380 memcpy(buf + *masked_prod, opaque, ring_size - *masked_prod); \
381 memcpy(buf, (unsigned char *)opaque + (ring_size - *masked_prod), \
382 size - (ring_size - *masked_prod)); \
383 } \
384 *masked_prod = name##_mask(*masked_prod + size, ring_size); \
385} \
386 \
387static inline RING_IDX name##_queued(RING_IDX prod, \
388 RING_IDX cons, \
389 RING_IDX ring_size) \
390{ \
391 RING_IDX size; \
392 \
393 if (prod == cons) \
394 return 0; \
395 \
396 prod = name##_mask(prod, ring_size); \
397 cons = name##_mask(cons, ring_size); \
398 \
399 if (prod == cons) \
400 return ring_size; \
401 \
402 if (prod > cons) \
403 size = prod - cons; \
404 else \
405 size = ring_size - (cons - prod); \
406 return size; \
407} \
408 \
409struct name##_data { \
410 unsigned char *in; \
411 unsigned char *out; \
412}
413
414#define DEFINE_XEN_FLEX_RING_AND_INTF(name) \
415struct name##_data_intf { \
416 RING_IDX in_cons, in_prod; \
417 \
418 uint8_t pad1[56]; \
419 \
420 RING_IDX out_cons, out_prod; \
421 \
422 uint8_t pad2[56]; \
423 \
424 RING_IDX ring_order; \
425 grant_ref_t ref[]; \
426}; \
427DEFINE_XEN_FLEX_RING(name)
428
429#endif
430