1
2
3
4
5
6
7
8
9
10#ifndef __XEN_PUBLIC_IO_RING_H__
11#define __XEN_PUBLIC_IO_RING_H__
12
13#include <xen/interface/grant_table.h>
14
15typedef unsigned int RING_IDX;
16
17
18#define __RD2(_x) (((_x) & 0x00000002) ? 0x2 : ((_x) & 0x1))
19#define __RD4(_x) (((_x) & 0x0000000c) ? __RD2((_x)>>2)<<2 : __RD2(_x))
20#define __RD8(_x) (((_x) & 0x000000f0) ? __RD4((_x)>>4)<<4 : __RD4(_x))
21#define __RD16(_x) (((_x) & 0x0000ff00) ? __RD8((_x)>>8)<<8 : __RD8(_x))
22#define __RD32(_x) (((_x) & 0xffff0000) ? __RD16((_x)>>16)<<16 : __RD16(_x))
23
24
25
26
27
28
29
30#define __CONST_RING_SIZE(_s, _sz) \
31 (__RD32(((_sz) - offsetof(struct _s##_sring, ring)) / \
32 sizeof(((struct _s##_sring *)0)->ring[0])))
33
34
35
36
37#define __RING_SIZE(_s, _sz) \
38 (__RD32(((_sz) - (long)&(_s)->ring + (long)(_s)) / sizeof((_s)->ring[0])))
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74#define DEFINE_RING_TYPES(__name, __req_t, __rsp_t) \
75 \
76 \
77union __name##_sring_entry { \
78 __req_t req; \
79 __rsp_t rsp; \
80}; \
81 \
82 \
83struct __name##_sring { \
84 RING_IDX req_prod, req_event; \
85 RING_IDX rsp_prod, rsp_event; \
86 uint8_t pad[48]; \
87 union __name##_sring_entry ring[1]; \
88}; \
89 \
90 \
91struct __name##_front_ring { \
92 RING_IDX req_prod_pvt; \
93 RING_IDX rsp_cons; \
94 unsigned int nr_ents; \
95 struct __name##_sring *sring; \
96}; \
97 \
98 \
99struct __name##_back_ring { \
100 RING_IDX rsp_prod_pvt; \
101 RING_IDX req_cons; \
102 unsigned int nr_ents; \
103 struct __name##_sring *sring; \
104};
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122#define SHARED_RING_INIT(_s) do { \
123 (_s)->req_prod = (_s)->rsp_prod = 0; \
124 (_s)->req_event = (_s)->rsp_event = 1; \
125 memset((_s)->pad, 0, sizeof((_s)->pad)); \
126} while(0)
127
128#define FRONT_RING_ATTACH(_r, _s, _i, __size) do { \
129 (_r)->req_prod_pvt = (_i); \
130 (_r)->rsp_cons = (_i); \
131 (_r)->nr_ents = __RING_SIZE(_s, __size); \
132 (_r)->sring = (_s); \
133} while (0)
134
135#define FRONT_RING_INIT(_r, _s, __size) FRONT_RING_ATTACH(_r, _s, 0, __size)
136
137#define BACK_RING_ATTACH(_r, _s, _i, __size) do { \
138 (_r)->rsp_prod_pvt = (_i); \
139 (_r)->req_cons = (_i); \
140 (_r)->nr_ents = __RING_SIZE(_s, __size); \
141 (_r)->sring = (_s); \
142} while (0)
143
144#define BACK_RING_INIT(_r, _s, __size) BACK_RING_ATTACH(_r, _s, 0, __size)
145
146
147#define RING_SIZE(_r) \
148 ((_r)->nr_ents)
149
150
151#define RING_FREE_REQUESTS(_r) \
152 (RING_SIZE(_r) - ((_r)->req_prod_pvt - (_r)->rsp_cons))
153
154
155
156
157#define RING_FULL(_r) \
158 (RING_FREE_REQUESTS(_r) == 0)
159
160
161#define RING_HAS_UNCONSUMED_RESPONSES(_r) \
162 ((_r)->sring->rsp_prod - (_r)->rsp_cons)
163
164#define RING_HAS_UNCONSUMED_REQUESTS(_r) \
165 ({ \
166 unsigned int req = (_r)->sring->req_prod - (_r)->req_cons; \
167 unsigned int rsp = RING_SIZE(_r) - \
168 ((_r)->req_cons - (_r)->rsp_prod_pvt); \
169 req < rsp ? req : rsp; \
170 })
171
172
173#define RING_GET_REQUEST(_r, _idx) \
174 (&((_r)->sring->ring[((_idx) & (RING_SIZE(_r) - 1))].req))
175
176
177
178
179
180
181
182
183
184
185#define RING_COPY_REQUEST(_r, _idx, _req) do { \
186 \
187 *(_req) = *(volatile typeof(_req))RING_GET_REQUEST(_r, _idx); \
188} while (0)
189
190#define RING_GET_RESPONSE(_r, _idx) \
191 (&((_r)->sring->ring[((_idx) & (RING_SIZE(_r) - 1))].rsp))
192
193
194#define RING_REQUEST_CONS_OVERFLOW(_r, _cons) \
195 (((_cons) - (_r)->rsp_prod_pvt) >= RING_SIZE(_r))
196
197
198#define RING_REQUEST_PROD_OVERFLOW(_r, _prod) \
199 (((_prod) - (_r)->rsp_prod_pvt) > RING_SIZE(_r))
200
201
202#define RING_PUSH_REQUESTS(_r) do { \
203 virt_wmb(); \
204 (_r)->sring->req_prod = (_r)->req_prod_pvt; \
205} while (0)
206
207#define RING_PUSH_RESPONSES(_r) do { \
208 virt_wmb(); \
209 (_r)->sring->rsp_prod = (_r)->rsp_prod_pvt; \
210} while (0)
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242#define RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(_r, _notify) do { \
243 RING_IDX __old = (_r)->sring->req_prod; \
244 RING_IDX __new = (_r)->req_prod_pvt; \
245 virt_wmb(); \
246 (_r)->sring->req_prod = __new; \
247 virt_mb(); \
248 (_notify) = ((RING_IDX)(__new - (_r)->sring->req_event) < \
249 (RING_IDX)(__new - __old)); \
250} while (0)
251
252#define RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(_r, _notify) do { \
253 RING_IDX __old = (_r)->sring->rsp_prod; \
254 RING_IDX __new = (_r)->rsp_prod_pvt; \
255 virt_wmb(); \
256 (_r)->sring->rsp_prod = __new; \
257 virt_mb(); \
258 (_notify) = ((RING_IDX)(__new - (_r)->sring->rsp_event) < \
259 (RING_IDX)(__new - __old)); \
260} while (0)
261
262#define RING_FINAL_CHECK_FOR_REQUESTS(_r, _work_to_do) do { \
263 (_work_to_do) = RING_HAS_UNCONSUMED_REQUESTS(_r); \
264 if (_work_to_do) break; \
265 (_r)->sring->req_event = (_r)->req_cons + 1; \
266 virt_mb(); \
267 (_work_to_do) = RING_HAS_UNCONSUMED_REQUESTS(_r); \
268} while (0)
269
270#define RING_FINAL_CHECK_FOR_RESPONSES(_r, _work_to_do) do { \
271 (_work_to_do) = RING_HAS_UNCONSUMED_RESPONSES(_r); \
272 if (_work_to_do) break; \
273 (_r)->sring->rsp_event = (_r)->rsp_cons + 1; \
274 virt_mb(); \
275 (_work_to_do) = RING_HAS_UNCONSUMED_RESPONSES(_r); \
276} while (0)
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320#ifndef XEN_PAGE_SHIFT
321
322
323
324
325#define XEN_PAGE_SHIFT 12
326#endif
327#define XEN_FLEX_RING_SIZE(order) \
328 (1UL << ((order) + XEN_PAGE_SHIFT - 1))
329
330#define DEFINE_XEN_FLEX_RING(name) \
331static inline RING_IDX name##_mask(RING_IDX idx, RING_IDX ring_size) \
332{ \
333 return idx & (ring_size - 1); \
334} \
335 \
336static inline unsigned char *name##_get_ring_ptr(unsigned char *buf, \
337 RING_IDX idx, \
338 RING_IDX ring_size) \
339{ \
340 return buf + name##_mask(idx, ring_size); \
341} \
342 \
343static inline void name##_read_packet(void *opaque, \
344 const unsigned char *buf, \
345 size_t size, \
346 RING_IDX masked_prod, \
347 RING_IDX *masked_cons, \
348 RING_IDX ring_size) \
349{ \
350 if (*masked_cons < masked_prod || \
351 size <= ring_size - *masked_cons) { \
352 memcpy(opaque, buf + *masked_cons, size); \
353 } else { \
354 memcpy(opaque, buf + *masked_cons, ring_size - *masked_cons); \
355 memcpy((unsigned char *)opaque + ring_size - *masked_cons, buf, \
356 size - (ring_size - *masked_cons)); \
357 } \
358 *masked_cons = name##_mask(*masked_cons + size, ring_size); \
359} \
360 \
361static inline void name##_write_packet(unsigned char *buf, \
362 const void *opaque, \
363 size_t size, \
364 RING_IDX *masked_prod, \
365 RING_IDX masked_cons, \
366 RING_IDX ring_size) \
367{ \
368 if (*masked_prod < masked_cons || \
369 size <= ring_size - *masked_prod) { \
370 memcpy(buf + *masked_prod, opaque, size); \
371 } else { \
372 memcpy(buf + *masked_prod, opaque, ring_size - *masked_prod); \
373 memcpy(buf, (unsigned char *)opaque + (ring_size - *masked_prod), \
374 size - (ring_size - *masked_prod)); \
375 } \
376 *masked_prod = name##_mask(*masked_prod + size, ring_size); \
377} \
378 \
379static inline RING_IDX name##_queued(RING_IDX prod, \
380 RING_IDX cons, \
381 RING_IDX ring_size) \
382{ \
383 RING_IDX size; \
384 \
385 if (prod == cons) \
386 return 0; \
387 \
388 prod = name##_mask(prod, ring_size); \
389 cons = name##_mask(cons, ring_size); \
390 \
391 if (prod == cons) \
392 return ring_size; \
393 \
394 if (prod > cons) \
395 size = prod - cons; \
396 else \
397 size = ring_size - (cons - prod); \
398 return size; \
399} \
400 \
401struct name##_data { \
402 unsigned char *in; \
403 unsigned char *out; \
404}
405
406#define DEFINE_XEN_FLEX_RING_AND_INTF(name) \
407struct name##_data_intf { \
408 RING_IDX in_cons, in_prod; \
409 \
410 uint8_t pad1[56]; \
411 \
412 RING_IDX out_cons, out_prod; \
413 \
414 uint8_t pad2[56]; \
415 \
416 RING_IDX ring_order; \
417 grant_ref_t ref[]; \
418}; \
419DEFINE_XEN_FLEX_RING(name)
420
421#endif
422