1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#ifndef __XEN_PUBLIC_IO_RING_H__
28#define __XEN_PUBLIC_IO_RING_H__
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45#if __XEN_INTERFACE_VERSION__ < 0x00030208
46#define xen_mb() mb()
47#define xen_rmb() rmb()
48#define xen_wmb() wmb()
49#endif
50
51typedef unsigned int RING_IDX;
52
53
54#define __RD2(_x) (((_x) & 0x00000002) ? 0x2 : ((_x) & 0x1))
55#define __RD4(_x) (((_x) & 0x0000000c) ? __RD2((_x)>>2)<<2 : __RD2(_x))
56#define __RD8(_x) (((_x) & 0x000000f0) ? __RD4((_x)>>4)<<4 : __RD4(_x))
57#define __RD16(_x) (((_x) & 0x0000ff00) ? __RD8((_x)>>8)<<8 : __RD8(_x))
58#define __RD32(_x) (((_x) & 0xffff0000) ? __RD16((_x)>>16)<<16 : __RD16(_x))
59
60
61
62
63
64
65
66#define __CONST_RING_SIZE(_s, _sz) \
67 (__RD32(((_sz) - offsetof(struct _s##_sring, ring)) / \
68 sizeof_field(struct _s##_sring, ring[0])))
69
70
71
72#define __RING_SIZE(_s, _sz) \
73 (__RD32(((_sz) - (long)(_s)->ring + (long)(_s)) / sizeof((_s)->ring[0])))
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107#define DEFINE_RING_TYPES(__name, __req_t, __rsp_t) \
108 \
109 \
110union __name##_sring_entry { \
111 __req_t req; \
112 __rsp_t rsp; \
113}; \
114 \
115 \
116struct __name##_sring { \
117 RING_IDX req_prod, req_event; \
118 RING_IDX rsp_prod, rsp_event; \
119 union { \
120 struct { \
121 uint8_t smartpoll_active; \
122 } netif; \
123 struct { \
124 uint8_t msg; \
125 } tapif_user; \
126 uint8_t pvt_pad[4]; \
127 } pvt; \
128 uint8_t __pad[44]; \
129 union __name##_sring_entry ring[1]; \
130}; \
131 \
132 \
133struct __name##_front_ring { \
134 RING_IDX req_prod_pvt; \
135 RING_IDX rsp_cons; \
136 unsigned int nr_ents; \
137 struct __name##_sring *sring; \
138}; \
139 \
140 \
141struct __name##_back_ring { \
142 RING_IDX rsp_prod_pvt; \
143 RING_IDX req_cons; \
144 unsigned int nr_ents; \
145 struct __name##_sring *sring; \
146}; \
147 \
148 \
149typedef struct __name##_sring __name##_sring_t; \
150typedef struct __name##_front_ring __name##_front_ring_t; \
151typedef struct __name##_back_ring __name##_back_ring_t
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169#define SHARED_RING_INIT(_s) do { \
170 (_s)->req_prod = (_s)->rsp_prod = 0; \
171 (_s)->req_event = (_s)->rsp_event = 1; \
172 (void)memset((_s)->pvt.pvt_pad, 0, sizeof((_s)->pvt.pvt_pad)); \
173 (void)memset((_s)->__pad, 0, sizeof((_s)->__pad)); \
174} while(0)
175
176#define FRONT_RING_INIT(_r, _s, __size) do { \
177 (_r)->req_prod_pvt = 0; \
178 (_r)->rsp_cons = 0; \
179 (_r)->nr_ents = __RING_SIZE(_s, __size); \
180 (_r)->sring = (_s); \
181} while (0)
182
183#define BACK_RING_INIT(_r, _s, __size) do { \
184 (_r)->rsp_prod_pvt = 0; \
185 (_r)->req_cons = 0; \
186 (_r)->nr_ents = __RING_SIZE(_s, __size); \
187 (_r)->sring = (_s); \
188} while (0)
189
190
191#define RING_SIZE(_r) \
192 ((_r)->nr_ents)
193
194
195#define RING_FREE_REQUESTS(_r) \
196 (RING_SIZE(_r) - ((_r)->req_prod_pvt - (_r)->rsp_cons))
197
198
199
200
201#define RING_FULL(_r) \
202 (RING_FREE_REQUESTS(_r) == 0)
203
204
205#define RING_HAS_UNCONSUMED_RESPONSES(_r) \
206 ((_r)->sring->rsp_prod - (_r)->rsp_cons)
207
208#ifdef __GNUC__
209#define RING_HAS_UNCONSUMED_REQUESTS(_r) ({ \
210 unsigned int req = (_r)->sring->req_prod - (_r)->req_cons; \
211 unsigned int rsp = RING_SIZE(_r) - \
212 ((_r)->req_cons - (_r)->rsp_prod_pvt); \
213 req < rsp ? req : rsp; \
214})
215#else
216
217#define RING_HAS_UNCONSUMED_REQUESTS(_r) \
218 ((((_r)->sring->req_prod - (_r)->req_cons) < \
219 (RING_SIZE(_r) - ((_r)->req_cons - (_r)->rsp_prod_pvt))) ? \
220 ((_r)->sring->req_prod - (_r)->req_cons) : \
221 (RING_SIZE(_r) - ((_r)->req_cons - (_r)->rsp_prod_pvt)))
222#endif
223
224
225#define RING_GET_REQUEST(_r, _idx) \
226 (&((_r)->sring->ring[((_idx) & (RING_SIZE(_r) - 1))].req))
227
228
229
230
231
232
233
234
235
236
237#define RING_COPY_REQUEST(_r, _idx, _req) do { \
238 \
239 *(_req) = *(volatile typeof(_req))RING_GET_REQUEST(_r, _idx); \
240} while (0)
241
242#define RING_GET_RESPONSE(_r, _idx) \
243 (&((_r)->sring->ring[((_idx) & (RING_SIZE(_r) - 1))].rsp))
244
245
246#define RING_REQUEST_CONS_OVERFLOW(_r, _cons) \
247 (((_cons) - (_r)->rsp_prod_pvt) >= RING_SIZE(_r))
248
249
250#define RING_REQUEST_PROD_OVERFLOW(_r, _prod) \
251 (((_prod) - (_r)->rsp_prod_pvt) > RING_SIZE(_r))
252
253#define RING_PUSH_REQUESTS(_r) do { \
254 xen_wmb(); \
255 (_r)->sring->req_prod = (_r)->req_prod_pvt; \
256} while (0)
257
258#define RING_PUSH_RESPONSES(_r) do { \
259 xen_wmb(); \
260 (_r)->sring->rsp_prod = (_r)->rsp_prod_pvt; \
261} while (0)
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293#define RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(_r, _notify) do { \
294 RING_IDX __old = (_r)->sring->req_prod; \
295 RING_IDX __new = (_r)->req_prod_pvt; \
296 xen_wmb(); \
297 (_r)->sring->req_prod = __new; \
298 xen_mb(); \
299 (_notify) = ((RING_IDX)(__new - (_r)->sring->req_event) < \
300 (RING_IDX)(__new - __old)); \
301} while (0)
302
303#define RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(_r, _notify) do { \
304 RING_IDX __old = (_r)->sring->rsp_prod; \
305 RING_IDX __new = (_r)->rsp_prod_pvt; \
306 xen_wmb(); \
307 (_r)->sring->rsp_prod = __new; \
308 xen_mb(); \
309 (_notify) = ((RING_IDX)(__new - (_r)->sring->rsp_event) < \
310 (RING_IDX)(__new - __old)); \
311} while (0)
312
313#define RING_FINAL_CHECK_FOR_REQUESTS(_r, _work_to_do) do { \
314 (_work_to_do) = RING_HAS_UNCONSUMED_REQUESTS(_r); \
315 if (_work_to_do) break; \
316 (_r)->sring->req_event = (_r)->req_cons + 1; \
317 xen_mb(); \
318 (_work_to_do) = RING_HAS_UNCONSUMED_REQUESTS(_r); \
319} while (0)
320
321#define RING_FINAL_CHECK_FOR_RESPONSES(_r, _work_to_do) do { \
322 (_work_to_do) = RING_HAS_UNCONSUMED_RESPONSES(_r); \
323 if (_work_to_do) break; \
324 (_r)->sring->rsp_event = (_r)->rsp_cons + 1; \
325 xen_mb(); \
326 (_work_to_do) = RING_HAS_UNCONSUMED_RESPONSES(_r); \
327} while (0)
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371#ifndef XEN_PAGE_SHIFT
372
373
374
375
376#define XEN_PAGE_SHIFT 12
377#endif
378#define XEN_FLEX_RING_SIZE(order) \
379 (1UL << ((order) + XEN_PAGE_SHIFT - 1))
380
381#define DEFINE_XEN_FLEX_RING(name) \
382static inline RING_IDX name##_mask(RING_IDX idx, RING_IDX ring_size) \
383{ \
384 return idx & (ring_size - 1); \
385} \
386 \
387static inline unsigned char *name##_get_ring_ptr(unsigned char *buf, \
388 RING_IDX idx, \
389 RING_IDX ring_size) \
390{ \
391 return buf + name##_mask(idx, ring_size); \
392} \
393 \
394static inline void name##_read_packet(void *opaque, \
395 const unsigned char *buf, \
396 size_t size, \
397 RING_IDX masked_prod, \
398 RING_IDX *masked_cons, \
399 RING_IDX ring_size) \
400{ \
401 if (*masked_cons < masked_prod || \
402 size <= ring_size - *masked_cons) { \
403 memcpy(opaque, buf + *masked_cons, size); \
404 } else { \
405 memcpy(opaque, buf + *masked_cons, ring_size - *masked_cons); \
406 memcpy((unsigned char *)opaque + ring_size - *masked_cons, buf, \
407 size - (ring_size - *masked_cons)); \
408 } \
409 *masked_cons = name##_mask(*masked_cons + size, ring_size); \
410} \
411 \
412static inline void name##_write_packet(unsigned char *buf, \
413 const void *opaque, \
414 size_t size, \
415 RING_IDX *masked_prod, \
416 RING_IDX masked_cons, \
417 RING_IDX ring_size) \
418{ \
419 if (*masked_prod < masked_cons || \
420 size <= ring_size - *masked_prod) { \
421 memcpy(buf + *masked_prod, opaque, size); \
422 } else { \
423 memcpy(buf + *masked_prod, opaque, ring_size - *masked_prod); \
424 memcpy(buf, (unsigned char *)opaque + (ring_size - *masked_prod), \
425 size - (ring_size - *masked_prod)); \
426 } \
427 *masked_prod = name##_mask(*masked_prod + size, ring_size); \
428} \
429 \
430static inline RING_IDX name##_queued(RING_IDX prod, \
431 RING_IDX cons, \
432 RING_IDX ring_size) \
433{ \
434 RING_IDX size; \
435 \
436 if (prod == cons) \
437 return 0; \
438 \
439 prod = name##_mask(prod, ring_size); \
440 cons = name##_mask(cons, ring_size); \
441 \
442 if (prod == cons) \
443 return ring_size; \
444 \
445 if (prod > cons) \
446 size = prod - cons; \
447 else \
448 size = ring_size - (cons - prod); \
449 return size; \
450} \
451 \
452struct name##_data { \
453 unsigned char *in; \
454 unsigned char *out; \
455}
456
457#define DEFINE_XEN_FLEX_RING_AND_INTF(name) \
458struct name##_data_intf { \
459 RING_IDX in_cons, in_prod; \
460 \
461 uint8_t pad1[56]; \
462 \
463 RING_IDX out_cons, out_prod; \
464 \
465 uint8_t pad2[56]; \
466 \
467 RING_IDX ring_order; \
468 grant_ref_t ref[]; \
469}; \
470DEFINE_XEN_FLEX_RING(name)
471
472#endif
473
474
475
476
477
478
479
480
481
482
483