1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/kernel.h>
25#include <linux/mm.h>
26#include "osd.h"
27#include "logging.h"
28#include "ring_buffer.h"
29
30
31
32
33
34
35#define BYTES_AVAIL_TO_WRITE(r, w, z) ((w) >= (r)) ? ((z) - ((w) - (r))) : ((r) - (w))
36
37
38
39
40
41
42
43
44
45
46
47
48static inline void
49get_ringbuffer_availbytes(struct hv_ring_buffer_info *rbi,
50 u32 *read, u32 *write)
51{
52 u32 read_loc, write_loc;
53
54
55 read_loc = rbi->ring_buffer->read_index;
56 write_loc = rbi->ring_buffer->write_index;
57
58 *write = BYTES_AVAIL_TO_WRITE(read_loc, write_loc, rbi->ring_datasize);
59 *read = rbi->ring_datasize - *write;
60}
61
62
63
64
65
66
67
68
69
70
71static inline u32
72get_next_write_location(struct hv_ring_buffer_info *ring_info)
73{
74 u32 next = ring_info->ring_buffer->write_index;
75
76
77
78 return next;
79}
80
81
82
83
84
85
86
87
88
89
90static inline void
91set_next_write_location(struct hv_ring_buffer_info *ring_info,
92 u32 next_write_location)
93{
94 ring_info->ring_buffer->write_index = next_write_location;
95}
96
97
98
99
100
101
102
103
104
105
106static inline u32
107get_next_read_location(struct hv_ring_buffer_info *ring_info)
108{
109 u32 next = ring_info->ring_buffer->read_index;
110
111
112
113 return next;
114}
115
116
117
118
119
120
121
122
123
124
125
126static inline u32
127get_next_readlocation_withoffset(struct hv_ring_buffer_info *ring_info,
128 u32 offset)
129{
130 u32 next = ring_info->ring_buffer->read_index;
131
132
133 next += offset;
134 next %= ring_info->ring_datasize;
135
136 return next;
137}
138
139
140
141
142
143
144
145
146
147
148static inline void
149set_next_read_location(struct hv_ring_buffer_info *ring_info,
150 u32 next_read_location)
151{
152 ring_info->ring_buffer->read_index = next_read_location;
153}
154
155
156
157
158
159
160
161
162
163
164
165static inline void *
166get_ring_buffer(struct hv_ring_buffer_info *ring_info)
167{
168 return (void *)ring_info->ring_buffer->buffer;
169}
170
171
172
173
174
175
176
177
178
179
180
181static inline u32
182get_ring_buffersize(struct hv_ring_buffer_info *ring_info)
183{
184 return ring_info->ring_datasize;
185}
186
187
188
189
190
191
192
193
194
195
196static inline u64
197get_ring_bufferindices(struct hv_ring_buffer_info *ring_info)
198{
199 return (u64)ring_info->ring_buffer->write_index << 32;
200}
201
202
203
204
205
206
207
208
209
210
211
212void dump_ring_info(struct hv_ring_buffer_info *ring_info, char *prefix)
213{
214 u32 bytes_avail_towrite;
215 u32 bytes_avail_toread;
216
217 get_ringbuffer_availbytes(ring_info,
218 &bytes_avail_toread,
219 &bytes_avail_towrite);
220
221 DPRINT(VMBUS,
222 DEBUG_RING_LVL,
223 "%s <<ringinfo %p buffer %p avail write %u "
224 "avail read %u read idx %u write idx %u>>",
225 prefix,
226 ring_info,
227 ring_info->ring_buffer->buffer,
228 bytes_avail_towrite,
229 bytes_avail_toread,
230 ring_info->ring_buffer->read_index,
231 ring_info->ring_buffer->write_index);
232}
233
234
235
236
237static u32
238copyto_ringbuffer(
239 struct hv_ring_buffer_info *ring_info,
240 u32 start_write_offset,
241 void *src,
242 u32 srclen);
243
244static u32
245copyfrom_ringbuffer(
246 struct hv_ring_buffer_info *ring_info,
247 void *dest,
248 u32 destlen,
249 u32 start_read_offset);
250
251
252
253
254
255
256
257
258
259
260
261
262void ringbuffer_get_debuginfo(struct hv_ring_buffer_info *ring_info,
263 struct hv_ring_buffer_debug_info *debug_info)
264{
265 u32 bytes_avail_towrite;
266 u32 bytes_avail_toread;
267
268 if (ring_info->ring_buffer) {
269 get_ringbuffer_availbytes(ring_info,
270 &bytes_avail_toread,
271 &bytes_avail_towrite);
272
273 debug_info->bytes_avail_toread = bytes_avail_toread;
274 debug_info->bytes_avail_towrite = bytes_avail_towrite;
275 debug_info->current_read_index =
276 ring_info->ring_buffer->read_index;
277 debug_info->current_write_index =
278 ring_info->ring_buffer->write_index;
279 debug_info->current_interrupt_mask =
280 ring_info->ring_buffer->interrupt_mask;
281 }
282}
283
284
285
286
287
288
289
290
291
292
293
294u32 get_ringbuffer_interrupt_mask(struct hv_ring_buffer_info *rbi)
295{
296 return rbi->ring_buffer->interrupt_mask;
297}
298
299
300
301
302
303
304
305
306
307
308int ringbuffer_init(struct hv_ring_buffer_info *ring_info,
309 void *buffer, u32 buflen)
310{
311 if (sizeof(struct hv_ring_buffer) != PAGE_SIZE)
312 return -EINVAL;
313
314 memset(ring_info, 0, sizeof(struct hv_ring_buffer_info));
315
316 ring_info->ring_buffer = (struct hv_ring_buffer *)buffer;
317 ring_info->ring_buffer->read_index =
318 ring_info->ring_buffer->write_index = 0;
319
320 ring_info->ring_size = buflen;
321 ring_info->ring_datasize = buflen - sizeof(struct hv_ring_buffer);
322
323 spin_lock_init(&ring_info->ring_lock);
324
325 return 0;
326}
327
328
329
330
331
332
333
334
335
336
337void ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info)
338{
339}
340
341
342
343
344
345
346
347
348
349
350int ringbuffer_write(struct hv_ring_buffer_info *outring_info,
351 struct scatterlist *sglist, u32 sgcount)
352{
353 int i = 0;
354 u32 bytes_avail_towrite;
355 u32 bytes_avail_toread;
356 u32 totalbytes_towrite = 0;
357
358 struct scatterlist *sg;
359 volatile u32 next_write_location;
360 u64 prev_indices = 0;
361 unsigned long flags;
362
363 for_each_sg(sglist, sg, sgcount, i)
364 {
365 totalbytes_towrite += sg->length;
366 }
367
368 totalbytes_towrite += sizeof(u64);
369
370 spin_lock_irqsave(&outring_info->ring_lock, flags);
371
372 get_ringbuffer_availbytes(outring_info,
373 &bytes_avail_toread,
374 &bytes_avail_towrite);
375
376 DPRINT_DBG(VMBUS, "Writing %u bytes...", totalbytes_towrite);
377
378
379
380
381
382
383 if (bytes_avail_towrite <= totalbytes_towrite) {
384 DPRINT_DBG(VMBUS,
385 "No more space left on outbound ring buffer "
386 "(needed %u, avail %u)",
387 totalbytes_towrite,
388 bytes_avail_towrite);
389
390 spin_unlock_irqrestore(&outring_info->ring_lock, flags);
391 return -1;
392 }
393
394
395 next_write_location = get_next_write_location(outring_info);
396
397 for_each_sg(sglist, sg, sgcount, i)
398 {
399 next_write_location = copyto_ringbuffer(outring_info,
400 next_write_location,
401 sg_virt(sg),
402 sg->length);
403 }
404
405
406 prev_indices = get_ring_bufferindices(outring_info);
407
408 next_write_location = copyto_ringbuffer(outring_info,
409 next_write_location,
410 &prev_indices,
411 sizeof(u64));
412
413
414 mb();
415
416
417 set_next_write_location(outring_info, next_write_location);
418
419
420
421 spin_unlock_irqrestore(&outring_info->ring_lock, flags);
422 return 0;
423}
424
425
426
427
428
429
430
431
432
433
434
435int ringbuffer_peek(struct hv_ring_buffer_info *Inring_info,
436 void *Buffer, u32 buflen)
437{
438 u32 bytes_avail_towrite;
439 u32 bytes_avail_toread;
440 u32 next_read_location = 0;
441 unsigned long flags;
442
443 spin_lock_irqsave(&Inring_info->ring_lock, flags);
444
445 get_ringbuffer_availbytes(Inring_info,
446 &bytes_avail_toread,
447 &bytes_avail_towrite);
448
449
450 if (bytes_avail_toread < buflen) {
451
452
453
454
455
456
457 spin_unlock_irqrestore(&Inring_info->ring_lock, flags);
458
459 return -1;
460 }
461
462
463 next_read_location = get_next_read_location(Inring_info);
464
465 next_read_location = copyfrom_ringbuffer(Inring_info,
466 Buffer,
467 buflen,
468 next_read_location);
469
470 spin_unlock_irqrestore(&Inring_info->ring_lock, flags);
471
472 return 0;
473}
474
475
476
477
478
479
480
481
482
483
484
485int ringbuffer_read(struct hv_ring_buffer_info *inring_info, void *buffer,
486 u32 buflen, u32 offset)
487{
488 u32 bytes_avail_towrite;
489 u32 bytes_avail_toread;
490 u32 next_read_location = 0;
491 u64 prev_indices = 0;
492 unsigned long flags;
493
494 if (buflen <= 0)
495 return -EINVAL;
496
497 spin_lock_irqsave(&inring_info->ring_lock, flags);
498
499 get_ringbuffer_availbytes(inring_info,
500 &bytes_avail_toread,
501 &bytes_avail_towrite);
502
503 DPRINT_DBG(VMBUS, "Reading %u bytes...", buflen);
504
505
506
507
508 if (bytes_avail_toread < buflen) {
509 DPRINT_DBG(VMBUS,
510 "got callback but not enough to read "
511 "<avail to read %d read size %d>!!",
512 bytes_avail_toread,
513 buflen);
514
515 spin_unlock_irqrestore(&inring_info->ring_lock, flags);
516
517 return -1;
518 }
519
520 next_read_location =
521 get_next_readlocation_withoffset(inring_info, offset);
522
523 next_read_location = copyfrom_ringbuffer(inring_info,
524 buffer,
525 buflen,
526 next_read_location);
527
528 next_read_location = copyfrom_ringbuffer(inring_info,
529 &prev_indices,
530 sizeof(u64),
531 next_read_location);
532
533
534
535
536 mb();
537
538
539 set_next_read_location(inring_info, next_read_location);
540
541
542
543 spin_unlock_irqrestore(&inring_info->ring_lock, flags);
544
545 return 0;
546}
547
548
549
550
551
552
553
554
555
556
557
558
559static u32
560copyto_ringbuffer(
561 struct hv_ring_buffer_info *ring_info,
562 u32 start_write_offset,
563 void *src,
564 u32 srclen)
565{
566 void *ring_buffer = get_ring_buffer(ring_info);
567 u32 ring_buffer_size = get_ring_buffersize(ring_info);
568 u32 frag_len;
569
570
571 if (srclen > ring_buffer_size - start_write_offset) {
572 DPRINT_DBG(VMBUS, "wrap-around detected!");
573
574 frag_len = ring_buffer_size - start_write_offset;
575 memcpy(ring_buffer + start_write_offset, src, frag_len);
576 memcpy(ring_buffer, src + frag_len, srclen - frag_len);
577 } else
578 memcpy(ring_buffer + start_write_offset, src, srclen);
579
580 start_write_offset += srclen;
581 start_write_offset %= ring_buffer_size;
582
583 return start_write_offset;
584}
585
586
587
588
589
590
591
592
593
594
595
596
597static u32
598copyfrom_ringbuffer(
599 struct hv_ring_buffer_info *ring_info,
600 void *dest,
601 u32 destlen,
602 u32 start_read_offset)
603{
604 void *ring_buffer = get_ring_buffer(ring_info);
605 u32 ring_buffer_size = get_ring_buffersize(ring_info);
606
607 u32 frag_len;
608
609
610 if (destlen > ring_buffer_size - start_read_offset) {
611 DPRINT_DBG(VMBUS, "src wrap-around detected!");
612
613 frag_len = ring_buffer_size - start_read_offset;
614
615 memcpy(dest, ring_buffer + start_read_offset, frag_len);
616 memcpy(dest + frag_len, ring_buffer, destlen - frag_len);
617 } else
618
619 memcpy(dest, ring_buffer + start_read_offset, destlen);
620
621
622 start_read_offset += destlen;
623 start_read_offset %= ring_buffer_size;
624
625 return start_read_offset;
626}
627
628
629
630