1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
25
26#include <linux/kernel.h>
27#include <linux/mm.h>
28#include <linux/hyperv.h>
29
30#include "hyperv_vmbus.h"
31
32void hv_begin_read(struct hv_ring_buffer_info *rbi)
33{
34 rbi->ring_buffer->interrupt_mask = 1;
35 mb();
36}
37
38u32 hv_end_read(struct hv_ring_buffer_info *rbi)
39{
40 u32 read;
41 u32 write;
42
43 rbi->ring_buffer->interrupt_mask = 0;
44 mb();
45
46
47
48
49
50
51 hv_get_ringbuffer_availbytes(rbi, &read, &write);
52
53 return read;
54}
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72static bool hv_need_to_signal(u32 old_write, struct hv_ring_buffer_info *rbi)
73{
74 mb();
75 if (rbi->ring_buffer->interrupt_mask)
76 return false;
77
78
79 rmb();
80
81
82
83
84 if (old_write == rbi->ring_buffer->read_index)
85 return true;
86
87 return false;
88}
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105static bool hv_need_to_signal_on_read(u32 old_rd,
106 struct hv_ring_buffer_info *rbi)
107{
108 u32 prev_write_sz;
109 u32 cur_write_sz;
110 u32 r_size;
111 u32 write_loc = rbi->ring_buffer->write_index;
112 u32 read_loc = rbi->ring_buffer->read_index;
113 u32 pending_sz = rbi->ring_buffer->pending_send_sz;
114
115
116
117
118 if (pending_sz == 0)
119 return false;
120
121 r_size = rbi->ring_datasize;
122 cur_write_sz = write_loc >= read_loc ? r_size - (write_loc - read_loc) :
123 read_loc - write_loc;
124
125 prev_write_sz = write_loc >= old_rd ? r_size - (write_loc - old_rd) :
126 old_rd - write_loc;
127
128
129 if ((prev_write_sz < pending_sz) && (cur_write_sz >= pending_sz))
130 return true;
131
132 return false;
133}
134
135
136
137
138
139
140
141static inline u32
142hv_get_next_write_location(struct hv_ring_buffer_info *ring_info)
143{
144 u32 next = ring_info->ring_buffer->write_index;
145
146 return next;
147}
148
149
150
151
152
153
154
155static inline void
156hv_set_next_write_location(struct hv_ring_buffer_info *ring_info,
157 u32 next_write_location)
158{
159 ring_info->ring_buffer->write_index = next_write_location;
160}
161
162
163
164
165
166
167static inline u32
168hv_get_next_read_location(struct hv_ring_buffer_info *ring_info)
169{
170 u32 next = ring_info->ring_buffer->read_index;
171
172 return next;
173}
174
175
176
177
178
179
180
181static inline u32
182hv_get_next_readlocation_withoffset(struct hv_ring_buffer_info *ring_info,
183 u32 offset)
184{
185 u32 next = ring_info->ring_buffer->read_index;
186
187 next += offset;
188 next %= ring_info->ring_datasize;
189
190 return next;
191}
192
193
194
195
196
197
198
199
200static inline void
201hv_set_next_read_location(struct hv_ring_buffer_info *ring_info,
202 u32 next_read_location)
203{
204 ring_info->ring_buffer->read_index = next_read_location;
205}
206
207
208
209
210
211
212
213
214static inline void *
215hv_get_ring_buffer(struct hv_ring_buffer_info *ring_info)
216{
217 return (void *)ring_info->ring_buffer->buffer;
218}
219
220
221
222
223
224
225
226
227static inline u32
228hv_get_ring_buffersize(struct hv_ring_buffer_info *ring_info)
229{
230 return ring_info->ring_datasize;
231}
232
233
234
235
236
237
238
239
240static inline u64
241hv_get_ring_bufferindices(struct hv_ring_buffer_info *ring_info)
242{
243 return (u64)ring_info->ring_buffer->write_index << 32;
244}
245
246
247
248
249
250
251
252
253
254static u32 hv_copyfrom_ringbuffer(
255 struct hv_ring_buffer_info *ring_info,
256 void *dest,
257 u32 destlen,
258 u32 start_read_offset)
259{
260 void *ring_buffer = hv_get_ring_buffer(ring_info);
261 u32 ring_buffer_size = hv_get_ring_buffersize(ring_info);
262
263 u32 frag_len;
264
265
266 if (destlen > ring_buffer_size - start_read_offset) {
267 frag_len = ring_buffer_size - start_read_offset;
268
269 memcpy(dest, ring_buffer + start_read_offset, frag_len);
270 memcpy(dest + frag_len, ring_buffer, destlen - frag_len);
271 } else
272
273 memcpy(dest, ring_buffer + start_read_offset, destlen);
274
275
276 start_read_offset += destlen;
277 start_read_offset %= ring_buffer_size;
278
279 return start_read_offset;
280}
281
282
283
284
285
286
287
288
289
290
291static u32 hv_copyto_ringbuffer(
292 struct hv_ring_buffer_info *ring_info,
293 u32 start_write_offset,
294 void *src,
295 u32 srclen)
296{
297 void *ring_buffer = hv_get_ring_buffer(ring_info);
298 u32 ring_buffer_size = hv_get_ring_buffersize(ring_info);
299 u32 frag_len;
300
301
302 if (srclen > ring_buffer_size - start_write_offset) {
303 frag_len = ring_buffer_size - start_write_offset;
304 memcpy(ring_buffer + start_write_offset, src, frag_len);
305 memcpy(ring_buffer, src + frag_len, srclen - frag_len);
306 } else
307 memcpy(ring_buffer + start_write_offset, src, srclen);
308
309 start_write_offset += srclen;
310 start_write_offset %= ring_buffer_size;
311
312 return start_write_offset;
313}
314
315
316
317
318
319
320
321
322void hv_ringbuffer_get_debuginfo(struct hv_ring_buffer_info *ring_info,
323 struct hv_ring_buffer_debug_info *debug_info)
324{
325 u32 bytes_avail_towrite;
326 u32 bytes_avail_toread;
327
328 if (ring_info->ring_buffer) {
329 hv_get_ringbuffer_availbytes(ring_info,
330 &bytes_avail_toread,
331 &bytes_avail_towrite);
332
333 debug_info->bytes_avail_toread = bytes_avail_toread;
334 debug_info->bytes_avail_towrite = bytes_avail_towrite;
335 debug_info->current_read_index =
336 ring_info->ring_buffer->read_index;
337 debug_info->current_write_index =
338 ring_info->ring_buffer->write_index;
339 debug_info->current_interrupt_mask =
340 ring_info->ring_buffer->interrupt_mask;
341 }
342}
343
344
345
346
347
348
349
350
351int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info,
352 void *buffer, u32 buflen)
353{
354 if (sizeof(struct hv_ring_buffer) != PAGE_SIZE)
355 return -EINVAL;
356
357 memset(ring_info, 0, sizeof(struct hv_ring_buffer_info));
358
359 ring_info->ring_buffer = (struct hv_ring_buffer *)buffer;
360 ring_info->ring_buffer->read_index =
361 ring_info->ring_buffer->write_index = 0;
362
363 ring_info->ring_size = buflen;
364 ring_info->ring_datasize = buflen - sizeof(struct hv_ring_buffer);
365
366 spin_lock_init(&ring_info->ring_lock);
367
368 return 0;
369}
370
371
372
373
374
375
376
377
378void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info)
379{
380}
381
382
383
384
385
386
387
388
389int hv_ringbuffer_write(struct hv_ring_buffer_info *outring_info,
390 struct scatterlist *sglist, u32 sgcount, bool *signal)
391{
392 int i = 0;
393 u32 bytes_avail_towrite;
394 u32 bytes_avail_toread;
395 u32 totalbytes_towrite = 0;
396
397 struct scatterlist *sg;
398 u32 next_write_location;
399 u32 old_write;
400 u64 prev_indices = 0;
401 unsigned long flags;
402
403 for_each_sg(sglist, sg, sgcount, i)
404 {
405 totalbytes_towrite += sg->length;
406 }
407
408 totalbytes_towrite += sizeof(u64);
409
410 spin_lock_irqsave(&outring_info->ring_lock, flags);
411
412 hv_get_ringbuffer_availbytes(outring_info,
413 &bytes_avail_toread,
414 &bytes_avail_towrite);
415
416
417
418
419
420 if (bytes_avail_towrite <= totalbytes_towrite) {
421 spin_unlock_irqrestore(&outring_info->ring_lock, flags);
422 return -EAGAIN;
423 }
424
425
426 next_write_location = hv_get_next_write_location(outring_info);
427
428 old_write = next_write_location;
429
430 for_each_sg(sglist, sg, sgcount, i)
431 {
432 next_write_location = hv_copyto_ringbuffer(outring_info,
433 next_write_location,
434 sg_virt(sg),
435 sg->length);
436 }
437
438
439 prev_indices = hv_get_ring_bufferindices(outring_info);
440
441 next_write_location = hv_copyto_ringbuffer(outring_info,
442 next_write_location,
443 &prev_indices,
444 sizeof(u64));
445
446
447 mb();
448
449
450 hv_set_next_write_location(outring_info, next_write_location);
451
452
453 spin_unlock_irqrestore(&outring_info->ring_lock, flags);
454
455 *signal = hv_need_to_signal(old_write, outring_info);
456 return 0;
457}
458
459
460
461
462
463
464
465
466
467int hv_ringbuffer_peek(struct hv_ring_buffer_info *Inring_info,
468 void *Buffer, u32 buflen)
469{
470 u32 bytes_avail_towrite;
471 u32 bytes_avail_toread;
472 u32 next_read_location = 0;
473 unsigned long flags;
474
475 spin_lock_irqsave(&Inring_info->ring_lock, flags);
476
477 hv_get_ringbuffer_availbytes(Inring_info,
478 &bytes_avail_toread,
479 &bytes_avail_towrite);
480
481
482 if (bytes_avail_toread < buflen) {
483
484 spin_unlock_irqrestore(&Inring_info->ring_lock, flags);
485
486 return -EAGAIN;
487 }
488
489
490 next_read_location = hv_get_next_read_location(Inring_info);
491
492 next_read_location = hv_copyfrom_ringbuffer(Inring_info,
493 Buffer,
494 buflen,
495 next_read_location);
496
497 spin_unlock_irqrestore(&Inring_info->ring_lock, flags);
498
499 return 0;
500}
501
502
503
504
505
506
507
508
509
510int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info, void *buffer,
511 u32 buflen, u32 offset, bool *signal)
512{
513 u32 bytes_avail_towrite;
514 u32 bytes_avail_toread;
515 u32 next_read_location = 0;
516 u64 prev_indices = 0;
517 unsigned long flags;
518 u32 old_read;
519
520 if (buflen <= 0)
521 return -EINVAL;
522
523 spin_lock_irqsave(&inring_info->ring_lock, flags);
524
525 hv_get_ringbuffer_availbytes(inring_info,
526 &bytes_avail_toread,
527 &bytes_avail_towrite);
528
529 old_read = bytes_avail_toread;
530
531
532 if (bytes_avail_toread < buflen) {
533 spin_unlock_irqrestore(&inring_info->ring_lock, flags);
534
535 return -EAGAIN;
536 }
537
538 next_read_location =
539 hv_get_next_readlocation_withoffset(inring_info, offset);
540
541 next_read_location = hv_copyfrom_ringbuffer(inring_info,
542 buffer,
543 buflen,
544 next_read_location);
545
546 next_read_location = hv_copyfrom_ringbuffer(inring_info,
547 &prev_indices,
548 sizeof(u64),
549 next_read_location);
550
551
552
553
554 mb();
555
556
557 hv_set_next_read_location(inring_info, next_read_location);
558
559 spin_unlock_irqrestore(&inring_info->ring_lock, flags);
560
561 *signal = hv_need_to_signal_on_read(old_read, inring_info);
562
563 return 0;
564}
565