1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
25
26#include <linux/kernel.h>
27#include <linux/mm.h>
28#include <linux/hyperv.h>
29#include <linux/uio.h>
30
31#include "hyperv_vmbus.h"
32
33void hv_begin_read(struct hv_ring_buffer_info *rbi)
34{
35 rbi->ring_buffer->interrupt_mask = 1;
36 mb();
37}
38
39u32 hv_end_read(struct hv_ring_buffer_info *rbi)
40{
41 u32 read;
42 u32 write;
43
44 rbi->ring_buffer->interrupt_mask = 0;
45 mb();
46
47
48
49
50
51
52 hv_get_ringbuffer_availbytes(rbi, &read, &write);
53
54 return read;
55}
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73static bool hv_need_to_signal(u32 old_write, struct hv_ring_buffer_info *rbi)
74{
75 mb();
76 if (rbi->ring_buffer->interrupt_mask)
77 return false;
78
79
80 rmb();
81
82
83
84
85 if (old_write == rbi->ring_buffer->read_index)
86 return true;
87
88 return false;
89}
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106static bool hv_need_to_signal_on_read(u32 prev_write_sz,
107 struct hv_ring_buffer_info *rbi)
108{
109 u32 cur_write_sz;
110 u32 r_size;
111 u32 write_loc = rbi->ring_buffer->write_index;
112 u32 read_loc = rbi->ring_buffer->read_index;
113 u32 pending_sz = rbi->ring_buffer->pending_send_sz;
114
115
116 if (pending_sz == 0)
117 return false;
118
119 r_size = rbi->ring_datasize;
120 cur_write_sz = write_loc >= read_loc ? r_size - (write_loc - read_loc) :
121 read_loc - write_loc;
122
123 if ((prev_write_sz < pending_sz) && (cur_write_sz >= pending_sz))
124 return true;
125
126 return false;
127}
128
129
130static inline u32
131hv_get_next_write_location(struct hv_ring_buffer_info *ring_info)
132{
133 u32 next = ring_info->ring_buffer->write_index;
134
135 return next;
136}
137
138
139static inline void
140hv_set_next_write_location(struct hv_ring_buffer_info *ring_info,
141 u32 next_write_location)
142{
143 ring_info->ring_buffer->write_index = next_write_location;
144}
145
146
147static inline u32
148hv_get_next_read_location(struct hv_ring_buffer_info *ring_info)
149{
150 u32 next = ring_info->ring_buffer->read_index;
151
152 return next;
153}
154
155
156
157
158
159static inline u32
160hv_get_next_readlocation_withoffset(struct hv_ring_buffer_info *ring_info,
161 u32 offset)
162{
163 u32 next = ring_info->ring_buffer->read_index;
164
165 next += offset;
166 next %= ring_info->ring_datasize;
167
168 return next;
169}
170
171
172static inline void
173hv_set_next_read_location(struct hv_ring_buffer_info *ring_info,
174 u32 next_read_location)
175{
176 ring_info->ring_buffer->read_index = next_read_location;
177}
178
179
180
181static inline void *
182hv_get_ring_buffer(struct hv_ring_buffer_info *ring_info)
183{
184 return (void *)ring_info->ring_buffer->buffer;
185}
186
187
188
189static inline u32
190hv_get_ring_buffersize(struct hv_ring_buffer_info *ring_info)
191{
192 return ring_info->ring_datasize;
193}
194
195
196static inline u64
197hv_get_ring_bufferindices(struct hv_ring_buffer_info *ring_info)
198{
199 return (u64)ring_info->ring_buffer->write_index << 32;
200}
201
202
203
204
205
206static u32 hv_copyfrom_ringbuffer(
207 struct hv_ring_buffer_info *ring_info,
208 void *dest,
209 u32 destlen,
210 u32 start_read_offset)
211{
212 void *ring_buffer = hv_get_ring_buffer(ring_info);
213 u32 ring_buffer_size = hv_get_ring_buffersize(ring_info);
214
215 u32 frag_len;
216
217
218 if (destlen > ring_buffer_size - start_read_offset) {
219 frag_len = ring_buffer_size - start_read_offset;
220
221 memcpy(dest, ring_buffer + start_read_offset, frag_len);
222 memcpy(dest + frag_len, ring_buffer, destlen - frag_len);
223 } else
224
225 memcpy(dest, ring_buffer + start_read_offset, destlen);
226
227
228 start_read_offset += destlen;
229 start_read_offset %= ring_buffer_size;
230
231 return start_read_offset;
232}
233
234
235
236
237
238
239static u32 hv_copyto_ringbuffer(
240 struct hv_ring_buffer_info *ring_info,
241 u32 start_write_offset,
242 void *src,
243 u32 srclen)
244{
245 void *ring_buffer = hv_get_ring_buffer(ring_info);
246 u32 ring_buffer_size = hv_get_ring_buffersize(ring_info);
247 u32 frag_len;
248
249
250 if (srclen > ring_buffer_size - start_write_offset) {
251 frag_len = ring_buffer_size - start_write_offset;
252 memcpy(ring_buffer + start_write_offset, src, frag_len);
253 memcpy(ring_buffer, src + frag_len, srclen - frag_len);
254 } else
255 memcpy(ring_buffer + start_write_offset, src, srclen);
256
257 start_write_offset += srclen;
258 start_write_offset %= ring_buffer_size;
259
260 return start_write_offset;
261}
262
263
264void hv_ringbuffer_get_debuginfo(struct hv_ring_buffer_info *ring_info,
265 struct hv_ring_buffer_debug_info *debug_info)
266{
267 u32 bytes_avail_towrite;
268 u32 bytes_avail_toread;
269
270 if (ring_info->ring_buffer) {
271 hv_get_ringbuffer_availbytes(ring_info,
272 &bytes_avail_toread,
273 &bytes_avail_towrite);
274
275 debug_info->bytes_avail_toread = bytes_avail_toread;
276 debug_info->bytes_avail_towrite = bytes_avail_towrite;
277 debug_info->current_read_index =
278 ring_info->ring_buffer->read_index;
279 debug_info->current_write_index =
280 ring_info->ring_buffer->write_index;
281 debug_info->current_interrupt_mask =
282 ring_info->ring_buffer->interrupt_mask;
283 }
284}
285
286
287int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info,
288 void *buffer, u32 buflen)
289{
290 if (sizeof(struct hv_ring_buffer) != PAGE_SIZE)
291 return -EINVAL;
292
293 memset(ring_info, 0, sizeof(struct hv_ring_buffer_info));
294
295 ring_info->ring_buffer = (struct hv_ring_buffer *)buffer;
296 ring_info->ring_buffer->read_index =
297 ring_info->ring_buffer->write_index = 0;
298
299
300 ring_info->ring_buffer->feature_bits.value = 1;
301
302 ring_info->ring_size = buflen;
303 ring_info->ring_datasize = buflen - sizeof(struct hv_ring_buffer);
304
305 spin_lock_init(&ring_info->ring_lock);
306
307 return 0;
308}
309
310
311void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info)
312{
313}
314
315
316int hv_ringbuffer_write(struct hv_ring_buffer_info *outring_info,
317 struct kvec *kv_list, u32 kv_count, bool *signal)
318{
319 int i = 0;
320 u32 bytes_avail_towrite;
321 u32 bytes_avail_toread;
322 u32 totalbytes_towrite = 0;
323
324 u32 next_write_location;
325 u32 old_write;
326 u64 prev_indices = 0;
327 unsigned long flags;
328
329 for (i = 0; i < kv_count; i++)
330 totalbytes_towrite += kv_list[i].iov_len;
331
332 totalbytes_towrite += sizeof(u64);
333
334 spin_lock_irqsave(&outring_info->ring_lock, flags);
335
336 hv_get_ringbuffer_availbytes(outring_info,
337 &bytes_avail_toread,
338 &bytes_avail_towrite);
339
340
341
342
343
344
345 if (bytes_avail_towrite <= totalbytes_towrite) {
346 spin_unlock_irqrestore(&outring_info->ring_lock, flags);
347 return -EAGAIN;
348 }
349
350
351 next_write_location = hv_get_next_write_location(outring_info);
352
353 old_write = next_write_location;
354
355 for (i = 0; i < kv_count; i++) {
356 next_write_location = hv_copyto_ringbuffer(outring_info,
357 next_write_location,
358 kv_list[i].iov_base,
359 kv_list[i].iov_len);
360 }
361
362
363 prev_indices = hv_get_ring_bufferindices(outring_info);
364
365 next_write_location = hv_copyto_ringbuffer(outring_info,
366 next_write_location,
367 &prev_indices,
368 sizeof(u64));
369
370
371 mb();
372
373
374 hv_set_next_write_location(outring_info, next_write_location);
375
376
377 spin_unlock_irqrestore(&outring_info->ring_lock, flags);
378
379 *signal = hv_need_to_signal(old_write, outring_info);
380 return 0;
381}
382
383int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info,
384 void *buffer, u32 buflen, u32 *buffer_actual_len,
385 u64 *requestid, bool *signal, bool raw)
386{
387 u32 bytes_avail_towrite;
388 u32 bytes_avail_toread;
389 u32 next_read_location = 0;
390 u64 prev_indices = 0;
391 unsigned long flags;
392 struct vmpacket_descriptor desc;
393 u32 offset;
394 u32 packetlen;
395 int ret = 0;
396
397 if (buflen <= 0)
398 return -EINVAL;
399
400 spin_lock_irqsave(&inring_info->ring_lock, flags);
401
402 *buffer_actual_len = 0;
403 *requestid = 0;
404
405 hv_get_ringbuffer_availbytes(inring_info,
406 &bytes_avail_toread,
407 &bytes_avail_towrite);
408
409
410 if (bytes_avail_toread < sizeof(desc)) {
411
412
413
414
415 goto out_unlock;
416 }
417
418 next_read_location = hv_get_next_read_location(inring_info);
419 next_read_location = hv_copyfrom_ringbuffer(inring_info, &desc,
420 sizeof(desc),
421 next_read_location);
422
423 offset = raw ? 0 : (desc.offset8 << 3);
424 packetlen = (desc.len8 << 3) - offset;
425 *buffer_actual_len = packetlen;
426 *requestid = desc.trans_id;
427
428 if (bytes_avail_toread < packetlen + offset) {
429 ret = -EAGAIN;
430 goto out_unlock;
431 }
432
433 if (packetlen > buflen) {
434 ret = -ENOBUFS;
435 goto out_unlock;
436 }
437
438 next_read_location =
439 hv_get_next_readlocation_withoffset(inring_info, offset);
440
441 next_read_location = hv_copyfrom_ringbuffer(inring_info,
442 buffer,
443 packetlen,
444 next_read_location);
445
446 next_read_location = hv_copyfrom_ringbuffer(inring_info,
447 &prev_indices,
448 sizeof(u64),
449 next_read_location);
450
451
452
453
454
455
456 mb();
457
458
459 hv_set_next_read_location(inring_info, next_read_location);
460
461 *signal = hv_need_to_signal_on_read(bytes_avail_towrite, inring_info);
462
463out_unlock:
464 spin_unlock_irqrestore(&inring_info->ring_lock, flags);
465 return ret;
466}
467