1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
25
26#include <linux/kernel.h>
27#include <linux/mm.h>
28#include <linux/hyperv.h>
29#include <linux/uio.h>
30#include <linux/vmalloc.h>
31#include <linux/slab.h>
32#include <linux/prefetch.h>
33
34#include "hyperv_vmbus.h"
35
36#define VMBUS_PKT_TRAILER 8
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63static void hv_signal_on_write(u32 old_write, struct vmbus_channel *channel)
64{
65 struct hv_ring_buffer_info *rbi = &channel->outbound;
66
67 virt_mb();
68 if (READ_ONCE(rbi->ring_buffer->interrupt_mask))
69 return;
70
71
72 virt_rmb();
73
74
75
76
77 if (old_write == READ_ONCE(rbi->ring_buffer->read_index))
78 vmbus_setevent(channel);
79}
80
81
82static inline u32
83hv_get_next_write_location(struct hv_ring_buffer_info *ring_info)
84{
85 u32 next = ring_info->ring_buffer->write_index;
86
87 return next;
88}
89
90
91static inline void
92hv_set_next_write_location(struct hv_ring_buffer_info *ring_info,
93 u32 next_write_location)
94{
95 ring_info->ring_buffer->write_index = next_write_location;
96}
97
98
99static inline void
100hv_set_next_read_location(struct hv_ring_buffer_info *ring_info,
101 u32 next_read_location)
102{
103 ring_info->ring_buffer->read_index = next_read_location;
104 ring_info->priv_read_index = next_read_location;
105}
106
107
108static inline u32
109hv_get_ring_buffersize(const struct hv_ring_buffer_info *ring_info)
110{
111 return ring_info->ring_datasize;
112}
113
114
115static inline u64
116hv_get_ring_bufferindices(struct hv_ring_buffer_info *ring_info)
117{
118 return (u64)ring_info->ring_buffer->write_index << 32;
119}
120
121
122
123
124
125static u32 hv_copyto_ringbuffer(
126 struct hv_ring_buffer_info *ring_info,
127 u32 start_write_offset,
128 const void *src,
129 u32 srclen)
130{
131 void *ring_buffer = hv_get_ring_buffer(ring_info);
132 u32 ring_buffer_size = hv_get_ring_buffersize(ring_info);
133
134 memcpy(ring_buffer + start_write_offset, src, srclen);
135
136 start_write_offset += srclen;
137 if (start_write_offset >= ring_buffer_size)
138 start_write_offset -= ring_buffer_size;
139
140 return start_write_offset;
141}
142
143
144void hv_ringbuffer_get_debuginfo(const struct hv_ring_buffer_info *ring_info,
145 struct hv_ring_buffer_debug_info *debug_info)
146{
147 u32 bytes_avail_towrite;
148 u32 bytes_avail_toread;
149
150 if (ring_info->ring_buffer) {
151 hv_get_ringbuffer_availbytes(ring_info,
152 &bytes_avail_toread,
153 &bytes_avail_towrite);
154
155 debug_info->bytes_avail_toread = bytes_avail_toread;
156 debug_info->bytes_avail_towrite = bytes_avail_towrite;
157 debug_info->current_read_index =
158 ring_info->ring_buffer->read_index;
159 debug_info->current_write_index =
160 ring_info->ring_buffer->write_index;
161 debug_info->current_interrupt_mask =
162 ring_info->ring_buffer->interrupt_mask;
163 }
164}
165EXPORT_SYMBOL_GPL(hv_ringbuffer_get_debuginfo);
166
167
168int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info,
169 struct page *pages, u32 page_cnt)
170{
171 int i;
172 struct page **pages_wraparound;
173
174 BUILD_BUG_ON((sizeof(struct hv_ring_buffer) != PAGE_SIZE));
175
176 memset(ring_info, 0, sizeof(struct hv_ring_buffer_info));
177
178
179
180
181
182 pages_wraparound = kzalloc(sizeof(struct page *) * (page_cnt * 2 - 1),
183 GFP_KERNEL);
184 if (!pages_wraparound)
185 return -ENOMEM;
186
187 pages_wraparound[0] = pages;
188 for (i = 0; i < 2 * (page_cnt - 1); i++)
189 pages_wraparound[i + 1] = &pages[i % (page_cnt - 1) + 1];
190
191 ring_info->ring_buffer = (struct hv_ring_buffer *)
192 vmap(pages_wraparound, page_cnt * 2 - 1, VM_MAP, PAGE_KERNEL);
193
194 kfree(pages_wraparound);
195
196
197 if (!ring_info->ring_buffer)
198 return -ENOMEM;
199
200 ring_info->ring_buffer->read_index =
201 ring_info->ring_buffer->write_index = 0;
202
203
204 ring_info->ring_buffer->feature_bits.value = 1;
205
206 ring_info->ring_size = page_cnt << PAGE_SHIFT;
207 ring_info->ring_datasize = ring_info->ring_size -
208 sizeof(struct hv_ring_buffer);
209
210 spin_lock_init(&ring_info->ring_lock);
211
212 return 0;
213}
214
215
216void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info)
217{
218 vunmap(ring_info->ring_buffer);
219}
220
221
222int hv_ringbuffer_write(struct vmbus_channel *channel,
223 const struct kvec *kv_list, u32 kv_count)
224{
225 int i;
226 u32 bytes_avail_towrite;
227 u32 totalbytes_towrite = sizeof(u64);
228 u32 next_write_location;
229 u32 old_write;
230 u64 prev_indices;
231 unsigned long flags;
232 struct hv_ring_buffer_info *outring_info = &channel->outbound;
233
234 if (channel->rescind)
235 return -ENODEV;
236
237 for (i = 0; i < kv_count; i++)
238 totalbytes_towrite += kv_list[i].iov_len;
239
240 spin_lock_irqsave(&outring_info->ring_lock, flags);
241
242 bytes_avail_towrite = hv_get_bytes_to_write(outring_info);
243
244
245
246
247
248
249 if (bytes_avail_towrite <= totalbytes_towrite) {
250 spin_unlock_irqrestore(&outring_info->ring_lock, flags);
251 return -EAGAIN;
252 }
253
254
255 next_write_location = hv_get_next_write_location(outring_info);
256
257 old_write = next_write_location;
258
259 for (i = 0; i < kv_count; i++) {
260 next_write_location = hv_copyto_ringbuffer(outring_info,
261 next_write_location,
262 kv_list[i].iov_base,
263 kv_list[i].iov_len);
264 }
265
266
267 prev_indices = hv_get_ring_bufferindices(outring_info);
268
269 next_write_location = hv_copyto_ringbuffer(outring_info,
270 next_write_location,
271 &prev_indices,
272 sizeof(u64));
273
274
275 virt_mb();
276
277
278 hv_set_next_write_location(outring_info, next_write_location);
279
280
281 spin_unlock_irqrestore(&outring_info->ring_lock, flags);
282
283 hv_signal_on_write(old_write, channel);
284
285 if (channel->rescind)
286 return -ENODEV;
287
288 return 0;
289}
290
291int hv_ringbuffer_read(struct vmbus_channel *channel,
292 void *buffer, u32 buflen, u32 *buffer_actual_len,
293 u64 *requestid, bool raw)
294{
295 struct vmpacket_descriptor *desc;
296 u32 packetlen, offset;
297
298 if (unlikely(buflen == 0))
299 return -EINVAL;
300
301 *buffer_actual_len = 0;
302 *requestid = 0;
303
304
305 desc = hv_pkt_iter_first(channel);
306 if (desc == NULL) {
307
308
309
310
311 return 0;
312 }
313
314 offset = raw ? 0 : (desc->offset8 << 3);
315 packetlen = (desc->len8 << 3) - offset;
316 *buffer_actual_len = packetlen;
317 *requestid = desc->trans_id;
318
319 if (unlikely(packetlen > buflen))
320 return -ENOBUFS;
321
322
323 memcpy(buffer, (const char *)desc + offset, packetlen);
324
325
326 __hv_pkt_iter_next(channel, desc);
327
328
329 hv_pkt_iter_close(channel);
330
331 return 0;
332}
333
334
335
336
337
338
339
340
341static u32 hv_pkt_iter_avail(const struct hv_ring_buffer_info *rbi)
342{
343 u32 priv_read_loc = rbi->priv_read_index;
344 u32 write_loc = READ_ONCE(rbi->ring_buffer->write_index);
345
346 if (write_loc >= priv_read_loc)
347 return write_loc - priv_read_loc;
348 else
349 return (rbi->ring_datasize - priv_read_loc) + write_loc;
350}
351
352
353
354
355
356
357struct vmpacket_descriptor *hv_pkt_iter_first(struct vmbus_channel *channel)
358{
359 struct hv_ring_buffer_info *rbi = &channel->inbound;
360 struct vmpacket_descriptor *desc;
361
362 if (hv_pkt_iter_avail(rbi) < sizeof(struct vmpacket_descriptor))
363 return NULL;
364
365 desc = hv_get_ring_buffer(rbi) + rbi->priv_read_index;
366 if (desc)
367 prefetch((char *)desc + (desc->len8 << 3));
368
369 return desc;
370}
371EXPORT_SYMBOL_GPL(hv_pkt_iter_first);
372
373
374
375
376
377
378
379struct vmpacket_descriptor *
380__hv_pkt_iter_next(struct vmbus_channel *channel,
381 const struct vmpacket_descriptor *desc)
382{
383 struct hv_ring_buffer_info *rbi = &channel->inbound;
384 u32 packetlen = desc->len8 << 3;
385 u32 dsize = rbi->ring_datasize;
386
387
388 rbi->priv_read_index += packetlen + VMBUS_PKT_TRAILER;
389 if (rbi->priv_read_index >= dsize)
390 rbi->priv_read_index -= dsize;
391
392
393 return hv_pkt_iter_first(channel);
394}
395EXPORT_SYMBOL_GPL(__hv_pkt_iter_next);
396
397
398
399
400void hv_pkt_iter_close(struct vmbus_channel *channel)
401{
402 struct hv_ring_buffer_info *rbi = &channel->inbound;
403 u32 orig_write_sz = hv_get_bytes_to_write(rbi);
404
405
406
407
408
409
410 virt_rmb();
411 rbi->ring_buffer->read_index = rbi->priv_read_index;
412
413
414
415
416
417
418
419
420
421
422
423
424 virt_mb();
425
426
427 if (rbi->ring_buffer->interrupt_mask)
428 return;
429
430 if (rbi->ring_buffer->feature_bits.feat_pending_send_sz) {
431 u32 pending_sz = READ_ONCE(rbi->ring_buffer->pending_send_sz);
432
433
434
435
436
437
438
439 if (orig_write_sz > pending_sz)
440 return;
441
442
443 if (hv_get_bytes_to_write(rbi) < pending_sz)
444 return;
445 }
446
447 vmbus_setevent(channel);
448}
449EXPORT_SYMBOL_GPL(hv_pkt_iter_close);
450