1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/module.h>
35#include <linux/sched.h>
36#include <linux/mm.h>
37#include <linux/vmalloc.h>
38#include <linux/uaccess.h>
39
40#include <xen/interface/xen.h>
41#include <xen/page.h>
42#include <xen/grant_table.h>
43#include <asm/xen/hypercall.h>
44
45#include <asm/pgtable.h>
46#include <asm/sync_bitops.h>
47
48
49
50#define NR_RESERVED_ENTRIES 8
51#define GNTTAB_LIST_END 0xffffffff
52#define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(struct grant_entry))
53
54static grant_ref_t **gnttab_list;
55static unsigned int nr_grant_frames;
56static unsigned int boot_max_nr_grant_frames;
57static int gnttab_free_count;
58static grant_ref_t gnttab_free_head;
59static DEFINE_SPINLOCK(gnttab_list_lock);
60
61static struct grant_entry *shared;
62
63static struct gnttab_free_callback *gnttab_free_callback_list;
64
65static int gnttab_expand(unsigned int req_entries);
66
67#define RPP (PAGE_SIZE / sizeof(grant_ref_t))
68
69static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
70{
71 return &gnttab_list[(entry) / RPP][(entry) % RPP];
72}
73
74#define gnttab_entry(entry) (*__gnttab_entry(entry))
75
76static int get_free_entries(unsigned count)
77{
78 unsigned long flags;
79 int ref, rc;
80 grant_ref_t head;
81
82 spin_lock_irqsave(&gnttab_list_lock, flags);
83
84 if ((gnttab_free_count < count) &&
85 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
86 spin_unlock_irqrestore(&gnttab_list_lock, flags);
87 return rc;
88 }
89
90 ref = head = gnttab_free_head;
91 gnttab_free_count -= count;
92 while (count-- > 1)
93 head = gnttab_entry(head);
94 gnttab_free_head = gnttab_entry(head);
95 gnttab_entry(head) = GNTTAB_LIST_END;
96
97 spin_unlock_irqrestore(&gnttab_list_lock, flags);
98
99 return ref;
100}
101
102static void do_free_callbacks(void)
103{
104 struct gnttab_free_callback *callback, *next;
105
106 callback = gnttab_free_callback_list;
107 gnttab_free_callback_list = NULL;
108
109 while (callback != NULL) {
110 next = callback->next;
111 if (gnttab_free_count >= callback->count) {
112 callback->next = NULL;
113 callback->fn(callback->arg);
114 } else {
115 callback->next = gnttab_free_callback_list;
116 gnttab_free_callback_list = callback;
117 }
118 callback = next;
119 }
120}
121
122static inline void check_free_callbacks(void)
123{
124 if (unlikely(gnttab_free_callback_list))
125 do_free_callbacks();
126}
127
128static void put_free_entry(grant_ref_t ref)
129{
130 unsigned long flags;
131 spin_lock_irqsave(&gnttab_list_lock, flags);
132 gnttab_entry(ref) = gnttab_free_head;
133 gnttab_free_head = ref;
134 gnttab_free_count++;
135 check_free_callbacks();
136 spin_unlock_irqrestore(&gnttab_list_lock, flags);
137}
138
139static void update_grant_entry(grant_ref_t ref, domid_t domid,
140 unsigned long frame, unsigned flags)
141{
142
143
144
145
146
147
148
149
150
151
152 shared[ref].frame = frame;
153 shared[ref].domid = domid;
154 wmb();
155 shared[ref].flags = flags;
156}
157
158
159
160
161void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
162 unsigned long frame, int readonly)
163{
164 update_grant_entry(ref, domid, frame,
165 GTF_permit_access | (readonly ? GTF_readonly : 0));
166}
167EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
168
169int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
170 int readonly)
171{
172 int ref;
173
174 ref = get_free_entries(1);
175 if (unlikely(ref < 0))
176 return -ENOSPC;
177
178 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
179
180 return ref;
181}
182EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
183
184int gnttab_query_foreign_access(grant_ref_t ref)
185{
186 u16 nflags;
187
188 nflags = shared[ref].flags;
189
190 return (nflags & (GTF_reading|GTF_writing));
191}
192EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
193
194int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
195{
196 u16 flags, nflags;
197
198 nflags = shared[ref].flags;
199 do {
200 flags = nflags;
201 if (flags & (GTF_reading|GTF_writing)) {
202 printk(KERN_ALERT "WARNING: g.e. still in use!\n");
203 return 0;
204 }
205 } while ((nflags = sync_cmpxchg(&shared[ref].flags, flags, 0)) != flags);
206
207 return 1;
208}
209EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
210
211void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
212 unsigned long page)
213{
214 if (gnttab_end_foreign_access_ref(ref, readonly)) {
215 put_free_entry(ref);
216 if (page != 0)
217 free_page(page);
218 } else {
219
220
221 printk(KERN_WARNING
222 "WARNING: leaking g.e. and page still in use!\n");
223 }
224}
225EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
226
227int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
228{
229 int ref;
230
231 ref = get_free_entries(1);
232 if (unlikely(ref < 0))
233 return -ENOSPC;
234 gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
235
236 return ref;
237}
238EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
239
240void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
241 unsigned long pfn)
242{
243 update_grant_entry(ref, domid, pfn, GTF_accept_transfer);
244}
245EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
246
247unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
248{
249 unsigned long frame;
250 u16 flags;
251
252
253
254
255
256 while (!((flags = shared[ref].flags) & GTF_transfer_committed)) {
257 if (sync_cmpxchg(&shared[ref].flags, flags, 0) == flags)
258 return 0;
259 cpu_relax();
260 }
261
262
263 while (!(flags & GTF_transfer_completed)) {
264 flags = shared[ref].flags;
265 cpu_relax();
266 }
267
268 rmb();
269 frame = shared[ref].frame;
270 BUG_ON(frame == 0);
271
272 return frame;
273}
274EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
275
276unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
277{
278 unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
279 put_free_entry(ref);
280 return frame;
281}
282EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
283
284void gnttab_free_grant_reference(grant_ref_t ref)
285{
286 put_free_entry(ref);
287}
288EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
289
290void gnttab_free_grant_references(grant_ref_t head)
291{
292 grant_ref_t ref;
293 unsigned long flags;
294 int count = 1;
295 if (head == GNTTAB_LIST_END)
296 return;
297 spin_lock_irqsave(&gnttab_list_lock, flags);
298 ref = head;
299 while (gnttab_entry(ref) != GNTTAB_LIST_END) {
300 ref = gnttab_entry(ref);
301 count++;
302 }
303 gnttab_entry(ref) = gnttab_free_head;
304 gnttab_free_head = head;
305 gnttab_free_count += count;
306 check_free_callbacks();
307 spin_unlock_irqrestore(&gnttab_list_lock, flags);
308}
309EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
310
311int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
312{
313 int h = get_free_entries(count);
314
315 if (h < 0)
316 return -ENOSPC;
317
318 *head = h;
319
320 return 0;
321}
322EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
323
324int gnttab_empty_grant_references(const grant_ref_t *private_head)
325{
326 return (*private_head == GNTTAB_LIST_END);
327}
328EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
329
330int gnttab_claim_grant_reference(grant_ref_t *private_head)
331{
332 grant_ref_t g = *private_head;
333 if (unlikely(g == GNTTAB_LIST_END))
334 return -ENOSPC;
335 *private_head = gnttab_entry(g);
336 return g;
337}
338EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
339
340void gnttab_release_grant_reference(grant_ref_t *private_head,
341 grant_ref_t release)
342{
343 gnttab_entry(release) = *private_head;
344 *private_head = release;
345}
346EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
347
348void gnttab_request_free_callback(struct gnttab_free_callback *callback,
349 void (*fn)(void *), void *arg, u16 count)
350{
351 unsigned long flags;
352 spin_lock_irqsave(&gnttab_list_lock, flags);
353 if (callback->next)
354 goto out;
355 callback->fn = fn;
356 callback->arg = arg;
357 callback->count = count;
358 callback->next = gnttab_free_callback_list;
359 gnttab_free_callback_list = callback;
360 check_free_callbacks();
361out:
362 spin_unlock_irqrestore(&gnttab_list_lock, flags);
363}
364EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
365
366void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
367{
368 struct gnttab_free_callback **pcb;
369 unsigned long flags;
370
371 spin_lock_irqsave(&gnttab_list_lock, flags);
372 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
373 if (*pcb == callback) {
374 *pcb = callback->next;
375 break;
376 }
377 }
378 spin_unlock_irqrestore(&gnttab_list_lock, flags);
379}
380EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
381
382static int grow_gnttab_list(unsigned int more_frames)
383{
384 unsigned int new_nr_grant_frames, extra_entries, i;
385 unsigned int nr_glist_frames, new_nr_glist_frames;
386
387 new_nr_grant_frames = nr_grant_frames + more_frames;
388 extra_entries = more_frames * GREFS_PER_GRANT_FRAME;
389
390 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
391 new_nr_glist_frames =
392 (new_nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
393 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
394 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
395 if (!gnttab_list[i])
396 goto grow_nomem;
397 }
398
399
400 for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
401 i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
402 gnttab_entry(i) = i + 1;
403
404 gnttab_entry(i) = gnttab_free_head;
405 gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
406 gnttab_free_count += extra_entries;
407
408 nr_grant_frames = new_nr_grant_frames;
409
410 check_free_callbacks();
411
412 return 0;
413
414grow_nomem:
415 for ( ; i >= nr_glist_frames; i--)
416 free_page((unsigned long) gnttab_list[i]);
417 return -ENOMEM;
418}
419
420static unsigned int __max_nr_grant_frames(void)
421{
422 struct gnttab_query_size query;
423 int rc;
424
425 query.dom = DOMID_SELF;
426
427 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
428 if ((rc < 0) || (query.status != GNTST_okay))
429 return 4;
430
431 return query.max_nr_frames;
432}
433
434static inline unsigned int max_nr_grant_frames(void)
435{
436 unsigned int xen_max = __max_nr_grant_frames();
437
438 if (xen_max > boot_max_nr_grant_frames)
439 return boot_max_nr_grant_frames;
440 return xen_max;
441}
442
443static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
444{
445 struct gnttab_setup_table setup;
446 unsigned long *frames;
447 unsigned int nr_gframes = end_idx + 1;
448 int rc;
449
450 frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
451 if (!frames)
452 return -ENOMEM;
453
454 setup.dom = DOMID_SELF;
455 setup.nr_frames = nr_gframes;
456 set_xen_guest_handle(setup.frame_list, frames);
457
458 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
459 if (rc == -ENOSYS) {
460 kfree(frames);
461 return -ENOSYS;
462 }
463
464 BUG_ON(rc || setup.status);
465
466 rc = arch_gnttab_map_shared(frames, nr_gframes, max_nr_grant_frames(),
467 &shared);
468 BUG_ON(rc);
469
470 kfree(frames);
471
472 return 0;
473}
474
475int gnttab_resume(void)
476{
477 if (max_nr_grant_frames() < nr_grant_frames)
478 return -ENOSYS;
479 return gnttab_map(0, nr_grant_frames - 1);
480}
481
482int gnttab_suspend(void)
483{
484 arch_gnttab_unmap_shared(shared, nr_grant_frames);
485 return 0;
486}
487
488static int gnttab_expand(unsigned int req_entries)
489{
490 int rc;
491 unsigned int cur, extra;
492
493 cur = nr_grant_frames;
494 extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
495 GREFS_PER_GRANT_FRAME);
496 if (cur + extra > max_nr_grant_frames())
497 return -ENOSPC;
498
499 rc = gnttab_map(cur, cur + extra - 1);
500 if (rc == 0)
501 rc = grow_gnttab_list(extra);
502
503 return rc;
504}
505
506static int __devinit gnttab_init(void)
507{
508 int i;
509 unsigned int max_nr_glist_frames, nr_glist_frames;
510 unsigned int nr_init_grefs;
511
512 if (!xen_domain())
513 return -ENODEV;
514
515 nr_grant_frames = 1;
516 boot_max_nr_grant_frames = __max_nr_grant_frames();
517
518
519
520
521 max_nr_glist_frames = (boot_max_nr_grant_frames *
522 GREFS_PER_GRANT_FRAME / RPP);
523
524 gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
525 GFP_KERNEL);
526 if (gnttab_list == NULL)
527 return -ENOMEM;
528
529 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
530 for (i = 0; i < nr_glist_frames; i++) {
531 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
532 if (gnttab_list[i] == NULL)
533 goto ini_nomem;
534 }
535
536 if (gnttab_resume() < 0)
537 return -ENODEV;
538
539 nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
540
541 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
542 gnttab_entry(i) = i + 1;
543
544 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
545 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
546 gnttab_free_head = NR_RESERVED_ENTRIES;
547
548 printk("Grant table initialized\n");
549 return 0;
550
551 ini_nomem:
552 for (i--; i >= 0; i--)
553 free_page((unsigned long)gnttab_list[i]);
554 kfree(gnttab_list);
555 return -ENOMEM;
556}
557
558core_initcall(gnttab_init);
559