1
2
3
4
5
6
7
8#include <linux/errno.h>
9#include <linux/sched.h>
10#include <linux/kernel.h>
11#include <linux/mm.h>
12#include <linux/smp.h>
13#include <linux/stddef.h>
14#include <linux/unistd.h>
15#include <linux/slab.h>
16#include <linux/user.h>
17#include <linux/elf.h>
18#include <linux/security.h>
19#include <linux/memblock.h>
20
21#include <asm/pgtable.h>
22#include <asm/processor.h>
23#include <asm/mmu.h>
24#include <asm/mmu_context.h>
25#include <asm/prom.h>
26#include <asm/machdep.h>
27#include <asm/cputable.h>
28#include <asm/sections.h>
29#include <asm/firmware.h>
30#include <asm/vdso.h>
31#include <asm/vdso_datapage.h>
32#include <asm/setup.h>
33
34#undef DEBUG
35
36#ifdef DEBUG
37#define DBG(fmt...) printk(fmt)
38#else
39#define DBG(fmt...)
40#endif
41
42
43#define MAX_SYMNAME 64
44
45
46#define VDSO_ALIGNMENT (1 << 16)
47
48static unsigned int vdso32_pages;
49static void *vdso32_kbase;
50static struct page **vdso32_pagelist;
51unsigned long vdso32_sigtramp;
52unsigned long vdso32_rt_sigtramp;
53
54#ifdef CONFIG_VDSO32
55extern char vdso32_start, vdso32_end;
56#endif
57
58#ifdef CONFIG_PPC64
59extern char vdso64_start, vdso64_end;
60static void *vdso64_kbase = &vdso64_start;
61static unsigned int vdso64_pages;
62static struct page **vdso64_pagelist;
63unsigned long vdso64_rt_sigtramp;
64#endif
65
66static int vdso_ready;
67
68
69
70
71
72
73static union {
74 struct vdso_data data;
75 u8 page[PAGE_SIZE];
76} vdso_data_store __page_aligned_data;
77struct vdso_data *vdso_data = &vdso_data_store.data;
78
79
80struct vdso_patch_def
81{
82 unsigned long ftr_mask, ftr_value;
83 const char *gen_name;
84 const char *fix_name;
85};
86
87
88
89
90
91
92static struct vdso_patch_def vdso_patches[] = {
93 {
94 CPU_FTR_COHERENT_ICACHE, CPU_FTR_COHERENT_ICACHE,
95 "__kernel_sync_dicache", "__kernel_sync_dicache_p5"
96 },
97#ifdef CONFIG_PPC32
98 {
99 CPU_FTR_USE_RTC, CPU_FTR_USE_RTC,
100 "__kernel_gettimeofday", NULL
101 },
102 {
103 CPU_FTR_USE_RTC, CPU_FTR_USE_RTC,
104 "__kernel_clock_gettime", NULL
105 },
106 {
107 CPU_FTR_USE_RTC, CPU_FTR_USE_RTC,
108 "__kernel_clock_getres", NULL
109 },
110 {
111 CPU_FTR_USE_RTC, CPU_FTR_USE_RTC,
112 "__kernel_get_tbfreq", NULL
113 },
114 {
115 CPU_FTR_USE_RTC, CPU_FTR_USE_RTC,
116 "__kernel_time", NULL
117 },
118#endif
119};
120
121
122
123
124
125struct lib32_elfinfo
126{
127 Elf32_Ehdr *hdr;
128 Elf32_Sym *dynsym;
129 unsigned long dynsymsize;
130 char *dynstr;
131 unsigned long text;
132};
133
134struct lib64_elfinfo
135{
136 Elf64_Ehdr *hdr;
137 Elf64_Sym *dynsym;
138 unsigned long dynsymsize;
139 char *dynstr;
140 unsigned long text;
141};
142
143
144
145
146
147
148int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
149{
150 struct mm_struct *mm = current->mm;
151 struct page **vdso_pagelist;
152 unsigned long vdso_pages;
153 unsigned long vdso_base;
154 int rc;
155
156 if (!vdso_ready)
157 return 0;
158
159#ifdef CONFIG_PPC64
160 if (is_32bit_task()) {
161 vdso_pagelist = vdso32_pagelist;
162 vdso_pages = vdso32_pages;
163 vdso_base = VDSO32_MBASE;
164 } else {
165 vdso_pagelist = vdso64_pagelist;
166 vdso_pages = vdso64_pages;
167
168
169
170
171
172 vdso_base = 0;
173 }
174#else
175 vdso_pagelist = vdso32_pagelist;
176 vdso_pages = vdso32_pages;
177 vdso_base = VDSO32_MBASE;
178#endif
179
180 current->mm->context.vdso_base = 0;
181
182
183
184
185 if (vdso_pages == 0)
186 return 0;
187
188 vdso_pages ++;
189
190
191
192
193
194
195
196 if (down_write_killable(&mm->mmap_sem))
197 return -EINTR;
198 vdso_base = get_unmapped_area(NULL, vdso_base,
199 (vdso_pages << PAGE_SHIFT) +
200 ((VDSO_ALIGNMENT - 1) & PAGE_MASK),
201 0, 0);
202 if (IS_ERR_VALUE(vdso_base)) {
203 rc = vdso_base;
204 goto fail_mmapsem;
205 }
206
207
208 vdso_base = ALIGN(vdso_base, VDSO_ALIGNMENT);
209
210
211
212
213
214
215 current->mm->context.vdso_base = vdso_base;
216
217
218
219
220
221
222
223
224
225
226
227 rc = install_special_mapping(mm, vdso_base, vdso_pages << PAGE_SHIFT,
228 VM_READ|VM_EXEC|
229 VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
230 vdso_pagelist);
231 if (rc) {
232 current->mm->context.vdso_base = 0;
233 goto fail_mmapsem;
234 }
235
236 up_write(&mm->mmap_sem);
237 return 0;
238
239 fail_mmapsem:
240 up_write(&mm->mmap_sem);
241 return rc;
242}
243
244const char *arch_vma_name(struct vm_area_struct *vma)
245{
246 if (vma->vm_mm && vma->vm_start == vma->vm_mm->context.vdso_base)
247 return "[vdso]";
248 return NULL;
249}
250
251
252
253#ifdef CONFIG_VDSO32
254static void * __init find_section32(Elf32_Ehdr *ehdr, const char *secname,
255 unsigned long *size)
256{
257 Elf32_Shdr *sechdrs;
258 unsigned int i;
259 char *secnames;
260
261
262 sechdrs = (void *)ehdr + ehdr->e_shoff;
263 secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset;
264
265
266 for (i = 1; i < ehdr->e_shnum; i++) {
267 if (strcmp(secnames+sechdrs[i].sh_name, secname) == 0) {
268 if (size)
269 *size = sechdrs[i].sh_size;
270 return (void *)ehdr + sechdrs[i].sh_offset;
271 }
272 }
273 *size = 0;
274 return NULL;
275}
276
277static Elf32_Sym * __init find_symbol32(struct lib32_elfinfo *lib,
278 const char *symname)
279{
280 unsigned int i;
281 char name[MAX_SYMNAME], *c;
282
283 for (i = 0; i < (lib->dynsymsize / sizeof(Elf32_Sym)); i++) {
284 if (lib->dynsym[i].st_name == 0)
285 continue;
286 strlcpy(name, lib->dynstr + lib->dynsym[i].st_name,
287 MAX_SYMNAME);
288 c = strchr(name, '@');
289 if (c)
290 *c = 0;
291 if (strcmp(symname, name) == 0)
292 return &lib->dynsym[i];
293 }
294 return NULL;
295}
296
297
298
299
300static unsigned long __init find_function32(struct lib32_elfinfo *lib,
301 const char *symname)
302{
303 Elf32_Sym *sym = find_symbol32(lib, symname);
304
305 if (sym == NULL) {
306 printk(KERN_WARNING "vDSO32: function %s not found !\n",
307 symname);
308 return 0;
309 }
310 return sym->st_value - VDSO32_LBASE;
311}
312
313static int __init vdso_do_func_patch32(struct lib32_elfinfo *v32,
314 struct lib64_elfinfo *v64,
315 const char *orig, const char *fix)
316{
317 Elf32_Sym *sym32_gen, *sym32_fix;
318
319 sym32_gen = find_symbol32(v32, orig);
320 if (sym32_gen == NULL) {
321 printk(KERN_ERR "vDSO32: Can't find symbol %s !\n", orig);
322 return -1;
323 }
324 if (fix == NULL) {
325 sym32_gen->st_name = 0;
326 return 0;
327 }
328 sym32_fix = find_symbol32(v32, fix);
329 if (sym32_fix == NULL) {
330 printk(KERN_ERR "vDSO32: Can't find symbol %s !\n", fix);
331 return -1;
332 }
333 sym32_gen->st_value = sym32_fix->st_value;
334 sym32_gen->st_size = sym32_fix->st_size;
335 sym32_gen->st_info = sym32_fix->st_info;
336 sym32_gen->st_other = sym32_fix->st_other;
337 sym32_gen->st_shndx = sym32_fix->st_shndx;
338
339 return 0;
340}
341#else
342static unsigned long __init find_function32(struct lib32_elfinfo *lib,
343 const char *symname)
344{
345 return 0;
346}
347
348static int __init vdso_do_func_patch32(struct lib32_elfinfo *v32,
349 struct lib64_elfinfo *v64,
350 const char *orig, const char *fix)
351{
352 return 0;
353}
354#endif
355
356
357#ifdef CONFIG_PPC64
358
359static void * __init find_section64(Elf64_Ehdr *ehdr, const char *secname,
360 unsigned long *size)
361{
362 Elf64_Shdr *sechdrs;
363 unsigned int i;
364 char *secnames;
365
366
367 sechdrs = (void *)ehdr + ehdr->e_shoff;
368 secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset;
369
370
371 for (i = 1; i < ehdr->e_shnum; i++) {
372 if (strcmp(secnames+sechdrs[i].sh_name, secname) == 0) {
373 if (size)
374 *size = sechdrs[i].sh_size;
375 return (void *)ehdr + sechdrs[i].sh_offset;
376 }
377 }
378 if (size)
379 *size = 0;
380 return NULL;
381}
382
383static Elf64_Sym * __init find_symbol64(struct lib64_elfinfo *lib,
384 const char *symname)
385{
386 unsigned int i;
387 char name[MAX_SYMNAME], *c;
388
389 for (i = 0; i < (lib->dynsymsize / sizeof(Elf64_Sym)); i++) {
390 if (lib->dynsym[i].st_name == 0)
391 continue;
392 strlcpy(name, lib->dynstr + lib->dynsym[i].st_name,
393 MAX_SYMNAME);
394 c = strchr(name, '@');
395 if (c)
396 *c = 0;
397 if (strcmp(symname, name) == 0)
398 return &lib->dynsym[i];
399 }
400 return NULL;
401}
402
403
404
405
406static unsigned long __init find_function64(struct lib64_elfinfo *lib,
407 const char *symname)
408{
409 Elf64_Sym *sym = find_symbol64(lib, symname);
410
411 if (sym == NULL) {
412 printk(KERN_WARNING "vDSO64: function %s not found !\n",
413 symname);
414 return 0;
415 }
416#ifdef VDS64_HAS_DESCRIPTORS
417 return *((u64 *)(vdso64_kbase + sym->st_value - VDSO64_LBASE)) -
418 VDSO64_LBASE;
419#else
420 return sym->st_value - VDSO64_LBASE;
421#endif
422}
423
424static int __init vdso_do_func_patch64(struct lib32_elfinfo *v32,
425 struct lib64_elfinfo *v64,
426 const char *orig, const char *fix)
427{
428 Elf64_Sym *sym64_gen, *sym64_fix;
429
430 sym64_gen = find_symbol64(v64, orig);
431 if (sym64_gen == NULL) {
432 printk(KERN_ERR "vDSO64: Can't find symbol %s !\n", orig);
433 return -1;
434 }
435 if (fix == NULL) {
436 sym64_gen->st_name = 0;
437 return 0;
438 }
439 sym64_fix = find_symbol64(v64, fix);
440 if (sym64_fix == NULL) {
441 printk(KERN_ERR "vDSO64: Can't find symbol %s !\n", fix);
442 return -1;
443 }
444 sym64_gen->st_value = sym64_fix->st_value;
445 sym64_gen->st_size = sym64_fix->st_size;
446 sym64_gen->st_info = sym64_fix->st_info;
447 sym64_gen->st_other = sym64_fix->st_other;
448 sym64_gen->st_shndx = sym64_fix->st_shndx;
449
450 return 0;
451}
452
453#endif
454
455
456static __init int vdso_do_find_sections(struct lib32_elfinfo *v32,
457 struct lib64_elfinfo *v64)
458{
459 void *sect;
460
461
462
463
464
465#ifdef CONFIG_VDSO32
466 v32->dynsym = find_section32(v32->hdr, ".dynsym", &v32->dynsymsize);
467 v32->dynstr = find_section32(v32->hdr, ".dynstr", NULL);
468 if (v32->dynsym == NULL || v32->dynstr == NULL) {
469 printk(KERN_ERR "vDSO32: required symbol section not found\n");
470 return -1;
471 }
472 sect = find_section32(v32->hdr, ".text", NULL);
473 if (sect == NULL) {
474 printk(KERN_ERR "vDSO32: the .text section was not found\n");
475 return -1;
476 }
477 v32->text = sect - vdso32_kbase;
478#endif
479
480#ifdef CONFIG_PPC64
481 v64->dynsym = find_section64(v64->hdr, ".dynsym", &v64->dynsymsize);
482 v64->dynstr = find_section64(v64->hdr, ".dynstr", NULL);
483 if (v64->dynsym == NULL || v64->dynstr == NULL) {
484 printk(KERN_ERR "vDSO64: required symbol section not found\n");
485 return -1;
486 }
487 sect = find_section64(v64->hdr, ".text", NULL);
488 if (sect == NULL) {
489 printk(KERN_ERR "vDSO64: the .text section was not found\n");
490 return -1;
491 }
492 v64->text = sect - vdso64_kbase;
493#endif
494
495 return 0;
496}
497
498static __init void vdso_setup_trampolines(struct lib32_elfinfo *v32,
499 struct lib64_elfinfo *v64)
500{
501
502
503
504
505#ifdef CONFIG_PPC64
506 vdso64_rt_sigtramp = find_function64(v64, "__kernel_sigtramp_rt64");
507#endif
508 vdso32_sigtramp = find_function32(v32, "__kernel_sigtramp32");
509 vdso32_rt_sigtramp = find_function32(v32, "__kernel_sigtramp_rt32");
510}
511
512static __init int vdso_fixup_datapage(struct lib32_elfinfo *v32,
513 struct lib64_elfinfo *v64)
514{
515#ifdef CONFIG_VDSO32
516 Elf32_Sym *sym32;
517#endif
518#ifdef CONFIG_PPC64
519 Elf64_Sym *sym64;
520
521 sym64 = find_symbol64(v64, "__kernel_datapage_offset");
522 if (sym64 == NULL) {
523 printk(KERN_ERR "vDSO64: Can't find symbol "
524 "__kernel_datapage_offset !\n");
525 return -1;
526 }
527 *((int *)(vdso64_kbase + sym64->st_value - VDSO64_LBASE)) =
528 (vdso64_pages << PAGE_SHIFT) -
529 (sym64->st_value - VDSO64_LBASE);
530#endif
531
532#ifdef CONFIG_VDSO32
533 sym32 = find_symbol32(v32, "__kernel_datapage_offset");
534 if (sym32 == NULL) {
535 printk(KERN_ERR "vDSO32: Can't find symbol "
536 "__kernel_datapage_offset !\n");
537 return -1;
538 }
539 *((int *)(vdso32_kbase + (sym32->st_value - VDSO32_LBASE))) =
540 (vdso32_pages << PAGE_SHIFT) -
541 (sym32->st_value - VDSO32_LBASE);
542#endif
543
544 return 0;
545}
546
547
548static __init int vdso_fixup_features(struct lib32_elfinfo *v32,
549 struct lib64_elfinfo *v64)
550{
551 unsigned long size;
552 void *start;
553
554#ifdef CONFIG_PPC64
555 start = find_section64(v64->hdr, "__ftr_fixup", &size);
556 if (start)
557 do_feature_fixups(cur_cpu_spec->cpu_features,
558 start, start + size);
559
560 start = find_section64(v64->hdr, "__mmu_ftr_fixup", &size);
561 if (start)
562 do_feature_fixups(cur_cpu_spec->mmu_features,
563 start, start + size);
564
565 start = find_section64(v64->hdr, "__fw_ftr_fixup", &size);
566 if (start)
567 do_feature_fixups(powerpc_firmware_features,
568 start, start + size);
569
570 start = find_section64(v64->hdr, "__lwsync_fixup", &size);
571 if (start)
572 do_lwsync_fixups(cur_cpu_spec->cpu_features,
573 start, start + size);
574#endif
575
576#ifdef CONFIG_VDSO32
577 start = find_section32(v32->hdr, "__ftr_fixup", &size);
578 if (start)
579 do_feature_fixups(cur_cpu_spec->cpu_features,
580 start, start + size);
581
582 start = find_section32(v32->hdr, "__mmu_ftr_fixup", &size);
583 if (start)
584 do_feature_fixups(cur_cpu_spec->mmu_features,
585 start, start + size);
586
587#ifdef CONFIG_PPC64
588 start = find_section32(v32->hdr, "__fw_ftr_fixup", &size);
589 if (start)
590 do_feature_fixups(powerpc_firmware_features,
591 start, start + size);
592#endif
593
594 start = find_section32(v32->hdr, "__lwsync_fixup", &size);
595 if (start)
596 do_lwsync_fixups(cur_cpu_spec->cpu_features,
597 start, start + size);
598#endif
599
600 return 0;
601}
602
603static __init int vdso_fixup_alt_funcs(struct lib32_elfinfo *v32,
604 struct lib64_elfinfo *v64)
605{
606 int i;
607
608 for (i = 0; i < ARRAY_SIZE(vdso_patches); i++) {
609 struct vdso_patch_def *patch = &vdso_patches[i];
610 int match = (cur_cpu_spec->cpu_features & patch->ftr_mask)
611 == patch->ftr_value;
612 if (!match)
613 continue;
614
615 DBG("replacing %s with %s...\n", patch->gen_name,
616 patch->fix_name ? "NONE" : patch->fix_name);
617
618
619
620
621
622
623
624 vdso_do_func_patch32(v32, v64, patch->gen_name,
625 patch->fix_name);
626#ifdef CONFIG_PPC64
627 vdso_do_func_patch64(v32, v64, patch->gen_name,
628 patch->fix_name);
629#endif
630 }
631
632 return 0;
633}
634
635
636static __init int vdso_setup(void)
637{
638 struct lib32_elfinfo v32;
639 struct lib64_elfinfo v64;
640
641 v32.hdr = vdso32_kbase;
642#ifdef CONFIG_PPC64
643 v64.hdr = vdso64_kbase;
644#endif
645 if (vdso_do_find_sections(&v32, &v64))
646 return -1;
647
648 if (vdso_fixup_datapage(&v32, &v64))
649 return -1;
650
651 if (vdso_fixup_features(&v32, &v64))
652 return -1;
653
654 if (vdso_fixup_alt_funcs(&v32, &v64))
655 return -1;
656
657 vdso_setup_trampolines(&v32, &v64);
658
659 return 0;
660}
661
662
663
664
665
666static void __init vdso_setup_syscall_map(void)
667{
668 unsigned int i;
669 extern unsigned long *sys_call_table;
670#ifdef CONFIG_PPC64
671 extern unsigned long *compat_sys_call_table;
672#endif
673 extern unsigned long sys_ni_syscall;
674
675
676 for (i = 0; i < NR_syscalls; i++) {
677#ifdef CONFIG_PPC64
678 if (sys_call_table[i] != sys_ni_syscall)
679 vdso_data->syscall_map_64[i >> 5] |=
680 0x80000000UL >> (i & 0x1f);
681 if (compat_sys_call_table[i] != sys_ni_syscall)
682 vdso_data->syscall_map_32[i >> 5] |=
683 0x80000000UL >> (i & 0x1f);
684#else
685 if (sys_call_table[i] != sys_ni_syscall)
686 vdso_data->syscall_map_32[i >> 5] |=
687 0x80000000UL >> (i & 0x1f);
688#endif
689 }
690}
691
692#ifdef CONFIG_PPC64
693int vdso_getcpu_init(void)
694{
695 unsigned long cpu, node, val;
696
697
698
699
700
701 cpu = get_cpu();
702 WARN_ON_ONCE(cpu > 0xffff);
703
704 node = cpu_to_node(cpu);
705 WARN_ON_ONCE(node > 0xffff);
706
707 val = (cpu & 0xfff) | ((node & 0xffff) << 16);
708 mtspr(SPRN_SPRG_VDSO_WRITE, val);
709 get_paca()->sprg_vdso = val;
710
711 put_cpu();
712
713 return 0;
714}
715
716early_initcall(vdso_getcpu_init);
717#endif
718
719static int __init vdso_init(void)
720{
721 int i;
722
723#ifdef CONFIG_PPC64
724
725
726
727 strcpy((char *)vdso_data->eye_catcher, "SYSTEMCFG:PPC64");
728 vdso_data->version.major = SYSTEMCFG_MAJOR;
729 vdso_data->version.minor = SYSTEMCFG_MINOR;
730 vdso_data->processor = mfspr(SPRN_PVR);
731
732
733
734
735 vdso_data->platform = 0x100;
736 if (firmware_has_feature(FW_FEATURE_LPAR))
737 vdso_data->platform |= 1;
738 vdso_data->physicalMemorySize = memblock_phys_mem_size();
739 vdso_data->dcache_size = ppc64_caches.l1d.size;
740 vdso_data->dcache_line_size = ppc64_caches.l1d.line_size;
741 vdso_data->icache_size = ppc64_caches.l1i.size;
742 vdso_data->icache_line_size = ppc64_caches.l1i.line_size;
743 vdso_data->dcache_block_size = ppc64_caches.l1d.block_size;
744 vdso_data->icache_block_size = ppc64_caches.l1i.block_size;
745 vdso_data->dcache_log_block_size = ppc64_caches.l1d.log_block_size;
746 vdso_data->icache_log_block_size = ppc64_caches.l1i.log_block_size;
747
748
749
750
751 vdso64_pages = (&vdso64_end - &vdso64_start) >> PAGE_SHIFT;
752 DBG("vdso64_kbase: %p, 0x%x pages\n", vdso64_kbase, vdso64_pages);
753#else
754 vdso_data->dcache_block_size = L1_CACHE_BYTES;
755 vdso_data->dcache_log_block_size = L1_CACHE_SHIFT;
756 vdso_data->icache_block_size = L1_CACHE_BYTES;
757 vdso_data->icache_log_block_size = L1_CACHE_SHIFT;
758#endif
759
760
761#ifdef CONFIG_VDSO32
762 vdso32_kbase = &vdso32_start;
763
764
765
766
767 vdso32_pages = (&vdso32_end - &vdso32_start) >> PAGE_SHIFT;
768 DBG("vdso32_kbase: %p, 0x%x pages\n", vdso32_kbase, vdso32_pages);
769#endif
770
771
772
773
774
775 vdso_setup_syscall_map();
776
777
778
779
780
781 if (vdso_setup()) {
782 printk(KERN_ERR "vDSO setup failure, not enabled !\n");
783 vdso32_pages = 0;
784#ifdef CONFIG_PPC64
785 vdso64_pages = 0;
786#endif
787 return 0;
788 }
789
790#ifdef CONFIG_VDSO32
791
792 vdso32_pagelist = kcalloc(vdso32_pages + 2, sizeof(struct page *),
793 GFP_KERNEL);
794 BUG_ON(vdso32_pagelist == NULL);
795 for (i = 0; i < vdso32_pages; i++) {
796 struct page *pg = virt_to_page(vdso32_kbase + i*PAGE_SIZE);
797 get_page(pg);
798 vdso32_pagelist[i] = pg;
799 }
800 vdso32_pagelist[i++] = virt_to_page(vdso_data);
801 vdso32_pagelist[i] = NULL;
802#endif
803
804#ifdef CONFIG_PPC64
805 vdso64_pagelist = kcalloc(vdso64_pages + 2, sizeof(struct page *),
806 GFP_KERNEL);
807 BUG_ON(vdso64_pagelist == NULL);
808 for (i = 0; i < vdso64_pages; i++) {
809 struct page *pg = virt_to_page(vdso64_kbase + i*PAGE_SIZE);
810 get_page(pg);
811 vdso64_pagelist[i] = pg;
812 }
813 vdso64_pagelist[i++] = virt_to_page(vdso_data);
814 vdso64_pagelist[i] = NULL;
815#endif
816
817 get_page(virt_to_page(vdso_data));
818
819 smp_wmb();
820 vdso_ready = 1;
821
822 return 0;
823}
824arch_initcall(vdso_init);
825