1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#define DISABLE_BRANCH_PROFILING
22
23#include <linux/time.h>
24#include <linux/init.h>
25#include <linux/kernel.h>
26#include <linux/timer.h>
27#include <linux/seqlock.h>
28#include <linux/jiffies.h>
29#include <linux/sysctl.h>
30#include <linux/clocksource.h>
31#include <linux/getcpu.h>
32#include <linux/cpu.h>
33#include <linux/smp.h>
34#include <linux/notifier.h>
35
36#include <asm/vsyscall.h>
37#include <asm/pgtable.h>
38#include <asm/page.h>
39#include <asm/unistd.h>
40#include <asm/fixmap.h>
41#include <asm/errno.h>
42#include <asm/io.h>
43#include <asm/segment.h>
44#include <asm/desc.h>
45#include <asm/topology.h>
46#include <asm/vgtod.h>
47
48#define __vsyscall(nr) \
49 __attribute__ ((unused, __section__(".vsyscall_" #nr))) notrace
50#define __syscall_clobber "r11","cx","memory"
51
52
53
54
55
56
57
58int __vgetcpu_mode __section_vgetcpu_mode;
59
60struct vsyscall_gtod_data __vsyscall_gtod_data __section_vsyscall_gtod_data =
61{
62 .lock = SEQLOCK_UNLOCKED,
63 .sysctl_enabled = 1,
64};
65
66void update_vsyscall_tz(void)
67{
68 unsigned long flags;
69
70 write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
71
72 vsyscall_gtod_data.sys_tz = sys_tz;
73 write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
74}
75
76void update_vsyscall(struct timespec *wall_time, struct timespec *wtm,
77 struct clocksource *clock, u32 mult)
78{
79 unsigned long flags;
80
81 write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
82
83 vsyscall_gtod_data.clock.vread = clock->vread;
84 vsyscall_gtod_data.clock.cycle_last = clock->cycle_last;
85 vsyscall_gtod_data.clock.mask = clock->mask;
86 vsyscall_gtod_data.clock.mult = mult;
87 vsyscall_gtod_data.clock.shift = clock->shift;
88 vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
89 vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
90 vsyscall_gtod_data.wall_to_monotonic = *wtm;
91 vsyscall_gtod_data.wall_time_coarse = __current_kernel_time();
92 write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
93}
94
95
96
97
98static __always_inline void do_get_tz(struct timezone * tz)
99{
100 *tz = __vsyscall_gtod_data.sys_tz;
101}
102
103static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
104{
105 int ret;
106 asm volatile("syscall"
107 : "=a" (ret)
108 : "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
109 : __syscall_clobber );
110 return ret;
111}
112
113static __always_inline long time_syscall(long *t)
114{
115 long secs;
116 asm volatile("syscall"
117 : "=a" (secs)
118 : "0" (__NR_time),"D" (t) : __syscall_clobber);
119 return secs;
120}
121
122static __always_inline void do_vgettimeofday(struct timeval * tv)
123{
124 cycle_t now, base, mask, cycle_delta;
125 unsigned seq;
126 unsigned long mult, shift, nsec;
127 cycle_t (*vread)(void);
128 do {
129 seq = read_seqbegin(&__vsyscall_gtod_data.lock);
130
131 vread = __vsyscall_gtod_data.clock.vread;
132 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
133 gettimeofday(tv,NULL);
134 return;
135 }
136
137 now = vread();
138 base = __vsyscall_gtod_data.clock.cycle_last;
139 mask = __vsyscall_gtod_data.clock.mask;
140 mult = __vsyscall_gtod_data.clock.mult;
141 shift = __vsyscall_gtod_data.clock.shift;
142
143 tv->tv_sec = __vsyscall_gtod_data.wall_time_sec;
144 nsec = __vsyscall_gtod_data.wall_time_nsec;
145 } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
146
147
148 cycle_delta = (now - base) & mask;
149
150 nsec += (cycle_delta * mult) >> shift;
151
152 while (nsec >= NSEC_PER_SEC) {
153 tv->tv_sec += 1;
154 nsec -= NSEC_PER_SEC;
155 }
156 tv->tv_usec = nsec / NSEC_PER_USEC;
157}
158
159int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
160{
161 if (tv)
162 do_vgettimeofday(tv);
163 if (tz)
164 do_get_tz(tz);
165 return 0;
166}
167
168
169
170time_t __vsyscall(1) vtime(time_t *t)
171{
172 unsigned seq;
173 time_t result;
174 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
175 return time_syscall(t);
176
177 do {
178 seq = read_seqbegin(&__vsyscall_gtod_data.lock);
179
180 result = __vsyscall_gtod_data.wall_time_sec;
181
182 } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
183
184 if (t)
185 *t = result;
186 return result;
187}
188
189
190
191
192
193
194
195
196
197long __vsyscall(2)
198vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
199{
200 unsigned int p;
201 unsigned long j = 0;
202
203
204
205
206
207
208
209
210
211 if (tcache && tcache->blob[0] == (j = __jiffies)) {
212 p = tcache->blob[1];
213 } else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
214
215 native_read_tscp(&p);
216 } else {
217
218 asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
219 }
220 if (tcache) {
221 tcache->blob[0] = j;
222 tcache->blob[1] = p;
223 }
224 if (cpu)
225 *cpu = p & 0xfff;
226 if (node)
227 *node = p >> 12;
228 return 0;
229}
230
231static long __vsyscall(3) venosys_1(void)
232{
233 return -ENOSYS;
234}
235
236#ifdef CONFIG_SYSCTL
237static ctl_table kernel_table2[] = {
238 { .procname = "vsyscall64",
239 .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
240 .mode = 0644,
241 .proc_handler = proc_dointvec },
242 {}
243};
244
245static ctl_table kernel_root_table2[] = {
246 { .procname = "kernel", .mode = 0555,
247 .child = kernel_table2 },
248 {}
249};
250#endif
251
252
253
254static void __cpuinit vsyscall_set_cpu(int cpu)
255{
256 unsigned long d;
257 unsigned long node = 0;
258#ifdef CONFIG_NUMA
259 node = cpu_to_node(cpu);
260#endif
261 if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP))
262 write_rdtscp_aux((node << 12) | cpu);
263
264
265
266
267 d = 0x0f40000000000ULL;
268 d |= cpu;
269 d |= (node & 0xf) << 12;
270 d |= (node >> 4) << 48;
271 write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S);
272}
273
274static void __cpuinit cpu_vsyscall_init(void *arg)
275{
276
277 vsyscall_set_cpu(raw_smp_processor_id());
278}
279
280static int __cpuinit
281cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
282{
283 long cpu = (long)arg;
284 if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
285 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 1);
286 return NOTIFY_DONE;
287}
288
289void __init map_vsyscall(void)
290{
291 extern char __vsyscall_0;
292 unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
293
294
295 __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
296}
297
298static int __init vsyscall_init(void)
299{
300 BUG_ON(((unsigned long) &vgettimeofday !=
301 VSYSCALL_ADDR(__NR_vgettimeofday)));
302 BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
303 BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
304 BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
305#ifdef CONFIG_SYSCTL
306 register_sysctl_table(kernel_root_table2);
307#endif
308 on_each_cpu(cpu_vsyscall_init, NULL, 1);
309
310 hotcpu_notifier(cpu_vsyscall_notifier, 30);
311 return 0;
312}
313
314__initcall(vsyscall_init);
315