1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/highmem.h>
34
35#include "rds.h"
36
37struct rds_page_remainder {
38 struct page *r_page;
39 unsigned long r_offset;
40};
41
42DEFINE_PER_CPU_SHARED_ALIGNED(struct rds_page_remainder, rds_page_remainders);
43
44
45
46
47
48
49
50
51
52int rds_page_copy_user(struct page *page, unsigned long offset,
53 void __user *ptr, unsigned long bytes,
54 int to_user)
55{
56 unsigned long ret;
57 void *addr;
58
59 if (to_user)
60 rds_stats_add(s_copy_to_user, bytes);
61 else
62 rds_stats_add(s_copy_from_user, bytes);
63
64 addr = kmap_atomic(page, KM_USER0);
65 if (to_user)
66 ret = __copy_to_user_inatomic(ptr, addr + offset, bytes);
67 else
68 ret = __copy_from_user_inatomic(addr + offset, ptr, bytes);
69 kunmap_atomic(addr, KM_USER0);
70
71 if (ret) {
72 addr = kmap(page);
73 if (to_user)
74 ret = copy_to_user(ptr, addr + offset, bytes);
75 else
76 ret = copy_from_user(addr + offset, ptr, bytes);
77 kunmap(page);
78 if (ret)
79 return -EFAULT;
80 }
81
82 return 0;
83}
84EXPORT_SYMBOL_GPL(rds_page_copy_user);
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
106 gfp_t gfp)
107{
108 struct rds_page_remainder *rem;
109 unsigned long flags;
110 struct page *page;
111 int ret;
112
113 gfp |= __GFP_HIGHMEM;
114
115
116 if (bytes >= PAGE_SIZE) {
117 page = alloc_page(gfp);
118 if (page == NULL) {
119 ret = -ENOMEM;
120 } else {
121 sg_set_page(scat, page, PAGE_SIZE, 0);
122 ret = 0;
123 }
124 goto out;
125 }
126
127 rem = &per_cpu(rds_page_remainders, get_cpu());
128 local_irq_save(flags);
129
130 while (1) {
131
132 if (rem->r_page && bytes > (PAGE_SIZE - rem->r_offset)) {
133 rds_stats_inc(s_page_remainder_miss);
134 __free_page(rem->r_page);
135 rem->r_page = NULL;
136 }
137
138
139 if (rem->r_page && bytes <= (PAGE_SIZE - rem->r_offset)) {
140 sg_set_page(scat, rem->r_page, bytes, rem->r_offset);
141 get_page(sg_page(scat));
142
143 if (rem->r_offset != 0)
144 rds_stats_inc(s_page_remainder_hit);
145
146 rem->r_offset += bytes;
147 if (rem->r_offset == PAGE_SIZE) {
148 __free_page(rem->r_page);
149 rem->r_page = NULL;
150 }
151 ret = 0;
152 break;
153 }
154
155
156 local_irq_restore(flags);
157 put_cpu();
158
159 page = alloc_page(gfp);
160
161 rem = &per_cpu(rds_page_remainders, get_cpu());
162 local_irq_save(flags);
163
164 if (page == NULL) {
165 ret = -ENOMEM;
166 break;
167 }
168
169
170 if (rem->r_page) {
171 __free_page(page);
172 continue;
173 }
174
175
176 rem->r_page = page;
177 rem->r_offset = 0;
178 }
179
180 local_irq_restore(flags);
181 put_cpu();
182out:
183 rdsdebug("bytes %lu ret %d %p %u %u\n", bytes, ret,
184 ret ? NULL : sg_page(scat), ret ? 0 : scat->offset,
185 ret ? 0 : scat->length);
186 return ret;
187}
188
189static int rds_page_remainder_cpu_notify(struct notifier_block *self,
190 unsigned long action, void *hcpu)
191{
192 struct rds_page_remainder *rem;
193 long cpu = (long)hcpu;
194
195 rem = &per_cpu(rds_page_remainders, cpu);
196
197 rdsdebug("cpu %ld action 0x%lx\n", cpu, action);
198
199 switch (action) {
200 case CPU_DEAD:
201 if (rem->r_page)
202 __free_page(rem->r_page);
203 rem->r_page = NULL;
204 break;
205 }
206
207 return 0;
208}
209
210static struct notifier_block rds_page_remainder_nb = {
211 .notifier_call = rds_page_remainder_cpu_notify,
212};
213
214void rds_page_exit(void)
215{
216 int i;
217
218 for_each_possible_cpu(i)
219 rds_page_remainder_cpu_notify(&rds_page_remainder_nb,
220 (unsigned long)CPU_DEAD,
221 (void *)(long)i);
222}
223