1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41#include "ehca_tools.h"
42#include "ipz_pt_fn.h"
43#include "ehca_classes.h"
44
45#define PAGES_PER_KPAGE (PAGE_SIZE >> EHCA_PAGESHIFT)
46
47struct kmem_cache *small_qp_cache;
48
49void *ipz_qpageit_get_inc(struct ipz_queue *queue)
50{
51 void *ret = ipz_qeit_get(queue);
52 queue->current_q_offset += queue->pagesize;
53 if (queue->current_q_offset > queue->queue_length) {
54 queue->current_q_offset -= queue->pagesize;
55 ret = NULL;
56 }
57 if (((u64)ret) % queue->pagesize) {
58 ehca_gen_err("ERROR!! not at PAGE-Boundary");
59 return NULL;
60 }
61 return ret;
62}
63
64void *ipz_qeit_eq_get_inc(struct ipz_queue *queue)
65{
66 void *ret = ipz_qeit_get(queue);
67 u64 last_entry_in_q = queue->queue_length - queue->qe_size;
68
69 queue->current_q_offset += queue->qe_size;
70 if (queue->current_q_offset > last_entry_in_q) {
71 queue->current_q_offset = 0;
72 queue->toggle_state = (~queue->toggle_state) & 1;
73 }
74
75 return ret;
76}
77
78int ipz_queue_abs_to_offset(struct ipz_queue *queue, u64 addr, u64 *q_offset)
79{
80 int i;
81 for (i = 0; i < queue->queue_length / queue->pagesize; i++) {
82 u64 page = (u64)virt_to_abs(queue->queue_pages[i]);
83 if (addr >= page && addr < page + queue->pagesize) {
84 *q_offset = addr - page + i * queue->pagesize;
85 return 0;
86 }
87 }
88 return -EINVAL;
89}
90
91#if PAGE_SHIFT < EHCA_PAGESHIFT
92#error Kernel pages must be at least as large than eHCA pages (4K) !
93#endif
94
95
96
97
98
99
100static int alloc_queue_pages(struct ipz_queue *queue, const u32 nr_of_pages)
101{
102 int k, f = 0;
103 u8 *kpage;
104
105 while (f < nr_of_pages) {
106 kpage = (u8 *)get_zeroed_page(GFP_KERNEL);
107 if (!kpage)
108 goto out;
109
110 for (k = 0; k < PAGES_PER_KPAGE && f < nr_of_pages; k++) {
111 queue->queue_pages[f] = (struct ipz_page *)kpage;
112 kpage += EHCA_PAGESIZE;
113 f++;
114 }
115 }
116 return 1;
117
118out:
119 for (f = 0; f < nr_of_pages && queue->queue_pages[f];
120 f += PAGES_PER_KPAGE)
121 free_page((unsigned long)(queue->queue_pages)[f]);
122 return 0;
123}
124
125static int alloc_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd)
126{
127 int order = ilog2(queue->pagesize) - 9;
128 struct ipz_small_queue_page *page;
129 unsigned long bit;
130
131 mutex_lock(&pd->lock);
132
133 if (!list_empty(&pd->free[order]))
134 page = list_entry(pd->free[order].next,
135 struct ipz_small_queue_page, list);
136 else {
137 page = kmem_cache_zalloc(small_qp_cache, GFP_KERNEL);
138 if (!page)
139 goto out;
140
141 page->page = get_zeroed_page(GFP_KERNEL);
142 if (!page->page) {
143 kmem_cache_free(small_qp_cache, page);
144 goto out;
145 }
146
147 list_add(&page->list, &pd->free[order]);
148 }
149
150 bit = find_first_zero_bit(page->bitmap, IPZ_SPAGE_PER_KPAGE >> order);
151 __set_bit(bit, page->bitmap);
152 page->fill++;
153
154 if (page->fill == IPZ_SPAGE_PER_KPAGE >> order)
155 list_move(&page->list, &pd->full[order]);
156
157 mutex_unlock(&pd->lock);
158
159 queue->queue_pages[0] = (void *)(page->page | (bit << (order + 9)));
160 queue->small_page = page;
161 queue->offset = bit << (order + 9);
162 return 1;
163
164out:
165 ehca_err(pd->ib_pd.device, "failed to allocate small queue page");
166 mutex_unlock(&pd->lock);
167 return 0;
168}
169
170static void free_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd)
171{
172 int order = ilog2(queue->pagesize) - 9;
173 struct ipz_small_queue_page *page = queue->small_page;
174 unsigned long bit;
175 int free_page = 0;
176
177 bit = ((unsigned long)queue->queue_pages[0] & ~PAGE_MASK)
178 >> (order + 9);
179
180 mutex_lock(&pd->lock);
181
182 __clear_bit(bit, page->bitmap);
183 page->fill--;
184
185 if (page->fill == 0) {
186 list_del(&page->list);
187 free_page = 1;
188 }
189
190 if (page->fill == (IPZ_SPAGE_PER_KPAGE >> order) - 1)
191
192 list_move_tail(&page->list, &pd->free[order]);
193
194 mutex_unlock(&pd->lock);
195
196 if (free_page) {
197 free_page(page->page);
198 kmem_cache_free(small_qp_cache, page);
199 }
200}
201
202int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue,
203 const u32 nr_of_pages, const u32 pagesize,
204 const u32 qe_size, const u32 nr_of_sg,
205 int is_small)
206{
207 if (pagesize > PAGE_SIZE) {
208 ehca_gen_err("FATAL ERROR: pagesize=%x "
209 "is greater than kernel page size", pagesize);
210 return 0;
211 }
212
213
214 queue->queue_length = nr_of_pages * pagesize;
215 queue->pagesize = pagesize;
216 queue->qe_size = qe_size;
217 queue->act_nr_of_sg = nr_of_sg;
218 queue->current_q_offset = 0;
219 queue->toggle_state = 1;
220 queue->small_page = NULL;
221
222
223 queue->queue_pages = kmalloc(nr_of_pages * sizeof(void *), GFP_KERNEL);
224 if (!queue->queue_pages) {
225 queue->queue_pages = vmalloc(nr_of_pages * sizeof(void *));
226 if (!queue->queue_pages) {
227 ehca_gen_err("Couldn't allocate queue page list");
228 return 0;
229 }
230 }
231 memset(queue->queue_pages, 0, nr_of_pages * sizeof(void *));
232
233
234 if (is_small) {
235 if (!alloc_small_queue_page(queue, pd))
236 goto ipz_queue_ctor_exit0;
237 } else
238 if (!alloc_queue_pages(queue, nr_of_pages))
239 goto ipz_queue_ctor_exit0;
240
241 return 1;
242
243ipz_queue_ctor_exit0:
244 ehca_gen_err("Couldn't alloc pages queue=%p "
245 "nr_of_pages=%x", queue, nr_of_pages);
246 if (is_vmalloc_addr(queue->queue_pages))
247 vfree(queue->queue_pages);
248 else
249 kfree(queue->queue_pages);
250
251 return 0;
252}
253
254int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue)
255{
256 int i, nr_pages;
257
258 if (!queue || !queue->queue_pages) {
259 ehca_gen_dbg("queue or queue_pages is NULL");
260 return 0;
261 }
262
263 if (queue->small_page)
264 free_small_queue_page(queue, pd);
265 else {
266 nr_pages = queue->queue_length / queue->pagesize;
267 for (i = 0; i < nr_pages; i += PAGES_PER_KPAGE)
268 free_page((unsigned long)queue->queue_pages[i]);
269 }
270
271 if (is_vmalloc_addr(queue->queue_pages))
272 vfree(queue->queue_pages);
273 else
274 kfree(queue->queue_pages);
275
276 return 1;
277}
278
279int ehca_init_small_qp_cache(void)
280{
281 small_qp_cache = kmem_cache_create("ehca_cache_small_qp",
282 sizeof(struct ipz_small_queue_page),
283 0, SLAB_HWCACHE_ALIGN, NULL);
284 if (!small_qp_cache)
285 return -ENOMEM;
286
287 return 0;
288}
289
290void ehca_cleanup_small_qp_cache(void)
291{
292 kmem_cache_destroy(small_qp_cache);
293}
294