1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/platform_device.h>
35#include "hns_roce_device.h"
36#include "hns_roce_hem.h"
37#include "hns_roce_common.h"
38
39#define HNS_ROCE_HEM_ALLOC_SIZE (1 << 17)
40#define HNS_ROCE_TABLE_CHUNK_SIZE (1 << 17)
41
42#define DMA_ADDR_T_SHIFT 12
43#define BT_BA_SHIFT 32
44
45struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, int npages,
46 gfp_t gfp_mask)
47{
48 struct hns_roce_hem_chunk *chunk = NULL;
49 struct hns_roce_hem *hem;
50 struct scatterlist *mem;
51 int order;
52 void *buf;
53
54 WARN_ON(gfp_mask & __GFP_HIGHMEM);
55
56 hem = kmalloc(sizeof(*hem),
57 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
58 if (!hem)
59 return NULL;
60
61 hem->refcount = 0;
62 INIT_LIST_HEAD(&hem->chunk_list);
63
64 order = get_order(HNS_ROCE_HEM_ALLOC_SIZE);
65
66 while (npages > 0) {
67 if (!chunk) {
68 chunk = kmalloc(sizeof(*chunk),
69 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
70 if (!chunk)
71 goto fail;
72
73 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
74 chunk->npages = 0;
75 chunk->nsg = 0;
76 list_add_tail(&chunk->list, &hem->chunk_list);
77 }
78
79 while (1 << order > npages)
80 --order;
81
82
83
84
85
86 mem = &chunk->mem[chunk->npages];
87 buf = dma_alloc_coherent(&hr_dev->pdev->dev, PAGE_SIZE << order,
88 &sg_dma_address(mem), gfp_mask);
89 if (!buf)
90 goto fail;
91
92 sg_set_buf(mem, buf, PAGE_SIZE << order);
93 WARN_ON(mem->offset);
94 sg_dma_len(mem) = PAGE_SIZE << order;
95
96 ++chunk->npages;
97 ++chunk->nsg;
98 npages -= 1 << order;
99 }
100
101 return hem;
102
103fail:
104 hns_roce_free_hem(hr_dev, hem);
105 return NULL;
106}
107
108void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
109{
110 struct hns_roce_hem_chunk *chunk, *tmp;
111 int i;
112
113 if (!hem)
114 return;
115
116 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
117 for (i = 0; i < chunk->npages; ++i)
118 dma_free_coherent(&hr_dev->pdev->dev,
119 chunk->mem[i].length,
120 lowmem_page_address(sg_page(&chunk->mem[i])),
121 sg_dma_address(&chunk->mem[i]));
122 kfree(chunk);
123 }
124
125 kfree(hem);
126}
127
128static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
129 struct hns_roce_hem_table *table, unsigned long obj)
130{
131 struct device *dev = &hr_dev->pdev->dev;
132 spinlock_t *lock = &hr_dev->bt_cmd_lock;
133 unsigned long end = 0;
134 unsigned long flags;
135 struct hns_roce_hem_iter iter;
136 void __iomem *bt_cmd;
137 u32 bt_cmd_h_val = 0;
138 u32 bt_cmd_val[2];
139 u32 bt_cmd_l = 0;
140 u64 bt_ba = 0;
141 int ret = 0;
142
143
144 unsigned long i = (obj & (table->num_obj - 1)) /
145 (HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size);
146
147 switch (table->type) {
148 case HEM_TYPE_QPC:
149 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
150 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
151 break;
152 case HEM_TYPE_MTPT:
153 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
154 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
155 HEM_TYPE_MTPT);
156 break;
157 case HEM_TYPE_CQC:
158 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
159 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
160 break;
161 case HEM_TYPE_SRQC:
162 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
163 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
164 HEM_TYPE_SRQC);
165 break;
166 default:
167 return ret;
168 }
169 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
170 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
171 roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
172 roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
173
174
175 for (hns_roce_hem_first(table->hem[i], &iter);
176 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
177 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT;
178
179 spin_lock_irqsave(lock, flags);
180
181 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
182
183 end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
184 while (1) {
185 if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
186 if (!(time_before(jiffies, end))) {
187 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
188 spin_unlock_irqrestore(lock, flags);
189 return -EBUSY;
190 }
191 } else {
192 break;
193 }
194 msleep(HW_SYNC_SLEEP_TIME_INTERVAL);
195 }
196
197 bt_cmd_l = (u32)bt_ba;
198 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
199 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
200 bt_ba >> BT_BA_SHIFT);
201
202 bt_cmd_val[0] = bt_cmd_l;
203 bt_cmd_val[1] = bt_cmd_h_val;
204 hns_roce_write64_k(bt_cmd_val,
205 hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
206 spin_unlock_irqrestore(lock, flags);
207 }
208
209 return ret;
210}
211
212int hns_roce_table_get(struct hns_roce_dev *hr_dev,
213 struct hns_roce_hem_table *table, unsigned long obj)
214{
215 struct device *dev = &hr_dev->pdev->dev;
216 int ret = 0;
217 unsigned long i;
218
219 i = (obj & (table->num_obj - 1)) / (HNS_ROCE_TABLE_CHUNK_SIZE /
220 table->obj_size);
221
222 mutex_lock(&table->mutex);
223
224 if (table->hem[i]) {
225 ++table->hem[i]->refcount;
226 goto out;
227 }
228
229 table->hem[i] = hns_roce_alloc_hem(hr_dev,
230 HNS_ROCE_TABLE_CHUNK_SIZE >> PAGE_SHIFT,
231 (table->lowmem ? GFP_KERNEL :
232 GFP_HIGHUSER) | __GFP_NOWARN);
233 if (!table->hem[i]) {
234 ret = -ENOMEM;
235 goto out;
236 }
237
238
239 if (hns_roce_set_hem(hr_dev, table, obj)) {
240 ret = -ENODEV;
241 dev_err(dev, "set HEM base address to HW failed.\n");
242 goto out;
243 }
244
245 ++table->hem[i]->refcount;
246out:
247 mutex_unlock(&table->mutex);
248 return ret;
249}
250
251void hns_roce_table_put(struct hns_roce_dev *hr_dev,
252 struct hns_roce_hem_table *table, unsigned long obj)
253{
254 struct device *dev = &hr_dev->pdev->dev;
255 unsigned long i;
256
257 i = (obj & (table->num_obj - 1)) /
258 (HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size);
259
260 mutex_lock(&table->mutex);
261
262 if (--table->hem[i]->refcount == 0) {
263
264 if (hr_dev->hw->clear_hem(hr_dev, table, obj))
265 dev_warn(dev, "Clear HEM base address failed.\n");
266
267 hns_roce_free_hem(hr_dev, table->hem[i]);
268 table->hem[i] = NULL;
269 }
270
271 mutex_unlock(&table->mutex);
272}
273
274void *hns_roce_table_find(struct hns_roce_hem_table *table, unsigned long obj,
275 dma_addr_t *dma_handle)
276{
277 struct hns_roce_hem_chunk *chunk;
278 unsigned long idx;
279 int i;
280 int offset, dma_offset;
281 struct hns_roce_hem *hem;
282 struct page *page = NULL;
283
284 if (!table->lowmem)
285 return NULL;
286
287 mutex_lock(&table->mutex);
288 idx = (obj & (table->num_obj - 1)) * table->obj_size;
289 hem = table->hem[idx / HNS_ROCE_TABLE_CHUNK_SIZE];
290 dma_offset = offset = idx % HNS_ROCE_TABLE_CHUNK_SIZE;
291
292 if (!hem)
293 goto out;
294
295 list_for_each_entry(chunk, &hem->chunk_list, list) {
296 for (i = 0; i < chunk->npages; ++i) {
297 if (dma_handle && dma_offset >= 0) {
298 if (sg_dma_len(&chunk->mem[i]) >
299 (u32)dma_offset)
300 *dma_handle = sg_dma_address(
301 &chunk->mem[i]) + dma_offset;
302 dma_offset -= sg_dma_len(&chunk->mem[i]);
303 }
304
305 if (chunk->mem[i].length > (u32)offset) {
306 page = sg_page(&chunk->mem[i]);
307 goto out;
308 }
309 offset -= chunk->mem[i].length;
310 }
311 }
312
313out:
314 mutex_unlock(&table->mutex);
315 return page ? lowmem_page_address(page) + offset : NULL;
316}
317
318int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
319 struct hns_roce_hem_table *table,
320 unsigned long start, unsigned long end)
321{
322 unsigned long inc = HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size;
323 unsigned long i = 0;
324 int ret = 0;
325
326
327 for (i = start; i <= end; i += inc) {
328 ret = hns_roce_table_get(hr_dev, table, i);
329 if (ret)
330 goto fail;
331 }
332
333 return 0;
334
335fail:
336 while (i > start) {
337 i -= inc;
338 hns_roce_table_put(hr_dev, table, i);
339 }
340 return ret;
341}
342
343void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
344 struct hns_roce_hem_table *table,
345 unsigned long start, unsigned long end)
346{
347 unsigned long i;
348
349 for (i = start; i <= end;
350 i += HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size)
351 hns_roce_table_put(hr_dev, table, i);
352}
353
354int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
355 struct hns_roce_hem_table *table, u32 type,
356 unsigned long obj_size, unsigned long nobj,
357 int use_lowmem)
358{
359 unsigned long obj_per_chunk;
360 unsigned long num_hem;
361
362 obj_per_chunk = HNS_ROCE_TABLE_CHUNK_SIZE / obj_size;
363 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
364
365 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
366 if (!table->hem)
367 return -ENOMEM;
368
369 table->type = type;
370 table->num_hem = num_hem;
371 table->num_obj = nobj;
372 table->obj_size = obj_size;
373 table->lowmem = use_lowmem;
374 mutex_init(&table->mutex);
375
376 return 0;
377}
378
379void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
380 struct hns_roce_hem_table *table)
381{
382 struct device *dev = &hr_dev->pdev->dev;
383 unsigned long i;
384
385 for (i = 0; i < table->num_hem; ++i)
386 if (table->hem[i]) {
387 if (hr_dev->hw->clear_hem(hr_dev, table,
388 i * HNS_ROCE_TABLE_CHUNK_SIZE / table->obj_size))
389 dev_err(dev, "Clear HEM base address failed.\n");
390
391 hns_roce_free_hem(hr_dev, table->hem[i]);
392 }
393
394 kfree(table->hem);
395}
396
397void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
398{
399 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
400 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
401 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
402 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
403 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
404}
405
