1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/module.h>
34#include <rdma/ib_umem.h>
35#include <rdma/ib_umem_odp.h>
36#include "mlx5_ib.h"
37
38
39
40
41
42
43
44
45
46void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
47 unsigned long max_page_shift,
48 int *count, int *shift,
49 int *ncont, int *order)
50{
51 unsigned long tmp;
52 unsigned long m;
53 u64 base = ~0, p = 0;
54 u64 len, pfn;
55 int i = 0;
56 struct scatterlist *sg;
57 int entry;
58
59 if (umem->is_odp) {
60 struct ib_umem_odp *odp = to_ib_umem_odp(umem);
61 unsigned int page_shift = odp->page_shift;
62
63 *ncont = ib_umem_odp_num_pages(odp);
64 *count = *ncont << (page_shift - PAGE_SHIFT);
65 *shift = page_shift;
66 if (order)
67 *order = ilog2(roundup_pow_of_two(*ncont));
68
69 return;
70 }
71
72 addr = addr >> PAGE_SHIFT;
73 tmp = (unsigned long)addr;
74 m = find_first_bit(&tmp, BITS_PER_LONG);
75 if (max_page_shift)
76 m = min_t(unsigned long, max_page_shift - PAGE_SHIFT, m);
77
78 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
79 len = sg_dma_len(sg) >> PAGE_SHIFT;
80 pfn = sg_dma_address(sg) >> PAGE_SHIFT;
81 if (base + p != pfn) {
82
83
84
85 tmp = (unsigned long)(pfn | p);
86 if (!IS_ALIGNED(tmp, 1 << m))
87 m = find_first_bit(&tmp, BITS_PER_LONG);
88
89 base = pfn;
90 p = 0;
91 }
92
93 p += len;
94 i += len;
95 }
96
97 if (i) {
98 m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m);
99
100 if (order)
101 *order = ilog2(roundup_pow_of_two(i) >> m);
102
103 *ncont = DIV_ROUND_UP(i, (1 << m));
104 } else {
105 m = 0;
106
107 if (order)
108 *order = 0;
109
110 *ncont = 0;
111 }
112 *shift = PAGE_SHIFT + m;
113 *count = i;
114}
115
116static u64 umem_dma_to_mtt(dma_addr_t umem_dma)
117{
118 u64 mtt_entry = umem_dma & ODP_DMA_ADDR_MASK;
119
120 if (umem_dma & ODP_READ_ALLOWED_BIT)
121 mtt_entry |= MLX5_IB_MTT_READ;
122 if (umem_dma & ODP_WRITE_ALLOWED_BIT)
123 mtt_entry |= MLX5_IB_MTT_WRITE;
124
125 return mtt_entry;
126}
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
142 int page_shift, size_t offset, size_t num_pages,
143 __be64 *pas, int access_flags)
144{
145 int shift = page_shift - PAGE_SHIFT;
146 int mask = (1 << shift) - 1;
147 int i, k, idx;
148 u64 cur = 0;
149 u64 base;
150 int len;
151 struct scatterlist *sg;
152 int entry;
153
154 if (umem->is_odp) {
155 WARN_ON(shift != 0);
156 WARN_ON(access_flags != (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE));
157
158 for (i = 0; i < num_pages; ++i) {
159 dma_addr_t pa =
160 to_ib_umem_odp(umem)->dma_list[offset + i];
161
162 pas[i] = cpu_to_be64(umem_dma_to_mtt(pa));
163 }
164 return;
165 }
166
167 i = 0;
168 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
169 len = sg_dma_len(sg) >> PAGE_SHIFT;
170 base = sg_dma_address(sg);
171
172
173 if (i + len < offset << shift) {
174 i += len;
175 continue;
176 }
177
178
179 if (i < offset << shift) {
180 k = (offset << shift) - i;
181 i = offset << shift;
182 } else {
183 k = 0;
184 }
185
186 for (; k < len; k++) {
187 if (!(i & mask)) {
188 cur = base + (k << PAGE_SHIFT);
189 cur |= access_flags;
190 idx = (i >> shift) - offset;
191
192 pas[idx] = cpu_to_be64(cur);
193 mlx5_ib_dbg(dev, "pas[%d] 0x%llx\n",
194 i >> shift, be64_to_cpu(pas[idx]));
195 }
196 i++;
197
198
199 if (i >> shift >= offset + num_pages)
200 return;
201 }
202 }
203}
204
205void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
206 int page_shift, __be64 *pas, int access_flags)
207{
208 return __mlx5_ib_populate_pas(dev, umem, page_shift, 0,
209 ib_umem_num_pages(umem), pas,
210 access_flags);
211}
212int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset)
213{
214 u64 page_size;
215 u64 page_mask;
216 u64 off_size;
217 u64 off_mask;
218 u64 buf_off;
219
220 page_size = (u64)1 << page_shift;
221 page_mask = page_size - 1;
222 buf_off = addr & page_mask;
223 off_size = page_size >> 6;
224 off_mask = off_size - 1;
225
226 if (buf_off & off_mask)
227 return -EINVAL;
228
229 *offset = buf_off >> ilog2(off_size);
230 return 0;
231}
232