1
2
3
4
5
6#include <linux/dma-buf.h>
7#include <linux/dma-resv.h>
8#include <linux/dma-mapping.h>
9#include <linux/module.h>
10
11#include "uverbs.h"
12
13MODULE_IMPORT_NS(DMA_BUF);
14
15int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf)
16{
17 struct sg_table *sgt;
18 struct scatterlist *sg;
19 unsigned long start, end, cur = 0;
20 unsigned int nmap = 0;
21 int i;
22
23 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv);
24
25 if (umem_dmabuf->sgt)
26 goto wait_fence;
27
28 sgt = dma_buf_map_attachment(umem_dmabuf->attach, DMA_BIDIRECTIONAL);
29 if (IS_ERR(sgt))
30 return PTR_ERR(sgt);
31
32
33
34 start = ALIGN_DOWN(umem_dmabuf->umem.address, PAGE_SIZE);
35 end = ALIGN(umem_dmabuf->umem.address + umem_dmabuf->umem.length,
36 PAGE_SIZE);
37 for_each_sgtable_dma_sg(sgt, sg, i) {
38 if (start < cur + sg_dma_len(sg) && cur < end)
39 nmap++;
40 if (cur <= start && start < cur + sg_dma_len(sg)) {
41 unsigned long offset = start - cur;
42
43 umem_dmabuf->first_sg = sg;
44 umem_dmabuf->first_sg_offset = offset;
45 sg_dma_address(sg) += offset;
46 sg_dma_len(sg) -= offset;
47 cur += offset;
48 }
49 if (cur < end && end <= cur + sg_dma_len(sg)) {
50 unsigned long trim = cur + sg_dma_len(sg) - end;
51
52 umem_dmabuf->last_sg = sg;
53 umem_dmabuf->last_sg_trim = trim;
54 sg_dma_len(sg) -= trim;
55 break;
56 }
57 cur += sg_dma_len(sg);
58 }
59
60 umem_dmabuf->umem.sgt_append.sgt.sgl = umem_dmabuf->first_sg;
61 umem_dmabuf->umem.sgt_append.sgt.nents = nmap;
62 umem_dmabuf->sgt = sgt;
63
64wait_fence:
65
66
67
68
69
70 return dma_resv_wait_timeout(umem_dmabuf->attach->dmabuf->resv,
71 DMA_RESV_USAGE_KERNEL,
72 false, MAX_SCHEDULE_TIMEOUT);
73}
74EXPORT_SYMBOL(ib_umem_dmabuf_map_pages);
75
76void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf)
77{
78 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv);
79
80 if (!umem_dmabuf->sgt)
81 return;
82
83
84 if (umem_dmabuf->first_sg) {
85 sg_dma_address(umem_dmabuf->first_sg) -=
86 umem_dmabuf->first_sg_offset;
87 sg_dma_len(umem_dmabuf->first_sg) +=
88 umem_dmabuf->first_sg_offset;
89 umem_dmabuf->first_sg = NULL;
90 umem_dmabuf->first_sg_offset = 0;
91 }
92 if (umem_dmabuf->last_sg) {
93 sg_dma_len(umem_dmabuf->last_sg) +=
94 umem_dmabuf->last_sg_trim;
95 umem_dmabuf->last_sg = NULL;
96 umem_dmabuf->last_sg_trim = 0;
97 }
98
99 dma_buf_unmap_attachment(umem_dmabuf->attach, umem_dmabuf->sgt,
100 DMA_BIDIRECTIONAL);
101
102 umem_dmabuf->sgt = NULL;
103}
104EXPORT_SYMBOL(ib_umem_dmabuf_unmap_pages);
105
106struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device,
107 unsigned long offset, size_t size,
108 int fd, int access,
109 const struct dma_buf_attach_ops *ops)
110{
111 struct dma_buf *dmabuf;
112 struct ib_umem_dmabuf *umem_dmabuf;
113 struct ib_umem *umem;
114 unsigned long end;
115 struct ib_umem_dmabuf *ret = ERR_PTR(-EINVAL);
116
117 if (check_add_overflow(offset, (unsigned long)size, &end))
118 return ret;
119
120 if (unlikely(!ops || !ops->move_notify))
121 return ret;
122
123 dmabuf = dma_buf_get(fd);
124 if (IS_ERR(dmabuf))
125 return ERR_CAST(dmabuf);
126
127 if (dmabuf->size < end)
128 goto out_release_dmabuf;
129
130 umem_dmabuf = kzalloc(sizeof(*umem_dmabuf), GFP_KERNEL);
131 if (!umem_dmabuf) {
132 ret = ERR_PTR(-ENOMEM);
133 goto out_release_dmabuf;
134 }
135
136 umem = &umem_dmabuf->umem;
137 umem->ibdev = device;
138 umem->length = size;
139 umem->address = offset;
140 umem->writable = ib_access_writable(access);
141 umem->is_dmabuf = 1;
142
143 if (!ib_umem_num_pages(umem))
144 goto out_free_umem;
145
146 umem_dmabuf->attach = dma_buf_dynamic_attach(
147 dmabuf,
148 device->dma_device,
149 ops,
150 umem_dmabuf);
151 if (IS_ERR(umem_dmabuf->attach)) {
152 ret = ERR_CAST(umem_dmabuf->attach);
153 goto out_free_umem;
154 }
155 return umem_dmabuf;
156
157out_free_umem:
158 kfree(umem_dmabuf);
159
160out_release_dmabuf:
161 dma_buf_put(dmabuf);
162 return ret;
163}
164EXPORT_SYMBOL(ib_umem_dmabuf_get);
165
166static void
167ib_umem_dmabuf_unsupported_move_notify(struct dma_buf_attachment *attach)
168{
169 struct ib_umem_dmabuf *umem_dmabuf = attach->importer_priv;
170
171 ibdev_warn_ratelimited(umem_dmabuf->umem.ibdev,
172 "Invalidate callback should not be called when memory is pinned\n");
173}
174
175static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = {
176 .allow_peer2peer = true,
177 .move_notify = ib_umem_dmabuf_unsupported_move_notify,
178};
179
180struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device,
181 unsigned long offset,
182 size_t size, int fd,
183 int access)
184{
185 struct ib_umem_dmabuf *umem_dmabuf;
186 int err;
187
188 umem_dmabuf = ib_umem_dmabuf_get(device, offset, size, fd, access,
189 &ib_umem_dmabuf_attach_pinned_ops);
190 if (IS_ERR(umem_dmabuf))
191 return umem_dmabuf;
192
193 dma_resv_lock(umem_dmabuf->attach->dmabuf->resv, NULL);
194 err = dma_buf_pin(umem_dmabuf->attach);
195 if (err)
196 goto err_release;
197 umem_dmabuf->pinned = 1;
198
199 err = ib_umem_dmabuf_map_pages(umem_dmabuf);
200 if (err)
201 goto err_unpin;
202 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv);
203
204 return umem_dmabuf;
205
206err_unpin:
207 dma_buf_unpin(umem_dmabuf->attach);
208err_release:
209 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv);
210 ib_umem_release(&umem_dmabuf->umem);
211 return ERR_PTR(err);
212}
213EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned);
214
215void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf)
216{
217 struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf;
218
219 dma_resv_lock(dmabuf->resv, NULL);
220 ib_umem_dmabuf_unmap_pages(umem_dmabuf);
221 if (umem_dmabuf->pinned)
222 dma_buf_unpin(umem_dmabuf->attach);
223 dma_resv_unlock(dmabuf->resv);
224
225 dma_buf_detach(dmabuf, umem_dmabuf->attach);
226 dma_buf_put(dmabuf);
227 kfree(umem_dmabuf);
228}
229
