1
2
3
4
5#include <string.h>
6#include <unistd.h>
7
8#include <rte_bus.h>
9#include <rte_bus_pci.h>
10#include <rte_common.h>
11#include <rte_eal.h>
12#include <rte_lcore.h>
13#include <rte_mempool.h>
14#include <rte_pci.h>
15#include <rte_rawdev.h>
16#include <rte_rawdev_pmd.h>
17
18#include <otx2_common.h>
19
20#include "otx2_dpi_rawdev.h"
21
22static const struct rte_pci_id pci_dma_map[] = {
23 {
24 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
25 PCI_DEVID_OCTEONTX2_DPI_VF)
26 },
27 {
28 .vendor_id = 0,
29 },
30};
31
32
33static inline int
34dma_engine_enb_dis(struct dpi_vf_s *dpivf, const bool enb)
35{
36 if (enb)
37 otx2_write64(0x1, dpivf->vf_bar0 + DPI_VDMA_EN);
38 else
39 otx2_write64(0x0, dpivf->vf_bar0 + DPI_VDMA_EN);
40
41 return DPI_DMA_QUEUE_SUCCESS;
42}
43
44
45static inline int
46dma_queue_finish(struct dpi_vf_s *dpivf)
47{
48 uint32_t timeout = 0, sleep = 1;
49 uint64_t reg = 0ULL;
50
51
52 reg = otx2_read64(dpivf->vf_bar0 + DPI_VDMA_SADDR);
53 while (!(reg & BIT_ULL(DPI_VDMA_SADDR_REQ_IDLE))) {
54 rte_delay_ms(sleep);
55 timeout++;
56 if (timeout >= DPI_QFINISH_TIMEOUT) {
57 otx2_dpi_dbg("Timeout!!! Closing Forcibly");
58 break;
59 }
60 reg = otx2_read64(dpivf->vf_bar0 + DPI_VDMA_SADDR);
61 }
62
63 if (otx2_dpi_queue_close(dpivf) < 0)
64 return -EACCES;
65
66 rte_mempool_put(dpivf->chunk_pool, dpivf->base_ptr);
67 dpivf->vf_bar0 = (uintptr_t)NULL;
68
69 return DPI_DMA_QUEUE_SUCCESS;
70}
71
72
73static __rte_always_inline enum dpi_dma_queue_result_e
74dma_queue_write(struct dpi_vf_s *dpi, uint16_t cmd_count, uint64_t *cmds)
75{
76 if ((cmd_count < 1) || (cmd_count > 64))
77 return DPI_DMA_QUEUE_INVALID_PARAM;
78
79 if (cmds == NULL)
80 return DPI_DMA_QUEUE_INVALID_PARAM;
81
82
83 if (dpi->index + cmd_count < dpi->pool_size_m1) {
84 uint64_t *ptr = dpi->base_ptr;
85
86 ptr += dpi->index;
87 dpi->index += cmd_count;
88 while (cmd_count--)
89 *ptr++ = *cmds++;
90 } else {
91 void *new_buffer;
92 uint64_t *ptr;
93 int count;
94
95
96 if (rte_mempool_get(dpi->chunk_pool, &new_buffer) ||
97 new_buffer == NULL) {
98 return DPI_DMA_QUEUE_NO_MEMORY;
99 }
100 ptr = dpi->base_ptr;
101
102
103
104 count = dpi->pool_size_m1 - dpi->index;
105 ptr += dpi->index;
106 cmd_count -= count;
107 while (count--)
108 *ptr++ = *cmds++;
109
110 *ptr++ = (uint64_t)new_buffer;
111 *ptr = 0;
112
113
114
115 dpi->base_ptr = new_buffer;
116 dpi->index = cmd_count;
117 ptr = new_buffer;
118 while (cmd_count--)
119 *ptr++ = *cmds++;
120
121 if (dpi->index >= dpi->pool_size_m1) {
122 if (rte_mempool_get(dpi->chunk_pool, &new_buffer) ||
123 new_buffer == NULL) {
124 return DPI_DMA_QUEUE_NO_MEMORY;
125 }
126
127 *ptr = (uint64_t)new_buffer;
128 dpi->base_ptr = new_buffer;
129 dpi->index = 0;
130 }
131 }
132 return DPI_DMA_QUEUE_SUCCESS;
133}
134
135
136static __rte_always_inline int
137dma_queue_submit(struct rte_rawdev *dev, uint16_t cmd_count, uint64_t *cmds)
138{
139 struct dpi_vf_s *dpivf = dev->dev_private;
140 enum dpi_dma_queue_result_e result;
141
142 result = dma_queue_write(dpivf, cmd_count, cmds);
143 rte_wmb();
144 if (likely(result == DPI_DMA_QUEUE_SUCCESS))
145 otx2_write64((uint64_t)cmd_count,
146 dpivf->vf_bar0 + DPI_VDMA_DBELL);
147
148 return result;
149}
150
151
152
153
154static int
155otx2_dpi_rawdev_enqueue_bufs(struct rte_rawdev *dev,
156 struct rte_rawdev_buf **buffers,
157 unsigned int count, rte_rawdev_obj_t context)
158{
159 struct dpi_dma_queue_ctx_s *ctx = (struct dpi_dma_queue_ctx_s *)context;
160 struct dpi_dma_buf_ptr_s *cmd;
161 uint32_t c = 0;
162
163 for (c = 0; c < count; c++) {
164 uint64_t dpi_cmd[DPI_DMA_CMD_SIZE] = {0};
165 union dpi_dma_instr_hdr_u *hdr;
166 uint16_t index = 0, i;
167
168 hdr = (union dpi_dma_instr_hdr_u *)&dpi_cmd[0];
169 cmd = (struct dpi_dma_buf_ptr_s *)buffers[c]->buf_addr;
170
171 hdr->s.xtype = ctx->xtype & DPI_XTYPE_MASK;
172 hdr->s.pt = ctx->pt & DPI_HDR_PT_MASK;
173
174
175
176 if ((hdr->s.pt == DPI_HDR_PT_ZBW_CA ||
177 hdr->s.pt == DPI_HDR_PT_ZBW_NC) && cmd->comp_ptr == NULL)
178 return c;
179
180 cmd->comp_ptr->cdata = DPI_REQ_CDATA;
181 hdr->s.ptr = (uint64_t)cmd->comp_ptr;
182 hdr->s.deallocv = ctx->deallocv;
183 hdr->s.tt = ctx->tt & DPI_W0_TT_MASK;
184 hdr->s.grp = ctx->grp & DPI_W0_GRP_MASK;
185
186
187
188
189 if (ctx->c_ring) {
190 ctx->c_ring->compl_data[ctx->c_ring->tail] =
191 cmd->comp_ptr;
192 STRM_INC(ctx->c_ring);
193 }
194
195 if (hdr->s.deallocv)
196 hdr->s.pvfe = 1;
197
198 if (hdr->s.pt == DPI_HDR_PT_WQP)
199 hdr->s.ptr = hdr->s.ptr | DPI_HDR_PT_WQP_STATUSNC;
200
201 index += 4;
202 hdr->s.fport = 0;
203 hdr->s.lport = 0;
204 if (ctx->xtype != DPI_XTYPE_INTERNAL_ONLY)
205 hdr->s.lport = ctx->pem_id;
206
207
208
209
210 if (ctx->xtype == DPI_XTYPE_INBOUND) {
211 hdr->s.nfst = cmd->wptr_cnt & DPI_MAX_POINTER;
212 hdr->s.nlst = cmd->rptr_cnt & DPI_MAX_POINTER;
213 for (i = 0; i < hdr->s.nfst; i++) {
214 dpi_cmd[index++] = cmd->wptr[i]->u[0];
215 dpi_cmd[index++] = cmd->wptr[i]->u[1];
216 }
217 for (i = 0; i < hdr->s.nlst; i++) {
218 dpi_cmd[index++] = cmd->rptr[i]->u[0];
219 dpi_cmd[index++] = cmd->rptr[i]->u[1];
220 }
221 } else {
222 hdr->s.nfst = cmd->rptr_cnt & DPI_MAX_POINTER;
223 hdr->s.nlst = cmd->wptr_cnt & DPI_MAX_POINTER;
224 for (i = 0; i < hdr->s.nfst; i++) {
225 dpi_cmd[index++] = cmd->rptr[i]->u[0];
226 dpi_cmd[index++] = cmd->rptr[i]->u[1];
227 }
228 for (i = 0; i < hdr->s.nlst; i++) {
229 dpi_cmd[index++] = cmd->wptr[i]->u[0];
230 dpi_cmd[index++] = cmd->wptr[i]->u[1];
231 }
232 }
233 if (dma_queue_submit(dev, index, dpi_cmd))
234 return c;
235 }
236 return c;
237}
238
239
240static int
241otx2_dpi_rawdev_dequeue_bufs(struct rte_rawdev *dev __rte_unused,
242 struct rte_rawdev_buf **buffers,
243 unsigned int count, rte_rawdev_obj_t context)
244{
245 struct dpi_dma_queue_ctx_s *ctx = (struct dpi_dma_queue_ctx_s *)context;
246 unsigned int i = 0, headp;
247
248
249 if (ctx->c_ring == NULL)
250 return 0;
251
252 headp = ctx->c_ring->head;
253 for (i = 0; i < count && (headp != ctx->c_ring->tail); i++) {
254 struct dpi_dma_req_compl_s *comp_ptr =
255 ctx->c_ring->compl_data[headp];
256
257 if (comp_ptr->cdata)
258 break;
259
260
261 buffers[i] = (void *)comp_ptr;
262 headp = (headp + 1) % ctx->c_ring->max_cnt;
263 }
264 ctx->c_ring->head = headp;
265
266 return i;
267}
268
269static int
270otx2_dpi_rawdev_start(struct rte_rawdev *dev)
271{
272 dev->started = DPI_QUEUE_START;
273
274 return DPI_DMA_QUEUE_SUCCESS;
275}
276
277static void
278otx2_dpi_rawdev_stop(struct rte_rawdev *dev)
279{
280 dev->started = DPI_QUEUE_STOP;
281}
282
283static int
284otx2_dpi_rawdev_close(struct rte_rawdev *dev)
285{
286 dma_engine_enb_dis(dev->dev_private, false);
287 dma_queue_finish(dev->dev_private);
288
289 return DPI_DMA_QUEUE_SUCCESS;
290}
291
292static int
293otx2_dpi_rawdev_reset(struct rte_rawdev *dev)
294{
295 return dev ? DPI_QUEUE_STOP : DPI_QUEUE_START;
296}
297
298static int
299otx2_dpi_rawdev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config,
300 size_t config_size)
301{
302 struct dpi_rawdev_conf_s *conf = config;
303 struct dpi_vf_s *dpivf = NULL;
304 void *buf = NULL;
305 uintptr_t pool;
306 uint32_t gaura;
307
308 if (conf == NULL || config_size != sizeof(*conf)) {
309 otx2_dpi_dbg("NULL or invalid configuration");
310 return -EINVAL;
311 }
312 dpivf = (struct dpi_vf_s *)dev->dev_private;
313 dpivf->chunk_pool = conf->chunk_pool;
314 if (rte_mempool_get(conf->chunk_pool, &buf) || (buf == NULL)) {
315 otx2_err("Unable allocate buffer");
316 return -ENODEV;
317 }
318 dpivf->base_ptr = buf;
319 otx2_write64(0x0, dpivf->vf_bar0 + DPI_VDMA_EN);
320 dpivf->pool_size_m1 = (DPI_CHUNK_SIZE >> 3) - 2;
321 pool = (uintptr_t)((struct rte_mempool *)conf->chunk_pool)->pool_id;
322 gaura = npa_lf_aura_handle_to_aura(pool);
323 otx2_write64(0, dpivf->vf_bar0 + DPI_VDMA_REQQ_CTL);
324 otx2_write64(((uint64_t)buf >> 7) << 7,
325 dpivf->vf_bar0 + DPI_VDMA_SADDR);
326 if (otx2_dpi_queue_open(dpivf, DPI_CHUNK_SIZE, gaura) < 0) {
327 otx2_err("Unable to open DPI VF %d", dpivf->vf_id);
328 rte_mempool_put(conf->chunk_pool, buf);
329 return -EACCES;
330 }
331 dma_engine_enb_dis(dpivf, true);
332
333 return DPI_DMA_QUEUE_SUCCESS;
334}
335
336static const struct rte_rawdev_ops dpi_rawdev_ops = {
337 .dev_configure = otx2_dpi_rawdev_configure,
338 .dev_start = otx2_dpi_rawdev_start,
339 .dev_stop = otx2_dpi_rawdev_stop,
340 .dev_close = otx2_dpi_rawdev_close,
341 .dev_reset = otx2_dpi_rawdev_reset,
342 .enqueue_bufs = otx2_dpi_rawdev_enqueue_bufs,
343 .dequeue_bufs = otx2_dpi_rawdev_dequeue_bufs,
344 .dev_selftest = test_otx2_dma_rawdev,
345};
346
347static int
348otx2_dpi_rawdev_probe(struct rte_pci_driver *pci_drv __rte_unused,
349 struct rte_pci_device *pci_dev)
350{
351 char name[RTE_RAWDEV_NAME_MAX_LEN];
352 struct dpi_vf_s *dpivf = NULL;
353 struct rte_rawdev *rawdev;
354 uint16_t vf_id;
355
356
357 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
358 return DPI_DMA_QUEUE_SUCCESS;
359
360 if (pci_dev->mem_resource[0].addr == NULL) {
361 otx2_dpi_dbg("Empty bars %p %p", pci_dev->mem_resource[0].addr,
362 pci_dev->mem_resource[2].addr);
363 return -ENODEV;
364 }
365
366 memset(name, 0, sizeof(name));
367 snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "DPI:%x:%02x.%x",
368 pci_dev->addr.bus, pci_dev->addr.devid,
369 pci_dev->addr.function);
370
371
372 rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct dpi_vf_s),
373 rte_socket_id());
374 if (rawdev == NULL) {
375 otx2_err("Rawdev allocation failed");
376 return -EINVAL;
377 }
378
379 rawdev->dev_ops = &dpi_rawdev_ops;
380 rawdev->device = &pci_dev->device;
381 rawdev->driver_name = pci_dev->driver->driver.name;
382
383 dpivf = rawdev->dev_private;
384 if (dpivf->state != DPI_QUEUE_STOP) {
385 otx2_dpi_dbg("Device already started!!!");
386 return -ENODEV;
387 }
388
389 vf_id = ((pci_dev->addr.devid & 0x1F) << 3) |
390 (pci_dev->addr.function & 0x7);
391 vf_id -= 1;
392 dpivf->dev = pci_dev;
393 dpivf->state = DPI_QUEUE_START;
394 dpivf->vf_id = vf_id;
395 dpivf->vf_bar0 = (uintptr_t)pci_dev->mem_resource[0].addr;
396 dpivf->vf_bar2 = (uintptr_t)pci_dev->mem_resource[2].addr;
397
398 return DPI_DMA_QUEUE_SUCCESS;
399}
400
401static int
402otx2_dpi_rawdev_remove(struct rte_pci_device *pci_dev)
403{
404 char name[RTE_RAWDEV_NAME_MAX_LEN];
405 struct rte_rawdev *rawdev;
406 struct dpi_vf_s *dpivf;
407
408 if (pci_dev == NULL) {
409 otx2_dpi_dbg("Invalid pci_dev of the device!");
410 return -EINVAL;
411 }
412
413 memset(name, 0, sizeof(name));
414 snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "DPI:%x:%02x.%x",
415 pci_dev->addr.bus, pci_dev->addr.devid,
416 pci_dev->addr.function);
417
418 rawdev = rte_rawdev_pmd_get_named_dev(name);
419 if (rawdev == NULL) {
420 otx2_dpi_dbg("Invalid device name (%s)", name);
421 return -EINVAL;
422 }
423
424 dpivf = (struct dpi_vf_s *)rawdev->dev_private;
425 dma_engine_enb_dis(dpivf, false);
426 dma_queue_finish(dpivf);
427
428
429 return rte_rawdev_pmd_release(rawdev);
430}
431
432static struct rte_pci_driver rte_dpi_rawdev_pmd = {
433 .id_table = pci_dma_map,
434 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_NEED_IOVA_AS_VA,
435 .probe = otx2_dpi_rawdev_probe,
436 .remove = otx2_dpi_rawdev_remove,
437};
438
439RTE_PMD_REGISTER_PCI(dpi_rawdev_pci_driver, rte_dpi_rawdev_pmd);
440RTE_PMD_REGISTER_PCI_TABLE(dpi_rawdev_pci_driver, pci_dma_map);
441RTE_PMD_REGISTER_KMOD_DEP(dpi_rawdev_pci_driver, "vfio-pci");
442
