1
2
3
4
5#include <unistd.h>
6
7#include <rte_common.h>
8#include <rte_log.h>
9#include <rte_dev.h>
10#include <rte_malloc.h>
11#include <rte_mempool.h>
12#include <rte_errno.h>
13#include <rte_pci.h>
14#include <rte_bus_pci.h>
15#include <rte_byteorder.h>
16#ifdef RTE_BBDEV_OFFLOAD_COST
17#include <rte_cycles.h>
18#endif
19
20#include <rte_bbdev.h>
21#include <rte_bbdev_pmd.h>
22
23#include "fpga_5gnr_fec.h"
24#include "rte_pmd_fpga_5gnr_fec.h"
25
26#ifdef RTE_LIBRTE_BBDEV_DEBUG
27RTE_LOG_REGISTER_DEFAULT(fpga_5gnr_fec_logtype, DEBUG);
28#else
29RTE_LOG_REGISTER_DEFAULT(fpga_5gnr_fec_logtype, NOTICE);
30#endif
31
32#ifdef RTE_LIBRTE_BBDEV_DEBUG
33
34
35static inline void
36print_ring_reg_debug_info(void *mmio_base, uint32_t offset)
37{
38 rte_bbdev_log_debug(
39 "FPGA MMIO base address @ %p | Ring Control Register @ offset = 0x%08"
40 PRIx32, mmio_base, offset);
41 rte_bbdev_log_debug(
42 "RING_BASE_ADDR = 0x%016"PRIx64,
43 fpga_reg_read_64(mmio_base, offset));
44 rte_bbdev_log_debug(
45 "RING_HEAD_ADDR = 0x%016"PRIx64,
46 fpga_reg_read_64(mmio_base, offset +
47 FPGA_5GNR_FEC_RING_HEAD_ADDR));
48 rte_bbdev_log_debug(
49 "RING_SIZE = 0x%04"PRIx16,
50 fpga_reg_read_16(mmio_base, offset +
51 FPGA_5GNR_FEC_RING_SIZE));
52 rte_bbdev_log_debug(
53 "RING_MISC = 0x%02"PRIx8,
54 fpga_reg_read_8(mmio_base, offset +
55 FPGA_5GNR_FEC_RING_MISC));
56 rte_bbdev_log_debug(
57 "RING_ENABLE = 0x%02"PRIx8,
58 fpga_reg_read_8(mmio_base, offset +
59 FPGA_5GNR_FEC_RING_ENABLE));
60 rte_bbdev_log_debug(
61 "RING_FLUSH_QUEUE_EN = 0x%02"PRIx8,
62 fpga_reg_read_8(mmio_base, offset +
63 FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN));
64 rte_bbdev_log_debug(
65 "RING_SHADOW_TAIL = 0x%04"PRIx16,
66 fpga_reg_read_16(mmio_base, offset +
67 FPGA_5GNR_FEC_RING_SHADOW_TAIL));
68 rte_bbdev_log_debug(
69 "RING_HEAD_POINT = 0x%04"PRIx16,
70 fpga_reg_read_16(mmio_base, offset +
71 FPGA_5GNR_FEC_RING_HEAD_POINT));
72}
73
74
75static inline void
76print_static_reg_debug_info(void *mmio_base)
77{
78 uint16_t config = fpga_reg_read_16(mmio_base,
79 FPGA_5GNR_FEC_CONFIGURATION);
80 uint8_t qmap_done = fpga_reg_read_8(mmio_base,
81 FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE);
82 uint16_t lb_factor = fpga_reg_read_16(mmio_base,
83 FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR);
84 uint16_t ring_desc_len = fpga_reg_read_16(mmio_base,
85 FPGA_5GNR_FEC_RING_DESC_LEN);
86
87 rte_bbdev_log_debug("UL.DL Weights = %u.%u",
88 ((uint8_t)config), ((uint8_t)(config >> 8)));
89 rte_bbdev_log_debug("UL.DL Load Balance = %u.%u",
90 ((uint8_t)lb_factor), ((uint8_t)(lb_factor >> 8)));
91 rte_bbdev_log_debug("Queue-PF/VF Mapping Table = %s",
92 (qmap_done > 0) ? "READY" : "NOT-READY");
93 rte_bbdev_log_debug("Ring Descriptor Size = %u bytes",
94 ring_desc_len*FPGA_RING_DESC_LEN_UNIT_BYTES);
95}
96
97
98static void
99print_dma_dec_desc_debug_info(union fpga_dma_desc *desc)
100{
101 rte_bbdev_log_debug("DMA response desc %p\n"
102 "\t-- done(%"PRIu32") | iter(%"PRIu32") | et_pass(%"PRIu32")"
103 " | crcb_pass (%"PRIu32") | error(%"PRIu32")\n"
104 "\t-- qm_idx(%"PRIu32") | max_iter(%"PRIu32") | "
105 "bg_idx (%"PRIu32") | harqin_en(%"PRIu32") | zc(%"PRIu32")\n"
106 "\t-- hbstroe_offset(%"PRIu32") | num_null (%"PRIu32") "
107 "| irq_en(%"PRIu32")\n"
108 "\t-- ncb(%"PRIu32") | desc_idx (%"PRIu32") | "
109 "drop_crc24b(%"PRIu32") | RV (%"PRIu32")\n"
110 "\t-- crc24b_ind(%"PRIu32") | et_dis (%"PRIu32")\n"
111 "\t-- harq_input_length(%"PRIu32") | rm_e(%"PRIu32")\n"
112 "\t-- cbs_in_op(%"PRIu32") | in_add (0x%08"PRIx32"%08"PRIx32")"
113 "| out_add (0x%08"PRIx32"%08"PRIx32")",
114 desc,
115 (uint32_t)desc->dec_req.done,
116 (uint32_t)desc->dec_req.iter,
117 (uint32_t)desc->dec_req.et_pass,
118 (uint32_t)desc->dec_req.crcb_pass,
119 (uint32_t)desc->dec_req.error,
120 (uint32_t)desc->dec_req.qm_idx,
121 (uint32_t)desc->dec_req.max_iter,
122 (uint32_t)desc->dec_req.bg_idx,
123 (uint32_t)desc->dec_req.harqin_en,
124 (uint32_t)desc->dec_req.zc,
125 (uint32_t)desc->dec_req.hbstroe_offset,
126 (uint32_t)desc->dec_req.num_null,
127 (uint32_t)desc->dec_req.irq_en,
128 (uint32_t)desc->dec_req.ncb,
129 (uint32_t)desc->dec_req.desc_idx,
130 (uint32_t)desc->dec_req.drop_crc24b,
131 (uint32_t)desc->dec_req.rv,
132 (uint32_t)desc->dec_req.crc24b_ind,
133 (uint32_t)desc->dec_req.et_dis,
134 (uint32_t)desc->dec_req.harq_input_length,
135 (uint32_t)desc->dec_req.rm_e,
136 (uint32_t)desc->dec_req.cbs_in_op,
137 (uint32_t)desc->dec_req.in_addr_hi,
138 (uint32_t)desc->dec_req.in_addr_lw,
139 (uint32_t)desc->dec_req.out_addr_hi,
140 (uint32_t)desc->dec_req.out_addr_lw);
141 uint32_t *word = (uint32_t *) desc;
142 rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n"
143 "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n",
144 word[0], word[1], word[2], word[3],
145 word[4], word[5], word[6], word[7]);
146}
147
148
149static void
150print_dma_enc_desc_debug_info(union fpga_dma_desc *desc)
151{
152 rte_bbdev_log_debug("DMA response desc %p\n"
153 "%"PRIu32" %"PRIu32"\n"
154 "K' %"PRIu32" E %"PRIu32" desc %"PRIu32" Z %"PRIu32"\n"
155 "BG %"PRIu32" Qm %"PRIu32" CRC %"PRIu32" IRQ %"PRIu32"\n"
156 "k0 %"PRIu32" Ncb %"PRIu32" F %"PRIu32"\n",
157 desc,
158 (uint32_t)desc->enc_req.done,
159 (uint32_t)desc->enc_req.error,
160
161 (uint32_t)desc->enc_req.k_,
162 (uint32_t)desc->enc_req.rm_e,
163 (uint32_t)desc->enc_req.desc_idx,
164 (uint32_t)desc->enc_req.zc,
165
166 (uint32_t)desc->enc_req.bg_idx,
167 (uint32_t)desc->enc_req.qm_idx,
168 (uint32_t)desc->enc_req.crc_en,
169 (uint32_t)desc->enc_req.irq_en,
170
171 (uint32_t)desc->enc_req.k0,
172 (uint32_t)desc->enc_req.ncb,
173 (uint32_t)desc->enc_req.num_null);
174 uint32_t *word = (uint32_t *) desc;
175 rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n"
176 "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n",
177 word[0], word[1], word[2], word[3],
178 word[4], word[5], word[6], word[7]);
179}
180
181#endif
182
183static int
184fpga_setup_queues(struct rte_bbdev *dev, uint16_t num_queues, int socket_id)
185{
186
187 uint32_t hw_q_num = 0;
188 uint32_t ring_size, payload, address, q_id, offset;
189 rte_iova_t phys_addr;
190 struct fpga_ring_ctrl_reg ring_reg;
191 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
192
193 address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE;
194 if (!(fpga_reg_read_32(fpga_dev->mmio_base, address) & 0x1)) {
195 rte_bbdev_log(ERR,
196 "Queue-PF/VF mapping is not set! Was PF configured for device (%s) ?",
197 dev->data->name);
198 return -EPERM;
199 }
200
201
202 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg));
203
204
205
206
207
208
209 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
210 uint32_t hw_q_id = fpga_reg_read_32(fpga_dev->mmio_base,
211 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2));
212
213 rte_bbdev_log_debug("%s: queue ID: %u, registry queue ID: %u",
214 dev->device->name, q_id, hw_q_id);
215
216 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) {
217 fpga_dev->q_bound_bit_map |= (1ULL << q_id);
218
219 offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
220 (sizeof(struct fpga_ring_ctrl_reg) * q_id);
221 fpga_ring_reg_write(fpga_dev->mmio_base,
222 offset, ring_reg);
223 ++hw_q_num;
224 }
225 }
226 if (hw_q_num == 0) {
227 rte_bbdev_log(ERR,
228 "No HW queues assigned to this device. Probably this is a VF configured for PF mode. Check device configuration!");
229 return -ENODEV;
230 }
231
232 if (num_queues > hw_q_num) {
233 rte_bbdev_log(ERR,
234 "Not enough queues for device %s! Requested: %u, available: %u",
235 dev->device->name, num_queues, hw_q_num);
236 return -EINVAL;
237 }
238
239 ring_size = FPGA_RING_MAX_SIZE * sizeof(struct fpga_dma_dec_desc);
240
241
242 RTE_BUILD_BUG_ON((RTE_CACHE_LINE_SIZE % 32) != 0);
243
244
245 fpga_dev->sw_rings = rte_zmalloc_socket(dev->device->driver->name,
246 num_queues * ring_size, RTE_CACHE_LINE_SIZE,
247 socket_id);
248 if (fpga_dev->sw_rings == NULL) {
249 rte_bbdev_log(ERR,
250 "Failed to allocate memory for %s:%u sw_rings",
251 dev->device->driver->name, dev->data->dev_id);
252 return -ENOMEM;
253 }
254
255 fpga_dev->sw_rings_phys = rte_malloc_virt2iova(fpga_dev->sw_rings);
256 fpga_dev->sw_ring_size = ring_size;
257 fpga_dev->sw_ring_max_depth = FPGA_RING_MAX_SIZE;
258
259
260 fpga_dev->flush_queue_status = rte_zmalloc_socket(NULL,
261 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, socket_id);
262 if (fpga_dev->flush_queue_status == NULL) {
263 rte_bbdev_log(ERR,
264 "Failed to allocate memory for %s:%u flush_queue_status",
265 dev->device->driver->name, dev->data->dev_id);
266 return -ENOMEM;
267 }
268
269
270 phys_addr = rte_malloc_virt2iova(fpga_dev->flush_queue_status);
271
272 address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_LW;
273 payload = (uint32_t)(phys_addr);
274 fpga_reg_write_32(fpga_dev->mmio_base, address, payload);
275
276 address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_HI;
277 payload = (uint32_t)(phys_addr >> 32);
278 fpga_reg_write_32(fpga_dev->mmio_base, address, payload);
279
280 return 0;
281}
282
283static int
284fpga_dev_close(struct rte_bbdev *dev)
285{
286 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
287
288 rte_free(fpga_dev->sw_rings);
289 rte_free(fpga_dev->flush_queue_status);
290
291 return 0;
292}
293
294static void
295fpga_dev_info_get(struct rte_bbdev *dev,
296 struct rte_bbdev_driver_info *dev_info)
297{
298 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
299 uint32_t q_id = 0;
300
301 static const struct rte_bbdev_op_cap bbdev_capabilities[] = {
302 {
303 .type = RTE_BBDEV_OP_LDPC_ENC,
304 .cap.ldpc_enc = {
305 .capability_flags =
306 RTE_BBDEV_LDPC_RATE_MATCH |
307 RTE_BBDEV_LDPC_ENC_INTERRUPTS |
308 RTE_BBDEV_LDPC_CRC_24B_ATTACH,
309 .num_buffers_src =
310 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
311 .num_buffers_dst =
312 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
313 }
314 },
315 {
316 .type = RTE_BBDEV_OP_LDPC_DEC,
317 .cap.ldpc_dec = {
318 .capability_flags =
319 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK |
320 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP |
321 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
322 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
323 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE |
324 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE |
325 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE |
326 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK |
327 RTE_BBDEV_LDPC_DEC_INTERRUPTS |
328 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS,
329 .llr_size = 6,
330 .llr_decimals = 2,
331 .num_buffers_src =
332 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
333 .num_buffers_hard_out =
334 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
335 .num_buffers_soft_out = 0,
336 }
337 },
338 RTE_BBDEV_END_OF_CAPABILITIES_LIST()
339 };
340
341
342 uint8_t timeout_counter = 0;
343 uint32_t harq_buf_ready = fpga_reg_read_32(d->mmio_base,
344 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS);
345 while (harq_buf_ready != 1) {
346 usleep(FPGA_TIMEOUT_CHECK_INTERVAL);
347 timeout_counter++;
348 harq_buf_ready = fpga_reg_read_32(d->mmio_base,
349 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS);
350 if (timeout_counter > FPGA_HARQ_RDY_TIMEOUT) {
351 rte_bbdev_log(ERR, "HARQ Buffer not ready %d",
352 harq_buf_ready);
353 harq_buf_ready = 1;
354 }
355 }
356 uint32_t harq_buf_size = fpga_reg_read_32(d->mmio_base,
357 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
358
359 static struct rte_bbdev_queue_conf default_queue_conf;
360 default_queue_conf.socket = dev->data->socket_id;
361 default_queue_conf.queue_size = FPGA_RING_MAX_SIZE;
362
363 dev_info->driver_name = dev->device->driver->name;
364 dev_info->queue_size_lim = FPGA_RING_MAX_SIZE;
365 dev_info->hardware_accelerated = true;
366 dev_info->min_alignment = 64;
367 dev_info->harq_buffer_size = (harq_buf_size >> 10) + 1;
368 dev_info->default_queue_conf = default_queue_conf;
369 dev_info->capabilities = bbdev_capabilities;
370 dev_info->cpu_flag_reqs = NULL;
371 dev_info->data_endianness = RTE_LITTLE_ENDIAN;
372
373
374 dev_info->max_num_queues = 0;
375 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
376 uint32_t hw_q_id = fpga_reg_read_32(d->mmio_base,
377 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2));
378 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID)
379 dev_info->max_num_queues++;
380 }
381}
382
383
384
385
386
387static inline int
388fpga_find_free_queue_idx(struct rte_bbdev *dev,
389 const struct rte_bbdev_queue_conf *conf)
390{
391 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
392 uint64_t q_idx;
393 uint8_t i = 0;
394 uint8_t range = FPGA_TOTAL_NUM_QUEUES >> 1;
395
396 if (conf->op_type == RTE_BBDEV_OP_LDPC_ENC) {
397 i = FPGA_NUM_DL_QUEUES;
398 range = FPGA_TOTAL_NUM_QUEUES;
399 }
400
401 for (; i < range; ++i) {
402 q_idx = 1ULL << i;
403
404 if (d->q_bound_bit_map & q_idx)
405
406 if (!(d->q_assigned_bit_map & q_idx)) {
407 d->q_assigned_bit_map |= q_idx;
408 return i;
409 }
410 }
411
412 rte_bbdev_log(INFO, "Failed to find free queue on %s", dev->data->name);
413
414 return -1;
415}
416
417static int
418fpga_queue_setup(struct rte_bbdev *dev, uint16_t queue_id,
419 const struct rte_bbdev_queue_conf *conf)
420{
421 uint32_t address, ring_offset;
422 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
423 struct fpga_queue *q;
424 int8_t q_idx;
425
426
427 q_idx = fpga_find_free_queue_idx(dev, conf);
428 if (q_idx == -1)
429 return -1;
430
431
432 q = rte_zmalloc_socket(dev->device->driver->name, sizeof(*q),
433 RTE_CACHE_LINE_SIZE, conf->socket);
434 if (q == NULL) {
435
436 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
437 rte_bbdev_log(ERR, "Failed to allocate queue memory");
438 return -ENOMEM;
439 }
440
441 q->d = d;
442 q->q_idx = q_idx;
443
444
445 q->ring_addr = RTE_PTR_ADD(d->sw_rings, (d->sw_ring_size * queue_id));
446 q->ring_ctrl_reg.ring_base_addr = d->sw_rings_phys +
447 (d->sw_ring_size * queue_id);
448
449
450 q->ring_head_addr = rte_zmalloc_socket(dev->device->driver->name,
451 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, conf->socket);
452 if (q->ring_head_addr == NULL) {
453
454 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
455 rte_free(q);
456 rte_bbdev_log(ERR,
457 "Failed to allocate memory for %s:%u completion_head",
458 dev->device->driver->name, dev->data->dev_id);
459 return -ENOMEM;
460 }
461
462 q->ring_ctrl_reg.ring_head_addr =
463 rte_malloc_virt2iova(q->ring_head_addr);
464
465
466 q->shadow_completion_head = 0;
467
468
469 if (conf->queue_size > FPGA_RING_MAX_SIZE) {
470
471 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
472 rte_free(q->ring_head_addr);
473 rte_free(q);
474 rte_bbdev_log(ERR,
475 "Size of queue is too big %d (MAX: %d ) for %s:%u",
476 conf->queue_size, FPGA_RING_MAX_SIZE,
477 dev->device->driver->name, dev->data->dev_id);
478 return -EINVAL;
479 }
480 q->ring_ctrl_reg.ring_size = conf->queue_size;
481
482
483
484 q->ring_ctrl_reg.max_ul_dec = 0;
485
486 q->ring_ctrl_reg.max_ul_dec_en = 0;
487
488
489 q->ring_ctrl_reg.enable = 1;
490
491
492 q->ring_ctrl_reg.head_point = q->tail = 0;
493
494
495 q->ring_ctrl_reg.shadow_tail = q->tail;
496
497
498 ring_offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
499 (sizeof(struct fpga_ring_ctrl_reg) * q_idx);
500
501
502 fpga_ring_reg_write(d->mmio_base, ring_offset, q->ring_ctrl_reg);
503
504
505 address = ring_offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL;
506 q->shadow_tail_addr = RTE_PTR_ADD(d->mmio_base, address);
507
508 q->head_free_desc = q->tail;
509
510
511 q->sw_ring_wrap_mask = conf->queue_size - 1;
512
513 rte_bbdev_log_debug("Setup dev%u q%u: queue_idx=%u",
514 dev->data->dev_id, queue_id, q->q_idx);
515
516 dev->data->queues[queue_id].queue_private = q;
517
518 rte_bbdev_log_debug("BBDEV queue[%d] set up for FPGA queue[%d]",
519 queue_id, q_idx);
520
521#ifdef RTE_LIBRTE_BBDEV_DEBUG
522
523 print_ring_reg_debug_info(d->mmio_base, ring_offset);
524#endif
525 return 0;
526}
527
528static int
529fpga_queue_release(struct rte_bbdev *dev, uint16_t queue_id)
530{
531 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
532 struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
533 struct fpga_ring_ctrl_reg ring_reg;
534 uint32_t offset;
535
536 rte_bbdev_log_debug("FPGA Queue[%d] released", queue_id);
537
538 if (q != NULL) {
539 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg));
540 offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
541 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
542
543 fpga_reg_write_8(d->mmio_base,
544 offset + FPGA_5GNR_FEC_RING_ENABLE, 0x00);
545
546 fpga_ring_reg_write(d->mmio_base, offset, ring_reg);
547
548
549 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q->q_idx));
550 rte_free(q->ring_head_addr);
551 rte_free(q);
552 dev->data->queues[queue_id].queue_private = NULL;
553 }
554
555 return 0;
556}
557
558
559static int
560fpga_queue_start(struct rte_bbdev *dev, uint16_t queue_id)
561{
562 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
563#ifdef RTE_LIBRTE_BBDEV_DEBUG
564 if (d == NULL) {
565 rte_bbdev_log(ERR, "Invalid device pointer");
566 return -1;
567 }
568#endif
569 struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
570 uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
571 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
572 uint8_t enable = 0x01;
573 uint16_t zero = 0x0000;
574
575
576 q->tail = q->head_free_desc = 0;
577
578
579 fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_HEAD_POINT,
580 zero);
581 fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL,
582 zero);
583
584
585 fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE,
586 enable);
587
588 rte_bbdev_log_debug("FPGA Queue[%d] started", queue_id);
589 return 0;
590}
591
592
593static int
594fpga_queue_stop(struct rte_bbdev *dev, uint16_t queue_id)
595{
596 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
597#ifdef RTE_LIBRTE_BBDEV_DEBUG
598 if (d == NULL) {
599 rte_bbdev_log(ERR, "Invalid device pointer");
600 return -1;
601 }
602#endif
603 struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
604 uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
605 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
606 uint8_t payload = 0x01;
607 uint8_t counter = 0;
608 uint8_t timeout = FPGA_QUEUE_FLUSH_TIMEOUT_US /
609 FPGA_TIMEOUT_CHECK_INTERVAL;
610
611
612 fpga_reg_write_8(d->mmio_base,
613 offset + FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN, payload);
614
615
616
617
618
619
620 while (!(*((volatile uint8_t *)d->flush_queue_status + q->q_idx)
621 & payload)) {
622 if (counter > timeout) {
623 rte_bbdev_log(ERR, "FPGA Queue Flush failed for queue %d",
624 queue_id);
625 return -1;
626 }
627 usleep(FPGA_TIMEOUT_CHECK_INTERVAL);
628 counter++;
629 }
630
631
632 payload = 0x00;
633 fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE,
634 payload);
635
636 rte_bbdev_log_debug("FPGA Queue[%d] stopped", queue_id);
637 return 0;
638}
639
640static inline uint16_t
641get_queue_id(struct rte_bbdev_data *data, uint8_t q_idx)
642{
643 uint16_t queue_id;
644
645 for (queue_id = 0; queue_id < data->num_queues; ++queue_id) {
646 struct fpga_queue *q = data->queues[queue_id].queue_private;
647 if (q != NULL && q->q_idx == q_idx)
648 return queue_id;
649 }
650
651 return -1;
652}
653
654
655static void
656fpga_dev_interrupt_handler(void *cb_arg)
657{
658 struct rte_bbdev *dev = cb_arg;
659 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
660 struct fpga_queue *q;
661 uint64_t ring_head;
662 uint64_t q_idx;
663 uint16_t queue_id;
664 uint8_t i;
665
666
667 for (i = 0; i < FPGA_TOTAL_NUM_QUEUES; ++i) {
668 q_idx = 1ULL << i;
669 if (fpga_dev->q_bound_bit_map & q_idx) {
670 queue_id = get_queue_id(dev->data, i);
671 if (queue_id == (uint16_t) -1)
672 continue;
673
674
675 q = dev->data->queues[queue_id].queue_private;
676 ring_head = *q->ring_head_addr;
677 if (q->shadow_completion_head != ring_head &&
678 q->irq_enable == 1) {
679 q->shadow_completion_head = ring_head;
680 rte_bbdev_pmd_callback_process(
681 dev,
682 RTE_BBDEV_EVENT_DEQUEUE,
683 &queue_id);
684 }
685 }
686 }
687}
688
689static int
690fpga_queue_intr_enable(struct rte_bbdev *dev, uint16_t queue_id)
691{
692 struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
693
694 if (!rte_intr_cap_multiple(dev->intr_handle))
695 return -ENOTSUP;
696
697 q->irq_enable = 1;
698
699 return 0;
700}
701
702static int
703fpga_queue_intr_disable(struct rte_bbdev *dev, uint16_t queue_id)
704{
705 struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
706 q->irq_enable = 0;
707
708 return 0;
709}
710
711static int
712fpga_intr_enable(struct rte_bbdev *dev)
713{
714 int ret;
715 uint8_t i;
716
717 if (!rte_intr_cap_multiple(dev->intr_handle)) {
718 rte_bbdev_log(ERR, "Multiple intr vector is not supported by FPGA (%s)",
719 dev->data->name);
720 return -ENOTSUP;
721 }
722
723
724
725
726
727
728
729
730
731 if (rte_intr_efd_enable(dev->intr_handle, FPGA_NUM_INTR_VEC)) {
732 rte_bbdev_log(ERR, "Failed to create fds for %u queues",
733 dev->data->num_queues);
734 return -1;
735 }
736
737
738
739
740
741
742 for (i = 0; i < FPGA_NUM_INTR_VEC; ++i) {
743 if (rte_intr_efds_index_set(dev->intr_handle, i,
744 rte_intr_fd_get(dev->intr_handle)))
745 return -rte_errno;
746 }
747
748 if (rte_intr_vec_list_alloc(dev->intr_handle, "intr_vec",
749 dev->data->num_queues)) {
750 rte_bbdev_log(ERR, "Failed to allocate %u vectors",
751 dev->data->num_queues);
752 return -ENOMEM;
753 }
754
755 ret = rte_intr_enable(dev->intr_handle);
756 if (ret < 0) {
757 rte_bbdev_log(ERR,
758 "Couldn't enable interrupts for device: %s",
759 dev->data->name);
760 return ret;
761 }
762
763 ret = rte_intr_callback_register(dev->intr_handle,
764 fpga_dev_interrupt_handler, dev);
765 if (ret < 0) {
766 rte_bbdev_log(ERR,
767 "Couldn't register interrupt callback for device: %s",
768 dev->data->name);
769 return ret;
770 }
771
772 return 0;
773}
774
775static const struct rte_bbdev_ops fpga_ops = {
776 .setup_queues = fpga_setup_queues,
777 .intr_enable = fpga_intr_enable,
778 .close = fpga_dev_close,
779 .info_get = fpga_dev_info_get,
780 .queue_setup = fpga_queue_setup,
781 .queue_stop = fpga_queue_stop,
782 .queue_start = fpga_queue_start,
783 .queue_release = fpga_queue_release,
784 .queue_intr_enable = fpga_queue_intr_enable,
785 .queue_intr_disable = fpga_queue_intr_disable
786};
787
788static inline void
789fpga_dma_enqueue(struct fpga_queue *q, uint16_t num_desc,
790 struct rte_bbdev_stats *queue_stats)
791{
792#ifdef RTE_BBDEV_OFFLOAD_COST
793 uint64_t start_time = 0;
794 queue_stats->acc_offload_cycles = 0;
795#else
796 RTE_SET_USED(queue_stats);
797#endif
798
799
800 q->tail = (q->tail + num_desc) & q->sw_ring_wrap_mask;
801
802 rte_wmb();
803
804#ifdef RTE_BBDEV_OFFLOAD_COST
805
806 start_time = rte_rdtsc_precise();
807#endif
808 mmio_write_16(q->shadow_tail_addr, q->tail);
809
810#ifdef RTE_BBDEV_OFFLOAD_COST
811 rte_wmb();
812 queue_stats->acc_offload_cycles += rte_rdtsc_precise() - start_time;
813#endif
814}
815
816
817static inline bool
818check_bit(uint32_t bitmap, uint32_t bitmask)
819{
820 return bitmap & bitmask;
821}
822
823
824
825
826static inline int
827check_desc_error(uint32_t error_code) {
828 switch (error_code) {
829 case DESC_ERR_NO_ERR:
830 return 0;
831 case DESC_ERR_K_P_OUT_OF_RANGE:
832 rte_bbdev_log(ERR, "Encode block size K' is out of range");
833 break;
834 case DESC_ERR_Z_C_NOT_LEGAL:
835 rte_bbdev_log(ERR, "Zc is illegal");
836 break;
837 case DESC_ERR_DESC_OFFSET_ERR:
838 rte_bbdev_log(ERR,
839 "Queue offset does not meet the expectation in the FPGA"
840 );
841 break;
842 case DESC_ERR_DESC_READ_FAIL:
843 rte_bbdev_log(ERR, "Unsuccessful completion for descriptor read");
844 break;
845 case DESC_ERR_DESC_READ_TIMEOUT:
846 rte_bbdev_log(ERR, "Descriptor read time-out");
847 break;
848 case DESC_ERR_DESC_READ_TLP_POISONED:
849 rte_bbdev_log(ERR, "Descriptor read TLP poisoned");
850 break;
851 case DESC_ERR_HARQ_INPUT_LEN:
852 rte_bbdev_log(ERR, "HARQ input length is invalid");
853 break;
854 case DESC_ERR_CB_READ_FAIL:
855 rte_bbdev_log(ERR, "Unsuccessful completion for code block");
856 break;
857 case DESC_ERR_CB_READ_TIMEOUT:
858 rte_bbdev_log(ERR, "Code block read time-out");
859 break;
860 case DESC_ERR_CB_READ_TLP_POISONED:
861 rte_bbdev_log(ERR, "Code block read TLP poisoned");
862 break;
863 case DESC_ERR_HBSTORE_ERR:
864 rte_bbdev_log(ERR, "Hbstroe exceeds HARQ buffer size.");
865 break;
866 default:
867 rte_bbdev_log(ERR, "Descriptor error unknown error code %u",
868 error_code);
869 break;
870 }
871 return 1;
872}
873
874
875
876
877
878static inline uint16_t
879get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
880{
881 if (rv_index == 0)
882 return 0;
883 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
884 if (n_cb == n) {
885 if (rv_index == 1)
886 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
887 else if (rv_index == 2)
888 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
889 else
890 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
891 }
892
893 if (rv_index == 1)
894 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
895 / n) * z_c;
896 else if (rv_index == 2)
897 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
898 / n) * z_c;
899 else
900 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
901 / n) * z_c;
902}
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928static inline int
929fpga_dma_desc_te_fill(struct rte_bbdev_enc_op *op,
930 struct fpga_dma_enc_desc *desc, struct rte_mbuf *input,
931 struct rte_mbuf *output, uint16_t k_, uint16_t e,
932 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset,
933 uint8_t cbs_in_op)
934{
935
936 desc->done = 0;
937 desc->error = 0;
938 desc->k_ = k_;
939 desc->rm_e = e;
940 desc->desc_idx = desc_offset;
941 desc->zc = op->ldpc_enc.z_c;
942 desc->bg_idx = op->ldpc_enc.basegraph - 1;
943 desc->qm_idx = op->ldpc_enc.q_m / 2;
944 desc->crc_en = check_bit(op->ldpc_enc.op_flags,
945 RTE_BBDEV_LDPC_CRC_24B_ATTACH);
946 desc->irq_en = 0;
947 desc->k0 = get_k0(op->ldpc_enc.n_cb, op->ldpc_enc.z_c,
948 op->ldpc_enc.basegraph, op->ldpc_enc.rv_index);
949 desc->ncb = op->ldpc_enc.n_cb;
950 desc->num_null = op->ldpc_enc.n_filler;
951
952 desc->in_addr_hi = (uint32_t)(
953 rte_pktmbuf_iova_offset(input, in_offset) >> 32);
954 desc->in_addr_lw = (uint32_t)(
955 rte_pktmbuf_iova_offset(input, in_offset));
956
957 desc->out_addr_hi = (uint32_t)(
958 rte_pktmbuf_iova_offset(output, out_offset) >> 32);
959 desc->out_addr_lw = (uint32_t)(
960 rte_pktmbuf_iova_offset(output, out_offset));
961
962 desc->op_addr = op;
963
964 desc->cbs_in_op = cbs_in_op;
965 return 0;
966}
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986static inline int
987fpga_dma_desc_ld_fill(struct rte_bbdev_dec_op *op,
988 struct fpga_dma_dec_desc *desc,
989 struct rte_mbuf *input, struct rte_mbuf *output,
990 uint16_t harq_in_length,
991 uint32_t in_offset, uint32_t out_offset,
992 uint32_t harq_offset,
993 uint16_t desc_offset,
994 uint8_t cbs_in_op)
995{
996
997 desc->done = 0;
998 desc->error = 0;
999
1000 desc->in_addr_hi = (uint32_t)(
1001 rte_pktmbuf_iova_offset(input, in_offset) >> 32);
1002 desc->in_addr_lw = (uint32_t)(
1003 rte_pktmbuf_iova_offset(input, in_offset));
1004 desc->rm_e = op->ldpc_dec.cb_params.e;
1005 desc->harq_input_length = harq_in_length;
1006 desc->et_dis = !check_bit(op->ldpc_dec.op_flags,
1007 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE);
1008 desc->rv = op->ldpc_dec.rv_index;
1009 desc->crc24b_ind = check_bit(op->ldpc_dec.op_flags,
1010 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK);
1011 desc->drop_crc24b = check_bit(op->ldpc_dec.op_flags,
1012 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP);
1013 desc->desc_idx = desc_offset;
1014 desc->ncb = op->ldpc_dec.n_cb;
1015 desc->num_null = op->ldpc_dec.n_filler;
1016 desc->hbstroe_offset = harq_offset >> 10;
1017 desc->zc = op->ldpc_dec.z_c;
1018 desc->harqin_en = check_bit(op->ldpc_dec.op_flags,
1019 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE);
1020 desc->bg_idx = op->ldpc_dec.basegraph - 1;
1021 desc->max_iter = op->ldpc_dec.iter_max;
1022 desc->qm_idx = op->ldpc_dec.q_m / 2;
1023 desc->out_addr_hi = (uint32_t)(
1024 rte_pktmbuf_iova_offset(output, out_offset) >> 32);
1025 desc->out_addr_lw = (uint32_t)(
1026 rte_pktmbuf_iova_offset(output, out_offset));
1027
1028 desc->op_addr = op;
1029
1030 desc->cbs_in_op = cbs_in_op;
1031
1032 return 0;
1033}
1034
1035
1036static inline int
1037validate_ldpc_enc_op(struct rte_bbdev_enc_op *op)
1038{
1039 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &op->ldpc_enc;
1040
1041 if (op->mempool == NULL) {
1042 rte_bbdev_log(ERR, "Invalid mempool pointer");
1043 return -1;
1044 }
1045 if (ldpc_enc->input.data == NULL) {
1046 rte_bbdev_log(ERR, "Invalid input pointer");
1047 return -1;
1048 }
1049 if (ldpc_enc->output.data == NULL) {
1050 rte_bbdev_log(ERR, "Invalid output pointer");
1051 return -1;
1052 }
1053 if (ldpc_enc->input.length == 0) {
1054 rte_bbdev_log(ERR, "CB size (%u) is null",
1055 ldpc_enc->input.length);
1056 return -1;
1057 }
1058 if ((ldpc_enc->basegraph > 2) || (ldpc_enc->basegraph == 0)) {
1059 rte_bbdev_log(ERR,
1060 "BG (%u) is out of range 1 <= value <= 2",
1061 ldpc_enc->basegraph);
1062 return -1;
1063 }
1064 if (ldpc_enc->rv_index > 3) {
1065 rte_bbdev_log(ERR,
1066 "rv_index (%u) is out of range 0 <= value <= 3",
1067 ldpc_enc->rv_index);
1068 return -1;
1069 }
1070 if (ldpc_enc->code_block_mode > RTE_BBDEV_CODE_BLOCK) {
1071 rte_bbdev_log(ERR,
1072 "code_block_mode (%u) is out of range 0 <= value <= 1",
1073 ldpc_enc->code_block_mode);
1074 return -1;
1075 }
1076
1077 if (ldpc_enc->input.length >
1078 RTE_BBDEV_LDPC_MAX_CB_SIZE >> 3) {
1079 rte_bbdev_log(ERR, "CB size (%u) is too big, max: %d",
1080 ldpc_enc->input.length,
1081 RTE_BBDEV_LDPC_MAX_CB_SIZE);
1082 return -1;
1083 }
1084 int z_c = ldpc_enc->z_c;
1085
1086 if ((z_c > 384) || (z_c < 4)) {
1087 rte_bbdev_log(ERR, "Zc (%u) is out of range", z_c);
1088 return -1;
1089 }
1090 if (z_c > 256) {
1091 if ((z_c % 32) != 0) {
1092 rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1093 return -1;
1094 }
1095 } else if (z_c > 128) {
1096 if ((z_c % 16) != 0) {
1097 rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1098 return -1;
1099 }
1100 } else if (z_c > 64) {
1101 if ((z_c % 8) != 0) {
1102 rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1103 return -1;
1104 }
1105 } else if (z_c > 32) {
1106 if ((z_c % 4) != 0) {
1107 rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1108 return -1;
1109 }
1110 } else if (z_c > 16) {
1111 if ((z_c % 2) != 0) {
1112 rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1113 return -1;
1114 }
1115 }
1116
1117 int n_filler = ldpc_enc->n_filler;
1118 int K = (ldpc_enc->basegraph == 1 ? 22 : 10) * ldpc_enc->z_c;
1119 int Kp = K - n_filler;
1120 int q_m = ldpc_enc->q_m;
1121 int n_cb = ldpc_enc->n_cb;
1122 int N = (ldpc_enc->basegraph == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1123 int k0 = get_k0(n_cb, z_c, ldpc_enc->basegraph,
1124 ldpc_enc->rv_index);
1125 int crc24 = 0;
1126 int32_t L, Lcb, cw, cw_rm;
1127 int32_t e = ldpc_enc->cb_params.e;
1128 if (check_bit(op->ldpc_enc.op_flags,
1129 RTE_BBDEV_LDPC_CRC_24B_ATTACH))
1130 crc24 = 24;
1131
1132 if (K < (int) (ldpc_enc->input.length * 8 + n_filler) + crc24) {
1133 rte_bbdev_log(ERR, "K and F not matching input size %u %u %u",
1134 K, n_filler, ldpc_enc->input.length);
1135 return -1;
1136 }
1137 if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1138 rte_bbdev_log(ERR, "TB mode not supported");
1139 return -1;
1140
1141 }
1142
1143
1144 if (Kp % 8 > 0) {
1145 rte_bbdev_log(ERR, "K' not byte aligned %u", Kp);
1146 return -1;
1147 }
1148 if ((crc24 > 0) && (Kp < 292)) {
1149 rte_bbdev_log(ERR, "Invalid CRC24 for small block %u", Kp);
1150 return -1;
1151 }
1152 if (Kp < 24) {
1153 rte_bbdev_log(ERR, "K' too small %u", Kp);
1154 return -1;
1155 }
1156 if (n_filler >= (K - 2 * z_c)) {
1157 rte_bbdev_log(ERR, "K - F invalid %u %u", K, n_filler);
1158 return -1;
1159 }
1160
1161 if ((n_cb > N) || (n_cb < 32) || (n_cb <= (Kp - crc24))) {
1162 rte_bbdev_log(ERR, "Ncb (%u) is out of range K %d N %d", n_cb, K, N);
1163 return -1;
1164 }
1165
1166 if (!check_bit(op->ldpc_enc.op_flags, RTE_BBDEV_LDPC_INTERLEAVER_BYPASS) &&
1167 ((q_m == 0) || ((q_m > 2) && ((q_m % 2) == 1)) || (q_m > 8))) {
1168 rte_bbdev_log(ERR, "Qm (%u) is out of range", q_m);
1169 return -1;
1170 }
1171
1172 if (((k0 % z_c) > 0) || (k0 >= n_cb) || ((k0 >= (Kp - 2 * z_c))
1173 && (k0 < (K - 2 * z_c)))) {
1174 rte_bbdev_log(ERR, "K0 (%u) is out of range", k0);
1175 return -1;
1176 }
1177
1178 if (e <= RTE_MAX(32, z_c)) {
1179 rte_bbdev_log(ERR, "E is too small %"PRIu32"", e);
1180 return -1;
1181 }
1182 if ((e > 0xFFFF)) {
1183 rte_bbdev_log(ERR, "E is too large for N3000 %"PRIu32" > 64k", e);
1184 return -1;
1185 }
1186 if (q_m > 0) {
1187 if (e % q_m > 0) {
1188 rte_bbdev_log(ERR, "E %"PRIu32" not multiple of qm %d", e, q_m);
1189 return -1;
1190 }
1191 }
1192
1193 if (k0 > (Kp - 2 * z_c))
1194 L = k0 + e;
1195 else
1196 L = k0 + e + n_filler;
1197 Lcb = RTE_MIN(L, n_cb);
1198 if (ldpc_enc->basegraph == 1) {
1199 if (Lcb <= 25 * z_c)
1200 cw = 25 * z_c;
1201 else if (Lcb <= 27 * z_c)
1202 cw = 27 * z_c;
1203 else if (Lcb <= 30 * z_c)
1204 cw = 30 * z_c;
1205 else if (Lcb <= 33 * z_c)
1206 cw = 33 * z_c;
1207 else if (Lcb <= 44 * z_c)
1208 cw = 44 * z_c;
1209 else if (Lcb <= 55 * z_c)
1210 cw = 55 * z_c;
1211 else
1212 cw = 66 * z_c;
1213 } else {
1214 if (Lcb <= 15 * z_c)
1215 cw = 15 * z_c;
1216 else if (Lcb <= 20 * z_c)
1217 cw = 20 * z_c;
1218 else if (Lcb <= 25 * z_c)
1219 cw = 25 * z_c;
1220 else if (Lcb <= 30 * z_c)
1221 cw = 30 * z_c;
1222 else
1223 cw = 50 * z_c;
1224 }
1225 if (n_cb < Kp - 2 * z_c)
1226 cw_rm = n_cb;
1227 else if ((Kp - 2 * z_c <= n_cb) && (n_cb < K - 2 * z_c))
1228 cw_rm = Kp - 2 * z_c;
1229 else if ((K - 2 * z_c <= n_cb) && (n_cb < cw))
1230 cw_rm = n_cb - n_filler;
1231 else
1232 cw_rm = cw - n_filler;
1233 if (cw_rm <= 32) {
1234 rte_bbdev_log(ERR,
1235 "Invalid Ratematching");
1236 return -1;
1237 }
1238 return 0;
1239}
1240
1241
1242static inline int
1243validate_ldpc_dec_op(struct rte_bbdev_dec_op *op)
1244{
1245 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &op->ldpc_dec;
1246 if (check_bit(ldpc_dec->op_flags,
1247 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK))
1248 return 0;
1249 if (ldpc_dec->input.data == NULL) {
1250 rte_bbdev_log(ERR, "Invalid input pointer");
1251 return -1;
1252 }
1253 if (ldpc_dec->hard_output.data == NULL) {
1254 rte_bbdev_log(ERR, "Invalid output pointer");
1255 return -1;
1256 }
1257 if (ldpc_dec->input.length == 0) {
1258 rte_bbdev_log(ERR, "input is null");
1259 return -1;
1260 }
1261 if ((ldpc_dec->basegraph > 2) || (ldpc_dec->basegraph == 0)) {
1262 rte_bbdev_log(ERR,
1263 "BG (%u) is out of range 1 <= value <= 2",
1264 ldpc_dec->basegraph);
1265 return -1;
1266 }
1267 if (ldpc_dec->iter_max == 0) {
1268 rte_bbdev_log(ERR,
1269 "iter_max (%u) is equal to 0",
1270 ldpc_dec->iter_max);
1271 return -1;
1272 }
1273 if (ldpc_dec->rv_index > 3) {
1274 rte_bbdev_log(ERR,
1275 "rv_index (%u) is out of range 0 <= value <= 3",
1276 ldpc_dec->rv_index);
1277 return -1;
1278 }
1279 if (ldpc_dec->code_block_mode > RTE_BBDEV_CODE_BLOCK) {
1280 rte_bbdev_log(ERR,
1281 "code_block_mode (%u) is out of range 0 <= value <= 1",
1282 ldpc_dec->code_block_mode);
1283 return -1;
1284 }
1285 if (check_bit(op->ldpc_dec.op_flags,
1286 RTE_BBDEV_LDPC_DECODE_BYPASS)) {
1287 rte_bbdev_log(ERR, "Avoid LDPC Decode bypass");
1288 return -1;
1289 }
1290 int z_c = ldpc_dec->z_c;
1291
1292 if ((z_c > 384) || (z_c < 4)) {
1293 rte_bbdev_log(ERR,
1294 "Zc (%u) is out of range",
1295 z_c);
1296 return -1;
1297 }
1298 if (z_c > 256) {
1299 if ((z_c % 32) != 0) {
1300 rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1301 return -1;
1302 }
1303 } else if (z_c > 128) {
1304 if ((z_c % 16) != 0) {
1305 rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1306 return -1;
1307 }
1308 } else if (z_c > 64) {
1309 if ((z_c % 8) != 0) {
1310 rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1311 return -1;
1312 }
1313 } else if (z_c > 32) {
1314 if ((z_c % 4) != 0) {
1315 rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1316 return -1;
1317 }
1318 } else if (z_c > 16) {
1319 if ((z_c % 2) != 0) {
1320 rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1321 return -1;
1322 }
1323 }
1324
1325 int n_filler = ldpc_dec->n_filler;
1326 int K = (ldpc_dec->basegraph == 1 ? 22 : 10) * ldpc_dec->z_c;
1327 int Kp = K - n_filler;
1328 int q_m = ldpc_dec->q_m;
1329 int n_cb = ldpc_dec->n_cb;
1330 int N = (ldpc_dec->basegraph == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1331 int k0 = get_k0(n_cb, z_c, ldpc_dec->basegraph,
1332 ldpc_dec->rv_index);
1333 int crc24 = 0;
1334 int32_t L, Lcb, cw, cw_rm;
1335 int32_t e = ldpc_dec->cb_params.e;
1336 if (check_bit(op->ldpc_dec.op_flags,
1337 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK))
1338 crc24 = 24;
1339
1340 if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1341 rte_bbdev_log(ERR,
1342 "TB mode not supported");
1343 return -1;
1344 }
1345
1346 ldpc_dec->harq_combined_input.length = RTE_MIN((uint32_t) n_cb,
1347 ldpc_dec->harq_combined_input.length);
1348 if ((ldpc_dec->harq_combined_input.length == 0) &&
1349 check_bit(ldpc_dec->op_flags,
1350 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) {
1351 rte_bbdev_log(ERR,
1352 "HARQ input length (%u) should not be null",
1353 ldpc_dec->harq_combined_input.length);
1354 return -1;
1355 }
1356 if ((ldpc_dec->harq_combined_input.length > 0) &&
1357 !check_bit(ldpc_dec->op_flags,
1358 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) {
1359 ldpc_dec->harq_combined_input.length = 0;
1360 }
1361
1362
1363 if (Kp % 8 > 0) {
1364 rte_bbdev_log(ERR,
1365 "K' not byte aligned %u",
1366 Kp);
1367 return -1;
1368 }
1369 if ((crc24 > 0) && (Kp < 292)) {
1370 rte_bbdev_log(ERR,
1371 "Invalid CRC24 for small block %u",
1372 Kp);
1373 return -1;
1374 }
1375 if (Kp < 24) {
1376 rte_bbdev_log(ERR,
1377 "K' too small %u",
1378 Kp);
1379 return -1;
1380 }
1381 if (n_filler >= (K - 2 * z_c)) {
1382 rte_bbdev_log(ERR,
1383 "K - F invalid %u %u",
1384 K, n_filler);
1385 return -1;
1386 }
1387
1388 if (n_cb != N) {
1389 rte_bbdev_log(ERR,
1390 "Ncb (%u) is out of range K %d N %d",
1391 n_cb, K, N);
1392 return -1;
1393 }
1394
1395 if (!check_bit(op->ldpc_dec.op_flags,
1396 RTE_BBDEV_LDPC_INTERLEAVER_BYPASS) &&
1397 ((q_m == 0) || ((q_m > 2) && ((q_m % 2) == 1))
1398 || (q_m > 8))) {
1399 rte_bbdev_log(ERR,
1400 "Qm (%u) is out of range",
1401 q_m);
1402 return -1;
1403 }
1404
1405 if (((k0 % z_c) > 0) || (k0 >= n_cb) || ((k0 >= (Kp - 2 * z_c))
1406 && (k0 < (K - 2 * z_c)))) {
1407 rte_bbdev_log(ERR,
1408 "K0 (%u) is out of range",
1409 k0);
1410 return -1;
1411 }
1412
1413 if (e <= RTE_MAX(32, z_c)) {
1414 rte_bbdev_log(ERR,
1415 "E is too small");
1416 return -1;
1417 }
1418 if ((e > 0xFFFF)) {
1419 rte_bbdev_log(ERR,
1420 "E is too large");
1421 return -1;
1422 }
1423 if (q_m > 0) {
1424 if (e % q_m > 0) {
1425 rte_bbdev_log(ERR,
1426 "E not multiple of qm %d", q_m);
1427 return -1;
1428 }
1429 }
1430
1431 if (k0 > (Kp - 2 * z_c))
1432 L = k0 + e;
1433 else
1434 L = k0 + e + n_filler;
1435 Lcb = RTE_MIN(n_cb, RTE_MAX(L,
1436 (int32_t) ldpc_dec->harq_combined_input.length));
1437 if (ldpc_dec->basegraph == 1) {
1438 if (Lcb <= 25 * z_c)
1439 cw = 25 * z_c;
1440 else if (Lcb <= 27 * z_c)
1441 cw = 27 * z_c;
1442 else if (Lcb <= 30 * z_c)
1443 cw = 30 * z_c;
1444 else if (Lcb <= 33 * z_c)
1445 cw = 33 * z_c;
1446 else if (Lcb <= 44 * z_c)
1447 cw = 44 * z_c;
1448 else if (Lcb <= 55 * z_c)
1449 cw = 55 * z_c;
1450 else
1451 cw = 66 * z_c;
1452 } else {
1453 if (Lcb <= 15 * z_c)
1454 cw = 15 * z_c;
1455 else if (Lcb <= 20 * z_c)
1456 cw = 20 * z_c;
1457 else if (Lcb <= 25 * z_c)
1458 cw = 25 * z_c;
1459 else if (Lcb <= 30 * z_c)
1460 cw = 30 * z_c;
1461 else
1462 cw = 50 * z_c;
1463 }
1464 cw_rm = cw - n_filler;
1465 if (cw_rm <= 32) {
1466 rte_bbdev_log(ERR,
1467 "Invalid Ratematching");
1468 return -1;
1469 }
1470 return 0;
1471}
1472
1473static inline char *
1474mbuf_append(struct rte_mbuf *m_head, struct rte_mbuf *m, uint16_t len)
1475{
1476 if (unlikely(len > rte_pktmbuf_tailroom(m)))
1477 return NULL;
1478
1479 char *tail = (char *)m->buf_addr + m->data_off + m->data_len;
1480 m->data_len = (uint16_t)(m->data_len + len);
1481 m_head->pkt_len = (m_head->pkt_len + len);
1482 return tail;
1483}
1484
1485static inline void
1486fpga_mutex_acquisition(struct fpga_queue *q)
1487{
1488 uint32_t mutex_ctrl, mutex_read, cnt = 0;
1489
1490 q->ddr_mutex_uuid = rand();
1491
1492 mutex_ctrl = (q->ddr_mutex_uuid << 16) + 1;
1493 do {
1494 if (cnt > 0)
1495 usleep(FPGA_TIMEOUT_CHECK_INTERVAL);
1496 rte_bbdev_log_debug("Acquiring Mutex for %x\n",
1497 q->ddr_mutex_uuid);
1498 fpga_reg_write_32(q->d->mmio_base,
1499 FPGA_5GNR_FEC_MUTEX,
1500 mutex_ctrl);
1501 mutex_read = fpga_reg_read_32(q->d->mmio_base,
1502 FPGA_5GNR_FEC_MUTEX);
1503 rte_bbdev_log_debug("Mutex %x cnt %d owner %x\n",
1504 mutex_read, cnt, q->ddr_mutex_uuid);
1505 cnt++;
1506 } while ((mutex_read >> 16) != q->ddr_mutex_uuid);
1507}
1508
1509static inline void
1510fpga_mutex_free(struct fpga_queue *q)
1511{
1512 uint32_t mutex_ctrl = q->ddr_mutex_uuid << 16;
1513 fpga_reg_write_32(q->d->mmio_base,
1514 FPGA_5GNR_FEC_MUTEX,
1515 mutex_ctrl);
1516}
1517
1518static inline int
1519fpga_harq_write_loopback(struct fpga_queue *q,
1520 struct rte_mbuf *harq_input, uint16_t harq_in_length,
1521 uint32_t harq_in_offset, uint32_t harq_out_offset)
1522{
1523 fpga_mutex_acquisition(q);
1524 uint32_t out_offset = harq_out_offset;
1525 uint32_t in_offset = harq_in_offset;
1526 uint32_t left_length = harq_in_length;
1527 uint32_t reg_32, increment = 0;
1528 uint64_t *input = NULL;
1529 uint32_t last_transaction = left_length
1530 % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1531 uint64_t last_word;
1532
1533 if (last_transaction > 0)
1534 left_length -= last_transaction;
1535
1536
1537
1538
1539
1540 reg_32 = fpga_reg_read_32(q->d->mmio_base,
1541 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
1542 if (reg_32 < harq_in_length) {
1543 left_length = reg_32;
1544 rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n");
1545 }
1546
1547 input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_input,
1548 uint8_t *, in_offset);
1549
1550 while (left_length > 0) {
1551 if (fpga_reg_read_8(q->d->mmio_base,
1552 FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) == 1) {
1553 fpga_reg_write_32(q->d->mmio_base,
1554 FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS,
1555 out_offset);
1556 fpga_reg_write_64(q->d->mmio_base,
1557 FPGA_5GNR_FEC_DDR4_WR_DATA_REGS,
1558 input[increment]);
1559 left_length -= FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1560 out_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1561 increment++;
1562 fpga_reg_write_8(q->d->mmio_base,
1563 FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1);
1564 }
1565 }
1566 while (last_transaction > 0) {
1567 if (fpga_reg_read_8(q->d->mmio_base,
1568 FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) == 1) {
1569 fpga_reg_write_32(q->d->mmio_base,
1570 FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS,
1571 out_offset);
1572 last_word = input[increment];
1573 last_word &= (uint64_t)(1 << (last_transaction * 4))
1574 - 1;
1575 fpga_reg_write_64(q->d->mmio_base,
1576 FPGA_5GNR_FEC_DDR4_WR_DATA_REGS,
1577 last_word);
1578 fpga_reg_write_8(q->d->mmio_base,
1579 FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1);
1580 last_transaction = 0;
1581 }
1582 }
1583 fpga_mutex_free(q);
1584 return 1;
1585}
1586
1587static inline int
1588fpga_harq_read_loopback(struct fpga_queue *q,
1589 struct rte_mbuf *harq_output, uint16_t harq_in_length,
1590 uint32_t harq_in_offset, uint32_t harq_out_offset)
1591{
1592 fpga_mutex_acquisition(q);
1593 uint32_t left_length, in_offset = harq_in_offset;
1594 uint64_t reg;
1595 uint32_t increment = 0;
1596 uint64_t *input = NULL;
1597 uint32_t last_transaction = harq_in_length
1598 % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1599
1600 if (last_transaction > 0)
1601 harq_in_length += (8 - last_transaction);
1602
1603 reg = fpga_reg_read_32(q->d->mmio_base,
1604 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
1605 if (reg < harq_in_length) {
1606 harq_in_length = reg;
1607 rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n");
1608 }
1609
1610 if (!mbuf_append(harq_output, harq_output, harq_in_length)) {
1611 rte_bbdev_log(ERR, "HARQ output buffer warning %d %d\n",
1612 harq_output->buf_len -
1613 rte_pktmbuf_headroom(harq_output),
1614 harq_in_length);
1615 harq_in_length = harq_output->buf_len -
1616 rte_pktmbuf_headroom(harq_output);
1617 if (!mbuf_append(harq_output, harq_output, harq_in_length)) {
1618 rte_bbdev_log(ERR, "HARQ output buffer issue %d %d\n",
1619 harq_output->buf_len, harq_in_length);
1620 return -1;
1621 }
1622 }
1623 left_length = harq_in_length;
1624
1625 input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_output,
1626 uint8_t *, harq_out_offset);
1627
1628 while (left_length > 0) {
1629 fpga_reg_write_32(q->d->mmio_base,
1630 FPGA_5GNR_FEC_DDR4_RD_ADDR_REGS, in_offset);
1631 fpga_reg_write_8(q->d->mmio_base,
1632 FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 1);
1633 reg = fpga_reg_read_8(q->d->mmio_base,
1634 FPGA_5GNR_FEC_DDR4_RD_RDY_REGS);
1635 while (reg != 1) {
1636 reg = fpga_reg_read_8(q->d->mmio_base,
1637 FPGA_5GNR_FEC_DDR4_RD_RDY_REGS);
1638 if (reg == FPGA_DDR_OVERFLOW) {
1639 rte_bbdev_log(ERR,
1640 "Read address is overflow!\n");
1641 return -1;
1642 }
1643 }
1644 input[increment] = fpga_reg_read_64(q->d->mmio_base,
1645 FPGA_5GNR_FEC_DDR4_RD_DATA_REGS);
1646 left_length -= FPGA_5GNR_FEC_DDR_RD_DATA_LEN_IN_BYTES;
1647 in_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1648 increment++;
1649 fpga_reg_write_8(q->d->mmio_base,
1650 FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 0);
1651 }
1652 fpga_mutex_free(q);
1653 return 1;
1654}
1655
1656static inline int
1657enqueue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op,
1658 uint16_t desc_offset)
1659{
1660 union fpga_dma_desc *desc;
1661 int ret;
1662 uint8_t c, crc24_bits = 0;
1663 struct rte_bbdev_op_ldpc_enc *enc = &op->ldpc_enc;
1664 uint16_t in_offset = enc->input.offset;
1665 uint16_t out_offset = enc->output.offset;
1666 struct rte_mbuf *m_in = enc->input.data;
1667 struct rte_mbuf *m_out = enc->output.data;
1668 struct rte_mbuf *m_out_head = enc->output.data;
1669 uint32_t in_length, out_length, e;
1670 uint16_t total_left = enc->input.length;
1671 uint16_t ring_offset;
1672 uint16_t K, k_;
1673
1674
1675 if (validate_ldpc_enc_op(op) == -1) {
1676 rte_bbdev_log(ERR, "LDPC encoder validation rejected");
1677 return -EINVAL;
1678 }
1679
1680
1681 op->status = 0;
1682
1683 if (m_in == NULL || m_out == NULL) {
1684 rte_bbdev_log(ERR, "Invalid mbuf pointer");
1685 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1686 return -EINVAL;
1687 }
1688
1689 if (enc->op_flags & RTE_BBDEV_LDPC_CRC_24B_ATTACH)
1690 crc24_bits = 24;
1691
1692 if (enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1693
1694
1695 c = enc->tb_params.c;
1696 e = enc->tb_params.ea;
1697 } else {
1698 c = 1;
1699 e = enc->cb_params.e;
1700 }
1701
1702
1703 K = (enc->basegraph == 1 ? 22 : 10) * enc->z_c;
1704 k_ = K - enc->n_filler;
1705 in_length = (k_ - crc24_bits) >> 3;
1706 out_length = (e + 7) >> 3;
1707
1708 total_left = rte_pktmbuf_data_len(m_in) - in_offset;
1709
1710
1711 if (total_left != in_length) {
1712 op->status |= 1 << RTE_BBDEV_DATA_ERROR;
1713 rte_bbdev_log(ERR,
1714 "Mismatch between mbuf length and included CBs sizes %d",
1715 total_left);
1716 }
1717
1718 mbuf_append(m_out_head, m_out, out_length);
1719
1720
1721 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask);
1722
1723 desc = q->ring_addr + ring_offset;
1724
1725 ret = fpga_dma_desc_te_fill(op, &desc->enc_req, m_in, m_out,
1726 k_, e, in_offset, out_offset, ring_offset, c);
1727 if (unlikely(ret < 0))
1728 return ret;
1729
1730
1731 total_left -= in_length;
1732 op->ldpc_enc.output.length += out_length;
1733
1734 if (total_left > 0) {
1735 rte_bbdev_log(ERR,
1736 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
1737 total_left, in_length);
1738 return -1;
1739 }
1740
1741#ifdef RTE_LIBRTE_BBDEV_DEBUG
1742 print_dma_enc_desc_debug_info(desc);
1743#endif
1744 return 1;
1745}
1746
1747static inline int
1748enqueue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op *op,
1749 uint16_t desc_offset)
1750{
1751 union fpga_dma_desc *desc;
1752 int ret;
1753 uint16_t ring_offset;
1754 uint8_t c;
1755 uint16_t e, in_length, out_length, k0, l, seg_total_left, sys_cols;
1756 uint16_t K, parity_offset, harq_in_length = 0, harq_out_length = 0;
1757 uint16_t crc24_overlap = 0;
1758 struct rte_bbdev_op_ldpc_dec *dec = &op->ldpc_dec;
1759 struct rte_mbuf *m_in = dec->input.data;
1760 struct rte_mbuf *m_out = dec->hard_output.data;
1761 struct rte_mbuf *m_out_head = dec->hard_output.data;
1762 uint16_t in_offset = dec->input.offset;
1763 uint16_t out_offset = dec->hard_output.offset;
1764 uint32_t harq_offset = 0;
1765
1766 if (validate_ldpc_dec_op(op) == -1) {
1767 rte_bbdev_log(ERR, "LDPC decoder validation rejected");
1768 return -EINVAL;
1769 }
1770
1771
1772 op->status = 0;
1773
1774
1775 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask);
1776 desc = q->ring_addr + ring_offset;
1777
1778 if (check_bit(dec->op_flags,
1779 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1780 struct rte_mbuf *harq_in = dec->harq_combined_input.data;
1781 struct rte_mbuf *harq_out = dec->harq_combined_output.data;
1782 harq_in_length = dec->harq_combined_input.length;
1783 uint32_t harq_in_offset = dec->harq_combined_input.offset;
1784 uint32_t harq_out_offset = dec->harq_combined_output.offset;
1785
1786 if (check_bit(dec->op_flags,
1787 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE
1788 )) {
1789 ret = fpga_harq_write_loopback(q, harq_in,
1790 harq_in_length, harq_in_offset,
1791 harq_out_offset);
1792 } else if (check_bit(dec->op_flags,
1793 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE
1794 )) {
1795 ret = fpga_harq_read_loopback(q, harq_out,
1796 harq_in_length, harq_in_offset,
1797 harq_out_offset);
1798 dec->harq_combined_output.length = harq_in_length;
1799 } else {
1800 rte_bbdev_log(ERR, "OP flag Err!");
1801 ret = -1;
1802 }
1803
1804 desc->dec_req.done = 1;
1805 desc->dec_req.error = 0;
1806 desc->dec_req.op_addr = op;
1807 desc->dec_req.cbs_in_op = 1;
1808
1809 desc->dec_req.desc_idx = (ring_offset + 1)
1810 & q->sw_ring_wrap_mask;
1811 return ret;
1812 }
1813
1814 if (m_in == NULL || m_out == NULL) {
1815 rte_bbdev_log(ERR, "Invalid mbuf pointer");
1816 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1817 return -1;
1818 }
1819
1820 c = 1;
1821 e = dec->cb_params.e;
1822
1823 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP))
1824 crc24_overlap = 24;
1825
1826 sys_cols = (dec->basegraph == 1) ? 22 : 10;
1827 K = sys_cols * dec->z_c;
1828 parity_offset = K - 2 * dec->z_c;
1829
1830 out_length = ((K - crc24_overlap - dec->n_filler) >> 3);
1831 in_length = e;
1832 seg_total_left = dec->input.length;
1833
1834 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) {
1835 harq_in_length = RTE_MIN(dec->harq_combined_input.length,
1836 (uint32_t)dec->n_cb);
1837 }
1838
1839 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) {
1840 k0 = get_k0(dec->n_cb, dec->z_c,
1841 dec->basegraph, dec->rv_index);
1842 if (k0 > parity_offset)
1843 l = k0 + e;
1844 else
1845 l = k0 + e + dec->n_filler;
1846 harq_out_length = RTE_MIN(RTE_MAX(harq_in_length, l),
1847 dec->n_cb);
1848 dec->harq_combined_output.length = harq_out_length;
1849 }
1850
1851 mbuf_append(m_out_head, m_out, out_length);
1852 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE))
1853 harq_offset = dec->harq_combined_input.offset;
1854 else if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE))
1855 harq_offset = dec->harq_combined_output.offset;
1856
1857 if ((harq_offset & 0x3FF) > 0) {
1858 rte_bbdev_log(ERR, "Invalid HARQ offset %d", harq_offset);
1859 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1860 return -1;
1861 }
1862
1863 ret = fpga_dma_desc_ld_fill(op, &desc->dec_req, m_in, m_out,
1864 harq_in_length, in_offset, out_offset, harq_offset,
1865 ring_offset, c);
1866 if (unlikely(ret < 0))
1867 return ret;
1868
1869 seg_total_left -= in_length;
1870 op->ldpc_dec.hard_output.length += out_length;
1871 if (seg_total_left > 0) {
1872 rte_bbdev_log(ERR,
1873 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
1874 seg_total_left, in_length);
1875 return -1;
1876 }
1877
1878#ifdef RTE_LIBRTE_BBDEV_DEBUG
1879 print_dma_dec_desc_debug_info(desc);
1880#endif
1881
1882 return 1;
1883}
1884
1885static uint16_t
1886fpga_enqueue_ldpc_enc(struct rte_bbdev_queue_data *q_data,
1887 struct rte_bbdev_enc_op **ops, uint16_t num)
1888{
1889 uint16_t i, total_enqueued_cbs = 0;
1890 int32_t avail;
1891 int enqueued_cbs;
1892 struct fpga_queue *q = q_data->queue_private;
1893 union fpga_dma_desc *desc;
1894
1895
1896 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) ==
1897 q->head_free_desc))
1898 return 0;
1899
1900
1901 avail = (q->head_free_desc > q->tail) ?
1902 q->head_free_desc - q->tail - 1 :
1903 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1;
1904
1905 for (i = 0; i < num; ++i) {
1906
1907
1908
1909
1910 if (unlikely(avail - 1 < 0))
1911 break;
1912 avail -= 1;
1913 enqueued_cbs = enqueue_ldpc_enc_one_op_cb(q, ops[i],
1914 total_enqueued_cbs);
1915
1916 if (enqueued_cbs < 0)
1917 break;
1918
1919 total_enqueued_cbs += enqueued_cbs;
1920
1921 rte_bbdev_log_debug("enqueuing enc ops [%d/%d] | head %d | tail %d",
1922 total_enqueued_cbs, num,
1923 q->head_free_desc, q->tail);
1924 }
1925
1926
1927
1928
1929 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1)
1930 & q->sw_ring_wrap_mask);
1931 desc->enc_req.irq_en = q->irq_enable;
1932
1933 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats);
1934
1935
1936 q_data->queue_stats.enqueued_count += i;
1937 q_data->queue_stats.enqueue_err_count += num - i;
1938
1939 return i;
1940}
1941
1942static uint16_t
1943fpga_enqueue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
1944 struct rte_bbdev_dec_op **ops, uint16_t num)
1945{
1946 uint16_t i, total_enqueued_cbs = 0;
1947 int32_t avail;
1948 int enqueued_cbs;
1949 struct fpga_queue *q = q_data->queue_private;
1950 union fpga_dma_desc *desc;
1951
1952
1953 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) ==
1954 q->head_free_desc))
1955 return 0;
1956
1957
1958 avail = (q->head_free_desc > q->tail) ?
1959 q->head_free_desc - q->tail - 1 :
1960 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1;
1961
1962 for (i = 0; i < num; ++i) {
1963
1964
1965
1966
1967 if (unlikely(avail - 1 < 0))
1968 break;
1969 avail -= 1;
1970 enqueued_cbs = enqueue_ldpc_dec_one_op_cb(q, ops[i],
1971 total_enqueued_cbs);
1972
1973 if (enqueued_cbs < 0)
1974 break;
1975
1976 total_enqueued_cbs += enqueued_cbs;
1977
1978 rte_bbdev_log_debug("enqueuing dec ops [%d/%d] | head %d | tail %d",
1979 total_enqueued_cbs, num,
1980 q->head_free_desc, q->tail);
1981 }
1982
1983
1984 q_data->queue_stats.enqueued_count += i;
1985 q_data->queue_stats.enqueue_err_count += num - i;
1986
1987
1988
1989
1990 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1)
1991 & q->sw_ring_wrap_mask);
1992 desc->enc_req.irq_en = q->irq_enable;
1993 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats);
1994 return i;
1995}
1996
1997
1998static inline int
1999dequeue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op **op,
2000 uint16_t desc_offset)
2001{
2002 union fpga_dma_desc *desc;
2003 int desc_error;
2004
2005 desc = q->ring_addr + ((q->head_free_desc + desc_offset)
2006 & q->sw_ring_wrap_mask);
2007
2008
2009 if (desc->enc_req.done == 0)
2010 return -1;
2011
2012
2013 rte_smp_rmb();
2014
2015 rte_bbdev_log_debug("DMA response desc %p", desc);
2016
2017#ifdef RTE_LIBRTE_BBDEV_DEBUG
2018 print_dma_enc_desc_debug_info(desc);
2019#endif
2020
2021 *op = desc->enc_req.op_addr;
2022
2023 desc_error = check_desc_error(desc->enc_req.error);
2024 (*op)->status = desc_error << RTE_BBDEV_DATA_ERROR;
2025
2026 return 1;
2027}
2028
2029
2030static inline int
2031dequeue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op **op,
2032 uint16_t desc_offset)
2033{
2034 union fpga_dma_desc *desc;
2035 int desc_error;
2036
2037 desc = q->ring_addr + ((q->head_free_desc + desc_offset)
2038 & q->sw_ring_wrap_mask);
2039
2040
2041 if (desc->dec_req.done == 0)
2042 return -1;
2043
2044
2045 rte_smp_rmb();
2046
2047#ifdef RTE_LIBRTE_BBDEV_DEBUG
2048 print_dma_dec_desc_debug_info(desc);
2049#endif
2050
2051 *op = desc->dec_req.op_addr;
2052
2053 if (check_bit((*op)->ldpc_dec.op_flags,
2054 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
2055 (*op)->status = 0;
2056 return 1;
2057 }
2058
2059
2060 (*op)->ldpc_dec.iter_count = desc->dec_req.iter + 1;
2061
2062 if (desc->dec_req.crc24b_ind && !(desc->dec_req.crcb_pass))
2063 (*op)->status = 1 << RTE_BBDEV_CRC_ERROR;
2064
2065 (*op)->status |= !(desc->dec_req.et_pass) << RTE_BBDEV_SYNDROME_ERROR;
2066
2067 desc_error = check_desc_error(desc->dec_req.error);
2068 (*op)->status |= desc_error << RTE_BBDEV_DATA_ERROR;
2069 return 1;
2070}
2071
2072static uint16_t
2073fpga_dequeue_ldpc_enc(struct rte_bbdev_queue_data *q_data,
2074 struct rte_bbdev_enc_op **ops, uint16_t num)
2075{
2076 struct fpga_queue *q = q_data->queue_private;
2077 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask;
2078 uint16_t i;
2079 uint16_t dequeued_cbs = 0;
2080 int ret;
2081
2082 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) {
2083 ret = dequeue_ldpc_enc_one_op_cb(q, &ops[i], dequeued_cbs);
2084
2085 if (ret < 0)
2086 break;
2087
2088 dequeued_cbs += ret;
2089
2090 rte_bbdev_log_debug("dequeuing enc ops [%d/%d] | head %d | tail %d",
2091 dequeued_cbs, num, q->head_free_desc, q->tail);
2092 }
2093
2094
2095 q->head_free_desc = (q->head_free_desc + dequeued_cbs) &
2096 q->sw_ring_wrap_mask;
2097
2098
2099 q_data->queue_stats.dequeued_count += i;
2100
2101 return i;
2102}
2103
2104static uint16_t
2105fpga_dequeue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
2106 struct rte_bbdev_dec_op **ops, uint16_t num)
2107{
2108 struct fpga_queue *q = q_data->queue_private;
2109 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask;
2110 uint16_t i;
2111 uint16_t dequeued_cbs = 0;
2112 int ret;
2113
2114 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) {
2115 ret = dequeue_ldpc_dec_one_op_cb(q, &ops[i], dequeued_cbs);
2116
2117 if (ret < 0)
2118 break;
2119
2120 dequeued_cbs += ret;
2121
2122 rte_bbdev_log_debug("dequeuing dec ops [%d/%d] | head %d | tail %d",
2123 dequeued_cbs, num, q->head_free_desc, q->tail);
2124 }
2125
2126
2127 q->head_free_desc = (q->head_free_desc + dequeued_cbs) &
2128 q->sw_ring_wrap_mask;
2129
2130
2131 q_data->queue_stats.dequeued_count += i;
2132
2133 return i;
2134}
2135
2136
2137
2138static void
2139fpga_5gnr_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv)
2140{
2141 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
2142
2143 dev->dev_ops = &fpga_ops;
2144 dev->enqueue_ldpc_enc_ops = fpga_enqueue_ldpc_enc;
2145 dev->enqueue_ldpc_dec_ops = fpga_enqueue_ldpc_dec;
2146 dev->dequeue_ldpc_enc_ops = fpga_dequeue_ldpc_enc;
2147 dev->dequeue_ldpc_dec_ops = fpga_dequeue_ldpc_dec;
2148
2149 ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->pf_device =
2150 !strcmp(drv->driver.name,
2151 RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME));
2152 ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->mmio_base =
2153 pci_dev->mem_resource[0].addr;
2154
2155 rte_bbdev_log_debug(
2156 "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64,
2157 drv->driver.name, dev->data->name,
2158 (void *)pci_dev->mem_resource[0].addr,
2159 pci_dev->mem_resource[0].phys_addr);
2160}
2161
2162static int
2163fpga_5gnr_fec_probe(struct rte_pci_driver *pci_drv,
2164 struct rte_pci_device *pci_dev)
2165{
2166 struct rte_bbdev *bbdev = NULL;
2167 char dev_name[RTE_BBDEV_NAME_MAX_LEN];
2168
2169 if (pci_dev == NULL) {
2170 rte_bbdev_log(ERR, "NULL PCI device");
2171 return -EINVAL;
2172 }
2173
2174 rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name));
2175
2176
2177 bbdev = rte_bbdev_allocate(pci_dev->device.name);
2178 if (bbdev == NULL)
2179 return -ENODEV;
2180
2181
2182 bbdev->data->dev_private = rte_zmalloc_socket(dev_name,
2183 sizeof(struct fpga_5gnr_fec_device),
2184 RTE_CACHE_LINE_SIZE,
2185 pci_dev->device.numa_node);
2186
2187 if (bbdev->data->dev_private == NULL) {
2188 rte_bbdev_log(CRIT,
2189 "Allocate of %zu bytes for device \"%s\" failed",
2190 sizeof(struct fpga_5gnr_fec_device), dev_name);
2191 rte_bbdev_release(bbdev);
2192 return -ENOMEM;
2193 }
2194
2195
2196 bbdev->device = &pci_dev->device;
2197 bbdev->intr_handle = pci_dev->intr_handle;
2198 bbdev->data->socket_id = pci_dev->device.numa_node;
2199
2200
2201 fpga_5gnr_fec_init(bbdev, pci_drv);
2202
2203 rte_bbdev_log_debug("bbdev id = %u [%s]",
2204 bbdev->data->dev_id, dev_name);
2205
2206 struct fpga_5gnr_fec_device *d = bbdev->data->dev_private;
2207 uint32_t version_id = fpga_reg_read_32(d->mmio_base,
2208 FPGA_5GNR_FEC_VERSION_ID);
2209 rte_bbdev_log(INFO, "FEC FPGA RTL v%u.%u",
2210 ((uint16_t)(version_id >> 16)), ((uint16_t)version_id));
2211
2212#ifdef RTE_LIBRTE_BBDEV_DEBUG
2213 if (!strcmp(pci_drv->driver.name,
2214 RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME)))
2215 print_static_reg_debug_info(d->mmio_base);
2216#endif
2217 return 0;
2218}
2219
2220static int
2221fpga_5gnr_fec_remove(struct rte_pci_device *pci_dev)
2222{
2223 struct rte_bbdev *bbdev;
2224 int ret;
2225 uint8_t dev_id;
2226
2227 if (pci_dev == NULL)
2228 return -EINVAL;
2229
2230
2231 bbdev = rte_bbdev_get_named_dev(pci_dev->device.name);
2232 if (bbdev == NULL) {
2233 rte_bbdev_log(CRIT,
2234 "Couldn't find HW dev \"%s\" to uninitialise it",
2235 pci_dev->device.name);
2236 return -ENODEV;
2237 }
2238 dev_id = bbdev->data->dev_id;
2239
2240
2241 rte_free(bbdev->data->dev_private);
2242
2243
2244 ret = rte_bbdev_close(dev_id);
2245 if (ret < 0)
2246 rte_bbdev_log(ERR,
2247 "Device %i failed to close during uninit: %i",
2248 dev_id, ret);
2249
2250
2251 ret = rte_bbdev_release(bbdev);
2252 if (ret)
2253 rte_bbdev_log(ERR, "Device %i failed to uninit: %i", dev_id,
2254 ret);
2255
2256 rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id);
2257
2258 return 0;
2259}
2260
2261static inline void
2262set_default_fpga_conf(struct rte_fpga_5gnr_fec_conf *def_conf)
2263{
2264
2265 memset(def_conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf));
2266
2267 def_conf->pf_mode_en = true;
2268
2269
2270 def_conf->ul_bandwidth = 3;
2271 def_conf->dl_bandwidth = 3;
2272
2273
2274 def_conf->dl_load_balance = 64;
2275 def_conf->ul_load_balance = 64;
2276}
2277
2278
2279int
2280rte_fpga_5gnr_fec_configure(const char *dev_name,
2281 const struct rte_fpga_5gnr_fec_conf *conf)
2282{
2283 uint32_t payload_32, address;
2284 uint16_t payload_16;
2285 uint8_t payload_8;
2286 uint16_t q_id, vf_id, total_q_id, total_ul_q_id, total_dl_q_id;
2287 struct rte_bbdev *bbdev = rte_bbdev_get_named_dev(dev_name);
2288 struct rte_fpga_5gnr_fec_conf def_conf;
2289
2290 if (bbdev == NULL) {
2291 rte_bbdev_log(ERR,
2292 "Invalid dev_name (%s), or device is not yet initialised",
2293 dev_name);
2294 return -ENODEV;
2295 }
2296
2297 struct fpga_5gnr_fec_device *d = bbdev->data->dev_private;
2298
2299 if (conf == NULL) {
2300 rte_bbdev_log(ERR,
2301 "FPGA Configuration was not provided. Default configuration will be loaded.");
2302 set_default_fpga_conf(&def_conf);
2303 conf = &def_conf;
2304 }
2305
2306
2307
2308
2309
2310
2311 payload_16 = (conf->dl_bandwidth << 8) | conf->ul_bandwidth;
2312 address = FPGA_5GNR_FEC_CONFIGURATION;
2313 fpga_reg_write_16(d->mmio_base, address, payload_16);
2314
2315
2316 payload_32 = FPGA_INVALID_HW_QUEUE_ID;
2317 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
2318 address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
2319 fpga_reg_write_32(d->mmio_base, address, payload_32);
2320 }
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376 if (conf->pf_mode_en) {
2377 payload_32 = 0x1;
2378 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
2379 address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
2380 fpga_reg_write_32(d->mmio_base, address, payload_32);
2381 }
2382 } else {
2383
2384 total_ul_q_id = total_dl_q_id = 0;
2385 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
2386 total_ul_q_id += conf->vf_ul_queues_number[vf_id];
2387 total_dl_q_id += conf->vf_dl_queues_number[vf_id];
2388 }
2389 total_q_id = total_dl_q_id + total_ul_q_id;
2390
2391
2392
2393
2394 if ((total_ul_q_id > FPGA_NUM_UL_QUEUES) ||
2395 (total_dl_q_id > FPGA_NUM_DL_QUEUES) ||
2396 (total_q_id > FPGA_TOTAL_NUM_QUEUES)) {
2397 rte_bbdev_log(ERR,
2398 "FPGA Configuration failed. Too many queues to configure: UL_Q %u, DL_Q %u, FPGA_Q %u",
2399 total_ul_q_id, total_dl_q_id,
2400 FPGA_TOTAL_NUM_QUEUES);
2401 return -EINVAL;
2402 }
2403 total_ul_q_id = 0;
2404 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
2405 for (q_id = 0; q_id < conf->vf_ul_queues_number[vf_id];
2406 ++q_id, ++total_ul_q_id) {
2407 address = (total_ul_q_id << 2) +
2408 FPGA_5GNR_FEC_QUEUE_MAP;
2409 payload_32 = ((0x80 + vf_id) << 16) | 0x1;
2410 fpga_reg_write_32(d->mmio_base, address,
2411 payload_32);
2412 }
2413 }
2414 total_dl_q_id = 0;
2415 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
2416 for (q_id = 0; q_id < conf->vf_dl_queues_number[vf_id];
2417 ++q_id, ++total_dl_q_id) {
2418 address = ((total_dl_q_id + FPGA_NUM_UL_QUEUES)
2419 << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
2420 payload_32 = ((0x80 + vf_id) << 16) | 0x1;
2421 fpga_reg_write_32(d->mmio_base, address,
2422 payload_32);
2423 }
2424 }
2425 }
2426
2427
2428 payload_16 = (conf->dl_load_balance << 8) | (conf->ul_load_balance);
2429 address = FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR;
2430 fpga_reg_write_16(d->mmio_base, address, payload_16);
2431
2432
2433 payload_16 = FPGA_RING_DESC_ENTRY_LENGTH;
2434 address = FPGA_5GNR_FEC_RING_DESC_LEN;
2435 fpga_reg_write_16(d->mmio_base, address, payload_16);
2436
2437
2438 payload_8 = 0x1;
2439 address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE;
2440 fpga_reg_write_8(d->mmio_base, address, payload_8);
2441
2442 rte_bbdev_log_debug("PF FPGA 5GNR FEC configuration complete for %s",
2443 dev_name);
2444
2445#ifdef RTE_LIBRTE_BBDEV_DEBUG
2446 print_static_reg_debug_info(d->mmio_base);
2447#endif
2448 return 0;
2449}
2450
2451
2452static struct rte_pci_id pci_id_fpga_5gnr_fec_pf_map[] = {
2453 {
2454 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID,
2455 FPGA_5GNR_FEC_PF_DEVICE_ID)
2456 },
2457 {.device_id = 0},
2458};
2459
2460static struct rte_pci_driver fpga_5gnr_fec_pci_pf_driver = {
2461 .probe = fpga_5gnr_fec_probe,
2462 .remove = fpga_5gnr_fec_remove,
2463 .id_table = pci_id_fpga_5gnr_fec_pf_map,
2464 .drv_flags = RTE_PCI_DRV_NEED_MAPPING
2465};
2466
2467
2468static struct rte_pci_id pci_id_fpga_5gnr_fec_vf_map[] = {
2469 {
2470 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID,
2471 FPGA_5GNR_FEC_VF_DEVICE_ID)
2472 },
2473 {.device_id = 0},
2474};
2475
2476static struct rte_pci_driver fpga_5gnr_fec_pci_vf_driver = {
2477 .probe = fpga_5gnr_fec_probe,
2478 .remove = fpga_5gnr_fec_remove,
2479 .id_table = pci_id_fpga_5gnr_fec_vf_map,
2480 .drv_flags = RTE_PCI_DRV_NEED_MAPPING
2481};
2482
2483
2484RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_PF_DRIVER_NAME, fpga_5gnr_fec_pci_pf_driver);
2485RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_PF_DRIVER_NAME,
2486 pci_id_fpga_5gnr_fec_pf_map);
2487RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_VF_DRIVER_NAME, fpga_5gnr_fec_pci_vf_driver);
2488RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_VF_DRIVER_NAME,
2489 pci_id_fpga_5gnr_fec_vf_map);
2490