1
2
3
4
5
6#include <stdbool.h>
7#include <stdio.h>
8#include <string.h>
9#include <stdint.h>
10#include <stdarg.h>
11#include <unistd.h>
12#include <inttypes.h>
13#include <errno.h>
14#include <sys/queue.h>
15
16#include <rte_common.h>
17#include <rte_log.h>
18#include <rte_debug.h>
19#include <rte_memory.h>
20#include <rte_memzone.h>
21#include <rte_malloc.h>
22#include <rte_atomic.h>
23#include <rte_launch.h>
24#include <rte_eal.h>
25#include <rte_eal_memconfig.h>
26#include <rte_per_lcore.h>
27#include <rte_lcore.h>
28#include <rte_branch_prediction.h>
29#include <rte_errno.h>
30#include <rte_string_fns.h>
31#include <rte_spinlock.h>
32#include <rte_tailq.h>
33#include <rte_eal_paging.h>
34
35#include "rte_mempool.h"
36#include "rte_mempool_trace.h"
37
38TAILQ_HEAD(rte_mempool_list, rte_tailq_entry);
39
40static struct rte_tailq_elem rte_mempool_tailq = {
41 .name = "RTE_MEMPOOL",
42};
43EAL_REGISTER_TAILQ(rte_mempool_tailq)
44
45#define CACHE_FLUSHTHRESH_MULTIPLIER 1.5
46#define CALC_CACHE_FLUSHTHRESH(c) \
47 ((typeof(c))((c) * CACHE_FLUSHTHRESH_MULTIPLIER))
48
49#if defined(RTE_ARCH_X86)
50
51
52
53
54static unsigned get_gcd(unsigned a, unsigned b)
55{
56 unsigned c;
57
58 if (0 == a)
59 return b;
60 if (0 == b)
61 return a;
62
63 if (a < b) {
64 c = a;
65 a = b;
66 b = c;
67 }
68
69 while (b != 0) {
70 c = a % b;
71 a = b;
72 b = c;
73 }
74
75 return a;
76}
77
78
79
80
81
82
83
84static unsigned int
85arch_mem_object_align(unsigned int obj_size)
86{
87 unsigned nrank, nchan;
88 unsigned new_obj_size;
89
90
91 nchan = rte_memory_get_nchannel();
92 if (nchan == 0)
93 nchan = 4;
94
95 nrank = rte_memory_get_nrank();
96 if (nrank == 0)
97 nrank = 1;
98
99
100 new_obj_size = (obj_size + RTE_MEMPOOL_ALIGN_MASK) / RTE_MEMPOOL_ALIGN;
101 while (get_gcd(new_obj_size, nrank * nchan) != 1)
102 new_obj_size++;
103 return new_obj_size * RTE_MEMPOOL_ALIGN;
104}
105#else
106static unsigned int
107arch_mem_object_align(unsigned int obj_size)
108{
109 return obj_size;
110}
111#endif
112
113struct pagesz_walk_arg {
114 int socket_id;
115 size_t min;
116};
117
118static int
119find_min_pagesz(const struct rte_memseg_list *msl, void *arg)
120{
121 struct pagesz_walk_arg *wa = arg;
122 bool valid;
123
124
125
126
127
128
129
130
131 valid = msl->socket_id == wa->socket_id;
132 valid |= wa->socket_id == SOCKET_ID_ANY && msl->external == 0;
133
134 if (valid && msl->page_sz < wa->min)
135 wa->min = msl->page_sz;
136
137 return 0;
138}
139
140static size_t
141get_min_page_size(int socket_id)
142{
143 struct pagesz_walk_arg wa;
144
145 wa.min = SIZE_MAX;
146 wa.socket_id = socket_id;
147
148 rte_memseg_list_walk(find_min_pagesz, &wa);
149
150 return wa.min == SIZE_MAX ? (size_t) rte_mem_page_size() : wa.min;
151}
152
153
154static void
155mempool_add_elem(struct rte_mempool *mp, __rte_unused void *opaque,
156 void *obj, rte_iova_t iova)
157{
158 struct rte_mempool_objhdr *hdr;
159 struct rte_mempool_objtlr *tlr __rte_unused;
160
161
162 hdr = RTE_PTR_SUB(obj, sizeof(*hdr));
163 hdr->mp = mp;
164 hdr->iova = iova;
165 STAILQ_INSERT_TAIL(&mp->elt_list, hdr, next);
166 mp->populated_size++;
167
168#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
169 hdr->cookie = RTE_MEMPOOL_HEADER_COOKIE2;
170 tlr = __mempool_get_trailer(obj);
171 tlr->cookie = RTE_MEMPOOL_TRAILER_COOKIE;
172#endif
173}
174
175
176uint32_t
177rte_mempool_obj_iter(struct rte_mempool *mp,
178 rte_mempool_obj_cb_t *obj_cb, void *obj_cb_arg)
179{
180 struct rte_mempool_objhdr *hdr;
181 void *obj;
182 unsigned n = 0;
183
184 STAILQ_FOREACH(hdr, &mp->elt_list, next) {
185 obj = (char *)hdr + sizeof(*hdr);
186 obj_cb(mp, obj_cb_arg, obj, n);
187 n++;
188 }
189
190 return n;
191}
192
193
194uint32_t
195rte_mempool_mem_iter(struct rte_mempool *mp,
196 rte_mempool_mem_cb_t *mem_cb, void *mem_cb_arg)
197{
198 struct rte_mempool_memhdr *hdr;
199 unsigned n = 0;
200
201 STAILQ_FOREACH(hdr, &mp->mem_list, next) {
202 mem_cb(mp, mem_cb_arg, hdr, n);
203 n++;
204 }
205
206 return n;
207}
208
209
210uint32_t
211rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
212 struct rte_mempool_objsz *sz)
213{
214 struct rte_mempool_objsz lsz;
215
216 sz = (sz != NULL) ? sz : &lsz;
217
218 sz->header_size = sizeof(struct rte_mempool_objhdr);
219 if ((flags & MEMPOOL_F_NO_CACHE_ALIGN) == 0)
220 sz->header_size = RTE_ALIGN_CEIL(sz->header_size,
221 RTE_MEMPOOL_ALIGN);
222
223#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
224 sz->trailer_size = sizeof(struct rte_mempool_objtlr);
225#else
226 sz->trailer_size = 0;
227#endif
228
229
230 sz->elt_size = RTE_ALIGN_CEIL(elt_size, sizeof(uint64_t));
231
232
233 if ((flags & MEMPOOL_F_NO_CACHE_ALIGN) == 0) {
234 sz->total_size = sz->header_size + sz->elt_size +
235 sz->trailer_size;
236 sz->trailer_size += ((RTE_MEMPOOL_ALIGN -
237 (sz->total_size & RTE_MEMPOOL_ALIGN_MASK)) &
238 RTE_MEMPOOL_ALIGN_MASK);
239 }
240
241
242
243
244
245 if ((flags & MEMPOOL_F_NO_SPREAD) == 0) {
246 unsigned new_size;
247 new_size = arch_mem_object_align
248 (sz->header_size + sz->elt_size + sz->trailer_size);
249 sz->trailer_size = new_size - sz->header_size - sz->elt_size;
250 }
251
252
253 sz->total_size = sz->header_size + sz->elt_size + sz->trailer_size;
254
255 return sz->total_size;
256}
257
258
259static void
260rte_mempool_memchunk_mz_free(__rte_unused struct rte_mempool_memhdr *memhdr,
261 void *opaque)
262{
263 const struct rte_memzone *mz = opaque;
264 rte_memzone_free(mz);
265}
266
267
268static void
269rte_mempool_free_memchunks(struct rte_mempool *mp)
270{
271 struct rte_mempool_memhdr *memhdr;
272 void *elt;
273
274 while (!STAILQ_EMPTY(&mp->elt_list)) {
275 rte_mempool_ops_dequeue_bulk(mp, &elt, 1);
276 (void)elt;
277 STAILQ_REMOVE_HEAD(&mp->elt_list, next);
278 mp->populated_size--;
279 }
280
281 while (!STAILQ_EMPTY(&mp->mem_list)) {
282 memhdr = STAILQ_FIRST(&mp->mem_list);
283 STAILQ_REMOVE_HEAD(&mp->mem_list, next);
284 if (memhdr->free_cb != NULL)
285 memhdr->free_cb(memhdr, memhdr->opaque);
286 rte_free(memhdr);
287 mp->nb_mem_chunks--;
288 }
289}
290
291static int
292mempool_ops_alloc_once(struct rte_mempool *mp)
293{
294 int ret;
295
296
297 if ((mp->flags & MEMPOOL_F_POOL_CREATED) == 0) {
298 ret = rte_mempool_ops_alloc(mp);
299 if (ret != 0)
300 return ret;
301 mp->flags |= MEMPOOL_F_POOL_CREATED;
302 }
303 return 0;
304}
305
306
307
308
309
310int
311rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
312 rte_iova_t iova, size_t len, rte_mempool_memchunk_free_cb_t *free_cb,
313 void *opaque)
314{
315 unsigned i = 0;
316 size_t off;
317 struct rte_mempool_memhdr *memhdr;
318 int ret;
319
320 ret = mempool_ops_alloc_once(mp);
321 if (ret != 0)
322 return ret;
323
324
325 if (mp->populated_size >= mp->size)
326 return -ENOSPC;
327
328 memhdr = rte_zmalloc("MEMPOOL_MEMHDR", sizeof(*memhdr), 0);
329 if (memhdr == NULL)
330 return -ENOMEM;
331
332 memhdr->mp = mp;
333 memhdr->addr = vaddr;
334 memhdr->iova = iova;
335 memhdr->len = len;
336 memhdr->free_cb = free_cb;
337 memhdr->opaque = opaque;
338
339 if (mp->flags & MEMPOOL_F_NO_CACHE_ALIGN)
340 off = RTE_PTR_ALIGN_CEIL(vaddr, 8) - vaddr;
341 else
342 off = RTE_PTR_ALIGN_CEIL(vaddr, RTE_MEMPOOL_ALIGN) - vaddr;
343
344 if (off > len) {
345 ret = 0;
346 goto fail;
347 }
348
349 i = rte_mempool_ops_populate(mp, mp->size - mp->populated_size,
350 (char *)vaddr + off,
351 (iova == RTE_BAD_IOVA) ? RTE_BAD_IOVA : (iova + off),
352 len - off, mempool_add_elem, NULL);
353
354
355 if (i == 0) {
356 ret = 0;
357 goto fail;
358 }
359
360 STAILQ_INSERT_TAIL(&mp->mem_list, memhdr, next);
361 mp->nb_mem_chunks++;
362
363 rte_mempool_trace_populate_iova(mp, vaddr, iova, len, free_cb, opaque);
364 return i;
365
366fail:
367 rte_free(memhdr);
368 return ret;
369}
370
371static rte_iova_t
372get_iova(void *addr)
373{
374 struct rte_memseg *ms;
375
376
377 ms = rte_mem_virt2memseg(addr, NULL);
378 if (ms == NULL || ms->iova == RTE_BAD_IOVA)
379
380 return rte_mem_virt2iova(addr);
381 return ms->iova + RTE_PTR_DIFF(addr, ms->addr);
382}
383
384
385
386
387int
388rte_mempool_populate_virt(struct rte_mempool *mp, char *addr,
389 size_t len, size_t pg_sz, rte_mempool_memchunk_free_cb_t *free_cb,
390 void *opaque)
391{
392 rte_iova_t iova;
393 size_t off, phys_len;
394 int ret, cnt = 0;
395
396 if (mp->flags & MEMPOOL_F_NO_IOVA_CONTIG)
397 return rte_mempool_populate_iova(mp, addr, RTE_BAD_IOVA,
398 len, free_cb, opaque);
399
400 for (off = 0; off < len &&
401 mp->populated_size < mp->size; off += phys_len) {
402
403 iova = get_iova(addr + off);
404
405
406 for (phys_len = RTE_MIN(
407 (size_t)(RTE_PTR_ALIGN_CEIL(addr + off + 1, pg_sz) -
408 (addr + off)),
409 len - off);
410 off + phys_len < len;
411 phys_len = RTE_MIN(phys_len + pg_sz, len - off)) {
412 rte_iova_t iova_tmp;
413
414 iova_tmp = get_iova(addr + off + phys_len);
415
416 if (iova_tmp == RTE_BAD_IOVA ||
417 iova_tmp != iova + phys_len)
418 break;
419 }
420
421 ret = rte_mempool_populate_iova(mp, addr + off, iova,
422 phys_len, free_cb, opaque);
423 if (ret == 0)
424 continue;
425 if (ret < 0)
426 goto fail;
427
428 free_cb = NULL;
429 cnt += ret;
430 }
431
432 rte_mempool_trace_populate_virt(mp, addr, len, pg_sz, free_cb, opaque);
433 return cnt;
434
435 fail:
436 rte_mempool_free_memchunks(mp);
437 return ret;
438}
439
440
441int
442rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz)
443{
444 bool need_iova_contig_obj;
445 bool alloc_in_ext_mem;
446 int ret;
447
448
449 ret = rte_malloc_heap_socket_is_external(mp->socket_id);
450 if (ret < 0)
451 return -EINVAL;
452 alloc_in_ext_mem = (ret == 1);
453 need_iova_contig_obj = !(mp->flags & MEMPOOL_F_NO_IOVA_CONTIG);
454
455 if (!need_iova_contig_obj)
456 *pg_sz = 0;
457 else if (rte_eal_has_hugepages() || alloc_in_ext_mem)
458 *pg_sz = get_min_page_size(mp->socket_id);
459 else
460 *pg_sz = rte_mem_page_size();
461
462 rte_mempool_trace_get_page_size(mp, *pg_sz);
463 return 0;
464}
465
466
467
468
469
470int
471rte_mempool_populate_default(struct rte_mempool *mp)
472{
473 unsigned int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY;
474 char mz_name[RTE_MEMZONE_NAMESIZE];
475 const struct rte_memzone *mz;
476 ssize_t mem_size;
477 size_t align, pg_sz, pg_shift = 0;
478 rte_iova_t iova;
479 unsigned mz_id, n;
480 int ret;
481 bool need_iova_contig_obj;
482 size_t max_alloc_size = SIZE_MAX;
483
484 ret = mempool_ops_alloc_once(mp);
485 if (ret != 0)
486 return ret;
487
488
489 if (mp->nb_mem_chunks != 0)
490 return -EEXIST;
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530 need_iova_contig_obj = !(mp->flags & MEMPOOL_F_NO_IOVA_CONTIG);
531 ret = rte_mempool_get_page_size(mp, &pg_sz);
532 if (ret < 0)
533 return ret;
534
535 if (pg_sz != 0)
536 pg_shift = rte_bsf32(pg_sz);
537
538 for (mz_id = 0, n = mp->size; n > 0; mz_id++, n -= ret) {
539 size_t min_chunk_size;
540
541 mem_size = rte_mempool_ops_calc_mem_size(
542 mp, n, pg_shift, &min_chunk_size, &align);
543
544 if (mem_size < 0) {
545 ret = mem_size;
546 goto fail;
547 }
548
549 ret = snprintf(mz_name, sizeof(mz_name),
550 RTE_MEMPOOL_MZ_FORMAT "_%d", mp->name, mz_id);
551 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
552 ret = -ENAMETOOLONG;
553 goto fail;
554 }
555
556
557
558
559 if (min_chunk_size == (size_t)mem_size)
560 mz_flags |= RTE_MEMZONE_IOVA_CONTIG;
561
562
563 do {
564 mz = rte_memzone_reserve_aligned(mz_name,
565 RTE_MIN((size_t)mem_size, max_alloc_size),
566 mp->socket_id, mz_flags, align);
567
568 if (mz != NULL || rte_errno != ENOMEM)
569 break;
570
571 max_alloc_size = RTE_MIN(max_alloc_size,
572 (size_t)mem_size) / 2;
573 } while (mz == NULL && max_alloc_size >= min_chunk_size);
574
575 if (mz == NULL) {
576 ret = -rte_errno;
577 goto fail;
578 }
579
580 if (need_iova_contig_obj)
581 iova = mz->iova;
582 else
583 iova = RTE_BAD_IOVA;
584
585 if (pg_sz == 0 || (mz_flags & RTE_MEMZONE_IOVA_CONTIG))
586 ret = rte_mempool_populate_iova(mp, mz->addr,
587 iova, mz->len,
588 rte_mempool_memchunk_mz_free,
589 (void *)(uintptr_t)mz);
590 else
591 ret = rte_mempool_populate_virt(mp, mz->addr,
592 mz->len, pg_sz,
593 rte_mempool_memchunk_mz_free,
594 (void *)(uintptr_t)mz);
595 if (ret == 0)
596 ret = -ENOBUFS;
597 if (ret < 0) {
598 rte_memzone_free(mz);
599 goto fail;
600 }
601 }
602
603 rte_mempool_trace_populate_default(mp);
604 return mp->size;
605
606 fail:
607 rte_mempool_free_memchunks(mp);
608 return ret;
609}
610
611
612static ssize_t
613get_anon_size(const struct rte_mempool *mp)
614{
615 ssize_t size;
616 size_t pg_sz, pg_shift;
617 size_t min_chunk_size;
618 size_t align;
619
620 pg_sz = rte_mem_page_size();
621 pg_shift = rte_bsf32(pg_sz);
622 size = rte_mempool_ops_calc_mem_size(mp, mp->size, pg_shift,
623 &min_chunk_size, &align);
624
625 return size;
626}
627
628
629static void
630rte_mempool_memchunk_anon_free(struct rte_mempool_memhdr *memhdr,
631 void *opaque)
632{
633 ssize_t size;
634
635
636
637
638
639
640 size = get_anon_size(memhdr->mp);
641 if (size < 0)
642 return;
643
644 rte_mem_unmap(opaque, size);
645}
646
647
648int
649rte_mempool_populate_anon(struct rte_mempool *mp)
650{
651 ssize_t size;
652 int ret;
653 char *addr;
654
655
656 if ((!STAILQ_EMPTY(&mp->mem_list)) || mp->nb_mem_chunks != 0) {
657 rte_errno = EINVAL;
658 return 0;
659 }
660
661 ret = mempool_ops_alloc_once(mp);
662 if (ret < 0) {
663 rte_errno = -ret;
664 return 0;
665 }
666
667 size = get_anon_size(mp);
668 if (size < 0) {
669 rte_errno = -size;
670 return 0;
671 }
672
673
674 addr = rte_mem_map(NULL, size, RTE_PROT_READ | RTE_PROT_WRITE,
675 RTE_MAP_SHARED | RTE_MAP_ANONYMOUS, -1, 0);
676 if (addr == NULL)
677 return 0;
678
679 if (rte_mem_lock(addr, size) < 0) {
680 rte_mem_unmap(addr, size);
681 return 0;
682 }
683
684 ret = rte_mempool_populate_virt(mp, addr, size, rte_mem_page_size(),
685 rte_mempool_memchunk_anon_free, addr);
686 if (ret == 0)
687 ret = -ENOBUFS;
688 if (ret < 0) {
689 rte_errno = -ret;
690 goto fail;
691 }
692
693 rte_mempool_trace_populate_anon(mp);
694 return mp->populated_size;
695
696 fail:
697 rte_mempool_free_memchunks(mp);
698 return 0;
699}
700
701
702void
703rte_mempool_free(struct rte_mempool *mp)
704{
705 struct rte_mempool_list *mempool_list = NULL;
706 struct rte_tailq_entry *te;
707
708 if (mp == NULL)
709 return;
710
711 mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list);
712 rte_mcfg_tailq_write_lock();
713
714 TAILQ_FOREACH(te, mempool_list, next) {
715 if (te->data == (void *)mp)
716 break;
717 }
718
719 if (te != NULL) {
720 TAILQ_REMOVE(mempool_list, te, next);
721 rte_free(te);
722 }
723 rte_mcfg_tailq_write_unlock();
724
725 rte_mempool_trace_free(mp);
726 rte_mempool_free_memchunks(mp);
727 rte_mempool_ops_free(mp);
728 rte_memzone_free(mp->mz);
729}
730
731static void
732mempool_cache_init(struct rte_mempool_cache *cache, uint32_t size)
733{
734 cache->size = size;
735 cache->flushthresh = CALC_CACHE_FLUSHTHRESH(size);
736 cache->len = 0;
737}
738
739
740
741
742
743
744struct rte_mempool_cache *
745rte_mempool_cache_create(uint32_t size, int socket_id)
746{
747 struct rte_mempool_cache *cache;
748
749 if (size == 0 || size > RTE_MEMPOOL_CACHE_MAX_SIZE) {
750 rte_errno = EINVAL;
751 return NULL;
752 }
753
754 cache = rte_zmalloc_socket("MEMPOOL_CACHE", sizeof(*cache),
755 RTE_CACHE_LINE_SIZE, socket_id);
756 if (cache == NULL) {
757 RTE_LOG(ERR, MEMPOOL, "Cannot allocate mempool cache.\n");
758 rte_errno = ENOMEM;
759 return NULL;
760 }
761
762 mempool_cache_init(cache, size);
763
764 rte_mempool_trace_cache_create(size, socket_id, cache);
765 return cache;
766}
767
768
769
770
771
772
773void
774rte_mempool_cache_free(struct rte_mempool_cache *cache)
775{
776 rte_mempool_trace_cache_free(cache);
777 rte_free(cache);
778}
779
780
781struct rte_mempool *
782rte_mempool_create_empty(const char *name, unsigned n, unsigned elt_size,
783 unsigned cache_size, unsigned private_data_size,
784 int socket_id, unsigned flags)
785{
786 char mz_name[RTE_MEMZONE_NAMESIZE];
787 struct rte_mempool_list *mempool_list;
788 struct rte_mempool *mp = NULL;
789 struct rte_tailq_entry *te = NULL;
790 const struct rte_memzone *mz = NULL;
791 size_t mempool_size;
792 unsigned int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY;
793 struct rte_mempool_objsz objsz;
794 unsigned lcore_id;
795 int ret;
796
797
798 RTE_BUILD_BUG_ON((sizeof(struct rte_mempool) &
799 RTE_CACHE_LINE_MASK) != 0);
800 RTE_BUILD_BUG_ON((sizeof(struct rte_mempool_cache) &
801 RTE_CACHE_LINE_MASK) != 0);
802#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
803 RTE_BUILD_BUG_ON((sizeof(struct rte_mempool_debug_stats) &
804 RTE_CACHE_LINE_MASK) != 0);
805 RTE_BUILD_BUG_ON((offsetof(struct rte_mempool, stats) &
806 RTE_CACHE_LINE_MASK) != 0);
807#endif
808
809 mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list);
810
811
812 if (n == 0) {
813 rte_errno = EINVAL;
814 return NULL;
815 }
816
817
818 if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE ||
819 CALC_CACHE_FLUSHTHRESH(cache_size) > n) {
820 rte_errno = EINVAL;
821 return NULL;
822 }
823
824
825 if (flags & MEMPOOL_F_NO_CACHE_ALIGN)
826 flags |= MEMPOOL_F_NO_SPREAD;
827
828
829 if (!rte_mempool_calc_obj_size(elt_size, flags, &objsz)) {
830 rte_errno = EINVAL;
831 return NULL;
832 }
833
834 rte_mcfg_mempool_write_lock();
835
836
837
838
839
840 private_data_size = (private_data_size +
841 RTE_MEMPOOL_ALIGN_MASK) & (~RTE_MEMPOOL_ALIGN_MASK);
842
843
844
845 te = rte_zmalloc("MEMPOOL_TAILQ_ENTRY", sizeof(*te), 0);
846 if (te == NULL) {
847 RTE_LOG(ERR, MEMPOOL, "Cannot allocate tailq entry!\n");
848 goto exit_unlock;
849 }
850
851 mempool_size = MEMPOOL_HEADER_SIZE(mp, cache_size);
852 mempool_size += private_data_size;
853 mempool_size = RTE_ALIGN_CEIL(mempool_size, RTE_MEMPOOL_ALIGN);
854
855 ret = snprintf(mz_name, sizeof(mz_name), RTE_MEMPOOL_MZ_FORMAT, name);
856 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
857 rte_errno = ENAMETOOLONG;
858 goto exit_unlock;
859 }
860
861 mz = rte_memzone_reserve(mz_name, mempool_size, socket_id, mz_flags);
862 if (mz == NULL)
863 goto exit_unlock;
864
865
866 mp = mz->addr;
867 memset(mp, 0, MEMPOOL_HEADER_SIZE(mp, cache_size));
868 ret = strlcpy(mp->name, name, sizeof(mp->name));
869 if (ret < 0 || ret >= (int)sizeof(mp->name)) {
870 rte_errno = ENAMETOOLONG;
871 goto exit_unlock;
872 }
873 mp->mz = mz;
874 mp->size = n;
875 mp->flags = flags;
876 mp->socket_id = socket_id;
877 mp->elt_size = objsz.elt_size;
878 mp->header_size = objsz.header_size;
879 mp->trailer_size = objsz.trailer_size;
880
881 mp->cache_size = cache_size;
882 mp->private_data_size = private_data_size;
883 STAILQ_INIT(&mp->elt_list);
884 STAILQ_INIT(&mp->mem_list);
885
886
887
888
889
890 mp->local_cache = (struct rte_mempool_cache *)
891 RTE_PTR_ADD(mp, MEMPOOL_HEADER_SIZE(mp, 0));
892
893
894 if (cache_size != 0) {
895 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++)
896 mempool_cache_init(&mp->local_cache[lcore_id],
897 cache_size);
898 }
899
900 te->data = mp;
901
902 rte_mcfg_tailq_write_lock();
903 TAILQ_INSERT_TAIL(mempool_list, te, next);
904 rte_mcfg_tailq_write_unlock();
905 rte_mcfg_mempool_write_unlock();
906
907 rte_mempool_trace_create_empty(name, n, elt_size, cache_size,
908 private_data_size, flags, mp);
909 return mp;
910
911exit_unlock:
912 rte_mcfg_mempool_write_unlock();
913 rte_free(te);
914 rte_mempool_free(mp);
915 return NULL;
916}
917
918
919struct rte_mempool *
920rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
921 unsigned cache_size, unsigned private_data_size,
922 rte_mempool_ctor_t *mp_init, void *mp_init_arg,
923 rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
924 int socket_id, unsigned flags)
925{
926 int ret;
927 struct rte_mempool *mp;
928
929 mp = rte_mempool_create_empty(name, n, elt_size, cache_size,
930 private_data_size, socket_id, flags);
931 if (mp == NULL)
932 return NULL;
933
934
935
936
937
938 if ((flags & MEMPOOL_F_SP_PUT) && (flags & MEMPOOL_F_SC_GET))
939 ret = rte_mempool_set_ops_byname(mp, "ring_sp_sc", NULL);
940 else if (flags & MEMPOOL_F_SP_PUT)
941 ret = rte_mempool_set_ops_byname(mp, "ring_sp_mc", NULL);
942 else if (flags & MEMPOOL_F_SC_GET)
943 ret = rte_mempool_set_ops_byname(mp, "ring_mp_sc", NULL);
944 else
945 ret = rte_mempool_set_ops_byname(mp, "ring_mp_mc", NULL);
946
947 if (ret)
948 goto fail;
949
950
951 if (mp_init)
952 mp_init(mp, mp_init_arg);
953
954 if (rte_mempool_populate_default(mp) < 0)
955 goto fail;
956
957
958 if (obj_init)
959 rte_mempool_obj_iter(mp, obj_init, obj_init_arg);
960
961 rte_mempool_trace_create(name, n, elt_size, cache_size,
962 private_data_size, mp_init, mp_init_arg, obj_init,
963 obj_init_arg, flags, mp);
964 return mp;
965
966 fail:
967 rte_mempool_free(mp);
968 return NULL;
969}
970
971
972unsigned int
973rte_mempool_avail_count(const struct rte_mempool *mp)
974{
975 unsigned count;
976 unsigned lcore_id;
977
978 count = rte_mempool_ops_get_count(mp);
979
980 if (mp->cache_size == 0)
981 return count;
982
983 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++)
984 count += mp->local_cache[lcore_id].len;
985
986
987
988
989
990 if (count > mp->size)
991 return mp->size;
992 return count;
993}
994
995
996unsigned int
997rte_mempool_in_use_count(const struct rte_mempool *mp)
998{
999 return mp->size - rte_mempool_avail_count(mp);
1000}
1001
1002
1003static unsigned
1004rte_mempool_dump_cache(FILE *f, const struct rte_mempool *mp)
1005{
1006 unsigned lcore_id;
1007 unsigned count = 0;
1008 unsigned cache_count;
1009
1010 fprintf(f, " internal cache infos:\n");
1011 fprintf(f, " cache_size=%"PRIu32"\n", mp->cache_size);
1012
1013 if (mp->cache_size == 0)
1014 return count;
1015
1016 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1017 cache_count = mp->local_cache[lcore_id].len;
1018 fprintf(f, " cache_count[%u]=%"PRIu32"\n",
1019 lcore_id, cache_count);
1020 count += cache_count;
1021 }
1022 fprintf(f, " total_cache_count=%u\n", count);
1023 return count;
1024}
1025
1026#ifndef __INTEL_COMPILER
1027#pragma GCC diagnostic ignored "-Wcast-qual"
1028#endif
1029
1030
1031void rte_mempool_check_cookies(const struct rte_mempool *mp,
1032 void * const *obj_table_const, unsigned n, int free)
1033{
1034#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1035 struct rte_mempool_objhdr *hdr;
1036 struct rte_mempool_objtlr *tlr;
1037 uint64_t cookie;
1038 void *tmp;
1039 void *obj;
1040 void **obj_table;
1041
1042
1043
1044 tmp = (void *) obj_table_const;
1045 obj_table = tmp;
1046
1047 while (n--) {
1048 obj = obj_table[n];
1049
1050 if (rte_mempool_from_obj(obj) != mp)
1051 rte_panic("MEMPOOL: object is owned by another "
1052 "mempool\n");
1053
1054 hdr = __mempool_get_header(obj);
1055 cookie = hdr->cookie;
1056
1057 if (free == 0) {
1058 if (cookie != RTE_MEMPOOL_HEADER_COOKIE1) {
1059 RTE_LOG(CRIT, MEMPOOL,
1060 "obj=%p, mempool=%p, cookie=%" PRIx64 "\n",
1061 obj, (const void *) mp, cookie);
1062 rte_panic("MEMPOOL: bad header cookie (put)\n");
1063 }
1064 hdr->cookie = RTE_MEMPOOL_HEADER_COOKIE2;
1065 } else if (free == 1) {
1066 if (cookie != RTE_MEMPOOL_HEADER_COOKIE2) {
1067 RTE_LOG(CRIT, MEMPOOL,
1068 "obj=%p, mempool=%p, cookie=%" PRIx64 "\n",
1069 obj, (const void *) mp, cookie);
1070 rte_panic("MEMPOOL: bad header cookie (get)\n");
1071 }
1072 hdr->cookie = RTE_MEMPOOL_HEADER_COOKIE1;
1073 } else if (free == 2) {
1074 if (cookie != RTE_MEMPOOL_HEADER_COOKIE1 &&
1075 cookie != RTE_MEMPOOL_HEADER_COOKIE2) {
1076 RTE_LOG(CRIT, MEMPOOL,
1077 "obj=%p, mempool=%p, cookie=%" PRIx64 "\n",
1078 obj, (const void *) mp, cookie);
1079 rte_panic("MEMPOOL: bad header cookie (audit)\n");
1080 }
1081 }
1082 tlr = __mempool_get_trailer(obj);
1083 cookie = tlr->cookie;
1084 if (cookie != RTE_MEMPOOL_TRAILER_COOKIE) {
1085 RTE_LOG(CRIT, MEMPOOL,
1086 "obj=%p, mempool=%p, cookie=%" PRIx64 "\n",
1087 obj, (const void *) mp, cookie);
1088 rte_panic("MEMPOOL: bad trailer cookie\n");
1089 }
1090 }
1091#else
1092 RTE_SET_USED(mp);
1093 RTE_SET_USED(obj_table_const);
1094 RTE_SET_USED(n);
1095 RTE_SET_USED(free);
1096#endif
1097}
1098
1099void
1100rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,
1101 void * const *first_obj_table_const, unsigned int n, int free)
1102{
1103#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1104 struct rte_mempool_info info;
1105 const size_t total_elt_sz =
1106 mp->header_size + mp->elt_size + mp->trailer_size;
1107 unsigned int i, j;
1108
1109 rte_mempool_ops_get_info(mp, &info);
1110
1111 for (i = 0; i < n; ++i) {
1112 void *first_obj = first_obj_table_const[i];
1113
1114 for (j = 0; j < info.contig_block_size; ++j) {
1115 void *obj;
1116
1117 obj = (void *)((uintptr_t)first_obj + j * total_elt_sz);
1118 rte_mempool_check_cookies(mp, &obj, 1, free);
1119 }
1120 }
1121#else
1122 RTE_SET_USED(mp);
1123 RTE_SET_USED(first_obj_table_const);
1124 RTE_SET_USED(n);
1125 RTE_SET_USED(free);
1126#endif
1127}
1128
1129#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1130static void
1131mempool_obj_audit(struct rte_mempool *mp, __rte_unused void *opaque,
1132 void *obj, __rte_unused unsigned idx)
1133{
1134 __mempool_check_cookies(mp, &obj, 1, 2);
1135}
1136
1137static void
1138mempool_audit_cookies(struct rte_mempool *mp)
1139{
1140 unsigned num;
1141
1142 num = rte_mempool_obj_iter(mp, mempool_obj_audit, NULL);
1143 if (num != mp->size) {
1144 rte_panic("rte_mempool_obj_iter(mempool=%p, size=%u) "
1145 "iterated only over %u elements\n",
1146 mp, mp->size, num);
1147 }
1148}
1149#else
1150#define mempool_audit_cookies(mp) do {} while(0)
1151#endif
1152
1153#ifndef __INTEL_COMPILER
1154#pragma GCC diagnostic error "-Wcast-qual"
1155#endif
1156
1157
1158static void
1159mempool_audit_cache(const struct rte_mempool *mp)
1160{
1161
1162 unsigned lcore_id;
1163
1164 if (mp->cache_size == 0)
1165 return;
1166
1167 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1168 const struct rte_mempool_cache *cache;
1169 cache = &mp->local_cache[lcore_id];
1170 if (cache->len > RTE_DIM(cache->objs)) {
1171 RTE_LOG(CRIT, MEMPOOL, "badness on cache[%u]\n",
1172 lcore_id);
1173 rte_panic("MEMPOOL: invalid cache len\n");
1174 }
1175 }
1176}
1177
1178
1179void
1180rte_mempool_audit(struct rte_mempool *mp)
1181{
1182 mempool_audit_cache(mp);
1183 mempool_audit_cookies(mp);
1184
1185
1186 RTE_SET_USED(mp);
1187}
1188
1189
1190void
1191rte_mempool_dump(FILE *f, struct rte_mempool *mp)
1192{
1193#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1194 struct rte_mempool_info info;
1195 struct rte_mempool_debug_stats sum;
1196 unsigned lcore_id;
1197#endif
1198 struct rte_mempool_memhdr *memhdr;
1199 struct rte_mempool_ops *ops;
1200 unsigned common_count;
1201 unsigned cache_count;
1202 size_t mem_len = 0;
1203
1204 RTE_ASSERT(f != NULL);
1205 RTE_ASSERT(mp != NULL);
1206
1207 fprintf(f, "mempool <%s>@%p\n", mp->name, mp);
1208 fprintf(f, " flags=%x\n", mp->flags);
1209 fprintf(f, " socket_id=%d\n", mp->socket_id);
1210 fprintf(f, " pool=%p\n", mp->pool_data);
1211 fprintf(f, " iova=0x%" PRIx64 "\n", mp->mz->iova);
1212 fprintf(f, " nb_mem_chunks=%u\n", mp->nb_mem_chunks);
1213 fprintf(f, " size=%"PRIu32"\n", mp->size);
1214 fprintf(f, " populated_size=%"PRIu32"\n", mp->populated_size);
1215 fprintf(f, " header_size=%"PRIu32"\n", mp->header_size);
1216 fprintf(f, " elt_size=%"PRIu32"\n", mp->elt_size);
1217 fprintf(f, " trailer_size=%"PRIu32"\n", mp->trailer_size);
1218 fprintf(f, " total_obj_size=%"PRIu32"\n",
1219 mp->header_size + mp->elt_size + mp->trailer_size);
1220
1221 fprintf(f, " private_data_size=%"PRIu32"\n", mp->private_data_size);
1222
1223 fprintf(f, " ops_index=%d\n", mp->ops_index);
1224 ops = rte_mempool_get_ops(mp->ops_index);
1225 fprintf(f, " ops_name: <%s>\n", (ops != NULL) ? ops->name : "NA");
1226
1227 STAILQ_FOREACH(memhdr, &mp->mem_list, next)
1228 mem_len += memhdr->len;
1229 if (mem_len != 0) {
1230 fprintf(f, " avg bytes/object=%#Lf\n",
1231 (long double)mem_len / mp->size);
1232 }
1233
1234 cache_count = rte_mempool_dump_cache(f, mp);
1235 common_count = rte_mempool_ops_get_count(mp);
1236 if ((cache_count + common_count) > mp->size)
1237 common_count = mp->size - cache_count;
1238 fprintf(f, " common_pool_count=%u\n", common_count);
1239
1240
1241#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
1242 rte_mempool_ops_get_info(mp, &info);
1243 memset(&sum, 0, sizeof(sum));
1244 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1245 sum.put_bulk += mp->stats[lcore_id].put_bulk;
1246 sum.put_objs += mp->stats[lcore_id].put_objs;
1247 sum.put_common_pool_bulk += mp->stats[lcore_id].put_common_pool_bulk;
1248 sum.put_common_pool_objs += mp->stats[lcore_id].put_common_pool_objs;
1249 sum.get_common_pool_bulk += mp->stats[lcore_id].get_common_pool_bulk;
1250 sum.get_common_pool_objs += mp->stats[lcore_id].get_common_pool_objs;
1251 sum.get_success_bulk += mp->stats[lcore_id].get_success_bulk;
1252 sum.get_success_objs += mp->stats[lcore_id].get_success_objs;
1253 sum.get_fail_bulk += mp->stats[lcore_id].get_fail_bulk;
1254 sum.get_fail_objs += mp->stats[lcore_id].get_fail_objs;
1255 sum.get_success_blks += mp->stats[lcore_id].get_success_blks;
1256 sum.get_fail_blks += mp->stats[lcore_id].get_fail_blks;
1257 }
1258 fprintf(f, " stats:\n");
1259 fprintf(f, " put_bulk=%"PRIu64"\n", sum.put_bulk);
1260 fprintf(f, " put_objs=%"PRIu64"\n", sum.put_objs);
1261 fprintf(f, " put_common_pool_bulk=%"PRIu64"\n", sum.put_common_pool_bulk);
1262 fprintf(f, " put_common_pool_objs=%"PRIu64"\n", sum.put_common_pool_objs);
1263 fprintf(f, " get_common_pool_bulk=%"PRIu64"\n", sum.get_common_pool_bulk);
1264 fprintf(f, " get_common_pool_objs=%"PRIu64"\n", sum.get_common_pool_objs);
1265 fprintf(f, " get_success_bulk=%"PRIu64"\n", sum.get_success_bulk);
1266 fprintf(f, " get_success_objs=%"PRIu64"\n", sum.get_success_objs);
1267 fprintf(f, " get_fail_bulk=%"PRIu64"\n", sum.get_fail_bulk);
1268 fprintf(f, " get_fail_objs=%"PRIu64"\n", sum.get_fail_objs);
1269 if (info.contig_block_size > 0) {
1270 fprintf(f, " get_success_blks=%"PRIu64"\n",
1271 sum.get_success_blks);
1272 fprintf(f, " get_fail_blks=%"PRIu64"\n", sum.get_fail_blks);
1273 }
1274#else
1275 fprintf(f, " no statistics available\n");
1276#endif
1277
1278 rte_mempool_audit(mp);
1279}
1280
1281
1282void
1283rte_mempool_list_dump(FILE *f)
1284{
1285 struct rte_mempool *mp = NULL;
1286 struct rte_tailq_entry *te;
1287 struct rte_mempool_list *mempool_list;
1288
1289 mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list);
1290
1291 rte_mcfg_mempool_read_lock();
1292
1293 TAILQ_FOREACH(te, mempool_list, next) {
1294 mp = (struct rte_mempool *) te->data;
1295 rte_mempool_dump(f, mp);
1296 }
1297
1298 rte_mcfg_mempool_read_unlock();
1299}
1300
1301
1302struct rte_mempool *
1303rte_mempool_lookup(const char *name)
1304{
1305 struct rte_mempool *mp = NULL;
1306 struct rte_tailq_entry *te;
1307 struct rte_mempool_list *mempool_list;
1308
1309 mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list);
1310
1311 rte_mcfg_mempool_read_lock();
1312
1313 TAILQ_FOREACH(te, mempool_list, next) {
1314 mp = (struct rte_mempool *) te->data;
1315 if (strncmp(name, mp->name, RTE_MEMPOOL_NAMESIZE) == 0)
1316 break;
1317 }
1318
1319 rte_mcfg_mempool_read_unlock();
1320
1321 if (te == NULL) {
1322 rte_errno = ENOENT;
1323 return NULL;
1324 }
1325
1326 return mp;
1327}
1328
1329void rte_mempool_walk(void (*func)(struct rte_mempool *, void *),
1330 void *arg)
1331{
1332 struct rte_tailq_entry *te = NULL;
1333 struct rte_mempool_list *mempool_list;
1334 void *tmp_te;
1335
1336 mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list);
1337
1338 rte_mcfg_mempool_read_lock();
1339
1340 TAILQ_FOREACH_SAFE(te, mempool_list, next, tmp_te) {
1341 (*func)((struct rte_mempool *) te->data, arg);
1342 }
1343
1344 rte_mcfg_mempool_read_unlock();
1345}
1346