1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/export.h>
34#include <linux/kernel.h>
35
36#include <asm/octeon/cvmx.h>
37#include <asm/octeon/cvmx-spinlock.h>
38#include <asm/octeon/cvmx-bootmem.h>
39
40
41
42
43static struct cvmx_bootmem_desc *cvmx_bootmem_desc;
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58#define CVMX_BOOTMEM_NAMED_GET_FIELD(addr, field) \
59 __cvmx_bootmem_desc_get(addr, \
60 offsetof(struct cvmx_bootmem_named_block_desc, field), \
61 sizeof_field(struct cvmx_bootmem_named_block_desc, field))
62
63
64
65
66
67
68
69
70
71
72
73
74
75static inline uint64_t __cvmx_bootmem_desc_get(uint64_t base, int offset,
76 int size)
77{
78 base = (1ull << 63) | (base + offset);
79 switch (size) {
80 case 4:
81 return cvmx_read64_uint32(base);
82 case 8:
83 return cvmx_read64_uint64(base);
84 default:
85 return 0;
86 }
87}
88
89
90
91
92
93
94
95#define NEXT_OFFSET 0
96#define SIZE_OFFSET 8
97
98static void cvmx_bootmem_phy_set_size(uint64_t addr, uint64_t size)
99{
100 cvmx_write64_uint64((addr + SIZE_OFFSET) | (1ull << 63), size);
101}
102
103static void cvmx_bootmem_phy_set_next(uint64_t addr, uint64_t next)
104{
105 cvmx_write64_uint64((addr + NEXT_OFFSET) | (1ull << 63), next);
106}
107
108static uint64_t cvmx_bootmem_phy_get_size(uint64_t addr)
109{
110 return cvmx_read64_uint64((addr + SIZE_OFFSET) | (1ull << 63));
111}
112
113static uint64_t cvmx_bootmem_phy_get_next(uint64_t addr)
114{
115 return cvmx_read64_uint64((addr + NEXT_OFFSET) | (1ull << 63));
116}
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131static void *cvmx_bootmem_alloc_range(uint64_t size, uint64_t alignment,
132 uint64_t min_addr, uint64_t max_addr)
133{
134 int64_t address;
135 address =
136 cvmx_bootmem_phy_alloc(size, min_addr, max_addr, alignment, 0);
137
138 if (address > 0)
139 return cvmx_phys_to_ptr(address);
140 else
141 return NULL;
142}
143
144void *cvmx_bootmem_alloc_address(uint64_t size, uint64_t address,
145 uint64_t alignment)
146{
147 return cvmx_bootmem_alloc_range(size, alignment, address,
148 address + size);
149}
150
151void *cvmx_bootmem_alloc_named_range(uint64_t size, uint64_t min_addr,
152 uint64_t max_addr, uint64_t align,
153 char *name)
154{
155 int64_t addr;
156
157 addr = cvmx_bootmem_phy_named_block_alloc(size, min_addr, max_addr,
158 align, name, 0);
159 if (addr >= 0)
160 return cvmx_phys_to_ptr(addr);
161 else
162 return NULL;
163}
164
165void *cvmx_bootmem_alloc_named(uint64_t size, uint64_t alignment, char *name)
166{
167 return cvmx_bootmem_alloc_named_range(size, 0, 0, alignment, name);
168}
169EXPORT_SYMBOL(cvmx_bootmem_alloc_named);
170
171void cvmx_bootmem_lock(void)
172{
173 cvmx_spinlock_lock((cvmx_spinlock_t *) &(cvmx_bootmem_desc->lock));
174}
175
176void cvmx_bootmem_unlock(void)
177{
178 cvmx_spinlock_unlock((cvmx_spinlock_t *) &(cvmx_bootmem_desc->lock));
179}
180
181int cvmx_bootmem_init(void *mem_desc_ptr)
182{
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197 if (!cvmx_bootmem_desc) {
198#if defined(CVMX_ABI_64)
199
200 cvmx_bootmem_desc = cvmx_phys_to_ptr(CAST64(mem_desc_ptr));
201#else
202 cvmx_bootmem_desc = (struct cvmx_bootmem_desc *) mem_desc_ptr;
203#endif
204 }
205
206 return 0;
207}
208
209
210
211
212
213
214
215
216
217int64_t cvmx_bootmem_phy_alloc(uint64_t req_size, uint64_t address_min,
218 uint64_t address_max, uint64_t alignment,
219 uint32_t flags)
220{
221
222 uint64_t head_addr;
223 uint64_t ent_addr;
224
225 uint64_t prev_addr = 0;
226 uint64_t new_ent_addr = 0;
227 uint64_t desired_min_addr;
228
229#ifdef DEBUG
230 cvmx_dprintf("cvmx_bootmem_phy_alloc: req_size: 0x%llx, "
231 "min_addr: 0x%llx, max_addr: 0x%llx, align: 0x%llx\n",
232 (unsigned long long)req_size,
233 (unsigned long long)address_min,
234 (unsigned long long)address_max,
235 (unsigned long long)alignment);
236#endif
237
238 if (cvmx_bootmem_desc->major_version > 3) {
239 cvmx_dprintf("ERROR: Incompatible bootmem descriptor "
240 "version: %d.%d at addr: %p\n",
241 (int)cvmx_bootmem_desc->major_version,
242 (int)cvmx_bootmem_desc->minor_version,
243 cvmx_bootmem_desc);
244 goto error_out;
245 }
246
247
248
249
250
251
252
253
254
255
256 if (!req_size)
257 goto error_out;
258
259
260 req_size = (req_size + (CVMX_BOOTMEM_ALIGNMENT_SIZE - 1)) &
261 ~(CVMX_BOOTMEM_ALIGNMENT_SIZE - 1);
262
263
264
265
266
267
268
269 if (address_min && !address_max)
270 address_max = address_min + req_size;
271 else if (!address_min && !address_max)
272 address_max = ~0ull;
273
274
275
276
277
278
279 if (alignment < CVMX_BOOTMEM_ALIGNMENT_SIZE)
280 alignment = CVMX_BOOTMEM_ALIGNMENT_SIZE;
281
282
283
284
285
286
287 if (alignment)
288 address_min = ALIGN(address_min, alignment);
289
290
291
292
293
294
295 if (req_size > address_max - address_min)
296 goto error_out;
297
298
299
300 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
301 cvmx_bootmem_lock();
302 head_addr = cvmx_bootmem_desc->head_addr;
303 ent_addr = head_addr;
304 for (; ent_addr;
305 prev_addr = ent_addr,
306 ent_addr = cvmx_bootmem_phy_get_next(ent_addr)) {
307 uint64_t usable_base, usable_max;
308 uint64_t ent_size = cvmx_bootmem_phy_get_size(ent_addr);
309
310 if (cvmx_bootmem_phy_get_next(ent_addr)
311 && ent_addr > cvmx_bootmem_phy_get_next(ent_addr)) {
312 cvmx_dprintf("Internal bootmem_alloc() error: ent: "
313 "0x%llx, next: 0x%llx\n",
314 (unsigned long long)ent_addr,
315 (unsigned long long)
316 cvmx_bootmem_phy_get_next(ent_addr));
317 goto error_out;
318 }
319
320
321
322
323
324
325 usable_base =
326 ALIGN(max(address_min, ent_addr), alignment);
327 usable_max = min(address_max, ent_addr + ent_size);
328
329
330
331
332
333 desired_min_addr = usable_base;
334
335
336
337
338 if (!((ent_addr + ent_size) > usable_base
339 && ent_addr < address_max
340 && req_size <= usable_max - usable_base))
341 continue;
342
343
344
345
346
347
348 if (flags & CVMX_BOOTMEM_FLAG_END_ALLOC) {
349 desired_min_addr = usable_max - req_size;
350
351
352
353
354 desired_min_addr &= ~(alignment - 1);
355 }
356
357
358 if (desired_min_addr == ent_addr) {
359 if (req_size < ent_size) {
360
361
362
363
364 new_ent_addr = ent_addr + req_size;
365 cvmx_bootmem_phy_set_next(new_ent_addr,
366 cvmx_bootmem_phy_get_next(ent_addr));
367 cvmx_bootmem_phy_set_size(new_ent_addr,
368 ent_size -
369 req_size);
370
371
372
373
374
375 cvmx_bootmem_phy_set_next(ent_addr,
376 new_ent_addr);
377 }
378
379
380
381
382
383 if (prev_addr)
384 cvmx_bootmem_phy_set_next(prev_addr,
385 cvmx_bootmem_phy_get_next(ent_addr));
386 else
387
388
389
390
391 cvmx_bootmem_desc->head_addr =
392 cvmx_bootmem_phy_get_next(ent_addr);
393
394 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
395 cvmx_bootmem_unlock();
396 return desired_min_addr;
397 }
398
399
400
401
402
403
404
405
406
407
408
409 new_ent_addr = desired_min_addr;
410 cvmx_bootmem_phy_set_next(new_ent_addr,
411 cvmx_bootmem_phy_get_next
412 (ent_addr));
413 cvmx_bootmem_phy_set_size(new_ent_addr,
414 cvmx_bootmem_phy_get_size
415 (ent_addr) -
416 (desired_min_addr -
417 ent_addr));
418 cvmx_bootmem_phy_set_size(ent_addr,
419 desired_min_addr - ent_addr);
420 cvmx_bootmem_phy_set_next(ent_addr, new_ent_addr);
421
422 }
423error_out:
424
425 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
426 cvmx_bootmem_unlock();
427 return -1;
428}
429
430int __cvmx_bootmem_phy_free(uint64_t phy_addr, uint64_t size, uint32_t flags)
431{
432 uint64_t cur_addr;
433 uint64_t prev_addr = 0;
434 int retval = 0;
435
436#ifdef DEBUG
437 cvmx_dprintf("__cvmx_bootmem_phy_free addr: 0x%llx, size: 0x%llx\n",
438 (unsigned long long)phy_addr, (unsigned long long)size);
439#endif
440 if (cvmx_bootmem_desc->major_version > 3) {
441 cvmx_dprintf("ERROR: Incompatible bootmem descriptor "
442 "version: %d.%d at addr: %p\n",
443 (int)cvmx_bootmem_desc->major_version,
444 (int)cvmx_bootmem_desc->minor_version,
445 cvmx_bootmem_desc);
446 return 0;
447 }
448
449
450 if (!size)
451 return 0;
452
453 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
454 cvmx_bootmem_lock();
455 cur_addr = cvmx_bootmem_desc->head_addr;
456 if (cur_addr == 0 || phy_addr < cur_addr) {
457
458 if (cur_addr && phy_addr + size > cur_addr)
459 goto bootmem_free_done;
460 else if (phy_addr + size == cur_addr) {
461
462 cvmx_bootmem_phy_set_next(phy_addr,
463 cvmx_bootmem_phy_get_next
464 (cur_addr));
465 cvmx_bootmem_phy_set_size(phy_addr,
466 cvmx_bootmem_phy_get_size
467 (cur_addr) + size);
468 cvmx_bootmem_desc->head_addr = phy_addr;
469
470 } else {
471
472 cvmx_bootmem_phy_set_next(phy_addr, cur_addr);
473 cvmx_bootmem_phy_set_size(phy_addr, size);
474 cvmx_bootmem_desc->head_addr = phy_addr;
475 }
476 retval = 1;
477 goto bootmem_free_done;
478 }
479
480
481 while (cur_addr && phy_addr > cur_addr) {
482 prev_addr = cur_addr;
483 cur_addr = cvmx_bootmem_phy_get_next(cur_addr);
484 }
485
486 if (!cur_addr) {
487
488
489
490
491
492 if (prev_addr + cvmx_bootmem_phy_get_size(prev_addr) ==
493 phy_addr) {
494 cvmx_bootmem_phy_set_size(prev_addr,
495 cvmx_bootmem_phy_get_size
496 (prev_addr) + size);
497 } else {
498 cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
499 cvmx_bootmem_phy_set_size(phy_addr, size);
500 cvmx_bootmem_phy_set_next(phy_addr, 0);
501 }
502 retval = 1;
503 goto bootmem_free_done;
504 } else {
505
506
507
508
509 if (prev_addr + cvmx_bootmem_phy_get_size(prev_addr) ==
510 phy_addr) {
511
512 cvmx_bootmem_phy_set_size(prev_addr,
513 cvmx_bootmem_phy_get_size
514 (prev_addr) + size);
515 if (phy_addr + size == cur_addr) {
516
517 cvmx_bootmem_phy_set_size(prev_addr,
518 cvmx_bootmem_phy_get_size(cur_addr) +
519 cvmx_bootmem_phy_get_size(prev_addr));
520 cvmx_bootmem_phy_set_next(prev_addr,
521 cvmx_bootmem_phy_get_next(cur_addr));
522 }
523 retval = 1;
524 goto bootmem_free_done;
525 } else if (phy_addr + size == cur_addr) {
526
527 cvmx_bootmem_phy_set_size(phy_addr,
528 cvmx_bootmem_phy_get_size
529 (cur_addr) + size);
530 cvmx_bootmem_phy_set_next(phy_addr,
531 cvmx_bootmem_phy_get_next
532 (cur_addr));
533 cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
534 retval = 1;
535 goto bootmem_free_done;
536 }
537
538
539 cvmx_bootmem_phy_set_size(phy_addr, size);
540 cvmx_bootmem_phy_set_next(phy_addr, cur_addr);
541 cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
542
543 }
544 retval = 1;
545
546bootmem_free_done:
547 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
548 cvmx_bootmem_unlock();
549 return retval;
550
551}
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566static struct cvmx_bootmem_named_block_desc *
567 cvmx_bootmem_phy_named_block_find(char *name, uint32_t flags)
568{
569 unsigned int i;
570 struct cvmx_bootmem_named_block_desc *named_block_array_ptr;
571
572#ifdef DEBUG
573 cvmx_dprintf("cvmx_bootmem_phy_named_block_find: %s\n", name);
574#endif
575
576
577
578
579 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
580 cvmx_bootmem_lock();
581
582
583 named_block_array_ptr = (struct cvmx_bootmem_named_block_desc *)
584 cvmx_phys_to_ptr(cvmx_bootmem_desc->named_block_array_addr);
585
586#ifdef DEBUG
587 cvmx_dprintf
588 ("cvmx_bootmem_phy_named_block_find: named_block_array_ptr: %p\n",
589 named_block_array_ptr);
590#endif
591 if (cvmx_bootmem_desc->major_version == 3) {
592 for (i = 0;
593 i < cvmx_bootmem_desc->named_block_num_blocks; i++) {
594 if ((name && named_block_array_ptr[i].size
595 && !strncmp(name, named_block_array_ptr[i].name,
596 cvmx_bootmem_desc->named_block_name_len
597 - 1))
598 || (!name && !named_block_array_ptr[i].size)) {
599 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
600 cvmx_bootmem_unlock();
601
602 return &(named_block_array_ptr[i]);
603 }
604 }
605 } else {
606 cvmx_dprintf("ERROR: Incompatible bootmem descriptor "
607 "version: %d.%d at addr: %p\n",
608 (int)cvmx_bootmem_desc->major_version,
609 (int)cvmx_bootmem_desc->minor_version,
610 cvmx_bootmem_desc);
611 }
612 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
613 cvmx_bootmem_unlock();
614
615 return NULL;
616}
617
618void *cvmx_bootmem_alloc_named_range_once(uint64_t size, uint64_t min_addr,
619 uint64_t max_addr, uint64_t align,
620 char *name,
621 void (*init) (void *))
622{
623 int64_t addr;
624 void *ptr;
625 uint64_t named_block_desc_addr;
626
627 named_block_desc_addr = (uint64_t)
628 cvmx_bootmem_phy_named_block_find(name,
629 (uint32_t)CVMX_BOOTMEM_FLAG_NO_LOCKING);
630
631 if (named_block_desc_addr) {
632 addr = CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_desc_addr,
633 base_addr);
634 return cvmx_phys_to_ptr(addr);
635 }
636
637 addr = cvmx_bootmem_phy_named_block_alloc(size, min_addr, max_addr,
638 align, name,
639 (uint32_t)CVMX_BOOTMEM_FLAG_NO_LOCKING);
640
641 if (addr < 0)
642 return NULL;
643 ptr = cvmx_phys_to_ptr(addr);
644
645 if (init)
646 init(ptr);
647 else
648 memset(ptr, 0, size);
649
650 return ptr;
651}
652EXPORT_SYMBOL(cvmx_bootmem_alloc_named_range_once);
653
654struct cvmx_bootmem_named_block_desc *cvmx_bootmem_find_named_block(char *name)
655{
656 return cvmx_bootmem_phy_named_block_find(name, 0);
657}
658EXPORT_SYMBOL(cvmx_bootmem_find_named_block);
659
660
661
662
663
664
665
666
667
668
669static int cvmx_bootmem_phy_named_block_free(char *name, uint32_t flags)
670{
671 struct cvmx_bootmem_named_block_desc *named_block_ptr;
672
673 if (cvmx_bootmem_desc->major_version != 3) {
674 cvmx_dprintf("ERROR: Incompatible bootmem descriptor version: "
675 "%d.%d at addr: %p\n",
676 (int)cvmx_bootmem_desc->major_version,
677 (int)cvmx_bootmem_desc->minor_version,
678 cvmx_bootmem_desc);
679 return 0;
680 }
681#ifdef DEBUG
682 cvmx_dprintf("cvmx_bootmem_phy_named_block_free: %s\n", name);
683#endif
684
685
686
687
688
689 cvmx_bootmem_lock();
690
691 named_block_ptr =
692 cvmx_bootmem_phy_named_block_find(name,
693 CVMX_BOOTMEM_FLAG_NO_LOCKING);
694 if (named_block_ptr) {
695#ifdef DEBUG
696 cvmx_dprintf("cvmx_bootmem_phy_named_block_free: "
697 "%s, base: 0x%llx, size: 0x%llx\n",
698 name,
699 (unsigned long long)named_block_ptr->base_addr,
700 (unsigned long long)named_block_ptr->size);
701#endif
702 __cvmx_bootmem_phy_free(named_block_ptr->base_addr,
703 named_block_ptr->size,
704 CVMX_BOOTMEM_FLAG_NO_LOCKING);
705 named_block_ptr->size = 0;
706
707 }
708
709 cvmx_bootmem_unlock();
710 return named_block_ptr != NULL;
711}
712
713int cvmx_bootmem_free_named(char *name)
714{
715 return cvmx_bootmem_phy_named_block_free(name, 0);
716}
717
718int64_t cvmx_bootmem_phy_named_block_alloc(uint64_t size, uint64_t min_addr,
719 uint64_t max_addr,
720 uint64_t alignment,
721 char *name,
722 uint32_t flags)
723{
724 int64_t addr_allocated;
725 struct cvmx_bootmem_named_block_desc *named_block_desc_ptr;
726
727#ifdef DEBUG
728 cvmx_dprintf("cvmx_bootmem_phy_named_block_alloc: size: 0x%llx, min: "
729 "0x%llx, max: 0x%llx, align: 0x%llx, name: %s\n",
730 (unsigned long long)size,
731 (unsigned long long)min_addr,
732 (unsigned long long)max_addr,
733 (unsigned long long)alignment,
734 name);
735#endif
736 if (cvmx_bootmem_desc->major_version != 3) {
737 cvmx_dprintf("ERROR: Incompatible bootmem descriptor version: "
738 "%d.%d at addr: %p\n",
739 (int)cvmx_bootmem_desc->major_version,
740 (int)cvmx_bootmem_desc->minor_version,
741 cvmx_bootmem_desc);
742 return -1;
743 }
744
745
746
747
748
749 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
750 cvmx_spinlock_lock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
751
752
753 named_block_desc_ptr =
754 cvmx_bootmem_phy_named_block_find(NULL,
755 flags | CVMX_BOOTMEM_FLAG_NO_LOCKING);
756
757
758
759
760
761 if (cvmx_bootmem_phy_named_block_find(name,
762 flags | CVMX_BOOTMEM_FLAG_NO_LOCKING) || !named_block_desc_ptr) {
763 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
764 cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
765 return -1;
766 }
767
768
769
770
771
772
773
774
775 size = ALIGN(size, CVMX_BOOTMEM_ALIGNMENT_SIZE);
776
777 addr_allocated = cvmx_bootmem_phy_alloc(size, min_addr, max_addr,
778 alignment,
779 flags | CVMX_BOOTMEM_FLAG_NO_LOCKING);
780 if (addr_allocated >= 0) {
781 named_block_desc_ptr->base_addr = addr_allocated;
782 named_block_desc_ptr->size = size;
783 strncpy(named_block_desc_ptr->name, name,
784 cvmx_bootmem_desc->named_block_name_len);
785 named_block_desc_ptr->name[cvmx_bootmem_desc->named_block_name_len - 1] = 0;
786 }
787
788 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING))
789 cvmx_spinlock_unlock((cvmx_spinlock_t *)&(cvmx_bootmem_desc->lock));
790 return addr_allocated;
791}
792
793struct cvmx_bootmem_desc *cvmx_bootmem_get_desc(void)
794{
795 return cvmx_bootmem_desc;
796}
797