1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/list.h>
34#include <linux/spinlock.h>
35#include <linux/highmem.h>
36#include <linux/mm_types.h>
37#include <linux/module.h>
38#include <linux/mm.h>
39#include <linux/seq_file.h>
40#include <linux/slab.h>
41
42#include <asm/atomic.h>
43
44#include "ttm/ttm_bo_driver.h"
45#include "ttm/ttm_page_alloc.h"
46
47#ifdef TTM_HAS_AGP
48#include <asm/agp.h>
49#endif
50
51#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *))
52#define SMALL_ALLOCATION 16
53#define FREE_ALL_PAGES (~0U)
54
55#define PAGE_FREE_INTERVAL 1000
56
57
58
59
60
61
62
63
64
65
66
67
68struct ttm_page_pool {
69 spinlock_t lock;
70 bool fill_lock;
71 struct list_head list;
72 gfp_t gfp_flags;
73 unsigned npages;
74 char *name;
75 unsigned long nfrees;
76 unsigned long nrefills;
77};
78
79
80
81
82
83
84
85struct ttm_pool_opts {
86 unsigned alloc_size;
87 unsigned max_size;
88 unsigned small;
89};
90
91#define NUM_POOLS 4
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106struct ttm_pool_manager {
107 struct kobject kobj;
108 struct shrinker mm_shrink;
109 struct ttm_pool_opts options;
110
111 union {
112 struct ttm_page_pool pools[NUM_POOLS];
113 struct {
114 struct ttm_page_pool wc_pool;
115 struct ttm_page_pool uc_pool;
116 struct ttm_page_pool wc_pool_dma32;
117 struct ttm_page_pool uc_pool_dma32;
118 } ;
119 };
120};
121
122static struct attribute ttm_page_pool_max = {
123 .name = "pool_max_size",
124 .mode = S_IRUGO | S_IWUSR
125};
126static struct attribute ttm_page_pool_small = {
127 .name = "pool_small_allocation",
128 .mode = S_IRUGO | S_IWUSR
129};
130static struct attribute ttm_page_pool_alloc_size = {
131 .name = "pool_allocation_size",
132 .mode = S_IRUGO | S_IWUSR
133};
134
135static struct attribute *ttm_pool_attrs[] = {
136 &ttm_page_pool_max,
137 &ttm_page_pool_small,
138 &ttm_page_pool_alloc_size,
139 NULL
140};
141
142static void ttm_pool_kobj_release(struct kobject *kobj)
143{
144 struct ttm_pool_manager *m =
145 container_of(kobj, struct ttm_pool_manager, kobj);
146 kfree(m);
147}
148
149static ssize_t ttm_pool_store(struct kobject *kobj,
150 struct attribute *attr, const char *buffer, size_t size)
151{
152 struct ttm_pool_manager *m =
153 container_of(kobj, struct ttm_pool_manager, kobj);
154 int chars;
155 unsigned val;
156 chars = sscanf(buffer, "%u", &val);
157 if (chars == 0)
158 return size;
159
160
161 val = val / (PAGE_SIZE >> 10);
162
163 if (attr == &ttm_page_pool_max)
164 m->options.max_size = val;
165 else if (attr == &ttm_page_pool_small)
166 m->options.small = val;
167 else if (attr == &ttm_page_pool_alloc_size) {
168 if (val > NUM_PAGES_TO_ALLOC*8) {
169 printk(KERN_ERR TTM_PFX
170 "Setting allocation size to %lu "
171 "is not allowed. Recommended size is "
172 "%lu\n",
173 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7),
174 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
175 return size;
176 } else if (val > NUM_PAGES_TO_ALLOC) {
177 printk(KERN_WARNING TTM_PFX
178 "Setting allocation size to "
179 "larger than %lu is not recommended.\n",
180 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
181 }
182 m->options.alloc_size = val;
183 }
184
185 return size;
186}
187
188static ssize_t ttm_pool_show(struct kobject *kobj,
189 struct attribute *attr, char *buffer)
190{
191 struct ttm_pool_manager *m =
192 container_of(kobj, struct ttm_pool_manager, kobj);
193 unsigned val = 0;
194
195 if (attr == &ttm_page_pool_max)
196 val = m->options.max_size;
197 else if (attr == &ttm_page_pool_small)
198 val = m->options.small;
199 else if (attr == &ttm_page_pool_alloc_size)
200 val = m->options.alloc_size;
201
202 val = val * (PAGE_SIZE >> 10);
203
204 return snprintf(buffer, PAGE_SIZE, "%u\n", val);
205}
206
207static const struct sysfs_ops ttm_pool_sysfs_ops = {
208 .show = &ttm_pool_show,
209 .store = &ttm_pool_store,
210};
211
212static struct kobj_type ttm_pool_kobj_type = {
213 .release = &ttm_pool_kobj_release,
214 .sysfs_ops = &ttm_pool_sysfs_ops,
215 .default_attrs = ttm_pool_attrs,
216};
217
218static struct ttm_pool_manager *_manager;
219
220#ifndef CONFIG_X86
221static int set_pages_array_wb(struct page **pages, int addrinarray)
222{
223#ifdef TTM_HAS_AGP
224 int i;
225
226 for (i = 0; i < addrinarray; i++)
227 unmap_page_from_agp(pages[i]);
228#endif
229 return 0;
230}
231
232static int set_pages_array_wc(struct page **pages, int addrinarray)
233{
234#ifdef TTM_HAS_AGP
235 int i;
236
237 for (i = 0; i < addrinarray; i++)
238 map_page_into_agp(pages[i]);
239#endif
240 return 0;
241}
242
243static int set_pages_array_uc(struct page **pages, int addrinarray)
244{
245#ifdef TTM_HAS_AGP
246 int i;
247
248 for (i = 0; i < addrinarray; i++)
249 map_page_into_agp(pages[i]);
250#endif
251 return 0;
252}
253#endif
254
255
256
257static struct ttm_page_pool *ttm_get_pool(int flags,
258 enum ttm_caching_state cstate)
259{
260 int pool_index;
261
262 if (cstate == tt_cached)
263 return NULL;
264
265 if (cstate == tt_wc)
266 pool_index = 0x0;
267 else
268 pool_index = 0x1;
269
270 if (flags & TTM_PAGE_FLAG_DMA32)
271 pool_index |= 0x2;
272
273 return &_manager->pools[pool_index];
274}
275
276
277static void ttm_pages_put(struct page *pages[], unsigned npages)
278{
279 unsigned i;
280 if (set_pages_array_wb(pages, npages))
281 printk(KERN_ERR TTM_PFX "Failed to set %d pages to wb!\n",
282 npages);
283 for (i = 0; i < npages; ++i)
284 __free_page(pages[i]);
285}
286
287static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,
288 unsigned freed_pages)
289{
290 pool->npages -= freed_pages;
291 pool->nfrees += freed_pages;
292}
293
294
295
296
297
298
299
300
301
302
303static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free)
304{
305 unsigned long irq_flags;
306 struct page *p;
307 struct page **pages_to_free;
308 unsigned freed_pages = 0,
309 npages_to_free = nr_free;
310
311 if (NUM_PAGES_TO_ALLOC < nr_free)
312 npages_to_free = NUM_PAGES_TO_ALLOC;
313
314 pages_to_free = kmalloc(npages_to_free * sizeof(struct page *),
315 GFP_KERNEL);
316 if (!pages_to_free) {
317 printk(KERN_ERR TTM_PFX
318 "Failed to allocate memory for pool free operation.\n");
319 return 0;
320 }
321
322restart:
323 spin_lock_irqsave(&pool->lock, irq_flags);
324
325 list_for_each_entry_reverse(p, &pool->list, lru) {
326 if (freed_pages >= npages_to_free)
327 break;
328
329 pages_to_free[freed_pages++] = p;
330
331 if (freed_pages >= NUM_PAGES_TO_ALLOC) {
332
333 __list_del(p->lru.prev, &pool->list);
334
335 ttm_pool_update_free_locked(pool, freed_pages);
336
337
338
339
340 spin_unlock_irqrestore(&pool->lock, irq_flags);
341
342 ttm_pages_put(pages_to_free, freed_pages);
343 if (likely(nr_free != FREE_ALL_PAGES))
344 nr_free -= freed_pages;
345
346 if (NUM_PAGES_TO_ALLOC >= nr_free)
347 npages_to_free = nr_free;
348 else
349 npages_to_free = NUM_PAGES_TO_ALLOC;
350
351 freed_pages = 0;
352
353
354 if (nr_free)
355 goto restart;
356
357
358
359
360
361 goto out;
362
363 }
364 }
365
366
367 if (freed_pages) {
368 __list_del(&p->lru, &pool->list);
369
370 ttm_pool_update_free_locked(pool, freed_pages);
371 nr_free -= freed_pages;
372 }
373
374 spin_unlock_irqrestore(&pool->lock, irq_flags);
375
376 if (freed_pages)
377 ttm_pages_put(pages_to_free, freed_pages);
378out:
379 kfree(pages_to_free);
380 return nr_free;
381}
382
383
384static int ttm_pool_get_num_unused_pages(void)
385{
386 unsigned i;
387 int total = 0;
388 for (i = 0; i < NUM_POOLS; ++i)
389 total += _manager->pools[i].npages;
390
391 return total;
392}
393
394
395
396
397static int ttm_pool_mm_shrink(struct shrinker *shrink, int shrink_pages, gfp_t gfp_mask)
398{
399 static atomic_t start_pool = ATOMIC_INIT(0);
400 unsigned i;
401 unsigned pool_offset = atomic_add_return(1, &start_pool);
402 struct ttm_page_pool *pool;
403
404 pool_offset = pool_offset % NUM_POOLS;
405
406 for (i = 0; i < NUM_POOLS; ++i) {
407 unsigned nr_free = shrink_pages;
408 if (shrink_pages == 0)
409 break;
410 pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];
411 shrink_pages = ttm_page_pool_free(pool, nr_free);
412 }
413
414 return ttm_pool_get_num_unused_pages();
415}
416
417static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
418{
419 manager->mm_shrink.shrink = &ttm_pool_mm_shrink;
420 manager->mm_shrink.seeks = 1;
421 register_shrinker(&manager->mm_shrink);
422}
423
424static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
425{
426 unregister_shrinker(&manager->mm_shrink);
427}
428
429static int ttm_set_pages_caching(struct page **pages,
430 enum ttm_caching_state cstate, unsigned cpages)
431{
432 int r = 0;
433
434 switch (cstate) {
435 case tt_uncached:
436 r = set_pages_array_uc(pages, cpages);
437 if (r)
438 printk(KERN_ERR TTM_PFX
439 "Failed to set %d pages to uc!\n",
440 cpages);
441 break;
442 case tt_wc:
443 r = set_pages_array_wc(pages, cpages);
444 if (r)
445 printk(KERN_ERR TTM_PFX
446 "Failed to set %d pages to wc!\n",
447 cpages);
448 break;
449 default:
450 break;
451 }
452 return r;
453}
454
455
456
457
458
459
460static void ttm_handle_caching_state_failure(struct list_head *pages,
461 int ttm_flags, enum ttm_caching_state cstate,
462 struct page **failed_pages, unsigned cpages)
463{
464 unsigned i;
465
466 for (i = 0; i < cpages; ++i) {
467 list_del(&failed_pages[i]->lru);
468 __free_page(failed_pages[i]);
469 }
470}
471
472
473
474
475
476
477
478static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
479 int ttm_flags, enum ttm_caching_state cstate, unsigned count)
480{
481 struct page **caching_array;
482 struct page *p;
483 int r = 0;
484 unsigned i, cpages;
485 unsigned max_cpages = min(count,
486 (unsigned)(PAGE_SIZE/sizeof(struct page *)));
487
488
489 caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL);
490
491 if (!caching_array) {
492 printk(KERN_ERR TTM_PFX
493 "Unable to allocate table for new pages.");
494 return -ENOMEM;
495 }
496
497 for (i = 0, cpages = 0; i < count; ++i) {
498 p = alloc_page(gfp_flags);
499
500 if (!p) {
501 printk(KERN_ERR TTM_PFX "Unable to get page %u.\n", i);
502
503
504
505 if (cpages) {
506 r = ttm_set_pages_caching(caching_array,
507 cstate, cpages);
508 if (r)
509 ttm_handle_caching_state_failure(pages,
510 ttm_flags, cstate,
511 caching_array, cpages);
512 }
513 r = -ENOMEM;
514 goto out;
515 }
516
517#ifdef CONFIG_HIGHMEM
518
519
520
521 if (!PageHighMem(p))
522#endif
523 {
524 caching_array[cpages++] = p;
525 if (cpages == max_cpages) {
526
527 r = ttm_set_pages_caching(caching_array,
528 cstate, cpages);
529 if (r) {
530 ttm_handle_caching_state_failure(pages,
531 ttm_flags, cstate,
532 caching_array, cpages);
533 goto out;
534 }
535 cpages = 0;
536 }
537 }
538
539 list_add(&p->lru, pages);
540 }
541
542 if (cpages) {
543 r = ttm_set_pages_caching(caching_array, cstate, cpages);
544 if (r)
545 ttm_handle_caching_state_failure(pages,
546 ttm_flags, cstate,
547 caching_array, cpages);
548 }
549out:
550 kfree(caching_array);
551
552 return r;
553}
554
555
556
557
558
559static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
560 int ttm_flags, enum ttm_caching_state cstate, unsigned count,
561 unsigned long *irq_flags)
562{
563 struct page *p;
564 int r;
565 unsigned cpages = 0;
566
567
568
569
570
571 if (pool->fill_lock)
572 return;
573
574 pool->fill_lock = true;
575
576
577
578 if (count < _manager->options.small
579 && count > pool->npages) {
580 struct list_head new_pages;
581 unsigned alloc_size = _manager->options.alloc_size;
582
583
584
585
586
587 spin_unlock_irqrestore(&pool->lock, *irq_flags);
588
589 INIT_LIST_HEAD(&new_pages);
590 r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,
591 cstate, alloc_size);
592 spin_lock_irqsave(&pool->lock, *irq_flags);
593
594 if (!r) {
595 list_splice(&new_pages, &pool->list);
596 ++pool->nrefills;
597 pool->npages += alloc_size;
598 } else {
599 printk(KERN_ERR TTM_PFX
600 "Failed to fill pool (%p).", pool);
601
602 list_for_each_entry(p, &pool->list, lru) {
603 ++cpages;
604 }
605 list_splice(&new_pages, &pool->list);
606 pool->npages += cpages;
607 }
608
609 }
610 pool->fill_lock = false;
611}
612
613
614
615
616
617
618static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
619 struct list_head *pages, int ttm_flags,
620 enum ttm_caching_state cstate, unsigned count)
621{
622 unsigned long irq_flags;
623 struct list_head *p;
624 unsigned i;
625
626 spin_lock_irqsave(&pool->lock, irq_flags);
627 ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags);
628
629 if (count >= pool->npages) {
630
631 list_splice_init(&pool->list, pages);
632 count -= pool->npages;
633 pool->npages = 0;
634 goto out;
635 }
636
637
638 if (count <= pool->npages/2) {
639 i = 0;
640 list_for_each(p, &pool->list) {
641 if (++i == count)
642 break;
643 }
644 } else {
645 i = pool->npages + 1;
646 list_for_each_prev(p, &pool->list) {
647 if (--i == count)
648 break;
649 }
650 }
651
652 list_cut_position(pages, &pool->list, p);
653 pool->npages -= count;
654 count = 0;
655out:
656 spin_unlock_irqrestore(&pool->lock, irq_flags);
657 return count;
658}
659
660
661
662
663
664int ttm_get_pages(struct list_head *pages, int flags,
665 enum ttm_caching_state cstate, unsigned count)
666{
667 struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
668 struct page *p = NULL;
669 gfp_t gfp_flags = GFP_USER;
670 int r;
671
672
673 if (flags & TTM_PAGE_FLAG_ZERO_ALLOC)
674 gfp_flags |= __GFP_ZERO;
675
676
677 if (pool == NULL) {
678 if (flags & TTM_PAGE_FLAG_DMA32)
679 gfp_flags |= GFP_DMA32;
680 else
681 gfp_flags |= GFP_HIGHUSER;
682
683 for (r = 0; r < count; ++r) {
684 p = alloc_page(gfp_flags);
685 if (!p) {
686
687 printk(KERN_ERR TTM_PFX
688 "Unable to allocate page.");
689 return -ENOMEM;
690 }
691
692 list_add(&p->lru, pages);
693 }
694 return 0;
695 }
696
697
698
699 gfp_flags |= pool->gfp_flags;
700
701
702 count = ttm_page_pool_get_pages(pool, pages, flags, cstate, count);
703
704
705 if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
706 list_for_each_entry(p, pages, lru) {
707 clear_page(page_address(p));
708 }
709 }
710
711
712 if (count > 0) {
713
714
715
716 r = ttm_alloc_new_pages(pages, gfp_flags, flags, cstate, count);
717 if (r) {
718
719
720 printk(KERN_ERR TTM_PFX
721 "Failed to allocate extra pages "
722 "for large request.");
723 ttm_put_pages(pages, 0, flags, cstate);
724 return r;
725 }
726 }
727
728
729 return 0;
730}
731
732
733void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags,
734 enum ttm_caching_state cstate)
735{
736 unsigned long irq_flags;
737 struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
738 struct page *p, *tmp;
739
740 if (pool == NULL) {
741
742
743 list_for_each_entry_safe(p, tmp, pages, lru) {
744 __free_page(p);
745 }
746
747 INIT_LIST_HEAD(pages);
748 return;
749 }
750 if (page_count == 0) {
751 list_for_each_entry_safe(p, tmp, pages, lru) {
752 ++page_count;
753 }
754 }
755
756 spin_lock_irqsave(&pool->lock, irq_flags);
757 list_splice_init(pages, &pool->list);
758 pool->npages += page_count;
759
760 page_count = 0;
761 if (pool->npages > _manager->options.max_size) {
762 page_count = pool->npages - _manager->options.max_size;
763
764
765 if (page_count < NUM_PAGES_TO_ALLOC)
766 page_count = NUM_PAGES_TO_ALLOC;
767 }
768 spin_unlock_irqrestore(&pool->lock, irq_flags);
769 if (page_count)
770 ttm_page_pool_free(pool, page_count);
771}
772
773static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags,
774 char *name)
775{
776 spin_lock_init(&pool->lock);
777 pool->fill_lock = false;
778 INIT_LIST_HEAD(&pool->list);
779 pool->npages = pool->nfrees = 0;
780 pool->gfp_flags = flags;
781 pool->name = name;
782}
783
784int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
785{
786 int ret;
787
788 WARN_ON(_manager);
789
790 printk(KERN_INFO TTM_PFX "Initializing pool allocator.\n");
791
792 _manager = kzalloc(sizeof(*_manager), GFP_KERNEL);
793
794 ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc");
795
796 ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc");
797
798 ttm_page_pool_init_locked(&_manager->wc_pool_dma32,
799 GFP_USER | GFP_DMA32, "wc dma");
800
801 ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
802 GFP_USER | GFP_DMA32, "uc dma");
803
804 _manager->options.max_size = max_pages;
805 _manager->options.small = SMALL_ALLOCATION;
806 _manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
807
808 ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type,
809 &glob->kobj, "pool");
810 if (unlikely(ret != 0)) {
811 kobject_put(&_manager->kobj);
812 _manager = NULL;
813 return ret;
814 }
815
816 ttm_pool_mm_shrink_init(_manager);
817
818 return 0;
819}
820
821void ttm_page_alloc_fini(void)
822{
823 int i;
824
825 printk(KERN_INFO TTM_PFX "Finalizing pool allocator.\n");
826 ttm_pool_mm_shrink_fini(_manager);
827
828 for (i = 0; i < NUM_POOLS; ++i)
829 ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES);
830
831 kobject_put(&_manager->kobj);
832 _manager = NULL;
833}
834
835int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
836{
837 struct ttm_page_pool *p;
838 unsigned i;
839 char *h[] = {"pool", "refills", "pages freed", "size"};
840 if (!_manager) {
841 seq_printf(m, "No pool allocator running.\n");
842 return 0;
843 }
844 seq_printf(m, "%6s %12s %13s %8s\n",
845 h[0], h[1], h[2], h[3]);
846 for (i = 0; i < NUM_POOLS; ++i) {
847 p = &_manager->pools[i];
848
849 seq_printf(m, "%6s %12ld %13ld %8d\n",
850 p->name, p->nrefills,
851 p->nfrees, p->npages);
852 }
853 return 0;
854}
855EXPORT_SYMBOL(ttm_page_alloc_debugfs);
856