1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31#include <linux/export.h>
32#include <drm/drmP.h>
33
34#if defined(CONFIG_X86)
35static void
36drm_clflush_page(struct page *page)
37{
38 uint8_t *page_virtual;
39 unsigned int i;
40 const int size = boot_cpu_data.x86_clflush_size;
41
42 if (unlikely(page == NULL))
43 return;
44
45 page_virtual = kmap_atomic(page);
46 for (i = 0; i < PAGE_SIZE; i += size)
47 clflush(page_virtual + i);
48 kunmap_atomic(page_virtual);
49}
50
51static void drm_cache_flush_clflush(struct page *pages[],
52 unsigned long num_pages)
53{
54 unsigned long i;
55
56 mb();
57 for (i = 0; i < num_pages; i++)
58 drm_clflush_page(*pages++);
59 mb();
60}
61
62static void
63drm_clflush_ipi_handler(void *null)
64{
65 wbinvd();
66}
67#endif
68
69void
70drm_clflush_pages(struct page *pages[], unsigned long num_pages)
71{
72
73#if defined(CONFIG_X86)
74 if (cpu_has_clflush) {
75 drm_cache_flush_clflush(pages, num_pages);
76 return;
77 }
78
79 if (on_each_cpu(drm_clflush_ipi_handler, NULL, 1) != 0)
80 printk(KERN_ERR "Timed out waiting for cache flush.\n");
81
82#elif defined(__powerpc__)
83 unsigned long i;
84 for (i = 0; i < num_pages; i++) {
85 struct page *page = pages[i];
86 void *page_virtual;
87
88 if (unlikely(page == NULL))
89 continue;
90
91 page_virtual = kmap_atomic(page);
92 flush_dcache_range((unsigned long)page_virtual,
93 (unsigned long)page_virtual + PAGE_SIZE);
94 kunmap_atomic(page_virtual);
95 }
96#else
97 printk(KERN_ERR "Architecture has no drm_cache.c support\n");
98 WARN_ON_ONCE(1);
99#endif
100}
101EXPORT_SYMBOL(drm_clflush_pages);
102
103void
104drm_clflush_sg(struct sg_table *st)
105{
106#if defined(CONFIG_X86)
107 if (cpu_has_clflush) {
108 struct sg_page_iter sg_iter;
109
110 mb();
111 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
112 drm_clflush_page(sg_page_iter_page(&sg_iter));
113 mb();
114
115 return;
116 }
117
118 if (on_each_cpu(drm_clflush_ipi_handler, NULL, 1) != 0)
119 printk(KERN_ERR "Timed out waiting for cache flush.\n");
120#else
121 printk(KERN_ERR "Architecture has no drm_cache.c support\n");
122 WARN_ON_ONCE(1);
123#endif
124}
125EXPORT_SYMBOL(drm_clflush_sg);
126
127void
128drm_clflush_virt_range(char *addr, unsigned long length)
129{
130#if defined(CONFIG_X86)
131 if (cpu_has_clflush) {
132 char *end = addr + length;
133 mb();
134 for (; addr < end; addr += boot_cpu_data.x86_clflush_size)
135 clflush(addr);
136 clflush(end - 1);
137 mb();
138 return;
139 }
140
141 if (on_each_cpu(drm_clflush_ipi_handler, NULL, 1) != 0)
142 printk(KERN_ERR "Timed out waiting for cache flush.\n");
143#else
144 printk(KERN_ERR "Architecture has no drm_cache.c support\n");
145 WARN_ON_ONCE(1);
146#endif
147}
148EXPORT_SYMBOL(drm_clflush_virt_range);
149