1
2#ifndef _ASM_X86_IO_H
3#define _ASM_X86_IO_H
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38#define ARCH_HAS_IOREMAP_WC
39#define ARCH_HAS_IOREMAP_WT
40
41#include <linux/string.h>
42#include <linux/compiler.h>
43#include <asm/page.h>
44#include <asm/early_ioremap.h>
45#include <asm/pgtable_types.h>
46
47#define build_mmio_read(name, size, type, reg, barrier) \
48static inline type name(const volatile void __iomem *addr) \
49{ type ret; asm volatile("mov" size " %1,%0":reg (ret) \
50:"m" (*(volatile type __force *)addr) barrier); return ret; }
51
52#define build_mmio_write(name, size, type, reg, barrier) \
53static inline void name(type val, volatile void __iomem *addr) \
54{ asm volatile("mov" size " %0,%1": :reg (val), \
55"m" (*(volatile type __force *)addr) barrier); }
56
57build_mmio_read(readb, "b", unsigned char, "=q", :"memory")
58build_mmio_read(readw, "w", unsigned short, "=r", :"memory")
59build_mmio_read(readl, "l", unsigned int, "=r", :"memory")
60
61build_mmio_read(__readb, "b", unsigned char, "=q", )
62build_mmio_read(__readw, "w", unsigned short, "=r", )
63build_mmio_read(__readl, "l", unsigned int, "=r", )
64
65build_mmio_write(writeb, "b", unsigned char, "q", :"memory")
66build_mmio_write(writew, "w", unsigned short, "r", :"memory")
67build_mmio_write(writel, "l", unsigned int, "r", :"memory")
68
69build_mmio_write(__writeb, "b", unsigned char, "q", )
70build_mmio_write(__writew, "w", unsigned short, "r", )
71build_mmio_write(__writel, "l", unsigned int, "r", )
72
73#define readb readb
74#define readw readw
75#define readl readl
76#define readb_relaxed(a) __readb(a)
77#define readw_relaxed(a) __readw(a)
78#define readl_relaxed(a) __readl(a)
79#define __raw_readb __readb
80#define __raw_readw __readw
81#define __raw_readl __readl
82
83#define writeb writeb
84#define writew writew
85#define writel writel
86#define writeb_relaxed(v, a) __writeb(v, a)
87#define writew_relaxed(v, a) __writew(v, a)
88#define writel_relaxed(v, a) __writel(v, a)
89#define __raw_writeb __writeb
90#define __raw_writew __writew
91#define __raw_writel __writel
92
93#ifdef CONFIG_X86_64
94
95build_mmio_read(readq, "q", u64, "=r", :"memory")
96build_mmio_read(__readq, "q", u64, "=r", )
97build_mmio_write(writeq, "q", u64, "r", :"memory")
98build_mmio_write(__writeq, "q", u64, "r", )
99
100#define readq_relaxed(a) __readq(a)
101#define writeq_relaxed(v, a) __writeq(v, a)
102
103#define __raw_readq __readq
104#define __raw_writeq __writeq
105
106
107#define readq readq
108#define writeq writeq
109
110#endif
111
112#define ARCH_HAS_VALID_PHYS_ADDR_RANGE
113extern int valid_phys_addr_range(phys_addr_t addr, size_t size);
114extern int valid_mmap_phys_addr_range(unsigned long pfn, size_t size);
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129static inline phys_addr_t virt_to_phys(volatile void *address)
130{
131 return __pa(address);
132}
133#define virt_to_phys virt_to_phys
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148static inline void *phys_to_virt(phys_addr_t address)
149{
150 return __va(address);
151}
152#define phys_to_virt phys_to_virt
153
154
155
156
157#define page_to_phys(page) ((dma_addr_t)page_to_pfn(page) << PAGE_SHIFT)
158
159
160
161
162
163
164static inline unsigned int isa_virt_to_bus(volatile void *address)
165{
166 return (unsigned int)virt_to_phys(address);
167}
168#define isa_bus_to_virt phys_to_virt
169
170
171
172
173
174
175
176#define virt_to_bus virt_to_phys
177#define bus_to_virt phys_to_virt
178
179
180
181
182
183extern void __iomem *ioremap_uc(resource_size_t offset, unsigned long size);
184#define ioremap_uc ioremap_uc
185extern void __iomem *ioremap_cache(resource_size_t offset, unsigned long size);
186#define ioremap_cache ioremap_cache
187extern void __iomem *ioremap_prot(resource_size_t offset, unsigned long size, unsigned long prot_val);
188#define ioremap_prot ioremap_prot
189extern void __iomem *ioremap_encrypted(resource_size_t phys_addr, unsigned long size);
190#define ioremap_encrypted ioremap_encrypted
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206void __iomem *ioremap(resource_size_t offset, unsigned long size);
207#define ioremap ioremap
208
209extern void iounmap(volatile void __iomem *addr);
210#define iounmap iounmap
211
212extern void set_iounmap_nonlazy(void);
213
214#ifdef __KERNEL__
215
216void memcpy_fromio(void *, const volatile void __iomem *, size_t);
217void memcpy_toio(volatile void __iomem *, const void *, size_t);
218void memset_io(volatile void __iomem *, int, size_t);
219
220#define memcpy_fromio memcpy_fromio
221#define memcpy_toio memcpy_toio
222#define memset_io memset_io
223
224#include <asm-generic/iomap.h>
225
226
227
228
229
230
231
232
233
234#define __ISA_IO_base ((char __iomem *)(PAGE_OFFSET))
235
236#endif
237
238extern void native_io_delay(void);
239
240extern int io_delay_type;
241extern void io_delay_init(void);
242
243#if defined(CONFIG_PARAVIRT)
244#include <asm/paravirt.h>
245#else
246
247static inline void slow_down_io(void)
248{
249 native_io_delay();
250#ifdef REALLY_SLOW_IO
251 native_io_delay();
252 native_io_delay();
253 native_io_delay();
254#endif
255}
256
257#endif
258
259#ifdef CONFIG_AMD_MEM_ENCRYPT
260#include <linux/jump_label.h>
261
262extern struct static_key_false sev_enable_key;
263static inline bool sev_key_active(void)
264{
265 return static_branch_unlikely(&sev_enable_key);
266}
267
268#else
269
270static inline bool sev_key_active(void) { return false; }
271
272#endif
273
274#define BUILDIO(bwl, bw, type) \
275static inline void out##bwl(unsigned type value, int port) \
276{ \
277 asm volatile("out" #bwl " %" #bw "0, %w1" \
278 : : "a"(value), "Nd"(port)); \
279} \
280 \
281static inline unsigned type in##bwl(int port) \
282{ \
283 unsigned type value; \
284 asm volatile("in" #bwl " %w1, %" #bw "0" \
285 : "=a"(value) : "Nd"(port)); \
286 return value; \
287} \
288 \
289static inline void out##bwl##_p(unsigned type value, int port) \
290{ \
291 out##bwl(value, port); \
292 slow_down_io(); \
293} \
294 \
295static inline unsigned type in##bwl##_p(int port) \
296{ \
297 unsigned type value = in##bwl(port); \
298 slow_down_io(); \
299 return value; \
300} \
301 \
302static inline void outs##bwl(int port, const void *addr, unsigned long count) \
303{ \
304 if (sev_key_active()) { \
305 unsigned type *value = (unsigned type *)addr; \
306 while (count) { \
307 out##bwl(*value, port); \
308 value++; \
309 count--; \
310 } \
311 } else { \
312 asm volatile("rep; outs" #bwl \
313 : "+S"(addr), "+c"(count) \
314 : "d"(port) : "memory"); \
315 } \
316} \
317 \
318static inline void ins##bwl(int port, void *addr, unsigned long count) \
319{ \
320 if (sev_key_active()) { \
321 unsigned type *value = (unsigned type *)addr; \
322 while (count) { \
323 *value = in##bwl(port); \
324 value++; \
325 count--; \
326 } \
327 } else { \
328 asm volatile("rep; ins" #bwl \
329 : "+D"(addr), "+c"(count) \
330 : "d"(port) : "memory"); \
331 } \
332}
333
334BUILDIO(b, b, char)
335BUILDIO(w, w, short)
336BUILDIO(l, , int)
337
338#define inb inb
339#define inw inw
340#define inl inl
341#define inb_p inb_p
342#define inw_p inw_p
343#define inl_p inl_p
344#define insb insb
345#define insw insw
346#define insl insl
347
348#define outb outb
349#define outw outw
350#define outl outl
351#define outb_p outb_p
352#define outw_p outw_p
353#define outl_p outl_p
354#define outsb outsb
355#define outsw outsw
356#define outsl outsl
357
358extern void *xlate_dev_mem_ptr(phys_addr_t phys);
359extern void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr);
360
361#define xlate_dev_mem_ptr xlate_dev_mem_ptr
362#define unxlate_dev_mem_ptr unxlate_dev_mem_ptr
363
364extern int ioremap_change_attr(unsigned long vaddr, unsigned long size,
365 enum page_cache_mode pcm);
366extern void __iomem *ioremap_wc(resource_size_t offset, unsigned long size);
367#define ioremap_wc ioremap_wc
368extern void __iomem *ioremap_wt(resource_size_t offset, unsigned long size);
369#define ioremap_wt ioremap_wt
370
371extern bool is_early_ioremap_ptep(pte_t *ptep);
372
373#define IO_SPACE_LIMIT 0xffff
374
375#include <asm-generic/io.h>
376#undef PCI_IOBASE
377
378#ifdef CONFIG_MTRR
379extern int __must_check arch_phys_wc_index(int handle);
380#define arch_phys_wc_index arch_phys_wc_index
381
382extern int __must_check arch_phys_wc_add(unsigned long base,
383 unsigned long size);
384extern void arch_phys_wc_del(int handle);
385#define arch_phys_wc_add arch_phys_wc_add
386#endif
387
388#ifdef CONFIG_X86_PAT
389extern int arch_io_reserve_memtype_wc(resource_size_t start, resource_size_t size);
390extern void arch_io_free_memtype_wc(resource_size_t start, resource_size_t size);
391#define arch_io_reserve_memtype_wc arch_io_reserve_memtype_wc
392#endif
393
394extern bool arch_memremap_can_ram_remap(resource_size_t offset,
395 unsigned long size,
396 unsigned long flags);
397#define arch_memremap_can_ram_remap arch_memremap_can_ram_remap
398
399extern bool phys_mem_access_encrypted(unsigned long phys_addr,
400 unsigned long size);
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415static inline void iosubmit_cmds512(void __iomem *dst, const void *src,
416 size_t count)
417{
418 const u8 *from = src;
419 const u8 *end = from + count * 64;
420
421 while (from < end) {
422 movdir64b(dst, from);
423 from += 64;
424 }
425}
426
427#endif
428