1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#ifndef __ASM_WORD_AT_A_TIME_H
17#define __ASM_WORD_AT_A_TIME_H
18
19#ifndef __AARCH64EB__
20
21#include <linux/kernel.h>
22
23struct word_at_a_time {
24 const unsigned long one_bits, high_bits;
25};
26
27#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) }
28
29static inline unsigned long has_zero(unsigned long a, unsigned long *bits,
30 const struct word_at_a_time *c)
31{
32 unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits;
33 *bits = mask;
34 return mask;
35}
36
37#define prep_zero_mask(a, bits, c) (bits)
38
39static inline unsigned long create_zero_mask(unsigned long bits)
40{
41 bits = (bits - 1) & ~bits;
42 return bits >> 7;
43}
44
45static inline unsigned long find_zero(unsigned long mask)
46{
47 return fls64(mask) >> 3;
48}
49
50#define zero_bytemask(mask) (mask)
51
52#else
53#include <asm-generic/word-at-a-time.h>
54#endif
55
56
57
58
59
60
61
62
63static inline unsigned long load_unaligned_zeropad(const void *addr)
64{
65 unsigned long ret, offset;
66
67
68 asm(
69 "1: ldr %0, %3\n"
70 "2:\n"
71 " .pushsection .fixup,\"ax\"\n"
72 " .align 2\n"
73 "3: and %1, %2, #0x7\n"
74 " bic %2, %2, #0x7\n"
75 " ldr %0, [%2]\n"
76 " lsl %1, %1, #0x3\n"
77#ifndef __AARCH64EB__
78 " lsr %0, %0, %1\n"
79#else
80 " lsl %0, %0, %1\n"
81#endif
82 " b 2b\n"
83 " .popsection\n"
84 " .pushsection __ex_table,\"a\"\n"
85 " .align 3\n"
86 " .quad 1b, 3b\n"
87 " .popsection"
88 : "=&r" (ret), "=&r" (offset)
89 : "r" (addr), "Q" (*(unsigned long *)addr));
90
91 return ret;
92}
93
94#endif
95