1
2
3
4
5
6
7
8
9#ifndef _ASM_BITOPS_H
10#define _ASM_BITOPS_H
11
12#ifndef _LINUX_BITOPS_H
13#error only <linux/bitops.h> can be included directly
14#endif
15
16#include <linux/compiler.h>
17#include <linux/irqflags.h>
18#include <linux/types.h>
19#include <asm/barrier.h>
20#include <asm/bug.h>
21#include <asm/byteorder.h>
22#include <asm/cpu-features.h>
23#include <asm/sgidefs.h>
24#include <asm/war.h>
25
26#if _MIPS_SZLONG == 32
27#define SZLONG_LOG 5
28#define SZLONG_MASK 31UL
29#define __LL "ll "
30#define __SC "sc "
31#define __INS "ins "
32#define __EXT "ext "
33#elif _MIPS_SZLONG == 64
34#define SZLONG_LOG 6
35#define SZLONG_MASK 63UL
36#define __LL "lld "
37#define __SC "scd "
38#define __INS "dins "
39#define __EXT "dext "
40#endif
41
42
43
44
45#define smp_mb__before_clear_bit() smp_llsc_mb()
46#define smp_mb__after_clear_bit() smp_llsc_mb()
47
48
49
50
51
52
53
54
55
56
57
58static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
59{
60 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
61 unsigned short bit = nr & SZLONG_MASK;
62 unsigned long temp;
63
64 if (kernel_uses_llsc && R10000_LLSC_WAR) {
65 __asm__ __volatile__(
66 " .set mips3 \n"
67 "1: " __LL "%0, %1 # set_bit \n"
68 " or %0, %2 \n"
69 " " __SC "%0, %1 \n"
70 " beqzl %0, 1b \n"
71 " .set mips0 \n"
72 : "=&r" (temp), "=m" (*m)
73 : "ir" (1UL << bit), "m" (*m));
74#ifdef CONFIG_CPU_MIPSR2
75 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
76 __asm__ __volatile__(
77 "1: " __LL "%0, %1 # set_bit \n"
78 " " __INS "%0, %4, %2, 1 \n"
79 " " __SC "%0, %1 \n"
80 " beqz %0, 2f \n"
81 " .subsection 2 \n"
82 "2: b 1b \n"
83 " .previous \n"
84 : "=&r" (temp), "=m" (*m)
85 : "ir" (bit), "m" (*m), "r" (~0));
86#endif
87 } else if (kernel_uses_llsc) {
88 __asm__ __volatile__(
89 " .set mips3 \n"
90 "1: " __LL "%0, %1 # set_bit \n"
91 " or %0, %2 \n"
92 " " __SC "%0, %1 \n"
93 " beqz %0, 2f \n"
94 " .subsection 2 \n"
95 "2: b 1b \n"
96 " .previous \n"
97 " .set mips0 \n"
98 : "=&r" (temp), "=m" (*m)
99 : "ir" (1UL << bit), "m" (*m));
100 } else {
101 volatile unsigned long *a = addr;
102 unsigned long mask;
103 unsigned long flags;
104
105 a += nr >> SZLONG_LOG;
106 mask = 1UL << bit;
107 raw_local_irq_save(flags);
108 *a |= mask;
109 raw_local_irq_restore(flags);
110 }
111}
112
113
114
115
116
117
118
119
120
121
122
123static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
124{
125 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
126 unsigned short bit = nr & SZLONG_MASK;
127 unsigned long temp;
128
129 if (kernel_uses_llsc && R10000_LLSC_WAR) {
130 __asm__ __volatile__(
131 " .set mips3 \n"
132 "1: " __LL "%0, %1 # clear_bit \n"
133 " and %0, %2 \n"
134 " " __SC "%0, %1 \n"
135 " beqzl %0, 1b \n"
136 " .set mips0 \n"
137 : "=&r" (temp), "=m" (*m)
138 : "ir" (~(1UL << bit)), "m" (*m));
139#ifdef CONFIG_CPU_MIPSR2
140 } else if (kernel_uses_llsc && __builtin_constant_p(bit)) {
141 __asm__ __volatile__(
142 "1: " __LL "%0, %1 # clear_bit \n"
143 " " __INS "%0, $0, %2, 1 \n"
144 " " __SC "%0, %1 \n"
145 " beqz %0, 2f \n"
146 " .subsection 2 \n"
147 "2: b 1b \n"
148 " .previous \n"
149 : "=&r" (temp), "=m" (*m)
150 : "ir" (bit), "m" (*m));
151#endif
152 } else if (kernel_uses_llsc) {
153 __asm__ __volatile__(
154 " .set mips3 \n"
155 "1: " __LL "%0, %1 # clear_bit \n"
156 " and %0, %2 \n"
157 " " __SC "%0, %1 \n"
158 " beqz %0, 2f \n"
159 " .subsection 2 \n"
160 "2: b 1b \n"
161 " .previous \n"
162 " .set mips0 \n"
163 : "=&r" (temp), "=m" (*m)
164 : "ir" (~(1UL << bit)), "m" (*m));
165 } else {
166 volatile unsigned long *a = addr;
167 unsigned long mask;
168 unsigned long flags;
169
170 a += nr >> SZLONG_LOG;
171 mask = 1UL << bit;
172 raw_local_irq_save(flags);
173 *a &= ~mask;
174 raw_local_irq_restore(flags);
175 }
176}
177
178
179
180
181
182
183
184
185
186static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
187{
188 smp_mb__before_clear_bit();
189 clear_bit(nr, addr);
190}
191
192
193
194
195
196
197
198
199
200
201static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
202{
203 unsigned short bit = nr & SZLONG_MASK;
204
205 if (kernel_uses_llsc && R10000_LLSC_WAR) {
206 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
207 unsigned long temp;
208
209 __asm__ __volatile__(
210 " .set mips3 \n"
211 "1: " __LL "%0, %1 # change_bit \n"
212 " xor %0, %2 \n"
213 " " __SC "%0, %1 \n"
214 " beqzl %0, 1b \n"
215 " .set mips0 \n"
216 : "=&r" (temp), "=m" (*m)
217 : "ir" (1UL << bit), "m" (*m));
218 } else if (kernel_uses_llsc) {
219 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
220 unsigned long temp;
221
222 __asm__ __volatile__(
223 " .set mips3 \n"
224 "1: " __LL "%0, %1 # change_bit \n"
225 " xor %0, %2 \n"
226 " " __SC "%0, %1 \n"
227 " beqz %0, 2f \n"
228 " .subsection 2 \n"
229 "2: b 1b \n"
230 " .previous \n"
231 " .set mips0 \n"
232 : "=&r" (temp), "=m" (*m)
233 : "ir" (1UL << bit), "m" (*m));
234 } else {
235 volatile unsigned long *a = addr;
236 unsigned long mask;
237 unsigned long flags;
238
239 a += nr >> SZLONG_LOG;
240 mask = 1UL << bit;
241 raw_local_irq_save(flags);
242 *a ^= mask;
243 raw_local_irq_restore(flags);
244 }
245}
246
247
248
249
250
251
252
253
254
255static inline int test_and_set_bit(unsigned long nr,
256 volatile unsigned long *addr)
257{
258 unsigned short bit = nr & SZLONG_MASK;
259 unsigned long res;
260
261 smp_llsc_mb();
262
263 if (kernel_uses_llsc && R10000_LLSC_WAR) {
264 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
265 unsigned long temp;
266
267 __asm__ __volatile__(
268 " .set mips3 \n"
269 "1: " __LL "%0, %1 # test_and_set_bit \n"
270 " or %2, %0, %3 \n"
271 " " __SC "%2, %1 \n"
272 " beqzl %2, 1b \n"
273 " and %2, %0, %3 \n"
274 " .set mips0 \n"
275 : "=&r" (temp), "=m" (*m), "=&r" (res)
276 : "r" (1UL << bit), "m" (*m)
277 : "memory");
278 } else if (kernel_uses_llsc) {
279 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
280 unsigned long temp;
281
282 __asm__ __volatile__(
283 " .set push \n"
284 " .set noreorder \n"
285 " .set mips3 \n"
286 "1: " __LL "%0, %1 # test_and_set_bit \n"
287 " or %2, %0, %3 \n"
288 " " __SC "%2, %1 \n"
289 " beqz %2, 2f \n"
290 " and %2, %0, %3 \n"
291 " .subsection 2 \n"
292 "2: b 1b \n"
293 " nop \n"
294 " .previous \n"
295 " .set pop \n"
296 : "=&r" (temp), "=m" (*m), "=&r" (res)
297 : "r" (1UL << bit), "m" (*m)
298 : "memory");
299 } else {
300 volatile unsigned long *a = addr;
301 unsigned long mask;
302 unsigned long flags;
303
304 a += nr >> SZLONG_LOG;
305 mask = 1UL << bit;
306 raw_local_irq_save(flags);
307 res = (mask & *a);
308 *a |= mask;
309 raw_local_irq_restore(flags);
310 }
311
312 smp_llsc_mb();
313
314 return res != 0;
315}
316
317
318
319
320
321
322
323
324
325static inline int test_and_set_bit_lock(unsigned long nr,
326 volatile unsigned long *addr)
327{
328 unsigned short bit = nr & SZLONG_MASK;
329 unsigned long res;
330
331 if (kernel_uses_llsc && R10000_LLSC_WAR) {
332 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
333 unsigned long temp;
334
335 __asm__ __volatile__(
336 " .set mips3 \n"
337 "1: " __LL "%0, %1 # test_and_set_bit \n"
338 " or %2, %0, %3 \n"
339 " " __SC "%2, %1 \n"
340 " beqzl %2, 1b \n"
341 " and %2, %0, %3 \n"
342 " .set mips0 \n"
343 : "=&r" (temp), "=m" (*m), "=&r" (res)
344 : "r" (1UL << bit), "m" (*m)
345 : "memory");
346 } else if (kernel_uses_llsc) {
347 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
348 unsigned long temp;
349
350 __asm__ __volatile__(
351 " .set push \n"
352 " .set noreorder \n"
353 " .set mips3 \n"
354 "1: " __LL "%0, %1 # test_and_set_bit \n"
355 " or %2, %0, %3 \n"
356 " " __SC "%2, %1 \n"
357 " beqz %2, 2f \n"
358 " and %2, %0, %3 \n"
359 " .subsection 2 \n"
360 "2: b 1b \n"
361 " nop \n"
362 " .previous \n"
363 " .set pop \n"
364 : "=&r" (temp), "=m" (*m), "=&r" (res)
365 : "r" (1UL << bit), "m" (*m)
366 : "memory");
367 } else {
368 volatile unsigned long *a = addr;
369 unsigned long mask;
370 unsigned long flags;
371
372 a += nr >> SZLONG_LOG;
373 mask = 1UL << bit;
374 raw_local_irq_save(flags);
375 res = (mask & *a);
376 *a |= mask;
377 raw_local_irq_restore(flags);
378 }
379
380 smp_llsc_mb();
381
382 return res != 0;
383}
384
385
386
387
388
389
390
391
392static inline int test_and_clear_bit(unsigned long nr,
393 volatile unsigned long *addr)
394{
395 unsigned short bit = nr & SZLONG_MASK;
396 unsigned long res;
397
398 smp_llsc_mb();
399
400 if (kernel_uses_llsc && R10000_LLSC_WAR) {
401 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
402 unsigned long temp;
403
404 __asm__ __volatile__(
405 " .set mips3 \n"
406 "1: " __LL "%0, %1 # test_and_clear_bit \n"
407 " or %2, %0, %3 \n"
408 " xor %2, %3 \n"
409 " " __SC "%2, %1 \n"
410 " beqzl %2, 1b \n"
411 " and %2, %0, %3 \n"
412 " .set mips0 \n"
413 : "=&r" (temp), "=m" (*m), "=&r" (res)
414 : "r" (1UL << bit), "m" (*m)
415 : "memory");
416#ifdef CONFIG_CPU_MIPSR2
417 } else if (kernel_uses_llsc && __builtin_constant_p(nr)) {
418 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
419 unsigned long temp;
420
421 __asm__ __volatile__(
422 "1: " __LL "%0, %1 # test_and_clear_bit \n"
423 " " __EXT "%2, %0, %3, 1 \n"
424 " " __INS "%0, $0, %3, 1 \n"
425 " " __SC "%0, %1 \n"
426 " beqz %0, 2f \n"
427 " .subsection 2 \n"
428 "2: b 1b \n"
429 " .previous \n"
430 : "=&r" (temp), "=m" (*m), "=&r" (res)
431 : "ir" (bit), "m" (*m)
432 : "memory");
433#endif
434 } else if (kernel_uses_llsc) {
435 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
436 unsigned long temp;
437
438 __asm__ __volatile__(
439 " .set push \n"
440 " .set noreorder \n"
441 " .set mips3 \n"
442 "1: " __LL "%0, %1 # test_and_clear_bit \n"
443 " or %2, %0, %3 \n"
444 " xor %2, %3 \n"
445 " " __SC "%2, %1 \n"
446 " beqz %2, 2f \n"
447 " and %2, %0, %3 \n"
448 " .subsection 2 \n"
449 "2: b 1b \n"
450 " nop \n"
451 " .previous \n"
452 " .set pop \n"
453 : "=&r" (temp), "=m" (*m), "=&r" (res)
454 : "r" (1UL << bit), "m" (*m)
455 : "memory");
456 } else {
457 volatile unsigned long *a = addr;
458 unsigned long mask;
459 unsigned long flags;
460
461 a += nr >> SZLONG_LOG;
462 mask = 1UL << bit;
463 raw_local_irq_save(flags);
464 res = (mask & *a);
465 *a &= ~mask;
466 raw_local_irq_restore(flags);
467 }
468
469 smp_llsc_mb();
470
471 return res != 0;
472}
473
474
475
476
477
478
479
480
481
482static inline int test_and_change_bit(unsigned long nr,
483 volatile unsigned long *addr)
484{
485 unsigned short bit = nr & SZLONG_MASK;
486 unsigned long res;
487
488 smp_llsc_mb();
489
490 if (kernel_uses_llsc && R10000_LLSC_WAR) {
491 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
492 unsigned long temp;
493
494 __asm__ __volatile__(
495 " .set mips3 \n"
496 "1: " __LL "%0, %1 # test_and_change_bit \n"
497 " xor %2, %0, %3 \n"
498 " " __SC "%2, %1 \n"
499 " beqzl %2, 1b \n"
500 " and %2, %0, %3 \n"
501 " .set mips0 \n"
502 : "=&r" (temp), "=m" (*m), "=&r" (res)
503 : "r" (1UL << bit), "m" (*m)
504 : "memory");
505 } else if (kernel_uses_llsc) {
506 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
507 unsigned long temp;
508
509 __asm__ __volatile__(
510 " .set push \n"
511 " .set noreorder \n"
512 " .set mips3 \n"
513 "1: " __LL "%0, %1 # test_and_change_bit \n"
514 " xor %2, %0, %3 \n"
515 " " __SC "\t%2, %1 \n"
516 " beqz %2, 2f \n"
517 " and %2, %0, %3 \n"
518 " .subsection 2 \n"
519 "2: b 1b \n"
520 " nop \n"
521 " .previous \n"
522 " .set pop \n"
523 : "=&r" (temp), "=m" (*m), "=&r" (res)
524 : "r" (1UL << bit), "m" (*m)
525 : "memory");
526 } else {
527 volatile unsigned long *a = addr;
528 unsigned long mask;
529 unsigned long flags;
530
531 a += nr >> SZLONG_LOG;
532 mask = 1UL << bit;
533 raw_local_irq_save(flags);
534 res = (mask & *a);
535 *a ^= mask;
536 raw_local_irq_restore(flags);
537 }
538
539 smp_llsc_mb();
540
541 return res != 0;
542}
543
544#include <asm-generic/bitops/non-atomic.h>
545
546
547
548
549
550
551
552
553
554
555static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
556{
557 smp_mb();
558 __clear_bit(nr, addr);
559}
560
561
562
563
564
565static inline unsigned long __fls(unsigned long word)
566{
567 int num;
568
569 if (BITS_PER_LONG == 32 &&
570 __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
571 __asm__(
572 " .set push \n"
573 " .set mips32 \n"
574 " clz %0, %1 \n"
575 " .set pop \n"
576 : "=r" (num)
577 : "r" (word));
578
579 return 31 - num;
580 }
581
582 if (BITS_PER_LONG == 64 &&
583 __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) {
584 __asm__(
585 " .set push \n"
586 " .set mips64 \n"
587 " dclz %0, %1 \n"
588 " .set pop \n"
589 : "=r" (num)
590 : "r" (word));
591
592 return 63 - num;
593 }
594
595 num = BITS_PER_LONG - 1;
596
597#if BITS_PER_LONG == 64
598 if (!(word & (~0ul << 32))) {
599 num -= 32;
600 word <<= 32;
601 }
602#endif
603 if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
604 num -= 16;
605 word <<= 16;
606 }
607 if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
608 num -= 8;
609 word <<= 8;
610 }
611 if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
612 num -= 4;
613 word <<= 4;
614 }
615 if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
616 num -= 2;
617 word <<= 2;
618 }
619 if (!(word & (~0ul << (BITS_PER_LONG-1))))
620 num -= 1;
621 return num;
622}
623
624
625
626
627
628
629
630
631static inline unsigned long __ffs(unsigned long word)
632{
633 return __fls(word & -word);
634}
635
636
637
638
639
640
641
642
643static inline int fls(int x)
644{
645 int r;
646
647 if (__builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) {
648 __asm__("clz %0, %1" : "=r" (x) : "r" (x));
649
650 return 32 - x;
651 }
652
653 r = 32;
654 if (!x)
655 return 0;
656 if (!(x & 0xffff0000u)) {
657 x <<= 16;
658 r -= 16;
659 }
660 if (!(x & 0xff000000u)) {
661 x <<= 8;
662 r -= 8;
663 }
664 if (!(x & 0xf0000000u)) {
665 x <<= 4;
666 r -= 4;
667 }
668 if (!(x & 0xc0000000u)) {
669 x <<= 2;
670 r -= 2;
671 }
672 if (!(x & 0x80000000u)) {
673 x <<= 1;
674 r -= 1;
675 }
676 return r;
677}
678
679#include <asm-generic/bitops/fls64.h>
680
681
682
683
684
685
686
687
688
689static inline int ffs(int word)
690{
691 if (!word)
692 return 0;
693
694 return fls(word & -word);
695}
696
697#include <asm-generic/bitops/ffz.h>
698#include <asm-generic/bitops/find.h>
699
700#ifdef __KERNEL__
701
702#include <asm-generic/bitops/sched.h>
703#include <asm-generic/bitops/hweight.h>
704#include <asm-generic/bitops/ext2-non-atomic.h>
705#include <asm-generic/bitops/ext2-atomic.h>
706#include <asm-generic/bitops/minix.h>
707
708#endif
709
710#endif
711
