LXR linux/arch/arc/include/asm/atomic.h

   1/*
   2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   3 *
   4 * This program is free software; you can redistribute it and/or modify
   5 * it under the terms of the GNU General Public License version 2 as
   6 * published by the Free Software Foundation.
   7 */
   8
   9#ifndef _ASM_ARC_ATOMIC_H
  10#define _ASM_ARC_ATOMIC_H
  11
  12#ifndef __ASSEMBLY__
  13
  14#include <linux/types.h>
  15#include <linux/compiler.h>
  16#include <asm/cmpxchg.h>
  17#include <asm/barrier.h>
  18#include <asm/smp.h>
  19
  20#ifndef CONFIG_ARC_PLAT_EZNPS
  21
  22#define atomic_read(v)  READ_ONCE((v)->counter)
  23#define ATOMIC_INIT(i)  { (i) }
  24
  25#ifdef CONFIG_ARC_HAS_LLSC
  26
  27#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
  28
  29#define ATOMIC_OP(op, c_op, asm_op)                                     \
  30static inline void atomic_##op(int i, atomic_t *v)                      \
  31{                                                                       \
  32        unsigned int val;                                               \
  33                                                                        \
  34        __asm__ __volatile__(                                           \
  35        "1:     llock   %[val], [%[ctr]]                \n"             \
  36        "       " #asm_op " %[val], %[val], %[i]        \n"             \
  37        "       scond   %[val], [%[ctr]]                \n"             \
  38        "       bnz     1b                              \n"             \
  39        : [val] "=&r"   (val) /* Early clobber to prevent reg reuse */  \
  40        : [ctr] "r"     (&v->counter), /* Not "m": llock only supports reg direct addr mode */  \
  41          [i]   "ir"    (i)                                             \
  42        : "cc");                                                        \
  43}                                                                       \
  44
  45#define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
  46static inline int atomic_##op##_return(int i, atomic_t *v)              \
  47{                                                                       \
  48        unsigned int val;                                               \
  49                                                                        \
  50        /*                                                              \
  51         * Explicit full memory barrier needed before/after as          \
  52         * LLOCK/SCOND thmeselves don't provide any such semantics      \
  53         */                                                             \
  54        smp_mb();                                                       \
  55                                                                        \
  56        __asm__ __volatile__(                                           \
  57        "1:     llock   %[val], [%[ctr]]                \n"             \
  58        "       " #asm_op " %[val], %[val], %[i]        \n"             \
  59        "       scond   %[val], [%[ctr]]                \n"             \
  60        "       bnz     1b                              \n"             \
  61        : [val] "=&r"   (val)                                           \
  62        : [ctr] "r"     (&v->counter),                                  \
  63          [i]   "ir"    (i)                                             \
  64        : "cc");                                                        \
  65                                                                        \
  66        smp_mb();                                                       \
  67                                                                        \
  68        return val;                                                     \
  69}
  70
  71#define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
  72static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
  73{                                                                       \
  74        unsigned int val, orig;                                         \
  75                                                                        \
  76        /*                                                              \
  77         * Explicit full memory barrier needed before/after as          \
  78         * LLOCK/SCOND thmeselves don't provide any such semantics      \
  79         */                                                             \
  80        smp_mb();                                                       \
  81                                                                        \
  82        __asm__ __volatile__(                                           \
  83        "1:     llock   %[orig], [%[ctr]]               \n"             \
  84        "       " #asm_op " %[val], %[orig], %[i]       \n"             \
  85        "       scond   %[val], [%[ctr]]                \n"             \
  86        "                                               \n"             \
  87        : [val] "=&r"   (val),                                          \
  88          [orig] "=&r" (orig)                                           \
  89        : [ctr] "r"     (&v->counter),                                  \
  90          [i]   "ir"    (i)                                             \
  91        : "cc");                                                        \
  92                                                                        \
  93        smp_mb();                                                       \
  94                                                                        \
  95        return orig;                                                    \
  96}
  97
  98#else   /* !CONFIG_ARC_HAS_LLSC */
  99
 100#ifndef CONFIG_SMP
 101
 102 /* violating atomic_xxx API locking protocol in UP for optimization sake */
 103#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
 104
 105#else
 106
 107static inline void atomic_set(atomic_t *v, int i)
 108{
 109        /*
 110         * Independent of hardware support, all of the atomic_xxx() APIs need
 111         * to follow the same locking rules to make sure that a "hardware"
 112         * atomic insn (e.g. LD) doesn't clobber an "emulated" atomic insn
 113         * sequence
 114         *
 115         * Thus atomic_set() despite being 1 insn (and seemingly atomic)
 116         * requires the locking.
 117         */
 118        unsigned long flags;
 119
 120        atomic_ops_lock(flags);
 121        WRITE_ONCE(v->counter, i);
 122        atomic_ops_unlock(flags);
 123}
 124
 125#endif
 126
 127/*
 128 * Non hardware assisted Atomic-R-M-W
 129 * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
 130 */
 131
 132#define ATOMIC_OP(op, c_op, asm_op)                                     \
 133static inline void atomic_##op(int i, atomic_t *v)                      \
 134{                                                                       \
 135        unsigned long flags;                                            \
 136                                                                        \
 137        atomic_ops_lock(flags);                                         \
 138        v->counter c_op i;                                              \
 139        atomic_ops_unlock(flags);                                       \
 140}
 141
 142#define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 143static inline int atomic_##op##_return(int i, atomic_t *v)              \
 144{                                                                       \
 145        unsigned long flags;                                            \
 146        unsigned long temp;                                             \
 147                                                                        \
 148        /*                                                              \
 149         * spin lock/unlock provides the needed smp_mb() before/after   \
 150         */                                                             \
 151        atomic_ops_lock(flags);                                         \
 152        temp = v->counter;                                              \
 153        temp c_op i;                                                    \
 154        v->counter = temp;                                              \
 155        atomic_ops_unlock(flags);                                       \
 156                                                                        \
 157        return temp;                                                    \
 158}
 159
 160#define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
 161static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
 162{                                                                       \
 163        unsigned long flags;                                            \
 164        unsigned long orig;                                             \
 165                                                                        \
 166        /*                                                              \
 167         * spin lock/unlock provides the needed smp_mb() before/after   \
 168         */                                                             \
 169        atomic_ops_lock(flags);                                         \
 170        orig = v->counter;                                              \
 171        v->counter c_op i;                                              \
 172        atomic_ops_unlock(flags);                                       \
 173                                                                        \
 174        return orig;                                                    \
 175}
 176
 177#endif /* !CONFIG_ARC_HAS_LLSC */
 178
 179#define ATOMIC_OPS(op, c_op, asm_op)                                    \
 180        ATOMIC_OP(op, c_op, asm_op)                                     \
 181        ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 182        ATOMIC_FETCH_OP(op, c_op, asm_op)
 183
 184ATOMIC_OPS(add, +=, add)
 185ATOMIC_OPS(sub, -=, sub)
 186
 187#define atomic_andnot atomic_andnot
 188
 189#undef ATOMIC_OPS
 190#define ATOMIC_OPS(op, c_op, asm_op)                                    \
 191        ATOMIC_OP(op, c_op, asm_op)                                     \
 192        ATOMIC_FETCH_OP(op, c_op, asm_op)
 193
 194ATOMIC_OPS(and, &=, and)
 195ATOMIC_OPS(andnot, &= ~, bic)
 196ATOMIC_OPS(or, |=, or)
 197ATOMIC_OPS(xor, ^=, xor)
 198
 199#else /* CONFIG_ARC_PLAT_EZNPS */
 200
 201static inline int atomic_read(const atomic_t *v)
 202{
 203        int temp;
 204
 205        __asm__ __volatile__(
 206        "       ld.di %0, [%1]"
 207        : "=r"(temp)
 208        : "r"(&v->counter)
 209        : "memory");
 210        return temp;
 211}
 212
 213static inline void atomic_set(atomic_t *v, int i)
 214{
 215        __asm__ __volatile__(
 216        "       st.di %0,[%1]"
 217        :
 218        : "r"(i), "r"(&v->counter)
 219        : "memory");
 220}
 221
 222#define ATOMIC_OP(op, c_op, asm_op)                                     \
 223static inline void atomic_##op(int i, atomic_t *v)                      \
 224{                                                                       \
 225        __asm__ __volatile__(                                           \
 226        "       mov r2, %0\n"                                           \
 227        "       mov r3, %1\n"                                           \
 228        "       .word %2\n"                                             \
 229        :                                                               \
 230        : "r"(i), "r"(&v->counter), "i"(asm_op)                         \
 231        : "r2", "r3", "memory");                                        \
 232}                                                                       \
 233
 234#define ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 235static inline int atomic_##op##_return(int i, atomic_t *v)              \
 236{                                                                       \
 237        unsigned int temp = i;                                          \
 238                                                                        \
 239        /* Explicit full memory barrier needed before/after */          \
 240        smp_mb();                                                       \
 241                                                                        \
 242        __asm__ __volatile__(                                           \
 243        "       mov r2, %0\n"                                           \
 244        "       mov r3, %1\n"                                           \
 245        "       .word %2\n"                                             \
 246        "       mov %0, r2"                                             \
 247        : "+r"(temp)                                                    \
 248        : "r"(&v->counter), "i"(asm_op)                                 \
 249        : "r2", "r3", "memory");                                        \
 250                                                                        \
 251        smp_mb();                                                       \
 252                                                                        \
 253        temp c_op i;                                                    \
 254                                                                        \
 255        return temp;                                                    \
 256}
 257
 258#define ATOMIC_FETCH_OP(op, c_op, asm_op)                               \
 259static inline int atomic_fetch_##op(int i, atomic_t *v)                 \
 260{                                                                       \
 261        unsigned int temp = i;                                          \
 262                                                                        \
 263        /* Explicit full memory barrier needed before/after */          \
 264        smp_mb();                                                       \
 265                                                                        \
 266        __asm__ __volatile__(                                           \
 267        "       mov r2, %0\n"                                           \
 268        "       mov r3, %1\n"                                           \
 269        "       .word %2\n"                                             \
 270        "       mov %0, r2"                                             \
 271        : "+r"(temp)                                                    \
 272        : "r"(&v->counter), "i"(asm_op)                                 \
 273        : "r2", "r3", "memory");                                        \
 274                                                                        \
 275        smp_mb();                                                       \
 276                                                                        \
 277        return temp;                                                    \
 278}
 279
 280#define ATOMIC_OPS(op, c_op, asm_op)                                    \
 281        ATOMIC_OP(op, c_op, asm_op)                                     \
 282        ATOMIC_OP_RETURN(op, c_op, asm_op)                              \
 283        ATOMIC_FETCH_OP(op, c_op, asm_op)
 284
 285ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3)
 286#define atomic_sub(i, v) atomic_add(-(i), (v))
 287#define atomic_sub_return(i, v) atomic_add_return(-(i), (v))
 288#define atomic_fetch_sub(i, v) atomic_fetch_add(-(i), (v))
 289
 290#undef ATOMIC_OPS
 291#define ATOMIC_OPS(op, c_op, asm_op)                                    \
 292        ATOMIC_OP(op, c_op, asm_op)                                     \
 293        ATOMIC_FETCH_OP(op, c_op, asm_op)
 294
 295ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3)
 296#define atomic_andnot(mask, v) atomic_and(~(mask), (v))
 297#define atomic_fetch_andnot(mask, v) atomic_fetch_and(~(mask), (v))
 298ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3)
 299ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)
 300
 301#endif /* CONFIG_ARC_PLAT_EZNPS */
 302
 303#undef ATOMIC_OPS
 304#undef ATOMIC_FETCH_OP
 305#undef ATOMIC_OP_RETURN
 306#undef ATOMIC_OP
 307
 308/**
 309 * __atomic_add_unless - add unless the number is a given value
 310 * @v: pointer of type atomic_t
 311 * @a: the amount to add to v...
 312 * @u: ...unless v is equal to u.
 313 *
 314 * Atomically adds @a to @v, so long as it was not @u.
 315 * Returns the old value of @v
 316 */
 317#define __atomic_add_unless(v, a, u)                                    \
 318({                                                                      \
 319        int c, old;                                                     \
 320                                                                        \
 321        /*                                                              \
 322         * Explicit full memory barrier needed before/after as          \
 323         * LLOCK/SCOND thmeselves don't provide any such semantics      \
 324         */                                                             \
 325        smp_mb();                                                       \
 326                                                                        \
 327        c = atomic_read(v);                                             \
 328        while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c)\
 329                c = old;                                                \
 330                                                                        \
 331        smp_mb();                                                       \
 332                                                                        \
 333        c;                                                              \
 334})
 335
 336#define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
 337
 338#define atomic_inc(v)                   atomic_add(1, v)
 339#define atomic_dec(v)                   atomic_sub(1, v)
 340
 341#define atomic_inc_and_test(v)          (atomic_add_return(1, v) == 0)
 342#define atomic_dec_and_test(v)          (atomic_sub_return(1, v) == 0)
 343#define atomic_inc_return(v)            atomic_add_return(1, (v))
 344#define atomic_dec_return(v)            atomic_sub_return(1, (v))
 345#define atomic_sub_and_test(i, v)       (atomic_sub_return(i, v) == 0)
 346
 347#define atomic_add_negative(i, v)       (atomic_add_return(i, v) < 0)
 348
 349
 350#ifdef CONFIG_GENERIC_ATOMIC64
 351
 352#include <asm-generic/atomic64.h>
 353
 354#else   /* Kconfig ensures this is only enabled with needed h/w assist */
 355
 356/*
 357 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD)
 358 *  - The address HAS to be 64-bit aligned
 359 *  - There are 2 semantics involved here:
 360 *    = exclusive implies no interim update between load/store to same addr
 361 *    = both words are observed/updated together: this is guaranteed even
 362 *      for regular 64-bit load (LDD) / store (STD). Thus atomic64_set()
 363 *      is NOT required to use LLOCKD+SCONDD, STD suffices
 364 */
 365
 366typedef struct {
 367        aligned_u64 counter;
 368} atomic64_t;
 369
 370#define ATOMIC64_INIT(a) { (a) }
 371
 372static inline long long atomic64_read(const atomic64_t *v)
 373{
 374        unsigned long long val;
 375
 376        __asm__ __volatile__(
 377        "       ldd   %0, [%1]  \n"
 378        : "=r"(val)
 379        : "r"(&v->counter));
 380
 381        return val;
 382}
 383
 384static inline void atomic64_set(atomic64_t *v, long long a)
 385{
 386        /*
 387         * This could have been a simple assignment in "C" but would need
 388         * explicit volatile. Otherwise gcc optimizers could elide the store
 389         * which borked atomic64 self-test
 390         * In the inline asm version, memory clobber needed for exact same
 391         * reason, to tell gcc about the store.
 392         *
 393         * This however is not needed for sibling atomic64_add() etc since both
 394         * load/store are explicitly done in inline asm. As long as API is used
 395         * for each access, gcc has no way to optimize away any load/store
 396         */
 397        __asm__ __volatile__(
 398        "       std   %0, [%1]  \n"
 399        :
 400        : "r"(a), "r"(&v->counter)
 401        : "memory");
 402}
 403
 404#define ATOMIC64_OP(op, op1, op2)                                       \
 405static inline void atomic64_##op(long long a, atomic64_t *v)            \
 406{                                                                       \
 407        unsigned long long val;                                         \
 408                                                                        \
 409        __asm__ __volatile__(                                           \
 410        "1:                             \n"                             \
 411        "       llockd  %0, [%1]        \n"                             \
 412        "       " #op1 " %L0, %L0, %L2  \n"                             \
 413        "       " #op2 " %H0, %H0, %H2  \n"                             \
 414        "       scondd   %0, [%1]       \n"                             \
 415        "       bnz     1b              \n"                             \
 416        : "=&r"(val)                                                    \
 417        : "r"(&v->counter), "ir"(a)                                     \
 418        : "cc");                                                \
 419}                                                                       \
 420
 421#define ATOMIC64_OP_RETURN(op, op1, op2)                                \
 422static inline long long atomic64_##op##_return(long long a, atomic64_t *v)      \
 423{                                                                       \
 424        unsigned long long val;                                         \
 425                                                                        \
 426        smp_mb();                                                       \
 427                                                                        \
 428        __asm__ __volatile__(                                           \
 429        "1:                             \n"                             \
 430        "       llockd   %0, [%1]       \n"                             \
 431        "       " #op1 " %L0, %L0, %L2  \n"                             \
 432        "       " #op2 " %H0, %H0, %H2  \n"                             \
 433        "       scondd   %0, [%1]       \n"                             \
 434        "       bnz     1b              \n"                             \
 435        : [val] "=&r"(val)                                              \
 436        : "r"(&v->counter), "ir"(a)                                     \
 437        : "cc");        /* memory clobber comes from smp_mb() */        \
 438                                                                        \
 439        smp_mb();                                                       \
 440                                                                        \
 441        return val;                                                     \
 442}
 443
 444#define ATOMIC64_FETCH_OP(op, op1, op2)                                 \
 445static inline long long atomic64_fetch_##op(long long a, atomic64_t *v) \
 446{                                                                       \
 447        unsigned long long val, orig;                                   \
 448                                                                        \
 449        smp_mb();                                                       \
 450                                                                        \
 451        __asm__ __volatile__(                                           \
 452        "1:                             \n"                             \
 453        "       llockd   %0, [%2]       \n"                             \
 454        "       " #op1 " %L1, %L0, %L3  \n"                             \
 455        "       " #op2 " %H1, %H0, %H3  \n"                             \
 456        "       scondd   %1, [%2]       \n"                             \
 457        "       bnz     1b              \n"                             \
 458        : "=&r"(orig), "=&r"(val)                                       \
 459        : "r"(&v->counter), "ir"(a)                                     \
 460        : "cc");        /* memory clobber comes from smp_mb() */        \
 461                                                                        \
 462        smp_mb();                                                       \
 463                                                                        \
 464        return orig;                                                    \
 465}
 466
 467#define ATOMIC64_OPS(op, op1, op2)                                      \
 468        ATOMIC64_OP(op, op1, op2)                                       \
 469        ATOMIC64_OP_RETURN(op, op1, op2)                                \
 470        ATOMIC64_FETCH_OP(op, op1, op2)
 471
 472#define atomic64_andnot atomic64_andnot
 473
 474ATOMIC64_OPS(add, add.f, adc)
 475ATOMIC64_OPS(sub, sub.f, sbc)
 476ATOMIC64_OPS(and, and, and)
 477ATOMIC64_OPS(andnot, bic, bic)
 478ATOMIC64_OPS(or, or, or)
 479ATOMIC64_OPS(xor, xor, xor)
 480
 481#undef ATOMIC64_OPS
 482#undef ATOMIC64_FETCH_OP
 483#undef ATOMIC64_OP_RETURN
 484#undef ATOMIC64_OP
 485
 486static inline long long
 487atomic64_cmpxchg(atomic64_t *ptr, long long expected, long long new)
 488{
 489        long long prev;
 490
 491        smp_mb();
 492
 493        __asm__ __volatile__(
 494        "1:     llockd  %0, [%1]        \n"
 495        "       brne    %L0, %L2, 2f    \n"
 496        "       brne    %H0, %H2, 2f    \n"
 497        "       scondd  %3, [%1]        \n"
 498        "       bnz     1b              \n"
 499        "2:                             \n"
 500        : "=&r"(prev)
 501        : "r"(ptr), "ir"(expected), "r"(new)
 502        : "cc");        /* memory clobber comes from smp_mb() */
 503
 504        smp_mb();
 505
 506        return prev;
 507}
 508
 509static inline long long atomic64_xchg(atomic64_t *ptr, long long new)
 510{
 511        long long prev;
 512
 513        smp_mb();
 514
 515        __asm__ __volatile__(
 516        "1:     llockd  %0, [%1]        \n"
 517        "       scondd  %2, [%1]        \n"
 518        "       bnz     1b              \n"
 519        "2:                             \n"
 520        : "=&r"(prev)
 521        : "r"(ptr), "r"(new)
 522        : "cc");        /* memory clobber comes from smp_mb() */
 523
 524        smp_mb();
 525
 526        return prev;
 527}
 528
 529/**
 530 * atomic64_dec_if_positive - decrement by 1 if old value positive
 531 * @v: pointer of type atomic64_t
 532 *
 533 * The function returns the old value of *v minus 1, even if
 534 * the atomic variable, v, was not decremented.
 535 */
 536
 537static inline long long atomic64_dec_if_positive(atomic64_t *v)
 538{
 539        long long val;
 540
 541        smp_mb();
 542
 543        __asm__ __volatile__(
 544        "1:     llockd  %0, [%1]        \n"
 545        "       sub.f   %L0, %L0, 1     # w0 - 1, set C on borrow\n"
 546        "       sub.c   %H0, %H0, 1     # if C set, w1 - 1\n"
 547        "       brlt    %H0, 0, 2f      \n"
 548        "       scondd  %0, [%1]        \n"
 549        "       bnz     1b              \n"
 550        "2:                             \n"
 551        : "=&r"(val)
 552        : "r"(&v->counter)
 553        : "cc");        /* memory clobber comes from smp_mb() */
 554
 555        smp_mb();
 556
 557        return val;
 558}
 559
 560/**
 561 * atomic64_add_unless - add unless the number is a given value
 562 * @v: pointer of type atomic64_t
 563 * @a: the amount to add to v...
 564 * @u: ...unless v is equal to u.
 565 *
 566 * if (v != u) { v += a; ret = 1} else {ret = 0}
 567 * Returns 1 iff @v was not @u (i.e. if add actually happened)
 568 */
 569static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u)
 570{
 571        long long val;
 572        int op_done;
 573
 574        smp_mb();
 575
 576        __asm__ __volatile__(
 577        "1:     llockd  %0, [%2]        \n"
 578        "       mov     %1, 1           \n"
 579        "       brne    %L0, %L4, 2f    # continue to add since v != u \n"
 580        "       breq.d  %H0, %H4, 3f    # return since v == u \n"
 581        "       mov     %1, 0           \n"
 582        "2:                             \n"
 583        "       add.f   %L0, %L0, %L3   \n"
 584        "       adc     %H0, %H0, %H3   \n"
 585        "       scondd  %0, [%2]        \n"
 586        "       bnz     1b              \n"
 587        "3:                             \n"
 588        : "=&r"(val), "=&r" (op_done)
 589        : "r"(&v->counter), "r"(a), "r"(u)
 590        : "cc");        /* memory clobber comes from smp_mb() */
 591
 592        smp_mb();
 593
 594        return op_done;
 595}
 596
 597#define atomic64_add_negative(a, v)     (atomic64_add_return((a), (v)) < 0)
 598#define atomic64_inc(v)                 atomic64_add(1LL, (v))
 599#define atomic64_inc_return(v)          atomic64_add_return(1LL, (v))
 600#define atomic64_inc_and_test(v)        (atomic64_inc_return(v) == 0)
 601#define atomic64_sub_and_test(a, v)     (atomic64_sub_return((a), (v)) == 0)
 602#define atomic64_dec(v)                 atomic64_sub(1LL, (v))
 603#define atomic64_dec_return(v)          atomic64_sub_return(1LL, (v))
 604#define atomic64_dec_and_test(v)        (atomic64_dec_return((v)) == 0)
 605#define atomic64_inc_not_zero(v)        atomic64_add_unless((v), 1LL, 0LL)
 606
 607#endif  /* !CONFIG_GENERIC_ATOMIC64 */
 608
 609#endif  /* !__ASSEMBLY__ */
 610
 611#endif
 612