linux/arch/ia64/kernel/unwind.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 1999-2004 Hewlett-Packard Co
   4 *      David Mosberger-Tang <davidm@hpl.hp.com>
   5 * Copyright (C) 2003 Fenghua Yu <fenghua.yu@intel.com>
   6 *      - Change pt_regs_off() to make it less dependent on pt_regs structure.
   7 */
   8/*
   9 * This file implements call frame unwind support for the Linux
  10 * kernel.  Parsing and processing the unwind information is
  11 * time-consuming, so this implementation translates the unwind
  12 * descriptors into unwind scripts.  These scripts are very simple
  13 * (basically a sequence of assignments) and efficient to execute.
  14 * They are cached for later re-use.  Each script is specific for a
  15 * given instruction pointer address and the set of predicate values
  16 * that the script depends on (most unwind descriptors are
  17 * unconditional and scripts often do not depend on predicates at
  18 * all).  This code is based on the unwind conventions described in
  19 * the "IA-64 Software Conventions and Runtime Architecture" manual.
  20 *
  21 * SMP conventions:
  22 *      o updates to the global unwind data (in structure "unw") are serialized
  23 *        by the unw.lock spinlock
  24 *      o each unwind script has its own read-write lock; a thread must acquire
  25 *        a read lock before executing a script and must acquire a write lock
  26 *        before modifying a script
  27 *      o if both the unw.lock spinlock and a script's read-write lock must be
  28 *        acquired, then the read-write lock must be acquired first.
  29 */
  30#include <linux/module.h>
  31#include <linux/bootmem.h>
  32#include <linux/elf.h>
  33#include <linux/kernel.h>
  34#include <linux/sched.h>
  35#include <linux/slab.h>
  36
  37#include <asm/unwind.h>
  38
  39#include <asm/delay.h>
  40#include <asm/page.h>
  41#include <asm/ptrace.h>
  42#include <asm/ptrace_offsets.h>
  43#include <asm/rse.h>
  44#include <asm/sections.h>
  45#include <linux/uaccess.h>
  46
  47#include "entry.h"
  48#include "unwind_i.h"
  49
  50#define UNW_LOG_CACHE_SIZE      7       /* each unw_script is ~256 bytes in size */
  51#define UNW_CACHE_SIZE          (1 << UNW_LOG_CACHE_SIZE)
  52
  53#define UNW_LOG_HASH_SIZE       (UNW_LOG_CACHE_SIZE + 1)
  54#define UNW_HASH_SIZE           (1 << UNW_LOG_HASH_SIZE)
  55
  56#define UNW_STATS       0       /* WARNING: this disabled interrupts for long time-spans!! */
  57
  58#ifdef UNW_DEBUG
  59  static unsigned int unw_debug_level = UNW_DEBUG;
  60#  define UNW_DEBUG_ON(n)       unw_debug_level >= n
  61   /* Do not code a printk level, not all debug lines end in newline */
  62#  define UNW_DPRINT(n, ...)  if (UNW_DEBUG_ON(n)) printk(__VA_ARGS__)
  63#  undef inline
  64#  define inline
  65#else /* !UNW_DEBUG */
  66#  define UNW_DEBUG_ON(n)  0
  67#  define UNW_DPRINT(n, ...)
  68#endif /* UNW_DEBUG */
  69
  70#if UNW_STATS
  71# define STAT(x...)     x
  72#else
  73# define STAT(x...)
  74#endif
  75
  76#define alloc_reg_state()       kmalloc(sizeof(struct unw_reg_state), GFP_ATOMIC)
  77#define free_reg_state(usr)     kfree(usr)
  78#define alloc_labeled_state()   kmalloc(sizeof(struct unw_labeled_state), GFP_ATOMIC)
  79#define free_labeled_state(usr) kfree(usr)
  80
  81typedef unsigned long unw_word;
  82typedef unsigned char unw_hash_index_t;
  83
  84static struct {
  85        spinlock_t lock;                        /* spinlock for unwind data */
  86
  87        /* list of unwind tables (one per load-module) */
  88        struct unw_table *tables;
  89
  90        unsigned long r0;                       /* constant 0 for r0 */
  91
  92        /* table of registers that prologues can save (and order in which they're saved): */
  93        const unsigned char save_order[8];
  94
  95        /* maps a preserved register index (preg_index) to corresponding switch_stack offset: */
  96        unsigned short sw_off[sizeof(struct unw_frame_info) / 8];
  97
  98        unsigned short lru_head;                /* index of lead-recently used script */
  99        unsigned short lru_tail;                /* index of most-recently used script */
 100
 101        /* index into unw_frame_info for preserved register i */
 102        unsigned short preg_index[UNW_NUM_REGS];
 103
 104        short pt_regs_offsets[32];
 105
 106        /* unwind table for the kernel: */
 107        struct unw_table kernel_table;
 108
 109        /* unwind table describing the gate page (kernel code that is mapped into user space): */
 110        size_t gate_table_size;
 111        unsigned long *gate_table;
 112
 113        /* hash table that maps instruction pointer to script index: */
 114        unsigned short hash[UNW_HASH_SIZE];
 115
 116        /* script cache: */
 117        struct unw_script cache[UNW_CACHE_SIZE];
 118
 119# ifdef UNW_DEBUG
 120        const char *preg_name[UNW_NUM_REGS];
 121# endif
 122# if UNW_STATS
 123        struct {
 124                struct {
 125                        int lookups;
 126                        int hinted_hits;
 127                        int normal_hits;
 128                        int collision_chain_traversals;
 129                } cache;
 130                struct {
 131                        unsigned long build_time;
 132                        unsigned long run_time;
 133                        unsigned long parse_time;
 134                        int builds;
 135                        int news;
 136                        int collisions;
 137                        int runs;
 138                } script;
 139                struct {
 140                        unsigned long init_time;
 141                        unsigned long unwind_time;
 142                        int inits;
 143                        int unwinds;
 144                } api;
 145        } stat;
 146# endif
 147} unw = {
 148        .tables = &unw.kernel_table,
 149        .lock = __SPIN_LOCK_UNLOCKED(unw.lock),
 150        .save_order = {
 151                UNW_REG_RP, UNW_REG_PFS, UNW_REG_PSP, UNW_REG_PR,
 152                UNW_REG_UNAT, UNW_REG_LC, UNW_REG_FPSR, UNW_REG_PRI_UNAT_GR
 153        },
 154        .preg_index = {
 155                offsetof(struct unw_frame_info, pri_unat_loc)/8,        /* PRI_UNAT_GR */
 156                offsetof(struct unw_frame_info, pri_unat_loc)/8,        /* PRI_UNAT_MEM */
 157                offsetof(struct unw_frame_info, bsp_loc)/8,
 158                offsetof(struct unw_frame_info, bspstore_loc)/8,
 159                offsetof(struct unw_frame_info, pfs_loc)/8,
 160                offsetof(struct unw_frame_info, rnat_loc)/8,
 161                offsetof(struct unw_frame_info, psp)/8,
 162                offsetof(struct unw_frame_info, rp_loc)/8,
 163                offsetof(struct unw_frame_info, r4)/8,
 164                offsetof(struct unw_frame_info, r5)/8,
 165                offsetof(struct unw_frame_info, r6)/8,
 166                offsetof(struct unw_frame_info, r7)/8,
 167                offsetof(struct unw_frame_info, unat_loc)/8,
 168                offsetof(struct unw_frame_info, pr_loc)/8,
 169                offsetof(struct unw_frame_info, lc_loc)/8,
 170                offsetof(struct unw_frame_info, fpsr_loc)/8,
 171                offsetof(struct unw_frame_info, b1_loc)/8,
 172                offsetof(struct unw_frame_info, b2_loc)/8,
 173                offsetof(struct unw_frame_info, b3_loc)/8,
 174                offsetof(struct unw_frame_info, b4_loc)/8,
 175                offsetof(struct unw_frame_info, b5_loc)/8,
 176                offsetof(struct unw_frame_info, f2_loc)/8,
 177                offsetof(struct unw_frame_info, f3_loc)/8,
 178                offsetof(struct unw_frame_info, f4_loc)/8,
 179                offsetof(struct unw_frame_info, f5_loc)/8,
 180                offsetof(struct unw_frame_info, fr_loc[16 - 16])/8,
 181                offsetof(struct unw_frame_info, fr_loc[17 - 16])/8,
 182                offsetof(struct unw_frame_info, fr_loc[18 - 16])/8,
 183                offsetof(struct unw_frame_info, fr_loc[19 - 16])/8,
 184                offsetof(struct unw_frame_info, fr_loc[20 - 16])/8,
 185                offsetof(struct unw_frame_info, fr_loc[21 - 16])/8,
 186                offsetof(struct unw_frame_info, fr_loc[22 - 16])/8,
 187                offsetof(struct unw_frame_info, fr_loc[23 - 16])/8,
 188                offsetof(struct unw_frame_info, fr_loc[24 - 16])/8,
 189                offsetof(struct unw_frame_info, fr_loc[25 - 16])/8,
 190                offsetof(struct unw_frame_info, fr_loc[26 - 16])/8,
 191                offsetof(struct unw_frame_info, fr_loc[27 - 16])/8,
 192                offsetof(struct unw_frame_info, fr_loc[28 - 16])/8,
 193                offsetof(struct unw_frame_info, fr_loc[29 - 16])/8,
 194                offsetof(struct unw_frame_info, fr_loc[30 - 16])/8,
 195                offsetof(struct unw_frame_info, fr_loc[31 - 16])/8,
 196        },
 197        .pt_regs_offsets = {
 198                [0] = -1,
 199                offsetof(struct pt_regs,  r1),
 200                offsetof(struct pt_regs,  r2),
 201                offsetof(struct pt_regs,  r3),
 202                [4] = -1, [5] = -1, [6] = -1, [7] = -1,
 203                offsetof(struct pt_regs,  r8),
 204                offsetof(struct pt_regs,  r9),
 205                offsetof(struct pt_regs, r10),
 206                offsetof(struct pt_regs, r11),
 207                offsetof(struct pt_regs, r12),
 208                offsetof(struct pt_regs, r13),
 209                offsetof(struct pt_regs, r14),
 210                offsetof(struct pt_regs, r15),
 211                offsetof(struct pt_regs, r16),
 212                offsetof(struct pt_regs, r17),
 213                offsetof(struct pt_regs, r18),
 214                offsetof(struct pt_regs, r19),
 215                offsetof(struct pt_regs, r20),
 216                offsetof(struct pt_regs, r21),
 217                offsetof(struct pt_regs, r22),
 218                offsetof(struct pt_regs, r23),
 219                offsetof(struct pt_regs, r24),
 220                offsetof(struct pt_regs, r25),
 221                offsetof(struct pt_regs, r26),
 222                offsetof(struct pt_regs, r27),
 223                offsetof(struct pt_regs, r28),
 224                offsetof(struct pt_regs, r29),
 225                offsetof(struct pt_regs, r30),
 226                offsetof(struct pt_regs, r31),
 227        },
 228        .hash = { [0 ... UNW_HASH_SIZE - 1] = -1 },
 229#ifdef UNW_DEBUG
 230        .preg_name = {
 231                "pri_unat_gr", "pri_unat_mem", "bsp", "bspstore", "ar.pfs", "ar.rnat", "psp", "rp",
 232                "r4", "r5", "r6", "r7",
 233                "ar.unat", "pr", "ar.lc", "ar.fpsr",
 234                "b1", "b2", "b3", "b4", "b5",
 235                "f2", "f3", "f4", "f5",
 236                "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
 237                "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"
 238        }
 239#endif
 240};
 241
 242static inline int
 243read_only (void *addr)
 244{
 245        return (unsigned long) ((char *) addr - (char *) &unw.r0) < sizeof(unw.r0);
 246}
 247
 248/*
 249 * Returns offset of rREG in struct pt_regs.
 250 */
 251static inline unsigned long
 252pt_regs_off (unsigned long reg)
 253{
 254        short off = -1;
 255
 256        if (reg < ARRAY_SIZE(unw.pt_regs_offsets))
 257                off = unw.pt_regs_offsets[reg];
 258
 259        if (off < 0) {
 260                UNW_DPRINT(0, "unwind.%s: bad scratch reg r%lu\n", __func__, reg);
 261                off = 0;
 262        }
 263        return (unsigned long) off;
 264}
 265
 266static inline struct pt_regs *
 267get_scratch_regs (struct unw_frame_info *info)
 268{
 269        if (!info->pt) {
 270                /* This should not happen with valid unwind info.  */
 271                UNW_DPRINT(0, "unwind.%s: bad unwind info: resetting info->pt\n", __func__);
 272                if (info->flags & UNW_FLAG_INTERRUPT_FRAME)
 273                        info->pt = (unsigned long) ((struct pt_regs *) info->psp - 1);
 274                else
 275                        info->pt = info->sp - 16;
 276        }
 277        UNW_DPRINT(3, "unwind.%s: sp 0x%lx pt 0x%lx\n", __func__, info->sp, info->pt);
 278        return (struct pt_regs *) info->pt;
 279}
 280
 281/* Unwind accessors.  */
 282
 283int
 284unw_access_gr (struct unw_frame_info *info, int regnum, unsigned long *val, char *nat, int write)
 285{
 286        unsigned long *addr, *nat_addr, nat_mask = 0, dummy_nat;
 287        struct unw_ireg *ireg;
 288        struct pt_regs *pt;
 289
 290        if ((unsigned) regnum - 1 >= 127) {
 291                if (regnum == 0 && !write) {
 292                        *val = 0;       /* read r0 always returns 0 */
 293                        *nat = 0;
 294                        return 0;
 295                }
 296                UNW_DPRINT(0, "unwind.%s: trying to access non-existent r%u\n",
 297                           __func__, regnum);
 298                return -1;
 299        }
 300
 301        if (regnum < 32) {
 302                if (regnum >= 4 && regnum <= 7) {
 303                        /* access a preserved register */
 304                        ireg = &info->r4 + (regnum - 4);
 305                        addr = ireg->loc;
 306                        if (addr) {
 307                                nat_addr = addr + ireg->nat.off;
 308                                switch (ireg->nat.type) {
 309                                      case UNW_NAT_VAL:
 310                                        /* simulate getf.sig/setf.sig */
 311                                        if (write) {
 312                                                if (*nat) {
 313                                                        /* write NaTVal and be done with it */
 314                                                        addr[0] = 0;
 315                                                        addr[1] = 0x1fffe;
 316                                                        return 0;
 317                                                }
 318                                                addr[1] = 0x1003e;
 319                                        } else {
 320                                                if (addr[0] == 0 && addr[1] == 0x1ffe) {
 321                                                        /* return NaT and be done with it */
 322                                                        *val = 0;
 323                                                        *nat = 1;
 324                                                        return 0;
 325                                                }
 326                                        }
 327                                        /* fall through */
 328                                      case UNW_NAT_NONE:
 329                                        dummy_nat = 0;
 330                                        nat_addr = &dummy_nat;
 331                                        break;
 332
 333                                      case UNW_NAT_MEMSTK:
 334                                        nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 335                                        break;
 336
 337                                      case UNW_NAT_REGSTK:
 338                                        nat_addr = ia64_rse_rnat_addr(addr);
 339                                        if ((unsigned long) addr < info->regstk.limit
 340                                            || (unsigned long) addr >= info->regstk.top)
 341                                        {
 342                                                UNW_DPRINT(0, "unwind.%s: %p outside of regstk "
 343                                                        "[0x%lx-0x%lx)\n",
 344                                                        __func__, (void *) addr,
 345                                                        info->regstk.limit,
 346                                                        info->regstk.top);
 347                                                return -1;
 348                                        }
 349                                        if ((unsigned long) nat_addr >= info->regstk.top)
 350                                                nat_addr = &info->sw->ar_rnat;
 351                                        nat_mask = (1UL << ia64_rse_slot_num(addr));
 352                                        break;
 353                                }
 354                        } else {
 355                                addr = &info->sw->r4 + (regnum - 4);
 356                                nat_addr = &info->sw->ar_unat;
 357                                nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 358                        }
 359                } else {
 360                        /* access a scratch register */
 361                        pt = get_scratch_regs(info);
 362                        addr = (unsigned long *) ((unsigned long)pt + pt_regs_off(regnum));
 363                        if (info->pri_unat_loc)
 364                                nat_addr = info->pri_unat_loc;
 365                        else
 366                                nat_addr = &info->sw->caller_unat;
 367                        nat_mask = (1UL << ((long) addr & 0x1f8)/8);
 368                }
 369        } else {
 370                /* access a stacked register */
 371                addr = ia64_rse_skip_regs((unsigned long *) info->bsp, regnum - 32);
 372                nat_addr = ia64_rse_rnat_addr(addr);
 373                if ((unsigned long) addr < info->regstk.limit
 374                    || (unsigned long) addr >= info->regstk.top)
 375                {
 376                        UNW_DPRINT(0, "unwind.%s: ignoring attempt to access register outside "
 377                                   "of rbs\n",  __func__);
 378                        return -1;
 379                }
 380                if ((unsigned long) nat_addr >= info->regstk.top)
 381                        nat_addr = &info->sw->ar_rnat;
 382                nat_mask = (1UL << ia64_rse_slot_num(addr));
 383        }
 384
 385        if (write) {
 386                if (read_only(addr)) {
 387                        UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 388                                __func__);
 389                } else {
 390                        *addr = *val;
 391                        if (*nat)
 392                                *nat_addr |= nat_mask;
 393                        else
 394                                *nat_addr &= ~nat_mask;
 395                }
 396        } else {
 397                if ((*nat_addr & nat_mask) == 0) {
 398                        *val = *addr;
 399                        *nat = 0;
 400                } else {
 401                        *val = 0;       /* if register is a NaT, *addr may contain kernel data! */
 402                        *nat = 1;
 403                }
 404        }
 405        return 0;
 406}
 407EXPORT_SYMBOL(unw_access_gr);
 408
 409int
 410unw_access_br (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
 411{
 412        unsigned long *addr;
 413        struct pt_regs *pt;
 414
 415        switch (regnum) {
 416                /* scratch: */
 417              case 0: pt = get_scratch_regs(info); addr = &pt->b0; break;
 418              case 6: pt = get_scratch_regs(info); addr = &pt->b6; break;
 419              case 7: pt = get_scratch_regs(info); addr = &pt->b7; break;
 420
 421                /* preserved: */
 422              case 1: case 2: case 3: case 4: case 5:
 423                addr = *(&info->b1_loc + (regnum - 1));
 424                if (!addr)
 425                        addr = &info->sw->b1 + (regnum - 1);
 426                break;
 427
 428              default:
 429                UNW_DPRINT(0, "unwind.%s: trying to access non-existent b%u\n",
 430                           __func__, regnum);
 431                return -1;
 432        }
 433        if (write)
 434                if (read_only(addr)) {
 435                        UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 436                                __func__);
 437                } else
 438                        *addr = *val;
 439        else
 440                *val = *addr;
 441        return 0;
 442}
 443EXPORT_SYMBOL(unw_access_br);
 444
 445int
 446unw_access_fr (struct unw_frame_info *info, int regnum, struct ia64_fpreg *val, int write)
 447{
 448        struct ia64_fpreg *addr = NULL;
 449        struct pt_regs *pt;
 450
 451        if ((unsigned) (regnum - 2) >= 126) {
 452                UNW_DPRINT(0, "unwind.%s: trying to access non-existent f%u\n",
 453                           __func__, regnum);
 454                return -1;
 455        }
 456
 457        if (regnum <= 5) {
 458                addr = *(&info->f2_loc + (regnum - 2));
 459                if (!addr)
 460                        addr = &info->sw->f2 + (regnum - 2);
 461        } else if (regnum <= 15) {
 462                if (regnum <= 11) {
 463                        pt = get_scratch_regs(info);
 464                        addr = &pt->f6  + (regnum - 6);
 465                }
 466                else
 467                        addr = &info->sw->f12 + (regnum - 12);
 468        } else if (regnum <= 31) {
 469                addr = info->fr_loc[regnum - 16];
 470                if (!addr)
 471                        addr = &info->sw->f16 + (regnum - 16);
 472        } else {
 473                struct task_struct *t = info->task;
 474
 475                if (write)
 476                        ia64_sync_fph(t);
 477                else
 478                        ia64_flush_fph(t);
 479                addr = t->thread.fph + (regnum - 32);
 480        }
 481
 482        if (write)
 483                if (read_only(addr)) {
 484                        UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 485                                __func__);
 486                } else
 487                        *addr = *val;
 488        else
 489                *val = *addr;
 490        return 0;
 491}
 492EXPORT_SYMBOL(unw_access_fr);
 493
 494int
 495unw_access_ar (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
 496{
 497        unsigned long *addr;
 498        struct pt_regs *pt;
 499
 500        switch (regnum) {
 501              case UNW_AR_BSP:
 502                addr = info->bsp_loc;
 503                if (!addr)
 504                        addr = &info->sw->ar_bspstore;
 505                break;
 506
 507              case UNW_AR_BSPSTORE:
 508                addr = info->bspstore_loc;
 509                if (!addr)
 510                        addr = &info->sw->ar_bspstore;
 511                break;
 512
 513              case UNW_AR_PFS:
 514                addr = info->pfs_loc;
 515                if (!addr)
 516                        addr = &info->sw->ar_pfs;
 517                break;
 518
 519              case UNW_AR_RNAT:
 520                addr = info->rnat_loc;
 521                if (!addr)
 522                        addr = &info->sw->ar_rnat;
 523                break;
 524
 525              case UNW_AR_UNAT:
 526                addr = info->unat_loc;
 527                if (!addr)
 528                        addr = &info->sw->caller_unat;
 529                break;
 530
 531              case UNW_AR_LC:
 532                addr = info->lc_loc;
 533                if (!addr)
 534                        addr = &info->sw->ar_lc;
 535                break;
 536
 537              case UNW_AR_EC:
 538                if (!info->cfm_loc)
 539                        return -1;
 540                if (write)
 541                        *info->cfm_loc =
 542                                (*info->cfm_loc & ~(0x3fUL << 52)) | ((*val & 0x3f) << 52);
 543                else
 544                        *val = (*info->cfm_loc >> 52) & 0x3f;
 545                return 0;
 546
 547              case UNW_AR_FPSR:
 548                addr = info->fpsr_loc;
 549                if (!addr)
 550                        addr = &info->sw->ar_fpsr;
 551                break;
 552
 553              case UNW_AR_RSC:
 554                pt = get_scratch_regs(info);
 555                addr = &pt->ar_rsc;
 556                break;
 557
 558              case UNW_AR_CCV:
 559                pt = get_scratch_regs(info);
 560                addr = &pt->ar_ccv;
 561                break;
 562
 563              case UNW_AR_CSD:
 564                pt = get_scratch_regs(info);
 565                addr = &pt->ar_csd;
 566                break;
 567
 568              case UNW_AR_SSD:
 569                pt = get_scratch_regs(info);
 570                addr = &pt->ar_ssd;
 571                break;
 572
 573              default:
 574                UNW_DPRINT(0, "unwind.%s: trying to access non-existent ar%u\n",
 575                           __func__, regnum);
 576                return -1;
 577        }
 578
 579        if (write) {
 580                if (read_only(addr)) {
 581                        UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 582                                __func__);
 583                } else
 584                        *addr = *val;
 585        } else
 586                *val = *addr;
 587        return 0;
 588}
 589EXPORT_SYMBOL(unw_access_ar);
 590
 591int
 592unw_access_pr (struct unw_frame_info *info, unsigned long *val, int write)
 593{
 594        unsigned long *addr;
 595
 596        addr = info->pr_loc;
 597        if (!addr)
 598                addr = &info->sw->pr;
 599
 600        if (write) {
 601                if (read_only(addr)) {
 602                        UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
 603                                __func__);
 604                } else
 605                        *addr = *val;
 606        } else
 607                *val = *addr;
 608        return 0;
 609}
 610EXPORT_SYMBOL(unw_access_pr);
 611
 612
 613/* Routines to manipulate the state stack.  */
 614
 615static inline void
 616push (struct unw_state_record *sr)
 617{
 618        struct unw_reg_state *rs;
 619
 620        rs = alloc_reg_state();
 621        if (!rs) {
 622                printk(KERN_ERR "unwind: cannot stack reg state!\n");
 623                return;
 624        }
 625        memcpy(rs, &sr->curr, sizeof(*rs));
 626        sr->curr.next = rs;
 627}
 628
 629static void
 630pop (struct unw_state_record *sr)
 631{
 632        struct unw_reg_state *rs = sr->curr.next;
 633
 634        if (!rs) {
 635                printk(KERN_ERR "unwind: stack underflow!\n");
 636                return;
 637        }
 638        memcpy(&sr->curr, rs, sizeof(*rs));
 639        free_reg_state(rs);
 640}
 641
 642/* Make a copy of the state stack.  Non-recursive to avoid stack overflows.  */
 643static struct unw_reg_state *
 644dup_state_stack (struct unw_reg_state *rs)
 645{
 646        struct unw_reg_state *copy, *prev = NULL, *first = NULL;
 647
 648        while (rs) {
 649                copy = alloc_reg_state();
 650                if (!copy) {
 651                        printk(KERN_ERR "unwind.dup_state_stack: out of memory\n");
 652                        return NULL;
 653                }
 654                memcpy(copy, rs, sizeof(*copy));
 655                if (first)
 656                        prev->next = copy;
 657                else
 658                        first = copy;
 659                rs = rs->next;
 660                prev = copy;
 661        }
 662        return first;
 663}
 664
 665/* Free all stacked register states (but not RS itself).  */
 666static void
 667free_state_stack (struct unw_reg_state *rs)
 668{
 669        struct unw_reg_state *p, *next;
 670
 671        for (p = rs->next; p != NULL; p = next) {
 672                next = p->next;
 673                free_reg_state(p);
 674        }
 675        rs->next = NULL;
 676}
 677
 678/* Unwind decoder routines */
 679
 680static enum unw_register_index __attribute_const__
 681decode_abreg (unsigned char abreg, int memory)
 682{
 683        switch (abreg) {
 684              case 0x04 ... 0x07: return UNW_REG_R4 + (abreg - 0x04);
 685              case 0x22 ... 0x25: return UNW_REG_F2 + (abreg - 0x22);
 686              case 0x30 ... 0x3f: return UNW_REG_F16 + (abreg - 0x30);
 687              case 0x41 ... 0x45: return UNW_REG_B1 + (abreg - 0x41);
 688              case 0x60: return UNW_REG_PR;
 689              case 0x61: return UNW_REG_PSP;
 690              case 0x62: return memory ? UNW_REG_PRI_UNAT_MEM : UNW_REG_PRI_UNAT_GR;
 691              case 0x63: return UNW_REG_RP;
 692              case 0x64: return UNW_REG_BSP;
 693              case 0x65: return UNW_REG_BSPSTORE;
 694              case 0x66: return UNW_REG_RNAT;
 695              case 0x67: return UNW_REG_UNAT;
 696              case 0x68: return UNW_REG_FPSR;
 697              case 0x69: return UNW_REG_PFS;
 698              case 0x6a: return UNW_REG_LC;
 699              default:
 700                break;
 701        }
 702        UNW_DPRINT(0, "unwind.%s: bad abreg=0x%x\n", __func__, abreg);
 703        return UNW_REG_LC;
 704}
 705
 706static void
 707set_reg (struct unw_reg_info *reg, enum unw_where where, int when, unsigned long val)
 708{
 709        reg->val = val;
 710        reg->where = where;
 711        if (reg->when == UNW_WHEN_NEVER)
 712                reg->when = when;
 713}
 714
 715static void
 716alloc_spill_area (unsigned long *offp, unsigned long regsize,
 717                  struct unw_reg_info *lo, struct unw_reg_info *hi)
 718{
 719        struct unw_reg_info *reg;
 720
 721        for (reg = hi; reg >= lo; --reg) {
 722                if (reg->where == UNW_WHERE_SPILL_HOME) {
 723                        reg->where = UNW_WHERE_PSPREL;
 724                        *offp -= regsize;
 725                        reg->val = *offp;
 726                }
 727        }
 728}
 729
 730static inline void
 731spill_next_when (struct unw_reg_info **regp, struct unw_reg_info *lim, unw_word t)
 732{
 733        struct unw_reg_info *reg;
 734
 735        for (reg = *regp; reg <= lim; ++reg) {
 736                if (reg->where == UNW_WHERE_SPILL_HOME) {
 737                        reg->when = t;
 738                        *regp = reg + 1;
 739                        return;
 740                }
 741        }
 742        UNW_DPRINT(0, "unwind.%s: excess spill!\n",  __func__);
 743}
 744
 745static inline void
 746finish_prologue (struct unw_state_record *sr)
 747{
 748        struct unw_reg_info *reg;
 749        unsigned long off;
 750        int i;
 751
 752        /*
 753         * First, resolve implicit register save locations (see Section "11.4.2.3 Rules
 754         * for Using Unwind Descriptors", rule 3):
 755         */
 756        for (i = 0; i < (int) ARRAY_SIZE(unw.save_order); ++i) {
 757                reg = sr->curr.reg + unw.save_order[i];
 758                if (reg->where == UNW_WHERE_GR_SAVE) {
 759                        reg->where = UNW_WHERE_GR;
 760                        reg->val = sr->gr_save_loc++;
 761                }
 762        }
 763
 764        /*
 765         * Next, compute when the fp, general, and branch registers get
 766         * saved.  This must come before alloc_spill_area() because
 767         * we need to know which registers are spilled to their home
 768         * locations.
 769         */
 770        if (sr->imask) {
 771                unsigned char kind, mask = 0, *cp = sr->imask;
 772                int t;
 773                static const unsigned char limit[3] = {
 774                        UNW_REG_F31, UNW_REG_R7, UNW_REG_B5
 775                };
 776                struct unw_reg_info *(regs[3]);
 777
 778                regs[0] = sr->curr.reg + UNW_REG_F2;
 779                regs[1] = sr->curr.reg + UNW_REG_R4;
 780                regs[2] = sr->curr.reg + UNW_REG_B1;
 781
 782                for (t = 0; t < sr->region_len; ++t) {
 783                        if ((t & 3) == 0)
 784                                mask = *cp++;
 785                        kind = (mask >> 2*(3-(t & 3))) & 3;
 786                        if (kind > 0)
 787                                spill_next_when(&regs[kind - 1], sr->curr.reg + limit[kind - 1],
 788                                                sr->region_start + t);
 789                }
 790        }
 791        /*
 792         * Next, lay out the memory stack spill area:
 793         */
 794        if (sr->any_spills) {
 795                off = sr->spill_offset;
 796                alloc_spill_area(&off, 16, sr->curr.reg + UNW_REG_F2, sr->curr.reg + UNW_REG_F31);
 797                alloc_spill_area(&off,  8, sr->curr.reg + UNW_REG_B1, sr->curr.reg + UNW_REG_B5);
 798                alloc_spill_area(&off,  8, sr->curr.reg + UNW_REG_R4, sr->curr.reg + UNW_REG_R7);
 799        }
 800}
 801
 802/*
 803 * Region header descriptors.
 804 */
 805
 806static void
 807desc_prologue (int body, unw_word rlen, unsigned char mask, unsigned char grsave,
 808               struct unw_state_record *sr)
 809{
 810        int i, region_start;
 811
 812        if (!(sr->in_body || sr->first_region))
 813                finish_prologue(sr);
 814        sr->first_region = 0;
 815
 816        /* check if we're done: */
 817        if (sr->when_target < sr->region_start + sr->region_len) {
 818                sr->done = 1;
 819                return;
 820        }
 821
 822        region_start = sr->region_start + sr->region_len;
 823
 824        for (i = 0; i < sr->epilogue_count; ++i)
 825                pop(sr);
 826        sr->epilogue_count = 0;
 827        sr->epilogue_start = UNW_WHEN_NEVER;
 828
 829        sr->region_start = region_start;
 830        sr->region_len = rlen;
 831        sr->in_body = body;
 832
 833        if (!body) {
 834                push(sr);
 835
 836                for (i = 0; i < 4; ++i) {
 837                        if (mask & 0x8)
 838                                set_reg(sr->curr.reg + unw.save_order[i], UNW_WHERE_GR,
 839                                        sr->region_start + sr->region_len - 1, grsave++);
 840                        mask <<= 1;
 841                }
 842                sr->gr_save_loc = grsave;
 843                sr->any_spills = 0;
 844                sr->imask = NULL;
 845                sr->spill_offset = 0x10;        /* default to psp+16 */
 846        }
 847}
 848
 849/*
 850 * Prologue descriptors.
 851 */
 852
 853static inline void
 854desc_abi (unsigned char abi, unsigned char context, struct unw_state_record *sr)
 855{
 856        if (abi == 3 && context == 'i') {
 857                sr->flags |= UNW_FLAG_INTERRUPT_FRAME;
 858                UNW_DPRINT(3, "unwind.%s: interrupt frame\n",  __func__);
 859        }
 860        else
 861                UNW_DPRINT(0, "unwind%s: ignoring unwabi(abi=0x%x,context=0x%x)\n",
 862                                __func__, abi, context);
 863}
 864
 865static inline void
 866desc_br_gr (unsigned char brmask, unsigned char gr, struct unw_state_record *sr)
 867{
 868        int i;
 869
 870        for (i = 0; i < 5; ++i) {
 871                if (brmask & 1)
 872                        set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_GR,
 873                                sr->region_start + sr->region_len - 1, gr++);
 874                brmask >>= 1;
 875        }
 876}
 877
 878static inline void
 879desc_br_mem (unsigned char brmask, struct unw_state_record *sr)
 880{
 881        int i;
 882
 883        for (i = 0; i < 5; ++i) {
 884                if (brmask & 1) {
 885                        set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_SPILL_HOME,
 886                                sr->region_start + sr->region_len - 1, 0);
 887                        sr->any_spills = 1;
 888                }
 889                brmask >>= 1;
 890        }
 891}
 892
 893static inline void
 894desc_frgr_mem (unsigned char grmask, unw_word frmask, struct unw_state_record *sr)
 895{
 896        int i;
 897
 898        for (i = 0; i < 4; ++i) {
 899                if ((grmask & 1) != 0) {
 900                        set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
 901                                sr->region_start + sr->region_len - 1, 0);
 902                        sr->any_spills = 1;
 903                }
 904                grmask >>= 1;
 905        }
 906        for (i = 0; i < 20; ++i) {
 907                if ((frmask & 1) != 0) {
 908                        int base = (i < 4) ? UNW_REG_F2 : UNW_REG_F16 - 4;
 909                        set_reg(sr->curr.reg + base + i, UNW_WHERE_SPILL_HOME,
 910                                sr->region_start + sr->region_len - 1, 0);
 911                        sr->any_spills = 1;
 912                }
 913                frmask >>= 1;
 914        }
 915}
 916
 917static inline void
 918desc_fr_mem (unsigned char frmask, struct unw_state_record *sr)
 919{
 920        int i;
 921
 922        for (i = 0; i < 4; ++i) {
 923                if ((frmask & 1) != 0) {
 924                        set_reg(sr->curr.reg + UNW_REG_F2 + i, UNW_WHERE_SPILL_HOME,
 925                                sr->region_start + sr->region_len - 1, 0);
 926                        sr->any_spills = 1;
 927                }
 928                frmask >>= 1;
 929        }
 930}
 931
 932static inline void
 933desc_gr_gr (unsigned char grmask, unsigned char gr, struct unw_state_record *sr)
 934{
 935        int i;
 936
 937        for (i = 0; i < 4; ++i) {
 938                if ((grmask & 1) != 0)
 939                        set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_GR,
 940                                sr->region_start + sr->region_len - 1, gr++);
 941                grmask >>= 1;
 942        }
 943}
 944
 945static inline void
 946desc_gr_mem (unsigned char grmask, struct unw_state_record *sr)
 947{
 948        int i;
 949
 950        for (i = 0; i < 4; ++i) {
 951                if ((grmask & 1) != 0) {
 952                        set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
 953                                sr->region_start + sr->region_len - 1, 0);
 954                        sr->any_spills = 1;
 955                }
 956                grmask >>= 1;
 957        }
 958}
 959
 960static inline void
 961desc_mem_stack_f (unw_word t, unw_word size, struct unw_state_record *sr)
 962{
 963        set_reg(sr->curr.reg + UNW_REG_PSP, UNW_WHERE_NONE,
 964                sr->region_start + min_t(int, t, sr->region_len - 1), 16*size);
 965}
 966
 967static inline void
 968desc_mem_stack_v (unw_word t, struct unw_state_record *sr)
 969{
 970        sr->curr.reg[UNW_REG_PSP].when = sr->region_start + min_t(int, t, sr->region_len - 1);
 971}
 972
 973static inline void
 974desc_reg_gr (unsigned char reg, unsigned char dst, struct unw_state_record *sr)
 975{
 976        set_reg(sr->curr.reg + reg, UNW_WHERE_GR, sr->region_start + sr->region_len - 1, dst);
 977}
 978
 979static inline void
 980desc_reg_psprel (unsigned char reg, unw_word pspoff, struct unw_state_record *sr)
 981{
 982        set_reg(sr->curr.reg + reg, UNW_WHERE_PSPREL, sr->region_start + sr->region_len - 1,
 983                0x10 - 4*pspoff);
 984}
 985
 986static inline void
 987desc_reg_sprel (unsigned char reg, unw_word spoff, struct unw_state_record *sr)
 988{
 989        set_reg(sr->curr.reg + reg, UNW_WHERE_SPREL, sr->region_start + sr->region_len - 1,
 990                4*spoff);
 991}
 992
 993static inline void
 994desc_rp_br (unsigned char dst, struct unw_state_record *sr)
 995{
 996        sr->return_link_reg = dst;
 997}
 998
 999static inline void
1000desc_reg_when (unsigned char regnum, unw_word t, struct unw_state_record *sr)
1001{
1002        struct unw_reg_info *reg = sr->curr.reg + regnum;
1003
1004        if (reg->where == UNW_WHERE_NONE)
1005                reg->where = UNW_WHERE_GR_SAVE;
1006        reg->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1007}
1008
1009static inline void
1010desc_spill_base (unw_word pspoff, struct unw_state_record *sr)
1011{
1012        sr->spill_offset = 0x10 - 4*pspoff;
1013}
1014
1015static inline unsigned char *
1016desc_spill_mask (unsigned char *imaskp, struct unw_state_record *sr)
1017{
1018        sr->imask = imaskp;
1019        return imaskp + (2*sr->region_len + 7)/8;
1020}
1021
1022/*
1023 * Body descriptors.
1024 */
1025static inline void
1026desc_epilogue (unw_word t, unw_word ecount, struct unw_state_record *sr)
1027{
1028        sr->epilogue_start = sr->region_start + sr->region_len - 1 - t;
1029        sr->epilogue_count = ecount + 1;
1030}
1031
1032static inline void
1033desc_copy_state (unw_word label, struct unw_state_record *sr)
1034{
1035        struct unw_labeled_state *ls;
1036
1037        for (ls = sr->labeled_states; ls; ls = ls->next) {
1038                if (ls->label == label) {
1039                        free_state_stack(&sr->curr);
1040                        memcpy(&sr->curr, &ls->saved_state, sizeof(sr->curr));
1041                        sr->curr.next = dup_state_stack(ls->saved_state.next);
1042                        return;
1043                }
1044        }
1045        printk(KERN_ERR "unwind: failed to find state labeled 0x%lx\n", label);
1046}
1047
1048static inline void
1049desc_label_state (unw_word label, struct unw_state_record *sr)
1050{
1051        struct unw_labeled_state *ls;
1052
1053        ls = alloc_labeled_state();
1054        if (!ls) {
1055                printk(KERN_ERR "unwind.desc_label_state(): out of memory\n");
1056                return;
1057        }
1058        ls->label = label;
1059        memcpy(&ls->saved_state, &sr->curr, sizeof(ls->saved_state));
1060        ls->saved_state.next = dup_state_stack(sr->curr.next);
1061
1062        /* insert into list of labeled states: */
1063        ls->next = sr->labeled_states;
1064        sr->labeled_states = ls;
1065}
1066
1067/*
1068 * General descriptors.
1069 */
1070
1071static inline int
1072desc_is_active (unsigned char qp, unw_word t, struct unw_state_record *sr)
1073{
1074        if (sr->when_target <= sr->region_start + min_t(int, t, sr->region_len - 1))
1075                return 0;
1076        if (qp > 0) {
1077                if ((sr->pr_val & (1UL << qp)) == 0)
1078                        return 0;
1079                sr->pr_mask |= (1UL << qp);
1080        }
1081        return 1;
1082}
1083
1084static inline void
1085desc_restore_p (unsigned char qp, unw_word t, unsigned char abreg, struct unw_state_record *sr)
1086{
1087        struct unw_reg_info *r;
1088
1089        if (!desc_is_active(qp, t, sr))
1090                return;
1091
1092        r = sr->curr.reg + decode_abreg(abreg, 0);
1093        r->where = UNW_WHERE_NONE;
1094        r->when = UNW_WHEN_NEVER;
1095        r->val = 0;
1096}
1097
1098static inline void
1099desc_spill_reg_p (unsigned char qp, unw_word t, unsigned char abreg, unsigned char x,
1100                     unsigned char ytreg, struct unw_state_record *sr)
1101{
1102        enum unw_where where = UNW_WHERE_GR;
1103        struct unw_reg_info *r;
1104
1105        if (!desc_is_active(qp, t, sr))
1106                return;
1107
1108        if (x)
1109                where = UNW_WHERE_BR;
1110        else if (ytreg & 0x80)
1111                where = UNW_WHERE_FR;
1112
1113        r = sr->curr.reg + decode_abreg(abreg, 0);
1114        r->where = where;
1115        r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1116        r->val = (ytreg & 0x7f);
1117}
1118
1119static inline void
1120desc_spill_psprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word pspoff,
1121                     struct unw_state_record *sr)
1122{
1123        struct unw_reg_info *r;
1124
1125        if (!desc_is_active(qp, t, sr))
1126                return;
1127
1128        r = sr->curr.reg + decode_abreg(abreg, 1);
1129        r->where = UNW_WHERE_PSPREL;
1130        r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1131        r->val = 0x10 - 4*pspoff;
1132}
1133
1134static inline void
1135desc_spill_sprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word spoff,
1136                       struct unw_state_record *sr)
1137{
1138        struct unw_reg_info *r;
1139
1140        if (!desc_is_active(qp, t, sr))
1141                return;
1142
1143        r = sr->curr.reg + decode_abreg(abreg, 1);
1144        r->where = UNW_WHERE_SPREL;
1145        r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1146        r->val = 4*spoff;
1147}
1148
1149#define UNW_DEC_BAD_CODE(code)                  printk(KERN_ERR "unwind: unknown code 0x%02x\n", \
1150                                                       code);
1151
1152/*
1153 * region headers:
1154 */
1155#define UNW_DEC_PROLOGUE_GR(fmt,r,m,gr,arg)     desc_prologue(0,r,m,gr,arg)
1156#define UNW_DEC_PROLOGUE(fmt,b,r,arg)           desc_prologue(b,r,0,32,arg)
1157/*
1158 * prologue descriptors:
1159 */
1160#define UNW_DEC_ABI(fmt,a,c,arg)                desc_abi(a,c,arg)
1161#define UNW_DEC_BR_GR(fmt,b,g,arg)              desc_br_gr(b,g,arg)
1162#define UNW_DEC_BR_MEM(fmt,b,arg)               desc_br_mem(b,arg)
1163#define UNW_DEC_FRGR_MEM(fmt,g,f,arg)           desc_frgr_mem(g,f,arg)
1164#define UNW_DEC_FR_MEM(fmt,f,arg)               desc_fr_mem(f,arg)
1165#define UNW_DEC_GR_GR(fmt,m,g,arg)              desc_gr_gr(m,g,arg)
1166#define UNW_DEC_GR_MEM(fmt,m,arg)               desc_gr_mem(m,arg)
1167#define UNW_DEC_MEM_STACK_F(fmt,t,s,arg)        desc_mem_stack_f(t,s,arg)
1168#define UNW_DEC_MEM_STACK_V(fmt,t,arg)          desc_mem_stack_v(t,arg)
1169#define UNW_DEC_REG_GR(fmt,r,d,arg)             desc_reg_gr(r,d,arg)
1170#define UNW_DEC_REG_PSPREL(fmt,r,o,arg)         desc_reg_psprel(r,o,arg)
1171#define UNW_DEC_REG_SPREL(fmt,r,o,arg)          desc_reg_sprel(r,o,arg)
1172#define UNW_DEC_REG_WHEN(fmt,r,t,arg)           desc_reg_when(r,t,arg)
1173#define UNW_DEC_PRIUNAT_WHEN_GR(fmt,t,arg)      desc_reg_when(UNW_REG_PRI_UNAT_GR,t,arg)
1174#define UNW_DEC_PRIUNAT_WHEN_MEM(fmt,t,arg)     desc_reg_when(UNW_REG_PRI_UNAT_MEM,t,arg)
1175#define UNW_DEC_PRIUNAT_GR(fmt,r,arg)           desc_reg_gr(UNW_REG_PRI_UNAT_GR,r,arg)
1176#define UNW_DEC_PRIUNAT_PSPREL(fmt,o,arg)       desc_reg_psprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1177#define UNW_DEC_PRIUNAT_SPREL(fmt,o,arg)        desc_reg_sprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1178#define UNW_DEC_RP_BR(fmt,d,arg)                desc_rp_br(d,arg)
1179#define UNW_DEC_SPILL_BASE(fmt,o,arg)           desc_spill_base(o,arg)
1180#define UNW_DEC_SPILL_MASK(fmt,m,arg)           (m = desc_spill_mask(m,arg))
1181/*
1182 * body descriptors:
1183 */
1184#define UNW_DEC_EPILOGUE(fmt,t,c,arg)           desc_epilogue(t,c,arg)
1185#define UNW_DEC_COPY_STATE(fmt,l,arg)           desc_copy_state(l,arg)
1186#define UNW_DEC_LABEL_STATE(fmt,l,arg)          desc_label_state(l,arg)
1187/*
1188 * general unwind descriptors:
1189 */
1190#define UNW_DEC_SPILL_REG_P(f,p,t,a,x,y,arg)    desc_spill_reg_p(p,t,a,x,y,arg)
1191#define UNW_DEC_SPILL_REG(f,t,a,x,y,arg)        desc_spill_reg_p(0,t,a,x,y,arg)
1192#define UNW_DEC_SPILL_PSPREL_P(f,p,t,a,o,arg)   desc_spill_psprel_p(p,t,a,o,arg)
1193#define UNW_DEC_SPILL_PSPREL(f,t,a,o,arg)       desc_spill_psprel_p(0,t,a,o,arg)
1194#define UNW_DEC_SPILL_SPREL_P(f,p,t,a,o,arg)    desc_spill_sprel_p(p,t,a,o,arg)
1195#define UNW_DEC_SPILL_SPREL(f,t,a,o,arg)        desc_spill_sprel_p(0,t,a,o,arg)
1196#define UNW_DEC_RESTORE_P(f,p,t,a,arg)          desc_restore_p(p,t,a,arg)
1197#define UNW_DEC_RESTORE(f,t,a,arg)              desc_restore_p(0,t,a,arg)
1198
1199#include "unwind_decoder.c"
1200
1201
1202/* Unwind scripts. */
1203
1204static inline unw_hash_index_t
1205hash (unsigned long ip)
1206{
1207        /* magic number = ((sqrt(5)-1)/2)*2^64 */
1208        static const unsigned long hashmagic = 0x9e3779b97f4a7c16UL;
1209
1210        return (ip >> 4) * hashmagic >> (64 - UNW_LOG_HASH_SIZE);
1211}
1212
1213static inline long
1214cache_match (struct unw_script *script, unsigned long ip, unsigned long pr)
1215{
1216        read_lock(&script->lock);
1217        if (ip == script->ip && ((pr ^ script->pr_val) & script->pr_mask) == 0)
1218                /* keep the read lock... */
1219                return 1;
1220        read_unlock(&script->lock);
1221        return 0;
1222}
1223
1224static inline struct unw_script *
1225script_lookup (struct unw_frame_info *info)
1226{
1227        struct unw_script *script = unw.cache + info->hint;
1228        unsigned short index;
1229        unsigned long ip, pr;
1230
1231        if (UNW_DEBUG_ON(0))
1232                return NULL;    /* Always regenerate scripts in debug mode */
1233
1234        STAT(++unw.stat.cache.lookups);
1235
1236        ip = info->ip;
1237        pr = info->pr;
1238
1239        if (cache_match(script, ip, pr)) {
1240                STAT(++unw.stat.cache.hinted_hits);
1241                return script;
1242        }
1243
1244        index = unw.hash[hash(ip)];
1245        if (index >= UNW_CACHE_SIZE)
1246                return NULL;
1247
1248        script = unw.cache + index;
1249        while (1) {
1250                if (cache_match(script, ip, pr)) {
1251                        /* update hint; no locking required as single-word writes are atomic */
1252                        STAT(++unw.stat.cache.normal_hits);
1253                        unw.cache[info->prev_script].hint = script - unw.cache;
1254                        return script;
1255                }
1256                if (script->coll_chain >= UNW_HASH_SIZE)
1257                        return NULL;
1258                script = unw.cache + script->coll_chain;
1259                STAT(++unw.stat.cache.collision_chain_traversals);
1260        }
1261}
1262
1263/*
1264 * On returning, a write lock for the SCRIPT is still being held.
1265 */
1266static inline struct unw_script *
1267script_new (unsigned long ip)
1268{
1269        struct unw_script *script, *prev, *tmp;
1270        unw_hash_index_t index;
1271        unsigned short head;
1272
1273        STAT(++unw.stat.script.news);
1274
1275        /*
1276         * Can't (easily) use cmpxchg() here because of ABA problem
1277         * that is intrinsic in cmpxchg()...
1278         */
1279        head = unw.lru_head;
1280        script = unw.cache + head;
1281        unw.lru_head = script->lru_chain;
1282
1283        /*
1284         * We'd deadlock here if we interrupted a thread that is holding a read lock on
1285         * script->lock.  Thus, if the write_trylock() fails, we simply bail out.  The
1286         * alternative would be to disable interrupts whenever we hold a read-lock, but
1287         * that seems silly.
1288         */
1289        if (!write_trylock(&script->lock))
1290                return NULL;
1291
1292        /* re-insert script at the tail of the LRU chain: */
1293        unw.cache[unw.lru_tail].lru_chain = head;
1294        unw.lru_tail = head;
1295
1296        /* remove the old script from the hash table (if it's there): */
1297        if (script->ip) {
1298                index = hash(script->ip);
1299                tmp = unw.cache + unw.hash[index];
1300                prev = NULL;
1301                while (1) {
1302                        if (tmp == script) {
1303                                if (prev)
1304                                        prev->coll_chain = tmp->coll_chain;
1305                                else
1306                                        unw.hash[index] = tmp->coll_chain;
1307                                break;
1308                        } else
1309                                prev = tmp;
1310                        if (tmp->coll_chain >= UNW_CACHE_SIZE)
1311                        /* old script wasn't in the hash-table */
1312                                break;
1313                        tmp = unw.cache + tmp->coll_chain;
1314                }
1315        }
1316
1317        /* enter new script in the hash table */
1318        index = hash(ip);
1319        script->coll_chain = unw.hash[index];
1320        unw.hash[index] = script - unw.cache;
1321
1322        script->ip = ip;        /* set new IP while we're holding the locks */
1323
1324        STAT(if (script->coll_chain < UNW_CACHE_SIZE) ++unw.stat.script.collisions);
1325
1326        script->flags = 0;
1327        script->hint = 0;
1328        script->count = 0;
1329        return script;
1330}
1331
1332static void
1333script_finalize (struct unw_script *script, struct unw_state_record *sr)
1334{
1335        script->pr_mask = sr->pr_mask;
1336        script->pr_val = sr->pr_val;
1337        /*
1338         * We could down-grade our write-lock on script->lock here but
1339         * the rwlock API doesn't offer atomic lock downgrading, so
1340         * we'll just keep the write-lock and release it later when
1341         * we're done using the script.
1342         */
1343}
1344
1345static inline void
1346script_emit (struct unw_script *script, struct unw_insn insn)
1347{
1348        if (script->count >= UNW_MAX_SCRIPT_LEN) {
1349                UNW_DPRINT(0, "unwind.%s: script exceeds maximum size of %u instructions!\n",
1350                        __func__, UNW_MAX_SCRIPT_LEN);
1351                return;
1352        }
1353        script->insn[script->count++] = insn;
1354}
1355
1356static inline void
1357emit_nat_info (struct unw_state_record *sr, int i, struct unw_script *script)
1358{
1359        struct unw_reg_info *r = sr->curr.reg + i;
1360        enum unw_insn_opcode opc;
1361        struct unw_insn insn;
1362        unsigned long val = 0;
1363
1364        switch (r->where) {
1365              case UNW_WHERE_GR:
1366                if (r->val >= 32) {
1367                        /* register got spilled to a stacked register */
1368                        opc = UNW_INSN_SETNAT_TYPE;
1369                        val = UNW_NAT_REGSTK;
1370                } else
1371                        /* register got spilled to a scratch register */
1372                        opc = UNW_INSN_SETNAT_MEMSTK;
1373                break;
1374
1375              case UNW_WHERE_FR:
1376                opc = UNW_INSN_SETNAT_TYPE;
1377                val = UNW_NAT_VAL;
1378                break;
1379
1380              case UNW_WHERE_BR:
1381                opc = UNW_INSN_SETNAT_TYPE;
1382                val = UNW_NAT_NONE;
1383                break;
1384
1385              case UNW_WHERE_PSPREL:
1386              case UNW_WHERE_SPREL:
1387                opc = UNW_INSN_SETNAT_MEMSTK;
1388                break;
1389
1390              default:
1391                UNW_DPRINT(0, "unwind.%s: don't know how to emit nat info for where = %u\n",
1392                           __func__, r->where);
1393                return;
1394        }
1395        insn.opc = opc;
1396        insn.dst = unw.preg_index[i];
1397        insn.val = val;
1398        script_emit(script, insn);
1399}
1400
1401static void
1402compile_reg (struct unw_state_record *sr, int i, struct unw_script *script)
1403{
1404        struct unw_reg_info *r = sr->curr.reg + i;
1405        enum unw_insn_opcode opc;
1406        unsigned long val, rval;
1407        struct unw_insn insn;
1408        long need_nat_info;
1409
1410        if (r->where == UNW_WHERE_NONE || r->when >= sr->when_target)
1411                return;
1412
1413        opc = UNW_INSN_MOVE;
1414        val = rval = r->val;
1415        need_nat_info = (i >= UNW_REG_R4 && i <= UNW_REG_R7);
1416
1417        switch (r->where) {
1418              case UNW_WHERE_GR:
1419                if (rval >= 32) {
1420                        opc = UNW_INSN_MOVE_STACKED;
1421                        val = rval - 32;
1422                } else if (rval >= 4 && rval <= 7) {
1423                        if (need_nat_info) {
1424                                opc = UNW_INSN_MOVE2;
1425                                need_nat_info = 0;
1426                        }
1427                        val = unw.preg_index[UNW_REG_R4 + (rval - 4)];
1428                } else if (rval == 0) {
1429                        opc = UNW_INSN_MOVE_CONST;
1430                        val = 0;
1431                } else {
1432                        /* register got spilled to a scratch register */
1433                        opc = UNW_INSN_MOVE_SCRATCH;
1434                        val = pt_regs_off(rval);
1435                }
1436                break;
1437
1438              case UNW_WHERE_FR:
1439                if (rval <= 5)
1440                        val = unw.preg_index[UNW_REG_F2  + (rval -  2)];
1441                else if (rval >= 16 && rval <= 31)
1442                        val = unw.preg_index[UNW_REG_F16 + (rval - 16)];
1443                else {
1444                        opc = UNW_INSN_MOVE_SCRATCH;
1445                        if (rval <= 11)
1446                                val = offsetof(struct pt_regs, f6) + 16*(rval - 6);
1447                        else
1448                                UNW_DPRINT(0, "unwind.%s: kernel may not touch f%lu\n",
1449                                           __func__, rval);
1450                }
1451                break;
1452
1453              case UNW_WHERE_BR:
1454                if (rval >= 1 && rval <= 5)
1455                        val = unw.preg_index[UNW_REG_B1 + (rval - 1)];
1456                else {
1457                        opc = UNW_INSN_MOVE_SCRATCH;
1458                        if (rval == 0)
1459                                val = offsetof(struct pt_regs, b0);
1460                        else if (rval == 6)
1461                                val = offsetof(struct pt_regs, b6);
1462                        else
1463                                val = offsetof(struct pt_regs, b7);
1464                }
1465                break;
1466
1467              case UNW_WHERE_SPREL:
1468                opc = UNW_INSN_ADD_SP;
1469                break;
1470
1471              case UNW_WHERE_PSPREL:
1472                opc = UNW_INSN_ADD_PSP;
1473                break;
1474
1475              default:
1476                UNW_DPRINT(0, "unwind%s: register %u has unexpected `where' value of %u\n",
1477                           __func__, i, r->where);
1478                break;
1479        }
1480        insn.opc = opc;
1481        insn.dst = unw.preg_index[i];
1482        insn.val = val;
1483        script_emit(script, insn);
1484        if (need_nat_info)
1485                emit_nat_info(sr, i, script);
1486
1487        if (i == UNW_REG_PSP) {
1488                /*
1489                 * info->psp must contain the _value_ of the previous
1490                 * sp, not it's save location.  We get this by
1491                 * dereferencing the value we just stored in
1492                 * info->psp:
1493                 */
1494                insn.opc = UNW_INSN_LOAD;
1495                insn.dst = insn.val = unw.preg_index[UNW_REG_PSP];
1496                script_emit(script, insn);
1497        }
1498}
1499
1500static inline const struct unw_table_entry *
1501lookup (struct unw_table *table, unsigned long rel_ip)
1502{
1503        const struct unw_table_entry *e = NULL;
1504        unsigned long lo, hi, mid;
1505
1506        /* do a binary search for right entry: */
1507        for (lo = 0, hi = table->length; lo < hi; ) {
1508                mid = (lo + hi) / 2;
1509                e = &table->array[mid];
1510                if (rel_ip < e->start_offset)
1511                        hi = mid;
1512                else if (rel_ip >= e->end_offset)
1513                        lo = mid + 1;
1514                else
1515                        break;
1516        }
1517        if (rel_ip < e->start_offset || rel_ip >= e->end_offset)
1518                return NULL;
1519        return e;
1520}
1521
1522/*
1523 * Build an unwind script that unwinds from state OLD_STATE to the
1524 * entrypoint of the function that called OLD_STATE.
1525 */
1526static inline struct unw_script *
1527build_script (struct unw_frame_info *info)
1528{
1529        const struct unw_table_entry *e = NULL;
1530        struct unw_script *script = NULL;
1531        struct unw_labeled_state *ls, *next;
1532        unsigned long ip = info->ip;
1533        struct unw_state_record sr;
1534        struct unw_table *table, *prev;
1535        struct unw_reg_info *r;
1536        struct unw_insn insn;
1537        u8 *dp, *desc_end;
1538        u64 hdr;
1539        int i;
1540        STAT(unsigned long start, parse_start;)
1541
1542        STAT(++unw.stat.script.builds; start = ia64_get_itc());
1543
1544        /* build state record */
1545        memset(&sr, 0, sizeof(sr));
1546        for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1547                r->when = UNW_WHEN_NEVER;
1548        sr.pr_val = info->pr;
1549
1550        UNW_DPRINT(3, "unwind.%s: ip 0x%lx\n", __func__, ip);
1551        script = script_new(ip);
1552        if (!script) {
1553                UNW_DPRINT(0, "unwind.%s: failed to create unwind script\n",  __func__);
1554                STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1555                return NULL;
1556        }
1557        unw.cache[info->prev_script].hint = script - unw.cache;
1558
1559        /* search the kernels and the modules' unwind tables for IP: */
1560
1561        STAT(parse_start = ia64_get_itc());
1562
1563        prev = NULL;
1564        for (table = unw.tables; table; table = table->next) {
1565                if (ip >= table->start && ip < table->end) {
1566                        /*
1567                         * Leave the kernel unwind table at the very front,
1568                         * lest moving it breaks some assumption elsewhere.
1569                         * Otherwise, move the matching table to the second
1570                         * position in the list so that traversals can benefit
1571                         * from commonality in backtrace paths.
1572                         */
1573                        if (prev && prev != unw.tables) {
1574                                /* unw is safe - we're already spinlocked */
1575                                prev->next = table->next;
1576                                table->next = unw.tables->next;
1577                                unw.tables->next = table;
1578                        }
1579                        e = lookup(table, ip - table->segment_base);
1580                        break;
1581                }
1582                prev = table;
1583        }
1584        if (!e) {
1585                /* no info, return default unwinder (leaf proc, no mem stack, no saved regs)  */
1586                UNW_DPRINT(1, "unwind.%s: no unwind info for ip=0x%lx (prev ip=0x%lx)\n",
1587                        __func__, ip, unw.cache[info->prev_script].ip);
1588                sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1589                sr.curr.reg[UNW_REG_RP].when = -1;
1590                sr.curr.reg[UNW_REG_RP].val = 0;
1591                compile_reg(&sr, UNW_REG_RP, script);
1592                script_finalize(script, &sr);
1593                STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1594                STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1595                return script;
1596        }
1597
1598        sr.when_target = (3*((ip & ~0xfUL) - (table->segment_base + e->start_offset))/16
1599                          + (ip & 0xfUL));
1600        hdr = *(u64 *) (table->segment_base + e->info_offset);
1601        dp =   (u8 *)  (table->segment_base + e->info_offset + 8);
1602        desc_end = dp + 8*UNW_LENGTH(hdr);
1603
1604        while (!sr.done && dp < desc_end)
1605                dp = unw_decode(dp, sr.in_body, &sr);
1606
1607        if (sr.when_target > sr.epilogue_start) {
1608                /*
1609                 * sp has been restored and all values on the memory stack below
1610                 * psp also have been restored.
1611                 */
1612                sr.curr.reg[UNW_REG_PSP].val = 0;
1613                sr.curr.reg[UNW_REG_PSP].where = UNW_WHERE_NONE;
1614                sr.curr.reg[UNW_REG_PSP].when = UNW_WHEN_NEVER;
1615                for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1616                        if ((r->where == UNW_WHERE_PSPREL && r->val <= 0x10)
1617                            || r->where == UNW_WHERE_SPREL)
1618                        {
1619                                r->val = 0;
1620                                r->where = UNW_WHERE_NONE;
1621                                r->when = UNW_WHEN_NEVER;
1622                        }
1623        }
1624
1625        script->flags = sr.flags;
1626
1627        /*
1628         * If RP did't get saved, generate entry for the return link
1629         * register.
1630         */
1631        if (sr.curr.reg[UNW_REG_RP].when >= sr.when_target) {
1632                sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1633                sr.curr.reg[UNW_REG_RP].when = -1;
1634                sr.curr.reg[UNW_REG_RP].val = sr.return_link_reg;
1635                UNW_DPRINT(1, "unwind.%s: using default for rp at ip=0x%lx where=%d val=0x%lx\n",
1636                           __func__, ip, sr.curr.reg[UNW_REG_RP].where,
1637                           sr.curr.reg[UNW_REG_RP].val);
1638        }
1639
1640#ifdef UNW_DEBUG
1641        UNW_DPRINT(1, "unwind.%s: state record for func 0x%lx, t=%u:\n",
1642                __func__, table->segment_base + e->start_offset, sr.when_target);
1643        for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r) {
1644                if (r->where != UNW_WHERE_NONE || r->when != UNW_WHEN_NEVER) {
1645                        UNW_DPRINT(1, "  %s <- ", unw.preg_name[r - sr.curr.reg]);
1646                        switch (r->where) {
1647                              case UNW_WHERE_GR:     UNW_DPRINT(1, "r%lu", r->val); break;
1648                              case UNW_WHERE_FR:     UNW_DPRINT(1, "f%lu", r->val); break;
1649                              case UNW_WHERE_BR:     UNW_DPRINT(1, "b%lu", r->val); break;
1650                              case UNW_WHERE_SPREL:  UNW_DPRINT(1, "[sp+0x%lx]", r->val); break;
1651                              case UNW_WHERE_PSPREL: UNW_DPRINT(1, "[psp+0x%lx]", r->val); break;
1652                              case UNW_WHERE_NONE:
1653                                UNW_DPRINT(1, "%s+0x%lx", unw.preg_name[r - sr.curr.reg], r->val);
1654                                break;
1655
1656                              default:
1657                                UNW_DPRINT(1, "BADWHERE(%d)", r->where);
1658                                break;
1659                        }
1660                        UNW_DPRINT(1, "\t\t%d\n", r->when);
1661                }
1662        }
1663#endif
1664
1665        STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1666
1667        /* translate state record into unwinder instructions: */
1668
1669        /*
1670         * First, set psp if we're dealing with a fixed-size frame;
1671         * subsequent instructions may depend on this value.
1672         */
1673        if (sr.when_target > sr.curr.reg[UNW_REG_PSP].when
1674            && (sr.curr.reg[UNW_REG_PSP].where == UNW_WHERE_NONE)
1675            && sr.curr.reg[UNW_REG_PSP].val != 0) {
1676                /* new psp is sp plus frame size */
1677                insn.opc = UNW_INSN_ADD;
1678                insn.dst = offsetof(struct unw_frame_info, psp)/8;
1679                insn.val = sr.curr.reg[UNW_REG_PSP].val;        /* frame size */
1680                script_emit(script, insn);
1681        }
1682
1683        /* determine where the primary UNaT is: */
1684        if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1685                i = UNW_REG_PRI_UNAT_MEM;
1686        else if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when)
1687                i = UNW_REG_PRI_UNAT_GR;
1688        else if (sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when > sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1689                i = UNW_REG_PRI_UNAT_MEM;
1690        else
1691                i = UNW_REG_PRI_UNAT_GR;
1692
1693        compile_reg(&sr, i, script);
1694
1695        for (i = UNW_REG_BSP; i < UNW_NUM_REGS; ++i)
1696                compile_reg(&sr, i, script);
1697
1698        /* free labeled register states & stack: */
1699
1700        STAT(parse_start = ia64_get_itc());
1701        for (ls = sr.labeled_states; ls; ls = next) {
1702                next = ls->next;
1703                free_state_stack(&ls->saved_state);
1704                free_labeled_state(ls);
1705        }
1706        free_state_stack(&sr.curr);
1707        STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1708
1709        script_finalize(script, &sr);
1710        STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1711        return script;
1712}
1713
1714/*
1715 * Apply the unwinding actions represented by OPS and update SR to
1716 * reflect the state that existed upon entry to the function that this
1717 * unwinder represents.
1718 */
1719static inline void
1720run_script (struct unw_script *script, struct unw_frame_info *state)
1721{
1722        struct unw_insn *ip, *limit, next_insn;
1723        unsigned long opc, dst, val, off;
1724        unsigned long *s = (unsigned long *) state;
1725        STAT(unsigned long start;)
1726
1727        STAT(++unw.stat.script.runs; start = ia64_get_itc());
1728        state->flags = script->flags;
1729        ip = script->insn;
1730        limit = script->insn + script->count;
1731        next_insn = *ip;
1732
1733        while (ip++ < limit) {
1734                opc = next_insn.opc;
1735                dst = next_insn.dst;
1736                val = next_insn.val;
1737                next_insn = *ip;
1738
1739          redo:
1740                switch (opc) {
1741                      case UNW_INSN_ADD:
1742                        s[dst] += val;
1743                        break;
1744
1745                      case UNW_INSN_MOVE2:
1746                        if (!s[val])
1747                                goto lazy_init;
1748                        s[dst+1] = s[val+1];
1749                        s[dst] = s[val];
1750                        break;
1751
1752                      case UNW_INSN_MOVE:
1753                        if (!s[val])
1754                                goto lazy_init;
1755                        s[dst] = s[val];
1756                        break;
1757
1758                      case UNW_INSN_MOVE_SCRATCH:
1759                        if (state->pt) {
1760                                s[dst] = (unsigned long) get_scratch_regs(state) + val;
1761                        } else {
1762                                s[dst] = 0;
1763                                UNW_DPRINT(0, "unwind.%s: no state->pt, dst=%ld, val=%ld\n",
1764                                           __func__, dst, val);
1765                        }
1766                        break;
1767
1768                      case UNW_INSN_MOVE_CONST:
1769                        if (val == 0)
1770                                s[dst] = (unsigned long) &unw.r0;
1771                        else {
1772                                s[dst] = 0;
1773                                UNW_DPRINT(0, "unwind.%s: UNW_INSN_MOVE_CONST bad val=%ld\n",
1774                                           __func__, val);
1775                        }
1776                        break;
1777
1778
1779                      case UNW_INSN_MOVE_STACKED:
1780                        s[dst] = (unsigned long) ia64_rse_skip_regs((unsigned long *)state->bsp,
1781                                                                    val);
1782                        break;
1783
1784                      case UNW_INSN_ADD_PSP:
1785                        s[dst] = state->psp + val;
1786                        break;
1787
1788                      case UNW_INSN_ADD_SP:
1789                        s[dst] = state->sp + val;
1790                        break;
1791
1792                      case UNW_INSN_SETNAT_MEMSTK:
1793                        if (!state->pri_unat_loc)
1794                                state->pri_unat_loc = &state->sw->caller_unat;
1795                        /* register off. is a multiple of 8, so the least 3 bits (type) are 0 */
1796                        s[dst+1] = ((unsigned long) state->pri_unat_loc - s[dst]) | UNW_NAT_MEMSTK;
1797                        break;
1798
1799                      case UNW_INSN_SETNAT_TYPE:
1800                        s[dst+1] = val;
1801                        break;
1802
1803                      case UNW_INSN_LOAD:
1804#ifdef UNW_DEBUG
1805                        if ((s[val] & (local_cpu_data->unimpl_va_mask | 0x7)) != 0
1806                            || s[val] < TASK_SIZE)
1807                        {
1808                                UNW_DPRINT(0, "unwind.%s: rejecting bad psp=0x%lx\n",
1809                                           __func__, s[val]);
1810                                break;
1811                        }
1812#endif
1813                        s[dst] = *(unsigned long *) s[val];
1814                        break;
1815                }
1816        }
1817        STAT(unw.stat.script.run_time += ia64_get_itc() - start);
1818        return;
1819
1820  lazy_init:
1821        off = unw.sw_off[val];
1822        s[val] = (unsigned long) state->sw + off;
1823        if (off >= offsetof(struct switch_stack, r4) && off <= offsetof(struct switch_stack, r7))
1824                /*
1825                 * We're initializing a general register: init NaT info, too.  Note that
1826                 * the offset is a multiple of 8 which gives us the 3 bits needed for
1827                 * the type field.
1828                 */
1829                s[val+1] = (offsetof(struct switch_stack, ar_unat) - off) | UNW_NAT_MEMSTK;
1830        goto redo;
1831}
1832
1833static int
1834find_save_locs (struct unw_frame_info *info)
1835{
1836        int have_write_lock = 0;
1837        struct unw_script *scr;
1838        unsigned long flags = 0;
1839
1840        if ((info->ip & (local_cpu_data->unimpl_va_mask | 0xf)) || info->ip < TASK_SIZE) {
1841                /* don't let obviously bad addresses pollute the cache */
1842                /* FIXME: should really be level 0 but it occurs too often. KAO */
1843                UNW_DPRINT(1, "unwind.%s: rejecting bad ip=0x%lx\n", __func__, info->ip);
1844                info->rp_loc = NULL;
1845                return -1;
1846        }
1847
1848        scr = script_lookup(info);
1849        if (!scr) {
1850                spin_lock_irqsave(&unw.lock, flags);
1851                scr = build_script(info);
1852                if (!scr) {
1853                        spin_unlock_irqrestore(&unw.lock, flags);
1854                        UNW_DPRINT(0,
1855                                   "unwind.%s: failed to locate/build unwind script for ip %lx\n",
1856                                   __func__, info->ip);
1857                        return -1;
1858                }
1859                have_write_lock = 1;
1860        }
1861        info->hint = scr->hint;
1862        info->prev_script = scr - unw.cache;
1863
1864        run_script(scr, info);
1865
1866        if (have_write_lock) {
1867                write_unlock(&scr->lock);
1868                spin_unlock_irqrestore(&unw.lock, flags);
1869        } else
1870                read_unlock(&scr->lock);
1871        return 0;
1872}
1873
1874static int
1875unw_valid(const struct unw_frame_info *info, unsigned long* p)
1876{
1877        unsigned long loc = (unsigned long)p;
1878        return (loc >= info->regstk.limit && loc < info->regstk.top) ||
1879               (loc >= info->memstk.top && loc < info->memstk.limit);
1880}
1881
1882int
1883unw_unwind (struct unw_frame_info *info)
1884{
1885        unsigned long prev_ip, prev_sp, prev_bsp;
1886        unsigned long ip, pr, num_regs;
1887        STAT(unsigned long start, flags;)
1888        int retval;
1889
1890        STAT(local_irq_save(flags); ++unw.stat.api.unwinds; start = ia64_get_itc());
1891
1892        prev_ip = info->ip;
1893        prev_sp = info->sp;
1894        prev_bsp = info->bsp;
1895
1896        /* validate the return IP pointer */
1897        if (!unw_valid(info, info->rp_loc)) {
1898                /* FIXME: should really be level 0 but it occurs too often. KAO */
1899                UNW_DPRINT(1, "unwind.%s: failed to locate return link (ip=0x%lx)!\n",
1900                           __func__, info->ip);
1901                STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1902                return -1;
1903        }
1904        /* restore the ip */
1905        ip = info->ip = *info->rp_loc;
1906        if (ip < GATE_ADDR) {
1907                UNW_DPRINT(2, "unwind.%s: reached user-space (ip=0x%lx)\n", __func__, ip);
1908                STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1909                return -1;
1910        }
1911
1912        /* validate the previous stack frame pointer */
1913        if (!unw_valid(info, info->pfs_loc)) {
1914                UNW_DPRINT(0, "unwind.%s: failed to locate ar.pfs!\n", __func__);
1915                STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1916                return -1;
1917        }
1918        /* restore the cfm: */
1919        info->cfm_loc = info->pfs_loc;
1920
1921        /* restore the bsp: */
1922        pr = info->pr;
1923        num_regs = 0;
1924        if ((info->flags & UNW_FLAG_INTERRUPT_FRAME)) {
1925                info->pt = info->sp + 16;
1926                if ((pr & (1UL << PRED_NON_SYSCALL)) != 0)
1927                        num_regs = *info->cfm_loc & 0x7f;               /* size of frame */
1928                info->pfs_loc =
1929                        (unsigned long *) (info->pt + offsetof(struct pt_regs, ar_pfs));
1930                UNW_DPRINT(3, "unwind.%s: interrupt_frame pt 0x%lx\n", __func__, info->pt);
1931        } else
1932                num_regs = (*info->cfm_loc >> 7) & 0x7f;        /* size of locals */
1933        info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->bsp, -num_regs);
1934        if (info->bsp < info->regstk.limit || info->bsp > info->regstk.top) {
1935                UNW_DPRINT(0, "unwind.%s: bsp (0x%lx) out of range [0x%lx-0x%lx]\n",
1936                        __func__, info->bsp, info->regstk.limit, info->regstk.top);
1937                STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1938                return -1;
1939        }
1940
1941        /* restore the sp: */
1942        info->sp = info->psp;
1943        if (info->sp < info->memstk.top || info->sp > info->memstk.limit) {
1944                UNW_DPRINT(0, "unwind.%s: sp (0x%lx) out of range [0x%lx-0x%lx]\n",
1945                        __func__, info->sp, info->memstk.top, info->memstk.limit);
1946                STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1947                return -1;
1948        }
1949
1950        if (info->ip == prev_ip && info->sp == prev_sp && info->bsp == prev_bsp) {
1951                UNW_DPRINT(0, "unwind.%s: ip, sp, bsp unchanged; stopping here (ip=0x%lx)\n",
1952                           __func__, ip);
1953                STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1954                return -1;
1955        }
1956
1957        /* as we unwind, the saved ar.unat becomes the primary unat: */
1958        info->pri_unat_loc = info->unat_loc;
1959
1960        /* finally, restore the predicates: */
1961        unw_get_pr(info, &info->pr);
1962
1963        retval = find_save_locs(info);
1964        STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1965        return retval;
1966}
1967EXPORT_SYMBOL(unw_unwind);
1968
1969int
1970unw_unwind_to_user (struct unw_frame_info *info)
1971{
1972        unsigned long ip, sp, pr = info->pr;
1973
1974        do {
1975                unw_get_sp(info, &sp);
1976                if ((long)((unsigned long)info->task + IA64_STK_OFFSET - sp)
1977                    < IA64_PT_REGS_SIZE) {
1978                        UNW_DPRINT(0, "unwind.%s: ran off the top of the kernel stack\n",
1979                                   __func__);
1980                        break;
1981                }
1982                if (unw_is_intr_frame(info) &&
1983                    (pr & (1UL << PRED_USER_STACK)))
1984                        return 0;
1985                if (unw_get_pr (info, &pr) < 0) {
1986                        unw_get_rp(info, &ip);
1987                        UNW_DPRINT(0, "unwind.%s: failed to read "
1988                                   "predicate register (ip=0x%lx)\n",
1989                                __func__, ip);
1990                        return -1;
1991                }
1992        } while (unw_unwind(info) >= 0);
1993        unw_get_ip(info, &ip);
1994        UNW_DPRINT(0, "unwind.%s: failed to unwind to user-level (ip=0x%lx)\n",
1995                   __func__, ip);
1996        return -1;
1997}
1998EXPORT_SYMBOL(unw_unwind_to_user);
1999
2000static void
2001init_frame_info (struct unw_frame_info *info, struct task_struct *t,
2002                 struct switch_stack *sw, unsigned long stktop)
2003{
2004        unsigned long rbslimit, rbstop, stklimit;
2005        STAT(unsigned long start, flags;)
2006
2007        STAT(local_irq_save(flags); ++unw.stat.api.inits; start = ia64_get_itc());
2008
2009        /*
2010         * Subtle stuff here: we _could_ unwind through the switch_stack frame but we
2011         * don't want to do that because it would be slow as each preserved register would
2012         * have to be processed.  Instead, what we do here is zero out the frame info and
2013         * start the unwind process at the function that created the switch_stack frame.
2014         * When a preserved value in switch_stack needs to be accessed, run_script() will
2015         * initialize the appropriate pointer on demand.
2016         */
2017        memset(info, 0, sizeof(*info));
2018
2019        rbslimit = (unsigned long) t + IA64_RBS_OFFSET;
2020        stklimit = (unsigned long) t + IA64_STK_OFFSET;
2021
2022        rbstop   = sw->ar_bspstore;
2023        if (rbstop > stklimit || rbstop < rbslimit)
2024                rbstop = rbslimit;
2025
2026        if (stktop <= rbstop)
2027                stktop = rbstop;
2028        if (stktop > stklimit)
2029                stktop = stklimit;
2030
2031        info->regstk.limit = rbslimit;
2032        info->regstk.top   = rbstop;
2033        info->memstk.limit = stklimit;
2034        info->memstk.top   = stktop;
2035        info->task = t;
2036        info->sw  = sw;
2037        info->sp = info->psp = stktop;
2038        info->pr = sw->pr;
2039        UNW_DPRINT(3, "unwind.%s:\n"
2040                   "  task   0x%lx\n"
2041                   "  rbs = [0x%lx-0x%lx)\n"
2042                   "  stk = [0x%lx-0x%lx)\n"
2043                   "  pr     0x%lx\n"
2044                   "  sw     0x%lx\n"
2045                   "  sp     0x%lx\n",
2046                   __func__, (unsigned long) t, rbslimit, rbstop, stktop, stklimit,
2047                   info->pr, (unsigned long) info->sw, info->sp);
2048        STAT(unw.stat.api.init_time += ia64_get_itc() - start; local_irq_restore(flags));
2049}
2050
2051void
2052unw_init_frame_info (struct unw_frame_info *info, struct task_struct *t, struct switch_stack *sw)
2053{
2054        unsigned long sol;
2055
2056        init_frame_info(info, t, sw, (unsigned long) (sw + 1) - 16);
2057        info->cfm_loc = &sw->ar_pfs;
2058        sol = (*info->cfm_loc >> 7) & 0x7f;
2059        info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sol);
2060        info->ip = sw->b0;
2061        UNW_DPRINT(3, "unwind.%s:\n"
2062                   "  bsp    0x%lx\n"
2063                   "  sol    0x%lx\n"
2064                   "  ip     0x%lx\n",
2065                   __func__, info->bsp, sol, info->ip);
2066        find_save_locs(info);
2067}
2068
2069EXPORT_SYMBOL(unw_init_frame_info);
2070
2071void
2072unw_init_from_blocked_task (struct unw_frame_info *info, struct task_struct *t)
2073{
2074        struct switch_stack *sw = (struct switch_stack *) (t->thread.ksp + 16);
2075
2076        UNW_DPRINT(1, "unwind.%s\n", __func__);
2077        unw_init_frame_info(info, t, sw);
2078}
2079EXPORT_SYMBOL(unw_init_from_blocked_task);
2080
2081static void
2082init_unwind_table (struct unw_table *table, const char *name, unsigned long segment_base,
2083                   unsigned long gp, const void *table_start, const void *table_end)
2084{
2085        const struct unw_table_entry *start = table_start, *end = table_end;
2086
2087        table->name = name;
2088        table->segment_base = segment_base;
2089        table->gp = gp;
2090        table->start = segment_base + start[0].start_offset;
2091        table->end = segment_base + end[-1].end_offset;
2092        table->array = start;
2093        table->length = end - start;
2094}
2095
2096void *
2097unw_add_unwind_table (const char *name, unsigned long segment_base, unsigned long gp,
2098                      const void *table_start, const void *table_end)
2099{
2100        const struct unw_table_entry *start = table_start, *end = table_end;
2101        struct unw_table *table;
2102        unsigned long flags;
2103
2104        if (end - start <= 0) {
2105                UNW_DPRINT(0, "unwind.%s: ignoring attempt to insert empty unwind table\n",
2106                           __func__);
2107                return NULL;
2108        }
2109
2110        table = kmalloc(sizeof(*table), GFP_USER);
2111        if (!table)
2112                return NULL;
2113
2114        init_unwind_table(table, name, segment_base, gp, table_start, table_end);
2115
2116        spin_lock_irqsave(&unw.lock, flags);
2117        {
2118                /* keep kernel unwind table at the front (it's searched most commonly): */
2119                table->next = unw.tables->next;
2120                unw.tables->next = table;
2121        }
2122        spin_unlock_irqrestore(&unw.lock, flags);
2123
2124        return table;
2125}
2126
2127void
2128unw_remove_unwind_table (void *handle)
2129{
2130        struct unw_table *table, *prev;
2131        struct unw_script *tmp;
2132        unsigned long flags;
2133        long index;
2134
2135        if (!handle) {
2136                UNW_DPRINT(0, "unwind.%s: ignoring attempt to remove non-existent unwind table\n",
2137                           __func__);
2138                return;
2139        }
2140
2141        table = handle;
2142        if (table == &unw.kernel_table) {
2143                UNW_DPRINT(0, "unwind.%s: sorry, freeing the kernel's unwind table is a "
2144                           "no-can-do!\n", __func__);
2145                return;
2146        }
2147
2148        spin_lock_irqsave(&unw.lock, flags);
2149        {
2150                /* first, delete the table: */
2151
2152                for (prev = (struct unw_table *) &unw.tables; prev; prev = prev->next)
2153                        if (prev->next == table)
2154                                break;
2155                if (!prev) {
2156                        UNW_DPRINT(0, "unwind.%s: failed to find unwind table %p\n",
2157                                   __func__, (void *) table);
2158                        spin_unlock_irqrestore(&unw.lock, flags);
2159                        return;
2160                }
2161                prev->next = table->next;
2162        }
2163        spin_unlock_irqrestore(&unw.lock, flags);
2164
2165        /* next, remove hash table entries for this table */
2166
2167        for (index = 0; index < UNW_HASH_SIZE; ++index) {
2168                tmp = unw.cache + unw.hash[index];
2169                if (unw.hash[index] >= UNW_CACHE_SIZE
2170                    || tmp->ip < table->start || tmp->ip >= table->end)
2171                        continue;
2172
2173                write_lock(&tmp->lock);
2174                {
2175                        if (tmp->ip >= table->start && tmp->ip < table->end) {
2176                                unw.hash[index] = tmp->coll_chain;
2177                                tmp->ip = 0;
2178                        }
2179                }
2180                write_unlock(&tmp->lock);
2181        }
2182
2183        kfree(table);
2184}
2185
2186static int __init
2187create_gate_table (void)
2188{
2189        const struct unw_table_entry *entry, *start, *end;
2190        unsigned long *lp, segbase = GATE_ADDR;
2191        size_t info_size, size;
2192        char *info;
2193        Elf64_Phdr *punw = NULL, *phdr = (Elf64_Phdr *) (GATE_ADDR + GATE_EHDR->e_phoff);
2194        int i;
2195
2196        for (i = 0; i < GATE_EHDR->e_phnum; ++i, ++phdr)
2197                if (phdr->p_type == PT_IA_64_UNWIND) {
2198                        punw = phdr;
2199                        break;
2200                }
2201
2202        if (!punw) {
2203                printk("%s: failed to find gate DSO's unwind table!\n", __func__);
2204                return 0;
2205        }
2206
2207        start = (const struct unw_table_entry *) punw->p_vaddr;
2208        end = (struct unw_table_entry *) ((char *) start + punw->p_memsz);
2209        size  = 0;
2210
2211        unw_add_unwind_table("linux-gate.so", segbase, 0, start, end);
2212
2213        for (entry = start; entry < end; ++entry)
2214                size += 3*8 + 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2215        size += 8;      /* reserve space for "end of table" marker */
2216
2217        unw.gate_table = kmalloc(size, GFP_KERNEL);
2218        if (!unw.gate_table) {
2219                unw.gate_table_size = 0;
2220                printk(KERN_ERR "%s: unable to create unwind data for gate page!\n", __func__);
2221                return 0;
2222        }
2223        unw.gate_table_size = size;
2224
2225        lp = unw.gate_table;
2226        info = (char *) unw.gate_table + size;
2227
2228        for (entry = start; entry < end; ++entry, lp += 3) {
2229                info_size = 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2230                info -= info_size;
2231                memcpy(info, (char *) segbase + entry->info_offset, info_size);
2232
2233                lp[0] = segbase + entry->start_offset;          /* start */
2234                lp[1] = segbase + entry->end_offset;            /* end */
2235                lp[2] = info - (char *) unw.gate_table;         /* info */
2236        }
2237        *lp = 0;        /* end-of-table marker */
2238        return 0;
2239}
2240
2241__initcall(create_gate_table);
2242
2243void __init
2244unw_init (void)
2245{
2246        extern char __gp[];
2247        extern void unw_hash_index_t_is_too_narrow (void);
2248        long i, off;
2249
2250        if (8*sizeof(unw_hash_index_t) < UNW_LOG_HASH_SIZE)
2251                unw_hash_index_t_is_too_narrow();
2252
2253        unw.sw_off[unw.preg_index[UNW_REG_PRI_UNAT_GR]] = SW(CALLER_UNAT);
2254        unw.sw_off[unw.preg_index[UNW_REG_BSPSTORE]] = SW(AR_BSPSTORE);
2255        unw.sw_off[unw.preg_index[UNW_REG_PFS]] = SW(AR_PFS);
2256        unw.sw_off[unw.preg_index[UNW_REG_RP]] = SW(B0);
2257        unw.sw_off[unw.preg_index[UNW_REG_UNAT]] = SW(CALLER_UNAT);
2258        unw.sw_off[unw.preg_index[UNW_REG_PR]] = SW(PR);
2259        unw.sw_off[unw.preg_index[UNW_REG_LC]] = SW(AR_LC);
2260        unw.sw_off[unw.preg_index[UNW_REG_FPSR]] = SW(AR_FPSR);
2261        for (i = UNW_REG_R4, off = SW(R4); i <= UNW_REG_R7; ++i, off += 8)
2262                unw.sw_off[unw.preg_index[i]] = off;
2263        for (i = UNW_REG_B1, off = SW(B1); i <= UNW_REG_B5; ++i, off += 8)
2264                unw.sw_off[unw.preg_index[i]] = off;
2265        for (i = UNW_REG_F2, off = SW(F2); i <= UNW_REG_F5; ++i, off += 16)
2266                unw.sw_off[unw.preg_index[i]] = off;
2267        for (i = UNW_REG_F16, off = SW(F16); i <= UNW_REG_F31; ++i, off += 16)
2268                unw.sw_off[unw.preg_index[i]] = off;
2269
2270        for (i = 0; i < UNW_CACHE_SIZE; ++i) {
2271                if (i > 0)
2272                        unw.cache[i].lru_chain = (i - 1);
2273                unw.cache[i].coll_chain = -1;
2274                rwlock_init(&unw.cache[i].lock);
2275        }
2276        unw.lru_head = UNW_CACHE_SIZE - 1;
2277        unw.lru_tail = 0;
2278
2279        init_unwind_table(&unw.kernel_table, "kernel", KERNEL_START, (unsigned long) __gp,
2280                          __start_unwind, __end_unwind);
2281}
2282
2283/*
2284 * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2285 *
2286 *      This system call has been deprecated.  The new and improved way to get
2287 *      at the kernel's unwind info is via the gate DSO.  The address of the
2288 *      ELF header for this DSO is passed to user-level via AT_SYSINFO_EHDR.
2289 *
2290 * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2291 *
2292 * This system call copies the unwind data into the buffer pointed to by BUF and returns
2293 * the size of the unwind data.  If BUF_SIZE is smaller than the size of the unwind data
2294 * or if BUF is NULL, nothing is copied, but the system call still returns the size of the
2295 * unwind data.
2296 *
2297 * The first portion of the unwind data contains an unwind table and rest contains the
2298 * associated unwind info (in no particular order).  The unwind table consists of a table
2299 * of entries of the form:
2300 *
2301 *      u64 start;      (64-bit address of start of function)
2302 *      u64 end;        (64-bit address of start of function)
2303 *      u64 info;       (BUF-relative offset to unwind info)
2304 *
2305 * The end of the unwind table is indicated by an entry with a START address of zero.
2306 *
2307 * Please see the IA-64 Software Conventions and Runtime Architecture manual for details
2308 * on the format of the unwind info.
2309 *
2310 * ERRORS
2311 *      EFAULT  BUF points outside your accessible address space.
2312 */
2313asmlinkage long
2314sys_getunwind (void __user *buf, size_t buf_size)
2315{
2316        if (buf && buf_size >= unw.gate_table_size)
2317                if (copy_to_user(buf, unw.gate_table, unw.gate_table_size) != 0)
2318                        return -EFAULT;
2319        return unw.gate_table_size;
2320}
2321