linux/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c
<<
>>
Prefs
   1/*
   2 * Copyright 2012 Red Hat Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Ben Skeggs
  23 */
  24#include <subdev/bios.h>
  25#include <subdev/bios/bit.h>
  26#include <subdev/bios/bmp.h>
  27#include <subdev/bios/conn.h>
  28#include <subdev/bios/dcb.h>
  29#include <subdev/bios/dp.h>
  30#include <subdev/bios/gpio.h>
  31#include <subdev/bios/init.h>
  32#include <subdev/bios/ramcfg.h>
  33
  34#include <subdev/devinit.h>
  35#include <subdev/gpio.h>
  36#include <subdev/i2c.h>
  37#include <subdev/vga.h>
  38
  39#include <linux/kernel.h>
  40
  41#define bioslog(lvl, fmt, args...) do {                                        \
  42        nvkm_printk(init->subdev, lvl, info, "0x%08x[%c]: "fmt,                \
  43                    init->offset, init_exec(init) ?                            \
  44                    '0' + (init->nested - 1) : ' ', ##args);                   \
  45} while(0)
  46#define cont(fmt, args...) do {                                                \
  47        if (init->subdev->debug >= NV_DBG_TRACE)                               \
  48                printk(fmt, ##args);                                           \
  49} while(0)
  50#define trace(fmt, args...) bioslog(TRACE, fmt, ##args)
  51#define warn(fmt, args...) bioslog(WARN, fmt, ##args)
  52#define error(fmt, args...) bioslog(ERROR, fmt, ##args)
  53
  54/******************************************************************************
  55 * init parser control flow helpers
  56 *****************************************************************************/
  57
  58static inline bool
  59init_exec(struct nvbios_init *init)
  60{
  61        return (init->execute == 1) || ((init->execute & 5) == 5);
  62}
  63
  64static inline void
  65init_exec_set(struct nvbios_init *init, bool exec)
  66{
  67        if (exec) init->execute &= 0xfd;
  68        else      init->execute |= 0x02;
  69}
  70
  71static inline void
  72init_exec_inv(struct nvbios_init *init)
  73{
  74        init->execute ^= 0x02;
  75}
  76
  77static inline void
  78init_exec_force(struct nvbios_init *init, bool exec)
  79{
  80        if (exec) init->execute |= 0x04;
  81        else      init->execute &= 0xfb;
  82}
  83
  84/******************************************************************************
  85 * init parser wrappers for normal register/i2c/whatever accessors
  86 *****************************************************************************/
  87
  88static inline int
  89init_or(struct nvbios_init *init)
  90{
  91        if (init_exec(init)) {
  92                if (init->or >= 0)
  93                        return init->or;
  94                error("script needs OR!!\n");
  95        }
  96        return 0;
  97}
  98
  99static inline int
 100init_link(struct nvbios_init *init)
 101{
 102        if (init_exec(init)) {
 103                if (init->link)
 104                        return init->link == 2;
 105                error("script needs OR link\n");
 106        }
 107        return 0;
 108}
 109
 110static inline int
 111init_head(struct nvbios_init *init)
 112{
 113        if (init_exec(init)) {
 114                if (init->head >= 0)
 115                        return init->head;
 116                error("script needs head\n");
 117        }
 118        return 0;
 119}
 120
 121static u8
 122init_conn(struct nvbios_init *init)
 123{
 124        struct nvkm_bios *bios = init->subdev->device->bios;
 125        struct nvbios_connE connE;
 126        u8  ver, hdr;
 127        u32 conn;
 128
 129        if (init_exec(init)) {
 130                if (init->outp) {
 131                        conn = init->outp->connector;
 132                        conn = nvbios_connEp(bios, conn, &ver, &hdr, &connE);
 133                        if (conn)
 134                                return connE.type;
 135                }
 136
 137                error("script needs connector type\n");
 138        }
 139
 140        return 0xff;
 141}
 142
 143static inline u32
 144init_nvreg(struct nvbios_init *init, u32 reg)
 145{
 146        struct nvkm_devinit *devinit = init->subdev->device->devinit;
 147
 148        /* C51 (at least) sometimes has the lower bits set which the VBIOS
 149         * interprets to mean that access needs to go through certain IO
 150         * ports instead.  The NVIDIA binary driver has been seen to access
 151         * these through the NV register address, so lets assume we can
 152         * do the same
 153         */
 154        reg &= ~0x00000003;
 155
 156        /* GF8+ display scripts need register addresses mangled a bit to
 157         * select a specific CRTC/OR
 158         */
 159        if (init->subdev->device->card_type >= NV_50) {
 160                if (reg & 0x80000000) {
 161                        reg += init_head(init) * 0x800;
 162                        reg &= ~0x80000000;
 163                }
 164
 165                if (reg & 0x40000000) {
 166                        reg += init_or(init) * 0x800;
 167                        reg &= ~0x40000000;
 168                        if (reg & 0x20000000) {
 169                                reg += init_link(init) * 0x80;
 170                                reg &= ~0x20000000;
 171                        }
 172                }
 173        }
 174
 175        if (reg & ~0x00fffffc)
 176                warn("unknown bits in register 0x%08x\n", reg);
 177
 178        return nvkm_devinit_mmio(devinit, reg);
 179}
 180
 181static u32
 182init_rd32(struct nvbios_init *init, u32 reg)
 183{
 184        struct nvkm_device *device = init->subdev->device;
 185        reg = init_nvreg(init, reg);
 186        if (reg != ~0 && init_exec(init))
 187                return nvkm_rd32(device, reg);
 188        return 0x00000000;
 189}
 190
 191static void
 192init_wr32(struct nvbios_init *init, u32 reg, u32 val)
 193{
 194        struct nvkm_device *device = init->subdev->device;
 195        reg = init_nvreg(init, reg);
 196        if (reg != ~0 && init_exec(init))
 197                nvkm_wr32(device, reg, val);
 198}
 199
 200static u32
 201init_mask(struct nvbios_init *init, u32 reg, u32 mask, u32 val)
 202{
 203        struct nvkm_device *device = init->subdev->device;
 204        reg = init_nvreg(init, reg);
 205        if (reg != ~0 && init_exec(init)) {
 206                u32 tmp = nvkm_rd32(device, reg);
 207                nvkm_wr32(device, reg, (tmp & ~mask) | val);
 208                return tmp;
 209        }
 210        return 0x00000000;
 211}
 212
 213static u8
 214init_rdport(struct nvbios_init *init, u16 port)
 215{
 216        if (init_exec(init))
 217                return nvkm_rdport(init->subdev->device, init->head, port);
 218        return 0x00;
 219}
 220
 221static void
 222init_wrport(struct nvbios_init *init, u16 port, u8 value)
 223{
 224        if (init_exec(init))
 225                nvkm_wrport(init->subdev->device, init->head, port, value);
 226}
 227
 228static u8
 229init_rdvgai(struct nvbios_init *init, u16 port, u8 index)
 230{
 231        struct nvkm_subdev *subdev = init->subdev;
 232        if (init_exec(init)) {
 233                int head = init->head < 0 ? 0 : init->head;
 234                return nvkm_rdvgai(subdev->device, head, port, index);
 235        }
 236        return 0x00;
 237}
 238
 239static void
 240init_wrvgai(struct nvbios_init *init, u16 port, u8 index, u8 value)
 241{
 242        struct nvkm_device *device = init->subdev->device;
 243
 244        /* force head 0 for updates to cr44, it only exists on first head */
 245        if (device->card_type < NV_50) {
 246                if (port == 0x03d4 && index == 0x44)
 247                        init->head = 0;
 248        }
 249
 250        if (init_exec(init)) {
 251                int head = init->head < 0 ? 0 : init->head;
 252                nvkm_wrvgai(device, head, port, index, value);
 253        }
 254
 255        /* select head 1 if cr44 write selected it */
 256        if (device->card_type < NV_50) {
 257                if (port == 0x03d4 && index == 0x44 && value == 3)
 258                        init->head = 1;
 259        }
 260}
 261
 262static struct i2c_adapter *
 263init_i2c(struct nvbios_init *init, int index)
 264{
 265        struct nvkm_i2c *i2c = init->subdev->device->i2c;
 266        struct nvkm_i2c_bus *bus;
 267
 268        if (index == 0xff) {
 269                index = NVKM_I2C_BUS_PRI;
 270                if (init->outp && init->outp->i2c_upper_default)
 271                        index = NVKM_I2C_BUS_SEC;
 272        } else
 273        if (index == 0x80) {
 274                index = NVKM_I2C_BUS_PRI;
 275        } else
 276        if (index == 0x81) {
 277                index = NVKM_I2C_BUS_SEC;
 278        }
 279
 280        bus = nvkm_i2c_bus_find(i2c, index);
 281        return bus ? &bus->i2c : NULL;
 282}
 283
 284static int
 285init_rdi2cr(struct nvbios_init *init, u8 index, u8 addr, u8 reg)
 286{
 287        struct i2c_adapter *adap = init_i2c(init, index);
 288        if (adap && init_exec(init))
 289                return nvkm_rdi2cr(adap, addr, reg);
 290        return -ENODEV;
 291}
 292
 293static int
 294init_wri2cr(struct nvbios_init *init, u8 index, u8 addr, u8 reg, u8 val)
 295{
 296        struct i2c_adapter *adap = init_i2c(init, index);
 297        if (adap && init_exec(init))
 298                return nvkm_wri2cr(adap, addr, reg, val);
 299        return -ENODEV;
 300}
 301
 302static struct nvkm_i2c_aux *
 303init_aux(struct nvbios_init *init)
 304{
 305        struct nvkm_i2c *i2c = init->subdev->device->i2c;
 306        if (!init->outp) {
 307                if (init_exec(init))
 308                        error("script needs output for aux\n");
 309                return NULL;
 310        }
 311        return nvkm_i2c_aux_find(i2c, init->outp->i2c_index);
 312}
 313
 314static u8
 315init_rdauxr(struct nvbios_init *init, u32 addr)
 316{
 317        struct nvkm_i2c_aux *aux = init_aux(init);
 318        u8 data;
 319
 320        if (aux && init_exec(init)) {
 321                int ret = nvkm_rdaux(aux, addr, &data, 1);
 322                if (ret == 0)
 323                        return data;
 324                trace("auxch read failed with %d\n", ret);
 325        }
 326
 327        return 0x00;
 328}
 329
 330static int
 331init_wrauxr(struct nvbios_init *init, u32 addr, u8 data)
 332{
 333        struct nvkm_i2c_aux *aux = init_aux(init);
 334        if (aux && init_exec(init)) {
 335                int ret = nvkm_wraux(aux, addr, &data, 1);
 336                if (ret)
 337                        trace("auxch write failed with %d\n", ret);
 338                return ret;
 339        }
 340        return -ENODEV;
 341}
 342
 343static void
 344init_prog_pll(struct nvbios_init *init, u32 id, u32 freq)
 345{
 346        struct nvkm_devinit *devinit = init->subdev->device->devinit;
 347        if (init_exec(init)) {
 348                int ret = nvkm_devinit_pll_set(devinit, id, freq);
 349                if (ret)
 350                        warn("failed to prog pll 0x%08x to %dkHz\n", id, freq);
 351        }
 352}
 353
 354/******************************************************************************
 355 * parsing of bios structures that are required to execute init tables
 356 *****************************************************************************/
 357
 358static u16
 359init_table(struct nvkm_bios *bios, u16 *len)
 360{
 361        struct bit_entry bit_I;
 362
 363        if (!bit_entry(bios, 'I', &bit_I)) {
 364                *len = bit_I.length;
 365                return bit_I.offset;
 366        }
 367
 368        if (bmp_version(bios) >= 0x0510) {
 369                *len = 14;
 370                return bios->bmp_offset + 75;
 371        }
 372
 373        return 0x0000;
 374}
 375
 376static u16
 377init_table_(struct nvbios_init *init, u16 offset, const char *name)
 378{
 379        struct nvkm_bios *bios = init->subdev->device->bios;
 380        u16 len, data = init_table(bios, &len);
 381        if (data) {
 382                if (len >= offset + 2) {
 383                        data = nvbios_rd16(bios, data + offset);
 384                        if (data)
 385                                return data;
 386
 387                        warn("%s pointer invalid\n", name);
 388                        return 0x0000;
 389                }
 390
 391                warn("init data too short for %s pointer", name);
 392                return 0x0000;
 393        }
 394
 395        warn("init data not found\n");
 396        return 0x0000;
 397}
 398
 399#define init_script_table(b) init_table_((b), 0x00, "script table")
 400#define init_macro_index_table(b) init_table_((b), 0x02, "macro index table")
 401#define init_macro_table(b) init_table_((b), 0x04, "macro table")
 402#define init_condition_table(b) init_table_((b), 0x06, "condition table")
 403#define init_io_condition_table(b) init_table_((b), 0x08, "io condition table")
 404#define init_io_flag_condition_table(b) init_table_((b), 0x0a, "io flag condition table")
 405#define init_function_table(b) init_table_((b), 0x0c, "function table")
 406#define init_xlat_table(b) init_table_((b), 0x10, "xlat table");
 407
 408static u16
 409init_script(struct nvkm_bios *bios, int index)
 410{
 411        struct nvbios_init init = { .subdev = &bios->subdev };
 412        u16 bmp_ver = bmp_version(bios), data;
 413
 414        if (bmp_ver && bmp_ver < 0x0510) {
 415                if (index > 1 || bmp_ver < 0x0100)
 416                        return 0x0000;
 417
 418                data = bios->bmp_offset + (bmp_ver < 0x0200 ? 14 : 18);
 419                return nvbios_rd16(bios, data + (index * 2));
 420        }
 421
 422        data = init_script_table(&init);
 423        if (data)
 424                return nvbios_rd16(bios, data + (index * 2));
 425
 426        return 0x0000;
 427}
 428
 429static u16
 430init_unknown_script(struct nvkm_bios *bios)
 431{
 432        u16 len, data = init_table(bios, &len);
 433        if (data && len >= 16)
 434                return nvbios_rd16(bios, data + 14);
 435        return 0x0000;
 436}
 437
 438static u8
 439init_ram_restrict_group_count(struct nvbios_init *init)
 440{
 441        return nvbios_ramcfg_count(init->subdev->device->bios);
 442}
 443
 444static u8
 445init_ram_restrict(struct nvbios_init *init)
 446{
 447        /* This appears to be the behaviour of the VBIOS parser, and *is*
 448         * important to cache the NV_PEXTDEV_BOOT0 on later chipsets to
 449         * avoid fucking up the memory controller (somehow) by reading it
 450         * on every INIT_RAM_RESTRICT_ZM_GROUP opcode.
 451         *
 452         * Preserving the non-caching behaviour on earlier chipsets just
 453         * in case *not* re-reading the strap causes similar breakage.
 454         */
 455        if (!init->ramcfg || init->subdev->device->bios->version.major < 0x70)
 456                init->ramcfg = 0x80000000 | nvbios_ramcfg_index(init->subdev);
 457        return (init->ramcfg & 0x7fffffff);
 458}
 459
 460static u8
 461init_xlat_(struct nvbios_init *init, u8 index, u8 offset)
 462{
 463        struct nvkm_bios *bios = init->subdev->device->bios;
 464        u16 table = init_xlat_table(init);
 465        if (table) {
 466                u16 data = nvbios_rd16(bios, table + (index * 2));
 467                if (data)
 468                        return nvbios_rd08(bios, data + offset);
 469                warn("xlat table pointer %d invalid\n", index);
 470        }
 471        return 0x00;
 472}
 473
 474/******************************************************************************
 475 * utility functions used by various init opcode handlers
 476 *****************************************************************************/
 477
 478static bool
 479init_condition_met(struct nvbios_init *init, u8 cond)
 480{
 481        struct nvkm_bios *bios = init->subdev->device->bios;
 482        u16 table = init_condition_table(init);
 483        if (table) {
 484                u32 reg = nvbios_rd32(bios, table + (cond * 12) + 0);
 485                u32 msk = nvbios_rd32(bios, table + (cond * 12) + 4);
 486                u32 val = nvbios_rd32(bios, table + (cond * 12) + 8);
 487                trace("\t[0x%02x] (R[0x%06x] & 0x%08x) == 0x%08x\n",
 488                      cond, reg, msk, val);
 489                return (init_rd32(init, reg) & msk) == val;
 490        }
 491        return false;
 492}
 493
 494static bool
 495init_io_condition_met(struct nvbios_init *init, u8 cond)
 496{
 497        struct nvkm_bios *bios = init->subdev->device->bios;
 498        u16 table = init_io_condition_table(init);
 499        if (table) {
 500                u16 port = nvbios_rd16(bios, table + (cond * 5) + 0);
 501                u8 index = nvbios_rd08(bios, table + (cond * 5) + 2);
 502                u8  mask = nvbios_rd08(bios, table + (cond * 5) + 3);
 503                u8 value = nvbios_rd08(bios, table + (cond * 5) + 4);
 504                trace("\t[0x%02x] (0x%04x[0x%02x] & 0x%02x) == 0x%02x\n",
 505                      cond, port, index, mask, value);
 506                return (init_rdvgai(init, port, index) & mask) == value;
 507        }
 508        return false;
 509}
 510
 511static bool
 512init_io_flag_condition_met(struct nvbios_init *init, u8 cond)
 513{
 514        struct nvkm_bios *bios = init->subdev->device->bios;
 515        u16 table = init_io_flag_condition_table(init);
 516        if (table) {
 517                u16 port = nvbios_rd16(bios, table + (cond * 9) + 0);
 518                u8 index = nvbios_rd08(bios, table + (cond * 9) + 2);
 519                u8  mask = nvbios_rd08(bios, table + (cond * 9) + 3);
 520                u8 shift = nvbios_rd08(bios, table + (cond * 9) + 4);
 521                u16 data = nvbios_rd16(bios, table + (cond * 9) + 5);
 522                u8 dmask = nvbios_rd08(bios, table + (cond * 9) + 7);
 523                u8 value = nvbios_rd08(bios, table + (cond * 9) + 8);
 524                u8 ioval = (init_rdvgai(init, port, index) & mask) >> shift;
 525                return (nvbios_rd08(bios, data + ioval) & dmask) == value;
 526        }
 527        return false;
 528}
 529
 530static inline u32
 531init_shift(u32 data, u8 shift)
 532{
 533        if (shift < 0x80)
 534                return data >> shift;
 535        return data << (0x100 - shift);
 536}
 537
 538static u32
 539init_tmds_reg(struct nvbios_init *init, u8 tmds)
 540{
 541        /* For mlv < 0x80, it is an index into a table of TMDS base addresses.
 542         * For mlv == 0x80 use the "or" value of the dcb_entry indexed by
 543         * CR58 for CR57 = 0 to index a table of offsets to the basic
 544         * 0x6808b0 address.
 545         * For mlv == 0x81 use the "or" value of the dcb_entry indexed by
 546         * CR58 for CR57 = 0 to index a table of offsets to the basic
 547         * 0x6808b0 address, and then flip the offset by 8.
 548         */
 549        const int pramdac_offset[13] = {
 550                0, 0, 0x8, 0, 0x2000, 0, 0, 0, 0x2008, 0, 0, 0, 0x2000 };
 551        const u32 pramdac_table[4] = {
 552                0x6808b0, 0x6808b8, 0x6828b0, 0x6828b8 };
 553
 554        if (tmds >= 0x80) {
 555                if (init->outp) {
 556                        u32 dacoffset = pramdac_offset[init->outp->or];
 557                        if (tmds == 0x81)
 558                                dacoffset ^= 8;
 559                        return 0x6808b0 + dacoffset;
 560                }
 561
 562                if (init_exec(init))
 563                        error("tmds opcodes need dcb\n");
 564        } else {
 565                if (tmds < ARRAY_SIZE(pramdac_table))
 566                        return pramdac_table[tmds];
 567
 568                error("tmds selector 0x%02x unknown\n", tmds);
 569        }
 570
 571        return 0;
 572}
 573
 574/******************************************************************************
 575 * init opcode handlers
 576 *****************************************************************************/
 577
 578/**
 579 * init_reserved - stub for various unknown/unused single-byte opcodes
 580 *
 581 */
 582static void
 583init_reserved(struct nvbios_init *init)
 584{
 585        struct nvkm_bios *bios = init->subdev->device->bios;
 586        u8 opcode = nvbios_rd08(bios, init->offset);
 587        u8 length, i;
 588
 589        switch (opcode) {
 590        case 0xaa:
 591                length = 4;
 592                break;
 593        default:
 594                length = 1;
 595                break;
 596        }
 597
 598        trace("RESERVED 0x%02x\t", opcode);
 599        for (i = 1; i < length; i++)
 600                cont(" 0x%02x", nvbios_rd08(bios, init->offset + i));
 601        cont("\n");
 602        init->offset += length;
 603}
 604
 605/**
 606 * INIT_DONE - opcode 0x71
 607 *
 608 */
 609static void
 610init_done(struct nvbios_init *init)
 611{
 612        trace("DONE\n");
 613        init->offset = 0x0000;
 614}
 615
 616/**
 617 * INIT_IO_RESTRICT_PROG - opcode 0x32
 618 *
 619 */
 620static void
 621init_io_restrict_prog(struct nvbios_init *init)
 622{
 623        struct nvkm_bios *bios = init->subdev->device->bios;
 624        u16 port = nvbios_rd16(bios, init->offset + 1);
 625        u8 index = nvbios_rd08(bios, init->offset + 3);
 626        u8  mask = nvbios_rd08(bios, init->offset + 4);
 627        u8 shift = nvbios_rd08(bios, init->offset + 5);
 628        u8 count = nvbios_rd08(bios, init->offset + 6);
 629        u32  reg = nvbios_rd32(bios, init->offset + 7);
 630        u8 conf, i;
 631
 632        trace("IO_RESTRICT_PROG\tR[0x%06x] = "
 633              "((0x%04x[0x%02x] & 0x%02x) >> %d) [{\n",
 634              reg, port, index, mask, shift);
 635        init->offset += 11;
 636
 637        conf = (init_rdvgai(init, port, index) & mask) >> shift;
 638        for (i = 0; i < count; i++) {
 639                u32 data = nvbios_rd32(bios, init->offset);
 640
 641                if (i == conf) {
 642                        trace("\t0x%08x *\n", data);
 643                        init_wr32(init, reg, data);
 644                } else {
 645                        trace("\t0x%08x\n", data);
 646                }
 647
 648                init->offset += 4;
 649        }
 650        trace("}]\n");
 651}
 652
 653/**
 654 * INIT_REPEAT - opcode 0x33
 655 *
 656 */
 657static void
 658init_repeat(struct nvbios_init *init)
 659{
 660        struct nvkm_bios *bios = init->subdev->device->bios;
 661        u8 count = nvbios_rd08(bios, init->offset + 1);
 662        u16 repeat = init->repeat;
 663
 664        trace("REPEAT\t0x%02x\n", count);
 665        init->offset += 2;
 666
 667        init->repeat = init->offset;
 668        init->repend = init->offset;
 669        while (count--) {
 670                init->offset = init->repeat;
 671                nvbios_exec(init);
 672                if (count)
 673                        trace("REPEAT\t0x%02x\n", count);
 674        }
 675        init->offset = init->repend;
 676        init->repeat = repeat;
 677}
 678
 679/**
 680 * INIT_IO_RESTRICT_PLL - opcode 0x34
 681 *
 682 */
 683static void
 684init_io_restrict_pll(struct nvbios_init *init)
 685{
 686        struct nvkm_bios *bios = init->subdev->device->bios;
 687        u16 port = nvbios_rd16(bios, init->offset + 1);
 688        u8 index = nvbios_rd08(bios, init->offset + 3);
 689        u8  mask = nvbios_rd08(bios, init->offset + 4);
 690        u8 shift = nvbios_rd08(bios, init->offset + 5);
 691        s8  iofc = nvbios_rd08(bios, init->offset + 6);
 692        u8 count = nvbios_rd08(bios, init->offset + 7);
 693        u32  reg = nvbios_rd32(bios, init->offset + 8);
 694        u8 conf, i;
 695
 696        trace("IO_RESTRICT_PLL\tR[0x%06x] =PLL= "
 697              "((0x%04x[0x%02x] & 0x%02x) >> 0x%02x) IOFCOND 0x%02x [{\n",
 698              reg, port, index, mask, shift, iofc);
 699        init->offset += 12;
 700
 701        conf = (init_rdvgai(init, port, index) & mask) >> shift;
 702        for (i = 0; i < count; i++) {
 703                u32 freq = nvbios_rd16(bios, init->offset) * 10;
 704
 705                if (i == conf) {
 706                        trace("\t%dkHz *\n", freq);
 707                        if (iofc > 0 && init_io_flag_condition_met(init, iofc))
 708                                freq *= 2;
 709                        init_prog_pll(init, reg, freq);
 710                } else {
 711                        trace("\t%dkHz\n", freq);
 712                }
 713
 714                init->offset += 2;
 715        }
 716        trace("}]\n");
 717}
 718
 719/**
 720 * INIT_END_REPEAT - opcode 0x36
 721 *
 722 */
 723static void
 724init_end_repeat(struct nvbios_init *init)
 725{
 726        trace("END_REPEAT\n");
 727        init->offset += 1;
 728
 729        if (init->repeat) {
 730                init->repend = init->offset;
 731                init->offset = 0;
 732        }
 733}
 734
 735/**
 736 * INIT_COPY - opcode 0x37
 737 *
 738 */
 739static void
 740init_copy(struct nvbios_init *init)
 741{
 742        struct nvkm_bios *bios = init->subdev->device->bios;
 743        u32  reg = nvbios_rd32(bios, init->offset + 1);
 744        u8 shift = nvbios_rd08(bios, init->offset + 5);
 745        u8 smask = nvbios_rd08(bios, init->offset + 6);
 746        u16 port = nvbios_rd16(bios, init->offset + 7);
 747        u8 index = nvbios_rd08(bios, init->offset + 9);
 748        u8  mask = nvbios_rd08(bios, init->offset + 10);
 749        u8  data;
 750
 751        trace("COPY\t0x%04x[0x%02x] &= 0x%02x |= "
 752              "((R[0x%06x] %s 0x%02x) & 0x%02x)\n",
 753              port, index, mask, reg, (shift & 0x80) ? "<<" : ">>",
 754              (shift & 0x80) ? (0x100 - shift) : shift, smask);
 755        init->offset += 11;
 756
 757        data  = init_rdvgai(init, port, index) & mask;
 758        data |= init_shift(init_rd32(init, reg), shift) & smask;
 759        init_wrvgai(init, port, index, data);
 760}
 761
 762/**
 763 * INIT_NOT - opcode 0x38
 764 *
 765 */
 766static void
 767init_not(struct nvbios_init *init)
 768{
 769        trace("NOT\n");
 770        init->offset += 1;
 771        init_exec_inv(init);
 772}
 773
 774/**
 775 * INIT_IO_FLAG_CONDITION - opcode 0x39
 776 *
 777 */
 778static void
 779init_io_flag_condition(struct nvbios_init *init)
 780{
 781        struct nvkm_bios *bios = init->subdev->device->bios;
 782        u8 cond = nvbios_rd08(bios, init->offset + 1);
 783
 784        trace("IO_FLAG_CONDITION\t0x%02x\n", cond);
 785        init->offset += 2;
 786
 787        if (!init_io_flag_condition_met(init, cond))
 788                init_exec_set(init, false);
 789}
 790
 791/**
 792 * INIT_GENERIC_CONDITION - opcode 0x3a
 793 *
 794 */
 795static void
 796init_generic_condition(struct nvbios_init *init)
 797{
 798        struct nvkm_bios *bios = init->subdev->device->bios;
 799        struct nvbios_dpout info;
 800        u8  cond = nvbios_rd08(bios, init->offset + 1);
 801        u8  size = nvbios_rd08(bios, init->offset + 2);
 802        u8  ver, hdr, cnt, len;
 803        u16 data;
 804
 805        trace("GENERIC_CONDITION\t0x%02x 0x%02x\n", cond, size);
 806        init->offset += 3;
 807
 808        switch (cond) {
 809        case 0: /* CONDITION_ID_INT_DP. */
 810                if (init_conn(init) != DCB_CONNECTOR_eDP)
 811                        init_exec_set(init, false);
 812                break;
 813        case 1: /* CONDITION_ID_USE_SPPLL0. */
 814        case 2: /* CONDITION_ID_USE_SPPLL1. */
 815                if ( init->outp &&
 816                    (data = nvbios_dpout_match(bios, DCB_OUTPUT_DP,
 817                                               (init->outp->or << 0) |
 818                                               (init->outp->sorconf.link << 6),
 819                                               &ver, &hdr, &cnt, &len, &info)))
 820                {
 821                        if (!(info.flags & cond))
 822                                init_exec_set(init, false);
 823                        break;
 824                }
 825
 826                if (init_exec(init))
 827                        warn("script needs dp output table data\n");
 828                break;
 829        case 5: /* CONDITION_ID_ASSR_SUPPORT. */
 830                if (!(init_rdauxr(init, 0x0d) & 1))
 831                        init_exec_set(init, false);
 832                break;
 833        case 7: /* CONDITION_ID_NO_PANEL_SEQ_DELAYS. */
 834                init_exec_set(init, false);
 835                break;
 836        default:
 837                warn("INIT_GENERIC_CONDITION: unknown 0x%02x\n", cond);
 838                init->offset += size;
 839                break;
 840        }
 841}
 842
 843/**
 844 * INIT_IO_MASK_OR - opcode 0x3b
 845 *
 846 */
 847static void
 848init_io_mask_or(struct nvbios_init *init)
 849{
 850        struct nvkm_bios *bios = init->subdev->device->bios;
 851        u8 index = nvbios_rd08(bios, init->offset + 1);
 852        u8    or = init_or(init);
 853        u8  data;
 854
 855        trace("IO_MASK_OR\t0x03d4[0x%02x] &= ~(1 << 0x%02x)\n", index, or);
 856        init->offset += 2;
 857
 858        data = init_rdvgai(init, 0x03d4, index);
 859        init_wrvgai(init, 0x03d4, index, data &= ~(1 << or));
 860}
 861
 862/**
 863 * INIT_IO_OR - opcode 0x3c
 864 *
 865 */
 866static void
 867init_io_or(struct nvbios_init *init)
 868{
 869        struct nvkm_bios *bios = init->subdev->device->bios;
 870        u8 index = nvbios_rd08(bios, init->offset + 1);
 871        u8    or = init_or(init);
 872        u8  data;
 873
 874        trace("IO_OR\t0x03d4[0x%02x] |= (1 << 0x%02x)\n", index, or);
 875        init->offset += 2;
 876
 877        data = init_rdvgai(init, 0x03d4, index);
 878        init_wrvgai(init, 0x03d4, index, data | (1 << or));
 879}
 880
 881/**
 882 * INIT_ANDN_REG - opcode 0x47
 883 *
 884 */
 885static void
 886init_andn_reg(struct nvbios_init *init)
 887{
 888        struct nvkm_bios *bios = init->subdev->device->bios;
 889        u32  reg = nvbios_rd32(bios, init->offset + 1);
 890        u32 mask = nvbios_rd32(bios, init->offset + 5);
 891
 892        trace("ANDN_REG\tR[0x%06x] &= ~0x%08x\n", reg, mask);
 893        init->offset += 9;
 894
 895        init_mask(init, reg, mask, 0);
 896}
 897
 898/**
 899 * INIT_OR_REG - opcode 0x48
 900 *
 901 */
 902static void
 903init_or_reg(struct nvbios_init *init)
 904{
 905        struct nvkm_bios *bios = init->subdev->device->bios;
 906        u32  reg = nvbios_rd32(bios, init->offset + 1);
 907        u32 mask = nvbios_rd32(bios, init->offset + 5);
 908
 909        trace("OR_REG\tR[0x%06x] |= 0x%08x\n", reg, mask);
 910        init->offset += 9;
 911
 912        init_mask(init, reg, 0, mask);
 913}
 914
 915/**
 916 * INIT_INDEX_ADDRESS_LATCHED - opcode 0x49
 917 *
 918 */
 919static void
 920init_idx_addr_latched(struct nvbios_init *init)
 921{
 922        struct nvkm_bios *bios = init->subdev->device->bios;
 923        u32 creg = nvbios_rd32(bios, init->offset + 1);
 924        u32 dreg = nvbios_rd32(bios, init->offset + 5);
 925        u32 mask = nvbios_rd32(bios, init->offset + 9);
 926        u32 data = nvbios_rd32(bios, init->offset + 13);
 927        u8 count = nvbios_rd08(bios, init->offset + 17);
 928
 929        trace("INDEX_ADDRESS_LATCHED\tR[0x%06x] : R[0x%06x]\n", creg, dreg);
 930        trace("\tCTRL &= 0x%08x |= 0x%08x\n", mask, data);
 931        init->offset += 18;
 932
 933        while (count--) {
 934                u8 iaddr = nvbios_rd08(bios, init->offset + 0);
 935                u8 idata = nvbios_rd08(bios, init->offset + 1);
 936
 937                trace("\t[0x%02x] = 0x%02x\n", iaddr, idata);
 938                init->offset += 2;
 939
 940                init_wr32(init, dreg, idata);
 941                init_mask(init, creg, ~mask, data | iaddr);
 942        }
 943}
 944
 945/**
 946 * INIT_IO_RESTRICT_PLL2 - opcode 0x4a
 947 *
 948 */
 949static void
 950init_io_restrict_pll2(struct nvbios_init *init)
 951{
 952        struct nvkm_bios *bios = init->subdev->device->bios;
 953        u16 port = nvbios_rd16(bios, init->offset + 1);
 954        u8 index = nvbios_rd08(bios, init->offset + 3);
 955        u8  mask = nvbios_rd08(bios, init->offset + 4);
 956        u8 shift = nvbios_rd08(bios, init->offset + 5);
 957        u8 count = nvbios_rd08(bios, init->offset + 6);
 958        u32  reg = nvbios_rd32(bios, init->offset + 7);
 959        u8  conf, i;
 960
 961        trace("IO_RESTRICT_PLL2\t"
 962              "R[0x%06x] =PLL= ((0x%04x[0x%02x] & 0x%02x) >> 0x%02x) [{\n",
 963              reg, port, index, mask, shift);
 964        init->offset += 11;
 965
 966        conf = (init_rdvgai(init, port, index) & mask) >> shift;
 967        for (i = 0; i < count; i++) {
 968                u32 freq = nvbios_rd32(bios, init->offset);
 969                if (i == conf) {
 970                        trace("\t%dkHz *\n", freq);
 971                        init_prog_pll(init, reg, freq);
 972                } else {
 973                        trace("\t%dkHz\n", freq);
 974                }
 975                init->offset += 4;
 976        }
 977        trace("}]\n");
 978}
 979
 980/**
 981 * INIT_PLL2 - opcode 0x4b
 982 *
 983 */
 984static void
 985init_pll2(struct nvbios_init *init)
 986{
 987        struct nvkm_bios *bios = init->subdev->device->bios;
 988        u32  reg = nvbios_rd32(bios, init->offset + 1);
 989        u32 freq = nvbios_rd32(bios, init->offset + 5);
 990
 991        trace("PLL2\tR[0x%06x] =PLL= %dkHz\n", reg, freq);
 992        init->offset += 9;
 993
 994        init_prog_pll(init, reg, freq);
 995}
 996
 997/**
 998 * INIT_I2C_BYTE - opcode 0x4c
 999 *
1000 */
1001static void
1002init_i2c_byte(struct nvbios_init *init)
1003{
1004        struct nvkm_bios *bios = init->subdev->device->bios;
1005        u8 index = nvbios_rd08(bios, init->offset + 1);
1006        u8  addr = nvbios_rd08(bios, init->offset + 2) >> 1;
1007        u8 count = nvbios_rd08(bios, init->offset + 3);
1008
1009        trace("I2C_BYTE\tI2C[0x%02x][0x%02x]\n", index, addr);
1010        init->offset += 4;
1011
1012        while (count--) {
1013                u8  reg = nvbios_rd08(bios, init->offset + 0);
1014                u8 mask = nvbios_rd08(bios, init->offset + 1);
1015                u8 data = nvbios_rd08(bios, init->offset + 2);
1016                int val;
1017
1018                trace("\t[0x%02x] &= 0x%02x |= 0x%02x\n", reg, mask, data);
1019                init->offset += 3;
1020
1021                val = init_rdi2cr(init, index, addr, reg);
1022                if (val < 0)
1023                        continue;
1024                init_wri2cr(init, index, addr, reg, (val & mask) | data);
1025        }
1026}
1027
1028/**
1029 * INIT_ZM_I2C_BYTE - opcode 0x4d
1030 *
1031 */
1032static void
1033init_zm_i2c_byte(struct nvbios_init *init)
1034{
1035        struct nvkm_bios *bios = init->subdev->device->bios;
1036        u8 index = nvbios_rd08(bios, init->offset + 1);
1037        u8  addr = nvbios_rd08(bios, init->offset + 2) >> 1;
1038        u8 count = nvbios_rd08(bios, init->offset + 3);
1039
1040        trace("ZM_I2C_BYTE\tI2C[0x%02x][0x%02x]\n", index, addr);
1041        init->offset += 4;
1042
1043        while (count--) {
1044                u8  reg = nvbios_rd08(bios, init->offset + 0);
1045                u8 data = nvbios_rd08(bios, init->offset + 1);
1046
1047                trace("\t[0x%02x] = 0x%02x\n", reg, data);
1048                init->offset += 2;
1049
1050                init_wri2cr(init, index, addr, reg, data);
1051        }
1052}
1053
1054/**
1055 * INIT_ZM_I2C - opcode 0x4e
1056 *
1057 */
1058static void
1059init_zm_i2c(struct nvbios_init *init)
1060{
1061        struct nvkm_bios *bios = init->subdev->device->bios;
1062        u8 index = nvbios_rd08(bios, init->offset + 1);
1063        u8  addr = nvbios_rd08(bios, init->offset + 2) >> 1;
1064        u8 count = nvbios_rd08(bios, init->offset + 3);
1065        u8 data[256], i;
1066
1067        trace("ZM_I2C\tI2C[0x%02x][0x%02x]\n", index, addr);
1068        init->offset += 4;
1069
1070        for (i = 0; i < count; i++) {
1071                data[i] = nvbios_rd08(bios, init->offset);
1072                trace("\t0x%02x\n", data[i]);
1073                init->offset++;
1074        }
1075
1076        if (init_exec(init)) {
1077                struct i2c_adapter *adap = init_i2c(init, index);
1078                struct i2c_msg msg = {
1079                        .addr = addr, .flags = 0, .len = count, .buf = data,
1080                };
1081                int ret;
1082
1083                if (adap && (ret = i2c_transfer(adap, &msg, 1)) != 1)
1084                        warn("i2c wr failed, %d\n", ret);
1085        }
1086}
1087
1088/**
1089 * INIT_TMDS - opcode 0x4f
1090 *
1091 */
1092static void
1093init_tmds(struct nvbios_init *init)
1094{
1095        struct nvkm_bios *bios = init->subdev->device->bios;
1096        u8 tmds = nvbios_rd08(bios, init->offset + 1);
1097        u8 addr = nvbios_rd08(bios, init->offset + 2);
1098        u8 mask = nvbios_rd08(bios, init->offset + 3);
1099        u8 data = nvbios_rd08(bios, init->offset + 4);
1100        u32 reg = init_tmds_reg(init, tmds);
1101
1102        trace("TMDS\tT[0x%02x][0x%02x] &= 0x%02x |= 0x%02x\n",
1103              tmds, addr, mask, data);
1104        init->offset += 5;
1105
1106        if (reg == 0)
1107                return;
1108
1109        init_wr32(init, reg + 0, addr | 0x00010000);
1110        init_wr32(init, reg + 4, data | (init_rd32(init, reg + 4) & mask));
1111        init_wr32(init, reg + 0, addr);
1112}
1113
1114/**
1115 * INIT_ZM_TMDS_GROUP - opcode 0x50
1116 *
1117 */
1118static void
1119init_zm_tmds_group(struct nvbios_init *init)
1120{
1121        struct nvkm_bios *bios = init->subdev->device->bios;
1122        u8  tmds = nvbios_rd08(bios, init->offset + 1);
1123        u8 count = nvbios_rd08(bios, init->offset + 2);
1124        u32  reg = init_tmds_reg(init, tmds);
1125
1126        trace("TMDS_ZM_GROUP\tT[0x%02x]\n", tmds);
1127        init->offset += 3;
1128
1129        while (count--) {
1130                u8 addr = nvbios_rd08(bios, init->offset + 0);
1131                u8 data = nvbios_rd08(bios, init->offset + 1);
1132
1133                trace("\t[0x%02x] = 0x%02x\n", addr, data);
1134                init->offset += 2;
1135
1136                init_wr32(init, reg + 4, data);
1137                init_wr32(init, reg + 0, addr);
1138        }
1139}
1140
1141/**
1142 * INIT_CR_INDEX_ADDRESS_LATCHED - opcode 0x51
1143 *
1144 */
1145static void
1146init_cr_idx_adr_latch(struct nvbios_init *init)
1147{
1148        struct nvkm_bios *bios = init->subdev->device->bios;
1149        u8 addr0 = nvbios_rd08(bios, init->offset + 1);
1150        u8 addr1 = nvbios_rd08(bios, init->offset + 2);
1151        u8  base = nvbios_rd08(bios, init->offset + 3);
1152        u8 count = nvbios_rd08(bios, init->offset + 4);
1153        u8 save0;
1154
1155        trace("CR_INDEX_ADDR C[%02x] C[%02x]\n", addr0, addr1);
1156        init->offset += 5;
1157
1158        save0 = init_rdvgai(init, 0x03d4, addr0);
1159        while (count--) {
1160                u8 data = nvbios_rd08(bios, init->offset);
1161
1162                trace("\t\t[0x%02x] = 0x%02x\n", base, data);
1163                init->offset += 1;
1164
1165                init_wrvgai(init, 0x03d4, addr0, base++);
1166                init_wrvgai(init, 0x03d4, addr1, data);
1167        }
1168        init_wrvgai(init, 0x03d4, addr0, save0);
1169}
1170
1171/**
1172 * INIT_CR - opcode 0x52
1173 *
1174 */
1175static void
1176init_cr(struct nvbios_init *init)
1177{
1178        struct nvkm_bios *bios = init->subdev->device->bios;
1179        u8 addr = nvbios_rd08(bios, init->offset + 1);
1180        u8 mask = nvbios_rd08(bios, init->offset + 2);
1181        u8 data = nvbios_rd08(bios, init->offset + 3);
1182        u8 val;
1183
1184        trace("CR\t\tC[0x%02x] &= 0x%02x |= 0x%02x\n", addr, mask, data);
1185        init->offset += 4;
1186
1187        val = init_rdvgai(init, 0x03d4, addr) & mask;
1188        init_wrvgai(init, 0x03d4, addr, val | data);
1189}
1190
1191/**
1192 * INIT_ZM_CR - opcode 0x53
1193 *
1194 */
1195static void
1196init_zm_cr(struct nvbios_init *init)
1197{
1198        struct nvkm_bios *bios = init->subdev->device->bios;
1199        u8 addr = nvbios_rd08(bios, init->offset + 1);
1200        u8 data = nvbios_rd08(bios, init->offset + 2);
1201
1202        trace("ZM_CR\tC[0x%02x] = 0x%02x\n", addr,  data);
1203        init->offset += 3;
1204
1205        init_wrvgai(init, 0x03d4, addr, data);
1206}
1207
1208/**
1209 * INIT_ZM_CR_GROUP - opcode 0x54
1210 *
1211 */
1212static void
1213init_zm_cr_group(struct nvbios_init *init)
1214{
1215        struct nvkm_bios *bios = init->subdev->device->bios;
1216        u8 count = nvbios_rd08(bios, init->offset + 1);
1217
1218        trace("ZM_CR_GROUP\n");
1219        init->offset += 2;
1220
1221        while (count--) {
1222                u8 addr = nvbios_rd08(bios, init->offset + 0);
1223                u8 data = nvbios_rd08(bios, init->offset + 1);
1224
1225                trace("\t\tC[0x%02x] = 0x%02x\n", addr, data);
1226                init->offset += 2;
1227
1228                init_wrvgai(init, 0x03d4, addr, data);
1229        }
1230}
1231
1232/**
1233 * INIT_CONDITION_TIME - opcode 0x56
1234 *
1235 */
1236static void
1237init_condition_time(struct nvbios_init *init)
1238{
1239        struct nvkm_bios *bios = init->subdev->device->bios;
1240        u8  cond = nvbios_rd08(bios, init->offset + 1);
1241        u8 retry = nvbios_rd08(bios, init->offset + 2);
1242        u8  wait = min((u16)retry * 50, 100);
1243
1244        trace("CONDITION_TIME\t0x%02x 0x%02x\n", cond, retry);
1245        init->offset += 3;
1246
1247        if (!init_exec(init))
1248                return;
1249
1250        while (wait--) {
1251                if (init_condition_met(init, cond))
1252                        return;
1253                mdelay(20);
1254        }
1255
1256        init_exec_set(init, false);
1257}
1258
1259/**
1260 * INIT_LTIME - opcode 0x57
1261 *
1262 */
1263static void
1264init_ltime(struct nvbios_init *init)
1265{
1266        struct nvkm_bios *bios = init->subdev->device->bios;
1267        u16 msec = nvbios_rd16(bios, init->offset + 1);
1268
1269        trace("LTIME\t0x%04x\n", msec);
1270        init->offset += 3;
1271
1272        if (init_exec(init))
1273                mdelay(msec);
1274}
1275
1276/**
1277 * INIT_ZM_REG_SEQUENCE - opcode 0x58
1278 *
1279 */
1280static void
1281init_zm_reg_sequence(struct nvbios_init *init)
1282{
1283        struct nvkm_bios *bios = init->subdev->device->bios;
1284        u32 base = nvbios_rd32(bios, init->offset + 1);
1285        u8 count = nvbios_rd08(bios, init->offset + 5);
1286
1287        trace("ZM_REG_SEQUENCE\t0x%02x\n", count);
1288        init->offset += 6;
1289
1290        while (count--) {
1291                u32 data = nvbios_rd32(bios, init->offset);
1292
1293                trace("\t\tR[0x%06x] = 0x%08x\n", base, data);
1294                init->offset += 4;
1295
1296                init_wr32(init, base, data);
1297                base += 4;
1298        }
1299}
1300
1301/**
1302 * INIT_PLL_INDIRECT - opcode 0x59
1303 *
1304 */
1305static void
1306init_pll_indirect(struct nvbios_init *init)
1307{
1308        struct nvkm_bios *bios = init->subdev->device->bios;
1309        u32  reg = nvbios_rd32(bios, init->offset + 1);
1310        u16 addr = nvbios_rd16(bios, init->offset + 5);
1311        u32 freq = (u32)nvbios_rd16(bios, addr) * 1000;
1312
1313        trace("PLL_INDIRECT\tR[0x%06x] =PLL= VBIOS[%04x] = %dkHz\n",
1314              reg, addr, freq);
1315        init->offset += 7;
1316
1317        init_prog_pll(init, reg, freq);
1318}
1319
1320/**
1321 * INIT_ZM_REG_INDIRECT - opcode 0x5a
1322 *
1323 */
1324static void
1325init_zm_reg_indirect(struct nvbios_init *init)
1326{
1327        struct nvkm_bios *bios = init->subdev->device->bios;
1328        u32  reg = nvbios_rd32(bios, init->offset + 1);
1329        u16 addr = nvbios_rd16(bios, init->offset + 5);
1330        u32 data = nvbios_rd32(bios, addr);
1331
1332        trace("ZM_REG_INDIRECT\tR[0x%06x] = VBIOS[0x%04x] = 0x%08x\n",
1333              reg, addr, data);
1334        init->offset += 7;
1335
1336        init_wr32(init, addr, data);
1337}
1338
1339/**
1340 * INIT_SUB_DIRECT - opcode 0x5b
1341 *
1342 */
1343static void
1344init_sub_direct(struct nvbios_init *init)
1345{
1346        struct nvkm_bios *bios = init->subdev->device->bios;
1347        u16 addr = nvbios_rd16(bios, init->offset + 1);
1348        u16 save;
1349
1350        trace("SUB_DIRECT\t0x%04x\n", addr);
1351
1352        if (init_exec(init)) {
1353                save = init->offset;
1354                init->offset = addr;
1355                if (nvbios_exec(init)) {
1356                        error("error parsing sub-table\n");
1357                        return;
1358                }
1359                init->offset = save;
1360        }
1361
1362        init->offset += 3;
1363}
1364
1365/**
1366 * INIT_JUMP - opcode 0x5c
1367 *
1368 */
1369static void
1370init_jump(struct nvbios_init *init)
1371{
1372        struct nvkm_bios *bios = init->subdev->device->bios;
1373        u16 offset = nvbios_rd16(bios, init->offset + 1);
1374
1375        trace("JUMP\t0x%04x\n", offset);
1376
1377        if (init_exec(init))
1378                init->offset = offset;
1379        else
1380                init->offset += 3;
1381}
1382
1383/**
1384 * INIT_I2C_IF - opcode 0x5e
1385 *
1386 */
1387static void
1388init_i2c_if(struct nvbios_init *init)
1389{
1390        struct nvkm_bios *bios = init->subdev->device->bios;
1391        u8 index = nvbios_rd08(bios, init->offset + 1);
1392        u8  addr = nvbios_rd08(bios, init->offset + 2);
1393        u8   reg = nvbios_rd08(bios, init->offset + 3);
1394        u8  mask = nvbios_rd08(bios, init->offset + 4);
1395        u8  data = nvbios_rd08(bios, init->offset + 5);
1396        u8 value;
1397
1398        trace("I2C_IF\tI2C[0x%02x][0x%02x][0x%02x] & 0x%02x == 0x%02x\n",
1399              index, addr, reg, mask, data);
1400        init->offset += 6;
1401        init_exec_force(init, true);
1402
1403        value = init_rdi2cr(init, index, addr, reg);
1404        if ((value & mask) != data)
1405                init_exec_set(init, false);
1406
1407        init_exec_force(init, false);
1408}
1409
1410/**
1411 * INIT_COPY_NV_REG - opcode 0x5f
1412 *
1413 */
1414static void
1415init_copy_nv_reg(struct nvbios_init *init)
1416{
1417        struct nvkm_bios *bios = init->subdev->device->bios;
1418        u32  sreg = nvbios_rd32(bios, init->offset + 1);
1419        u8  shift = nvbios_rd08(bios, init->offset + 5);
1420        u32 smask = nvbios_rd32(bios, init->offset + 6);
1421        u32  sxor = nvbios_rd32(bios, init->offset + 10);
1422        u32  dreg = nvbios_rd32(bios, init->offset + 14);
1423        u32 dmask = nvbios_rd32(bios, init->offset + 18);
1424        u32 data;
1425
1426        trace("COPY_NV_REG\tR[0x%06x] &= 0x%08x |= "
1427              "((R[0x%06x] %s 0x%02x) & 0x%08x ^ 0x%08x)\n",
1428              dreg, dmask, sreg, (shift & 0x80) ? "<<" : ">>",
1429              (shift & 0x80) ? (0x100 - shift) : shift, smask, sxor);
1430        init->offset += 22;
1431
1432        data = init_shift(init_rd32(init, sreg), shift);
1433        init_mask(init, dreg, ~dmask, (data & smask) ^ sxor);
1434}
1435
1436/**
1437 * INIT_ZM_INDEX_IO - opcode 0x62
1438 *
1439 */
1440static void
1441init_zm_index_io(struct nvbios_init *init)
1442{
1443        struct nvkm_bios *bios = init->subdev->device->bios;
1444        u16 port = nvbios_rd16(bios, init->offset + 1);
1445        u8 index = nvbios_rd08(bios, init->offset + 3);
1446        u8  data = nvbios_rd08(bios, init->offset + 4);
1447
1448        trace("ZM_INDEX_IO\tI[0x%04x][0x%02x] = 0x%02x\n", port, index, data);
1449        init->offset += 5;
1450
1451        init_wrvgai(init, port, index, data);
1452}
1453
1454/**
1455 * INIT_COMPUTE_MEM - opcode 0x63
1456 *
1457 */
1458static void
1459init_compute_mem(struct nvbios_init *init)
1460{
1461        struct nvkm_devinit *devinit = init->subdev->device->devinit;
1462
1463        trace("COMPUTE_MEM\n");
1464        init->offset += 1;
1465
1466        init_exec_force(init, true);
1467        if (init_exec(init))
1468                nvkm_devinit_meminit(devinit);
1469        init_exec_force(init, false);
1470}
1471
1472/**
1473 * INIT_RESET - opcode 0x65
1474 *
1475 */
1476static void
1477init_reset(struct nvbios_init *init)
1478{
1479        struct nvkm_bios *bios = init->subdev->device->bios;
1480        u32   reg = nvbios_rd32(bios, init->offset + 1);
1481        u32 data1 = nvbios_rd32(bios, init->offset + 5);
1482        u32 data2 = nvbios_rd32(bios, init->offset + 9);
1483        u32 savepci19;
1484
1485        trace("RESET\tR[0x%08x] = 0x%08x, 0x%08x", reg, data1, data2);
1486        init->offset += 13;
1487        init_exec_force(init, true);
1488
1489        savepci19 = init_mask(init, 0x00184c, 0x00000f00, 0x00000000);
1490        init_wr32(init, reg, data1);
1491        udelay(10);
1492        init_wr32(init, reg, data2);
1493        init_wr32(init, 0x00184c, savepci19);
1494        init_mask(init, 0x001850, 0x00000001, 0x00000000);
1495
1496        init_exec_force(init, false);
1497}
1498
1499/**
1500 * INIT_CONFIGURE_MEM - opcode 0x66
1501 *
1502 */
1503static u16
1504init_configure_mem_clk(struct nvbios_init *init)
1505{
1506        u16 mdata = bmp_mem_init_table(init->subdev->device->bios);
1507        if (mdata)
1508                mdata += (init_rdvgai(init, 0x03d4, 0x3c) >> 4) * 66;
1509        return mdata;
1510}
1511
1512static void
1513init_configure_mem(struct nvbios_init *init)
1514{
1515        struct nvkm_bios *bios = init->subdev->device->bios;
1516        u16 mdata, sdata;
1517        u32 addr, data;
1518
1519        trace("CONFIGURE_MEM\n");
1520        init->offset += 1;
1521
1522        if (bios->version.major > 2) {
1523                init_done(init);
1524                return;
1525        }
1526        init_exec_force(init, true);
1527
1528        mdata = init_configure_mem_clk(init);
1529        sdata = bmp_sdr_seq_table(bios);
1530        if (nvbios_rd08(bios, mdata) & 0x01)
1531                sdata = bmp_ddr_seq_table(bios);
1532        mdata += 6; /* skip to data */
1533
1534        data = init_rdvgai(init, 0x03c4, 0x01);
1535        init_wrvgai(init, 0x03c4, 0x01, data | 0x20);
1536
1537        for (; (addr = nvbios_rd32(bios, sdata)) != 0xffffffff; sdata += 4) {
1538                switch (addr) {
1539                case 0x10021c: /* CKE_NORMAL */
1540                case 0x1002d0: /* CMD_REFRESH */
1541                case 0x1002d4: /* CMD_PRECHARGE */
1542                        data = 0x00000001;
1543                        break;
1544                default:
1545                        data = nvbios_rd32(bios, mdata);
1546                        mdata += 4;
1547                        if (data == 0xffffffff)
1548                                continue;
1549                        break;
1550                }
1551
1552                init_wr32(init, addr, data);
1553        }
1554
1555        init_exec_force(init, false);
1556}
1557
1558/**
1559 * INIT_CONFIGURE_CLK - opcode 0x67
1560 *
1561 */
1562static void
1563init_configure_clk(struct nvbios_init *init)
1564{
1565        struct nvkm_bios *bios = init->subdev->device->bios;
1566        u16 mdata, clock;
1567
1568        trace("CONFIGURE_CLK\n");
1569        init->offset += 1;
1570
1571        if (bios->version.major > 2) {
1572                init_done(init);
1573                return;
1574        }
1575        init_exec_force(init, true);
1576
1577        mdata = init_configure_mem_clk(init);
1578
1579        /* NVPLL */
1580        clock = nvbios_rd16(bios, mdata + 4) * 10;
1581        init_prog_pll(init, 0x680500, clock);
1582
1583        /* MPLL */
1584        clock = nvbios_rd16(bios, mdata + 2) * 10;
1585        if (nvbios_rd08(bios, mdata) & 0x01)
1586                clock *= 2;
1587        init_prog_pll(init, 0x680504, clock);
1588
1589        init_exec_force(init, false);
1590}
1591
1592/**
1593 * INIT_CONFIGURE_PREINIT - opcode 0x68
1594 *
1595 */
1596static void
1597init_configure_preinit(struct nvbios_init *init)
1598{
1599        struct nvkm_bios *bios = init->subdev->device->bios;
1600        u32 strap;
1601
1602        trace("CONFIGURE_PREINIT\n");
1603        init->offset += 1;
1604
1605        if (bios->version.major > 2) {
1606                init_done(init);
1607                return;
1608        }
1609        init_exec_force(init, true);
1610
1611        strap = init_rd32(init, 0x101000);
1612        strap = ((strap << 2) & 0xf0) | ((strap & 0x40) >> 6);
1613        init_wrvgai(init, 0x03d4, 0x3c, strap);
1614
1615        init_exec_force(init, false);
1616}
1617
1618/**
1619 * INIT_IO - opcode 0x69
1620 *
1621 */
1622static void
1623init_io(struct nvbios_init *init)
1624{
1625        struct nvkm_bios *bios = init->subdev->device->bios;
1626        u16 port = nvbios_rd16(bios, init->offset + 1);
1627        u8  mask = nvbios_rd16(bios, init->offset + 3);
1628        u8  data = nvbios_rd16(bios, init->offset + 4);
1629        u8 value;
1630
1631        trace("IO\t\tI[0x%04x] &= 0x%02x |= 0x%02x\n", port, mask, data);
1632        init->offset += 5;
1633
1634        /* ummm.. yes.. should really figure out wtf this is and why it's
1635         * needed some day..  it's almost certainly wrong, but, it also
1636         * somehow makes things work...
1637         */
1638        if (bios->subdev.device->card_type >= NV_50 &&
1639            port == 0x03c3 && data == 0x01) {
1640                init_mask(init, 0x614100, 0xf0800000, 0x00800000);
1641                init_mask(init, 0x00e18c, 0x00020000, 0x00020000);
1642                init_mask(init, 0x614900, 0xf0800000, 0x00800000);
1643                init_mask(init, 0x000200, 0x40000000, 0x00000000);
1644                mdelay(10);
1645                init_mask(init, 0x00e18c, 0x00020000, 0x00000000);
1646                init_mask(init, 0x000200, 0x40000000, 0x40000000);
1647                init_wr32(init, 0x614100, 0x00800018);
1648                init_wr32(init, 0x614900, 0x00800018);
1649                mdelay(10);
1650                init_wr32(init, 0x614100, 0x10000018);
1651                init_wr32(init, 0x614900, 0x10000018);
1652        }
1653
1654        value = init_rdport(init, port) & mask;
1655        init_wrport(init, port, data | value);
1656}
1657
1658/**
1659 * INIT_SUB - opcode 0x6b
1660 *
1661 */
1662static void
1663init_sub(struct nvbios_init *init)
1664{
1665        struct nvkm_bios *bios = init->subdev->device->bios;
1666        u8 index = nvbios_rd08(bios, init->offset + 1);
1667        u16 addr, save;
1668
1669        trace("SUB\t0x%02x\n", index);
1670
1671        addr = init_script(bios, index);
1672        if (addr && init_exec(init)) {
1673                save = init->offset;
1674                init->offset = addr;
1675                if (nvbios_exec(init)) {
1676                        error("error parsing sub-table\n");
1677                        return;
1678                }
1679                init->offset = save;
1680        }
1681
1682        init->offset += 2;
1683}
1684
1685/**
1686 * INIT_RAM_CONDITION - opcode 0x6d
1687 *
1688 */
1689static void
1690init_ram_condition(struct nvbios_init *init)
1691{
1692        struct nvkm_bios *bios = init->subdev->device->bios;
1693        u8  mask = nvbios_rd08(bios, init->offset + 1);
1694        u8 value = nvbios_rd08(bios, init->offset + 2);
1695
1696        trace("RAM_CONDITION\t"
1697              "(R[0x100000] & 0x%02x) == 0x%02x\n", mask, value);
1698        init->offset += 3;
1699
1700        if ((init_rd32(init, 0x100000) & mask) != value)
1701                init_exec_set(init, false);
1702}
1703
1704/**
1705 * INIT_NV_REG - opcode 0x6e
1706 *
1707 */
1708static void
1709init_nv_reg(struct nvbios_init *init)
1710{
1711        struct nvkm_bios *bios = init->subdev->device->bios;
1712        u32  reg = nvbios_rd32(bios, init->offset + 1);
1713        u32 mask = nvbios_rd32(bios, init->offset + 5);
1714        u32 data = nvbios_rd32(bios, init->offset + 9);
1715
1716        trace("NV_REG\tR[0x%06x] &= 0x%08x |= 0x%08x\n", reg, mask, data);
1717        init->offset += 13;
1718
1719        init_mask(init, reg, ~mask, data);
1720}
1721
1722/**
1723 * INIT_MACRO - opcode 0x6f
1724 *
1725 */
1726static void
1727init_macro(struct nvbios_init *init)
1728{
1729        struct nvkm_bios *bios = init->subdev->device->bios;
1730        u8  macro = nvbios_rd08(bios, init->offset + 1);
1731        u16 table;
1732
1733        trace("MACRO\t0x%02x\n", macro);
1734
1735        table = init_macro_table(init);
1736        if (table) {
1737                u32 addr = nvbios_rd32(bios, table + (macro * 8) + 0);
1738                u32 data = nvbios_rd32(bios, table + (macro * 8) + 4);
1739                trace("\t\tR[0x%06x] = 0x%08x\n", addr, data);
1740                init_wr32(init, addr, data);
1741        }
1742
1743        init->offset += 2;
1744}
1745
1746/**
1747 * INIT_RESUME - opcode 0x72
1748 *
1749 */
1750static void
1751init_resume(struct nvbios_init *init)
1752{
1753        trace("RESUME\n");
1754        init->offset += 1;
1755        init_exec_set(init, true);
1756}
1757
1758/**
1759 * INIT_STRAP_CONDITION - opcode 0x73
1760 *
1761 */
1762static void
1763init_strap_condition(struct nvbios_init *init)
1764{
1765        struct nvkm_bios *bios = init->subdev->device->bios;
1766        u32 mask = nvbios_rd32(bios, init->offset + 1);
1767        u32 value = nvbios_rd32(bios, init->offset + 5);
1768
1769        trace("STRAP_CONDITION\t(R[0x101000] & 0x%08x) == 0x%08x\n", mask, value);
1770        init->offset += 9;
1771
1772        if ((init_rd32(init, 0x101000) & mask) != value)
1773                init_exec_set(init, false);
1774}
1775
1776/**
1777 * INIT_TIME - opcode 0x74
1778 *
1779 */
1780static void
1781init_time(struct nvbios_init *init)
1782{
1783        struct nvkm_bios *bios = init->subdev->device->bios;
1784        u16 usec = nvbios_rd16(bios, init->offset + 1);
1785
1786        trace("TIME\t0x%04x\n", usec);
1787        init->offset += 3;
1788
1789        if (init_exec(init)) {
1790                if (usec < 1000)
1791                        udelay(usec);
1792                else
1793                        mdelay((usec + 900) / 1000);
1794        }
1795}
1796
1797/**
1798 * INIT_CONDITION - opcode 0x75
1799 *
1800 */
1801static void
1802init_condition(struct nvbios_init *init)
1803{
1804        struct nvkm_bios *bios = init->subdev->device->bios;
1805        u8 cond = nvbios_rd08(bios, init->offset + 1);
1806
1807        trace("CONDITION\t0x%02x\n", cond);
1808        init->offset += 2;
1809
1810        if (!init_condition_met(init, cond))
1811                init_exec_set(init, false);
1812}
1813
1814/**
1815 * INIT_IO_CONDITION - opcode 0x76
1816 *
1817 */
1818static void
1819init_io_condition(struct nvbios_init *init)
1820{
1821        struct nvkm_bios *bios = init->subdev->device->bios;
1822        u8 cond = nvbios_rd08(bios, init->offset + 1);
1823
1824        trace("IO_CONDITION\t0x%02x\n", cond);
1825        init->offset += 2;
1826
1827        if (!init_io_condition_met(init, cond))
1828                init_exec_set(init, false);
1829}
1830
1831/**
1832 * INIT_ZM_REG16 - opcode 0x77
1833 *
1834 */
1835static void
1836init_zm_reg16(struct nvbios_init *init)
1837{
1838        struct nvkm_bios *bios = init->subdev->device->bios;
1839        u32 addr = nvbios_rd32(bios, init->offset + 1);
1840        u16 data = nvbios_rd16(bios, init->offset + 5);
1841
1842        trace("ZM_REG\tR[0x%06x] = 0x%04x\n", addr, data);
1843        init->offset += 7;
1844
1845        init_wr32(init, addr, data);
1846}
1847
1848/**
1849 * INIT_INDEX_IO - opcode 0x78
1850 *
1851 */
1852static void
1853init_index_io(struct nvbios_init *init)
1854{
1855        struct nvkm_bios *bios = init->subdev->device->bios;
1856        u16 port = nvbios_rd16(bios, init->offset + 1);
1857        u8 index = nvbios_rd16(bios, init->offset + 3);
1858        u8  mask = nvbios_rd08(bios, init->offset + 4);
1859        u8  data = nvbios_rd08(bios, init->offset + 5);
1860        u8 value;
1861
1862        trace("INDEX_IO\tI[0x%04x][0x%02x] &= 0x%02x |= 0x%02x\n",
1863              port, index, mask, data);
1864        init->offset += 6;
1865
1866        value = init_rdvgai(init, port, index) & mask;
1867        init_wrvgai(init, port, index, data | value);
1868}
1869
1870/**
1871 * INIT_PLL - opcode 0x79
1872 *
1873 */
1874static void
1875init_pll(struct nvbios_init *init)
1876{
1877        struct nvkm_bios *bios = init->subdev->device->bios;
1878        u32  reg = nvbios_rd32(bios, init->offset + 1);
1879        u32 freq = nvbios_rd16(bios, init->offset + 5) * 10;
1880
1881        trace("PLL\tR[0x%06x] =PLL= %dkHz\n", reg, freq);
1882        init->offset += 7;
1883
1884        init_prog_pll(init, reg, freq);
1885}
1886
1887/**
1888 * INIT_ZM_REG - opcode 0x7a
1889 *
1890 */
1891static void
1892init_zm_reg(struct nvbios_init *init)
1893{
1894        struct nvkm_bios *bios = init->subdev->device->bios;
1895        u32 addr = nvbios_rd32(bios, init->offset + 1);
1896        u32 data = nvbios_rd32(bios, init->offset + 5);
1897
1898        trace("ZM_REG\tR[0x%06x] = 0x%08x\n", addr, data);
1899        init->offset += 9;
1900
1901        if (addr == 0x000200)
1902                data |= 0x00000001;
1903
1904        init_wr32(init, addr, data);
1905}
1906
1907/**
1908 * INIT_RAM_RESTRICT_PLL - opcde 0x87
1909 *
1910 */
1911static void
1912init_ram_restrict_pll(struct nvbios_init *init)
1913{
1914        struct nvkm_bios *bios = init->subdev->device->bios;
1915        u8  type = nvbios_rd08(bios, init->offset + 1);
1916        u8 count = init_ram_restrict_group_count(init);
1917        u8 strap = init_ram_restrict(init);
1918        u8 cconf;
1919
1920        trace("RAM_RESTRICT_PLL\t0x%02x\n", type);
1921        init->offset += 2;
1922
1923        for (cconf = 0; cconf < count; cconf++) {
1924                u32 freq = nvbios_rd32(bios, init->offset);
1925
1926                if (cconf == strap) {
1927                        trace("%dkHz *\n", freq);
1928                        init_prog_pll(init, type, freq);
1929                } else {
1930                        trace("%dkHz\n", freq);
1931                }
1932
1933                init->offset += 4;
1934        }
1935}
1936
1937/**
1938 * INIT_RESET_BEGUN - opcode 0x8c
1939 *
1940 */
1941static void
1942init_reset_begun(struct nvbios_init *init)
1943{
1944        trace("RESET_BEGUN\n");
1945        init->offset += 1;
1946}
1947
1948/**
1949 * INIT_RESET_END - opcode 0x8d
1950 *
1951 */
1952static void
1953init_reset_end(struct nvbios_init *init)
1954{
1955        trace("RESET_END\n");
1956        init->offset += 1;
1957}
1958
1959/**
1960 * INIT_GPIO - opcode 0x8e
1961 *
1962 */
1963static void
1964init_gpio(struct nvbios_init *init)
1965{
1966        struct nvkm_gpio *gpio = init->subdev->device->gpio;
1967
1968        trace("GPIO\n");
1969        init->offset += 1;
1970
1971        if (init_exec(init))
1972                nvkm_gpio_reset(gpio, DCB_GPIO_UNUSED);
1973}
1974
1975/**
1976 * INIT_RAM_RESTRICT_ZM_GROUP - opcode 0x8f
1977 *
1978 */
1979static void
1980init_ram_restrict_zm_reg_group(struct nvbios_init *init)
1981{
1982        struct nvkm_bios *bios = init->subdev->device->bios;
1983        u32 addr = nvbios_rd32(bios, init->offset + 1);
1984        u8  incr = nvbios_rd08(bios, init->offset + 5);
1985        u8   num = nvbios_rd08(bios, init->offset + 6);
1986        u8 count = init_ram_restrict_group_count(init);
1987        u8 index = init_ram_restrict(init);
1988        u8 i, j;
1989
1990        trace("RAM_RESTRICT_ZM_REG_GROUP\t"
1991              "R[0x%08x] 0x%02x 0x%02x\n", addr, incr, num);
1992        init->offset += 7;
1993
1994        for (i = 0; i < num; i++) {
1995                trace("\tR[0x%06x] = {\n", addr);
1996                for (j = 0; j < count; j++) {
1997                        u32 data = nvbios_rd32(bios, init->offset);
1998
1999                        if (j == index) {
2000                                trace("\t\t0x%08x *\n", data);
2001                                init_wr32(init, addr, data);
2002                        } else {
2003                                trace("\t\t0x%08x\n", data);
2004                        }
2005
2006                        init->offset += 4;
2007                }
2008                trace("\t}\n");
2009                addr += incr;
2010        }
2011}
2012
2013/**
2014 * INIT_COPY_ZM_REG - opcode 0x90
2015 *
2016 */
2017static void
2018init_copy_zm_reg(struct nvbios_init *init)
2019{
2020        struct nvkm_bios *bios = init->subdev->device->bios;
2021        u32 sreg = nvbios_rd32(bios, init->offset + 1);
2022        u32 dreg = nvbios_rd32(bios, init->offset + 5);
2023
2024        trace("COPY_ZM_REG\tR[0x%06x] = R[0x%06x]\n", dreg, sreg);
2025        init->offset += 9;
2026
2027        init_wr32(init, dreg, init_rd32(init, sreg));
2028}
2029
2030/**
2031 * INIT_ZM_REG_GROUP - opcode 0x91
2032 *
2033 */
2034static void
2035init_zm_reg_group(struct nvbios_init *init)
2036{
2037        struct nvkm_bios *bios = init->subdev->device->bios;
2038        u32 addr = nvbios_rd32(bios, init->offset + 1);
2039        u8 count = nvbios_rd08(bios, init->offset + 5);
2040
2041        trace("ZM_REG_GROUP\tR[0x%06x] =\n", addr);
2042        init->offset += 6;
2043
2044        while (count--) {
2045                u32 data = nvbios_rd32(bios, init->offset);
2046                trace("\t0x%08x\n", data);
2047                init_wr32(init, addr, data);
2048                init->offset += 4;
2049        }
2050}
2051
2052/**
2053 * INIT_XLAT - opcode 0x96
2054 *
2055 */
2056static void
2057init_xlat(struct nvbios_init *init)
2058{
2059        struct nvkm_bios *bios = init->subdev->device->bios;
2060        u32 saddr = nvbios_rd32(bios, init->offset + 1);
2061        u8 sshift = nvbios_rd08(bios, init->offset + 5);
2062        u8  smask = nvbios_rd08(bios, init->offset + 6);
2063        u8  index = nvbios_rd08(bios, init->offset + 7);
2064        u32 daddr = nvbios_rd32(bios, init->offset + 8);
2065        u32 dmask = nvbios_rd32(bios, init->offset + 12);
2066        u8  shift = nvbios_rd08(bios, init->offset + 16);
2067        u32 data;
2068
2069        trace("INIT_XLAT\tR[0x%06x] &= 0x%08x |= "
2070              "(X%02x((R[0x%06x] %s 0x%02x) & 0x%02x) << 0x%02x)\n",
2071              daddr, dmask, index, saddr, (sshift & 0x80) ? "<<" : ">>",
2072              (sshift & 0x80) ? (0x100 - sshift) : sshift, smask, shift);
2073        init->offset += 17;
2074
2075        data = init_shift(init_rd32(init, saddr), sshift) & smask;
2076        data = init_xlat_(init, index, data) << shift;
2077        init_mask(init, daddr, ~dmask, data);
2078}
2079
2080/**
2081 * INIT_ZM_MASK_ADD - opcode 0x97
2082 *
2083 */
2084static void
2085init_zm_mask_add(struct nvbios_init *init)
2086{
2087        struct nvkm_bios *bios = init->subdev->device->bios;
2088        u32 addr = nvbios_rd32(bios, init->offset + 1);
2089        u32 mask = nvbios_rd32(bios, init->offset + 5);
2090        u32  add = nvbios_rd32(bios, init->offset + 9);
2091        u32 data;
2092
2093        trace("ZM_MASK_ADD\tR[0x%06x] &= 0x%08x += 0x%08x\n", addr, mask, add);
2094        init->offset += 13;
2095
2096        data =  init_rd32(init, addr);
2097        data = (data & mask) | ((data + add) & ~mask);
2098        init_wr32(init, addr, data);
2099}
2100
2101/**
2102 * INIT_AUXCH - opcode 0x98
2103 *
2104 */
2105static void
2106init_auxch(struct nvbios_init *init)
2107{
2108        struct nvkm_bios *bios = init->subdev->device->bios;
2109        u32 addr = nvbios_rd32(bios, init->offset + 1);
2110        u8 count = nvbios_rd08(bios, init->offset + 5);
2111
2112        trace("AUXCH\tAUX[0x%08x] 0x%02x\n", addr, count);
2113        init->offset += 6;
2114
2115        while (count--) {
2116                u8 mask = nvbios_rd08(bios, init->offset + 0);
2117                u8 data = nvbios_rd08(bios, init->offset + 1);
2118                trace("\tAUX[0x%08x] &= 0x%02x |= 0x%02x\n", addr, mask, data);
2119                mask = init_rdauxr(init, addr) & mask;
2120                init_wrauxr(init, addr, mask | data);
2121                init->offset += 2;
2122        }
2123}
2124
2125/**
2126 * INIT_AUXCH - opcode 0x99
2127 *
2128 */
2129static void
2130init_zm_auxch(struct nvbios_init *init)
2131{
2132        struct nvkm_bios *bios = init->subdev->device->bios;
2133        u32 addr = nvbios_rd32(bios, init->offset + 1);
2134        u8 count = nvbios_rd08(bios, init->offset + 5);
2135
2136        trace("ZM_AUXCH\tAUX[0x%08x] 0x%02x\n", addr, count);
2137        init->offset += 6;
2138
2139        while (count--) {
2140                u8 data = nvbios_rd08(bios, init->offset + 0);
2141                trace("\tAUX[0x%08x] = 0x%02x\n", addr, data);
2142                init_wrauxr(init, addr, data);
2143                init->offset += 1;
2144        }
2145}
2146
2147/**
2148 * INIT_I2C_LONG_IF - opcode 0x9a
2149 *
2150 */
2151static void
2152init_i2c_long_if(struct nvbios_init *init)
2153{
2154        struct nvkm_bios *bios = init->subdev->device->bios;
2155        u8 index = nvbios_rd08(bios, init->offset + 1);
2156        u8  addr = nvbios_rd08(bios, init->offset + 2) >> 1;
2157        u8 reglo = nvbios_rd08(bios, init->offset + 3);
2158        u8 reghi = nvbios_rd08(bios, init->offset + 4);
2159        u8  mask = nvbios_rd08(bios, init->offset + 5);
2160        u8  data = nvbios_rd08(bios, init->offset + 6);
2161        struct i2c_adapter *adap;
2162
2163        trace("I2C_LONG_IF\t"
2164              "I2C[0x%02x][0x%02x][0x%02x%02x] & 0x%02x == 0x%02x\n",
2165              index, addr, reglo, reghi, mask, data);
2166        init->offset += 7;
2167
2168        adap = init_i2c(init, index);
2169        if (adap) {
2170                u8 i[2] = { reghi, reglo };
2171                u8 o[1] = {};
2172                struct i2c_msg msg[] = {
2173                        { .addr = addr, .flags = 0, .len = 2, .buf = i },
2174                        { .addr = addr, .flags = I2C_M_RD, .len = 1, .buf = o }
2175                };
2176                int ret;
2177
2178                ret = i2c_transfer(adap, msg, 2);
2179                if (ret == 2 && ((o[0] & mask) == data))
2180                        return;
2181        }
2182
2183        init_exec_set(init, false);
2184}
2185
2186/**
2187 * INIT_GPIO_NE - opcode 0xa9
2188 *
2189 */
2190static void
2191init_gpio_ne(struct nvbios_init *init)
2192{
2193        struct nvkm_bios *bios = init->subdev->device->bios;
2194        struct nvkm_gpio *gpio = bios->subdev.device->gpio;
2195        struct dcb_gpio_func func;
2196        u8 count = nvbios_rd08(bios, init->offset + 1);
2197        u8 idx = 0, ver, len;
2198        u16 data, i;
2199
2200        trace("GPIO_NE\t");
2201        init->offset += 2;
2202
2203        for (i = init->offset; i < init->offset + count; i++)
2204                cont("0x%02x ", nvbios_rd08(bios, i));
2205        cont("\n");
2206
2207        while ((data = dcb_gpio_parse(bios, 0, idx++, &ver, &len, &func))) {
2208                if (func.func != DCB_GPIO_UNUSED) {
2209                        for (i = init->offset; i < init->offset + count; i++) {
2210                                if (func.func == nvbios_rd08(bios, i))
2211                                        break;
2212                        }
2213
2214                        trace("\tFUNC[0x%02x]", func.func);
2215                        if (i == (init->offset + count)) {
2216                                cont(" *");
2217                                if (init_exec(init))
2218                                        nvkm_gpio_reset(gpio, func.func);
2219                        }
2220                        cont("\n");
2221                }
2222        }
2223
2224        init->offset += count;
2225}
2226
2227static struct nvbios_init_opcode {
2228        void (*exec)(struct nvbios_init *);
2229} init_opcode[] = {
2230        [0x32] = { init_io_restrict_prog },
2231        [0x33] = { init_repeat },
2232        [0x34] = { init_io_restrict_pll },
2233        [0x36] = { init_end_repeat },
2234        [0x37] = { init_copy },
2235        [0x38] = { init_not },
2236        [0x39] = { init_io_flag_condition },
2237        [0x3a] = { init_generic_condition },
2238        [0x3b] = { init_io_mask_or },
2239        [0x3c] = { init_io_or },
2240        [0x47] = { init_andn_reg },
2241        [0x48] = { init_or_reg },
2242        [0x49] = { init_idx_addr_latched },
2243        [0x4a] = { init_io_restrict_pll2 },
2244        [0x4b] = { init_pll2 },
2245        [0x4c] = { init_i2c_byte },
2246        [0x4d] = { init_zm_i2c_byte },
2247        [0x4e] = { init_zm_i2c },
2248        [0x4f] = { init_tmds },
2249        [0x50] = { init_zm_tmds_group },
2250        [0x51] = { init_cr_idx_adr_latch },
2251        [0x52] = { init_cr },
2252        [0x53] = { init_zm_cr },
2253        [0x54] = { init_zm_cr_group },
2254        [0x56] = { init_condition_time },
2255        [0x57] = { init_ltime },
2256        [0x58] = { init_zm_reg_sequence },
2257        [0x59] = { init_pll_indirect },
2258        [0x5a] = { init_zm_reg_indirect },
2259        [0x5b] = { init_sub_direct },
2260        [0x5c] = { init_jump },
2261        [0x5e] = { init_i2c_if },
2262        [0x5f] = { init_copy_nv_reg },
2263        [0x62] = { init_zm_index_io },
2264        [0x63] = { init_compute_mem },
2265        [0x65] = { init_reset },
2266        [0x66] = { init_configure_mem },
2267        [0x67] = { init_configure_clk },
2268        [0x68] = { init_configure_preinit },
2269        [0x69] = { init_io },
2270        [0x6b] = { init_sub },
2271        [0x6d] = { init_ram_condition },
2272        [0x6e] = { init_nv_reg },
2273        [0x6f] = { init_macro },
2274        [0x71] = { init_done },
2275        [0x72] = { init_resume },
2276        [0x73] = { init_strap_condition },
2277        [0x74] = { init_time },
2278        [0x75] = { init_condition },
2279        [0x76] = { init_io_condition },
2280        [0x77] = { init_zm_reg16 },
2281        [0x78] = { init_index_io },
2282        [0x79] = { init_pll },
2283        [0x7a] = { init_zm_reg },
2284        [0x87] = { init_ram_restrict_pll },
2285        [0x8c] = { init_reset_begun },
2286        [0x8d] = { init_reset_end },
2287        [0x8e] = { init_gpio },
2288        [0x8f] = { init_ram_restrict_zm_reg_group },
2289        [0x90] = { init_copy_zm_reg },
2290        [0x91] = { init_zm_reg_group },
2291        [0x92] = { init_reserved },
2292        [0x96] = { init_xlat },
2293        [0x97] = { init_zm_mask_add },
2294        [0x98] = { init_auxch },
2295        [0x99] = { init_zm_auxch },
2296        [0x9a] = { init_i2c_long_if },
2297        [0xa9] = { init_gpio_ne },
2298        [0xaa] = { init_reserved },
2299};
2300
2301int
2302nvbios_exec(struct nvbios_init *init)
2303{
2304        struct nvkm_bios *bios = init->subdev->device->bios;
2305
2306        init->nested++;
2307        while (init->offset) {
2308                u8 opcode = nvbios_rd08(bios, init->offset);
2309                if (opcode >= ARRAY_SIZE(init_opcode) ||
2310                    !init_opcode[opcode].exec) {
2311                        error("unknown opcode 0x%02x\n", opcode);
2312                        return -EINVAL;
2313                }
2314
2315                init_opcode[opcode].exec(init);
2316        }
2317        init->nested--;
2318        return 0;
2319}
2320
2321int
2322nvbios_post(struct nvkm_subdev *subdev, bool execute)
2323{
2324        struct nvkm_bios *bios = subdev->device->bios;
2325        int ret = 0;
2326        int i = -1;
2327        u16 data;
2328
2329        if (execute)
2330                nvkm_debug(subdev, "running init tables\n");
2331        while (!ret && (data = (init_script(bios, ++i)))) {
2332                ret = nvbios_init(subdev, data,
2333                        init.execute = execute ? 1 : 0;
2334                      );
2335        }
2336
2337        /* the vbios parser will run this right after the normal init
2338         * tables, whereas the binary driver appears to run it later.
2339         */
2340        if (!ret && (data = init_unknown_script(bios))) {
2341                ret = nvbios_init(subdev, data,
2342                        init.execute = execute ? 1 : 0;
2343                      );
2344        }
2345
2346        return ret;
2347}
2348