// SPDX-License-Identifier: GPL-2.0

/*
 * Copyright 2016-2020 HabanaLabs, Ltd.
 * All Rights Reserved.
 */

#include "gaudiP.h"
#include "../include/hw_ip/mmu/mmu_general.h"
#include "../include/hw_ip/mmu/mmu_v1_1.h"
#include "../include/gaudi/gaudi_masks.h"
#include "../include/gaudi/gaudi_fw_if.h"
#include "../include/gaudi/gaudi_reg_map.h"
#include "../include/gaudi/gaudi_async_ids_map_extended.h"

#include <linux/module.h>
#include <linux/pci.h>
#include <linux/firmware.h>
#include <linux/hwmon.h>
#include <linux/iommu.h>
#include <linux/seq_file.h>

/*
 * Gaudi security scheme:
 *
 * 1. Host is protected by:
 *        - Range registers
 *        - MMU
 *
 * 2. DDR is protected by:
 *        - Range registers (protect the first 512MB)
 *
 * 3. Configuration is protected by:
 *        - Range registers
 *        - Protection bits
 *
 * MMU is always enabled.
 *
 * QMAN DMA channels 0,1 (PCI DMAN):
 *     - DMA is not secured.
 *     - PQ and CQ are secured.
 *     - CP is secured: The driver needs to parse CB but WREG should be allowed
 *                      because of TDMA (tensor DMA). Hence, WREG is always not
 *                      secured.
 *
 * When the driver needs to use DMA it will check that Gaudi is idle, set DMA
 * channel 0 to be secured, execute the DMA and change it back to not secured.
 * Currently, the driver doesn't use the DMA while there are compute jobs
 * running.
 *
 * The current use cases for the driver to use the DMA are:
 *     - Clear SRAM on context switch (happens on context switch when device is
 *       idle)
 *     - MMU page tables area clear (happens on init)
 *
 * QMAN DMA 2-7, TPC, MME, NIC:
 * PQ is secured and is located on the Host (HBM CON TPC3 bug)
 * CQ, CP and the engine are not secured
 *
 */

#define GAUDI_BOOT_FIT_FILE     "habanalabs/gaudi/gaudi-boot-fit.itb"
#define GAUDI_LINUX_FW_FILE     "habanalabs/gaudi/gaudi-fit.itb"
#define GAUDI_TPC_FW_FILE       "habanalabs/gaudi/gaudi_tpc.bin"

#define GAUDI_DMA_POOL_BLK_SIZE         0x100 /* 256 bytes */

#define GAUDI_RESET_TIMEOUT_MSEC        2000            /* 2000ms */
#define GAUDI_RESET_WAIT_MSEC           1               /* 1ms */
#define GAUDI_CPU_RESET_WAIT_MSEC       200             /* 200ms */
#define GAUDI_TEST_QUEUE_WAIT_USEC      100000          /* 100ms */

#define GAUDI_PLDM_RESET_WAIT_MSEC      1000            /* 1s */
#define GAUDI_PLDM_HRESET_TIMEOUT_MSEC  20000           /* 20s */
#define GAUDI_PLDM_TEST_QUEUE_WAIT_USEC 1000000         /* 1s */
#define GAUDI_PLDM_MMU_TIMEOUT_USEC     (MMU_CONFIG_TIMEOUT_USEC * 100)
#define GAUDI_PLDM_QMAN0_TIMEOUT_USEC   (HL_DEVICE_TIMEOUT_USEC * 30)
#define GAUDI_PLDM_TPC_KERNEL_WAIT_USEC (HL_DEVICE_TIMEOUT_USEC * 30)
#define GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC 1000000         /* 1s */
#define GAUDI_MSG_TO_CPU_TIMEOUT_USEC   4000000         /* 4s */
#define GAUDI_WAIT_FOR_BL_TIMEOUT_USEC  15000000        /* 15s */

#define GAUDI_QMAN0_FENCE_VAL           0x72E91AB9

#define GAUDI_MAX_STRING_LEN            20

#define GAUDI_CB_POOL_CB_CNT            512
#define GAUDI_CB_POOL_CB_SIZE           0x20000 /* 128KB */

#define GAUDI_ALLOC_CPU_MEM_RETRY_CNT   3

#define GAUDI_NUM_OF_TPC_INTR_CAUSE     20

#define GAUDI_NUM_OF_QM_ERR_CAUSE       16

#define GAUDI_NUM_OF_QM_ARB_ERR_CAUSE   3

#define GAUDI_ARB_WDT_TIMEOUT           0x1000000

#define GAUDI_CLK_GATE_DEBUGFS_MASK     (\
                BIT(GAUDI_ENGINE_ID_MME_0) |\
                BIT(GAUDI_ENGINE_ID_MME_2) |\
                GENMASK_ULL(GAUDI_ENGINE_ID_TPC_7, GAUDI_ENGINE_ID_TPC_0))

#define HBM_SCRUBBING_TIMEOUT_US        1000000 /* 1s */

#define GAUDI_PLL_MAX 10

static const char gaudi_irq_name[GAUDI_MSI_ENTRIES][GAUDI_MAX_STRING_LEN] = {
                "gaudi cq 0_0", "gaudi cq 0_1", "gaudi cq 0_2", "gaudi cq 0_3",
                "gaudi cq 1_0", "gaudi cq 1_1", "gaudi cq 1_2", "gaudi cq 1_3",
                "gaudi cq 5_0", "gaudi cq 5_1", "gaudi cq 5_2", "gaudi cq 5_3",
                "gaudi cpu eq"
};

static const u8 gaudi_dma_assignment[GAUDI_DMA_MAX] = {
        [GAUDI_PCI_DMA_1] = GAUDI_ENGINE_ID_DMA_0,
        [GAUDI_PCI_DMA_2] = GAUDI_ENGINE_ID_DMA_1,
        [GAUDI_HBM_DMA_1] = GAUDI_ENGINE_ID_DMA_2,
        [GAUDI_HBM_DMA_2] = GAUDI_ENGINE_ID_DMA_3,
        [GAUDI_HBM_DMA_3] = GAUDI_ENGINE_ID_DMA_4,
        [GAUDI_HBM_DMA_4] = GAUDI_ENGINE_ID_DMA_5,
        [GAUDI_HBM_DMA_5] = GAUDI_ENGINE_ID_DMA_6,
        [GAUDI_HBM_DMA_6] = GAUDI_ENGINE_ID_DMA_7
};

static const u8 gaudi_cq_assignment[NUMBER_OF_CMPLT_QUEUES] = {
        [0] = GAUDI_QUEUE_ID_DMA_0_0,
        [1] = GAUDI_QUEUE_ID_DMA_0_1,
        [2] = GAUDI_QUEUE_ID_DMA_0_2,
        [3] = GAUDI_QUEUE_ID_DMA_0_3,
        [4] = GAUDI_QUEUE_ID_DMA_1_0,
        [5] = GAUDI_QUEUE_ID_DMA_1_1,
        [6] = GAUDI_QUEUE_ID_DMA_1_2,
        [7] = GAUDI_QUEUE_ID_DMA_1_3,
};

static const u16 gaudi_packet_sizes[MAX_PACKET_ID] = {
        [PACKET_WREG_32]        = sizeof(struct packet_wreg32),
        [PACKET_WREG_BULK]      = sizeof(struct packet_wreg_bulk),
        [PACKET_MSG_LONG]       = sizeof(struct packet_msg_long),
        [PACKET_MSG_SHORT]      = sizeof(struct packet_msg_short),
        [PACKET_CP_DMA]         = sizeof(struct packet_cp_dma),
        [PACKET_REPEAT]         = sizeof(struct packet_repeat),
        [PACKET_MSG_PROT]       = sizeof(struct packet_msg_prot),
        [PACKET_FENCE]          = sizeof(struct packet_fence),
        [PACKET_LIN_DMA]        = sizeof(struct packet_lin_dma),
        [PACKET_NOP]            = sizeof(struct packet_nop),
        [PACKET_STOP]           = sizeof(struct packet_stop),
        [PACKET_ARB_POINT]      = sizeof(struct packet_arb_point),
        [PACKET_WAIT]           = sizeof(struct packet_wait),
        [PACKET_LOAD_AND_EXE]   = sizeof(struct packet_load_and_exe)
};

static inline bool validate_packet_id(enum packet_id id)
{
        switch (id) {
        case PACKET_WREG_32:
        case PACKET_WREG_BULK:
        case PACKET_MSG_LONG:
        case PACKET_MSG_SHORT:
        case PACKET_CP_DMA:
        case PACKET_REPEAT:
        case PACKET_MSG_PROT:
        case PACKET_FENCE:
        case PACKET_LIN_DMA:
        case PACKET_NOP:
        case PACKET_STOP:
        case PACKET_ARB_POINT:
        case PACKET_WAIT:
        case PACKET_LOAD_AND_EXE:
                return true;
        default:
                return false;
        }
}

static const char * const
gaudi_tpc_interrupts_cause[GAUDI_NUM_OF_TPC_INTR_CAUSE] = {
        "tpc_address_exceed_slm",
        "tpc_div_by_0",
        "tpc_spu_mac_overflow",
        "tpc_spu_addsub_overflow",
        "tpc_spu_abs_overflow",
        "tpc_spu_fp_dst_nan_inf",
        "tpc_spu_fp_dst_denorm",
        "tpc_vpu_mac_overflow",
        "tpc_vpu_addsub_overflow",
        "tpc_vpu_abs_overflow",
        "tpc_vpu_fp_dst_nan_inf",
        "tpc_vpu_fp_dst_denorm",
        "tpc_assertions",
        "tpc_illegal_instruction",
        "tpc_pc_wrap_around",
        "tpc_qm_sw_err",
        "tpc_hbw_rresp_err",
        "tpc_hbw_bresp_err",
        "tpc_lbw_rresp_err",
        "tpc_lbw_bresp_err"
};

static const char * const
gaudi_qman_error_cause[GAUDI_NUM_OF_QM_ERR_CAUSE] = {
        "PQ AXI HBW error",
        "CQ AXI HBW error",
        "CP AXI HBW error",
        "CP error due to undefined OPCODE",
        "CP encountered STOP OPCODE",
        "CP AXI LBW error",
        "CP WRREG32 or WRBULK returned error",
        "N/A",
        "FENCE 0 inc over max value and clipped",
        "FENCE 1 inc over max value and clipped",
        "FENCE 2 inc over max value and clipped",
        "FENCE 3 inc over max value and clipped",
        "FENCE 0 dec under min value and clipped",
        "FENCE 1 dec under min value and clipped",
        "FENCE 2 dec under min value and clipped",
        "FENCE 3 dec under min value and clipped"
};

static const char * const
gaudi_qman_arb_error_cause[GAUDI_NUM_OF_QM_ARB_ERR_CAUSE] = {
        "Choice push while full error",
        "Choice Q watchdog error",
        "MSG AXI LBW returned with error"
};

enum gaudi_sm_sei_cause {
        GAUDI_SM_SEI_SO_OVERFLOW,
        GAUDI_SM_SEI_LBW_4B_UNALIGNED,
        GAUDI_SM_SEI_AXI_RESPONSE_ERR
};

static enum hl_queue_type gaudi_queue_type[GAUDI_QUEUE_ID_SIZE] = {
        QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_0 */
        QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_1 */
        QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_2 */
        QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_3 */
        QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_0 */
        QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_1 */
        QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_2 */
        QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_3 */
        QUEUE_TYPE_CPU, /* GAUDI_QUEUE_ID_CPU_PQ */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_0_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_0_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_0_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_0_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_1_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_1_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_1_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_1_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_2_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_2_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_2_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_2_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_3_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_3_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_3_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_3_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_4_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_4_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_4_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_4_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_5_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_5_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_5_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_5_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_6_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_6_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_6_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_6_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_7_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_7_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_7_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_7_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_8_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_8_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_8_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_8_3 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_0 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_1 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_2 */
        QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_3 */
};

struct ecc_info_extract_params {
        u64 block_address;
        u32 num_memories;
        bool derr;
        bool disable_clock_gating;
};

static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid,
                                                                u64 phys_addr);
static int gaudi_send_job_on_qman0(struct hl_device *hdev,
                                        struct hl_cs_job *job);
static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr,
                                        u32 size, u64 val);
static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base,
                                        u32 num_regs, u32 val);
static int gaudi_schedule_register_memset(struct hl_device *hdev,
                u32 hw_queue_id, u64 reg_base, u32 num_regs, u32 val);
static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel,
                                u32 tpc_id);
static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev);
static int gaudi_cpucp_info_get(struct hl_device *hdev);
static void gaudi_disable_clock_gating(struct hl_device *hdev);
static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid);
static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id,
                                u32 size, bool eb);
static u32 gaudi_gen_wait_cb(struct hl_device *hdev,
                                struct hl_gen_wait_properties *prop);

static inline enum hl_collective_mode
get_collective_mode(struct hl_device *hdev, u32 queue_id)
{
        if (gaudi_queue_type[queue_id] == QUEUE_TYPE_EXT)
                return HL_COLLECTIVE_MASTER;

        if (queue_id >= GAUDI_QUEUE_ID_DMA_5_0 &&
                        queue_id <= GAUDI_QUEUE_ID_DMA_5_3)
                return HL_COLLECTIVE_SLAVE;

        if (queue_id >= GAUDI_QUEUE_ID_TPC_7_0 &&
                        queue_id <= GAUDI_QUEUE_ID_TPC_7_3)
                return HL_COLLECTIVE_SLAVE;

        if (queue_id >= GAUDI_QUEUE_ID_NIC_0_0 &&
                        queue_id <= GAUDI_QUEUE_ID_NIC_9_3)
                return HL_COLLECTIVE_SLAVE;

        return HL_COLLECTIVE_NOT_SUPPORTED;
}

static inline void set_default_power_values(struct hl_device *hdev)
{
        struct asic_fixed_properties *prop = &hdev->asic_prop;

        if (hdev->card_type == cpucp_card_type_pmc) {
                prop->max_power_default = MAX_POWER_DEFAULT_PMC;
                prop->dc_power_default = DC_POWER_DEFAULT_PMC;
        } else {
                prop->max_power_default = MAX_POWER_DEFAULT_PCI;
                prop->dc_power_default = DC_POWER_DEFAULT_PCI;
        }
}

static int gaudi_set_fixed_properties(struct hl_device *hdev)
{
        struct asic_fixed_properties *prop = &hdev->asic_prop;
        u32 num_sync_stream_queues = 0;
        int i;

        prop->max_queues = GAUDI_QUEUE_ID_SIZE;
        prop->hw_queues_props = kcalloc(prop->max_queues,
                        sizeof(struct hw_queue_properties),
                        GFP_KERNEL);

        if (!prop->hw_queues_props)
                return -ENOMEM;

        for (i = 0 ; i < prop->max_queues ; i++) {
                if (gaudi_queue_type[i] == QUEUE_TYPE_EXT) {
                        prop->hw_queues_props[i].type = QUEUE_TYPE_EXT;
                        prop->hw_queues_props[i].driver_only = 0;
                        prop->hw_queues_props[i].supports_sync_stream = 1;
                        prop->hw_queues_props[i].cb_alloc_flags =
                                CB_ALLOC_KERNEL;
                        num_sync_stream_queues++;
                } else if (gaudi_queue_type[i] == QUEUE_TYPE_CPU) {
                        prop->hw_queues_props[i].type = QUEUE_TYPE_CPU;
                        prop->hw_queues_props[i].driver_only = 1;
                        prop->hw_queues_props[i].supports_sync_stream = 0;
                        prop->hw_queues_props[i].cb_alloc_flags =
                                CB_ALLOC_KERNEL;
                } else if (gaudi_queue_type[i] == QUEUE_TYPE_INT) {
                        prop->hw_queues_props[i].type = QUEUE_TYPE_INT;
                        prop->hw_queues_props[i].driver_only = 0;
                        prop->hw_queues_props[i].supports_sync_stream = 0;
                        prop->hw_queues_props[i].cb_alloc_flags =
                                CB_ALLOC_USER;

                }
                prop->hw_queues_props[i].collective_mode =
                                                get_collective_mode(hdev, i);
        }

        prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;
        prop->collective_first_sob = 0;
        prop->collective_first_mon = 0;

        /* 2 SOBs per internal queue stream are reserved for collective */
        prop->sync_stream_first_sob =
                        ALIGN(NUMBER_OF_SOBS_IN_GRP, HL_MAX_SOBS_PER_MONITOR)
                        * QMAN_STREAMS * HL_RSVD_SOBS;

        /* 1 monitor per internal queue stream are reserved for collective
         * 2 monitors per external queue stream are reserved for collective
         */
        prop->sync_stream_first_mon =
                        (NUMBER_OF_COLLECTIVE_QUEUES * QMAN_STREAMS) +
                        (NUMBER_OF_EXT_HW_QUEUES * 2);

        prop->dram_base_address = DRAM_PHYS_BASE;
        prop->dram_size = GAUDI_HBM_SIZE_32GB;
        prop->dram_end_address = prop->dram_base_address +
                                        prop->dram_size;
        prop->dram_user_base_address = DRAM_BASE_ADDR_USER;

        prop->sram_base_address = SRAM_BASE_ADDR;
        prop->sram_size = SRAM_SIZE;
        prop->sram_end_address = prop->sram_base_address +
                                        prop->sram_size;
        prop->sram_user_base_address = prop->sram_base_address +
                                        SRAM_USER_BASE_OFFSET;

        prop->mmu_pgt_addr = MMU_PAGE_TABLES_ADDR;
        if (hdev->pldm)
                prop->mmu_pgt_size = 0x800000; /* 8MB */
        else
                prop->mmu_pgt_size = MMU_PAGE_TABLES_SIZE;
        prop->mmu_pte_size = HL_PTE_SIZE;
        prop->mmu_hop_table_size = HOP_TABLE_SIZE;
        prop->mmu_hop0_tables_total_size = HOP0_TABLES_TOTAL_SIZE;
        prop->dram_page_size = PAGE_SIZE_2MB;
        prop->dram_supports_virtual_memory = false;

        prop->pmmu.hop0_shift = HOP0_SHIFT;
        prop->pmmu.hop1_shift = HOP1_SHIFT;
        prop->pmmu.hop2_shift = HOP2_SHIFT;
        prop->pmmu.hop3_shift = HOP3_SHIFT;
        prop->pmmu.hop4_shift = HOP4_SHIFT;
        prop->pmmu.hop0_mask = HOP0_MASK;
        prop->pmmu.hop1_mask = HOP1_MASK;
        prop->pmmu.hop2_mask = HOP2_MASK;
        prop->pmmu.hop3_mask = HOP3_MASK;
        prop->pmmu.hop4_mask = HOP4_MASK;
        prop->pmmu.start_addr = VA_HOST_SPACE_START;
        prop->pmmu.end_addr =
                        (VA_HOST_SPACE_START + VA_HOST_SPACE_SIZE / 2) - 1;
        prop->pmmu.page_size = PAGE_SIZE_4KB;
        prop->pmmu.num_hops = MMU_ARCH_5_HOPS;

        /* PMMU and HPMMU are the same except of page size */
        memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));
        prop->pmmu_huge.page_size = PAGE_SIZE_2MB;

        /* shifts and masks are the same in PMMU and DMMU */
        memcpy(&prop->dmmu, &prop->pmmu, sizeof(prop->pmmu));
        prop->dmmu.start_addr = (VA_HOST_SPACE_START + VA_HOST_SPACE_SIZE / 2);
        prop->dmmu.end_addr = VA_HOST_SPACE_END;
        prop->dmmu.page_size = PAGE_SIZE_2MB;

        prop->cfg_size = CFG_SIZE;
        prop->max_asid = MAX_ASID;
        prop->num_of_events = GAUDI_EVENT_SIZE;
        prop->tpc_enabled_mask = TPC_ENABLED_MASK;

        set_default_power_values(hdev);

        prop->cb_pool_cb_cnt = GAUDI_CB_POOL_CB_CNT;
        prop->cb_pool_cb_size = GAUDI_CB_POOL_CB_SIZE;

        prop->pcie_dbi_base_address = mmPCIE_DBI_BASE;
        prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;

        strncpy(prop->cpucp_info.card_name, GAUDI_DEFAULT_CARD_NAME,
                                        CARD_NAME_MAX_LEN);

        prop->max_pending_cs = GAUDI_MAX_PENDING_CS;

        prop->first_available_user_sob[HL_GAUDI_WS_DCORE] =
                        prop->sync_stream_first_sob +
                        (num_sync_stream_queues * HL_RSVD_SOBS);
        prop->first_available_user_mon[HL_GAUDI_WS_DCORE] =
                        prop->sync_stream_first_mon +
                        (num_sync_stream_queues * HL_RSVD_MONS);

        prop->first_available_user_msix_interrupt = USHRT_MAX;

        for (i = 0 ; i < HL_MAX_DCORES ; i++)
                prop->first_available_cq[i] = USHRT_MAX;

        prop->fw_cpu_boot_dev_sts0_valid = false;
        prop->fw_cpu_boot_dev_sts1_valid = false;
        prop->hard_reset_done_by_fw = false;
        prop->gic_interrupts_enable = true;

        return 0;
}

static int gaudi_pci_bars_map(struct hl_device *hdev)
{
        static const char * const name[] = {"SRAM", "CFG", "HBM"};
        bool is_wc[3] = {false, false, true};
        int rc;

        rc = hl_pci_bars_map(hdev, name, is_wc);
        if (rc)
                return rc;

        hdev->rmmio = hdev->pcie_bar[CFG_BAR_ID] +
                        (CFG_BASE - SPI_FLASH_BASE_ADDR);

        return 0;
}

static u64 gaudi_set_hbm_bar_base(struct hl_device *hdev, u64 addr)
{
        struct gaudi_device *gaudi = hdev->asic_specific;
        struct hl_inbound_pci_region pci_region;
        u64 old_addr = addr;
        int rc;

        if ((gaudi) && (gaudi->hbm_bar_cur_addr == addr))
                return old_addr;

        if (hdev->asic_prop.iatu_done_by_fw)
                return U64_MAX;

        /* Inbound Region 2 - Bar 4 - Point to HBM */
        pci_region.mode = PCI_BAR_MATCH_MODE;
        pci_region.bar = HBM_BAR_ID;
        pci_region.addr = addr;
        rc = hl_pci_set_inbound_region(hdev, 2, &pci_region);
        if (rc)
                return U64_MAX;

        if (gaudi) {
                old_addr = gaudi->hbm_bar_cur_addr;
                gaudi->hbm_bar_cur_addr = addr;
        }

        return old_addr;
}

static int gaudi_init_iatu(struct hl_device *hdev)
{
        struct hl_inbound_pci_region inbound_region;
        struct hl_outbound_pci_region outbound_region;
        int rc;

        if (hdev->asic_prop.iatu_done_by_fw)
                return 0;

        /* Inbound Region 0 - Bar 0 - Point to SRAM + CFG */
        inbound_region.mode = PCI_BAR_MATCH_MODE;
        inbound_region.bar = SRAM_BAR_ID;
        inbound_region.addr = SRAM_BASE_ADDR;
        rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);
        if (rc)
                goto done;

        /* Inbound Region 1 - Bar 2 - Point to SPI FLASH */
        inbound_region.mode = PCI_BAR_MATCH_MODE;
        inbound_region.bar = CFG_BAR_ID;
        inbound_region.addr = SPI_FLASH_BASE_ADDR;
        rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region);
        if (rc)
                goto done;

        /* Inbound Region 2 - Bar 4 - Point to HBM */
        inbound_region.mode = PCI_BAR_MATCH_MODE;
        inbound_region.bar = HBM_BAR_ID;
        inbound_region.addr = DRAM_PHYS_BASE;
        rc = hl_pci_set_inbound_region(hdev, 2, &inbound_region);
        if (rc)
                goto done;

        hdev->asic_funcs->set_dma_mask_from_fw(hdev);

        /* Outbound Region 0 - Point to Host */
        outbound_region.addr = HOST_PHYS_BASE;
        outbound_region.size = HOST_PHYS_SIZE;
        rc = hl_pci_set_outbound_region(hdev, &outbound_region);

done:
        return rc;
}

static enum hl_device_hw_state gaudi_get_hw_state(struct hl_device *hdev)
{
        return RREG32(mmHW_STATE);
}

static int gaudi_early_init(struct hl_device *hdev)
{
        struct asic_fixed_properties *prop = &hdev->asic_prop;
        struct pci_dev *pdev = hdev->pdev;
        u32 fw_boot_status;
        int rc;

        rc = gaudi_set_fixed_properties(hdev);
        if (rc) {
                dev_err(hdev->dev, "Failed setting fixed properties\n");
                return rc;
        }

        /* Check BAR sizes */
        if (pci_resource_len(pdev, SRAM_BAR_ID) != SRAM_BAR_SIZE) {
                dev_err(hdev->dev,
                        "Not " HL_NAME "? BAR %d size %llu, expecting %llu\n",
                        SRAM_BAR_ID,
                        (unsigned long long) pci_resource_len(pdev,
                                                        SRAM_BAR_ID),
                        SRAM_BAR_SIZE);
                rc = -ENODEV;
                goto free_queue_props;
        }

        if (pci_resource_len(pdev, CFG_BAR_ID) != CFG_BAR_SIZE) {
                dev_err(hdev->dev,
                        "Not " HL_NAME "? BAR %d size %llu, expecting %llu\n",
                        CFG_BAR_ID,
                        (unsigned long long) pci_resource_len(pdev,
                                                                CFG_BAR_ID),
                        CFG_BAR_SIZE);
                rc = -ENODEV;
                goto free_queue_props;
        }

        prop->dram_pci_bar_size = pci_resource_len(pdev, HBM_BAR_ID);

        /* If FW security is enabled at this point it means no access to ELBI */
        if (hdev->asic_prop.fw_security_enabled) {
                hdev->asic_prop.iatu_done_by_fw = true;

                /*
                 * GIC-security-bit can ONLY be set by CPUCP, so in this stage
                 * decision can only be taken based on PCI ID security.
                 */
                hdev->asic_prop.gic_interrupts_enable = false;
                goto pci_init;
        }

        rc = hl_pci_elbi_read(hdev, CFG_BASE + mmCPU_BOOT_DEV_STS0,
                                &fw_boot_status);
        if (rc)
                goto free_queue_props;

        /* Check whether FW is configuring iATU */
        if ((fw_boot_status & CPU_BOOT_DEV_STS0_ENABLED) &&
                        (fw_boot_status & CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN))
                hdev->asic_prop.iatu_done_by_fw = true;

pci_init:
        rc = hl_pci_init(hdev);
        if (rc)
                goto free_queue_props;

        /* Before continuing in the initialization, we need to read the preboot
         * version to determine whether we run with a security-enabled firmware
         */
        rc = hl_fw_read_preboot_status(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
                                        mmCPU_BOOT_DEV_STS0,
                                        mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0,
                                        mmCPU_BOOT_ERR1,
                                        GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC);
        if (rc) {
                if (hdev->reset_on_preboot_fail)
                        hdev->asic_funcs->hw_fini(hdev, true);
                goto pci_fini;
        }

        if (gaudi_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
                dev_info(hdev->dev,
                        "H/W state is dirty, must reset before initializing\n");
                hdev->asic_funcs->hw_fini(hdev, true);
        }

        return 0;

pci_fini:
        hl_pci_fini(hdev);
free_queue_props:
        kfree(hdev->asic_prop.hw_queues_props);
        return rc;
}

static int gaudi_early_fini(struct hl_device *hdev)
{
        kfree(hdev->asic_prop.hw_queues_props);
        hl_pci_fini(hdev);

        return 0;
}

/**
 * gaudi_fetch_psoc_frequency - Fetch PSOC frequency values
 *
 * @hdev: pointer to hl_device structure
 *
 */
static int gaudi_fetch_psoc_frequency(struct hl_device *hdev)
{
        struct asic_fixed_properties *prop = &hdev->asic_prop;
        u32 nr = 0, nf = 0, od = 0, div_fctr = 0, pll_clk, div_sel;
        u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS], freq;
        int rc;

        if (hdev->asic_prop.fw_security_enabled) {
                rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI_CPU_PLL, pll_freq_arr);

                if (rc)
                        return rc;

                freq = pll_freq_arr[2];
        } else {
                /* Backward compatibility */
                div_fctr = RREG32(mmPSOC_CPU_PLL_DIV_FACTOR_2);
                div_sel = RREG32(mmPSOC_CPU_PLL_DIV_SEL_2);
                nr = RREG32(mmPSOC_CPU_PLL_NR);
                nf = RREG32(mmPSOC_CPU_PLL_NF);
                od = RREG32(mmPSOC_CPU_PLL_OD);

                if (div_sel == DIV_SEL_REF_CLK ||
                                div_sel == DIV_SEL_DIVIDED_REF) {
                        if (div_sel == DIV_SEL_REF_CLK)
                                freq = PLL_REF_CLK;
                        else
                                freq = PLL_REF_CLK / (div_fctr + 1);
                } else if (div_sel == DIV_SEL_PLL_CLK ||
                        div_sel == DIV_SEL_DIVIDED_PLL) {
                        pll_clk = PLL_REF_CLK * (nf + 1) /
                                        ((nr + 1) * (od + 1));
                        if (div_sel == DIV_SEL_PLL_CLK)
                                freq = pll_clk;
                        else
                                freq = pll_clk / (div_fctr + 1);
                } else {
                        dev_warn(hdev->dev,
                                "Received invalid div select value: %d",
                                div_sel);
                        freq = 0;
                }
        }

        prop->psoc_timestamp_frequency = freq;
        prop->psoc_pci_pll_nr = nr;
        prop->psoc_pci_pll_nf = nf;
        prop->psoc_pci_pll_od = od;
        prop->psoc_pci_pll_div_factor = div_fctr;

        return 0;
}

static int _gaudi_init_tpc_mem(struct hl_device *hdev,
                dma_addr_t tpc_kernel_src_addr, u32 tpc_kernel_size)
{
        struct asic_fixed_properties *prop = &hdev->asic_prop;
        struct packet_lin_dma *init_tpc_mem_pkt;
        struct hl_cs_job *job;
        struct hl_cb *cb;
        u64 dst_addr;
        u32 cb_size, ctl;
        u8 tpc_id;
        int rc;

        cb = hl_cb_kernel_create(hdev, PAGE_SIZE, false);
        if (!cb)
                return -EFAULT;

        init_tpc_mem_pkt = cb->kernel_address;
        cb_size = sizeof(*init_tpc_mem_pkt);
        memset(init_tpc_mem_pkt, 0, cb_size);

        init_tpc_mem_pkt->tsize = cpu_to_le32(tpc_kernel_size);

        ctl = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_LIN_DMA);
        ctl |= FIELD_PREP(GAUDI_PKT_LIN_DMA_CTL_LIN_MASK, 1);
        ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1);
        ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1);

        init_tpc_mem_pkt->ctl = cpu_to_le32(ctl);

        init_tpc_mem_pkt->src_addr = cpu_to_le64(tpc_kernel_src_addr);
        dst_addr = (prop->sram_user_base_address &
                        GAUDI_PKT_LIN_DMA_DST_ADDR_MASK) >>
                        GAUDI_PKT_LIN_DMA_DST_ADDR_SHIFT;
        init_tpc_mem_pkt->dst_addr |= cpu_to_le64(dst_addr);

        job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
        if (!job) {
                dev_err(hdev->dev, "Failed to allocate a new job\n");
                rc = -ENOMEM;
                goto release_cb;
        }

        job->id = 0;
        job->user_cb = cb;
        atomic_inc(&job->user_cb->cs_cnt);
        job->user_cb_size = cb_size;
        job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0;
        job->patched_cb = job->user_cb;
        job->job_cb_size = job->user_cb_size + sizeof(struct packet_msg_prot);

        hl_debugfs_add_job(hdev, job);

        rc = gaudi_send_job_on_qman0(hdev, job);

        if (rc)
                goto free_job;

        for (tpc_id = 0 ; tpc_id < TPC_NUMBER_OF_ENGINES ; tpc_id++) {
                rc = gaudi_run_tpc_kernel(hdev, dst_addr, tpc_id);
                if (rc)
                        break;
        }

free_job:
        hl_userptr_delete_list(hdev, &job->userptr_list);
        hl_debugfs_remove_job(hdev, job);
        kfree(job);
        atomic_dec(&cb->cs_cnt);

release_cb:
        hl_cb_put(cb);
        hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);

        return rc;
}

/*
 * gaudi_init_tpc_mem() - Initialize TPC memories.
 * @hdev: Pointer to hl_device structure.
 *
 * Copy TPC kernel fw from firmware file and run it to initialize TPC memories.
 *
 * Return: 0 for success, negative value for error.
 */
static int gaudi_init_tpc_mem(struct hl_device *hdev)
{
        const struct firmware *fw;
        size_t fw_size;
        void *cpu_addr;
        dma_addr_t dma_handle;
        int rc, count = 5;

again:
        rc = request_firmware(&fw, GAUDI_TPC_FW_FILE, hdev->dev);
        if (rc == -EINTR && count-- > 0) {
                msleep(50);
                goto again;
        }

        if (rc) {
                dev_err(hdev->dev, "Failed to load firmware file %s\n",
                                GAUDI_TPC_FW_FILE);
                goto out;
        }

        fw_size = fw->size;
        cpu_addr = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, fw_size,
                        &dma_handle, GFP_KERNEL | __GFP_ZERO);
        if (!cpu_addr) {
                dev_err(hdev->dev,
                        "Failed to allocate %zu of dma memory for TPC kernel\n",
                        fw_size);
                rc = -ENOMEM;
                goto out;
        }

        memcpy(cpu_addr, fw->data, fw_size);

        rc = _gaudi_init_tpc_mem(hdev, dma_handle, fw_size);

        hdev->asic_funcs->asic_dma_free_coherent(hdev, fw->size, cpu_addr,
                        dma_handle);

out:
        release_firmware(fw);
        return rc;
}

static void gaudi_collective_map_sobs(struct hl_device *hdev, u32 stream)
{
        struct gaudi_device *gaudi = hdev->asic_specific;
        struct gaudi_collective_properties *prop = &gaudi->collective_props;
        struct hl_hw_queue *q;
        u32 i, sob_id, sob_group_id, queue_id;

        /* Iterate through SOB groups and assign a SOB for each slave queue */
        sob_group_id =
                stream * HL_RSVD_SOBS + prop->curr_sob_group_idx[stream];
        sob_id = prop->hw_sob_group[sob_group_id].base_sob_id;

        queue_id = GAUDI_QUEUE_ID_NIC_0_0 + stream;
        for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) {
                q = &hdev->kernel_queues[queue_id + (4 * i)];
                q->sync_stream_prop.collective_sob_id = sob_id + i;
        }

        /* Both DMA5 and TPC7 use the same resources since only a single
         * engine need to participate in the reduction process
         */
        queue_id = GAUDI_QUEUE_ID_DMA_5_0 + stream;
        q = &hdev->kernel_queues[queue_id];
        q->sync_stream_prop.collective_sob_id =
                        sob_id + NIC_NUMBER_OF_ENGINES;

        queue_id = GAUDI_QUEUE_ID_TPC_7_0 + stream;
        q = &hdev->kernel_queues[queue_id];
        q->sync_stream_prop.collective_sob_id =
                        sob_id + NIC_NUMBER_OF_ENGINES;
}

static void gaudi_sob_group_hw_reset(struct kref *ref)
{
        struct gaudi_hw_sob_group *hw_sob_group =
                container_of(ref, struct gaudi_hw_sob_group, kref);
        struct hl_device *hdev = hw_sob_group->hdev;
        u64 base_addr;
        int rc;

        base_addr = CFG_BASE + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 +
                        hw_sob_group->base_sob_id * 4;
        rc = gaudi_schedule_register_memset(hdev, hw_sob_group->queue_id,
                        base_addr, NUMBER_OF_SOBS_IN_GRP, 0);
        if (rc)
                dev_err(hdev->dev,
                        "failed resetting sob group - sob base %u, count %u",
                        hw_sob_group->base_sob_id, NUMBER_OF_SOBS_IN_GRP);

        kref_init(&hw_sob_group->kref);
}

static void gaudi_sob_group_reset_error(struct kref *ref)
{
        struct gaudi_hw_sob_group *hw_sob_group =
                container_of(ref, struct gaudi_hw_sob_group, kref);
        struct hl_device *hdev = hw_sob_group->hdev;

        dev_crit(hdev->dev,
                "SOB release shouldn't be called here, base_sob_id: %d\n",
                hw_sob_group->base_sob_id);

}

static void gaudi_collective_mstr_sob_mask_set(struct gaudi_device *gaudi)
{
        struct gaudi_collective_properties *prop;
        int i;

        prop = &gaudi->collective_props;

        memset(prop->mstr_sob_mask, 0, sizeof(prop->mstr_sob_mask));

        for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++)
                if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i))
                        prop->mstr_sob_mask[i / HL_MAX_SOBS_PER_MONITOR] |=
                                        BIT(i % HL_MAX_SOBS_PER_MONITOR);
        /* Set collective engine bit */
        prop->mstr_sob_mask[i / HL_MAX_SOBS_PER_MONITOR] |=
                                BIT(i % HL_MAX_SOBS_PER_MONITOR);
}

static int gaudi_collective_init(struct hl_device *hdev)
{
        u32 i, sob_id, reserved_sobs_per_group;
        struct gaudi_collective_properties *prop;
        struct gaudi_device *gaudi;

        gaudi = hdev->asic_specific;
        prop = &gaudi->collective_props;
        sob_id = hdev->asic_prop.collective_first_sob;

        /* First sob in group must be aligned to HL_MAX_SOBS_PER_MONITOR */
        reserved_sobs_per_group =
                ALIGN(NUMBER_OF_SOBS_IN_GRP, HL_MAX_SOBS_PER_MONITOR);

        /* Init SOB groups */
        for (i = 0 ; i < NUM_SOB_GROUPS; i++) {
                prop->hw_sob_group[i].hdev = hdev;
                prop->hw_sob_group[i].base_sob_id = sob_id;
                sob_id += reserved_sobs_per_group;
                gaudi_sob_group_hw_reset(&prop->hw_sob_group[i].kref);
        }

        for (i = 0 ; i < QMAN_STREAMS; i++) {
                prop->next_sob_group_val[i] = 1;
                prop->curr_sob_group_idx[i] = 0;
                gaudi_collective_map_sobs(hdev, i);
        }

        gaudi_collective_mstr_sob_mask_set(gaudi);

        return 0;
}

static void gaudi_reset_sob_group(struct hl_device *hdev, u16 sob_group)
{
        struct gaudi_device *gaudi = hdev->asic_specific;
        struct gaudi_collective_properties *cprop = &gaudi->collective_props;

        kref_put(&cprop->hw_sob_group[sob_group].kref,
                                        gaudi_sob_group_hw_reset);
}

static void gaudi_collective_master_init_job(struct hl_device *hdev,
                struct hl_cs_job *job, u32 stream, u32 sob_group_offset)
{
        u32 master_sob_base, master_monitor, queue_id, cb_size = 0;
        struct gaudi_collective_properties *cprop;
        struct hl_gen_wait_properties wait_prop;
        struct hl_sync_stream_properties *prop;
        struct gaudi_device *gaudi;

        gaudi = hdev->asic_specific;
        cprop = &gaudi->collective_props;
        queue_id = job->hw_queue_id;
        prop = &hdev->kernel_queues[queue_id].sync_stream_prop;

        master_sob_base =
                cprop->hw_sob_group[sob_group_offset].base_sob_id;
        master_monitor = prop->collective_mstr_mon_id[0];

        cprop->hw_sob_group[sob_group_offset].queue_id = queue_id;

        dev_dbg(hdev->dev,
                "Generate master wait CBs, sob %d (mask %#x), val:0x%x, mon %u, q %d\n",
                master_sob_base, cprop->mstr_sob_mask[0],
                cprop->next_sob_group_val[stream],
                master_monitor, queue_id);

        wait_prop.data = (void *) job->patched_cb;
        wait_prop.sob_base = master_sob_base;
        wait_prop.sob_mask = cprop->mstr_sob_mask[0];
        wait_prop.sob_val = cprop->next_sob_group_val[stream];
        wait_prop.mon_id = master_monitor;
        wait_prop.q_idx = queue_id;
        wait_prop.size = cb_size;
        cb_size += gaudi_gen_wait_cb(hdev, &wait_prop);

        master_sob_base += HL_MAX_SOBS_PER_MONITOR;
        master_monitor = prop->collective_mstr_mon_id[1];

        dev_dbg(hdev->dev,
                "Generate master wait CBs, sob %d (mask %#x), val:0x%x, mon %u, q %d\n",
                master_sob_base, cprop->mstr_sob_mask[1],
                cprop->next_sob_group_val[stream],
                master_monitor, queue_id);

        wait_prop.sob_base = master_sob_base;
        wait_prop.sob_mask = cprop->mstr_sob_mask[1];
        wait_prop.mon_id = master_monitor;
        wait_prop.size = cb_size;
        cb_size += gaudi_gen_wait_cb(hdev, &wait_prop);
}

static void gaudi_collective_slave_init_job(struct hl_device *hdev,
                struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl)
{
        struct hl_gen_wait_properties wait_prop;
        struct hl_sync_stream_properties *prop;
        u32 queue_id, cb_size = 0;

        queue_id = job->hw_queue_id;
        prop = &hdev->kernel_queues[queue_id].sync_stream_prop;

        /* Add to wait CBs using slave monitor */
        wait_prop.data = (void *) job->user_cb;
        wait_prop.sob_base = cs_cmpl->hw_sob->sob_id;
        wait_prop.sob_mask = 0x1;
        wait_prop.sob_val = cs_cmpl->sob_val;
        wait_prop.mon_id = prop->collective_slave_mon_id;
        wait_prop.q_idx = queue_id;
        wait_prop.size = cb_size;

        dev_dbg(hdev->dev,
                "Generate slave wait CB, sob %d, val:0x%x, mon %d, q %d\n",
                cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val,
                prop->collective_slave_mon_id, queue_id);

        cb_size += gaudi_gen_wait_cb(hdev, &wait_prop);

        dev_dbg(hdev->dev,
                "generate signal CB, sob_id: %d, sob val: 1, q_idx: %d\n",
                prop->collective_sob_id, queue_id);

        cb_size += gaudi_gen_signal_cb(hdev, job->user_cb,
                        prop->collective_sob_id, cb_size, false);
}

static void gaudi_collective_wait_init_cs(struct hl_cs *cs)
{
        struct hl_cs_compl *signal_cs_cmpl =
                container_of(cs->signal_fence, struct hl_cs_compl, base_fence);
        struct hl_cs_compl *cs_cmpl =
                container_of(cs->fence, struct hl_cs_compl, base_fence);
        struct gaudi_collective_properties *cprop;
        u32 stream, queue_id, sob_group_offset;
        struct gaudi_device *gaudi;
        struct hl_device *hdev;
        struct hl_cs_job *job;
        struct hl_ctx *ctx;

        ctx = cs->ctx;
        hdev = ctx->hdev;
        gaudi = hdev->asic_specific;
        cprop = &gaudi->collective_props;

        /* copy the SOB id and value of the signal CS */
        cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;
        cs_cmpl->sob_val = signal_cs_cmpl->sob_val;

        /* Calculate the stream from collective master queue (1st job) */
        job = list_first_entry(&cs->job_list, struct hl_cs_job, cs_node);
        stream = job->hw_queue_id % 4;
        sob_group_offset =
                stream * HL_RSVD_SOBS + cprop->curr_sob_group_idx[stream];

        list_for_each_entry(job, &cs->job_list, cs_node) {
                queue_id = job->hw_queue_id;

                if (hdev->kernel_queues[queue_id].collective_mode ==
                                HL_COLLECTIVE_MASTER)
                        gaudi_collective_master_init_job(hdev, job, stream,
                                                sob_group_offset);
                else
                        gaudi_collective_slave_init_job(hdev, job, cs_cmpl);
        }

        cs_cmpl->sob_group = sob_group_offset;

        /* Handle sob group kref and wraparound */
        kref_get(&cprop->hw_sob_group[sob_group_offset].kref);
        cprop->next_sob_group_val[stream]++;

        if (cprop->next_sob_group_val[stream] == HL_MAX_SOB_VAL) {
                /*
                 * Decrement as we reached the max value.
                 * The release function won't be called here as we've
                 * just incremented the refcount.
                 */
                kref_put(&cprop->hw_sob_group[sob_group_offset].kref,
                                gaudi_sob_group_reset_error);
                cprop->next_sob_group_val[stream] = 1;
                /* only two SOBs are currently in use */
                cprop->curr_sob_group_idx[stream] =
                        (cprop->curr_sob_group_idx[stream] + 1) &
                                                        (HL_RSVD_SOBS - 1);

                gaudi_collective_map_sobs(hdev, stream);

                dev_dbg(hdev->dev, "switched to SOB group %d, stream: %d\n",
                                cprop->curr_sob_group_idx[stream], stream);
        }

        /* Increment kref since all slave queues are now waiting on it */
        kref_get(&cs_cmpl->hw_sob->kref);
        /*
         * Must put the signal fence after the SOB refcnt increment so
         * the SOB refcnt won't turn 0 and reset the SOB before the
         * wait CS was submitted.
         */
        mb();
        hl_fence_put(cs->signal_fence);
        cs->signal_fence = NULL;
}

static int gaudi_collective_wait_create_job(struct hl_device *hdev,
                struct hl_ctx *ctx, struct hl_cs *cs,
                enum hl_collective_mode mode, u32 queue_id, u32 wait_queue_id)
{
        struct hw_queue_properties *hw_queue_prop;
        struct hl_cs_counters_atomic *cntr;
        struct hl_cs_job *job;
        struct hl_cb *cb;
        u32 cb_size;
        bool patched_cb;

        cntr = &hdev->aggregated_cs_counters;

        if (mode == HL_COLLECTIVE_MASTER) {
                /* CB size of collective master queue contains
                 * 4 msg short packets for monitor 1 configuration
                 * 1 fence packet
                 * 4 msg short packets for monitor 2 configuration
                 * 1 fence packet
                 * 2 msg prot packets for completion and MSI-X
                 */
                cb_size = sizeof(struct packet_msg_short) * 8 +
                                sizeof(struct packet_fence) * 2 +
                                sizeof(struct packet_msg_prot) * 2;
                patched_cb = true;
        } else {
                /* CB size of collective slave queues contains
                 * 4 msg short packets for monitor configuration
                 * 1 fence packet
                 * 1 additional msg short packet for sob signal
                 */
                cb_size = sizeof(struct packet_msg_short) * 5 +
                                sizeof(struct packet_fence);
                patched_cb = false;
        }

        hw_queue_prop = &hdev->asic_prop.hw_queues_props[queue_id];
        job = hl_cs_allocate_job(hdev, hw_queue_prop->type, true);
        if (!job) {
                atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
                atomic64_inc(&cntr->out_of_mem_drop_cnt);
                dev_err(hdev->dev, "Failed to allocate a new job\n");
                return -ENOMEM;
        }

        /* Allocate internal mapped CB for non patched CBs */
        cb = hl_cb_kernel_create(hdev, cb_size,
                        hdev->mmu_enable && !patched_cb);
        if (!cb) {
                atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);
                atomic64_inc(&cntr->out_of_mem_drop_cnt);
                kfree(job);
                return -EFAULT;
        }

        job->id = 0;
        job->cs = cs;
        job->user_cb = cb;
        atomic_inc(&job->user_cb->cs_cnt);
        job->user_cb_size = cb_size;
        job->hw_queue_id = queue_id;

        /*
         * No need in parsing, user CB is the patched CB.
         * We call hl_cb_destroy() out of two reasons - we don't need
         * the CB in the CB idr anymore and to decrement its refcount as
         * it was incremented inside hl_cb_kernel_create().
         */
        if (patched_cb)
                job->patched_cb = job->user_cb;
        else
                job->patched_cb = NULL;

        job->job_cb_size = job->user_cb_size;
        hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);

        /* increment refcount as for external queues we get completion */
        if (hw_queue_prop->type == QUEUE_TYPE_EXT)
                cs_get(cs);

        cs->jobs_in_queue_cnt[job->hw_queue_id]++;

        list_add_tail(&job->cs_node, &cs->job_list);

        hl_debugfs_add_job(hdev, job);

        return 0;
}

static int gaudi_collective_wait_create_jobs(struct hl_device *hdev,
                struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id,
                u32 collective_engine_id)
{
        struct gaudi_device *gaudi = hdev->asic_specific;
        struct hw_queue_properties *hw_queue_prop;
        u32 queue_id, collective_queue, num_jobs;
        u32 stream, nic_queue, nic_idx = 0;
        bool skip;
        int i, rc = 0;

        /* Verify wait queue id is configured as master */
        hw_queue_prop = &hdev->asic_prop.hw_queues_props[wait_queue_id];
        if (!(hw_queue_prop->collective_mode == HL_COLLECTIVE_MASTER)) {
                dev_err(hdev->dev,
                        "Queue %d is not configured as collective master\n",
                        wait_queue_id);
                return -EINVAL;
        }

        /* Verify engine id is supported */
        if (collective_engine_id != GAUDI_ENGINE_ID_DMA_5 &&
                        collective_engine_id != GAUDI_ENGINE_ID_TPC_7) {
                dev_err(hdev->dev,
                        "Collective wait does not support engine %u\n",
                        collective_engine_id);
                return -EINVAL;
        }

        stream = wait_queue_id % 4;

        if (collective_engine_id == GAUDI_ENGINE_ID_DMA_5)
                collective_queue = GAUDI_QUEUE_ID_DMA_5_0 + stream;
        else
                collective_queue = GAUDI_QUEUE_ID_TPC_7_0 + stream;

        num_jobs = NUMBER_OF_SOBS_IN_GRP + 1;
        nic_queue = GAUDI_QUEUE_ID_NIC_0_0 + stream;

        /* First job goes to the collective master queue, it will wait for
         * the collective slave queues to finish execution.
         * The synchronization is done using two monitors:
         * First monitor for NICs 0-7, second monitor for NICs 8-9 and the
         * reduction engine (DMA5/TPC7).
         *
         * Rest of the jobs goes to the collective slave queues which will
         * all wait for the user to signal sob 'cs_cmpl->sob_val'.
         */
        for (i = 0 ; i < num_jobs ; i++) {
                if (i == 0) {
                        queue_id = wait_queue_id;
                        rc = gaudi_collective_wait_create_job(hdev, ctx, cs,
                                HL_COLLECTIVE_MASTER, queue_id, wait_queue_id);
                } else {
                        if (nic_idx < NIC_NUMBER_OF_ENGINES) {
                                if (gaudi->hw_cap_initialized &
                                        BIT(HW_CAP_NIC_SHIFT + nic_idx))
                                        skip = false;
                                else
                                        skip = true;

                                queue_id = nic_queue;
                                nic_queue += 4;
                                nic_idx++;

                                if (skip)
                                        continue;
                        } else {
                                queue_id = collective_queue;
                        }

                        rc = gaudi_collective_wait_create_job(hdev, ctx, cs,
                                HL_COLLECTIVE_SLAVE, queue_id, wait_queue_id);
                }

                if (rc)
                        return rc;
        }

        return rc;
}

static int gaudi_late_init(struct hl_device *hdev)
{
        struct gaudi_device *gaudi = hdev->asic_specific;
        int rc;

        rc = gaudi->cpucp_info_get(hdev);
        if (rc) {
                dev_err(hdev->dev, "Failed to get cpucp info\n");
                return rc;
        }

        if ((hdev->card_type == cpucp_card_type_pci) &&
                        (hdev->nic_ports_mask & 0x3)) {
                dev_info(hdev->dev,
                        "PCI card detected, only 8 ports are enabled\n");
                hdev->nic_ports_mask &= ~0x3;

                /* Stop and disable unused NIC QMANs */
                WREG32(mmNIC0_QM0_GLBL_CFG1, NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK |
                                        NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK |
                                        NIC0_QM0_GLBL_CFG1_CP_STOP_MASK);

                WREG32(mmNIC0_QM1_GLBL_CFG1, NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK |
                                        NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK |
                                        NIC0_QM0_GLBL_CFG1_CP_STOP_MASK);

                WREG32(mmNIC0_QM0_GLBL_CFG0, 0);
                WREG32(mmNIC0_QM1_GLBL_CFG0, 0);

                gaudi->hw_cap_initialized &= ~(HW_CAP_NIC0 | HW_CAP_NIC1);
        }

        rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS);
        if (rc) {
                dev_err(hdev->dev, "Failed to enable PCI access from CPU\n");
                return rc;
        }

        rc = gaudi_fetch_psoc_frequency(hdev);
        if (rc) {
                dev_err(hdev->dev, "Failed to fetch psoc frequency\n");
                goto disable_pci_access;
        }

        rc = gaudi_mmu_clear_pgt_range(hdev);
        if (rc) {
                dev_err(hdev->dev, "Failed to clear MMU page tables range\n");
                goto disable_pci_access;
        }

        rc = gaudi_init_tpc_mem(hdev);
        if (rc) {
                dev_err(hdev->dev, "Failed to initialize TPC memories\n");
                goto disable_pci_access;
        }

        rc = gaudi_collective_init(hdev);
        if (rc) {
                dev_err(hdev->dev, "Failed to init collective\n");
                goto disable_pci_access;
        }

        return 0;

disable_pci_access:
        hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS);

        return rc;
}

static void gaudi_late_fini(struct hl_device *hdev)
{
        const struct hwmon_channel_info **channel_info_arr;
        int i = 0;

        if (!hdev->hl_chip_info->info)
                return;

        channel_info_arr = hdev->hl_chip_info->info;

        while (channel_info_arr[i]) {
                kfree(channel_info_arr[i]->config);
                kfree(channel_info_arr[i]);
                i++;
        }

        kfree(channel_info_arr);

        hdev->hl_chip_info->info = NULL;
}

static int gaudi_alloc_cpu_accessible_dma_mem(struct hl_device *hdev)
{
        dma_addr_t dma_addr_arr[GAUDI_ALLOC_CPU_MEM_RETRY_CNT] = {}, end_addr;
        void *virt_addr_arr[GAUDI_ALLOC_CPU_MEM_RETRY_CNT] = {};
        int i, j, rc = 0;

        /*
         * The device CPU works with 40-bits addresses, while bit 39 must be set
         * to '1' when accessing the host.
         * Bits 49:39 of the full host address are saved for a later
         * configuration of the HW to perform extension to 50 bits.
         * Because there is a single HW register that holds the extension bits,
         * these bits must be identical in all allocated range.
         */

        for (i = 0 ; i < GAUDI_ALLOC_CPU_MEM_RETRY_CNT ; i++) {
                virt_addr_arr[i] =
                        hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
                                                HL_CPU_ACCESSIBLE_MEM_SIZE,
                                                &dma_addr_arr[i],
                                                GFP_KERNEL | __GFP_ZERO);
                if (!virt_addr_arr[i]) {
                        rc = -ENOMEM;
                        goto free_dma_mem_arr;
                }

                end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1;
                if (GAUDI_CPU_PCI_MSB_ADDR(dma_addr_arr[i]) ==
                                GAUDI_CPU_PCI_MSB_ADDR(end_addr))
                        break;
        }

        if (i == GAUDI_ALLOC_CPU_MEM_RETRY_CNT) {
                dev_err(hdev->dev,
                        "MSB of CPU accessible DMA memory are not identical in all range\n");
                rc = -EFAULT;
                goto free_dma_mem_arr;
        }

        hdev->cpu_accessible_dma_mem = virt_addr_arr[i];
        hdev->cpu_accessible_dma_address = dma_addr_arr[i];
        hdev->cpu_pci_msb_addr =
                GAUDI_CPU_PCI_MSB_ADDR(hdev->cpu_accessible_dma_address);

        if (!hdev->asic_prop.fw_security_enabled)
                GAUDI_PCI_TO_CPU_ADDR(hdev->cpu_accessible_dma_address);

free_dma_mem_arr:
        for (j = 0 ; j < i ; j++)
                hdev->asic_funcs->asic_dma_free_coherent(hdev,
                                                HL_CPU_ACCESSIBLE_MEM_SIZE,
                                                virt_addr_arr[j],
                                                dma_addr_arr[j]);

        return rc;
}

static void gaudi_free_internal_qmans_pq_mem(struct hl_device *hdev)
{
        struct gaudi_device *gaudi = hdev->asic_specific;
        struct gaudi_internal_qman_info *q;
        u32 i;

        for (i = 0 ; i < GAUDI_QUEUE_ID_SIZE ; i++) {
                q = &gaudi->internal_qmans[i];
                if (!q->pq_kernel_addr)
                        continue;
                hdev->asic_funcs->asic_dma_free_coherent(hdev, q->pq_size,
                                                        q->pq_kernel_addr,
                                                        q->pq_dma_addr);
        }
}

static int gaudi_alloc_internal_qmans_pq_mem(struct hl_device *hdev)
{
        struct gaudi_device *gaudi = hdev->asic_specific;
        struct gaudi_internal_qman_info *q;
        int rc, i;

        for (i = 0 ; i < GAUDI_QUEUE_ID_SIZE ; i++) {
                if (gaudi_queue_type[i] != QUEUE_TYPE_INT)
                        continue;

                q = &gaudi->internal_qmans[i];

                switch (i) {
                case GAUDI_QUEUE_ID_DMA_2_0 ... GAUDI_QUEUE_ID_DMA_7_3:
                        q->pq_size = HBM_DMA_QMAN_SIZE_IN_BYTES;
                        break;
                case GAUDI_QUEUE_ID_MME_0_0 ... GAUDI_QUEUE_ID_MME_1_3:
                        q->pq_size = MME_QMAN_SIZE_IN_BYTES;
                        break;
                case GAUDI_QUEUE_ID_TPC_0_0 ... GAUDI_QUEUE_ID_TPC_7_3:
                        q->pq_size = TPC_QMAN_SIZE_IN_BYTES;
                        break;
                case GAUDI_QUEUE_ID_NIC_0_0 ... GAUDI_QUEUE_ID_NIC_9_3:
                        q->pq_size = NIC_QMAN_SIZE_IN_BYTES;
                        break;
                default:
                        dev_err(hdev->dev, "Bad internal queue index %d", i);
                        rc = -EINVAL;
                        goto free_internal_qmans_pq_mem;
                }

                q->pq_kernel_addr = hdev->asic_funcs->asic_dma_alloc_coherent(
                                                hdev, q->pq_size,
                                                &q->pq_dma_addr,
                                                GFP_KERNEL | __GFP_ZERO);
                if (!q->pq_kernel_addr) {
                        rc = -ENOMEM;
                        goto free_internal_qmans_pq_mem;
                }
        }

        return 0;

free_internal_qmans_pq_mem:
        gaudi_free_internal_qmans_pq_mem(hdev);
        return rc;
}

static void gaudi_set_pci_memory_regions(struct hl_device *hdev)
{
        struct asic_fixed_properties *prop = &hdev->asic_prop;
        struct pci_mem_region *region;

        /* CFG */
        region = &hdev->pci_mem_region[PCI_REGION_CFG];
        region->region_base = CFG_BASE;
        region->region_size = CFG_SIZE;
        region->offset_in_bar = CFG_BASE - SPI_FLASH_BASE_ADDR;
        region->bar_size = CFG_BAR_SIZE;
        region->bar_id = CFG_BAR_ID;
        region->used = 1;

        /* SRAM */
        region = &hdev->pci_mem_region[PCI_REGION_SRAM];
        region->region_base = SRAM_BASE_ADDR;
        region->region_size = SRAM_SIZE;
        region->offset_in_bar = 0;
        region->bar_size = SRAM_BAR_SIZE;
        region->bar_id = SRAM_BAR_ID;
        region->used = 1;

        /* DRAM */
        region = &hdev->pci_mem_region[PCI_REGION_DRAM];
        region->region_base = DRAM_PHYS_BASE;
        region->region_size = hdev->asic_prop.dram_size;
        region->offset_in_bar = 0;
        region->bar_size = prop->dram_pci_bar_size;
        region->bar_id = HBM_BAR_ID;
        region->used = 1;

        /* SP SRAM */
        region = &hdev->pci_mem_region[PCI_REGION_SP_SRAM];
        region->region_base = PSOC_SCRATCHPAD_ADDR;
        region->region_size = PSOC_SCRATCHPAD_SIZE;
        region->offset_in_bar = PSOC_SCRATCHPAD_ADDR - SPI_FLASH_BASE_ADDR;
        region->bar_size = CFG_BAR_SIZE;
        region->bar_id = CFG_BAR_ID;
        region->used = 1;
}

static int gaudi_sw_init(struct hl_device *hdev)
{
        struct gaudi_device *gaudi;
        u32 i, event_id = 0;
        int rc;

        /* Allocate device structure */
        gaudi = kzalloc(sizeof(*gaudi), GFP_KERNEL);
        if (!gaudi)
                return -ENOMEM;

        for (i = 0 ; i < ARRAY_SIZE(gaudi_irq_map_table) ; i++) {
                if (gaudi_irq_map_table[i].valid) {
                        if (event_id == GAUDI_EVENT_SIZE) {
                                dev_err(hdev->dev,
                                        "Event array exceeds the limit of %u events\n",
                                        GAUDI_EVENT_SIZE);
                                rc = -EINVAL;
                                goto free_gaudi_device;
                        }

                        gaudi->events[event_id++] =
                                        gaudi_irq_map_table[i].fc_id;
                }
        }

        gaudi->cpucp_info_get = gaudi_cpucp_info_get;

        gaudi->max_freq_value = GAUDI_MAX_CLK_FREQ;

        hdev->asic_specific = gaudi;

        /* Create DMA pool for small allocations */
        hdev->dma_pool = dma_pool_create(dev_name(hdev->dev),
                        &hdev->pdev->dev, GAUDI_DMA_POOL_BLK_SIZE, 8, 0);
        if (!hdev->dma_pool) {
                dev_err(hdev->dev, "failed to create DMA pool\n");
                rc = -ENOMEM;
                goto free_gaudi_device;
        }

        rc = gaudi_alloc_cpu_accessible_dma_mem(hdev);
        if (rc)
                goto free_dma_pool;

        hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);
        if (!hdev->cpu_accessible_dma_pool) {
                dev_err(hdev->dev,
                        "Failed to create CPU accessible DMA pool\n");
                rc = -ENOMEM;
                goto free_cpu_dma_mem;
        }

        rc = gen_pool_add(hdev->cpu_accessible_dma_pool,
                                (uintptr_t) hdev->cpu_accessible_dma_mem,
                                HL_CPU_ACCESSIBLE_MEM_SIZE, -1);
        if (rc) {
                dev_err(hdev->dev,
                        "Failed to add memory to CPU accessible DMA pool\n");
                rc = -EFAULT;
                goto free_cpu_accessible_dma_pool;
        }

        rc = gaudi_alloc_internal_qmans_pq_mem(hdev);
        if (rc)
                goto free_cpu_accessible_dma_pool;

        spin_lock_init(&gaudi->hw_queues_lock);
        mutex_init(&gaudi->clk_gate_mutex);

        hdev->supports_sync_stream = true;
        hdev->supports_coresight = true;
        hdev->supports_staged_submission = true;

        gaudi_set_pci_memory_regions(hdev);

        return 0;

free_cpu_accessible_dma_pool:
        gen_pool_destroy(hdev->cpu_accessible_dma_pool);
free_cpu_dma_mem:
        if (!hdev->asic_prop.fw_security_enabled)
                GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address,
                                        hdev->cpu_pci_msb_addr);
        hdev->asic_funcs->asic_dma_free_coherent(hdev,
                        HL_CPU_ACCESSIBLE_MEM_SIZE,
                        hdev->cpu_accessible_dma_mem,
                        hdev->cpu_accessible_dma_address);
free_dma_pool:
        dma_pool_destroy(hdev->dma_pool);
free_gaudi_device:
        kfree(gaudi);
        return rc;
}

static int gaudi_sw_fini(struct hl_device *hdev)
{
        struct gaudi_device *gaudi = hdev->asic_specific;

        gaudi_free_internal_qmans_pq_mem(hdev);

        gen_pool_destroy(hdev->cpu_accessible_dma_pool);

        if (!hdev->asic_prop.fw_security_enabled)
                GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address,
                                        hdev->cpu_pci_msb_addr);

        hdev->asic_funcs->asic_dma_free_coherent(hdev,
                        HL_CPU_ACCESSIBLE_MEM_SIZE,
                        hdev->cpu_accessible_dma_mem,
                        hdev->cpu_accessible_dma_address);

        dma_pool_destroy(hdev->dma_pool);

        mutex_destroy(&gaudi->clk_gate_mutex);

        kfree(gaudi);

        return 0;
}

static irqreturn_t gaudi_irq_handler_single(int irq, void *arg)
{
        struct hl_device *hdev = arg;
        int i;

        if (hdev->disabled)
                return IRQ_HANDLED;

        for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
                hl_irq_handler_cq(irq, &hdev->completion_queue[i]);

        hl_irq_handler_eq(irq, &hdev->event_queue);

        return IRQ_HANDLED;
}

/*
 * For backward compatibility, new MSI interrupts should be set after the
 * existing CPU and NIC interrupts.
 */
static int gaudi_pci_irq_vector(struct hl_device *hdev, unsigned int nr,
                                bool cpu_eq)
{
        int msi_vec;

        if ((nr != GAUDI_EVENT_QUEUE_MSI_IDX) && (cpu_eq))
                dev_crit(hdev->dev, "CPU EQ must use IRQ %d\n",
                                GAUDI_EVENT_QUEUE_MSI_IDX);

        msi_vec = ((nr < GAUDI_EVENT_QUEUE_MSI_IDX) || (cpu_eq)) ? nr :
                        (nr + NIC_NUMBER_OF_ENGINES + 1);

        return pci_irq_vector(hdev->pdev, msi_vec);
}

static int gaudi_enable_msi_single(struct hl_device *hdev)
{
        int rc, irq;

        dev_dbg(hdev->dev, "Working in single MSI IRQ mode\n");

        irq = gaudi_pci_irq_vector(hdev, 0, false);
        rc = request_irq(irq, gaudi_irq_handler_single, 0,
                        "gaudi single msi", hdev);
        if (rc)
                dev_err(hdev->dev,
                        "Failed to request single MSI IRQ\n");

        return rc;
}

static int gaudi_enable_msi_multi(struct hl_device *hdev)
{
        int cq_cnt = hdev->asic_prop.completion_queues_count;
        int rc, i, irq_cnt_init, irq;

        for (i = 0, irq_cnt_init = 0 ; i < cq_cnt ; i++, irq_cnt_init++) {
                irq = gaudi_pci_irq_vector(hdev, i, false);
                rc = request_irq(irq, hl_irq_handler_cq, 0, gaudi_irq_name[i],
                                &hdev->completion_queue[i]);
                if (rc) {
                        dev_err(hdev->dev, "Failed to request IRQ %d", irq);
                        goto free_irqs;
                }
        }

        irq = gaudi_pci_irq_vector(hdev, GAUDI_EVENT_QUEUE_MSI_IDX, true);
        rc = request_irq(irq, hl_irq_handler_eq, 0, gaudi_irq_name[cq_cnt],
                                &hdev->event_queue);
        if (rc) {
                dev_err(hdev->dev, "Failed to request IRQ %d", irq);
                goto free_irqs;
        }

        return 0;

free_irqs:
        for (i = 0 ; i < irq_cnt_init ; i++)
                free_irq(gaudi_pci_irq_vector(hdev, i, false),
                                &hdev->completion_queue[i]);
        return rc;
}

static int gaudi_enable_msi(struct hl_device *hdev)
{
        struct gaudi_device *gaudi = hdev->asic_specific;
        int rc;

        if (gaudi->hw_cap_initialized & HW_CAP_MSI)
                return 0;

        rc = pci_alloc_irq_vectors(hdev->pdev, 1, 1, PCI_IRQ_MSI);
        if (rc < 0) {
                dev_err(hdev->dev, "MSI: Failed to enable support %d\n", rc);
                return rc;
        }

        if (rc < NUMBER_OF_INTERRUPTS) {
                gaudi->multi_msi_mode = false;
                rc = gaudi_enable_msi_single(hdev);
        } else {
                gaudi->multi_msi_mode = true;
                rc = gaudi_enable_msi_multi(hdev);
        }

        if (rc)
                goto free_pci_irq_vectors;

        gaudi->hw_cap_initialized |= HW_CAP_MSI;

        return 0;

free_pci_irq_vectors:
        pci_free_irq_vectors(hdev->pdev);
        return rc;
}

static void gaudi_sync_irqs(struct hl_device *hdev)
{
        struct gaudi_device *gaudi = hdev->asic_specific;
        int i, cq_cnt = hdev->asic_prop.completion_queues_count;

        if (!(gaudi->hw_cap_initialized & HW_CAP_MSI))
                return;

        /* Wait for all pending IRQs to be finished */
        if (gaudi->multi_msi_mode) {
                for (i = 0 ; i < cq_cnt ; i++)
                        synchronize_irq(gaudi_pci_irq_vector(hdev, i, false));

                synchronize_irq(gaudi_pci_irq_vector(hdev,
                                                GAUDI_EVENT_QUEUE_MSI_IDX,
                                                true));
        } else {
                synchronize_irq(gaudi_pci_irq_vector(hdev, 0, false));
        }
}

static void gaudi_disable_msi(struct hl_device *hdev)
{
        struct gaudi_device *gaudi = hdev->asic_specific;
        int i, irq, cq_cnt = hdev->asic_prop.completion_queues_count;

        if (!(gaudi->hw_cap_initialized & HW_CAP_MSI))
                return;

        gaudi_sync_irqs(hdev);

        if (gaudi->multi_msi_mode) {
                irq = gaudi_pci_irq_vector(hdev, GAUDI_EVENT_QUEUE_MSI_IDX,
                                                true);
                free_irq(irq, &hdev->event_queue);

                for (i = 0 ; i < cq_cnt ; i++) {
                        irq = gaudi_pci_irq_vector(hdev, i, false);
                        free_irq(irq, &hdev->completion_queue[i]);
                }
        } else {
                free_irq(gaudi_pci_irq_vector(hdev, 0, false), hdev);
        }

        pci_free_irq_vectors(hdev->pdev);

        gaudi->hw_cap_initialized &= ~HW_CAP_MSI;
}

static void gaudi_init_scrambler_sram(struct hl_device *hdev)
{
        struct gaudi_device *gaudi = hdev->asic_specific;

        if (hdev->asic_prop.fw_security_enabled)
                return;

        if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
                                                CPU_BOOT_DEV_STS0_SRAM_SCR_EN)
                return;

        if (gaudi->hw_cap_initialized & HW_CAP_SRAM_SCRAMBLER)
                return;

        if (!hdev->sram_scrambler_enable)
                return;

        WREG32(mmNIF_RTR_CTRL_0_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_1_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_2_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_3_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_4_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_5_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_6_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_7_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);

        WREG32(mmSIF_RTR_CTRL_0_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_1_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_2_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_3_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_4_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_5_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_6_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_7_SCRAM_SRAM_EN,
                        1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);

        WREG32(mmDMA_IF_E_N_DOWN_CH0_SCRAM_SRAM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_E_N_DOWN_CH1_SCRAM_SRAM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_E_S_DOWN_CH0_SCRAM_SRAM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_E_S_DOWN_CH1_SCRAM_SRAM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_W_N_DOWN_CH0_SCRAM_SRAM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_W_N_DOWN_CH1_SCRAM_SRAM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_W_S_DOWN_CH0_SCRAM_SRAM_EN,

                        1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_W_S_DOWN_CH1_SCRAM_SRAM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);

        gaudi->hw_cap_initialized |= HW_CAP_SRAM_SCRAMBLER;
}

static void gaudi_init_scrambler_hbm(struct hl_device *hdev)
{
        struct gaudi_device *gaudi = hdev->asic_specific;

        if (hdev->asic_prop.fw_security_enabled)
                return;

        if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 &
                                        CPU_BOOT_DEV_STS0_DRAM_SCR_EN)
                return;

        if (gaudi->hw_cap_initialized & HW_CAP_HBM_SCRAMBLER)
                return;

        if (!hdev->dram_scrambler_enable)
                return;

        WREG32(mmNIF_RTR_CTRL_0_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_1_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_2_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_3_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_4_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_5_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_6_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_7_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);

        WREG32(mmSIF_RTR_CTRL_0_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_1_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_2_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_3_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_4_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_5_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_6_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_7_SCRAM_HBM_EN,
                        1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);

        WREG32(mmDMA_IF_E_N_DOWN_CH0_SCRAM_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_E_N_DOWN_CH1_SCRAM_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_E_S_DOWN_CH0_SCRAM_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_E_S_DOWN_CH1_SCRAM_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_W_N_DOWN_CH0_SCRAM_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_W_N_DOWN_CH1_SCRAM_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_W_S_DOWN_CH0_SCRAM_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_W_S_DOWN_CH1_SCRAM_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);

        gaudi->hw_cap_initialized |= HW_CAP_HBM_SCRAMBLER;
}

static void gaudi_init_e2e(struct hl_device *hdev)
{
        if (hdev->asic_prop.fw_security_enabled)
                return;

        if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 &
                                        CPU_BOOT_DEV_STS0_E2E_CRED_EN)
                return;

        WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_WR_SIZE, 247 >> 3);
        WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_RD_SIZE, 785 >> 3);
        WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_WR_SIZE, 49);
        WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_RD_SIZE, 101);

        WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_WR_SIZE, 275 >> 3);
        WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_RD_SIZE, 614 >> 3);
        WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_WR_SIZE, 1);
        WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_RD_SIZE, 39);

        WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_WR_SIZE, 1);
        WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_RD_SIZE, 1);
        WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_WR_SIZE, 1);
        WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_RD_SIZE, 32);

        WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_WR_SIZE, 176 >> 3);
        WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_RD_SIZE, 32 >> 3);
        WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_WR_SIZE, 19);
        WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_RD_SIZE, 32);

        WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_WR_SIZE, 176 >> 3);
        WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_RD_SIZE, 32 >> 3);
        WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_WR_SIZE, 19);
        WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_RD_SIZE, 32);

        WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_WR_SIZE, 1);
        WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_RD_SIZE, 1);
        WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_WR_SIZE, 1);
        WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_RD_SIZE, 32);

        WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_WR_SIZE, 275 >> 3);
        WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_RD_SIZE, 614 >> 3);
        WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_WR_SIZE, 1);
        WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_RD_SIZE, 39);

        WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_WR_SIZE, 297 >> 3);
        WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_RD_SIZE, 908 >> 3);
        WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_WR_SIZE, 19);
        WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_RD_SIZE, 19);

        WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_WR_SIZE, 318 >> 3);
        WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_RD_SIZE, 956 >> 3);
        WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_WR_SIZE, 79);
        WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_RD_SIZE, 163);

        WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_WR_SIZE, 275 >> 3);
        WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_RD_SIZE, 614 >> 3);
        WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_WR_SIZE, 1);
        WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_RD_SIZE, 39);

        WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_WR_SIZE, 1);
        WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_RD_SIZE, 1);
        WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_WR_SIZE, 1);
        WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_RD_SIZE, 32);

        WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_WR_SIZE, 176 >> 3);
        WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_RD_SIZE, 32 >> 3);
        WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_WR_SIZE, 19);
        WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_RD_SIZE, 32);

        WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_WR_SIZE, 176 >> 3);
        WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_RD_SIZE, 32 >> 3);
        WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_WR_SIZE, 19);
        WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_RD_SIZE, 32);

        WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_WR_SIZE, 1);
        WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_RD_SIZE, 1);
        WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_WR_SIZE, 1);
        WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_RD_SIZE, 32);

        WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_WR_SIZE, 275 >> 3);
        WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_RD_SIZE, 614 >> 3);
        WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_WR_SIZE, 1);
        WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_RD_SIZE, 39);

        WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_WR_SIZE, 318 >> 3);
        WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_RD_SIZE, 956 >> 3);
        WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_WR_SIZE, 79);
        WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_RD_SIZE, 79);

        WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);
        WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);
        WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_WR_SIZE, 162);
        WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_RD_SIZE, 338);

        WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);
        WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);
        WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_WR_SIZE, 162);
        WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_RD_SIZE, 338);

        WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);
        WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);
        WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_WR_SIZE, 162);
        WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_RD_SIZE, 338);

        WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);
        WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);
        WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_WR_SIZE, 162);
        WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_RD_SIZE, 338);

        WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);
        WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);
        WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_WR_SIZE, 162);
        WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_RD_SIZE, 338);

        WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);
        WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);
        WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_WR_SIZE, 162);
        WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_RD_SIZE, 338);

        WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);
        WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);
        WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_WR_SIZE, 162);
        WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_RD_SIZE, 338);

        WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);
        WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);
        WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_WR_SIZE, 162);
        WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_RD_SIZE, 338);

        if (!hdev->dram_scrambler_enable) {
                WREG32(mmSIF_RTR_CTRL_0_NL_HBM_SEL_0, 0x21);
                WREG32(mmSIF_RTR_CTRL_0_NL_HBM_SEL_1, 0x22);
                WREG32(mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmSIF_RTR_CTRL_0_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmSIF_RTR_CTRL_1_NL_HBM_SEL_0, 0x21);
                WREG32(mmSIF_RTR_CTRL_1_NL_HBM_SEL_1, 0x22);
                WREG32(mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmSIF_RTR_CTRL_1_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmSIF_RTR_CTRL_2_NL_HBM_SEL_0, 0x21);
                WREG32(mmSIF_RTR_CTRL_2_NL_HBM_SEL_1, 0x22);
                WREG32(mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmSIF_RTR_CTRL_2_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmSIF_RTR_CTRL_3_NL_HBM_SEL_0, 0x21);
                WREG32(mmSIF_RTR_CTRL_3_NL_HBM_SEL_1, 0x22);
                WREG32(mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmSIF_RTR_CTRL_3_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmSIF_RTR_CTRL_4_NL_HBM_SEL_0, 0x21);
                WREG32(mmSIF_RTR_CTRL_4_NL_HBM_SEL_1, 0x22);
                WREG32(mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmSIF_RTR_CTRL_4_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmSIF_RTR_CTRL_5_NL_HBM_SEL_0, 0x21);
                WREG32(mmSIF_RTR_CTRL_5_NL_HBM_SEL_1, 0x22);
                WREG32(mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmSIF_RTR_CTRL_5_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmSIF_RTR_CTRL_6_NL_HBM_SEL_0, 0x21);
                WREG32(mmSIF_RTR_CTRL_6_NL_HBM_SEL_1, 0x22);
                WREG32(mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmSIF_RTR_CTRL_6_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmSIF_RTR_CTRL_7_NL_HBM_SEL_0, 0x21);
                WREG32(mmSIF_RTR_CTRL_7_NL_HBM_SEL_1, 0x22);
                WREG32(mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmSIF_RTR_CTRL_7_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmNIF_RTR_CTRL_0_NL_HBM_SEL_0, 0x21);
                WREG32(mmNIF_RTR_CTRL_0_NL_HBM_SEL_1, 0x22);
                WREG32(mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmNIF_RTR_CTRL_0_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmNIF_RTR_CTRL_1_NL_HBM_SEL_0, 0x21);
                WREG32(mmNIF_RTR_CTRL_1_NL_HBM_SEL_1, 0x22);
                WREG32(mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmNIF_RTR_CTRL_1_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmNIF_RTR_CTRL_2_NL_HBM_SEL_0, 0x21);
                WREG32(mmNIF_RTR_CTRL_2_NL_HBM_SEL_1, 0x22);
                WREG32(mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmNIF_RTR_CTRL_2_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmNIF_RTR_CTRL_3_NL_HBM_SEL_0, 0x21);
                WREG32(mmNIF_RTR_CTRL_3_NL_HBM_SEL_1, 0x22);
                WREG32(mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmNIF_RTR_CTRL_3_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmNIF_RTR_CTRL_4_NL_HBM_SEL_0, 0x21);
                WREG32(mmNIF_RTR_CTRL_4_NL_HBM_SEL_1, 0x22);
                WREG32(mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmNIF_RTR_CTRL_4_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmNIF_RTR_CTRL_5_NL_HBM_SEL_0, 0x21);
                WREG32(mmNIF_RTR_CTRL_5_NL_HBM_SEL_1, 0x22);
                WREG32(mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmNIF_RTR_CTRL_5_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmNIF_RTR_CTRL_6_NL_HBM_SEL_0, 0x21);
                WREG32(mmNIF_RTR_CTRL_6_NL_HBM_SEL_1, 0x22);
                WREG32(mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmNIF_RTR_CTRL_6_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmNIF_RTR_CTRL_7_NL_HBM_SEL_0, 0x21);
                WREG32(mmNIF_RTR_CTRL_7_NL_HBM_SEL_1, 0x22);
                WREG32(mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmNIF_RTR_CTRL_7_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmDMA_IF_E_N_DOWN_CH0_NL_HBM_SEL_0, 0x21);
                WREG32(mmDMA_IF_E_N_DOWN_CH0_NL_HBM_SEL_1, 0x22);
                WREG32(mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmDMA_IF_E_N_DOWN_CH0_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmDMA_IF_E_N_DOWN_CH1_NL_HBM_SEL_0, 0x21);
                WREG32(mmDMA_IF_E_N_DOWN_CH1_NL_HBM_SEL_1, 0x22);
                WREG32(mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmDMA_IF_E_N_DOWN_CH1_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmDMA_IF_E_S_DOWN_CH0_NL_HBM_SEL_0, 0x21);
                WREG32(mmDMA_IF_E_S_DOWN_CH0_NL_HBM_SEL_1, 0x22);
                WREG32(mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmDMA_IF_E_S_DOWN_CH0_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmDMA_IF_E_S_DOWN_CH1_NL_HBM_SEL_0, 0x21);
                WREG32(mmDMA_IF_E_S_DOWN_CH1_NL_HBM_SEL_1, 0x22);
                WREG32(mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmDMA_IF_E_S_DOWN_CH1_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmDMA_IF_W_N_DOWN_CH0_NL_HBM_SEL_0, 0x21);
                WREG32(mmDMA_IF_W_N_DOWN_CH0_NL_HBM_SEL_1, 0x22);
                WREG32(mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmDMA_IF_W_N_DOWN_CH0_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmDMA_IF_W_N_DOWN_CH1_NL_HBM_SEL_0, 0x21);
                WREG32(mmDMA_IF_W_N_DOWN_CH1_NL_HBM_SEL_1, 0x22);
                WREG32(mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmDMA_IF_W_N_DOWN_CH1_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmDMA_IF_W_S_DOWN_CH0_NL_HBM_SEL_0, 0x21);
                WREG32(mmDMA_IF_W_S_DOWN_CH0_NL_HBM_SEL_1, 0x22);
                WREG32(mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmDMA_IF_W_S_DOWN_CH0_NL_HBM_PC_SEL_3, 0x20);

                WREG32(mmDMA_IF_W_S_DOWN_CH1_NL_HBM_SEL_0, 0x21);
                WREG32(mmDMA_IF_W_S_DOWN_CH1_NL_HBM_SEL_1, 0x22);
                WREG32(mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_18, 0x1F);
                WREG32(mmDMA_IF_W_S_DOWN_CH1_NL_HBM_PC_SEL_3, 0x20);
        }

        WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_EN,
                        1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_EN,
                        1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);

        WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
        WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_EN,
                        1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
}

static void gaudi_init_hbm_cred(struct hl_device *hdev)
{
        uint32_t hbm0_wr, hbm1_wr, hbm0_rd, hbm1_rd;

        if (hdev->asic_prop.fw_security_enabled)
                return;

        if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 &
                                                CPU_BOOT_DEV_STS0_HBM_CRED_EN)
                return;

        hbm0_wr = 0x33333333;
        hbm0_rd = 0x77777777;
        hbm1_wr = 0x55555555;
        hbm1_rd = 0xDDDDDDDD;

        WREG32(mmDMA_IF_E_N_HBM0_WR_CRED_CNT, hbm0_wr);
        WREG32(mmDMA_IF_E_N_HBM1_WR_CRED_CNT, hbm1_wr);
        WREG32(mmDMA_IF_E_N_HBM0_RD_CRED_CNT, hbm0_rd);
        WREG32(mmDMA_IF_E_N_HBM1_RD_CRED_CNT, hbm1_rd);

        WREG32(mmDMA_IF_E_S_HBM0_WR_CRED_CNT, hbm0_wr);
        WREG32(mmDMA_IF_E_S_HBM1_WR_CRED_CNT, hbm1_wr);
        WREG32(mmDMA_IF_E_S_HBM0_RD_CRED_CNT, hbm0_rd);
        WREG32(mmDMA_IF_E_S_HBM1_RD_CRED_CNT, hbm1_rd);

        WREG32(mmDMA_IF_W_N_HBM0_WR_CRED_CNT, hbm0_wr);
        WREG32(mmDMA_IF_W_N_HBM1_WR_CRED_CNT, hbm1_wr);
        WREG32(mmDMA_IF_W_N_HBM0_RD_CRED_CNT, hbm0_rd);
        WREG32(mmDMA_IF_W_N_HBM1_RD_CRED_CNT, hbm1_rd);

        WREG32(mmDMA_IF_W_S_HBM0_WR_CRED_CNT, hbm0_wr);
        WREG32(mmDMA_IF_W_S_HBM1_WR_CRED_CNT, hbm1_wr);
        WREG32(mmDMA_IF_W_S_HBM0_RD_CRED_CNT, hbm0_rd);
        WREG32(mmDMA_IF_W_S_HBM1_RD_CRED_CNT, hbm1_rd);

        WREG32(mmDMA_IF_E_N_HBM_CRED_EN_0,
                        (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
                        (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
        WREG32(mmDMA_IF_E_S_HBM_CRED_EN_0,
                        (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
                        (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
        WREG32(mmDMA_IF_W_N_HBM_CRED_EN_0,
                        (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
                        (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
        WREG32(mmDMA_IF_W_S_HBM_CRED_EN_0,
                        (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
                        (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));

        WREG32(mmDMA_IF_E_N_HBM_CRED_EN_1,
                        (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
                        (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
        WREG32(mmDMA_IF_E_S_HBM_CRED_EN_1,
                        (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
                        (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
        WREG32(mmDMA_IF_W_N_HBM_CRED_EN_1,
                        (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
                        (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
        WREG32(mmDMA_IF_W_S_HBM_CRED_EN_1,
                        (1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
                        (1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
}

static void gaudi_init_golden_registers(struct hl_device *hdev)
{
        u32 tpc_offset;
        int tpc_id, i;

        gaudi_init_e2e(hdev);
        gaudi_init_hbm_cred(hdev);

        for (tpc_id = 0, tpc_offset = 0;
                                tpc_id < TPC_NUMBER_OF_ENGINES;
                                tpc_id++, tpc_offset += TPC_CFG_OFFSET) {
                /* Mask all arithmetic interrupts from TPC */
                WREG32(mmTPC0_CFG_TPC_INTR_MASK + tpc_offset, 0x8FFF);
                /* Set 16 cache lines */
                WREG32_FIELD(TPC0_CFG_MSS_CONFIG, tpc_offset,
                                ICACHE_FETCH_LINE_NUM, 2);
        }

        /* Make sure 1st 128 bytes in SRAM are 0 for Tensor DMA */
        for (i = 0 ; i < 128 ; i += 8)
                writeq(0, hdev->pcie_bar[SRAM_BAR_ID] + i);

        WREG32(mmMME0_CTRL_EUS_ROLLUP_CNT_ADD, 3);
        WREG32(mmMME1_CTRL_EUS_ROLLUP_CNT_ADD, 3);
        WREG32(mmMME2_CTRL_EUS_ROLLUP_CNT_ADD, 3);
        WREG32(mmMME3_CTRL_EUS_ROLLUP_CNT_ADD, 3);
}

static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id,
                                        int qman_id, dma_addr_t qman_pq_addr)
{
        struct cpu_dyn_regs *dyn_regs =
                        &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
        u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi;
        u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi;
        u32 q_off, dma_qm_offset;
        u32 dma_qm_err_cfg, irq_handler_offset;

        dma_qm_offset = dma_id * DMA_QMAN_OFFSET;

        mtr_base_en_lo = lower_32_bits(CFG_BASE +
                                mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
        mtr_base_en_hi = upper_32_bits(CFG_BASE +
                                mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
        so_base_en_lo = lower_32_bits(CFG_BASE +
                                mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
        so_base_en_hi = upper_32_bits(CFG_BASE +
                                mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
        mtr_base_ws_lo = lower_32_bits(CFG_BASE +
                                mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
        mtr_base_ws_hi = upper_32_bits(CFG_BASE +
                                mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
        so_base_ws_lo = lower_32_bits(CFG_BASE +
                                mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);
        so_base_ws_hi = upper_32_bits(CFG_BASE +
                                mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);

        q_off = dma_qm_offset + qman_id * 4;

        WREG32(mmDMA0_QM_PQ_BASE_LO_0 + q_off, lower_32_bits(qman_pq_addr));
        WREG32(mmDMA0_QM_PQ_BASE_HI_0 + q_off, upper_32_bits(qman_pq_addr));

        WREG32(mmDMA0_QM_PQ_SIZE_0 + q_off, ilog2(HL_QUEUE_LENGTH));
        WREG32(mmDMA0_QM_PQ_PI_0 + q_off, 0);
        WREG32(mmDMA0_QM_PQ_CI_0 + q_off, 0);

        WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, QMAN_LDMA_SIZE_OFFSET);
        WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,
                                                        QMAN_LDMA_SRC_OFFSET);
        WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,
                                                        QMAN_LDMA_DST_OFFSET);

        WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo);
        WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi);
        WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo);
        WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi);
        WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_0 + q_off, mtr_base_ws_lo);
        WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_0 + q_off, mtr_base_ws_hi);
        WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_0 + q_off, so_base_ws_lo);
        WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_0 + q_off, so_base_ws_hi);

        WREG32(mmDMA0_QM_CP_BARRIER_CFG_0 + q_off, 0x100);

        /* The following configuration is needed only once per QMAN */
        if (qman_id == 0) {
                irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
                                mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
                                le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl);

                /* Configure RAZWI IRQ */
                dma_qm_err_cfg = PCI_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
                if (hdev->stop_on_err)
                        dma_qm_err_cfg |=
                                PCI_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;

                WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg);

                WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset,
                        lower_32_bits(CFG_BASE + irq_handler_offset));
                WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset,
                        upper_32_bits(CFG_BASE + irq_handler_offset));

                WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset,
                        gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id +
                                                                        dma_id);

                WREG32(mmDMA0_QM_ARB_ERR_MSG_EN + dma_qm_offset,
                                QM_ARB_ERR_MSG_EN_MASK);

                /* Increase ARB WDT to support streams architecture */
                WREG32(mmDMA0_QM_ARB_SLV_CHOISE_WDT + dma_qm_offset,
                                GAUDI_ARB_WDT_TIMEOUT);

                WREG32(mmDMA0_QM_GLBL_PROT + dma_qm_offset,
                                QMAN_EXTERNAL_MAKE_TRUSTED);

                WREG32(mmDMA0_QM_GLBL_CFG1 + dma_qm_offset, 0);
        }
}

static void gaudi_init_dma_core(struct hl_device *hdev, int dma_id)
{
        struct cpu_dyn_regs *dyn_regs =
                        &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
        u32 dma_err_cfg = 1 << DMA0_CORE_ERR_CFG_ERR_MSG_EN_SHIFT;
        u32 dma_offset = dma_id * DMA_CORE_OFFSET;
        u32 irq_handler_offset;

        /* Set to maximum possible according to physical size */
        WREG32(mmDMA0_CORE_RD_MAX_OUTSTAND + dma_offset, 0);
        WREG32(mmDMA0_CORE_RD_MAX_SIZE + dma_offset, 0);

        /* WA for H/W bug H3-2116 */
        WREG32(mmDMA0_CORE_LBW_MAX_OUTSTAND + dma_offset, 15);

        /* STOP_ON bit implies no completion to operation in case of RAZWI */
        if (hdev->stop_on_err)
                dma_err_cfg |= 1 << DMA0_CORE_ERR_CFG_STOP_ON_ERR_SHIFT;

        WREG32(mmDMA0_CORE_ERR_CFG + dma_offset, dma_err_cfg);

        irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
                        mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
                        le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl);

        WREG32(mmDMA0_CORE_ERRMSG_ADDR_LO + dma_offset,
                lower_32_bits(CFG_BASE + irq_handler_offset));
        WREG32(mmDMA0_CORE_ERRMSG_ADDR_HI + dma_offset,
                upper_32_bits(CFG_BASE + irq_handler_offset));

        WREG32(mmDMA0_CORE_ERRMSG_WDATA + dma_offset,
                gaudi_irq_map_table[GAUDI_EVENT_DMA0_CORE].cpu_id + dma_id);
        WREG32(mmDMA0_CORE_PROT + dma_offset,
                        1 << DMA0_CORE_PROT_ERR_VAL_SHIFT);
        /* If the channel is secured, it should be in MMU bypass mode */
        WREG32(mmDMA0_CORE_SECURE_PROPS + dma_offset,
                        1 << DMA0_CORE_SECURE_PROPS_MMBP_SHIFT);
        WREG32(mmDMA0_CORE_CFG_0 + dma_offset, 1 << DMA0_CORE_CFG_0_EN_SHIFT);
}

static void gaudi_enable_qman(struct hl_device *hdev, int dma_id,
                                u32 enable_mask)
{
        u32 dma_qm_offset = dma_id * DMA_QMAN_OFFSET;

        WREG32(mmDMA0_QM_GLBL_CFG0 + dma_qm_offset, enable_mask);
}

static void gaudi_init_pci_dma_qmans(struct hl_device *hdev)
{
        struct gaudi_device *gaudi = hdev->asic_specific;
        struct hl_hw_queue *q;
        int i, j, dma_id, cpu_skip, nic_skip, cq_id = 0, q_idx, msi_vec = 0;

        if (gaudi->hw_cap_initialized & HW_CAP_PCI_DMA)
                return;

        for (i = 0 ; i < PCI_DMA_NUMBER_OF_CHNLS ; i++) {
                dma_id = gaudi_dma_assignment[i];
                /*
                 * For queues after the CPU Q need to add 1 to get the correct
                 * queue. In addition, need to add the CPU EQ and NIC IRQs in
                 * order to get the correct MSI register.
                 */
                if (dma_id > 1) {
                        cpu_skip = 1;
                        nic_skip = NIC_NUMBER_OF_ENGINES;
                } else {
                        cpu_skip = 0;
                        nic_skip = 0;
                }

                for (j = 0 ; j < QMAN_STREAMS ; j++) {
                        q_idx = 4 * dma_id + j + cpu_skip;
                        q = &hdev->kernel_queues[q_idx];
                        q->cq_id = cq_id++;
                        q->msi_vec = nic_skip + cpu_skip + msi_vec++;
                        gaudi_init_pci_dma_qman(hdev, dma_id, j,
                                                q->bus_address);
                }

                gaudi_init_dma_core(hdev, dma_id);

                gaudi_enable_qman(hdev, dma_id, PCI_DMA_QMAN_ENABLE);
        }

        gaudi->hw_cap_initialized |= HW_CAP_PCI_DMA;
}

static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id,
                                        int qman_id, u64 qman_base_addr)
{
        struct cpu_dyn_regs *dyn_regs =
                        &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
        u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi;
        u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi;
        u32 dma_qm_err_cfg, irq_handler_offset;
        u32 q_off, dma_qm_offset;

        dma_qm_offset = dma_id * DMA_QMAN_OFFSET;

        mtr_base_en_lo = lower_32_bits(CFG_BASE +
                        mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
        mtr_base_en_hi = upper_32_bits(CFG_BASE +
                                mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
        so_base_en_lo = lower_32_bits(CFG_BASE +
                                mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
        so_base_en_hi = upper_32_bits(CFG_BASE +
                                mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
        mtr_base_ws_lo = lower_32_bits(CFG_BASE +
                                mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
        mtr_base_ws_hi = upper_32_bits(CFG_BASE +
                                mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
        so_base_ws_lo = lower_32_bits(CFG_BASE +
                                mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);
        so_base_ws_hi = upper_32_bits(CFG_BASE +
                                mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);

        q_off = dma_qm_offset + qman_id * 4;

        if (qman_id < 4) {
                WREG32(mmDMA0_QM_PQ_BASE_LO_0 + q_off,
                                        lower_32_bits(qman_base_addr));
                WREG32(mmDMA0_QM_PQ_BASE_HI_0 + q_off,
                                        upper_32_bits(qman_base_addr));

                WREG32(mmDMA0_QM_PQ_SIZE_0 + q_off, ilog2(HBM_DMA_QMAN_LENGTH));
                WREG32(mmDMA0_QM_PQ_PI_0 + q_off, 0);
                WREG32(mmDMA0_QM_PQ_CI_0 + q_off, 0);

                WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off,
                                                        QMAN_CPDMA_SIZE_OFFSET);
                WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,
                                                        QMAN_CPDMA_SRC_OFFSET);
                WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,
                                                        QMAN_CPDMA_DST_OFFSET);
        } else {
                irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
                                mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
                                le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl);

                WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off,
                                                        QMAN_LDMA_SIZE_OFFSET);
                WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,
                                                        QMAN_LDMA_SRC_OFFSET);
                WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,
                                                        QMAN_LDMA_DST_OFFSET);

                /* Configure RAZWI IRQ */
                dma_qm_err_cfg = HBM_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
                if (hdev->stop_on_err)
                        dma_qm_err_cfg |=
                                HBM_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;

                WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg);

                WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset,
                        lower_32_bits(CFG_BASE + irq_handler_offset));
                WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset,
                        upper_32_bits(CFG_BASE + irq_handler_offset));

                WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset,
                        gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id +
                                                                        dma_id);

                WREG32(mmDMA0_QM_ARB_ERR_MSG_EN + dma_qm_offset,
                                QM_ARB_ERR_MSG_EN_MASK);

                /* Increase ARB WDT to support streams architecture */
                WREG32(mmDMA0_QM_ARB_SLV_CHOISE_WDT + dma_qm_offset,
                                GAUDI_ARB_WDT_TIMEOUT);

                WREG32(mmDMA0_QM_GLBL_CFG1 + dma_qm_offset, 0);
                WREG32(mmDMA0_QM_GLBL_PROT + dma_qm_offset,
                                QMAN_INTERNAL_MAKE_TRUSTED);
        }

        WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo);
        WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi);
        WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo);
        WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi);

        /* Configure DMA5 CP_MSG_BASE 2/3 for sync stream collective */
        if (gaudi_dma_assignment[dma_id] == GAUDI_ENGINE_ID_DMA_5) {
                WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_0 + q_off,
                                mtr_base_ws_lo);
                WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_0 + q_off,
                                mtr_base_ws_hi);
                WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_0 + q_off,
                                so_base_ws_lo);
                WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_0 + q_off,
                                so_base_ws_hi);
        }
}

static void gaudi_init_hbm_dma_qmans(struct hl_device *hdev)
{
        struct gaudi_device *gaudi = hdev->asic_specific;
        struct gaudi_internal_qman_info *q;
        u64 qman_base_addr;
        int i, j, dma_id, internal_q_index;

        if (gaudi->hw_cap_initialized & HW_CAP_HBM_DMA)
                return;

        for (i = 0 ; i < HBM_DMA_NUMBER_OF_CHNLS ; i++) {
                dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_1 + i];

                for (j = 0 ; j < QMAN_STREAMS ; j++) {
                         /*
                          * Add the CPU queue in order to get the correct queue
                          * number as all internal queue are placed after it
                          */
                        internal_q_index = dma_id * QMAN_STREAMS + j + 1;

                        q = &gaudi->internal_qmans[internal_q_index];
                        qman_base_addr = (u64) q->pq_dma_addr;
                        gaudi_init_hbm_dma_qman(hdev, dma_id, j,
                                                qman_base_addr);
                }

                /* Initializing lower CP for HBM DMA QMAN */
                gaudi_init_hbm_dma_qman(hdev, dma_id, 4, 0);

                gaudi_init_dma_core(hdev, dma_id);

                gaudi_enable_qman(hdev, dma_id, HBM_DMA_QMAN_ENABLE);
        }

        gaudi->hw_cap_initialized |= HW_CAP_HBM_DMA;
}

static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset,
                                        int qman_id, u64 qman_base_addr)
{
        struct cpu_dyn_regs *dyn_regs =
                        &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
        u32 mtr_base_lo, mtr_base_hi;
        u32 so_base_lo, so_base_hi;
        u32 irq_handler_offset;
        u32 q_off, mme_id;
        u32 mme_qm_err_cfg;

        mtr_base_lo = lower_32_bits(CFG_BASE +
                                mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
        mtr_base_hi = upper_32_bits(CFG_BASE +
                                mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
        so_base_lo = lower_32_bits(CFG_BASE +
                                mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
        so_base_hi = upper_32_bits(CFG_BASE +
                                mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);

        q_off = mme_offset + qman_id * 4;

        if (qman_id < 4) {
                WREG32(mmMME0_QM_PQ_BASE_LO_0 + q_off,
                                        lower_32_bits(qman_base_addr));
                WREG32(mmMME0_QM_PQ_BASE_HI_0 + q_off,
                                        upper_32_bits(qman_base_addr));

                WREG32(mmMME0_QM_PQ_SIZE_0 + q_off, ilog2(MME_QMAN_LENGTH));
                WREG32(mmMME0_QM_PQ_PI_0 + q_off, 0);
                WREG32(mmMME0_QM_PQ_CI_0 + q_off, 0);

                WREG32(mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off,
                                                        QMAN_CPDMA_SIZE_OFFSET);
                WREG32(mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,
                                                        QMAN_CPDMA_SRC_OFFSET);
                WREG32(mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,
                                                        QMAN_CPDMA_DST_OFFSET);
        } else {
                irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
                                mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
                                le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl);

                WREG32(mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off,
                                                        QMAN_LDMA_SIZE_OFFSET);
                WREG32(mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,
                                                        QMAN_LDMA_SRC_OFFSET);
                WREG32(mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,
                                                        QMAN_LDMA_DST_OFFSET);

                /* Configure RAZWI IRQ */
                mme_id = mme_offset /
                                (mmMME1_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0) / 2;

                mme_qm_err_cfg = MME_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
                if (hdev->stop_on_err)
                        mme_qm_err_cfg |=
                                MME_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;

                WREG32(mmMME0_QM_GLBL_ERR_CFG + mme_offset, mme_qm_err_cfg);

                WREG32(mmMME0_QM_GLBL_ERR_ADDR_LO + mme_offset,
                        lower_32_bits(CFG_BASE + irq_handler_offset));
                WREG32(mmMME0_QM_GLBL_ERR_ADDR_HI + mme_offset,
                        upper_32_bits(CFG_BASE + irq_handler_offset));

                WREG32(mmMME0_QM_GLBL_ERR_WDATA + mme_offset,
                        gaudi_irq_map_table[GAUDI_EVENT_MME0_QM].cpu_id +
                                                                        mme_id);

                WREG32(mmMME0_QM_ARB_ERR_MSG_EN + mme_offset,
                                QM_ARB_ERR_MSG_EN_MASK);

                /* Increase ARB WDT to support streams architecture */
                WREG32(mmMME0_QM_ARB_SLV_CHOISE_WDT + mme_offset,
                                GAUDI_ARB_WDT_TIMEOUT);

                WREG32(mmMME0_QM_GLBL_CFG1 + mme_offset, 0);
                WREG32(mmMME0_QM_GLBL_PROT + mme_offset,
                                QMAN_INTERNAL_MAKE_TRUSTED);
        }

        WREG32(mmMME0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_lo);
        WREG32(mmMME0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_hi);
        WREG32(mmMME0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_lo);
        WREG32(mmMME0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_hi);
}

static void gaudi_init_mme_qmans(struct hl_device *hdev)
{
        struct gaudi_device *gaudi = hdev->asic_specific;
        struct gaudi_internal_qman_info *q;
        u64 qman_base_addr;
        u32 mme_offset;
        int i, internal_q_index;

        if (gaudi->hw_cap_initialized & HW_CAP_MME)
                return;

        /*
         * map GAUDI_QUEUE_ID_MME_0_X to the N_W_MME (mmMME2_QM_BASE)
         * and GAUDI_QUEUE_ID_MME_1_X to the S_W_MME (mmMME0_QM_BASE)
         */

        mme_offset = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0;

        for (i = 0 ; i < MME_NUMBER_OF_QMANS ; i++) {
                internal_q_index = GAUDI_QUEUE_ID_MME_0_0 + i;
                q = &gaudi->internal_qmans[internal_q_index];
                qman_base_addr = (u64) q->pq_dma_addr;
                gaudi_init_mme_qman(hdev, mme_offset, (i & 0x3),
                                        qman_base_addr);
                if (i == 3)
                        mme_offset = 0;
        }

        /* Initializing lower CP for MME QMANs */
        mme_offset = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0;
        gaudi_init_mme_qman(hdev, mme_offset, 4, 0);
        gaudi_init_mme_qman(hdev, 0, 4, 0);

        WREG32(mmMME2_QM_GLBL_CFG0, QMAN_MME_ENABLE);
        WREG32(mmMME0_QM_GLBL_CFG0, QMAN_MME_ENABLE);

        gaudi->hw_cap_initialized |= HW_CAP_MME;
}

static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset,
                                int qman_id, u64 qman_base_addr)
{
        struct cpu_dyn_regs *dyn_regs =
                        &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
        u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi;
        u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi;
        u32 tpc_qm_err_cfg, irq_handler_offset;