/*
 * Copyright 2025 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 */
#include "amdgpu.h"
#include "gmc_v12_1.h"
#include "soc15_common.h"
#include "soc_v1_0.h"
#include "soc_v1_0_enum.h"
#include "oss/osssys_7_1_0_offset.h"
#include "oss/osssys_7_1_0_sh_mask.h"
#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"

static void gmc_v12_1_get_mtypes(struct amdgpu_device *adev,
				 unsigned int *mtype_local,
				 unsigned int *mtype_remote);

static int gmc_v12_1_vm_fault_interrupt_state(struct amdgpu_device *adev,
					      struct amdgpu_irq_src *src,
					      unsigned int type,
					      enum amdgpu_interrupt_state state)
{
	struct amdgpu_vmhub *hub;
	u32 tmp, reg, i, j;

	switch (state) {
	case AMDGPU_IRQ_STATE_DISABLE:
		for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
			hub = &adev->vmhub[j];
			for (i = 0; i < 16; i++) {
				reg = hub->vm_context0_cntl + i;

				/* This works because this interrupt is only
				 * enabled at init/resume and disabled in
				 * fini/suspend, so the overall state doesn't
				 * change over the course of suspend/resume.
				 */
				if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
					continue;

				if (j >= AMDGPU_MMHUB0(0))
					tmp = RREG32_SOC15_IP(MMHUB, reg);
				else
					tmp = RREG32_XCC(reg, j);

				tmp &= ~hub->vm_cntx_cntl_vm_fault;

				if (j >= AMDGPU_MMHUB0(0))
					WREG32_SOC15_IP(MMHUB, reg, tmp);
				else
					WREG32_XCC(reg, tmp, j);
			}
		}
		break;
	case AMDGPU_IRQ_STATE_ENABLE:
		for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
			hub = &adev->vmhub[j];
			for (i = 0; i < 16; i++) {
				reg = hub->vm_context0_cntl + i;

				/* This works because this interrupt is only
				 * enabled at init/resume and disabled in
				 * fini/suspend, so the overall state doesn't
				 * change over the course of suspend/resume.
				 */
				if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
					continue;

				if (j >= AMDGPU_MMHUB0(0))
					tmp = RREG32_SOC15_IP(MMHUB, reg);
				else
					tmp = RREG32_XCC(reg, j);

				tmp |= hub->vm_cntx_cntl_vm_fault;

				if (j >= AMDGPU_MMHUB0(0))
					WREG32_SOC15_IP(MMHUB, reg, tmp);
				else
					WREG32_XCC(reg, tmp, j);
			}
		}
		break;
	default:
		break;
	}

	return 0;
}

/*
 * NodeID definition in interrupt cookie for gmc v12.1.0
 *
 * DIE		IH Cookie NodeID
 * MID0		0x0
 * AID0		0x1
 * AID0.XCD0	0x2
 * AID0.XCD1	0x3
 * AID0.XCD2	0x4
 * AID0.XCD3	0x5
 * MID1		0x8
 * AID1		0x9
 * AID1.XCD0	0xA
 * AID1.XCD1	0xB
 * AID1.XCD2	0xC
 * AID1.XCD3	0xD
 */
static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev,
				       struct amdgpu_irq_src *source,
				       struct amdgpu_iv_entry *entry)
{
	struct amdgpu_task_info *task_info;
	bool retry_fault = false, write_fault = false;
	unsigned int vmhub, node_id;
	struct amdgpu_vmhub *hub;
	uint32_t cam_index = 0;
	const char *hub_name;
	int xcc_id = 0;
	uint32_t status = 0, status_hi = 0;
	const char *die_name;
	char die_name_buf[32];
	u64 addr;

	node_id = entry->node_id;

	addr = (u64)entry->src_data[0] << 12;
	addr |= ((u64)entry->src_data[1] & 0x1fff) << 44;

	if (entry->src_id == UTCL2_1_0__SRCID__RETRY) {
		retry_fault = true;
		write_fault = !!(entry->src_data[1] & AMDGPU_GMC121_FAULT_SOURCE_DATA_WRITE);
	}

	if (entry->client_id == SOC_V1_0_IH_CLIENTID_VMC) {
		hub_name = "mmhub0";
		vmhub = AMDGPU_MMHUB0(node_id / 8);
	} else {
		hub_name = "gfxhub0";
		if (adev->gfx.funcs->ih_node_to_logical_xcc) {
			xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev,
								node_id);
			if (xcc_id < 0)
				xcc_id = 0;
		}
		vmhub = xcc_id;
	}

	hub = &adev->vmhub[vmhub];

	if (retry_fault) {
		cam_index = entry->src_data[3] & 0x3ff;

		int ret = amdgpu_gmc_handle_retry_fault(adev, entry, addr, cam_index, node_id,
							write_fault);
		/* Returning 1 here also prevents sending the IV to the KFD */
		if (ret == 1)
			return 1;
	}

	if (kgd2kfd_vmfault_fast_path(adev, entry, retry_fault))
		return 1;

	if (!printk_ratelimit())
		return 0;

	dev_err(adev->dev,
		"[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n", hub_name,
		retry_fault ? "retry" : "no-retry",
		entry->src_id, entry->ring_id, entry->vmid, entry->pasid);

	task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid);
	if (task_info) {
		amdgpu_vm_print_task_info(adev, task_info);
		amdgpu_vm_put_task_info(task_info);
	}

	dev_err(adev->dev, "  in page starting at address 0x%016llx from IH client %d (%s)\n",
		addr, entry->client_id, soc_v1_0_ih_clientid_name[entry->client_id]);

	if (adev->irq.ih_funcs &&
	    adev->irq.ih_funcs->node_id_to_die_name) {
		die_name = adev->irq.ih_funcs->node_id_to_die_name(adev, node_id,
								   die_name_buf,
								   sizeof(die_name_buf));
		if (die_name)
			dev_err(adev->dev,
				"  cookie node_id %d fault from die %s\n",
				node_id, die_name);
	}

	if (amdgpu_sriov_vf(adev))
		return 0;

	/*
	 * Issue a dummy read to wait for the status register to
	 * be updated to avoid reading an incorrect value due to
	 * the new fast GRBM interface.
	 */
	if (entry->vmid_src == AMDGPU_GFXHUB(0))
		RREG32(hub->vm_l2_pro_fault_status);

	status = RREG32(hub->vm_l2_pro_fault_status);
	if (hub->vmhub_funcs &&
	    hub->vmhub_funcs->print_l2_protection_fault_status_hi)
		status_hi = RREG32(hub->vm_l2_pro_fault_status_hi);

	/* Only print L2 fault status if the status register could be read and
	 * contains useful information
	 */
	if (!status && !status_hi)
		return 0;

	WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);

	amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, vmhub);

	if (!hub->vmhub_funcs || !hub->vmhub_funcs->print_l2_protection_fault_status) {
		dev_warn_once(adev->dev, "vmhub %d print fault status func not defined\n", vmhub);
		return 0;
	}

	hub->vmhub_funcs->print_l2_protection_fault_status(adev, status);
	if (hub->vmhub_funcs->print_l2_protection_fault_status_hi)
		hub->vmhub_funcs->print_l2_protection_fault_status_hi(adev,
								     status_hi);

	return 0;
}

static bool gmc_v12_1_get_vmid_pasid_mapping_info(struct amdgpu_device *adev,
						  uint8_t vmid, uint8_t inst,
						  uint16_t *p_pasid)
{
	uint16_t index;

	if (inst/4)
		index = 0xA + inst%4;
	else
		index = 0x2 + inst%4;

	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_LUT_INDEX), index);

	*p_pasid = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid) & 0xffff;

	return !!(*p_pasid);
}

/*
 * GART
 * VMID 0 is the physical GPU addresses as used by the kernel.
 * VMIDs 1-15 are used for userspace clients and are handled
 * by the amdgpu vm/hsa code.
 */

/**
 * gmc_v12_1_use_invalidate_semaphore - judge whether to use semaphore
 *
 * @adev: amdgpu_device pointer
 * @vmhub: vmhub type
 *
 */
static bool gmc_v12_1_use_invalidate_semaphore(struct amdgpu_device *adev,
				       uint32_t vmhub)
{
	return ((!AMDGPU_IS_GFXHUB(vmhub)) &&
		(!amdgpu_sriov_vf(adev)));
}

static void gmc_v12_1_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
				   unsigned int vmhub, uint32_t flush_type)
{
	bool use_semaphore = gmc_v12_1_use_invalidate_semaphore(adev, vmhub);
	struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
	u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
	u32 tmp;
	/* Use register 17 for GART */
	const unsigned eng = 17;
	unsigned int i;
	u32 inst;

	inst = AMDGPU_IS_GFXHUB(vmhub) ? vmhub - AMDGPU_GFXHUB_START : 0;

	spin_lock(&adev->gmc.invalidate_lock);

	if (use_semaphore) {
		for (i = 0; i < adev->usec_timeout; i++) {
			/* a read return value of 1 means semaphore acuqire */
			if (AMDGPU_IS_GFXHUB(vmhub))
				tmp = RREG32_SOC15_IP_NO_KIQ(GC,
							     hub->vm_inv_eng0_sem +
							     hub->eng_distance * eng,
							     inst);
			else
				tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB,
							     hub->vm_inv_eng0_sem +
							     hub->eng_distance * eng,
							     inst);
			if (tmp & 0x1)
				break;
			udelay(1);
		}

		if (i >= adev->usec_timeout)
			DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
	}

	if (AMDGPU_IS_GFXHUB(vmhub))
		WREG32_SOC15_IP_NO_KIQ(GC,
				       hub->vm_inv_eng0_req + hub->eng_distance * eng,
				       inv_req, inst);
	else
		WREG32_SOC15_IP_NO_KIQ(MMHUB,
				       hub->vm_inv_eng0_req + hub->eng_distance * eng,
				       inv_req, inst);

	/* Wait for ACK with a delay.*/
	for (i = 0; i < adev->usec_timeout; i++) {
		if (AMDGPU_IS_GFXHUB(vmhub))
			tmp = RREG32_SOC15_IP_NO_KIQ(GC,
						     hub->vm_inv_eng0_ack +
						     hub->eng_distance * eng,
						     inst);
		else
			tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB,
						     hub->vm_inv_eng0_ack +
						     hub->eng_distance * eng,
						     inst);
		tmp &= 1 << vmid;
		if (tmp)
			break;

		udelay(1);
	}

	if (use_semaphore) {
		if (AMDGPU_IS_GFXHUB(vmhub))
			WREG32_SOC15_IP_NO_KIQ(GC,
					       hub->vm_inv_eng0_sem + hub->eng_distance * eng,
					       0, inst);
		else
			WREG32_SOC15_IP_NO_KIQ(MMHUB,
					       hub->vm_inv_eng0_sem + hub->eng_distance * eng,
					       0, inst);
	}

	/* Issue additional private vm invalidation to MMHUB */
	if (!AMDGPU_IS_GFXHUB(vmhub) &&
	    (hub->vm_l2_bank_select_reserved_cid2) &&
		!amdgpu_sriov_vf(adev)) {
		inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
		/* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */
		inv_req |= (1 << 25);
		/* Issue private invalidation */
		WREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2, inv_req);
		/* Read back to ensure invalidation is done*/
		RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
	}

	spin_unlock(&adev->gmc.invalidate_lock);

	if (i < adev->usec_timeout)
		return;

	dev_err(adev->dev, "Timeout waiting for VM flush ACK!\n");
}

/**
 * gmc_v12_1_flush_gpu_tlb - gart tlb flush callback
 *
 * @adev: amdgpu_device pointer
 * @vmid: vm instance to flush
 * @vmhub: which hub to flush
 * @flush_type: the flush type
 *
 * Flush the TLB for the requested page table.
 */
static void gmc_v12_1_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
				    uint32_t vmhub, uint32_t flush_type)
{
	u32 inst;

	if (AMDGPU_IS_GFXHUB(vmhub) &&
	    !adev->gfx.is_poweron)
		return;

	if (vmhub >= AMDGPU_MMHUB0(0))
		inst = 0;
	else
		inst = vmhub;

	/* This is necessary for SRIOV as well as for GFXOFF to function
	 * properly under bare metal
	 */
	if (adev->mes.ring[MES_PIPE_INST(inst, 0)].sched.ready &&
	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) {
		struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
		const unsigned eng = 17;
		u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
		u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
		u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;

		amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req,
				1 << vmid, inst);
		return;
	}

	gmc_v12_1_flush_vm_hub(adev, vmid, vmhub, flush_type);
	return;
}

/**
 * gmc_v12_1_flush_gpu_tlb_pasid - tlb flush via pasid
 *
 * @adev: amdgpu_device pointer
 * @pasid: pasid to be flush
 * @flush_type: the flush type
 * @all_hub: flush all hubs
 * @inst: XCC instance to use for the invalidation
 *
 * Flush the TLB for the requested pasid.
 */
static void gmc_v12_1_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
					  uint16_t pasid, uint32_t flush_type,
					  bool all_hub, uint32_t inst)
{
	uint16_t queried;
	int vmid, i;

	if (adev->mes.ring[0].sched.ready &&
	    (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x6f) {
		struct mes_inv_tlbs_pasid_input input = {0};
		input.xcc_id = inst;
		input.pasid = pasid;
		input.flush_type = flush_type;

		/* MES will invalidate hubs for the device(including slave xcc) from master, ignore request from slave */
		if (!amdgpu_gfx_is_master_xcc(adev, inst))
			return;

		input.hub_id = AMDGPU_GFXHUB(0);
		adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);

		if (all_hub) {
			/* invalidate mm_hub */
			if (test_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask)) {
				input.hub_id = AMDGPU_MMHUB0(0);
				adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
			}
			if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) {
				input.hub_id = AMDGPU_MMHUB1(0);
				adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
			}
		}
		return;
	}

	for (vmid = 1; vmid < 16; vmid++) {
		bool valid;

		valid = gmc_v12_1_get_vmid_pasid_mapping_info(adev, vmid, inst,
							      &queried);
		if (!valid || queried != pasid)
			continue;

		if (all_hub) {
			for_each_set_bit(i, adev->vmhubs_mask,
					 AMDGPU_MAX_VMHUBS)
				gmc_v12_1_flush_gpu_tlb(adev, vmid, i,
							flush_type);
		} else {
			gmc_v12_1_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(inst),
						flush_type);
		}
	}
}

static uint64_t gmc_v12_1_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
					     unsigned vmid, uint64_t pd_addr)
{
	bool use_semaphore = gmc_v12_1_use_invalidate_semaphore(ring->adev, ring->vm_hub);
	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
	uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
	unsigned eng = ring->vm_inv_eng;

	if (use_semaphore)
		/* a read return value of 1 means semaphore acuqire */
		amdgpu_ring_emit_reg_wait(ring,
					  hub->vm_inv_eng0_sem +
					  hub->eng_distance * eng, 0x1, 0x1);

	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
			      (hub->ctx_addr_distance * vmid),
			      lower_32_bits(pd_addr));

	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
			      (hub->ctx_addr_distance * vmid),
			      upper_32_bits(pd_addr));

	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
					    hub->eng_distance * eng,
					    hub->vm_inv_eng0_ack +
					    hub->eng_distance * eng,
					    req, 1 << vmid);

	if (use_semaphore)
		/*
		 * add semaphore release after invalidation,
		 * write with 0 means semaphore release
		 */
		amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
				      hub->eng_distance * eng, 0);

	return pd_addr;
}

static void gmc_v12_1_emit_pasid_mapping(struct amdgpu_ring *ring,
					 unsigned vmid, unsigned pasid)
{
	struct amdgpu_device *adev = ring->adev;
	uint32_t reg;

	if (ring->vm_hub == AMDGPU_GFXHUB(0))
		reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid;
	else
		reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT_MM) + vmid;

	amdgpu_ring_emit_wreg(ring, reg, pasid);
}

/*
 * PTE format:
 * 63 P
 * 62:59 reserved
 * 58 D
 * 57 G
 * 56 T
 * 55:54 M
 * 53:52 SW
 * 51:48 reserved for future
 * 47:12 4k physical page base address
 * 11:7 fragment
 * 6 write
 * 5 read
 * 4 exe
 * 3 Z
 * 2 snooped
 * 1 system
 * 0 valid
 *
 * PDE format:
 * 63 P
 * 62:58 block fragment size
 * 57 reserved
 * 56 A
 * 55:54 M
 * 53:52 reserved
 * 51:48 reserved for future
 * 47:6 physical base address of PD or PTE
 * 5:3 reserved
 * 2 C
 * 1 system
 * 0 valid
 */

static void gmc_v12_1_get_vm_pde(struct amdgpu_device *adev, int level,
				 uint64_t *addr, uint64_t *flags)
{
	if (!(*flags & AMDGPU_PDE_PTE_GFX12) && !(*flags & AMDGPU_PTE_SYSTEM))
		*addr = adev->vm_manager.vram_base_offset + *addr -
			adev->gmc.vram_start;

	if (*addr & ~adev->gmc.pte_addr_mask) {
		dev_err(adev->dev, "Invalid PDE address: %llx\n", *addr);
		*addr = 0;
		*flags &= ~AMDGPU_PTE_VALID;
	}

	*flags |= AMDGPU_PTE_SNOOPED;

	if (!adev->gmc.translate_further)
		return;

	if (level == AMDGPU_VM_PDB1) {
		/* Set the block fragment size */
		if (!(*flags & AMDGPU_PDE_PTE_GFX12))
			*flags |= AMDGPU_PDE_BFS_GFX12(0x9);
	}
}

static void gmc_v12_1_get_npa_flags(struct amdgpu_device *adev,
				    uint64_t *flags)
{
	unsigned int mtype_local, mtype_remote;

	gmc_v12_1_get_mtypes(adev, &mtype_local, &mtype_remote);

	*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, mtype_remote);
	/* VSCT = 0011 to identify NPA. Additionally PTE.B = 1 */
	*flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PTE_PRT_GFX12 |
		   AMDGPU_PTE_BUS_ATOMICS;
	*flags &= ~AMDGPU_PTE_VALID;
	*flags &= ~AMDGPU_PTE_SYSTEM;
	*flags &= ~AMDGPU_PTE_EXECUTABLE;
}

/*
 * Resolve the MTYPEs used for local and remote memory accesses on GFX 12.1.
 * Both default to an ASIC-dependent value that can be overridden through the
 * amdgpu_mtype_local and amdgpu_mtype_remote module parameters.
 */
static void gmc_v12_1_get_mtypes(struct amdgpu_device *adev,
				 unsigned int *mtype_local,
				 unsigned int *mtype_remote)
{
	bool is_aid_a1 = SOC_V1_0_DIE_REV_AID(adev->rev_id) == 1;

	/* Local memory: ASIC default depends on the AID stepping. */
	*mtype_local = is_aid_a1 ? MTYPE_RW : MTYPE_NC;
	if (amdgpu_mtype_local == 0)
		*mtype_local = MTYPE_RW;
	else if (amdgpu_mtype_local == 1)
		*mtype_local = MTYPE_NC;
	else if (amdgpu_mtype_local == 2)
		DRM_INFO_ONCE("MTYPE_CC not supported for local memory\n");

	/* Remote memory defaults to MTYPE_UC on GFX 12.1. */
	*mtype_remote = MTYPE_UC;
	if (amdgpu_mtype_remote == 0)
		*mtype_remote = MTYPE_NC;
	else if (amdgpu_mtype_remote == 1)
		*mtype_remote = MTYPE_UC;

	DRM_INFO_ONCE("Using %s for local memory and %s for remote memory\n",
		      *mtype_local == MTYPE_RW ? "MTYPE_RW" : "MTYPE_NC",
		      *mtype_remote == MTYPE_NC ? "MTYPE_NC" : "MTYPE_UC");
}

/*
 * The compute MQD coherent_aql_mtype field (offset 509) must be programmed to
 * 0 whenever the driver maps local or remote memory as MTYPE_NC, and to 1 in
 * all other cases.
 */
u32 gmc_v12_1_get_coherent_aql_mtype(struct amdgpu_device *adev)
{
	unsigned int mtype_local, mtype_remote;

	gmc_v12_1_get_mtypes(adev, &mtype_local, &mtype_remote);

	return (mtype_local == MTYPE_NC || mtype_remote == MTYPE_NC) ? 0 : 1;
}

static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev,
					  struct amdgpu_bo *bo,
					  uint64_t *flags)
{
	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
	bool is_vram = bo->tbo.resource &&
		       bo->tbo.resource->mem_type == TTM_PL_VRAM;
	bool coherent = bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
				     AMDGPU_GEM_CREATE_EXT_COHERENT);
	bool ext_coherent = bo->flags & AMDGPU_GEM_CREATE_EXT_COHERENT;
	uint32_t gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0);
	bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED;
	unsigned int mtype, mtype_local, mtype_remote;
	bool snoop = false;
	bool is_local = false;
	bool is_aid_a1, is_spx;

	switch (gc_ip_version) {
	case IP_VERSION(12, 1, 0):
		gmc_v12_1_get_mtypes(adev, &mtype_local, &mtype_remote);

		is_local = (is_vram && adev == bo_adev);
		snoop = true;
		if (uncached) {
			mtype = MTYPE_UC;
		} else if (ext_coherent) {
			is_aid_a1 = SOC_V1_0_DIE_REV_AID(adev->rev_id) == 1;
			is_spx = amdgpu_xcp_query_partition_mode(adev->xcp_mgr,
								 AMDGPU_XCP_FL_NONE) ==
				 AMDGPU_SPX_PARTITION_MODE;
			/* AID A0 requires MTYPE_UC for extended-scope coherent
			 * local memory in DPX/QPX/CPX modes.
			 */
			if (is_local && (is_aid_a1 || is_spx))
				mtype = mtype_local;
			else
				mtype = MTYPE_UC;
		} else {
			mtype = is_local ? mtype_local : mtype_remote;
		}
		break;
	default:
		if (uncached || coherent)
			mtype = MTYPE_UC;
		else
			mtype = MTYPE_NC;
	}

	if (mtype != MTYPE_NC)
		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, mtype);

	if (is_local || adev->have_atomics_support)
		*flags |= AMDGPU_PTE_BUS_ATOMICS;

	*flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
}

static void gmc_v12_1_get_vm_pte(struct amdgpu_device *adev,
				 struct amdgpu_vm *vm,
				 struct amdgpu_bo *bo,
				 uint32_t vm_flags,
				 uint64_t *flags)
{
	struct ttm_resource *mem;

	if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE)
		*flags |= AMDGPU_PTE_EXECUTABLE;
	else
		*flags &= ~AMDGPU_PTE_EXECUTABLE;

	switch (vm_flags & AMDGPU_VM_MTYPE_MASK) {
	case AMDGPU_VM_MTYPE_DEFAULT:
		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC);
		break;
	case AMDGPU_VM_MTYPE_NC:
	default:
		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC);
		break;
	case AMDGPU_VM_MTYPE_RW:
		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_RW);
		break;
	case AMDGPU_VM_MTYPE_UC:
		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_UC);
		break;
	}

	if (bo) {
		mem = bo->tbo.resource;
		if (mem && mem->mem_type == AMDGPU_PL_NPA) {
			dev_dbg(adev->dev,
				"Setting PTE for NPA BO, mem->type: %d, mem->start: %lx, mem->size: %u, cur_flags: %llx\n",
				mem->mem_type, mem->start, (u32)mem->size, *flags);
			gmc_v12_1_get_npa_flags(adev, flags);
		} else if (*flags & AMDGPU_PTE_VALID) {
			gmc_v12_1_get_coherence_flags(adev, bo, flags);
		}
	}
}

static const struct amdgpu_gmc_funcs gmc_v12_1_gmc_funcs = {
	.flush_gpu_tlb = gmc_v12_1_flush_gpu_tlb,
	.flush_gpu_tlb_pasid = gmc_v12_1_flush_gpu_tlb_pasid,
	.emit_flush_gpu_tlb = gmc_v12_1_emit_flush_gpu_tlb,
	.emit_pasid_mapping = gmc_v12_1_emit_pasid_mapping,
	.get_vm_pde = gmc_v12_1_get_vm_pde,
	.get_vm_pte = gmc_v12_1_get_vm_pte,
	.query_mem_partition_mode = &amdgpu_gmc_query_memory_partition,
	.request_mem_partition_mode = &amdgpu_gmc_request_memory_partition,
};

void gmc_v12_1_set_gmc_funcs(struct amdgpu_device *adev)
{
	adev->gmc.gmc_funcs = &gmc_v12_1_gmc_funcs;
}

static const struct amdgpu_irq_src_funcs gmc_v12_1_irq_funcs = {
	.set = gmc_v12_1_vm_fault_interrupt_state,
	.process = gmc_v12_1_process_interrupt,
};

static const struct amdgpu_irq_src_funcs gmc_v12_1_ecc_funcs = {
	.process = amdgpu_umc_uniras_process_ecc_irq,
};

void gmc_v12_1_set_irq_funcs(struct amdgpu_device *adev)
{
	adev->gmc.vm_fault.num_types = 1;
	adev->gmc.vm_fault.funcs = &gmc_v12_1_irq_funcs;

	adev->gmc.ecc_irq.num_types = 1;
	adev->gmc.ecc_irq.funcs = &gmc_v12_1_ecc_funcs;
}

void gmc_v12_1_init_vram_info(struct amdgpu_device *adev)
{
	/* TODO: query vram_info from ip discovery binary */
	adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM4;
	adev->gmc.vram_width = 384 * 64;
}

void gmc_v12_1_init_nps_details(struct amdgpu_device *adev)
{
	enum amdgpu_memory_partition mode;
	uint32_t supp_modes;
	int i;

	adev->gmc.supported_nps_modes = 0;

	if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU))
		return;

	mode = amdgpu_gmc_get_memory_partition(adev, &supp_modes);

	/* Mode detected by hardware and supported modes available */
	if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) && supp_modes) {
		while ((i = ffs(supp_modes))) {
			if (AMDGPU_ALL_NPS_MASK & BIT(i))
				adev->gmc.supported_nps_modes |= BIT(i);
			supp_modes &= supp_modes - 1;
		}
	} else {
		switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
		case IP_VERSION(12, 1, 0):
			adev->gmc.supported_nps_modes =
				BIT(AMDGPU_NPS1_PARTITION_MODE) |
				BIT(AMDGPU_NPS2_PARTITION_MODE);
			break;
		default:
			break;
		}
	}
}