/* SPDX-License-Identifier: MIT */
/*
 * Copyright © 2022 Intel Corporation
 */

#ifndef _XE_VM_TYPES_H_
#define _XE_VM_TYPES_H_

#include <drm/drm_gpusvm.h>
#include <drm/drm_gpuvm.h>
#include <drm/drm_pagemap_util.h>

#include <linux/dma-resv.h>
#include <linux/kref.h>
#include <linux/mmu_notifier.h>
#include <linux/scatterlist.h>

#include "xe_device_types.h"
#include "xe_pt_types.h"
#include "xe_range_fence.h"
#include "xe_tlb_inval_types.h"
#include "xe_userptr.h"

struct drm_pagemap;

struct xe_bo;
struct xe_pagefault;
struct xe_svm_range;
struct xe_sync_entry;
struct xe_user_fence;
struct xe_vm;
struct xe_vm_pgtable_update_op;

#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
#define TEST_VM_OPS_ERROR
#define FORCE_OP_ERROR	BIT(31)

#define FORCE_OP_ERROR_LOCK	0
#define FORCE_OP_ERROR_PREPARE	1
#define FORCE_OP_ERROR_RUN	2
#define FORCE_OP_ERROR_COUNT	3
#endif

#define XE_VMA_READ_ONLY	DRM_GPUVA_USERBITS
#define XE_VMA_DESTROYED	(DRM_GPUVA_USERBITS << 1)
#define XE_VMA_ATOMIC_PTE_BIT	(DRM_GPUVA_USERBITS << 2)
#define XE_VMA_PTE_4K		(DRM_GPUVA_USERBITS << 3)
#define XE_VMA_PTE_2M		(DRM_GPUVA_USERBITS << 4)
#define XE_VMA_PTE_1G		(DRM_GPUVA_USERBITS << 5)
#define XE_VMA_PTE_64K		(DRM_GPUVA_USERBITS << 6)
#define XE_VMA_PTE_COMPACT	(DRM_GPUVA_USERBITS << 7)
#define XE_VMA_DUMPABLE		(DRM_GPUVA_USERBITS << 8)
#define XE_VMA_SYSTEM_ALLOCATOR	(DRM_GPUVA_USERBITS << 9)
#define XE_VMA_MADV_AUTORESET	(DRM_GPUVA_USERBITS << 10)

/**
 * struct xe_vma_mem_attr - memory attributes associated with vma
 */
struct xe_vma_mem_attr {
	/** @preferred_loc: preferred memory_location */
	struct xe_vma_preferred_loc {
		/** @preferred_loc.migration_policy: Pages migration policy */
		u32 migration_policy;

		/**
		 * @preferred_loc.devmem_fd: used for determining pagemap_fd
		 * requested by user DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM and
		 * DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE mean system memory or
		 * closest device memory respectively.
		 */
		u32 devmem_fd;
		/**
		 * @preferred_loc.dpagemap: Reference-counted pointer to the drm_pagemap preferred
		 * for migration on a SVM page-fault. The pointer is protected by the
		 * vm lock, and is %NULL if @devmem_fd should be consulted for special
		 * values.
		 */
		struct drm_pagemap *dpagemap;
	} preferred_loc;

	/**
	 * @atomic_access: The atomic access type for the vma
	 * See %DRM_XE_VMA_ATOMIC_UNDEFINED, %DRM_XE_VMA_ATOMIC_DEVICE,
	 * %DRM_XE_VMA_ATOMIC_GLOBAL, and %DRM_XE_VMA_ATOMIC_CPU for possible
	 * values. These are defined in uapi/drm/xe_drm.h.
	 */
	u32 atomic_access;

	/**
	 * @default_pat_index: The pat index for VMA set during first bind by user.
	 */
	u16 default_pat_index;

	/**
	 * @pat_index: The pat index to use when encoding the PTEs for this vma.
	 * same as default_pat_index unless overwritten by madvise.
	 */
	u16 pat_index;

	/**
	 * @purgeable_state: Purgeable hint for this VMA mapping
	 *
	 * Per-VMA purgeable state from madvise. Valid states are WILLNEED (0)
	 * or DONTNEED (1). Shared BOs require all VMAs to be DONTNEED before
	 * the BO can be purged. PURGED state exists only at BO level.
	 *
	 * Protected by BO dma-resv lock. Set via DRM_IOCTL_XE_MADVISE.
	 */
	u32 purgeable_state;
};

struct xe_vma {
	/** @gpuva: Base GPUVA object */
	struct drm_gpuva gpuva;

	/**
	 * @combined_links: links into lists which are mutually exclusive.
	 * Locking: vm lock in write mode OR vm lock in read mode and the vm's
	 * resv.
	 */
	union {
		/** @rebind: link into VM if this VMA needs rebinding. */
		struct list_head rebind;
		/** @destroy: link to contested list when VM is being closed. */
		struct list_head destroy;
	} combined_links;

	union {
		/** @destroy_cb: callback to destroy VMA when unbind job is done */
		struct dma_fence_cb destroy_cb;
		/** @destroy_work: worker to destroy this BO */
		struct work_struct destroy_work;
	};

	/**
	 * @fault_lock: Synchronizes fault processing. Locking order: inside
	 * vm->lock, outside dma-resv.
	 */
	struct mutex fault_lock;

	/**
	 * @tile_invalidated: Tile mask of binding are invalidated for this VMA.
	 * protected by BO's resv and for userptrs, vm->svm.gpusvm.notifier_lock in
	 * write mode for writing or vm->svm.gpusvm.notifier_lock in read mode and
	 * the vm->resv. For stable reading, BO's resv or userptr
	 * vm->svm.gpusvm.notifier_lock in read mode is required. Can be
	 * opportunistically read with READ_ONCE outside of locks.
	 */
	u8 tile_invalidated;

	/** @tile_mask: Tile mask of where to create binding for this VMA */
	u8 tile_mask;

	/**
	 * @tile_present: Tile mask of binding are present for this VMA.
	 * protected by vm->lock, vm->resv and for userptrs,
	 * vm->svm.gpusvm.notifier_lock for writing. Needs either for reading,
	 * but if reading is done under the vm->lock only, it needs to be held
	 * in write mode.
	 */
	u8 tile_present;

	/** @tile_staged: bind is staged for this VMA */
	u8 tile_staged;

	/**
	 * @skip_invalidation: Used in madvise to avoid invalidation
	 * if mem attributes doesn't change
	 */
	bool skip_invalidation;

	/**
	 * @ufence: The user fence that was provided with MAP.
	 * Needs to be signalled before UNMAP can be processed.
	 */
	struct xe_user_fence *ufence;

	/**
	 * @attr: The attributes of vma which determines the migration policy
	 * and encoding of the PTEs for this vma.
	 */
	struct xe_vma_mem_attr attr;
};

/**
 * struct xe_userptr_vma - A userptr vma subclass
 * @vma: The vma.
 * @userptr: Additional userptr information.
 */
struct xe_userptr_vma {
	struct xe_vma vma;
	struct xe_userptr userptr;
};

struct xe_device;

/**
 * struct xe_vm_fault_entry - Elements of vm->faults.list
 * @list: link into @xe_vm.faults.list
 * @address: address of the fault
 * @address_precision: precision of faulted address
 * @access_type: type of address access that resulted in fault
 * @fault_type: type of fault reported
 * @fault_level: fault level of the fault
 * @srcid: ID of the faulting hardware unit
 */
struct xe_vm_fault_entry {
	struct list_head list;
	u64 address;
	u32 address_precision;
	u8 access_type;
	u8 fault_type;
	u8 fault_level;
	u8 srcid;
};

struct xe_vm {
	/** @gpuvm: base GPUVM used to track VMAs */
	struct drm_gpuvm gpuvm;

	/** @svm: Shared virtual memory state */
	struct {
		/** @svm.gpusvm: base GPUSVM used to track fault allocations */
		struct drm_gpusvm gpusvm;
		/**
		 * @svm.range_lock: Protects insertion and removal of ranges
		 * from GPU SVM tree.
		 */
		struct mutex range_lock;
		/**
		 * @svm.garbage_collector: Garbage collector which is used unmap
		 * SVM range's GPU bindings and destroy the ranges.
		 */
		struct {
			/**
			 * @svm.garbage_collector.lock: Ensures only one thread
			 * runs the garbage collector at a time. Locking order:
			 * inside vm->lock, outside range->lock and dma-resv.
			 */
			struct mutex lock;
			/**
			 * @svm.garbage_collector.list_lock: Protect's range
			 * list
			 */
			spinlock_t list_lock;
			/**
			 * @svm.garbage_collector.range_list: List of SVM ranges
			 * in the garbage collector.
			 */
			struct list_head range_list;
			/**
			 * @svm.garbage_collector.work: Worker which the
			 * garbage collector runs on.
			 */
			struct work_struct work;
		} garbage_collector;
		struct xe_pagemap *pagemaps[XE_MAX_TILES_PER_DEVICE];
		/** @svm.peer: Used for pagemap connectivity computations. */
		struct drm_pagemap_peer peer;
	} svm;

	struct xe_device *xe;

	/* exec queue used for (un)binding vma's */
	struct xe_exec_queue *q;

	/** @lru_bulk_move: Bulk LRU move list for this VM's BOs */
	struct ttm_lru_bulk_move lru_bulk_move;

	u64 size;

	struct xe_pt *pt_root[XE_MAX_TILES_PER_DEVICE];
	struct xe_pt *scratch_pt[XE_MAX_TILES_PER_DEVICE][XE_VM_MAX_LEVEL];

	/**
	 * @flags: flags for this VM, statically setup a creation time aside
	 * from XE_VM_FLAG_BANNED which requires vm->lock to set / read safely
	 */
#define XE_VM_FLAG_64K			BIT(0)
#define XE_VM_FLAG_LR_MODE		BIT(1)
#define XE_VM_FLAG_MIGRATION		BIT(2)
#define XE_VM_FLAG_SCRATCH_PAGE		BIT(3)
#define XE_VM_FLAG_FAULT_MODE		BIT(4)
#define XE_VM_FLAG_BANNED		BIT(5)
#define XE_VM_FLAG_TILE_ID(flags)	FIELD_GET(GENMASK(7, 6), flags)
#define XE_VM_FLAG_SET_TILE_ID(tile)	FIELD_PREP(GENMASK(7, 6), (tile)->id)
#define XE_VM_FLAG_GSC			BIT(8)
#define XE_VM_FLAG_NO_VM_OVERCOMMIT     BIT(9)
	unsigned long flags;

	/**
	 * @lock: outer most lock, protects objects of anything attached to this
	 * VM
	 */
	struct rw_semaphore lock;
	/**
	 * @snap_mutex: Mutex used to guard insertions and removals from gpuva,
	 * so we can take a snapshot safely from devcoredump.
	 */
	struct mutex snap_mutex;

	/**
	 * @rebind_list: list of VMAs that need rebinding. Protected by the
	 * vm->lock in write mode, OR (the vm->lock in read mode and the
	 * vm resv).
	 */
	struct list_head rebind_list;

	/**
	 * @destroy_work: worker to destroy VM, needed as a dma_fence signaling
	 * from an irq context can be last put and the destroy needs to be able
	 * to sleep.
	 */
	struct work_struct destroy_work;

	/**
	 * @rftree: range fence tree to track updates to page table structure.
	 * Used to implement conflict tracking between independent bind engines.
	 */
	struct xe_range_fence_tree rftree;

	const struct xe_pt_ops *pt_ops;

	/** @userptr: user pointer state */
	struct xe_userptr_vm userptr;

	/** @preempt: preempt state */
	struct {
		/**
		 * @min_run_period_ms: The minimum run period before preempting
		 * an engine again
		 */
		unsigned int min_run_period_ms;
		/** @exec_queues: list of exec queues attached to this VM */
		struct list_head exec_queues;
		/** @num_exec_queues: number exec queues attached to this VM */
		int num_exec_queues;
		/**
		 * @rebind_deactivated: Whether rebind has been temporarily deactivated
		 * due to no work available. Protected by the vm resv.
		 */
		bool rebind_deactivated;
		/**
		 * @rebind_work: worker to rebind invalidated userptrs / evicted
		 * BOs
		 */
		struct work_struct rebind_work;
		/**
		 * @preempt.pm_activate_link: Link to list of rebind workers to be
		 * kicked on resume.
		 */
		struct list_head pm_activate_link;
	} preempt;

	/** @exec_queues: Manages list of exec queues attached to this VM, protected by lock. */
	struct {
		/**
		 * @exec_queues.list: list of exec queues attached to this VM,
		 * per GT
		 */
		struct list_head list[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE];
		/**
		 * @exec_queues.count: count of exec queues attached to this VM,
		 * per GT
		 */
		int count[XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE];
		/** @exec_queues.lock: lock to protect exec_queues list */
		struct rw_semaphore lock;
	} exec_queues;

	/** @um: unified memory state */
	struct {
		/** @asid: address space ID, unique to each VM */
		u32 asid;
	} usm;

	/** @error_capture: allow to track errors */
	struct {
		/** @capture_once: capture only one error per VM */
		bool capture_once;
	} error_capture;

	/** @faults: List of all faults associated with this VM */
	struct {
		/** @faults.lock: lock protecting @faults.list */
		spinlock_t lock;
		/** @faults.list: list of xe_vm_fault_entry entries */
		struct list_head list;
		/** @faults.len: length of @faults.list */
		unsigned int len;
	} faults;

	/**
	 * @validation: Validation data only valid with the vm resv held.
	 * Note: This is really task state of the task holding the vm resv,
	 * and moving forward we should
	 * come up with a better way of passing this down the call-
	 * chain.
	 */
	struct {
		/**
		 * @validation.validating: The task that is currently making bos resident.
		 * for this vm.
		 * Protected by the VM's resv for writing. Opportunistic reading can be done
		 * using READ_ONCE. Note: This is a workaround for the
		 * TTM eviction_valuable() callback not being passed a struct
		 * ttm_operation_context(). Future work might want to address this.
		 */
		struct task_struct *validating;
		/**
		 *  @validation.exec The drm_exec context used when locking the vm resv.
		 *  Protected by the vm's resv.
		 */
		struct drm_exec *_exec;
	} validation;

	/**
	 * @tlb_flush_seqno: Required TLB flush seqno for the next exec.
	 * protected by the vm resv.
	 */
	u64 tlb_flush_seqno;
	/** @batch_invalidate_tlb: Always invalidate TLB before batch start */
	bool batch_invalidate_tlb;
	/** @xef: Xe file handle for tracking this VM's drm client */
	struct xe_file *xef;
};

/** struct xe_vma_op_map - VMA map operation */
struct xe_vma_op_map {
	/** @vma: VMA to map */
	struct xe_vma *vma;
	/** @vma_flags: VMA flags for this operation */
	unsigned int vma_flags;
	/** @immediate: Immediate bind */
	bool immediate;
	/** @invalidate_on_bind: Invalidate on bind */
	bool invalidate_on_bind;
	/** @request_decompress: schedule decompression for GPU map */
	bool request_decompress;
	/** @pat_index: The pat index to use for this operation. */
	u16 pat_index;
};

/** struct xe_vma_op_remap - VMA remap operation */
struct xe_vma_op_remap {
	/** @prev: VMA preceding part of a split mapping */
	struct xe_vma *prev;
	/** @next: VMA subsequent part of a split mapping */
	struct xe_vma *next;
	/** @start: start of the VMA unmap */
	u64 start;
	/** @range: range of the VMA unmap */
	u64 range;
	/** @old_start: Original start of the VMA we unmap */
	u64 old_start;
	/** @old_range: Original range of the VMA we unmap */
	u64 old_range;
	/** @skip_prev: skip prev rebind */
	bool skip_prev;
	/** @skip_next: skip next rebind */
	bool skip_next;
	/** @unmap_done: unmap operation in done */
	bool unmap_done;
};

/** struct xe_vma_op_prefetch - VMA prefetch operation */
struct xe_vma_op_prefetch {
	/** @region: memory region to prefetch to */
	u32 region;
};

/** struct xe_vma_op_map_range - VMA map range operation */
struct xe_vma_op_map_range {
	/** @vma: VMA to map (system allocator VMA) */
	struct xe_vma *vma;
	/** @range: SVM range to map */
	struct xe_svm_range *range;
};

/** struct xe_vma_op_unmap_range - VMA unmap range operation */
struct xe_vma_op_unmap_range {
	/** @range: SVM range to unmap */
	struct xe_svm_range *range;
};

/** struct xe_vma_op_prefetch_range - VMA prefetch range operation */
struct xe_vma_op_prefetch_range {
	/** @range: xarray for SVM ranges data */
	struct xarray range;
	/** @ranges_count: number of svm ranges to map */
	u32 ranges_count;
	/**
	 * @dpagemap: Pointer to the dpagemap structure containing memory to prefetch.
	 * NULL if prefetch requested region is smem
	 */
	struct drm_pagemap *dpagemap;
};

/** enum xe_vma_op_flags - flags for VMA operation */
enum xe_vma_op_flags {
	/** @XE_VMA_OP_COMMITTED: VMA operation committed */
	XE_VMA_OP_COMMITTED		= BIT(0),
	/** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */
	XE_VMA_OP_PREV_COMMITTED	= BIT(1),
	/** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */
	XE_VMA_OP_NEXT_COMMITTED	= BIT(2),
};

/** enum xe_vma_subop - VMA sub-operation */
enum xe_vma_subop {
	/** @XE_VMA_SUBOP_MAP_RANGE: Map range */
	XE_VMA_SUBOP_MAP_RANGE,
	/** @XE_VMA_SUBOP_UNMAP_RANGE: Unmap range */
	XE_VMA_SUBOP_UNMAP_RANGE,
};

/** struct xe_vma_op - VMA operation */
struct xe_vma_op {
	/** @base: GPUVA base operation */
	struct drm_gpuva_op base;
	/** @link: async operation link */
	struct list_head link;
	/** @flags: operation flags */
	enum xe_vma_op_flags flags;
	/** @subop: user defined sub-operation */
	enum xe_vma_subop subop;
	/** @tile_mask: Tile mask for operation */
	u8 tile_mask;

	union {
		/** @map: VMA map operation specific data */
		struct xe_vma_op_map map;
		/** @remap: VMA remap operation specific data */
		struct xe_vma_op_remap remap;
		/** @prefetch: VMA prefetch operation specific data */
		struct xe_vma_op_prefetch prefetch;
		/** @map_range: VMA map range operation specific data */
		struct xe_vma_op_map_range map_range;
		/** @unmap_range: VMA unmap range operation specific data */
		struct xe_vma_op_unmap_range unmap_range;
		/** @prefetch_range: VMA prefetch range operation specific data */
		struct xe_vma_op_prefetch_range prefetch_range;
	};
};

/** struct xe_vma_ops - VMA operations */
struct xe_vma_ops {
	/** @list: list of VMA operations */
	struct list_head list;
	/** @vm: VM */
	struct xe_vm *vm;
	/** @q: exec queue for VMA operations */
	struct xe_exec_queue *q;
	/** @syncs: syncs these operation */
	struct xe_sync_entry *syncs;
	/** @num_syncs: number of syncs */
	u32 num_syncs;
	/** @pt_update_ops: page table update operations */
	struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE];
	/** @start: start address of ops */
	u64 start;
	/** @last: last address of ops */
	u64 last;
	/** @flag: signify the properties within xe_vma_ops*/
#define XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH	BIT(0)
#define XE_VMA_OPS_FLAG_MADVISE			BIT(1)
#define XE_VMA_OPS_ARRAY_OF_BINDS		BIT(2)
#define XE_VMA_OPS_FLAG_SKIP_TLB_WAIT		BIT(3)
#define XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP		BIT(4)
#define XE_VMA_OPS_FLAG_MODIFIES_GPUVA		BIT(5)
#define XE_VMA_OPS_FLAG_DOWNGRADE_LOCK		BIT(6)
#define XE_VMA_OPS_FLAG_HAS_SVM_VALID_RANGE	BIT(7)
#define XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP	BIT(8)
#define XE_VMA_OPS_FLAG_WAIT_VM_KERNEL		BIT(9)
#define XE_VMA_OPS_FLAG_NEEDS_INVALIDATION	BIT(10)
#define XE_VMA_OPS_FLAG_NEEDS_SVM_LOCK		BIT(11)
	u32 flags;
#ifdef TEST_VM_OPS_ERROR
	/** @inject_error: inject error to test error handling */
	bool inject_error;
#endif
};

#endif