/* SPDX-License-Identifier: MIT */ /* * Copyright © 2025 Intel Corporation */ #ifndef _XE_PAGEFAULT_TYPES_H_ #define _XE_PAGEFAULT_TYPES_H_ #include <linux/workqueue.h> struct xe_device; struct xe_gt; struct xe_pagefault; /** enum xe_pagefault_access_type - Xe page fault access type */ enum xe_pagefault_access_type { /** @XE_PAGEFAULT_ACCESS_TYPE_READ: Read access type */ XE_PAGEFAULT_ACCESS_TYPE_READ = 0, /** @XE_PAGEFAULT_ACCESS_TYPE_WRITE: Write access type */ XE_PAGEFAULT_ACCESS_TYPE_WRITE = 1, /** @XE_PAGEFAULT_ACCESS_TYPE_ATOMIC: Atomic access type */ XE_PAGEFAULT_ACCESS_TYPE_ATOMIC = 2, }; /** enum xe_pagefault_type - Xe page fault type */ enum xe_pagefault_type { /** @XE_PAGEFAULT_TYPE_NOT_PRESENT: Not present */ XE_PAGEFAULT_TYPE_NOT_PRESENT = 0, /** @XE_PAGEFAULT_TYPE_WRITE_ACCESS_VIOLATION: Write access violation */ XE_PAGEFAULT_TYPE_WRITE_ACCESS_VIOLATION = 1, /** @XE_PAGEFAULT_TYPE_ATOMIC_ACCESS_VIOLATION: Atomic access violation */ XE_PAGEFAULT_TYPE_ATOMIC_ACCESS_VIOLATION = 2, }; /** * enum xe_pagefault_error - Xe page fault servicing error * * Uniquely identifies the high-level point at which servicing of a page * fault failed. Encoded into the reserved low bits of * &xe_pagefault.consumer.page_addr (which is always page aligned) so the * failure reason can be threaded back up to xe_pagefault_print() without * growing the size of struct xe_pagefault. See xe_pagefault_set_error() and * xe_pagefault_error_to_str(). */ enum xe_pagefault_error { /** @XE_PAGEFAULT_ERROR_NONE: No error recorded */ XE_PAGEFAULT_ERROR_NONE = 0, /** * @XE_PAGEFAULT_ERROR_VM_NOT_FOUND: VM lookup by ASID failed, e.g. * the VM's file descriptor was already closed and the ASID has been * torn down */ XE_PAGEFAULT_ERROR_VM_NOT_FOUND, /** * @XE_PAGEFAULT_ERROR_VM_NOT_IN_FAULT_MODE: VM found by ASID lookup * but is not in fault mode */ XE_PAGEFAULT_ERROR_VM_NOT_IN_FAULT_MODE, /** @XE_PAGEFAULT_ERROR_VM_CLOSED: VM found but already closed */ XE_PAGEFAULT_ERROR_VM_CLOSED, /** @XE_PAGEFAULT_ERROR_VMA_NOT_FOUND: No VMA covers the faulted address */ XE_PAGEFAULT_ERROR_VMA_NOT_FOUND, /** * @XE_PAGEFAULT_ERROR_READ_ONLY_VIOLATION: Write/atomic fault on a * read-only VMA */ XE_PAGEFAULT_ERROR_READ_ONLY_VIOLATION, /** * @XE_PAGEFAULT_ERROR_VMA_NEEDS_VRAM_CHECK: Failed determining if VMA * requires VRAM for an atomic access */ XE_PAGEFAULT_ERROR_VMA_NEEDS_VRAM_CHECK, /** * @XE_PAGEFAULT_ERROR_VMA_ATOMIC_USERPTR: Atomic access requires VRAM * but VMA is a userptr, which is unsupported */ XE_PAGEFAULT_ERROR_VMA_ATOMIC_USERPTR, /** @XE_PAGEFAULT_ERROR_VMA_USERPTR_PIN: Userptr page pin/repin failed */ XE_PAGEFAULT_ERROR_VMA_USERPTR_PIN, /** * @XE_PAGEFAULT_ERROR_VMA_VALIDATE: Failed to lock/validate VMA's BO * or migrate it to VRAM */ XE_PAGEFAULT_ERROR_VMA_VALIDATE, /** @XE_PAGEFAULT_ERROR_VMA_REBIND: Failed to rebind VMA into page tables */ XE_PAGEFAULT_ERROR_VMA_REBIND, /** * @XE_PAGEFAULT_ERROR_SVM_GARBAGE_COLLECTOR: Failed processing * pending SVM garbage collection (unmaps) prior to servicing the * fault */ XE_PAGEFAULT_ERROR_SVM_GARBAGE_COLLECTOR, /** * @XE_PAGEFAULT_ERROR_SVM_RANGE_NOT_FOUND: Failed to find or insert * an SVM range covering the faulted address */ XE_PAGEFAULT_ERROR_SVM_RANGE_NOT_FOUND, /** @XE_PAGEFAULT_ERROR_SVM_REBIND: Failed to rebind an SVM range into page tables */ XE_PAGEFAULT_ERROR_SVM_REBIND, /** * @XE_PAGEFAULT_ERROR_SVM_NEEDS_VRAM_CHECK: Failed determining if SVM * VMA requires VRAM for an atomic access */ XE_PAGEFAULT_ERROR_SVM_NEEDS_VRAM_CHECK, /** * @XE_PAGEFAULT_ERROR_SVM_VMA_NOT_FOUND: SVM VMA re-lookup after a * range split failed to find a covering VMA */ XE_PAGEFAULT_ERROR_SVM_VMA_NOT_FOUND, /** * @XE_PAGEFAULT_ERROR_SVM_SERVICE_FAILED: SVM range population, * migration, or bind failed */ XE_PAGEFAULT_ERROR_SVM_SERVICE_FAILED, }; /** struct xe_pagefault_ops - Xe pagefault ops (producer) */ struct xe_pagefault_ops { /** * @ack_fault_begin: Ack fault begin * @private: producer private data * * Page fault producer begins acknowledgment from the consumer. */ void (*ack_fault_begin)(void *private); /** * @ack_fault: Ack fault * @pf: Page fault * @err: Error state of fault * * Page fault producer receives acknowledgment from the consumer and * sends the result to the HW/FW interface. */ void (*ack_fault)(struct xe_pagefault *pf, int err); /** * @ack_fault_end: Ack fault end * @private: producer private data * * Page fault producer ends acknowledgment from the consumer. */ void (*ack_fault_end)(void *private); /** * @print: Print pagefault info * @pf: Page fault * @err_str: Consumer servicing error string * * Print pagefault identification fields decoded from producer, * along with consumer-specific @err_str prints. * Must be implemented by all producers. */ void (*print)(struct xe_pagefault *pf, const char *err_str); }; /** * struct xe_pagefault - Xe page fault * * Generic page fault structure for communication between producer and consumer. * Carefully sized to be 64 bytes. Upon a device page fault, the producer * populates this structure, and the consumer copies it into the page-fault * queue for deferred handling. */ struct xe_pagefault { /** * @gt: GT of fault */ struct xe_gt *gt; /** * @consumer: State for the software handling the fault. Populated by * the producer and may be modified by the consumer to communicate * information back to the producer upon fault acknowledgment. After * fault acknowledgment, the producer should only access consumer fields * via well defined helpers. */ struct { /** * @consumer.page_addr: address of page fault, populated by * consumer after fault completion */ u64 page_addr; union { struct { /** * @consumer.alloc_state: page fault allocation * state */ u8 alloc_state; /** * @consumer.access_type: access type, u8 rather * than enum to keep size compact */ u8 access_type; #define XE_PAGEFAULT_ACCESS_TYPE_MASK GENMASK(1, 0) #define XE_PAGEFAULT_ACCESS_PREFETCH BIT(7) /** * @consumer.fault_type_level: fault type and * level, u8 rather than enum to keep size * compact */ u8 fault_type_level; #define XE_PAGEFAULT_TYPE_LEVEL_NACK 0xff /* Producer indicates nack fault */ #define XE_PAGEFAULT_LEVEL_MASK GENMASK(2, 0) #define XE_PAGEFAULT_TYPE_MASK GENMASK(6, 3) #define XE_PAGEFAULT_REQUEUE_MASK BIT(7) /** @consumer.engine_class_instance: engine class and instance */ u8 engine_class_instance; #define XE_PAGEFAULT_ENGINE_CLASS_MASK GENMASK(3, 0) #define XE_PAGEFAULT_ENGINE_INSTANCE_MASK GENMASK(7, 4) /** * @consumer.id: address space ID and SRCID, folded into one * to keep size compact */ u32 id; #define XE_PAGEFAULT_ASID_MASK GENMASK(23, 0) #define XE_PAGEFAULT_SRCID_MASK GENMASK(31, 24) }; /** * @consumer.end_addr: end address of page fault, * populated by consumer after fault completion */ u64 end_addr; }; /** * @consumer.next: next pagefault chained to this fault, * protected by pf_queue lock */ struct xe_pagefault *next; } consumer; /** * @producer: State for the producer (i.e., HW/FW interface). Populated * by the producer and should not be modified—or even inspected—by the * consumer, except for calling operations. */ struct { /** @producer.private: private pointer */ void *private; /** @producer.ops: operations */ const struct xe_pagefault_ops *ops; #define XE_PAGEFAULT_PRODUCER_MSG_LEN_DW 4 /** * @producer.msg: page fault message, used by producer in fault * acknowledgment to formulate response to HW/FW interface. * Included in the page-fault message because the producer * typically receives the fault in a context where memory cannot * be allocated (e.g., atomic context or the reclaim path). */ u32 msg[XE_PAGEFAULT_PRODUCER_MSG_LEN_DW]; } producer; }; /** * struct xe_pagefault_queue - Xe pagefault queue (consumer) * * Used to capture all device page faults for deferred processing. Size this * queue to absorb the device’s worst-case number of outstanding faults. */ struct xe_pagefault_queue { /** @xe: Back-pointer to the Xe device */ struct xe_device *xe; /** * @data: Data in queue containing struct xe_pagefault, protected by * @lock */ void *data; /** @size: Size of queue in bytes */ u32 size; /** @head: Head pointer in bytes, moved by producer, protected by @lock */ u32 head; /** @tail: Tail pointer in bytes, moved by consumer, protected by @lock */ u32 tail; /** @lock: protects page fault queue, workers caches */ spinlock_t lock; }; /** * struct xe_pagefault_work - Xe page fault work item (consumer) * * Represents a worker that pops a &struct xe_pagefault from the page fault * queue and processes it. */ struct xe_pagefault_work { /** @xe: Back-pointer to the Xe device */ struct xe_device *xe; /** @id: Identifier for this work item */ int id; /** * @cache: Page fault cache for the currently processed fault * * Protected by the page fault queue lock. */ struct { /** @cache.start: Start address of the current page fault */ u64 start; /** @cache.end: End address of the current page fault */ u64 end; /** @cache.asid: Address space ID of the current page fault */ u32 asid; /** @cache.pf: Pointer to the current page fault */ struct xe_pagefault *pf; } cache; /** @work: Work item used to process the page fault */ struct work_struct work; }; #endif