// SPDX-License-Identifier: GPL-2.0-or-later
/*
* CXL Flash Device Driver
*
* Written by: Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation
* Uma Krishnan <ukrishn@linux.vnet.ibm.com>, IBM Corporation
*
* Copyright (C) 2018 IBM Corporation
*/
#include <linux/file.h>
#include <linux/idr.h>
#include <linux/module.h>
#include <linux/mount.h>
#include <linux/pseudo_fs.h>
#include <linux/poll.h>
#include <linux/sched/signal.h>
#include <linux/interrupt.h>
#include <linux/irqdomain.h>
#include <asm/xive.h>
#include <misc/ocxl.h>
#include <uapi/misc/cxl.h>
#include "backend.h"
#include "ocxl_hw.h"
/*
* Pseudo-filesystem to allocate inodes.
*/
#define OCXLFLASH_FS_MAGIC 0x1697698f
static int ocxlflash_fs_cnt;
static struct vfsmount *ocxlflash_vfs_mount;
static int ocxlflash_fs_init_fs_context(struct fs_context *fc)
{
return init_pseudo(fc, OCXLFLASH_FS_MAGIC) ? 0 : -ENOMEM;
}
static struct file_system_type ocxlflash_fs_type = {
.name = "ocxlflash",
.owner = THIS_MODULE,
.init_fs_context = ocxlflash_fs_init_fs_context,
.kill_sb = kill_anon_super,
};
/*
* ocxlflash_release_mapping() - release the memory mapping
* @ctx: Context whose mapping is to be released.
*/
static void ocxlflash_release_mapping(struct ocxlflash_context *ctx)
{
if (ctx->mapping)
simple_release_fs(&ocxlflash_vfs_mount, &ocxlflash_fs_cnt);
ctx->mapping = NULL;
}
/*
* ocxlflash_getfile() - allocate pseudo filesystem, inode, and the file
* @dev: Generic device of the host.
* @name: Name of the pseudo filesystem.
* @fops: File operations.
* @priv: Private data.
* @flags: Flags for the file.
*
* Return: pointer to the file on success, ERR_PTR on failure
*/
static struct file *ocxlflash_getfile(struct device *dev, const char *name,
const struct file_operations *fops,
void *priv, int flags)
{
struct file *file;
struct inode *inode;
int rc;
if (fops->owner && !try_module_get(fops->owner)) {
dev_err(dev, "%s: Owner does not exist\n", __func__);
rc = -ENOENT;
goto err1;
}
rc = simple_pin_fs(&ocxlflash_fs_type, &ocxlflash_vfs_mount,
&ocxlflash_fs_cnt);
if (unlikely(rc < 0)) {
dev_err(dev, "%s: Cannot mount ocxlflash pseudofs rc=%d\n",
__func__, rc);
goto err2;
}
inode = alloc_anon_inode(ocxlflash_vfs_mount->mnt_sb);
if (IS_ERR(inode)) {
rc = PTR_ERR(inode);
dev_err(dev, "%s: alloc_anon_inode failed rc=%d\n",
__func__, rc);
goto err3;
}
file = alloc_file_pseudo(inode, ocxlflash_vfs_mount, name,
flags & (O_ACCMODE | O_NONBLOCK), fops);
if (IS_ERR(file)) {
rc = PTR_ERR(file);
dev_err(dev, "%s: alloc_file failed rc=%d\n",
__func__, rc);
goto err4;
}
file->private_data = priv;
out:
return file;
err4:
iput(inode);
err3:
simple_release_fs(&ocxlflash_vfs_mount, &ocxlflash_fs_cnt);
err2:
module_put(fops->owner);
err1:
file = ERR_PTR(rc);
goto out;
}
/**
* ocxlflash_psa_map() - map the process specific MMIO space
* @ctx_cookie: Adapter context for which the mapping needs to be done.
*
* Return: MMIO pointer of the mapped region
*/
static void __iomem *ocxlflash_psa_map(void *ctx_cookie)
{
struct ocxlflash_context *ctx = ctx_cookie;
struct device *dev = ctx->hw_afu->dev;
mutex_lock(&ctx->state_mutex);
if (ctx->state != STARTED) {
dev_err(dev, "%s: Context not started, state=%d\n", __func__,
ctx->state);
mutex_unlock(&ctx->state_mutex);
return NULL;
}
mutex_unlock(&ctx->state_mutex);
return ioremap(ctx->psn_phys, ctx->psn_size);
}
/**
* ocxlflash_psa_unmap() - unmap the process specific MMIO space
* @addr: MMIO pointer to unmap.
*/
static void ocxlflash_psa_unmap(void __iomem *addr)
{
iounmap(addr);
}
/**
* ocxlflash_process_element() - get process element of the adapter context
* @ctx_cookie: Adapter context associated with the process element.
*
* Return: process element of the adapter context
*/
static int ocxlflash_process_element(void *ctx_cookie)
{
struct ocxlflash_context *ctx = ctx_cookie;
return ctx->pe;
}
/**
* afu_map_irq() - map the interrupt of the adapter context
* @flags: Flags.
* @ctx: Adapter context.
* @num: Per-context AFU interrupt number.
* @handler: Interrupt handler to register.
* @cookie: Interrupt handler private data.
* @name: Name of the interrupt.
*
* Return: 0 on success, -errno on failure
*/
static int afu_map_irq(u64 flags, struct ocxlflash_context *ctx, int num,
irq_handler_t handler, void *cookie, char *name)
{
struct ocxl_hw_afu *afu = ctx->hw_afu;
struct device *dev = afu->dev;
struct ocxlflash_irqs *irq;
struct xive_irq_data *xd;
u32 virq;
int rc = 0;
if (num < 0 || num >= ctx->num_irqs) {
dev_err(dev, "%s: Interrupt %d not allocated\n", __func__, num);
rc = -ENOENT;
goto out;
}
irq = &ctx->irqs[num];
virq = irq_create_mapping(NULL, irq->hwirq);
if (unlikely(!virq)) {
dev_err(dev, "%s: irq_create_mapping failed\n", __func__);
rc = -ENOMEM;
goto out;
}
rc = request_irq(virq, handler, 0, name, cookie);
if (unlikely(rc)) {
dev_err(dev, "%s: request_irq failed rc=%d\n", __func__, rc);
goto err1;
}
xd = irq_get_handler_data(virq);
if (unlikely(!xd)) {
dev_err(dev, "%s: Can't get interrupt data\n", __func__);
rc = -ENXIO;
goto err2;
}
irq->virq = virq;
irq->vtrig = xd->trig_mmio;
out:
return rc;
err2:
free_irq(virq, cookie);
err1:
irq_dispose_mapping(virq);
goto out;
}
/**
* ocxlflash_map_afu_irq() - map the interrupt of the adapter context
* @ctx_cookie: Adapter context.
* @num: Per-context AFU interrupt number.
* @handler: Interrupt handler to register.
* @cookie: Interrupt handler private data.
* @name: Name of the interrupt.
*
* Return: 0 on success, -errno on failure
*/
static int ocxlflash_map_afu_irq(void *ctx_cookie, int num,
irq_handler_t handler, void *cookie,
char *name)
{
return afu_map_irq(0, ctx_cookie, num, handler, cookie, name);
}
/**
* afu_unmap_irq() - unmap the interrupt
* @flags: Flags.
* @ctx: Adapter context.
* @num: Per-context AFU interrupt number.
* @cookie: Interrupt handler private data.
*/
static void afu_unmap_irq(u64 flags, struct ocxlflash_context *ctx, int num,
void *cookie)
{
struct ocxl_hw_afu *afu = ctx->hw_afu;
struct device *dev = afu->dev;
struct ocxlflash_irqs *irq;
if (num < 0 || num >= ctx->num_irqs) {
dev_err(dev, "%s: Interrupt %d not allocated\n", __func__, num);
return;
}
irq = &ctx->irqs[num];
if (irq_find_mapping(NULL, irq->hwirq)) {
free_irq(irq->virq, cookie);
irq_dispose_mapping(irq->virq);
}
memset(irq, 0, sizeof(*irq));
}
/**
* ocxlflash_unmap_afu_irq() - unmap the interrupt
* @ctx_cookie: Adapter context.
* @num: Per-context AFU interrupt number.
* @cookie: Interrupt handler private data.
*/
static void ocxlflash_unmap_afu_irq(void *ctx_cookie, int num, void *cookie)
{
return afu_unmap_irq(0, ctx_cookie, num, cookie);
}
/**
* ocxlflash_get_irq_objhndl() - get the object handle for an interrupt
* @ctx_cookie: Context associated with the interrupt.
* @irq: Interrupt number.
*
* Return: effective address of the mapped region
*/
static u64 ocxlflash_get_irq_objhndl(void *ctx_cookie, int irq)
{
struct ocxlflash_context *ctx = ctx_cookie;
if (irq < 0 || irq >= ctx->num_irqs)
return 0;
return (__force u64)ctx->irqs[irq].vtrig;
}
/**
* ocxlflash_xsl_fault() - callback when translation error is triggered
* @data: Private data provided at callback registration, the context.
* @addr: Address that triggered the error.
* @dsisr: Value of dsisr register.
*/
static void ocxlflash_xsl_fault(void *data, u64 addr, u64 dsisr)
{
struct ocxlflash_context *ctx = data;
spin_lock(&ctx->slock);
ctx->fault_addr = addr;
ctx->fault_dsisr = dsisr;
ctx->pending_fault = true;
spin_unlock(&ctx->slock);
wake_up_all(&ctx->wq);
}
/**
* start_context() - local routine to start a context
* @ctx: Adapter context to be started.
*
* Assign the context specific MMIO space, add and enable the PE.
*
* Return: 0 on success, -errno on failure
*/
static int start_context(struct ocxlflash_context *ctx)
{
struct ocxl_hw_afu *afu = ctx->hw_afu;
struct ocxl_afu_config *acfg = &afu->acfg;
void *link_token = afu->link_token;
struct pci_dev *pdev = afu->pdev;
struct device *dev = afu->dev;
bool master = ctx->master;
struct mm_struct *mm;
int rc = 0;
u32 pid;
mutex_lock(&ctx->state_mutex);
if (ctx->state != OPENED) {
dev_err(dev, "%s: Context state invalid, state=%d\n",
__func__, ctx->state);
rc = -EINVAL;
goto out;
}
if (master) {
ctx->psn_size = acfg->global_mmio_size;
ctx->psn_phys = afu->gmmio_phys;
} else {
ctx->psn_size = acfg->pp_mmio_stride;
ctx->psn_phys = afu->ppmmio_phys + (ctx->pe * ctx->psn_size);
}
/* pid and mm not set for master contexts */
if (master) {
pid = 0;
mm = NULL;
} else {
pid = current->mm->context.id;
mm = current->mm;
}
rc = ocxl_link_add_pe(link_token, ctx->pe, pid, 0, 0,
pci_dev_id(pdev), mm, ocxlflash_xsl_fault,
ctx);
if (unlikely(rc)) {
dev_err(dev, "%s: ocxl_link_add_pe failed rc=%d\n",
__func__, rc);
goto out;
}
ctx->state = STARTED;
out:
mutex_unlock(&ctx->state_mutex);
return rc;
}
/**
* ocxlflash_start_context() - start a kernel context
* @ctx_cookie: Adapter context to be started.
*
* Return: 0 on success, -errno on failure
*/
static int ocxlflash_start_context(void *ctx_cookie)
{
struct ocxlflash_context *ctx = ctx_cookie;
return start_context(ctx);
}
/**
* ocxlflash_stop_context() - stop a context
* @ctx_cookie: Adapter context to be stopped.
*
* Return: 0 on success, -errno on failure
*/
static int ocxlflash_stop_context(void *ctx_cookie)
{
struct ocxlflash_context *ctx = ctx_cookie;
struct ocxl_hw_afu *afu = ctx->hw_afu;
struct ocxl_afu_config *acfg = &afu->acfg;
struct pci_dev *pdev = afu->pdev;
struct device *dev = afu->dev;
enum ocxlflash_ctx_state state;
int rc = 0;
mutex_lock(&ctx->state_mutex);
state = ctx->state;
ctx->state = CLOSED;
mutex_unlock(&ctx->state_mutex);
if (state != STARTED)
goto out;
rc = ocxl_config_terminate_pasid(pdev, acfg->dvsec_afu_control_pos,
ctx->pe);
if (unlikely(rc)) {
dev_err(dev, "%s: ocxl_config_terminate_pasid failed rc=%d\n",
__func__, rc);
/* If EBUSY, PE could be referenced in future by the AFU */
if (rc == -EBUSY)
goto out;
}
rc = ocxl_link_remove_pe(afu->link_token, ctx->pe);
if (unlikely(rc)) {
dev_err(dev, "%s: ocxl_link_remove_pe failed rc=%d\n",
__func__, rc);
goto out;
}
out:
return rc;
}
/**
* ocxlflash_afu_reset() - reset the AFU
* @ctx_cookie: Adapter context.
*/
static int ocxlflash_afu_reset(void *ctx_cookie)
{
struct ocxlflash_context *ctx = ctx_cookie;
struct device *dev = ctx->hw_afu->dev;
/* Pending implementation from OCXL transport services */
dev_err_once(dev, "%s: afu_reset() fop not supported\n", __func__);
/* Silently return success until it is implemented */
return 0;
}
/**
* ocxlflash_set_master() - sets the context as master
* @ctx_cookie: Adapter context to set as master.
*/
static void ocxlflash_set_master(void *ctx_cookie)
{
struct ocxlflash_context *ctx = ctx_cookie;
ctx->master = true;
}
/**
* ocxlflash_get_context() - obtains the context associated with the host
* @pdev: PCI device associated with the host.
* @afu_cookie: Hardware AFU associated with the host.
*
* Return: returns the pointer to host adapter context
*/
static void *ocxlflash_get_context(struct pci_dev *pdev, void *afu_cookie)
{
struct ocxl_hw_afu *afu = afu_cookie;
return afu->ocxl_ctx;
}
/**
* ocxlflash_dev_context_init() - allocate and initialize an adapter context
* @pdev: PCI device associated with the host.
* @afu_cookie: Hardware AFU associated with the host.
*
* Return: returns the adapter context on success, ERR_PTR on failure
*/
static void *ocxlflash_dev_context_init(struct pci_dev *pdev, void *afu_cookie)
{
struct ocxl_hw_afu *afu = afu_cookie;
struct device *dev = afu->dev;
struct ocxlflash_context *ctx;
int rc;
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
if (unlikely(!ctx)) {
dev_err(dev, "%s: Context allocation failed\n", __func__);
rc = -ENOMEM;
goto err1;
}
idr_preload(GFP_KERNEL);
rc = idr_alloc(&afu->idr, ctx, 0, afu->max_pasid, GFP_NOWAIT);
idr_preload_end();
if (unlikely(rc < 0)) {
dev_err(dev, "%s: idr_alloc failed rc=%d\n", __func__, rc);
goto err2;
}
spin_lock_init(&ctx->slock);
init_waitqueue_head(&ctx->wq);
mutex_init(&ctx->state_mutex);
ctx->state = OPENED;
ctx->pe = rc;
ctx->master = false;
ctx->mapping = NULL;
ctx->hw_afu = afu;
ctx->irq_bitmap = 0;
ctx->pending_irq = false;
ctx->pending_fault = false;
out:
return ctx;
err2:
kfree(ctx);
err1:
ctx = ERR_PTR(rc);
goto out;
}
/**
* ocxlflash_release_context() - releases an adapter context
* @ctx_cookie: Adapter context to be released.
*
* Return: 0 on success, -errno on failure
*/
static int ocxlflash_release_context(void *ctx_cookie)
{
struct ocxlflash_context *ctx = ctx_cookie;
struct device *dev;
int rc = 0;
if (!ctx)
goto out;
dev = ctx->hw_afu->dev;
mutex_lock(&ctx->state_mutex);
if (ctx->state >= STARTED) {
dev_err(dev, "%s: Context in use, state=%d\n", __func__,
ctx->state);
mutex_unlock(&ctx->state_mutex);
rc = -EBUSY;
goto out;
}
mutex_unlock(&ctx->state_mutex);
idr_remove(&ctx->hw_afu->idr, ctx->pe);
ocxlflash_release_mapping(ctx);
kfree(ctx);
out:
return rc;
}
/**
* ocxlflash_perst_reloads_same_image() - sets the image reload policy
* @afu_cookie: Hardware AFU associated with the host.
* @image: Whether to load the same image on PERST.
*/
static void ocxlflash_perst_reloads_same_image(void *afu_cookie, bool image)
{
struct ocxl_hw_afu *afu = afu_cookie;
afu->perst_same_image = image;
}
/**
* ocxlflash_read_adapter_vpd() - reads the adapter VPD
* @pdev: PCI device associated with the host.
* @buf: Buffer to get the VPD data.
* @count: Size of buffer (maximum bytes that can be read).
*
* Return: size of VPD on success, -errno on failure
*/
static ssize_t ocxlflash_read_adapter_vpd(struct pci_dev *pdev, void *buf,
size_t count)
{
return pci_read_vpd(pdev, 0, count, buf);
}
/**
* free_afu_irqs() - internal service to free interrupts
* @ctx: Adapter context.
*/
static void free_afu_irqs(struct ocxlflash_context *ctx)
{
struct ocxl_hw_afu *afu = ctx->hw_afu;
struct device *dev = afu->dev;
int i;
if (!ctx->irqs) {
dev_err(dev, "%s: Interrupts not allocated\n", __func__);
return;
}
for (i = ctx->num_irqs; i >= 0; i--)
ocxl_link_free_irq(afu->link_token, ctx->irqs[i].hwirq);
kfree(ctx->irqs);
ctx->irqs = NULL;
}
/**
* alloc_afu_irqs() - internal service to allocate interrupts
* @ctx: Context associated with the request.
* @num: Number of interrupts requested.
*
* Return: 0 on success, -errno on failure
*/
static int alloc_afu_irqs(struct ocxlflash_context *ctx, int num)
{
struct ocxl_hw_afu *afu = ctx->hw_afu;
struct device *dev = afu->dev;
struct ocxlflash_irqs *irqs;
int rc = 0;
int hwirq;
int i;
if (ctx->irqs) {
dev_err(dev, "%s: Interrupts already allocated\n", __func__);
rc = -EEXIST;
goto out;
}
if (num > OCXL_MAX_IRQS) {
dev_err(dev, "%s: Too many interrupts num=%d\n", __func__, num);
rc = -EINVAL;
goto out;
}
irqs = kcalloc(num, sizeof(*irqs), GFP_KERNEL);
if (unlikely(!irqs)) {
dev_err(dev, "%s: Context irqs allocation failed\n", __func__);
rc = -ENOMEM;
goto out;
}
for (i = 0; i < num; i++) {
rc = ocxl_link_irq_alloc(afu->link_token, &hwirq);
if (unlikely(rc)) {
dev_err(dev, "%s: ocxl_link_irq_alloc failed rc=%d\n",
__func__, rc);
goto err;
}
irqs[i].hwirq = hwirq;
}
ctx->irqs = irqs;
ctx->num_irqs = num;
out:
return rc;
err:
for (i = i-1; i >= 0; i--)
ocxl_link_free_irq(afu->link_token, irqs[i].hwirq);
kfree(irqs);
goto out;
}
/**
* ocxlflash_allocate_afu_irqs() - allocates the requested number of interrupts
* @ctx_cookie: Context associated with the request.
* @num: Number of interrupts requested.
*
* Return: 0 on success, -errno on failure
*/
static int ocxlflash_allocate_afu_irqs(void *ctx_cookie, int num)
{
return alloc_afu_irqs(ctx_cookie, num);
}
/**
* ocxlflash_free_afu_irqs() - frees the interrupts of an adapter context
* @ctx_cookie: Adapter context.
*/
static void ocxlflash_free_afu_irqs(void *ctx_cookie)
{
free_afu_irqs(ctx_cookie);
}
/**
* ocxlflash_unconfig_afu() - unconfigure the AFU
* @afu: AFU associated with the host.
*/
static void ocxlflash_unconfig_afu(struct ocxl_hw_afu *afu)
{
if (afu->gmmio_virt) {
iounmap(afu->gmmio_virt);
afu->gmmio_virt = NULL;
}
}
/**
* ocxlflash_destroy_afu() - destroy the AFU structure
* @afu_cookie: AFU to be freed.
*/
static void ocxlflash_destroy_afu(void *afu_cookie)
{
struct ocxl_hw_afu *afu = afu_cookie;
int pos;
if (!afu)
return;
ocxlflash_release_context(afu->ocxl_ctx);
idr_destroy(&afu->idr);
/* Disable the AFU */
pos = afu->acfg.dvsec_afu_control_pos;
ocxl_config_set_afu_state(afu->pdev, pos, 0);
ocxlflash_unconfig_afu(afu);
kfree(afu);
}
/**
* ocxlflash_config_fn() - configure the host function
* @pdev: PCI device associated with the host.
* @afu: AFU associated with the host.
*
* Return: 0 on success, -errno on failure
*/
static int ocxlflash_config_fn(struct pci_dev *pdev, struct ocxl_hw_afu *afu)
{
struct ocxl_fn_config *fcfg = &afu->fcfg;
struct device *dev = &pdev->dev;
u16 base, enabled, supported;
int rc = 0;
/* Read DVSEC config of the function */
rc = ocxl_config_read_function(pdev, fcfg);
if (unlikely(rc)) {
dev_err(dev, "%s: ocxl_config_read_function failed rc=%d\n",
__func__, rc);
goto out;
}
/* Check if function has AFUs defined, only 1 per function supported */
if (fcfg->max_afu_index >= 0) {
afu->is_present = true;
if (fcfg->max_afu_index != 0)
dev_warn(dev, "%s: Unexpected AFU index value %d\n",
__func__, fcfg->max_afu_index);
}
rc = ocxl_config_get_actag_info(pdev, &base, &enabled, &supported);
if (unlikely(rc)) {
dev_err(dev, "%s: ocxl_config_get_actag_info failed rc=%d\n",
__func__, rc);
goto out;
}
afu->fn_actag_base = base;
afu->fn_actag_enabled = enabled;
ocxl_config_set_actag(pdev, fcfg->dvsec_function_pos, base, enabled);
dev_dbg(dev, "%s: Function acTag range base=%u enabled=%u\n",
__func__, base, enabled);
rc = ocxl_link_setup(pdev, 0, &afu->link_token);
if (unlikely(rc)) {
dev_err(dev, "%s: ocxl_link_setup failed rc=%d\n",
__func__, rc);
goto out;
}
rc = ocxl_config_set_TL(pdev, fcfg->dvsec_tl_pos);
if (unlikely(rc)) {
dev_err(dev, "%s: ocxl_config_set_TL failed rc=%d\n",
__func__, rc);
goto err;
}
out:
return rc;
err:
ocxl_link_release(pdev, afu->link_token);
goto out;
}
/**
* ocxlflash_unconfig_fn() - unconfigure the host function
* @pdev: PCI device associated with the host.
* @afu: AFU associated with the host.
*/
static void ocxlflash_unconfig_fn(struct pci_dev *pdev, struct ocxl_hw_afu *afu)
{
ocxl_link_release(pdev, afu->link_token);
}
/**
* ocxlflash_map_mmio() - map the AFU MMIO space
* @afu: AFU associated with the host.
*
* Return: 0 on success, -errno on failure
*/
static int ocxlflash_map_mmio(struct ocxl_hw_afu *afu)
{
struct ocxl_afu_config *acfg = &afu->acfg;
struct pci_dev *pdev = afu->pdev;
struct device *dev = afu->dev;
phys_addr_t gmmio, ppmmio;
int rc = 0;
rc = pci_request_region(pdev, acfg->global_mmio_bar, "ocxlflash");
if (unlikely(rc)) {
dev_err(dev, "%s: pci_request_region for global failed rc=%d\n",
__func__, rc);
goto out;
}
gmmio = pci_resource_start(pdev, acfg->global_mmio_bar);
gmmio += acfg->global_mmio_offset;
rc = pci_request_region(pdev, acfg->pp_mmio_bar, "ocxlflash");
if (unlikely(rc)) {
dev_err(dev, "%s: pci_request_region for pp bar failed rc=%d\n",
__func__, rc);
goto err1;
}
ppmmio = pci_resource_start(pdev, acfg->pp_mmio_bar);
ppmmio += acfg->pp_mmio_offset;
afu->gmmio_virt = ioremap(gmmio, acfg->global_mmio_size);
if (unlikely(!afu->gmmio_virt)) {
dev_err(dev, "%s: MMIO mapping failed\n", __func__);
rc = -ENOMEM;
goto err2;
}
afu->gmmio_phys = gmmio;
afu->ppmmio_phys = ppmmio;
out:
return rc;
err2:
pci_release_region(pdev, acfg->pp_mmio_bar);
err1:
pci_release_region(pdev, acfg->global_mmio_bar);
goto out;
}
/**
* ocxlflash_config_afu() - configure the host AFU
* @pdev: PCI device associated with the host.
* @afu: AFU associated with the host.
*
* Must be called _after_ host function configuration.
*
* Return: 0 on success, -errno on failure
*/
static int ocxlflash_config_afu(struct pci_dev *pdev, struct ocxl_hw_afu *afu)
{
struct ocxl_afu_config *acfg = &afu->acfg;
struct ocxl_fn_config *fcfg = &afu->fcfg;
struct device *dev = &pdev->dev;
int count;
int base;
int pos;
int rc = 0;
/* This HW AFU function does not have any AFUs defined */
if (!afu->is_present)
goto out;
/* Read AFU config at index 0 */
rc = ocxl_config_read_afu(pdev, fcfg, acfg, 0);
if (unlikely(rc)) {
dev_err(dev, "%s: ocxl_config_read_afu failed rc=%d\n",
__func__, rc);
goto out;
}
/* Only one AFU per function is supported, so actag_base is same */
base = afu->fn_actag_base;
count = min_t(int, acfg->actag_supported, afu->fn_actag_enabled);
pos = acfg->dvsec_afu_control_pos;
ocxl_config_set_afu_actag(pdev, pos, base, count);
dev_dbg(dev, "%s: acTag base=%d enabled=%d\n", __func__, base, count);
afu->afu_actag_base = base;
afu->afu_actag_enabled = count;
afu->max_pasid = 1 << acfg->pasid_supported_log;
ocxl_config_set_afu_pasid(pdev, pos, 0, acfg->pasid_supported_log);
rc = ocxlflash_map_mmio(afu);
if (unlikely(rc)) {
dev_err(dev, "%s: ocxlflash_map_mmio failed rc=%d\n",
__func__, rc);
goto out;
}
/* Enable the AFU */
ocxl_config_set_afu_state(pdev, acfg->dvsec_afu_control_pos, 1);
out:
return rc;
}
/**
* ocxlflash_create_afu() - create the AFU for OCXL
* @pdev: PCI device associated with the host.
*
* Return: AFU on success, NULL on failure
*/
static void *ocxlflash_create_afu(struct pci_dev *pdev)
{
struct device *dev = &pdev->dev;
struct ocxlflash_context *ctx;
struct ocxl_hw_afu *afu;
int rc;
afu = kzalloc(sizeof(*afu), GFP_KERNEL);
if (unlikely(!afu)) {
dev_err(dev, "%s: HW AFU allocation failed\n", __func__);
goto out;
}
afu->pdev = pdev;
afu->dev = dev;
idr_init(&afu->idr);
rc = ocxlflash_config_fn(pdev, afu);
if (unlikely(rc)) {
dev_err(dev, "%s: Function configuration failed rc=%d\n",
__func__, rc);
goto err1;
}
rc = ocxlflash_config_afu(pdev, afu);
if (unlikely(rc)) {
dev_err(dev, "%s: AFU configuration failed rc=%d\n",
__func__, rc);
goto err2;
}
ctx = ocxlflash_dev_context_init(pdev, afu);
if (IS_ERR(ctx)) {
rc = PTR_ERR(ctx);
dev_err(dev, "%s: ocxlflash_dev_context_init failed rc=%d\n",
__func__, rc);
goto err3;
}
afu->ocxl_ctx = ctx;
out:
return afu;
err3:
ocxlflash_unconfig_afu(afu);
err2:
ocxlflash_unconfig_fn(pdev, afu);
err1:
idr_destroy(&afu->idr);
kfree(afu);
afu = NULL;
goto out;
}
/**
* ctx_event_pending() - check for any event pending on the context
* @ctx: Context to be checked.
*
* Return: true if there is an event pending, false if none pending
*/
static inline bool ctx_event_pending(struct ocxlflash_context *ctx)
{
if (ctx->pending_irq || ctx->pending_fault)
return true;
return false;
}
/**
* afu_poll() - poll the AFU for events on the context
* @file: File associated with the adapter context.
* @poll: Poll structure from the user.
*
* Return: poll mask
*/
static unsigned int afu_poll(struct file *file, struct poll_table_struct *poll)
{
struct ocxlflash_context *ctx = file->private_data;
struct device *dev = ctx->hw_afu->dev;
ulong lock_flags;
int mask = 0;
poll_wait(file, &ctx->wq, poll);
spin_lock_irqsave(&ctx->slock, lock_flags);
if (ctx_event_pending(ctx))
mask |= POLLIN | POLLRDNORM;
else if (ctx->state == CLOSED)
mask |= POLLERR;
spin_unlock_irqrestore(&ctx->slock, lock_flags);
dev_dbg(dev, "%s: Poll wait completed for pe %i mask %i\n",
__func__, ctx->pe, mask);
return mask;
}
/**
* afu_read() - perform a read on the context for any event
* @file: File associated with the adapter context.
* @buf: Buffer to receive the data.
* @count: Size of buffer (maximum bytes that can be read).
* @off: Offset.
*
* Return: size of the data read on success, -errno on failure
*/
static ssize_t afu_read(struct file *file, char __user *buf, size_t count,
loff_t *off)
{
struct ocxlflash_context *ctx = file->private_data;
struct device *dev = ctx->hw_afu->dev;
struct cxl_event event;
ulong lock_flags;
ssize_t esize;
ssize_t rc;
int bit;
DEFINE_WAIT(event_wait);
if (*off != 0) {
dev_err(dev, "%s: Non-zero offset not supported, off=%lld\n",
__func__, *off);
rc = -EINVAL;
goto out;
}
spin_lock_irqsave(&ctx->slock, lock_flags);
for (;;) {
prepare_to_wait(&ctx->wq, &event_wait, TASK_INTERRUPTIBLE);
if (ctx_event_pending(ctx) || (ctx->state == CLOSED))
break;
if (file->f_flags & O_NONBLOCK) {
dev_err(dev, "%s: File cannot be blocked on I/O\n",
__func__);
rc = -EAGAIN;
goto err;
}
if (signal_pending(current)) {
dev_err(dev, "%s: Signal pending on the process\n",
__func__);
rc = -ERESTARTSYS;
goto err;
}
spin_unlock_irqrestore(&ctx->slock, lock_flags);
schedule();
spin_lock_irqsave(&ctx->slock, lock_flags);
}
finish_wait(&ctx->wq, &event_wait);
memset(&event, 0, sizeof(event));
event.header.process_element = ctx->pe;
event.header.size = sizeof(struct cxl_event_header);
if (ctx->pending_irq) {
esize = sizeof(struct cxl_event_afu_interrupt);
event.header.size += esize;
event.header.type = CXL_EVENT_AFU_INTERRUPT;
bit = find_first_bit(&ctx->irq_bitmap, ctx->num_irqs);
clear_bit(bit, &ctx->irq_bitmap);
event.irq.irq = bit + 1;
if (bitmap_empty(&ctx->irq_bitmap, ctx->num_irqs))
ctx->pending_irq = false;
} else if (ctx->pending_fault) {
event.header.size += sizeof(struct cxl_event_data_storage);
event.header.type = CXL_EVENT_DATA_STORAGE;
event.fault.addr = ctx->fault_addr;
event.fault.dsisr = ctx->fault_dsisr;
ctx->pending_fault = false;
}
spin_unlock_irqrestore(&ctx->slock, lock_flags);
if (copy_to_user(buf, &event, event.header.size)) {
dev_err(dev, "%s: copy_to_user failed\n", __func__);
rc = -EFAULT;
goto out;
}
rc = event.header.size;
out:
return rc;
err:
finish_wait(&ctx->wq, &event_wait);
spin_unlock_irqrestore(&ctx->slock, lock_flags);
goto out;
}
/**
* afu_release() - release and free the context
* @inode: File inode pointer.
* @file: File associated with the context.
*
* Return: 0 on success, -errno on failure
*/
static int afu_release(struct inode *inode, struct file *file)
{
struct ocxlflash_context *ctx = file->private_data;
int i;
/* Unmap and free the interrupts associated with the context */
for (i = ctx->num_irqs; i >= 0; i--)
afu_unmap_irq(0, ctx, i, ctx);
free_afu_irqs(ctx);
return ocxlflash_release_context(ctx);
}
/**
* ocxlflash_mmap_fault() - mmap fault handler
* @vmf: VM fault associated with current fault.
*
* Return: 0 on success, -errno on failure
*/
static vm_fault_t ocxlflash_mmap_fault(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
struct ocxlflash_context *ctx = vma->vm_file->private_data;
struct device *dev = ctx->hw_afu->dev;
u64 mmio_area, offset;
offset = vmf->pgoff << PAGE_SHIFT;
if (offset >= ctx->psn_size)
return VM_FAULT_SIGBUS;
mutex_lock(&ctx->state_mutex);
if (ctx->state != STARTED) {
dev_err(dev, "%s: Context not started, state=%d\n",
__func__, ctx->state);
mutex_unlock(&ctx->state_mutex);
return VM_FAULT_SIGBUS;
}
mutex_unlock(&ctx->state_mutex);
mmio_area = ctx->psn_phys;
mmio_area += offset;
return vmf_insert_pfn(vma, vmf->address, mmio_area >> PAGE_SHIFT);
}
static const struct vm_operations_struct ocxlflash_vmops = {
.fault = ocxlflash_mmap_fault,
};
/**
* afu_mmap() - map the fault handler operations
* @file: File associated with the context.
* @vma: VM area associated with mapping.
*
* Return: 0 on success, -errno on failure
*/
static int afu_mmap(struct file *file, struct vm_area_struct *vma)
{
struct ocxlflash_context *ctx = file->private_data;
if ((vma_pages(vma) + vma->vm_pgoff) >
(ctx->psn_size >> PAGE_SHIFT))
return -EINVAL;
vm_flags_set(vma, VM_IO | VM_PFNMAP);
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
vma->vm_ops = &ocxlflash_vmops;
return 0;
}
static const struct file_operations ocxl_afu_fops = {
.owner = THIS_MODULE,
.poll = afu_poll,
.read = afu_read,
.release = afu_release,
.mmap = afu_mmap,
};
#define PATCH_FOPS(NAME) \
do { if (!fops->NAME) fops->NAME = ocxl_afu_fops.NAME; } while (0)
/**
* ocxlflash_get_fd() - get file descriptor for an adapter context
* @ctx_cookie: Adapter context.
* @fops: File operations to be associated.
* @fd: File descriptor to be returned back.
*
* Return: pointer to the file on success, ERR_PTR on failure
*/
static struct file *ocxlflash_get_fd(void *ctx_cookie,
struct file_operations *fops, int *fd)
{
struct ocxlflash_context *ctx = ctx_cookie;
struct device *dev = ctx->hw_afu->dev;
struct file *file;
int flags, fdtmp;
int rc = 0;
char *name = NULL;
/* Only allow one fd per context */
if (ctx->mapping) {
dev_err(dev, "%s: Context is already mapped to an fd\n",
__func__);
rc = -EEXIST;
goto err1;
}
flags = O_RDWR | O_CLOEXEC;
/* This code is similar to anon_inode_getfd() */
rc = get_unused_fd_flags(flags);
if (unlikely(rc < 0)) {
dev_err(dev, "%s: get_unused_fd_flags failed rc=%d\n",
__func__, rc);
goto err1;
}
fdtmp = rc;
/* Patch the file ops that are not defined */
if (fops) {
PATCH_FOPS(poll);
PATCH_FOPS(read);
PATCH_FOPS(release);
PATCH_FOPS(mmap);
} else /* Use default ops */
fops = (struct file_operations *)&ocxl_afu_fops;
name = kasprintf(GFP_KERNEL, "ocxlflash:%d", ctx->pe);
file = ocxlflash_getfile(dev, name, fops, ctx, flags);
kfree(name);
if (IS_ERR(file)) {
rc = PTR_ERR(file);
dev_err(dev, "%s: ocxlflash_getfile failed rc=%d\n",
__func__, rc);
goto err2;
}
ctx->mapping = file->f_mapping;
*fd = fdtmp;
out:
return file;
err2:
put_unused_fd(fdtmp);
err1:
file = ERR_PTR(rc);
goto out;
}
/**
* ocxlflash_fops_get_context() - get the context associated with the file
* @file: File associated with the adapter context.
*
* Return: pointer to the context
*/
static void *ocxlflash_fops_get_context(struct file *file)
{
return file->private_data;
}
/**
* ocxlflash_afu_irq() - interrupt handler for user contexts
* @irq: Interrupt number.
* @data: Private data provided at interrupt registration, the context.
*
* Return: Always return IRQ_HANDLED.
*/
static irqreturn_t ocxlflash_afu_irq(int irq, void *data)
{
struct ocxlflash_context *ctx = data;
struct device *dev = ctx->hw_afu->dev;
int i;
dev_dbg(dev, "%s: Interrupt raised for pe %i virq %i\n",
__func__, ctx->pe, irq);
for (i = 0; i < ctx->num_irqs; i++) {
if (ctx->irqs[i].virq == irq)
break;
}
if (unlikely(i >= ctx->num_irqs)) {
dev_err(dev, "%s: Received AFU IRQ out of range\n", __func__);
goto out;
}
spin_lock(&ctx->slock);
set_bit(i - 1, &ctx->irq_bitmap);
ctx->pending_irq = true;
spin_unlock(&ctx->slock);
wake_up_all(&ctx->wq);
out:
return IRQ_HANDLED;
}
/**
* ocxlflash_start_work() - start a user context
* @ctx_cookie: Context to be started.
* @num_irqs: Number of interrupts requested.
*
* Return: 0 on success, -errno on failure
*/
static int ocxlflash_start_work(void *ctx_cookie, u64 num_irqs)
{
struct ocxlflash_context *ctx = ctx_cookie;
struct ocxl_hw_afu *afu = ctx->hw_afu;
struct device *dev = afu->dev;
char *name;
int rc = 0;
int i;
rc = alloc_afu_irqs(ctx, num_irqs);
if (unlikely(rc < 0)) {
dev_err(dev, "%s: alloc_afu_irqs failed rc=%d\n", __func__, rc);
goto out;
}
for (i = 0; i < num_irqs; i++) {
name = kasprintf(GFP_KERNEL, "ocxlflash-%s-pe%i-%i",
dev_name(dev), ctx->pe, i);
rc = afu_map_irq(0, ctx, i, ocxlflash_afu_irq, ctx, name);
kfree(name);
if (unlikely(rc < 0)) {
dev_err(dev, "%s: afu_map_irq failed rc=%d\n",
__func__, rc);
goto err;
}
}
rc = start_context(ctx);
if (unlikely(rc)) {
dev_err(dev, "%s: start_context failed rc=%d\n", __func__, rc);
goto err;
}
out:
return rc;
err:
for (i = i-1; i >= 0; i--)
afu_unmap_irq(0, ctx, i, ctx);
free_afu_irqs(ctx);
goto out;
};
/**
* ocxlflash_fd_mmap() - mmap handler for adapter file descriptor
* @file: File installed with adapter file descriptor.
* @vma: VM area associated with mapping.
*
* Return: 0 on success, -errno on failure
*/
static int ocxlflash_fd_mmap(struct file *file, struct vm_area_struct *vma)
{
return afu_mmap(file, vma);
}
/**
* ocxlflash_fd_release() - release the context associated with the file
* @inode: File inode pointer.
* @file: File associated with the adapter context.
*
* Return: 0 on success, -errno on failure
*/
static int ocxlflash_fd_release(struct inode *inode, struct file *file)
{
return afu_release(inode, file);
}
/* Backend ops to ocxlflash services */
const struct cxlflash_backend_ops cxlflash_ocxl_ops = {
.module = THIS_MODULE,
.psa_map = ocxlflash_psa_map,
.psa_unmap = ocxlflash_psa_unmap,
.process_element = ocxlflash_process_element,
.map_afu_irq = ocxlflash_map_afu_irq,
.unmap_afu_irq = ocxlflash_unmap_afu_irq,
.get_irq_objhndl = ocxlflash_get_irq_objhndl,
.start_context = ocxlflash_start_context,
.stop_context = ocxlflash_stop_context,
.afu_reset = ocxlflash_afu_reset,
.set_master = ocxlflash_set_master,
.get_context = ocxlflash_get_context,
.dev_context_init = ocxlflash_dev_context_init,
.release_context = ocxlflash_release_context,
.perst_reloads_same_image = ocxlflash_perst_reloads_same_image,
.read_adapter_vpd = ocxlflash_read_adapter_vpd,
.allocate_afu_irqs = ocxlflash_allocate_afu_irqs,
.free_afu_irqs = ocxlflash_free_afu_irqs,
.create_afu = ocxlflash_create_afu,
.destroy_afu = ocxlflash_destroy_afu,
.get_fd = ocxlflash_get_fd,
.fops_get_context = ocxlflash_fops_get_context,
.start_work = ocxlflash_start_work,
.fd_mmap = ocxlflash_fd_mmap,
.fd_release = ocxlflash_fd_release,
};