#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include "vnic_dev.h"
#include "vnic_wq.h"
#include "vnic_rq.h"
#include "vnic_cq.h"
#include "vnic_intr.h"
#include "vnic_resource.h"
#include "vnic_devcmd.h"
#include "enic.h"
#include "enic_admin.h"
#include "cq_desc.h"
#include "cq_enet_desc.h"
#include "wq_enet_desc.h"
#include "rq_enet_desc.h"
#include "enic_mbox.h"
#define ENIC_ADMIN_RQ_REFILL_RETRY_MS 100
static void enic_admin_wq_buf_clean(struct vnic_wq *wq,
struct vnic_wq_buf *buf)
{
struct enic *enic = vnic_dev_priv(wq->vdev);
if (buf->os_buf) {
dma_unmap_single(&enic->pdev->dev, buf->dma_addr,
buf->len, DMA_TO_DEVICE);
kfree(buf->os_buf);
buf->os_buf = NULL;
}
}
static void enic_admin_rq_buf_clean(struct vnic_rq *rq,
struct vnic_rq_buf *buf)
{
struct enic *enic = vnic_dev_priv(rq->vdev);
if (!buf->os_buf)
return;
dma_unmap_single(&enic->pdev->dev, buf->dma_addr, buf->len,
DMA_FROM_DEVICE);
kfree(buf->os_buf);
buf->os_buf = NULL;
}
static int enic_admin_rq_post_one(struct enic *enic, gfp_t gfp)
{
struct vnic_rq *rq = &enic->admin_rq;
struct rq_enet_desc *desc;
dma_addr_t dma_addr;
void *buf;
buf = kzalloc(ENIC_ADMIN_BUF_SIZE, gfp);
if (!buf)
return -ENOMEM;
dma_addr = dma_map_single(&enic->pdev->dev, buf, ENIC_ADMIN_BUF_SIZE,
DMA_FROM_DEVICE);
if (dma_mapping_error(&enic->pdev->dev, dma_addr)) {
kfree(buf);
return -ENOMEM;
}
desc = vnic_rq_next_desc(rq);
rq_enet_desc_enc(desc, (u64)dma_addr | VNIC_PADDR_TARGET,
RQ_ENET_TYPE_ONLY_SOP, ENIC_ADMIN_BUF_SIZE);
vnic_rq_post(rq, buf, 0, dma_addr, ENIC_ADMIN_BUF_SIZE, 0);
return 0;
}
static int enic_admin_rq_fill(struct enic *enic, gfp_t gfp)
{
struct vnic_rq *rq = &enic->admin_rq;
int err;
while (vnic_rq_desc_avail(rq) > 0) {
err = enic_admin_rq_post_one(enic, gfp);
if (err)
return err;
}
return 0;
}
static void enic_admin_rq_drain(struct enic *enic)
{
vnic_rq_clean(&enic->admin_rq, enic_admin_rq_buf_clean);
}
static unsigned int enic_admin_cq_color(void *cq_desc, unsigned int desc_size)
{
u8 type_color = *((u8 *)cq_desc + desc_size - 1);
return (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK;
}
unsigned int enic_admin_wq_cq_service(struct enic *enic)
{
struct vnic_cq *cq = &enic->admin_cq[0];
unsigned int work = 0;
void *desc;
desc = vnic_cq_to_clean(cq);
while (enic_admin_cq_color(desc, cq->ring.desc_size) !=
cq->last_color) {
vnic_cq_inc_to_clean(cq);
work++;
desc = vnic_cq_to_clean(cq);
}
return work;
}
#define ENIC_ADMIN_MSG_MAX 256
static void enic_admin_msg_enqueue(struct enic *enic, void *buf,
unsigned int len)
{
struct enic_admin_msg *msg;
msg = kmalloc_flex(*msg, data, len);
if (!msg)
return;
msg->len = len;
memcpy(msg->data, buf, len);
spin_lock(&enic->admin_msg_lock);
if (enic->admin_msg_count >= ENIC_ADMIN_MSG_MAX) {
spin_unlock(&enic->admin_msg_lock);
kfree(msg);
if (net_ratelimit())
netdev_warn(enic->netdev,
"admin msg backlog full (%u); dropping\n",
ENIC_ADMIN_MSG_MAX);
return;
}
list_add_tail(&msg->list, &enic->admin_msg_list);
enic->admin_msg_count++;
spin_unlock(&enic->admin_msg_lock);
}
unsigned int enic_admin_rq_cq_service(struct enic *enic)
{
struct vnic_cq *cq = &enic->admin_cq[1];
struct vnic_rq *rq = &enic->admin_rq;
struct cq_enet_rq_desc *rq_desc;
struct vnic_rq_buf *buf;
u16 bwf, bytes_written;
unsigned int work = 0;
void *desc;
desc = vnic_cq_to_clean(cq);
while (enic_admin_cq_color(desc, cq->ring.desc_size) !=
cq->last_color) {
dma_rmb();
buf = rq->to_clean;
rq_desc = desc;
bwf = le16_to_cpu(rq_desc->bytes_written_flags);
bytes_written = bwf & CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
if (bytes_written > buf->len)
goto next_desc;
dma_sync_single_for_cpu(&enic->pdev->dev,
buf->dma_addr, buf->len,
DMA_FROM_DEVICE);
if (bwf & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) {
netdev_warn_once(enic->netdev,
"admin RQ: truncated message dropped\n");
goto next_desc;
}
if (!(rq_desc->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK)) {
netdev_warn_once(enic->netdev,
"admin RQ: bad FCS, dropping message\n");
goto next_desc;
}
if (enic->admin_rq_handler) {
u16 sender_vlan;
sender_vlan = le16_to_cpu(rq_desc->vlan);
if (bytes_written >= sizeof(struct enic_mbox_hdr)) {
struct enic_mbox_hdr *hdr = buf->os_buf;
hdr->src_vnic_id = (sender_vlan == 0) ?
cpu_to_le16(ENIC_MBOX_DST_PF) :
cpu_to_le16(sender_vlan - 1);
}
enic_admin_msg_enqueue(enic, buf->os_buf,
bytes_written);
}
next_desc:
enic_admin_rq_buf_clean(rq, rq->to_clean);
rq->to_clean = rq->to_clean->next;
rq->ring.desc_avail++;
vnic_cq_inc_to_clean(cq);
work++;
desc = vnic_cq_to_clean(cq);
}
if (enic_admin_rq_fill(enic, GFP_KERNEL)) {
if (net_ratelimit())
netdev_warn(enic->netdev,
"admin RQ refill failed; scheduling retry\n");
schedule_delayed_work(&enic->admin_poll_work,
msecs_to_jiffies(ENIC_ADMIN_RQ_REFILL_RETRY_MS));
}
return work;
}
static irqreturn_t enic_admin_isr_msix(int irq, void *data)
{
struct enic *enic = data;
schedule_delayed_work(&enic->admin_poll_work, 0);
return IRQ_HANDLED;
}
static void enic_admin_msg_work_handler(struct work_struct *work)
{
struct enic *enic = container_of(work, struct enic, admin_msg_work);
struct enic_admin_msg *msg, *tmp;
LIST_HEAD(local_list);
spin_lock_bh(&enic->admin_msg_lock);
list_splice_init(&enic->admin_msg_list, &local_list);
enic->admin_msg_count = 0;
spin_unlock_bh(&enic->admin_msg_lock);
list_for_each_entry_safe(msg, tmp, &local_list, list) {
if (enic->admin_rq_handler)
enic->admin_rq_handler(enic, msg->data, msg->len);
list_del(&msg->list);
kfree(msg);
}
}
static void enic_admin_poll_work_handler(struct work_struct *work)
{
struct enic *enic = container_of(to_delayed_work(work), struct enic,
admin_poll_work);
unsigned int credits;
unsigned int rq_work;
credits = vnic_intr_credits(&enic->admin_intr);
rq_work = enic_admin_rq_cq_service(enic);
if (rq_work > 0)
schedule_work(&enic->admin_msg_work);
vnic_intr_return_credits(&enic->admin_intr,
credits,
1 , 0);
}
static int enic_admin_setup_intr(struct enic *enic)
{
unsigned int intr_index = enic->intr_count;
int err;
if (vnic_dev_get_intr_mode(enic->vdev) != VNIC_DEV_INTR_MODE_MSIX ||
intr_index >= enic->intr_avail)
return -ENODEV;
err = vnic_intr_alloc(enic->vdev, &enic->admin_intr, intr_index);
if (err) {
netdev_warn(enic->netdev,
"Failed to alloc admin intr at index %u: %d\n",
intr_index, err);
return err;
}
enic->admin_intr_index = intr_index;
vnic_intr_mask(&enic->admin_intr);
snprintf(enic->msix[intr_index].devname,
sizeof(enic->msix[intr_index].devname),
"%s-admin", pci_name(enic->pdev));
enic->msix[intr_index].isr = enic_admin_isr_msix;
enic->msix[intr_index].devid = enic;
err = request_irq(enic->msix_entry[intr_index].vector,
enic->msix[intr_index].isr, 0,
enic->msix[intr_index].devname,
enic->msix[intr_index].devid);
if (err) {
netdev_warn(enic->netdev,
"Failed to request admin MSI-X irq: %d\n", err);
vnic_intr_free(&enic->admin_intr);
return err;
}
enic->msix[intr_index].requested = 1;
netdev_dbg(enic->netdev,
"admin channel using MSI-X interrupt (index %u)\n",
intr_index);
return 0;
}
static void enic_admin_teardown_intr(struct enic *enic)
{
unsigned int intr_index = enic->admin_intr_index;
free_irq(enic->msix_entry[intr_index].vector,
enic->msix[intr_index].devid);
cancel_delayed_work_sync(&enic->admin_poll_work);
enic->msix[intr_index].requested = 0;
}
static int enic_admin_qp_type_set(struct enic *enic, u32 enable)
{
u64 a0 = QP_TYPE_ADMIN, a1 = enable;
int wait = 1000;
int err;
spin_lock_bh(&enic->devcmd_lock);
err = vnic_dev_cmd(enic->vdev, CMD_QP_TYPE_SET, &a0, &a1, wait);
spin_unlock_bh(&enic->devcmd_lock);
return err;
}
static int enic_admin_alloc_resources(struct enic *enic)
{
int err;
err = vnic_wq_alloc_with_type(enic->vdev, &enic->admin_wq, 0,
ENIC_ADMIN_DESC_COUNT,
sizeof(struct wq_enet_desc),
RES_TYPE_ADMIN_WQ);
if (err)
return err;
err = vnic_rq_alloc_with_type(enic->vdev, &enic->admin_rq, 0,
ENIC_ADMIN_DESC_COUNT,
sizeof(struct rq_enet_desc),
RES_TYPE_ADMIN_RQ);
if (err)
goto free_wq;
err = vnic_cq_alloc_with_type(enic->vdev, &enic->admin_cq[0], 0,
ENIC_ADMIN_DESC_COUNT,
sizeof(struct cq_desc),
RES_TYPE_ADMIN_CQ);
if (err)
goto free_rq;
err = vnic_cq_alloc_with_type(enic->vdev, &enic->admin_cq[1], 1,
ENIC_ADMIN_DESC_COUNT,
16 << enic->ext_cq,
RES_TYPE_ADMIN_CQ);
if (err)
goto free_cq0;
return 0;
free_cq0:
vnic_cq_free(&enic->admin_cq[0]);
free_rq:
vnic_rq_free(&enic->admin_rq);
free_wq:
vnic_wq_free(&enic->admin_wq);
return err;
}
static void enic_admin_free_resources(struct enic *enic)
{
vnic_intr_free(&enic->admin_intr);
vnic_cq_free(&enic->admin_cq[1]);
vnic_cq_free(&enic->admin_cq[0]);
vnic_rq_free(&enic->admin_rq);
vnic_wq_free(&enic->admin_wq);
}
static void enic_admin_init_resources(struct enic *enic)
{
unsigned int intr_offset = enic->admin_intr_index;
vnic_wq_init(&enic->admin_wq,
0, 0, 0);
vnic_rq_init(&enic->admin_rq,
1, 0, 0);
vnic_cq_init(&enic->admin_cq[0],
VNIC_CQ_FC_DISABLE,
VNIC_CQ_COLOR_ENABLE,
0, 0, 1,
VNIC_CQ_INTR_DISABLE,
VNIC_CQ_ENTRY_ENABLE,
VNIC_CQ_MSG_DISABLE,
intr_offset,
0 );
vnic_cq_init(&enic->admin_cq[1],
VNIC_CQ_FC_DISABLE,
VNIC_CQ_COLOR_ENABLE,
0, 0, 1,
VNIC_CQ_INTR_ENABLE,
VNIC_CQ_ENTRY_ENABLE,
VNIC_CQ_MSG_DISABLE,
intr_offset,
0 );
vnic_intr_init(&enic->admin_intr,
0, 0, 1);
}
static void enic_admin_msg_drain(struct enic *enic)
{
struct enic_admin_msg *msg, *tmp;
spin_lock_bh(&enic->admin_msg_lock);
list_for_each_entry_safe(msg, tmp, &enic->admin_msg_list, list) {
list_del(&msg->list);
kfree(msg);
}
enic->admin_msg_count = 0;
spin_unlock_bh(&enic->admin_msg_lock);
}
int enic_admin_channel_open(struct enic *enic)
{
int err;
if (!enic->has_admin_channel)
return -ENODEV;
WRITE_ONCE(enic->mbox_send_disabled, true);
err = enic_admin_alloc_resources(enic);
if (err) {
netdev_err(enic->netdev,
"Failed to alloc admin channel resources: %d\n",
err);
return err;
}
spin_lock_init(&enic->admin_msg_lock);
INIT_LIST_HEAD(&enic->admin_msg_list);
INIT_WORK(&enic->admin_msg_work, enic_admin_msg_work_handler);
INIT_DELAYED_WORK(&enic->admin_poll_work, enic_admin_poll_work_handler);
err = enic_admin_setup_intr(enic);
if (err) {
netdev_err(enic->netdev,
"Admin channel requires MSI-X, SR-IOV unavailable: %d\n",
err);
goto free_resources;
}
enic_admin_init_resources(enic);
vnic_wq_enable(&enic->admin_wq);
vnic_rq_enable(&enic->admin_rq);
err = enic_admin_rq_fill(enic, GFP_KERNEL);
if (err) {
netdev_err(enic->netdev,
"Failed to fill admin RQ buffers: %d\n", err);
goto disable_queues;
}
err = enic_admin_qp_type_set(enic, QP_ENABLE);
if (err) {
netdev_err(enic->netdev,
"Failed to set admin QP type: %d\n", err);
goto disable_queues;
}
vnic_intr_unmask(&enic->admin_intr);
WRITE_ONCE(enic->mbox_send_disabled, false);
netdev_dbg(enic->netdev,
"admin channel open: intr=%u wq_avail=%u rq_avail=%u cq0_color=%u cq1_color=%u\n",
enic->admin_intr_index,
vnic_wq_desc_avail(&enic->admin_wq),
vnic_rq_desc_avail(&enic->admin_rq),
enic->admin_cq[0].last_color,
enic->admin_cq[1].last_color);
enic->admin_chan_up = true;
return 0;
disable_queues:
enic_admin_teardown_intr(enic);
enic_admin_qp_type_set(enic, QP_DISABLE);
if (vnic_wq_disable(&enic->admin_wq))
netdev_warn(enic->netdev, "Failed to disable admin WQ\n");
if (vnic_rq_disable(&enic->admin_rq))
netdev_warn(enic->netdev, "Failed to disable admin RQ\n");
cancel_work_sync(&enic->admin_msg_work);
enic_admin_msg_drain(enic);
enic_admin_rq_drain(enic);
free_resources:
enic_admin_free_resources(enic);
return err;
}
void enic_admin_channel_close(struct enic *enic)
{
int err;
if (!enic->has_admin_channel)
return;
if (!enic->admin_chan_up)
return;
WRITE_ONCE(enic->mbox_send_disabled, true);
netdev_dbg(enic->netdev, "admin channel close\n");
vnic_intr_mask(&enic->admin_intr);
enic_admin_teardown_intr(enic);
cancel_work_sync(&enic->link_notify_work);
cancel_work_sync(&enic->admin_msg_work);
enic_admin_msg_drain(enic);
enic_admin_qp_type_set(enic, QP_DISABLE);
err = vnic_wq_disable(&enic->admin_wq);
if (err)
netdev_warn(enic->netdev,
"Failed to disable admin WQ: %d\n", err);
err = vnic_rq_disable(&enic->admin_rq);
if (err)
netdev_warn(enic->netdev,
"Failed to disable admin RQ: %d\n", err);
vnic_wq_clean(&enic->admin_wq, enic_admin_wq_buf_clean);
enic_admin_rq_drain(enic);
vnic_cq_clean(&enic->admin_cq[0]);
vnic_cq_clean(&enic->admin_cq[1]);
vnic_intr_clean(&enic->admin_intr);
enic->admin_rq_handler = NULL;
enic_admin_free_resources(enic);
enic->admin_chan_up = false;
}