#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/dma-mapping.h>
#include <linux/delay.h>
#include <linux/completion.h>
#include "vnic_dev.h"
#include "vnic_wq.h"
#include "vnic_cq.h"
#include "enic.h"
#include "enic_admin.h"
#include "enic_mbox.h"
#include "wq_enet_desc.h"
#define ENIC_MBOX_POLL_TIMEOUT_US 5000000
#define ENIC_MBOX_POLL_INTERVAL_US 100
static void enic_mbox_fill_hdr(struct enic_mbox_hdr *hdr, u8 msg_type,
u16 dst_vnic_id, u16 msg_len, u64 msg_num)
{
memset(hdr, 0, sizeof(*hdr));
hdr->dst_vnic_id = cpu_to_le16(dst_vnic_id);
hdr->msg_type = msg_type;
hdr->msg_len = cpu_to_le16(msg_len);
hdr->msg_num = cpu_to_le64(msg_num);
}
static int enic_mbox_send_msg_id(struct enic *enic, u8 msg_type,
u16 dst_vnic_id, void *payload,
u16 payload_len, u64 msg_num, bool reuse_msg_num,
u8 expected_reply)
{
size_t total_len = sizeof(struct enic_mbox_hdr) + payload_len;
struct vnic_wq *wq = &enic->admin_wq;
struct wq_enet_desc *desc;
bool reply_expected = false;
unsigned long timeout;
dma_addr_t dma_addr;
u16 vlan_tag;
void *buf;
int err;
if (payload_len > ENIC_ADMIN_BUF_SIZE - sizeof(struct enic_mbox_hdr))
return -EINVAL;
mutex_lock(&enic->mbox_lock);
if (!enic->has_admin_channel || READ_ONCE(enic->mbox_send_disabled)) {
err = -ENODEV;
goto unlock;
}
if (vnic_wq_desc_avail(wq) == 0) {
err = -ENOSPC;
goto unlock;
}
buf = kmalloc(total_len, GFP_KERNEL);
if (!buf) {
err = -ENOMEM;
goto unlock;
}
if (!reuse_msg_num)
msg_num = ++enic->mbox_msg_num;
if (expected_reply) {
reinit_completion(&enic->mbox_comp);
spin_lock_bh(&enic->mbox_state_lock);
enic->mbox_expected_reply = expected_reply;
enic->mbox_expected_msg_num = msg_num;
spin_unlock_bh(&enic->mbox_state_lock);
reply_expected = true;
}
enic_mbox_fill_hdr(buf, msg_type, dst_vnic_id, total_len, msg_num);
if (payload_len) {
void *dst = buf + sizeof(struct enic_mbox_hdr);
memcpy(dst, payload, payload_len);
}
dma_addr = dma_map_single(&enic->pdev->dev, buf, total_len,
DMA_TO_DEVICE);
if (dma_mapping_error(&enic->pdev->dev, dma_addr)) {
kfree(buf);
err = -ENOMEM;
goto unlock;
}
if (dst_vnic_id == ENIC_MBOX_DST_PF)
vlan_tag = 0;
else
vlan_tag = dst_vnic_id + 1;
desc = vnic_wq_next_desc(wq);
wq_enet_desc_enc(desc, (u64)dma_addr | VNIC_PADDR_TARGET,
total_len,
0, 0, 0,
1, 1,
0,
1, vlan_tag,
0);
vnic_wq_post(wq, buf, dma_addr, total_len,
1, 1,
1, 1,
0, 0);
vnic_wq_doorbell(wq);
timeout = jiffies + usecs_to_jiffies(ENIC_MBOX_POLL_TIMEOUT_US);
err = -ETIMEDOUT;
while (time_before(jiffies, timeout)) {
if (enic_admin_wq_cq_service(enic)) {
err = 0;
break;
}
usleep_range(ENIC_MBOX_POLL_INTERVAL_US,
ENIC_MBOX_POLL_INTERVAL_US + 50);
}
if (err && enic_admin_wq_cq_service(enic))
err = 0;
if (!err) {
wq->to_clean = wq->to_clean->next;
wq->ring.desc_avail++;
dma_unmap_single(&enic->pdev->dev, dma_addr, total_len,
DMA_TO_DEVICE);
kfree(buf);
} else {
netdev_err(enic->netdev,
"MBOX send timed out (type %u dst %u), disabling channel\n",
msg_type, dst_vnic_id);
WRITE_ONCE(enic->mbox_send_disabled, true);
}
netdev_dbg(enic->netdev,
"MBOX send msg_type %u dst %u vlan %u err %d\n",
msg_type, dst_vnic_id, vlan_tag, err);
unlock:
if (err && reply_expected) {
spin_lock_bh(&enic->mbox_state_lock);
if (enic->mbox_expected_reply == expected_reply &&
enic->mbox_expected_msg_num == msg_num) {
enic->mbox_expected_reply = 0;
enic->mbox_expected_msg_num = 0;
}
spin_unlock_bh(&enic->mbox_state_lock);
}
mutex_unlock(&enic->mbox_lock);
return err;
}
int enic_mbox_send_msg(struct enic *enic, u8 msg_type, u16 dst_vnic_id,
void *payload, u16 payload_len)
{
return enic_mbox_send_msg_id(enic, msg_type, dst_vnic_id, payload,
payload_len, 0, false, 0);
}
static int enic_mbox_send_reply(struct enic *enic, u8 msg_type,
u16 dst_vnic_id, void *payload, u16 payload_len,
u64 msg_num)
{
return enic_mbox_send_msg_id(enic, msg_type, dst_vnic_id, payload,
payload_len, msg_num, true, 0);
}
static int enic_mbox_vf_send_request(struct enic *enic, u8 request_type,
u8 expected_reply, void *payload,
u16 payload_len)
{
return enic_mbox_send_msg_id(enic, request_type, ENIC_MBOX_DST_PF,
payload, payload_len, 0, false,
expected_reply);
}
static int enic_mbox_wait_reply(struct enic *enic, unsigned long timeout_ms)
{
unsigned long left;
int err = 0;
left = wait_for_completion_timeout(&enic->mbox_comp,
msecs_to_jiffies(timeout_ms));
if (left)
return 0;
spin_lock_bh(&enic->mbox_state_lock);
if (enic->mbox_expected_reply) {
enic->mbox_expected_reply = 0;
enic->mbox_expected_msg_num = 0;
err = -ETIMEDOUT;
}
spin_unlock_bh(&enic->mbox_state_lock);
return err;
}
int enic_mbox_send_link_state(struct enic *enic, u16 vf_id, u32 link_state)
{
struct enic_mbox_pf_link_state_notif_msg notif = {};
if (!enic->vf_state || vf_id >= enic->num_vfs ||
!enic->vf_state[vf_id].registered) {
netdev_dbg(enic->netdev,
"MBOX: skip link state to unregistered VF %u\n",
vf_id);
return 0;
}
notif.link_state = cpu_to_le32(link_state);
return enic_mbox_send_msg(enic, ENIC_MBOX_PF_LINK_STATE_NOTIF, vf_id,
¬if, sizeof(notif));
}
static int enic_mbox_pf_handle_capability(struct enic *enic, void *msg,
u16 vf_id, u64 msg_num)
{
struct enic_mbox_vf_capability_reply_msg reply = {};
reply.reply.ret_major = cpu_to_le16(0);
reply.version = cpu_to_le32(ENIC_MBOX_CAP_VERSION_1);
return enic_mbox_send_reply(enic, ENIC_MBOX_VF_CAPABILITY_REPLY, vf_id,
&reply, sizeof(reply), msg_num);
}
static int enic_mbox_pf_handle_register(struct enic *enic, void *msg,
u16 vf_id, u64 msg_num)
{
struct enic_mbox_vf_register_reply_msg reply = {};
u32 link_state;
int err;
if (!enic->vf_state || vf_id >= enic->num_vfs) {
if (net_ratelimit())
netdev_warn(enic->netdev,
"MBOX: register from invalid VF %u\n",
vf_id);
return -EINVAL;
}
if (enic->vf_state[vf_id].registered) {
netdev_dbg(enic->netdev,
"MBOX: VF %u re-register, cleaning previous state\n",
vf_id);
enic->vf_state[vf_id].registered = false;
}
reply.reply.ret_major = cpu_to_le16(0);
err = enic_mbox_send_reply(enic, ENIC_MBOX_VF_REGISTER_REPLY, vf_id,
&reply, sizeof(reply), msg_num);
if (err)
return err;
enic->vf_state[vf_id].registered = true;
if (net_ratelimit())
netdev_info(enic->netdev, "VF %u registered via MBOX\n", vf_id);
link_state = netif_carrier_ok(enic->netdev) ?
ENIC_MBOX_LINK_STATE_ENABLE :
ENIC_MBOX_LINK_STATE_DISABLE;
err = enic_mbox_send_link_state(enic, vf_id, link_state);
if (err && net_ratelimit())
netdev_warn(enic->netdev,
"VF %u: failed to send initial link state: %d\n",
vf_id, err);
return 0;
}
static int enic_mbox_pf_handle_unregister(struct enic *enic, void *msg,
u16 vf_id, u64 msg_num)
{
struct enic_mbox_vf_register_reply_msg reply = {};
int err;
if (!enic->vf_state || vf_id >= enic->num_vfs) {
if (net_ratelimit())
netdev_warn(enic->netdev,
"MBOX: unregister from invalid VF %u\n",
vf_id);
return -EINVAL;
}
enic->vf_state[vf_id].registered = false;
reply.reply.ret_major = cpu_to_le16(0);
err = enic_mbox_send_reply(enic, ENIC_MBOX_VF_UNREGISTER_REPLY, vf_id,
&reply, sizeof(reply), msg_num);
if (net_ratelimit())
netdev_info(enic->netdev,
"VF %u unregistered via MBOX\n", vf_id);
return err;
}
static void enic_mbox_pf_process_msg(struct enic *enic,
struct enic_mbox_hdr *hdr, void *payload)
{
u16 vf_id = le16_to_cpu(hdr->src_vnic_id);
u16 msg_len = le16_to_cpu(hdr->msg_len);
int err = 0;
if (!enic->vf_state) {
netdev_dbg(enic->netdev,
"MBOX: PF received msg but SRIOV not active\n");
return;
}
if (vf_id >= enic->num_vfs) {
if (net_ratelimit())
netdev_warn(enic->netdev,
"MBOX: PF received msg from invalid VF %u\n",
vf_id);
return;
}
switch (hdr->msg_type) {
case ENIC_MBOX_VF_CAPABILITY_REQUEST:
err = enic_mbox_pf_handle_capability(enic, payload, vf_id,
le64_to_cpu(hdr->msg_num));
break;
case ENIC_MBOX_VF_REGISTER_REQUEST:
err = enic_mbox_pf_handle_register(enic, payload, vf_id,
le64_to_cpu(hdr->msg_num));
break;
case ENIC_MBOX_VF_UNREGISTER_REQUEST:
err = enic_mbox_pf_handle_unregister(enic, payload, vf_id,
le64_to_cpu(hdr->msg_num));
break;
case ENIC_MBOX_PF_LINK_STATE_ACK: {
struct enic_mbox_pf_link_state_ack_msg *ack = payload;
if (msg_len < sizeof(*hdr) + sizeof(*ack))
break;
if (le16_to_cpu(ack->ack.ret_major) && net_ratelimit())
netdev_warn(enic->netdev,
"MBOX: VF %u link state ACK error %u/%u\n",
vf_id,
le16_to_cpu(ack->ack.ret_major),
le16_to_cpu(ack->ack.ret_minor));
break;
}
default:
netdev_dbg(enic->netdev,
"MBOX: PF unhandled msg type %u from VF %u\n",
hdr->msg_type, vf_id);
err = -EOPNOTSUPP;
break;
}
if (err && net_ratelimit())
netdev_warn(enic->netdev,
"MBOX: PF handler for msg type %u from VF %u failed: %d\n",
hdr->msg_type, vf_id, err);
}
static void enic_mbox_vf_handle_reply(struct enic *enic, u8 reply_type,
void *payload, u64 msg_num)
{
struct enic_mbox_generic_reply *reply = payload;
u16 ret_major = le16_to_cpu(reply->ret_major);
u64 expected_msg_num;
u8 expected_type;
spin_lock_bh(&enic->mbox_state_lock);
expected_type = enic->mbox_expected_reply;
expected_msg_num = enic->mbox_expected_msg_num;
if (expected_type != reply_type || expected_msg_num != msg_num) {
spin_unlock_bh(&enic->mbox_state_lock);
netdev_warn(enic->netdev,
"MBOX: stale reply %u/%llu (expected %u/%llu), drop\n",
reply_type, (unsigned long long)msg_num,
expected_type, (unsigned long long)expected_msg_num);
return;
}
if (!ret_major) {
switch (reply_type) {
case ENIC_MBOX_VF_CAPABILITY_REPLY: {
struct enic_mbox_vf_capability_reply_msg *cap = payload;
WRITE_ONCE(enic->pf_cap_version,
le32_to_cpu(cap->version));
break;
}
case ENIC_MBOX_VF_REGISTER_REPLY:
WRITE_ONCE(enic->vf_registered, true);
break;
case ENIC_MBOX_VF_UNREGISTER_REPLY:
WRITE_ONCE(enic->vf_registered, false);
break;
}
}
enic->mbox_expected_reply = 0;
enic->mbox_expected_msg_num = 0;
complete(&enic->mbox_comp);
spin_unlock_bh(&enic->mbox_state_lock);
if (ret_major)
netdev_warn(enic->netdev,
"MBOX: PF rejected reply type %u: %u/%u\n",
reply_type, ret_major,
le16_to_cpu(reply->ret_minor));
}
static void enic_mbox_vf_handle_link_state(struct enic *enic, void *payload,
u64 msg_num)
{
struct enic_mbox_pf_link_state_notif_msg *notif = payload;
struct enic_mbox_pf_link_state_ack_msg ack = {};
u32 link_state = le32_to_cpu(notif->link_state);
int err;
spin_lock_bh(&enic->vf_link_state_lock);
switch (link_state) {
case ENIC_MBOX_LINK_STATE_ENABLE:
enic->vf_link_state = ENIC_VF_LINK_STATE_UP;
if (enic->vf_link_running &&
!netif_carrier_ok(enic->netdev))
netif_carrier_on(enic->netdev);
netdev_dbg(enic->netdev, "MBOX: link state -> UP\n");
break;
case ENIC_MBOX_LINK_STATE_DISABLE:
enic->vf_link_state = ENIC_VF_LINK_STATE_DOWN;
if (enic->vf_link_running &&
netif_carrier_ok(enic->netdev))
netif_carrier_off(enic->netdev);
netdev_dbg(enic->netdev, "MBOX: link state -> DOWN\n");
break;
default:
netdev_warn(enic->netdev, "MBOX: unknown link state %u\n",
link_state);
ack.ack.ret_major = cpu_to_le16(ENIC_MBOX_ERR_GENERIC);
break;
}
spin_unlock_bh(&enic->vf_link_state_lock);
err = enic_mbox_send_reply(enic, ENIC_MBOX_PF_LINK_STATE_ACK,
ENIC_MBOX_DST_PF, &ack, sizeof(ack), msg_num);
if (err && net_ratelimit())
netdev_warn(enic->netdev,
"MBOX: failed to send link state ACK: %d\n", err);
}
void enic_mbox_vf_link_state_reset(struct enic *enic)
{
spin_lock_bh(&enic->vf_link_state_lock);
enic->vf_link_state = ENIC_VF_LINK_STATE_UNKNOWN;
if (enic->vf_link_running && netif_carrier_ok(enic->netdev))
netif_carrier_off(enic->netdev);
spin_unlock_bh(&enic->vf_link_state_lock);
}
void enic_mbox_vf_link_state_set_running(struct enic *enic, bool running)
{
spin_lock_bh(&enic->vf_link_state_lock);
enic->vf_link_running = running;
if (running && enic->vf_link_state == ENIC_VF_LINK_STATE_UP) {
if (!netif_carrier_ok(enic->netdev))
netif_carrier_on(enic->netdev);
} else if (netif_carrier_ok(enic->netdev)) {
netif_carrier_off(enic->netdev);
}
spin_unlock_bh(&enic->vf_link_state_lock);
}
static bool enic_mbox_vf_payload_ok(struct enic *enic, u8 msg_type,
u16 payload_len, size_t min_len)
{
if (payload_len < min_len) {
netdev_warn(enic->netdev,
"MBOX: short payload for type %u (%u < %zu)\n",
msg_type, payload_len, min_len);
return false;
}
return true;
}
static void enic_mbox_vf_process_msg(struct enic *enic,
struct enic_mbox_hdr *hdr, void *payload,
u16 payload_len)
{
u64 msg_num = le64_to_cpu(hdr->msg_num);
switch (hdr->msg_type) {
case ENIC_MBOX_VF_CAPABILITY_REPLY: {
size_t exp = sizeof(struct enic_mbox_vf_capability_reply_msg);
if (!enic_mbox_vf_payload_ok(enic, hdr->msg_type,
payload_len, exp))
return;
enic_mbox_vf_handle_reply(enic, hdr->msg_type, payload, msg_num);
break;
}
case ENIC_MBOX_VF_REGISTER_REPLY: {
size_t exp = sizeof(struct enic_mbox_vf_register_reply_msg);
if (!enic_mbox_vf_payload_ok(enic, hdr->msg_type,
payload_len, exp))
return;
enic_mbox_vf_handle_reply(enic, hdr->msg_type, payload, msg_num);
break;
}
case ENIC_MBOX_VF_UNREGISTER_REPLY: {
size_t exp = sizeof(struct enic_mbox_vf_register_reply_msg);
if (!enic_mbox_vf_payload_ok(enic, hdr->msg_type,
payload_len, exp))
return;
enic_mbox_vf_handle_reply(enic, hdr->msg_type, payload, msg_num);
break;
}
case ENIC_MBOX_PF_LINK_STATE_NOTIF: {
size_t exp = sizeof(struct enic_mbox_pf_link_state_notif_msg);
if (!enic_mbox_vf_payload_ok(enic, hdr->msg_type,
payload_len, exp))
return;
enic_mbox_vf_handle_link_state(enic, payload, msg_num);
break;
}
default:
netdev_dbg(enic->netdev,
"MBOX: VF unhandled msg type %u\n",
hdr->msg_type);
break;
}
}
static void enic_mbox_recv_handler(struct enic *enic, void *buf,
unsigned int len)
{
struct enic_mbox_hdr *hdr = buf;
void *payload;
u16 msg_len;
if (len < sizeof(*hdr)) {
if (net_ratelimit())
netdev_warn(enic->netdev,
"MBOX: truncated message (len %u < %zu)\n",
len, sizeof(*hdr));
return;
}
if (hdr->msg_type >= ENIC_MBOX_MAX) {
if (net_ratelimit())
netdev_warn(enic->netdev,
"MBOX: unknown msg type %u\n",
hdr->msg_type);
return;
}
msg_len = le16_to_cpu(hdr->msg_len);
if (msg_len < sizeof(*hdr) || msg_len > len) {
if (net_ratelimit())
netdev_warn(enic->netdev,
"MBOX: invalid msg_len %u (buf len %u)\n",
msg_len, len);
return;
}
netdev_dbg(enic->netdev,
"MBOX recv: type %u from vnic %u len %u\n",
hdr->msg_type, le16_to_cpu(hdr->src_vnic_id), msg_len);
payload = buf + sizeof(*hdr);
if (enic->vf_state) {
enic_mbox_pf_process_msg(enic, hdr, payload);
} else if (le16_to_cpu(hdr->src_vnic_id) == ENIC_MBOX_DST_PF) {
enic_mbox_vf_process_msg(enic, hdr, payload,
msg_len - (u16)sizeof(*hdr));
} else if (net_ratelimit()) {
netdev_warn(enic->netdev,
"MBOX: VF dropping non-PF message from vnic %u\n",
le16_to_cpu(hdr->src_vnic_id));
}
}
int enic_mbox_vf_capability_check(struct enic *enic)
{
struct enic_mbox_vf_capability_msg req = {};
u32 version;
int err;
WRITE_ONCE(enic->pf_cap_version, 0);
req.version = cpu_to_le32(ENIC_MBOX_CAP_VERSION_1);
err = enic_mbox_vf_send_request(enic,
ENIC_MBOX_VF_CAPABILITY_REQUEST,
ENIC_MBOX_VF_CAPABILITY_REPLY,
&req, sizeof(req));
if (err)
return err;
err = enic_mbox_wait_reply(enic, 3000);
version = READ_ONCE(enic->pf_cap_version);
if (err) {
netdev_warn(enic->netdev,
"MBOX: no capability reply from PF\n");
return err;
}
if (version < ENIC_MBOX_CAP_VERSION_1) {
netdev_warn(enic->netdev,
"MBOX: PF rejected capability request or reported unsupported version %u\n",
version);
return -EOPNOTSUPP;
}
return 0;
}
int enic_mbox_vf_register(struct enic *enic)
{
bool registered;
int err;
WRITE_ONCE(enic->vf_registered, false);
err = enic_mbox_vf_send_request(enic, ENIC_MBOX_VF_REGISTER_REQUEST,
ENIC_MBOX_VF_REGISTER_REPLY, NULL, 0);
if (err)
return err;
err = enic_mbox_wait_reply(enic, 3000);
registered = READ_ONCE(enic->vf_registered);
if (err) {
netdev_warn(enic->netdev,
"MBOX: VF registration with PF timed out\n");
return err;
}
if (!registered)
return -ENODEV;
return 0;
}
int enic_mbox_vf_unregister(struct enic *enic)
{
bool registered;
int err;
if (!READ_ONCE(enic->vf_registered))
return 0;
err = enic_mbox_vf_send_request(enic,
ENIC_MBOX_VF_UNREGISTER_REQUEST,
ENIC_MBOX_VF_UNREGISTER_REPLY,
NULL, 0);
if (err)
return err;
err = enic_mbox_wait_reply(enic, 3000);
registered = READ_ONCE(enic->vf_registered);
if (err)
return err;
if (registered)
return -EACCES;
return 0;
}
void enic_mbox_init(struct enic *enic)
{
bool reinit = enic->mbox_initialized;
if (!reinit) {
mutex_init(&enic->mbox_lock);
init_completion(&enic->mbox_comp);
spin_lock_init(&enic->mbox_state_lock);
enic->mbox_msg_num = 0;
enic->mbox_initialized = true;
} else {
reinit_completion(&enic->mbox_comp);
}
spin_lock_bh(&enic->mbox_state_lock);
enic->mbox_expected_reply = 0;
enic->mbox_expected_msg_num = 0;
spin_unlock_bh(&enic->mbox_state_lock);
enic->admin_rq_handler = enic_mbox_recv_handler;
}