#include <linux/bitfield.h>
#include <linux/bits.h>
#include <linux/cache.h>
#include <linux/if_ether.h>
#include <linux/iopoll.h>
#include <linux/log2.h>
#include <linux/sizes.h>
#include "mpnic.h"
#include "mpnic_txrx.h"
#define MPNIC_MEM_INIT_POLL_US 500
#define MPNIC_MEM_INIT_TO_US 5000
#define MPNIC_MEM_INIT_BDQ_VAL 0x3
#define MPNIC_MEM_INIT_RCM_VAL 0xf
#define MPNIC_MEM_INIT_RDE_VAL 0x7ffff
#define MPNIC_MEM_INIT_RPC_VAL 0x1
#define MPNIC_MEM_INIT_TCM_VAL 0xf
#define MPNIC_MEM_INIT_TDE_VAL 0x3
#define MPNIC_MEM_INIT_TDF_VAL 0x1
#define MPNIC_MEM_INIT_RXB_VAL 0x1
#define MPNIC_TQS_ARB_INIT_VAL 0xf
#define MPNIC_BDQ_SRAM_DESCS 64u
#define MPNIC_TDF_SRAM_DESCS 128u
#define MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL 800
#define MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL \
(SZ_1M / 128 - 2 * MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL)
#define MPNIC_TXB_INIT_BMC_THRESH 100
#define MPNIC_TXB_INIT_P0_THRESH 100
#define MPNIC_TXB_INIT_P0_ARB_WEIGHTS 0x64
#define MPNIC_RXB_INIT_HOST_DROP_THRESH 0x50
#define MPNIC_RXB_INIT_POOL_TC_CRDTS_P0 (0xd20 * 8)
#define MPNIC_RXB_INIT_BMC_CRDTS 0x20
#define MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH \
(SZ_8M / 128 - MPNIC_RXB_INIT_POOL_TC_CRDTS_P0 - \
MPNIC_RXB_INIT_BMC_CRDTS)
#define MPNIC_RXB_INIT_COMMON_CRDT_THRSH 0x50
#define MPNIC_TXB_PORT_MODE_SINGLE 1
#define MPNIC_TXB_TC_IDX_MAC_0 8
#define MPNIC_TXB_TC_IDX_BMC 16
#define MPNIC_TQS_MTU_CTL0_MAX 0x2800
#define MPNIC_TQS_DWRR_INIT_QUANTUM (15 * MPNIC_TQS_MTU_CTL0_MAX / 256)
#define MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH (15 * MPNIC_TQS_MTU_CTL0_MAX)
#define MPNIC_TQS_GLBL_TXB_CRDT_UNIT SZ_1K
#define MPNIC_TQS_GLBL_TXB_CRDT_BMC \
(MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL * 128 / MPNIC_TQS_GLBL_TXB_CRDT_UNIT)
#define MPNIC_TQS_GLBL_TXB_CRDT_MAC \
(MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL * 128 / \
MPNIC_TQS_GLBL_TXB_CRDT_UNIT)
struct mpnic_init_poll {
u64 exp_val;
u32 addr;
};
struct mpnic_poll_state {
int poll_idx;
u64 val;
};
static void mpnic_tdf_glbl_init(struct mpnic_dev *mpd)
{
mpnic_wr64(mpd, MPNIC_TWQ_DEF_PRI_TWD, MPNIC_TWD_FLAG_REQ_COMPLETION);
}
static void mpnic_txb_init(struct mpnic_dev *mpd)
{
int i;
mpnic_wr64(mpd, MPNIC_TXB_BMC, MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL);
for (i = 0; i < MPNIC_TXB_P0_CNT; i++)
mpnic_wr64(mpd, MPNIC_TXB_P0(i), 0);
mpnic_wr64(mpd, MPNIC_TXB_P0(MPNIC_TXB_TC_IDX_MAC_0),
MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL);
mpnic_wr64(mpd, MPNIC_TXB_P0(MPNIC_TXB_TC_IDX_BMC),
MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL);
mpnic_wr64(mpd, MPNIC_TXB_P0_THRESH(MPNIC_TXB_TC_IDX_BMC),
MPNIC_TXB_INIT_BMC_THRESH);
mpnic_wr64(mpd, MPNIC_TXB_P0_THRESH(MPNIC_TXB_TC_IDX_MAC_0),
MPNIC_TXB_INIT_P0_THRESH);
mpnic_wr64(mpd, MPNIC_TXB_P0_ARB_WEIGHTS(MPNIC_TXB_TC_IDX_MAC_0),
MPNIC_TXB_INIT_P0_ARB_WEIGHTS);
mpnic_wr64(mpd, MPNIC_TXB_PORT_CONFIG,
FIELD_PREP(MPNIC_TXB_PORT_CONFIG_PORT_MODE,
MPNIC_TXB_PORT_MODE_SINGLE));
}
static void mpnic_rxb_init(struct mpnic_dev *mpd)
{
mpnic_wr64(mpd, MPNIC_RXB_PORT_CLASS_CFG(0),
FIELD_PREP(MPNIC_RXB_PORT_CLASS_CFG_DEFAULT_L2_ACTION,
MPNIC_L2_ACTION_PASS));
mpnic_wr64(mpd, MPNIC_RXB_PORT_CFG(0),
FIELD_PREP(MPNIC_RXB_PORT_CFG_FCS_STRIP_MODE,
MPNIC_FCS_MODE_STRIP));
mpnic_wr64(mpd, MPNIC_RXB_HOST_DROP_THRESH(0),
MPNIC_RXB_INIT_HOST_DROP_THRESH);
mpnic_wr64(mpd, MPNIC_RXB_TC_CRDTS(0),
MPNIC_RXB_INIT_POOL_TC_CRDTS_P0);
mpnic_wr64(mpd, MPNIC_RXB_COMMON_CRDT_CTRL_TC(0),
FIELD_PREP(MPNIC_RXB_COMMON_CRDT_CTRL_TC_THRESH,
MPNIC_RXB_INIT_COMMON_CRDT_THRSH) |
FIELD_PREP(MPNIC_RXB_COMMON_CRDT_CTRL_TC_MAX_CRDTS,
MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH) |
MPNIC_RXB_COMMON_CRDT_CTRL_TC_TC_EN);
mpnic_wr64(mpd, MPNIC_RXB_POOL_COMMON_CRDTS(0),
MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH);
mpnic_wr64(mpd, MPNIC_RXB_BMC_CRDTS(0), MPNIC_RXB_INIT_BMC_CRDTS);
mpnic_wr64(mpd, MPNIC_RXB_MEM_INIT_REQ, MPNIC_MEM_INIT_RXB_VAL);
}
static u64 mpnic_desc_cfg(unsigned int sram_descs, unsigned int q_idx)
{
return FIELD_PREP(MPNIC_DESC_CFG_NUM_DESCS, __ffs(sram_descs) - 6) |
FIELD_PREP(MPNIC_DESC_CFG_START_ADDR,
q_idx * (sram_descs >> 6));
}
static void mpnic_desc_sram_init(struct mpnic_dev *mpd)
{
int i;
for (i = 0; i < MPNIC_MAX_TXQS * 2; i++)
mpnic_wr64(mpd, MPNIC_TDF_DESC_CFG(i),
mpnic_desc_cfg(MPNIC_TDF_SRAM_DESCS, i));
for (i = 0; i < MPNIC_MAX_RXQS; i++) {
mpnic_wr64(mpd, MPNIC_HPQ_DESC_CFG(i),
mpnic_desc_cfg(MPNIC_BDQ_SRAM_DESCS, i));
mpnic_wr64(mpd, MPNIC_PPQ_DESC_CFG(i),
mpnic_desc_cfg(MPNIC_BDQ_SRAM_DESCS,
MPNIC_MAX_RXQS + i));
}
}
static void mpnic_rxglb_init(struct mpnic_dev *mpd)
{
mpnic_wr64(mpd, MPNIC_BDQ_GLBL_CTL0,
FIELD_PREP(MPNIC_BDQ_GLBL_CTL0_PREFETCH_SPACE_THRESH, 32) |
FIELD_PREP(MPNIC_BDQ_GLBL_CTL0_MAX_REQ_SIZE, 64));
mpnic_wr64(mpd, MPNIC_RDE_CTL,
FIELD_PREP(MPNIC_RDE_CTL_HPQ_DROP_THRESHOLD,
MPNIC_RX_HPQ_DROP_THRS - 1) |
FIELD_PREP(MPNIC_RDE_CTL_PPQ_DROP_THRESHOLD,
MPNIC_RX_PPQ_DROP_THRS - 1) |
FIELD_PREP(MPNIC_RDE_CTL_HPQ_LOCAL_DROP_THRESHOLD,
MPNIC_RX_HPQ_DROP_THRS - 1) |
FIELD_PREP(MPNIC_RDE_CTL_PPQ_LOCAL_DROP_THRESHOLD,
MPNIC_RX_PPQ_DROP_THRS - 1));
mpnic_wr64(mpd, MPNIC_RSC_GLOBAL_CONF,
MPNIC_RSC_GLOBAL_CONF_RSC_DISABLE);
mpnic_wr64(mpd, MPNIC_BDQ_MEM_INIT_REQ, MPNIC_MEM_INIT_BDQ_VAL);
mpnic_wr64(mpd, MPNIC_RCM_MEM_INIT_REQ, MPNIC_MEM_INIT_RCM_VAL);
mpnic_wr64(mpd, MPNIC_RDE_MEM_INIT_REQ, MPNIC_MEM_INIT_RDE_VAL);
mpnic_wr64(mpd, MPNIC_RPC_MEM_INIT_REQ, MPNIC_MEM_INIT_RPC_VAL);
}
static void mpnic_txglb_init(struct mpnic_dev *mpd)
{
mpnic_wr64(mpd, MPNIC_TOF_TCAM_DEST_REMAP, 0);
mpnic_wr64(mpd, MPNIC_TCM_MEM_INIT_REQ, MPNIC_MEM_INIT_TCM_VAL);
mpnic_wr64(mpd, MPNIC_TDE_MEM_INIT_REQ, MPNIC_MEM_INIT_TDE_VAL);
mpnic_wr64(mpd, MPNIC_TDF_MEM_INIT_REQ, MPNIC_MEM_INIT_TDF_VAL);
}
static void mpnic_tqs_sram_init(struct mpnic_dev *mpd)
{
mpnic_wr64(mpd, MPNIC_TQS_GROUP_INIT_CTL,
FIELD_PREP(MPNIC_TQS_GROUP_INIT_CTL_QUANTUM,
MPNIC_TQS_DWRR_INIT_QUANTUM) |
MPNIC_TQS_GROUP_INIT_CTL_INIT);
mpnic_wr64(mpd, MPNIC_TQS_SET_INIT_CTL,
FIELD_PREP(MPNIC_TQS_SET_INIT_CTL_QUANTUM,
MPNIC_TQS_DWRR_INIT_QUANTUM) |
MPNIC_TQS_SET_INIT_CTL_INIT);
mpnic_wr64(mpd, MPNIC_TQS_PORT_INIT_CTL,
FIELD_PREP(MPNIC_TQS_PORT_INIT_CTL_QUANTUM,
MPNIC_TQS_DWRR_INIT_QUANTUM) |
MPNIC_TQS_PORT_INIT_CTL_INIT);
mpnic_wr64(mpd, MPNIC_TQS_SRAM_INIT_CTL,
FIELD_PREP(MPNIC_TQS_SRAM_INIT_CTL_QUANTUM,
MPNIC_TQS_DWRR_INIT_QUANTUM) |
MPNIC_TQS_SRAM_INIT_CTL_INIT);
}
static void mpnic_tqs_init(struct mpnic_dev *mpd)
{
u64 val;
mpnic_wr64(mpd, MPNIC_TQS_MTU_CTL0, MPNIC_TQS_MTU_CTL0_MAX);
mpnic_wr64(mpd, MPNIC_TQS_MTU_CTL1, MPNIC_TQS_MTU_CTL0_MAX);
mpnic_wr64(mpd, MPNIC_TQS_GLBL_CTL0,
MPNIC_TQS_GLBL_CTL0_TWD_ERROR_CHECK_EN);
mpnic_tqs_sram_init(mpd);
mpnic_wr64(mpd, MPNIC_TQS_GLBL_P0_0,
FIELD_PREP(MPNIC_TQS_GLBL_P0_0_TXB_MAX_CRDTS_0,
MPNIC_TQS_GLBL_TXB_CRDT_MAC));
mpnic_wr64(mpd, MPNIC_TQS_GLBL_P0_1, 0);
mpnic_wr64(mpd, MPNIC_TQS_GLBL_BMC,
FIELD_PREP(MPNIC_TQS_GLBL_BMC_TXB_MAX_CRDTS,
MPNIC_TQS_GLBL_TXB_CRDT_BMC));
mpnic_wr64(mpd, MPNIC_TQS_PORT_CTL(0), 0);
mpnic_wr64(mpd, MPNIC_TQS_SET_P0_MAP0(0), ~0ULL);
mpnic_wr64(mpd, MPNIC_TQS_SET_P0_MAP1(0), ~0ULL);
mpnic_wr64(mpd, MPNIC_TQS_CEV_MIN_SCHED_THRESH_0,
FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_QUEUE,
MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH) |
FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_GROUP,
MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH));
mpnic_wr64(mpd, MPNIC_TQS_CEV_MIN_SCHED_THRESH_1,
FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_SET,
MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH) |
FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_PORT,
MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH));
mpnic_wr64(mpd, MPNIC_TQS_GLBL_SHAPING, MPNIC_TQS_GLBL_SHAPING_DISABLE);
val = mpnic_rd64(mpd, MPNIC_TQS_SLOWDOWN_CTL);
val |= MPNIC_TQS_SLOWDOWN_CTL_ENABLE;
mpnic_wr64(mpd, MPNIC_TQS_SLOWDOWN_CTL, val);
val = mpnic_rd64(mpd, MPNIC_TQS_ARB_CTL);
val |= MPNIC_TQS_ARB_CTL_SET_CRDT_BUCKET_EN |
MPNIC_TQS_ARB_CTL_SET_IMM_DECR_EN |
MPNIC_TQS_ARB_CTL_GROUP_CRDT_BUCKET_EN |
MPNIC_TQS_ARB_CTL_GROUP_IMM_DECR_EN |
MPNIC_TQS_ARB_CTL_QUEUE_CRDT_BUCKET_EN |
MPNIC_TQS_ARB_CTL_QUEUE_IMM_DECR_EN;
mpnic_wr64(mpd, MPNIC_TQS_ARB_CTL, val);
}
static void mpnic_mps_init(struct mpnic_dev *mpd, u32 reg, unsigned int mps,
unsigned int cls)
{
u64 val = mpnic_rd64(mpd, reg);
val &= ~(MPNIC_RNI_RDE_CTL_MPS | MPNIC_RNI_RDE_CTL_CLS);
val |= FIELD_PREP(MPNIC_RNI_RDE_CTL_MPS, mps) |
FIELD_PREP(MPNIC_RNI_RDE_CTL_CLS, cls);
mpnic_wr64(mpd, reg, val);
}
static void mpnic_mrrs_init(struct mpnic_dev *mpd, u32 reg, unsigned int mrrs,
unsigned int cls)
{
u64 val = mpnic_rd64(mpd, reg);
val &= ~(MPNIC_TNI_GLBL_TDF_CTL_MRRS | MPNIC_TNI_GLBL_TDF_CTL_CLS);
val |= FIELD_PREP(MPNIC_TNI_GLBL_TDF_CTL_MRRS, mrrs) |
FIELD_PREP(MPNIC_TNI_GLBL_TDF_CTL_CLS, cls);
mpnic_wr64(mpd, reg, val);
}
static void mpnic_axi_init(struct mpnic_dev *mpd)
{
int mps, cls, mrrs;
mps = clamp(ilog2(mpd->mps) - 7, 0, 3);
cls = clamp(ilog2(L1_CACHE_BYTES) - 6, 0, 3);
mpnic_mps_init(mpd, MPNIC_RNI_RDE_CTL, mps, cls);
mpnic_mps_init(mpd, MPNIC_RNI_RCM_CTL, mps, cls);
mpnic_mps_init(mpd, MPNIC_TNI_GLBL_TCM_CTL, mps, cls);
mrrs = clamp(ilog2(mpd->readrq) - 7, 0, 3);
mpnic_mrrs_init(mpd, MPNIC_RNI_RBP_CTL, mrrs, cls);
mpnic_mrrs_init(mpd, MPNIC_TNI_GLBL_TDF_CTL, mrrs, cls);
mrrs = clamp(ilog2(mpd->readrq) - 7, 0, 5);
mpnic_mrrs_init(mpd, MPNIC_TNI_GLBL_TDE_CTL, mrrs, cls);
}
static void mpnic_ro_init(struct mpnic_dev *mpd)
{
u64 attr = mpd->relaxed_ord ? MPNIC_OB_ATTR_RO : 0;
mpnic_wr64(mpd, MPNIC_OB_ATTR_TDE_H, attr);
mpnic_wr64(mpd, MPNIC_OB_ATTR_TDE_P, attr);
mpnic_wr64(mpd, MPNIC_OB_ATTR_TDF, attr);
mpnic_wr64(mpd, MPNIC_OB_ATTR_RBP_HPQ, attr);
mpnic_wr64(mpd, MPNIC_OB_ATTR_RBP_PPQ, attr);
mpnic_wr64(mpd, MPNIC_OB_ATTR_RDE_H, attr);
mpnic_wr64(mpd, MPNIC_OB_ATTR_RDE_P, attr);
}
static bool mpnic_init_status_ready(struct mpnic_dev *mpd,
const struct mpnic_init_poll *polls,
struct mpnic_poll_state *state)
{
u64 val;
int i;
for (i = state->poll_idx; polls[i].addr; i++) {
val = mpnic_rd64(mpd, polls[i].addr);
if ((val & polls[i].exp_val) != polls[i].exp_val) {
state->poll_idx = i;
state->val = val;
return false;
}
}
return true;
}
static int mpnic_mem_init_poll(struct mpnic_dev *mpd)
{
static const struct mpnic_init_poll polls[] = {
{ MPNIC_TQS_ARB_INIT_VAL, MPNIC_TQS_SRAM_STS },
{ MPNIC_MEM_INIT_BDQ_VAL, MPNIC_BDQ_MEM_INIT_DONE },
{ MPNIC_MEM_INIT_RCM_VAL, MPNIC_RCM_MEM_INIT_DONE },
{ MPNIC_MEM_INIT_RDE_VAL, MPNIC_RDE_MEM_INIT_DONE },
{ MPNIC_MEM_INIT_RPC_VAL, MPNIC_RPC_MEM_INIT_DONE },
{ MPNIC_MEM_INIT_TCM_VAL, MPNIC_TCM_MEM_INIT_DONE },
{ MPNIC_MEM_INIT_TDE_VAL, MPNIC_TDE_MEM_INIT_DONE },
{ MPNIC_MEM_INIT_TDF_VAL, MPNIC_TDF_MEM_INIT_DONE },
{ MPNIC_MEM_INIT_RXB_VAL, MPNIC_RXB_MEM_INIT_DONE },
{ 0 },
};
struct mpnic_poll_state state = {};
bool done;
int err;
err = read_poll_timeout(mpnic_init_status_ready, done, done,
MPNIC_MEM_INIT_POLL_US, MPNIC_MEM_INIT_TO_US,
false, mpd, polls, &state);
if (err)
dev_err(mpd->dev, "Poll timeout for reg 0x%x: 0x%llx\n",
polls[state.poll_idx].addr, state.val);
return err;
}
int mpnic_dev_init(struct mpnic_dev *mpd)
{
int err;
mpnic_tdf_glbl_init(mpd);
mpnic_txb_init(mpd);
mpnic_rxb_init(mpd);
mpnic_desc_sram_init(mpd);
mpnic_axi_init(mpd);
mpnic_ro_init(mpd);
mpnic_rxglb_init(mpd);
mpnic_txglb_init(mpd);
mpnic_tqs_init(mpd);
err = mpnic_mem_init_poll(mpd);
if (err) {
dev_err(mpd->dev, "Device initialization failed: %d\n", err);
return err;
}
if (!mpnic_present(mpd))
return -EIO;
return 0;
}