#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/iopoll.h>
#include <linux/etherdevice.h>
#include <linux/vmalloc.h>
#include <net/netdev_queues.h>
#include <net/page_pool/helpers.h>
#include "rnpgbe_lib.h"
#include "rnpgbe.h"
#include "rnpgbe_mbx_fw.h"
static void rnpgbe_mbx_work(struct work_struct *work)
{
struct mucse *mucse = container_of(work, struct mucse, mbx_work);
int irq_seq;
irq_seq = atomic_read(&mucse->mbx_irq_seq);
mucse_fw_irq_handler(&mucse->hw);
if (irq_seq != atomic_read(&mucse->mbx_irq_seq))
queue_work(system_percpu_wq, &mucse->mbx_work);
}
static irqreturn_t rnpgbe_msix_other(int irq, void *data)
{
struct mucse *mucse = (struct mucse *)data;
atomic_inc(&mucse->mbx_irq_seq);
queue_work(system_percpu_wq, &mucse->mbx_work);
return IRQ_HANDLED;
}
static void rnpgbe_irq_disable_queues(struct mucse_q_vector *q_vector)
{
struct mucse_ring *ring;
mucse_for_each_ring(ring, q_vector->tx) {
writel(INT_VALID, ring->trig);
writel((RX_INT_MASK | TX_INT_MASK), ring->irq_mask);
}
if (q_vector->tx.ring)
readl(q_vector->tx.ring->irq_mask);
}
static void rnpgbe_irq_enable_queues(struct mucse_q_vector *q_vector)
{
struct mucse_ring *ring;
mucse_for_each_ring(ring, q_vector->tx) {
writel(0, ring->irq_mask);
writel(INT_VALID | TX_INT_MASK | RX_INT_MASK, ring->trig);
}
}
static bool rnpgbe_clean_tx_irq(struct mucse_q_vector *q_vector,
struct mucse_ring *tx_ring,
int napi_budget)
{
int budget = q_vector->mucse->tx_work_limit;
u64 total_bytes = 0, total_packets = 0;
struct mucse *mucse = q_vector->mucse;
struct mucse_tx_buffer *tx_buffer;
struct rnpgbe_tx_desc *tx_desc;
int i = tx_ring->next_to_clean;
tx_buffer = &tx_ring->tx_buffer_info[i];
tx_desc = M_TX_DESC(tx_ring, i);
i -= tx_ring->count;
do {
struct rnpgbe_tx_desc *eop_desc = tx_buffer->next_to_watch;
if (!eop_desc)
break;
dma_rmb();
if (!(eop_desc->vlan_cmd & cpu_to_le32(M_TXD_STAT_DD)))
break;
tx_buffer->next_to_watch = NULL;
total_bytes += tx_buffer->bytecount;
total_packets += tx_buffer->gso_segs;
napi_consume_skb(tx_buffer->skb, napi_budget);
if (tx_buffer->mapped_as_page) {
dma_unmap_page(tx_ring->dev,
dma_unmap_addr(tx_buffer, dma),
dma_unmap_len(tx_buffer, len),
DMA_TO_DEVICE);
} else {
dma_unmap_single(tx_ring->dev,
dma_unmap_addr(tx_buffer, dma),
dma_unmap_len(tx_buffer, len),
DMA_TO_DEVICE);
}
tx_buffer->skb = NULL;
dma_unmap_len_set(tx_buffer, len, 0);
while (tx_desc != eop_desc) {
tx_buffer++;
tx_desc++;
i++;
if (unlikely(!i)) {
i -= tx_ring->count;
tx_buffer = tx_ring->tx_buffer_info;
tx_desc = M_TX_DESC(tx_ring, 0);
}
if (dma_unmap_len(tx_buffer, len)) {
dma_unmap_page(tx_ring->dev,
dma_unmap_addr(tx_buffer, dma),
dma_unmap_len(tx_buffer, len),
DMA_TO_DEVICE);
dma_unmap_len_set(tx_buffer, len, 0);
}
}
tx_buffer++;
tx_desc++;
i++;
if (unlikely(!i)) {
i -= tx_ring->count;
tx_buffer = tx_ring->tx_buffer_info;
tx_desc = M_TX_DESC(tx_ring, 0);
}
prefetch(tx_desc);
budget--;
} while (likely(budget > 0));
i += tx_ring->count;
tx_ring->next_to_clean = i;
u64_stats_update_begin(&tx_ring->stats->syncp);
tx_ring->stats->bytes += total_bytes;
tx_ring->stats->packets += total_packets;
u64_stats_update_end(&tx_ring->stats->syncp);
#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
__netif_txq_completed_wake(txring_txq(tx_ring),
total_packets, total_bytes,
mucse_desc_unused(tx_ring),
TX_WAKE_THRESHOLD,
!netif_running(mucse->netdev));
return !!budget;
}
static bool mucse_alloc_mapped_page(struct mucse_ring *rx_ring,
struct mucse_rx_buffer *bi)
{
struct page *page = bi->page;
dma_addr_t dma;
if (page) {
dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
bi->page_offset,
RNPGBE_RX_DESC_DATA_LEN,
DMA_FROM_DEVICE);
return true;
}
page = page_pool_dev_alloc_pages(rx_ring->page_pool);
if (unlikely(!page))
return false;
dma = page_pool_get_dma_addr(page);
bi->dma = dma;
bi->page = page;
bi->page_offset = RNPGBE_SKB_PAD;
return true;
}
static void mucse_update_rx_tail(struct mucse_ring *rx_ring,
u32 val)
{
rx_ring->next_to_use = val;
writel(val, rx_ring->tail);
}
static void rnpgbe_alloc_rx_buffers(struct mucse_ring *rx_ring,
u16 cleaned_count)
{
u64 fun_id = ((u64)(rx_ring->pfvfnum) << 56);
union rnpgbe_rx_desc *rx_desc;
u16 i = rx_ring->next_to_use;
struct mucse_rx_buffer *bi;
u64 addr;
if (!cleaned_count)
return;
rx_desc = M_RX_DESC(rx_ring, i);
bi = &rx_ring->rx_buffer_info[i];
i -= rx_ring->count;
do {
if (!mucse_alloc_mapped_page(rx_ring, bi))
break;
addr = (u64)(bi->dma + bi->page_offset);
rx_desc->pkt_addr = cpu_to_le64(addr | fun_id);
rx_desc->resv_cmd = 0;
rx_desc++;
bi++;
i++;
if (unlikely(!i)) {
rx_desc = M_RX_DESC(rx_ring, 0);
bi = rx_ring->rx_buffer_info;
i -= rx_ring->count;
}
cleaned_count--;
} while (cleaned_count);
i += rx_ring->count;
dma_wmb();
mucse_update_rx_tail(rx_ring, i);
}
static struct mucse_rx_buffer *rnpgbe_get_buffer(struct mucse_ring *rx_ring,
struct sk_buff **skb,
const unsigned int size)
{
struct mucse_rx_buffer *rx_buffer;
rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
*skb = rx_buffer->skb;
prefetchw(page_address(rx_buffer->page) + rx_buffer->page_offset);
dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma,
rx_buffer->page_offset, size,
DMA_FROM_DEVICE);
return rx_buffer;
}
static void rnpgbe_add_rx_frag(struct mucse_rx_buffer *rx_buffer,
struct sk_buff *skb,
unsigned int size)
{
unsigned int truesize = PAGE_SIZE;
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
rx_buffer->page_offset, size, truesize);
}
static struct sk_buff *rnpgbe_build_skb(struct mucse_rx_buffer *rx_buffer,
unsigned int size)
{
void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
unsigned int truesize = PAGE_SIZE;
struct sk_buff *skb;
net_prefetch(va);
skb = napi_build_skb(va - RNPGBE_SKB_PAD, truesize);
if (unlikely(!skb))
return NULL;
skb_reserve(skb, RNPGBE_SKB_PAD);
__skb_put(skb, size);
skb_mark_for_recycle(skb);
return skb;
}
static bool rnpgbe_is_non_eop(struct mucse_ring *rx_ring,
union rnpgbe_rx_desc *rx_desc,
struct sk_buff *skb)
{
u32 ntc = rx_ring->next_to_clean + 1;
ntc = (ntc < rx_ring->count) ? ntc : 0;
rx_ring->next_to_clean = ntc;
prefetch(M_RX_DESC(rx_ring, ntc));
if (likely(rnpgbe_test_staterr(rx_desc, M_RXD_STAT_EOP)))
return false;
if (skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS) {
rx_ring->rx_buffer_info[ntc].skb = skb;
} else {
rx_ring->drop_bytes = skb->len;
dev_kfree_skb_any(skb);
rx_ring->drop_reason = MUCSE_RX_DROP_RESOURCE;
}
rx_desc->wb.cmd = 0;
return true;
}
static bool rnpgbe_cleanup_headers(struct sk_buff *skb)
{
if (unlikely(!pskb_may_pull(skb, ETH_HLEN))) {
dev_kfree_skb_any(skb);
return true;
}
if (eth_skb_pad(skb))
return true;
return false;
}
static void rnpgbe_process_skb_fields(struct mucse_ring *rx_ring,
struct sk_buff *skb)
{
struct net_device *dev = rx_ring->netdev;
skb_record_rx_queue(skb, rx_ring->queue_index);
skb->protocol = eth_type_trans(skb, dev);
}
static void rnpgbe_rx_alloc_retry(struct timer_list *timer)
{
struct mucse_q_vector *q_vector =
timer_container_of(q_vector, timer, rx_alloc_timer);
napi_schedule(&q_vector->napi);
}
static bool rnpgbe_rx_refill_pending(struct mucse_q_vector *q_vector)
{
struct mucse_ring *ring;
mucse_for_each_ring(ring, q_vector->rx) {
if (mucse_desc_unused_rx(ring) >= M_RX_BUFFER_WRITE)
return true;
}
return false;
}
static void rnpgbe_schedule_rx_retry(struct mucse *mucse)
{
struct mucse_q_vector *q_vector;
for (int i = 0; i < mucse->num_q_vectors; i++) {
q_vector = mucse->q_vector[i];
if (rnpgbe_rx_refill_pending(q_vector))
mod_timer(&q_vector->rx_alloc_timer,
jiffies + msecs_to_jiffies(500));
}
}
static int rnpgbe_clean_rx_irq(struct mucse_q_vector *q_vector,
struct mucse_ring *rx_ring,
int budget)
{
unsigned int total_rx_bytes = 0, total_rx_packets = 0;
u16 cleaned_count = mucse_desc_unused_rx(rx_ring);
unsigned int total_rx_length_errors = 0;
unsigned int total_rx_dropped = 0;
unsigned int work_done = 0;
while (likely(work_done < budget)) {
struct mucse_rx_buffer *rx_buffer;
union rnpgbe_rx_desc *rx_desc;
struct sk_buff *skb;
unsigned int size;
if (cleaned_count >= M_RX_BUFFER_WRITE) {
rnpgbe_alloc_rx_buffers(rx_ring, cleaned_count);
cleaned_count = 0;
}
rx_desc = M_RX_DESC(rx_ring, rx_ring->next_to_clean);
if (!rnpgbe_test_staterr(rx_desc, M_RXD_STAT_DD))
break;
dma_rmb();
size = le16_to_cpu(rx_desc->wb.len);
if (unlikely(!size || size > RNPGBE_RX_DESC_DATA_LEN)) {
struct mucse_rx_buffer *err_rx_buffer;
u16 idx = rx_ring->next_to_clean;
cleaned_count++;
rx_ring->drop_reason = MUCSE_RX_DROP_LENGTH;
rx_ring->drop_bytes = 0;
err_rx_buffer = &rx_ring->rx_buffer_info[idx];
if (unlikely(err_rx_buffer->skb)) {
dev_kfree_skb_any(err_rx_buffer->skb);
err_rx_buffer->skb = NULL;
}
if (rnpgbe_test_staterr(rx_desc, M_RXD_STAT_EOP)) {
total_rx_length_errors++;
rx_ring->drop_reason = MUCSE_RX_DROP_NONE;
work_done++;
}
rx_desc->wb.cmd = 0;
rx_ring->next_to_clean++;
if (rx_ring->next_to_clean >= rx_ring->count)
rx_ring->next_to_clean = 0;
continue;
}
if (unlikely(rx_ring->drop_reason != MUCSE_RX_DROP_NONE)) {
cleaned_count++;
if (rx_ring->drop_reason == MUCSE_RX_DROP_RESOURCE)
rx_ring->drop_bytes += size;
if (rnpgbe_test_staterr(rx_desc, M_RXD_STAT_EOP)) {
if (rx_ring->drop_reason ==
MUCSE_RX_DROP_LENGTH) {
total_rx_length_errors++;
} else {
total_rx_bytes += rx_ring->drop_bytes;
total_rx_packets++;
total_rx_dropped++;
}
rx_ring->drop_reason = MUCSE_RX_DROP_NONE;
rx_ring->drop_bytes = 0;
work_done++;
}
rx_desc->wb.cmd = 0;
rx_ring->next_to_clean++;
if (rx_ring->next_to_clean >= rx_ring->count)
rx_ring->next_to_clean = 0;
continue;
}
rx_buffer = rnpgbe_get_buffer(rx_ring, &skb, size);
if (skb)
rnpgbe_add_rx_frag(rx_buffer, skb, size);
else
skb = rnpgbe_build_skb(rx_buffer, size);
if (!skb) {
cleaned_count++;
if (rnpgbe_test_staterr(rx_desc, M_RXD_STAT_EOP)) {
total_rx_bytes += size;
total_rx_packets++;
total_rx_dropped++;
work_done++;
} else {
rx_ring->drop_bytes = size;
rx_ring->drop_reason = MUCSE_RX_DROP_RESOURCE;
}
rx_desc->wb.cmd = 0;
rx_ring->next_to_clean++;
if (rx_ring->next_to_clean >= rx_ring->count)
rx_ring->next_to_clean = 0;
continue;
}
rx_buffer->page = NULL;
rx_buffer->skb = NULL;
cleaned_count++;
if (rnpgbe_is_non_eop(rx_ring, rx_desc, skb))
continue;
size = skb->len;
if (rnpgbe_cleanup_headers(skb)) {
if (size >= ETH_HLEN) {
total_rx_bytes += size;
total_rx_packets++;
}
total_rx_dropped++;
rx_desc->wb.cmd = 0;
work_done++;
continue;
}
total_rx_bytes += skb->len;
rnpgbe_process_skb_fields(rx_ring, skb);
rx_desc->wb.cmd = 0;
napi_gro_receive(&q_vector->napi, skb);
total_rx_packets++;
work_done++;
}
u64_stats_update_begin(&rx_ring->stats->syncp);
rx_ring->stats->packets += total_rx_packets;
rx_ring->stats->bytes += total_rx_bytes;
rx_ring->stats->length_errors += total_rx_length_errors;
atomic64_add(total_rx_dropped, &rx_ring->stats->dropped);
u64_stats_update_end(&rx_ring->stats->syncp);
return work_done;
}
static int rnpgbe_poll(struct napi_struct *napi, int budget)
{
struct mucse_q_vector *q_vector =
container_of(napi, struct mucse_q_vector, napi);
bool clean_complete = true;
struct mucse_ring *ring;
bool refill_pending;
int per_ring_budget;
int work_done = 0;
mucse_for_each_ring(ring, q_vector->tx) {
if (!rnpgbe_clean_tx_irq(q_vector, ring, budget))
clean_complete = false;
}
if (unlikely(!budget))
return 0;
if (q_vector->rx.count > 1)
per_ring_budget = max(budget / q_vector->rx.count, 1);
else
per_ring_budget = budget;
mucse_for_each_ring(ring, q_vector->rx) {
int cleaned = 0;
cleaned = rnpgbe_clean_rx_irq(q_vector, ring, per_ring_budget);
work_done += cleaned;
if (cleaned >= per_ring_budget)
clean_complete = false;
}
if (!clean_complete)
return budget;
refill_pending = rnpgbe_rx_refill_pending(q_vector);
if (likely(napi_complete_done(napi, work_done))) {
if (refill_pending)
mod_timer(&q_vector->rx_alloc_timer,
jiffies + msecs_to_jiffies(500));
rnpgbe_irq_enable_queues(q_vector);
}
return work_done;
}
int rnpgbe_request_mbx_irq(struct mucse *mucse)
{
struct pci_dev *pdev = mucse->pdev;
int err = 0;
snprintf(mucse->mbx_name, sizeof(mucse->mbx_name),
"rnpgbe-mbx:%s", pci_name(pdev));
INIT_WORK(&mucse->mbx_work, rnpgbe_mbx_work);
atomic_set(&mucse->mbx_irq_seq, 0);
err = request_irq(pci_irq_vector(pdev, 0), rnpgbe_msix_other, 0,
mucse->mbx_name, mucse);
if (!err)
mucse->mbx_irq_requested = true;
return err;
}
void rnpgbe_free_mbx_irq(struct mucse *mucse)
{
struct pci_dev *pdev = mucse->pdev;
if (mucse->mbx_irq_requested) {
mucse->mbx_irq_requested = false;
free_irq(pci_irq_vector(pdev, 0), mucse);
cancel_work_sync(&mucse->mbx_work);
}
cancel_delayed_work_sync(&mucse->serv_task);
}
static void rnpgbe_set_num_queues(struct mucse *mucse)
{
mucse->num_tx_queues = 1;
mucse->num_rx_queues = 1;
}
static int rnpgbe_set_interrupt_capability(struct mucse *mucse)
{
int v_budget;
v_budget = min3(mucse->num_tx_queues, mucse->num_rx_queues,
MAX_Q_VECTORS);
v_budget = min_t(int, v_budget, num_online_cpus());
v_budget += 1;
v_budget = pci_alloc_irq_vectors(mucse->pdev, 2, v_budget,
PCI_IRQ_MSIX);
if (v_budget < 0)
return v_budget;
mucse->num_q_vectors = v_budget - 1;
return 0;
}
static void mucse_add_ring(struct mucse_ring *ring,
struct mucse_ring_container *head)
{
ring->next = head->ring;
head->ring = ring;
head->count++;
}
static int rnpgbe_alloc_q_vector(struct mucse *mucse,
int eth_queue_idx, int vector_idx, int r_idx,
int r_count, int step)
{
int rxr_idx = r_idx, txr_idx = r_idx;
struct mucse_hw *hw = &mucse->hw;
struct mucse_q_vector *q_vector;
int txr_count, rxr_count, idx;
struct mucse_ring *ring;
int ring_count;
txr_count = r_count;
rxr_count = r_count;
ring_count = txr_count + rxr_count;
q_vector = kzalloc_flex(*q_vector, ring, ring_count);
if (!q_vector)
return -ENOMEM;
netif_napi_add(mucse->netdev, &q_vector->napi, rnpgbe_poll);
mucse->q_vector[vector_idx] = q_vector;
q_vector->mucse = mucse;
timer_setup(&q_vector->rx_alloc_timer, rnpgbe_rx_alloc_retry, 0);
q_vector->hw_vector = vector_idx;
q_vector->hw_vector++;
ring = q_vector->ring;
for (idx = 0; idx < txr_count; idx++) {
ring->dev = &mucse->pdev->dev;
mucse_add_ring(ring, &q_vector->tx);
ring->count = mucse->tx_ring_item_count;
ring->netdev = mucse->netdev;
ring->queue_index = eth_queue_idx + idx;
ring->rnpgbe_queue_idx = txr_idx;
ring->ring_addr = hw->hw_addr + RING_OFFSET(txr_idx);
ring->irq_mask = ring->ring_addr + RNPGBE_DMA_INT_MASK;
ring->trig = ring->ring_addr + RNPGBE_DMA_INT_TRIG;
ring->q_vector = q_vector;
ring->pfvfnum = hw->pfvfnum;
ring->stats = &mucse->tx_stats[ring->queue_index];
mucse->tx_ring[ring->queue_index] = ring;
txr_idx += step;
ring++;
}
for (idx = 0; idx < rxr_count; idx++) {
ring->dev = &mucse->pdev->dev;
mucse_add_ring(ring, &q_vector->rx);
ring->count = mucse->rx_ring_item_count;
ring->netdev = mucse->netdev;
ring->queue_index = eth_queue_idx + idx;
ring->rnpgbe_queue_idx = rxr_idx;
ring->ring_addr = hw->hw_addr + RING_OFFSET(rxr_idx);
ring->irq_mask = ring->ring_addr + RNPGBE_DMA_INT_MASK;
ring->trig = ring->ring_addr + RNPGBE_DMA_INT_TRIG;
ring->q_vector = q_vector;
ring->pfvfnum = hw->pfvfnum;
ring->stats = &mucse->rx_stats[ring->queue_index];
mucse->rx_ring[ring->queue_index] = ring;
rxr_idx += step;
ring++;
}
return 0;
}
static void rnpgbe_free_q_vector(struct mucse *mucse, int vector_idx)
{
struct mucse_q_vector *q_vector = mucse->q_vector[vector_idx];
struct mucse_ring *ring;
timer_delete_sync(&q_vector->rx_alloc_timer);
mucse_for_each_ring(ring, q_vector->tx)
mucse->tx_ring[ring->queue_index] = NULL;
mucse_for_each_ring(ring, q_vector->rx)
mucse->rx_ring[ring->queue_index] = NULL;
mucse->q_vector[vector_idx] = NULL;
netif_napi_del(&q_vector->napi);
kfree(q_vector);
}
static int rnpgbe_alloc_q_vectors(struct mucse *mucse)
{
int err, ring_cnt, v_remaining = mucse->num_q_vectors;
int r_remaining = min_t(int, mucse->num_tx_queues,
mucse->num_rx_queues);
int num_q_vectors = 0;
int eth_queue_idx = 0;
int vector_idx = 0;
int ring_step = 1;
int ring_idx = 0;
for (; r_remaining > 0 && v_remaining > 0; v_remaining--) {
ring_cnt = DIV_ROUND_UP(r_remaining, v_remaining);
err = rnpgbe_alloc_q_vector(mucse, eth_queue_idx,
vector_idx, ring_idx, ring_cnt,
ring_step);
if (err)
goto err_free_q_vector;
ring_idx += ring_step * ring_cnt;
eth_queue_idx += ring_cnt;
r_remaining -= ring_cnt;
num_q_vectors++;
vector_idx++;
}
mucse->num_q_vectors = num_q_vectors;
mucse->num_tx_queues = eth_queue_idx;
mucse->num_rx_queues = eth_queue_idx;
return 0;
err_free_q_vector:
mucse->num_tx_queues = 0;
mucse->num_rx_queues = 0;
mucse->num_q_vectors = 0;
while (vector_idx--)
rnpgbe_free_q_vector(mucse, vector_idx);
return err;
}
static void rnpgbe_reset_interrupt_capability(struct mucse *mucse)
{
if (!mucse->pdev->msix_enabled)
return;
pci_free_irq_vectors(mucse->pdev);
}
int rnpgbe_init_interrupt_scheme(struct mucse *mucse)
{
int err;
rnpgbe_set_num_queues(mucse);
err = rnpgbe_set_interrupt_capability(mucse);
if (err)
return err;
err = rnpgbe_alloc_q_vectors(mucse);
if (err) {
rnpgbe_reset_interrupt_capability(mucse);
return err;
}
return 0;
}
static void rnpgbe_free_q_vectors(struct mucse *mucse)
{
int vector_idx = mucse->num_q_vectors;
mucse->num_q_vectors = 0;
while (vector_idx--)
rnpgbe_free_q_vector(mucse, vector_idx);
}
void rnpgbe_clear_interrupt_scheme(struct mucse *mucse)
{
rnpgbe_free_q_vectors(mucse);
rnpgbe_reset_interrupt_capability(mucse);
}
static irqreturn_t rnpgbe_msix_clean_rings(int irq, void *data)
{
struct mucse_q_vector *q_vector = (struct mucse_q_vector *)data;
rnpgbe_irq_disable_queues(q_vector);
napi_schedule_irqoff(&q_vector->napi);
return IRQ_HANDLED;
}
int rnpgbe_request_irq(struct mucse *mucse)
{
struct net_device *netdev = mucse->netdev;
struct pci_dev *pdev = mucse->pdev;
struct mucse_q_vector *q_vector;
int err, i;
for (i = 0; i < mucse->num_q_vectors; i++) {
q_vector = mucse->q_vector[i];
snprintf(q_vector->name, sizeof(q_vector->name),
"%s-%s-%d", netdev->name, "TxRx", i);
err = request_irq(pci_irq_vector(pdev, i + 1),
rnpgbe_msix_clean_rings, 0,
q_vector->name, q_vector);
if (err) {
dev_err(&pdev->dev, "MSI-X req err %d: %d\n",
i + 1, err);
goto err_free_irqs;
}
}
return 0;
err_free_irqs:
while (i--) {
q_vector = mucse->q_vector[i];
synchronize_irq(pci_irq_vector(pdev, i + 1));
free_irq(pci_irq_vector(pdev, i + 1), q_vector);
}
return err;
}
void rnpgbe_free_irq(struct mucse *mucse)
{
struct pci_dev *pdev = mucse->pdev;
struct mucse_q_vector *q_vector;
for (int i = 0; i < mucse->num_q_vectors; i++) {
q_vector = mucse->q_vector[i];
if (!q_vector)
continue;
free_irq(pci_irq_vector(pdev, i + 1), q_vector);
}
}
static void rnpgbe_set_ring_vector(struct mucse *mucse,
u8 queue, u8 vector)
{
struct mucse_hw *hw = &mucse->hw;
u32 data;
data = hw->pfvfnum << 24;
data |= (vector << 8);
data |= vector;
writel(data, hw->ring_msix_base + RING_VECTOR(queue));
}
static void rnpgbe_configure_msix(struct mucse *mucse)
{
struct mucse_q_vector *q_vector;
for (int i = 0; i < mucse->num_q_vectors; i++) {
struct mucse_ring *ring;
q_vector = mucse->q_vector[i];
mucse_for_each_ring(ring, q_vector->tx) {
rnpgbe_set_ring_vector(mucse, ring->rnpgbe_queue_idx,
q_vector->hw_vector);
}
}
}
static void rnpgbe_irq_enable(struct mucse *mucse)
{
for (int i = 0; i < mucse->num_q_vectors; i++)
rnpgbe_irq_enable_queues(mucse->q_vector[i]);
}
static void rnpgbe_irq_disable(struct mucse *mucse)
{
struct pci_dev *pdev = mucse->pdev;
for (int i = 0; i < mucse->num_q_vectors; i++) {
rnpgbe_irq_disable_queues(mucse->q_vector[i]);
synchronize_irq(pci_irq_vector(pdev, i + 1));
}
}
static void rnpgbe_napi_enable_all(struct mucse *mucse)
{
for (int i = 0; i < mucse->num_q_vectors; i++)
napi_enable(&mucse->q_vector[i]->napi);
}
static void rnpgbe_napi_disable_all(struct mucse *mucse)
{
for (int i = 0; i < mucse->num_q_vectors; i++)
napi_disable(&mucse->q_vector[i]->napi);
}
static void rnpgbe_stop_tx_ring(struct mucse_ring *tx_ring)
{
if (!tx_ring->tx_buffer_info)
return;
mucse_ring_wr32(tx_ring, RNPGBE_TX_START, 0);
(void)mucse_ring_rd32(tx_ring, RNPGBE_TX_START);
}
static int rnpgbe_wait_tx_dma_idle(struct mucse *mucse)
{
struct mucse_hw *hw = &mucse->hw;
u32 dma_status;
int err;
err = readl_poll_timeout(hw->hw_addr + RNPGBE_DMA_STATUS,
dma_status,
(dma_status & RNPGBE_DMA_TX_STATUS) ==
RNPGBE_DMA_TX_STATUS,
10, 100000);
if (err) {
set_bit(__MUCSE_AXI_FAULT, &mucse->state);
dev_err(&mucse->pdev->dev,
"TX DMA failed to quiesce, status %#x\n", dma_status);
}
return err;
}
static int rnpgbe_wait_rx_dma_idle(struct mucse *mucse)
{
struct mucse_hw *hw = &mucse->hw;
u32 dma_status;
int err;
err = readl_poll_timeout(hw->hw_addr + RNPGBE_DMA_STATUS,
dma_status,
(dma_status & RNPGBE_DMA_RX_STATUS) ==
RNPGBE_DMA_RX_STATUS,
10, 100000);
if (err) {
set_bit(__MUCSE_AXI_FAULT, &mucse->state);
dev_err(&mucse->pdev->dev,
"RX DMA failed to quiesce, status %#x\n", dma_status);
}
return err;
}
static int rnpgbe_stop_all_tx_rings(struct mucse *mucse)
{
struct mucse_hw *hw = &mucse->hw;
u32 dma_axi_ctl;
int err = 0;
for (int i = 0; i < mucse->num_tx_queues; i++)
rnpgbe_stop_tx_ring(mucse->tx_ring[i]);
if (mucse->num_tx_queues &&
!test_bit(__MUCSE_AXI_FAULT, &mucse->state))
err = rnpgbe_wait_tx_dma_idle(mucse);
dma_axi_ctl = mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);
dma_axi_ctl &= ~TX_AXI_RW_EN;
mucse_hw_wr32(hw, RNPGBE_DMA_AXI_EN, dma_axi_ctl);
(void)mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);
if (!test_bit(__MUCSE_AXI_FAULT, &mucse->state))
return 0;
return err ? err : -EIO;
}
static void rnpgbe_clean_tx_ring(struct mucse_ring *tx_ring)
{
struct mucse_tx_buffer *tx_buffer;
u16 i = tx_ring->next_to_clean;
unsigned long size;
if (!tx_ring->tx_buffer_info)
return;
tx_buffer = &tx_ring->tx_buffer_info[i];
while (i != tx_ring->next_to_use) {
struct rnpgbe_tx_desc *eop_desc, *tx_desc;
dev_kfree_skb_any(tx_buffer->skb);
if (dma_unmap_len(tx_buffer, len)) {
if (tx_buffer->mapped_as_page) {
dma_unmap_page(tx_ring->dev,
dma_unmap_addr(tx_buffer, dma),
dma_unmap_len(tx_buffer, len),
DMA_TO_DEVICE);
} else {
dma_unmap_single(tx_ring->dev,
dma_unmap_addr(tx_buffer, dma),
dma_unmap_len(tx_buffer, len),
DMA_TO_DEVICE);
}
}
eop_desc = tx_buffer->next_to_watch;
tx_desc = M_TX_DESC(tx_ring, i);
while (tx_desc != eop_desc) {
tx_buffer++;
tx_desc++;
i++;
if (unlikely(i == tx_ring->count)) {
i = 0;
tx_buffer = tx_ring->tx_buffer_info;
tx_desc = M_TX_DESC(tx_ring, 0);
}
if (dma_unmap_len(tx_buffer, len))
dma_unmap_page(tx_ring->dev,
dma_unmap_addr(tx_buffer, dma),
dma_unmap_len(tx_buffer, len),
DMA_TO_DEVICE);
}
tx_buffer++;
i++;
if (unlikely(i == tx_ring->count)) {
i = 0;
tx_buffer = tx_ring->tx_buffer_info;
}
}
netdev_tx_reset_queue(txring_txq(tx_ring));
size = sizeof(struct mucse_tx_buffer) * tx_ring->count;
memset(tx_ring->tx_buffer_info, 0, size);
memset(tx_ring->desc, 0, tx_ring->size);
tx_ring->next_to_use = 0;
tx_ring->next_to_clean = 0;
}
void rnpgbe_clean_all_tx_rings(struct mucse *mucse)
{
rnpgbe_stop_all_tx_rings(mucse);
for (int i = 0; i < mucse->num_tx_queues; i++)
rnpgbe_clean_tx_ring(mucse->tx_ring[i]);
}
static void rnpgbe_stop_rx_ring(struct mucse_ring *rx_ring)
{
mucse_ring_wr32(rx_ring, RNPGBE_RX_START, 0);
(void)mucse_ring_rd32(rx_ring, RNPGBE_RX_START);
}
static int rnpgbe_stop_all_rx_rings(struct mucse *mucse)
{
struct mucse_hw *hw = &mucse->hw;
u32 dma_axi_ctl;
int err = 0;
for (int i = 0; i < mucse->num_rx_queues; i++)
rnpgbe_stop_rx_ring(mucse->rx_ring[i]);
if (mucse->num_rx_queues &&
!test_bit(__MUCSE_AXI_FAULT, &mucse->state))
err = rnpgbe_wait_rx_dma_idle(mucse);
dma_axi_ctl = mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);
dma_axi_ctl &= ~RX_AXI_RW_EN;
mucse_hw_wr32(hw, RNPGBE_DMA_AXI_EN, dma_axi_ctl);
(void)mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);
if (!test_bit(__MUCSE_AXI_FAULT, &mucse->state))
return 0;
return err ? err : -EIO;
}
static void rnpgbe_clean_rx_ring(struct mucse_ring *rx_ring)
{
struct mucse_rx_buffer *rx_buffer;
u16 i;
if (!rx_ring->rx_buffer_info)
return;
for (i = 0; i < rx_ring->count; i++) {
rx_buffer = &rx_ring->rx_buffer_info[i];
if (rx_buffer->skb) {
struct sk_buff *skb = rx_buffer->skb;
dev_kfree_skb(skb);
rx_buffer->skb = NULL;
}
if (rx_buffer->page) {
page_pool_put_full_page(rx_ring->page_pool,
rx_buffer->page, false);
rx_buffer->page = NULL;
}
}
rx_ring->next_to_clean = 0;
rx_ring->next_to_use = 0;
rx_ring->drop_bytes = 0;
rx_ring->drop_reason = MUCSE_RX_DROP_NONE;
}
static void rnpgbe_clean_all_rx_rings(struct mucse *mucse)
{
for (int i = 0; i < mucse->num_rx_queues; i++)
rnpgbe_clean_rx_ring(mucse->rx_ring[i]);
}
static void rnpgbe_cancel_rx_retry_timers(struct mucse *mucse)
{
for (int i = 0; i < mucse->num_q_vectors; i++)
timer_delete_sync(&mucse->q_vector[i]->rx_alloc_timer);
}
void rnpgbe_down(struct mucse *mucse)
{
struct net_device *netdev = mucse->netdev;
struct mucse_hw *hw = &mucse->hw;
unsigned long flags;
int err;
spin_lock_irqsave(&mucse->link_lock, flags);
mucse->link_event_ready = false;
mucse->link_pending = false;
hw->link = false;
hw->speed = 0;
hw->duplex = 0;
spin_unlock_irqrestore(&mucse->link_lock, flags);
cancel_delayed_work_sync(&mucse->serv_task);
netif_carrier_off(netdev);
rnpgbe_irq_disable(mucse);
rnpgbe_napi_disable_all(mucse);
synchronize_net();
rnpgbe_irq_disable(mucse);
netif_tx_disable(netdev);
rnpgbe_stop_all_rx_rings(mucse);
rnpgbe_cancel_rx_retry_timers(mucse);
rnpgbe_clean_all_tx_rings(mucse);
err = rnpgbe_send_notify(hw, false, mucse_fw_link_report_en);
if (err) {
dev_warn(&hw->pdev->dev, "Send link report to hw failed %d\n",
err);
dev_warn(&hw->pdev->dev, "Fw will still report link event\n");
}
err = rnpgbe_send_notify(hw, false, mucse_fw_portup);
if (err) {
dev_warn(&hw->pdev->dev, "Send port down to hw failed %d\n",
err);
dev_warn(&hw->pdev->dev, "Port is not truly down\n");
}
rnpgbe_clean_all_rx_rings(mucse);
}
int rnpgbe_up_complete(struct mucse *mucse)
{
struct net_device *netdev = mucse->netdev;
struct mucse_hw *hw = &mucse->hw;
unsigned long flags;
int err;
rnpgbe_configure_msix(mucse);
rnpgbe_napi_enable_all(mucse);
rnpgbe_schedule_rx_retry(mucse);
rnpgbe_irq_enable(mucse);
netif_tx_start_all_queues(netdev);
err = rnpgbe_send_notify(hw, true, mucse_fw_portup);
if (err) {
dev_err(&hw->pdev->dev,
"Failed to notify firmware that port is up: %d\n", err);
return err;
}
spin_lock_irqsave(&mucse->link_lock, flags);
mucse->link_pending = false;
hw->link = false;
hw->speed = 0;
hw->duplex = 0;
mucse_hw_wr32(hw, RNPGBE_LINK_ST, M_DEFAULT_ST);
mucse->link_event_ready = true;
spin_unlock_irqrestore(&mucse->link_lock, flags);
err = rnpgbe_send_notify(hw, true, mucse_fw_link_report_en);
if (err) {
dev_err(&hw->pdev->dev,
"Failed to enable firmware link reporting: %d\n",
err);
return err;
}
return 0;
}
static void rnpgbe_free_tx_resources(struct mucse_ring *tx_ring)
{
vfree(tx_ring->tx_buffer_info);
tx_ring->tx_buffer_info = NULL;
if (!tx_ring->desc)
return;
dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc,
tx_ring->dma);
tx_ring->desc = NULL;
}
static int rnpgbe_setup_tx_resources(struct mucse_ring *tx_ring)
{
struct device *dev = tx_ring->dev;
int size;
size = sizeof(struct mucse_tx_buffer) * tx_ring->count;
tx_ring->tx_buffer_info = vzalloc(size);
if (!tx_ring->tx_buffer_info)
goto err_return;
tx_ring->size = tx_ring->count * sizeof(struct rnpgbe_tx_desc);
tx_ring->size = ALIGN(tx_ring->size, 4096);
tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, &tx_ring->dma,
GFP_KERNEL);
if (!tx_ring->desc)
goto err_free_buffer;
tx_ring->next_to_use = 0;
tx_ring->next_to_clean = 0;
return 0;
err_free_buffer:
vfree(tx_ring->tx_buffer_info);
err_return:
tx_ring->tx_buffer_info = NULL;
return -ENOMEM;
}
static void rnpgbe_configure_tx_ring(struct mucse *mucse,
struct mucse_ring *ring)
{
struct mucse_hw *hw = &mucse->hw;
mucse_ring_wr32(ring, RNPGBE_TX_BASE_ADDR_LO, (u32)ring->dma);
mucse_ring_wr32(ring, RNPGBE_TX_BASE_ADDR_HI,
(u32)(((u64)ring->dma) >> 32) |
((u32)hw->pfvfnum << 24));
mucse_ring_wr32(ring, RNPGBE_TX_LEN, ring->count);
ring->next_to_clean = mucse_ring_rd32(ring, RNPGBE_TX_HEAD) %
ring->count;
ring->next_to_use = ring->next_to_clean;
ring->tail = ring->ring_addr + RNPGBE_TX_TAIL;
writel(ring->next_to_use, ring->tail);
mucse_ring_wr32(ring, RNPGBE_TX_FETCH_CTRL, M_DEFAULT_TX_FETCH);
mucse_ring_wr32(ring, RNPGBE_TX_INT_TIMER,
M_DEFAULT_INT_TIMER * hw->cycles_per_us);
mucse_ring_wr32(ring, RNPGBE_TX_INT_PKTCNT, M_DEFAULT_INT_PKTCNT);
}
int rnpgbe_configure_tx(struct mucse *mucse)
{
struct mucse_hw *hw = &mucse->hw;
u32 i, dma_axi_ctl;
int err;
err = rnpgbe_stop_all_tx_rings(mucse);
if (err)
return err;
for (i = 0; i < mucse->num_tx_queues; i++)
rnpgbe_configure_tx_ring(mucse, mucse->tx_ring[i]);
wmb();
dma_axi_ctl = mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);
dma_axi_ctl |= TX_AXI_RW_EN;
mucse_hw_wr32(hw, RNPGBE_DMA_AXI_EN, dma_axi_ctl);
(void)mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);
for (i = 0; i < mucse->num_tx_queues; i++)
mucse_ring_wr32(mucse->tx_ring[i], RNPGBE_TX_START, 1);
return 0;
}
int rnpgbe_setup_all_tx_resources(struct mucse *mucse)
{
int i, err = 0;
for (i = 0; i < mucse->num_tx_queues; i++) {
err = rnpgbe_setup_tx_resources(mucse->tx_ring[i]);
if (!err)
continue;
goto err_free_res;
}
return 0;
err_free_res:
while (i--)
rnpgbe_free_tx_resources(mucse->tx_ring[i]);
return err;
}
void rnpgbe_free_all_tx_resources(struct mucse *mucse)
{
for (int i = 0; i < (mucse->num_tx_queues); i++)
rnpgbe_free_tx_resources(mucse->tx_ring[i]);
}
static int rnpgbe_tx_map(struct mucse_ring *tx_ring,
struct mucse_tx_buffer *first, u32 mac_ip_len,
u32 tx_flags)
{
struct mucse_tx_buffer *tx_buffer;
struct rnpgbe_tx_desc *tx_desc;
struct skb_shared_info *shinfo;
unsigned int data_len, size;
struct sk_buff *skb;
skb_frag_t *frag;
dma_addr_t dma;
int nr_frags;
int frag_idx;
u64 fun_id;
u16 i;
skb = first->skb;
shinfo = skb_shinfo(skb);
fun_id = (u64)tx_ring->pfvfnum << 56;
nr_frags = shinfo->nr_frags;
i = tx_ring->next_to_use;
frag_idx = 0;
tx_desc = M_TX_DESC(tx_ring, i);
size = skb_headlen(skb);
data_len = skb->data_len;
if (size) {
dma = dma_map_single(tx_ring->dev, skb->data, size,
DMA_TO_DEVICE);
first->mapped_as_page = false;
} else if (data_len) {
while (frag_idx < nr_frags &&
!skb_frag_size(&shinfo->frags[frag_idx]))
frag_idx++;
if (frag_idx == nr_frags)
goto err_unmap;
frag = &shinfo->frags[frag_idx];
size = skb_frag_size(frag);
dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
size, DMA_TO_DEVICE);
first->mapped_as_page = true;
data_len -= size;
frag_idx++;
} else {
goto err_unmap;
}
tx_buffer = first;
dma_unmap_len_set(tx_buffer, len, 0);
dma_unmap_addr_set(tx_buffer, dma, 0);
for (;; frag_idx++) {
if (dma_mapping_error(tx_ring->dev, dma))
goto err_unmap;
dma_unmap_len_set(tx_buffer, len, size);
dma_unmap_addr_set(tx_buffer, dma, dma);
tx_desc->pkt_addr = cpu_to_le64(dma | fun_id);
while (unlikely(size > M_MAX_DATA_PER_TXD)) {
tx_desc->vlan_cmd_bsz = build_ctob(tx_flags,
mac_ip_len,
M_MAX_DATA_PER_TXD);
i++;
tx_desc++;
if (i == tx_ring->count) {
tx_desc = M_TX_DESC(tx_ring, 0);
i = 0;
}
tx_buffer = &tx_ring->tx_buffer_info[i];
dma_unmap_len_set(tx_buffer, len, 0);
dma += M_MAX_DATA_PER_TXD;
size -= M_MAX_DATA_PER_TXD;
tx_desc->pkt_addr = cpu_to_le64(dma | fun_id);
}
if (likely(!data_len))
break;
tx_desc->vlan_cmd_bsz = build_ctob(tx_flags, mac_ip_len, size);
i++;
tx_desc++;
if (i == tx_ring->count) {
tx_desc = M_TX_DESC(tx_ring, 0);
i = 0;
}
while (frag_idx < nr_frags &&
!skb_frag_size(&shinfo->frags[frag_idx]))
frag_idx++;
if (frag_idx == nr_frags)
goto err_unmap;
frag = &shinfo->frags[frag_idx];
size = skb_frag_size(frag);
data_len -= size;
dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
DMA_TO_DEVICE);
tx_buffer = &tx_ring->tx_buffer_info[i];
tx_buffer->mapped_as_page = true;
}
tx_desc->vlan_cmd_bsz = build_ctob(tx_flags | M_TXD_CMD_EOP |
M_TXD_CMD_RS,
mac_ip_len, size);
dma_wmb();
first->next_to_watch = tx_desc;
i++;
if (i == tx_ring->count)
i = 0;
tx_ring->next_to_use = i;
skb_tx_timestamp(skb);
netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
writel(i, tx_ring->tail);
return 0;
err_unmap:
for (;;) {
tx_buffer = &tx_ring->tx_buffer_info[i];
if (dma_unmap_len(tx_buffer, len)) {
if (tx_buffer->mapped_as_page) {
dma_unmap_page(tx_ring->dev,
dma_unmap_addr(tx_buffer, dma),
dma_unmap_len(tx_buffer, len),
DMA_TO_DEVICE);
} else {
dma_unmap_single(tx_ring->dev,
dma_unmap_addr(tx_buffer, dma),
dma_unmap_len(tx_buffer, len),
DMA_TO_DEVICE);
}
}
dma_unmap_len_set(tx_buffer, len, 0);
dma_unmap_addr_set(tx_buffer, dma, 0);
if (tx_buffer == first)
break;
if (i == 0)
i += tx_ring->count;
i--;
}
dev_kfree_skb_any(first->skb);
first->skb = NULL;
tx_ring->next_to_use = i;
return -ENOMEM;
}
netdev_tx_t rnpgbe_xmit_frame_ring(struct sk_buff *skb,
struct mucse_ring *tx_ring)
{
u32 mac_ip_len = M_DEFAULT_MAC_IP_LEN;
struct mucse_tx_buffer *first;
u32 tx_flags = 0;
unsigned short f;
u16 count;
count = TXD_USE_COUNT(skb_headlen(skb));
for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
skb_frag_t *frag_temp = &skb_shinfo(skb)->frags[f];
count += TXD_USE_COUNT(skb_frag_size(frag_temp));
}
if (!netif_txq_maybe_stop(txring_txq(tx_ring),
mucse_desc_unused(tx_ring),
count + RESV_DESC_NEEDED,
count + RESV_DESC_NEEDED))
return NETDEV_TX_BUSY;
first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
first->skb = skb;
first->bytecount = skb->len;
first->gso_segs = 1;
if (rnpgbe_tx_map(tx_ring, first, mac_ip_len, tx_flags)) {
atomic64_inc(&tx_ring->stats->dropped);
goto out;
}
netif_txq_maybe_stop(txring_txq(tx_ring),
mucse_desc_unused(tx_ring),
DESC_NEEDED, DESC_NEEDED);
out:
return NETDEV_TX_OK;
}
void rnpgbe_get_stats64(struct net_device *netdev,
struct rtnl_link_stats64 *stats)
{
struct mucse *mucse = netdev_priv(netdev);
int i;
for (i = 0; i < RNPGBE_MAX_QUEUES; i++) {
unsigned int start;
u64 bytes, packets, dropped;
struct mucse_queue_stats *queue_stats = &mucse->tx_stats[i];
do {
start = u64_stats_fetch_begin(&queue_stats->syncp);
packets = queue_stats->packets;
bytes = queue_stats->bytes;
} while (u64_stats_fetch_retry(&queue_stats->syncp, start));
dropped = atomic64_read(&queue_stats->dropped);
stats->tx_packets += packets;
stats->tx_dropped += dropped;
stats->tx_bytes += bytes;
}
for (i = 0; i < RNPGBE_MAX_QUEUES; i++) {
unsigned int start;
u64 bytes, packets, dropped, length_errors;
struct mucse_queue_stats *queue_stats = &mucse->rx_stats[i];
do {
start = u64_stats_fetch_begin(&queue_stats->syncp);
packets = queue_stats->packets;
bytes = queue_stats->bytes;
length_errors = queue_stats->length_errors;
} while (u64_stats_fetch_retry(&queue_stats->syncp, start));
dropped = atomic64_read(&queue_stats->dropped);
stats->rx_packets += packets;
stats->rx_dropped += dropped;
stats->rx_bytes += bytes;
stats->rx_errors += length_errors;
stats->rx_length_errors += length_errors;
}
}
static int mucse_alloc_page_pool(struct mucse_ring *rx_ring)
{
struct page_pool_params pp_params = {
.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
.order = 0,
.pool_size = rx_ring->count,
.nid = dev_to_node(rx_ring->dev),
.dev = rx_ring->dev,
.napi = &rx_ring->q_vector->napi,
.dma_dir = DMA_FROM_DEVICE,
.offset = RNPGBE_SKB_PAD,
.max_len = RNPGBE_RX_DESC_DATA_LEN,
.netdev = rx_ring->netdev,
.queue_idx = rx_ring->queue_index,
};
int ret = 0;
rx_ring->page_pool = page_pool_create(&pp_params);
if (IS_ERR(rx_ring->page_pool)) {
ret = PTR_ERR(rx_ring->page_pool);
rx_ring->page_pool = NULL;
}
return ret;
}
static int rnpgbe_setup_rx_resources(struct mucse_ring *rx_ring)
{
struct device *dev = rx_ring->dev;
int size;
size = sizeof(struct mucse_rx_buffer) * rx_ring->count;
rx_ring->rx_buffer_info = vzalloc(size);
if (!rx_ring->rx_buffer_info)
goto err_return;
rx_ring->size = rx_ring->count * sizeof(union rnpgbe_rx_desc);
rx_ring->size = ALIGN(rx_ring->size, 4096);
rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, &rx_ring->dma,
GFP_KERNEL);
if (!rx_ring->desc)
goto err_free_buffer;
rx_ring->next_to_clean = 0;
rx_ring->next_to_use = 0;
if (mucse_alloc_page_pool(rx_ring))
goto err_free_desc;
return 0;
err_free_desc:
dma_free_coherent(dev, rx_ring->size, rx_ring->desc,
rx_ring->dma);
rx_ring->desc = NULL;
err_free_buffer:
vfree(rx_ring->rx_buffer_info);
err_return:
rx_ring->rx_buffer_info = NULL;
return -ENOMEM;
}
static void rnpgbe_free_rx_resources(struct mucse_ring *rx_ring)
{
vfree(rx_ring->rx_buffer_info);
rx_ring->rx_buffer_info = NULL;
if (!rx_ring->desc)
return;
dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc,
rx_ring->dma);
rx_ring->desc = NULL;
if (rx_ring->page_pool) {
page_pool_destroy(rx_ring->page_pool);
rx_ring->page_pool = NULL;
}
}
int rnpgbe_setup_all_rx_resources(struct mucse *mucse)
{
int i, err = 0;
for (i = 0; i < mucse->num_rx_queues; i++) {
err = rnpgbe_setup_rx_resources(mucse->rx_ring[i]);
if (!err)
continue;
goto err_setup_rx;
}
return 0;
err_setup_rx:
while (i--)
rnpgbe_free_rx_resources(mucse->rx_ring[i]);
return err;
}
void rnpgbe_free_all_rx_resources(struct mucse *mucse)
{
for (int i = 0; i < mucse->num_rx_queues; i++)
rnpgbe_free_rx_resources(mucse->rx_ring[i]);
}
static void rnpgbe_configure_rx_ring(struct mucse *mucse,
struct mucse_ring *ring)
{
struct mucse_hw *hw = &mucse->hw;
mucse_ring_wr32(ring, RNPGBE_RX_BASE_ADDR_LO, (u32)ring->dma);
mucse_ring_wr32(ring, RNPGBE_RX_BASE_ADDR_HI,
(u32)((u64)ring->dma >> 32) |
((u32)hw->pfvfnum << 24));
mucse_ring_wr32(ring, RNPGBE_RX_LEN, ring->count);
ring->tail = ring->ring_addr + RNPGBE_RX_TAIL;
ring->next_to_clean = mucse_ring_rd32(ring, RNPGBE_RX_HEAD) %
ring->count;
ring->next_to_use = ring->next_to_clean;
ring->drop_bytes = 0;
ring->drop_reason = MUCSE_RX_DROP_NONE;
mucse_ring_wr32(ring, RNPGBE_RX_SG_LEN, M_DEFAULT_SG);
mucse_ring_wr32(ring, RNPGBE_RX_FETCH, M_DEFAULT_RX_FETCH);
mucse_ring_wr32(ring, RNPGBE_RX_TIMEOUT_TH, 0);
mucse_ring_wr32(ring, RNPGBE_RX_INT_TIMER,
M_DEFAULT_INT_TIMER_R * hw->cycles_per_us);
mucse_ring_wr32(ring, RNPGBE_RX_INT_PKTCNT, M_DEFAULT_RX_INT_PKTCNT);
rnpgbe_alloc_rx_buffers(ring, mucse_desc_unused_rx(ring));
}
int rnpgbe_configure_rx(struct mucse *mucse)
{
struct mucse_hw *hw = &mucse->hw;
u32 dma_axi_ctl;
int err;
err = rnpgbe_stop_all_rx_rings(mucse);
if (err)
return err;
for (int i = 0; i < mucse->num_rx_queues; i++)
rnpgbe_configure_rx_ring(mucse, mucse->rx_ring[i]);
wmb();
dma_axi_ctl = mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);
dma_axi_ctl |= RX_AXI_RW_EN;
mucse_hw_wr32(hw, RNPGBE_DMA_AXI_EN, dma_axi_ctl);
(void)mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);
for (int i = 0; i < mucse->num_rx_queues; i++)
mucse_ring_wr32(mucse->rx_ring[i], RNPGBE_RX_START, 1);
return 0;
}
static bool rnpgbe_process_link_event(struct mucse *mucse, bool *link,
int *speed, u8 *duplex)
{
struct mucse_hw *hw = &mucse->hw;
unsigned long flags;
spin_lock_irqsave(&mucse->link_lock, flags);
if (!mucse->link_event_ready || !mucse->link_pending) {
spin_unlock_irqrestore(&mucse->link_lock, flags);
return false;
}
mucse->link_pending = false;
*link = hw->link;
*speed = hw->speed;
*duplex = hw->duplex;
spin_unlock_irqrestore(&mucse->link_lock, flags);
return true;
}
static void rnpgbe_link_is_up(struct mucse *mucse, int speed, u8 duplex)
{
struct net_device *netdev = mucse->netdev;
if (netif_carrier_ok(netdev))
return;
netdev_info(netdev, "NIC Link is Up %d Mbps, %s Duplex\n",
speed, duplex ? "Full" : "Half");
netif_carrier_on(netdev);
}
static void rnpgbe_link_is_down(struct mucse *mucse)
{
struct net_device *netdev = mucse->netdev;
if (!netif_carrier_ok(netdev))
return;
netdev_info(netdev, "NIC Link is Down\n");
netif_carrier_off(netdev);
}
static void rnpgbe_process_link_subtask(struct mucse *mucse)
{
bool link;
int speed;
u8 duplex;
if (!rnpgbe_process_link_event(mucse, &link, &speed, &duplex))
return;
if (link)
rnpgbe_link_is_up(mucse, speed, duplex);
else
rnpgbe_link_is_down(mucse);
}
void rnpgbe_service_task(struct work_struct *work)
{
struct mucse *mucse = container_of(work, struct mucse, serv_task.work);
rnpgbe_process_link_subtask(mucse);
}