// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2020 - 2025 Mucse Corporation. */

#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/iopoll.h>
#include <linux/etherdevice.h>
#include <linux/vmalloc.h>
#include <net/netdev_queues.h>
#include <net/page_pool/helpers.h>

#include "rnpgbe_lib.h"
#include "rnpgbe.h"
#include "rnpgbe_mbx_fw.h"

static void rnpgbe_mbx_work(struct work_struct *work)
{
	struct mucse *mucse = container_of(work, struct mucse, mbx_work);
	int irq_seq;

	irq_seq = atomic_read(&mucse->mbx_irq_seq);
	mucse_fw_irq_handler(&mucse->hw);

	/* A mailbox interrupt while this work runs needs another invocation. */
	if (irq_seq != atomic_read(&mucse->mbx_irq_seq))
		queue_work(system_percpu_wq, &mucse->mbx_work);
}

/**
 * rnpgbe_msix_other - Other irq handler
 * @irq: interrupt number
 * @data: private data
 *
 * Return: IRQ_HANDLED
 **/
static irqreturn_t rnpgbe_msix_other(int irq, void *data)
{
	struct mucse *mucse = (struct mucse *)data;

	atomic_inc(&mucse->mbx_irq_seq);
	queue_work(system_percpu_wq, &mucse->mbx_work);

	return IRQ_HANDLED;
}

static void rnpgbe_irq_disable_queues(struct mucse_q_vector *q_vector)
{
	struct mucse_ring *ring;

	/*
	 * TX/RX pairs share interrupt control registers; update them through
	 * the TX ring list.
	 */
	mucse_for_each_ring(ring, q_vector->tx) {
		writel(INT_VALID, ring->trig);
		writel((RX_INT_MASK | TX_INT_MASK), ring->irq_mask);
	}
	/* flush posted writes to ensure hardware sees the mask */
	if (q_vector->tx.ring)
		readl(q_vector->tx.ring->irq_mask);
}

static void rnpgbe_irq_enable_queues(struct mucse_q_vector *q_vector)
{
	struct mucse_ring *ring;

	/*
	 * TX/RX pairs share interrupt control registers; update them through
	 * the TX ring list.
	 */
	mucse_for_each_ring(ring, q_vector->tx) {
		writel(0, ring->irq_mask);

		/* Re-trigger hw to re-check events lost while masked. */
		writel(INT_VALID | TX_INT_MASK | RX_INT_MASK, ring->trig);
	}
}

/**
 * rnpgbe_clean_tx_irq - Reclaim resources after transmit completes
 * @q_vector: structure containing interrupt and ring information
 * @tx_ring: tx ring to clean
 * @napi_budget: Used to determine if we are in netpoll
 *
 * Return: true if cleaning completed within the budget, otherwise false
 **/
static bool rnpgbe_clean_tx_irq(struct mucse_q_vector *q_vector,
				struct mucse_ring *tx_ring,
				int napi_budget)
{
	int budget = q_vector->mucse->tx_work_limit;
	u64 total_bytes = 0, total_packets = 0;
	struct mucse *mucse = q_vector->mucse;
	struct mucse_tx_buffer *tx_buffer;
	struct rnpgbe_tx_desc *tx_desc;
	int i = tx_ring->next_to_clean;

	tx_buffer = &tx_ring->tx_buffer_info[i];
	tx_desc = M_TX_DESC(tx_ring, i);
	i -= tx_ring->count;

	do {
		struct rnpgbe_tx_desc *eop_desc = tx_buffer->next_to_watch;

		/* if next_to_watch is not set then there is no work pending */
		if (!eop_desc)
			break;

		/* prevent any other reads prior to eop_desc */
		dma_rmb();

		/* if eop DD is not set pending work has not been completed */
		if (!(eop_desc->vlan_cmd & cpu_to_le32(M_TXD_STAT_DD)))
			break;
		/* clear next_to_watch to prevent false hangs */
		tx_buffer->next_to_watch = NULL;
		total_bytes += tx_buffer->bytecount;
		total_packets += tx_buffer->gso_segs;
		napi_consume_skb(tx_buffer->skb, napi_budget);
		if (tx_buffer->mapped_as_page) {
			dma_unmap_page(tx_ring->dev,
				       dma_unmap_addr(tx_buffer, dma),
				       dma_unmap_len(tx_buffer, len),
				       DMA_TO_DEVICE);
		} else {
			dma_unmap_single(tx_ring->dev,
					 dma_unmap_addr(tx_buffer, dma),
					 dma_unmap_len(tx_buffer, len),
					 DMA_TO_DEVICE);
		}
		tx_buffer->skb = NULL;
		dma_unmap_len_set(tx_buffer, len, 0);

		/* unmap remaining buffers */
		while (tx_desc != eop_desc) {
			tx_buffer++;
			tx_desc++;
			i++;
			if (unlikely(!i)) {
				i -= tx_ring->count;
				tx_buffer = tx_ring->tx_buffer_info;
				tx_desc = M_TX_DESC(tx_ring, 0);
			}

			/* unmap any remaining paged data */
			if (dma_unmap_len(tx_buffer, len)) {
				dma_unmap_page(tx_ring->dev,
					       dma_unmap_addr(tx_buffer, dma),
					       dma_unmap_len(tx_buffer, len),
					       DMA_TO_DEVICE);
				dma_unmap_len_set(tx_buffer, len, 0);
			}
		}

		/* move us one more past the eop_desc for start of next pkt */
		tx_buffer++;
		tx_desc++;
		i++;
		if (unlikely(!i)) {
			i -= tx_ring->count;
			tx_buffer = tx_ring->tx_buffer_info;
			tx_desc = M_TX_DESC(tx_ring, 0);
		}

		prefetch(tx_desc);
		budget--;
	} while (likely(budget > 0));

	i += tx_ring->count;
	tx_ring->next_to_clean = i;
	u64_stats_update_begin(&tx_ring->stats->syncp);
	tx_ring->stats->bytes += total_bytes;
	tx_ring->stats->packets += total_packets;
	u64_stats_update_end(&tx_ring->stats->syncp);

#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
	__netif_txq_completed_wake(txring_txq(tx_ring),
				   total_packets, total_bytes,
				   mucse_desc_unused(tx_ring),
				   TX_WAKE_THRESHOLD,
				   !netif_running(mucse->netdev));

	return !!budget;
}

static bool mucse_alloc_mapped_page(struct mucse_ring *rx_ring,
				    struct mucse_rx_buffer *bi)
{
	struct page *page = bi->page;
	dma_addr_t dma;

	if (page) {
		/* Buffer is being reused without going back through the
		 * page_pool. Do dma_sync for hw use.
		 */
		dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
						 bi->page_offset,
						 RNPGBE_RX_DESC_DATA_LEN,
						 DMA_FROM_DEVICE);
		return true;
	}

	page = page_pool_dev_alloc_pages(rx_ring->page_pool);
	if (unlikely(!page))
		return false;
	dma = page_pool_get_dma_addr(page);

	bi->dma = dma;
	bi->page = page;
	bi->page_offset = RNPGBE_SKB_PAD;

	return true;
}

static void mucse_update_rx_tail(struct mucse_ring *rx_ring,
				 u32 val)
{
	rx_ring->next_to_use = val;
	writel(val, rx_ring->tail);
}

/**
 * rnpgbe_alloc_rx_buffers - Replace used receive buffers
 * @rx_ring: ring to place buffers on
 * @cleaned_count: number of buffers to replace
 **/
static void rnpgbe_alloc_rx_buffers(struct mucse_ring *rx_ring,
				    u16 cleaned_count)
{
	u64 fun_id = ((u64)(rx_ring->pfvfnum) << 56);
	union rnpgbe_rx_desc *rx_desc;
	u16 i = rx_ring->next_to_use;
	struct mucse_rx_buffer *bi;
	u64 addr;
	/* nothing to do */
	if (!cleaned_count)
		return;

	rx_desc = M_RX_DESC(rx_ring, i);
	bi = &rx_ring->rx_buffer_info[i];
	i -= rx_ring->count;

	do {
		if (!mucse_alloc_mapped_page(rx_ring, bi))
			break;

		addr = (u64)(bi->dma + bi->page_offset);
		rx_desc->pkt_addr = cpu_to_le64(addr | fun_id);
		/* clean dd */
		rx_desc->resv_cmd = 0;
		rx_desc++;
		bi++;
		i++;
		if (unlikely(!i)) {
			rx_desc = M_RX_DESC(rx_ring, 0);
			bi = rx_ring->rx_buffer_info;
			i -= rx_ring->count;
		}
		cleaned_count--;
	} while (cleaned_count);

	i += rx_ring->count;

	/* Publish the tail even when allocation stopped early, so a stale
	 * tail from a previous session cannot point hardware at a
	 * reallocated ring.
	 */
	dma_wmb();
	mucse_update_rx_tail(rx_ring, i);
}

/**
 * rnpgbe_get_buffer - Get the rx_buffer to be used
 * @rx_ring: pointer to rx ring
 * @skb: pointer skb for this packet
 * @size: data size in this desc
 *
 * Return: rx_buffer.
 **/
static struct mucse_rx_buffer *rnpgbe_get_buffer(struct mucse_ring *rx_ring,
						 struct sk_buff **skb,
						 const unsigned int size)
{
	struct mucse_rx_buffer *rx_buffer;

	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
	*skb = rx_buffer->skb;
	prefetchw(page_address(rx_buffer->page) + rx_buffer->page_offset);
	/* we are reusing so sync this buffer for CPU use */
	dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma,
				      rx_buffer->page_offset, size,
				      DMA_FROM_DEVICE);

	return rx_buffer;
}

/**
 * rnpgbe_add_rx_frag - Add non-linear data to the skb
 * @rx_buffer: pointer to rx_buffer
 * @skb: pointer skb for this packet
 * @size: data size in this desc
 **/
static void rnpgbe_add_rx_frag(struct mucse_rx_buffer *rx_buffer,
			       struct sk_buff *skb,
			       unsigned int size)
{
	unsigned int truesize = PAGE_SIZE;

	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
			rx_buffer->page_offset, size, truesize);
}

/**
 * rnpgbe_build_skb - Try to build a skb based on rx_buffer
 * @rx_buffer: pointer to rx_buffer
 * @size: data size in this desc
 *
 * Return: skb for this rx_buffer
 **/
static struct sk_buff *rnpgbe_build_skb(struct mucse_rx_buffer *rx_buffer,
					unsigned int size)
{
	void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
	unsigned int truesize = PAGE_SIZE;
	struct sk_buff *skb;

	net_prefetch(va);
	/* build an skb around the page buffer */
	skb = napi_build_skb(va - RNPGBE_SKB_PAD, truesize);
	if (unlikely(!skb))
		return NULL;
	/* update pointers within the skb to store the data */
	skb_reserve(skb, RNPGBE_SKB_PAD);
	__skb_put(skb, size);
	skb_mark_for_recycle(skb);

	return skb;
}

/**
 * rnpgbe_is_non_eop - Process handling of non-EOP buffers
 * @rx_ring: rx ring being processed
 * @rx_desc: rx descriptor for current buffer
 * @skb: current socket buffer containing buffer in progress
 *
 * Advance next_to_clean. For a non-EOP descriptor, carry the skb into the
 * next descriptor unless the fragment limit has been reached. In that case,
 * free the skb and mark the remaining descriptors for drop.
 *
 * Return: true if the descriptor is not EOP. The skb may have been consumed
 * when true is returned.
 **/
static bool rnpgbe_is_non_eop(struct mucse_ring *rx_ring,
			      union rnpgbe_rx_desc *rx_desc,
			      struct sk_buff *skb)
{
	u32 ntc = rx_ring->next_to_clean + 1;

	/* fetch, update, and store next to clean */
	ntc = (ntc < rx_ring->count) ? ntc : 0;
	rx_ring->next_to_clean = ntc;
	prefetch(M_RX_DESC(rx_ring, ntc));
	/* if we are the last buffer then there is nothing else to do */
	if (likely(rnpgbe_test_staterr(rx_desc, M_RXD_STAT_EOP)))
		return false;
	if (skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS) {
		/* place skb in next buffer to be received */
		rx_ring->rx_buffer_info[ntc].skb = skb;
	} else {
		rx_ring->drop_bytes = skb->len;
		/* too much frags, force free */
		dev_kfree_skb_any(skb);
		rx_ring->drop_reason = MUCSE_RX_DROP_RESOURCE;
	}
	/* we should clean it since we used all info in it */
	rx_desc->wb.cmd = 0;

	return true;
}

/**
 * rnpgbe_cleanup_headers - Correct corrupted or empty headers
 * @skb: current socket buffer containing buffer in progress
 *
 * Return: true if an error was encountered and skb was freed.
 **/
static bool rnpgbe_cleanup_headers(struct sk_buff *skb)
{
	if (unlikely(!pskb_may_pull(skb, ETH_HLEN))) {
		dev_kfree_skb_any(skb);
		return true;
	}

	/* if eth_skb_pad returns an error the skb was freed */
	if (eth_skb_pad(skb))
		return true;

	return false;
}

/**
 * rnpgbe_process_skb_fields - Set the RX queue and protocol fields
 * @rx_ring: RX descriptor ring containing the queue information
 * @skb: skb currently being received
 *
 * Records the RX queue that received the skb and sets its protocol from
 * the Ethernet header.
 **/
static void rnpgbe_process_skb_fields(struct mucse_ring *rx_ring,
				      struct sk_buff *skb)
{
	struct net_device *dev = rx_ring->netdev;

	skb_record_rx_queue(skb, rx_ring->queue_index);
	skb->protocol = eth_type_trans(skb, dev);
}

/**
 * rnpgbe_rx_alloc_retry - Retry RX buffer allocation
 * @timer: RX allocation retry timer
 *
 * Schedules NAPI after RX buffer allocation fails.
 **/
static void rnpgbe_rx_alloc_retry(struct timer_list *timer)
{
	struct mucse_q_vector *q_vector =
		timer_container_of(q_vector, timer, rx_alloc_timer);

	napi_schedule(&q_vector->napi);
}

static bool rnpgbe_rx_refill_pending(struct mucse_q_vector *q_vector)
{
	struct mucse_ring *ring;

	mucse_for_each_ring(ring, q_vector->rx) {
		if (mucse_desc_unused_rx(ring) >= M_RX_BUFFER_WRITE)
			return true;
	}

	return false;
}

static void rnpgbe_schedule_rx_retry(struct mucse *mucse)
{
	struct mucse_q_vector *q_vector;

	for (int i = 0; i < mucse->num_q_vectors; i++) {
		q_vector = mucse->q_vector[i];
		if (rnpgbe_rx_refill_pending(q_vector))
			mod_timer(&q_vector->rx_alloc_timer,
				  jiffies + msecs_to_jiffies(500));
	}
}

/**
 * rnpgbe_clean_rx_irq - Clean completed descriptors from Rx ring
 * @q_vector: structure containing interrupt and ring information
 * @rx_ring: rx descriptor ring to transact packets on
 * @budget: total limit on number of packets to process
 *
 * rnpgbe_clean_rx_irq tries to check dd in desc, handle this desc
 * if dd is set which means data is write-back by hw
 *
 * Return: amount of work completed.
 **/
static int rnpgbe_clean_rx_irq(struct mucse_q_vector *q_vector,
			       struct mucse_ring *rx_ring,
			       int budget)
{
	unsigned int total_rx_bytes = 0, total_rx_packets = 0;
	u16 cleaned_count = mucse_desc_unused_rx(rx_ring);
	unsigned int total_rx_length_errors = 0;
	unsigned int total_rx_dropped = 0;
	unsigned int work_done = 0;

	while (likely(work_done < budget)) {
		struct mucse_rx_buffer *rx_buffer;
		union rnpgbe_rx_desc *rx_desc;
		struct sk_buff *skb;
		unsigned int size;

		if (cleaned_count >= M_RX_BUFFER_WRITE) {
			rnpgbe_alloc_rx_buffers(rx_ring, cleaned_count);
			cleaned_count = 0;
		}
		rx_desc = M_RX_DESC(rx_ring, rx_ring->next_to_clean);

		if (!rnpgbe_test_staterr(rx_desc, M_RXD_STAT_DD))
			break;

		/* This memory barrier is needed to keep us from reading
		 * any other fields out of the rx_desc until we know the
		 * descriptor has been written back
		 */
		dma_rmb();
		/* Hardware guarantees the first descriptor of each packet is at
		 * least 33 bytes, including multi-descriptor packets.
		 * Multi-descriptor packets are only used for jumbo frames over
		 * 1536 bytes (M_DEFAULT_SG = 96). Each descriptor is at most
		 * 1536 bytes. Small packets use a single descriptor.
		 */
		size = le16_to_cpu(rx_desc->wb.len);

		if (unlikely(!size || size > RNPGBE_RX_DESC_DATA_LEN)) {
			struct mucse_rx_buffer *err_rx_buffer;
			u16 idx = rx_ring->next_to_clean;

			cleaned_count++;
			/* Length errors override resource drops. */
			rx_ring->drop_reason = MUCSE_RX_DROP_LENGTH;
			rx_ring->drop_bytes = 0;

			/* Free the partial skb from a previous non-EOP
			 * descriptor before advancing next_to_clean.
			 */
			err_rx_buffer = &rx_ring->rx_buffer_info[idx];
			if (unlikely(err_rx_buffer->skb)) {
				dev_kfree_skb_any(err_rx_buffer->skb);
				err_rx_buffer->skb = NULL;
			}

			/* drop data until eop */
			if (rnpgbe_test_staterr(rx_desc, M_RXD_STAT_EOP)) {
				total_rx_length_errors++;
				rx_ring->drop_reason = MUCSE_RX_DROP_NONE;
				work_done++;
			}

			rx_desc->wb.cmd = 0;
			rx_ring->next_to_clean++;
			if (rx_ring->next_to_clean >= rx_ring->count)
				rx_ring->next_to_clean = 0;
			continue;
		}

		if (unlikely(rx_ring->drop_reason != MUCSE_RX_DROP_NONE)) {
			cleaned_count++;
			if (rx_ring->drop_reason == MUCSE_RX_DROP_RESOURCE)
				rx_ring->drop_bytes += size;

			/* Count the packet once at EOP, using its final drop
			 * reason even across NAPI polls.
			 */
			if (rnpgbe_test_staterr(rx_desc, M_RXD_STAT_EOP)) {
				if (rx_ring->drop_reason ==
				    MUCSE_RX_DROP_LENGTH) {
					total_rx_length_errors++;
				} else {
					total_rx_bytes += rx_ring->drop_bytes;
					total_rx_packets++;
					total_rx_dropped++;
				}
				rx_ring->drop_reason = MUCSE_RX_DROP_NONE;
				rx_ring->drop_bytes = 0;
				work_done++;
			}

			rx_desc->wb.cmd = 0;
			rx_ring->next_to_clean++;
			if (rx_ring->next_to_clean >= rx_ring->count)
				rx_ring->next_to_clean = 0;
			continue;
		}

		rx_buffer = rnpgbe_get_buffer(rx_ring, &skb, size);

		if (skb)
			rnpgbe_add_rx_frag(rx_buffer, skb, size);
		else
			skb = rnpgbe_build_skb(rx_buffer, size);

		if (!skb) {
			cleaned_count++;

			/* drop until eop if multiple descriptors */
			if (rnpgbe_test_staterr(rx_desc, M_RXD_STAT_EOP)) {
				total_rx_bytes += size;
				total_rx_packets++;
				total_rx_dropped++;
				work_done++;
			} else {
				rx_ring->drop_bytes = size;
				rx_ring->drop_reason = MUCSE_RX_DROP_RESOURCE;
			}

			rx_desc->wb.cmd = 0;
			rx_ring->next_to_clean++;
			if (rx_ring->next_to_clean >= rx_ring->count)
				rx_ring->next_to_clean = 0;

			continue;
		}

		rx_buffer->page = NULL;
		rx_buffer->skb = NULL;
		cleaned_count++;

		if (rnpgbe_is_non_eop(rx_ring, rx_desc, skb))
			continue;

		/* verify the packet layout is correct */
		size = skb->len;
		if (rnpgbe_cleanup_headers(skb)) {
			if (size >= ETH_HLEN) {
				total_rx_bytes += size;
				total_rx_packets++;
			}
			/* we should clean it since we used all info in it */
			total_rx_dropped++;
			rx_desc->wb.cmd = 0;
			work_done++;
			continue;
		}

		/* probably a little skewed due to removing CRC */
		total_rx_bytes += skb->len;
		rnpgbe_process_skb_fields(rx_ring, skb);
		rx_desc->wb.cmd = 0;
		napi_gro_receive(&q_vector->napi, skb);
		/* Update packet statistics and NAPI work accounting. */
		total_rx_packets++;
		work_done++;
	}

	u64_stats_update_begin(&rx_ring->stats->syncp);
	rx_ring->stats->packets += total_rx_packets;
	rx_ring->stats->bytes += total_rx_bytes;
	rx_ring->stats->length_errors += total_rx_length_errors;
	atomic64_add(total_rx_dropped, &rx_ring->stats->dropped);
	u64_stats_update_end(&rx_ring->stats->syncp);

	return work_done;
}

/**
 * rnpgbe_poll - NAPI polling callback
 * @napi: structure for representing this polling device
 * @budget: polling budget
 *
 * Clean completed TX and RX packets and complete NAPI polling when all
 * queues are clean.
 *
 * Return: number of RX packets processed, or @budget if polling continues
 **/
static int rnpgbe_poll(struct napi_struct *napi, int budget)
{
	struct mucse_q_vector *q_vector =
		container_of(napi, struct mucse_q_vector, napi);
	bool clean_complete = true;
	struct mucse_ring *ring;
	bool refill_pending;
	int per_ring_budget;
	int work_done = 0;

	mucse_for_each_ring(ring, q_vector->tx) {
		if (!rnpgbe_clean_tx_irq(q_vector, ring, budget))
			clean_complete = false;
	}

	/* Exit if we are called by netpoll */
	if (unlikely(!budget))
		return 0;

	if (q_vector->rx.count > 1)
		per_ring_budget = max(budget / q_vector->rx.count, 1);
	else
		per_ring_budget = budget;

	mucse_for_each_ring(ring, q_vector->rx) {
		int cleaned = 0;

		cleaned = rnpgbe_clean_rx_irq(q_vector, ring, per_ring_budget);
		work_done += cleaned;
		if (cleaned >= per_ring_budget)
			clean_complete = false;
	}

	if (!clean_complete)
		return budget;

	/* Read RX indices while this poll still owns the NAPI instance. */
	refill_pending = rnpgbe_rx_refill_pending(q_vector);
	if (likely(napi_complete_done(napi, work_done))) {
		if (refill_pending)
			mod_timer(&q_vector->rx_alloc_timer,
				  jiffies + msecs_to_jiffies(500));
		rnpgbe_irq_enable_queues(q_vector);
	}

	return work_done;
}

/**
 * rnpgbe_request_mbx_irq - Register mbx routine
 * @mucse: pointer to private structure
 *
 * Register the dedicated mailbox handler on vector 0.
 *
 * Return: 0 on success, negative on failure
 **/
int rnpgbe_request_mbx_irq(struct mucse *mucse)
{
	struct pci_dev *pdev = mucse->pdev;
	int err = 0;

	snprintf(mucse->mbx_name, sizeof(mucse->mbx_name),
		 "rnpgbe-mbx:%s", pci_name(pdev));
	INIT_WORK(&mucse->mbx_work, rnpgbe_mbx_work);
	atomic_set(&mucse->mbx_irq_seq, 0);

	err = request_irq(pci_irq_vector(pdev, 0), rnpgbe_msix_other, 0,
			  mucse->mbx_name, mucse);
	if (!err)
		mucse->mbx_irq_requested = true;

	return err;
}

/**
 * rnpgbe_free_mbx_irq - Remove mbx routine
 * @mucse: pointer to private structure
 **/
void rnpgbe_free_mbx_irq(struct mucse *mucse)
{
	struct pci_dev *pdev = mucse->pdev;

	if (mucse->mbx_irq_requested) {
		mucse->mbx_irq_requested = false;
		free_irq(pci_irq_vector(pdev, 0), mucse);
		cancel_work_sync(&mucse->mbx_work);
	}
	cancel_delayed_work_sync(&mucse->serv_task);
}

/**
 * rnpgbe_set_num_queues - Set initial TX/RX queue counts
 * @mucse: pointer to private structure
 *
 * Start with one TX queue and one RX queue.
 **/
static void rnpgbe_set_num_queues(struct mucse *mucse)
{
	mucse->num_tx_queues = 1;
	mucse->num_rx_queues = 1;
}

/**
 * rnpgbe_set_interrupt_capability - Set MSI-X interrupt capability
 * @mucse: pointer to private structure
 *
 * Allocate one MSI-X vector for the mailbox and one or more vectors for
 * the data path.
 *
 * Return: 0 on success, negative on failure
 **/
static int rnpgbe_set_interrupt_capability(struct mucse *mucse)
{
	int v_budget;

	v_budget = min3(mucse->num_tx_queues, mucse->num_rx_queues,
			MAX_Q_VECTORS);
	v_budget = min_t(int, v_budget, num_online_cpus());
	/* add one vector for mbx */
	v_budget += 1;

	/* Vector 0 is reserved for mailbox events and must not share NAPI. */
	v_budget = pci_alloc_irq_vectors(mucse->pdev, 2, v_budget,
					 PCI_IRQ_MSIX);
	if (v_budget < 0)
		return v_budget;

	/* q_vectors do not include the mailbox vector. */
	mucse->num_q_vectors = v_budget - 1;

	return 0;
}

/**
 * mucse_add_ring - Add ring to ring container
 * @ring: ring to be added
 * @head: ring container
 **/
static void mucse_add_ring(struct mucse_ring *ring,
			   struct mucse_ring_container *head)
{
	ring->next = head->ring;
	head->ring = ring;
	head->count++;
}

/**
 * rnpgbe_alloc_q_vector - Allocate memory for a single interrupt vector
 * @mucse: pointer to private structure
 * @eth_queue_idx: queue_index idx for this q_vector
 * @vector_idx: q_vector array index
 * @r_idx: starting hardware ring index
 * @r_count: number of TX/RX ring pairs
 * @step: ring step
 *
 * Return: 0 on success. If allocation fails we return -ENOMEM.
 **/
static int rnpgbe_alloc_q_vector(struct mucse *mucse,
				 int eth_queue_idx, int vector_idx, int r_idx,
				 int r_count, int step)
{
	int rxr_idx = r_idx, txr_idx = r_idx;
	struct mucse_hw *hw = &mucse->hw;
	struct mucse_q_vector *q_vector;
	int txr_count, rxr_count, idx;
	struct mucse_ring *ring;
	int ring_count;

	/*
	 * TX and RX rings are always allocated as pairs with the same hardware
	 * ring index. Interrupt control is shared by the pair, so the TX ring
	 * list is used to update the common registers.
	 */
	txr_count = r_count;
	rxr_count = r_count;
	ring_count = txr_count + rxr_count;

	q_vector = kzalloc_flex(*q_vector, ring, ring_count);
	if (!q_vector)
		return -ENOMEM;

	netif_napi_add(mucse->netdev, &q_vector->napi, rnpgbe_poll);
	/* tie q_vector and mucse together */
	mucse->q_vector[vector_idx] = q_vector;
	q_vector->mucse = mucse;
	timer_setup(&q_vector->rx_alloc_timer, rnpgbe_rx_alloc_retry, 0);
	q_vector->hw_vector = vector_idx;
	/* Vector 0 is reserved for the mailbox. */
	q_vector->hw_vector++;

	ring = q_vector->ring;

	for (idx = 0; idx < txr_count; idx++) {
		ring->dev = &mucse->pdev->dev;
		mucse_add_ring(ring, &q_vector->tx);
		ring->count = mucse->tx_ring_item_count;
		ring->netdev = mucse->netdev;
		ring->queue_index = eth_queue_idx + idx;
		ring->rnpgbe_queue_idx = txr_idx;
		ring->ring_addr = hw->hw_addr + RING_OFFSET(txr_idx);
		ring->irq_mask = ring->ring_addr + RNPGBE_DMA_INT_MASK;
		ring->trig = ring->ring_addr + RNPGBE_DMA_INT_TRIG;
		ring->q_vector = q_vector;
		ring->pfvfnum = hw->pfvfnum;
		ring->stats = &mucse->tx_stats[ring->queue_index];
		mucse->tx_ring[ring->queue_index] = ring;
		txr_idx += step;
		ring++;
	}

	for (idx = 0; idx < rxr_count; idx++) {
		ring->dev = &mucse->pdev->dev;
		mucse_add_ring(ring, &q_vector->rx);
		ring->count = mucse->rx_ring_item_count;
		ring->netdev = mucse->netdev;
		ring->queue_index = eth_queue_idx + idx;
		ring->rnpgbe_queue_idx = rxr_idx;
		ring->ring_addr = hw->hw_addr + RING_OFFSET(rxr_idx);
		ring->irq_mask = ring->ring_addr + RNPGBE_DMA_INT_MASK;
		ring->trig = ring->ring_addr + RNPGBE_DMA_INT_TRIG;
		ring->q_vector = q_vector;
		ring->pfvfnum = hw->pfvfnum;
		ring->stats = &mucse->rx_stats[ring->queue_index];
		mucse->rx_ring[ring->queue_index] = ring;
		rxr_idx += step;
		ring++;
	}

	return 0;
}

/**
 * rnpgbe_free_q_vector - Free memory allocated for specific interrupt vector
 * @mucse: pointer to private structure
 * @vector_idx: q_vector array index
 *
 * This function frees the memory allocated to the q_vector.  In addition if
 * NAPI is enabled it will delete any references to the NAPI struct prior
 * to freeing the q_vector.
 **/
static void rnpgbe_free_q_vector(struct mucse *mucse, int vector_idx)
{
	struct mucse_q_vector *q_vector = mucse->q_vector[vector_idx];
	struct mucse_ring *ring;

	timer_delete_sync(&q_vector->rx_alloc_timer);

	mucse_for_each_ring(ring, q_vector->tx)
		mucse->tx_ring[ring->queue_index] = NULL;
	mucse_for_each_ring(ring, q_vector->rx)
		mucse->rx_ring[ring->queue_index] = NULL;
	mucse->q_vector[vector_idx] = NULL;
	netif_napi_del(&q_vector->napi);
	kfree(q_vector);
}

/**
 * rnpgbe_alloc_q_vectors - Allocate memory for interrupt vectors
 * @mucse: pointer to private structure
 *
 * Return: 0 on success, or -ENOMEM on allocation failure.
 **/
static int rnpgbe_alloc_q_vectors(struct mucse *mucse)
{
	int err, ring_cnt, v_remaining = mucse->num_q_vectors;
	int r_remaining = min_t(int, mucse->num_tx_queues,
			      mucse->num_rx_queues);
	int num_q_vectors = 0;
	int eth_queue_idx = 0;
	int vector_idx = 0;
	int ring_step = 1;
	int ring_idx = 0;

	for (; r_remaining > 0 && v_remaining > 0; v_remaining--) {
		ring_cnt = DIV_ROUND_UP(r_remaining, v_remaining);
		err = rnpgbe_alloc_q_vector(mucse, eth_queue_idx,
					    vector_idx, ring_idx, ring_cnt,
					    ring_step);
		if (err)
			goto err_free_q_vector;
		ring_idx += ring_step * ring_cnt;
		eth_queue_idx += ring_cnt;
		r_remaining -= ring_cnt;
		num_q_vectors++;
		vector_idx++;
	}
	/* Record the number of queue vectors actually allocated */
	mucse->num_q_vectors = num_q_vectors;
	mucse->num_tx_queues = eth_queue_idx;
	mucse->num_rx_queues = eth_queue_idx;

	return 0;

err_free_q_vector:
	mucse->num_tx_queues = 0;
	mucse->num_rx_queues = 0;
	mucse->num_q_vectors = 0;

	while (vector_idx--)
		rnpgbe_free_q_vector(mucse, vector_idx);

	return err;
}

/**
 * rnpgbe_reset_interrupt_capability - Reset irq capability setup
 * @mucse: pointer to private structure
 **/
static void rnpgbe_reset_interrupt_capability(struct mucse *mucse)
{
	if (!mucse->pdev->msix_enabled)
		return;

	pci_free_irq_vectors(mucse->pdev);
}

/**
 * rnpgbe_init_interrupt_scheme - Initialize the interrupt scheme
 * @mucse: pointer to private structure
 *
 * Set up one TX/RX queue pair with a dedicated mailbox vector.
 *
 * Return: 0 on success, negative on failure
 **/
int rnpgbe_init_interrupt_scheme(struct mucse *mucse)
{
	int err;

	rnpgbe_set_num_queues(mucse);

	err = rnpgbe_set_interrupt_capability(mucse);
	if (err)
		return err;

	err = rnpgbe_alloc_q_vectors(mucse);
	if (err) {
		rnpgbe_reset_interrupt_capability(mucse);
		return err;
	}

	return 0;
}

/**
 * rnpgbe_free_q_vectors - Free memory allocated for interrupt vectors
 * @mucse: pointer to private structure
 *
 * This function frees the memory allocated to the q_vectors.  In addition if
 * NAPI is enabled it will delete any references to the NAPI struct prior
 * to freeing the q_vector.
 **/
static void rnpgbe_free_q_vectors(struct mucse *mucse)
{
	int vector_idx = mucse->num_q_vectors;

	mucse->num_q_vectors = 0;

	while (vector_idx--)
		rnpgbe_free_q_vector(mucse, vector_idx);
}

/**
 * rnpgbe_clear_interrupt_scheme - Clear the current interrupt scheme settings
 * @mucse: pointer to private structure
 *
 * Clear interrupt specific resources and reset the structure
 **/
void rnpgbe_clear_interrupt_scheme(struct mucse *mucse)
{
	rnpgbe_free_q_vectors(mucse);
	rnpgbe_reset_interrupt_capability(mucse);
}

/**
 * rnpgbe_msix_clean_rings - MSI-x interrupt handler for ring irq
 * @irq: interrupt number
 * @data: private data
 *
 * rnpgbe_msix_clean_rings handle irq from ring, start napi
 * Return: IRQ_HANDLED
 **/
static irqreturn_t rnpgbe_msix_clean_rings(int irq, void *data)
{
	struct mucse_q_vector *q_vector = (struct mucse_q_vector *)data;

	rnpgbe_irq_disable_queues(q_vector);
	napi_schedule_irqoff(&q_vector->napi);

	return IRQ_HANDLED;
}

/**
 * rnpgbe_request_irq - Register data-path MSI-X handlers
 * @mucse: pointer to private structure
 *
 * Register interrupt handlers for the data-path MSI-X vectors.
 *
 * Return: 0 on success, negative value on failure
 **/
int rnpgbe_request_irq(struct mucse *mucse)
{
	struct net_device *netdev = mucse->netdev;
	struct pci_dev *pdev = mucse->pdev;
	struct mucse_q_vector *q_vector;
	int err, i;

	for (i = 0; i < mucse->num_q_vectors; i++) {
		q_vector = mucse->q_vector[i];

		snprintf(q_vector->name, sizeof(q_vector->name),
			 "%s-%s-%d", netdev->name, "TxRx", i);

		err = request_irq(pci_irq_vector(pdev, i + 1),
				  rnpgbe_msix_clean_rings, 0,
				  q_vector->name, q_vector);
		if (err) {
			dev_err(&pdev->dev, "MSI-X req err %d: %d\n",
				i + 1, err);
			goto err_free_irqs;
		}
	}

	return 0;
err_free_irqs:
	while (i--) {
		q_vector = mucse->q_vector[i];
		synchronize_irq(pci_irq_vector(pdev, i + 1));
		free_irq(pci_irq_vector(pdev, i + 1), q_vector);
	}

	return err;
}

/**
 * rnpgbe_free_irq - Free data-path MSI-X handlers
 * @mucse: pointer to private structure
 **/
void rnpgbe_free_irq(struct mucse *mucse)
{
	struct pci_dev *pdev = mucse->pdev;
	struct mucse_q_vector *q_vector;

	for (int i = 0; i < mucse->num_q_vectors; i++) {
		q_vector = mucse->q_vector[i];
		if (!q_vector)
			continue;

		free_irq(pci_irq_vector(pdev, i + 1), q_vector);
	}
}

/**
 * rnpgbe_set_ring_vector - Set the ring_vector registers,
 * mapping interrupt causes to vectors
 * @mucse: pointer to private structure
 * @queue: queue to map the corresponding interrupt to
 * @vector: the vector num to map to the corresponding queue
 *
 */
static void rnpgbe_set_ring_vector(struct mucse *mucse,
				   u8 queue, u8 vector)
{
	struct mucse_hw *hw = &mucse->hw;
	u32 data;

	data = hw->pfvfnum << 24;
	data |= (vector << 8);
	data |= vector;
	writel(data, hw->ring_msix_base + RING_VECTOR(queue));
}

/**
 * rnpgbe_configure_msix - Configure MSI-X hardware
 * @mucse: pointer to private structure
 *
 * rnpgbe_configure_msix sets up the hardware to properly generate MSI-X
 * interrupts.
 **/
static void rnpgbe_configure_msix(struct mucse *mucse)
{
	struct mucse_q_vector *q_vector;

	for (int i = 0; i < mucse->num_q_vectors; i++) {
		struct mucse_ring *ring;

		q_vector = mucse->q_vector[i];
		/* TX/RX pairs share vector registers.
		 * Update them through TX.
		 */
		mucse_for_each_ring(ring, q_vector->tx) {
			rnpgbe_set_ring_vector(mucse, ring->rnpgbe_queue_idx,
					       q_vector->hw_vector);
		}
	}
}

static void rnpgbe_irq_enable(struct mucse *mucse)
{
	for (int i = 0; i < mucse->num_q_vectors; i++)
		rnpgbe_irq_enable_queues(mucse->q_vector[i]);
}

/**
 * rnpgbe_irq_disable - Mask queue interrupt generation on the NIC
 * @mucse: board private structure
 **/
static void rnpgbe_irq_disable(struct mucse *mucse)
{
	struct pci_dev *pdev = mucse->pdev;

	for (int i = 0; i < mucse->num_q_vectors; i++) {
		rnpgbe_irq_disable_queues(mucse->q_vector[i]);
		synchronize_irq(pci_irq_vector(pdev, i + 1));
	}
}

static void rnpgbe_napi_enable_all(struct mucse *mucse)
{
	for (int i = 0; i < mucse->num_q_vectors; i++)
		napi_enable(&mucse->q_vector[i]->napi);
}

static void rnpgbe_napi_disable_all(struct mucse *mucse)
{
	for (int i = 0; i < mucse->num_q_vectors; i++)
		napi_disable(&mucse->q_vector[i]->napi);
}

static void rnpgbe_stop_tx_ring(struct mucse_ring *tx_ring)
{
	if (!tx_ring->tx_buffer_info)
		return;

	/* Stop hw. No new descriptors are fetched after TX_START=0.
	 * DMA for descriptors fetched before the stop may still be in flight.
	 */
	mucse_ring_wr32(tx_ring, RNPGBE_TX_START, 0);
	/* Flush posted write to ensure hardware sees TX_START=0 */
	(void)mucse_ring_rd32(tx_ring, RNPGBE_TX_START);
}

static int rnpgbe_wait_tx_dma_idle(struct mucse *mucse)
{
	struct mucse_hw *hw = &mucse->hw;
	u32 dma_status;
	int err;

	/* A timeout indicates a terminal AXI fault. Hardware stops all PCIe
	 * DMA requests in this state, including requests accepted before the
	 * fault. Teardown may therefore release mappings and descriptor memory.
	 * Recovery requires a chip-level reset.
	 */
	err = readl_poll_timeout(hw->hw_addr + RNPGBE_DMA_STATUS,
				 dma_status,
				 (dma_status & RNPGBE_DMA_TX_STATUS) ==
				 RNPGBE_DMA_TX_STATUS,
				 10, 100000);
	if (err) {
		set_bit(__MUCSE_AXI_FAULT, &mucse->state);
		dev_err(&mucse->pdev->dev,
			"TX DMA failed to quiesce, status %#x\n", dma_status);
	}

	return err;
}

static int rnpgbe_wait_rx_dma_idle(struct mucse *mucse)
{
	struct mucse_hw *hw = &mucse->hw;
	u32 dma_status;
	int err;

	/* A timeout indicates a terminal AXI fault. Hardware stops all PCIe
	 * DMA requests in this state, including requests accepted before the
	 * fault. Teardown may therefore release mappings and descriptor memory.
	 * Recovery requires a chip-level reset.
	 */
	err = readl_poll_timeout(hw->hw_addr + RNPGBE_DMA_STATUS,
				 dma_status,
				 (dma_status & RNPGBE_DMA_RX_STATUS) ==
				 RNPGBE_DMA_RX_STATUS,
				 10, 100000);
	if (err) {
		set_bit(__MUCSE_AXI_FAULT, &mucse->state);
		dev_err(&mucse->pdev->dev,
			"RX DMA failed to quiesce, status %#x\n", dma_status);
	}

	return err;
}

/**
 * rnpgbe_stop_all_tx_rings - Stop TX DMA on all queues
 * @mucse: board private structure
 *
 * Return: 0 when TX DMA was quiesced, negative errno otherwise
 **/
static int rnpgbe_stop_all_tx_rings(struct mucse *mucse)
{
	struct mucse_hw *hw = &mucse->hw;
	u32 dma_axi_ctl;
	int err = 0;

	for (int i = 0; i < mucse->num_tx_queues; i++)
		rnpgbe_stop_tx_ring(mucse->tx_ring[i]);

	if (mucse->num_tx_queues &&
	    !test_bit(__MUCSE_AXI_FAULT, &mucse->state))
		err = rnpgbe_wait_tx_dma_idle(mucse);

	dma_axi_ctl = mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);
	dma_axi_ctl &= ~TX_AXI_RW_EN;
	mucse_hw_wr32(hw, RNPGBE_DMA_AXI_EN, dma_axi_ctl);
	/* Flush the posted write before continuing. */
	(void)mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);

	if (!test_bit(__MUCSE_AXI_FAULT, &mucse->state))
		return 0;

	return err ? err : -EIO;
}

/**
 * rnpgbe_clean_tx_ring - Free Tx Buffers
 * @tx_ring: ring to be cleaned
 **/
static void rnpgbe_clean_tx_ring(struct mucse_ring *tx_ring)
{
	struct mucse_tx_buffer *tx_buffer;
	u16 i = tx_ring->next_to_clean;
	unsigned long size;

	/* ring already cleared, nothing to do */
	if (!tx_ring->tx_buffer_info)
		return;

	tx_buffer = &tx_ring->tx_buffer_info[i];

	while (i != tx_ring->next_to_use) {
		struct rnpgbe_tx_desc *eop_desc, *tx_desc;

		dev_kfree_skb_any(tx_buffer->skb);
		/* unmap skb header data */
		if (dma_unmap_len(tx_buffer, len)) {
			if (tx_buffer->mapped_as_page) {
				dma_unmap_page(tx_ring->dev,
					       dma_unmap_addr(tx_buffer, dma),
					       dma_unmap_len(tx_buffer, len),
					       DMA_TO_DEVICE);
			} else {
				dma_unmap_single(tx_ring->dev,
						 dma_unmap_addr(tx_buffer, dma),
						 dma_unmap_len(tx_buffer, len),
						 DMA_TO_DEVICE);
			}
		}
		eop_desc = tx_buffer->next_to_watch;
		tx_desc = M_TX_DESC(tx_ring, i);
		/* unmap remaining buffers */
		while (tx_desc != eop_desc) {
			tx_buffer++;
			tx_desc++;
			i++;
			if (unlikely(i == tx_ring->count)) {
				i = 0;
				tx_buffer = tx_ring->tx_buffer_info;
				tx_desc = M_TX_DESC(tx_ring, 0);
			}

			/* unmap any remaining paged data */
			if (dma_unmap_len(tx_buffer, len))
				dma_unmap_page(tx_ring->dev,
					       dma_unmap_addr(tx_buffer, dma),
					       dma_unmap_len(tx_buffer, len),
					       DMA_TO_DEVICE);
		}
		/* move us one more past the eop_desc for start of next pkt */
		tx_buffer++;
		i++;
		if (unlikely(i == tx_ring->count)) {
			i = 0;
			tx_buffer = tx_ring->tx_buffer_info;
		}
	}

	netdev_tx_reset_queue(txring_txq(tx_ring));
	size = sizeof(struct mucse_tx_buffer) * tx_ring->count;
	memset(tx_ring->tx_buffer_info, 0, size);
	/* Zero out the descriptor ring */
	memset(tx_ring->desc, 0, tx_ring->size);
	tx_ring->next_to_use = 0;
	tx_ring->next_to_clean = 0;
}

/**
 * rnpgbe_clean_all_tx_rings - Stop TX DMA and free Tx buffers for all queues
 * @mucse: board private structure
 **/
void rnpgbe_clean_all_tx_rings(struct mucse *mucse)
{
	/* rnpgbe_stop_all_tx_rings() logs and latches any AXI fault. Hardware
	 * stops DMA requests in this state, so cleanup intentionally ignores
	 * an error and releases the mappings and descriptor memory.
	 */
	rnpgbe_stop_all_tx_rings(mucse);

	for (int i = 0; i < mucse->num_tx_queues; i++)
		rnpgbe_clean_tx_ring(mucse->tx_ring[i]);
}

static void rnpgbe_stop_rx_ring(struct mucse_ring *rx_ring)
{
	/* Stop hw. hardware design guarantees:
	 * - No new descriptors will be fetched after RX_START=0
	 * - No DMA will be initiated for already-fetched descriptors
	 */
	mucse_ring_wr32(rx_ring, RNPGBE_RX_START, 0);
	/* Flush posted write to ensure hardware sees RX_START=0 */
	(void)mucse_ring_rd32(rx_ring, RNPGBE_RX_START);
}

/**
 * rnpgbe_stop_all_rx_rings - Stop RX DMA for all queues
 * @mucse: board private structure
 *
 * Return: 0 if RX DMA is quiesced and no AXI fault is latched,
 * negative errno otherwise
 **/
static int rnpgbe_stop_all_rx_rings(struct mucse *mucse)
{
	struct mucse_hw *hw = &mucse->hw;
	u32 dma_axi_ctl;
	int err = 0;

	for (int i = 0; i < mucse->num_rx_queues; i++)
		rnpgbe_stop_rx_ring(mucse->rx_ring[i]);

	if (mucse->num_rx_queues &&
	    !test_bit(__MUCSE_AXI_FAULT, &mucse->state))
		err = rnpgbe_wait_rx_dma_idle(mucse);

	dma_axi_ctl = mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);
	dma_axi_ctl &= ~RX_AXI_RW_EN;
	mucse_hw_wr32(hw, RNPGBE_DMA_AXI_EN, dma_axi_ctl);
	/* Flush the posted write before continuing. */
	(void)mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);

	if (!test_bit(__MUCSE_AXI_FAULT, &mucse->state))
		return 0;

	return err ? err : -EIO;
}

/**
 * rnpgbe_clean_rx_ring - Free Rx Buffers per Queue
 * @rx_ring: ring to free buffers from
 **/
static void rnpgbe_clean_rx_ring(struct mucse_ring *rx_ring)
{
	struct mucse_rx_buffer *rx_buffer;
	u16 i;

	/* ring already cleared, nothing to do */
	if (!rx_ring->rx_buffer_info)
		return;
	/* Free all the Rx ring sk_buffs */
	for (i = 0; i < rx_ring->count; i++) {
		rx_buffer = &rx_ring->rx_buffer_info[i];

		if (rx_buffer->skb) {
			struct sk_buff *skb = rx_buffer->skb;

			dev_kfree_skb(skb);
			rx_buffer->skb = NULL;
		}

		if (rx_buffer->page) {
			page_pool_put_full_page(rx_ring->page_pool,
						rx_buffer->page, false);
			rx_buffer->page = NULL;
		}
	}

	rx_ring->next_to_clean = 0;
	rx_ring->next_to_use = 0;
	rx_ring->drop_bytes = 0;
	rx_ring->drop_reason = MUCSE_RX_DROP_NONE;
}

/**
 * rnpgbe_clean_all_rx_rings - Free Rx buffers for all queues
 * @mucse: board private structure
 **/
static void rnpgbe_clean_all_rx_rings(struct mucse *mucse)
{
	for (int i = 0; i < mucse->num_rx_queues; i++)
		rnpgbe_clean_rx_ring(mucse->rx_ring[i]);
}

static void rnpgbe_cancel_rx_retry_timers(struct mucse *mucse)
{
	for (int i = 0; i < mucse->num_q_vectors; i++)
		timer_delete_sync(&mucse->q_vector[i]->rx_alloc_timer);
}

void rnpgbe_down(struct mucse *mucse)
{
	struct net_device *netdev = mucse->netdev;
	struct mucse_hw *hw = &mucse->hw;
	unsigned long flags;
	int err;

	spin_lock_irqsave(&mucse->link_lock, flags);
	mucse->link_event_ready = false;
	mucse->link_pending = false;
	hw->link = false;
	hw->speed = 0;
	hw->duplex = 0;
	spin_unlock_irqrestore(&mucse->link_lock, flags);
	cancel_delayed_work_sync(&mucse->serv_task);

	netif_carrier_off(netdev);
	rnpgbe_irq_disable(mucse);
	rnpgbe_napi_disable_all(mucse);
	synchronize_net();
	rnpgbe_irq_disable(mucse);
	netif_tx_disable(netdev);
	rnpgbe_stop_all_rx_rings(mucse);
	rnpgbe_cancel_rx_retry_timers(mucse);
	rnpgbe_clean_all_tx_rings(mucse);

	err = rnpgbe_send_notify(hw, false, mucse_fw_link_report_en);
	if (err) {
		dev_warn(&hw->pdev->dev, "Send link report to hw failed %d\n",
			 err);
		dev_warn(&hw->pdev->dev, "Fw will still report link event\n");
	}

	/* Firmware owns GMAC_CONTROL_RE and clears it while processing
	 * port-down. RX DMA is already quiesced, so no local GMAC update is
	 * needed before freeing the RX buffers.
	 */
	err = rnpgbe_send_notify(hw, false, mucse_fw_portup);
	if (err) {
		dev_warn(&hw->pdev->dev, "Send port down to hw failed %d\n",
			 err);
		dev_warn(&hw->pdev->dev, "Port is not truly down\n");
	}
	rnpgbe_clean_all_rx_rings(mucse);
}

/**
 * rnpgbe_up_complete - Final step for port up
 * @mucse: pointer to private structure
 *
 * Return: 0 on success, negative errno if firmware setup fails
 **/
int rnpgbe_up_complete(struct mucse *mucse)
{
	struct net_device *netdev = mucse->netdev;
	struct mucse_hw *hw = &mucse->hw;
	unsigned long flags;
	int err;

	rnpgbe_configure_msix(mucse);
	rnpgbe_napi_enable_all(mucse);
	rnpgbe_schedule_rx_retry(mucse);
	rnpgbe_irq_enable(mucse);
	netif_tx_start_all_queues(netdev);
	err = rnpgbe_send_notify(hw, true, mucse_fw_portup);
	if (err) {
		dev_err(&hw->pdev->dev,
			"Failed to notify firmware that port is up: %d\n", err);
		return err;
	}
	/* Open link-event handling only after the data path is ready. Reset
	 * the snapshot at the same time so firmware reports the current state.
	 */
	spin_lock_irqsave(&mucse->link_lock, flags);
	mucse->link_pending = false;
	hw->link = false;
	hw->speed = 0;
	hw->duplex = 0;
	mucse_hw_wr32(hw, RNPGBE_LINK_ST, M_DEFAULT_ST);
	mucse->link_event_ready = true;
	spin_unlock_irqrestore(&mucse->link_lock, flags);
	/* Firmware checks RNPGBE_LINK_ST (driver's last-received link state)
	 * and only asserts LINK_CHANGE_EVT when it differs from the actual link
	 * state AND link_report_en is true.
	 */
	err = rnpgbe_send_notify(hw, true, mucse_fw_link_report_en);
	if (err) {
		dev_err(&hw->pdev->dev,
			"Failed to enable firmware link reporting: %d\n",
			 err);
		return err;
	}

	return 0;
}

/**
 * rnpgbe_free_tx_resources - Free Tx Resources per Queue
 * @tx_ring: tx descriptor ring for a specific queue
 *
 * Free all transmit software resources
 **/
static void rnpgbe_free_tx_resources(struct mucse_ring *tx_ring)
{
	vfree(tx_ring->tx_buffer_info);
	tx_ring->tx_buffer_info = NULL;
	/* if not set, then don't free */
	if (!tx_ring->desc)
		return;

	dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc,
			  tx_ring->dma);
	tx_ring->desc = NULL;
}

/**
 * rnpgbe_setup_tx_resources - allocate Tx resources (Descriptors)
 * @tx_ring: tx descriptor ring (for a specific queue) to setup
 *
 * Return: 0 on success, negative on failure
 **/
static int rnpgbe_setup_tx_resources(struct mucse_ring *tx_ring)
{
	struct device *dev = tx_ring->dev;
	int size;

	size = sizeof(struct mucse_tx_buffer) * tx_ring->count;

	tx_ring->tx_buffer_info = vzalloc(size);
	if (!tx_ring->tx_buffer_info)
		goto err_return;
	/* round up to nearest 4K */
	tx_ring->size = tx_ring->count * sizeof(struct rnpgbe_tx_desc);
	tx_ring->size = ALIGN(tx_ring->size, 4096);
	tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, &tx_ring->dma,
					   GFP_KERNEL);
	if (!tx_ring->desc)
		goto err_free_buffer;

	tx_ring->next_to_use = 0;
	tx_ring->next_to_clean = 0;

	return 0;

err_free_buffer:
	vfree(tx_ring->tx_buffer_info);
err_return:
	tx_ring->tx_buffer_info = NULL;
	return -ENOMEM;
}

/**
 * rnpgbe_configure_tx_ring - Configure Tx ring after Reset
 * @mucse: pointer to private structure
 * @ring: structure containing ring specific data
 *
 * Configure the Tx descriptor ring after a reset.
 **/
static void rnpgbe_configure_tx_ring(struct mucse *mucse,
				     struct mucse_ring *ring)
{
	struct mucse_hw *hw = &mucse->hw;

	mucse_ring_wr32(ring, RNPGBE_TX_BASE_ADDR_LO, (u32)ring->dma);
	mucse_ring_wr32(ring, RNPGBE_TX_BASE_ADDR_HI,
			(u32)(((u64)ring->dma) >> 32) |
			((u32)hw->pfvfnum << 24));
	mucse_ring_wr32(ring, RNPGBE_TX_LEN, ring->count);
	ring->next_to_clean = mucse_ring_rd32(ring, RNPGBE_TX_HEAD) %
			      ring->count;
	ring->next_to_use = ring->next_to_clean;
	ring->tail = ring->ring_addr + RNPGBE_TX_TAIL;
	writel(ring->next_to_use, ring->tail);
	mucse_ring_wr32(ring, RNPGBE_TX_FETCH_CTRL, M_DEFAULT_TX_FETCH);
	mucse_ring_wr32(ring, RNPGBE_TX_INT_TIMER,
			M_DEFAULT_INT_TIMER * hw->cycles_per_us);
	mucse_ring_wr32(ring, RNPGBE_TX_INT_PKTCNT, M_DEFAULT_INT_PKTCNT);
}

/**
 * rnpgbe_configure_tx - Configure Transmit Unit after Reset
 * @mucse: pointer to private structure
 *
 * Configure the Tx DMA after a reset.
 *
 * Return: 0 on success, negative errno if TX DMA did not quiesce
 **/
int rnpgbe_configure_tx(struct mucse *mucse)
{
	struct mucse_hw *hw = &mucse->hw;
	u32 i, dma_axi_ctl;
	int err;

	err = rnpgbe_stop_all_tx_rings(mucse);
	if (err)
		return err;

	for (i = 0; i < mucse->num_tx_queues; i++)
		rnpgbe_configure_tx_ring(mucse, mucse->tx_ring[i]);

	/* Ensure all ring configuration is visible before enabling TX DMA. */
	wmb();
	dma_axi_ctl = mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);
	dma_axi_ctl |= TX_AXI_RW_EN;
	mucse_hw_wr32(hw, RNPGBE_DMA_AXI_EN, dma_axi_ctl);
	/* Ensure TX_AXI_RW_EN is visible before starting the rings. */
	(void)mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);

	for (i = 0; i < mucse->num_tx_queues; i++)
		mucse_ring_wr32(mucse->tx_ring[i], RNPGBE_TX_START, 1);

	return 0;
}

/**
 * rnpgbe_setup_all_tx_resources - allocate all queues Tx resources
 * @mucse: pointer to private structure
 *
 * Allocate memory for tx_ring.
 *
 * Return: 0 on success, negative on failure
 **/
int rnpgbe_setup_all_tx_resources(struct mucse *mucse)
{
	int i, err = 0;

	for (i = 0; i < mucse->num_tx_queues; i++) {
		err = rnpgbe_setup_tx_resources(mucse->tx_ring[i]);
		if (!err)
			continue;

		goto err_free_res;
	}

	return 0;
err_free_res:
	while (i--)
		rnpgbe_free_tx_resources(mucse->tx_ring[i]);
	return err;
}

/**
 * rnpgbe_free_all_tx_resources - Free Tx Resources for All Queues
 * @mucse: pointer to private structure
 *
 * Free all transmit software resources
 **/
void rnpgbe_free_all_tx_resources(struct mucse *mucse)
{
	for (int i = 0; i < (mucse->num_tx_queues); i++)
		rnpgbe_free_tx_resources(mucse->tx_ring[i]);
}

static int rnpgbe_tx_map(struct mucse_ring *tx_ring,
			 struct mucse_tx_buffer *first, u32 mac_ip_len,
			 u32 tx_flags)
{
	struct mucse_tx_buffer *tx_buffer;
	struct rnpgbe_tx_desc *tx_desc;
	struct skb_shared_info *shinfo;
	unsigned int data_len, size;
	struct sk_buff *skb;
	skb_frag_t *frag;
	dma_addr_t dma;
	int nr_frags;
	int frag_idx;
	/* Hardware requires this in the upper 8 bits of
	 * the descriptor address
	 */
	u64 fun_id;
	u16 i;

	skb = first->skb;
	shinfo = skb_shinfo(skb);
	fun_id = (u64)tx_ring->pfvfnum << 56;
	nr_frags = shinfo->nr_frags;
	i = tx_ring->next_to_use;
	frag_idx = 0;
	tx_desc = M_TX_DESC(tx_ring, i);
	size = skb_headlen(skb);
	data_len = skb->data_len;

	if (size) {
		dma = dma_map_single(tx_ring->dev, skb->data, size,
				     DMA_TO_DEVICE);
		first->mapped_as_page = false;
	} else if (data_len) {
		while (frag_idx < nr_frags &&
		       !skb_frag_size(&shinfo->frags[frag_idx]))
			frag_idx++;

		if (frag_idx == nr_frags)
			goto err_unmap;

		frag = &shinfo->frags[frag_idx];
		size = skb_frag_size(frag);

		dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
				       size, DMA_TO_DEVICE);
		first->mapped_as_page = true;
		data_len -= size;
		frag_idx++;
	} else {
		goto err_unmap;
	}

	tx_buffer = first;

	dma_unmap_len_set(tx_buffer, len, 0);
	dma_unmap_addr_set(tx_buffer, dma, 0);

	for (;; frag_idx++) {
		if (dma_mapping_error(tx_ring->dev, dma))
			goto err_unmap;

		/* record length, and DMA address */
		dma_unmap_len_set(tx_buffer, len, size);
		dma_unmap_addr_set(tx_buffer, dma, dma);

		tx_desc->pkt_addr = cpu_to_le64(dma | fun_id);

		while (unlikely(size > M_MAX_DATA_PER_TXD)) {
			tx_desc->vlan_cmd_bsz = build_ctob(tx_flags,
							   mac_ip_len,
							   M_MAX_DATA_PER_TXD);
			i++;
			tx_desc++;
			if (i == tx_ring->count) {
				tx_desc = M_TX_DESC(tx_ring, 0);
				i = 0;
			}

			tx_buffer = &tx_ring->tx_buffer_info[i];
			dma_unmap_len_set(tx_buffer, len, 0);
			dma += M_MAX_DATA_PER_TXD;
			size -= M_MAX_DATA_PER_TXD;
			tx_desc->pkt_addr = cpu_to_le64(dma | fun_id);
		}

		if (likely(!data_len))
			break;
		tx_desc->vlan_cmd_bsz = build_ctob(tx_flags, mac_ip_len, size);
		i++;
		tx_desc++;
		if (i == tx_ring->count) {
			tx_desc = M_TX_DESC(tx_ring, 0);
			i = 0;
		}

		while (frag_idx < nr_frags &&
		       !skb_frag_size(&shinfo->frags[frag_idx]))
			frag_idx++;

		if (frag_idx == nr_frags)
			goto err_unmap;

		frag = &shinfo->frags[frag_idx];
		size = skb_frag_size(frag);
		data_len -= size;
		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
				       DMA_TO_DEVICE);
		tx_buffer = &tx_ring->tx_buffer_info[i];
		tx_buffer->mapped_as_page = true;
	}

	/* write last descriptor with RS and EOP bits */
	tx_desc->vlan_cmd_bsz = build_ctob(tx_flags | M_TXD_CMD_EOP |
					   M_TXD_CMD_RS,
					   mac_ip_len, size);

	/* Force memory writes to complete before letting h/w know there
	 * are new descriptors to fetch.  (Only applicable for weak-ordered
	 * memory model archs, such as IA-64).
	 *
	 * We also need this memory barrier to make certain all of the
	 * status bits have been updated before next_to_watch is written.
	 */
	dma_wmb();
	/* set next_to_watch value indicating a packet is present */
	first->next_to_watch = tx_desc;
	i++;
	if (i == tx_ring->count)
		i = 0;
	tx_ring->next_to_use = i;
	skb_tx_timestamp(skb);
	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
	/* notify HW of packet */
	writel(i, tx_ring->tail);

	return 0;
err_unmap:
	for (;;) {
		tx_buffer = &tx_ring->tx_buffer_info[i];
		if (dma_unmap_len(tx_buffer, len)) {
			if (tx_buffer->mapped_as_page) {
				dma_unmap_page(tx_ring->dev,
					       dma_unmap_addr(tx_buffer, dma),
					       dma_unmap_len(tx_buffer, len),
					       DMA_TO_DEVICE);
			} else {
				dma_unmap_single(tx_ring->dev,
						 dma_unmap_addr(tx_buffer, dma),
						 dma_unmap_len(tx_buffer, len),
						 DMA_TO_DEVICE);
			}
		}
		dma_unmap_len_set(tx_buffer, len, 0);
		dma_unmap_addr_set(tx_buffer, dma, 0);
		if (tx_buffer == first)
			break;
		if (i == 0)
			i += tx_ring->count;
		i--;
	}
	dev_kfree_skb_any(first->skb);
	first->skb = NULL;
	tx_ring->next_to_use = i;

	return -ENOMEM;
}

netdev_tx_t rnpgbe_xmit_frame_ring(struct sk_buff *skb,
				   struct mucse_ring *tx_ring)
{
	/* hw requires non-zero mac_ip_len even without offload;
	 * Will be updated per-packet when offload is added
	 */
	u32 mac_ip_len = M_DEFAULT_MAC_IP_LEN;
	struct mucse_tx_buffer *first;
	u32 tx_flags = 0;
	unsigned short f;
	u16 count;

	count = TXD_USE_COUNT(skb_headlen(skb));
	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
		skb_frag_t *frag_temp = &skb_shinfo(skb)->frags[f];

		count += TXD_USE_COUNT(skb_frag_size(frag_temp));
	}

	if (!netif_txq_maybe_stop(txring_txq(tx_ring),
				  mucse_desc_unused(tx_ring),
				  count + RESV_DESC_NEEDED,
				  count + RESV_DESC_NEEDED))
		return NETDEV_TX_BUSY;

	/* record the location of the first descriptor for this packet */
	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
	first->skb = skb;
	first->bytecount = skb->len;
	first->gso_segs = 1;

	if (rnpgbe_tx_map(tx_ring, first, mac_ip_len, tx_flags)) {
		atomic64_inc(&tx_ring->stats->dropped);

		goto out;
	}

	/* NETDEV_TX_BUSY is expensive. So stop advancing the TX queue. */
	netif_txq_maybe_stop(txring_txq(tx_ring),
			     mucse_desc_unused(tx_ring),
			     DESC_NEEDED, DESC_NEEDED);
out:
	return NETDEV_TX_OK;
}

/**
 * rnpgbe_get_stats64 - Get stats for this netdev
 * @netdev: network interface device structure
 * @stats: stats data
 **/
void rnpgbe_get_stats64(struct net_device *netdev,
			struct rtnl_link_stats64 *stats)
{
	struct mucse *mucse = netdev_priv(netdev);
	int i;

	for (i = 0; i < RNPGBE_MAX_QUEUES; i++) {
		unsigned int start;
		u64 bytes, packets, dropped;
		struct mucse_queue_stats *queue_stats = &mucse->tx_stats[i];

		do {
			start = u64_stats_fetch_begin(&queue_stats->syncp);
			packets = queue_stats->packets;
			bytes = queue_stats->bytes;
		} while (u64_stats_fetch_retry(&queue_stats->syncp, start));
		dropped = atomic64_read(&queue_stats->dropped);

		stats->tx_packets += packets;
		stats->tx_dropped += dropped;
		stats->tx_bytes += bytes;
	}

	for (i = 0; i < RNPGBE_MAX_QUEUES; i++) {
		unsigned int start;
		u64 bytes, packets, dropped, length_errors;
		struct mucse_queue_stats *queue_stats = &mucse->rx_stats[i];

		do {
			start = u64_stats_fetch_begin(&queue_stats->syncp);
			packets = queue_stats->packets;
			bytes = queue_stats->bytes;
			length_errors = queue_stats->length_errors;
		} while (u64_stats_fetch_retry(&queue_stats->syncp, start));
		dropped = atomic64_read(&queue_stats->dropped);

		stats->rx_packets += packets;
		stats->rx_dropped += dropped;
		stats->rx_bytes += bytes;
		stats->rx_errors += length_errors;
		stats->rx_length_errors += length_errors;
	}
}

static int mucse_alloc_page_pool(struct mucse_ring *rx_ring)
{
	struct page_pool_params pp_params = {
		.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
		.order = 0,
		.pool_size = rx_ring->count,
		.nid = dev_to_node(rx_ring->dev),
		.dev = rx_ring->dev,
		.napi = &rx_ring->q_vector->napi,
		.dma_dir = DMA_FROM_DEVICE,
		.offset = RNPGBE_SKB_PAD,
		.max_len = RNPGBE_RX_DESC_DATA_LEN,
		.netdev = rx_ring->netdev,
		.queue_idx = rx_ring->queue_index,
	};
	int ret = 0;

	rx_ring->page_pool = page_pool_create(&pp_params);
	if (IS_ERR(rx_ring->page_pool)) {
		ret = PTR_ERR(rx_ring->page_pool);
		rx_ring->page_pool = NULL;
	}

	return ret;
}

/**
 * rnpgbe_setup_rx_resources - allocate Rx resources (Descriptors)
 * @rx_ring:    rx descriptor ring (for a specific queue) to setup
 *
 * Return: 0 on success, negative on failure
 **/
static int rnpgbe_setup_rx_resources(struct mucse_ring *rx_ring)
{
	struct device *dev = rx_ring->dev;
	int size;

	size = sizeof(struct mucse_rx_buffer) * rx_ring->count;

	rx_ring->rx_buffer_info = vzalloc(size);

	if (!rx_ring->rx_buffer_info)
		goto err_return;
	/* Round up to nearest 4K */
	rx_ring->size = rx_ring->count * sizeof(union rnpgbe_rx_desc);
	rx_ring->size = ALIGN(rx_ring->size, 4096);
	rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, &rx_ring->dma,
					   GFP_KERNEL);
	if (!rx_ring->desc)
		goto err_free_buffer;

	rx_ring->next_to_clean = 0;
	rx_ring->next_to_use = 0;

	if (mucse_alloc_page_pool(rx_ring))
		goto err_free_desc;

	return 0;
err_free_desc:
	dma_free_coherent(dev, rx_ring->size, rx_ring->desc,
			  rx_ring->dma);
	rx_ring->desc = NULL;
err_free_buffer:
	vfree(rx_ring->rx_buffer_info);
err_return:
	rx_ring->rx_buffer_info = NULL;
	return -ENOMEM;
}

/**
 * rnpgbe_free_rx_resources - Free Rx Resources
 * @rx_ring: ring to clean the resources from
 *
 * RX DMA must be quiesced and all ring-held buffers released before this
 * function is called.
 **/
static void rnpgbe_free_rx_resources(struct mucse_ring *rx_ring)
{
	vfree(rx_ring->rx_buffer_info);
	rx_ring->rx_buffer_info = NULL;
	/* if not set, then don't free */
	if (!rx_ring->desc)
		return;

	dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc,
			  rx_ring->dma);
	rx_ring->desc = NULL;
	if (rx_ring->page_pool) {
		page_pool_destroy(rx_ring->page_pool);
		rx_ring->page_pool = NULL;
	}
}

/**
 * rnpgbe_setup_all_rx_resources - allocate all queues Rx resources
 * @mucse: pointer to private structure
 *
 * Return: 0 on success, negative on failure
 **/
int rnpgbe_setup_all_rx_resources(struct mucse *mucse)
{
	int i, err = 0;

	for (i = 0; i < mucse->num_rx_queues; i++) {
		err = rnpgbe_setup_rx_resources(mucse->rx_ring[i]);
		if (!err)
			continue;

		goto err_setup_rx;
	}

	return 0;
err_setup_rx:
	while (i--)
		rnpgbe_free_rx_resources(mucse->rx_ring[i]);
	return err;
}

/**
 * rnpgbe_free_all_rx_resources - Free Rx Resources for All Queues
 * @mucse: pointer to private structure
 *
 * Free all receive software resources
 **/
void rnpgbe_free_all_rx_resources(struct mucse *mucse)
{
	for (int i = 0; i < mucse->num_rx_queues; i++)
		rnpgbe_free_rx_resources(mucse->rx_ring[i]);
}

/**
 * rnpgbe_configure_rx_ring - Configure Rx ring info to hw
 * @mucse: pointer to private structure
 * @ring: structure containing ring specific data
 *
 * Configure the Rx descriptor ring after a reset.
 **/
static void rnpgbe_configure_rx_ring(struct mucse *mucse,
				     struct mucse_ring *ring)
{
	struct mucse_hw *hw = &mucse->hw;

	/* Set the descriptor registers after RX DMA has quiesced. */
	mucse_ring_wr32(ring, RNPGBE_RX_BASE_ADDR_LO, (u32)ring->dma);
	mucse_ring_wr32(ring, RNPGBE_RX_BASE_ADDR_HI,
			(u32)((u64)ring->dma >> 32) |
			((u32)hw->pfvfnum << 24));
	mucse_ring_wr32(ring, RNPGBE_RX_LEN, ring->count);
	ring->tail = ring->ring_addr + RNPGBE_RX_TAIL;
	ring->next_to_clean = mucse_ring_rd32(ring, RNPGBE_RX_HEAD) %
			      ring->count;
	ring->next_to_use = ring->next_to_clean;
	ring->drop_bytes = 0;
	ring->drop_reason = MUCSE_RX_DROP_NONE;
	mucse_ring_wr32(ring, RNPGBE_RX_SG_LEN, M_DEFAULT_SG);
	mucse_ring_wr32(ring, RNPGBE_RX_FETCH, M_DEFAULT_RX_FETCH);
	mucse_ring_wr32(ring, RNPGBE_RX_TIMEOUT_TH, 0);
	mucse_ring_wr32(ring, RNPGBE_RX_INT_TIMER,
			M_DEFAULT_INT_TIMER_R * hw->cycles_per_us);
	mucse_ring_wr32(ring, RNPGBE_RX_INT_PKTCNT, M_DEFAULT_RX_INT_PKTCNT);
	/* Fill the ring before RX DMA is enabled. Batch-sized allocation
	 * shortfalls are retried after NAPI is enabled; smaller shortfalls are
	 * replenished once traffic frees a complete refill batch.
	 */
	rnpgbe_alloc_rx_buffers(ring, mucse_desc_unused_rx(ring));
}

/**
 * rnpgbe_configure_rx - Configure Receive Unit after Reset
 * @mucse: pointer to private structure
 *
 * Configure the Rx unit after a reset.
 *
 * Return: 0 on success, negative errno if RX DMA does not quiesce or
 * an AXI fault is already latched
 **/
int rnpgbe_configure_rx(struct mucse *mucse)
{
	struct mucse_hw *hw = &mucse->hw;
	u32 dma_axi_ctl;
	int err;

	err = rnpgbe_stop_all_rx_rings(mucse);
	if (err)
		return err;

	for (int i = 0; i < mucse->num_rx_queues; i++)
		rnpgbe_configure_rx_ring(mucse, mucse->rx_ring[i]);

	/* Ensure all ring configuration is visible before enabling RX DMA. */
	wmb();
	dma_axi_ctl = mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);
	dma_axi_ctl |= RX_AXI_RW_EN;
	mucse_hw_wr32(hw, RNPGBE_DMA_AXI_EN, dma_axi_ctl);
	/* Ensure RX_AXI_RW_EN is visible before starting the rings. */
	(void)mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN);

	for (int i = 0; i < mucse->num_rx_queues; i++)
		mucse_ring_wr32(mucse->rx_ring[i], RNPGBE_RX_START, 1);

	return 0;
}

/**
 * rnpgbe_process_link_event - Consume a pending link event
 * @mucse: pointer to the device private structure
 * @link: link status snapshot
 * @speed: link speed snapshot
 * @duplex: link duplex snapshot
 *
 * Return: true if a link event was consumed, false otherwise
 **/
static bool rnpgbe_process_link_event(struct mucse *mucse, bool *link,
				      int *speed, u8 *duplex)
{
	struct mucse_hw *hw = &mucse->hw;
	unsigned long flags;

	spin_lock_irqsave(&mucse->link_lock, flags);
	if (!mucse->link_event_ready || !mucse->link_pending) {
		spin_unlock_irqrestore(&mucse->link_lock, flags);
		return false;
	}
	mucse->link_pending = false;
	*link = hw->link;
	*speed = hw->speed;
	*duplex = hw->duplex;
	spin_unlock_irqrestore(&mucse->link_lock, flags);

	return true;
}

/**
 * rnpgbe_link_is_up - Update netif_carrier status and
 * print link up message
 * @mucse: pointer to the device private structure
 * @speed: link speed snapshot
 * @duplex: link duplex snapshot
 **/
static void rnpgbe_link_is_up(struct mucse *mucse, int speed, u8 duplex)
{
	struct net_device *netdev = mucse->netdev;

	/* Only continue if link was previously down */
	if (netif_carrier_ok(netdev))
		return;

	netdev_info(netdev, "NIC Link is Up %d Mbps, %s Duplex\n",
		    speed, duplex ? "Full" : "Half");
	netif_carrier_on(netdev);
}

/**
 * rnpgbe_link_is_down - Update netif_carrier status and
 * print link down message
 * @mucse: pointer to the private structure
 **/
static void rnpgbe_link_is_down(struct mucse *mucse)
{
	struct net_device *netdev = mucse->netdev;

	/* Only continue if link was up previously */
	if (!netif_carrier_ok(netdev))
		return;
	netdev_info(netdev, "NIC Link is Down\n");
	netif_carrier_off(netdev);
}

/**
 * rnpgbe_process_link_subtask - Process a link-state update
 * @mucse: pointer to the device private structure
 **/
static void rnpgbe_process_link_subtask(struct mucse *mucse)
{
	bool link;
	int speed;
	u8 duplex;

	if (!rnpgbe_process_link_event(mucse, &link, &speed, &duplex))
		return;

	if (link)
		rnpgbe_link_is_up(mucse, speed, duplex);
	else
		rnpgbe_link_is_down(mucse);
}

/**
 * rnpgbe_service_task - Manages and runs subtasks
 * @work: pointer to work_struct containing our data
 **/
void rnpgbe_service_task(struct work_struct *work)
{
	struct mucse *mucse = container_of(work, struct mucse, serv_task.work);

	rnpgbe_process_link_subtask(mucse);
}