// SPDX-License-Identifier: GPL-2.0-or-later
/* Broadcom NetXtreme-C/E network driver.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation.
 */

#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <net/netdev_queues.h>
#include <net/ip.h>
#include <net/ipv6.h>
#include <net/udp.h>
#include <net/tso.h>
#include <linux/bnxt/hsi.h>

#include "bnxt.h"
#include "bnxt_gso.h"

static u32 bnxt_sw_gso_lhint(unsigned int len)
{
	if (len <= 512)
		return TX_BD_FLAGS_LHINT_512_AND_SMALLER;
	else if (len <= 1023)
		return TX_BD_FLAGS_LHINT_512_TO_1023;
	else if (len <= 2047)
		return TX_BD_FLAGS_LHINT_1024_TO_2047;
	else
		return TX_BD_FLAGS_LHINT_2048_AND_LARGER;
}

static struct tx_bd *bnxt_sw_gso_data_bd(struct bnxt *bp,
					 struct bnxt_tx_ring_info *txr,
					 u16 prod, dma_addr_t addr,
					 unsigned int len)
{
	struct tx_bd *txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
	struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];

	txbd->tx_bd_haddr = cpu_to_le64(addr);
	txbd->tx_bd_len_flags_type = cpu_to_le32(len << TX_BD_LEN_SHIFT);
	txbd->tx_bd_opaque = 0;

	dma_unmap_addr_set(tx_buf, mapping, addr);
	dma_unmap_len_set(tx_buf, len, 0);
	tx_buf->skb = NULL;
	tx_buf->is_sw_gso = 0;

	return txbd;
}

/* Transmit an skb requiring software UDP segmentation.
 *
 * Returns 1 if the skb was queued and new BDs were produced, 0 if the skb
 * was dropped, or -1 if the ring is full and the skb should be retried.
 * The caller owns the doorbell for all three cases.
 */
int bnxt_sw_udp_gso_xmit(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
			 struct netdev_queue *txq, struct sk_buff *skb)
{
	unsigned int last_unmap_len __maybe_unused = 0;
	dma_addr_t last_unmap_addr __maybe_unused = 0;
	struct bnxt_sw_tx_bd *last_unmap_buf = NULL;
	unsigned int hdr_len, mss, num_segs;
	struct pci_dev *pdev = bp->pdev;
	unsigned int total_payload;
	struct tso_dma_map map;
	u32 vlan_tag_flags = 0;
	int i, bds_needed;
	struct tso_t tso;
	u16 cfa_action;
	__le32 csum;
	u16 prod;

	hdr_len = tso_start(skb, &tso);
	mss = skb_shinfo(skb)->gso_size;
	total_payload = skb->len - hdr_len;
	num_segs = DIV_ROUND_UP(total_payload, mss);

	if (unlikely(num_segs <= 1))
		goto drop;

	/* bnxt_features_check() makes the stack segment these, but paths
	 * that skip validate_xmit_skb() (e.g. generic XDP_TX of a GRO'd
	 * skb) can still get here. Padding every segment would need more
	 * BDs than bds_needed reserves below, so drop instead.
	 */
	if (unlikely(bnxt_sw_gso_pad_len(hdr_len, mss)))
		goto drop;

	/* Upper bound on the number of descriptors needed.
	 *
	 * Each segment uses 1 long BD + 1 ext BD + payload BDs, which is
	 * at most num_segs + nr_frags (each frag boundary crossing adds at
	 * most 1 extra BD). The last segment may need 1 pad BD.
	 */
	bds_needed = 3 * num_segs + skb_shinfo(skb)->nr_frags + 1;

	if (unlikely(bnxt_tx_avail(bp, txr) < bds_needed)) {
		netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
				   bp->tx_wake_thresh);
		return -1;
	}

	/* BD backpressure alone cannot prevent overwriting in-flight
	 * headers in the inline buffer. Check slot availability directly.
	 */
	if (!netif_txq_maybe_stop(txq, bnxt_inline_avail(txr),
				  num_segs, num_segs))
		return -1;

	if (unlikely(tso_dma_map_init(&map, &pdev->dev, skb, hdr_len)))
		goto drop;

	cfa_action = bnxt_xmit_get_cfa_action(skb);
	if (skb_vlan_tag_present(skb)) {
		vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
				 skb_vlan_tag_get(skb);
		if (skb->vlan_proto == htons(ETH_P_8021Q))
			vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
	}

	csum = cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
	if (!tso.ipv6)
		csum |= cpu_to_le32(TX_BD_FLAGS_IP_CKSUM);

	prod = txr->tx_prod;

	for (i = 0; i < num_segs; i++) {
		unsigned int seg_payload = min_t(unsigned int, mss,
						 total_payload - i * mss);
		u16 slot = (txr->tx_inline_prod + i) &
			   (BNXT_SW_USO_MAX_SEGS - 1);
		struct bnxt_sw_tx_bd *tx_buf;
		unsigned int mapping_len;
		dma_addr_t this_hdr_dma;
		unsigned int chunk_len;
		unsigned int offset;
		dma_addr_t dma_addr;
		struct tx_bd *txbd;
		struct udphdr *uh;
		unsigned int pad;
		void *this_hdr;
		int bd_count;
		bool last;
		u32 flags;

		last = (i == num_segs - 1);
		pad = bnxt_sw_gso_pad_len(hdr_len, seg_payload);
		offset = slot * TSO_HEADER_SIZE;
		this_hdr = txr->tx_inline_buf + offset;
		this_hdr_dma = txr->tx_inline_dma + offset;

		tso_build_hdr(skb, this_hdr, &tso, seg_payload, last);

		/* Zero stale csum fields copied from the original skb;
		 * HW offload recomputes from scratch.
		 */
		uh = this_hdr + skb_transport_offset(skb);
		uh->check = 0;
		if (!tso.ipv6) {
			struct iphdr *iph = this_hdr + skb_network_offset(skb);

			iph->check = 0;
		}

		/* Pad bytes follow the header in the inline slot. Zero them
		 * so stale header bytes from an earlier packet are not sent.
		 */
		if (pad)
			memset(this_hdr + hdr_len, 0, pad);

		dma_sync_single_for_device(&pdev->dev, this_hdr_dma,
					   hdr_len + pad, DMA_TO_DEVICE);

		bd_count = tso_dma_map_count(&map, seg_payload);
		if (pad)
			bd_count++;

		tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
		txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];

		tx_buf->skb = skb;
		tx_buf->nr_frags = bd_count;
		tx_buf->is_push = 0;
		tx_buf->is_ts_pkt = 0;

		dma_unmap_addr_set(tx_buf, mapping, this_hdr_dma);
		dma_unmap_len_set(tx_buf, len, 0);

		if (last) {
			tx_buf->is_sw_gso = BNXT_SW_GSO_LAST;
			tso_dma_map_completion_save(&map, &tx_buf->sw_gso_cstate);
		} else {
			tx_buf->is_sw_gso = BNXT_SW_GSO_MID;
		}

		flags = (hdr_len << TX_BD_LEN_SHIFT) |
			TX_BD_TYPE_LONG_TX_BD |
			TX_BD_CNT(2 + bd_count);

		flags |= bnxt_sw_gso_lhint(hdr_len + seg_payload);

		txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
		txbd->tx_bd_haddr = cpu_to_le64(this_hdr_dma);
		txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod,
						   2 + bd_count);

		prod = NEXT_TX(prod);
		bnxt_init_ext_bd(bp, txr, prod, csum,
				 vlan_tag_flags, cfa_action);

		/* set dma_unmap_len on the LAST BD touching each
		 * region. Since completions are in-order, the last segment
		 * completes after all earlier ones, so the unmap is safe.
		 */
		while (tso_dma_map_next(&map, &dma_addr, &chunk_len,
					&mapping_len, seg_payload)) {
			prod = NEXT_TX(prod);
			txbd = bnxt_sw_gso_data_bd(bp, txr, prod, dma_addr,
						   chunk_len);
			tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];

			if (mapping_len) {
				if (last_unmap_buf) {
					dma_unmap_addr_set(last_unmap_buf,
							   mapping,
							   last_unmap_addr);
					dma_unmap_len_set(last_unmap_buf,
							  len,
							  last_unmap_len);
				}
				last_unmap_addr = dma_addr;
				last_unmap_len = mapping_len;
			}
			last_unmap_buf = tx_buf;

			seg_payload -= chunk_len;
		}

		if (pad) {
			prod = NEXT_TX(prod);
			txbd = bnxt_sw_gso_data_bd(bp, txr, prod,
						   this_hdr_dma + hdr_len, pad);
		}

		txbd->tx_bd_len_flags_type |=
			cpu_to_le32(TX_BD_FLAGS_PACKET_END);

		prod = NEXT_TX(prod);
	}

	if (last_unmap_buf) {
		dma_unmap_addr_set(last_unmap_buf, mapping, last_unmap_addr);
		dma_unmap_len_set(last_unmap_buf, len, last_unmap_len);
	}

	txr->tx_inline_prod += num_segs;

	netdev_tx_sent_queue(txq, skb->len);

	WRITE_ONCE(txr->tx_prod, prod);

	if (unlikely(bnxt_tx_avail(bp, txr) <= bp->tx_wake_thresh))
		netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
				   bp->tx_wake_thresh);

	return 1;

drop:
	dev_kfree_skb_any(skb);
	dev_core_stats_tx_dropped_inc(bp->dev);
	return 0;
}