// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2019 Mellanox Technologies. */

#ifdef HAVE_XSK_ZERO_COPY_SUPPORT
#include "rx.h"
#include "en/xdp.h"
#ifdef HAVE_XDP_SOCK_DRV_H
#include <net/xdp_sock_drv.h>
#else
#include <net/xdp_sock.h>
#endif
#include <linux/filter.h>
#ifdef HAVE_NET_PAGE_POOL_OLD_H
#include <net/page_pool.h>
#endif
#ifdef HAVE_NET_PAGE_POOL_TYPES_H
#include <net/page_pool/types.h>
#include <net/page_pool/helpers.h>
#endif

/* RX data path */

#ifdef HAVE_XSK_BUFF_ALLOC
static struct mlx5e_xdp_buff *xsk_buff_to_mxbuf(struct xdp_buff *xdp)
{
	/* mlx5e_xdp_buff shares its layout with xdp_buff_xsk
	 * and private mlx5e_xdp_buff fields fall into xdp_buff_xsk->cb
	 */
	return (struct mlx5e_xdp_buff *)xdp;
}
#endif

#ifndef HAVE_XSK_BUFF_ALLOC
bool mlx5e_xsk_pages_enough_umem(struct mlx5e_rq *rq, int count)
{
	/* Check in advance that we have enough frames, instead of allocating
	 * one-by-one, failing and moving frames to the Reuse Ring.
	 */
	return xsk_umem_has_addrs_rq(rq->umem, count);
}

int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq,
		struct mlx5e_alloc_unit *au)
{
	struct xdp_umem *umem = rq->umem;

	dma_addr_t addr;
	u64 handle;

	if (!xsk_umem_peek_addr_rq(umem, &handle))
		return -ENOMEM;

#ifdef HAVE_XSK_UMEM_ADJUST_OFFSET
	au->xsk.handle = xsk_umem_adjust_offset(umem, handle,
			rq->buff.umem_headroom);
#else
	au->xsk.handle = handle + rq->buff.umem_headroom;
#endif
	au->xsk.data = xdp_umem_get_data(umem, au->xsk.handle);

	/* No need to add headroom to the DMA address. In striding RQ case, we
	 * just provide pages for UMR, and headroom is counted at the setup
	 * stage when creating a WQE. In non-striding RQ case, headroom is
	 * accounted in mlx5e_alloc_rx_wqe.
	 */
	addr = xdp_umem_get_dma(umem, handle);
#if !defined(HAVE_PAGE_POOL_GET_DMA_ADDR) || !defined(HAVE_XSK_BUFF_GET_FRAME_DMA)
	au->addr = addr;
#endif

#ifdef HAVE_XSK_UMEM_RELEASE_ADDR_RQ
	xsk_umem_release_addr_rq(umem);
#else
	xsk_umem_discard_addr_rq(umem);
#endif

	dma_sync_single_for_device(rq->pdev, addr, PAGE_SIZE,
			DMA_BIDIRECTIONAL);
	return 0;
}

static inline void mlx5e_xsk_recycle_frame(struct mlx5e_rq *rq, u64 handle)
{
	xsk_umem_fq_reuse(rq->umem, handle & rq->umem->chunk_mask);
}

/* XSKRQ uses pages from UMEM, they must not be released. They are returned to
 * the userspace if possible, and if not, this function is called to reuse them
 * in the driver.
 */
void mlx5e_xsk_page_release(struct mlx5e_rq *rq,
		struct mlx5e_alloc_unit *au)
{
	mlx5e_xsk_recycle_frame(rq, au->xsk.handle);
}

/* Return a frame back to the hardware to fill in again. It is used by XDP when
 * the XDP program returns XDP_TX or XDP_REDIRECT not to an XSKMAP.
 */
void mlx5e_xsk_zca_free(struct zero_copy_allocator *zca, unsigned long handle)
{
	struct mlx5e_rq *rq = container_of(zca, struct mlx5e_rq, zca);

	mlx5e_xsk_recycle_frame(rq, handle);
}

void mlx5e_fill_xdp_buff_for_old_xsk(struct mlx5e_rq *rq, void *va, u16 headroom,
				     u32 len, struct xdp_buff *xdp,
				     struct mlx5e_alloc_unit *au);
#endif /* HAVE_XSK_BUFF_ALLOC */
#ifdef HAVE_XSK_BUFF_ALLOC_BATCH
int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
{
	struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix);
	struct mlx5e_icosq *icosq = rq->icosq;
	struct mlx5_wq_cyc *wq = &icosq->wq;
	struct mlx5e_umr_wqe *umr_wqe;
	int batch, i;
	u32 offset; /* 17-bit value with MTT. */
	u16 pi;

#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
	if (unlikely(!xsk_buff_can_alloc(rq->xsk_pool, rq->mpwqe.pages_per_wqe)))
#else
	if (unlikely(!xsk_buff_can_alloc(rq->umem, MLX5_MPWRQ_MAX_PAGES_PER_WQE)))
#endif
		goto err;

#ifdef HAVE_NO_REFCNT_BIAS
	BUILD_BUG_ON(sizeof(wi->alloc_units[0]) != sizeof(wi->alloc_units[0].xsk));
	XSK_CHECK_PRIV_TYPE(struct mlx5e_xdp_buff);
	batch = xsk_buff_alloc_batch(
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
				     rq->xsk_pool,
#else
				     rq->umem,
#endif
				     (struct xdp_buff **)wi->alloc_units,
				     rq->mpwqe.pages_per_wqe);

	/* If batch < pages_per_wqe, either:
	 * 1. Some (or all) descriptors were invalid.
	 * 2. dma_need_sync is true, and it fell back to allocating one frame.
	 * In either case, try to continue allocating frames one by one, until
	 * the first error, which will mean there are no more valid descriptors.
	 */
#else
	batch = 0;
#endif
	for (; batch < rq->mpwqe.pages_per_wqe; batch++) {
		wi->alloc_units[batch].xsk = 
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
			xsk_buff_alloc(rq->xsk_pool);
#else
			xsk_buff_alloc(rq->umem);
#endif
		if (unlikely(!wi->alloc_units[batch].xsk))
			goto err_reuse_batch;
	}

	pi = mlx5e_icosq_get_next_pi(icosq, rq->mpwqe.umr_wqebbs);
	umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi);
	memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe));

	if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED)) {
		for (i = 0; i < batch; i++) {
#ifdef HAVE_XDP_METADATA_OPS
			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units[i].xsk);
#endif
			dma_addr_t addr =
#if defined(HAVE_PAGE_POOL_GET_DMA_ADDR) && defined(HAVE_XSK_BUFF_GET_FRAME_DMA)
				xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);
#else
				wi->alloc_units[i].addr;
#endif

			umr_wqe->inline_mtts[i] = (struct mlx5_mtt) {
				.ptag = cpu_to_be64(addr | MLX5_EN_WR),
			};
#ifdef HAVE_XDP_METADATA_OPS
			mxbuf->rq = rq;
#endif
		}
	} else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED)) {
		for (i = 0; i < batch; i++) {
#ifdef HAVE_XDP_METADATA_OPS
			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units[i].xsk);
#endif
			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);

			umr_wqe->inline_ksms[i] = (struct mlx5_ksm) {
				.key = rq->mkey_be,
				.va = cpu_to_be64(addr),
			};
#ifdef HAVE_XDP_METADATA_OPS
			mxbuf->rq = rq;
#endif
		}
	} else if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE)) {
		u32 mapping_size = 1 << (rq->mpwqe.page_shift - 2);

		for (i = 0; i < batch; i++) {
#ifdef HAVE_XDP_METADATA_OPS
			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units[i].xsk);
#endif
			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);

			umr_wqe->inline_ksms[i << 2] = (struct mlx5_ksm) {
				.key = rq->mkey_be,
				.va = cpu_to_be64(addr),
			};
			umr_wqe->inline_ksms[(i << 2) + 1] = (struct mlx5_ksm) {
				.key = rq->mkey_be,
				.va = cpu_to_be64(addr + mapping_size),
			};
			umr_wqe->inline_ksms[(i << 2) + 2] = (struct mlx5_ksm) {
				.key = rq->mkey_be,
				.va = cpu_to_be64(addr + mapping_size * 2),
			};
			umr_wqe->inline_ksms[(i << 2) + 3] = (struct mlx5_ksm) {
				.key = rq->mkey_be,
				.va = cpu_to_be64(rq->wqe_overflow.addr),
			};
#ifdef HAVE_XDP_METADATA_OPS
			mxbuf->rq = rq;
#endif
		}
	} else {
		__be32 pad_size = cpu_to_be32((1 << rq->mpwqe.page_shift) -
					      rq->xsk_pool->chunk_size);
		__be32 frame_size = cpu_to_be32(rq->xsk_pool->chunk_size);

		for (i = 0; i < batch; i++) {
#ifdef HAVE_XDP_METADATA_OPS
			struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units[i].xsk);
#endif
			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);

			umr_wqe->inline_klms[i << 1] = (struct mlx5_klm) {
				.key = rq->mkey_be,
				.va = cpu_to_be64(addr),
				.bcount = frame_size,
			};
			umr_wqe->inline_klms[(i << 1) + 1] = (struct mlx5_klm) {
				.key = rq->mkey_be,
				.va = cpu_to_be64(rq->wqe_overflow.addr),
				.bcount = pad_size,
			};
#ifdef HAVE_XDP_METADATA_OPS
			mxbuf->rq = rq;
#endif
		}
	}

#ifdef HAVE_XDP_SUPPORT
	bitmap_zero(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe);
#endif
	wi->consumed_strides = 0;

	umr_wqe->ctrl.opmod_idx_opcode =
		cpu_to_be32((icosq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | MLX5_OPCODE_UMR);

	/* Optimized for speed: keep in sync with mlx5e_mpwrq_umr_entry_size. */
	offset = ix * rq->mpwqe.mtts_per_wqe;
	if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED))
		offset = offset * sizeof(struct mlx5_mtt) / MLX5_OCTWORD;
	else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_OVERSIZED))
		offset = offset * sizeof(struct mlx5_klm) * 2 / MLX5_OCTWORD;
	else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE))
		offset = offset * sizeof(struct mlx5_ksm) * 4 / MLX5_OCTWORD;
	umr_wqe->uctrl.xlt_offset = cpu_to_be16(offset);

	icosq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) {
		.wqe_type = MLX5E_ICOSQ_WQE_UMR_RX,
		.num_wqebbs = rq->mpwqe.umr_wqebbs,
		.umr.rq = rq,
	};

	icosq->pc += rq->mpwqe.umr_wqebbs;

	icosq->doorbell_cseg = &umr_wqe->ctrl;

	return 0;

err_reuse_batch:
	while (--batch >= 0)
		xsk_buff_free(wi->alloc_units[batch].xsk);

err:
	rq->stats->buff_alloc_err++;
	return -ENOMEM;
}
#endif
#ifdef HAVE_NO_REFCNT_BIAS
int mlx5e_xsk_alloc_rx_wqes_batched(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
{
	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
	struct xdp_buff **buffs;
	u32 contig, alloc;
	int i;

	/* mlx5e_init_frags_partition creates a 1:1 mapping between
	 * rq->wqe.frags and rq->wqe.alloc_units, which allows us to
	 * allocate XDP buffers straight into alloc_units.
	 */
	BUILD_BUG_ON(sizeof(rq->wqe.alloc_units[0]) !=
	             sizeof(rq->wqe.alloc_units[0].xsk));

	buffs = (struct xdp_buff **)rq->wqe.alloc_units;
	contig = mlx5_wq_cyc_get_size(wq) - ix;
	if (wqe_bulk <= contig) {
		alloc = xsk_buff_alloc_batch(rq->xsk_pool, buffs + ix, wqe_bulk);
	} else {
		alloc = xsk_buff_alloc_batch(rq->xsk_pool, buffs + ix, contig);
		if (likely(alloc == contig))
			alloc += xsk_buff_alloc_batch(rq->xsk_pool, buffs, wqe_bulk - contig);
	}

	for (i = 0; i < alloc; i++) {
		int j = mlx5_wq_cyc_ctr2ix(wq, ix + i);
		struct mlx5e_wqe_frag_info *frag;
		struct mlx5e_rx_wqe_cyc *wqe;
		dma_addr_t addr;

		wqe = mlx5_wq_cyc_get_wqe(wq, j);
		/* Assumes log_num_frags == 0. */
		frag = &rq->wqe.frags[j];

		addr = xsk_buff_xdp_get_frame_dma(frag->au->xsk);
		wqe->data[0].addr = cpu_to_be64(addr + rq->buff.headroom);
	}

	return alloc;
}
#endif

int mlx5e_xsk_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
{
	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
	int i;

	for (i = 0; i < wqe_bulk; i++) {
		int j = mlx5_wq_cyc_ctr2ix(wq, ix + i);
		struct mlx5e_wqe_frag_info *frag;
		struct mlx5e_rx_wqe_cyc *wqe;
		dma_addr_t addr;

		wqe = mlx5_wq_cyc_get_wqe(wq, j);
		/* Assumes log_num_frags == 0. */
		frag = &rq->wqe.frags[j];

#ifdef HAVE_XSK_BUFF_ALLOC
		frag->au->xsk =
#ifdef HAVE_NETDEV_BPF_XSK_BUFF_POOL
			xsk_buff_alloc(rq->xsk_pool);
#else
			xsk_buff_alloc(rq->umem);
#endif
		if (unlikely(!(frag->au->xsk)))
			return i;
#endif

		addr =
#if defined(HAVE_PAGE_POOL_GET_DMA_ADDR) && defined(HAVE_XSK_BUFF_GET_FRAME_DMA)
			xsk_buff_xdp_get_frame_dma(frag->au->xsk);
#else
			frag->au->addr;
#endif

		wqe->data[0].addr = cpu_to_be64(addr + rq->buff.headroom);
	}

	return wqe_bulk;
}

static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, struct xdp_buff *xdp)
{
	u32 totallen = xdp->data_end - xdp->data_meta;
	u32 metalen = xdp->data - xdp->data_meta;
	struct sk_buff *skb;

	skb = napi_alloc_skb(rq->cq.napi, totallen);
	if (unlikely(!skb)) {
		rq->stats->buff_alloc_err++;
		return NULL;
	}

	skb_put_data(skb, xdp->data_meta, totallen);

	if (metalen) {
		skb_metadata_set(skb, metalen);
		__skb_pull(skb, metalen);
	}

	return skb;
}

struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
		struct mlx5e_mpw_info *wi,
		struct mlx5_cqe64 *cqe,
		u16 cqe_bcnt,
		u32 head_offset,
		u32 page_idx)
{
#ifdef HAVE_XSK_BUFF_ALLOC
	struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->alloc_units[page_idx].xsk);
#else
	 struct xdp_buff xdp_old;
	 struct xdp_buff *xdp = &xdp_old;
	 struct mlx5e_alloc_unit *au = &wi->alloc_units[page_idx];
	 u16 rx_headroom = rq->buff.headroom - rq->buff.umem_headroom;
#ifdef HAVE_PAGE_POOL_GET_DMA_ADDR
	 dma_addr_t addr;
#endif
	 void *va, *data;
	 u32 frag_size;
#endif
#ifndef HAVE_XSK_BUFF_ALLOC
	 u32 cqe_bcnt32 = cqe_bcnt;
#endif
	struct bpf_prog *prog;

        /* Check packet size. Note LRO doesn't use linear SKB */
        if (unlikely(cqe_bcnt > rq->hw_mtu + rq->pet_hdr_size)) {
                rq->stats->oversize_pkts_sw_drop++;
                return NULL;
        }
		/* head_offset is not used in this function, because xdp->data and the
         * DMA address point directly to the necessary place. Furthermore, in
         * the current implementation, UMR pages are mapped to XSK frames, so
         * head_offset should always be 0.
         */
        WARN_ON_ONCE(head_offset);

#ifdef HAVE_XSK_BUFF_ALLOC
#ifdef HAVE_XDP_METADATA_OPS
        /* mxbuf->rq is set on allocation, but cqe is per-packet so set it here */
        mxbuf->cqe = cqe;
#endif
        xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt);
#ifdef HAVE_XSK_BUFF_DMA_SYNC_FOR_CPU_2_PARAMS
        xsk_buff_dma_sync_for_cpu(&mxbuf->xdp, rq->xsk_pool);
#else
	xsk_buff_dma_sync_for_cpu(&mxbuf->xdp);
#endif
#else
	va        = au->xsk.data;
	data      = va + rx_headroom;
	frag_size = rq->buff.headroom + cqe_bcnt32;
#ifdef HAVE_PAGE_POOL_GET_DMA_ADDR
	addr = page_pool_get_dma_addr(au->page);
	dma_sync_single_for_cpu(rq->pdev, addr, frag_size, DMA_BIDIRECTIONAL);
#else
	dma_sync_single_for_cpu(rq->pdev, au->addr, frag_size, DMA_BIDIRECTIONAL);
#endif
	mlx5e_fill_xdp_buff_for_old_xsk(rq, va, rx_headroom, cqe_bcnt, xdp, au);
#endif
#ifdef HAVE_XSK_BUFF_ALLOC
        net_prefetch(mxbuf->xdp.data);
#else
	net_prefetch(xdp->data);
#endif

        /* Possible flows:
         * - XDP_REDIRECT to XSKMAP:
         *   The page is owned by the userspace from now.
         * - XDP_TX and other XDP_REDIRECTs:
         *   The page was returned by ZCA and recycled.
         * - XDP_DROP:
         *   Recycle the page.
         * - XDP_PASS:
         *   Allocate an SKB, copy the data and recycle the page.
         *
         * Pages to be recycled go to the Reuse Ring on MPWQE deallocation. Its
         * size is the same as the Driver RX Ring's size, and pages for WQEs are
         * allocated first from the Reuse Ring, so it has enough space.
         */

        prog = rcu_dereference(rq->xdp_prog);
        if (likely(prog &&
#ifdef HAVE_XSK_BUFF_ALLOC
		   mlx5e_xdp_handle(rq, &wi->alloc_units[page_idx], prog, mxbuf
#else
		   mlx5e_xdp_handle_old(rq, &wi->alloc_units[page_idx], prog, xdp
#endif
		))) {
                if (likely(__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)))
                        __set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */
                return NULL; /* page/packet was consumed by XDP */
        }

        /* XDP_PASS: copy the data from the UMEM to a new SKB and reuse the
         * frame. On SKB allocation failure, NULL is returned.
         */
        return mlx5e_xsk_construct_skb(rq,
#ifdef HAVE_XSK_BUFF_ALLOC
				       &mxbuf->xdp);
#else
					xdp);
#endif
}

struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
                                              struct mlx5e_wqe_frag_info *wi,
                                              struct mlx5_cqe64 *cqe,
                                              u32 cqe_bcnt)
{
#ifdef HAVE_XSK_BUFF_ALLOC
        struct mlx5e_xdp_buff *mxbuf = xsk_buff_to_mxbuf(wi->au->xsk);
#else
	struct xdp_buff xdp_old;
	struct xdp_buff *xdp = &xdp_old;
	struct mlx5e_alloc_unit *au = wi->au;
	u16 rx_headroom = rq->buff.headroom - rq->buff.umem_headroom;
#ifdef HAVE_PAGE_POOL_GET_DMA_ADDR
	dma_addr_t addr;
#endif
	void *va, *data;
	u32 frag_size;
#endif
        struct bpf_prog *prog;

        /* wi->offset is not used in this function, because xdp->data and the
         * DMA address point directly to the necessary place. Furthermore, the
         * XSK allocator allocates frames per packet, instead of pages, so
         * wi->offset should always be 0.
         */
        WARN_ON_ONCE(wi->offset);
#ifdef HAVE_XSK_BUFF_ALLOC
#ifdef HAVE_XDP_METADATA_OPS
        /* mxbuf->rq is set on allocation, but cqe is per-packet so set it here */
        mxbuf->cqe = cqe;
 #endif
        xsk_buff_set_size(&mxbuf->xdp, cqe_bcnt);
#ifdef HAVE_XSK_BUFF_DMA_SYNC_FOR_CPU_2_PARAMS
        xsk_buff_dma_sync_for_cpu(&mxbuf->xdp, rq->xsk_pool);
#else
	xsk_buff_dma_sync_for_cpu(&mxbuf->xdp);
#endif
#else
	va        = au->xsk.data;
	data      = va + rx_headroom;
	frag_size = rq->buff.headroom + cqe_bcnt;
#ifdef HAVE_PAGE_POOL_GET_DMA_ADDR
	addr = page_pool_get_dma_addr(au->page);
	dma_sync_single_for_cpu(rq->pdev, addr, frag_size, DMA_BIDIRECTIONAL);
#else
	dma_sync_single_for_cpu(rq->pdev, au->addr, frag_size, DMA_BIDIRECTIONAL);
#endif
	mlx5e_fill_xdp_buff_for_old_xsk(rq, va, rx_headroom, cqe_bcnt, xdp, au);
#endif
#ifdef HAVE_XSK_BUFF_ALLOC
        net_prefetch(mxbuf->xdp.data);
#else
        net_prefetch(xdp->data);
#endif

        prog = rcu_dereference(rq->xdp_prog);
        if (likely(prog &&
#ifdef HAVE_XSK_BUFF_ALLOC
		   mlx5e_xdp_handle(rq, wi->au, prog, mxbuf
#else
		   mlx5e_xdp_handle_old(rq, wi->au, prog, xdp
#endif
				       )))
                return NULL; /* page/packet was consumed by XDP */

        /* XDP_PASS: copy the data from the UMEM to a new SKB. The frame reuse
         * will be handled by mlx5e_free_rx_wqe.
         * On SKB allocation failure, NULL is returned.
         */
        return mlx5e_xsk_construct_skb(rq,
#ifdef HAVE_XSK_BUFF_ALLOC
					&mxbuf->xdp);
#else
					xdp);
#endif
}
#endif /* HAVE_XSK_ZERO_COPY_SUPPORT*/
