path: root/drivers/net/ethernet/google/gve/gve_rx.c

// SPDX-License-Identifier: (GPL-2.0 OR MIT)
/* Google virtual Ethernet (gve) driver
 *
 * Copyright (C) 2015-2021 Google, Inc.
 */

#include "gve.h"
#include "gve_adminq.h"
#include "gve_utils.h"
#include <linux/etherdevice.h>
#include <linux/filter.h>
#include <net/xdp.h>
#include <net/xdp_sock_drv.h>

static void gve_rx_free_buffer(struct device *dev,
			       struct gve_rx_slot_page_info *page_info,
			       union gve_rx_data_slot *data_slot)
{
	dma_addr_t dma = (dma_addr_t)(be64_to_cpu(data_slot->addr) &
				      GVE_DATA_SLOT_ADDR_PAGE_MASK);

	page_ref_sub(page_info->page, page_info->pagecnt_bias - 1);
	gve_free_page(dev, page_info->page, dma, DMA_FROM_DEVICE);
}

static void gve_rx_unfill_pages(struct gve_priv *priv,
				struct gve_rx_ring *rx,
				struct gve_rx_alloc_rings_cfg *cfg)
{
	u32 slots = rx->mask + 1;
	int i;

	if (!rx->data.page_info)
		return;

	if (rx->data.raw_addressing) {
		for (i = 0; i < slots; i++)
			gve_rx_free_buffer(&priv->pdev->dev, &rx->data.page_info[i],
					   &rx->data.data_ring[i]);
	} else {
		for (i = 0; i < slots; i++)
			page_ref_sub(rx->data.page_info[i].page,
				     rx->data.page_info[i].pagecnt_bias - 1);

		for (i = 0; i < rx->qpl_copy_pool_mask + 1; i++) {
			page_ref_sub(rx->qpl_copy_pool[i].page,
				     rx->qpl_copy_pool[i].pagecnt_bias - 1);
			put_page(rx->qpl_copy_pool[i].page);
		}
	}
	kvfree(rx->data.page_info);
	rx->data.page_info = NULL;
}

static void gve_rx_ctx_clear(struct gve_rx_ctx *ctx)
{
	ctx->skb_head = NULL;
	ctx->skb_tail = NULL;
	ctx->total_size = 0;
	ctx->frag_cnt = 0;
	ctx->drop_pkt = false;
}

static void gve_rx_init_ring_state_gqi(struct gve_rx_ring *rx)
{
	rx->desc.seqno = 1;
	rx->cnt = 0;
	gve_rx_ctx_clear(&rx->ctx);
}

static void gve_rx_reset_ring_gqi(struct gve_priv *priv, int idx)
{
	struct gve_rx_ring *rx = &priv->rx[idx];
	const u32 slots = priv->rx_desc_cnt;
	size_t size;

	/* Reset desc ring */
	if (rx->desc.desc_ring) {
		size = slots * sizeof(rx->desc.desc_ring[0]);
		memset(rx->desc.desc_ring, 0, size);
	}

	/* Reset q_resources */
	if (rx->q_resources)
		memset(rx->q_resources, 0, sizeof(*rx->q_resources));

	gve_rx_init_ring_state_gqi(rx);
}

void gve_rx_stop_ring_gqi(struct gve_priv *priv, int idx)
{
	int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);

	if (!gve_rx_was_added_to_block(priv, idx))
		return;

	gve_remove_napi(priv, ntfy_idx);
	gve_rx_remove_from_block(priv, idx);
	gve_rx_reset_ring_gqi(priv, idx);
}

void gve_rx_free_ring_gqi(struct gve_priv *priv, struct gve_rx_ring *rx,
			  struct gve_rx_alloc_rings_cfg *cfg)
{
	struct device *dev = &priv->pdev->dev;
	u32 slots = rx->mask + 1;
	int idx = rx->q_num;
	size_t bytes;
	u32 qpl_id;

	if (rx->desc.desc_ring) {
		bytes = sizeof(struct gve_rx_desc) * cfg->ring_size;
		dma_free_coherent(dev, bytes, rx->desc.desc_ring, rx->desc.bus);
		rx->desc.desc_ring = NULL;
	}

	if (rx->q_resources) {
		dma_free_coherent(dev, sizeof(*rx->q_resources),
				  rx->q_resources, rx->q_resources_bus);
		rx->q_resources = NULL;
	}

	gve_rx_unfill_pages(priv, rx, cfg);

	if (rx->data.data_ring) {
		bytes = sizeof(*rx->data.data_ring) * slots;
		dma_free_coherent(dev, bytes, rx->data.data_ring,
				  rx->data.data_bus