// SPDX-License-Identifier: (GPL-2.0 OR MIT)
/* Google virtual Ethernet (gve) driver
*
* Copyright (C) 2015-2021 Google, Inc.
*/
#include "gve.h"
#include "gve_adminq.h"
#include "gve_utils.h"
#include "gve_dqo.h"
#include <net/ip.h>
#include <linux/tcp.h>
#include <linux/slab.h>
#include <linux/skbuff.h>
/* Returns true if tx_bufs are available. */
static bool gve_has_free_tx_qpl_bufs(struct gve_tx_ring *tx, int count)
{
int num_avail;
if (!tx->dqo.qpl)
return true;
num_avail = tx->dqo.num_tx_qpl_bufs -
(tx->dqo_tx.alloc_tx_qpl_buf_cnt -
tx->dqo_tx.free_tx_qpl_buf_cnt);
if (count <= num_avail)
return true;
/* Update cached value from dqo_compl. */
tx->dqo_tx.free_tx_qpl_buf_cnt =
atomic_read_acquire(&tx->dqo_compl.free_tx_qpl_buf_cnt);
num_avail = tx->dqo.num_tx_qpl_bufs -
(tx->dqo_tx.alloc_tx_qpl_buf_cnt -
tx->dqo_tx.free_tx_qpl_buf_cnt);
return count <= num_avail;
}
static s16
gve_alloc_tx_qpl_buf(struct gve_tx_ring *tx)
{
s16 index;
index = tx->dqo_tx.free_tx_qpl_buf_head;
/* No TX buffers available, try to steal the list from the
* completion handler.
*/
if (unlikely(index == -1)) {
tx->dqo_tx.free_tx_qpl_buf_head =
atomic_xchg(&tx->dqo_compl.free_tx_qpl_buf_head, -1);
index = tx->dqo_tx.free_tx_qpl_buf_head;
if (unlikely(index == -1))
return index;
}
/* Remove TX buf from free list */
tx->dqo_tx.free_tx_qpl_buf_head = tx->dqo.tx_qpl_buf_next[index];
return index;
}
static void
gve_free_tx_qpl_bufs(struct gve_tx_ring *tx,
struct gve_tx_pending_packet_dqo *pkt)
{
s16 index;
int i;
if (!pkt->num_bufs)
return;
index = pkt->tx_qpl_buf_ids[0];
/* Create a linked list of buffers to be added to the free list */
for (i = 1; i < pkt->num_bufs; i++) {
tx->dqo.tx_qpl_buf_next[index] = pkt->tx_qpl_buf_ids[i];
index = pkt->tx_qpl_buf_ids[i];
}
while (true) {
s16 old_head = atomic_read_acquire(&tx->dqo_compl.free_tx_qpl_buf_head);
tx->dqo.tx_qpl_buf_next[index] = old_head;
if (atomic_cmpxchg(&tx->dqo_compl.free_tx_qpl_buf_head,
old_head,
pkt->tx_qpl_buf_ids[0]) == old_head) {
break;
}
}
atomic_add(pkt->num_bufs