path: root/kernel/smp.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Generic helpers for smp ipi calls
 *
 * (C) Jens Axboe <jens.axboe@oracle.com> 2008
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/irq_work.h>
#include <linux/rcupdate.h>
#include <linux/rculist.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/percpu.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/gfp.h>
#include <linux/smp.h>
#include <linux/cpu.h>
#include <linux/sched.h>
#include <linux/sched/idle.h>
#include <linux/hypervisor.h>
#include <linux/sched/clock.h>
#include <linux/nmi.h>
#include <linux/sched/debug.h>
#include <linux/jump_label.h>

#include "smpboot.h"
#include "sched/smp.h"

#define CSD_TYPE(_csd)	((_csd)->node.u_flags & CSD_FLAG_TYPE_MASK)

#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
union cfd_seq_cnt {
	u64		val;
	struct {
		u64	src:16;
		u64	dst:16;
#define CFD_SEQ_NOCPU	0xffff
		u64	type:4;
#define CFD_SEQ_QUEUE	0
#define CFD_SEQ_IPI	1
#define CFD_SEQ_NOIPI	2
#define CFD_SEQ_PING	3
#define CFD_SEQ_PINGED	4
#define CFD_SEQ_HANDLE	5
#define CFD_SEQ_DEQUEUE	6
#define CFD_SEQ_IDLE	7
#define CFD_SEQ_GOTIPI	8
#define CFD_SEQ_HDLEND	9
		u64	cnt:28;
	}		u;
};

static char *seq_type[] = {
	[CFD_SEQ_QUEUE]		= "queue",
	[CFD_SEQ_IPI]		= "ipi",
	[CFD_SEQ_NOIPI]		= "noipi",
	[CFD_SEQ_PING]		= "ping",
	[CFD_SEQ_PINGED]	= "pinged",
	[CFD_SEQ_HANDLE]	= "handle",
	[CFD_SEQ_DEQUEUE]	= "dequeue (src CPU 0 == empty)",
	[CFD_SEQ_IDLE]		= "idle",
	[CFD_SEQ_GOTIPI]	= "gotipi",
	[CFD_SEQ_HDLEND]	= "hdlend (src CPU 0 == early)",
};

struct cfd_seq_local {
	u64	ping;
	u64	pinged;
	u64	handle;
	u64	dequeue;
	u64	idle;
	u64	gotipi;
	u64	hdlend;
};
#endif

struct cfd_percpu {
	call_single_data_t	csd;
#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
	u64	seq_queue;
	u64	seq_ipi;
	u64	seq_noipi;
#endif
};

struct call_function_data {
	struct cfd_percpu	__percpu *pcpu;
	cpumask_var_t		cpumask;
	cpumask_var_t		cpumask_ipi;
};

static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data);

static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);

static void flush_smp_call_function_queue(bool warn_cpu_offline);

int smpcfd_prepare_cpu(unsigned int cpu)
{
	struct call_function_data *cfd = &per_cpu(cfd_data, cpu);

	if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
				     cpu_to_node(cpu)))
		return -ENOMEM;
	if (!zalloc_cpumask_var_node(&cfd->cpumask_ipi, GFP_KERNEL,
				     cpu_to_node(cpu))) {
		free_cpumask_var(cfd->cpumask);
		return -ENOMEM;
	}
	cfd->pcpu = alloc_percpu(struct cfd_percpu);
	if (!cfd->pcpu) {
		free_cpumask_var(cfd->cpumask);
		free_cpumask_var(cfd->cpumask_ipi);
		return -ENOMEM;
	}

	return 0;
}

int smpcfd_dead_cpu(unsigned int cpu)
{
	struct call_function_data *cfd = &per_cpu(cfd_data, cpu);

	free_cpumask_var(cfd->cpumask);
	free_cpumask_var(cfd->cpumask_ipi);
	free_percpu(cfd->pcpu);
	return 0;
}

int smpcfd_dying_cpu(unsigned int cpu)
{
	/*
	 * The IPIs for the smp-call-function callbacks queued by other
	 * CPUs might arrive late, either due to hardware latencies or
	 * because this CPU disabled interrupts (inside stop-machine)
	 * before the IPIs were sent. So flush out any pending callbacks
	 * explicitly (without waiting for the IPIs to arrive), to
	 * ensure that the outgoing CPU doesn't go offline with work
	 * still pending.
	 */
	flush_smp_call_function_queue(false);
	irq_work_run();
	return 0;
}

void __init call_function_init(void)
{
	int i;

	for_each_possible_cpu(