path: root/block/blk-mq.c

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>
#include <linux/llist.h>
#include <linux/list_sort.h>
#include <linux/cpu.h>
#include <linux/cache.h>
#include <linux/sched/sysctl.h>
#include <linux/delay.h>

#include <trace/events/block.h>

#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-tag.h"

static DEFINE_MUTEX(all_q_mutex);
static LIST_HEAD(all_q_list);

static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx);

static struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
					   unsigned int cpu)
{
	return per_cpu_ptr(q->queue_ctx, cpu);
}

/*
 * This assumes per-cpu software queueing queues. They could be per-node
 * as well, for instance. For now this is hardcoded as-is. Note that we don't
 * care about preemption, since we know the ctx's are persistent. This does
 * mean that we can't rely on ctx always matching the currently running CPU.
 */
static struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q)
{
	return __blk_mq_get_ctx(q, get_cpu());
}

static void blk_mq_put_ctx(struct blk_mq_ctx *ctx)
{
	put_cpu();
}

/*
 * Check if any of the ctx's have pending work in this hardware queue
 */
static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
{
	unsigned int i;

	for (i = 0; i < hctx->nr_ctx_map; i++)
		if (hctx->ctx_map[i])
			return true;

	return false;
}

/*
 * Mark this ctx as having pending work in this hardware queue
 */
static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
				     struct blk_mq_ctx *ctx)
{
	if (!test_bit(ctx->index_hw, hctx->ctx_map))
		set_bit(ctx->index_hw, hctx->ctx_map);
}

static struct request *__blk_mq_alloc_request(struct blk_mq_hw_ctx *hctx,
					      gfp_t gfp, bool reserved)
{
	struct request *rq;
	unsigned int tag;

	tag = blk_mq_get_tag(hctx->tags, gfp, reserved);
	if (tag != BLK_MQ_TAG_FAIL) {
		rq = hctx->tags->rqs[tag];
		blk_rq_init(hctx->queue, rq);
		rq->tag = tag;

		return rq;
	}

	return NULL;
}

static int blk_mq_queue_enter(struct request_queue *q)
{
	int ret;

	__percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
	smp_wmb();
	/* we have problems to freeze the queue if it's initializing */
	if (!blk_queue_bypass(q) || !blk_queue_init_done(q))
		return 0;

	__percpu_counter_add(&q->mq_usage_counter, -1, 1000000);

	spin_lock_irq(q->queue_lock);
	ret = wait_event_interruptible_lock_irq(q->mq_freeze_wq,
		!blk_queue_bypass(q) || blk_queue_dying(q),
		*q->queue_lock);
	/* inc usage with lock hold to avoid freeze_queue runs here */
	if (!ret && !blk_queue_dying(q))
		__percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
	else if (blk_queue_dying(q))
		ret = -ENODEV;
	spin_unlock_irq(q->queue_lock);

	return ret;
}

static void blk_mq_queue_exit(struct request_queue *q)
{
	__percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
}

static void __blk_mq_drain_queue(struct request_queue *q)
{
	while (true) {
		s64 count;

		spin_lock_irq(q->queue_lock);
		count = percpu_counter_sum(&q->mq_usage_counter);
		spin_unlock_irq(q->queue_lock);

		if (count == 0)
			break;
		blk_mq_run_queues(q, false);
		msleep(10);
	}
}

/*
 * Guarantee no request is in use, so we can change any data structure of
 * the queue afterward.
 */
static void blk_mq_freeze_queue(struct request_queue *q)
{
	bool drain;

	spin_lock_irq(q->queue_lock);
	drain = !q->bypass_depth++;
	queue_flag_set(QUEUE_FLAG_BYPASS, q);
	spin_unlo