path: root/drivers/lightnvm/rrpc.c

/*
 * Copyright (C) 2015 IT University of Copenhagen
 * Initial release: Matias Bjorling <m@bjorling.me>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License version
 * 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * Implementation of a Round-robin page-based Hybrid FTL for Open-channel SSDs.
 */

#include "rrpc.h"

static struct kmem_cache *rrpc_gcb_cache, *rrpc_rq_cache;
static DECLARE_RWSEM(rrpc_lock);

static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
				struct nvm_rq *rqd, unsigned long flags);

#define rrpc_for_each_lun(rrpc, rlun, i) \
		for ((i) = 0, rlun = &(rrpc)->luns[0]; \
			(i) < (rrpc)->nr_luns; (i)++, rlun = &(rrpc)->luns[(i)])

static void rrpc_page_invalidate(struct rrpc *rrpc, struct rrpc_addr *a)
{
	struct nvm_tgt_dev *dev = rrpc->dev;
	struct rrpc_block *rblk = a->rblk;
	unsigned int pg_offset;

	lockdep_assert_held(&rrpc->rev_lock);

	if (a->addr == ADDR_EMPTY || !rblk)
		return;

	spin_lock(&rblk->lock);

	div_u64_rem(a->addr, dev->geo.sec_per_blk, &pg_offset);
	WARN_ON(test_and_set_bit(pg_offset, rblk->invalid_pages));
	rblk->nr_invalid_pages++;

	spin_unlock(&rblk->lock);

	rrpc->rev_trans_map[a->addr - rrpc->poffset].addr = ADDR_EMPTY;
}

static void rrpc_invalidate_range(struct rrpc *rrpc, sector_t slba,
							unsigned int len)
{
	sector_t i;

	spin_lock(&rrpc->rev_lock);
	for (i = slba; i < slba + len; i++) {
		struct rrpc_addr *gp = &rrpc->trans_map[i];

		rrpc_page_invalidate(rrpc, gp);
		gp->rblk = NULL;
	}
	spin_unlock(&rrpc->rev_lock);
}

static struct nvm_rq *rrpc_inflight_laddr_acquire(struct rrpc *rrpc,
					sector_t laddr, unsigned int pages)
{
	struct nvm_rq *rqd;
	struct rrpc_inflight_rq *inf;

	rqd = mempool_alloc(rrpc->rq_pool, GFP_ATOMIC);
	if (!rqd)
		return ERR_PTR(-ENOMEM);

	inf = rrpc_get_inflight_rq(rqd);
	if (rrpc_lock_laddr(rrpc, laddr, pages, inf)) {
		mempool_free(rqd, rrpc->rq_pool);
		return NULL;
	}

	return rqd;
}

static void rrpc_inflight_laddr_release(struct rrpc *rrpc, struct nvm_rq *rqd)
{
	struct rrpc_inflight_rq *inf = rrpc_get_inflight_rq(rqd);

	rrpc_unlock_laddr(rrpc, inf);

	mempool_free(rqd, rrpc->rq_pool);
}

static void rrpc_discard(struct rrpc *rrpc, struct bio *bio)
{
	sector_t slba = bio->bi_iter.bi_sector / NR_PHY_IN_LOG;
	sector_t len = bio->bi_iter.bi_size / RRPC_EXPOSED_PAGE_SIZE;
	struct nvm_rq *rqd;

	while (1) {
		rqd = rrpc_inflight_laddr_acquire(rrpc, slba, len);
		if (rqd)
			break;

		schedule();
	}

	if (IS_ERR(rqd)) {
		pr_err("rrpc: unable to acquire inflight IO\n");
		bio_io_error(bio);
		return;
	}

	rrpc_invalidate_range(rrpc, slba, len);
	rrpc_inflight_laddr_release(rrpc, rqd);
}

static int block_is_full(struct rrpc *rrpc, struct rrpc_block *rblk)
{
	struct nvm_tgt_dev *dev = rrpc->dev;

	return (rblk->next_page == dev->geo.sec_per_blk);
}

/* Calculate relative addr for the given block, considering instantiated LUNs */
static u64 block_to_rel_addr(struct rrpc *rrpc, struct rrpc_block *rblk)
{
	struct nvm_tgt_dev *dev = rrpc->dev;
	struct nvm_block *blk = rblk->parent;
	int lun_blk = blk->id % (dev->geo.blks_per_lun * rrpc->nr_luns);

	return lun_blk * dev->geo.sec_per_blk;
}

/* Calculate global addr for the given block */
static u64 block_to_addr(struct rrpc *rrpc, struct rrpc_block *rblk)
{
	struct nvm_tgt_dev *dev = rrpc->dev;
	struct nvm_block *blk = rblk->parent;

	return blk->id * dev->geo.sec_per_blk;
}

static struct ppa_addr rrpc_ppa_to_gaddr(struct nvm_tgt_dev *dev, u64 addr)
{
	struct ppa_addr paddr;

	paddr.ppa = addr;
	return linear_to_generic_addr(&dev->geo, paddr);
}

/* requires lun->lock taken */
static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *new_rblk,
						struct rrpc_block **cur_rblk)
{
	struct rrpc *rrpc = rlun->rrpc;

	if (*cur_rblk) {
		spin_lock(&(*cur_rblk)->lock);
		WARN_ON(!block_is_full(rrpc, *cur_rblk));
		spin_unlock(&(*cur_rblk)->lock);
	}
	*cur_rblk = new_rblk;
}

static struct nvm_block *__rrpc_get_blk(struct rrpc *rrpc,
							struct rrpc_lun *rlun)
{
	struct nvm_lun *lun = rlun->parent;
	struct nvm_block *blk = NULL;

	if (list_empty(&lun->free_list))
		goto out;

	blk = list_first_entry(&lun->free_list, struct nvm_block, list);

	list_move_tail(&blk->list, &lun->used_list);
	blk->state = NVM_BLK_ST_TGT;
	lun->nr_free_blocks--;

out:
	return blk;
}

static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun,
							unsigned long flags)
{
	struct nvm_tgt_dev *dev = rrpc->dev;
	struct nvm_lun *lun = rlun->parent;
	struct nvm_block *blk;
	struct rrpc_block *rblk;
	int is_gc = flags & NVM_IOTYPE_GC;

	spin_lock(&rlun->lock);
	if (!is_gc && lun->nr_free_blocks < rlun->reserved_blocks) {
		pr_err("nvm: rrpc: cannot give block to non GC request\n");
		spin_unlock(&rlun->lock);
		return NULL;
	}

	blk = __rrpc_get_blk(rrpc, rlun);
	if (!blk) {
		pr_err("nvm: rrpc: cannot get new block\n");
		spin_unlock(&rlun->lock);
		return NULL;
	}
	spin_unlock(&rlun->lock);

	rblk = rrpc_get_rblk(rlun, blk->id);
	blk->priv = rblk;
	bitmap_zero(rblk->invalid_pages, dev->geo.sec_per_blk);
	rblk->next_page = 0;
	rblk->nr_invalid_pages = 0;
	atomic_set(&rblk->data_cmnt_size, 0);

	return rblk;
}

static void rrpc_put_blk(struct rrpc *rrpc, struct rrpc_block *rblk)
{
	struct nvm_block *blk = rblk->parent;
	struct rrpc_lun *rlun = rblk->rlun;
	struct nvm_lun *lun = rlun->parent;

	spin_lock(&rlun->lock);
	if (blk->state & NVM_BLK_ST_TGT) {
		list_move_tail(&blk->list, &lun->free_list);
		lun->nr_free_blocks++;
		blk->state = NVM_BLK_ST_FREE;
	} else if (blk->state & NVM_BLK_ST_BAD) {
		list_move_tail(&blk->list, &lun->bb_list);
		blk->state = NVM_BLK_ST_BAD;
	} else {
		WARN_ON_ONCE(1);
		pr_err("rrpc: erroneous block type (%lu -> %u)\n",
							blk->id, blk->state);
		list_move_tail(&blk->list, &lun->bb_list);
	}
	spin_unlock(&rlun->lock);
}

static void rrpc_put_blks(struct rrpc *rrpc)
{
	struct rrpc_lun *rlun;
	int i;

	for (i = 0; i < rrpc->nr_luns; i++) {
		rlun = &rrpc->luns[i];
		if (rlun->cur)
			rrpc_put_blk(rrpc, rlun->cur);
		if (rlun->gc_cur)
			rrpc_put_blk(rrpc, rlun->gc_cur);
	}
}

static struct rrpc_lun *get_next_lun(struct rrpc *rrpc)
{
	int next = atomic_inc_return(&rrpc->next_lun);

	return &rrpc->luns[next % rrpc->nr_luns];
}

static void rrpc_gc_kick(struct rrpc *rrpc)
{
	struct rrpc_lun *rlun;
	unsigned int i;

	for (i = 0; i < rrpc->nr_luns; i++) {
		rlun = &rrpc->luns[i];
		queue_work(rrpc->krqd_wq, &rlun->ws_gc);
	}
}

/*
 * timed GC every interval.
 */
static void rrpc_gc_timer(unsigned long data)
{
	struct rrpc *rrpc = (struct rrpc *)data;

	rrpc_gc_kick(rrpc);
	mod_timer(&rrpc->gc_timer, jiffies + msecs_to_jiffies(10));
}

static void rrpc_end_sync_bio(struct bio *bio)
{
	struct completion *waiting = bio->bi_private;

	if (bio->bi_error)
		pr_err("nvm: gc requ