path: root/fs/bcachefs/move.c

// SPDX-License-Identifier: GPL-2.0

#include "bcachefs.h"
#include "alloc_background.h"
#include "alloc_foreground.h"
#include "backpointers.h"
#include "bkey_buf.h"
#include "btree_gc.h"
#include "btree_io.h"
#include "btree_update.h"
#include "btree_update_interior.h"
#include "btree_write_buffer.h"
#include "compress.h"
#include "disk_groups.h"
#include "ec.h"
#include "errcode.h"
#include "error.h"
#include "inode.h"
#include "io_read.h"
#include "io_write.h"
#include "journal_reclaim.h"
#include "keylist.h"
#include "move.h"
#include "replicas.h"
#include "snapshot.h"
#include "super-io.h"
#include "trace.h"

#include <linux/ioprio.h>
#include <linux/kthread.h>

const char * const bch2_data_ops_strs[] = {
#define x(t, n, ...) [n] = #t,
	BCH_DATA_OPS()
#undef x
	NULL
};

static void trace_move_extent2(struct bch_fs *c, struct bkey_s_c k,
			       struct bch_io_opts *io_opts,
			       struct data_update_opts *data_opts)
{
	if (trace_move_extent_enabled()) {
		struct printbuf buf = PRINTBUF;

		bch2_bkey_val_to_text(&buf, c, k);
		prt_newline(&buf);
		bch2_data_update_opts_to_text(&buf, c, io_opts, data_opts);
		trace_move_extent(c, buf.buf);
		printbuf_exit(&buf);
	}
}

static void trace_move_extent_read2(struct bch_fs *c, struct bkey_s_c k)
{
	if (trace_move_extent_read_enabled()) {
		struct printbuf buf = PRINTBUF;

		bch2_bkey_val_to_text(&buf, c, k);
		trace_move_extent_read(c, buf.buf);
		printbuf_exit(&buf);
	}
}

struct moving_io {
	struct list_head		read_list;
	struct list_head		io_list;
	struct move_bucket_in_flight	*b;
	struct closure			cl;
	bool				read_completed;

	unsigned			read_sectors;
	unsigned			write_sectors;

	struct bch_read_bio		rbio;

	struct data_update		write;
	/* Must be last since it is variable size */
	struct bio_vec			bi_inline_vecs[];
};

static void move_free(struct moving_io *io)
{
	struct moving_context *ctxt = io->write.ctxt;

	if (io->b)
		atomic_dec(&io->b->count);

	bch2_data_update_exit(&io->write);

	mutex_lock(&ctxt->lock);
	list_del(&io->io_list);
	wake_up(&ctxt->wait);
	mutex_unlock(&ctxt->lock);

	kfree(io);
}

static void move_write_done(struct bch_write_op *op)
{
	struct moving_io *io = container_of(op, struct moving_io, write.op);
	struct moving_context *ctxt = io->write.ctxt;

	if (io->write.op.error)
		ctxt->write_error = true;

	atomic_sub(io->write_sectors, &io->write.ctxt->write_sectors);
	atomic_dec(&io->write.ctxt->write_ios);
	move_free(io);
	closure_put(&ctxt->cl);
}

static void move_write(struct moving_io *io)
{
	if (unlikely(io->rbio.bio.bi_status || io->rbio.hole)) {
		move_free(io);
		return;
	}

	if (trace_move_extent_write_enabled()) {
		struct bch_fs *c = io->write.op.c;
		struct printbuf buf = PRINTBUF;

		bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(io->write.k.k));
		trace_move_extent_write(c, buf.buf);
		printbuf_exit(&buf);
	}

	closure_get(&io->write.ctxt->cl);
	atomic_add(io->write_sectors, &io->write.ctxt->write_sectors);
	atomic_inc(&io->write.ctxt->write_ios);

	bch2_data_update_read_done(&io->write