path: root/fs/bcachefs/btree_key_cache.c

// SPDX-License-Identifier: GPL-2.0

#include "bcachefs.h"
#include "btree_cache.h"
#include "btree_iter.h"
#include "btree_key_cache.h"
#include "btree_locking.h"
#include "btree_update.h"
#include "errcode.h"
#include "error.h"
#include "journal.h"
#include "journal_reclaim.h"
#include "trace.h"

#include <linux/sched/mm.h>

static inline bool btree_uses_pcpu_readers(enum btree_id id)
{
	return id == BTREE_ID_subvolumes;
}

static struct kmem_cache *bch2_key_cache;

static int bch2_btree_key_cache_cmp_fn(struct rhashtable_compare_arg *arg,
				       const void *obj)
{
	const struct bkey_cached *ck = obj;
	const struct bkey_cached_key *key = arg->key;

	return ck->key.btree_id != key->btree_id ||
		!bpos_eq(ck->key.pos, key->pos);
}

static const struct rhashtable_params bch2_btree_key_cache_params = {
	.head_offset	= offsetof(struct bkey_cached, hash),
	.key_offset	= offsetof(struct bkey_cached, key),
	.key_len	= sizeof(struct bkey_cached_key),
	.obj_cmpfn	= bch2_btree_key_cache_cmp_fn,
};

__flatten
inline struct bkey_cached *
bch2_btree_key_cache_find(struct bch_fs *c, enum btree_id btree_id, struct bpos pos)
{
	struct bkey_cached_key key = {
		.btree_id	= btree_id,
		.pos		= pos,
	};

	return rhashtable_lookup_fast(&c->btree_key_cache.table, &key,
				      bch2_btree_key_cache_params);
}

static bool bkey_cached_lock_for_evict(struct bkey_cached *ck)
{
	if (!six_trylock_intent(&ck->c.lock))
		return false;

	if (test_bit(BKEY_CACHED_DIRTY, &ck->flags)) {
		six_unlock_intent(&ck->c.lock);
		return false;
	}

	if (!six_trylock_write(&ck->c.lock)) {
		six_unlock_intent(&ck->c.lock);
		return false;
	}

	return true;
}

static void bkey_cached_evict(struct btree_key_cache *c,
			      struct bkey_cached *ck)
{
	BUG_ON(rhashtable_remove_fast(&c->table, &ck->hash,
				      bch2_btree_key_cache_params));
	memset(&ck->key, ~0, sizeof(ck->key));

	atomic_long_dec(&c->nr_keys);
}

static void bkey_cached_free(struct btree_key_cache *bc,
			     struct bkey_cached *ck)
{
	struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache);

	BUG_ON(test_bit(BKEY_CACHED_DIRTY, &ck->flags));

	ck->btree_trans_barrier_seq =
		start_poll_synchronize_srcu(&c->btree_trans_barrier);

	if (ck->c.lock.readers) {
		list_move_tail(&ck->list, &bc->freed_pcpu);
		bc->nr_freed_pcpu++;
	} else {
		list_move_tail(&ck->list, &bc->freed_nonpcpu);
		bc->nr_freed_nonpcpu++;
	}
	atomic_long_inc(&bc->nr_freed);

	kfree(ck->k);
	ck->k		= NULL;
	ck->u64s	= 0;

	six_unlock_write(&ck->c.lock);
	six_unlock_intent(&ck->c.lock);
}

#ifdef __KERNEL__
static void __bkey_cached_move_to_freelist_ordered(struct btree_key_cache *bc,
						   struct bkey_cached *ck)
{
	struct bkey_cached *pos;

	bc->nr_freed_nonpcpu++;

	list_for_each_entry_reverse(pos, &bc->freed_nonpcpu, list) {
		if (ULONG_CMP_GE(ck->btree_trans_barrier_seq,
				 pos->btree_trans_barrier_seq)) {
			list_move(&ck->list, &pos->list);
			return;
		}
	}

	list_move(&ck->list, &bc->freed_nonpcpu);
}
#endif

static void bkey_cached_move_to_freelist(struct btree_key_cache *bc,
					 struct bkey_cached *ck)
{
	BUG_ON(test_bit(BKEY_CACHED_DIRTY, &ck->flags));

	if (!ck->c.lock.readers) {
#ifdef __KERNEL__
		struct btree_key_cache_freelist *f;
		bool freed = false;

		preempt_disable();
		f = this_cpu_ptr(bc->pcpu_freed);

		if (f->nr < ARRAY_SIZE(f->objs)) {
			f->objs[f->nr++] = ck;
			freed = true;
		}
		preempt_enable();

		if (!freed) {
			mutex_lock(&bc->lock);
			preempt_disable();
			f = this_cpu_ptr(bc->pcpu_freed);

			while (f->nr > ARRAY_SIZE(f->objs) / 2) {
				struct bkey_cached *ck2 =