// SPDX-License-Identifier: GPL-2.0
/*
* Some low level IO code, and hacks for various block layer limitations
*
* Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
* Copyright 2012 Google, Inc.
*/
#include "bcachefs.h"
#include "alloc_background.h"
#include "alloc_foreground.h"
#include "btree_update.h"
#include "buckets.h"
#include "checksum.h"
#include "clock.h"
#include "compress.h"
#include "data_update.h"
#include "disk_groups.h"
#include "ec.h"
#include "error.h"
#include "io_read.h"
#include "io_misc.h"
#include "io_write.h"
#include "reflink.h"
#include "subvolume.h"
#include "trace.h"
#include <linux/random.h>
#include <linux/sched/mm.h>
#ifdef CONFIG_BCACHEFS_DEBUG
static unsigned bch2_read_corrupt_ratio;
module_param_named(read_corrupt_ratio, bch2_read_corrupt_ratio, uint, 0644);
MODULE_PARM_DESC(read_corrupt_ratio, "");
#endif
#ifndef CONFIG_BCACHEFS_NO_LATENCY_ACCT
static bool bch2_target_congested(struct bch_fs *c, u16 target)
{
const struct bch_devs_mask *devs;
unsigned d, nr = 0, total = 0;
u64 now = local_clock(), last;
s64 congested;
struct bch_dev *ca;
if (!target)
return false;
rcu_read_lock();
devs = bch2_target_to_mask(c, target) ?:
&c->rw_devs[BCH_DATA_user];
for_each_set_bit(d, devs->d, BCH_SB_MEMBERS_MAX) {
ca = rcu_dereference(c->devs[d]);
if (!ca)
continue;
congested = atomic_read(&ca->congested);
last = READ_ONCE(ca->congested_last);
if (time_after64(now, last))
congested -= (now - last) >> 12;
total += max(congested, 0LL);
nr++;
}
rcu_read_unlock();
return get_random_u32_below(nr * CONGESTED_MAX) < total;
}
#else
static bool bch2_target_congested(struct bch_fs *c, u16 target)
{
return false;
}
#endif
/* Cache promotion on read */
struct promote_op {
struct rcu_head rcu;
u64 start_time;
