// SPDX-License-Identifier: GPL-2.0
/*
* Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
* Copyright 2012 Google, Inc.
*/
#include "bcachefs.h"
#include "alloc_foreground.h"
#include "bkey_buf.h"
#include "bset.h"
#include "btree_update.h"
#include "buckets.h"
#include "checksum.h"
#include "clock.h"
#include "compress.h"
#include "debug.h"
#include "ec.h"
#include "error.h"
#include "extent_update.h"
#include "inode.h"
#include "io_write.h"
#include "journal.h"
#include "keylist.h"
#include "move.h"
#include "nocow_locking.h"
#include "rebalance.h"
#include "subvolume.h"
#include "super.h"
#include "super-io.h"
#include "trace.h"
#include <linux/blkdev.h>
#include <linux/prefetch.h>
#include <linux/random.h>
#include <linux/sched/mm.h>
#ifndef CONFIG_BCACHEFS_NO_LATENCY_ACCT
static inline void bch2_congested_acct(struct bch_dev *ca, u64 io_latency,
u64 now, int rw)
{
u64 latency_capable =
ca->io_latency[rw].quantiles.entries[QUANTILE_IDX(1)].m;
/* ideally we'd be taking into account the device's variance here: */
u64 latency_threshold = latency_capable << (rw == READ ? 2 : 3);
s64 latency_over = io_latency - latency_threshold;
if (latency_threshold && latency_over > 0) {
/*
* bump up congested by approximately latency_over * 4 /
* latency_threshold - we don't need much accuracy here so don't
* bother with the divide:
*/
if (atomic_read(&ca->congested) < CONGESTED_MAX)
atomic_add(latency_over >>
max_t(int, ilog2(laten
| 