smb/client: introduce cpu_policy={nearest,affine}

This commit introduces the cpu_policy= mount option.
The 2 possible values for this option are "nearest" and "affine".

If "cpu_policy=nearest" is passed (or "cpu_policy=" not passed at all,
or an invalid value passed to it), the cifsd kernel thread will run just
as before, where, internally, kthread_run() will let the scheduler
select the best CPU it sees fit.

For "cpu_policy=affine", a global variable "cifsd_rr_last_cpu" tracks
the last used CPU by a started cifsd thread, and it's incremented each
time a new cifsd thread is about to start, so it gets run on a different
CPU, on a round-robin manner.  Since cifsd is started for each
non-shared TCP_Server_Info (i.e. connection to different server or
adding a new channel for an existing connection).

For small workloads this shouldn't make much difference, or at all.

For big workloads, on servers with lots of CPU, and/or with multichannel
enabled, the performance improvements starts to show, as we maintain now
the whole message flow (from request creation to response processing)
within a single CPU (all workqueues are scheduled to run on the same CPU
as their TCP_Server_Info::tsk), allowing any data cached when sending
to be a cache-hit on receiving, thus increasing performance.

Signed-off-by: Enzo Matsumiya <ematsumiya@suse.de>

11 files changed, 106 insertions, 14 deletions

diff --git a/fs/smb/client/cached_dir.c b/fs/smb/client/cached_dir.c
index bfc964b36c72..5a5e6571e031 100644
--- a/fs/smb/client/cached_dir.c
+++ b/fs/smb/client/cached_dir.c
@@ -515,8 +515,8 @@ int cached_dir_lease_break(struct cifs_tcon *tcon, __u8 lease_key[16])
 			cfid->on_list = false;
 			cfids->num_entries--;
 
-			queue_work(cifsiod_wq,
-				   &cfid->lease_break);
+			queue_work_on(tcon->ses->server->tsk_cpu,
+				      cifsiod_wq, &cfid->lease_break);
 			spin_unlock(&cfids->cfid_list_lock);
 			return true;
 		}
diff --git a/fs/smb/client/cifsglob.h b/fs/smb/client/cifsglob.h
index 0d84bb1a8cd9..caeac04da97c 100644
--- a/fs/smb/client/cifsglob.h
+++ b/fs/smb/client/cifsglob.h
@@ -639,7 +639,9 @@ struct TCP_Server_Info {
 	spinlock_t req_lock;  /* protect the two values above */
 	struct mutex _srv_mutex;
 	unsigned int nofs_flag;
+	bool cpu_policy_affine:1;
 	struct task_struct *tsk;
+#define tsk_cpu tsk->thread_info.cpu
 	char server_GUID[16];
 	__u16 sec_mode;
 	bool sign; /* is signing enabled on this connection? */
diff --git a/fs/smb/client/cifssmb.c b/fs/smb/client/cifssmb.c
index 9d963caec35c..13a16bc335bc 100644
--- a/fs/smb/client/cifssmb.c
+++ b/fs/smb/client/cifssmb.c
@@ -1304,7 +1304,7 @@ cifs_readv_callback(struct mid_q_entry *mid)
 		rdata->result = -EIO;
 	}
 
-	queue_work(cifsiod_wq, &rdata->work);
+	queue_work_on(rdata->server->tsk_cpu, cifsiod_wq, &rdata->work);
 	release_mid(mid);
 	add_credits(server, &credits, 0);
 }
@@ -1652,7 +1652,7 @@ cifs_writev_callback(struct mid_q_entry *mid)
 		break;
 	}
 
-	queue_work(cifsiod_wq, &wdata->work);
+	queue_work_on(wdata->server->tsk_cpu, cifsiod_wq, &wdata->work);
 	release_mid(mid);
 	add_credits(tcon->ses->server, &credits, 0);
 }
diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c
index 8e9a672320ab..c808e2662857 100644
--- a/fs/smb/client/connect.c
+++ b/fs/smb/client/connect.c
@@ -8,6 +8,7 @@
 #include <linux/fs.h>
 #include <linux/net.h>
 #include <linux/string.h>
+#include <linux/sched.h>
 #include <linux/sched/mm.h>
 #include <linux/sched/signal.h>
 #include <linux/list.h>
@@ -30,6 +31,7 @@
 #include <linux/module.h>
 #include <keys/user-type.h>
 #include <net/ipv6.h>
+#include <net/route.h>
 #include <linux/parser.h>
 #include <linux/bvec.h>
 #include "cifspdu.h"
@@ -67,6 +69,8 @@ static int generic_ip_connect(struct TCP_Server_Info *server);
 static void tlink_rb_insert(struct rb_root *root, struct tcon_link *new_tlink);
 static void cifs_prune_tlinks(struct work_struct *work);
 
+static int cifsd_rr_last_cpu = -1;
+
 /*
  * Resolve hostname and set ip addr in tcp ses. Useful for hostnames that may
  * get their ip addresses changed at some point.
@@ -1550,6 +1554,45 @@ cifs_put_tcp_session(struct TCP_Server_Info *server, int from_reconnect)
 		send_sig(SIGKILL, task, 1);
 }
 
+/**
+ * __smb_get_next_cpu: return the next online CPU, globally accounted by @cifsd_rr_last_cpu
+ */
+static inline int __smb_get_next_cpu(void)
+{
+	int cpu;
+
+	cpu = cpumask_next_wrap(cifsd_rr_last_cpu, cpu_online_mask, -1, false);
+	if (unlikely(cpu >= nr_cpu_ids)) {
+		cpu = cpumask_next(raw_smp_processor_id(), cpu_online_mask);
+		if (unlikely(cpu >= nr_cpu_ids)) {
+			/* last resort */
+			cpu = raw_smp_processor_id();
+		}
+	}
+
+	cifsd_rr_last_cpu = cpu;
+	return cifsd_rr_last_cpu;
+}
+
+/**
+ * smb_get_cifsd_task: return a task_struct for cifsd, based on @server::cpu_policy_affine
+ *
+ * @server::cpu_policy_affine behaviour:
+ * true: create the task via kthread_run() which will let the scheduler pick the best CPU to use
+ * false: create the task via kthread_run_on_cpu(), with its @cpu argument set to the next
+ *	  online CPU, which is chosen in a round-robin manner in __smb_get_next_cpu() and tracked
+ *	  via global %cifsd_rr_last_cpu
+ */
+static inline struct task_struct *smb_get_cifsd_task(struct TCP_Server_Info *server)
+{
+	if (!server->cpu_policy_affine || num_online_cpus() == 1)
+		return kthread_run(cifs_demultiplex_thread, server, "cifsd");
+
+	/* cpu_policy=affine */
+	return kthread_run_on_cpu(cifs_demultiplex_thread, server, __smb_get_next_cpu(),
+				  "cifsd");
+}
+
 struct TCP_Server_Info *
 cifs_get_tcp_session(struct smb3_fs_context *ctx,
 		     struct TCP_Server_Info *primary_server)
@@ -1680,14 +1723,21 @@ smbd_connected:
 	 * this will succeed. No need for try_module_get().
 	 */
 	__module_get(THIS_MODULE);
-	tcp_ses->tsk = kthread_run(cifs_demultiplex_thread,
-				  tcp_ses, "cifsd");
+
+	tcp_ses->cpu_policy_affine = tcp_ses->primary_server ?
+				     tcp_ses->primary_server->cpu_policy_affine :
+				     ctx->cpu_policy_affine;
+	tcp_ses->tsk = smb_get_cifsd_task(tcp_ses);
 	if (IS_ERR(tcp_ses->tsk)) {
 		rc = PTR_ERR(tcp_ses->tsk);
 		cifs_dbg(VFS, "error %d create cifsd thread\n", rc);
 		module_put(THIS_MODULE);
 		goto out_err_crypto_release;
 	}
+
+	cifs_dbg(FYI, "%s: running cifsd task 0x%p on CPU#%d\n",
+		 __func__, tcp_ses->tsk, tcp_ses->tsk_cpu);
+
 	tcp_ses->min_offload = ctx->min_offload;
 	/*
 	 * at this point we are the only ones with the pointer
diff --git a/fs/smb/client/dfs_cache.c b/fs/smb/client/dfs_cache.c
index 1513b2709889..8ce8e3ccd05d 100644
--- a/fs/smb/client/dfs_cache.c
+++ b/fs/smb/client/dfs_cache.c
@@ -1300,6 +1300,6 @@ void dfs_cache_refresh(struct work_struct *work)
 		mutex_unlock(&server->refpath_lock);
 	}
 
-	queue_delayed_work(dfscache_wq, &tcon->dfs_cache_work,
-			   atomic_read(&dfs_cache_ttl) * HZ);
+	queue_delayed_work_on(server->tsk_cpu, dfscache_wq, &tcon->dfs_cache_work,
+			      atomic_read(&dfs_cache_ttl) * HZ);
 }
diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c
index df88b8c04d03..81422012128e 100644
--- a/fs/smb/client/file.c
+++ b/fs/smb/client/file.c
@@ -679,7 +679,7 @@ void _cifsFileInfo_put(struct cifsFileInfo *cifs_file,
 	cifs_del_pending_open(&open);
 
 	if (offload)
-		queue_work(fileinfo_put_wq, &cifs_file->put);
+		queue_work_on(server->tsk_cpu, fileinfo_put_wq, &cifs_file->put);
 	else
 		cifsFileInfo_put_final(cifs_file);
 }
diff --git a/fs/smb/client/fs_context.c b/fs/smb/client/fs_context.c
index 1bda75609b64..23dd35a9cefd 100644
--- a/fs/smb/client/fs_context.c
+++ b/fs/smb/client/fs_context.c
@@ -25,6 +25,7 @@
 #include <linux/mount.h>
 #include <linux/parser.h>
 #include <linux/utsname.h>
+#include <linux/cpumask.h>
 #include "cifsfs.h"
 #include "cifspdu.h"
 #include "cifsglob.h"
@@ -172,6 +173,7 @@ const struct fs_parameter_spec smb3_fs_parameters[] = {
 	fsparam_string("vers", Opt_vers),
 	fsparam_string("sec", Opt_sec),
 	fsparam_string("cache", Opt_cache),
+	fsparam_string("cpu_policy", Opt_cpu_policy),
 
 	/* Arguments that should be ignored */
 	fsparam_flag("guest", Opt_ignore),
@@ -292,6 +294,28 @@ cifs_parse_cache_flavor(struct fs_context *fc, char *value, struct smb3_fs_conte
 	return 0;
 }
 
+static const match_table_t smb_cpu_policies = {
+	{ Opt_cpu_policy_affine, "affine" },
+	{ Opt_cpu_policy_nearest, "nearest" },
+};
+
+static void smb_parse_cpu_policy(struct fs_context *fc, char *val, struct smb3_fs_context *ctx)
+{
+	substring_t args[MAX_OPT_ARGS];
+
+	switch (match_token(val, smb_cpu_policies, args)) {
+	default:
+		cifs_errorf(fc, "bad cpu_policy= option '%s', using default 'nearest'\n", val);
+		fallthrough;
+	case Opt_cpu_policy_nearest:
+		ctx->cpu_policy_affine = 0;
+		break;
+	case Opt_cpu_policy_affine:
+		ctx->cpu_policy_affine = 1;
+		break;
+	}
+}
+
 #define DUP_CTX_STR(field)						\
 do {									\
 	if (ctx->field) {						\
@@ -1314,6 +1338,14 @@ static int smb3_fs_context_parse_param(struct fs_context *fc,
 		if (cifs_parse_cache_flavor(fc, param->string, ctx) != 0)
 			goto cifs_parse_mount_err;
 		break;
+	case Opt_cpu_policy:
+		smb_parse_cpu_policy(fc, param->string, ctx);
+
+		if (ctx->cpu_policy_affine && num_present_cpus() == 1)
+			pr_warn("cpu_policy=affine: it appears there's only 1 CPU present, until "
+				"there are at least 2 CPUs online, this setting will not have any "
+				"effect\n");
+		break;
 	case Opt_witness:
 #ifndef CONFIG_CIFS_SWN_UPCALL
 		cifs_errorf(fc, "Witness support needs CONFIG_CIFS_SWN_UPCALL config option\n");
diff --git a/fs/smb/client/fs_context.h b/fs/smb/client/fs_context.h
index f4eaf8558902..21bac6318c07 100644
--- a/fs/smb/client/fs_context.h
+++ b/fs/smb/client/fs_context.h
@@ -54,6 +54,11 @@ enum cifs_sec_param {
 	Opt_sec_err
 };
 
+enum {
+	Opt_cpu_policy_nearest,
+	Opt_cpu_policy_affine,
+};
+
 enum cifs_param {
 	/* Mount options that take no arguments */
 	Opt_user_xattr,
@@ -146,6 +151,7 @@ enum cifs_param {
 	Opt_vers,
 	Opt_sec,
 	Opt_cache,
+	Opt_cpu_policy,
 
 	/* Mount options to be ignored */
 	Opt_ignore,
@@ -245,6 +251,7 @@ struct smb3_fs_context {
 	unsigned int wsize;
 	unsigned int min_offload;
 	bool sockopt_tcp_nodelay:1;
+	bool cpu_policy_affine:1;
 	/* attribute cache timemout for files and directories in jiffies */
 	unsigned long acregmax;
 	unsigned long acdirmax;
diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c
index 3935a60db5c3..f03a802ce4a9 100644
--- a/fs/smb/client/smb2misc.c
+++ b/fs/smb/client/smb2misc.c
@@ -540,7 +540,8 @@ smb2_queue_pending_open_break(struct tcon_link *tlink, __u8 *lease_key,
 	lw->tlink = tlink;
 	lw->lease_state = new_lease_state;
 	memcpy(lw->lease_key, lease_key, SMB2_LEASE_KEY_SIZE);
-	queue_work(cifsiod_wq, &lw->lease_break);
+	queue_work_on(tlink->tl_tcon->ses->server->tsk_cpu,
+		      cifsiod_wq, &lw->lease_break);
 }
 
 static bool
@@ -797,7 +798,7 @@ __smb2_handle_cancelled_cmd(struct cifs_tcon *tcon, __u16 cmd, __u64 mid,
 	cancelled->cmd = cmd;
 	cancelled->mid = mid;
 	INIT_WORK(&cancelled->work, smb2_cancelled_close_fid);
-	WARN_ON(queue_work(cifsiod_wq, &cancelled->work) == false);
+	WARN_ON(!queue_work_on(tcon->ses->server->tsk_cpu, cifsiod_wq, &cancelled->work));
 
 	return 0;
 }
diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c
index 5065398665f1..92ac69ea6957 100644
--- a/fs/smb/client/smb2ops.c
+++ b/fs/smb/client/smb2ops.c
@@ -4941,7 +4941,7 @@ receive_encrypted_read(struct TCP_Server_Info *server, struct mid_q_entry **mid,
 		dw->buf = server->smallbuf;
 		server->smallbuf = (char *)cifs_small_buf_get();
 
-		queue_work(decrypt_wq, &dw->decrypt);
+		queue_work_on(server->tsk_cpu, decrypt_wq, &dw->decrypt);
 		*num_mids = 0; /* worker thread takes care of finding mid */
 		return -1;
 	}
diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c
index 7063b395d22f..cfedb3293542 100644
--- a/fs/smb/client/smb2pdu.c
+++ b/fs/smb/client/smb2pdu.c
@@ -4209,7 +4209,7 @@ smb2_readv_callback(struct mid_q_entry *mid)
 				     tcon->tid, tcon->ses->Suid,
 				     rdata->offset, rdata->got_bytes);
 
-	queue_work(cifsiod_wq, &rdata->work);
+	queue_work_on(rdata->server->tsk_cpu, cifsiod_wq, &rdata->work);
 	release_mid(mid);
 	add_credits(server, &credits, 0);
 }
@@ -4448,7 +4448,7 @@ smb2_writev_callback(struct mid_q_entry *mid)
 				      tcon->tid, tcon->ses->Suid,
 				      wdata->offset, wdata->bytes);
 
-	queue_work(cifsiod_wq, &wdata->work);
+	queue_work_on(wdata->server->tsk_cpu, cifsiod_wq, &wdata->work);
 	release_mid(mid);
 	add_credits(server, &credits, 0);
 }