smb: client: chan pick WIP (46s)

Signed-off-by: Enzo Matsumiya <ematsumiya@suse.de>

9 files changed, 131 insertions, 122 deletions

diff --git a/fs/smb/client/cifsglob.h b/fs/smb/client/cifsglob.h
index b3e1d0319c0b..24d05d16d70e 100644
--- a/fs/smb/client/cifsglob.h
+++ b/fs/smb/client/cifsglob.h
@@ -773,8 +773,7 @@ struct TCP_Server_Info {
 	__u32 reconnect_instance; /* incremented on each reconnect */
 	__le32 session_key_id; /* retrieved from negotiate response and send in session setup request */
 	struct session_key session_key;
-	atomic_long_t lstrq; /* when we last sent a request to this server (jiffies) */
-	atomic_long_t lstrp; /* when we got last response from this server (jiffies) */
+	unsigned long lstrp; /* when we got last response from this server (jiffies) */
 	struct cifs_secmech secmech; /* crypto sec mech functs, descriptors */
 #define	CIFS_NEGFLAVOR_UNENCAP	1	/* wct == 17, but no ext_sec */
 #define	CIFS_NEGFLAVOR_EXTENDED	2	/* wct == 17, ext_sec bit set */
@@ -1101,7 +1100,7 @@ release_iface(struct kref *ref)
 }
 
 struct cifs_chan {
-	struct list_head lru_head;
+	struct list_head head;
 	unsigned int in_reconnect : 1; /* if session setup in progress for this channel */
 	struct TCP_Server_Info *server;
 	struct cifs_server_iface *iface; /* interface in use */
@@ -1174,10 +1173,12 @@ struct cifs_ses {
 
 	struct cifs_chan chans[CIFS_MAX_CHANNELS]; /* this only stores channels for management */
 	struct list_head chans_lru; /* for cifs_pick_channel() */
+	struct list_head chans_lru_io; /* for cifs_pick_io_channel() */
 	size_t chan_count;
 	size_t chan_max;
 	size_t fast_picks;
 	size_t slow_picks;
+	size_t chan_changes;
 
 	/*
 	 * chans_need_reconnect is a bitmap indicating which of the channels
diff --git a/fs/smb/client/cifsproto.h b/fs/smb/client/cifsproto.h
index ea11f718a69d..3fd18ef176ef 100644
--- a/fs/smb/client/cifsproto.h
+++ b/fs/smb/client/cifsproto.h
@@ -101,6 +101,7 @@ extern int cifs_call_async(struct TCP_Server_Info *server,
 			mid_handle_t *handle, void *cbdata, const int flags,
 			const struct cifs_credits *exist_credits);
 extern struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses);
+extern struct TCP_Server_Info *cifs_pick_io_channel(struct cifs_ses *ses);
 extern int cifs_send_recv(const unsigned int xid, struct cifs_ses *ses,
 			  struct TCP_Server_Info *server,
 			  struct smb_rqst *rqst, int *resp_buf_type,
diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c
index 261d78fe2ff2..90c8286ae39d 100644
--- a/fs/smb/client/connect.c
+++ b/fs/smb/client/connect.c
@@ -282,7 +282,7 @@ cifs_abort_connection(struct TCP_Server_Info *server)
 	kfree_sensitive(server->session_key.response);
 	server->session_key.response = NULL;
 	server->session_key.len = 0;
-	atomic_long_set(&server->lstrp, jiffies);
+	server->lstrp = jiffies;
 
 	/* mark submitted MIDs for retry and issue callback */
 	INIT_LIST_HEAD(&retry_list);
@@ -583,7 +583,6 @@ cifs_echo_request(struct work_struct *work)
 	int rc;
 	struct TCP_Server_Info *server = container_of(work,
 					struct TCP_Server_Info, echo.work);
-	unsigned long lstrp = atomic_long_read(&server->lstrp);
 
 	/*
 	 * We cannot send an echo if it is disabled.
@@ -594,7 +593,7 @@ cifs_echo_request(struct work_struct *work)
 	    server->tcpStatus == CifsExiting ||
 	    server->tcpStatus == CifsNew ||
 	    (server->ops->can_echo && !server->ops->can_echo(server)) ||
-	    time_before(jiffies, lstrp + server->echo_interval - HZ))
+	    time_before(jiffies, server->lstrp + server->echo_interval - HZ))
 		goto requeue_echo;
 
 	rc = server->ops->echo ? server->ops->echo(server) : -ENOSYS;
@@ -643,8 +642,6 @@ allocate_buffers(struct TCP_Server_Info *server)
 static bool
 server_unresponsive(struct TCP_Server_Info *server)
 {
-	unsigned long lstrp = atomic_long_read(&server->lstrp);
-
 	/*
 	 * If we're in the process of mounting a share or reconnecting a session
 	 * and the server abruptly shut down (e.g. socket wasn't closed, packet
@@ -653,7 +650,7 @@ server_unresponsive(struct TCP_Server_Info *server)
 	 */
 	spin_lock(&server->srv_lock);
 	if (server->tcpStatus == CifsInNegotiate &&
-	    time_after(jiffies, lstrp + 20 * HZ)) {
+	    time_after(jiffies, server->lstrp + 20 * HZ)) {
 		spin_unlock(&server->srv_lock);
 		cifs_reconnect(server, false);
 		return true;
@@ -672,7 +669,7 @@ server_unresponsive(struct TCP_Server_Info *server)
 	if ((server->tcpStatus == CifsGood ||
 	    server->tcpStatus == CifsNeedNegotiate) &&
 	    (!server->ops->can_echo || server->ops->can_echo(server)) &&
-	    time_after(jiffies, lstrp + 3 * server->echo_interval)) {
+	    time_after(jiffies, server->lstrp + 3 * server->echo_interval)) {
 		spin_unlock(&server->srv_lock);
 		cifs_server_dbg(VFS, "has not responded in %lu seconds. Reconnecting...\n",
 			 (3 * server->echo_interval) / HZ);
@@ -1348,7 +1345,7 @@ next_pdu:
 			}
 		}
 
-		atomic_long_set(&server->lstrp, jiffies);
+		server->lstrp = jiffies;
 
 		for (i = 0; i < num_mids; i++) {
 			if (mids[i] != NULL) {
@@ -1842,8 +1839,7 @@ cifs_get_tcp_session(struct smb3_fs_context *ctx,
 	tcp_ses->sequence_number = 0;
 	tcp_ses->channel_sequence_num = 0; /* only tracked for primary channel */
 	tcp_ses->reconnect_instance = 1;
-	atomic_long_set(&tcp_ses->lstrq, 0);
-	atomic_long_set(&tcp_ses->lstrp, jiffies);
+	tcp_ses->lstrp = jiffies;
 	tcp_ses->compression.requested = ctx->compress;
 	spin_lock_init(&tcp_ses->req_lock);
 	spin_lock_init(&tcp_ses->srv_lock);
@@ -2184,11 +2180,11 @@ void __cifs_put_smb_ses(struct cifs_ses *ses)
 	list_del_init(&ses->smb_ses_list);
 	spin_unlock(&cifs_tcp_ses_lock);
 
-	pr_err("%s: fast %zu, slow %zu\n", __func__, ses->fast_picks, ses->slow_picks);
+	pr_err("%s: channel changes %zu, fast %zu, slow %zu\n", __func__, ses->chan_changes, ses->fast_picks, ses->slow_picks);
 
 	/* close any extra channels */
 	for (i = 1; i < ses->chan_count; i++) {
-		list_del_init(&ses->chans[i].lru_head);
+		list_del_init(&ses->chans[i].head);
 		if (ses->chans[i].iface) {
 			kref_put(&ses->chans[i].iface->refcount, release_iface);
 			ses->chans[i].iface = NULL;
@@ -2199,7 +2195,7 @@ void __cifs_put_smb_ses(struct cifs_ses *ses)
 
 	/* we now account for primary channel in iface->refcount */
 	if (ses->chans[0].iface) {
-		list_del_init(&ses->chans[0].lru_head);
+		list_del_init(&ses->chans[0].head);
 		kref_put(&ses->chans[0].iface->refcount, release_iface);
 		ses->chans[0].server = NULL;
 	}
@@ -2516,7 +2512,7 @@ retry_old_session:
 	/* add server as first channel */
 	spin_lock(&ses->chan_lock);
 	ses->chans[0].server = server;
-	list_add(&ses->chans[0].lru_head, &ses->chans_lru);
+	list_add(&ses->chans[0].head, &ses->chans_lru);
 	ses->chan_count = 1;
 	ses->chan_max = ctx->multichannel ? ctx->max_channels:1;
 	ses->chans_need_reconnect = 1;
@@ -4235,7 +4231,7 @@ retry:
 		return 0;
 	}
 
-	atomic_long_set(&server->lstrp, jiffies);
+	server->lstrp = jiffies;
 	server->tcpStatus = CifsInNegotiate;
 	spin_unlock(&server->srv_lock);
 
diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c
index 9835672267d2..aae2d5ff5176 100644
--- a/fs/smb/client/file.c
+++ b/fs/smb/client/file.c
@@ -60,7 +60,7 @@ static void cifs_prepare_write(struct netfs_io_subrequest *subreq)
 		wdata->have_xid = true;
 	}
 
-	server = cifs_pick_channel(tlink_tcon(open_file->tlink)->ses);
+	server = cifs_pick_io_channel(tlink_tcon(open_file->tlink)->ses);
 	wdata->server = server;
 
 retry:
@@ -157,7 +157,7 @@ static int cifs_prepare_read(struct netfs_io_subrequest *subreq)
 		rdata->have_xid = true;
 	}
 
-	server = cifs_pick_channel(tlink_tcon(req->cfile->tlink)->ses);
+	server = cifs_pick_io_channel(tlink_tcon(req->cfile->tlink)->ses);
 	rdata->server = server;
 
 	if (cifs_sb->ctx->rsize == 0) {
diff --git a/fs/smb/client/misc.c b/fs/smb/client/misc.c
index dbe0a03a0c89..a0962aeef79f 100644
--- a/fs/smb/client/misc.c
+++ b/fs/smb/client/misc.c
@@ -75,6 +75,7 @@ sesInfoAlloc(void)
 		INIT_LIST_HEAD(&ret_buf->smb_ses_list);
 		INIT_LIST_HEAD(&ret_buf->tcon_list);
 		INIT_LIST_HEAD(&ret_buf->chans_lru);
+		INIT_LIST_HEAD(&ret_buf->chans_lru_io);
 		mutex_init(&ret_buf->session_mutex);
 		spin_lock_init(&ret_buf->chan_lock);
 	}
diff --git a/fs/smb/client/sess.c b/fs/smb/client/sess.c
index facbe8c413e4..68e405bfe9cc 100644
--- a/fs/smb/client/sess.c
+++ b/fs/smb/client/sess.c
@@ -275,7 +275,7 @@ cifs_disable_secondary_channels(struct cifs_ses *ses)
 		ses->chans[i].iface = NULL;
 		ses->chans[i].server = NULL;
 
-		list_del_init(&ses->chans[i].lru_head);
+		list_del_init(&ses->chans[i].head);
 		spin_unlock(&ses->chan_lock);
 
 		if (iface) {
@@ -541,7 +541,7 @@ cifs_ses_add_channel(struct cifs_ses *ses,
 	chan = &ses->chans[ses->chan_count];
 	chan->server = chan_server;
 	chan->iface = iface;
-	INIT_LIST_HEAD(&chan->lru_head);
+	INIT_LIST_HEAD(&chan->head);
 	ses->chan_count++;
 
 	/* Mark this channel as needing connect/setup */
@@ -567,7 +567,7 @@ cifs_ses_add_channel(struct cifs_ses *ses,
 		rc = cifs_setup_session(xid, ses, chan->server, ses->local_nls);
 		if (!rc) {
 			/* channel is ready to be used */
-			list_add(&chan->lru_head, &ses->chans_lru);
+			list_add(&chan->head, &ses->chans_lru);
 		}
 	}
 
@@ -581,7 +581,7 @@ out:
 
 		/* we rely on all bits beyond chan_count to be clear */
 		cifs_chan_clear_need_reconnect(ses, chan->server);
-		list_del_init(&chan->lru_head);
+		list_del_init(&chan->head);
 		ses->chan_count--;
 		/*
 		 * chan_count should never reach 0 as at least the primary
diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c
index 56f766b15e36..4d431f0d136a 100644
--- a/fs/smb/client/smb2ops.c
+++ b/fs/smb/client/smb2ops.c
@@ -4827,7 +4827,7 @@ static void smb2_decrypt_offload(struct work_struct *work)
 		goto free_pages;
 	}
 
-	atomic_long_set(&dw->server->lstrp, jiffies);
+	dw->server->lstrp = jiffies;
 	mid = smb2_find_dequeue_mid(dw->server, dw->buf);
 	if (mid == NULL)
 		cifs_dbg(FYI, "mid not found\n");
diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c
index bcc3e4060826..776e1d3d4c2f 100644
--- a/fs/smb/client/smb2pdu.c
+++ b/fs/smb/client/smb2pdu.c
@@ -181,7 +181,7 @@ cifs_chan_skip_or_disable(struct cifs_ses *ses,
 			goto skip_terminate;
 		}
 
-		list_del_init(&ses->chans[chan_index].lru_head);
+		list_del_init(&ses->chans[chan_index].head);
 		ses->chans[chan_index].server = NULL;
 		server->terminate = true;
 		spin_unlock(&ses->chan_lock);
@@ -3387,7 +3387,11 @@ SMB2_ioctl(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid,
 replay_again:
 	/* reinitialize for possible replay */
 	flags = 0;
-	server = cifs_pick_channel(ses);
+	if (opcode == FSCTL_SRV_COPYCHUNK || opcode == FSCTL_SRV_COPYCHUNK_WRITE ||
+	    opcode == FSCTL_SET_ZERO_DATA)
+		server = cifs_pick_io_channel(ses);
+	else
+		server = cifs_pick_channel(ses);
 
 	if (!server)
 		return -EIO;
diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c
index 08f9a8d5ffda..f25f075293c3 100644
--- a/fs/smb/client/transport.c
+++ b/fs/smb/client/transport.c
@@ -195,9 +195,6 @@ smb_send_kvec(struct TCP_Server_Info *server, struct msghdr *smb_msg,
 		smb_msg->msg_flags = MSG_NOSIGNAL;
 
 	while (msg_data_left(smb_msg)) {
-		/* update lastrq anytime we're about to send a request through the wire */
-		atomic_long_set(&server->lstrq, jiffies);
-
 		/*
 		 * If blocking send, we try 3 times, since each can block
 		 * for 5 seconds. For nonblocking  we have to try more
@@ -1010,131 +1007,140 @@ cifs_cancelled_callback(struct mid_q_entry *mid)
 }
 
 enum {
-	CIFS_CHAN_GOOD = 0,
+	CIFS_CHAN_GOOD,
 	CIFS_CHAN_SLOW,
 	CIFS_CHAN_STUCK,
 };
 
-static inline int chan_responsiveness(struct cifs_chan *chan)
+static inline int chan_responsiveness(struct cifs_chan *chan, unsigned int in_flight)
 {
-	unsigned long lstrq = atomic_long_read(&chan->server->lstrq);
-	unsigned long lstrp = atomic_long_read(&chan->server->lstrp);
+	unsigned long lstrp = chan->server->lstrp;
+	unsigned long echo = msecs_to_jiffies(chan->server->echo_interval * 1000);
 	unsigned long now = jiffies;
-	unsigned long threshold = 500;
-
-	lstrq = jiffies_to_msecs(now - lstrq);
-	lstrp = jiffies_to_msecs(now - lstrp);
+	unsigned long threshold;
 
-	/* traffic in last 500ms, check in flight */
-	if (lstrq < threshold && lstrp < threshold) {
-		if (chan->server->in_flight < 5)
-			return CIFS_CHAN_GOOD;
-		return CIFS_CHAN_SLOW;
-	}
+	if (in_flight == 0)
+		return CIFS_CHAN_GOOD;
 
-	/*
-	 * No comms in last 500ms.  Set threshold to echo interval timeframe, and check if last
-	 * request was sent before the last response, and if last response was within the new
-	 * threshold.
-	 *
-	 * If so, but there are requests in flight, channel is stuck.
-	 */
-	threshold = jiffies_to_msecs(chan->server->echo_interval * HZ);
-	if (lstrp < lstrq && lstrp < threshold) {
-		if (!chan->server->in_flight)
-			return CIFS_CHAN_GOOD;
-		return CIFS_CHAN_STUCK;
-	}
+	threshold = msecs_to_jiffies(clamp_t(unsigned long, slow_rsp_threshold * 1000, 500, 3000));
+	if (time_after(lstrp + threshold, now))
+		return CIFS_CHAN_GOOD;
 
-	/*
-	 * Last request was sent after last response, if it was sent more than threshold/2 ms or if
-	 * there are no requests in flight, consider it stuck.
-	 * Otherwise we just have a slow channel.
-	 */
-	threshold /= 2;
-	if (lstrq < lstrp && lstrq < threshold && chan->server->in_flight)
+	if (time_after(lstrp + echo, now))
 		return CIFS_CHAN_SLOW;
 
 	return CIFS_CHAN_STUCK;
 }
 
-static struct cifs_chan *slow_pick_channel(struct cifs_ses *ses)
+static struct cifs_chan *fast_pick_channel(struct list_head *list)
 {
-	struct cifs_chan *cur, *best = &ses->chans[0];
-	int i, cur_res, best_res;
-
-	best_res = chan_responsiveness(best);
-
-	for (i = 1; i < ses->chan_count; i++) {
-		cur = &ses->chans[i];
-		cur_res = chan_responsiveness(cur);
-		if (cur_res <= CIFS_CHAN_SLOW) {
-			if (cur->server->in_flight <= best->server->in_flight) {
-				if (cur_res == CIFS_CHAN_GOOD) {
-					best = cur;
-					best_res = cur_res;
-				}
+	struct cifs_chan *cur, *next = NULL;
+
+	if (likely(!list_is_singular(list))) {
+		cur = list_first_entry(list, struct cifs_chan, head);
+check_next:
+		if (cur->server->in_flight < 5) {
+			if (chan_responsiveness(cur, cur->server->in_flight) == CIFS_CHAN_GOOD)
+				return cur;
+
+			if (!next) {
+				next = list_next_entry_circular(cur, list, head);
+				cur = next;
+				goto check_next;
 			}
-		} else {
-			list_del_init(&cur->lru_head);
 		}
 	}
 
-	return best;
+	return NULL;
 }
 
-static struct cifs_chan *fast_pick_channel(struct cifs_ses *ses)
+static struct cifs_chan *slow_pick_channel(struct cifs_ses *ses)
 {
-	struct cifs_chan *cur, *tmp;
-	int cur_res, tmp_res;
-
-	if (ses->chan_count == 1)
-		return &ses->chans[0];
-
-	cur = list_first_entry(&ses->chans_lru, struct cifs_chan, lru_head);
-	if (likely(!list_is_singular(&ses->chans_lru))) {
-		tmp = list_prev_entry_circular(cur, &ses->chans_lru, lru_head);
-		cur_res = chan_responsiveness(cur);
-		tmp_res = chan_responsiveness(tmp);
-		if (likely(cur_res == CIFS_CHAN_GOOD)) {
-			if (tmp_res == CIFS_CHAN_GOOD) {
-				if (likely(cur->server->in_flight <= tmp->server->in_flight))
-					return cur;
-				return tmp;
-			}
-		}
-	}
+	int i;
 
-	return NULL;
+	for (i = 1; i < ses->chan_count; i++)
+		if (chan_responsiveness(&ses->chans[i], ses->chans[i].server->in_flight) == CIFS_CHAN_GOOD)
+			return &ses->chans[i];
+
+	return &ses->chans[0];
 }
 
-/*
- * Return a channel (master if none) of @ses, based on best latency, that can be used to send
- * regular requests.
- */
-struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses)
+#if 0
+static struct TCP_Server_Info *pick_channel(struct cifs_ses *ses, bool io)
 {
 	struct cifs_chan *chan;
 
-	if (likely(ses && ses->server)) {
-		spin_lock(&ses->chan_lock);
-		chan = fast_pick_channel(ses);
-		if (likely(chan)) {
-			ses->fast_picks++;
-			goto out;
-		}
+	spin_lock(&ses->chan_lock);
+	if (ses->chan_count == 1) {
+		chan = &ses->chans[0];
+		goto out;
+	}
+
+	chan = fast_pick_channel(&ses->chans_lru);
+	if (likely(chan)) {
+		ses->fast_picks++;
+		goto out;
+	}
 
-		/* slow_pick_channel() never returns NULL, no need to check */
-		chan = slow_pick_channel(ses);
-		ses->slow_picks++;
+	chan = fast_pick_channel(&ses->chans_lru_io);
+	if (chan) {
+		ses->fast_picks++;
+		goto out;
+	}
+
+	ses->slow_picks++;
+	chan = slow_pick_channel(ses);
 out:
-		list_move_tail(&chan->lru_head, &ses->chans_lru);
-		spin_unlock(&ses->chan_lock);
+	if (!io)
+		list_move_tail(&chan->head, &ses->chans_lru);
+	else
+		list_move_tail(&chan->head, &ses->chans_lru_io);
+	spin_unlock(&ses->chan_lock);
+	return chan->server;
+}
+#endif
+
+static struct TCP_Server_Info *pick_channel(struct cifs_ses *ses, bool io)
+{
+	struct cifs_chan *chan;
 
-		return chan->server;
+	spin_lock(&ses->chan_lock);
+	if (ses->chan_count == 1) {
+		chan = &ses->chans[0];
+		goto out;
 	}
 
-	return NULL;
+	chan = fast_pick_channel(&ses->chans_lru);
+	if (likely(chan)) {
+		ses->fast_picks++;
+		goto out;
+	}
+
+	chan = fast_pick_channel(&ses->chans_lru_io);
+	if (chan) {
+		ses->fast_picks++;
+		goto out;
+	}
+
+	ses->slow_picks++;
+	chan = slow_pick_channel(ses);
+out:
+	if (!io)
+		list_move_tail(&chan->head, &ses->chans_lru);
+	else
+		list_move_tail(&chan->head, &ses->chans_lru_io);
+	spin_unlock(&ses->chan_lock);
+	return chan->server;
+}
+
+struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses)
+{
+	return pick_channel(ses, false);
+}
+
+struct TCP_Server_Info *cifs_pick_io_channel(struct cifs_ses *ses)
+{
+	return pick_channel(ses, true);
 }
 
 int