diff options
Diffstat (limited to 'drivers/platform/surface/aggregator/ssh_packet_layer.c')
| -rw-r--r-- | drivers/platform/surface/aggregator/ssh_packet_layer.c | 2074 |
1 files changed, 2074 insertions, 0 deletions
diff --git a/drivers/platform/surface/aggregator/ssh_packet_layer.c b/drivers/platform/surface/aggregator/ssh_packet_layer.c new file mode 100644 index 000000000000..15d96eac6811 --- /dev/null +++ b/drivers/platform/surface/aggregator/ssh_packet_layer.c @@ -0,0 +1,2074 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * SSH packet transport layer. + * + * Copyright (C) 2019-2020 Maximilian Luz <luzmaximilian@gmail.com> + */ + +#include <asm/unaligned.h> +#include <linux/atomic.h> +#include <linux/error-injection.h> +#include <linux/jiffies.h> +#include <linux/kfifo.h> +#include <linux/kref.h> +#include <linux/kthread.h> +#include <linux/ktime.h> +#include <linux/limits.h> +#include <linux/list.h> +#include <linux/lockdep.h> +#include <linux/serdev.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/workqueue.h> + +#include <linux/surface_aggregator/serial_hub.h> + +#include "ssh_msgb.h" +#include "ssh_packet_layer.h" +#include "ssh_parser.h" + +#include "trace.h" + +/* + * To simplify reasoning about the code below, we define a few concepts. The + * system below is similar to a state-machine for packets, however, there are + * too many states to explicitly write them down. To (somewhat) manage the + * states and packets we rely on flags, reference counting, and some simple + * concepts. State transitions are triggered by actions. + * + * >> Actions << + * + * - submit + * - transmission start (process next item in queue) + * - transmission finished (guaranteed to never be parallel to transmission + * start) + * - ACK received + * - NAK received (this is equivalent to issuing re-submit for all pending + * packets) + * - timeout (this is equivalent to re-issuing a submit or canceling) + * - cancel (non-pending and pending) + * + * >> Data Structures, Packet Ownership, General Overview << + * + * The code below employs two main data structures: The packet queue, + * containing all packets scheduled for transmission, and the set of pending + * packets, containing all packets awaiting an ACK. + * + * Shared ownership of a packet is controlled via reference counting. Inside + * the transport system are a total of five packet owners: + * + * - the packet queue, + * - the pending set, + * - the transmitter thread, + * - the receiver thread (via ACKing), and + * - the timeout work item. + * + * Normal operation is as follows: The initial reference of the packet is + * obtained by submitting the packet and queuing it. The receiver thread takes + * packets from the queue. By doing this, it does not increment the refcount + * but takes over the reference (removing it from the queue). If the packet is + * sequenced (i.e. needs to be ACKed by the client), the transmitter thread + * sets-up the timeout and adds the packet to the pending set before starting + * to transmit it. As the timeout is handled by a reaper task, no additional + * reference for it is needed. After the transmit is done, the reference held + * by the transmitter thread is dropped. If the packet is unsequenced (i.e. + * does not need an ACK), the packet is completed by the transmitter thread + * before dropping that reference. + * + * On receival of an ACK, the receiver thread removes and obtains the + * reference to the packet from the pending set. The receiver thread will then + * complete the packet and drop its reference. + * + * On receival of a NAK, the receiver thread re-submits all currently pending + * packets. + * + * Packet timeouts are detected by the timeout reaper. This is a task, + * scheduled depending on the earliest packet timeout expiration date, + * checking all currently pending packets if their timeout has expired. If the + * timeout of a packet has expired, it is re-submitted and the number of tries + * of this packet is incremented. If this number reaches its limit, the packet + * will be completed with a failure. + * + * On transmission failure (such as repeated packet timeouts), the completion + * callback is immediately run by on thread on which the error was detected. + * + * To ensure that a packet eventually leaves the system it is marked as + * "locked" directly before it is going to be completed or when it is + * canceled. Marking a packet as "locked" has the effect that passing and + * creating new references of the packet is disallowed. This means that the + * packet cannot be added to the queue, the pending set, and the timeout, or + * be picked up by the transmitter thread or receiver thread. To remove a + * packet from the system it has to be marked as locked and subsequently all + * references from the data structures (queue, pending) have to be removed. + * References held by threads will eventually be dropped automatically as + * their execution progresses. + * + * Note that the packet completion callback is, in case of success and for a + * sequenced packet, guaranteed to run on the receiver thread, thus providing + * a way to reliably identify responses to the packet. The packet completion + * callback is only run once and it does not indicate that the packet has + * fully left the system (for this, one should rely on the release method, + * triggered when the reference count of the packet reaches zero). In case of + * re-submission (and with somewhat unlikely timing), it may be possible that + * the packet is being re-transmitted while the completion callback runs. + * Completion will occur both on success and internal error, as well as when + * the packet is canceled. + * + * >> Flags << + * + * Flags are used to indicate the state and progression of a packet. Some flags + * have stricter guarantees than other: + * + * - locked + * Indicates if the packet is locked. If the packet is locked, passing and/or + * creating additional references to the packet is forbidden. The packet thus + * may not be queued, dequeued, or removed or added to the pending set. Note + * that the packet state flags may still change (e.g. it may be marked as + * ACKed, transmitted, ...). + * + * - completed + * Indicates if the packet completion callback has been executed or is about + * to be executed. This flag is used to ensure that the packet completion + * callback is only run once. + * + * - queued + * Indicates if a packet is present in the submission queue or not. This flag + * must only be modified with the queue lock held, and must be coherent to the + * presence of the packet in the queue. + * + * - pending + * Indicates if a packet is present in the set of pending packets or not. + * This flag must only be modified with the pending lock held, and must be + * coherent to the presence of the packet in the pending set. + * + * - transmitting + * Indicates if the packet is currently transmitting. In case of + * re-transmissions, it is only safe to wait on the "transmitted" completion + * after this flag has been set. The completion will be set both in success + * and error case. + * + * - transmitted + * Indicates if the packet has been transmitted. This flag is not cleared by + * the system, thus it indicates the first transmission only. + * + * - acked + * Indicates if the packet has been acknowledged by the client. There are no + * other guarantees given. For example, the packet may still be canceled + * and/or the completion may be triggered an error even though this bit is + * set. Rely on the status provided to the completion callback instead. + * + * - canceled + * Indicates if the packet has been canceled from the outside. There are no + * other guarantees given. Specifically, the packet may be completed by + * another part of the system before the cancellation attempts to complete it. + * + * >> General Notes << + * + * - To avoid deadlocks, if both queue and pending locks are required, the + * pending lock must be acquired before the queue lock. + * + * - The packet priority must be accessed only while holding the queue lock. + * + * - The packet timestamp must be accessed only while holding the pending + * lock. + */ + +/* + * SSH_PTL_MAX_PACKET_TRIES - Maximum transmission attempts for packet. + * + * Maximum number of transmission attempts per sequenced packet in case of + * time-outs. Must be smaller than 16. If the packet times out after this + * amount of tries, the packet will be completed with %-ETIMEDOUT as status + * code. + */ +#define SSH_PTL_MAX_PACKET_TRIES 3 + +/* + * SSH_PTL_TX_TIMEOUT - Packet transmission timeout. + * + * Timeout in jiffies for packet transmission via the underlying serial + * device. If transmitting the packet takes longer than this timeout, the + * packet will be completed with -ETIMEDOUT. It will not be re-submitted. + */ +#define SSH_PTL_TX_TIMEOUT HZ + +/* + * SSH_PTL_PACKET_TIMEOUT - Packet response timeout. + * + * Timeout as ktime_t delta for ACKs. If we have not received an ACK in this + * time-frame after starting transmission, the packet will be re-submitted. + */ +#define SSH_PTL_PACKET_TIMEOUT ms_to_ktime(1000) + +/* + * SSH_PTL_PACKET_TIMEOUT_RESOLUTION - Packet timeout granularity. + * + * Time-resolution for timeouts. Should be larger than one jiffy to avoid + * direct re-scheduling of reaper work_struct. + */ +#define SSH_PTL_PACKET_TIMEOUT_RESOLUTION ms_to_ktime(max(2000 / HZ, 50)) + +/* + * SSH_PTL_MAX_PENDING - Maximum number of pending packets. + * + * Maximum number of sequenced packets concurrently waiting for an ACK. + * Packets marked as blocking will not be transmitted while this limit is + * reached. + */ +#define SSH_PTL_MAX_PENDING 1 + +/* + * SSH_PTL_RX_BUF_LEN - Evaluation-buffer size in bytes. + */ +#define SSH_PTL_RX_BUF_LEN 4096 + +/* + * SSH_PTL_RX_FIFO_LEN - Fifo input-buffer size in bytes. + */ +#define SSH_PTL_RX_FIFO_LEN 4096 + +#ifdef CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION + +/** + * ssh_ptl_should_drop_ack_packet() - Error injection hook to drop ACK packets. + * + * Useful to test detection and handling of automated re-transmits by the EC. + * Specifically of packets that the EC considers not-ACKed but the driver + * already considers ACKed (due to dropped ACK). In this case, the EC + * re-transmits the packet-to-be-ACKed and the driver should detect it as + * duplicate/already handled. Note that the driver should still send an ACK + * for the re-transmitted packet. + */ +static noinline bool ssh_ptl_should_drop_ack_packet(void) +{ + return false; +} +ALLOW_ERROR_INJECTION(ssh_ptl_should_drop_ack_packet, TRUE); + +/** + * ssh_ptl_should_drop_nak_packet() - Error injection hook to drop NAK packets. + * + * Useful to test/force automated (timeout-based) re-transmit by the EC. + * Specifically, packets that have not reached the driver completely/with valid + * checksums. Only useful in combination with receival of (injected) bad data. + */ +static noinline bool ssh_ptl_should_drop_nak_packet(void) +{ + return false; +} +ALLOW_ERROR_INJECTION(ssh_ptl_should_drop_nak_packet, TRUE); + +/** + * ssh_ptl_should_drop_dsq_packet() - Error injection hook to drop sequenced + * data packet. + * + * Useful to test re-transmit timeout of the driver. If the data packet has not + * been ACKed after a certain time, the driver should re-transmit the packet up + * to limited number of times defined in SSH_PTL_MAX_PACKET_TRIES. + */ +static noinline bool ssh_ptl_should_drop_dsq_packet(void) +{ + return false; +} +ALLOW_ERROR_INJECTION(ssh_ptl_should_drop_dsq_packet, TRUE); + +/** + * ssh_ptl_should_fail_write() - Error injection hook to make + * serdev_device_write() fail. + * + * Hook to simulate errors in serdev_device_write when transmitting packets. + */ +static noinline int ssh_ptl_should_fail_write(void) +{ + return 0; +} +ALLOW_ERROR_INJECTION(ssh_ptl_should_fail_write, ERRNO); + +/** + * ssh_ptl_should_corrupt_tx_data() - Error injection hook to simulate invalid + * data being sent to the EC. + * + * Hook to simulate corrupt/invalid data being sent from host (driver) to EC. + * Causes the packet data to be actively corrupted by overwriting it with + * pre-defined values, such that it becomes invalid, causing the EC to respond + * with a NAK packet. Useful to test handling of NAK packets received by the + * driver. + */ +static noinline bool ssh_ptl_should_corrupt_tx_data(void) +{ + return false; +} +ALLOW_ERROR_INJECTION(ssh_ptl_should_corrupt_tx_data, TRUE); + +/** + * ssh_ptl_should_corrupt_rx_syn() - Error injection hook to simulate invalid + * data being sent by the EC. + * + * Hook to simulate invalid SYN bytes, i.e. an invalid start of messages and + * test handling thereof in the driver. + */ +static noinline bool ssh_ptl_should_corrupt_rx_syn(void) +{ + return false; +} +ALLOW_ERROR_INJECTION(ssh_ptl_should_corrupt_rx_syn, TRUE); + +/** + * ssh_ptl_should_corrupt_rx_data() - Error injection hook to simulate invalid + * data being sent by the EC. + * + * Hook to simulate invalid data/checksum of the message frame and test handling + * thereof in the driver. + */ +static noinline bool ssh_ptl_should_corrupt_rx_data(void) +{ + return false; +} +ALLOW_ERROR_INJECTION(ssh_ptl_should_corrupt_rx_data, TRUE); + +static bool __ssh_ptl_should_drop_ack_packet(struct ssh_packet *packet) +{ + if (likely(!ssh_ptl_should_drop_ack_packet())) + return false; + + trace_ssam_ei_tx_drop_ack_packet(packet); + ptl_info(packet->ptl, "packet error injection: dropping ACK packet %p\n", + packet); + + return true; +} + +static bool __ssh_ptl_should_drop_nak_packet(struct ssh_packet *packet) +{ + if (likely(!ssh_ptl_should_drop_nak_packet())) + return false; + + trace_ssam_ei_tx_drop_nak_packet(packet); + ptl_info(packet->ptl, "packet error injection: dropping NAK packet %p\n", + packet); + + return true; +} + +static bool __ssh_ptl_should_drop_dsq_packet(struct ssh_packet *packet) +{ + if (likely(!ssh_ptl_should_drop_dsq_packet())) + return false; + + trace_ssam_ei_tx_drop_dsq_packet(packet); + ptl_info(packet->ptl, + "packet error injection: dropping sequenced data packet %p\n", + packet); + + return true; +} + +static bool ssh_ptl_should_drop_packet(struct ssh_packet *packet) +{ + /* Ignore packets that don't carry any data (i.e. flush). */ + if (!packet->data.ptr || !packet->data.len) + return false; + + switch (packet->data.ptr[SSH_MSGOFFSET_FRAME(type)]) { + case SSH_FRAME_TYPE_ACK: + return __ssh_ptl_should_drop_ack_packet(packet); + + case SSH_FRAME_TYPE_NAK: + return __ssh_ptl_should_drop_nak_packet(packet); + + case SSH_FRAME_TYPE_DATA_SEQ: + return __ssh_ptl_should_drop_dsq_packet(packet); + + default: + return false; + } +} + +static int ssh_ptl_write_buf(struct ssh_ptl *ptl, struct ssh_packet *packet, + const unsigned char *buf, size_t count) +{ + int status; + + status = ssh_ptl_should_fail_write(); + if (unlikely(status)) { + trace_ssam_ei_tx_fail_write(packet, status); + ptl_info(packet->ptl, + "packet error injection: simulating transmit error %d, packet %p\n", + status, packet); + + return status; + } + + return serdev_device_write_buf(ptl->serdev, buf, count); +} + +static void ssh_ptl_tx_inject_invalid_data(struct ssh_packet *packet) +{ + /* Ignore packets that don't carry any data (i.e. flush). */ + if (!packet->data.ptr || !packet->data.len) + return; + + /* Only allow sequenced data packets to be modified. */ + if (packet->data.ptr[SSH_MSGOFFSET_FRAME(type)] != SSH_FRAME_TYPE_DATA_SEQ) + return; + + if (likely(!ssh_ptl_should_corrupt_tx_data())) + return; + + trace_ssam_ei_tx_corrupt_data(packet); + ptl_info(packet->ptl, + "packet error injection: simulating invalid transmit data on packet %p\n", + packet); + + /* + * NB: The value 0xb3 has been chosen more or less randomly so that it + * doesn't have any (major) overlap with the SYN bytes (aa 55) and is + * non-trivial (i.e. non-zero, non-0xff). + */ + memset(packet->data.ptr, 0xb3, packet->data.len); +} + +static void ssh_ptl_rx_inject_invalid_syn(struct ssh_ptl *ptl, + struct ssam_span *data) +{ + struct ssam_span frame; + + /* Check if there actually is something to corrupt. */ + if (!sshp_find_syn(data, &frame)) + return; + + if (likely(!ssh_ptl_should_corrupt_rx_syn())) + return; + + trace_ssam_ei_rx_corrupt_syn(data->len); + + data->ptr[1] = 0xb3; /* Set second byte of SYN to "random" value. */ +} + +static void ssh_ptl_rx_inject_invalid_data(struct ssh_ptl *ptl, + struct ssam_span *frame) +{ + size_t payload_len, message_len; + struct ssh_frame *sshf; + + /* Ignore incomplete messages, will get handled once it's complete. */ + if (frame->len < SSH_MESSAGE_LENGTH(0)) + return; + + /* Ignore incomplete messages, part 2. */ + payload_len = get_unaligned_le16(&frame->ptr[SSH_MSGOFFSET_FRAME(len)]); + message_len = SSH_MESSAGE_LENGTH(payload_len); + if (frame->len < message_len) + return; + + if (likely(!ssh_ptl_should_corrupt_rx_data())) + return; + + sshf = (struct ssh_frame *)&frame->ptr[SSH_MSGOFFSET_FRAME(type)]; + trace_ssam_ei_rx_corrupt_data(sshf); + + /* + * Flip bits in first byte of payload checksum. This is basically + * equivalent to a payload/frame data error without us having to worry + * about (the, arguably pretty small, probability of) accidental + * checksum collisions. + */ + frame->ptr[frame->len - 2] = ~frame->ptr[frame->len - 2]; +} + +#else /* CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION */ + +static inline bool ssh_ptl_should_drop_packet(struct ssh_packet *packet) +{ + return false; +} + +static inline int ssh_ptl_write_buf(struct ssh_ptl *ptl, + struct ssh_packet *packet, + const unsigned char *buf, + size_t count) +{ + return serdev_device_write_buf(ptl->serdev, buf, count); +} + +static inline void ssh_ptl_tx_inject_invalid_data(struct ssh_packet *packet) +{ +} + +static inline void ssh_ptl_rx_inject_invalid_syn(struct ssh_ptl *ptl, + struct ssam_span *data) +{ +} + +static inline void ssh_ptl_rx_inject_invalid_data(struct ssh_ptl *ptl, + struct ssam_span *frame) +{ +} + +#endif /* CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION */ + +static void __ssh_ptl_packet_release(struct kref *kref) +{ + struct ssh_packet *p = container_of(kref, struct ssh_packet, refcnt); + + trace_ssam_packet_release(p); + + ptl_dbg_cond(p->ptl, "ptl: releasing packet %p\n", p); + p->ops->release(p); +} + +/** + * ssh_packet_get() - Increment reference count of packet. + * @packet: The packet to increment the reference count of. + * + * Increments the reference count of the given packet. See ssh_packet_put() + * for the counter-part of this function. + * + * Return: Returns the packet provided as input. + */ +struct ssh_packet *ssh_packet_get(struct ssh_packet *packet) +{ + if (packet) + kref_get(&packet->refcnt); + return packet; +} +EXPORT_SYMBOL_GPL(ssh_packet_get); + +/** + * ssh_packet_put() - Decrement reference count of packet. + * @packet: The packet to decrement the reference count of. + * + * If the reference count reaches zero, the ``release`` callback specified in + * the packet's &struct ssh_packet_ops, i.e. ``packet->ops->release``, will be + * called. + * + * See ssh_packet_get() for the counter-part of this function. + */ +void ssh_packet_put(struct ssh_packet *packet) +{ + if (packet) + kref_put(&packet->refcnt, __ssh_ptl_packet_release); +} +EXPORT_SYMBOL_GPL(ssh_packet_put); + +static u8 ssh_packet_get_seq(struct ssh_packet *packet) +{ + return packet->data.ptr[SSH_MSGOFFSET_FRAME(seq)]; +} + +/** + * ssh_packet_init() - Initialize SSH packet. + * @packet: The packet to initialize. + * @type: Type-flags of the packet. + * @priority: Priority of the packet. See SSH_PACKET_PRIORITY() for details. + * @ops: Packet operations. + * + * Initializes the given SSH packet. Sets the transmission buffer pointer to + * %NULL and the transmission buffer length to zero. For data-type packets, + * this buffer has to be set separately via ssh_packet_set_data() before + * submission, and must contain a valid SSH message, i.e. frame with optional + * payload of any type. + */ +void ssh_packet_init(struct ssh_packet *packet, unsigned long type, + u8 priority, const struct ssh_packet_ops *ops) +{ + kref_init(&packet->refcnt); + + packet->ptl = NULL; + INIT_LIST_HEAD(&packet->queue_node); + INIT_LIST_HEAD(&packet->pending_node); + + packet->state = type & SSH_PACKET_FLAGS_TY_MASK; + packet->priority = priority; + packet->timestamp = KTIME_MAX; + + packet->data.ptr = NULL; + packet->data.len = 0; + + packet->ops = ops; +} + +static struct kmem_cache *ssh_ctrl_packet_cache; + +/** + * ssh_ctrl_packet_cache_init() - Initialize the control packet cache. + */ +int ssh_ctrl_packet_cache_init(void) +{ + const unsigned int size = sizeof(struct ssh_packet) + SSH_MSG_LEN_CTRL; + const unsigned int align = __alignof__(struct ssh_packet); + struct kmem_cache *cache; + + cache = kmem_cache_create("ssam_ctrl_packet", size, align, 0, NULL); + if (!cache) + return -ENOMEM; + + ssh_ctrl_packet_cache = cache; + return 0; +} + +/** + * ssh_ctrl_packet_cache_destroy() - Deinitialize the control packet cache. + */ +void ssh_ctrl_packet_cache_destroy(void) +{ + kmem_cache_destroy(ssh_ctrl_packet_cache); + ssh_ctrl_packet_cache = NULL; +} + +/** + * ssh_ctrl_packet_alloc() - Allocate packet from control packet cache. + * @packet: Where the pointer to the newly allocated packet should be stored. + * @buffer: The buffer corresponding to this packet. + * @flags: Flags used for allocation. + * + * Allocates a packet and corresponding transport buffer from the control + * packet cache. Sets the packet's buffer reference to the allocated buffer. + * The packet must be freed via ssh_ctrl_packet_free(), which will also free + * the corresponding buffer. The corresponding buffer must not be freed + * separately. Intended to be used with %ssh_ptl_ctrl_packet_ops as packet + * operations. + * + * Return: Returns zero on success, %-ENOMEM if the allocation failed. + */ +static int ssh_ctrl_packet_alloc(struct ssh_packet **packet, + struct ssam_span *buffer, gfp_t flags) +{ + *packet = kmem_cache_alloc(ssh_ctrl_packet_cache, flags); + if (!*packet) + return -ENOMEM; + + buffer->ptr = (u8 *)(*packet + 1); + buffer->len = SSH_MSG_LEN_CTRL; + + trace_ssam_ctrl_packet_alloc(*packet, buffer->len); + return 0; +} + +/** + * ssh_ctrl_packet_free() - Free packet allocated from control packet cache. + * @p: The packet to free. + */ +static void ssh_ctrl_packet_free(struct ssh_packet *p) +{ + trace_ssam_ctrl_packet_free(p); + kmem_cache_free(ssh_ctrl_packet_cache, p); +} + +static const struct ssh_packet_ops ssh_ptl_ctrl_packet_ops = { + .complete = NULL, + .release = ssh_ctrl_packet_free, +}; + +static void ssh_ptl_timeout_reaper_mod(struct ssh_ptl *ptl, ktime_t now, + ktime_t expires) +{ + unsigned long delta = msecs_to_jiffies(ktime_ms_delta(expires, now)); + ktime_t aexp = ktime_add(expires, SSH_PTL_PACKET_TIMEOUT_RESOLUTION); + + spin_lock(&ptl->rtx_timeout.lock); + + /* Re-adjust / schedule reaper only if it is above resolution delta. */ + if (ktime_before(aexp, ptl->rtx_timeout.expires)) { + ptl->rtx_timeout.expires = expires; + mod_delayed_work(system_wq, &ptl->rtx_timeout.reaper, delta); + } + + spin_unlock(&ptl->rtx_timeout.lock); +} + +/* Must be called with queue lock held. */ +static void ssh_packet_next_try(struct ssh_packet *p) +{ + u8 base = ssh_packet_priority_get_base(p->priority); + u8 try = ssh_packet_priority_get_try(p->priority); + + lockdep_assert_held(&p->ptl->queue.lock); + + /* + * Ensure that we write the priority in one go via WRITE_ONCE() so we + * can access it via READ_ONCE() for tracing. Note that other access + * is guarded by the queue lock, so no need to use READ_ONCE() there. + */ + WRITE_ONCE(p->priority, __SSH_PACKET_PRIORITY(base, try + 1)); +} + +/* Must be called with queue lock held. */ +static struct list_head *__ssh_ptl_queue_find_entrypoint(struct ssh_packet *p) +{ + struct list_head *head; + struct ssh_packet *q; + + lockdep_assert_held(&p->ptl->queue.lock); + + /* + * We generally assume that there are less control (ACK/NAK) packets + * and re-submitted data packets as there are normal data packets (at + * least in situations in which many packets are queued; if there + * aren't many packets queued the decision on how to iterate should be + * basically irrelevant; the number of control/data packets is more or + * less limited via the maximum number of pending packets). Thus, when + * inserting a control or re-submitted data packet, (determined by + * their priority), we search from front to back. Normal data packets + * are, usually queued directly at the tail of the queue, so for those + * search from back to front. + */ + + if (p->priority > SSH_PACKET_PRIORITY(DATA, 0)) { + list_for_each(head, &p->ptl->queue.head) { + q = list_entry(head, struct ssh_packet, queue_node); + + if (q->priority < p->priority) + break; + } + } else { + list_for_each_prev(head, &p->ptl->queue.head) { + q = list_entry(head, struct ssh_packet, queue_node); + + if (q->priority >= p->priority) { + head = head->next; + break; + } + } + } + + return head; +} + +/* Must be called with queue lock held. */ +static int __ssh_ptl_queue_push(struct ssh_packet *packet) +{ + struct ssh_ptl *ptl = packet->ptl; + struct list_head *head; + + lockdep_assert_held(&ptl->queue.lock); + + if (test_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state)) + return -ESHUTDOWN; + + /* Avoid further transitions when canceling/completing. */ + if (test_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state)) + return -EINVAL; + + /* If this packet has already been queued, do not add it. */ + if (test_and_set_bit(SSH_PACKET_SF_QUEUED_BIT, &packet->state)) + return -EALREADY; + + head = __ssh_ptl_queue_find_entrypoint(packet); + + list_add_tail(&ssh_packet_get(packet)->queue_node, head); + return 0; +} + +static int ssh_ptl_queue_push(struct ssh_packet *packet) +{ + int status; + + spin_lock(&packet->ptl->queue.lock); + status = __ssh_ptl_queue_push(packet); + spin_unlock(&packet->ptl->queue.lock); + + return status; +} + +static void ssh_ptl_queue_remove(struct ssh_packet *packet) +{ + struct ssh_ptl *ptl = packet->ptl; + + spin_lock(&ptl->queue.lock); + + if (!test_and_clear_bit(SSH_PACKET_SF_QUEUED_BIT, &packet->state)) { + spin_unlock(&ptl->queue.lock); + return; + } + + list_del(&packet->queue_node); + + spin_unlock(&ptl->queue.lock); + ssh_packet_put(packet); +} + +static void ssh_ptl_pending_push(struct ssh_packet *p) +{ + struct ssh_ptl *ptl = p->ptl; + const ktime_t timestamp = ktime_get_coarse_boottime(); + const ktime_t timeout = ptl->rtx_timeout.timeout; + + /* + * Note: We can get the time for the timestamp before acquiring the + * lock as this is the only place we're setting it and this function + * is called only from the transmitter thread. Thus it is not possible + * to overwrite the timestamp with an outdated value below. + */ + + spin_lock(&ptl->pending.lock); + + /* If we are canceling/completing this packet, do not add it. */ + if (test_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state)) { + spin_unlock(&ptl->pending.lock); + return; + } + + /* + * On re-submission, the packet has already been added the pending + * set. We still need to update the timestamp as the packet timeout is + * reset for each (re-)submission. + */ + p->timestamp = timestamp; + + /* In case it is already pending (e.g. re-submission), do not add it. */ + if (!test_and_set_bit(SSH_PACKET_SF_PENDING_BIT, &p->state)) { + atomic_inc(&ptl->pending.count); + list_add_tail(&ssh_packet_get(p)->pending_node, &ptl->pending.head); + } + + spin_unlock(&ptl->pending.lock); + + /* Arm/update timeout reaper. */ + ssh_ptl_timeout_reaper_mod(ptl, timestamp, timestamp + timeout); +} + +static void ssh_ptl_pending_remove(struct ssh_packet *packet) +{ + struct ssh_ptl *ptl = packet->ptl; + + spin_lock(&ptl->pending.lock); + + if (!test_and_clear_bit(SSH_PACKET_SF_PENDING_BIT, &packet->state)) { + spin_unlock(&ptl->pending.lock); + return; + } + + list_del(&packet->pending_node); + atomic_dec(&ptl->pending.count); + + spin_unlock(&ptl->pending.lock); + + ssh_packet_put(packet); +} + +/* Warning: Does not check/set "completed" bit. */ +static void __ssh_ptl_complete(struct ssh_packet *p, int status) +{ + struct ssh_ptl *ptl = READ_ONCE(p->ptl); + + trace_ssam_packet_complete(p, status); + ptl_dbg_cond(ptl, "ptl: completing packet %p (status: %d)\n", p, status); + + if (p->ops->complete) + p->ops->complete(p, status); +} + +static void ssh_ptl_remove_and_complete(struct ssh_packet *p, int status) +{ + /* + * A call to this function should in general be preceded by + * set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->flags) to avoid re-adding the + * packet to the structures it's going to be removed from. + * + * The set_bit call does not need explicit memory barriers as the + * implicit barrier of the test_and_set_bit() call below ensure that the + * flag is visible before we actually attempt to remove the packet. + */ + + if (test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state)) + return; + + ssh_ptl_queue_remove(p); + ssh_ptl_pending_remove(p); + + __ssh_ptl_complete(p, status); +} + +static bool ssh_ptl_tx_can_process(struct ssh_packet *packet) +{ + struct ssh_ptl *ptl = packet->ptl; + + if (test_bit(SSH_PACKET_TY_FLUSH_BIT, &packet->state)) + return !atomic_read(&ptl->pending.count); + + /* We can always process non-blocking packets. */ + if (!test_bit(SSH_PACKET_TY_BLOCKING_BIT, &packet->state)) + return true; + + /* If we are already waiting for this packet, send it again. */ + if (test_bit(SSH_PACKET_SF_PENDING_BIT, &packet->state)) + return true; + + /* Otherwise: Check if we have the capacity to send. */ + return atomic_read(&ptl->pending.count) < SSH_PTL_MAX_PENDING; +} + +static struct ssh_packet *ssh_ptl_tx_pop(struct ssh_ptl *ptl) +{ + struct ssh_packet *packet = ERR_PTR(-ENOENT); + struct ssh_packet *p, *n; + + spin_lock(&ptl->queue.lock); + list_for_each_entry_safe(p, n, &ptl->queue.head, queue_node) { + /* + * If we are canceling or completing this packet, ignore it. + * It's going to be removed from this queue shortly. + */ + if (test_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state)) + continue; + + /* + * Packets should be ordered non-blocking/to-be-resent first. + * If we cannot process this packet, assume that we can't + * process any following packet either and abort. + */ + if (!ssh_ptl_tx_can_process(p)) { + packet = ERR_PTR(-EBUSY); + break; + } + + /* + * We are allowed to change the state now. Remove it from the + * queue and mark it as being transmitted. + */ + + list_del(&p->queue_node); + + set_bit(SSH_PACKET_SF_TRANSMITTING_BIT, &p->state); + /* Ensure that state never gets zero. */ + smp_mb__before_atomic(); + clear_bit(SSH_PACKET_SF_QUEUED_BIT, &p->state); + + /* + * Update number of tries. This directly influences the + * priority in case the packet is re-submitted (e.g. via + * timeout/NAK). Note that all reads and writes to the + * priority after the first submission are guarded by the + * queue lock. + */ + ssh_packet_next_try(p); + + packet = p; + break; + } + spin_unlock(&ptl->queue.lock); + + return packet; +} + +static struct ssh_packet *ssh_ptl_tx_next(struct ssh_ptl *ptl) +{ + struct ssh_packet *p; + + p = ssh_ptl_tx_pop(ptl); + if (IS_ERR(p)) + return p; + + if (test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &p->state)) { + ptl_dbg(ptl, "ptl: transmitting sequenced packet %p\n", p); + ssh_ptl_pending_push(p); + } else { + ptl_dbg(ptl, "ptl: transmitting non-sequenced packet %p\n", p); + } + + return p; +} + +static void ssh_ptl_tx_compl_success(struct ssh_packet *packet) +{ + struct ssh_ptl *ptl = packet->ptl; + + ptl_dbg(ptl, "ptl: successfully transmitted packet %p\n", packet); + + /* Transition state to "transmitted". */ + set_bit(SSH_PACKET_SF_TRANSMITTED_BIT, &packet->state); + /* Ensure that state never gets zero. */ + smp_mb__before_atomic(); + clear_bit(SSH_PACKET_SF_TRANSMITTING_BIT, &packet->state); + + /* If the packet is unsequenced, we're done: Lock and complete. */ + if (!test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &packet->state)) { + set_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state); + ssh_ptl_remove_and_complete(packet, 0); + } + + /* + * Notify that a packet transmission has finished. In general we're only + * waiting for one packet (if any), so wake_up_all should be fine. + */ + wake_up_all(&ptl->tx.packet_wq); +} + +static void ssh_ptl_tx_compl_error(struct ssh_packet *packet, int status) +{ + /* Transmission failure: Lock the packet and try to complete it. */ + set_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state); + /* Ensure that state never gets zero. */ + smp_mb__before_atomic(); + clear_bit(SSH_PACKET_SF_TRANSMITTING_BIT, &packet->state); + + ptl_err(packet->ptl, "ptl: transmission error: %d\n", status); + ptl_dbg(packet->ptl, "ptl: failed to transmit packet: %p\n", packet); + + ssh_ptl_remove_and_complete(packet, status); + + /* + * Notify that a packet transmission has finished. In general we're only + * waiting for one packet (if any), so wake_up_all should be fine. + */ + wake_up_all(&packet->ptl->tx.packet_wq); +} + +static long ssh_ptl_tx_wait_packet(struct ssh_ptl *ptl) +{ + int status; + + status = wait_for_completion_interruptible(&ptl->tx.thread_cplt_pkt); + reinit_completion(&ptl->tx.thread_cplt_pkt); + + /* + * Ensure completion is cleared before continuing to avoid lost update + * problems. + */ + smp_mb__after_atomic(); + + return status; +} + +static long ssh_ptl_tx_wait_transfer(struct ssh_ptl *ptl, long timeout) +{ + long status; + + status = wait_for_completion_interruptible_timeout(&ptl->tx.thread_cplt_tx, + timeout); + reinit_completion(&ptl->tx.thread_cplt_tx); + + /* + * Ensure completion is cleared before continuing to avoid lost update + * problems. + */ + smp_mb__after_atomic(); + + return status; +} + +static int ssh_ptl_tx_packet(struct ssh_ptl *ptl, struct ssh_packet *packet) +{ + long timeout = SSH_PTL_TX_TIMEOUT; + size_t offset = 0; + + /* Note: Flush-packets don't have any data. */ + if (unlikely(!packet->data.ptr)) + return 0; + + /* Error injection: drop packet to simul |