diff options
Diffstat (limited to 'drivers/char/xillybus/xillybus_core.c')
| -rw-r--r-- | drivers/char/xillybus/xillybus_core.c | 2103 |
1 files changed, 2103 insertions, 0 deletions
diff --git a/drivers/char/xillybus/xillybus_core.c b/drivers/char/xillybus/xillybus_core.c new file mode 100644 index 000000000000..b827fa095f1b --- /dev/null +++ b/drivers/char/xillybus/xillybus_core.c @@ -0,0 +1,2103 @@ +/* + * linux/drivers/misc/xillybus_core.c + * + * Copyright 2011 Xillybus Ltd, http://xillybus.com + * + * Driver for the Xillybus FPGA/host framework. + * + * This driver interfaces with a special IP core in an FPGA, setting up + * a pipe between a hardware FIFO in the programmable logic and a device + * file in the host. The number of such pipes and their attributes are + * set up on the logic. This driver detects these automatically and + * creates the device files accordingly. + * + * This program is free software; you can redistribute it and/or modify + * it under the smems of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + */ + +#include <linux/list.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/io.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/sched.h> +#include <linux/fs.h> +#include <linux/cdev.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/crc32.h> +#include <linux/poll.h> +#include <linux/delay.h> +#include <linux/slab.h> +#include <linux/workqueue.h> +#include "xillybus.h" + +MODULE_DESCRIPTION("Xillybus core functions"); +MODULE_AUTHOR("Eli Billauer, Xillybus Ltd."); +MODULE_VERSION("1.07"); +MODULE_ALIAS("xillybus_core"); +MODULE_LICENSE("GPL v2"); + +/* General timeout is 100 ms, rx timeout is 10 ms */ +#define XILLY_RX_TIMEOUT (10*HZ/1000) +#define XILLY_TIMEOUT (100*HZ/1000) + +#define fpga_msg_ctrl_reg 0x0008 +#define fpga_dma_control_reg 0x0020 +#define fpga_dma_bufno_reg 0x0024 +#define fpga_dma_bufaddr_lowaddr_reg 0x0028 +#define fpga_dma_bufaddr_highaddr_reg 0x002c +#define fpga_buf_ctrl_reg 0x0030 +#define fpga_buf_offset_reg 0x0034 +#define fpga_endian_reg 0x0040 + +#define XILLYMSG_OPCODE_RELEASEBUF 1 +#define XILLYMSG_OPCODE_QUIESCEACK 2 +#define XILLYMSG_OPCODE_FIFOEOF 3 +#define XILLYMSG_OPCODE_FATAL_ERROR 4 +#define XILLYMSG_OPCODE_NONEMPTY 5 + +static const char xillyname[] = "xillybus"; + +static struct class *xillybus_class; + +/* + * ep_list_lock is the last lock to be taken; No other lock requests are + * allowed while holding it. It merely protects list_of_endpoints, and not + * the endpoints listed in it. + */ + +static LIST_HEAD(list_of_endpoints); +static struct mutex ep_list_lock; +static struct workqueue_struct *xillybus_wq; + +/* + * Locking scheme: Mutexes protect invocations of character device methods. + * If both locks are taken, wr_mutex is taken first, rd_mutex second. + * + * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the + * buffers' end_offset fields against changes made by IRQ handler (and in + * theory, other file request handlers, but the mutex handles that). Nothing + * else. + * They are held for short direct memory manipulations. Needless to say, + * no mutex locking is allowed when a spinlock is held. + * + * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset. + * + * register_mutex is endpoint-specific, and is held when non-atomic + * register operations are performed. wr_mutex and rd_mutex may be + * held when register_mutex is taken, but none of the spinlocks. Note that + * register_mutex doesn't protect against sporadic buf_ctrl_reg writes + * which are unrelated to buf_offset_reg, since they are harmless. + * + * Blocking on the wait queues is allowed with mutexes held, but not with + * spinlocks. + * + * Only interruptible blocking is allowed on mutexes and wait queues. + * + * All in all, the locking order goes (with skips allowed, of course): + * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock + */ + +static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf) +{ + int opcode; + int msg_channel, msg_bufno, msg_data, msg_dir; + + opcode = (buf[0] >> 24) & 0xff; + msg_dir = buf[0] & 1; + msg_channel = (buf[0] >> 1) & 0x7ff; + msg_bufno = (buf[0] >> 12) & 0x3ff; + msg_data = buf[1] & 0xfffffff; + + dev_warn(endpoint->dev, + "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n", + opcode, msg_channel, msg_dir, msg_bufno, msg_data); +} + +/* + * xillybus_isr assumes the interrupt is allocated exclusively to it, + * which is the natural case MSI and several other hardware-oriented + * interrupts. Sharing is not allowed. + */ + +irqreturn_t xillybus_isr(int irq, void *data) +{ + struct xilly_endpoint *ep = data; + u32 *buf; + unsigned int buf_size; + int i; + int opcode; + unsigned int msg_channel, msg_bufno, msg_data, msg_dir; + struct xilly_channel *channel; + + buf = ep->msgbuf_addr; + buf_size = ep->msg_buf_size/sizeof(u32); + + ep->ephw->hw_sync_sgl_for_cpu(ep, + ep->msgbuf_dma_addr, + ep->msg_buf_size, + DMA_FROM_DEVICE); + + for (i = 0; i < buf_size; i += 2) { + if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) { + malformed_message(ep, &buf[i]); + dev_warn(ep->dev, + "Sending a NACK on counter %x (instead of %x) on entry %d\n", + ((buf[i+1] >> 28) & 0xf), + ep->msg_counter, + i/2); + + if (++ep->failed_messages > 10) { + dev_err(ep->dev, + "Lost sync with interrupt messages. Stopping.\n"); + } else { + ep->ephw->hw_sync_sgl_for_device( + ep, + ep->msgbuf_dma_addr, + ep->msg_buf_size, + DMA_FROM_DEVICE); + + iowrite32(0x01, /* Message NACK */ + ep->registers + fpga_msg_ctrl_reg); + } + return IRQ_HANDLED; + } else if (buf[i] & (1 << 22)) /* Last message */ + break; + } + + if (i >= buf_size) { + dev_err(ep->dev, "Bad interrupt message. Stopping.\n"); + return IRQ_HANDLED; + } + + buf_size = i + 2; + + for (i = 0; i < buf_size; i += 2) { /* Scan through messages */ + opcode = (buf[i] >> 24) & 0xff; + + msg_dir = buf[i] & 1; + msg_channel = (buf[i] >> 1) & 0x7ff; + msg_bufno = (buf[i] >> 12) & 0x3ff; + msg_data = buf[i+1] & 0xfffffff; + + switch (opcode) { + case XILLYMSG_OPCODE_RELEASEBUF: + if ((msg_channel > ep->num_channels) || + (msg_channel == 0)) { + malformed_message(ep, &buf[i]); + break; + } + + channel = ep->channels[msg_channel]; + + if (msg_dir) { /* Write channel */ + if (msg_bufno >= channel->num_wr_buffers) { + malformed_message(ep, &buf[i]); + break; + } + spin_lock(&channel->wr_spinlock); + channel->wr_buffers[msg_bufno]->end_offset = + msg_data; + channel->wr_fpga_buf_idx = msg_bufno; + channel->wr_empty = 0; + channel->wr_sleepy = 0; + spin_unlock(&channel->wr_spinlock); + + wake_up_interruptible(&channel->wr_wait); + + } else { + /* Read channel */ + + if (msg_bufno >= channel->num_rd_buffers) { + malformed_message(ep, &buf[i]); + break; + } + + spin_lock(&channel->rd_spinlock); + channel->rd_fpga_buf_idx = msg_bufno; + channel->rd_full = 0; + spin_unlock(&channel->rd_spinlock); + + wake_up_interruptible(&channel->rd_wait); + if (!channel->rd_synchronous) + queue_delayed_work( + xillybus_wq, + &channel->rd_workitem, + XILLY_RX_TIMEOUT); + } + + break; + case XILLYMSG_OPCODE_NONEMPTY: + if ((msg_channel > ep->num_channels) || + (msg_channel == 0) || (!msg_dir) || + !ep->channels[msg_channel]->wr_supports_nonempty) { + malformed_message(ep, &buf[i]); + break; + } + + channel = ep->channels[msg_channel]; + + if (msg_bufno >= channel->num_wr_buffers) { + malformed_message(ep, &buf[i]); + break; + } + spin_lock(&channel->wr_spinlock); + if (msg_bufno == channel->wr_host_buf_idx) + channel->wr_ready = 1; + spin_unlock(&channel->wr_spinlock); + + wake_up_interruptible(&channel->wr_ready_wait); + + break; + case XILLYMSG_OPCODE_QUIESCEACK: + ep->idtlen = msg_data; + wake_up_interruptible(&ep->ep_wait); + + break; + case XILLYMSG_OPCODE_FIFOEOF: + if ((msg_channel > ep->num_channels) || + (msg_channel == 0) || (!msg_dir) || + !ep->channels[msg_channel]->num_wr_buffers) { + malformed_message(ep, &buf[i]); + break; + } + channel = ep->channels[msg_channel]; + spin_lock(&channel->wr_spinlock); + channel->wr_eof = msg_bufno; + channel->wr_sleepy = 0; + + channel->wr_hangup = channel->wr_empty && + (channel->wr_host_buf_idx == msg_bufno); + + spin_unlock(&channel->wr_spinlock); + + wake_up_interruptible(&channel->wr_wait); + + break; + case XILLYMSG_OPCODE_FATAL_ERROR: + ep->fatal_error = 1; + wake_up_interruptible(&ep->ep_wait); /* For select() */ + dev_err(ep->dev, + "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n"); + break; + default: + malformed_message(ep, &buf[i]); + break; + } + } + + ep->ephw->hw_sync_sgl_for_device(ep, + ep->msgbuf_dma_addr, + ep->msg_buf_size, + DMA_FROM_DEVICE); + + ep->msg_counter = (ep->msg_counter + 1) & 0xf; + ep->failed_messages = 0; + iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */ + + return IRQ_HANDLED; +} +EXPORT_SYMBOL(xillybus_isr); + +/* + * A few trivial memory management functions. + * NOTE: These functions are used only on probe and remove, and therefore + * no locks are applied! + */ + +static void xillybus_autoflush(struct work_struct *work); + +struct xilly_alloc_state { + void *salami; + int left_of_salami; + int nbuffer; + enum dma_data_direction direction; + u32 regdirection; +}; + +static int xilly_get_dma_buffers(struct xilly_endpoint *ep, + struct xilly_alloc_state *s, + struct xilly_buffer **buffers, + int bufnum, int bytebufsize) +{ + int i, rc; + dma_addr_t dma_addr; + struct device *dev = ep->dev; + struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */ + + if (buffers) { /* Not the message buffer */ + this_buffer = devm_kcalloc(dev, bufnum, + sizeof(struct xilly_buffer), + GFP_KERNEL); + if (!this_buffer) + return -ENOMEM; + } + + for (i = 0; i < bufnum; i++) { + /* + * Buffers are expected in descending size order, so there + * is either enough space for this buffer or none at all. + */ + + if ((s->left_of_salami < bytebufsize) && + (s->left_of_salami > 0)) { + dev_err(ep->dev, + "Corrupt buffer allocation in IDT. Aborting.\n"); + return -ENODEV; + } + + if (s->left_of_salami == 0) { + int allocorder, allocsize; + + allocsize = PAGE_SIZE; + allocorder = 0; + while (bytebufsize > allocsize) { + allocsize *= 2; + allocorder++; + } + + s->salami = (void *) devm_get_free_pages( + dev, + GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO, + allocorder); + if (!s->salami) + return -ENOMEM; + + s->left_of_salami = allocsize; + } + + rc = ep->ephw->map_single(ep, s->salami, + bytebufsize, s->direction, + &dma_addr); + if (rc) + return rc; + + iowrite32((u32) (dma_addr & 0xffffffff), + ep->registers + fpga_dma_bufaddr_lowaddr_reg); + iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)), + ep->registers + fpga_dma_bufaddr_highaddr_reg); + + if (buffers) { /* Not the message buffer */ + this_buffer->addr = s->salami; + this_buffer->dma_addr = dma_addr; + buffers[i] = this_buffer++; + + iowrite32(s->regdirection | s->nbuffer++, + ep->registers + fpga_dma_bufno_reg); + } else { + ep->msgbuf_addr = s->salami; + ep->msgbuf_dma_addr = dma_addr; + ep->msg_buf_size = bytebufsize; + + iowrite32(s->regdirection, + ep->registers + fpga_dma_bufno_reg); + } + + s->left_of_salami -= bytebufsize; + s->salami += bytebufsize; + } + return 0; +} + +static int xilly_setupchannels(struct xilly_endpoint *ep, + unsigned char *chandesc, + int entries) +{ + struct device *dev = ep->dev; + int i, entry, rc; + struct xilly_channel *channel; + int channelnum, bufnum, bufsize, format, is_writebuf; + int bytebufsize; + int synchronous, allowpartial, exclusive_open, seekable; + int supports_nonempty; + int msg_buf_done = 0; + + struct xilly_alloc_state rd_alloc = { + .salami = NULL, + .left_of_salami = 0, + .nbuffer = 1, + .direction = DMA_TO_DEVICE, + .regdirection = 0, + }; + + struct xilly_alloc_state wr_alloc = { + .salami = NULL, + .left_of_salami = 0, + .nbuffer = 1, + .direction = DMA_FROM_DEVICE, + .regdirection = 0x80000000, + }; + + channel = devm_kcalloc(dev, ep->num_channels, + sizeof(struct xilly_channel), GFP_KERNEL); + if (!channel) + return -ENOMEM; + + ep->channels = devm_kcalloc(dev, ep->num_channels + 1, + sizeof(struct xilly_channel *), + GFP_KERNEL); + if (!ep->channels) + return -ENOMEM; + + ep->channels[0] = NULL; /* Channel 0 is message buf. */ + + /* Initialize all channels with defaults */ + + for (i = 1; i <= ep->num_channels; i++) { + channel->wr_buffers = NULL; + channel->rd_buffers = NULL; + channel->num_wr_buffers = 0; + channel->num_rd_buffers = 0; + channel->wr_fpga_buf_idx = -1; + channel->wr_host_buf_idx = 0; + channel->wr_host_buf_pos = 0; + channel->wr_empty = 1; + channel->wr_ready = 0; + channel->wr_sleepy = 1; + channel->rd_fpga_buf_idx = 0; + channel->rd_host_buf_idx = 0; + channel->rd_host_buf_pos = 0; + channel->rd_full = 0; + channel->wr_ref_count = 0; + channel->rd_ref_count = 0; + + spin_lock_init(&channel->wr_spinlock); + spin_lock_init(&channel->rd_spinlock); + mutex_init(&channel->wr_mutex); + mutex_init(&channel->rd_mutex); + init_waitqueue_head(&channel->rd_wait); + init_waitqueue_head(&channel->wr_wait); + init_waitqueue_head(&channel->wr_ready_wait); + + INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush); + + channel->endpoint = ep; + channel->chan_num = i; + + channel->log2_element_size = 0; + + ep->channels[i] = channel++; + } + + for (entry = 0; entry < entries; entry++, chandesc += 4) { + struct xilly_buffer **buffers = NULL; + + is_writebuf = chandesc[0] & 0x01; + channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7); + format = (chandesc[1] >> 4) & 0x03; + allowpartial = (chandesc[1] >> 6) & 0x01; + synchronous = (chandesc[1] >> 7) & 0x01; + bufsize = 1 << (chandesc[2] & 0x1f); + bufnum = 1 << (chandesc[3] & 0x0f); + exclusive_open = (chandesc[2] >> 7) & 0x01; + seekable = (chandesc[2] >> 6) & 0x01; + supports_nonempty = (chandesc[2] >> 5) & 0x01; + + if ((channelnum > ep->num_channels) || + ((channelnum == 0) && !is_writebuf)) { + dev_err(ep->dev, + "IDT requests channel out of range. Aborting.\n"); + return -ENODEV; + } + + channel = ep->channels[channelnum]; /* NULL for msg channel */ + + if (!is_writebuf || channelnum > 0) { + channel->log2_element_size = ((format > 2) ? + 2 : format); + + bytebufsize = channel->rd_buf_size = bufsize * + (1 << channel->log2_element_size); + + buffers = devm_kcalloc(dev, bufnum, + sizeof(struct xilly_buffer *), + GFP_KERNEL); + if (!buffers) + return -ENOMEM; + } else { + bytebufsize = bufsize << 2; + } + + if (!is_writebuf) { + channel->num_rd_buffers = bufnum; + channel->rd_allow_partial = allowpartial; + channel->rd_synchronous = synchronous; + channel->rd_exclusive_open = exclusive_open; + channel->seekable = seekable; + + channel->rd_buffers = buffers; + rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers, + bufnum, bytebufsize); + } else if (channelnum > 0) { + channel->num_wr_buffers = bufnum; + + channel->seekable = seekable; + channel->wr_supports_nonempty = supports_nonempty; + + channel->wr_allow_partial = allowpartial; + channel->wr_synchronous = synchronous; + channel->wr_exclusive_open = exclusive_open; + + channel->wr_buffers = buffers; + rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers, + bufnum, bytebufsize); + } else { + rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL, + bufnum, bytebufsize); + msg_buf_done++; + } + + if (rc) + return -ENOMEM; + } + + if (!msg_buf_done) { + dev_err(ep->dev, + "Corrupt IDT: No message buffer. Aborting.\n"); + return -ENODEV; + } + return 0; +} + +static int xilly_scan_idt(struct xilly_endpoint *endpoint, + struct xilly_idt_handle *idt_handle) +{ + int count = 0; + unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr; + unsigned char *end_of_idt = idt + endpoint->idtlen - 4; + unsigned char *scan; + int len; + + scan = idt; + idt_handle->idt = idt; + + scan++; /* Skip version number */ + + while ((scan <= end_of_idt) && *scan) { + while ((scan <= end_of_idt) && *scan++) + /* Do nothing, just scan thru string */; + count++; + } + + scan++; + + if (scan > end_of_idt) { + dev_err(endpoint->dev, + "IDT device name list overflow. Aborting.\n"); + return -ENODEV; + } + idt_handle->chandesc = scan; + + len = endpoint->idtlen - (3 + ((int) (scan - idt))); + + if (len & 0x03) { + dev_err(endpoint->dev, + "Corrupt IDT device name list. Aborting.\n"); + return -ENODEV; + } + + idt_handle->entries = len >> 2; + endpoint->num_channels = count; + + return 0; +} + +static int xilly_obtain_idt(struct xilly_endpoint *endpoint) +{ + struct xilly_channel *channel; + unsigned char *version; + long t; + + channel = endpoint->channels[1]; /* This should be generated ad-hoc */ + + channel->wr_sleepy = 1; + + iowrite32(1 | + (3 << 24), /* Opcode 3 for channel 0 = Send IDT */ + endpoint->registers + fpga_buf_ctrl_reg); + + t = wait_event_interruptible_timeout(channel->wr_wait, + (!channel->wr_sleepy), + XILLY_TIMEOUT); + + if (t <= 0) { + dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n"); + + if (endpoint->fatal_error) + return -EIO; + + return -ENODEV; + } + + endpoint->ephw->hw_sync_sgl_for_cpu( + channel->endpoint, + channel->wr_buffers[0]->dma_addr, + channel->wr_buf_size, + DMA_FROM_DEVICE); + + if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) { + dev_err(endpoint->dev, + "IDT length mismatch (%d != %d). Aborting.\n", + channel->wr_buffers[0]->end_offset, endpoint->idtlen); + return -ENODEV; + } + + if (crc32_le(~0, channel->wr_buffers[0]->addr, + endpoint->idtlen+1) != 0) { + dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n"); + return -ENODEV; + } + + version = channel->wr_buffers[0]->addr; + + /* Check version number. Accept anything below 0x82 for now. */ + if (*version > 0x82) { + dev_err(endpoint->dev, + "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgarde. Aborting.\n", + *version); + return -ENODEV; + } + + return 0; +} + +static ssize_t xillybus_read(struct file *filp, char __user *userbuf, + size_t count, loff_t *f_pos) +{ + ssize_t rc; + unsigned long flags; + int bytes_done = 0; + int no_time_left = 0; + long deadline, left_to_sleep; + struct xilly_channel *channel = filp->private_data; + + int empty, reached_eof, exhausted, ready; + /* Initializations are there only to silence warnings */ + + int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0; + int waiting_bufidx; + + if (channel->endpoint->fatal_error) + return -EIO; + + deadline = jiffies + 1 + XILLY_RX_TIMEOUT; + + rc = mutex_lock_interruptible(&channel->wr_mutex); + if (rc) + return rc; + + while (1) { /* Note that we may drop mutex within this loop */ + int bytes_to_do = count - bytes_done; + + spin_lock_irqsave(&channel->wr_spinlock, flags); + + empty = channel->wr_empty; + ready = !empty || channel->wr_ready; + + if (!empty) { + bufidx = channel->wr_host_buf_idx; + bufpos = channel->wr_host_buf_pos; + howmany = ((channel->wr_buffers[bufidx]->end_offset + + 1) << channel->log2_element_size) + - bufpos; + + /* Update wr_host_* to its post-operation state */ + if (howmany > bytes_to_do) { + bufferdone = 0; + + howmany = bytes_to_do; + channel->wr_host_buf_pos += howmany; + } else { + bufferdone = 1; + + channel->wr_host_buf_pos = 0; + + if (bufidx == channel->wr_fpga_buf_idx) { + channel->wr_empty = 1; + channel->wr_sleepy = 1; + channel->wr_ready = 0; + } + + if (bufidx >= (channel->num_wr_buffers - 1)) + channel->wr_host_buf_idx = 0; + else + channel->wr_host_buf_idx++; + } + } + + /* + * Marking our situation after the possible changes above, + * for use after releasing the spinlock. + * + * empty = empty before change + * exhasted = empty after possible change + */ + + reached_eof = channel->wr_empty && + (channel->wr_host_buf_idx == channel->wr_eof); + channel->wr_hangup = reached_eof; + exhausted = channel->wr_empty; + waiting_bufidx = channel->wr_host_buf_idx; + + spin_unlock_irqrestore(&channel->wr_spinlock, flags); + + if (!empty) { /* Go on, now without the spinlock */ + + if (bufpos == 0) /* Position zero means it's virgin */ + channel->endpoint->ephw->hw_sync_sgl_for_cpu( + channel->endpoint, + channel->wr_buffers[bufidx]->dma_addr, + channel->wr_buf_size, + DMA_FROM_DEVICE); + + if (copy_to_user( + userbuf, + channel->wr_buffers[bufidx]->addr + + bufpos, howmany)) + rc = -EFAULT; + + userbuf += howmany; + bytes_done += howmany; + + if (bufferdone) { + channel->endpoint->ephw->hw_sync_sgl_for_device( + channel->endpoint, + channel->wr_buffers[bufidx]->dma_addr, + channel->wr_buf_size, + DMA_FROM_DEVICE); + + /* + * Tell FPGA the buffer is done with. It's an + * atomic operation to the FPGA, so what + * happens with other channels doesn't matter, + * and the certain channel is protected with + * the channel-specific mutex. + */ + + iowrite32(1 | (channel->chan_num << 1) | + (bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + + if (rc) { + mutex_unlock(&channel->wr_mutex); + return rc; + } + } + + /* This includes a zero-count return = EOF */ + if ((bytes_done >= count) || reached_eof) + break; + + if (!exhausted) + continue; /* More in RAM buffer(s)? Just go on. */ + + if ((bytes_done > 0) && + (no_time_left || + (channel->wr_synchronous && channel->wr_allow_partial))) + break; + + /* + * Nonblocking read: The "ready" flag tells us that the FPGA + * has data to send. In non-blocking mode, if it isn't on, + * just return. But if there is, we jump directly to the point + * where we ask for the FPGA to send all it has, and wait + * until that data arrives. So in a sense, we *do* block in + * nonblocking mode, but only for a very short time. + */ + + if (!no_time_left && (filp->f_flags & O_NONBLOCK)) { + if (bytes_done > 0) + break; + + if (ready) + goto desperate; + + rc = -EAGAIN; + break; + } + + if (!no_time_left || (bytes_done > 0)) { + /* + * Note that in case of an element-misaligned read + * request, offsetlimit will include the last element, + * which will be partially read from. + */ + int offsetlimit = ((count - bytes_done) - 1) >> + channel->log2_element_size; + int buf_elements = channel->wr_buf_size >> + channel->log2_element_size; + + /* + * In synchronous mode, always send an offset limit. + * Just don't send a value too big. + */ + + if (channel->wr_synchronous) { + /* Don't request more than one buffer */ + if (channel->wr_allow_partial && + (offsetlimit >= buf_elements)) + offsetlimit = buf_elements - 1; + + /* Don't request more than all buffers */ + if (!channel->wr_allow_partial && + (offsetlimit >= + (buf_elements * channel->num_wr_buffers))) + offsetlimit = buf_elements * + channel->num_wr_buffers - 1; + } + + /* + * In asynchronous mode, force early flush of a buffer + * only if that will allow returning a full count. The + * "offsetlimit < ( ... )" rather than "<=" excludes + * requesting a full buffer, which would obviously + * cause a buffer transmission anyhow + */ + + if (channel->wr_synchronous || + (offsetlimit < (buf_elements - 1))) { + mutex_lock(&channel->endpoint->register_mutex); + + iowrite32(offsetlimit, + channel->endpoint->registers + + fpga_buf_offset_reg); + + iowrite32(1 | (channel->chan_num << 1) | + (2 << 24) | /* 2 = offset limit */ + (waiting_bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + + mutex_unlock(&channel->endpoint-> + register_mutex); + } + } + + /* + * If partial completion is disallowed, there is no point in + * timeout sleeping. Neither if no_time_left is set and + * there's no data. + */ + + if (!channel->wr_allow_partial || + (no_time_left && (bytes_done == 0))) { + /* + * This do-loop will run more than once if another + * thread reasserted wr_sleepy before we got the mutex + * back, so we try again. + */ + + do { + mutex_unlock(&channel->wr_mutex); + + if (wait_event_interruptible( + channel->wr_wait, + (!channel->wr_sleepy))) + goto interrupted; + + if (mutex_lock_interruptible( + &channel->wr_mutex)) + goto interrupted; + } while (channel->wr_sleepy); + + continue; + +interrupted: /* Mutex is not held if got here */ + if (channel->endpoint->fatal_error) + return -EIO; + if (bytes_done) + return bytes_done; + if (filp->f_flags & O_NONBLOCK) + return -EAGAIN; /* Don't admit snoozing */ + return -EINTR; + } + + left_to_sleep = deadline - ((long) jiffies); + + /* + * If our time is out, skip the waiting. We may miss wr_sleepy + * being deasserted but hey, almost missing the train is like + * missing it. + */ + + if (left_to_sleep > 0) { + left_to_sleep = + wait_event_interruptible_timeout( + channel->wr_wait, + (!channel->wr_sleepy), + left_to_sleep); + + if (left_to_sleep > 0) /* wr_sleepy deasserted */ + continue; + + if (left_to_sleep < 0) { /* Interrupt */ + mutex_unlock(&channel->wr_mutex); + if (channel->endpoint->fatal_error) + return -EIO; + if (bytes_done) + return bytes_done; + return -EINTR; + } + } + +desperate: + no_time_left = 1; /* We're out of sleeping time. Desperate! */ + + if (bytes_done == 0) { + /* + * Reaching here means that we allow partial return, + * that we've run out of time, and that we have + * nothing to return. + * So tell the FPGA to send anything it has or gets. + */ + + iowrite32(1 | (channel->chan_num << 1) | + (3 << 24) | /* Opcode 3, flush it all! */ + (waiting_bufidx << 12), + channel->endpoint->registers + + fpga_buf_ctrl_reg); + } + + /* + * Reaching here means that we *do* have data in the buffer, + * but the "partial" flag disallows returning less than + * required. And we don't have as much. So loop again, + * which is likely to end up blocking indefinitely until + * enough data has arrived. + */ + } + + mutex_unlock(&channel->wr_mutex); + + if (channel->endpoint->fatal_error) + return -EIO; + + if (rc) + return rc; + + return bytes_done; +} + +/* + * The timeout argument takes values as follows: + * >0 : Flush with timeout + * ==0 : Flush, and wait idefinitely for the flush to complete + * <0 : Autoflush: Flush only if there's a single buffer occupied + */ + +static int xillybus_myflush(struct xilly_channel *channel, long timeout) +{ + int rc; + unsigned long flags; + + int end_offset_plus1; + int bufidx, bufidx_minus1; + int i; + int empty; + int new_rd_host_buf_pos; + + if (channel->endpoint->fatal_error) + return -EIO; + rc = mutex_lock_interruptible(&channel->rd_mutex); + if (rc) + return rc; + + /* + * Don't flush a closed channel. This can happen when the work queued + * autoflush thread fires off after the file has closed. This is not + * an error, just something to dismiss. + */ + + if (!channel->rd_ref_count) + goto done; + + bufidx = channel->rd_host_buf_idx; + + bufidx_minus1 = (bufidx == 0) ? + channel->num_rd_buffers - 1 : + bufidx - 1; + + end_offset_plus1 = channel->rd_host_buf_pos >> + channel->log2_element_size; + + new_rd_host_buf_pos = channel->rd_host_buf_pos - + (end_offset_plus1 << channel->log2_element_size); + + /* Submit the current buffer if it's nonempty */ + if (end_offset_plus1) { + unsigned char *tail = channel->rd_buffers[bufidx]->addr + + (end_offset_plus1 << channel->log2_element_size); + + /* Copy unflushed data, so we can put it in next buffer */ + for (i = 0; i < new_rd_host_buf_pos; i++) + channel->rd_leftovers[i] = *tail++; + + spin_lock_irqsave(&channel->rd_spinlock, flags); + + /* Autoflush only if a single buffer is occupied */ + + if ((timeout < 0) && + (channel->rd_full || + (bufidx_minus1 != channel->rd_fpga_buf_idx))) { + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + /* + * A new work item may be queued by the ISR exactly + * now, since the execution of a work item allows the + * queuing of a new one while it's running. + */ + goto done; + } + + /* The 4th element is never needed for data, so it's a flag */ + channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0); + + /* Set up rd_full to reflect a certain moment's state */ + + if (bufidx == channel->rd_fpga_buf_idx) + channel->rd_full = 1; + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + + if (bufidx >= (channel->num_rd_buffers - 1)) + channel->rd_host_buf_idx = 0; + else + channel->rd_host_buf_idx++; + + channel->endpoint->ephw->hw_sync_sgl_for_device( + channel->endpoint, + channel->rd_buffers[bufidx]->dma_addr, + channel->rd_buf_size, + DMA_TO_DEVICE); + + mutex_lock(&channel->endpoint->register_mutex); + + iowrite32(end_offset_plus1 - 1, + channel->endpoint->registers + fpga_buf_offset_reg); + + iowrite32((channel->chan_num << 1) | /* Channel ID */ + (2 << 24) | /* Opcode 2, submit buffer */ + (bufidx << 12), + channel->endpoint->registers + fpga_buf_ctrl_reg); + + mutex_unlock(&channel->endpoint->register_mutex); + } else if (bufidx == 0) { + bufidx = channel->num_rd_buffers - 1; + } else { + bufidx--; + } + + channel->rd_host_buf_pos = new_rd_host_buf_pos; + + if (timeout < 0) + goto done; /* Autoflush */ + + /* + * bufidx is now the last buffer written to (or equal to + * rd_fpga_buf_idx if buffer was never written to), and + * channel->rd_host_buf_idx the one after it. + * + * If bufidx == channel->rd_fpga_buf_idx we're either empty or full. + */ + + while (1) { /* Loop waiting for draining of buffers */ + spin_lock_irqsave(&channel->rd_spinlock, flags); + + if (bufidx != channel->rd_fpga_buf_idx) + channel->rd_full = 1; /* + * Not really full, + * but needs waiting. + */ + + empty = !channel->rd_full; + + spin_unlock_irqrestore(&channel->rd_spinlock, flags); + + if (empty) + break; + + /* + * Indefinite sleep with mutex taken. With data waiting for + * flushing user should not be surprised if open() for write + * sleeps. + */ + if (timeout == 0) + wait_event_interruptible(channel->rd_wait, + (!channel->rd_full)); + + else if (wait_event_interruptible_timeout( + channel->rd_wait, + (!channel->rd_full), + timeout) == 0) { + dev_warn(channel->endpoint->dev, + "Timed out while flushing. Output data may be lost.\n"); + + rc = -ETIMEDOUT; + break; + } + + if (cha |