if (rl->count[BLK_RW_SYNC] >= q->nr_requests) {
blk_set_queue_full(q, BLK_RW_SYNC);
- } else if (rl->count[BLK_RW_SYNC]+1 <= q->nr_requests) {
+ } else {
blk_clear_queue_full(q, BLK_RW_SYNC);
wake_up(&rl->wait[BLK_RW_SYNC]);
}
if (rl->count[BLK_RW_ASYNC] >= q->nr_requests) {
blk_set_queue_full(q, BLK_RW_ASYNC);
- } else if (rl->count[BLK_RW_ASYNC]+1 <= q->nr_requests) {
+ } else {
blk_clear_queue_full(q, BLK_RW_ASYNC);
wake_up(&rl->wait[BLK_RW_ASYNC]);
}
static ssize_t queue_discard_max_show(struct request_queue *q, char *page)
{
- return queue_var_show(q->limits.max_discard_sectors << 9, page);
+ return sprintf(page, "%llu\n",
+ (unsigned long long)q->limits.max_discard_sectors << 9);
}
static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page)
return ret;
ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue");
- if (ret < 0)
+ if (ret < 0) {
+ blk_trace_remove_sysfs(dev);
return ret;
+ }
kobject_uevent(&q->kobj, KOBJ_ADD);
void blk_delete_timer(struct request *);
void blk_add_timer(struct request *);
void __generic_unplug_device(struct request_queue *);
- void blk_run_queue_async(struct request_queue *q);
/*
* Internal atomic flags for request handling
rq = list_entry_rq(q->queue_head.next);
return rq;
}
-
+ /*
+ * Flush request is running and flush request isn't queueable
+ * in the drive, we can hold the queue till flush request is
+ * finished. Even we don't do this, driver can't dispatch next
+ * requests and will requeue them. And this can improve
+ * throughput too. For example, we have request flush1, write1,
+ * flush 2. flush1 is dispatched, then queue is hold, write1
+ * isn't inserted to queue. After flush1 is finished, flush2
+ * will be dispatched. Since disk cache is already clean,
+ * flush2 will be finished very soon, so looks like flush2 is
+ * folded to flush1.
+ * Since the queue is hold, a flag is set to indicate the queue
+ * should be restarted later. Please see flush_end_io() for
+ * details.
+ */
+ if (q->flush_pending_idx != q->flush_running_idx &&
+ !queue_flush_queueable(q)) {
+ q->flush_queue_delayed = 1;
+ return NULL;
+ }
if (!q->elevator->ops->elevator_dispatch_fn(q, 0))
return NULL;
}
unsigned char misaligned;
unsigned char discard_misaligned;
unsigned char cluster;
- signed char discard_zeroes_data;
+ unsigned char discard_zeroes_data;
};
struct request_queue
* for flush operations
*/
unsigned int flush_flags;
+ unsigned int flush_not_queueable:1;
+ unsigned int flush_queue_delayed:1;
unsigned int flush_pending_idx:1;
unsigned int flush_running_idx:1;
unsigned long flush_pending_since;
#define QUEUE_FLAG_SYNCFULL 3 /* read queue has been filled */
#define QUEUE_FLAG_ASYNCFULL 4 /* write queue has been filled */
#define QUEUE_FLAG_DEAD 5 /* queue being torn down */
- #define QUEUE_FLAG_REENTER 6 /* Re-entrancy avoidance */
- #define QUEUE_FLAG_ELVSWITCH 7 /* don't use elevator, just do FIFO */
- #define QUEUE_FLAG_BIDI 8 /* queue supports bidi requests */
- #define QUEUE_FLAG_NOMERGES 9 /* disable merge attempts */
- #define QUEUE_FLAG_SAME_COMP 10 /* force complete on same CPU */
- #define QUEUE_FLAG_FAIL_IO 11 /* fake timeout */
- #define QUEUE_FLAG_STACKABLE 12 /* supports request stacking */
- #define QUEUE_FLAG_NONROT 13 /* non-rotational device (SSD) */
+ #define QUEUE_FLAG_ELVSWITCH 6 /* don't use elevator, just do FIFO */
+ #define QUEUE_FLAG_BIDI 7 /* queue supports bidi requests */
+ #define QUEUE_FLAG_NOMERGES 8 /* disable merge attempts */
+ #define QUEUE_FLAG_SAME_COMP 9 /* force complete on same CPU */
+ #define QUEUE_FLAG_FAIL_IO 10 /* fake timeout */
+ #define QUEUE_FLAG_STACKABLE 11 /* supports request stacking */
+ #define QUEUE_FLAG_NONROT 12 /* non-rotational device (SSD) */
#define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */
- #define QUEUE_FLAG_IO_STAT 15 /* do IO stats */
- #define QUEUE_FLAG_DISCARD 16 /* supports DISCARD */
- #define QUEUE_FLAG_NOXMERGES 17 /* No extended merges */
- #define QUEUE_FLAG_ADD_RANDOM 18 /* Contributes to random pool */
- #define QUEUE_FLAG_SECDISCARD 19 /* supports SECDISCARD */
+ #define QUEUE_FLAG_IO_STAT 13 /* do IO stats */
+ #define QUEUE_FLAG_DISCARD 14 /* supports DISCARD */
+ #define QUEUE_FLAG_NOXMERGES 15 /* No extended merges */
+ #define QUEUE_FLAG_ADD_RANDOM 16 /* Contributes to random pool */
+ #define QUEUE_FLAG_SECDISCARD 17 /* supports SECDISCARD */
#define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
(1 << QUEUE_FLAG_STACKABLE) | \
extern void __blk_stop_queue(struct request_queue *q);
extern void __blk_run_queue(struct request_queue *q);
extern void blk_run_queue(struct request_queue *);
+ extern void blk_run_queue_async(struct request_queue *q);
extern int blk_rq_map_user(struct request_queue *, struct request *,
struct rq_map_data *, void __user *, unsigned long,
gfp_t);
extern void blk_queue_rq_timed_out(struct request_queue *, rq_timed_out_fn *);
extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
extern void blk_queue_flush(struct request_queue *q, unsigned int flush);
+extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev);
extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
{
unsigned int alignment = (sector << 9) & (lim->discard_granularity - 1);
+ if (!lim->max_discard_sectors)
+ return 0;
+
return (lim->discard_granularity + lim->discard_alignment - alignment)
& (lim->discard_granularity - 1);
}
static inline unsigned int queue_discard_zeroes_data(struct request_queue *q)
{
- if (q->limits.discard_zeroes_data == 1)
+ if (q->limits.max_discard_sectors && q->limits.discard_zeroes_data == 1)
return 1;
return 0;
return bdev->bd_block_size;
}
+static inline bool queue_flush_queueable(struct request_queue *q)
+{
+ return !q->flush_not_queueable;
+}
+
typedef struct {struct page *v;} Sector;
unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *);
projects / linux.git / commitdiff