From e03a72e13648ac6277bf2bab6b8324d51f89c0fa Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Mon, 11 Jan 2010 03:21:51 -0500 Subject: block: Stop using byte offsets All callers of the stacking functions use 512-byte sector units rather than byte offsets. Simplify the code so the stacking functions take sectors when specifying data offsets. Signed-off-by: Martin K. Petersen Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 5c8018977efa..ffb13ad35716 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1112,18 +1112,13 @@ static inline int queue_alignment_offset(struct request_queue *q) return q->limits.alignment_offset; } -static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t offset) +static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector) { unsigned int granularity = max(lim->physical_block_size, lim->io_min); + unsigned int alignment = (sector << 9) & (granularity - 1); - offset &= granularity - 1; - return (granularity + lim->alignment_offset - offset) & (granularity - 1); -} - -static inline int queue_sector_alignment_offset(struct request_queue *q, - sector_t sector) -{ - return queue_limit_alignment_offset(&q->limits, sector << 9); + return (granularity + lim->alignment_offset - alignment) + & (granularity - 1); } static inline int bdev_alignment_offset(struct block_device *bdev) @@ -1147,10 +1142,8 @@ static inline int queue_discard_alignment(struct request_queue *q) return q->limits.discard_alignment; } -static inline int queue_sector_discard_alignment(struct request_queue *q, - sector_t sector) +static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector) { - struct queue_limits *lim = &q->limits; unsigned int alignment = (sector << 9) & (lim->discard_granularity - 1); return (lim->discard_granularity + lim->discard_alignment - alignment) -- cgit v1.2.3 From 488991e28e55b4fbca8067edf0259f69d1a6f92c Mon Sep 17 00:00:00 2001 From: "Alan D. Brunelle" Date: Fri, 29 Jan 2010 09:04:08 +0100 Subject: block: Added in stricter no merge semantics for block I/O Updated 'nomerges' tunable to accept a value of '2' - indicating that _no_ merges at all are to be attempted (not even the simple one-hit cache). The following table illustrates the additional benefit - 5 minute runs of a random I/O load were applied to a dozen devices on a 16-way x86_64 system. nomerges Throughput %System Improvement (tput / %sys) -------- ------------ ----------- ------------------------- 0 12.45 MB/sec 0.669365609 1 12.50 MB/sec 0.641519199 0.40% / 2.71% 2 12.52 MB/sec 0.639849750 0.56% / 2.96% Signed-off-by: Alan D. Brunelle Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index ffb13ad35716..f71f5c58620c 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -463,6 +463,7 @@ struct request_queue #define QUEUE_FLAG_IO_STAT 15 /* do IO stats */ #define QUEUE_FLAG_CQ 16 /* hardware does queuing */ #define QUEUE_FLAG_DISCARD 17 /* supports DISCARD */ +#define QUEUE_FLAG_NOXMERGES 18 /* No extended merges */ #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ (1 << QUEUE_FLAG_CLUSTER) | \ @@ -589,6 +590,8 @@ enum { #define blk_queue_queuing(q) test_bit(QUEUE_FLAG_CQ, &(q)->queue_flags) #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags) #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) +#define blk_queue_noxmerges(q) \ + test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags) #define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags) #define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags) #define blk_queue_flushing(q) ((q)->ordseq) -- cgit v1.2.3 From 79da0644a8e0838522828f106e4049639eea6baf Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 23 Feb 2010 08:40:43 +0100 Subject: Revert "block: improve queue_should_plug() by looking at IO depths" This reverts commit fb1e75389bd06fd5987e9cda1b4e0305c782f854. "Benjamin S." reports that the patch in question causes a big drop in sequential throughput for him, dropping from 200MB/sec down to only 70MB/sec. Needs to be investigated more fully, for now lets just revert the offending commit. Conflicts: include/linux/blkdev.h Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 5c8018977efa..1896e868854f 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -461,8 +461,7 @@ struct request_queue #define QUEUE_FLAG_NONROT 14 /* non-rotational device (SSD) */ #define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */ #define QUEUE_FLAG_IO_STAT 15 /* do IO stats */ -#define QUEUE_FLAG_CQ 16 /* hardware does queuing */ -#define QUEUE_FLAG_DISCARD 17 /* supports DISCARD */ +#define QUEUE_FLAG_DISCARD 16 /* supports DISCARD */ #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ (1 << QUEUE_FLAG_CLUSTER) | \ @@ -586,7 +585,6 @@ enum { #define blk_queue_plugged(q) test_bit(QUEUE_FLAG_PLUGGED, &(q)->queue_flags) #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags) -#define blk_queue_queuing(q) test_bit(QUEUE_FLAG_CQ, &(q)->queue_flags) #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags) #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) #define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags) -- cgit v1.2.3 From e751e76a5f7adeee7438e68b0965559ad2864d0d Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Feb 2010 00:20:36 -0500 Subject: block: Remove unused accessor function blk_queue_max_hw_sectors is no longer called by any subsystem and can be removed. Signed-off-by: Martin K. Petersen Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 2f17793048e7..c051cea2df07 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -922,7 +922,6 @@ extern void blk_cleanup_queue(struct request_queue *); extern void blk_queue_make_request(struct request_queue *, make_request_fn *); extern void blk_queue_bounce_limit(struct request_queue *, u64); extern void blk_queue_max_sectors(struct request_queue *, unsigned int); -extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int); extern void blk_queue_max_phys_segments(struct request_queue *, unsigned short); extern void blk_queue_max_hw_segments(struct request_queue *, unsigned short); extern void blk_queue_max_segment_size(struct request_queue *, unsigned int); -- cgit v1.2.3 From eb28d31bc97e6374d81f404da309401ffaed467b Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Feb 2010 00:20:37 -0500 Subject: block: Add BLK_ prefix to definitions Add a BLK_ prefix to block layer constants. Signed-off-by: Martin K. Petersen Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index c051cea2df07..5d378627f446 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1016,11 +1016,15 @@ extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm); #define MAX_PHYS_SEGMENTS 128 #define MAX_HW_SEGMENTS 128 #define SAFE_MAX_SECTORS 255 -#define BLK_DEF_MAX_SECTORS 1024 - #define MAX_SEGMENT_SIZE 65536 -#define BLK_SEG_BOUNDARY_MASK 0xFFFFFFFFUL +enum blk_default_limits { + BLK_MAX_SEGMENTS = 128, + BLK_SAFE_MAX_SECTORS = 255, + BLK_DEF_MAX_SECTORS = 1024, + BLK_MAX_SEGMENT_SIZE = 65536, + BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL, +}; #define blkdev_entry_to_request(entry) list_entry((entry), struct request, queuelist) -- cgit v1.2.3 From 086fa5ff0854c676ec333760f4c0154b3b242616 Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Feb 2010 00:20:38 -0500 Subject: block: Rename blk_queue_max_sectors to blk_queue_max_hw_sectors The block layer calling convention is blk_queue_. blk_queue_max_sectors predates this practice, leading to some confusion. Rename the function to appropriately reflect that its intended use is to set max_hw_sectors. Also introduce a temporary wrapper for backwards compability. This can be removed after the merge window is closed. Signed-off-by: Martin K. Petersen Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 5d378627f446..57bc48446e92 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -921,7 +921,14 @@ extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *); extern void blk_cleanup_queue(struct request_queue *); extern void blk_queue_make_request(struct request_queue *, make_request_fn *); extern void blk_queue_bounce_limit(struct request_queue *, u64); -extern void blk_queue_max_sectors(struct request_queue *, unsigned int); +extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int); + +/* Temporary compatibility wrapper */ +static inline void blk_queue_max_sectors(struct request_queue *q, unsigned int max) +{ + blk_queue_max_hw_sectors(q, max); +} + extern void blk_queue_max_phys_segments(struct request_queue *, unsigned short); extern void blk_queue_max_hw_segments(struct request_queue *, unsigned short); extern void blk_queue_max_segment_size(struct request_queue *, unsigned int); -- cgit v1.2.3 From 8a78362c4eefc1deddbefe2c7f38aabbc2429d6b Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Fri, 26 Feb 2010 00:20:39 -0500 Subject: block: Consolidate phys_segment and hw_segment limits Except for SCSI no device drivers distinguish between physical and hardware segment limits. Consolidate the two into a single segment limit. Signed-off-by: Martin K. Petersen Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 57bc48446e92..ebd22dbed861 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -316,8 +316,7 @@ struct queue_limits { unsigned int discard_alignment; unsigned short logical_block_size; - unsigned short max_hw_segments; - unsigned short max_phys_segments; + unsigned short max_segments; unsigned char misaligned; unsigned char discard_misaligned; @@ -929,8 +928,19 @@ static inline void blk_queue_max_sectors(struct request_queue *q, unsigned int m blk_queue_max_hw_sectors(q, max); } -extern void blk_queue_max_phys_segments(struct request_queue *, unsigned short); -extern void blk_queue_max_hw_segments(struct request_queue *, unsigned short); +extern void blk_queue_max_segments(struct request_queue *, unsigned short); + +static inline void blk_queue_max_phys_segments(struct request_queue *q, unsigned short max) +{ + blk_queue_max_segments(q, max); +} + +static inline void blk_queue_max_hw_segments(struct request_queue *q, unsigned short max) +{ + blk_queue_max_segments(q, max); +} + + extern void blk_queue_max_segment_size(struct request_queue *, unsigned int); extern void blk_queue_max_discard_sectors(struct request_queue *q, unsigned int max_discard_sectors); @@ -1055,14 +1065,9 @@ static inline unsigned int queue_max_hw_sectors(struct request_queue *q) return q->limits.max_hw_sectors; } -static inline unsigned short queue_max_hw_segments(struct request_queue *q) -{ - return q->limits.max_hw_segments; -} - -static inline unsigned short queue_max_phys_segments(struct request_queue *q) +static inline unsigned short queue_max_segments(struct request_queue *q) { - return q->limits.max_phys_segments; + return q->limits.max_segments; } static inline unsigned int queue_max_segment_size(struct request_queue *q) -- cgit v1.2.3 From ee714f2dd33e726346e34f5cda12543162f4753e Mon Sep 17 00:00:00 2001 From: "Martin K. Petersen" Date: Wed, 10 Mar 2010 00:48:32 -0500 Subject: block: Finalize conversion of block limits functions Remove compatibility wrappers and update remaining drivers. Signed-off-by: Martin K. Petersen Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 24 ------------------------ 1 file changed, 24 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index ebd22dbed861..41551c9341b6 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -921,26 +921,7 @@ extern void blk_cleanup_queue(struct request_queue *); extern void blk_queue_make_request(struct request_queue *, make_request_fn *); extern void blk_queue_bounce_limit(struct request_queue *, u64); extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int); - -/* Temporary compatibility wrapper */ -static inline void blk_queue_max_sectors(struct request_queue *q, unsigned int max) -{ - blk_queue_max_hw_sectors(q, max); -} - extern void blk_queue_max_segments(struct request_queue *, unsigned short); - -static inline void blk_queue_max_phys_segments(struct request_queue *q, unsigned short max) -{ - blk_queue_max_segments(q, max); -} - -static inline void blk_queue_max_hw_segments(struct request_queue *q, unsigned short max) -{ - blk_queue_max_segments(q, max); -} - - extern void blk_queue_max_segment_size(struct request_queue *, unsigned int); extern void blk_queue_max_discard_sectors(struct request_queue *q, unsigned int max_discard_sectors); @@ -1030,11 +1011,6 @@ static inline int sb_issue_discard(struct super_block *sb, extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm); -#define MAX_PHYS_SEGMENTS 128 -#define MAX_HW_SEGMENTS 128 -#define SAFE_MAX_SECTORS 255 -#define MAX_SEGMENT_SIZE 65536 - enum blk_default_limits { BLK_MAX_SEGMENTS = 128, BLK_SAFE_MAX_SECTORS = 255, -- cgit v1.2.3 From 181fdde3b4268cb7b4af76ba6337e7ec8accbb36 Mon Sep 17 00:00:00 2001 From: Richard Kennedy Date: Fri, 19 Mar 2010 08:58:16 +0100 Subject: block: remove 16 bytes of padding from struct request on 64bits Remove alignment padding to shrink struct request from 336 to 320 bytes so needing one fewer cacheline and therefore removing 48 bytes from struct request_queue. Signed-off-by: Richard Kennedy Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 41551c9341b6..6690e8bae7bb 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -158,7 +158,6 @@ enum rq_flag_bits { struct request { struct list_head queuelist; struct call_single_data csd; - int cpu; struct request_queue *q; @@ -166,9 +165,11 @@ struct request { enum rq_cmd_type_bits cmd_type; unsigned long atomic_flags; + int cpu; + /* the following two fields are internal, NEVER access directly */ - sector_t __sector; /* sector cursor */ unsigned int __data_len; /* total data len */ + sector_t __sector; /* sector cursor */ struct bio *bio; struct bio *biotail; @@ -201,20 +202,20 @@ struct request { unsigned short ioprio; + int ref_count; + void *special; /* opaque pointer available for LLD use */ char *buffer; /* kaddr of the current segment if available */ int tag; int errors; - int ref_count; - /* * when request is used as a packet command carrier */ - unsigned short cmd_len; unsigned char __cmd[BLK_MAX_CDB]; unsigned char *cmd; + unsigned short cmd_len; unsigned int extra_len; /* length of alignment and padding */ unsigned int sense_len; -- cgit v1.2.3 From 9195291e5f05e01d67f9a09c756b8aca8f009089 Mon Sep 17 00:00:00 2001 From: Divyesh Shah Date: Thu, 1 Apr 2010 15:01:41 -0700 Subject: blkio: Increment the blkio cgroup stats for real now We also add start_time_ns and io_start_time_ns fields to struct request here to record the time when a request is created and when it is dispatched to device. We use ns uints here as ms and jiffies are not very useful for non-rotational media. Signed-off-by: Divyesh Shah Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 6690e8bae7bb..f3fff8bf85ee 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -194,7 +194,10 @@ struct request { struct gendisk *rq_disk; unsigned long start_time; - +#ifdef CONFIG_BLK_CGROUP + unsigned long long start_time_ns; + unsigned long long io_start_time_ns; /* when passed to hardware */ +#endif /* Number of scatter-gather DMA addr+len pairs after * physical address coalescing is performed. */ @@ -1196,6 +1199,21 @@ static inline void put_dev_sector(Sector p) struct work_struct; int kblockd_schedule_work(struct request_queue *q, struct work_struct *work); +#ifdef CONFIG_BLK_CGROUP +static inline void set_start_time_ns(struct request *req) +{ + req->start_time_ns = sched_clock(); +} + +static inline void set_io_start_time_ns(struct request *req) +{ + req->io_start_time_ns = sched_clock(); +} +#else +static inline void set_start_time_ns(struct request *req) {} +static inline void set_io_start_time_ns(struct request *req) {} +#endif + #define MODULE_ALIAS_BLOCKDEV(major,minor) \ MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor)) #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \ -- cgit v1.2.3 From 84c124da9ff50bd71fab9c939ee5b7cd8bef2bd9 Mon Sep 17 00:00:00 2001 From: Divyesh Shah Date: Fri, 9 Apr 2010 08:31:19 +0200 Subject: blkio: Changes to IO controller additional stats patches that include some minor fixes and addresses all comments. Changelog: (most based on Vivek Goyal's comments) o renamed blkiocg_reset_write to blkiocg_reset_stats o more clarification in the documentation on io_service_time and io_wait_time o Initialize blkg->stats_lock o rename io_add_stat to blkio_add_stat and declare it static o use bool for direction and sync o derive direction and sync info from existing rq methods o use 12 for major:minor string length o define io_service_time better to cover the NCQ case o add a separate reset_stats interface o make the indexed stats a 2d array to simplify macro and function pointer code o blkio.time now exports in jiffies as before o Added stats description in patch description and Documentation/cgroup/blkio-controller.txt o Prefix all stats functions with blkio and make them static as applicable o replace IO_TYPE_MAX with IO_TYPE_TOTAL o Moved #define constant to top of blk-cgroup.c o Pass dev_t around instead of char * o Add note to documentation file about resetting stats o use BLK_CGROUP_MODULE in addition to BLK_CGROUP config option in #ifdef statements o Avoid struct request specific knowledge in blk-cgroup. blk-cgroup.h now has rq_direction() and rq_sync() functions which are used by CFQ and when using io-controller at a higher level, bio_* functions can be added. Signed-off-by: Divyesh Shah Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index f3fff8bf85ee..d483c494672a 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1209,9 +1209,27 @@ static inline void set_io_start_time_ns(struct request *req) { req->io_start_time_ns = sched_clock(); } + +static inline uint64_t rq_start_time_ns(struct request *req) +{ + return req->start_time_ns; +} + +static inline uint64_t rq_io_start_time_ns(struct request *req) +{ + return req->io_start_time_ns; +} #else static inline void set_start_time_ns(struct request *req) {} static inline void set_io_start_time_ns(struct request *req) {} +static inline uint64_t rq_start_time_ns(struct request *req) +{ + return 0; +} +static inline uint64_t rq_io_start_time_ns(struct request *req) +{ + return 0; +} #endif #define MODULE_ALIAS_BLOCKDEV(major,minor) \ -- cgit v1.2.3 From 7f1dc8a2d2f45fc557b27fd56115338b1d34fc24 Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Wed, 21 Apr 2010 17:44:16 +0200 Subject: blkio: Fix blkio crash during rq stat update blkio + cfq was crashing even when two sequential readers were put in two separate cgroups (group_isolation=0). The reason being that cfqq can migrate across groups based on its being sync-noidle or not, it can happen that at request insertion time, cfqq belonged to one cfqg and at request dispatch time, it belonged to root group. In this case request stats per cgroup can go wrong and it also runs into BUG_ON(). This patch implements rq stashing away a cfq group pointer and not relying on cfqq->cfqg pointer alone for rq stat accounting. [ 65.163523] ------------[ cut here ]------------ [ 65.164301] kernel BUG at block/blk-cgroup.c:117! [ 65.164301] invalid opcode: 0000 [#1] SMP [ 65.164301] last sysfs file: /sys/devices/pci0000:00/0000:00:05.0/0000:60:00.1/host9/rport-9:0-0/target9:0:0/9:0:0:2/block/sde/stat [ 65.164301] CPU 1 [ 65.164301] Modules linked in: dm_round_robin dm_multipath qla2xxx scsi_transport_fc dm_zero dm_mirror dm_region_hash dm_log dm_mod [last unloaded: scsi_wait_scan] [ 65.164301] [ 65.164301] Pid: 4505, comm: fio Not tainted 2.6.34-rc4-blk-for-35 #34 0A98h/HP xw8600 Workstation [ 65.164301] RIP: 0010:[] [] blkiocg_update_io_remove_stats+0x5b/0xaf [ 65.164301] RSP: 0018:ffff8800ba5a79e8 EFLAGS: 00010046 [ 65.164301] RAX: 0000000000000096 RBX: ffff8800bb268d60 RCX: 0000000000000000 [ 65.164301] RDX: ffff8800bb268eb8 RSI: 0000000000000000 RDI: ffff8800bb268e00 [ 65.164301] RBP: ffff8800ba5a7a08 R08: 0000000000000064 R09: 0000000000000001 [ 65.164301] R10: 0000000000079640 R11: ffff8800a0bd5bf0 R12: ffff8800bab4af01 [ 65.164301] R13: ffff8800bab4af00 R14: ffff8800bb1d8928 R15: 0000000000000000 [ 65.164301] FS: 00007f18f75056f0(0000) GS:ffff880001e40000(0000) knlGS:0000000000000000 [ 65.164301] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 65.164301] CR2: 000000000040e7f0 CR3: 00000000ba52b000 CR4: 00000000000006e0 [ 65.164301] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 65.164301] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [ 65.164301] Process fio (pid: 4505, threadinfo ffff8800ba5a6000, task ffff8800ba45ae80) [ 65.164301] Stack: [ 65.164301] ffff8800ba5a7a08 ffff8800ba722540 ffff8800bab4af68 ffff8800bab4af68 [ 65.164301] <0> ffff8800ba5a7a38 ffffffff8121d814 ffff8800ba722540 ffff8800bab4af68 [ 65.164301] <0> ffff8800ba722540 ffff8800a08f6800 ffff8800ba5a7a68 ffffffff8121d8ca [ 65.164301] Call Trace: [ 65.164301] [] cfq_remove_request+0xe4/0x116 [ 65.164301] [] cfq_dispatch_insert+0x84/0xe1 [ 65.164301] [] cfq_dispatch_requests+0x767/0x8e8 [ 65.164301] [] ? submit_bio+0xc3/0xcc [ 65.164301] [] ? sync_page_killable+0x0/0x35 [ 65.164301] [] blk_peek_request+0x191/0x1a7 [ 65.164301] [] ? dm_get_live_table+0x44/0x4f [dm_mod] [ 65.164301] [] dm_request_fn+0x38/0x14c [dm_mod] [ 65.164301] [] ? sync_page_killable+0x0/0x35 [ 65.164301] [] __generic_unplug_device+0x32/0x37 [ 65.164301] [] generic_unplug_device+0x2e/0x3c [ 65.164301] [] dm_unplug_all+0x42/0x5b [dm_mod] [ 65.164301] [] blk_unplug+0x29/0x2d [ 65.164301] [] blk_backing_dev_unplug+0x12/0x14 [ 65.164301] [] block_sync_page+0x35/0x39 [ 65.164301] [] sync_page+0x41/0x4a [ 65.164301] [] sync_page_killable+0xe/0x35 [ 65.164301] [] __wait_on_bit_lock+0x46/0x8f [ 65.164301] [] __lock_page_killable+0x66/0x6d [ 65.164301] [] ? wake_bit_function+0x0/0x33 [ 65.164301] [] lock_page_killable+0x2c/0x2e [ 65.164301] [] generic_file_aio_read+0x361/0x4f0 [ 65.164301] [] do_sync_read+0xcb/0x108 [ 65.164301] [] ? security_file_permission+0x16/0x18 [ 65.164301] [] vfs_read+0xab/0x108 [ 65.164301] [] sys_read+0x4a/0x6e [ 65.164301] [] system_call_fastpath+0x16/0x1b [ 65.164301] Code: 00 74 1c 48 8b 8b 60 01 00 00 48 85 c9 75 04 0f 0b eb fe 48 ff c9 48 89 8b 60 01 00 00 eb 1a 48 8b 8b 58 01 00 00 48 85 c9 75 04 <0f> 0b eb fe 48 ff c9 48 89 8b 58 01 00 00 45 84 e4 74 16 48 8b [ 65.164301] RIP [] blkiocg_update_io_remove_stats+0x5b/0xaf [ 65.164301] RSP [ 65.164301] ---[ end trace 1b2b828753032e68 ]--- Signed-off-by: Vivek Goyal Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index d483c494672a..5cf17a49ce38 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -186,11 +186,12 @@ struct request { }; /* - * two pointers are available for the IO schedulers, if they need + * Three pointers are available for the IO schedulers, if they need * more they have to dynamically allocate it. */ void *elevator_private; void *elevator_private2; + void *elevator_private3; struct gendisk *rq_disk; unsigned long start_time; -- cgit v1.2.3 From fbd9b09a177a481eda256447c881f014f29034fe Mon Sep 17 00:00:00 2001 From: Dmitry Monakhov Date: Wed, 28 Apr 2010 17:55:06 +0400 Subject: blkdev: generalize flags for blkdev_issue_fn functions The patch just convert all blkdev_issue_xxx function to common set of flags. Wait/allocation semantics preserved. Signed-off-by: Dmitry Monakhov Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 5cf17a49ce38..59b9aed0ee7d 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -998,12 +998,16 @@ static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt, return NULL; return bqt->tag_index[tag]; } - -extern int blkdev_issue_flush(struct block_device *, sector_t *); -#define DISCARD_FL_WAIT 0x01 /* wait for completion */ -#define DISCARD_FL_BARRIER 0x02 /* issue DISCARD_BARRIER request */ -extern int blkdev_issue_discard(struct block_device *, sector_t sector, - sector_t nr_sects, gfp_t, int flags); +enum{ + BLKDEV_WAIT, /* wait for completion */ + BLKDEV_BARRIER, /*issue request with barrier */ +}; +#define BLKDEV_IFL_WAIT (1 << BLKDEV_WAIT) +#define BLKDEV_IFL_BARRIER (1 << BLKDEV_BARRIER) +extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *, + unsigned long); +extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector, + sector_t nr_sects, gfp_t gfp_mask, unsigned long flags); static inline int sb_issue_discard(struct super_block *sb, sector_t block, sector_t nr_blocks) @@ -1011,7 +1015,7 @@ static inline int sb_issue_discard(struct super_block *sb, block <<= (sb->s_blocksize_bits - 9); nr_blocks <<= (sb->s_blocksize_bits - 9); return blkdev_issue_discard(sb->s_bdev, block, nr_blocks, GFP_KERNEL, - DISCARD_FL_BARRIER); + BLKDEV_IFL_WAIT | BLKDEV_IFL_BARRIER); } extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm); -- cgit v1.2.3 From 3f14d792f9a8fede64ce918dbb517f934497a4f8 Mon Sep 17 00:00:00 2001 From: Dmitry Monakhov Date: Wed, 28 Apr 2010 17:55:09 +0400 Subject: blkdev: add blkdev_issue_zeroout helper function - Add bio_batch helper primitive. This is rather generic primitive for submitting/waiting a complex request which consists of several bios. - blkdev_issue_zeroout() generate number of zero filed write bios. Signed-off-by: Dmitry Monakhov Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 59b9aed0ee7d..3ac2bd2fc485 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1008,7 +1008,8 @@ extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *, unsigned long); extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, unsigned long flags); - +extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, + sector_t nr_sects, gfp_t gfp_mask, unsigned long flags); static inline int sb_issue_discard(struct super_block *sb, sector_t block, sector_t nr_blocks) { -- cgit v1.2.3 From 01effb0dc1451fad55925873ffbfb88fa4eadce0 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 11 May 2010 08:57:42 +0200 Subject: block: allow initialization of previously allocated request_queue blk_init_queue() allocates the request_queue structure and then initializes it as needed (request_fn, elevator, etc). Split initialization out to blk_init_allocated_queue_node. Introduce blk_init_allocated_queue wrapper function to model existing blk_init_queue and blk_init_queue_node interfaces. Export elv_register_queue to allow a newly added elevator to be registered with sysfs. Export elv_unregister_queue for symmetry. These changes allow DM to initialize a device's request_queue with more precision. In particular, DM no longer unconditionally initializes a full request_queue (elevator et al). It only does so for a request-based DM device. Signed-off-by: Mike Snitzer Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 3ac2bd2fc485..346fd4856733 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -921,7 +921,12 @@ extern void blk_abort_queue(struct request_queue *); */ extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id); +extern struct request_queue *blk_init_allocated_queue_node(struct request_queue *, + request_fn_proc *, + spinlock_t *, int node_id); extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *); +extern struct request_queue *blk_init_allocated_queue(struct request_queue *, + request_fn_proc *, spinlock_t *); extern void blk_cleanup_queue(struct request_queue *); extern void blk_queue_make_request(struct request_queue *, make_request_fn *); extern void blk_queue_bounce_limit(struct request_queue *, u64); -- cgit v1.2.3 From b3a27d0529c6e5206f1b60f60263e3ecfd0d77cb Mon Sep 17 00:00:00 2001 From: Nitin Gupta Date: Mon, 17 May 2010 11:02:43 +0530 Subject: swap: Add swap slot free callback to block_device_operations This callback is required when RAM based devices are used as swap disks. One such device is ramzswap which is used as compressed in-memory swap disk. For such devices, we need a callback as soon as a swap slot is no longer used to allow freeing memory allocated for this slot. Without this callback, stale data can quickly accumulate in memory defeating the whole purpose of such devices. Signed-off-by: Nitin Gupta Acked-by: Linus Torvalds Acked-by: Nigel Cunningham Acked-by: Pekka Enberg Reviewed-by: Minchan Kim Signed-off-by: Greg Kroah-Hartman --- include/linux/blkdev.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 6690e8bae7bb..413284a51ccc 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1287,6 +1287,8 @@ struct block_device_operations { unsigned long long); int (*revalidate_disk) (struct gendisk *); int (*getgeo)(struct block_device *, struct hd_geometry *); + /* this callback is with swap_lock and sometimes page table lock held */ + void (*swap_slot_free_notify) (struct block_device *, unsigned long); struct module *owner; }; -- cgit v1.2.3 From c3e33e043f5e9c583aa59d5591a614b2a8243d3a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 15 May 2010 20:09:29 +0200 Subject: block,ide: simplify bdops->set_capacity() to ->unlock_native_capacity() bdops->set_capacity() is unnecessarily generic. All that's required is a simple one way notification to lower level driver telling it to try to unlock native capacity. There's no reason to pass in target capacity or return the new capacity. The former is always the inherent native capacity and the latter can be handled via the usual device resize / revalidation path. In fact, the current API is always used that way. Replace ->set_capacity() with ->unlock_native_capacity() which take only @disk and doesn't return anything. IDE which is the only current user of the API is converted accordingly. Signed-off-by: Tejun Heo Cc: Ben Hutchings Cc: Bartlomiej Zolnierkiewicz Acked-by: David S. Miller Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 346fd4856733..be411c12ebbe 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1330,8 +1330,7 @@ struct block_device_operations { int (*direct_access) (struct block_device *, sector_t, void **, unsigned long *); int (*media_changed) (struct gendisk *); - unsigned long long (*set_capacity) (struct gendisk *, - unsigned long long); + void (*unlock_native_capacity) (struct gendisk *); int (*revalidate_disk) (struct gendisk *); int (*getgeo)(struct block_device *, struct hd_geometry *); struct module *owner; -- cgit v1.2.3 From 28f4197e5d4707311febeec8a0eb97cb5fd93c97 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 1 Jun 2010 12:23:18 +0200 Subject: block: disable preemption before using sched_clock() Commit 9195291e5f05e01d67f9a09c756b8aca8f009089 added calls to sched_clock() from preemptible code. sched_clock() is both the wrong interface AND cannot be called without preempt disabled. Apply a temporary fix to get rid of the warnings, a real patch is in the works. Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 8b7f5e0914ad..09a840264d6f 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1211,14 +1211,23 @@ struct work_struct; int kblockd_schedule_work(struct request_queue *q, struct work_struct *work); #ifdef CONFIG_BLK_CGROUP +/* + * This should not be using sched_clock(). A real patch is in progress + * to fix this up, until that is in place we need to disable preemption + * around sched_clock() in this function and set_io_start_time_ns(). + */ static inline void set_start_time_ns(struct request *req) { + preempt_disable(); req->start_time_ns = sched_clock(); + preempt_enable(); } static inline void set_io_start_time_ns(struct request *req) { + preempt_disable(); req->io_start_time_ns = sched_clock(); + preempt_enable(); } static inline uint64_t rq_start_time_ns(struct request *req) -- cgit v1.2.3