diff options
Diffstat (limited to 'block/blk-merge.c')
| -rw-r--r-- | block/blk-merge.c | 107 |
1 files changed, 65 insertions, 42 deletions
diff --git a/block/blk-merge.c b/block/blk-merge.c index ad763ec313b6..e0b28e9298c9 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -107,17 +107,18 @@ static unsigned int bio_allowed_max_sectors(const struct queue_limits *lim) static struct bio *bio_submit_split(struct bio *bio, int split_sectors) { - if (unlikely(split_sectors < 0)) { - bio->bi_status = errno_to_blk_status(split_sectors); - bio_endio(bio); - return NULL; - } + if (unlikely(split_sectors < 0)) + goto error; if (split_sectors) { struct bio *split; split = bio_split(bio, split_sectors, GFP_NOIO, &bio->bi_bdev->bd_disk->bio_split); + if (IS_ERR(split)) { + split_sectors = PTR_ERR(split); + goto error; + } split->bi_opf |= REQ_NOMERGE; blkcg_bio_issue_init(split); bio_chain(split, bio); @@ -128,6 +129,10 @@ static struct bio *bio_submit_split(struct bio *bio, int split_sectors) } return bio; +error: + bio->bi_status = errno_to_blk_status(split_sectors); + bio_endio(bio); + return NULL; } struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, @@ -166,17 +171,6 @@ struct bio *bio_split_discard(struct bio *bio, const struct queue_limits *lim, return bio_submit_split(bio, split_sectors); } -struct bio *bio_split_write_zeroes(struct bio *bio, - const struct queue_limits *lim, unsigned *nsegs) -{ - *nsegs = 0; - if (!lim->max_write_zeroes_sectors) - return bio; - if (bio_sectors(bio) <= lim->max_write_zeroes_sectors) - return bio; - return bio_submit_split(bio, lim->max_write_zeroes_sectors); -} - static inline unsigned int blk_boundary_sectors(const struct queue_limits *lim, bool is_atomic) { @@ -211,7 +205,9 @@ static inline unsigned get_max_io_size(struct bio *bio, * We ignore lim->max_sectors for atomic writes because it may less * than the actual bio size, which we cannot tolerate. */ - if (is_atomic) + if (bio_op(bio) == REQ_OP_WRITE_ZEROES) + max_sectors = lim->max_write_zeroes_sectors; + else if (is_atomic) max_sectors = lim->atomic_write_max_sectors; else max_sectors = lim->max_sectors; @@ -296,6 +292,14 @@ static bool bvec_split_segs(const struct queue_limits *lim, return len > 0 || bv->bv_len > max_len; } +static unsigned int bio_split_alignment(struct bio *bio, + const struct queue_limits *lim) +{ + if (op_is_write(bio_op(bio)) && lim->zone_write_granularity) + return lim->zone_write_granularity; + return lim->logical_block_size; +} + /** * bio_split_rw_at - check if and where to split a read/write bio * @bio: [in] bio to be split @@ -358,7 +362,7 @@ split: * split size so that each bio is properly block size aligned, even if * we do not use the full hardware limits. */ - bytes = ALIGN_DOWN(bytes, lim->logical_block_size); + bytes = ALIGN_DOWN(bytes, bio_split_alignment(bio, lim)); /* * Bio splitting may cause subtle trouble such as hang when doing sync @@ -388,16 +392,35 @@ struct bio *bio_split_rw(struct bio *bio, const struct queue_limits *lim, struct bio *bio_split_zone_append(struct bio *bio, const struct queue_limits *lim, unsigned *nr_segs) { - unsigned int max_sectors = queue_limits_max_zone_append_sectors(lim); int split_sectors; split_sectors = bio_split_rw_at(bio, lim, nr_segs, - max_sectors << SECTOR_SHIFT); + lim->max_zone_append_sectors << SECTOR_SHIFT); if (WARN_ON_ONCE(split_sectors > 0)) split_sectors = -EINVAL; return bio_submit_split(bio, split_sectors); } +struct bio *bio_split_write_zeroes(struct bio *bio, + const struct queue_limits *lim, unsigned *nsegs) +{ + unsigned int max_sectors = get_max_io_size(bio, lim); + + *nsegs = 0; + + /* + * An unset limit should normally not happen, as bio submission is keyed + * off having a non-zero limit. But SCSI can clear the limit in the + * I/O completion handler, and we can race and see this. Splitting to a + * zero limit obviously doesn't make sense, so band-aid it here. + */ + if (!max_sectors) + return bio; + if (bio_sectors(bio) <= max_sectors) + return bio; + return bio_submit_split(bio, max_sectors); +} + /** * bio_split_to_limits - split a bio to fit the queue limits * @bio: bio to be split @@ -411,10 +434,9 @@ struct bio *bio_split_zone_append(struct bio *bio, */ struct bio *bio_split_to_limits(struct bio *bio) { - const struct queue_limits *lim = &bdev_get_queue(bio->bi_bdev)->limits; unsigned int nr_segs; - return __bio_split_to_limits(bio, lim, &nr_segs); + return __bio_split_to_limits(bio, bdev_limits(bio->bi_bdev), &nr_segs); } EXPORT_SYMBOL(bio_split_to_limits); @@ -797,7 +819,7 @@ static inline void blk_update_mixed_merge(struct request *req, static void blk_account_io_merge_request(struct request *req) { - if (blk_do_io_stat(req)) { + if (req->rq_flags & RQF_IO_STAT) { part_stat_lock(); part_stat_inc(req->part, merges[op_stat_group(req_op(req))]); part_stat_local_dec(req->part, @@ -845,12 +867,13 @@ static struct request *attempt_merge(struct request_queue *q, if (rq_data_dir(req) != rq_data_dir(next)) return NULL; - /* Don't merge requests with different write hints. */ - if (req->write_hint != next->write_hint) - return NULL; - - if (req->ioprio != next->ioprio) - return NULL; + if (req->bio && next->bio) { + /* Don't merge requests with different write hints. */ + if (req->bio->bi_write_hint != next->bio->bi_write_hint) + return NULL; + if (req->bio->bi_ioprio != next->bio->bi_ioprio) + return NULL; + } if (!blk_atomic_write_mergeable_rqs(req, next)) return NULL; @@ -979,12 +1002,13 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) if (!bio_crypt_rq_ctx_compatible(rq, bio)) return false; - /* Don't merge requests with different write hints. */ - if (rq->write_hint != bio->bi_write_hint) - return false; - - if (rq->ioprio != bio_prio(bio)) - return false; + if (rq->bio) { + /* Don't merge requests with different write hints. */ + if (rq->bio->bi_write_hint != bio->bi_write_hint) + return false; + if (rq->bio->bi_ioprio != bio->bi_ioprio) + return false; + } if (blk_atomic_write_mergeable_rq_bio(rq, bio) == false) return false; @@ -1005,12 +1029,11 @@ enum elv_merge blk_try_merge(struct request *rq, struct bio *bio) static void blk_account_io_merge_bio(struct request *req) { - if (!blk_do_io_stat(req)) - return; - - part_stat_lock(); - part_stat_inc(req->part, merges[op_stat_group(req_op(req))]); - part_stat_unlock(); + if (req->rq_flags & RQF_IO_STAT) { + part_stat_lock(); + part_stat_inc(req->part, merges[op_stat_group(req_op(req))]); + part_stat_unlock(); + } } enum bio_merge_status bio_attempt_back_merge(struct request *req, @@ -1156,7 +1179,7 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, struct blk_plug *plug = current->plug; struct request *rq; - if (!plug || rq_list_empty(plug->mq_list)) + if (!plug || rq_list_empty(&plug->mq_list)) return false; rq_list_for_each(&plug->mq_list, rq) { |