From 29e350944fdc2dfca102500790d8ad6d6ff4f69d Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Sun, 2 Apr 2006 12:46:35 -0700 Subject: splice: add SPLICE_F_NONBLOCK flag It doesn't make the splice itself necessarily nonblocking (because the actual file descriptors that are spliced from/to may block unless they have the O_NONBLOCK flag set), but it makes the splice pipe operations nonblocking. Signed-off-by: Linus Torvalds --- include/linux/pipe_fs_i.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux/pipe_fs_i.h') diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index 75c7f55023ab..d218fc729319 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -60,5 +60,8 @@ void free_pipe_info(struct inode* inode); * add the splice flags here. */ #define SPLICE_F_MOVE (0x01) /* move pages instead of copying */ +#define SPLICE_F_NONBLOCK (0x02) /* don't block on the pipe splicing (but */ + /* we may still block on the fd we splice */ + /* from/to, of course */ #endif -- cgit v1.2.3 From 4f6f0bd2ffa4e31c3524f5e65c84a29b6ab73307 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 2 Apr 2006 23:04:46 +0200 Subject: [PATCH] splice: improve writeback and clean up page stealing By cleaning up the writeback logic (killing write_one_page() and the manual set_page_dirty()), we can get rid of ->stolen inside the pipe_buffer and just keep it local in pipe_to_file(). This also adds dirty page balancing logic and O_SYNC handling. Signed-off-by: Jens Axboe --- fs/pipe.c | 1 - fs/splice.c | 64 +++++++++++++++++++++++++++++++++++------------ include/linux/pipe_fs_i.h | 1 - 3 files changed, 48 insertions(+), 18 deletions(-) (limited to 'include/linux/pipe_fs_i.h') diff --git a/fs/pipe.c b/fs/pipe.c index 109a102c150d..5093408746b8 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -124,7 +124,6 @@ static void anon_pipe_buf_unmap(struct pipe_inode_info *info, struct pipe_buffer static int anon_pipe_buf_steal(struct pipe_inode_info *info, struct pipe_buffer *buf) { - buf->stolen = 1; return 0; } diff --git a/fs/splice.c b/fs/splice.c index a555d0a83fe9..07f4d863c2d4 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -22,7 +22,10 @@ #include #include #include +#include +#include #include +#include /* * Passed to the actors @@ -38,11 +41,15 @@ static int page_cache_pipe_buf_steal(struct pipe_inode_info *info, struct pipe_buffer *buf) { struct page *page = buf->page; + struct address_space *mapping = page_mapping(page); WARN_ON(!PageLocked(page)); WARN_ON(!PageUptodate(page)); - if (!remove_mapping(page_mapping(page), page)) + if (PagePrivate(page)) + try_to_release_page(page, mapping_gfp_mask(mapping)); + + if (!remove_mapping(mapping, page)) return 1; if (PageLRU(page)) { @@ -55,7 +62,6 @@ static int page_cache_pipe_buf_steal(struct pipe_inode_info *info, spin_unlock_irq(&zone->lru_lock); } - buf->stolen = 1; return 0; } @@ -64,7 +70,6 @@ static void page_cache_pipe_buf_release(struct pipe_inode_info *info, { page_cache_release(buf->page); buf->page = NULL; - buf->stolen = 0; } static void *page_cache_pipe_buf_map(struct file *file, @@ -91,8 +96,7 @@ static void *page_cache_pipe_buf_map(struct file *file, static void page_cache_pipe_buf_unmap(struct pipe_inode_info *info, struct pipe_buffer *buf) { - if (!buf->stolen) - unlock_page(buf->page); + unlock_page(buf->page); kunmap(buf->page); } @@ -319,7 +323,8 @@ ssize_t generic_file_splice_read(struct file *in, struct inode *pipe, } /* - * Send 'len' bytes to socket from 'file' at position 'pos' using sendpage(). + * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' + * using sendpage(). */ static int pipe_to_sendpage(struct pipe_inode_info *info, struct pipe_buffer *buf, struct splice_desc *sd) @@ -379,7 +384,7 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, struct page *page; pgoff_t index; char *src; - int ret; + int ret, stolen; /* * after this, page will be locked and unmapped @@ -390,6 +395,7 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, index = sd->pos >> PAGE_CACHE_SHIFT; offset = sd->pos & ~PAGE_CACHE_MASK; + stolen = 0; /* * reuse buf page, if SPLICE_F_MOVE is set @@ -399,6 +405,7 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, goto find_page; page = buf->page; + stolen = 1; if (add_to_page_cache_lru(page, mapping, index, mapping_gfp_mask(mapping))) goto find_page; @@ -443,10 +450,13 @@ find_page: } ret = mapping->a_ops->prepare_write(file, page, 0, sd->len); - if (ret) + if (ret == AOP_TRUNCATED_PAGE) { + page_cache_release(page); + goto find_page; + } else if (ret) goto out; - if (!buf->stolen) { + if (!stolen) { char *dst = kmap_atomic(page, KM_USER0); memcpy(dst + offset, src + buf->offset, sd->len); @@ -455,16 +465,18 @@ find_page: } ret = mapping->a_ops->commit_write(file, page, 0, sd->len); - if (ret < 0) + if (ret == AOP_TRUNCATED_PAGE) { + page_cache_release(page); + goto find_page; + } else if (ret) goto out; - set_page_dirty(page); - ret = write_one_page(page, 0); + balance_dirty_pages_ratelimited(mapping); out: - if (ret < 0) - unlock_page(page); - if (!buf->stolen) + if (!stolen) { page_cache_release(page); + unlock_page(page); + } buf->ops->unmap(info, buf); return ret; } @@ -576,7 +588,27 @@ static ssize_t move_from_pipe(struct inode *inode, struct file *out, ssize_t generic_file_splice_write(struct inode *inode, struct file *out, size_t len, unsigned int flags) { - return move_from_pipe(inode, out, len, flags, pipe_to_file); + struct address_space *mapping = out->f_mapping; + ssize_t ret = move_from_pipe(inode, out, len, flags, pipe_to_file); + + /* + * if file or inode is SYNC and we actually wrote some data, sync it + */ + if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(mapping->host)) + && ret > 0) { + struct inode *inode = mapping->host; + int err; + + mutex_lock(&inode->i_mutex); + err = generic_osync_inode(mapping->host, mapping, + OSYNC_METADATA|OSYNC_DATA); + mutex_unlock(&inode->i_mutex); + + if (err) + ret = err; + } + + return ret; } ssize_t generic_splice_sendpage(struct inode *inode, struct file *out, diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index d218fc729319..32865004751c 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -9,7 +9,6 @@ struct pipe_buffer { struct page *page; unsigned int offset, len; struct pipe_buf_operations *ops; - unsigned int stolen; }; struct pipe_buf_operations { -- cgit v1.2.3 From b2b39fa478db6db89b7ccafb0649973845b0eb75 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 2 Apr 2006 23:05:41 +0200 Subject: [PATCH] splice: add a SPLICE_F_MORE flag This lets userspace indicate whether more data will be coming in a subsequent splice call. Signed-off-by: Jens Axboe --- fs/splice.c | 5 +++-- include/linux/pipe_fs_i.h | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux/pipe_fs_i.h') diff --git a/fs/splice.c b/fs/splice.c index 34591924c783..e84544558946 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -354,6 +354,7 @@ static int pipe_to_sendpage(struct pipe_inode_info *info, unsigned int offset; ssize_t ret; void *ptr; + int more; /* * sub-optimal, but we are limited by the pipe ->map. we don't @@ -366,9 +367,9 @@ static int pipe_to_sendpage(struct pipe_inode_info *info, return PTR_ERR(ptr); offset = pos & ~PAGE_CACHE_MASK; + more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len; - ret = file->f_op->sendpage(file, buf->page, offset, sd->len, &pos, - sd->len < sd->total_len); + ret = file->f_op->sendpage(file, buf->page, offset, sd->len, &pos,more); buf->ops->unmap(info, buf); if (ret == sd->len) diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index 32865004751c..70ae9332ec16 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -62,5 +62,6 @@ void free_pipe_info(struct inode* inode); #define SPLICE_F_NONBLOCK (0x02) /* don't block on the pipe splicing (but */ /* we may still block on the fd we splice */ /* from/to, of course */ +#define SPLICE_F_MORE (0x04) /* expect more data */ #endif -- cgit v1.2.3 From 3e7ee3e7b36fa4e2d88d8fb0a2577be95fc4636d Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 2 Apr 2006 23:11:04 +0200 Subject: [PATCH] splice: fix page stealing LRU handling. Originally from Nick Piggin, just adapted to the newer branch. You can't check PageLRU without holding zone->lru_lock. The page release code can get away with it only because the page refcount is 0 at that point. Also, you can't reliably remove pages from the LRU unless the refcount is 0. Ever. Signed-off-by: Nick Piggin Signed-off-by: Jens Axboe --- fs/pipe.c | 3 +++ fs/splice.c | 30 +++++++++++------------------- include/linux/pipe_fs_i.h | 4 ++++ 3 files changed, 18 insertions(+), 19 deletions(-) (limited to 'include/linux/pipe_fs_i.h') diff --git a/fs/pipe.c b/fs/pipe.c index 5093408746b8..795df987cd38 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -95,6 +95,8 @@ static void anon_pipe_buf_release(struct pipe_inode_info *info, struct pipe_buff { struct page *page = buf->page; + buf->flags &= ~PIPE_BUF_FLAG_STOLEN; + /* * If nobody else uses this page, and we don't already have a * temporary page, let's keep track of it as a one-deep @@ -124,6 +126,7 @@ static void anon_pipe_buf_unmap(struct pipe_inode_info *info, struct pipe_buffer static int anon_pipe_buf_steal(struct pipe_inode_info *info, struct pipe_buffer *buf) { + buf->flags |= PIPE_BUF_FLAG_STOLEN; return 0; } diff --git a/fs/splice.c b/fs/splice.c index b5fb2f3e3ac6..bfa42a277bb8 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -67,16 +67,7 @@ static int page_cache_pipe_buf_steal(struct pipe_inode_info *info, if (!remove_mapping(mapping, page)) return 1; - if (PageLRU(page)) { - struct zone *zone = page_zone(page); - - spin_lock_irq(&zone->lru_lock); - BUG_ON(!PageLRU(page)); - __ClearPageLRU(page); - del_page_from_lru(zone, page); - spin_unlock_irq(&zone->lru_lock); - } - + buf->flags |= PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU; return 0; } @@ -85,6 +76,7 @@ static void page_cache_pipe_buf_release(struct pipe_inode_info *info, { page_cache_release(buf->page); buf->page = NULL; + buf->flags &= ~(PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU); } static void *page_cache_pipe_buf_map(struct file *file, @@ -414,11 +406,12 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, { struct file *file = sd->file; struct address_space *mapping = file->f_mapping; + gfp_t gfp_mask = mapping_gfp_mask(mapping); unsigned int offset; struct page *page; pgoff_t index; char *src; - int ret, stolen; + int ret; /* * after this, page will be locked and unmapped @@ -429,7 +422,6 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, index = sd->pos >> PAGE_CACHE_SHIFT; offset = sd->pos & ~PAGE_CACHE_MASK; - stolen = 0; /* * reuse buf page, if SPLICE_F_MOVE is set @@ -443,15 +435,15 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, goto find_page; page = buf->page; - stolen = 1; - if (add_to_page_cache_lru(page, mapping, index, - mapping_gfp_mask(mapping))) + if (add_to_page_cache(page, mapping, index, gfp_mask)) goto find_page; + + if (!(buf->flags & PIPE_BUF_FLAG_LRU)) + lru_cache_add(page); } else { find_page: ret = -ENOMEM; - page = find_or_create_page(mapping, index, - mapping_gfp_mask(mapping)); + page = find_or_create_page(mapping, index, gfp_mask); if (!page) goto out; @@ -494,7 +486,7 @@ find_page: } else if (ret) goto out; - if (!stolen) { + if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) { char *dst = kmap_atomic(page, KM_USER0); memcpy(dst + offset, src + buf->offset, sd->len); @@ -511,7 +503,7 @@ find_page: balance_dirty_pages_ratelimited(mapping); out: - if (!stolen) { + if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) { page_cache_release(page); unlock_page(page); } diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index 70ae9332ec16..ec384958d509 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -5,10 +5,14 @@ #define PIPE_BUFFERS (16) +#define PIPE_BUF_FLAG_STOLEN 0x01 +#define PIPE_BUF_FLAG_LRU 0x02 + struct pipe_buffer { struct page *page; unsigned int offset, len; struct pipe_buf_operations *ops; + unsigned int flags; }; struct pipe_buf_operations { -- cgit v1.2.3