From 63e95b5c4f16e156b98adcf2f7d820ba941c82a3 Mon Sep 17 00:00:00 2001 From: Ross Zwisler Date: Tue, 8 Nov 2016 11:32:20 +1100 Subject: dax: coordinate locking for offsets in PMD range DAX radix tree locking currently locks entries based on the unique combination of the 'mapping' pointer and the pgoff_t 'index' for the entry. This works for PTEs, but as we move to PMDs we will need to have all the offsets within the range covered by the PMD to map to the same bit lock. To accomplish this, for ranges covered by a PMD entry we will instead lock based on the page offset of the beginning of the PMD entry. The 'mapping' pointer is still used in the same way. Signed-off-by: Ross Zwisler Reviewed-by: Christoph Hellwig Reviewed-by: Jan Kara Signed-off-by: Dave Chinner --- include/linux/dax.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/dax.h b/include/linux/dax.h index add6c4bc568f..a41a747d6112 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -22,7 +22,7 @@ int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, int dax_fault(struct vm_area_struct *, struct vm_fault *, get_block_t); int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index); void dax_wake_mapping_entry_waiter(struct address_space *mapping, - pgoff_t index, bool wake_all); + pgoff_t index, void *entry, bool wake_all); #ifdef CONFIG_FS_DAX struct page *read_dax_sector(struct block_device *bdev, sector_t n); -- cgit v1.2.3 From b9fde0462e34a05b25c3d68d344971865659abae Mon Sep 17 00:00:00 2001 From: Ross Zwisler Date: Tue, 8 Nov 2016 11:32:35 +1100 Subject: dax: remove dax_pmd_fault() dax_pmd_fault() is the old struct buffer_head + get_block_t based 2 MiB DAX fault handler. This fault handler has been disabled for several kernel releases, and support for PMDs will be reintroduced using the struct iomap interface instead. Signed-off-by: Ross Zwisler Reviewed-by: Christoph Hellwig Reviewed-by: Jan Kara Signed-off-by: Dave Chinner --- fs/dax.c | 213 ---------------------------------------------------- include/linux/dax.h | 6 +- 2 files changed, 1 insertion(+), 218 deletions(-) (limited to 'include/linux') diff --git a/fs/dax.c b/fs/dax.c index 72387023545e..3d0b1032c555 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -915,219 +915,6 @@ int dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, } EXPORT_SYMBOL_GPL(dax_fault); -#if defined(CONFIG_TRANSPARENT_HUGEPAGE) -/* - * The 'colour' (ie low bits) within a PMD of a page offset. This comes up - * more often than one might expect in the below function. - */ -#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1) - -static void __dax_dbg(struct buffer_head *bh, unsigned long address, - const char *reason, const char *fn) -{ - if (bh) { - char bname[BDEVNAME_SIZE]; - bdevname(bh->b_bdev, bname); - pr_debug("%s: %s addr: %lx dev %s state %lx start %lld " - "length %zd fallback: %s\n", fn, current->comm, - address, bname, bh->b_state, (u64)bh->b_blocknr, - bh->b_size, reason); - } else { - pr_debug("%s: %s addr: %lx fallback: %s\n", fn, - current->comm, address, reason); - } -} - -#define dax_pmd_dbg(bh, address, reason) __dax_dbg(bh, address, reason, "dax_pmd") - -/** - * dax_pmd_fault - handle a PMD fault on a DAX file - * @vma: The virtual memory area where the fault occurred - * @vmf: The description of the fault - * @get_block: The filesystem method used to translate file offsets to blocks - * - * When a page fault occurs, filesystems may call this helper in their - * pmd_fault handler for DAX files. - */ -int dax_pmd_fault(struct vm_area_struct *vma, unsigned long address, - pmd_t *pmd, unsigned int flags, get_block_t get_block) -{ - struct file *file = vma->vm_file; - struct address_space *mapping = file->f_mapping; - struct inode *inode = mapping->host; - struct buffer_head bh; - unsigned blkbits = inode->i_blkbits; - unsigned long pmd_addr = address & PMD_MASK; - bool write = flags & FAULT_FLAG_WRITE; - struct block_device *bdev; - pgoff_t size, pgoff; - sector_t block; - int result = 0; - bool alloc = false; - - /* dax pmd mappings require pfn_t_devmap() */ - if (!IS_ENABLED(CONFIG_FS_DAX_PMD)) - return VM_FAULT_FALLBACK; - - /* Fall back to PTEs if we're going to COW */ - if (write && !(vma->vm_flags & VM_SHARED)) { - split_huge_pmd(vma, pmd, address); - dax_pmd_dbg(NULL, address, "cow write"); - return VM_FAULT_FALLBACK; - } - /* If the PMD would extend outside the VMA */ - if (pmd_addr < vma->vm_start) { - dax_pmd_dbg(NULL, address, "vma start unaligned"); - return VM_FAULT_FALLBACK; - } - if ((pmd_addr + PMD_SIZE) > vma->vm_end) { - dax_pmd_dbg(NULL, address, "vma end unaligned"); - return VM_FAULT_FALLBACK; - } - - pgoff = linear_page_index(vma, pmd_addr); - size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; - if (pgoff >= size) - return VM_FAULT_SIGBUS; - /* If the PMD would cover blocks out of the file */ - if ((pgoff | PG_PMD_COLOUR) >= size) { - dax_pmd_dbg(NULL, address, - "offset + huge page size > file size"); - return VM_FAULT_FALLBACK; - } - - memset(&bh, 0, sizeof(bh)); - bh.b_bdev = inode->i_sb->s_bdev; - block = (sector_t)pgoff << (PAGE_SHIFT - blkbits); - - bh.b_size = PMD_SIZE; - - if (get_block(inode, block, &bh, 0) != 0) - return VM_FAULT_SIGBUS; - - if (!buffer_mapped(&bh) && write) { - if (get_block(inode, block, &bh, 1) != 0) - return VM_FAULT_SIGBUS; - alloc = true; - WARN_ON_ONCE(buffer_unwritten(&bh) || buffer_new(&bh)); - } - - bdev = bh.b_bdev; - - if (bh.b_size < PMD_SIZE) { - dax_pmd_dbg(&bh, address, "allocated block too small"); - return VM_FAULT_FALLBACK; - } - - /* - * If we allocated new storage, make sure no process has any - * zero pages covering this hole - */ - if (alloc) { - loff_t lstart = pgoff << PAGE_SHIFT; - loff_t lend = lstart + PMD_SIZE - 1; /* inclusive */ - - truncate_pagecache_range(inode, lstart, lend); - } - - if (!write && !buffer_mapped(&bh)) { - spinlock_t *ptl; - pmd_t entry; - struct page *zero_page = mm_get_huge_zero_page(vma->vm_mm); - - if (unlikely(!zero_page)) { - dax_pmd_dbg(&bh, address, "no zero page"); - goto fallback; - } - - ptl = pmd_lock(vma->vm_mm, pmd); - if (!pmd_none(*pmd)) { - spin_unlock(ptl); - dax_pmd_dbg(&bh, address, "pmd already present"); - goto fallback; - } - - dev_dbg(part_to_dev(bdev->bd_part), - "%s: %s addr: %lx pfn: sect: %llx\n", - __func__, current->comm, address, - (unsigned long long) to_sector(&bh, inode)); - - entry = mk_pmd(zero_page, vma->vm_page_prot); - entry = pmd_mkhuge(entry); - set_pmd_at(vma->vm_mm, pmd_addr, pmd, entry); - result = VM_FAULT_NOPAGE; - spin_unlock(ptl); - } else { - struct blk_dax_ctl dax = { - .sector = to_sector(&bh, inode), - .size = PMD_SIZE, - }; - long length = dax_map_atomic(bdev, &dax); - - if (length < 0) { - dax_pmd_dbg(&bh, address, "dax-error fallback"); - goto fallback; - } - if (length < PMD_SIZE) { - dax_pmd_dbg(&bh, address, "dax-length too small"); - dax_unmap_atomic(bdev, &dax); - goto fallback; - } - if (pfn_t_to_pfn(dax.pfn) & PG_PMD_COLOUR) { - dax_pmd_dbg(&bh, address, "pfn unaligned"); - dax_unmap_atomic(bdev, &dax); - goto fallback; - } - - if (!pfn_t_devmap(dax.pfn)) { - dax_unmap_atomic(bdev, &dax); - dax_pmd_dbg(&bh, address, "pfn not in memmap"); - goto fallback; - } - dax_unmap_atomic(bdev, &dax); - - /* - * For PTE faults we insert a radix tree entry for reads, and - * leave it clean. Then on the first write we dirty the radix - * tree entry via the dax_pfn_mkwrite() path. This sequence - * allows the dax_pfn_mkwrite() call to be simpler and avoid a - * call into get_block() to translate the pgoff to a sector in - * order to be able to create a new radix tree entry. - * - * The PMD path doesn't have an equivalent to - * dax_pfn_mkwrite(), though, so for a read followed by a - * write we traverse all the way through dax_pmd_fault() - * twice. This means we can just skip inserting a radix tree - * entry completely on the initial read and just wait until - * the write to insert a dirty entry. - */ - if (write) { - /* - * We should insert radix-tree entry and dirty it here. - * For now this is broken... - */ - } - - dev_dbg(part_to_dev(bdev->bd_part), - "%s: %s addr: %lx pfn: %lx sect: %llx\n", - __func__, current->comm, address, - pfn_t_to_pfn(dax.pfn), - (unsigned long long) dax.sector); - result |= vmf_insert_pfn_pmd(vma, address, pmd, - dax.pfn, write); - } - - out: - return result; - - fallback: - count_vm_event(THP_FAULT_FALLBACK); - result = VM_FAULT_FALLBACK; - goto out; -} -EXPORT_SYMBOL_GPL(dax_pmd_fault); -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ - /** * dax_pfn_mkwrite - handle first write to DAX page * @vma: The virtual memory area where the fault occurred diff --git a/include/linux/dax.h b/include/linux/dax.h index a41a747d6112..0f74866edae6 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -48,16 +48,12 @@ static inline int __dax_zero_page_range(struct block_device *bdev, } #endif -#if defined(CONFIG_TRANSPARENT_HUGEPAGE) -int dax_pmd_fault(struct vm_area_struct *, unsigned long addr, pmd_t *, - unsigned int flags, get_block_t); -#else static inline int dax_pmd_fault(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd, unsigned int flags, get_block_t gb) { return VM_FAULT_FALLBACK; } -#endif + int dax_pfn_mkwrite(struct vm_area_struct *, struct vm_fault *); #define dax_mkwrite(vma, vmf, gb) dax_fault(vma, vmf, gb) -- cgit v1.2.3 From 11c59c92f44d9272db7655a462608658a6d95013 Mon Sep 17 00:00:00 2001 From: Ross Zwisler Date: Tue, 8 Nov 2016 11:32:46 +1100 Subject: dax: correct dax iomap code namespace The recently added DAX functions that use the new struct iomap data structure were named iomap_dax_rw(), iomap_dax_fault() and iomap_dax_actor(). These are actually defined in fs/dax.c, though, so should be part of the "dax" namespace and not the "iomap" namespace. Rename them to dax_iomap_rw(), dax_iomap_fault() and dax_iomap_actor() respectively. Signed-off-by: Ross Zwisler Suggested-by: Dave Chinner Reviewed-by: Christoph Hellwig Reviewed-by: Jan Kara Signed-off-by: Dave Chinner --- fs/dax.c | 16 ++++++++-------- fs/ext2/file.c | 6 +++--- fs/xfs/xfs_file.c | 8 ++++---- include/linux/dax.h | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/fs/dax.c b/fs/dax.c index 3d0b1032c555..fdbd7a1ec6cf 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1031,7 +1031,7 @@ EXPORT_SYMBOL_GPL(dax_truncate_page); #ifdef CONFIG_FS_IOMAP static loff_t -iomap_dax_actor(struct inode *inode, loff_t pos, loff_t length, void *data, +dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data, struct iomap *iomap) { struct iov_iter *iter = data; @@ -1088,7 +1088,7 @@ iomap_dax_actor(struct inode *inode, loff_t pos, loff_t length, void *data, } /** - * iomap_dax_rw - Perform I/O to a DAX file + * dax_iomap_rw - Perform I/O to a DAX file * @iocb: The control block for this I/O * @iter: The addresses to do I/O from or to * @ops: iomap ops passed from the file system @@ -1098,7 +1098,7 @@ iomap_dax_actor(struct inode *inode, loff_t pos, loff_t length, void *data, * and evicting any page cache pages in the region under I/O. */ ssize_t -iomap_dax_rw(struct kiocb *iocb, struct iov_iter *iter, +dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, struct iomap_ops *ops) { struct address_space *mapping = iocb->ki_filp->f_mapping; @@ -1128,7 +1128,7 @@ iomap_dax_rw(struct kiocb *iocb, struct iov_iter *iter, while (iov_iter_count(iter)) { ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops, - iter, iomap_dax_actor); + iter, dax_iomap_actor); if (ret <= 0) break; pos += ret; @@ -1138,10 +1138,10 @@ iomap_dax_rw(struct kiocb *iocb, struct iov_iter *iter, iocb->ki_pos += done; return done ? done : ret; } -EXPORT_SYMBOL_GPL(iomap_dax_rw); +EXPORT_SYMBOL_GPL(dax_iomap_rw); /** - * iomap_dax_fault - handle a page fault on a DAX file + * dax_iomap_fault - handle a page fault on a DAX file * @vma: The virtual memory area where the fault occurred * @vmf: The description of the fault * @ops: iomap ops passed from the file system @@ -1150,7 +1150,7 @@ EXPORT_SYMBOL_GPL(iomap_dax_rw); * or mkwrite handler for DAX files. Assumes the caller has done all the * necessary locking for the page fault to proceed successfully. */ -int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, +int dax_iomap_fault(struct vm_area_struct *vma, struct vm_fault *vmf, struct iomap_ops *ops) { struct address_space *mapping = vma->vm_file->f_mapping; @@ -1252,5 +1252,5 @@ int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, return VM_FAULT_SIGBUS | major; return VM_FAULT_NOPAGE | major; } -EXPORT_SYMBOL_GPL(iomap_dax_fault); +EXPORT_SYMBOL_GPL(dax_iomap_fault); #endif /* CONFIG_FS_IOMAP */ diff --git a/fs/ext2/file.c b/fs/ext2/file.c index fb88b51ca947..b0f241528a30 100644 --- a/fs/ext2/file.c +++ b/fs/ext2/file.c @@ -38,7 +38,7 @@ static ssize_t ext2_dax_read_iter(struct kiocb *iocb, struct iov_iter *to) return 0; /* skip atime */ inode_lock_shared(inode); - ret = iomap_dax_rw(iocb, to, &ext2_iomap_ops); + ret = dax_iomap_rw(iocb, to, &ext2_iomap_ops); inode_unlock_shared(inode); file_accessed(iocb->ki_filp); @@ -62,7 +62,7 @@ static ssize_t ext2_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) if (ret) goto out_unlock; - ret = iomap_dax_rw(iocb, from, &ext2_iomap_ops); + ret = dax_iomap_rw(iocb, from, &ext2_iomap_ops); if (ret > 0 && iocb->ki_pos > i_size_read(inode)) { i_size_write(inode, iocb->ki_pos); mark_inode_dirty(inode); @@ -99,7 +99,7 @@ static int ext2_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf) } down_read(&ei->dax_sem); - ret = iomap_dax_fault(vma, vmf, &ext2_iomap_ops); + ret = dax_iomap_fault(vma, vmf, &ext2_iomap_ops); up_read(&ei->dax_sem); if (vmf->flags & FAULT_FLAG_WRITE) diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index a314fc7b56fa..e7f35d548cfc 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -344,7 +344,7 @@ xfs_file_dax_read( return 0; /* skip atime */ xfs_rw_ilock(ip, XFS_IOLOCK_SHARED); - ret = iomap_dax_rw(iocb, to, &xfs_iomap_ops); + ret = dax_iomap_rw(iocb, to, &xfs_iomap_ops); xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED); file_accessed(iocb->ki_filp); @@ -691,7 +691,7 @@ xfs_file_dax_write( trace_xfs_file_dax_write(ip, count, pos); - ret = iomap_dax_rw(iocb, from, &xfs_iomap_ops); + ret = dax_iomap_rw(iocb, from, &xfs_iomap_ops); if (ret > 0 && iocb->ki_pos > i_size_read(inode)) { i_size_write(inode, iocb->ki_pos); error = xfs_setfilesize(ip, pos, ret); @@ -1640,7 +1640,7 @@ xfs_filemap_page_mkwrite( xfs_ilock(XFS_I(inode), XFS_MMAPLOCK_SHARED); if (IS_DAX(inode)) { - ret = iomap_dax_fault(vma, vmf, &xfs_iomap_ops); + ret = dax_iomap_fault(vma, vmf, &xfs_iomap_ops); } else { ret = iomap_page_mkwrite(vma, vmf, &xfs_iomap_ops); ret = block_page_mkwrite_return(ret); @@ -1674,7 +1674,7 @@ xfs_filemap_fault( * changes to xfs_get_blocks_direct() to map unwritten extent * ioend for conversion on read-only mappings. */ - ret = iomap_dax_fault(vma, vmf, &xfs_iomap_ops); + ret = dax_iomap_fault(vma, vmf, &xfs_iomap_ops); } else ret = filemap_fault(vma, vmf); xfs_iunlock(XFS_I(inode), XFS_MMAPLOCK_SHARED); diff --git a/include/linux/dax.h b/include/linux/dax.h index 0f74866edae6..a3dfee4cb03f 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -11,13 +11,13 @@ struct iomap_ops; /* We use lowest available exceptional entry bit for locking */ #define RADIX_DAX_ENTRY_LOCK (1 << RADIX_TREE_EXCEPTIONAL_SHIFT) -ssize_t iomap_dax_rw(struct kiocb *iocb, struct iov_iter *iter, +ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, struct iomap_ops *ops); ssize_t dax_do_io(struct kiocb *, struct inode *, struct iov_iter *, get_block_t, dio_iodone_t, int flags); int dax_zero_page_range(struct inode *, loff_t from, unsigned len, get_block_t); int dax_truncate_page(struct inode *, loff_t from, get_block_t); -int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, +int dax_iomap_fault(struct vm_area_struct *vma, struct vm_fault *vmf, struct iomap_ops *ops); int dax_fault(struct vm_area_struct *, struct vm_fault *, get_block_t); int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index); -- cgit v1.2.3 From fa28f7296a7ce38ed15dc06bd2149e04c8db9d4b Mon Sep 17 00:00:00 2001 From: Ross Zwisler Date: Tue, 8 Nov 2016 11:33:35 +1100 Subject: dax: move RADIX_DAX_* defines to dax.h The RADIX_DAX_* defines currently mostly live in fs/dax.c, with just RADIX_DAX_ENTRY_LOCK being in include/linux/dax.h so it can be used in mm/filemap.c. When we add PMD support, though, mm/filemap.c will also need access to the RADIX_DAX_PTE type so it can properly construct a 4k sized empty entry. Instead of shifting the defines between dax.c and dax.h as they are individually used in other code, just move them wholesale to dax.h so they'll be available when we need them. Signed-off-by: Ross Zwisler Reviewed-by: Christoph Hellwig Reviewed-by: Jan Kara Signed-off-by: Dave Chinner --- fs/dax.c | 14 -------------- include/linux/dax.h | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/fs/dax.c b/fs/dax.c index 6edd89b3b69c..c45cc4d8e996 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -34,20 +34,6 @@ #include #include "internal.h" -/* - * We use lowest available bit in exceptional entry for locking, other two - * bits to determine entry type. In total 3 special bits. - */ -#define RADIX_DAX_SHIFT (RADIX_TREE_EXCEPTIONAL_SHIFT + 3) -#define RADIX_DAX_PTE (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1)) -#define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2)) -#define RADIX_DAX_TYPE_MASK (RADIX_DAX_PTE | RADIX_DAX_PMD) -#define RADIX_DAX_TYPE(entry) ((unsigned long)entry & RADIX_DAX_TYPE_MASK) -#define RADIX_DAX_SECTOR(entry) (((unsigned long)entry >> RADIX_DAX_SHIFT)) -#define RADIX_DAX_ENTRY(sector, pmd) ((void *)((unsigned long)sector << \ - RADIX_DAX_SHIFT | (pmd ? RADIX_DAX_PMD : RADIX_DAX_PTE) | \ - RADIX_TREE_EXCEPTIONAL_ENTRY)) - /* We choose 4096 entries - same as per-zone page wait tables */ #define DAX_WAIT_TABLE_BITS 12 #define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS) diff --git a/include/linux/dax.h b/include/linux/dax.h index a3dfee4cb03f..e9ea78c1cf98 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -8,8 +8,21 @@ struct iomap_ops; -/* We use lowest available exceptional entry bit for locking */ +/* + * We use lowest available bit in exceptional entry for locking, other two + * bits to determine entry type. In total 3 special bits. + */ +#define RADIX_DAX_SHIFT (RADIX_TREE_EXCEPTIONAL_SHIFT + 3) #define RADIX_DAX_ENTRY_LOCK (1 << RADIX_TREE_EXCEPTIONAL_SHIFT) +#define RADIX_DAX_PTE (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1)) +#define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2)) +#define RADIX_DAX_TYPE_MASK (RADIX_DAX_PTE | RADIX_DAX_PMD) +#define RADIX_DAX_TYPE(entry) ((unsigned long)entry & RADIX_DAX_TYPE_MASK) +#define RADIX_DAX_SECTOR(entry) (((unsigned long)entry >> RADIX_DAX_SHIFT)) +#define RADIX_DAX_ENTRY(sector, pmd) ((void *)((unsigned long)sector << \ + RADIX_DAX_SHIFT | (pmd ? RADIX_DAX_PMD : RADIX_DAX_PTE) | \ + RADIX_TREE_EXCEPTIONAL_ENTRY)) + ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, struct iomap_ops *ops); -- cgit v1.2.3 From 642261ac995e01d7837db1f4b90181496f7e6835 Mon Sep 17 00:00:00 2001 From: Ross Zwisler Date: Tue, 8 Nov 2016 11:34:45 +1100 Subject: dax: add struct iomap based DAX PMD support DAX PMDs have been disabled since Jan Kara introduced DAX radix tree based locking. This patch allows DAX PMDs to participate in the DAX radix tree based locking scheme so that they can be re-enabled using the new struct iomap based fault handlers. There are currently three types of DAX 4k entries: 4k zero pages, 4k DAX mappings that have an associated block allocation, and 4k DAX empty entries. The empty entries exist to provide locking for the duration of a given page fault. This patch adds three equivalent 2MiB DAX entries: Huge Zero Page (HZP) entries, PMD DAX entries that have associated block allocations, and 2 MiB DAX empty entries. Unlike the 4k case where we insert a struct page* into the radix tree for 4k zero pages, for HZP we insert a DAX exceptional entry with the new RADIX_DAX_HZP flag set. This is because we use a single 2 MiB zero page in every 2MiB hole mapping, and it doesn't make sense to have that same struct page* with multiple entries in multiple trees. This would cause contention on the single page lock for the one Huge Zero Page, and it would break the page->index and page->mapping associations that are assumed to be valid in many other places in the kernel. One difficult use case is when one thread is trying to use 4k entries in radix tree for a given offset, and another thread is using 2 MiB entries for that same offset. The current code handles this by making the 2 MiB user fall back to 4k entries for most cases. This was done because it is the simplest solution, and because the use of 2MiB pages is already opportunistic. If we were to try to upgrade from 4k pages to 2MiB pages for a given range, we run into the problem of how we lock out 4k page faults for the entire 2MiB range while we clean out the radix tree so we can insert the 2MiB entry. We can solve this problem if we need to, but I think that the cases where both 2MiB entries and 4K entries are being used for the same range will be rare enough and the gain small enough that it probably won't be worth the complexity. Signed-off-by: Ross Zwisler Reviewed-by: Jan Kara Signed-off-by: Dave Chinner --- fs/dax.c | 378 +++++++++++++++++++++++++++++++++++++++++++++++----- include/linux/dax.h | 55 ++++++-- mm/filemap.c | 3 +- 3 files changed, 386 insertions(+), 50 deletions(-) (limited to 'include/linux') diff --git a/fs/dax.c b/fs/dax.c index 0582c7c2ae40..281e91a63367 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -76,6 +76,26 @@ static void dax_unmap_atomic(struct block_device *bdev, blk_queue_exit(bdev->bd_queue); } +static int dax_is_pmd_entry(void *entry) +{ + return (unsigned long)entry & RADIX_DAX_PMD; +} + +static int dax_is_pte_entry(void *entry) +{ + return !((unsigned long)entry & RADIX_DAX_PMD); +} + +static int dax_is_zero_entry(void *entry) +{ + return (unsigned long)entry & RADIX_DAX_HZP; +} + +static int dax_is_empty_entry(void *entry) +{ + return (unsigned long)entry & RADIX_DAX_EMPTY; +} + struct page *read_dax_sector(struct block_device *bdev, sector_t n) { struct page *page = alloc_pages(GFP_KERNEL, 0); @@ -281,7 +301,7 @@ static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping, * queue to the start of that PMD. This ensures that all offsets in * the range covered by the PMD map to the same bit lock. */ - if (RADIX_DAX_TYPE(entry) == RADIX_DAX_PMD) + if (dax_is_pmd_entry(entry)) index &= ~((1UL << (PMD_SHIFT - PAGE_SHIFT)) - 1); key->mapping = mapping; @@ -413,36 +433,116 @@ static void put_unlocked_mapping_entry(struct address_space *mapping, * radix tree entry locked. If the radix tree doesn't contain given index, * create empty exceptional entry for the index and return with it locked. * + * When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will + * either return that locked entry or will return an error. This error will + * happen if there are any 4k entries (either zero pages or DAX entries) + * within the 2MiB range that we are requesting. + * + * We always favor 4k entries over 2MiB entries. There isn't a flow where we + * evict 4k entries in order to 'upgrade' them to a 2MiB entry. A 2MiB + * insertion will fail if it finds any 4k entries already in the tree, and a + * 4k insertion will cause an existing 2MiB entry to be unmapped and + * downgraded to 4k entries. This happens for both 2MiB huge zero pages as + * well as 2MiB empty entries. + * + * The exception to this downgrade path is for 2MiB DAX PMD entries that have + * real storage backing them. We will leave these real 2MiB DAX entries in + * the tree, and PTE writes will simply dirty the entire 2MiB DAX entry. + * * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For * persistent memory the benefit is doubtful. We can add that later if we can * show it helps. */ -static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index) +static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index, + unsigned long size_flag) { + bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */ void *entry, **slot; restart: spin_lock_irq(&mapping->tree_lock); entry = get_unlocked_mapping_entry(mapping, index, &slot); + + if (entry) { + if (size_flag & RADIX_DAX_PMD) { + if (!radix_tree_exceptional_entry(entry) || + dax_is_pte_entry(entry)) { + put_unlocked_mapping_entry(mapping, index, + entry); + entry = ERR_PTR(-EEXIST); + goto out_unlock; + } + } else { /* trying to grab a PTE entry */ + if (radix_tree_exceptional_entry(entry) && + dax_is_pmd_entry(entry) && + (dax_is_zero_entry(entry) || + dax_is_empty_entry(entry))) { + pmd_downgrade = true; + } + } + } + /* No entry for given index? Make sure radix tree is big enough. */ - if (!entry) { + if (!entry || pmd_downgrade) { int err; + if (pmd_downgrade) { + /* + * Make sure 'entry' remains valid while we drop + * mapping->tree_lock. + */ + entry = lock_slot(mapping, slot); + } + spin_unlock_irq(&mapping->tree_lock); err = radix_tree_preload( mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM); - if (err) + if (err) { + if (pmd_downgrade) + put_locked_mapping_entry(mapping, index, entry); return ERR_PTR(err); - entry = (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | - RADIX_DAX_ENTRY_LOCK); + } + + /* + * Besides huge zero pages the only other thing that gets + * downgraded are empty entries which don't need to be + * unmapped. + */ + if (pmd_downgrade && dax_is_zero_entry(entry)) + unmap_mapping_range(mapping, + (index << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0); + spin_lock_irq(&mapping->tree_lock); - err = radix_tree_insert(&mapping->page_tree, index, entry); + + if (pmd_downgrade) { + radix_tree_delete(&mapping->page_tree, index); + mapping->nrexceptional--; + dax_wake_mapping_entry_waiter(mapping, index, entry, + true); + } + + entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY); + + err = __radix_tree_insert(&mapping->page_tree, index, + dax_radix_order(entry), entry); radix_tree_preload_end(); if (err) { spin_unlock_irq(&mapping->tree_lock); - /* Someone already created the entry? */ - if (err == -EEXIST) + /* + * Someone already created the entry? This is a + * normal failure when inserting PMDs in a range + * that already contains PTEs. In that case we want + * to return -EEXIST immediately. + */ + if (err == -EEXIST && !(size_flag & RADIX_DAX_PMD)) goto restart; + /* + * Our insertion of a DAX PMD entry failed, most + * likely because it collided with a PTE sized entry + * at a different index in the PMD range. We haven't + * inserted anything into the radix tree and have no + * waiters to wake. + */ return ERR_PTR(err); } /* Good, we have inserted empty locked entry into the tree. */ @@ -466,6 +566,7 @@ restart: return page; } entry = lock_slot(mapping, slot); + out_unlock: spin_unlock_irq(&mapping->tree_lock); return entry; } @@ -473,9 +574,9 @@ restart: /* * We do not necessarily hold the mapping->tree_lock when we call this * function so it is possible that 'entry' is no longer a valid item in the - * radix tree. This is okay, though, because all we really need to do is to - * find the correct waitqueue where tasks might be sleeping waiting for that - * old 'entry' and wake them. + * radix tree. This is okay because all we really need to do is to find the + * correct waitqueue where tasks might be waiting for that old 'entry' and + * wake them. */ void dax_wake_mapping_entry_waiter(struct address_space *mapping, pgoff_t index, void *entry, bool wake_all) @@ -588,11 +689,17 @@ static int copy_user_dax(struct block_device *bdev, sector_t sector, size_t size return 0; } -#define DAX_PMD_INDEX(page_index) (page_index & (PMD_MASK >> PAGE_SHIFT)) - +/* + * By this point grab_mapping_entry() has ensured that we have a locked entry + * of the appropriate size so we don't have to worry about downgrading PMDs to + * PTEs. If we happen to be trying to insert a PTE and there is a PMD + * already in the tree, we will skip the insertion and just dirty the PMD as + * appropriate. + */ static void *dax_insert_mapping_entry(struct address_space *mapping, struct vm_fault *vmf, - void *entry, sector_t sector) + void *entry, sector_t sector, + unsigned long flags) { struct radix_tree_root *page_tree = &mapping->page_tree; int error = 0; @@ -615,22 +722,35 @@ static void *dax_insert_mapping_entry(struct address_space *mapping, error = radix_tree_preload(vmf->gfp_mask & ~__GFP_HIGHMEM); if (error) return ERR_PTR(error); + } else if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_HZP)) { + /* replacing huge zero page with PMD block mapping */ + unmap_mapping_range(mapping, + (vmf->pgoff << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0); } spin_lock_irq(&mapping->tree_lock); - new_entry = (void *)((unsigned long)RADIX_DAX_ENTRY(sector, false) | - RADIX_DAX_ENTRY_LOCK); + new_entry = dax_radix_locked_entry(sector, flags); + if (hole_fill) { __delete_from_page_cache(entry, NULL); /* Drop pagecache reference */ put_page(entry); - error = radix_tree_insert(page_tree, index, new_entry); + error = __radix_tree_insert(page_tree, index, + dax_radix_order(new_entry), new_entry); if (error) { new_entry = ERR_PTR(error); goto unlock; } mapping->nrexceptional++; - } else { + } else if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) { + /* + * Only swap our new entry into the radix tree if the current + * entry is a zero page or an empty entry. If a normal PTE or + * PMD entry is already in the tree, we leave it alone. This + * means that if we are trying to insert a PTE and the + * existing entry is a PMD, we will just leave the PMD in the + * tree and dirty it if necessary. + */ void **slot; void *ret; @@ -660,7 +780,6 @@ static int dax_writeback_one(struct block_device *bdev, struct address_space *mapping, pgoff_t index, void *entry) { struct radix_tree_root *page_tree = &mapping->page_tree; - int type = RADIX_DAX_TYPE(entry); struct radix_tree_node *node; struct blk_dax_ctl dax; void **slot; @@ -681,13 +800,21 @@ static int dax_writeback_one(struct block_device *bdev, if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)) goto unlock; - if (WARN_ON_ONCE(type != RADIX_DAX_PTE && type != RADIX_DAX_PMD)) { + if (WARN_ON_ONCE(dax_is_empty_entry(entry) || + dax_is_zero_entry(entry))) { ret = -EIO; goto unlock; } - dax.sector = RADIX_DAX_SECTOR(entry); - dax.size = (type == RADIX_DAX_PMD ? PMD_SIZE : PAGE_SIZE); + /* + * Even if dax_writeback_mapping_range() was given a wbc->range_start + * in the middle of a PMD, the 'index' we are given will be aligned to + * the start index of the PMD, as will the sector we pull from + * 'entry'. This allows us to flush for PMD_SIZE and not have to + * worry about partial PMD writebacks. + */ + dax.sector = dax_radix_sector(entry); + dax.size = PAGE_SIZE << dax_radix_order(entry); spin_unlock_irq(&mapping->tree_lock); /* @@ -726,12 +853,11 @@ int dax_writeback_mapping_range(struct address_space *mapping, struct block_device *bdev, struct writeback_control *wbc) { struct inode *inode = mapping->host; - pgoff_t start_index, end_index, pmd_index; + pgoff_t start_index, end_index; pgoff_t indices[PAGEVEC_SIZE]; struct pagevec pvec; bool done = false; int i, ret = 0; - void *entry; if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT)) return -EIO; @@ -741,15 +867,6 @@ int dax_writeback_mapping_range(struct address_space *mapping, start_index = wbc->range_start >> PAGE_SHIFT; end_index = wbc->range_end >> PAGE_SHIFT; - pmd_index = DAX_PMD_INDEX(start_index); - - rcu_read_lock(); - entry = radix_tree_lookup(&mapping->page_tree, pmd_index); - rcu_read_unlock(); - - /* see if the start of our range is covered by a PMD entry */ - if (entry && RADIX_DAX_TYPE(entry) == RADIX_DAX_PMD) - start_index = pmd_index; tag_pages_for_writeback(mapping, start_index, end_index); @@ -794,7 +911,7 @@ static int dax_insert_mapping(struct address_space *mapping, return PTR_ERR(dax.addr); dax_unmap_atomic(bdev, &dax); - ret = dax_insert_mapping_entry(mapping, vmf, entry, dax.sector); + ret = dax_insert_mapping_entry(mapping, vmf, entry, dax.sector, 0); if (IS_ERR(ret)) return PTR_ERR(ret); *entryp = ret; @@ -841,7 +958,7 @@ int dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, bh.b_bdev = inode->i_sb->s_bdev; bh.b_size = PAGE_SIZE; - entry = grab_mapping_entry(mapping, vmf->pgoff); + entry = grab_mapping_entry(mapping, vmf->pgoff, 0); if (IS_ERR(entry)) { error = PTR_ERR(entry); goto out; @@ -1162,7 +1279,7 @@ int dax_iomap_fault(struct vm_area_struct *vma, struct vm_fault *vmf, if (pos >= i_size_read(inode)) return VM_FAULT_SIGBUS; - entry = grab_mapping_entry(mapping, vmf->pgoff); + entry = grab_mapping_entry(mapping, vmf->pgoff, 0); if (IS_ERR(entry)) { error = PTR_ERR(entry); goto out; @@ -1264,4 +1381,191 @@ int dax_iomap_fault(struct vm_area_struct *vma, struct vm_fault *vmf, return VM_FAULT_NOPAGE | major; } EXPORT_SYMBOL_GPL(dax_iomap_fault); + +#ifdef CONFIG_FS_DAX_PMD +/* + * The 'colour' (ie low bits) within a PMD of a page offset. This comes up + * more often than one might expect in the below functions. + */ +#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1) + +static int dax_pmd_insert_mapping(struct vm_area_struct *vma, pmd_t *pmd, + struct vm_fault *vmf, unsigned long address, + struct iomap *iomap, loff_t pos, bool write, void **entryp) +{ + struct address_space *mapping = vma->vm_file->f_mapping; + struct block_device *bdev = iomap->bdev; + struct blk_dax_ctl dax = { + .sector = dax_iomap_sector(iomap, pos), + .size = PMD_SIZE, + }; + long length = dax_map_atomic(bdev, &dax); + void *ret; + + if (length < 0) /* dax_map_atomic() failed */ + return VM_FAULT_FALLBACK; + if (length < PMD_SIZE) + goto unmap_fallback; + if (pfn_t_to_pfn(dax.pfn) & PG_PMD_COLOUR) + goto unmap_fallback; + if (!pfn_t_devmap(dax.pfn)) + goto unmap_fallback; + + dax_unmap_atomic(bdev, &dax); + + ret = dax_insert_mapping_entry(mapping, vmf, *entryp, dax.sector, + RADIX_DAX_PMD); + if (IS_ERR(ret)) + return VM_FAULT_FALLBACK; + *entryp = ret; + + return vmf_insert_pfn_pmd(vma, address, pmd, dax.pfn, write); + + unmap_fallback: + dax_unmap_atomic(bdev, &dax); + return VM_FAULT_FALLBACK; +} + +static int dax_pmd_load_hole(struct vm_area_struct *vma, pmd_t *pmd, + struct vm_fault *vmf, unsigned long address, + struct iomap *iomap, void **entryp) +{ + struct address_space *mapping = vma->vm_file->f_mapping; + unsigned long pmd_addr = address & PMD_MASK; + struct page *zero_page; + spinlock_t *ptl; + pmd_t pmd_entry; + void *ret; + + zero_page = mm_get_huge_zero_page(vma->vm_mm); + + if (unlikely(!zero_page)) + return VM_FAULT_FALLBACK; + + ret = dax_insert_mapping_entry(mapping, vmf, *entryp, 0, + RADIX_DAX_PMD | RADIX_DAX_HZP); + if (IS_ERR(ret)) + return VM_FAULT_FALLBACK; + *entryp = ret; + + ptl = pmd_lock(vma->vm_mm, pmd); + if (!pmd_none(*pmd)) { + spin_unlock(ptl); + return VM_FAULT_FALLBACK; + } + + pmd_entry = mk_pmd(zero_page, vma->vm_page_prot); + pmd_entry = pmd_mkhuge(pmd_entry); + set_pmd_at(vma->vm_mm, pmd_addr, pmd, pmd_entry); + spin_unlock(ptl); + return VM_FAULT_NOPAGE; +} + +int dax_iomap_pmd_fault(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmd, unsigned int flags, struct iomap_ops *ops) +{ + struct address_space *mapping = vma->vm_file->f_mapping; + unsigned long pmd_addr = address & PMD_MASK; + bool write = flags & FAULT_FLAG_WRITE; + unsigned int iomap_flags = write ? IOMAP_WRITE : 0; + struct inode *inode = mapping->host; + int result = VM_FAULT_FALLBACK; + struct iomap iomap = { 0 }; + pgoff_t max_pgoff, pgoff; + struct vm_fault vmf; + void *entry; + loff_t pos; + int error; + + /* Fall back to PTEs if we're going to COW */ + if (write && !(vma->vm_flags & VM_SHARED)) + goto fallback; + + /* If the PMD would extend outside the VMA */ + if (pmd_addr < vma->vm_start) + goto fallback; + if ((pmd_addr + PMD_SIZE) > vma->vm_end) + goto fallback; + + /* + * Check whether offset isn't beyond end of file now. Caller is + * supposed to hold locks serializing us with truncate / punch hole so + * this is a reliable test. + */ + pgoff = linear_page_index(vma, pmd_addr); + max_pgoff = (i_size_read(inode) - 1) >> PAGE_SHIFT; + + if (pgoff > max_pgoff) + return VM_FAULT_SIGBUS; + + /* If the PMD would extend beyond the file size */ + if ((pgoff | PG_PMD_COLOUR) > max_pgoff) + goto fallback; + + /* + * grab_mapping_entry() will make sure we get a 2M empty entry, a DAX + * PMD or a HZP entry. If it can't (because a 4k page is already in + * the tree, for instance), it will return -EEXIST and we just fall + * back to 4k entries. + */ + entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD); + if (IS_ERR(entry)) + goto fallback; + + /* + * Note that we don't use iomap_apply here. We aren't doing I/O, only + * setting up a mapping, so really we're using iomap_begin() as a way + * to look up our filesystem block. + */ + pos = (loff_t)pgoff << PAGE_SHIFT; + error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap); + if (error) + goto unlock_entry; + if (iomap.offset + iomap.length < pos + PMD_SIZE) + goto finish_iomap; + + vmf.pgoff = pgoff; + vmf.flags = flags; + vmf.gfp_mask = mapping_gfp_mask(mapping) | __GFP_IO; + + switch (iomap.type) { + case IOMAP_MAPPED: + result = dax_pmd_insert_mapping(vma, pmd, &vmf, address, + &iomap, pos, write, &entry); + break; + case IOMAP_UNWRITTEN: + case IOMAP_HOLE: + if (WARN_ON_ONCE(write)) + goto finish_iomap; + result = dax_pmd_load_hole(vma, pmd, &vmf, address, &iomap, + &entry); + break; + default: + WARN_ON_ONCE(1); + break; + } + + finish_iomap: + if (ops->iomap_end) { + if (result == VM_FAULT_FALLBACK) { + ops->iomap_end(inode, pos, PMD_SIZE, 0, iomap_flags, + &iomap); + } else { + error = ops->iomap_end(inode, pos, PMD_SIZE, PMD_SIZE, + iomap_flags, &iomap); + if (error) + result = VM_FAULT_FALLBACK; + } + } + unlock_entry: + put_locked_mapping_entry(mapping, pgoff, entry); + fallback: + if (result == VM_FAULT_FALLBACK) { + split_huge_pmd(vma, pmd, address); + count_vm_event(THP_FAULT_FALLBACK); + } + return result; +} +EXPORT_SYMBOL_GPL(dax_iomap_pmd_fault); +#endif /* CONFIG_FS_DAX_PMD */ #endif /* CONFIG_FS_IOMAP */ diff --git a/include/linux/dax.h b/include/linux/dax.h index e9ea78c1cf98..8d1a5c47945f 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -9,20 +9,32 @@ struct iomap_ops; /* - * We use lowest available bit in exceptional entry for locking, other two - * bits to determine entry type. In total 3 special bits. + * We use lowest available bit in exceptional entry for locking, one bit for + * the entry size (PMD) and two more to tell us if the entry is a huge zero + * page (HZP) or an empty entry that is just used for locking. In total four + * special bits. + * + * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the HZP and + * EMPTY bits aren't set the entry is a normal DAX entry with a filesystem + * block allocation. */ -#define RADIX_DAX_SHIFT (RADIX_TREE_EXCEPTIONAL_SHIFT + 3) +#define RADIX_DAX_SHIFT (RADIX_TREE_EXCEPTIONAL_SHIFT + 4) #define RADIX_DAX_ENTRY_LOCK (1 << RADIX_TREE_EXCEPTIONAL_SHIFT) -#define RADIX_DAX_PTE (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1)) -#define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2)) -#define RADIX_DAX_TYPE_MASK (RADIX_DAX_PTE | RADIX_DAX_PMD) -#define RADIX_DAX_TYPE(entry) ((unsigned long)entry & RADIX_DAX_TYPE_MASK) -#define RADIX_DAX_SECTOR(entry) (((unsigned long)entry >> RADIX_DAX_SHIFT)) -#define RADIX_DAX_ENTRY(sector, pmd) ((void *)((unsigned long)sector << \ - RADIX_DAX_SHIFT | (pmd ? RADIX_DAX_PMD : RADIX_DAX_PTE) | \ - RADIX_TREE_EXCEPTIONAL_ENTRY)) +#define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1)) +#define RADIX_DAX_HZP (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2)) +#define RADIX_DAX_EMPTY (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 3)) +static inline unsigned long dax_radix_sector(void *entry) +{ + return (unsigned long)entry >> RADIX_DAX_SHIFT; +} + +static inline void *dax_radix_locked_entry(sector_t sector, unsigned long flags) +{ + return (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | flags | + ((unsigned long)sector << RADIX_DAX_SHIFT) | + RADIX_DAX_ENTRY_LOCK); +} ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, struct iomap_ops *ops); @@ -67,6 +79,27 @@ static inline int dax_pmd_fault(struct vm_area_struct *vma, unsigned long addr, return VM_FAULT_FALLBACK; } +#ifdef CONFIG_FS_DAX_PMD +static inline unsigned int dax_radix_order(void *entry) +{ + if ((unsigned long)entry & RADIX_DAX_PMD) + return PMD_SHIFT - PAGE_SHIFT; + return 0; +} +int dax_iomap_pmd_fault(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmd, unsigned int flags, struct iomap_ops *ops); +#else +static inline unsigned int dax_radix_order(void *entry) +{ + return 0; +} +static inline int dax_iomap_pmd_fault(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmd, unsigned int flags, + struct iomap_ops *ops) +{ + return VM_FAULT_FALLBACK; +} +#endif int dax_pfn_mkwrite(struct vm_area_struct *, struct vm_fault *); #define dax_mkwrite(vma, vmf, gb) dax_fault(vma, vmf, gb) diff --git a/mm/filemap.c b/mm/filemap.c index 1ffb7dcd1b5d..00ab94a882de 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -137,8 +137,7 @@ static int page_cache_tree_insert(struct address_space *mapping, } else { /* DAX can replace empty locked entry with a hole */ WARN_ON_ONCE(p != - (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | - RADIX_DAX_ENTRY_LOCK)); + dax_radix_locked_entry(0, RADIX_DAX_EMPTY)); /* DAX accounts exceptional entries as normal pages */ if (node) workingset_node_pages_dec(node); -- cgit v1.2.3 From 9484ab1bf4464faae695321dd4fa66365beda74e Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Thu, 10 Nov 2016 10:26:50 +1100 Subject: dax: Introduce IOMAP_FAULT flag Introduce a flag telling iomap operations whether they are handling a fault or other IO. That may influence behavior wrt inode size and similar things. Signed-off-by: Jan Kara Reviewed-by: Dave Chinner Reviewed-by: Christoph Hellwig Signed-off-by: Dave Chinner --- fs/dax.c | 4 ++-- fs/iomap.c | 5 +++-- include/linux/iomap.h | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/fs/dax.c b/fs/dax.c index 281e91a63367..28af41b9da3a 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1266,7 +1266,7 @@ int dax_iomap_fault(struct vm_area_struct *vma, struct vm_fault *vmf, loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT; sector_t sector; struct iomap iomap = { 0 }; - unsigned flags = 0; + unsigned flags = IOMAP_FAULT; int error, major = 0; int locked_status = 0; void *entry; @@ -1467,7 +1467,7 @@ int dax_iomap_pmd_fault(struct vm_area_struct *vma, unsigned long address, struct address_space *mapping = vma->vm_file->f_mapping; unsigned long pmd_addr = address & PMD_MASK; bool write = flags & FAULT_FLAG_WRITE; - unsigned int iomap_flags = write ? IOMAP_WRITE : 0; + unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT; struct inode *inode = mapping->host; int result = VM_FAULT_FALLBACK; struct iomap iomap = { 0 }; diff --git a/fs/iomap.c b/fs/iomap.c index 013d1d36fbbf..51a02573405e 100644 --- a/fs/iomap.c +++ b/fs/iomap.c @@ -468,8 +468,9 @@ int iomap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf, offset = page_offset(page); while (length > 0) { - ret = iomap_apply(inode, offset, length, IOMAP_WRITE, - ops, page, iomap_page_mkwrite_actor); + ret = iomap_apply(inode, offset, length, + IOMAP_WRITE | IOMAP_FAULT, ops, page, + iomap_page_mkwrite_actor); if (unlikely(ret <= 0)) goto out_unlock; offset += ret; diff --git a/include/linux/iomap.h b/include/linux/iomap.h index e63e288dee83..b9e7b8ec8c1d 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -44,6 +44,7 @@ struct iomap { */ #define IOMAP_WRITE (1 << 0) #define IOMAP_ZERO (1 << 1) +#define IOMAP_FAULT (1 << 3) /* mapping for page fault */ struct iomap_ops { /* -- cgit v1.2.3 From 1c7dcf69eea3224474fe884a03b3e19b82d1101e Mon Sep 17 00:00:00 2001 From: David Gstir Date: Sun, 13 Nov 2016 22:20:44 +0100 Subject: fscrypt: Add in-place encryption mode ext4 and f2fs require a bounce page when encrypting pages. However, not all filesystems will need that (eg. UBIFS). This is handled via a flag on fscrypt_operations where a fs implementation can select in-place encryption over using a bounce page (which is the default). Signed-off-by: David Gstir Signed-off-by: Richard Weinberger Signed-off-by: Theodore Ts'o --- fs/crypto/crypto.c | 25 +++++++++++++++---------- include/linux/fscrypto.h | 6 ++++++ 2 files changed, 21 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index 98f87fe8f186..f38dc8aac2fe 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -217,8 +217,9 @@ static struct page *alloc_bounce_page(struct fscrypt_ctx *ctx, gfp_t gfp_flags) * @plaintext_page: The page to encrypt. Must be locked. * @gfp_flags: The gfp flag for memory allocation * - * Allocates a ciphertext page and encrypts plaintext_page into it using the ctx - * encryption context. + * Encrypts plaintext_page using the ctx encryption context. If + * the filesystem supports it, encryption is performed in-place, otherwise a + * new ciphertext_page is allocated and returned. * * Called on the page write path. The caller must call * fscrypt_restore_control_page() on the returned ciphertext page to @@ -231,7 +232,7 @@ struct page *fscrypt_encrypt_page(struct inode *inode, struct page *plaintext_page, gfp_t gfp_flags) { struct fscrypt_ctx *ctx; - struct page *ciphertext_page = NULL; + struct page *ciphertext_page = plaintext_page; int err; BUG_ON(!PageLocked(plaintext_page)); @@ -240,10 +241,12 @@ struct page *fscrypt_encrypt_page(struct inode *inode, if (IS_ERR(ctx)) return (struct page *)ctx; - /* The encryption operation will require a bounce page. */ - ciphertext_page = alloc_bounce_page(ctx, gfp_flags); - if (IS_ERR(ciphertext_page)) - goto errout; + if (!(inode->i_sb->s_cop->flags & FS_CFLG_INPLACE_ENCRYPTION)) { + /* The encryption operation will require a bounce page. */ + ciphertext_page = alloc_bounce_page(ctx, gfp_flags); + if (IS_ERR(ciphertext_page)) + goto errout; + } ctx->w.control_page = plaintext_page; err = do_page_crypto(inode, FS_ENCRYPT, plaintext_page->index, @@ -253,9 +256,11 @@ struct page *fscrypt_encrypt_page(struct inode *inode, ciphertext_page = ERR_PTR(err); goto errout; } - SetPagePrivate(ciphertext_page); - set_page_private(ciphertext_page, (unsigned long)ctx); - lock_page(ciphertext_page); + if (!(inode->i_sb->s_cop->flags & FS_CFLG_INPLACE_ENCRYPTION)) { + SetPagePrivate(ciphertext_page); + set_page_private(ciphertext_page, (unsigned long)ctx); + lock_page(ciphertext_page); + } return ciphertext_page; errout: diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index ff8b11b26f31..5a65b0e3773f 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -153,10 +153,16 @@ struct fscrypt_name { #define fname_name(p) ((p)->disk_name.name) #define fname_len(p) ((p)->disk_name.len) +/* + * fscrypt superblock flags + */ +#define FS_CFLG_INPLACE_ENCRYPTION (1U << 1) + /* * crypto opertions for filesystems */ struct fscrypt_operations { + unsigned int flags; int (*get_context)(struct inode *, void *, size_t); int (*key_prefix)(struct inode *, u8 **); int (*prepare_context)(struct inode *); -- cgit v1.2.3 From b50f7b268bad8088dfe7579a65cd910d8cc5c40f Mon Sep 17 00:00:00 2001 From: David Gstir Date: Sun, 13 Nov 2016 22:20:45 +0100 Subject: fscrypt: Allow fscrypt_decrypt_page() to function with non-writeback pages Some filesystem might pass pages which do not have page->mapping->host set to the encrypted inode. We want the caller to explicitly pass the corresponding inode. Signed-off-by: David Gstir Signed-off-by: Richard Weinberger Signed-off-by: Theodore Ts'o --- fs/crypto/crypto.c | 11 ++++++----- fs/ext4/inode.c | 5 +++-- include/linux/fscrypto.h | 5 +++-- 3 files changed, 12 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index f38dc8aac2fe..222a70520565 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -271,7 +271,8 @@ EXPORT_SYMBOL(fscrypt_encrypt_page); /** * f2crypt_decrypt_page() - Decrypts a page in-place - * @page: The page to decrypt. Must be locked. + * @inode: The encrypted inode to decrypt. + * @page: The page to decrypt. Must be locked. * * Decrypts page in-place using the ctx encryption context. * @@ -279,12 +280,12 @@ EXPORT_SYMBOL(fscrypt_encrypt_page); * * Return: Zero on success, non-zero otherwise. */ -int fscrypt_decrypt_page(struct page *page) +int fscrypt_decrypt_page(struct inode *inode, struct page *page) { BUG_ON(!PageLocked(page)); - return do_page_crypto(page->mapping->host, - FS_DECRYPT, page->index, page, page, GFP_NOFS); + return do_page_crypto(inode, FS_DECRYPT, page->index, page, page, + GFP_NOFS); } EXPORT_SYMBOL(fscrypt_decrypt_page); @@ -419,7 +420,7 @@ static void completion_pages(struct work_struct *work) bio_for_each_segment_all(bv, bio, i) { struct page *page = bv->bv_page; - int ret = fscrypt_decrypt_page(page); + int ret = fscrypt_decrypt_page(page->mapping->host, page); if (ret) { WARN_ON_ONCE(1); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 9c064727ed62..4b7b842ec024 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -1166,7 +1166,7 @@ static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len, if (unlikely(err)) page_zero_new_buffers(page, from, to); else if (decrypt) - err = fscrypt_decrypt_page(page); + err = fscrypt_decrypt_page(page->mapping->host, page); return err; } #endif @@ -3743,7 +3743,8 @@ static int __ext4_block_zero_page_range(handle_t *handle, /* We expect the key to be set. */ BUG_ON(!fscrypt_has_encryption_key(inode)); BUG_ON(blocksize != PAGE_SIZE); - WARN_ON_ONCE(fscrypt_decrypt_page(page)); + WARN_ON_ONCE(fscrypt_decrypt_page(page->mapping->host, + page)); } } if (ext4_should_journal_data(inode)) { diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index 5a65b0e3773f..8be8e9657c63 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -249,7 +249,7 @@ int fscrypt_initialize(void); extern struct fscrypt_ctx *fscrypt_get_ctx(struct inode *, gfp_t); extern void fscrypt_release_ctx(struct fscrypt_ctx *); extern struct page *fscrypt_encrypt_page(struct inode *, struct page *, gfp_t); -extern int fscrypt_decrypt_page(struct page *); +extern int fscrypt_decrypt_page(struct inode *, struct page *); extern void fscrypt_decrypt_bio_pages(struct fscrypt_ctx *, struct bio *); extern void fscrypt_pullback_bio_page(struct page **, bool); extern void fscrypt_restore_control_page(struct page *); @@ -298,7 +298,8 @@ static inline struct page *fscrypt_notsupp_encrypt_page(struct inode *i, return ERR_PTR(-EOPNOTSUPP); } -static inline int fscrypt_notsupp_decrypt_page(struct page *p) +static inline int fscrypt_notsupp_decrypt_page(struct inode *i, + struct page *p) { return -EOPNOTSUPP; } -- cgit v1.2.3 From 7821d4dd4589ce5af54f3e46d04a29439ba3c2e5 Mon Sep 17 00:00:00 2001 From: David Gstir Date: Sun, 13 Nov 2016 22:20:46 +0100 Subject: fscrypt: Enable partial page encryption Not all filesystems work on full pages, thus we should allow them to hand partial pages to fscrypt for en/decryption. Signed-off-by: David Gstir Signed-off-by: Richard Weinberger Signed-off-by: Theodore Ts'o --- fs/crypto/crypto.c | 42 ++++++++++++++++++++++++++---------------- fs/ext4/inode.c | 6 ++++-- fs/ext4/page-io.c | 2 +- fs/f2fs/data.c | 2 ++ include/linux/fscrypto.h | 16 +++++++++++----- 5 files changed, 44 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index 222a70520565..e170aa05011d 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -149,6 +149,7 @@ typedef enum { static int do_page_crypto(struct inode *inode, fscrypt_direction_t rw, pgoff_t index, struct page *src_page, struct page *dest_page, + unsigned int src_len, unsigned int src_offset, gfp_t gfp_flags) { struct { @@ -179,10 +180,10 @@ static int do_page_crypto(struct inode *inode, memset(xts_tweak.padding, 0, sizeof(xts_tweak.padding)); sg_init_table(&dst, 1); - sg_set_page(&dst, dest_page, PAGE_SIZE, 0); + sg_set_page(&dst, dest_page, src_len, src_offset); sg_init_table(&src, 1); - sg_set_page(&src, src_page, PAGE_SIZE, 0); - skcipher_request_set_crypt(req, &src, &dst, PAGE_SIZE, &xts_tweak); + sg_set_page(&src, src_page, src_len, src_offset); + skcipher_request_set_crypt(req, &src, &dst, src_len, &xts_tweak); if (rw == FS_DECRYPT) res = crypto_skcipher_decrypt(req); else @@ -213,9 +214,11 @@ static struct page *alloc_bounce_page(struct fscrypt_ctx *ctx, gfp_t gfp_flags) /** * fscypt_encrypt_page() - Encrypts a page - * @inode: The inode for which the encryption should take place - * @plaintext_page: The page to encrypt. Must be locked. - * @gfp_flags: The gfp flag for memory allocation + * @inode: The inode for which the encryption should take place + * @plaintext_page: The page to encrypt. Must be locked. + * @plaintext_len: Length of plaintext within page + * @plaintext_offset: Offset of plaintext within page + * @gfp_flags: The gfp flag for memory allocation * * Encrypts plaintext_page using the ctx encryption context. If * the filesystem supports it, encryption is performed in-place, otherwise a @@ -229,13 +232,17 @@ static struct page *alloc_bounce_page(struct fscrypt_ctx *ctx, gfp_t gfp_flags) * error value or NULL. */ struct page *fscrypt_encrypt_page(struct inode *inode, - struct page *plaintext_page, gfp_t gfp_flags) + struct page *plaintext_page, + unsigned int plaintext_len, + unsigned int plaintext_offset, + gfp_t gfp_flags) + { struct fscrypt_ctx *ctx; struct page *ciphertext_page = plaintext_page; int err; - BUG_ON(!PageLocked(plaintext_page)); + BUG_ON(plaintext_len % FS_CRYPTO_BLOCK_SIZE != 0); ctx = fscrypt_get_ctx(inode, gfp_flags); if (IS_ERR(ctx)) @@ -251,6 +258,7 @@ struct page *fscrypt_encrypt_page(struct inode *inode, ctx->w.control_page = plaintext_page; err = do_page_crypto(inode, FS_ENCRYPT, plaintext_page->index, plaintext_page, ciphertext_page, + plaintext_len, plaintext_offset, gfp_flags); if (err) { ciphertext_page = ERR_PTR(err); @@ -270,9 +278,11 @@ errout: EXPORT_SYMBOL(fscrypt_encrypt_page); /** - * f2crypt_decrypt_page() - Decrypts a page in-place - * @inode: The encrypted inode to decrypt. + * fscrypt_decrypt_page() - Decrypts a page in-place + * @inode: Encrypted inode to decrypt. * @page: The page to decrypt. Must be locked. + * @len: Number of bytes in @page to be decrypted. + * @offs: Start of data in @page. * * Decrypts page in-place using the ctx encryption context. * @@ -280,11 +290,10 @@ EXPORT_SYMBOL(fscrypt_encrypt_page); * * Return: Zero on success, non-zero otherwise. */ -int fscrypt_decrypt_page(struct inode *inode, struct page *page) +int fscrypt_decrypt_page(struct inode *inode, struct page *page, + unsigned int len, unsigned int offs) { - BUG_ON(!PageLocked(page)); - - return do_page_crypto(inode, FS_DECRYPT, page->index, page, page, + return do_page_crypto(inode, FS_DECRYPT, page->index, page, page, len, offs, GFP_NOFS); } EXPORT_SYMBOL(fscrypt_decrypt_page); @@ -312,7 +321,7 @@ int fscrypt_zeroout_range(struct inode *inode, pgoff_t lblk, while (len--) { err = do_page_crypto(inode, FS_ENCRYPT, lblk, ZERO_PAGE(0), ciphertext_page, - GFP_NOFS); + PAGE_SIZE, 0, GFP_NOFS); if (err) goto errout; @@ -420,7 +429,8 @@ static void completion_pages(struct work_struct *work) bio_for_each_segment_all(bv, bio, i) { struct page *page = bv->bv_page; - int ret = fscrypt_decrypt_page(page->mapping->host, page); + int ret = fscrypt_decrypt_page(page->mapping->host, page, + PAGE_SIZE, 0); if (ret) { WARN_ON_ONCE(1); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 4b7b842ec024..1d498c5e2990 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -1166,7 +1166,8 @@ static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len, if (unlikely(err)) page_zero_new_buffers(page, from, to); else if (decrypt) - err = fscrypt_decrypt_page(page->mapping->host, page); + err = fscrypt_decrypt_page(page->mapping->host, page, + PAGE_SIZE, 0); return err; } #endif @@ -3743,8 +3744,9 @@ static int __ext4_block_zero_page_range(handle_t *handle, /* We expect the key to be set. */ BUG_ON(!fscrypt_has_encryption_key(inode)); BUG_ON(blocksize != PAGE_SIZE); + BUG_ON(!PageLocked(page)); WARN_ON_ONCE(fscrypt_decrypt_page(page->mapping->host, - page)); + page, PAGE_SIZE, 0)); } } if (ext4_should_journal_data(inode)) { diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index 0094923e5ebf..3d1d3d0f4303 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -470,7 +470,7 @@ int ext4_bio_write_page(struct ext4_io_submit *io, gfp_t gfp_flags = GFP_NOFS; retry_encrypt: - data_page = fscrypt_encrypt_page(inode, page, gfp_flags); + data_page = fscrypt_encrypt_page(inode, page, PAGE_SIZE, 0, gfp_flags); if (IS_ERR(data_page)) { ret = PTR_ERR(data_page); if (ret == -ENOMEM && wbc->sync_mode == WB_SYNC_ALL) { diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 9ae194fd2fdb..fac207254e8d 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -1194,7 +1194,9 @@ int do_write_data_page(struct f2fs_io_info *fio) f2fs_wait_on_encrypted_page_writeback(F2FS_I_SB(inode), fio->old_blkaddr); retry_encrypt: + BUG_ON(!PageLocked(fio->page)); fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page, + PAGE_SIZE, 0, gfp_flags); if (IS_ERR(fio->encrypted_page)) { err = PTR_ERR(fio->encrypted_page); diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index 8be8e9657c63..e05931272cea 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -248,8 +248,11 @@ int fscrypt_initialize(void); extern struct fscrypt_ctx *fscrypt_get_ctx(struct inode *, gfp_t); extern void fscrypt_release_ctx(struct fscrypt_ctx *); -extern struct page *fscrypt_encrypt_page(struct inode *, struct page *, gfp_t); -extern int fscrypt_decrypt_page(struct inode *, struct page *); +extern struct page *fscrypt_encrypt_page(struct inode *, struct page *, + unsigned int, unsigned int, + gfp_t); +extern int fscrypt_decrypt_page(struct inode *, struct page *, unsigned int, + unsigned int); extern void fscrypt_decrypt_bio_pages(struct fscrypt_ctx *, struct bio *); extern void fscrypt_pullback_bio_page(struct page **, bool); extern void fscrypt_restore_control_page(struct page *); @@ -293,13 +296,16 @@ static inline void fscrypt_notsupp_release_ctx(struct fscrypt_ctx *c) } static inline struct page *fscrypt_notsupp_encrypt_page(struct inode *i, - struct page *p, gfp_t f) + struct page *p, + unsigned int len, + unsigned int offs, + gfp_t f) { return ERR_PTR(-EOPNOTSUPP); } -static inline int fscrypt_notsupp_decrypt_page(struct inode *i, - struct page *p) +static inline int fscrypt_notsupp_decrypt_page(struct inode *i, struct page *p, + unsigned int len, unsigned int offs) { return -EOPNOTSUPP; } -- cgit v1.2.3 From 0b93e1b94b86f826d18a2aaf219a53e271274d49 Mon Sep 17 00:00:00 2001 From: David Gstir Date: Sun, 13 Nov 2016 22:20:47 +0100 Subject: fscrypt: Constify struct inode pointer Some filesystems, such as UBIFS, maintain a const pointer for struct inode. Signed-off-by: David Gstir Signed-off-by: Richard Weinberger Signed-off-by: Theodore Ts'o --- fs/crypto/crypto.c | 10 +++++----- fs/crypto/fname.c | 4 ++-- include/linux/fscrypto.h | 22 +++++++++++----------- 3 files changed, 18 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index e170aa05011d..f5c5e84ea9db 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -88,7 +88,7 @@ EXPORT_SYMBOL(fscrypt_release_ctx); * Return: An allocated and initialized encryption context on success; error * value or NULL otherwise. */ -struct fscrypt_ctx *fscrypt_get_ctx(struct inode *inode, gfp_t gfp_flags) +struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *inode, gfp_t gfp_flags) { struct fscrypt_ctx *ctx = NULL; struct fscrypt_info *ci = inode->i_crypt_info; @@ -146,7 +146,7 @@ typedef enum { FS_ENCRYPT, } fscrypt_direction_t; -static int do_page_crypto(struct inode *inode, +static int do_page_crypto(const struct inode *inode, fscrypt_direction_t rw, pgoff_t index, struct page *src_page, struct page *dest_page, unsigned int src_len, unsigned int src_offset, @@ -231,7 +231,7 @@ static struct page *alloc_bounce_page(struct fscrypt_ctx *ctx, gfp_t gfp_flags) * Return: An allocated page with the encrypted content on success. Else, an * error value or NULL. */ -struct page *fscrypt_encrypt_page(struct inode *inode, +struct page *fscrypt_encrypt_page(const struct inode *inode, struct page *plaintext_page, unsigned int plaintext_len, unsigned int plaintext_offset, @@ -290,7 +290,7 @@ EXPORT_SYMBOL(fscrypt_encrypt_page); * * Return: Zero on success, non-zero otherwise. */ -int fscrypt_decrypt_page(struct inode *inode, struct page *page, +int fscrypt_decrypt_page(const struct inode *inode, struct page *page, unsigned int len, unsigned int offs) { return do_page_crypto(inode, FS_DECRYPT, page->index, page, page, len, offs, @@ -298,7 +298,7 @@ int fscrypt_decrypt_page(struct inode *inode, struct page *page, } EXPORT_SYMBOL(fscrypt_decrypt_page); -int fscrypt_zeroout_range(struct inode *inode, pgoff_t lblk, +int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, sector_t pblk, unsigned int len) { struct fscrypt_ctx *ctx; diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c index 9a28133ac3b8..faeaf0be9400 100644 --- a/fs/crypto/fname.c +++ b/fs/crypto/fname.c @@ -220,7 +220,7 @@ static int digest_decode(const char *src, int len, char *dst) return cp - dst; } -u32 fscrypt_fname_encrypted_size(struct inode *inode, u32 ilen) +u32 fscrypt_fname_encrypted_size(const struct inode *inode, u32 ilen) { int padding = 32; struct fscrypt_info *ci = inode->i_crypt_info; @@ -238,7 +238,7 @@ EXPORT_SYMBOL(fscrypt_fname_encrypted_size); * Allocates an output buffer that is sufficient for the crypto operation * specified by the context and the direction. */ -int fscrypt_fname_alloc_buffer(struct inode *inode, +int fscrypt_fname_alloc_buffer(const struct inode *inode, u32 ilen, struct fscrypt_str *crypto_str) { unsigned int olen = fscrypt_fname_encrypted_size(inode, ilen); diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index e05931272cea..e9be944a324c 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -212,7 +212,7 @@ static inline struct page *fscrypt_control_page(struct page *page) #endif } -static inline int fscrypt_has_encryption_key(struct inode *inode) +static inline int fscrypt_has_encryption_key(const struct inode *inode) { #if IS_ENABLED(CONFIG_FS_ENCRYPTION) return (inode->i_crypt_info != NULL); @@ -246,17 +246,17 @@ static inline void fscrypt_set_d_op(struct dentry *dentry) extern struct kmem_cache *fscrypt_info_cachep; int fscrypt_initialize(void); -extern struct fscrypt_ctx *fscrypt_get_ctx(struct inode *, gfp_t); +extern struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *, gfp_t); extern void fscrypt_release_ctx(struct fscrypt_ctx *); -extern struct page *fscrypt_encrypt_page(struct inode *, struct page *, +extern struct page *fscrypt_encrypt_page(const struct inode *, struct page *, unsigned int, unsigned int, gfp_t); -extern int fscrypt_decrypt_page(struct inode *, struct page *, unsigned int, +extern int fscrypt_decrypt_page(const struct inode *, struct page *, unsigned int, unsigned int); extern void fscrypt_decrypt_bio_pages(struct fscrypt_ctx *, struct bio *); extern void fscrypt_pullback_bio_page(struct page **, bool); extern void fscrypt_restore_control_page(struct page *); -extern int fscrypt_zeroout_range(struct inode *, pgoff_t, sector_t, +extern int fscrypt_zeroout_range(const struct inode *, pgoff_t, sector_t, unsigned int); /* policy.c */ extern int fscrypt_process_policy(struct file *, const struct fscrypt_policy *); @@ -273,8 +273,8 @@ extern void fscrypt_put_encryption_info(struct inode *, struct fscrypt_info *); extern int fscrypt_setup_filename(struct inode *, const struct qstr *, int lookup, struct fscrypt_name *); extern void fscrypt_free_filename(struct fscrypt_name *); -extern u32 fscrypt_fname_encrypted_size(struct inode *, u32); -extern int fscrypt_fname_alloc_buffer(struct inode *, u32, +extern u32 fscrypt_fname_encrypted_size(const struct inode *, u32); +extern int fscrypt_fname_alloc_buffer(const struct inode *, u32, struct fscrypt_str *); extern void fscrypt_fname_free_buffer(struct fscrypt_str *); extern int fscrypt_fname_disk_to_usr(struct inode *, u32, u32, @@ -284,7 +284,7 @@ extern int fscrypt_fname_usr_to_disk(struct inode *, const struct qstr *, #endif /* crypto.c */ -static inline struct fscrypt_ctx *fscrypt_notsupp_get_ctx(struct inode *i, +static inline struct fscrypt_ctx *fscrypt_notsupp_get_ctx(const struct inode *i, gfp_t f) { return ERR_PTR(-EOPNOTSUPP); @@ -295,7 +295,7 @@ static inline void fscrypt_notsupp_release_ctx(struct fscrypt_ctx *c) return; } -static inline struct page *fscrypt_notsupp_encrypt_page(struct inode *i, +static inline struct page *fscrypt_notsupp_encrypt_page(const struct inode *i, struct page *p, unsigned int len, unsigned int offs, @@ -304,7 +304,7 @@ static inline struct page *fscrypt_notsupp_encrypt_page(struct inode *i, return ERR_PTR(-EOPNOTSUPP); } -static inline int fscrypt_notsupp_decrypt_page(struct inode *i, struct page *p, +static inline int fscrypt_notsupp_decrypt_page(const struct inode *i, struct page *p, unsigned int len, unsigned int offs) { return -EOPNOTSUPP; @@ -326,7 +326,7 @@ static inline void fscrypt_notsupp_restore_control_page(struct page *p) return; } -static inline int fscrypt_notsupp_zeroout_range(struct inode *i, pgoff_t p, +static inline int fscrypt_notsupp_zeroout_range(const struct inode *i, pgoff_t p, sector_t s, unsigned int f) { return -EOPNOTSUPP; -- cgit v1.2.3 From 9c4bb8a3a9b4de21753053d667310c2b7cb39916 Mon Sep 17 00:00:00 2001 From: David Gstir Date: Sun, 13 Nov 2016 22:20:48 +0100 Subject: fscrypt: Let fs select encryption index/tweak Avoid re-use of page index as tweak for AES-XTS when multiple parts of same page are encrypted. This will happen on multiple (partial) calls of fscrypt_encrypt_page on same page. page->index is only valid for writeback pages. Signed-off-by: David Gstir Signed-off-by: Richard Weinberger Signed-off-by: Theodore Ts'o --- fs/crypto/crypto.c | 11 +++++++---- fs/ext4/inode.c | 4 ++-- fs/ext4/page-io.c | 3 ++- fs/f2fs/data.c | 5 +++-- include/linux/fscrypto.h | 9 +++++---- 5 files changed, 19 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index f5c5e84ea9db..b6029785714c 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -218,6 +218,8 @@ static struct page *alloc_bounce_page(struct fscrypt_ctx *ctx, gfp_t gfp_flags) * @plaintext_page: The page to encrypt. Must be locked. * @plaintext_len: Length of plaintext within page * @plaintext_offset: Offset of plaintext within page + * @index: Index for encryption. This is mainly the page index, but + * but might be different for multiple calls on same page. * @gfp_flags: The gfp flag for memory allocation * * Encrypts plaintext_page using the ctx encryption context. If @@ -235,7 +237,7 @@ struct page *fscrypt_encrypt_page(const struct inode *inode, struct page *plaintext_page, unsigned int plaintext_len, unsigned int plaintext_offset, - gfp_t gfp_flags) + pgoff_t index, gfp_t gfp_flags) { struct fscrypt_ctx *ctx; @@ -256,7 +258,7 @@ struct page *fscrypt_encrypt_page(const struct inode *inode, } ctx->w.control_page = plaintext_page; - err = do_page_crypto(inode, FS_ENCRYPT, plaintext_page->index, + err = do_page_crypto(inode, FS_ENCRYPT, index, plaintext_page, ciphertext_page, plaintext_len, plaintext_offset, gfp_flags); @@ -283,6 +285,7 @@ EXPORT_SYMBOL(fscrypt_encrypt_page); * @page: The page to decrypt. Must be locked. * @len: Number of bytes in @page to be decrypted. * @offs: Start of data in @page. + * @index: Index for encryption. * * Decrypts page in-place using the ctx encryption context. * @@ -291,7 +294,7 @@ EXPORT_SYMBOL(fscrypt_encrypt_page); * Return: Zero on success, non-zero otherwise. */ int fscrypt_decrypt_page(const struct inode *inode, struct page *page, - unsigned int len, unsigned int offs) + unsigned int len, unsigned int offs, pgoff_t index) { return do_page_crypto(inode, FS_DECRYPT, page->index, page, page, len, offs, GFP_NOFS); @@ -430,7 +433,7 @@ static void completion_pages(struct work_struct *work) bio_for_each_segment_all(bv, bio, i) { struct page *page = bv->bv_page; int ret = fscrypt_decrypt_page(page->mapping->host, page, - PAGE_SIZE, 0); + PAGE_SIZE, 0, page->index); if (ret) { WARN_ON_ONCE(1); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 1d498c5e2990..1485ac273bfb 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -1167,7 +1167,7 @@ static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len, page_zero_new_buffers(page, from, to); else if (decrypt) err = fscrypt_decrypt_page(page->mapping->host, page, - PAGE_SIZE, 0); + PAGE_SIZE, 0, page->index); return err; } #endif @@ -3746,7 +3746,7 @@ static int __ext4_block_zero_page_range(handle_t *handle, BUG_ON(blocksize != PAGE_SIZE); BUG_ON(!PageLocked(page)); WARN_ON_ONCE(fscrypt_decrypt_page(page->mapping->host, - page, PAGE_SIZE, 0)); + page, PAGE_SIZE, 0, page->index)); } } if (ext4_should_journal_data(inode)) { diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index 3d1d3d0f4303..902a3e3059b3 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -470,7 +470,8 @@ int ext4_bio_write_page(struct ext4_io_submit *io, gfp_t gfp_flags = GFP_NOFS; retry_encrypt: - data_page = fscrypt_encrypt_page(inode, page, PAGE_SIZE, 0, gfp_flags); + data_page = fscrypt_encrypt_page(inode, page, PAGE_SIZE, 0, + page->index, gfp_flags); if (IS_ERR(data_page)) { ret = PTR_ERR(data_page); if (ret == -ENOMEM && wbc->sync_mode == WB_SYNC_ALL) { diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index fac207254e8d..435590c4b341 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -1196,8 +1196,9 @@ int do_write_data_page(struct f2fs_io_info *fio) retry_encrypt: BUG_ON(!PageLocked(fio->page)); fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page, - PAGE_SIZE, 0, - gfp_flags); + PAGE_SIZE, 0, + fio->page->index, + gfp_flags); if (IS_ERR(fio->encrypted_page)) { err = PTR_ERR(fio->encrypted_page); if (err == -ENOMEM) { diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index e9be944a324c..98c71e973a96 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -250,9 +250,9 @@ extern struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *, gfp_t); extern void fscrypt_release_ctx(struct fscrypt_ctx *); extern struct page *fscrypt_encrypt_page(const struct inode *, struct page *, unsigned int, unsigned int, - gfp_t); + pgoff_t, gfp_t); extern int fscrypt_decrypt_page(const struct inode *, struct page *, unsigned int, - unsigned int); + unsigned int, pgoff_t); extern void fscrypt_decrypt_bio_pages(struct fscrypt_ctx *, struct bio *); extern void fscrypt_pullback_bio_page(struct page **, bool); extern void fscrypt_restore_control_page(struct page *); @@ -299,13 +299,14 @@ static inline struct page *fscrypt_notsupp_encrypt_page(const struct inode *i, struct page *p, unsigned int len, unsigned int offs, - gfp_t f) + pgoff_t index, gfp_t f) { return ERR_PTR(-EOPNOTSUPP); } static inline int fscrypt_notsupp_decrypt_page(const struct inode *i, struct page *p, - unsigned int len, unsigned int offs) + unsigned int len, unsigned int offs, + pgoff_t index) { return -EOPNOTSUPP; } -- cgit v1.2.3 From dd936e4313fa3f60abd6e67abb3cb66fc9a018d1 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Sun, 20 Nov 2016 20:48:36 -0500 Subject: dax: rip out get_block based IO support No one uses functions using the get_block callback anymore. Rip them out and update documentation. Reviewed-by: Ross Zwisler Signed-off-by: Jan Kara Signed-off-by: Theodore Ts'o --- Documentation/filesystems/dax.txt | 22 +-- fs/dax.c | 315 -------------------------------------- include/linux/dax.h | 12 -- 3 files changed, 11 insertions(+), 338 deletions(-) (limited to 'include/linux') diff --git a/Documentation/filesystems/dax.txt b/Documentation/filesystems/dax.txt index 23d18b8a49d5..a7e6e14aeb08 100644 --- a/Documentation/filesystems/dax.txt +++ b/Documentation/filesystems/dax.txt @@ -58,22 +58,22 @@ Implementation Tips for Filesystem Writers Filesystem support consists of - adding support to mark inodes as being DAX by setting the S_DAX flag in i_flags -- implementing the direct_IO address space operation, and calling - dax_do_io() instead of blockdev_direct_IO() if S_DAX is set +- implementing ->read_iter and ->write_iter operations which use dax_iomap_rw() + when inode has S_DAX flag set - implementing an mmap file operation for DAX files which sets the VM_MIXEDMAP and VM_HUGEPAGE flags on the VMA, and setting the vm_ops to - include handlers for fault, pmd_fault and page_mkwrite (which should - probably call dax_fault(), dax_pmd_fault() and dax_mkwrite(), passing the - appropriate get_block() callback) -- calling dax_truncate_page() instead of block_truncate_page() for DAX files -- calling dax_zero_page_range() instead of zero_user() for DAX files + include handlers for fault, pmd_fault, page_mkwrite, pfn_mkwrite. These + handlers should probably call dax_iomap_fault() (for fault and page_mkwrite + handlers), dax_iomap_pmd_fault(), dax_pfn_mkwrite() passing the appropriate + iomap operations. +- calling iomap_zero_range() passing appropriate iomap operations instead of + block_truncate_page() for DAX files - ensuring that there is sufficient locking between reads, writes, truncates and page faults -The get_block() callback passed to the DAX functions may return -uninitialised extents. If it does, it must ensure that simultaneous -calls to get_block() (for example by a page-fault racing with a read() -or a write()) work correctly. +The iomap handlers for allocating blocks must make sure that allocated blocks +are zeroed out and converted to written extents before being returned to avoid +exposure of uninitialized data through mmap. These filesystems may be used for inspiration: - ext2: see Documentation/filesystems/ext2.txt diff --git a/fs/dax.c b/fs/dax.c index 28af41b9da3a..ad131cd2605d 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -116,168 +116,6 @@ struct page *read_dax_sector(struct block_device *bdev, sector_t n) return page; } -static bool buffer_written(struct buffer_head *bh) -{ - return buffer_mapped(bh) && !buffer_unwritten(bh); -} - -static sector_t to_sector(const struct buffer_head *bh, - const struct inode *inode) -{ - sector_t sector = bh->b_blocknr << (inode->i_blkbits - 9); - - return sector; -} - -static ssize_t dax_io(struct inode *inode, struct iov_iter *iter, - loff_t start, loff_t end, get_block_t get_block, - struct buffer_head *bh) -{ - loff_t pos = start, max = start, bh_max = start; - bool hole = false; - struct block_device *bdev = NULL; - int rw = iov_iter_rw(iter), rc; - long map_len = 0; - struct blk_dax_ctl dax = { - .addr = ERR_PTR(-EIO), - }; - unsigned blkbits = inode->i_blkbits; - sector_t file_blks = (i_size_read(inode) + (1 << blkbits) - 1) - >> blkbits; - - if (rw == READ) - end = min(end, i_size_read(inode)); - - while (pos < end) { - size_t len; - if (pos == max) { - long page = pos >> PAGE_SHIFT; - sector_t block = page << (PAGE_SHIFT - blkbits); - unsigned first = pos - (block << blkbits); - long size; - - if (pos == bh_max) { - bh->b_size = PAGE_ALIGN(end - pos); - bh->b_state = 0; - rc = get_block(inode, block, bh, rw == WRITE); - if (rc) - break; - bh_max = pos - first + bh->b_size; - bdev = bh->b_bdev; - /* - * We allow uninitialized buffers for writes - * beyond EOF as those cannot race with faults - */ - WARN_ON_ONCE( - (buffer_new(bh) && block < file_blks) || - (rw == WRITE && buffer_unwritten(bh))); - } else { - unsigned done = bh->b_size - - (bh_max - (pos - first)); - bh->b_blocknr += done >> blkbits; - bh->b_size -= done; - } - - hole = rw == READ && !buffer_written(bh); - if (hole) { - size = bh->b_size - first; - } else { - dax_unmap_atomic(bdev, &dax); - dax.sector = to_sector(bh, inode); - dax.size = bh->b_size; - map_len = dax_map_atomic(bdev, &dax); - if (map_len < 0) { - rc = map_len; - break; - } - dax.addr += first; - size = map_len - first; - } - /* - * pos + size is one past the last offset for IO, - * so pos + size can overflow loff_t at extreme offsets. - * Cast to u64 to catch this and get the true minimum. - */ - max = min_t(u64, pos + size, end); - } - - if (iov_iter_rw(iter) == WRITE) { - len = copy_from_iter_pmem(dax.addr, max - pos, iter); - } else if (!hole) - len = copy_to_iter((void __force *) dax.addr, max - pos, - iter); - else - len = iov_iter_zero(max - pos, iter); - - if (!len) { - rc = -EFAULT; - break; - } - - pos += len; - if (!IS_ERR(dax.addr)) - dax.addr += len; - } - - dax_unmap_atomic(bdev, &dax); - - return (pos == start) ? rc : pos - start; -} - -/** - * dax_do_io - Perform I/O to a DAX file - * @iocb: The control block for this I/O - * @inode: The file which the I/O is directed at - * @iter: The addresses to do I/O from or to - * @get_block: The filesystem method used to translate file offsets to blocks - * @end_io: A filesystem callback for I/O completion - * @flags: See below - * - * This function uses the same locking scheme as do_blockdev_direct_IO: - * If @flags has DIO_LOCKING set, we assume that the i_mutex is held by the - * caller for writes. For reads, we take and release the i_mutex ourselves. - * If DIO_LOCKING is not set, the filesystem takes care of its own locking. - * As with do_blockdev_direct_IO(), we increment i_dio_count while the I/O - * is in progress. - */ -ssize_t dax_do_io(struct kiocb *iocb, struct inode *inode, - struct iov_iter *iter, get_block_t get_block, - dio_iodone_t end_io, int flags) -{ - struct buffer_head bh; - ssize_t retval = -EINVAL; - loff_t pos = iocb->ki_pos; - loff_t end = pos + iov_iter_count(iter); - - memset(&bh, 0, sizeof(bh)); - bh.b_bdev = inode->i_sb->s_bdev; - - if ((flags & DIO_LOCKING) && iov_iter_rw(iter) == READ) - inode_lock(inode); - - /* Protects against truncate */ - if (!(flags & DIO_SKIP_DIO_COUNT)) - inode_dio_begin(inode); - - retval = dax_io(inode, iter, pos, end, get_block, &bh); - - if ((flags & DIO_LOCKING) && iov_iter_rw(iter) == READ) - inode_unlock(inode); - - if (end_io) { - int err; - - err = end_io(iocb, pos, retval, bh.b_private); - if (err) - retval = err; - } - - if (!(flags & DIO_SKIP_DIO_COUNT)) - inode_dio_end(inode); - return retval; -} -EXPORT_SYMBOL_GPL(dax_do_io); - /* * DAX radix tree locking */ @@ -919,105 +757,6 @@ static int dax_insert_mapping(struct address_space *mapping, return vm_insert_mixed(vma, vaddr, dax.pfn); } -/** - * dax_fault - handle a page fault on a DAX file - * @vma: The virtual memory area where the fault occurred - * @vmf: The description of the fault - * @get_block: The filesystem method used to translate file offsets to blocks - * - * When a page fault occurs, filesystems may call this helper in their - * fault handler for DAX files. dax_fault() assumes the caller has done all - * the necessary locking for the page fault to proceed successfully. - */ -int dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, - get_block_t get_block) -{ - struct file *file = vma->vm_file; - struct address_space *mapping = file->f_mapping; - struct inode *inode = mapping->host; - void *entry; - struct buffer_head bh; - unsigned long vaddr = (unsigned long)vmf->virtual_address; - unsigned blkbits = inode->i_blkbits; - sector_t block; - pgoff_t size; - int error; - int major = 0; - - /* - * Check whether offset isn't beyond end of file now. Caller is supposed - * to hold locks serializing us with truncate / punch hole so this is - * a reliable test. - */ - size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; - if (vmf->pgoff >= size) - return VM_FAULT_SIGBUS; - - memset(&bh, 0, sizeof(bh)); - block = (sector_t)vmf->pgoff << (PAGE_SHIFT - blkbits); - bh.b_bdev = inode->i_sb->s_bdev; - bh.b_size = PAGE_SIZE; - - entry = grab_mapping_entry(mapping, vmf->pgoff, 0); - if (IS_ERR(entry)) { - error = PTR_ERR(entry); - goto out; - } - - error = get_block(inode, block, &bh, 0); - if (!error && (bh.b_size < PAGE_SIZE)) - error = -EIO; /* fs corruption? */ - if (error) - goto unlock_entry; - - if (vmf->cow_page) { - struct page *new_page = vmf->cow_page; - if (buffer_written(&bh)) - error = copy_user_dax(bh.b_bdev, to_sector(&bh, inode), - bh.b_size, new_page, vaddr); - else - clear_user_highpage(new_page, vaddr); - if (error) - goto unlock_entry; - if (!radix_tree_exceptional_entry(entry)) { - vmf->page = entry; - return VM_FAULT_LOCKED; - } - vmf->entry = entry; - return VM_FAULT_DAX_LOCKED; - } - - if (!buffer_mapped(&bh)) { - if (vmf->flags & FAULT_FLAG_WRITE) { - error = get_block(inode, block, &bh, 1); - count_vm_event(PGMAJFAULT); - mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT); - major = VM_FAULT_MAJOR; - if (!error && (bh.b_size < PAGE_SIZE)) - error = -EIO; - if (error) - goto unlock_entry; - } else { - return dax_load_hole(mapping, entry, vmf); - } - } - - /* Filesystem should not return unwritten buffers to us! */ - WARN_ON_ONCE(buffer_unwritten(&bh) || buffer_new(&bh)); - error = dax_insert_mapping(mapping, bh.b_bdev, to_sector(&bh, inode), - bh.b_size, &entry, vma, vmf); - unlock_entry: - put_locked_mapping_entry(mapping, vmf->pgoff, entry); - out: - if (error == -ENOMEM) - return VM_FAULT_OOM | major; - /* -EBUSY is fine, somebody else faulted on the same PTE */ - if ((error < 0) && (error != -EBUSY)) - return VM_FAULT_SIGBUS | major; - return VM_FAULT_NOPAGE | major; -} -EXPORT_SYMBOL_GPL(dax_fault); - /** * dax_pfn_mkwrite - handle first write to DAX page * @vma: The virtual memory area where the fault occurred @@ -1078,60 +817,6 @@ int __dax_zero_page_range(struct block_device *bdev, sector_t sector, } EXPORT_SYMBOL_GPL(__dax_zero_page_range); -/** - * dax_zero_page_range - zero a range within a page of a DAX file - * @inode: The file being truncated - * @from: The file offset that is being truncated to - * @length: The number of bytes to zero - * @get_block: The filesystem method used to translate file offsets to blocks - * - * This function can be called by a filesystem when it is zeroing part of a - * page in a DAX file. This is intended for hole-punch operations. If - * you are truncating a file, the helper function dax_truncate_page() may be - * more convenient. - */ -int dax_zero_page_range(struct inode *inode, loff_t from, unsigned length, - get_block_t get_block) -{ - struct buffer_head bh; - pgoff_t index = from >> PAGE_SHIFT; - unsigned offset = from & (PAGE_SIZE-1); - int err; - - /* Block boundary? Nothing to do */ - if (!length) - return 0; - if (WARN_ON_ONCE((offset + length) > PAGE_SIZE)) - return -EINVAL; - - memset(&bh, 0, sizeof(bh)); - bh.b_bdev = inode->i_sb->s_bdev; - bh.b_size = PAGE_SIZE; - err = get_block(inode, index, &bh, 0); - if (err < 0 || !buffer_written(&bh)) - return err; - - return __dax_zero_page_range(bh.b_bdev, to_sector(&bh, inode), - offset, length); -} -EXPORT_SYMBOL_GPL(dax_zero_page_range); - -/** - * dax_truncate_page - handle a partial page being truncated in a DAX file - * @inode: The file being truncated - * @from: The file offset that is being truncated to - * @get_block: The filesystem method used to translate file offsets to blocks - * - * Similar to block_truncate_page(), this function can be called by a - * filesystem when it is truncating a DAX file to handle the partial page. - */ -int dax_truncate_page(struct inode *inode, loff_t from, get_block_t get_block) -{ - unsigned length = PAGE_ALIGN(from) - from; - return dax_zero_page_range(inode, from, length, get_block); -} -EXPORT_SYMBOL_GPL(dax_truncate_page); - #ifdef CONFIG_FS_IOMAP static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos) { diff --git a/include/linux/dax.h b/include/linux/dax.h index 8d1a5c47945f..0afade8bd3d7 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -38,13 +38,8 @@ static inline void *dax_radix_locked_entry(sector_t sector, unsigned long flags) ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, struct iomap_ops *ops); -ssize_t dax_do_io(struct kiocb *, struct inode *, struct iov_iter *, - get_block_t, dio_iodone_t, int flags); -int dax_zero_page_range(struct inode *, loff_t from, unsigned len, get_block_t); -int dax_truncate_page(struct inode *, loff_t from, get_block_t); int dax_iomap_fault(struct vm_area_struct *vma, struct vm_fault *vmf, struct iomap_ops *ops); -int dax_fault(struct vm_area_struct *, struct vm_fault *, get_block_t); int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index); void dax_wake_mapping_entry_waiter(struct address_space *mapping, pgoff_t index, void *entry, bool wake_all); @@ -73,12 +68,6 @@ static inline int __dax_zero_page_range(struct block_device *bdev, } #endif -static inline int dax_pmd_fault(struct vm_area_struct *vma, unsigned long addr, - pmd_t *pmd, unsigned int flags, get_block_t gb) -{ - return VM_FAULT_FALLBACK; -} - #ifdef CONFIG_FS_DAX_PMD static inline unsigned int dax_radix_order(void *entry) { @@ -101,7 +90,6 @@ static inline int dax_iomap_pmd_fault(struct vm_area_struct *vma, } #endif int dax_pfn_mkwrite(struct vm_area_struct *, struct vm_fault *); -#define dax_mkwrite(vma, vmf, gb) dax_fault(vma, vmf, gb) static inline bool vma_is_dax(struct vm_area_struct *vma) { -- cgit v1.2.3 From db717d8e26c2d1b0dba3e08668a1e6a7f665adde Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 26 Nov 2016 19:07:49 -0500 Subject: fscrypto: move ioctl processing more fully into common code Multiple bugs were recently fixed in the "set encryption policy" ioctl. To make it clear that fscrypt_process_policy() and fscrypt_get_policy() implement ioctls and therefore their implementations must take standard security and correctness precautions, rename them to fscrypt_ioctl_set_policy() and fscrypt_ioctl_get_policy(). Make the latter take in a struct file * to make it consistent with the former. Signed-off-by: Eric Biggers Signed-off-by: Theodore Ts'o --- fs/crypto/policy.c | 34 +++++++++++++++++++++------------- fs/ext4/ext4.h | 4 ++-- fs/ext4/ioctl.c | 34 +++++----------------------------- fs/f2fs/f2fs.h | 4 ++-- fs/f2fs/file.c | 19 ++----------------- include/linux/fscrypto.h | 12 ++++++------ 6 files changed, 38 insertions(+), 69 deletions(-) (limited to 'include/linux') diff --git a/fs/crypto/policy.c b/fs/crypto/policy.c index 6865663aac69..b96a10e3cf78 100644 --- a/fs/crypto/policy.c +++ b/fs/crypto/policy.c @@ -93,16 +93,19 @@ static int create_encryption_context_from_policy(struct inode *inode, return inode->i_sb->s_cop->set_context(inode, &ctx, sizeof(ctx), NULL); } -int fscrypt_process_policy(struct file *filp, - const struct fscrypt_policy *policy) +int fscrypt_ioctl_set_policy(struct file *filp, const void __user *arg) { + struct fscrypt_policy policy; struct inode *inode = file_inode(filp); int ret; + if (copy_from_user(&policy, arg, sizeof(policy))) + return -EFAULT; + if (!inode_owner_or_capable(inode)) return -EACCES; - if (policy->version != 0) + if (policy.version != 0) return -EINVAL; ret = mnt_want_write_file(filp); @@ -120,9 +123,9 @@ int fscrypt_process_policy(struct file *filp, ret = -ENOTEMPTY; else ret = create_encryption_context_from_policy(inode, - policy); + &policy); } else if (!is_encryption_context_consistent_with_policy(inode, - policy)) { + &policy)) { printk(KERN_WARNING "%s: Policy inconsistent with encryption context\n", __func__); @@ -134,11 +137,13 @@ int fscrypt_process_policy(struct file *filp, mnt_drop_write_file(filp); return ret; } -EXPORT_SYMBOL(fscrypt_process_policy); +EXPORT_SYMBOL(fscrypt_ioctl_set_policy); -int fscrypt_get_policy(struct inode *inode, struct fscrypt_policy *policy) +int fscrypt_ioctl_get_policy(struct file *filp, void __user *arg) { + struct inode *inode = file_inode(filp); struct fscrypt_context ctx; + struct fscrypt_policy policy; int res; if (!inode->i_sb->s_cop->get_context || @@ -151,15 +156,18 @@ int fscrypt_get_policy(struct inode *inode, struct fscrypt_policy *policy) if (ctx.format != FS_ENCRYPTION_CONTEXT_FORMAT_V1) return -EINVAL; - policy->version = 0; - policy->contents_encryption_mode = ctx.contents_encryption_mode; - policy->filenames_encryption_mode = ctx.filenames_encryption_mode; - policy->flags = ctx.flags; - memcpy(&policy->master_key_descriptor, ctx.master_key_descriptor, + policy.version = 0; + policy.contents_encryption_mode = ctx.contents_encryption_mode; + policy.filenames_encryption_mode = ctx.filenames_encryption_mode; + policy.flags = ctx.flags; + memcpy(policy.master_key_descriptor, ctx.master_key_descriptor, FS_KEY_DESCRIPTOR_SIZE); + + if (copy_to_user(arg, &policy, sizeof(policy))) + return -EFAULT; return 0; } -EXPORT_SYMBOL(fscrypt_get_policy); +EXPORT_SYMBOL(fscrypt_ioctl_get_policy); int fscrypt_has_permitted_context(struct inode *parent, struct inode *child) { diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 282a51b07c57..bd8bc3be93a5 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -2338,8 +2338,8 @@ static inline void ext4_fname_free_filename(struct ext4_filename *fname) { } #define fscrypt_pullback_bio_page fscrypt_notsupp_pullback_bio_page #define fscrypt_restore_control_page fscrypt_notsupp_restore_control_page #define fscrypt_zeroout_range fscrypt_notsupp_zeroout_range -#define fscrypt_process_policy fscrypt_notsupp_process_policy -#define fscrypt_get_policy fscrypt_notsupp_get_policy +#define fscrypt_ioctl_set_policy fscrypt_notsupp_ioctl_set_policy +#define fscrypt_ioctl_get_policy fscrypt_notsupp_ioctl_get_policy #define fscrypt_has_permitted_context fscrypt_notsupp_has_permitted_context #define fscrypt_inherit_context fscrypt_notsupp_inherit_context #define fscrypt_get_encryption_info fscrypt_notsupp_get_encryption_info diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index bf5ae8ebbc97..70083863ce51 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -765,22 +765,12 @@ resizefs_out: } case EXT4_IOC_PRECACHE_EXTENTS: return ext4_ext_precache(inode); - case EXT4_IOC_SET_ENCRYPTION_POLICY: { -#ifdef CONFIG_EXT4_FS_ENCRYPTION - struct fscrypt_policy policy; + case EXT4_IOC_SET_ENCRYPTION_POLICY: if (!ext4_has_feature_encrypt(sb)) return -EOPNOTSUPP; + return fscrypt_ioctl_set_policy(filp, (const void __user *)arg); - if (copy_from_user(&policy, - (struct fscrypt_policy __user *)arg, - sizeof(policy))) - return -EFAULT; - return fscrypt_process_policy(filp, &policy); -#else - return -EOPNOTSUPP; -#endif - } case EXT4_IOC_GET_ENCRYPTION_PWSALT: { int err, err2; struct ext4_sb_info *sbi = EXT4_SB(sb); @@ -817,23 +807,9 @@ resizefs_out: return -EFAULT; return 0; } - case EXT4_IOC_GET_ENCRYPTION_POLICY: { -#ifdef CONFIG_EXT4_FS_ENCRYPTION - struct fscrypt_policy policy; - int err = 0; - - if (!ext4_encrypted_inode(inode)) - return -ENOENT; - err = fscrypt_get_policy(inode, &policy); - if (err) - return err; - if (copy_to_user((void __user *)arg, &policy, sizeof(policy))) - return -EFAULT; - return 0; -#else - return -EOPNOTSUPP; -#endif - } + case EXT4_IOC_GET_ENCRYPTION_POLICY: + return fscrypt_ioctl_get_policy(filp, (void __user *)arg); + case EXT4_IOC_FSGETXATTR: { struct fsxattr fa; diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 9e8de18a168a..8e94b7bda42b 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -2453,8 +2453,8 @@ static inline bool f2fs_may_encrypt(struct inode *inode) #define fscrypt_pullback_bio_page fscrypt_notsupp_pullback_bio_page #define fscrypt_restore_control_page fscrypt_notsupp_restore_control_page #define fscrypt_zeroout_range fscrypt_notsupp_zeroout_range -#define fscrypt_process_policy fscrypt_notsupp_process_policy -#define fscrypt_get_policy fscrypt_notsupp_get_policy +#define fscrypt_ioctl_set_policy fscrypt_notsupp_ioctl_set_policy +#define fscrypt_ioctl_get_policy fscrypt_notsupp_ioctl_get_policy #define fscrypt_has_permitted_context fscrypt_notsupp_has_permitted_context #define fscrypt_inherit_context fscrypt_notsupp_inherit_context #define fscrypt_get_encryption_info fscrypt_notsupp_get_encryption_info diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index c7865073cd26..f0c83f74557d 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -1752,31 +1752,16 @@ static bool uuid_is_nonzero(__u8 u[16]) static int f2fs_ioc_set_encryption_policy(struct file *filp, unsigned long arg) { - struct fscrypt_policy policy; struct inode *inode = file_inode(filp); - if (copy_from_user(&policy, (struct fscrypt_policy __user *)arg, - sizeof(policy))) - return -EFAULT; - f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); - return fscrypt_process_policy(filp, &policy); + return fscrypt_ioctl_set_policy(filp, (const void __user *)arg); } static int f2fs_ioc_get_encryption_policy(struct file *filp, unsigned long arg) { - struct fscrypt_policy policy; - struct inode *inode = file_inode(filp); - int err; - - err = fscrypt_get_policy(inode, &policy); - if (err) - return err; - - if (copy_to_user((struct fscrypt_policy __user *)arg, &policy, sizeof(policy))) - return -EFAULT; - return 0; + return fscrypt_ioctl_get_policy(filp, (void __user *)arg); } static int f2fs_ioc_get_encryption_pwsalt(struct file *filp, unsigned long arg) diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index 98c71e973a96..be94684dc05f 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -259,8 +259,8 @@ extern void fscrypt_restore_control_page(struct page *); extern int fscrypt_zeroout_range(const struct inode *, pgoff_t, sector_t, unsigned int); /* policy.c */ -extern int fscrypt_process_policy(struct file *, const struct fscrypt_policy *); -extern int fscrypt_get_policy(struct inode *, struct fscrypt_policy *); +extern int fscrypt_ioctl_set_policy(struct file *, const void __user *); +extern int fscrypt_ioctl_get_policy(struct file *, void __user *); extern int fscrypt_has_permitted_context(struct inode *, struct inode *); extern int fscrypt_inherit_context(struct inode *, struct inode *, void *, bool); @@ -334,14 +334,14 @@ static inline int fscrypt_notsupp_zeroout_range(const struct inode *i, pgoff_t p } /* policy.c */ -static inline int fscrypt_notsupp_process_policy(struct file *f, - const struct fscrypt_policy *p) +static inline int fscrypt_notsupp_ioctl_set_policy(struct file *f, + const void __user *arg) { return -EOPNOTSUPP; } -static inline int fscrypt_notsupp_get_policy(struct inode *i, - struct fscrypt_policy *p) +static inline int fscrypt_notsupp_ioctl_get_policy(struct file *f, + void __user *arg) { return -EOPNOTSUPP; } -- cgit v1.2.3 From 3325bea5b26ac67e2521383f10e5ea0156c9a4b6 Mon Sep 17 00:00:00 2001 From: Theodore Ts'o Date: Sat, 26 Nov 2016 20:32:46 -0500 Subject: fscrypt: rename get_crypt_info() to fscrypt_get_crypt_info() To avoid namespace collisions, rename get_crypt_info() to fscrypt_get_crypt_info(). The function is only used inside the fs/crypto directory, so declare it in the new header file, fscrypt_private.h. Signed-off-by: Theodore Ts'o Reviewed-by: Eric Biggers --- fs/crypto/fname.c | 4 ++-- fs/crypto/fscrypt_private.h | 19 +++++++++++++++++++ fs/crypto/keyinfo.c | 6 +++--- include/linux/fscrypto.h | 1 - 4 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 fs/crypto/fscrypt_private.h (limited to 'include/linux') diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c index 6b45d9caeeb0..56ad9d195f18 100644 --- a/fs/crypto/fname.c +++ b/fs/crypto/fname.c @@ -12,7 +12,7 @@ #include #include -#include +#include "fscrypt_private.h" /** * fname_crypt_complete() - completion callback for filename crypto @@ -350,7 +350,7 @@ int fscrypt_setup_filename(struct inode *dir, const struct qstr *iname, fname->disk_name.len = iname->len; return 0; } - ret = get_crypt_info(dir); + ret = fscrypt_get_crypt_info(dir); if (ret && ret != -EOPNOTSUPP) return ret; diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h new file mode 100644 index 000000000000..7c31108728e4 --- /dev/null +++ b/fs/crypto/fscrypt_private.h @@ -0,0 +1,19 @@ +/* + * fscrypt_private.h + * + * Copyright (C) 2015, Google, Inc. + * + * This contains encryption key functions. + * + * Written by Michael Halcrow, Ildar Muslukhov, and Uday Savagaonkar, 2015. + */ + +#ifndef _FSCRYPT_PRIVATE_H +#define _FSCRYPT_PRIVATE_H + +#include + +/* keyinfo.c */ +extern int fscrypt_get_crypt_info(struct inode *); + +#endif /* _FSCRYPT_PRIVATE_H */ diff --git a/fs/crypto/keyinfo.c b/fs/crypto/keyinfo.c index 67fb6d8876d0..35d3317a27b3 100644 --- a/fs/crypto/keyinfo.c +++ b/fs/crypto/keyinfo.c @@ -10,7 +10,7 @@ #include #include -#include +#include "fscrypt_private.h" static void derive_crypt_complete(struct crypto_async_request *req, int rc) { @@ -178,7 +178,7 @@ static void put_crypt_info(struct fscrypt_info *ci) kmem_cache_free(fscrypt_info_cachep, ci); } -int get_crypt_info(struct inode *inode) +int fscrypt_get_crypt_info(struct inode *inode) { struct fscrypt_info *crypt_info; struct fscrypt_context ctx; @@ -327,7 +327,7 @@ int fscrypt_get_encryption_info(struct inode *inode) (ci->ci_keyring_key->flags & ((1 << KEY_FLAG_INVALIDATED) | (1 << KEY_FLAG_REVOKED) | (1 << KEY_FLAG_DEAD))))) - return get_crypt_info(inode); + return fscrypt_get_crypt_info(inode); return 0; } EXPORT_SYMBOL(fscrypt_get_encryption_info); diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index be94684dc05f..2f8894f0696c 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -265,7 +265,6 @@ extern int fscrypt_has_permitted_context(struct inode *, struct inode *); extern int fscrypt_inherit_context(struct inode *, struct inode *, void *, bool); /* keyinfo.c */ -extern int get_crypt_info(struct inode *); extern int fscrypt_get_encryption_info(struct inode *); extern void fscrypt_put_encryption_info(struct inode *, struct fscrypt_info *); -- cgit v1.2.3 From b98701df349b7003efd52d9330acbb7be5a255c6 Mon Sep 17 00:00:00 2001 From: Theodore Ts'o Date: Sat, 26 Nov 2016 20:43:09 -0500 Subject: fscrypt: unexport fscrypt_initialize() The fscrypt_initalize() function isn't used outside fs/crypto, so there's no point making it be an exported symbol. Signed-off-by: Theodore Ts'o Reviewed-by: Eric Biggers --- fs/crypto/crypto.c | 1 - fs/crypto/fscrypt_private.h | 3 +++ include/linux/fscrypto.h | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index b6029785714c..56f98f45cece 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -540,7 +540,6 @@ fail: mutex_unlock(&fscrypt_init_mutex); return res; } -EXPORT_SYMBOL(fscrypt_initialize); /** * fscrypt_init() - Set up for fs encryption. diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h index 7c31108728e4..bb92f0c0961b 100644 --- a/fs/crypto/fscrypt_private.h +++ b/fs/crypto/fscrypt_private.h @@ -13,6 +13,9 @@ #include +/* crypto.c */ +int fscrypt_initialize(void); + /* keyinfo.c */ extern int fscrypt_get_crypt_info(struct inode *); diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index 2f8894f0696c..ce2ebdee6a89 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -244,7 +244,6 @@ static inline void fscrypt_set_d_op(struct dentry *dentry) #if IS_ENABLED(CONFIG_FS_ENCRYPTION) /* crypto.c */ extern struct kmem_cache *fscrypt_info_cachep; -int fscrypt_initialize(void); extern struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *, gfp_t); extern void fscrypt_release_ctx(struct fscrypt_ctx *); -- cgit v1.2.3 From cc4e0df038ddb73510c01712abf302b3f0130147 Mon Sep 17 00:00:00 2001 From: Theodore Ts'o Date: Sat, 26 Nov 2016 22:05:18 -0500 Subject: fscrypt: move non-public structures and constants to fscrypt_private.h Signed-off-by: Theodore Ts'o Reviewed-by: Eric Biggers --- fs/crypto/crypto.c | 2 +- fs/crypto/fscrypt_private.h | 71 +++++++++++++++++++++++++++++++++++++++++++++ fs/crypto/policy.c | 2 +- include/linux/fscrypto.h | 68 ++----------------------------------------- 4 files changed, 76 insertions(+), 67 deletions(-) (limited to 'include/linux') diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index 56f98f45cece..4d9d221b1d60 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -27,7 +27,7 @@ #include #include #include -#include +#include "fscrypt_private.h" static unsigned int num_prealloc_crypto_pages = 32; static unsigned int num_prealloc_crypto_ctxs = 128; diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h index bb92f0c0961b..c98b2a7fb6d3 100644 --- a/fs/crypto/fscrypt_private.h +++ b/fs/crypto/fscrypt_private.h @@ -13,6 +13,77 @@ #include +#define FS_FNAME_CRYPTO_DIGEST_SIZE 32 + +/* Encryption parameters */ +#define FS_XTS_TWEAK_SIZE 16 +#define FS_AES_128_ECB_KEY_SIZE 16 +#define FS_AES_256_GCM_KEY_SIZE 32 +#define FS_AES_256_CBC_KEY_SIZE 32 +#define FS_AES_256_CTS_KEY_SIZE 32 +#define FS_AES_256_XTS_KEY_SIZE 64 +#define FS_MAX_KEY_SIZE 64 + +#define FS_KEY_DESC_PREFIX "fscrypt:" +#define FS_KEY_DESC_PREFIX_SIZE 8 + +#define FS_KEY_DERIVATION_NONCE_SIZE 16 + +/** + * Encryption context for inode + * + * Protector format: + * 1 byte: Protector format (1 = this version) + * 1 byte: File contents encryption mode + * 1 byte: File names encryption mode + * 1 byte: Flags + * 8 bytes: Master Key descriptor + * 16 bytes: Encryption Key derivation nonce + */ +struct fscrypt_context { + u8 format; + u8 contents_encryption_mode; + u8 filenames_encryption_mode; + u8 flags; + u8 master_key_descriptor[FS_KEY_DESCRIPTOR_SIZE]; + u8 nonce[FS_KEY_DERIVATION_NONCE_SIZE]; +} __packed; + +#define FS_ENCRYPTION_CONTEXT_FORMAT_V1 1 + +/* This is passed in from userspace into the kernel keyring */ +struct fscrypt_key { + u32 mode; + u8 raw[FS_MAX_KEY_SIZE]; + u32 size; +} __packed; + +/* + * A pointer to this structure is stored in the file system's in-core + * representation of an inode. + */ +struct fscrypt_info { + u8 ci_data_mode; + u8 ci_filename_mode; + u8 ci_flags; + struct crypto_skcipher *ci_ctfm; + struct key *ci_keyring_key; + u8 ci_master_key[FS_KEY_DESCRIPTOR_SIZE]; +}; + +#define FS_CTX_REQUIRES_FREE_ENCRYPT_FL 0x00000001 +#define FS_WRITE_PATH_FL 0x00000002 + +struct fscrypt_completion_result { + struct completion completion; + int res; +}; + +#define DECLARE_FS_COMPLETION_RESULT(ecr) \ + struct fscrypt_completion_result ecr = { \ + COMPLETION_INITIALIZER((ecr).completion), 0 } + + /* crypto.c */ int fscrypt_initialize(void); diff --git a/fs/crypto/policy.c b/fs/crypto/policy.c index b96a10e3cf78..6ed7c2eebeec 100644 --- a/fs/crypto/policy.c +++ b/fs/crypto/policy.c @@ -10,8 +10,8 @@ #include #include -#include #include +#include "fscrypt_private.h" static int inode_has_encryption_context(struct inode *inode) { diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index ce2ebdee6a89..71e8a20711ec 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -18,9 +18,6 @@ #include #include -#define FS_KEY_DERIVATION_NONCE_SIZE 16 -#define FS_ENCRYPTION_CONTEXT_FORMAT_V1 1 - #define FS_POLICY_FLAGS_PAD_4 0x00 #define FS_POLICY_FLAGS_PAD_8 0x01 #define FS_POLICY_FLAGS_PAD_16 0x02 @@ -35,56 +32,10 @@ #define FS_ENCRYPTION_MODE_AES_256_CBC 3 #define FS_ENCRYPTION_MODE_AES_256_CTS 4 -/** - * Encryption context for inode - * - * Protector format: - * 1 byte: Protector format (1 = this version) - * 1 byte: File contents encryption mode - * 1 byte: File names encryption mode - * 1 byte: Flags - * 8 bytes: Master Key descriptor - * 16 bytes: Encryption Key derivation nonce - */ -struct fscrypt_context { - u8 format; - u8 contents_encryption_mode; - u8 filenames_encryption_mode; - u8 flags; - u8 master_key_descriptor[FS_KEY_DESCRIPTOR_SIZE]; - u8 nonce[FS_KEY_DERIVATION_NONCE_SIZE]; -} __packed; - -/* Encryption parameters */ -#define FS_XTS_TWEAK_SIZE 16 -#define FS_AES_128_ECB_KEY_SIZE 16 -#define FS_AES_256_GCM_KEY_SIZE 32 -#define FS_AES_256_CBC_KEY_SIZE 32 -#define FS_AES_256_CTS_KEY_SIZE 32 -#define FS_AES_256_XTS_KEY_SIZE 64 -#define FS_MAX_KEY_SIZE 64 - -#define FS_KEY_DESC_PREFIX "fscrypt:" -#define FS_KEY_DESC_PREFIX_SIZE 8 - -/* This is passed in from userspace into the kernel keyring */ -struct fscrypt_key { - u32 mode; - u8 raw[FS_MAX_KEY_SIZE]; - u32 size; -} __packed; - -struct fscrypt_info { - u8 ci_data_mode; - u8 ci_filename_mode; - u8 ci_flags; - struct crypto_skcipher *ci_ctfm; - struct key *ci_keyring_key; - u8 ci_master_key[FS_KEY_DESCRIPTOR_SIZE]; -}; +#define FS_CRYPTO_BLOCK_SIZE 16 -#define FS_CTX_REQUIRES_FREE_ENCRYPT_FL 0x00000001 -#define FS_WRITE_PATH_FL 0x00000002 +struct fscrypt_info; +struct fscrypt_ctx; struct fscrypt_ctx { union { @@ -102,19 +53,6 @@ struct fscrypt_ctx { u8 mode; /* Encryption mode for tfm */ }; -struct fscrypt_completion_result { - struct completion completion; - int res; -}; - -#define DECLARE_FS_COMPLETION_RESULT(ecr) \ - struct fscrypt_completion_result ecr = { \ - COMPLETION_INITIALIZER((ecr).completion), 0 } - -#define FS_FNAME_NUM_SCATTER_ENTRIES 4 -#define FS_CRYPTO_BLOCK_SIZE 16 -#define FS_FNAME_CRYPTO_DIGEST_SIZE 32 - /** * For encrypted symlinks, the ciphertext length is stored at the beginning * of the string in little-endian format. -- cgit v1.2.3 From 41d5319af3368127b55f6587f1c747dd6a7b9b04 Mon Sep 17 00:00:00 2001 From: Theodore Ts'o Date: Sat, 26 Nov 2016 22:18:59 -0500 Subject: fscrypt: move the policy flags and encryption mode definitions to uapi header These constants are part of the UAPI, so they belong in include/uapi/linux/fs.h instead of include/linux/fscrypto.h Signed-off-by: Theodore Ts'o Reviewed-by: Eric Biggers --- include/linux/fscrypto.h | 14 -------------- include/uapi/linux/fs.h | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) (limited to 'include/linux') diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index 71e8a20711ec..42ef82d60790 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -18,20 +18,6 @@ #include #include -#define FS_POLICY_FLAGS_PAD_4 0x00 -#define FS_POLICY_FLAGS_PAD_8 0x01 -#define FS_POLICY_FLAGS_PAD_16 0x02 -#define FS_POLICY_FLAGS_PAD_32 0x03 -#define FS_POLICY_FLAGS_PAD_MASK 0x03 -#define FS_POLICY_FLAGS_VALID 0x03 - -/* Encryption algorithms */ -#define FS_ENCRYPTION_MODE_INVALID 0 -#define FS_ENCRYPTION_MODE_AES_256_XTS 1 -#define FS_ENCRYPTION_MODE_AES_256_GCM 2 -#define FS_ENCRYPTION_MODE_AES_256_CBC 3 -#define FS_ENCRYPTION_MODE_AES_256_CTS 4 - #define FS_CRYPTO_BLOCK_SIZE 16 struct fscrypt_info; diff --git a/include/uapi/linux/fs.h b/include/uapi/linux/fs.h index acb2b6152ba0..0496d37abe28 100644 --- a/include/uapi/linux/fs.h +++ b/include/uapi/linux/fs.h @@ -254,6 +254,20 @@ struct fsxattr { /* Policy provided via an ioctl on the topmost directory */ #define FS_KEY_DESCRIPTOR_SIZE 8 +#define FS_POLICY_FLAGS_PAD_4 0x00 +#define FS_POLICY_FLAGS_PAD_8 0x01 +#define FS_POLICY_FLAGS_PAD_16 0x02 +#define FS_POLICY_FLAGS_PAD_32 0x03 +#define FS_POLICY_FLAGS_PAD_MASK 0x03 +#define FS_POLICY_FLAGS_VALID 0x03 + +/* Encryption algorithms */ +#define FS_ENCRYPTION_MODE_INVALID 0 +#define FS_ENCRYPTION_MODE_AES_256_XTS 1 +#define FS_ENCRYPTION_MODE_AES_256_GCM 2 +#define FS_ENCRYPTION_MODE_AES_256_CBC 3 +#define FS_ENCRYPTION_MODE_AES_256_CTS 4 + struct fscrypt_policy { __u8 version; __u8 contents_encryption_mode; -- cgit v1.2.3 From 1400451f04f2ff28b658b92557495e5090914aee Mon Sep 17 00:00:00 2001 From: David Gstir Date: Tue, 6 Dec 2016 23:53:55 +0100 Subject: fscrypt: Cleanup fscrypt_{decrypt,encrypt}_page() - Improve documentation - Add BUG_ON(len == 0) to avoid accidental switch of offs and len parameters - Improve variable names for readability Signed-off-by: David Gstir Signed-off-by: Theodore Ts'o --- fs/crypto/crypto.c | 93 +++++++++++++++++++++++++++--------------------- include/linux/fscrypto.h | 8 ++--- 2 files changed, 56 insertions(+), 45 deletions(-) (limited to 'include/linux') diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index f287f76cc906..8c104e712bb2 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -147,9 +147,9 @@ typedef enum { } fscrypt_direction_t; static int do_page_crypto(const struct inode *inode, - fscrypt_direction_t rw, pgoff_t index, + fscrypt_direction_t rw, u64 lblk_num, struct page *src_page, struct page *dest_page, - unsigned int src_len, unsigned int src_offset, + unsigned int len, unsigned int offs, gfp_t gfp_flags) { struct { @@ -163,6 +163,8 @@ static int do_page_crypto(const struct inode *inode, struct crypto_skcipher *tfm = ci->ci_ctfm; int res = 0; + BUG_ON(len == 0); + req = skcipher_request_alloc(tfm, gfp_flags); if (!req) { printk_ratelimited(KERN_ERR @@ -176,14 +178,14 @@ static int do_page_crypto(const struct inode *inode, page_crypt_complete, &ecr); BUILD_BUG_ON(sizeof(xts_tweak) != FS_XTS_TWEAK_SIZE); - xts_tweak.index = cpu_to_le64(index); + xts_tweak.index = cpu_to_le64(lblk_num); memset(xts_tweak.padding, 0, sizeof(xts_tweak.padding)); sg_init_table(&dst, 1); - sg_set_page(&dst, dest_page, src_len, src_offset); + sg_set_page(&dst, dest_page, len, offs); sg_init_table(&src, 1); - sg_set_page(&src, src_page, src_len, src_offset); - skcipher_request_set_crypt(req, &src, &dst, src_len, &xts_tweak); + sg_set_page(&src, src_page, len, offs); + skcipher_request_set_crypt(req, &src, &dst, len, &xts_tweak); if (rw == FS_DECRYPT) res = crypto_skcipher_decrypt(req); else @@ -214,44 +216,53 @@ static struct page *alloc_bounce_page(struct fscrypt_ctx *ctx, gfp_t gfp_flags) /** * fscypt_encrypt_page() - Encrypts a page - * @inode: The inode for which the encryption should take place - * @plaintext_page: The page to encrypt. Must be locked. - * @plaintext_len: Length of plaintext within page - * @plaintext_offset: Offset of plaintext within page - * @index: Index for encryption. This is mainly the page index, but - * but might be different for multiple calls on same page. - * @gfp_flags: The gfp flag for memory allocation + * @inode: The inode for which the encryption should take place + * @page: The page to encrypt. Must be locked for bounce-page + * encryption. + * @len: Length of data to encrypt in @page and encrypted + * data in returned page. + * @offs: Offset of data within @page and returned + * page holding encrypted data. + * @lblk_num: Logical block number. This must be unique for multiple + * calls with same inode, except when overwriting + * previously written data. + * @gfp_flags: The gfp flag for memory allocation * - * Encrypts plaintext_page using the ctx encryption context. If - * the filesystem supports it, encryption is performed in-place, otherwise a - * new ciphertext_page is allocated and returned. + * Encrypts @page using the ctx encryption context. Performs encryption + * either in-place or into a newly allocated bounce page. + * Called on the page write path. * - * Called on the page write path. The caller must call + * Bounce page allocation is the default. + * In this case, the contents of @page are encrypted and stored in an + * allocated bounce page. @page has to be locked and the caller must call * fscrypt_restore_control_page() on the returned ciphertext page to * release the bounce buffer and the encryption context. * - * Return: An allocated page with the encrypted content on success. Else, an + * In-place encryption is used by setting the FS_CFLG_INPLACE_ENCRYPTION flag in + * fscrypt_operations. Here, the input-page is returned with its content + * encrypted. + * + * Return: A page with the encrypted content on success. Else, an * error value or NULL. */ struct page *fscrypt_encrypt_page(const struct inode *inode, - struct page *plaintext_page, - unsigned int plaintext_len, - unsigned int plaintext_offset, - pgoff_t index, gfp_t gfp_flags) + struct page *page, + unsigned int len, + unsigned int offs, + u64 lblk_num, gfp_t gfp_flags) { struct fscrypt_ctx *ctx; - struct page *ciphertext_page = plaintext_page; + struct page *ciphertext_page = page; int err; - BUG_ON(plaintext_len % FS_CRYPTO_BLOCK_SIZE != 0); + BUG_ON(len % FS_CRYPTO_BLOCK_SIZE != 0); if (inode->i_sb->s_cop->flags & FS_CFLG_INPLACE_ENCRYPTION) { /* with inplace-encryption we just encrypt the page */ - err = do_page_crypto(inode, FS_ENCRYPT, index, - plaintext_page, ciphertext_page, - plaintext_len, plaintext_offset, - gfp_flags); + err = do_page_crypto(inode, FS_ENCRYPT, lblk_num, + page, ciphertext_page, + len, offs, gfp_flags); if (err) return ERR_PTR(err); @@ -267,11 +278,10 @@ struct page *fscrypt_encrypt_page(const struct inode *inode, if (IS_ERR(ciphertext_page)) goto errout; - ctx->w.control_page = plaintext_page; - err = do_page_crypto(inode, FS_ENCRYPT, index, - plaintext_page, ciphertext_page, - plaintext_len, plaintext_offset, - gfp_flags); + ctx->w.control_page = page; + err = do_page_crypto(inode, FS_ENCRYPT, lblk_num, + page, ciphertext_page, + len, offs, gfp_flags); if (err) { ciphertext_page = ERR_PTR(err); goto errout; @@ -289,11 +299,12 @@ EXPORT_SYMBOL(fscrypt_encrypt_page); /** * fscrypt_decrypt_page() - Decrypts a page in-place - * @inode: Encrypted inode to decrypt. - * @page: The page to decrypt. Must be locked. - * @len: Number of bytes in @page to be decrypted. - * @offs: Start of data in @page. - * @index: Index for encryption. + * @inode: The corresponding inode for the page to decrypt. + * @page: The page to decrypt. Must be locked in case + * it is a writeback page. + * @len: Number of bytes in @page to be decrypted. + * @offs: Start of data in @page. + * @lblk_num: Logical block number. * * Decrypts page in-place using the ctx encryption context. * @@ -302,10 +313,10 @@ EXPORT_SYMBOL(fscrypt_encrypt_page); * Return: Zero on success, non-zero otherwise. */ int fscrypt_decrypt_page(const struct inode *inode, struct page *page, - unsigned int len, unsigned int offs, pgoff_t index) + unsigned int len, unsigned int offs, u64 lblk_num) { - return do_page_crypto(inode, FS_DECRYPT, index, page, page, len, offs, - GFP_NOFS); + return do_page_crypto(inode, FS_DECRYPT, lblk_num, page, page, len, + offs, GFP_NOFS); } EXPORT_SYMBOL(fscrypt_decrypt_page); diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index 42ef82d60790..2d9abfa22b94 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -173,9 +173,9 @@ extern struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *, gfp_t); extern void fscrypt_release_ctx(struct fscrypt_ctx *); extern struct page *fscrypt_encrypt_page(const struct inode *, struct page *, unsigned int, unsigned int, - pgoff_t, gfp_t); + u64, gfp_t); extern int fscrypt_decrypt_page(const struct inode *, struct page *, unsigned int, - unsigned int, pgoff_t); + unsigned int, u64); extern void fscrypt_decrypt_bio_pages(struct fscrypt_ctx *, struct bio *); extern void fscrypt_pullback_bio_page(struct page **, bool); extern void fscrypt_restore_control_page(struct page *); @@ -221,14 +221,14 @@ static inline struct page *fscrypt_notsupp_encrypt_page(const struct inode *i, struct page *p, unsigned int len, unsigned int offs, - pgoff_t index, gfp_t f) + u64 lblk_num, gfp_t f) { return ERR_PTR(-EOPNOTSUPP); } static inline int fscrypt_notsupp_decrypt_page(const struct inode *i, struct page *p, unsigned int len, unsigned int offs, - pgoff_t index) + u64 lblk_num) { return -EOPNOTSUPP; } -- cgit v1.2.3 From bd7b8290388dd58a8c0a3710b171e58ef952ca4d Mon Sep 17 00:00:00 2001 From: David Gstir Date: Tue, 6 Dec 2016 23:53:56 +0100 Subject: fscrypt: Cleanup page locking requirements for fscrypt_{decrypt,encrypt}_page() Rename the FS_CFLG_INPLACE_ENCRYPTION flag to FS_CFLG_OWN_PAGES which, when set, indicates that the fs uses pages under its own control as opposed to writeback pages which require locking and a bounce buffer for encryption. Signed-off-by: David Gstir Signed-off-by: Theodore Ts'o --- fs/crypto/crypto.c | 11 ++++++++--- fs/ext4/inode.c | 1 - fs/f2fs/data.c | 1 - include/linux/fscrypto.h | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index 8c104e712bb2..5ffc59436397 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -238,7 +238,7 @@ static struct page *alloc_bounce_page(struct fscrypt_ctx *ctx, gfp_t gfp_flags) * fscrypt_restore_control_page() on the returned ciphertext page to * release the bounce buffer and the encryption context. * - * In-place encryption is used by setting the FS_CFLG_INPLACE_ENCRYPTION flag in + * In-place encryption is used by setting the FS_CFLG_OWN_PAGES flag in * fscrypt_operations. Here, the input-page is returned with its content * encrypted. * @@ -258,7 +258,7 @@ struct page *fscrypt_encrypt_page(const struct inode *inode, BUG_ON(len % FS_CRYPTO_BLOCK_SIZE != 0); - if (inode->i_sb->s_cop->flags & FS_CFLG_INPLACE_ENCRYPTION) { + if (inode->i_sb->s_cop->flags & FS_CFLG_OWN_PAGES) { /* with inplace-encryption we just encrypt the page */ err = do_page_crypto(inode, FS_ENCRYPT, lblk_num, page, ciphertext_page, @@ -269,6 +269,8 @@ struct page *fscrypt_encrypt_page(const struct inode *inode, return ciphertext_page; } + BUG_ON(!PageLocked(page)); + ctx = fscrypt_get_ctx(inode, gfp_flags); if (IS_ERR(ctx)) return (struct page *)ctx; @@ -301,7 +303,7 @@ EXPORT_SYMBOL(fscrypt_encrypt_page); * fscrypt_decrypt_page() - Decrypts a page in-place * @inode: The corresponding inode for the page to decrypt. * @page: The page to decrypt. Must be locked in case - * it is a writeback page. + * it is a writeback page (FS_CFLG_OWN_PAGES unset). * @len: Number of bytes in @page to be decrypted. * @offs: Start of data in @page. * @lblk_num: Logical block number. @@ -315,6 +317,9 @@ EXPORT_SYMBOL(fscrypt_encrypt_page); int fscrypt_decrypt_page(const struct inode *inode, struct page *page, unsigned int len, unsigned int offs, u64 lblk_num) { + if (!(inode->i_sb->s_cop->flags & FS_CFLG_OWN_PAGES)) + BUG_ON(!PageLocked(page)); + return do_page_crypto(inode, FS_DECRYPT, lblk_num, page, page, len, offs, GFP_NOFS); } diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 1485ac273bfb..fb2b514f675b 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -3744,7 +3744,6 @@ static int __ext4_block_zero_page_range(handle_t *handle, /* We expect the key to be set. */ BUG_ON(!fscrypt_has_encryption_key(inode)); BUG_ON(blocksize != PAGE_SIZE); - BUG_ON(!PageLocked(page)); WARN_ON_ONCE(fscrypt_decrypt_page(page->mapping->host, page, PAGE_SIZE, 0, page->index)); } diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 435590c4b341..9f0ba90b92e4 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -1194,7 +1194,6 @@ int do_write_data_page(struct f2fs_io_info *fio) f2fs_wait_on_encrypted_page_writeback(F2FS_I_SB(inode), fio->old_blkaddr); retry_encrypt: - BUG_ON(!PageLocked(fio->page)); fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page, PAGE_SIZE, 0, fio->page->index, diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index 2d9abfa22b94..188b4fa59cbf 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -80,7 +80,7 @@ struct fscrypt_name { /* * fscrypt superblock flags */ -#define FS_CFLG_INPLACE_ENCRYPTION (1U << 1) +#define FS_CFLG_OWN_PAGES (1U << 1) /* * crypto opertions for filesystems -- cgit v1.2.3 From f32d7ac20a5864483c1f96e4970daa083e18bfd1 Mon Sep 17 00:00:00 2001 From: David Gstir Date: Tue, 6 Dec 2016 23:53:57 +0100 Subject: fscrypt: Delay bounce page pool allocation until needed Since fscrypt users can now indicated if fscrypt_encrypt_page() should use a bounce page, we can delay the bounce page pool initialization util it is really needed. That is until fscrypt_operations has no FS_CFLG_OWN_PAGES flag set. Signed-off-by: David Gstir Signed-off-by: Theodore Ts'o --- fs/crypto/crypto.c | 9 +++++++-- fs/crypto/fscrypt_private.h | 2 +- fs/crypto/keyinfo.c | 2 +- include/linux/fscrypto.h | 1 - 4 files changed, 9 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index 5ffc59436397..bc1d4781b9ec 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -525,17 +525,22 @@ static void fscrypt_destroy(void) /** * fscrypt_initialize() - allocate major buffers for fs encryption. + * @cop_flags: fscrypt operations flags * * We only call this when we start accessing encrypted files, since it * results in memory getting allocated that wouldn't otherwise be used. * * Return: Zero on success, non-zero otherwise. */ -int fscrypt_initialize(void) +int fscrypt_initialize(unsigned int cop_flags) { int i, res = -ENOMEM; - if (fscrypt_bounce_page_pool) + /* + * No need to allocate a bounce page pool if there already is one or + * this FS won't use it. + */ + if (cop_flags & FS_CFLG_OWN_PAGES || fscrypt_bounce_page_pool) return 0; mutex_lock(&fscrypt_init_mutex); diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h index c98b2a7fb6d3..7ba10cd45a2e 100644 --- a/fs/crypto/fscrypt_private.h +++ b/fs/crypto/fscrypt_private.h @@ -85,7 +85,7 @@ struct fscrypt_completion_result { /* crypto.c */ -int fscrypt_initialize(void); +int fscrypt_initialize(unsigned int cop_flags); /* keyinfo.c */ extern int fscrypt_get_crypt_info(struct inode *); diff --git a/fs/crypto/keyinfo.c b/fs/crypto/keyinfo.c index 35d3317a27b3..6eeea1dcba41 100644 --- a/fs/crypto/keyinfo.c +++ b/fs/crypto/keyinfo.c @@ -188,7 +188,7 @@ int fscrypt_get_crypt_info(struct inode *inode) u8 *raw_key = NULL; int res; - res = fscrypt_initialize(); + res = fscrypt_initialize(inode->i_sb->s_cop->flags); if (res) return res; diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index 188b4fa59cbf..1adc1c758d31 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -168,7 +168,6 @@ static inline void fscrypt_set_d_op(struct dentry *dentry) #if IS_ENABLED(CONFIG_FS_ENCRYPTION) /* crypto.c */ extern struct kmem_cache *fscrypt_info_cachep; - extern struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *, gfp_t); extern void fscrypt_release_ctx(struct fscrypt_ctx *); extern struct page *fscrypt_encrypt_page(const struct inode *, struct page *, -- cgit v1.2.3 From 6a34e4d2be07255cc59e6ccaf820669cfd7f815c Mon Sep 17 00:00:00 2001 From: David Gstir Date: Tue, 6 Dec 2016 23:53:58 +0100 Subject: fscrypt: Rename FS_WRITE_PATH_FL to FS_CTX_HAS_BOUNCE_BUFFER_FL ... to better explain its purpose after introducing in-place encryption without bounce buffer. Signed-off-by: David Gstir Signed-off-by: Theodore Ts'o --- fs/crypto/crypto.c | 6 +++--- fs/crypto/fscrypt_private.h | 2 +- include/linux/fscrypto.h | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index bc1d4781b9ec..ac8e4f6a3773 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -63,7 +63,7 @@ void fscrypt_release_ctx(struct fscrypt_ctx *ctx) { unsigned long flags; - if (ctx->flags & FS_WRITE_PATH_FL && ctx->w.bounce_page) { + if (ctx->flags & FS_CTX_HAS_BOUNCE_BUFFER_FL && ctx->w.bounce_page) { mempool_free(ctx->w.bounce_page, fscrypt_bounce_page_pool); ctx->w.bounce_page = NULL; } @@ -121,7 +121,7 @@ struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *inode, gfp_t gfp_flags) } else { ctx->flags &= ~FS_CTX_REQUIRES_FREE_ENCRYPT_FL; } - ctx->flags &= ~FS_WRITE_PATH_FL; + ctx->flags &= ~FS_CTX_HAS_BOUNCE_BUFFER_FL; return ctx; } EXPORT_SYMBOL(fscrypt_get_ctx); @@ -210,7 +210,7 @@ static struct page *alloc_bounce_page(struct fscrypt_ctx *ctx, gfp_t gfp_flags) ctx->w.bounce_page = mempool_alloc(fscrypt_bounce_page_pool, gfp_flags); if (ctx->w.bounce_page == NULL) return ERR_PTR(-ENOMEM); - ctx->flags |= FS_WRITE_PATH_FL; + ctx->flags |= FS_CTX_HAS_BOUNCE_BUFFER_FL; return ctx->w.bounce_page; } diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h index 7ba10cd45a2e..aeab032d7d35 100644 --- a/fs/crypto/fscrypt_private.h +++ b/fs/crypto/fscrypt_private.h @@ -72,7 +72,7 @@ struct fscrypt_info { }; #define FS_CTX_REQUIRES_FREE_ENCRYPT_FL 0x00000001 -#define FS_WRITE_PATH_FL 0x00000002 +#define FS_CTX_HAS_BOUNCE_BUFFER_FL 0x00000002 struct fscrypt_completion_result { struct completion completion; diff --git a/include/linux/fscrypto.h b/include/linux/fscrypto.h index 1adc1c758d31..c074b670aa99 100644 --- a/include/linux/fscrypto.h +++ b/include/linux/fscrypto.h @@ -21,7 +21,6 @@ #define FS_CRYPTO_BLOCK_SIZE 16 struct fscrypt_info; -struct fscrypt_ctx; struct fscrypt_ctx { union { -- cgit v1.2.3