From 90f9d70a582c02f50b4dd847166cd5b037219891 Mon Sep 17 00:00:00 2001 From: "bibo,mao" Date: Wed, 31 Jan 2007 17:50:31 +0800 Subject: [IA64] enable singlestep on system call As is pointed out in http://www.gelato.org/community/view_linear.php?id=1_1036&from=authors&value=Ian%20Wienand#1_1039, if single step on break instruction, the break fault has higher priority than the single-step trap. When the break fault handler is entered, it advances the IP by 1 instruction so break instruction single-stepping is skipped, actually it is next instruction which is single stepped. This patch modifies this, it adds TIF_SINGLESTEP bit for thread flags, and generate a fake sigtrap when single stepping break instruction. Test case in attachment can verify this. Any comments is welcome. Signed-off-by: bibo, mao Signed-off-by: Tony Luck --- include/asm-ia64/thread_info.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-ia64/thread_info.h b/include/asm-ia64/thread_info.h index 9b505b25544f..91698599f918 100644 --- a/include/asm-ia64/thread_info.h +++ b/include/asm-ia64/thread_info.h @@ -84,6 +84,7 @@ struct thread_info { #define TIF_NEED_RESCHED 2 /* rescheduling necessary */ #define TIF_SYSCALL_TRACE 3 /* syscall trace active */ #define TIF_SYSCALL_AUDIT 4 /* syscall auditing active */ +#define TIF_SINGLESTEP 5 /* restore singlestep on return to user mode */ #define TIF_POLLING_NRFLAG 16 /* true if poll_idle() is polling TIF_NEED_RESCHED */ #define TIF_MEMDIE 17 #define TIF_MCA_INIT 18 /* this task is processing MCA or INIT */ @@ -92,7 +93,8 @@ struct thread_info { #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) -#define _TIF_SYSCALL_TRACEAUDIT (_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT) +#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) +#define _TIF_SYSCALL_TRACEAUDIT (_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP) #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) -- cgit v1.2.3 From f43691ef8a816018a0294c5a9fa9d22512886c49 Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Mon, 15 Jan 2007 09:33:55 -0700 Subject: [IA64] remove bogus prototype ia64_esi_init() This function doesn't exist. Signed-off-by: Alex Williamson Signed-off-by: Tony Luck --- include/asm-ia64/esi.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include') diff --git a/include/asm-ia64/esi.h b/include/asm-ia64/esi.h index 84aac0e0b583..40991c6ba647 100644 --- a/include/asm-ia64/esi.h +++ b/include/asm-ia64/esi.h @@ -19,7 +19,6 @@ enum esi_proc_type { ESI_PROC_REENTRANT /* MP-safe and reentrant */ }; -extern int ia64_esi_init (void); extern struct ia64_sal_retval esi_call_phys (void *, u64 *); extern int ia64_esi_call(efi_guid_t, struct ia64_sal_retval *, enum esi_proc_type, -- cgit v1.2.3 From 139b830477ccdca21b68c40f9a83ec327e65eb56 Mon Sep 17 00:00:00 2001 From: Bob Picco Date: Tue, 30 Jan 2007 02:11:09 -0800 Subject: [IA64] register memory ranges in a consistent manner While pursuing and unrelated issue with 64Mb granules I noticed a problem related to inconsistent use of add_active_range. There doesn't appear any reason to me why FLATMEM versus DISCONTIG_MEM should register memory to add_active_range with different code. So I've changed the code into a common implementation. The other subtle issue fixed by this patch was calling add_active_range in count_node_pages before granule aligning is performed. We were lucky with 16MB granules but not so with 64MB granules. count_node_pages has reserved regions filtered out and as a consequence linked kernel text and data aren't covered by calls to count_node_pages. So linked kernel regions wasn't reported to add_active_regions. This resulted in free_initmem causing numerous bad_page reports. This won't occur with this patch because now all known memory regions are reported by register_active_ranges. Acked-by: Mel Gorman Signed-off-by: Bob Picco Acked-by: Simon Horman Signed-off-by: Andrew Morton Signed-off-by: Tony Luck --- arch/ia64/mm/discontig.c | 4 +++- arch/ia64/mm/init.c | 19 +++++++++++++++++-- include/asm-ia64/meminit.h | 3 ++- 3 files changed, 22 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c index d3edb12f3cf9..999cefd2b226 100644 --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c @@ -473,6 +473,9 @@ void __init find_memory(void) node_clear(node, memory_less_mask); mem_data[node].min_pfn = ~0UL; } + + efi_memmap_walk(register_active_ranges, NULL); + /* * Initialize the boot memory maps in reverse order since that's * what the bootmem allocator expects @@ -660,7 +663,6 @@ static __init int count_node_pages(unsigned long start, unsigned long len, int n { unsigned long end = start + len; - add_active_range(node, start >> PAGE_SHIFT, end >> PAGE_SHIFT); mem_data[node].num_physpages += len >> PAGE_SHIFT; if (start <= __pa(MAX_DMA_ADDRESS)) mem_data[node].num_dma_physpages += diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index 1373fae7657f..8b7599808dd5 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -595,13 +596,27 @@ find_largest_hole (u64 start, u64 end, void *arg) return 0; } +#endif /* CONFIG_VIRTUAL_MEM_MAP */ + int __init register_active_ranges(u64 start, u64 end, void *arg) { - add_active_range(0, __pa(start) >> PAGE_SHIFT, __pa(end) >> PAGE_SHIFT); + int nid = paddr_to_nid(__pa(start)); + + if (nid < 0) + nid = 0; +#ifdef CONFIG_KEXEC + if (start > crashk_res.start && start < crashk_res.end) + start = crashk_res.end; + if (end > crashk_res.start && end < crashk_res.end) + end = crashk_res.start; +#endif + + if (start < end) + add_active_range(nid, __pa(start) >> PAGE_SHIFT, + __pa(end) >> PAGE_SHIFT); return 0; } -#endif /* CONFIG_VIRTUAL_MEM_MAP */ static int __init count_reserved_pages (u64 start, u64 end, void *arg) diff --git a/include/asm-ia64/meminit.h b/include/asm-ia64/meminit.h index c8df75901083..6dd476b652c6 100644 --- a/include/asm-ia64/meminit.h +++ b/include/asm-ia64/meminit.h @@ -51,12 +51,13 @@ extern void efi_memmap_init(unsigned long *, unsigned long *); #define IGNORE_PFN0 1 /* XXX fix me: ignore pfn 0 until TLB miss handler is updated... */ +extern int register_active_ranges(u64 start, u64 end, void *arg); + #ifdef CONFIG_VIRTUAL_MEM_MAP # define LARGE_GAP 0x40000000 /* Use virtual mem map if hole is > than this */ extern unsigned long vmalloc_end; extern struct page *vmem_map; extern int find_largest_hole (u64 start, u64 end, void *arg); - extern int register_active_ranges (u64 start, u64 end, void *arg); extern int create_mem_map_page_table (u64 start, u64 end, void *arg); extern int vmemmap_find_next_valid_pfn(int, int); #else -- cgit v1.2.3 From 71120061f271f00d8280659bf12e065ca6533d4d Mon Sep 17 00:00:00 2001 From: Kirill Korotaev Date: Mon, 5 Feb 2007 16:20:00 -0800 Subject: [IA64] virt_to_page() can be called with NULL arg It does not return NULL when arg is NULL. Signed-off-by: Alexey Kuznetsov Signed-off-by: Kirill Korotaev Signed-off-by: Andrew Morton Signed-off-by: Tony Luck --- include/asm-ia64/pgalloc.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/asm-ia64/pgalloc.h b/include/asm-ia64/pgalloc.h index 393e04c42a2c..560c287b1233 100644 --- a/include/asm-ia64/pgalloc.h +++ b/include/asm-ia64/pgalloc.h @@ -137,7 +137,8 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t * pmd_entry, pte_t * pte) static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long addr) { - return virt_to_page(pgtable_quicklist_alloc()); + void *pg = pgtable_quicklist_alloc(); + return pg ? virt_to_page(pg) : NULL; } static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, -- cgit v1.2.3 From 86afa9eb88af2248bcc91d5b3568c63fdea65d6c Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Mon, 5 Feb 2007 16:07:57 -0800 Subject: [IA64] Hook up getcpu system call for IA64 getcpu system call returns cpu# and node# on which this system call and its caller are running. This patch hooks up its implementation on IA64. Signed-off-by: Fenghua Yu Signed-off-by: Tony Luck --- arch/ia64/kernel/entry.S | 2 ++ include/asm-ia64/unistd.h | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/arch/ia64/kernel/entry.S b/arch/ia64/kernel/entry.S index 15234ed3a341..e7873eeae448 100644 --- a/arch/ia64/kernel/entry.S +++ b/arch/ia64/kernel/entry.S @@ -1610,5 +1610,7 @@ sys_call_table: data8 sys_sync_file_range // 1300 data8 sys_tee data8 sys_vmsplice + data8 sys_ni_syscall // reserved for move_pages + data8 sys_getcpu .org sys_call_table + 8*NR_syscalls // guard against failures to increase NR_syscalls diff --git a/include/asm-ia64/unistd.h b/include/asm-ia64/unistd.h index 53c5c0ee122c..a9e1fa4cac4d 100644 --- a/include/asm-ia64/unistd.h +++ b/include/asm-ia64/unistd.h @@ -291,11 +291,13 @@ #define __NR_sync_file_range 1300 #define __NR_tee 1301 #define __NR_vmsplice 1302 +/* 1303 reserved for move_pages */ +#define __NR_getcpu 1304 #ifdef __KERNEL__ -#define NR_syscalls 279 /* length of syscall table */ +#define NR_syscalls 281 /* length of syscall table */ #define __ARCH_WANT_SYS_RT_SIGACTION -- cgit v1.2.3 From cde14bbfb3aa79b479db35bd29e6c083513d8614 Mon Sep 17 00:00:00 2001 From: Jan Beulich Date: Mon, 5 Feb 2007 18:46:40 -0800 Subject: [IA64] swiotlb bug fixes This patch fixes - marking I-cache clean of pages DMAed to now only done for IA64 - broken multiple inclusion in include/asm-x86_64/swiotlb.h - missing call to mark_clean in swiotlb_sync_sg() - a (perhaps only theoretical) issue in swiotlb_dma_supported() when io_tlb_end is exactly at the end of memory Signed-off-by: Jan Beulich Signed-off-by: Andrew Morton Signed-off-by: Tony Luck --- arch/ia64/mm/init.c | 19 +++++++++++++++++++ include/asm-ia64/dma.h | 2 ++ include/asm-x86_64/swiotlb.h | 7 ++++--- lib/swiotlb.c | 33 ++++++++------------------------- 4 files changed, 33 insertions(+), 28 deletions(-) (limited to 'include') diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index 8b7599808dd5..faaca21a3718 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -129,6 +129,25 @@ lazy_mmu_prot_update (pte_t pte) set_bit(PG_arch_1, &page->flags); /* mark page as clean */ } +/* + * Since DMA is i-cache coherent, any (complete) pages that were written via + * DMA can be marked as "clean" so that lazy_mmu_prot_update() doesn't have to + * flush them when they get mapped into an executable vm-area. + */ +void +dma_mark_clean(void *addr, size_t size) +{ + unsigned long pg_addr, end; + + pg_addr = PAGE_ALIGN((unsigned long) addr); + end = (unsigned long) addr + size; + while (pg_addr + PAGE_SIZE <= end) { + struct page *page = virt_to_page(pg_addr); + set_bit(PG_arch_1, &page->flags); + pg_addr += PAGE_SIZE; + } +} + inline void ia64_set_rbs_bot (void) { diff --git a/include/asm-ia64/dma.h b/include/asm-ia64/dma.h index dad3a735df8b..4d97f60f1ef5 100644 --- a/include/asm-ia64/dma.h +++ b/include/asm-ia64/dma.h @@ -19,4 +19,6 @@ extern unsigned long MAX_DMA_ADDRESS; #define free_dma(x) +void dma_mark_clean(void *addr, size_t size); + #endif /* _ASM_IA64_DMA_H */ diff --git a/include/asm-x86_64/swiotlb.h b/include/asm-x86_64/swiotlb.h index ba94ab3d2673..f9c589539a82 100644 --- a/include/asm-x86_64/swiotlb.h +++ b/include/asm-x86_64/swiotlb.h @@ -1,6 +1,5 @@ #ifndef _ASM_SWIOTLB_H -#define _ASM_SWTIOLB_H 1 - +#define _ASM_SWIOTLB_H 1 #include @@ -52,4 +51,6 @@ extern int swiotlb; extern void pci_swiotlb_init(void); -#endif /* _ASM_SWTIOLB_H */ +static inline void dma_mark_clean(void *addr, size_t size) {} + +#endif /* _ASM_SWIOTLB_H */ diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 10625785eefd..34278338aad0 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -557,25 +557,6 @@ swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir) return dev_addr; } -/* - * Since DMA is i-cache coherent, any (complete) pages that were written via - * DMA can be marked as "clean" so that lazy_mmu_prot_update() doesn't have to - * flush them when they get mapped into an executable vm-area. - */ -static void -mark_clean(void *addr, size_t size) -{ - unsigned long pg_addr, end; - - pg_addr = PAGE_ALIGN((unsigned long) addr); - end = (unsigned long) addr + size; - while (pg_addr + PAGE_SIZE <= end) { - struct page *page = virt_to_page(pg_addr); - set_bit(PG_arch_1, &page->flags); - pg_addr += PAGE_SIZE; - } -} - /* * Unmap a single streaming mode DMA translation. The dma_addr and size must * match what was provided for in a previous swiotlb_map_single call. All @@ -594,7 +575,7 @@ swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size, if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end) unmap_single(hwdev, dma_addr, size, dir); else if (dir == DMA_FROM_DEVICE) - mark_clean(dma_addr, size); + dma_mark_clean(dma_addr, size); } /* @@ -617,7 +598,7 @@ swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr, if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end) sync_single(hwdev, dma_addr, size, dir, target); else if (dir == DMA_FROM_DEVICE) - mark_clean(dma_addr, size); + dma_mark_clean(dma_addr, size); } void @@ -648,7 +629,7 @@ swiotlb_sync_single_range(struct device *hwdev, dma_addr_t dev_addr, if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end) sync_single(hwdev, dma_addr, size, dir, target); else if (dir == DMA_FROM_DEVICE) - mark_clean(dma_addr, size); + dma_mark_clean(dma_addr, size); } void @@ -698,7 +679,6 @@ swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nelems, dev_addr = virt_to_phys(addr); if (swiotlb_force || address_needs_mapping(hwdev, dev_addr)) { void *map = map_single(hwdev, addr, sg->length, dir); - sg->dma_address = virt_to_bus(map); if (!map) { /* Don't panic here, we expect map_sg users to do proper error handling. */ @@ -707,6 +687,7 @@ swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nelems, sg[0].dma_length = 0; return 0; } + sg->dma_address = virt_to_bus(map); } else sg->dma_address = dev_addr; sg->dma_length = sg->length; @@ -730,7 +711,7 @@ swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nelems, if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg)) unmap_single(hwdev, (void *) phys_to_virt(sg->dma_address), sg->dma_length, dir); else if (dir == DMA_FROM_DEVICE) - mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length); + dma_mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length); } /* @@ -752,6 +733,8 @@ swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sg, if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg)) sync_single(hwdev, (void *) sg->dma_address, sg->dma_length, dir, target); + else if (dir == DMA_FROM_DEVICE) + dma_mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length); } void @@ -783,7 +766,7 @@ swiotlb_dma_mapping_error(dma_addr_t dma_addr) int swiotlb_dma_supported (struct device *hwdev, u64 mask) { - return (virt_to_phys (io_tlb_end) - 1) <= mask; + return virt_to_phys(io_tlb_end - 1) <= mask; } EXPORT_SYMBOL(swiotlb_init); -- cgit v1.2.3 From 51099005ab8e09d68a13fea8d55bc739c1040ca6 Mon Sep 17 00:00:00 2001 From: Jan Beulich Date: Mon, 5 Feb 2007 18:53:04 -0800 Subject: [IA64] swiotlb abstraction (e.g. for Xen) Add abstraction so that the file can be used by environments other than IA64 and EM64T, namely for Xen. Signed-off-by: Jan Beulich Signed-off-by: Andrew Morton Signed-off-by: Tony Luck --- include/asm-ia64/swiotlb.h | 9 +++ include/asm-x86_64/swiotlb.h | 1 + lib/swiotlb.c | 184 +++++++++++++++++++++++++++++++++++-------- 3 files changed, 159 insertions(+), 35 deletions(-) create mode 100644 include/asm-ia64/swiotlb.h (limited to 'include') diff --git a/include/asm-ia64/swiotlb.h b/include/asm-ia64/swiotlb.h new file mode 100644 index 000000000000..452c162dee4e --- /dev/null +++ b/include/asm-ia64/swiotlb.h @@ -0,0 +1,9 @@ +#ifndef _ASM_SWIOTLB_H +#define _ASM_SWIOTLB_H 1 + +#include + +#define SWIOTLB_ARCH_NEED_LATE_INIT +#define SWIOTLB_ARCH_NEED_ALLOC + +#endif /* _ASM_SWIOTLB_H */ diff --git a/include/asm-x86_64/swiotlb.h b/include/asm-x86_64/swiotlb.h index f9c589539a82..ab913ffcad56 100644 --- a/include/asm-x86_64/swiotlb.h +++ b/include/asm-x86_64/swiotlb.h @@ -44,6 +44,7 @@ extern void swiotlb_init(void); extern int swiotlb_force; #ifdef CONFIG_SWIOTLB +#define SWIOTLB_ARCH_NEED_ALLOC extern int swiotlb; #else #define swiotlb 0 diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 067eed5b2758..50a438010182 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -35,8 +36,10 @@ #define OFFSET(val,align) ((unsigned long) \ ( (val) & ( (align) - 1))) +#ifndef SG_ENT_VIRT_ADDRESS #define SG_ENT_VIRT_ADDRESS(sg) (page_address((sg)->page) + (sg)->offset) #define SG_ENT_PHYS_ADDRESS(sg) virt_to_bus(SG_ENT_VIRT_ADDRESS(sg)) +#endif /* * Maximum allowable number of contiguous slabs to map, @@ -101,13 +104,25 @@ static unsigned int io_tlb_index; * We need to save away the original address corresponding to a mapped entry * for the sync operations. */ -static unsigned char **io_tlb_orig_addr; +#ifndef SWIOTLB_ARCH_HAS_IO_TLB_ADDR_T +typedef char *io_tlb_addr_t; +#define swiotlb_orig_addr_null(buffer) (!(buffer)) +#define ptr_to_io_tlb_addr(ptr) (ptr) +#define page_to_io_tlb_addr(pg, off) (page_address(pg) + (off)) +#define sg_to_io_tlb_addr(sg) SG_ENT_VIRT_ADDRESS(sg) +#endif +static io_tlb_addr_t *io_tlb_orig_addr; /* * Protect the above data structures in the map and unmap calls */ static DEFINE_SPINLOCK(io_tlb_lock); +#ifdef SWIOTLB_EXTRA_VARIABLES +SWIOTLB_EXTRA_VARIABLES; +#endif + +#ifndef SWIOTLB_ARCH_HAS_SETUP_IO_TLB_NPAGES static int __init setup_io_tlb_npages(char *str) { @@ -122,9 +137,25 @@ setup_io_tlb_npages(char *str) swiotlb_force = 1; return 1; } +#endif __setup("swiotlb=", setup_io_tlb_npages); /* make io_tlb_overflow tunable too? */ +#ifndef swiotlb_adjust_size +#define swiotlb_adjust_size(size) ((void)0) +#endif + +#ifndef swiotlb_adjust_seg +#define swiotlb_adjust_seg(start, size) ((void)0) +#endif + +#ifndef swiotlb_print_info +#define swiotlb_print_info(bytes) \ + printk(KERN_INFO "Placing %luMB software IO TLB between 0x%lx - " \ + "0x%lx\n", bytes >> 20, \ + virt_to_bus(io_tlb_start), virt_to_bus(io_tlb_end)) +#endif + /* * Statically reserve bounce buffer space and initialize bounce buffer data * structures for the software IO TLB used to implement the DMA API. @@ -138,6 +169,8 @@ swiotlb_init_with_default_size(size_t default_size) io_tlb_nslabs = (default_size >> IO_TLB_SHIFT); io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE); } + swiotlb_adjust_size(io_tlb_nslabs); + swiotlb_adjust_size(io_tlb_overflow); bytes = io_tlb_nslabs << IO_TLB_SHIFT; @@ -155,10 +188,14 @@ swiotlb_init_with_default_size(size_t default_size) * between io_tlb_start and io_tlb_end. */ io_tlb_list = alloc_bootmem(io_tlb_nslabs * sizeof(int)); - for (i = 0; i < io_tlb_nslabs; i++) + for (i = 0; i < io_tlb_nslabs; i++) { + if ( !(i % IO_TLB_SEGSIZE) ) + swiotlb_adjust_seg(io_tlb_start + (i << IO_TLB_SHIFT), + IO_TLB_SEGSIZE << IO_TLB_SHIFT); io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); + } io_tlb_index = 0; - io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(char *)); + io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(io_tlb_addr_t)); /* * Get the overflow emergency buffer @@ -166,17 +203,21 @@ swiotlb_init_with_default_size(size_t default_size) io_tlb_overflow_buffer = alloc_bootmem_low(io_tlb_overflow); if (!io_tlb_overflow_buffer) panic("Cannot allocate SWIOTLB overflow buffer!\n"); + swiotlb_adjust_seg(io_tlb_overflow_buffer, io_tlb_overflow); - printk(KERN_INFO "Placing software IO TLB between 0x%lx - 0x%lx\n", - virt_to_bus(io_tlb_start), virt_to_bus(io_tlb_end)); + swiotlb_print_info(bytes); } +#ifndef __swiotlb_init_with_default_size +#define __swiotlb_init_with_default_size swiotlb_init_with_default_size +#endif void __init swiotlb_init(void) { - swiotlb_init_with_default_size(64 * (1<<20)); /* default to 64MB */ + __swiotlb_init_with_default_size(64 * (1<<20)); /* default to 64MB */ } +#ifdef SWIOTLB_ARCH_NEED_LATE_INIT /* * Systems with larger DMA zones (those that don't support ISA) can * initialize the swiotlb later using the slab allocator if needed. @@ -234,12 +275,12 @@ swiotlb_late_init_with_default_size(size_t default_size) io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); io_tlb_index = 0; - io_tlb_orig_addr = (unsigned char **)__get_free_pages(GFP_KERNEL, - get_order(io_tlb_nslabs * sizeof(char *))); + io_tlb_orig_addr = (io_tlb_addr_t *)__get_free_pages(GFP_KERNEL, + get_order(io_tlb_nslabs * sizeof(io_tlb_addr_t))); if (!io_tlb_orig_addr) goto cleanup3; - memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(char *)); + memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(io_tlb_addr_t)); /* * Get the overflow emergency buffer @@ -249,19 +290,17 @@ swiotlb_late_init_with_default_size(size_t default_size) if (!io_tlb_overflow_buffer) goto cleanup4; - printk(KERN_INFO "Placing %luMB software IO TLB between 0x%lx - " - "0x%lx\n", bytes >> 20, - virt_to_bus(io_tlb_start), virt_to_bus(io_tlb_end)); + swiotlb_print_info(bytes); return 0; cleanup4: - free_pages((unsigned long)io_tlb_orig_addr, get_order(io_tlb_nslabs * - sizeof(char *))); + free_pages((unsigned long)io_tlb_orig_addr, + get_order(io_tlb_nslabs * sizeof(io_tlb_addr_t))); io_tlb_orig_addr = NULL; cleanup3: - free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs * - sizeof(int))); + free_pages((unsigned long)io_tlb_list, + get_order(io_tlb_nslabs * sizeof(int))); io_tlb_list = NULL; cleanup2: io_tlb_end = NULL; @@ -271,7 +310,9 @@ cleanup1: io_tlb_nslabs = req_nslabs; return -ENOMEM; } +#endif +#ifndef SWIOTLB_ARCH_HAS_NEEDS_MAPPING static inline int address_needs_mapping(struct device *hwdev, dma_addr_t addr) { @@ -282,11 +323,35 @@ address_needs_mapping(struct device *hwdev, dma_addr_t addr) return (addr & ~mask) != 0; } +static inline int range_needs_mapping(const void *ptr, size_t size) +{ + return swiotlb_force; +} + +static inline int order_needs_mapping(unsigned int order) +{ + return 0; +} +#endif + +static void +__sync_single(io_tlb_addr_t buffer, char *dma_addr, size_t size, int dir) +{ +#ifndef SWIOTLB_ARCH_HAS_SYNC_SINGLE + if (dir == DMA_TO_DEVICE) + memcpy(dma_addr, buffer, size); + else + memcpy(buffer, dma_addr, size); +#else + __swiotlb_arch_sync_single(buffer, dma_addr, size, dir); +#endif +} + /* * Allocates bounce buffer and returns its kernel virtual address. */ static void * -map_single(struct device *hwdev, char *buffer, size_t size, int dir) +map_single(struct device *hwdev, io_tlb_addr_t buffer, size_t size, int dir) { unsigned long flags; char *dma_addr; @@ -359,7 +424,7 @@ map_single(struct device *hwdev, char *buffer, size_t size, int dir) */ io_tlb_orig_addr[index] = buffer; if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) - memcpy(dma_addr, buffer, size); + __sync_single(buffer, dma_addr, size, DMA_TO_DEVICE); return dma_addr; } @@ -373,17 +438,18 @@ unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir) unsigned long flags; int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; - char *buffer = io_tlb_orig_addr[index]; + io_tlb_addr_t buffer = io_tlb_orig_addr[index]; /* * First, sync the memory before unmapping the entry */ - if (buffer && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))) + if (!swiotlb_orig_addr_null(buffer) + && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))) /* * bounce... copy the data back into the original buffer * and * delete the bounce buffer. */ - memcpy(buffer, dma_addr, size); + __sync_single(buffer, dma_addr, size, DMA_FROM_DEVICE); /* * Return the buffer to the free list by setting the corresponding @@ -416,18 +482,18 @@ sync_single(struct device *hwdev, char *dma_addr, size_t size, int dir, int target) { int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; - char *buffer = io_tlb_orig_addr[index]; + io_tlb_addr_t buffer = io_tlb_orig_addr[index]; switch (target) { case SYNC_FOR_CPU: if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)) - memcpy(buffer, dma_addr, size); + __sync_single(buffer, dma_addr, size, DMA_FROM_DEVICE); else BUG_ON(dir != DMA_TO_DEVICE); break; case SYNC_FOR_DEVICE: if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) - memcpy(dma_addr, buffer, size); + __sync_single(buffer, dma_addr, size, DMA_TO_DEVICE); else BUG_ON(dir != DMA_FROM_DEVICE); break; @@ -436,6 +502,8 @@ sync_single(struct device *hwdev, char *dma_addr, size_t size, } } +#ifdef SWIOTLB_ARCH_NEED_ALLOC + void * swiotlb_alloc_coherent(struct device *hwdev, size_t size, dma_addr_t *dma_handle, gfp_t flags) @@ -451,7 +519,10 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, */ flags |= GFP_DMA; - ret = (void *)__get_free_pages(flags, order); + if (!order_needs_mapping(order)) + ret = (void *)__get_free_pages(flags, order); + else + ret = NULL; if (ret && address_needs_mapping(hwdev, virt_to_bus(ret))) { /* * The allocated memory isn't reachable by the device. @@ -489,6 +560,7 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, *dma_handle = dev_addr; return ret; } +EXPORT_SYMBOL(swiotlb_alloc_coherent); void swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, @@ -501,6 +573,9 @@ swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, /* DMA_TO_DEVICE to avoid memcpy in unmap_single */ swiotlb_unmap_single (hwdev, dma_handle, size, DMA_TO_DEVICE); } +EXPORT_SYMBOL(swiotlb_free_coherent); + +#endif static void swiotlb_full(struct device *dev, size_t size, int dir, int do_panic) @@ -542,13 +617,14 @@ swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir) * we can safely return the device addr and not worry about bounce * buffering it. */ - if (!address_needs_mapping(hwdev, dev_addr) && !swiotlb_force) + if (!range_needs_mapping(ptr, size) + && !address_needs_mapping(hwdev, dev_addr)) return dev_addr; /* * Oh well, have to allocate and map a bounce buffer. */ - map = map_single(hwdev, ptr, size, dir); + map = map_single(hwdev, ptr_to_io_tlb_addr(ptr), size, dir); if (!map) { swiotlb_full(hwdev, size, dir, 1); map = io_tlb_overflow_buffer; @@ -676,17 +752,16 @@ int swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nelems, int dir) { - void *addr; dma_addr_t dev_addr; int i; BUG_ON(dir == DMA_NONE); for (i = 0; i < nelems; i++, sg++) { - addr = SG_ENT_VIRT_ADDRESS(sg); - dev_addr = virt_to_bus(addr); - if (swiotlb_force || address_needs_mapping(hwdev, dev_addr)) { - void *map = map_single(hwdev, addr, sg->length, dir); + dev_addr = SG_ENT_PHYS_ADDRESS(sg); + if (range_needs_mapping(SG_ENT_VIRT_ADDRESS(sg), sg->length) + || address_needs_mapping(hwdev, dev_addr)) { + void *map = map_single(hwdev, sg_to_io_tlb_addr(sg), sg->length, dir); if (!map) { /* Don't panic here, we expect map_sg users to do proper error handling. */ @@ -760,6 +835,44 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE); } +#ifdef SWIOTLB_ARCH_NEED_MAP_PAGE + +dma_addr_t +swiotlb_map_page(struct device *hwdev, struct page *page, + unsigned long offset, size_t size, + enum dma_data_direction direction) +{ + dma_addr_t dev_addr; + char *map; + + dev_addr = page_to_bus(page) + offset; + if (address_needs_mapping(hwdev, dev_addr)) { + map = map_single(hwdev, page_to_io_tlb_addr(page, offset), size, direction); + if (!map) { + swiotlb_full(hwdev, size, direction, 1); + map = io_tlb_overflow_buffer; + } + dev_addr = virt_to_bus(map); + } + + return dev_addr; +} + +void +swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, + size_t size, enum dma_data_direction direction) +{ + char *dma_addr = bus_to_virt(dev_addr); + + BUG_ON(direction == DMA_NONE); + if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end) + unmap_single(hwdev, dma_addr, size, direction); + else if (direction == DMA_FROM_DEVICE) + dma_mark_clean(dma_addr, size); +} + +#endif + int swiotlb_dma_mapping_error(dma_addr_t dma_addr) { @@ -772,10 +885,13 @@ swiotlb_dma_mapping_error(dma_addr_t dma_addr) * during bus mastering, then you would pass 0x00ffffff as the mask to * this function. */ +#ifndef __swiotlb_dma_supported +#define __swiotlb_dma_supported(hwdev, mask) (virt_to_bus(io_tlb_end - 1) <= (mask)) +#endif int swiotlb_dma_supported(struct device *hwdev, u64 mask) { - return virt_to_bus(io_tlb_end - 1) <= mask; + return __swiotlb_dma_supported(hwdev, mask); } EXPORT_SYMBOL(swiotlb_init); @@ -790,6 +906,4 @@ EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_device); EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu); EXPORT_SYMBOL(swiotlb_sync_sg_for_device); EXPORT_SYMBOL(swiotlb_dma_mapping_error); -EXPORT_SYMBOL(swiotlb_alloc_coherent); -EXPORT_SYMBOL(swiotlb_free_coherent); EXPORT_SYMBOL(swiotlb_dma_supported); -- cgit v1.2.3