diff options
Diffstat (limited to 'arch/mips/mm/tlbex.c')
| -rw-r--r-- | arch/mips/mm/tlbex.c | 165 |
1 files changed, 136 insertions, 29 deletions
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 1c8ac49ec72c..4d46d3787576 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -5,8 +5,8 @@ * * Synthesize TLB refill handlers at runtime. * - * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer - * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki + * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer + * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) * Copyright (C) 2008, 2009 Cavium Networks, Inc. * Copyright (C) 2011 MIPS Technologies, Inc. @@ -29,6 +29,7 @@ #include <linux/init.h> #include <linux/cache.h> +#include <asm/mmu_context.h> #include <asm/cacheflush.h> #include <asm/pgtable.h> #include <asm/war.h> @@ -212,7 +213,7 @@ static void __cpuinit uasm_bgezl_label(struct uasm_label **l, /* * pgtable bits are assigned dynamically depending on processor feature * and statically based on kernel configuration. This spits out the actual - * values the kernel is using. Required to make sense from disassembled + * values the kernel is using. Required to make sense from disassembled * TLB exception handlers. */ static void output_pgtable_bits_defines(void) @@ -305,6 +306,78 @@ static struct uasm_reloc relocs[128] __cpuinitdata; static int check_for_high_segbits __cpuinitdata; #endif +static void __cpuinit insn_fixup(unsigned int **start, unsigned int **stop, + unsigned int i_const) +{ + unsigned int **p; + + for (p = start; p < stop; p++) { +#ifndef CONFIG_CPU_MICROMIPS + unsigned int *ip; + + ip = *p; + *ip = (*ip & 0xffff0000) | i_const; +#else + unsigned short *ip; + + ip = ((unsigned short *)((unsigned int)*p - 1)); + if ((*ip & 0xf000) == 0x4000) { + *ip &= 0xfff1; + *ip |= (i_const << 1); + } else if ((*ip & 0xf000) == 0x6000) { + *ip &= 0xfff1; + *ip |= ((i_const >> 2) << 1); + } else { + ip++; + *ip = i_const; + } +#endif + local_flush_icache_range((unsigned long)ip, + (unsigned long)ip + sizeof(*ip)); + } +} + +#define asid_insn_fixup(section, const) \ +do { \ + extern unsigned int *__start_ ## section; \ + extern unsigned int *__stop_ ## section; \ + insn_fixup(&__start_ ## section, &__stop_ ## section, const); \ +} while(0) + +/* + * Caller is assumed to flush the caches before the first context switch. + */ +static void __cpuinit setup_asid(unsigned int inc, unsigned int mask, + unsigned int version_mask, + unsigned int first_version) +{ + extern asmlinkage void handle_ri_rdhwr_vivt(void); + unsigned long *vivt_exc; + +#ifdef CONFIG_CPU_MICROMIPS + /* + * Worst case optimised microMIPS addiu instructions support + * only a 3-bit immediate value. + */ + if(inc > 7) + panic("Invalid ASID increment value!"); +#endif + asid_insn_fixup(__asid_inc, inc); + asid_insn_fixup(__asid_mask, mask); + asid_insn_fixup(__asid_version_mask, version_mask); + asid_insn_fixup(__asid_first_version, first_version); + + /* Patch up the 'handle_ri_rdhwr_vivt' handler. */ + vivt_exc = (unsigned long *) &handle_ri_rdhwr_vivt; +#ifdef CONFIG_CPU_MICROMIPS + vivt_exc = (unsigned long *)((unsigned long) vivt_exc - 1); +#endif + vivt_exc++; + *vivt_exc = (*vivt_exc & ~mask) | mask; + + current_cpu_data.asid_cache = first_version; +} + static int check_for_high_segbits __cpuinitdata; static unsigned int kscratch_used_mask __cpuinitdata; @@ -464,8 +537,8 @@ static u32 final_handler[64] __cpuinitdata; * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: * 2. A timing hazard exists for the TLBP instruction. * - * stalling_instruction - * TLBP + * stalling_instruction + * TLBP * * The JTLB is being read for the TLBP throughout the stall generated by the * previous instruction. This is not really correct as the stalling instruction @@ -476,7 +549,7 @@ static u32 final_handler[64] __cpuinitdata; * The software work-around is to not allow the instruction preceding the TLBP * to stall - make it an NOP or some other instruction guaranteed not to stall. * - * Errata 2 will not be fixed. This errata is also on the R5000. + * Errata 2 will not be fixed. This errata is also on the R5000. * * As if we MIPS hackers wouldn't know how to nop pipelines happy ... */ @@ -581,6 +654,7 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, case CPU_4KC: case CPU_4KEC: case CPU_M14KC: + case CPU_M14KEC: case CPU_SB1: case CPU_SB1A: case CPU_4KSC: @@ -748,7 +822,7 @@ static __cpuinit void build_huge_update_entries(u32 **p, */ small_sequence = (HPAGE_SIZE >> 7) < 0x10000; - /* We can clobber tmp. It isn't used after this.*/ + /* We can clobber tmp. It isn't used after this.*/ if (!small_sequence) uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); @@ -830,12 +904,12 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, /* Clear lower 23 bits of context. */ uasm_i_dins(p, ptr, 0, 0, 23); - /* 1 0 1 0 1 << 6 xkphys cached */ + /* 1 0 1 0 1 << 6 xkphys cached */ uasm_i_ori(p, ptr, ptr, 0x540); uasm_i_drotr(p, ptr, ptr, 11); } #elif defined(CONFIG_SMP) -# ifdef CONFIG_MIPS_MT_SMTC +# ifdef CONFIG_MIPS_MT_SMTC /* * SMTC uses TCBind value as "CPU" index */ @@ -955,7 +1029,7 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ #ifdef CONFIG_SMP -#ifdef CONFIG_MIPS_MT_SMTC +#ifdef CONFIG_MIPS_MT_SMTC /* * SMTC uses TCBind value as "CPU" index */ @@ -965,7 +1039,7 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) #else /* * smp_processor_id() << 3 is stored in CONTEXT. - */ + */ uasm_i_mfc0(p, ptr, C0_CONTEXT); UASM_i_LA_mostly(p, tmp, pgdc); uasm_i_srl(p, ptr, ptr, 23); @@ -1153,7 +1227,7 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, if (pgd_reg == -1) { vmalloc_branch_delay_filled = 1; - /* 1 0 1 0 1 << 6 xkphys cached */ + /* 1 0 1 0 1 << 6 xkphys cached */ uasm_i_ori(p, ptr, ptr, 0x540); uasm_i_drotr(p, ptr, ptr, 11); } @@ -1171,9 +1245,9 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, uasm_l_vmalloc_done(l, *p); /* - * tmp ptr - * fall-through case = badvaddr *pgd_current - * vmalloc case = badvaddr swapper_pg_dir + * tmp ptr + * fall-through case = badvaddr *pgd_current + * vmalloc case = badvaddr swapper_pg_dir */ if (vmalloc_branch_delay_filled) @@ -1212,7 +1286,7 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); /* * The in the LWX case we don't want to do the load in the - * delay slot. It cannot issue in the same cycle and may be + * delay slot. It cannot issue in the same cycle and may be * speculative and unneeded. */ if (use_lwx_insns()) @@ -1457,17 +1531,17 @@ u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned; u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned; u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; #ifdef CONFIG_MIPS_PGD_C0_CONTEXT -u32 tlbmiss_handler_setup_pgd[16] __cacheline_aligned; +u32 tlbmiss_handler_setup_pgd_array[16] __cacheline_aligned; static void __cpuinit build_r4000_setup_pgd(void) { const int a0 = 4; const int a1 = 5; - u32 *p = tlbmiss_handler_setup_pgd; + u32 *p = tlbmiss_handler_setup_pgd_array; struct uasm_label *l = labels; struct uasm_reloc *r = relocs; - memset(tlbmiss_handler_setup_pgd, 0, sizeof(tlbmiss_handler_setup_pgd)); + memset(tlbmiss_handler_setup_pgd_array, 0, sizeof(tlbmiss_handler_setup_pgd_array)); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); @@ -1495,15 +1569,15 @@ static void __cpuinit build_r4000_setup_pgd(void) uasm_i_jr(&p, 31); UASM_i_MTC0(&p, a0, 31, pgd_reg); } - if (p - tlbmiss_handler_setup_pgd > ARRAY_SIZE(tlbmiss_handler_setup_pgd)) - panic("tlbmiss_handler_setup_pgd space exceeded"); + if (p - tlbmiss_handler_setup_pgd_array > ARRAY_SIZE(tlbmiss_handler_setup_pgd_array)) + panic("tlbmiss_handler_setup_pgd_array space exceeded"); uasm_resolve_relocs(relocs, labels); - pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n", - (unsigned int)(p - tlbmiss_handler_setup_pgd)); + pr_debug("Wrote tlbmiss_handler_setup_pgd_array (%u instructions).\n", + (unsigned int)(p - tlbmiss_handler_setup_pgd_array)); dump_handler("tlbmiss_handler", - tlbmiss_handler_setup_pgd, - ARRAY_SIZE(tlbmiss_handler_setup_pgd)); + tlbmiss_handler_setup_pgd_array, + ARRAY_SIZE(tlbmiss_handler_setup_pgd_array)); } #endif @@ -2029,6 +2103,13 @@ static void __cpuinit build_r4000_tlb_load_handler(void) uasm_l_nopage_tlbl(&l, p); build_restore_work_registers(&p); +#ifdef CONFIG_CPU_MICROMIPS + if ((unsigned long)tlb_do_page_fault_0 & 1) { + uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0)); + uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0)); + uasm_i_jr(&p, K0); + } else +#endif uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); uasm_i_nop(&p); @@ -2076,6 +2157,13 @@ static void __cpuinit build_r4000_tlb_store_handler(void) uasm_l_nopage_tlbs(&l, p); build_restore_work_registers(&p); +#ifdef CONFIG_CPU_MICROMIPS + if ((unsigned long)tlb_do_page_fault_1 & 1) { + uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); + uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); + uasm_i_jr(&p, K0); + } else +#endif uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); uasm_i_nop(&p); @@ -2124,6 +2212,13 @@ static void __cpuinit build_r4000_tlb_modify_handler(void) uasm_l_nopage_tlbm(&l, p); build_restore_work_registers(&p); +#ifdef CONFIG_CPU_MICROMIPS + if ((unsigned long)tlb_do_page_fault_1 & 1) { + uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); + uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); + uasm_i_jr(&p, K0); + } else +#endif uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); uasm_i_nop(&p); @@ -2161,8 +2256,12 @@ void __cpuinit build_tlb_refill_handler(void) case CPU_TX3922: case CPU_TX3927: #ifndef CONFIG_MIPS_PGD_C0_CONTEXT - build_r3000_tlb_refill_handler(); + setup_asid(0x40, 0xfc0, 0xf000, ASID_FIRST_VERSION_R3000); + if (cpu_has_local_ebase) + build_r3000_tlb_refill_handler(); if (!run_once) { + if (!cpu_has_local_ebase) + build_r3000_tlb_refill_handler(); build_r3000_tlb_load_handler(); build_r3000_tlb_store_handler(); build_r3000_tlb_modify_handler(); @@ -2183,6 +2282,11 @@ void __cpuinit build_tlb_refill_handler(void) break; default: +#ifndef CONFIG_MIPS_MT_SMTC + setup_asid(0x1, 0xff, 0xff00, ASID_FIRST_VERSION_R4000); +#else + setup_asid(0x1, smtc_asid_mask, 0xff00, ASID_FIRST_VERSION_R4000); +#endif if (!run_once) { scratch_reg = allocate_kscratch(); #ifdef CONFIG_MIPS_PGD_C0_CONTEXT @@ -2191,9 +2295,12 @@ void __cpuinit build_tlb_refill_handler(void) build_r4000_tlb_load_handler(); build_r4000_tlb_store_handler(); build_r4000_tlb_modify_handler(); + if (!cpu_has_local_ebase) + build_r4000_tlb_refill_handler(); run_once++; } - build_r4000_tlb_refill_handler(); + if (cpu_has_local_ebase) + build_r4000_tlb_refill_handler(); } } @@ -2206,7 +2313,7 @@ void __cpuinit flush_tlb_handlers(void) local_flush_icache_range((unsigned long)handle_tlbm, (unsigned long)handle_tlbm + sizeof(handle_tlbm)); #ifdef CONFIG_MIPS_PGD_C0_CONTEXT - local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd, - (unsigned long)tlbmiss_handler_setup_pgd + sizeof(handle_tlbm)); + local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd_array, + (unsigned long)tlbmiss_handler_setup_pgd_array + sizeof(handle_tlbm)); #endif } |