AcoSail/kernel_optimize_test
8
0
Fork 0
forked from luck/tmp_suning_uos_patched
Code Pull Requests Activity
da8cadb31b
kernel_optimize_test/arch/frv/mm/tlb-miss.S
David Howells 3c835670ab FRV: arrange things such that BRA can reach from the trap table 
Arrange the sections in the FRV arch so that a BRA instruction with a
16-bit displacement can always reach from the trap table to entry.S,
tlb-miss.S and break.S.

The problem otherwise is that the linker can insert sufficient code between
the slots in the trap table and the targets of the branch instructions in
those slots that the displacement field in the instruction isn't
sufficiently large.  This is because the branch targets were in the .text
section along with most of the other code in the kernel.

Signed-off-by: David Howells <dhowells@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-11-29 09:24:54 -08:00

631 lines
18 KiB
ArmAsm
Raw Blame History

/* tlb-miss.S: TLB miss handlers
*
* Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
* Written by David Howells (dhowells@redhat.com)
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
*/
#include <linux/sys.h>
#include <linux/linkage.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/highmem.h>
#include <asm/spr-regs.h>
.section .text.tlbmiss
.balign 4
.globl __entry_insn_mmu_miss
__entry_insn_mmu_miss:
break
nop
.globl __entry_insn_mmu_exception
__entry_insn_mmu_exception:
break
nop
.globl __entry_data_mmu_miss
__entry_data_mmu_miss:
break
nop
.globl __entry_data_mmu_exception
__entry_data_mmu_exception:
break
nop
###############################################################################
#
# handle a lookup failure of one sort or another in a kernel TLB handler
# On entry:
# GR29 - faulting address
# SCR2 - saved CCR
#
###############################################################################
.type __tlb_kernel_fault,@function
__tlb_kernel_fault:
# see if we're supposed to re-enable single-step mode upon return
sethi.p %hi(__break_tlb_miss_return_break),gr30
setlo %lo(__break_tlb_miss_return_break),gr30
movsg pcsr,gr31
subcc gr31,gr30,gr0,icc0
beq icc0,#0,__tlb_kernel_fault_sstep
movsg scr2,gr30
movgs gr30,ccr
movgs gr29,scr2 /* save EAR0 value */
sethi.p %hi(__kernel_current_task),gr29
setlo %lo(__kernel_current_task),gr29
ldi.p @(gr29,#0),gr29 /* restore GR29 */
bra __entry_kernel_handle_mmu_fault
# we've got to re-enable single-stepping
__tlb_kernel_fault_sstep:
sethi.p %hi(__break_tlb_miss_real_return_info),gr30
setlo %lo(__break_tlb_miss_real_return_info),gr30
lddi @(gr30,0),gr30
movgs gr30,pcsr
movgs gr31,psr
movsg scr2,gr30
movgs gr30,ccr
movgs gr29,scr2 /* save EAR0 value */
sethi.p %hi(__kernel_current_task),gr29
setlo %lo(__kernel_current_task),gr29
ldi.p @(gr29,#0),gr29 /* restore GR29 */
bra __entry_kernel_handle_mmu_fault_sstep
.size __tlb_kernel_fault, .-__tlb_kernel_fault
###############################################################################
#
# handle a lookup failure of one sort or another in a user TLB handler
# On entry:
# GR28 - faulting address
# SCR2 - saved CCR
#
###############################################################################
.type __tlb_user_fault,@function
__tlb_user_fault:
# see if we're supposed to re-enable single-step mode upon return
sethi.p %hi(__break_tlb_miss_return_break),gr30
setlo %lo(__break_tlb_miss_return_break),gr30
movsg pcsr,gr31
subcc gr31,gr30,gr0,icc0
beq icc0,#0,__tlb_user_fault_sstep
movsg scr2,gr30
movgs gr30,ccr
bra __entry_uspace_handle_mmu_fault
# we've got to re-enable single-stepping
__tlb_user_fault_sstep:
sethi.p %hi(__break_tlb_miss_real_return_info),gr30
setlo %lo(__break_tlb_miss_real_return_info),gr30
lddi @(gr30,0),gr30
movgs gr30,pcsr
movgs gr31,psr
movsg scr2,gr30
movgs gr30,ccr
bra __entry_uspace_handle_mmu_fault_sstep
.size __tlb_user_fault, .-__tlb_user_fault
###############################################################################
#
# Kernel instruction TLB miss handler
# On entry:
# GR1 - kernel stack pointer
# GR28 - saved exception frame pointer
# GR29 - faulting address
# GR31 - EAR0 ^ SCR0
# SCR0 - base of virtual range covered by cached PGE from last ITLB miss (or 0xffffffff)
# DAMR3 - mapped page directory
# DAMR4 - mapped page table as matched by SCR0
#
###############################################################################
.globl __entry_kernel_insn_tlb_miss
.type __entry_kernel_insn_tlb_miss,@function
__entry_kernel_insn_tlb_miss:
#if 0
sethi.p %hi(0xe1200004),gr30
setlo %lo(0xe1200004),gr30
st gr0,@(gr30,gr0)
sethi.p %hi(0xffc00100),gr30
setlo %lo(0xffc00100),gr30
sth gr30,@(gr30,gr0)
membar
#endif
movsg ccr,gr30 /* save CCR */
movgs gr30,scr2
# see if the cached page table mapping is appropriate
srlicc.p gr31,#26,gr0,icc0
setlos 0x3ffc,gr30
srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */
bne icc0,#0,__itlb_k_PTD_miss
__itlb_k_PTD_mapped:
# access the PTD with EAR0[25:14]
# - DAMLR4 points to the virtual address of the appropriate page table
# - the PTD holds 4096 PTEs
# - the PTD must be accessed uncached
# - the PTE must be marked accessed if it was valid
#
and gr31,gr30,gr31
movsg damlr4,gr30
add gr30,gr31,gr31
ldi @(gr31,#0),gr30 /* fetch the PTE */
andicc gr30,#_PAGE_PRESENT,gr0,icc0
ori.p gr30,#_PAGE_ACCESSED,gr30
beq icc0,#0,__tlb_kernel_fault /* jump if PTE invalid */
sti.p gr30,@(gr31,#0) /* update the PTE */
andi gr30,#~_PAGE_ACCESSED,gr30
# we're using IAMR1 as an extra TLB entry
# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
# - need to check DAMR1 lest we cause an multiple-DAT-hit exception
# - IAMPR1 has no WP bit, and we mustn't lose WP information
movsg iampr1,gr31
andicc gr31,#xAMPRx_V,gr0,icc0
setlos.p 0xfffff000,gr31
beq icc0,#0,__itlb_k_nopunt /* punt not required */
movsg iamlr1,gr31
movgs gr31,tplr /* set TPLR.CXN */
tlbpr gr31,gr0,#4,#0 /* delete matches from TLB, IAMR1, DAMR1 */
movsg dampr1,gr31
ori gr31,#xAMPRx_V,gr31 /* entry was invalidated by tlbpr #4 */
movgs gr31,tppr
movsg iamlr1,gr31 /* set TPLR.CXN */
movgs gr31,tplr
tlbpr gr31,gr0,#2,#0 /* save to the TLB */
movsg tpxr,gr31 /* check the TLB write error flag */
andicc.p gr31,#TPXR_E,gr0,icc0
setlos #0xfffff000,gr31
bne icc0,#0,__tlb_kernel_fault
__itlb_k_nopunt:
# assemble the new TLB entry
and gr29,gr31,gr29
movsg cxnr,gr31
or gr29,gr31,gr29
movgs gr29,iamlr1 /* xAMLR = address | context number */
movgs gr30,iampr1
movgs gr29,damlr1
movgs gr30,dampr1
# return, restoring registers
movsg scr2,gr30
movgs gr30,ccr
sethi.p %hi(__kernel_current_task),gr29
setlo %lo(__kernel_current_task),gr29
ldi @(gr29,#0),gr29
rett #0
beq icc0,#3,0 /* prevent icache prefetch */
# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
# appropriate page table and map that instead
# - access the PGD with EAR0[31:26]
# - DAMLR3 points to the virtual address of the page directory
# - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
__itlb_k_PTD_miss:
srli gr29,#26,gr31 /* calculate PGE offset */
slli gr31,#8,gr31 /* and clear bottom bits */
movsg damlr3,gr30
ld @(gr31,gr30),gr30 /* access the PGE */
andicc.p gr30,#_PAGE_PRESENT,gr0,icc0
andicc gr30,#xAMPRx_SS,gr0,icc1
# map this PTD instead and record coverage address
ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
beq icc0,#0,__tlb_kernel_fault /* jump if PGE not present */
slli.p gr31,#18,gr31
bne icc1,#0,__itlb_k_bigpage
movgs gr30,dampr4
movgs gr31,scr0
# we can now resume normal service
setlos 0x3ffc,gr30
srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */
bra __itlb_k_PTD_mapped
__itlb_k_bigpage:
break
nop
.size __entry_kernel_insn_tlb_miss, .-__entry_kernel_insn_tlb_miss
###############################################################################
#
# Kernel data TLB miss handler
# On entry:
# GR1 - kernel stack pointer
# GR28 - saved exception frame pointer
# GR29 - faulting address
# GR31 - EAR0 ^ SCR1
# SCR1 - base of virtual range covered by cached PGE from last DTLB miss (or 0xffffffff)
# DAMR3 - mapped page directory
# DAMR5 - mapped page table as matched by SCR1
#
###############################################################################
.globl __entry_kernel_data_tlb_miss
.type __entry_kernel_data_tlb_miss,@function
__entry_kernel_data_tlb_miss:
#if 0
sethi.p %hi(0xe1200004),gr30
setlo %lo(0xe1200004),gr30
st gr0,@(gr30,gr0)
sethi.p %hi(0xffc00100),gr30
setlo %lo(0xffc00100),gr30
sth gr30,@(gr30,gr0)
membar
#endif
movsg ccr,gr30 /* save CCR */
movgs gr30,scr2
# see if the cached page table mapping is appropriate
srlicc.p gr31,#26,gr0,icc0
setlos 0x3ffc,gr30
srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */
bne icc0,#0,__dtlb_k_PTD_miss
__dtlb_k_PTD_mapped:
# access the PTD with EAR0[25:14]
# - DAMLR5 points to the virtual address of the appropriate page table
# - the PTD holds 4096 PTEs
# - the PTD must be accessed uncached
# - the PTE must be marked accessed if it was valid
#
and gr31,gr30,gr31
movsg damlr5,gr30
add gr30,gr31,gr31
ldi @(gr31,#0),gr30 /* fetch the PTE */
andicc gr30,#_PAGE_PRESENT,gr0,icc0
ori.p gr30,#_PAGE_ACCESSED,gr30
beq icc0,#0,__tlb_kernel_fault /* jump if PTE invalid */
sti.p gr30,@(gr31,#0) /* update the PTE */
andi gr30,#~_PAGE_ACCESSED,gr30
# we're using DAMR1 as an extra TLB entry
# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
# - need to check IAMR1 lest we cause an multiple-DAT-hit exception
movsg dampr1,gr31
andicc gr31,#xAMPRx_V,gr0,icc0
setlos.p 0xfffff000,gr31
beq icc0,#0,__dtlb_k_nopunt /* punt not required */
movsg damlr1,gr31
movgs gr31,tplr /* set TPLR.CXN */
tlbpr gr31,gr0,#4,#0 /* delete matches from TLB, IAMR1, DAMR1 */
movsg dampr1,gr31
ori gr31,#xAMPRx_V,gr31 /* entry was invalidated by tlbpr #4 */
movgs gr31,tppr
movsg damlr1,gr31 /* set TPLR.CXN */
movgs gr31,tplr
tlbpr gr31,gr0,#2,#0 /* save to the TLB */
movsg tpxr,gr31 /* check the TLB write error flag */
andicc.p gr31,#TPXR_E,gr0,icc0
setlos #0xfffff000,gr31
bne icc0,#0,__tlb_kernel_fault
__dtlb_k_nopunt:
# assemble the new TLB entry
and gr29,gr31,gr29
movsg cxnr,gr31
or gr29,gr31,gr29
movgs gr29,iamlr1 /* xAMLR = address | context number */
movgs gr30,iampr1
movgs gr29,damlr1
movgs gr30,dampr1
# return, restoring registers
movsg scr2,gr30
movgs gr30,ccr
sethi.p %hi(__kernel_current_task),gr29
setlo %lo(__kernel_current_task),gr29
ldi @(gr29,#0),gr29
rett #0
beq icc0,#3,0 /* prevent icache prefetch */
# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
# appropriate page table and map that instead
# - access the PGD with EAR0[31:26]
# - DAMLR3 points to the virtual address of the page directory
# - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
__dtlb_k_PTD_miss:
srli gr29,#26,gr31 /* calculate PGE offset */
slli gr31,#8,gr31 /* and clear bottom bits */
movsg damlr3,gr30
ld @(gr31,gr30),gr30 /* access the PGE */
andicc.p gr30,#_PAGE_PRESENT,gr0,icc0
andicc gr30,#xAMPRx_SS,gr0,icc1
# map this PTD instead and record coverage address
ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
beq icc0,#0,__tlb_kernel_fault /* jump if PGE not present */
slli.p gr31,#18,gr31
bne icc1,#0,__dtlb_k_bigpage
movgs gr30,dampr5
movgs gr31,scr1
# we can now resume normal service
setlos 0x3ffc,gr30
srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */
bra __dtlb_k_PTD_mapped
__dtlb_k_bigpage:
break
nop
.size __entry_kernel_data_tlb_miss, .-__entry_kernel_data_tlb_miss
###############################################################################
#
# Userspace instruction TLB miss handler (with PGE prediction)
# On entry:
# GR28 - faulting address
# GR31 - EAR0 ^ SCR0
# SCR0 - base of virtual range covered by cached PGE from last ITLB miss (or 0xffffffff)
# DAMR3 - mapped page directory
# DAMR4 - mapped page table as matched by SCR0
#
###############################################################################
.globl __entry_user_insn_tlb_miss
.type __entry_user_insn_tlb_miss,@function
__entry_user_insn_tlb_miss:
#if 0
sethi.p %hi(0xe1200004),gr30
setlo %lo(0xe1200004),gr30
st gr0,@(gr30,gr0)
sethi.p %hi(0xffc00100),gr30
setlo %lo(0xffc00100),gr30
sth gr30,@(gr30,gr0)
membar
#endif
movsg ccr,gr30 /* save CCR */
movgs gr30,scr2
# see if the cached page table mapping is appropriate
srlicc.p gr31,#26,gr0,icc0
setlos 0x3ffc,gr30
srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
bne icc0,#0,__itlb_u_PTD_miss
__itlb_u_PTD_mapped:
# access the PTD with EAR0[25:14]
# - DAMLR4 points to the virtual address of the appropriate page table
# - the PTD holds 4096 PTEs
# - the PTD must be accessed uncached
# - the PTE must be marked accessed if it was valid
#
and gr31,gr30,gr31
movsg damlr4,gr30
add gr30,gr31,gr31
ldi @(gr31,#0),gr30 /* fetch the PTE */
andicc gr30,#_PAGE_PRESENT,gr0,icc0
ori.p gr30,#_PAGE_ACCESSED,gr30
beq icc0,#0,__tlb_user_fault /* jump if PTE invalid */
sti.p gr30,@(gr31,#0) /* update the PTE */
andi gr30,#~_PAGE_ACCESSED,gr30
# we're using IAMR1/DAMR1 as an extra TLB entry
# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
movsg dampr1,gr31
andicc gr31,#xAMPRx_V,gr0,icc0
setlos.p 0xfffff000,gr31
beq icc0,#0,__itlb_u_nopunt /* punt not required */
movsg dampr1,gr31
movgs gr31,tppr
movsg damlr1,gr31 /* set TPLR.CXN */
movgs gr31,tplr
tlbpr gr31,gr0,#2,#0 /* save to the TLB */
movsg tpxr,gr31 /* check the TLB write error flag */
andicc.p gr31,#TPXR_E,gr0,icc0
setlos #0xfffff000,gr31
bne icc0,#0,__tlb_user_fault
__itlb_u_nopunt:
# assemble the new TLB entry
and gr28,gr31,gr28
movsg cxnr,gr31
or gr28,gr31,gr28
movgs gr28,iamlr1 /* xAMLR = address | context number */
movgs gr30,iampr1
movgs gr28,damlr1
movgs gr30,dampr1
# return, restoring registers
movsg scr2,gr30
movgs gr30,ccr
rett #0
beq icc0,#3,0 /* prevent icache prefetch */
# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
# appropriate page table and map that instead
# - access the PGD with EAR0[31:26]
# - DAMLR3 points to the virtual address of the page directory
# - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
__itlb_u_PTD_miss:
srli gr28,#26,gr31 /* calculate PGE offset */
slli gr31,#8,gr31 /* and clear bottom bits */
movsg damlr3,gr30
ld @(gr31,gr30),gr30 /* access the PGE */
andicc.p gr30,#_PAGE_PRESENT,gr0,icc0
andicc gr30,#xAMPRx_SS,gr0,icc1
# map this PTD instead and record coverage address
ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
beq icc0,#0,__tlb_user_fault /* jump if PGE not present */
slli.p gr31,#18,gr31
bne icc1,#0,__itlb_u_bigpage
movgs gr30,dampr4
movgs gr31,scr0
# we can now resume normal service
setlos 0x3ffc,gr30
srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
bra __itlb_u_PTD_mapped
__itlb_u_bigpage:
break
nop
.size __entry_user_insn_tlb_miss, .-__entry_user_insn_tlb_miss
###############################################################################
#
# Userspace data TLB miss handler
# On entry:
# GR28 - faulting address
# GR31 - EAR0 ^ SCR1
# SCR1 - base of virtual range covered by cached PGE from last DTLB miss (or 0xffffffff)
# DAMR3 - mapped page directory
# DAMR5 - mapped page table as matched by SCR1
#
###############################################################################
.globl __entry_user_data_tlb_miss
.type __entry_user_data_tlb_miss,@function
__entry_user_data_tlb_miss:
#if 0
sethi.p %hi(0xe1200004),gr30
setlo %lo(0xe1200004),gr30
st gr0,@(gr30,gr0)
sethi.p %hi(0xffc00100),gr30
setlo %lo(0xffc00100),gr30
sth gr30,@(gr30,gr0)
membar
#endif
movsg ccr,gr30 /* save CCR */
movgs gr30,scr2
# see if the cached page table mapping is appropriate
srlicc.p gr31,#26,gr0,icc0
setlos 0x3ffc,gr30
srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
bne icc0,#0,__dtlb_u_PTD_miss
__dtlb_u_PTD_mapped:
# access the PTD with EAR0[25:14]
# - DAMLR5 points to the virtual address of the appropriate page table
# - the PTD holds 4096 PTEs
# - the PTD must be accessed uncached
# - the PTE must be marked accessed if it was valid
#
and gr31,gr30,gr31
movsg damlr5,gr30
__dtlb_u_using_iPTD:
add gr30,gr31,gr31
ldi @(gr31,#0),gr30 /* fetch the PTE */
andicc gr30,#_PAGE_PRESENT,gr0,icc0
ori.p gr30,#_PAGE_ACCESSED,gr30
beq icc0,#0,__tlb_user_fault /* jump if PTE invalid */
sti.p gr30,@(gr31,#0) /* update the PTE */
andi gr30,#~_PAGE_ACCESSED,gr30
# we're using DAMR1 as an extra TLB entry
# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
movsg dampr1,gr31
andicc gr31,#xAMPRx_V,gr0,icc0
setlos.p 0xfffff000,gr31
beq icc0,#0,__dtlb_u_nopunt /* punt not required */
movsg dampr1,gr31
movgs gr31,tppr
movsg damlr1,gr31 /* set TPLR.CXN */
movgs gr31,tplr
tlbpr gr31,gr0,#2,#0 /* save to the TLB */
movsg tpxr,gr31 /* check the TLB write error flag */
andicc.p gr31,#TPXR_E,gr0,icc0
setlos #0xfffff000,gr31
bne icc0,#0,__tlb_user_fault
__dtlb_u_nopunt:
# assemble the new TLB entry
and gr28,gr31,gr28
movsg cxnr,gr31
or gr28,gr31,gr28
movgs gr28,iamlr1 /* xAMLR = address | context number */
movgs gr30,iampr1
movgs gr28,damlr1
movgs gr30,dampr1
# return, restoring registers
movsg scr2,gr30
movgs gr30,ccr
rett #0
beq icc0,#3,0 /* prevent icache prefetch */
# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
# appropriate page table and map that instead
# - first of all, check the insn PGE cache - we may well get a hit there
# - access the PGD with EAR0[31:26]
# - DAMLR3 points to the virtual address of the page directory
# - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
__dtlb_u_PTD_miss:
movsg scr0,gr31 /* consult the insn-PGE-cache key */
xor gr28,gr31,gr31
srlicc gr31,#26,gr0,icc0
srli gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
bne icc0,#0,__dtlb_u_iPGE_miss
# what we're looking for is covered by the insn-PGE-cache
setlos 0x3ffc,gr30
and gr31,gr30,gr31
movsg damlr4,gr30
bra __dtlb_u_using_iPTD
__dtlb_u_iPGE_miss:
srli gr28,#26,gr31 /* calculate PGE offset */
slli gr31,#8,gr31 /* and clear bottom bits */
movsg damlr3,gr30
ld @(gr31,gr30),gr30 /* access the PGE */
andicc.p gr30,#_PAGE_PRESENT,gr0,icc0
andicc gr30,#xAMPRx_SS,gr0,icc1
# map this PTD instead and record coverage address
ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
beq icc0,#0,__tlb_user_fault /* jump if PGE not present */
slli.p gr31,#18,gr31
bne icc1,#0,__dtlb_u_bigpage
movgs gr30,dampr5
movgs gr31,scr1
# we can now resume normal service
setlos 0x3ffc,gr30
srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
bra __dtlb_u_PTD_mapped
__dtlb_u_bigpage:
break
nop
.size __entry_user_data_tlb_miss, .-__entry_user_data_tlb_miss
View Git Blame Copy Permalink