diff options
Diffstat (limited to 'arch/powerpc/include')
| -rw-r--r-- | arch/powerpc/include/asm/exception-64e.h | 52 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/paca.h | 7 |
2 files changed, 38 insertions, 21 deletions
diff --git a/arch/powerpc/include/asm/exception-64e.h b/arch/powerpc/include/asm/exception-64e.h index 6d53f311d942..ac13addb8495 100644 --- a/arch/powerpc/include/asm/exception-64e.h +++ b/arch/powerpc/include/asm/exception-64e.h @@ -48,30 +48,33 @@ #define EX_R14 (4 * 8) #define EX_R15 (5 * 8) -/* The TLB miss exception uses different slots */ +/* + * The TLB miss exception uses different slots. + * + * The bolted variant uses only the first six fields, + * which in combination with pgd and kernel_pgd fits in + * one 64-byte cache line. + */ #define EX_TLB_R10 ( 0 * 8) #define EX_TLB_R11 ( 1 * 8) -#define EX_TLB_R12 ( 2 * 8) -#define EX_TLB_R13 ( 3 * 8) -#define EX_TLB_R14 ( 4 * 8) -#define EX_TLB_R15 ( 5 * 8) -#define EX_TLB_R16 ( 6 * 8) -#define EX_TLB_CR ( 7 * 8) +#define EX_TLB_R14 ( 2 * 8) +#define EX_TLB_R15 ( 3 * 8) +#define EX_TLB_R16 ( 4 * 8) +#define EX_TLB_CR ( 5 * 8) +#define EX_TLB_R12 ( 6 * 8) +#define EX_TLB_R13 ( 7 * 8) #define EX_TLB_DEAR ( 8 * 8) /* Level 0 and 2 only */ #define EX_TLB_ESR ( 9 * 8) /* Level 0 and 2 only */ #define EX_TLB_SRR0 (10 * 8) #define EX_TLB_SRR1 (11 * 8) -#define EX_TLB_MMUCR0 (12 * 8) /* Level 0 */ -#define EX_TLB_MAS1 (12 * 8) /* Level 0 */ -#define EX_TLB_MAS2 (13 * 8) /* Level 0 */ #ifdef CONFIG_BOOK3E_MMU_TLB_STATS -#define EX_TLB_R8 (14 * 8) -#define EX_TLB_R9 (15 * 8) -#define EX_TLB_LR (16 * 8) -#define EX_TLB_SIZE (17 * 8) +#define EX_TLB_R8 (12 * 8) +#define EX_TLB_R9 (13 * 8) +#define EX_TLB_LR (14 * 8) +#define EX_TLB_SIZE (15 * 8) #else -#define EX_TLB_SIZE (14 * 8) +#define EX_TLB_SIZE (12 * 8) #endif #define START_EXCEPTION(label) \ @@ -168,6 +171,16 @@ exc_##label##_book3e: ld r9,EX_TLB_R9(r12); \ ld r8,EX_TLB_R8(r12); \ mtlr r16; +#define TLB_MISS_PROLOG_STATS_BOLTED \ + mflr r10; \ + std r8,PACA_EXTLB+EX_TLB_R8(r13); \ + std r9,PACA_EXTLB+EX_TLB_R9(r13); \ + std r10,PACA_EXTLB+EX_TLB_LR(r13); +#define TLB_MISS_RESTORE_STATS_BOLTED \ + ld r16,PACA_EXTLB+EX_TLB_LR(r13); \ + ld r9,PACA_EXTLB+EX_TLB_R9(r13); \ + ld r8,PACA_EXTLB+EX_TLB_R8(r13); \ + mtlr r16; #define TLB_MISS_STATS_D(name) \ addi r9,r13,MMSTAT_DSTATS+name; \ bl .tlb_stat_inc; @@ -183,17 +196,20 @@ exc_##label##_book3e: 61: addi r9,r13,MMSTAT_ISTATS+name; \ 62: bl .tlb_stat_inc; #define TLB_MISS_STATS_SAVE_INFO \ - std r14,EX_TLB_ESR(r12); /* save ESR */ \ - - + std r14,EX_TLB_ESR(r12); /* save ESR */ +#define TLB_MISS_STATS_SAVE_INFO_BOLTED \ + std r14,PACA_EXTLB+EX_TLB_ESR(r13); /* save ESR */ #else #define TLB_MISS_PROLOG_STATS #define TLB_MISS_RESTORE_STATS +#define TLB_MISS_PROLOG_STATS_BOLTED +#define TLB_MISS_RESTORE_STATS_BOLTED #define TLB_MISS_STATS_D(name) #define TLB_MISS_STATS_I(name) #define TLB_MISS_STATS_X(name) #define TLB_MISS_STATS_Y(name) #define TLB_MISS_STATS_SAVE_INFO +#define TLB_MISS_STATS_SAVE_INFO_BOLTED #endif #define SET_IVOR(vector_number, vector_offset) \ diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h index 74126765106a..c1f65f597920 100644 --- a/arch/powerpc/include/asm/paca.h +++ b/arch/powerpc/include/asm/paca.h @@ -103,11 +103,12 @@ struct paca_struct { #endif /* CONFIG_PPC_STD_MMU_64 */ #ifdef CONFIG_PPC_BOOK3E - pgd_t *pgd; /* Current PGD */ - pgd_t *kernel_pgd; /* Kernel PGD */ u64 exgen[8] __attribute__((aligned(0x80))); + /* Keep pgd in the same cacheline as the start of extlb */ + pgd_t *pgd __attribute__((aligned(0x80))); /* Current PGD */ + pgd_t *kernel_pgd; /* Kernel PGD */ /* We can have up to 3 levels of reentrancy in the TLB miss handler */ - u64 extlb[3][EX_TLB_SIZE / sizeof(u64)] __attribute__((aligned(0x80))); + u64 extlb[3][EX_TLB_SIZE / sizeof(u64)]; u64 exmc[8]; /* used for machine checks */ u64 excrit[8]; /* used for crit interrupts */ u64 exdbg[8]; /* used for debug interrupts */ |