Merge tag 'mips_5.4' of git://git.kernel.org/pub/scm/linux/kernel/git/mips/linux

Pull MIPS updates from Paul Burton:
 "Main MIPS changes:

   - boot_mem_map is removed, providing a nice cleanup made possible by
     the recent removal of bootmem.

   - Some fixes to atomics, in general providing compiler barriers for
     smp_mb__{before,after}_atomic plus fixes specific to Loongson CPUs
     or MIPS32 systems using cmpxchg64().

   - Conversion to the new generic VDSO infrastructure courtesy of
     Vincenzo Frascino.

   - Removal of undefined behavior in set_io_port_base(), fixing the
     behavior of some MIPS kernel configurations when built with recent
     clang versions.

   - Initial MIPS32 huge page support, functional on at least Ingenic
     SoCs.

   - pte_special() is now supported for some configurations, allowing
     among other things generic fast GUP to be used.

   - Miscellaneous fixes & cleanups.

  And platform specific changes:

   - Major improvements to Ingenic SoC support from Paul Cercueil,
     mostly enabled by the inclusion of the new TCU (timer-counter unit)
     drivers he's spent a very patient year or so working on. Plus some
     fixes for X1000 SoCs from Zhou Yanjie.

   - Netgear R6200 v1 systems are now supported by the bcm47xx platform.

   - DT updates for BMIPS, Lantiq & Microsemi Ocelot systems"

* tag 'mips_5.4' of git://git.kernel.org/pub/scm/linux/kernel/git/mips/linux: (89 commits)
  MIPS: Detect bad _PFN_SHIFT values
  MIPS: Disable pte_special() for MIPS32 with RiXi
  MIPS: ralink: deactivate PCI support for SOC_MT7621
  mips: compat: vdso: Use legacy syscalls as fallback
  MIPS: Drop Loongson _CACHE_* definitions
  MIPS: tlbex: Remove cpu_has_local_ebase
  MIPS: tlbex: Simplify r3k check
  MIPS: Select R3k-style TLB in Kconfig
  MIPS: PCI: refactor ioc3 special handling
  mips: remove ioremap_cachable
  mips/atomic: Fix smp_mb__{before,after}_atomic()
  mips/atomic: Fix loongson_llsc_mb() wreckage
  mips/atomic: Fix cmpxchg64 barriers
  MIPS: Octeon: remove duplicated include from dma-octeon.c
  firmware: bcm47xx_nvram: Allow COMPILE_TEST
  firmware: bcm47xx_nvram: Correct size_t printf format
  MIPS: Treat Loongson Extensions as ASEs
  MIPS: Remove dev_err() usage after platform_get_irq()
  MIPS: dts: mscc: describe the PTP ready interrupt
  MIPS: dts: mscc: describe the PTP register range
  ...

7 files changed, 108 insertions, 347 deletions

diff --git a/arch/mips/mm/Makefile b/arch/mips/mm/Makefile
index 1e8d335025d7..46f483e952c8 100644
--- a/arch/mips/mm/Makefile
+++ b/arch/mips/mm/Makefile
@@ -28,11 +28,11 @@ obj-$(CONFIG_HIGHMEM)		+= highmem.o
 obj-$(CONFIG_HUGETLB_PAGE)	+= hugetlbpage.o
 obj-$(CONFIG_DMA_NONCOHERENT)	+= dma-noncoherent.o
 
+obj-$(CONFIG_CPU_R3K_TLB)	+= tlb-r3k.o
 obj-$(CONFIG_CPU_R4K_CACHE_TLB) += c-r4k.o cex-gen.o tlb-r4k.o
-obj-$(CONFIG_CPU_R3000)		+= c-r3k.o tlb-r3k.o
-obj-$(CONFIG_CPU_R8000)		+= c-r4k.o cex-gen.o tlb-r8k.o
+obj-$(CONFIG_CPU_R3000)		+= c-r3k.o
 obj-$(CONFIG_CPU_SB1)		+= c-r4k.o cerr-sb1.o cex-sb1.o tlb-r4k.o
-obj-$(CONFIG_CPU_TX39XX)	+= c-tx39.o tlb-r3k.o
+obj-$(CONFIG_CPU_TX39XX)	+= c-tx39.o
 obj-$(CONFIG_CPU_CAVIUM_OCTEON) += c-octeon.o cex-oct.o tlb-r4k.o
 
 obj-$(CONFIG_IP22_CPU_SCACHE)	+= sc-ip22.o
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
index 5166e38cd1c6..89b9c851d822 100644
--- a/arch/mips/mm/c-r4k.c
+++ b/arch/mips/mm/c-r4k.c
@@ -1098,7 +1098,6 @@ static void probe_pcache(void)
 		c->options |= MIPS_CPU_CACHE_CDEX_P;
 		break;
 
-	case CPU_R5432:
 	case CPU_R5500:
 		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
 		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
@@ -1134,7 +1133,6 @@ static void probe_pcache(void)
 	case CPU_R4400PC:
 	case CPU_R4400SC:
 	case CPU_R4400MC:
-	case CPU_R4300:
 		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
 		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
 		c->icache.ways = 1;
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 8a038b30d3c4..090fa653dfa9 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -269,37 +269,46 @@ void __init fixrange_init(unsigned long start, unsigned long end,
 #endif
 }
 
-unsigned __weak platform_maar_init(unsigned num_pairs)
+struct maar_walk_info {
+	struct maar_config cfg[16];
+	unsigned int num_cfg;
+};
+
+static int maar_res_walk(unsigned long start_pfn, unsigned long nr_pages,
+			 void *data)
 {
-	struct maar_config cfg[BOOT_MEM_MAP_MAX];
-	unsigned i, num_configured, num_cfg = 0;
-
-	for (i = 0; i < boot_mem_map.nr_map; i++) {
-		switch (boot_mem_map.map[i].type) {
-		case BOOT_MEM_RAM:
-		case BOOT_MEM_INIT_RAM:
-			break;
-		default:
-			continue;
-		}
+	struct maar_walk_info *wi = data;
+	struct maar_config *cfg = &wi->cfg[wi->num_cfg];
+	unsigned int maar_align;
 
-		/* Round lower up */
-		cfg[num_cfg].lower = boot_mem_map.map[i].addr;
-		cfg[num_cfg].lower = (cfg[num_cfg].lower + 0xffff) & ~0xffff;
+	/* MAAR registers hold physical addresses right shifted by 4 bits */
+	maar_align = BIT(MIPS_MAAR_ADDR_SHIFT + 4);
 
-		/* Round upper down */
-		cfg[num_cfg].upper = boot_mem_map.map[i].addr +
-					boot_mem_map.map[i].size;
-		cfg[num_cfg].upper = (cfg[num_cfg].upper & ~0xffff) - 1;
+	/* Fill in the MAAR config entry */
+	cfg->lower = ALIGN(PFN_PHYS(start_pfn), maar_align);
+	cfg->upper = ALIGN_DOWN(PFN_PHYS(start_pfn + nr_pages), maar_align) - 1;
+	cfg->attrs = MIPS_MAAR_S;
+
+	/* Ensure we don't overflow the cfg array */
+	if (!WARN_ON(wi->num_cfg >= ARRAY_SIZE(wi->cfg)))
+		wi->num_cfg++;
+
+	return 0;
+}
 
-		cfg[num_cfg].attrs = MIPS_MAAR_S;
-		num_cfg++;
-	}
 
-	num_configured = maar_config(cfg, num_cfg, num_pairs);
-	if (num_configured < num_cfg)
-		pr_warn("Not enough MAAR pairs (%u) for all bootmem regions (%u)\n",
-			num_pairs, num_cfg);
+unsigned __weak platform_maar_init(unsigned num_pairs)
+{
+	unsigned int num_configured;
+	struct maar_walk_info wi;
+
+	wi.num_cfg = 0;
+	walk_system_ram_range(0, max_pfn, &wi, maar_res_walk);
+
+	num_configured = maar_config(wi.cfg, wi.num_cfg, num_pairs);
+	if (num_configured < wi.num_cfg)
+		pr_warn("Not enough MAAR pairs (%u) for all memory regions (%u)\n",
+			num_pairs, wi.num_cfg);
 
 	return num_configured;
 }
@@ -382,33 +391,6 @@ void maar_init(void)
 }
 
 #ifndef CONFIG_NEED_MULTIPLE_NODES
-int page_is_ram(unsigned long pagenr)
-{
-	int i;
-
-	for (i = 0; i < boot_mem_map.nr_map; i++) {
-		unsigned long addr, end;
-
-		switch (boot_mem_map.map[i].type) {
-		case BOOT_MEM_RAM:
-		case BOOT_MEM_INIT_RAM:
-			break;
-		default:
-			/* not usable memory */
-			continue;
-		}
-
-		addr = PFN_UP(boot_mem_map.map[i].addr);
-		end = PFN_DOWN(boot_mem_map.map[i].addr +
-			       boot_mem_map.map[i].size);
-
-		if (pagenr >= addr && pagenr < end)
-			return 1;
-	}
-
-	return 0;
-}
-
 void __init paging_init(void)
 {
 	unsigned long max_zone_pfns[MAX_NR_ZONES];
@@ -443,7 +425,7 @@ void __init paging_init(void)
 static struct kcore_list kcore_kseg0;
 #endif
 
-static inline void mem_init_free_highmem(void)
+static inline void __init mem_init_free_highmem(void)
 {
 #ifdef CONFIG_HIGHMEM
 	unsigned long tmp;
@@ -454,7 +436,7 @@ static inline void mem_init_free_highmem(void)
 	for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {
 		struct page *page = pfn_to_page(tmp);
 
-		if (!page_is_ram(tmp))
+		if (!memblock_is_memory(PFN_PHYS(tmp)))
 			SetPageReserved(page);
 		else
 			free_highmem_page(page);
@@ -464,6 +446,12 @@ static inline void mem_init_free_highmem(void)
 
 void __init mem_init(void)
 {
+	/*
+	 * When _PFN_SHIFT is greater than PAGE_SHIFT we won't have enough PTE
+	 * bits to hold a full 32b physical address on MIPS32 systems.
+	 */
+	BUILD_BUG_ON(IS_ENABLED(CONFIG_32BIT) && (_PFN_SHIFT > PAGE_SHIFT));
+
 #ifdef CONFIG_HIGHMEM
 #ifdef CONFIG_DISCONTIGMEM
 #error "CONFIG_HIGHMEM and CONFIG_DISCONTIGMEM dont work together yet"
diff --git a/arch/mips/mm/pgtable-32.c b/arch/mips/mm/pgtable-32.c
index e2a33adc0f29..6416a531a4c3 100644
--- a/arch/mips/mm/pgtable-32.c
+++ b/arch/mips/mm/pgtable-32.c
@@ -12,6 +12,7 @@
 #include <asm/fixmap.h>
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
+#include <asm/tlbflush.h>
 
 void pgd_init(unsigned long page)
 {
@@ -30,6 +31,25 @@ void pgd_init(unsigned long page)
 	}
 }
 
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE)
+pmd_t mk_pmd(struct page *page, pgprot_t prot)
+{
+	pmd_t pmd;
+
+	pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot);
+
+	return pmd;
+}
+
+
+void set_pmd_at(struct mm_struct *mm, unsigned long addr,
+		pmd_t *pmdp, pmd_t pmd)
+{
+	*pmdp = pmd;
+	flush_tlb_all();
+}
+#endif /* defined(CONFIG_TRANSPARENT_HUGEPAGE) */
+
 void __init pagetable_init(void)
 {
 	unsigned long vaddr;
diff --git a/arch/mips/mm/sc-mips.c b/arch/mips/mm/sc-mips.c
index 394673991bab..dbdbfe5d8408 100644
--- a/arch/mips/mm/sc-mips.c
+++ b/arch/mips/mm/sc-mips.c
@@ -221,13 +221,26 @@ static inline int __init mips_sc_probe(void)
 	else
 		return 0;
 
-	/*
-	 * According to config2 it would be 5-ways, but that is contradicted
-	 * by all documentation.
-	 */
-	if (current_cpu_type() == CPU_JZRISC &&
-				mips_machtype == MACH_INGENIC_JZ4770)
-		c->scache.ways = 4;
+	if (current_cpu_type() == CPU_XBURST) {
+		switch (mips_machtype) {
+		/*
+		 * According to config2 it would be 5-ways, but that is
+		 * contradicted by all documentation.
+		 */
+		case MACH_INGENIC_JZ4770:
+			c->scache.ways = 4;
+			break;
+
+		/*
+		 * According to config2 it would be 5-ways and 512-sets,
+		 * but that is contradicted by all documentation.
+		 */
+		case MACH_INGENIC_X1000:
+			c->scache.sets = 256;
+			c->scache.ways = 4;
+			break;
+		}
+	}
 
 	c->scache.waysize = c->scache.sets * c->scache.linesz;
 	c->scache.waybit = __ffs(c->scache.waysize);
diff --git a/arch/mips/mm/tlb-r8k.c b/arch/mips/mm/tlb-r8k.c
deleted file mode 100644
index c1e9e144007e..000000000000
--- a/arch/mips/mm/tlb-r8k.c
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * This file is subject to the terms and conditions of the GNU General Public
- * License.  See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
- * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
- * Carsten Langgaard, carstenl@mips.com
- * Copyright (C) 2002 MIPS Technologies, Inc.  All rights reserved.
- */
-#include <linux/sched.h>
-#include <linux/smp.h>
-#include <linux/mm.h>
-
-#include <asm/cpu.h>
-#include <asm/bootinfo.h>
-#include <asm/mmu_context.h>
-#include <asm/pgtable.h>
-
-extern void build_tlb_refill_handler(void);
-
-#define TFP_TLB_SIZE		384
-#define TFP_TLB_SET_SHIFT	7
-
-/* CP0 hazard avoidance. */
-#define BARRIER __asm__ __volatile__(".set noreorder\n\t" \
-				     "nop; nop; nop; nop; nop; nop;\n\t" \
-				     ".set reorder\n\t")
-
-void local_flush_tlb_all(void)
-{
-	unsigned long flags;
-	unsigned long old_ctx;
-	int entry;
-
-	local_irq_save(flags);
-	/* Save old context and create impossible VPN2 value */
-	old_ctx = read_c0_entryhi();
-	write_c0_entrylo(0);
-
-	for (entry = 0; entry < TFP_TLB_SIZE; entry++) {
-		write_c0_tlbset(entry >> TFP_TLB_SET_SHIFT);
-		write_c0_vaddr(entry << PAGE_SHIFT);
-		write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1)));
-		mtc0_tlbw_hazard();
-		tlb_write();
-	}
-	tlbw_use_hazard();
-	write_c0_entryhi(old_ctx);
-	local_irq_restore(flags);
-}
-
-void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
-	unsigned long end)
-{
-	struct mm_struct *mm = vma->vm_mm;
-	int cpu = smp_processor_id();
-	unsigned long flags;
-	int oldpid, newpid, size;
-
-	if (!cpu_context(cpu, mm))
-		return;
-
-	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
-	size = (size + 1) >> 1;
-
-	local_irq_save(flags);
-
-	if (size > TFP_TLB_SIZE / 2) {
-		drop_mmu_context(mm);
-		goto out_restore;
-	}
-
-	oldpid = read_c0_entryhi();
-	newpid = cpu_asid(cpu, mm);
-
-	write_c0_entrylo(0);
-
-	start &= PAGE_MASK;
-	end += (PAGE_SIZE - 1);
-	end &= PAGE_MASK;
-	while (start < end) {
-		signed long idx;
-
-		write_c0_vaddr(start);
-		write_c0_entryhi(start);
-		start += PAGE_SIZE;
-		tlb_probe();
-		idx = read_c0_tlbset();
-		if (idx < 0)
-			continue;
-
-		write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
-		tlb_write();
-	}
-	write_c0_entryhi(oldpid);
-
-out_restore:
-	local_irq_restore(flags);
-}
-
-/* Usable for KV1 addresses only! */
-void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
-{
-	unsigned long size, flags;
-
-	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
-	size = (size + 1) >> 1;
-
-	if (size > TFP_TLB_SIZE / 2) {
-		local_flush_tlb_all();
-		return;
-	}
-
-	local_irq_save(flags);
-
-	write_c0_entrylo(0);
-
-	start &= PAGE_MASK;
-	end += (PAGE_SIZE - 1);
-	end &= PAGE_MASK;
-	while (start < end) {
-		signed long idx;
-
-		write_c0_vaddr(start);
-		write_c0_entryhi(start);
-		start += PAGE_SIZE;
-		tlb_probe();
-		idx = read_c0_tlbset();
-		if (idx < 0)
-			continue;
-
-		write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
-		tlb_write();
-	}
-
-	local_irq_restore(flags);
-}
-
-void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
-{
-	int cpu = smp_processor_id();
-	unsigned long flags;
-	int oldpid, newpid;
-	signed long idx;
-
-	if (!cpu_context(cpu, vma->vm_mm))
-		return;
-
-	newpid = cpu_asid(cpu, vma->vm_mm);
-	page &= PAGE_MASK;
-	local_irq_save(flags);
-	oldpid = read_c0_entryhi();
-	write_c0_vaddr(page);
-	write_c0_entryhi(newpid);
-	tlb_probe();
-	idx = read_c0_tlbset();
-	if (idx < 0)
-		goto finish;
-
-	write_c0_entrylo(0);
-	write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
-	tlb_write();
-
-finish:
-	write_c0_entryhi(oldpid);
-	local_irq_restore(flags);
-}
-
-/*
- * We will need multiple versions of update_mmu_cache(), one that just
- * updates the TLB with the new pte(s), and another which also checks
- * for the R4k "end of page" hardware bug and does the needy.
- */
-void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
-{
-	unsigned long flags;
-	pgd_t *pgdp;
-	pmd_t *pmdp;
-	pte_t *ptep;
-	int pid;
-
-	/*
-	 * Handle debugger faulting in for debugee.
-	 */
-	if (current->active_mm != vma->vm_mm)
-		return;
-
-	pid = read_c0_entryhi() & cpu_asid_mask(&current_cpu_data);
-
-	local_irq_save(flags);
-	address &= PAGE_MASK;
-	write_c0_vaddr(address);
-	write_c0_entryhi(pid);
-	pgdp = pgd_offset(vma->vm_mm, address);
-	pmdp = pmd_offset(pgdp, address);
-	ptep = pte_offset_map(pmdp, address);
-	tlb_probe();
-
-	write_c0_entrylo(pte_val(*ptep++) >> 6);
-	tlb_write();
-
-	write_c0_entryhi(pid);
-	local_irq_restore(flags);
-}
-
-static void probe_tlb(unsigned long config)
-{
-	struct cpuinfo_mips *c = &current_cpu_data;
-
-	c->tlbsize = 3 * 128;		/* 3 sets each 128 entries */
-}
-
-void tlb_init(void)
-{
-	unsigned int config = read_c0_config();
-	unsigned long status;
-
-	probe_tlb(config);
-
-	status = read_c0_status();
-	status &= ~(ST0_UPS | ST0_KPS);
-#ifdef CONFIG_PAGE_SIZE_4KB
-	status |= (TFP_PAGESIZE_4K << 32) | (TFP_PAGESIZE_4K << 36);
-#elif defined(CONFIG_PAGE_SIZE_8KB)
-	status |= (TFP_PAGESIZE_8K << 32) | (TFP_PAGESIZE_8K << 36);
-#elif defined(CONFIG_PAGE_SIZE_16KB)
-	status |= (TFP_PAGESIZE_16K << 32) | (TFP_PAGESIZE_16K << 36);
-#elif defined(CONFIG_PAGE_SIZE_64KB)
-	status |= (TFP_PAGESIZE_64K << 32) | (TFP_PAGESIZE_64K << 36);
-#endif
-	write_c0_status(status);
-
-	write_c0_wired(0);
-
-	local_flush_tlb_all();
-
-	build_tlb_refill_handler();
-}
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 144ceb0fba88..e01cb33bfa1a 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -545,7 +545,6 @@ void build_tlb_write_entry(u32 **p, struct uasm_label **l,
 		tlbw(p);
 		break;
 
-	case CPU_R4300:
 	case CPU_5KC:
 	case CPU_TX49XX:
 	case CPU_PR4450:
@@ -604,13 +603,12 @@ void build_tlb_write_entry(u32 **p, struct uasm_label **l,
 
 	case CPU_VR4131:
 	case CPU_VR4133:
-	case CPU_R5432:
 		uasm_i_nop(p);
 		uasm_i_nop(p);
 		tlbw(p);
 		break;
 
-	case CPU_JZRISC:
+	case CPU_XBURST:
 		tlbw(p);
 		uasm_i_nop(p);
 		break;
@@ -631,7 +629,7 @@ static __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
 		return;
 	}
 
-	if (cpu_has_rixi && _PAGE_NO_EXEC) {
+	if (cpu_has_rixi && !!_PAGE_NO_EXEC) {
 		if (fill_includes_sw_bits) {
 			UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL));
 		} else {
@@ -2609,21 +2607,11 @@ void build_tlb_refill_handler(void)
 	check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
 #endif
 
-	switch (current_cpu_type()) {
-	case CPU_R2000:
-	case CPU_R3000:
-	case CPU_R3000A:
-	case CPU_R3081E:
-	case CPU_TX3912:
-	case CPU_TX3922:
-	case CPU_TX3927:
+	if (cpu_has_3kex) {
 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
-		if (cpu_has_local_ebase)
-			build_r3000_tlb_refill_handler();
 		if (!run_once) {
-			if (!cpu_has_local_ebase)
-				build_r3000_tlb_refill_handler();
 			build_setup_pgd();
+			build_r3000_tlb_refill_handler();
 			build_r3000_tlb_load_handler();
 			build_r3000_tlb_store_handler();
 			build_r3000_tlb_modify_handler();
@@ -2633,34 +2621,27 @@ void build_tlb_refill_handler(void)
 #else
 		panic("No R3000 TLB refill handler");
 #endif
-		break;
+		return;
+	}
 
-	case CPU_R8000:
-		panic("No R8000 TLB refill handler yet");
-		break;
+	if (cpu_has_ldpte)
+		setup_pw();
 
-	default:
+	if (!run_once) {
+		scratch_reg = allocate_kscratch();
+		build_setup_pgd();
+		build_r4000_tlb_load_handler();
+		build_r4000_tlb_store_handler();
+		build_r4000_tlb_modify_handler();
 		if (cpu_has_ldpte)
-			setup_pw();
-
-		if (!run_once) {
-			scratch_reg = allocate_kscratch();
-			build_setup_pgd();
-			build_r4000_tlb_load_handler();
-			build_r4000_tlb_store_handler();
-			build_r4000_tlb_modify_handler();
-			if (cpu_has_ldpte)
-				build_loongson3_tlb_refill_handler();
-			else if (!cpu_has_local_ebase)
-				build_r4000_tlb_refill_handler();
-			flush_tlb_handlers();
-			run_once++;
-		}
-		if (cpu_has_local_ebase)
+			build_loongson3_tlb_refill_handler();
+		else
 			build_r4000_tlb_refill_handler();
-		if (cpu_has_xpa)
-			config_xpa_params();
-		if (cpu_has_htw)
-			config_htw_params();
+		flush_tlb_handlers();
+		run_once++;
 	}
+	if (cpu_has_xpa)
+		config_xpa_params();
+	if (cpu_has_htw)
+		config_htw_params();
 }