diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c index f1387272a551..04d7cb3f1488 100644 --- a/arch/riscv/mm/cacheflush.c +++ b/arch/riscv/mm/cacheflush.c @@ -80,12 +80,31 @@ void flush_icache_mm(struct mm_struct *mm, bool local) #endif /* CONFIG_SMP */ #ifdef CONFIG_MMU +static u32 xuantie_flush_icache_range_flag = 0; +static inline void __flush_icache_range(unsigned long start, unsigned long end) +{ + register unsigned long i asm("a0") = start & ~(L1_CACHE_BYTES - 1); + + for (; i < end; i += L1_CACHE_BYTES) + asm volatile (".long 0x0305000b"); /* icache.iva a0 */ + + asm volatile (".long 0x01b0000b"); /* sync_is */ +} void flush_icache_pte(pte_t pte) { - struct folio *folio = page_folio(pte_page(pte)); + struct page *page = pte_page(pte); + struct folio *folio = page_folio(page); if (!test_bit(PG_dcache_clean, &folio->flags)) { - flush_icache_all(); + if (xuantie_flush_icache_range_flag) { + unsigned long start = (unsigned long)page_address(page); + unsigned long end = start + page_size(page); + + /* remove flush_icache_all for performance */ + __flush_icache_range(start, end); + } + else + flush_icache_all(); set_bit(PG_dcache_clean, &folio->flags); } } @@ -139,3 +158,15 @@ void __init riscv_init_cbo_blocksizes(void) if (cboz_block_size) riscv_cboz_block_size = cboz_block_size; } + +static int __init xuantie_flush_icache_init(void) +{ + struct device_node *cpu = of_find_node_by_path("/cpus"); + if (cpu) { + of_property_read_u32(cpu, "flush-icache-range", &xuantie_flush_icache_range_flag); + of_node_put(cpu); + } + + return 0; +} +arch_initcall(xuantie_flush_icache_init); diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 77be59aadc73..dd0d9324c465 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -1,11 +1,14 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include #include #include +static u32 xuantie_flush_tlb_range_flag = 0; + static inline void local_flush_tlb_all_asid(unsigned long asid) { __asm__ __volatile__ ("sfence.vma x0, %0" @@ -48,10 +51,14 @@ static void __ipi_flush_tlb_all(void *info) void flush_tlb_all(void) { - if (riscv_use_ipi_for_rfence()) - on_each_cpu(__ipi_flush_tlb_all, NULL, 1); - else - sbi_remote_sfence_vma(NULL, 0, -1); + if (xuantie_flush_tlb_range_flag) { + __asm__ __volatile__ ("sfence.vma" : : : "memory"); + } else { + if (riscv_use_ipi_for_rfence()) + on_each_cpu(__ipi_flush_tlb_all, NULL, 1); + else + sbi_remote_sfence_vma(NULL, 0, -1); + } } struct flush_tlb_range_data { @@ -129,18 +136,49 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start, void flush_tlb_mm(struct mm_struct *mm) { - __flush_tlb_range(mm, 0, -1, PAGE_SIZE); + if (xuantie_flush_tlb_range_flag) { + unsigned long asid = atomic_long_read(&mm->context.id) & asid_mask; + __asm__ __volatile__ ("sfence.vma zero, %0" + : + : "r"(asid) + : "memory"); + } else { + __flush_tlb_range(mm, 0, -1, PAGE_SIZE); + } } void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) { - __flush_tlb_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE); + if (xuantie_flush_tlb_range_flag) { + unsigned long asid = atomic_long_read(&vma->vm_mm->context.id) & asid_mask; + addr &= PAGE_MASK; + __asm__ __volatile__ ("sfence.vma %0, %1" + : + : "r"(addr), "r"(asid) + : "memory"); + } else { + __flush_tlb_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE); + } } void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - __flush_tlb_range(vma->vm_mm, start, end - start, PAGE_SIZE); + if (xuantie_flush_tlb_range_flag) { + unsigned long asid = atomic_long_read(&vma->vm_mm->context.id) & asid_mask; + start &= PAGE_MASK; + end += PAGE_SIZE - 1; + end &= PAGE_MASK; + while (start < end) { + __asm__ __volatile__ ("sfence.vma %0, %1" + : + : "r"(start), "r"(asid) + : "memory"); + start += PAGE_SIZE; + } + } else { + __flush_tlb_range(vma->vm_mm, start, end - start, PAGE_SIZE); + } } #ifdef CONFIG_TRANSPARENT_HUGEPAGE void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, @@ -149,3 +187,15 @@ void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, __flush_tlb_range(vma->vm_mm, start, end - start, PMD_SIZE); } #endif + +static int __init xuantie_flush_tlb_init(void) +{ + struct device_node *cpu = of_find_node_by_path("/cpus"); + if (cpu) { + of_property_read_u32(cpu, "flush-tlb-range", &xuantie_flush_tlb_range_flag); + of_node_put(cpu); + } + + return 0; +} +arch_initcall(xuantie_flush_tlb_init);