From 92778f2136ec900a873b76c69db8efbdb1998fea Mon Sep 17 00:00:00 2001 From: Jakub Jermar Date: Mon, 4 Dec 2006 21:14:07 +0000 Subject: [PATCH] Initial support for handling illegal virtual aliases on sparc64. --- kernel/Makefile | 4 +- kernel/arch/sparc64/Makefile.inc | 1 + kernel/arch/sparc64/include/cpu.h | 1 + kernel/arch/sparc64/include/interrupt.h | 7 ++- kernel/arch/sparc64/include/mm/as.h | 2 +- kernel/arch/sparc64/include/mm/cache.h | 10 ++++ kernel/arch/sparc64/src/cpu/cpu.c | 5 ++ kernel/arch/sparc64/src/mm/as.c | 43 +++++++++++++- .../sparc64/{include/mm/cache.h => src/mm/cache.c} | 65 +++++++++++++++++++-- .../arch/sparc64/src/mm/{cache.S => cache_asm.S} | 0 kernel/arch/sparc64/src/mm/tlb.c | 11 ++-- kernel/arch/sparc64/src/mm/tsb.c | 7 +-- kernel/arch/sparc64/src/smp/ipi.c | 4 ++ kernel/arch/sparc64/src/start.S | 12 ++-- kernel/arch/sparc64/src/trap/interrupt.c | 3 + kernel/generic/include/mm/as.h | 11 ++++ kernel/generic/src/mm/as.c | 66 ++++++++++++++++------ kernel/kernel.config | 4 +- 18 files changed, 208 insertions(+), 48 deletions(-) copy kernel/arch/sparc64/{include/mm/cache.h => src/mm/cache.c} (56%) rename kernel/arch/sparc64/src/mm/{cache.S => cache_asm.S} (100%) diff --git a/kernel/Makefile b/kernel/Makefile index fa6d40f92..68dd6c31b 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -91,8 +91,8 @@ ifeq ($(CONFIG_NS16550),y) DEFS += -DCONFIG_NS16550 endif -ifeq ($(CONFIG_VIRT_IDX_CACHE),y) - DEFS += -DCONFIG_VIRT_IDX_CACHE +ifeq ($(CONFIG_VIRT_IDX_DCACHE),y) + DEFS += -DCONFIG_VIRT_IDX_DCACHE endif ifeq ($(CONFIG_POWEROFF),y) diff --git a/kernel/arch/sparc64/Makefile.inc b/kernel/arch/sparc64/Makefile.inc index 94be699e9..36dfb0751 100644 --- a/kernel/arch/sparc64/Makefile.inc +++ b/kernel/arch/sparc64/Makefile.inc @@ -84,6 +84,7 @@ ARCH_SOURCES = \ arch/$(ARCH)/src/dummy.s \ arch/$(ARCH)/src/mm/as.c \ arch/$(ARCH)/src/mm/cache.c \ + arch/$(ARCH)/src/mm/cache_asm.S \ arch/$(ARCH)/src/mm/frame.c \ arch/$(ARCH)/src/mm/page.c \ arch/$(ARCH)/src/mm/tlb.c \ diff --git a/kernel/arch/sparc64/include/cpu.h b/kernel/arch/sparc64/include/cpu.h index 01ae25f97..d45fe5868 100644 --- a/kernel/arch/sparc64/include/cpu.h +++ b/kernel/arch/sparc64/include/cpu.h @@ -55,6 +55,7 @@ struct cpu_arch { uint32_t mid; /**< Processor ID as read from UPA_CONFIG. */ ver_reg_t ver; uint32_t clock_frequency; /**< Processor frequency in MHz. */ + int dcache_active; /**< When non-zero, the D-cache is not being shot down. */ }; #endif diff --git a/kernel/arch/sparc64/include/interrupt.h b/kernel/arch/sparc64/include/interrupt.h index 059dd0eee..bdfb06625 100644 --- a/kernel/arch/sparc64/include/interrupt.h +++ b/kernel/arch/sparc64/include/interrupt.h @@ -43,8 +43,13 @@ #define IVT_ITEMS 15 #define IVT_FIRST 1 +/* This needs to be defined for inter-architecture API portability. */ #define VECTOR_TLB_SHOOTDOWN_IPI 0 -#define IPI_TLB_SHOOTDOWN VECTOR_TLB_SHOOTDOWN_IPI + +enum { + IPI_TLB_SHOOTDOWN = VECTOR_TLB_SHOOTDOWN_IPI, + IPI_DCACHE_SHOOTDOWN +}; struct istate { uint64_t tnpc; diff --git a/kernel/arch/sparc64/include/mm/as.h b/kernel/arch/sparc64/include/mm/as.h index 0ac24642d..0742bc951 100644 --- a/kernel/arch/sparc64/include/mm/as.h +++ b/kernel/arch/sparc64/include/mm/as.h @@ -52,7 +52,7 @@ typedef struct { #ifdef CONFIG_TSB tsb_entry_t *itsb; tsb_entry_t *dtsb; -#endif +#endif /* CONFIG_TSB */ } as_arch_t; #ifdef CONFIG_TSB diff --git a/kernel/arch/sparc64/include/mm/cache.h b/kernel/arch/sparc64/include/mm/cache.h index a50650d5b..0b1294296 100644 --- a/kernel/arch/sparc64/include/mm/cache.h +++ b/kernel/arch/sparc64/include/mm/cache.h @@ -35,6 +35,16 @@ #ifndef KERN_sparc64_CACHE_H_ #define KERN_sparc64_CACHE_H_ +#ifdef CONFIG_SMP +extern void dcache_shootdown_start(void); +extern void dcache_shootdown_finalize(void); +extern void dcache_shootdown_ipi_recv(void); +#else /* CONFIG_SMP */ +#define dcache_shootdown_start(); +#define dcache_shootdown_finalize(); +#define dcache_shootdown_ipi_recv(); +#endif /* CONFIG_SMP */ + extern void dcache_flush(void); #endif diff --git a/kernel/arch/sparc64/src/cpu/cpu.c b/kernel/arch/sparc64/src/cpu/cpu.c index 7fecc5766..7c8ec87b2 100644 --- a/kernel/arch/sparc64/src/cpu/cpu.c +++ b/kernel/arch/sparc64/src/cpu/cpu.c @@ -92,6 +92,11 @@ void cpu_arch_init(void) dtlb_insert_mapping((uintptr_t) CPU->stack, KA2PA(CPU->stack), PAGESIZE_8K, true, true); } + /* + * Set the D-cache active flag. + * Needed for the D-cache to work. + */ + CPU->arch.dcache_active = 1; } /** Read version information from the current processor. */ diff --git a/kernel/arch/sparc64/src/mm/as.c b/kernel/arch/sparc64/src/mm/as.c index b6630f326..548530f5b 100644 --- a/kernel/arch/sparc64/src/mm/as.c +++ b/kernel/arch/sparc64/src/mm/as.c @@ -47,7 +47,11 @@ #include #include #include -#endif +#endif /* CONFIG_TSB */ + +#ifdef CONFIG_VIRT_IDX_DCACHE +#include +#endif /* CONFIG_VIRT_IDX_DCACHE */ /** Architecture dependent address space init. */ void as_arch_init(void) @@ -158,6 +162,23 @@ void as_install_arch(as_t *as) tsb_base.base = ((uintptr_t) as->arch.dtsb) >> PAGE_WIDTH; dtsb_base_write(tsb_base.value); #endif +#ifdef CONFIG_VIRT_IDX_DCACHE + if (as->dcache_flush_on_install) { + /* + * Some mappings in this address space are illegal address + * aliases. Upon their creation, the flush_dcache_on_install + * flag was set. + * + * We are now obliged to flush the D-cache in order to guarantee + * that there will be at most one cache line for each address + * alias. + * + * This flush performs a cleanup after another address space in + * which the alias might have existed. + */ + dcache_flush(); + } +#endif /* CONFIG_VIRT_IDX_DCACHE */ } /** Perform sparc64-specific tasks when an address space is removed from the processor. @@ -192,6 +213,26 @@ void as_deinstall_arch(as_t *as) dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_NUCLEUS, tsb); } #endif +#ifdef CONFIG_VIRT_IDX_DCACHE + if (as->dcache_flush_on_deinstall) { + /* + * Some mappings in this address space are illegal address + * aliases. Upon their creation, the flush_dcache_on_deinstall + * flag was set. + * + * We are now obliged to flush the D-cache in order to guarantee + * that there will be at most one cache line for each address + * alias. + * + * This flush performs a cleanup after this address space. It is + * necessary because other address spaces that contain the same + * alias are not necessarily aware of the need to carry out the + * cache flush. The only address spaces that are aware of it are + * those that created the illegal alias. + */ + dcache_flush(); + } +#endif /* CONFIG_VIRT_IDX_DCACHE */ } /** @} diff --git a/kernel/arch/sparc64/include/mm/cache.h b/kernel/arch/sparc64/src/mm/cache.c similarity index 56% copy from kernel/arch/sparc64/include/mm/cache.h copy to kernel/arch/sparc64/src/mm/cache.c index a50650d5b..7076c8153 100644 --- a/kernel/arch/sparc64/include/mm/cache.h +++ b/kernel/arch/sparc64/src/mm/cache.c @@ -29,15 +29,70 @@ /** @addtogroup sparc64mm * @{ */ -/** @file +/** + * @file + * @brief D-cache shootdown algorithm. */ -#ifndef KERN_sparc64_CACHE_H_ -#define KERN_sparc64_CACHE_H_ +#include -extern void dcache_flush(void); +#ifdef CONFIG_SMP -#endif +#include +#include +#include +#include +#include + +/** + * This spinlock is used by the processors to synchronize during the D-cache + * shootdown. + */ +SPINLOCK_INITIALIZE(dcachelock); + +/** Initialize the D-cache shootdown sequence. + * + * Start the shootdown sequence by sending out an IPI and wait until all + * processors spin on the dcachelock spinlock. + */ +void dcache_shootdown_start(void) +{ + int i; + + CPU->arch.dcache_active = 0; + spinlock_lock(&dcachelock); + + ipi_broadcast(IPI_DCACHE_SHOOTDOWN); + +busy_wait: + for (i = 0; i < config.cpu_count; i++) + if (cpus[i].arch.dcache_active) + goto busy_wait; +} + +/** Finish the D-cache shootdown sequence. */ +void dcache_shootdown_finalize(void) +{ + spinlock_unlock(&dcachelock); + CPU->arch.dcache_active = 1; +} + +/** Process the D-cache shootdown IPI. */ +void dcache_shootdown_ipi_recv(void) +{ + ASSERT(CPU); + + CPU->arch.dcache_active = 0; + spinlock_lock(&dcachelock); + spinlock_unlock(&dcachelock); + + dcache_flush(); + + CPU->arch.dcache_active = 1; +} + +#endif /* CONFIG_SMP */ /** @} */ + diff --git a/kernel/arch/sparc64/src/mm/cache.S b/kernel/arch/sparc64/src/mm/cache_asm.S similarity index 100% rename from kernel/arch/sparc64/src/mm/cache.S rename to kernel/arch/sparc64/src/mm/cache_asm.S diff --git a/kernel/arch/sparc64/src/mm/tlb.c b/kernel/arch/sparc64/src/mm/tlb.c index 066c8d3db..3a66ad0c4 100644 --- a/kernel/arch/sparc64/src/mm/tlb.c +++ b/kernel/arch/sparc64/src/mm/tlb.c @@ -111,9 +111,9 @@ void dtlb_insert_mapping(uintptr_t page, uintptr_t frame, int pagesize, bool loc data.pfn = fr.pfn; data.l = locked; data.cp = cacheable; -#ifdef CONFIG_VIRT_IDX_CACHE +#ifdef CONFIG_VIRT_IDX_DCACHE data.cv = cacheable; -#endif /* CONFIG_VIRT_IDX_CACHE */ +#endif /* CONFIG_VIRT_IDX_DCACHE */ data.p = true; data.w = true; data.g = false; @@ -148,9 +148,9 @@ void dtlb_pte_copy(pte_t *t, bool ro) data.pfn = fr.pfn; data.l = false; data.cp = t->c; -#ifdef CONFIG_VIRT_IDX_CACHE +#ifdef CONFIG_VIRT_IDX_DCACHE data.cv = t->c; -#endif /* CONFIG_VIRT_IDX_CACHE */ +#endif /* CONFIG_VIRT_IDX_DCACHE */ data.p = t->k; /* p like privileged */ data.w = ro ? false : t->w; data.g = t->g; @@ -184,9 +184,6 @@ void itlb_pte_copy(pte_t *t) data.pfn = fr.pfn; data.l = false; data.cp = t->c; -#ifdef CONFIG_VIRT_IDX_CACHE - data.cv = t->c; -#endif /* CONFIG_VIRT_IDX_CACHE */ data.p = t->k; /* p like privileged */ data.w = false; data.g = t->g; diff --git a/kernel/arch/sparc64/src/mm/tsb.c b/kernel/arch/sparc64/src/mm/tsb.c index 79b88340d..d29fa9975 100644 --- a/kernel/arch/sparc64/src/mm/tsb.c +++ b/kernel/arch/sparc64/src/mm/tsb.c @@ -100,9 +100,6 @@ void itsb_pte_copy(pte_t *t) tsb->data.size = PAGESIZE_8K; tsb->data.pfn = t->frame >> FRAME_WIDTH; tsb->data.cp = t->c; -#ifdef CONFIG_VIRT_IDX_CACHE - tsb->data.cv = t->c; -#endif /* CONFIG_VIRT_IDX_CACHE */ tsb->data.p = t->k; /* p as privileged */ tsb->data.v = t->p; @@ -142,9 +139,9 @@ void dtsb_pte_copy(pte_t *t, bool ro) tsb->data.size = PAGESIZE_8K; tsb->data.pfn = t->frame >> FRAME_WIDTH; tsb->data.cp = t->c; -#ifdef CONFIG_VIRT_IDX_CACHE +#ifdef CONFIG_VIRT_IDX_DCACHE tsb->data.cv = t->c; -#endif /* CONFIG_VIRT_IDX_CACHE */ +#endif /* CONFIG_VIRT_IDX_DCACHE */ tsb->data.p = t->k; /* p as privileged */ tsb->data.w = ro ? false : t->w; tsb->data.v = t->p; diff --git a/kernel/arch/sparc64/src/smp/ipi.c b/kernel/arch/sparc64/src/smp/ipi.c index 4bcc8106b..8f497deb6 100644 --- a/kernel/arch/sparc64/src/smp/ipi.c +++ b/kernel/arch/sparc64/src/smp/ipi.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -120,6 +121,9 @@ void ipi_broadcast_arch(int ipi) case IPI_TLB_SHOOTDOWN: func = tlb_shootdown_ipi_recv; break; + case IPI_DCACHE_SHOOTDOWN: + func = dcache_shootdown_ipi_recv; + break; default: panic("Unknown IPI (%d).\n", ipi); break; diff --git a/kernel/arch/sparc64/src/start.S b/kernel/arch/sparc64/src/start.S index 188d27866..77ce9946e 100644 --- a/kernel/arch/sparc64/src/start.S +++ b/kernel/arch/sparc64/src/start.S @@ -122,11 +122,11 @@ kernel_image_start: stxa %g1, [VA_DMMU_TAG_ACCESS] %asi membar #Sync -#ifdef CONFIG_VIRT_IDX_CACHE +#ifdef CONFIG_VIRT_IDX_DCACHE #define TTE_LOW_DATA(imm) (TTE_CP | TTE_CV | TTE_P | LMA | (imm)) -#else /* CONFIG_VIRT_IDX_CACHE */ +#else /* CONFIG_VIRT_IDX_DCACHE */ #define TTE_LOW_DATA(imm) (TTE_CP | TTE_P | LMA | (imm)) -#endif /* CONFIG_VIRT_IDX_CACHE */ +#endif /* CONFIG_VIRT_IDX_DCACHE */ #define SET_TLB_DATA(r1, r2, imm) \ set TTE_LOW_DATA(imm), %r1; \ @@ -360,8 +360,8 @@ physmem_base: */ .global kernel_8k_tlb_data_template kernel_8k_tlb_data_template: -#ifdef CONFIG_VIRT_IDX_CACHE +#ifdef CONFIG_VIRT_IDX_DCACHE .quad ((1 << TTE_V_SHIFT) | (PAGESIZE_8K << TTE_SIZE_SHIFT) | TTE_CP | TTE_CV | TTE_P | TTE_W) -#else /* CONFIG_VIRT_IDX_CACHE */ +#else /* CONFIG_VIRT_IDX_DCACHE */ .quad ((1 << TTE_V_SHIFT) | (PAGESIZE_8K << TTE_SIZE_SHIFT) | TTE_CP | TTE_P | TTE_W) -#endif /* CONFIG_VIRT_IDX_CACHE */ +#endif /* CONFIG_VIRT_IDX_DCACHE */ diff --git a/kernel/arch/sparc64/src/trap/interrupt.c b/kernel/arch/sparc64/src/trap/interrupt.c index e741bbc5a..a5e180745 100644 --- a/kernel/arch/sparc64/src/trap/interrupt.c +++ b/kernel/arch/sparc64/src/trap/interrupt.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include @@ -90,6 +91,8 @@ void interrupt(int n, istate_t *istate) #ifdef CONFIG_SMP if (data0 == (uintptr_t) tlb_shootdown_ipi_recv) { tlb_shootdown_ipi_recv(); + } else if (data0 == (uintptr_t) dcache_shootdown_ipi_recv) { + dcache_shootdown_ipi_recv(); } #endif } else { diff --git a/kernel/generic/include/mm/as.h b/kernel/generic/include/mm/as.h index 3b731cb36..f863d8402 100644 --- a/kernel/generic/include/mm/as.h +++ b/kernel/generic/include/mm/as.h @@ -94,6 +94,11 @@ struct as { /** Address space identifier. Constant on architectures that do not support ASIDs.*/ asid_t asid; +#ifdef CONFIG_VIRT_IDX_DCACHE + bool dcache_flush_on_install; + bool dcache_flush_on_deinstall; +#endif /* CONFIG_VIRT_IDX_DCACHE */ + /** Architecture specific content. */ as_arch_t arch; }; @@ -160,6 +165,12 @@ struct as_area { /** Data to be used by the backend. */ mem_backend_data_t backend_data; + + /** + * Virtual color of the original address space area that was at the beginning + * of the share chain. + */ + int orig_color; }; extern as_t *AS_KERNEL; diff --git a/kernel/generic/src/mm/as.c b/kernel/generic/src/mm/as.c index 2f3d726ec..70be56de7 100644 --- a/kernel/generic/src/mm/as.c +++ b/kernel/generic/src/mm/as.c @@ -78,6 +78,10 @@ #include #include +#ifdef CONFIG_VIRT_IDX_DCACHE +#include +#endif /* CONFIG_VIRT_IDX_DCACHE */ + /** * Each architecture decides what functions will be used to carry out * address space operations such as creating or locking page tables. @@ -162,6 +166,11 @@ as_t *as_create(int flags) as->cpu_refcount = 0; as->page_table = page_table_create(flags); +#ifdef CONFIG_VIRT_IDX_DCACHE + as->dcache_flush_on_install = false; + as->dcache_flush_on_deinstall = false; +#endif /* CONFIG_VIRT_IDX_DCACHE */ + return as; } @@ -269,6 +278,18 @@ as_area_t *as_area_create(as_t *as, int flags, size_t size, uintptr_t base, int else memsetb((uintptr_t) &a->backend_data, sizeof(a->backend_data), 0); +#ifdef CONFIG_VIRT_IDX_DCACHE + /* + * When the area is being created with the AS_AREA_ATTR_PARTIAL flag, the + * orig_color is probably wrong until the flag is reset. In other words, it is + * initialized with the color of the area being created and not with the color + * of the original address space area at the beginning of the share chain. Of + * course, the correct color is set by as_area_share() before the flag is + * reset. + */ + a->orig_color = PAGE_COLOR(base); +#endif /* CONFIG_VIRT_IDX_DCACHE */ + btree_create(&a->used_space); btree_insert(&as->as_area_btree, base, (void *) a, NULL); @@ -554,9 +575,7 @@ int as_area_destroy(as_t *as, uintptr_t address) * such address space area, EPERM if there was a problem in accepting the area * or ENOMEM if there was a problem in allocating destination address space * area. ENOTSUP is returned if the address space area backend does not support - * sharing. It can be also returned if the architecture uses virtually indexed - * caches and the source and destination areas start at pages with different - * page colors. + * sharing. */ int as_area_share(as_t *src_as, uintptr_t src_base, size_t acc_size, as_t *dst_as, uintptr_t dst_base, int dst_flags_mask) @@ -564,6 +583,7 @@ int as_area_share(as_t *src_as, uintptr_t src_base, size_t acc_size, ipl_t ipl; int src_flags; size_t src_size; + int src_orig_color; as_area_t *src_area, *dst_area; share_info_t *sh_info; mem_backend_t *src_backend; @@ -581,19 +601,6 @@ int as_area_share(as_t *src_as, uintptr_t src_base, size_t acc_size, return ENOENT; } -#if 0 /* disable the check for now */ -#ifdef CONFIG_VIRT_IDX_CACHE - if (PAGE_COLOR(src_area->base) != PAGE_COLOR(dst_base)) { - /* - * Refuse to create illegal address alias. - */ - mutex_unlock(&src_area->lock); - mutex_unlock(&src_as->lock); - interrupts_restore(ipl); - return ENOTSUP; - } -#endif /* CONFIG_VIRT_IDX_CACHE */ -#endif if (!src_area->backend || !src_area->backend->share) { /* @@ -610,6 +617,7 @@ int as_area_share(as_t *src_as, uintptr_t src_base, size_t acc_size, src_flags = src_area->flags; src_backend = src_area->backend; src_backend_data = src_area->backend_data; + src_orig_color = src_area->orig_color; /* Share the cacheable flag from the original mapping */ if (src_flags & AS_AREA_CACHEABLE) @@ -664,17 +672,39 @@ int as_area_share(as_t *src_as, uintptr_t src_base, size_t acc_size, interrupts_restore(ipl); return ENOMEM; } - + /* * Now the destination address space area has been * fully initialized. Clear the AS_AREA_ATTR_PARTIAL * attribute and set the sh_info. */ + mutex_lock(&dst_as->lock); mutex_lock(&dst_area->lock); dst_area->attributes &= ~AS_AREA_ATTR_PARTIAL; dst_area->sh_info = sh_info; + dst_area->orig_color = src_orig_color; +#ifdef CONFIG_VIRT_IDX_DCACHE + if (src_orig_color != PAGE_COLOR(dst_base)) { + /* + * We have just detected an attempt to create an invalid address + * alias. We allow this and set a special flag that tells the + * architecture specific code to flush the D-cache when the + * offending address space is installed and deinstalled + * (cleanup). + * + * In order for the flags to take effect immediately, we also + * perform a global D-cache shootdown. + */ + dcache_shootdown_start(); + dst_as->dcache_flush_on_install = true; + dst_as->dcache_flush_on_deinstall = true; + dcache_flush(); + dcache_shootdown_finalize(); + } +#endif /* CONFIG_VIRT_IDX_DCACHE */ mutex_unlock(&dst_area->lock); - + mutex_unlock(&dst_as->lock); + interrupts_restore(ipl); return 0; diff --git a/kernel/kernel.config b/kernel/kernel.config index 6dec3a4e6..99a76dd7a 100644 --- a/kernel/kernel.config +++ b/kernel/kernel.config @@ -97,8 +97,8 @@ # Support for NS16550 serial port ! [ARCH=sparc64] CONFIG_NS16550 (y/n) -# Virtually indexed cache support -! [ARCH=sparc64] CONFIG_VIRT_IDX_CACHE (n/y) +# Virtually indexed D-cache support +! [ARCH=sparc64] CONFIG_VIRT_IDX_DCACHE (y/n) ## Debugging configuration directives -- 2.11.4.GIT