From 124fe20d94630b6f173dae5eb815e6e6e350c72d Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Mon, 10 Aug 2015 23:07:05 -0400 Subject: mm: enhance region_is_ram() to region_intersects() region_is_ram() is used to prevent the establishment of aliased mappings to physical "System RAM" with incompatible cache settings. However, it uses "-1" to indicate both "unknown" memory ranges (ranges not described by platform firmware) and "mixed" ranges (where the parameters describe a range that partially overlaps "System RAM"). Fix this up by explicitly tracking the "unknown" vs "mixed" resource cases and returning REGION_INTERSECTS, REGION_MIXED, or REGION_DISJOINT. This re-write also adds support for detecting when the requested region completely eclipses all of a resource. Note, the implementation treats overlaps between "unknown" and the requested memory type as REGION_INTERSECTS. Finally, other memory types can be passed in by name, for now the only usage "System RAM". Suggested-by: Luis R. Rodriguez Reviewed-by: Toshi Kani Signed-off-by: Dan Williams --- include/linux/mm.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/mm.h b/include/linux/mm.h index 2e872f92dbac..84b05ebedb2d 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -369,7 +369,14 @@ static inline int put_page_unless_one(struct page *page) } extern int page_is_ram(unsigned long pfn); -extern int region_is_ram(resource_size_t phys_addr, unsigned long size); + +enum { + REGION_INTERSECTS, + REGION_DISJOINT, + REGION_MIXED, +}; + +int region_intersects(resource_size_t offset, size_t size, const char *type); /* Support for virtually mapped pages */ struct page *vmalloc_to_page(const void *addr); -- cgit v1.2.3 From 2584cf83578c26db144730ef498f4070f82ee3ea Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Mon, 10 Aug 2015 23:07:05 -0400 Subject: arch, drivers: don't include directly, use instead Preparation for uniform definition of ioremap, ioremap_wc, ioremap_wt, and ioremap_cache, tree-wide. Acked-by: Christoph Hellwig Signed-off-by: Dan Williams --- include/linux/io-mapping.h | 2 +- include/linux/mtd/map.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/io-mapping.h b/include/linux/io-mapping.h index c27dde7215b5..e399029b68c5 100644 --- a/include/linux/io-mapping.h +++ b/include/linux/io-mapping.h @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include /* diff --git a/include/linux/mtd/map.h b/include/linux/mtd/map.h index 29975c73a953..366cf77953b5 100644 --- a/include/linux/mtd/map.h +++ b/include/linux/mtd/map.h @@ -27,9 +27,9 @@ #include #include #include +#include #include -#include #include #ifdef CONFIG_MTD_MAP_BANK_WIDTH_1 -- cgit v1.2.3 From 92281dee825f6d2eb07c441437e4196a44b0861c Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Mon, 10 Aug 2015 23:07:06 -0400 Subject: arch: introduce memremap() Existing users of ioremap_cache() are mapping memory that is known in advance to not have i/o side effects. These users are forced to cast away the __iomem annotation, or otherwise neglect to fix the sparse errors thrown when dereferencing pointers to this memory. Provide memremap() as a non __iomem annotated ioremap_*() in the case when ioremap is otherwise a pointer to cacheable memory. Empirically, ioremap_() call sites are seeking memory-like semantics (e.g. speculative reads, and prefetching permitted). memremap() is a break from the ioremap implementation pattern of adding a new memremap_() for each mapping type and having silent compatibility fall backs. Instead, the implementation defines flags that are passed to the central memremap() and if a mapping type is not supported by an arch memremap returns NULL. We introduce a memremap prototype as a trivial wrapper of ioremap_cache() and ioremap_wt(). Later, once all ioremap_cache() and ioremap_wt() usage has been removed from drivers we teach archs to implement arch_memremap() with the ability to strictly enforce the mapping type. Cc: Arnd Bergmann Reviewed-by: Christoph Hellwig Signed-off-by: Dan Williams --- include/linux/io.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'include/linux') diff --git a/include/linux/io.h b/include/linux/io.h index fb5a99800e77..3fcf6256c088 100644 --- a/include/linux/io.h +++ b/include/linux/io.h @@ -121,4 +121,13 @@ static inline int arch_phys_wc_index(int handle) #endif #endif +enum { + /* See memremap() kernel-doc for usage description... */ + MEMREMAP_WB = 1 << 0, + MEMREMAP_WT = 1 << 1, +}; + +void *memremap(resource_size_t offset, size_t size, unsigned long flags); +void memunmap(void *addr); + #endif /* _LINUX_IO_H */ -- cgit v1.2.3 From e836a256e8fd579c9d7a3685f22981225a1ca451 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Wed, 12 Aug 2015 18:42:56 -0400 Subject: pmem: convert to generic memremap Kill arch_memremap_pmem() and just let the architecture specify the flags to be passed to memremap(). Default to writethrough by default. Suggested-by: Christoph Hellwig Reviewed-by: Christoph Hellwig Reviewed-by: Ross Zwisler Signed-off-by: Dan Williams --- include/linux/pmem.h | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pmem.h b/include/linux/pmem.h index d2114045a6c4..093c35ecefcc 100644 --- a/include/linux/pmem.h +++ b/include/linux/pmem.h @@ -28,12 +28,6 @@ static inline bool __arch_has_wmb_pmem(void) return false; } -static inline void __pmem *arch_memremap_pmem(resource_size_t offset, - unsigned long size) -{ - return NULL; -} - static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src, size_t n) { @@ -43,8 +37,8 @@ static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src, /* * Architectures that define ARCH_HAS_PMEM_API must provide - * implementations for arch_memremap_pmem(), arch_memcpy_to_pmem(), - * arch_wmb_pmem(), and __arch_has_wmb_pmem(). + * implementations for arch_memcpy_to_pmem(), arch_wmb_pmem(), and + * __arch_has_wmb_pmem(). */ static inline void memcpy_from_pmem(void *dst, void __pmem const *src, size_t size) @@ -54,7 +48,7 @@ static inline void memcpy_from_pmem(void *dst, void __pmem const *src, size_t si static inline void memunmap_pmem(void __pmem *addr) { - iounmap((void __force __iomem *) addr); + memunmap((void __force *) addr); } /** @@ -85,18 +79,12 @@ static inline bool arch_has_pmem_api(void) * default_memremap_pmem + default_memcpy_to_pmem is sufficient for * making data durable relative to i/o completion. */ -static void default_memcpy_to_pmem(void __pmem *dst, const void *src, +static inline void default_memcpy_to_pmem(void __pmem *dst, const void *src, size_t size) { memcpy((void __force *) dst, src, size); } -static void __pmem *default_memremap_pmem(resource_size_t offset, - unsigned long size) -{ - return (void __pmem __force *)ioremap_wt(offset, size); -} - /** * memremap_pmem - map physical persistent memory for pmem api * @offset: physical address of persistent memory @@ -112,9 +100,11 @@ static void __pmem *default_memremap_pmem(resource_size_t offset, static inline void __pmem *memremap_pmem(resource_size_t offset, unsigned long size) { - if (arch_has_pmem_api()) - return arch_memremap_pmem(offset, size); - return default_memremap_pmem(offset, size); +#ifdef ARCH_MEMREMAP_PMEM + return (void __pmem *) memremap(offset, size, ARCH_MEMREMAP_PMEM); +#else + return (void __pmem *) memremap(offset, size, MEMREMAP_WT); +#endif } /** -- cgit v1.2.3 From 7d3dcf26a6559fa82af3f53e2c8b163cec95fdaf Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 10 Aug 2015 23:07:07 -0400 Subject: devres: add devm_memremap Signed-off-by: Christoph Hellwig Signed-off-by: Dan Williams --- include/linux/io.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux') diff --git a/include/linux/io.h b/include/linux/io.h index 3fcf6256c088..d8d749abd665 100644 --- a/include/linux/io.h +++ b/include/linux/io.h @@ -80,6 +80,10 @@ int check_signature(const volatile void __iomem *io_addr, const unsigned char *signature, int length); void devm_ioremap_release(struct device *dev, void *res); +void *devm_memremap(struct device *dev, resource_size_t offset, + size_t size, unsigned long flags); +void devm_memunmap(struct device *dev, void *addr); + /* * Some systems do not have legacy ISA devices. * /dev/port is not a valid interface on these systems. -- cgit v1.2.3 From 708ab62bef1ed3a3cf065a4138bd87f5d083cfeb Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 10 Aug 2015 23:07:08 -0400 Subject: pmem: switch to devm_ allocations Signed-off-by: Christoph Hellwig [djbw: tools/testing/nvdimm/ and memunmap_pmem support] Reviewed-by: Ross Zwisler Signed-off-by: Dan Williams --- include/linux/pmem.h | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pmem.h b/include/linux/pmem.h index 093c35ecefcc..20c367cd76e6 100644 --- a/include/linux/pmem.h +++ b/include/linux/pmem.h @@ -46,9 +46,9 @@ static inline void memcpy_from_pmem(void *dst, void __pmem const *src, size_t si memcpy(dst, (void __force const *) src, size); } -static inline void memunmap_pmem(void __pmem *addr) +static inline void memunmap_pmem(struct device *dev, void __pmem *addr) { - memunmap((void __force *) addr); + devm_memunmap(dev, (void __force *) addr); } /** @@ -97,13 +97,15 @@ static inline void default_memcpy_to_pmem(void __pmem *dst, const void *src, * wmb_pmem() arrange for the data to be written through the * cache to persistent media. */ -static inline void __pmem *memremap_pmem(resource_size_t offset, - unsigned long size) +static inline void __pmem *memremap_pmem(struct device *dev, + resource_size_t offset, unsigned long size) { #ifdef ARCH_MEMREMAP_PMEM - return (void __pmem *) memremap(offset, size, ARCH_MEMREMAP_PMEM); + return (void __pmem *) devm_memremap(dev, offset, size, + ARCH_MEMREMAP_PMEM); #else - return (void __pmem *) memremap(offset, size, MEMREMAP_WT); + return (void __pmem *) devm_memremap(dev, offset, size, + MEMREMAP_WT); #endif } -- cgit v1.2.3 From 40603526569b304dd92f720f2f8ab11e828ea145 Mon Sep 17 00:00:00 2001 From: Ross Zwisler Date: Tue, 18 Aug 2015 13:55:36 -0600 Subject: pmem, x86: move x86 PMEM API to new pmem.h header Move the x86 PMEM API implementation out of asm/cacheflush.h and into its own header asm/pmem.h. This will allow members of the PMEM API to be more easily identified on this and other architectures. Signed-off-by: Ross Zwisler Suggested-by: Christoph Hellwig Reviewed-by: Christoph Hellwig Signed-off-by: Dan Williams --- include/linux/pmem.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/pmem.h b/include/linux/pmem.h index 20c367cd76e6..c2af613ec297 100644 --- a/include/linux/pmem.h +++ b/include/linux/pmem.h @@ -16,7 +16,7 @@ #include #ifdef CONFIG_ARCH_HAS_PMEM_API -#include +#include #else static inline void arch_wmb_pmem(void) { -- cgit v1.2.3 From 18279b467a9d89afe44afbc19d768e834dbf4545 Mon Sep 17 00:00:00 2001 From: Ross Zwisler Date: Tue, 18 Aug 2015 13:55:37 -0600 Subject: pmem: remove layer when calling arch_has_wmb_pmem() Prior to this change arch_has_wmb_pmem() was only called by arch_has_pmem_api(). Both arch_has_wmb_pmem() and arch_has_pmem_api() checked to make sure that CONFIG_ARCH_HAS_PMEM_API was enabled. Instead, remove the old arch_has_wmb_pmem() wrapper to be rid of one extra layer of indirection and the redundant CONFIG_ARCH_HAS_PMEM_API check. Rename __arch_has_wmb_pmem() to arch_has_wmb_pmem() since we no longer have a wrapper, and just have arch_has_pmem_api() call the architecture specific arch_has_wmb_pmem() directly. Signed-off-by: Ross Zwisler Reviewed-by: Christoph Hellwig Signed-off-by: Dan Williams --- include/linux/pmem.h | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pmem.h b/include/linux/pmem.h index c2af613ec297..a0706ea04efd 100644 --- a/include/linux/pmem.h +++ b/include/linux/pmem.h @@ -23,7 +23,7 @@ static inline void arch_wmb_pmem(void) BUG(); } -static inline bool __arch_has_wmb_pmem(void) +static inline bool arch_has_wmb_pmem(void) { return false; } @@ -38,7 +38,7 @@ static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src, /* * Architectures that define ARCH_HAS_PMEM_API must provide * implementations for arch_memcpy_to_pmem(), arch_wmb_pmem(), and - * __arch_has_wmb_pmem(). + * arch_has_wmb_pmem(). */ static inline void memcpy_from_pmem(void *dst, void __pmem const *src, size_t size) @@ -52,7 +52,7 @@ static inline void memunmap_pmem(struct device *dev, void __pmem *addr) } /** - * arch_has_wmb_pmem - true if wmb_pmem() ensures durability + * arch_has_pmem_api - true if wmb_pmem() ensures durability * * For a given cpu implementation within an architecture it is possible * that wmb_pmem() resolves to a nop. In the case this returns @@ -60,13 +60,6 @@ static inline void memunmap_pmem(struct device *dev, void __pmem *addr) * fall back to a different data consistency model, or otherwise notify * the user. */ -static inline bool arch_has_wmb_pmem(void) -{ - if (IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API)) - return __arch_has_wmb_pmem(); - return false; -} - static inline bool arch_has_pmem_api(void) { return IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && arch_has_wmb_pmem(); -- cgit v1.2.3 From 5de490daec8b6354b90d5c9d3e2415b195f5adb6 Mon Sep 17 00:00:00 2001 From: Ross Zwisler Date: Tue, 18 Aug 2015 13:55:39 -0600 Subject: pmem: add copy_from_iter_pmem() and clear_pmem() Add support for two new PMEM APIs, copy_from_iter_pmem() and clear_pmem(). copy_from_iter_pmem() is used to copy data from an iterator into a PMEM buffer. clear_pmem() zeros a PMEM memory range. Both of these new APIs must be explicitly ordered using a wmb_pmem() function call and are implemented in such a way that the wmb_pmem() will make the stores to PMEM durable. Because both APIs are unordered they can be called as needed without introducing any unwanted memory barriers. Signed-off-by: Ross Zwisler Signed-off-by: Dan Williams --- include/linux/pmem.h | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pmem.h b/include/linux/pmem.h index a0706ea04efd..a9d84bf335ee 100644 --- a/include/linux/pmem.h +++ b/include/linux/pmem.h @@ -14,6 +14,7 @@ #define __PMEM_H__ #include +#include #ifdef CONFIG_ARCH_HAS_PMEM_API #include @@ -33,12 +34,24 @@ static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src, { BUG(); } + +static inline size_t arch_copy_from_iter_pmem(void __pmem *addr, size_t bytes, + struct iov_iter *i) +{ + BUG(); + return 0; +} + +static inline void arch_clear_pmem(void __pmem *addr, size_t size) +{ + BUG(); +} #endif /* * Architectures that define ARCH_HAS_PMEM_API must provide - * implementations for arch_memcpy_to_pmem(), arch_wmb_pmem(), and - * arch_has_wmb_pmem(). + * implementations for arch_memcpy_to_pmem(), arch_wmb_pmem(), + * arch_copy_from_iter_pmem(), arch_clear_pmem() and arch_has_wmb_pmem(). */ static inline void memcpy_from_pmem(void *dst, void __pmem const *src, size_t size) @@ -78,6 +91,20 @@ static inline void default_memcpy_to_pmem(void __pmem *dst, const void *src, memcpy((void __force *) dst, src, size); } +static inline size_t default_copy_from_iter_pmem(void __pmem *addr, + size_t bytes, struct iov_iter *i) +{ + return copy_from_iter_nocache((void __force *)addr, bytes, i); +} + +static inline void default_clear_pmem(void __pmem *addr, size_t size) +{ + if (size == PAGE_SIZE && ((unsigned long)addr & ~PAGE_MASK) == 0) + clear_page((void __force *)addr); + else + memset((void __force *)addr, 0, size); +} + /** * memremap_pmem - map physical persistent memory for pmem api * @offset: physical address of persistent memory @@ -134,4 +161,37 @@ static inline void wmb_pmem(void) if (arch_has_pmem_api()) arch_wmb_pmem(); } + +/** + * copy_from_iter_pmem - copy data from an iterator to PMEM + * @addr: PMEM destination address + * @bytes: number of bytes to copy + * @i: iterator with source data + * + * Copy data from the iterator 'i' to the PMEM buffer starting at 'addr'. + * This function requires explicit ordering with a wmb_pmem() call. + */ +static inline size_t copy_from_iter_pmem(void __pmem *addr, size_t bytes, + struct iov_iter *i) +{ + if (arch_has_pmem_api()) + return arch_copy_from_iter_pmem(addr, bytes, i); + return default_copy_from_iter_pmem(addr, bytes, i); +} + +/** + * clear_pmem - zero a PMEM memory range + * @addr: virtual start address + * @size: number of bytes to zero + * + * Write zeros into the memory range starting at 'addr' for 'size' bytes. + * This function requires explicit ordering with a wmb_pmem() call. + */ +static inline void clear_pmem(void __pmem *addr, size_t size) +{ + if (arch_has_pmem_api()) + arch_clear_pmem(addr, size); + else + default_clear_pmem(addr, size); +} #endif /* __PMEM_H__ */ -- cgit v1.2.3 From e2e05394e4a3420dab96f728df4531893494e15d Mon Sep 17 00:00:00 2001 From: Ross Zwisler Date: Tue, 18 Aug 2015 13:55:41 -0600 Subject: pmem, dax: have direct_access use __pmem annotation Update the annotation for the kaddr pointer returned by direct_access() so that it is a __pmem pointer. This is consistent with the PMEM driver and with how this direct_access() pointer is used in the DAX code. Signed-off-by: Ross Zwisler Reviewed-by: Christoph Hellwig Signed-off-by: Dan Williams --- include/linux/blkdev.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index d4068c17d0df..c401ecdff9cb 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1555,8 +1555,8 @@ struct block_device_operations { int (*rw_page)(struct block_device *, sector_t, struct page *, int rw); int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); - long (*direct_access)(struct block_device *, sector_t, - void **, unsigned long *pfn, long size); + long (*direct_access)(struct block_device *, sector_t, void __pmem **, + unsigned long *pfn, long size); unsigned int (*check_events) (struct gendisk *disk, unsigned int clearing); /* ->media_changed() is DEPRECATED, use ->check_events() instead */ @@ -1574,8 +1574,8 @@ extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int, extern int bdev_read_page(struct block_device *, sector_t, struct page *); extern int bdev_write_page(struct block_device *, sector_t, struct page *, struct writeback_control *); -extern long bdev_direct_access(struct block_device *, sector_t, void **addr, - unsigned long *pfn, long size); +extern long bdev_direct_access(struct block_device *, sector_t, + void __pmem **addr, unsigned long *pfn, long size); #else /* CONFIG_BLOCK */ struct block_device; -- cgit v1.2.3