LXR linux/arch/powerpc/lib/pmem.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright(c) 2017 IBM Corporation. All rights reserved.
   4 */
   5
   6#include <linux/string.h>
   7#include <linux/export.h>
   8#include <linux/uaccess.h>
   9#include <linux/libnvdimm.h>
  10
  11#include <asm/cacheflush.h>
  12
  13static inline void __clean_pmem_range(unsigned long start, unsigned long stop)
  14{
  15        unsigned long shift = l1_dcache_shift();
  16        unsigned long bytes = l1_dcache_bytes();
  17        void *addr = (void *)(start & ~(bytes - 1));
  18        unsigned long size = stop - (unsigned long)addr + (bytes - 1);
  19        unsigned long i;
  20
  21        for (i = 0; i < size >> shift; i++, addr += bytes)
  22                asm volatile(PPC_DCBSTPS(%0, %1): :"i"(0), "r"(addr): "memory");
  23}
  24
  25static inline void __flush_pmem_range(unsigned long start, unsigned long stop)
  26{
  27        unsigned long shift = l1_dcache_shift();
  28        unsigned long bytes = l1_dcache_bytes();
  29        void *addr = (void *)(start & ~(bytes - 1));
  30        unsigned long size = stop - (unsigned long)addr + (bytes - 1);
  31        unsigned long i;
  32
  33        for (i = 0; i < size >> shift; i++, addr += bytes)
  34                asm volatile(PPC_DCBFPS(%0, %1): :"i"(0), "r"(addr): "memory");
  35}
  36
  37static inline void clean_pmem_range(unsigned long start, unsigned long stop)
  38{
  39        if (cpu_has_feature(CPU_FTR_ARCH_207S))
  40                return __clean_pmem_range(start, stop);
  41}
  42
  43static inline void flush_pmem_range(unsigned long start, unsigned long stop)
  44{
  45        if (cpu_has_feature(CPU_FTR_ARCH_207S))
  46                return __flush_pmem_range(start, stop);
  47}
  48
  49/*
  50 * CONFIG_ARCH_HAS_PMEM_API symbols
  51 */
  52void arch_wb_cache_pmem(void *addr, size_t size)
  53{
  54        unsigned long start = (unsigned long) addr;
  55        clean_pmem_range(start, start + size);
  56}
  57EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
  58
  59void arch_invalidate_pmem(void *addr, size_t size)
  60{
  61        unsigned long start = (unsigned long) addr;
  62        flush_pmem_range(start, start + size);
  63}
  64EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
  65
  66/*
  67 * CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols
  68 */
  69long __copy_from_user_flushcache(void *dest, const void __user *src,
  70                unsigned size)
  71{
  72        unsigned long copied, start = (unsigned long) dest;
  73
  74        copied = __copy_from_user(dest, src, size);
  75        clean_pmem_range(start, start + size);
  76
  77        return copied;
  78}
  79
  80void memcpy_flushcache(void *dest, const void *src, size_t size)
  81{
  82        unsigned long start = (unsigned long) dest;
  83
  84        memcpy(dest, src, size);
  85        clean_pmem_range(start, start + size);
  86}
  87EXPORT_SYMBOL(memcpy_flushcache);
  88
  89void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
  90        size_t len)
  91{
  92        memcpy_flushcache(to, page_to_virt(page) + offset, len);
  93}
  94EXPORT_SYMBOL(memcpy_page_flushcache);
  95