LXR linux/arch/tile/lib/cacheflush.c

   1/*
   2 * Copyright 2010 Tilera Corporation. All Rights Reserved.
   3 *
   4 *   This program is free software; you can redistribute it and/or
   5 *   modify it under the terms of the GNU General Public License
   6 *   as published by the Free Software Foundation, version 2.
   7 *
   8 *   This program is distributed in the hope that it will be useful, but
   9 *   WITHOUT ANY WARRANTY; without even the implied warranty of
  10 *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  11 *   NON INFRINGEMENT.  See the GNU General Public License for
  12 *   more details.
  13 */
  14
  15#include <asm/page.h>
  16#include <asm/cacheflush.h>
  17#include <arch/icache.h>
  18
  19
  20void __flush_icache_range(unsigned long start, unsigned long end)
  21{
  22        invalidate_icache((const void *)start, end - start, PAGE_SIZE);
  23}
  24
  25
  26/* Force a load instruction to issue. */
  27static inline void force_load(char *p)
  28{
  29        *(volatile char *)p;
  30}
  31
  32/*
  33 * Flush and invalidate a VA range that is homed remotely on a single
  34 * core (if "!hfh") or homed via hash-for-home (if "hfh"), waiting
  35 * until the memory controller holds the flushed values.
  36 */
  37void finv_buffer_remote(void *buffer, size_t size, int hfh)
  38{
  39        char *p, *base;
  40        size_t step_size, load_count;
  41        const unsigned long STRIPE_WIDTH = 8192;
  42
  43        /*
  44         * Flush and invalidate the buffer out of the local L1/L2
  45         * and request the home cache to flush and invalidate as well.
  46         */
  47        __finv_buffer(buffer, size);
  48
  49        /*
  50         * Wait for the home cache to acknowledge that it has processed
  51         * all the flush-and-invalidate requests.  This does not mean
  52         * that the flushed data has reached the memory controller yet,
  53         * but it does mean the home cache is processing the flushes.
  54         */
  55        __insn_mf();
  56
  57        /*
  58         * Issue a load to the last cache line, which can't complete
  59         * until all the previously-issued flushes to the same memory
  60         * controller have also completed.  If we weren't striping
  61         * memory, that one load would be sufficient, but since we may
  62         * be, we also need to back up to the last load issued to
  63         * another memory controller, which would be the point where
  64         * we crossed an 8KB boundary (the granularity of striping
  65         * across memory controllers).  Keep backing up and doing this
  66         * until we are before the beginning of the buffer, or have
  67         * hit all the controllers.
  68         *
  69         * If we are flushing a hash-for-home buffer, it's even worse.
  70         * Each line may be homed on a different tile, and each tile
  71         * may have up to four lines that are on different
  72         * controllers.  So as we walk backwards, we have to touch
  73         * enough cache lines to satisfy these constraints.  In
  74         * practice this ends up being close enough to "load from
  75         * every cache line on a full memory stripe on each
  76         * controller" that we simply do that, to simplify the logic.
  77         *
  78         * FIXME: See bug 9535 for some issues with this code.
  79         */
  80        if (hfh) {
  81                step_size = L2_CACHE_BYTES;
  82                load_count = (STRIPE_WIDTH / L2_CACHE_BYTES) *
  83                              (1 << CHIP_LOG_NUM_MSHIMS());
  84        } else {
  85                step_size = STRIPE_WIDTH;
  86                load_count = (1 << CHIP_LOG_NUM_MSHIMS());
  87        }
  88
  89        /* Load the last byte of the buffer. */
  90        p = (char *)buffer + size - 1;
  91        force_load(p);
  92
  93        /* Bump down to the end of the previous stripe or cache line. */
  94        p -= step_size;
  95        p = (char *)((unsigned long)p | (step_size - 1));
  96
  97        /* Figure out how far back we need to go. */
  98        base = p - (step_size * (load_count - 2));
  99        if ((long)base < (long)buffer)
 100                base = buffer;
 101
 102        /*
 103         * Fire all the loads we need.  The MAF only has eight entries
 104         * so we can have at most eight outstanding loads, so we
 105         * unroll by that amount.
 106         */
 107#pragma unroll 8
 108        for (; p >= base; p -= step_size)
 109                force_load(p);
 110
 111        /*
 112         * Repeat, but with inv's instead of loads, to get rid of the
 113         * data we just loaded into our own cache and the old home L3.
 114         * No need to unroll since inv's don't target a register.
 115         */
 116        p = (char *)buffer + size - 1;
 117        __insn_inv(p);
 118        p -= step_size;
 119        p = (char *)((unsigned long)p | (step_size - 1));
 120        for (; p >= base; p -= step_size)
 121                __insn_inv(p);
 122
 123        /* Wait for the load+inv's (and thus finvs) to have completed. */
 124        __insn_mf();
 125}
 126