1/* 2 * Copyright 2010 Tilera Corporation. All Rights Reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation, version 2. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 11 * NON INFRINGEMENT. See the GNU General Public License for 12 * more details. 13 */ 14 15#include <linux/export.h> 16#include <asm/page.h> 17#include <asm/cacheflush.h> 18#include <arch/icache.h> 19#include <arch/spr_def.h> 20 21 22void __flush_icache_range(unsigned long start, unsigned long end) 23{ 24 invalidate_icache((const void *)start, end - start, PAGE_SIZE); 25} 26 27 28/* Force a load instruction to issue. */ 29static inline void force_load(char *p) 30{ 31 *(volatile char *)p; 32} 33 34/* 35 * Flush and invalidate a VA range that is homed remotely on a single 36 * core (if "!hfh") or homed via hash-for-home (if "hfh"), waiting 37 * until the memory controller holds the flushed values. 38 */ 39void __attribute__((optimize("omit-frame-pointer"))) 40finv_buffer_remote(void *buffer, size_t size, int hfh) 41{ 42 char *p, *base; 43 size_t step_size, load_count; 44 45 /* 46 * On TILEPro the striping granularity is a fixed 8KB; on 47 * TILE-Gx it is configurable, and we rely on the fact that 48 * the hypervisor always configures maximum striping, so that 49 * bits 9 and 10 of the PA are part of the stripe function, so 50 * every 512 bytes we hit a striping boundary. 51 * 52 */ 53#ifdef __tilegx__ 54 const unsigned long STRIPE_WIDTH = 512; 55#else 56 const unsigned long STRIPE_WIDTH = 8192; 57#endif 58 59#ifdef __tilegx__ 60 /* 61 * On TILE-Gx, we must disable the dstream prefetcher before doing 62 * a cache flush; otherwise, we could end up with data in the cache 63 * that we don't want there. Note that normally we'd do an mf 64 * after the SPR write to disabling the prefetcher, but we do one 65 * below, before any further loads, so there's no need to do it 66 * here. 67 */ 68 uint_reg_t old_dstream_pf = __insn_mfspr(SPR_DSTREAM_PF); 69 __insn_mtspr(SPR_DSTREAM_PF, 0); 70#endif 71 72 /* 73 * Flush and invalidate the buffer out of the local L1/L2 74 * and request the home cache to flush and invalidate as well. 75 */ 76 __finv_buffer(buffer, size); 77 78 /* 79 * Wait for the home cache to acknowledge that it has processed 80 * all the flush-and-invalidate requests. This does not mean 81 * that the flushed data has reached the memory controller yet, 82 * but it does mean the home cache is processing the flushes. 83 */ 84 __insn_mf(); 85 86 /* 87 * Issue a load to the last cache line, which can't complete 88 * until all the previously-issued flushes to the same memory 89 * controller have also completed. If we weren't striping 90 * memory, that one load would be sufficient, but since we may 91 * be, we also need to back up to the last load issued to 92 * another memory controller, which would be the point where 93 * we crossed a "striping" boundary (the granularity of striping 94 * across memory controllers). Keep backing up and doing this 95 * until we are before the beginning of the buffer, or have 96 * hit all the controllers. 97 * 98 * If we are flushing a hash-for-home buffer, it's even worse. 99 * Each line may be homed on a different tile, and each tile 100 * may have up to four lines that are on different 101 * controllers. So as we walk backwards, we have to touch 102 * enough cache lines to satisfy these constraints. In 103 * practice this ends up being close enough to "load from 104 * every cache line on a full memory stripe on each 105 * controller" that we simply do that, to simplify the logic. 106 * 107 * On TILE-Gx the hash-for-home function is much more complex, 108 * with the upshot being we can't readily guarantee we have 109 * hit both entries in the 128-entry AMT that were hit by any 110 * load in the entire range, so we just re-load them all. 111 * With larger buffers, we may want to consider using a hypervisor 112 * trap to issue loads directly to each hash-for-home tile for 113 * each controller (doing it from Linux would trash the TLB). 114 */ 115 if (hfh) { 116 step_size = L2_CACHE_BYTES; 117#ifdef __tilegx__ 118 load_count = (size + L2_CACHE_BYTES - 1) / L2_CACHE_BYTES; 119#else 120 load_count = (STRIPE_WIDTH / L2_CACHE_BYTES) * 121 (1 << CHIP_LOG_NUM_MSHIMS()); 122#endif 123 } else { 124 step_size = STRIPE_WIDTH; 125 load_count = (1 << CHIP_LOG_NUM_MSHIMS()); 126 } 127 128 /* Load the last byte of the buffer. */ 129 p = (char *)buffer + size - 1; 130 force_load(p); 131 132 /* Bump down to the end of the previous stripe or cache line. */ 133 p -= step_size; 134 p = (char *)((unsigned long)p | (step_size - 1)); 135 136 /* Figure out how far back we need to go. */ 137 base = p - (step_size * (load_count - 2)); 138 if ((unsigned long)base < (unsigned long)buffer) 139 base = buffer; 140 141 /* 142 * Fire all the loads we need. The MAF only has eight entries 143 * so we can have at most eight outstanding loads, so we 144 * unroll by that amount. 145 */ 146#pragma unroll 8 147 for (; p >= base; p -= step_size) 148 force_load(p); 149 150 /* 151 * Repeat, but with finv's instead of loads, to get rid of the 152 * data we just loaded into our own cache and the old home L3. 153 * No need to unroll since finv's don't target a register. 154 * The finv's are guaranteed not to actually flush the data in 155 * the buffer back to their home, since we just read it, so the 156 * lines are clean in cache; we will only invalidate those lines. 157 */ 158 p = (char *)buffer + size - 1; 159 __insn_finv(p); 160 p -= step_size; 161 p = (char *)((unsigned long)p | (step_size - 1)); 162 for (; p >= base; p -= step_size) 163 __insn_finv(p); 164 165 /* Wait for these finv's (and thus the first finvs) to be done. */ 166 __insn_mf(); 167 168#ifdef __tilegx__ 169 /* Reenable the prefetcher. */ 170 __insn_mtspr(SPR_DSTREAM_PF, old_dstream_pf); 171#endif 172} 173EXPORT_SYMBOL_GPL(finv_buffer_remote); 174