linux/drivers/gpu/drm/gma500/mmu.c
<<
>>
Prefs
   1/**************************************************************************
   2 * Copyright (c) 2007, Intel Corporation.
   3 *
   4 * This program is free software; you can redistribute it and/or modify it
   5 * under the terms and conditions of the GNU General Public License,
   6 * version 2, as published by the Free Software Foundation.
   7 *
   8 * This program is distributed in the hope it will be useful, but WITHOUT
   9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11 * more details.
  12 *
  13 * You should have received a copy of the GNU General Public License along with
  14 * this program; if not, write to the Free Software Foundation, Inc.,
  15 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  16 *
  17 **************************************************************************/
  18#include <drm/drmP.h>
  19#include "psb_drv.h"
  20#include "psb_reg.h"
  21#include "mmu.h"
  22
  23/*
  24 * Code for the SGX MMU:
  25 */
  26
  27/*
  28 * clflush on one processor only:
  29 * clflush should apparently flush the cache line on all processors in an
  30 * SMP system.
  31 */
  32
  33/*
  34 * kmap atomic:
  35 * The usage of the slots must be completely encapsulated within a spinlock, and
  36 * no other functions that may be using the locks for other purposed may be
  37 * called from within the locked region.
  38 * Since the slots are per processor, this will guarantee that we are the only
  39 * user.
  40 */
  41
  42/*
  43 * TODO: Inserting ptes from an interrupt handler:
  44 * This may be desirable for some SGX functionality where the GPU can fault in
  45 * needed pages. For that, we need to make an atomic insert_pages function, that
  46 * may fail.
  47 * If it fails, the caller need to insert the page using a workqueue function,
  48 * but on average it should be fast.
  49 */
  50
  51static inline uint32_t psb_mmu_pt_index(uint32_t offset)
  52{
  53        return (offset >> PSB_PTE_SHIFT) & 0x3FF;
  54}
  55
  56static inline uint32_t psb_mmu_pd_index(uint32_t offset)
  57{
  58        return offset >> PSB_PDE_SHIFT;
  59}
  60
  61#if defined(CONFIG_X86)
  62static inline void psb_clflush(void *addr)
  63{
  64        __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory");
  65}
  66
  67static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
  68{
  69        if (!driver->has_clflush)
  70                return;
  71
  72        mb();
  73        psb_clflush(addr);
  74        mb();
  75}
  76#else
  77
  78static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
  79{;
  80}
  81
  82#endif
  83
  84static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)
  85{
  86        struct drm_device *dev = driver->dev;
  87        struct drm_psb_private *dev_priv = dev->dev_private;
  88
  89        if (atomic_read(&driver->needs_tlbflush) || force) {
  90                uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL);
  91                PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
  92
  93                /* Make sure data cache is turned off before enabling it */
  94                wmb();
  95                PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
  96                (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
  97                if (driver->msvdx_mmu_invaldc)
  98                        atomic_set(driver->msvdx_mmu_invaldc, 1);
  99        }
 100        atomic_set(&driver->needs_tlbflush, 0);
 101}
 102
 103#if 0
 104static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
 105{
 106        down_write(&driver->sem);
 107        psb_mmu_flush_pd_locked(driver, force);
 108        up_write(&driver->sem);
 109}
 110#endif
 111
 112void psb_mmu_flush(struct psb_mmu_driver *driver)
 113{
 114        struct drm_device *dev = driver->dev;
 115        struct drm_psb_private *dev_priv = dev->dev_private;
 116        uint32_t val;
 117
 118        down_write(&driver->sem);
 119        val = PSB_RSGX32(PSB_CR_BIF_CTRL);
 120        if (atomic_read(&driver->needs_tlbflush))
 121                PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
 122        else
 123                PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL);
 124
 125        /* Make sure data cache is turned off and MMU is flushed before
 126           restoring bank interface control register */
 127        wmb();
 128        PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC),
 129                   PSB_CR_BIF_CTRL);
 130        (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
 131
 132        atomic_set(&driver->needs_tlbflush, 0);
 133        if (driver->msvdx_mmu_invaldc)
 134                atomic_set(driver->msvdx_mmu_invaldc, 1);
 135        up_write(&driver->sem);
 136}
 137
 138void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)
 139{
 140        struct drm_device *dev = pd->driver->dev;
 141        struct drm_psb_private *dev_priv = dev->dev_private;
 142        uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 :
 143                          PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4;
 144
 145        down_write(&pd->driver->sem);
 146        PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset);
 147        wmb();
 148        psb_mmu_flush_pd_locked(pd->driver, 1);
 149        pd->hw_context = hw_context;
 150        up_write(&pd->driver->sem);
 151
 152}
 153
 154static inline unsigned long psb_pd_addr_end(unsigned long addr,
 155                                            unsigned long end)
 156{
 157        addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;
 158        return (addr < end) ? addr : end;
 159}
 160
 161static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type)
 162{
 163        uint32_t mask = PSB_PTE_VALID;
 164
 165        if (type & PSB_MMU_CACHED_MEMORY)
 166                mask |= PSB_PTE_CACHED;
 167        if (type & PSB_MMU_RO_MEMORY)
 168                mask |= PSB_PTE_RO;
 169        if (type & PSB_MMU_WO_MEMORY)
 170                mask |= PSB_PTE_WO;
 171
 172        return (pfn << PAGE_SHIFT) | mask;
 173}
 174
 175struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
 176                                    int trap_pagefaults, int invalid_type)
 177{
 178        struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL);
 179        uint32_t *v;
 180        int i;
 181
 182        if (!pd)
 183                return NULL;
 184
 185        pd->p = alloc_page(GFP_DMA32);
 186        if (!pd->p)
 187                goto out_err1;
 188        pd->dummy_pt = alloc_page(GFP_DMA32);
 189        if (!pd->dummy_pt)
 190                goto out_err2;
 191        pd->dummy_page = alloc_page(GFP_DMA32);
 192        if (!pd->dummy_page)
 193                goto out_err3;
 194
 195        if (!trap_pagefaults) {
 196                pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),
 197                                                   invalid_type);
 198                pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),
 199                                                   invalid_type);
 200        } else {
 201                pd->invalid_pde = 0;
 202                pd->invalid_pte = 0;
 203        }
 204
 205        v = kmap(pd->dummy_pt);
 206        for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
 207                v[i] = pd->invalid_pte;
 208
 209        kunmap(pd->dummy_pt);
 210
 211        v = kmap(pd->p);
 212        for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
 213                v[i] = pd->invalid_pde;
 214
 215        kunmap(pd->p);
 216
 217        clear_page(kmap(pd->dummy_page));
 218        kunmap(pd->dummy_page);
 219
 220        pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024);
 221        if (!pd->tables)
 222                goto out_err4;
 223
 224        pd->hw_context = -1;
 225        pd->pd_mask = PSB_PTE_VALID;
 226        pd->driver = driver;
 227
 228        return pd;
 229
 230out_err4:
 231        __free_page(pd->dummy_page);
 232out_err3:
 233        __free_page(pd->dummy_pt);
 234out_err2:
 235        __free_page(pd->p);
 236out_err1:
 237        kfree(pd);
 238        return NULL;
 239}
 240
 241static void psb_mmu_free_pt(struct psb_mmu_pt *pt)
 242{
 243        __free_page(pt->p);
 244        kfree(pt);
 245}
 246
 247void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)
 248{
 249        struct psb_mmu_driver *driver = pd->driver;
 250        struct drm_device *dev = driver->dev;
 251        struct drm_psb_private *dev_priv = dev->dev_private;
 252        struct psb_mmu_pt *pt;
 253        int i;
 254
 255        down_write(&driver->sem);
 256        if (pd->hw_context != -1) {
 257                PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4);
 258                psb_mmu_flush_pd_locked(driver, 1);
 259        }
 260
 261        /* Should take the spinlock here, but we don't need to do that
 262           since we have the semaphore in write mode. */
 263
 264        for (i = 0; i < 1024; ++i) {
 265                pt = pd->tables[i];
 266                if (pt)
 267                        psb_mmu_free_pt(pt);
 268        }
 269
 270        vfree(pd->tables);
 271        __free_page(pd->dummy_page);
 272        __free_page(pd->dummy_pt);
 273        __free_page(pd->p);
 274        kfree(pd);
 275        up_write(&driver->sem);
 276}
 277
 278static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
 279{
 280        struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);
 281        void *v;
 282        uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;
 283        uint32_t clflush_count = PAGE_SIZE / clflush_add;
 284        spinlock_t *lock = &pd->driver->lock;
 285        uint8_t *clf;
 286        uint32_t *ptes;
 287        int i;
 288
 289        if (!pt)
 290                return NULL;
 291
 292        pt->p = alloc_page(GFP_DMA32);
 293        if (!pt->p) {
 294                kfree(pt);
 295                return NULL;
 296        }
 297
 298        spin_lock(lock);
 299
 300        v = kmap_atomic(pt->p);
 301        clf = (uint8_t *) v;
 302        ptes = (uint32_t *) v;
 303        for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
 304                *ptes++ = pd->invalid_pte;
 305
 306#if defined(CONFIG_X86)
 307        if (pd->driver->has_clflush && pd->hw_context != -1) {
 308                mb();
 309                for (i = 0; i < clflush_count; ++i) {
 310                        psb_clflush(clf);
 311                        clf += clflush_add;
 312                }
 313                mb();
 314        }
 315#endif
 316        kunmap_atomic(v);
 317        spin_unlock(lock);
 318
 319        pt->count = 0;
 320        pt->pd = pd;
 321        pt->index = 0;
 322
 323        return pt;
 324}
 325
 326struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
 327                                             unsigned long addr)
 328{
 329        uint32_t index = psb_mmu_pd_index(addr);
 330        struct psb_mmu_pt *pt;
 331        uint32_t *v;
 332        spinlock_t *lock = &pd->driver->lock;
 333
 334        spin_lock(lock);
 335        pt = pd->tables[index];
 336        while (!pt) {
 337                spin_unlock(lock);
 338                pt = psb_mmu_alloc_pt(pd);
 339                if (!pt)
 340                        return NULL;
 341                spin_lock(lock);
 342
 343                if (pd->tables[index]) {
 344                        spin_unlock(lock);
 345                        psb_mmu_free_pt(pt);
 346                        spin_lock(lock);
 347                        pt = pd->tables[index];
 348                        continue;
 349                }
 350
 351                v = kmap_atomic(pd->p);
 352                pd->tables[index] = pt;
 353                v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;
 354                pt->index = index;
 355                kunmap_atomic((void *) v);
 356
 357                if (pd->hw_context != -1) {
 358                        psb_mmu_clflush(pd->driver, (void *)&v[index]);
 359                        atomic_set(&pd->driver->needs_tlbflush, 1);
 360                }
 361        }
 362        pt->v = kmap_atomic(pt->p);
 363        return pt;
 364}
 365
 366static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd,
 367                                              unsigned long addr)
 368{
 369        uint32_t index = psb_mmu_pd_index(addr);
 370        struct psb_mmu_pt *pt;
 371        spinlock_t *lock = &pd->driver->lock;
 372
 373        spin_lock(lock);
 374        pt = pd->tables[index];
 375        if (!pt) {
 376                spin_unlock(lock);
 377                return NULL;
 378        }
 379        pt->v = kmap_atomic(pt->p);
 380        return pt;
 381}
 382
 383static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
 384{
 385        struct psb_mmu_pd *pd = pt->pd;
 386        uint32_t *v;
 387
 388        kunmap_atomic(pt->v);
 389        if (pt->count == 0) {
 390                v = kmap_atomic(pd->p);
 391                v[pt->index] = pd->invalid_pde;
 392                pd->tables[pt->index] = NULL;
 393
 394                if (pd->hw_context != -1) {
 395                        psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);
 396                        atomic_set(&pd->driver->needs_tlbflush, 1);
 397                }
 398                kunmap_atomic(v);
 399                spin_unlock(&pd->driver->lock);
 400                psb_mmu_free_pt(pt);
 401                return;
 402        }
 403        spin_unlock(&pd->driver->lock);
 404}
 405
 406static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,
 407                                   uint32_t pte)
 408{
 409        pt->v[psb_mmu_pt_index(addr)] = pte;
 410}
 411
 412static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,
 413                                          unsigned long addr)
 414{
 415        pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;
 416}
 417
 418struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)
 419{
 420        struct psb_mmu_pd *pd;
 421
 422        down_read(&driver->sem);
 423        pd = driver->default_pd;
 424        up_read(&driver->sem);
 425
 426        return pd;
 427}
 428
 429/* Returns the physical address of the PD shared by sgx/msvdx */
 430uint32_t psb_get_default_pd_addr(struct psb_mmu_driver *driver)
 431{
 432        struct psb_mmu_pd *pd;
 433
 434        pd = psb_mmu_get_default_pd(driver);
 435        return page_to_pfn(pd->p) << PAGE_SHIFT;
 436}
 437
 438void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)
 439{
 440        struct drm_device *dev = driver->dev;
 441        struct drm_psb_private *dev_priv = dev->dev_private;
 442
 443        PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL);
 444        psb_mmu_free_pagedir(driver->default_pd);
 445        kfree(driver);
 446}
 447
 448struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev,
 449                                           int trap_pagefaults,
 450                                           int invalid_type,
 451                                           atomic_t *msvdx_mmu_invaldc)
 452{
 453        struct psb_mmu_driver *driver;
 454        struct drm_psb_private *dev_priv = dev->dev_private;
 455
 456        driver = kmalloc(sizeof(*driver), GFP_KERNEL);
 457
 458        if (!driver)
 459                return NULL;
 460
 461        driver->dev = dev;
 462        driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,
 463                                              invalid_type);
 464        if (!driver->default_pd)
 465                goto out_err1;
 466
 467        spin_lock_init(&driver->lock);
 468        init_rwsem(&driver->sem);
 469        down_write(&driver->sem);
 470        atomic_set(&driver->needs_tlbflush, 1);
 471        driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;
 472
 473        driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL);
 474        PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,
 475                   PSB_CR_BIF_CTRL);
 476        PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,
 477                   PSB_CR_BIF_CTRL);
 478
 479        driver->has_clflush = 0;
 480
 481#if defined(CONFIG_X86)
 482        if (boot_cpu_has(X86_FEATURE_CLFLUSH)) {
 483                uint32_t tfms, misc, cap0, cap4, clflush_size;
 484
 485                /*
 486                 * clflush size is determined at kernel setup for x86_64 but not
 487                 * for i386. We have to do it here.
 488                 */
 489
 490                cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);
 491                clflush_size = ((misc >> 8) & 0xff) * 8;
 492                driver->has_clflush = 1;
 493                driver->clflush_add =
 494                    PAGE_SIZE * clflush_size / sizeof(uint32_t);
 495                driver->clflush_mask = driver->clflush_add - 1;
 496                driver->clflush_mask = ~driver->clflush_mask;
 497        }
 498#endif
 499
 500        up_write(&driver->sem);
 501        return driver;
 502
 503out_err1:
 504        kfree(driver);
 505        return NULL;
 506}
 507
 508#if defined(CONFIG_X86)
 509static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
 510                               uint32_t num_pages, uint32_t desired_tile_stride,
 511                               uint32_t hw_tile_stride)
 512{
 513        struct psb_mmu_pt *pt;
 514        uint32_t rows = 1;
 515        uint32_t i;
 516        unsigned long addr;
 517        unsigned long end;
 518        unsigned long next;
 519        unsigned long add;
 520        unsigned long row_add;
 521        unsigned long clflush_add = pd->driver->clflush_add;
 522        unsigned long clflush_mask = pd->driver->clflush_mask;
 523
 524        if (!pd->driver->has_clflush)
 525                return;
 526
 527        if (hw_tile_stride)
 528                rows = num_pages / desired_tile_stride;
 529        else
 530                desired_tile_stride = num_pages;
 531
 532        add = desired_tile_stride << PAGE_SHIFT;
 533        row_add = hw_tile_stride << PAGE_SHIFT;
 534        mb();
 535        for (i = 0; i < rows; ++i) {
 536
 537                addr = address;
 538                end = addr + add;
 539
 540                do {
 541                        next = psb_pd_addr_end(addr, end);
 542                        pt = psb_mmu_pt_map_lock(pd, addr);
 543                        if (!pt)
 544                                continue;
 545                        do {
 546                                psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);
 547                        } while (addr += clflush_add,
 548                                 (addr & clflush_mask) < next);
 549
 550                        psb_mmu_pt_unmap_unlock(pt);
 551                } while (addr = next, next != end);
 552                address += row_add;
 553        }
 554        mb();
 555}
 556#else
 557static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
 558                               uint32_t num_pages, uint32_t desired_tile_stride,
 559                               uint32_t hw_tile_stride)
 560{
 561        drm_ttm_cache_flush();
 562}
 563#endif
 564
 565void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
 566                                 unsigned long address, uint32_t num_pages)
 567{
 568        struct psb_mmu_pt *pt;
 569        unsigned long addr;
 570        unsigned long end;
 571        unsigned long next;
 572        unsigned long f_address = address;
 573
 574        down_read(&pd->driver->sem);
 575
 576        addr = address;
 577        end = addr + (num_pages << PAGE_SHIFT);
 578
 579        do {
 580                next = psb_pd_addr_end(addr, end);
 581                pt = psb_mmu_pt_alloc_map_lock(pd, addr);
 582                if (!pt)
 583                        goto out;
 584                do {
 585                        psb_mmu_invalidate_pte(pt, addr);
 586                        --pt->count;
 587                } while (addr += PAGE_SIZE, addr < next);
 588                psb_mmu_pt_unmap_unlock(pt);
 589
 590        } while (addr = next, next != end);
 591
 592out:
 593        if (pd->hw_context != -1)
 594                psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
 595
 596        up_read(&pd->driver->sem);
 597
 598        if (pd->hw_context != -1)
 599                psb_mmu_flush(pd->driver);
 600
 601        return;
 602}
 603
 604void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,
 605                          uint32_t num_pages, uint32_t desired_tile_stride,
 606                          uint32_t hw_tile_stride)
 607{
 608        struct psb_mmu_pt *pt;
 609        uint32_t rows = 1;
 610        uint32_t i;
 611        unsigned long addr;
 612        unsigned long end;
 613        unsigned long next;
 614        unsigned long add;
 615        unsigned long row_add;
 616        unsigned long f_address = address;
 617
 618        if (hw_tile_stride)
 619                rows = num_pages / desired_tile_stride;
 620        else
 621                desired_tile_stride = num_pages;
 622
 623        add = desired_tile_stride << PAGE_SHIFT;
 624        row_add = hw_tile_stride << PAGE_SHIFT;
 625
 626        down_read(&pd->driver->sem);
 627
 628        /* Make sure we only need to flush this processor's cache */
 629
 630        for (i = 0; i < rows; ++i) {
 631
 632                addr = address;
 633                end = addr + add;
 634
 635                do {
 636                        next = psb_pd_addr_end(addr, end);
 637                        pt = psb_mmu_pt_map_lock(pd, addr);
 638                        if (!pt)
 639                                continue;
 640                        do {
 641                                psb_mmu_invalidate_pte(pt, addr);
 642                                --pt->count;
 643
 644                        } while (addr += PAGE_SIZE, addr < next);
 645                        psb_mmu_pt_unmap_unlock(pt);
 646
 647                } while (addr = next, next != end);
 648                address += row_add;
 649        }
 650        if (pd->hw_context != -1)
 651                psb_mmu_flush_ptes(pd, f_address, num_pages,
 652                                   desired_tile_stride, hw_tile_stride);
 653
 654        up_read(&pd->driver->sem);
 655
 656        if (pd->hw_context != -1)
 657                psb_mmu_flush(pd->driver);
 658}
 659
 660int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
 661                                unsigned long address, uint32_t num_pages,
 662                                int type)
 663{
 664        struct psb_mmu_pt *pt;
 665        uint32_t pte;
 666        unsigned long addr;
 667        unsigned long end;
 668        unsigned long next;
 669        unsigned long f_address = address;
 670        int ret = -ENOMEM;
 671
 672        down_read(&pd->driver->sem);
 673
 674        addr = address;
 675        end = addr + (num_pages << PAGE_SHIFT);
 676
 677        do {
 678                next = psb_pd_addr_end(addr, end);
 679                pt = psb_mmu_pt_alloc_map_lock(pd, addr);
 680                if (!pt) {
 681                        ret = -ENOMEM;
 682                        goto out;
 683                }
 684                do {
 685                        pte = psb_mmu_mask_pte(start_pfn++, type);
 686                        psb_mmu_set_pte(pt, addr, pte);
 687                        pt->count++;
 688                } while (addr += PAGE_SIZE, addr < next);
 689                psb_mmu_pt_unmap_unlock(pt);
 690
 691        } while (addr = next, next != end);
 692        ret = 0;
 693
 694out:
 695        if (pd->hw_context != -1)
 696                psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
 697
 698        up_read(&pd->driver->sem);
 699
 700        if (pd->hw_context != -1)
 701                psb_mmu_flush(pd->driver);
 702
 703        return 0;
 704}
 705
 706int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
 707                         unsigned long address, uint32_t num_pages,
 708                         uint32_t desired_tile_stride, uint32_t hw_tile_stride,
 709                         int type)
 710{
 711        struct psb_mmu_pt *pt;
 712        uint32_t rows = 1;
 713        uint32_t i;
 714        uint32_t pte;
 715        unsigned long addr;
 716        unsigned long end;
 717        unsigned long next;
 718        unsigned long add;
 719        unsigned long row_add;
 720        unsigned long f_address = address;
 721        int ret = -ENOMEM;
 722
 723        if (hw_tile_stride) {
 724                if (num_pages % desired_tile_stride != 0)
 725                        return -EINVAL;
 726                rows = num_pages / desired_tile_stride;
 727        } else {
 728                desired_tile_stride = num_pages;
 729        }
 730
 731        add = desired_tile_stride << PAGE_SHIFT;
 732        row_add = hw_tile_stride << PAGE_SHIFT;
 733
 734        down_read(&pd->driver->sem);
 735
 736        for (i = 0; i < rows; ++i) {
 737
 738                addr = address;
 739                end = addr + add;
 740
 741                do {
 742                        next = psb_pd_addr_end(addr, end);
 743                        pt = psb_mmu_pt_alloc_map_lock(pd, addr);
 744                        if (!pt)
 745                                goto out;
 746                        do {
 747                                pte = psb_mmu_mask_pte(page_to_pfn(*pages++),
 748                                                       type);
 749                                psb_mmu_set_pte(pt, addr, pte);
 750                                pt->count++;
 751                        } while (addr += PAGE_SIZE, addr < next);
 752                        psb_mmu_pt_unmap_unlock(pt);
 753
 754                } while (addr = next, next != end);
 755
 756                address += row_add;
 757        }
 758
 759        ret = 0;
 760out:
 761        if (pd->hw_context != -1)
 762                psb_mmu_flush_ptes(pd, f_address, num_pages,
 763                                   desired_tile_stride, hw_tile_stride);
 764
 765        up_read(&pd->driver->sem);
 766
 767        if (pd->hw_context != -1)
 768                psb_mmu_flush(pd->driver);
 769
 770        return ret;
 771}
 772
 773int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,
 774                           unsigned long *pfn)
 775{
 776        int ret;
 777        struct psb_mmu_pt *pt;
 778        uint32_t tmp;
 779        spinlock_t *lock = &pd->driver->lock;
 780
 781        down_read(&pd->driver->sem);
 782        pt = psb_mmu_pt_map_lock(pd, virtual);
 783        if (!pt) {
 784                uint32_t *v;
 785
 786                spin_lock(lock);
 787                v = kmap_atomic(pd->p);
 788                tmp = v[psb_mmu_pd_index(virtual)];
 789                kunmap_atomic(v);
 790                spin_unlock(lock);
 791
 792                if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) ||
 793                    !(pd->invalid_pte & PSB_PTE_VALID)) {
 794                        ret = -EINVAL;
 795                        goto out;
 796                }
 797                ret = 0;
 798                *pfn = pd->invalid_pte >> PAGE_SHIFT;
 799                goto out;
 800        }
 801        tmp = pt->v[psb_mmu_pt_index(virtual)];
 802        if (!(tmp & PSB_PTE_VALID)) {
 803                ret = -EINVAL;
 804        } else {
 805                ret = 0;
 806                *pfn = tmp >> PAGE_SHIFT;
 807        }
 808        psb_mmu_pt_unmap_unlock(pt);
 809out:
 810        up_read(&pd->driver->sem);
 811        return ret;
 812}
 813