LXR linux/mm/memfd.c

   1/*
   2 * memfd_create system call and file sealing support
   3 *
   4 * Code was originally included in shmem.c, and broken out to facilitate
   5 * use by hugetlbfs as well as tmpfs.
   6 *
   7 * This file is released under the GPL.
   8 */
   9
  10#include <linux/fs.h>
  11#include <linux/vfs.h>
  12#include <linux/pagemap.h>
  13#include <linux/file.h>
  14#include <linux/mm.h>
  15#include <linux/sched/signal.h>
  16#include <linux/khugepaged.h>
  17#include <linux/syscalls.h>
  18#include <linux/hugetlb.h>
  19#include <linux/shmem_fs.h>
  20#include <linux/memfd.h>
  21#include <uapi/linux/memfd.h>
  22
  23/*
  24 * We need a tag: a new tag would expand every xa_node by 8 bytes,
  25 * so reuse a tag which we firmly believe is never set or cleared on tmpfs
  26 * or hugetlbfs because they are memory only filesystems.
  27 */
  28#define MEMFD_TAG_PINNED        PAGECACHE_TAG_TOWRITE
  29#define LAST_SCAN               4       /* about 150ms max */
  30
  31static void memfd_tag_pins(struct xa_state *xas)
  32{
  33        struct page *page;
  34        int latency = 0;
  35        int cache_count;
  36
  37        lru_add_drain();
  38
  39        xas_lock_irq(xas);
  40        xas_for_each(xas, page, ULONG_MAX) {
  41                cache_count = 1;
  42                if (!xa_is_value(page) &&
  43                    PageTransHuge(page) && !PageHuge(page))
  44                        cache_count = HPAGE_PMD_NR;
  45
  46                if (!xa_is_value(page) &&
  47                    page_count(page) - total_mapcount(page) != cache_count)
  48                        xas_set_mark(xas, MEMFD_TAG_PINNED);
  49                if (cache_count != 1)
  50                        xas_set(xas, page->index + cache_count);
  51
  52                latency += cache_count;
  53                if (latency < XA_CHECK_SCHED)
  54                        continue;
  55                latency = 0;
  56
  57                xas_pause(xas);
  58                xas_unlock_irq(xas);
  59                cond_resched();
  60                xas_lock_irq(xas);
  61        }
  62        xas_unlock_irq(xas);
  63}
  64
  65/*
  66 * Setting SEAL_WRITE requires us to verify there's no pending writer. However,
  67 * via get_user_pages(), drivers might have some pending I/O without any active
  68 * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages
  69 * and see whether it has an elevated ref-count. If so, we tag them and wait for
  70 * them to be dropped.
  71 * The caller must guarantee that no new user will acquire writable references
  72 * to those pages to avoid races.
  73 */
  74static int memfd_wait_for_pins(struct address_space *mapping)
  75{
  76        XA_STATE(xas, &mapping->i_pages, 0);
  77        struct page *page;
  78        int error, scan;
  79
  80        memfd_tag_pins(&xas);
  81
  82        error = 0;
  83        for (scan = 0; scan <= LAST_SCAN; scan++) {
  84                int latency = 0;
  85                int cache_count;
  86
  87                if (!xas_marked(&xas, MEMFD_TAG_PINNED))
  88                        break;
  89
  90                if (!scan)
  91                        lru_add_drain_all();
  92                else if (schedule_timeout_killable((HZ << scan) / 200))
  93                        scan = LAST_SCAN;
  94
  95                xas_set(&xas, 0);
  96                xas_lock_irq(&xas);
  97                xas_for_each_marked(&xas, page, ULONG_MAX, MEMFD_TAG_PINNED) {
  98                        bool clear = true;
  99
 100                        cache_count = 1;
 101                        if (!xa_is_value(page) &&
 102                            PageTransHuge(page) && !PageHuge(page))
 103                                cache_count = HPAGE_PMD_NR;
 104
 105                        if (!xa_is_value(page) && cache_count !=
 106                            page_count(page) - total_mapcount(page)) {
 107                                /*
 108                                 * On the last scan, we clean up all those tags
 109                                 * we inserted; but make a note that we still
 110                                 * found pages pinned.
 111                                 */
 112                                if (scan == LAST_SCAN)
 113                                        error = -EBUSY;
 114                                else
 115                                        clear = false;
 116                        }
 117                        if (clear)
 118                                xas_clear_mark(&xas, MEMFD_TAG_PINNED);
 119
 120                        latency += cache_count;
 121                        if (latency < XA_CHECK_SCHED)
 122                                continue;
 123                        latency = 0;
 124
 125                        xas_pause(&xas);
 126                        xas_unlock_irq(&xas);
 127                        cond_resched();
 128                        xas_lock_irq(&xas);
 129                }
 130                xas_unlock_irq(&xas);
 131        }
 132
 133        return error;
 134}
 135
 136static unsigned int *memfd_file_seals_ptr(struct file *file)
 137{
 138        if (shmem_file(file))
 139                return &SHMEM_I(file_inode(file))->seals;
 140
 141#ifdef CONFIG_HUGETLBFS
 142        if (is_file_hugepages(file))
 143                return &HUGETLBFS_I(file_inode(file))->seals;
 144#endif
 145
 146        return NULL;
 147}
 148
 149#define F_ALL_SEALS (F_SEAL_SEAL | \
 150                     F_SEAL_SHRINK | \
 151                     F_SEAL_GROW | \
 152                     F_SEAL_WRITE | \
 153                     F_SEAL_FUTURE_WRITE)
 154
 155static int memfd_add_seals(struct file *file, unsigned int seals)
 156{
 157        struct inode *inode = file_inode(file);
 158        unsigned int *file_seals;
 159        int error;
 160
 161        /*
 162         * SEALING
 163         * Sealing allows multiple parties to share a tmpfs or hugetlbfs file
 164         * but restrict access to a specific subset of file operations. Seals
 165         * can only be added, but never removed. This way, mutually untrusted
 166         * parties can share common memory regions with a well-defined policy.
 167         * A malicious peer can thus never perform unwanted operations on a
 168         * shared object.
 169         *
 170         * Seals are only supported on special tmpfs or hugetlbfs files and
 171         * always affect the whole underlying inode. Once a seal is set, it
 172         * may prevent some kinds of access to the file. Currently, the
 173         * following seals are defined:
 174         *   SEAL_SEAL: Prevent further seals from being set on this file
 175         *   SEAL_SHRINK: Prevent the file from shrinking
 176         *   SEAL_GROW: Prevent the file from growing
 177         *   SEAL_WRITE: Prevent write access to the file
 178         *
 179         * As we don't require any trust relationship between two parties, we
 180         * must prevent seals from being removed. Therefore, sealing a file
 181         * only adds a given set of seals to the file, it never touches
 182         * existing seals. Furthermore, the "setting seals"-operation can be
 183         * sealed itself, which basically prevents any further seal from being
 184         * added.
 185         *
 186         * Semantics of sealing are only defined on volatile files. Only
 187         * anonymous tmpfs and hugetlbfs files support sealing. More
 188         * importantly, seals are never written to disk. Therefore, there's
 189         * no plan to support it on other file types.
 190         */
 191
 192        if (!(file->f_mode & FMODE_WRITE))
 193                return -EPERM;
 194        if (seals & ~(unsigned int)F_ALL_SEALS)
 195                return -EINVAL;
 196
 197        inode_lock(inode);
 198
 199        file_seals = memfd_file_seals_ptr(file);
 200        if (!file_seals) {
 201                error = -EINVAL;
 202                goto unlock;
 203        }
 204
 205        if (*file_seals & F_SEAL_SEAL) {
 206                error = -EPERM;
 207                goto unlock;
 208        }
 209
 210        if ((seals & F_SEAL_WRITE) && !(*file_seals & F_SEAL_WRITE)) {
 211                error = mapping_deny_writable(file->f_mapping);
 212                if (error)
 213                        goto unlock;
 214
 215                error = memfd_wait_for_pins(file->f_mapping);
 216                if (error) {
 217                        mapping_allow_writable(file->f_mapping);
 218                        goto unlock;
 219                }
 220        }
 221
 222        *file_seals |= seals;
 223        error = 0;
 224
 225unlock:
 226        inode_unlock(inode);
 227        return error;
 228}
 229
 230static int memfd_get_seals(struct file *file)
 231{
 232        unsigned int *seals = memfd_file_seals_ptr(file);
 233
 234        return seals ? *seals : -EINVAL;
 235}
 236
 237long memfd_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
 238{
 239        long error;
 240
 241        switch (cmd) {
 242        case F_ADD_SEALS:
 243                /* disallow upper 32bit */
 244                if (arg > UINT_MAX)
 245                        return -EINVAL;
 246
 247                error = memfd_add_seals(file, arg);
 248                break;
 249        case F_GET_SEALS:
 250                error = memfd_get_seals(file);
 251                break;
 252        default:
 253                error = -EINVAL;
 254                break;
 255        }
 256
 257        return error;
 258}
 259
 260#define MFD_NAME_PREFIX "memfd:"
 261#define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1)
 262#define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN)
 263
 264#define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_HUGETLB)
 265
 266SYSCALL_DEFINE2(memfd_create,
 267                const char __user *, uname,
 268                unsigned int, flags)
 269{
 270        unsigned int *file_seals;
 271        struct file *file;
 272        int fd, error;
 273        char *name;
 274        long len;
 275
 276        if (!(flags & MFD_HUGETLB)) {
 277                if (flags & ~(unsigned int)MFD_ALL_FLAGS)
 278                        return -EINVAL;
 279        } else {
 280                /* Allow huge page size encoding in flags. */
 281                if (flags & ~(unsigned int)(MFD_ALL_FLAGS |
 282                                (MFD_HUGE_MASK << MFD_HUGE_SHIFT)))
 283                        return -EINVAL;
 284        }
 285
 286        /* length includes terminating zero */
 287        len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1);
 288        if (len <= 0)
 289                return -EFAULT;
 290        if (len > MFD_NAME_MAX_LEN + 1)
 291                return -EINVAL;
 292
 293        name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_KERNEL);
 294        if (!name)
 295                return -ENOMEM;
 296
 297        strcpy(name, MFD_NAME_PREFIX);
 298        if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) {
 299                error = -EFAULT;
 300                goto err_name;
 301        }
 302
 303        /* terminating-zero may have changed after strnlen_user() returned */
 304        if (name[len + MFD_NAME_PREFIX_LEN - 1]) {
 305                error = -EFAULT;
 306                goto err_name;
 307        }
 308
 309        fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0);
 310        if (fd < 0) {
 311                error = fd;
 312                goto err_name;
 313        }
 314
 315        if (flags & MFD_HUGETLB) {
 316                file = hugetlb_file_setup(name, 0, VM_NORESERVE,
 317                                        HUGETLB_ANONHUGE_INODE,
 318                                        (flags >> MFD_HUGE_SHIFT) &
 319                                        MFD_HUGE_MASK);
 320        } else
 321                file = shmem_file_setup(name, 0, VM_NORESERVE);
 322        if (IS_ERR(file)) {
 323                error = PTR_ERR(file);
 324                goto err_fd;
 325        }
 326        file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE;
 327        file->f_flags |= O_LARGEFILE;
 328
 329        if (flags & MFD_ALLOW_SEALING) {
 330                file_seals = memfd_file_seals_ptr(file);
 331                *file_seals &= ~F_SEAL_SEAL;
 332        }
 333
 334        fd_install(fd, file);
 335        kfree(name);
 336        return fd;
 337
 338err_fd:
 339        put_unused_fd(fd);
 340err_name:
 341        kfree(name);
 342        return error;
 343}
 344