LXR dpdk/usertools/dpdk-hugepages.py

   1#! /usr/bin/env python3
   2# SPDX-License-Identifier: BSD-3-Clause
   3# Copyright (c) 2020 Microsoft Corporation
   4"""Script to query and setup huge pages for DPDK applications."""
   5
   6import argparse
   7import glob
   8import os
   9import re
  10import sys
  11from math import log2
  12
  13# Standard binary prefix
  14BINARY_PREFIX = "KMG"
  15
  16# systemd mount point for huge pages
  17HUGE_MOUNT = "/dev/hugepages"
  18
  19
  20def fmt_memsize(kb):
  21    '''Format memory size in kB into conventional format'''
  22    logk = int(log2(kb) / 10)
  23    suffix = BINARY_PREFIX[logk]
  24    unit = 2**(logk * 10)
  25    return '{}{}b'.format(int(kb / unit), suffix)
  26
  27
  28def get_memsize(arg):
  29    '''Convert memory size with suffix to kB'''
  30    match = re.match(r'(\d+)([' + BINARY_PREFIX + r']?)$', arg.upper())
  31    if match is None:
  32        sys.exit('{} is not a valid size'.format(arg))
  33    num = float(match.group(1))
  34    suffix = match.group(2)
  35    if suffix == "":
  36        return int(num / 1024)
  37    idx = BINARY_PREFIX.find(suffix)
  38    return int(num * (2**(idx * 10)))
  39
  40
  41def is_numa():
  42    '''Test if NUMA is necessary on this system'''
  43    return os.path.exists('/sys/devices/system/node')
  44
  45
  46def get_valid_page_sizes(path):
  47    '''Extract valid hugepage sizes'''
  48    dir = os.path.dirname(path)
  49    pg_sizes = (d.split("-")[1] for d in os.listdir(dir))
  50    return " ".join(pg_sizes)
  51
  52
  53def get_hugepages(path):
  54    '''Read number of reserved pages'''
  55    with open(path + '/nr_hugepages') as nr_hugepages:
  56        return int(nr_hugepages.read())
  57    return 0
  58
  59
  60def set_hugepages(path, reqpages):
  61    '''Write the number of reserved huge pages'''
  62    filename = path + '/nr_hugepages'
  63    try:
  64        with open(filename, 'w') as nr_hugepages:
  65            nr_hugepages.write('{}\n'.format(reqpages))
  66    except PermissionError:
  67        sys.exit('Permission denied: need to be root!')
  68    except FileNotFoundError:
  69        sys.exit("Invalid page size. Valid page sizes: {}".format(
  70                 get_valid_page_sizes(path)))
  71    gotpages = get_hugepages(path)
  72    if gotpages != reqpages:
  73        sys.exit('Unable to set pages ({} instead of {} in {}).'.format(
  74                 gotpages, reqpages, filename))
  75
  76
  77def show_numa_pages():
  78    '''Show huge page reservations on Numa system'''
  79    print('Node Pages Size Total')
  80    for numa_path in glob.glob('/sys/devices/system/node/node*'):
  81        node = numa_path[29:]  # slice after /sys/devices/system/node/node
  82        path = numa_path + '/hugepages'
  83        if not os.path.exists(path):
  84            continue
  85        for hdir in os.listdir(path):
  86            pages = get_hugepages(path + '/' + hdir)
  87            if pages > 0:
  88                kb = int(hdir[10:-2])  # slice out of hugepages-NNNkB
  89                print('{:<4} {:<5} {:<6} {}'.format(node, pages,
  90                                                    fmt_memsize(kb),
  91                                                    fmt_memsize(pages * kb)))
  92
  93
  94def show_non_numa_pages():
  95    '''Show huge page reservations on non Numa system'''
  96    print('Pages Size Total')
  97    path = '/sys/kernel/mm/hugepages'
  98    for hdir in os.listdir(path):
  99        pages = get_hugepages(path + '/' + hdir)
 100        if pages > 0:
 101            kb = int(hdir[10:-2])
 102            print('{:<5} {:<6} {}'.format(pages, fmt_memsize(kb),
 103                                          fmt_memsize(pages * kb)))
 104
 105
 106def show_pages():
 107    '''Show existing huge page settings'''
 108    if is_numa():
 109        show_numa_pages()
 110    else:
 111        show_non_numa_pages()
 112
 113
 114def clear_pages():
 115    '''Clear all existing huge page mappings'''
 116    if is_numa():
 117        dirs = glob.glob(
 118            '/sys/devices/system/node/node*/hugepages/hugepages-*')
 119    else:
 120        dirs = glob.glob('/sys/kernel/mm/hugepages/hugepages-*')
 121
 122    for path in dirs:
 123        set_hugepages(path, 0)
 124
 125
 126def default_pagesize():
 127    '''Get default huge page size from /proc/meminfo'''
 128    with open('/proc/meminfo') as meminfo:
 129        for line in meminfo:
 130            if line.startswith('Hugepagesize:'):
 131                return int(line.split()[1])
 132    return None
 133
 134
 135def set_numa_pages(pages, hugepgsz, node=None):
 136    '''Set huge page reservation on Numa system'''
 137    if node:
 138        nodes = ['/sys/devices/system/node/node{}/hugepages'.format(node)]
 139    else:
 140        nodes = glob.glob('/sys/devices/system/node/node*/hugepages')
 141
 142    for node_path in nodes:
 143        huge_path = '{}/hugepages-{}kB'.format(node_path, hugepgsz)
 144        set_hugepages(huge_path, pages)
 145
 146
 147def set_non_numa_pages(pages, hugepgsz):
 148    '''Set huge page reservation on non Numa system'''
 149    path = '/sys/kernel/mm/hugepages/hugepages-{}kB'.format(hugepgsz)
 150    set_hugepages(path, pages)
 151
 152
 153def reserve_pages(pages, hugepgsz, node=None):
 154    '''Set the number of huge pages to be reserved'''
 155    if node or is_numa():
 156        set_numa_pages(pages, hugepgsz, node=node)
 157    else:
 158        set_non_numa_pages(pages, hugepgsz)
 159
 160
 161def get_mountpoints():
 162    '''Get list of where hugepage filesystem is mounted'''
 163    mounted = []
 164    with open('/proc/mounts') as mounts:
 165        for line in mounts:
 166            fields = line.split()
 167            if fields[2] != 'hugetlbfs':
 168                continue
 169            mounted.append(fields[1])
 170    return mounted
 171
 172
 173def mount_huge(pagesize, mountpoint, user, group):
 174    '''Mount the huge TLB file system'''
 175    if mountpoint in get_mountpoints():
 176        print(mountpoint, "already mounted")
 177        return
 178    cmd = "mount -t hugetlbfs"
 179    if pagesize:
 180        cmd += ' -o pagesize={}'.format(pagesize * 1024)
 181    if user:
 182        cmd += ' -o uid=' + user
 183    if group:
 184        cmd += ' -o gid=' + group
 185    cmd += ' nodev ' + mountpoint
 186    os.system(cmd)
 187
 188
 189def umount_huge(mountpoint):
 190    '''Unmount the huge TLB file system (if mounted)'''
 191    if mountpoint in get_mountpoints():
 192        os.system("umount " + mountpoint)
 193
 194
 195def show_mount():
 196    '''Show where huge page filesystem is mounted'''
 197    mounted = get_mountpoints()
 198    if mounted:
 199        print("Hugepages mounted on", *mounted)
 200    else:
 201        print("Hugepages not mounted")
 202
 203
 204def main():
 205    '''Process the command line arguments and setup huge pages'''
 206    parser = argparse.ArgumentParser(
 207        formatter_class=argparse.RawDescriptionHelpFormatter,
 208        description="Setup huge pages",
 209        epilog="""
 210Examples:
 211
 212To display current huge page settings:
 213    %(prog)s -s
 214
 215To a complete setup of with 2 Gigabyte of 1G huge pages:
 216    %(prog)s -p 1G --setup 2G
 217""")
 218    parser.add_argument(
 219        '--show',
 220        '-s',
 221        action='store_true',
 222        help="print the current huge page configuration")
 223    parser.add_argument(
 224        '--clear', '-c', action='store_true', help="clear existing huge pages")
 225    parser.add_argument(
 226        '--mount',
 227        '-m',
 228        action='store_true',
 229        help='mount the huge page filesystem')
 230    parser.add_argument(
 231        '--unmount',
 232        '-u',
 233        action='store_true',
 234        help='unmount the system huge page directory')
 235    parser.add_argument(
 236        '--directory',
 237        '-d',
 238        metavar='DIR',
 239        default=HUGE_MOUNT,
 240        help='mount point')
 241    parser.add_argument(
 242        '--user',
 243        '-U',
 244        metavar='UID',
 245        help='set the mounted directory owner user')
 246    parser.add_argument(
 247        '--group',
 248        '-G',
 249        metavar='GID',
 250        help='set the mounted directory owner group')
 251    parser.add_argument(
 252        '--node', '-n', help='select numa node to reserve pages on')
 253    parser.add_argument(
 254        '--pagesize',
 255        '-p',
 256        metavar='SIZE',
 257        help='choose huge page size to use')
 258    parser.add_argument(
 259        '--reserve',
 260        '-r',
 261        metavar='SIZE',
 262        help='reserve huge pages. Size is in bytes with K, M, or G suffix')
 263    parser.add_argument(
 264        '--setup',
 265        metavar='SIZE',
 266        help='setup huge pages by doing clear, unmount, reserve and mount')
 267    args = parser.parse_args()
 268
 269    if args.setup:
 270        args.clear = True
 271        args.unmount = True
 272        args.reserve = args.setup
 273        args.mount = True
 274
 275    if args.pagesize:
 276        pagesize_kb = get_memsize(args.pagesize)
 277    else:
 278        pagesize_kb = default_pagesize()
 279    if not pagesize_kb:
 280        sys.exit("Invalid page size: {}kB".format(pagesize_kb))
 281
 282    if args.clear:
 283        clear_pages()
 284    if args.unmount:
 285        umount_huge(args.directory)
 286
 287    if args.reserve:
 288        reserve_kb = get_memsize(args.reserve)
 289        if reserve_kb % pagesize_kb != 0:
 290            sys.exit(
 291                'Huge reservation {}kB is not a multiple of page size {}kB'.
 292                format(reserve_kb, pagesize_kb))
 293        reserve_pages(
 294            int(reserve_kb / pagesize_kb), pagesize_kb, node=args.node)
 295    if args.mount:
 296        mount_huge(pagesize_kb, args.directory, args.user, args.group)
 297    if args.show:
 298        show_pages()
 299        print()
 300        show_mount()
 301
 302
 303if __name__ == "__main__":
 304    main()
 305