1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 5# 6 7""" 8Move config options from headers to defconfig files. 9 10See doc/develop/moveconfig.rst for documentation. 11""" 12 13import asteval 14import collections 15import copy 16import difflib 17import filecmp 18import fnmatch 19import glob 20import multiprocessing 21import optparse 22import os 23import queue 24import re 25import shutil 26import subprocess 27import sys 28import tempfile 29import threading 30import time 31 32from buildman import bsettings 33from buildman import kconfiglib 34from buildman import toolchain 35 36SHOW_GNU_MAKE = 'scripts/show-gnu-make' 37SLEEP_TIME=0.03 38 39STATE_IDLE = 0 40STATE_DEFCONFIG = 1 41STATE_AUTOCONF = 2 42STATE_SAVEDEFCONFIG = 3 43 44ACTION_MOVE = 0 45ACTION_NO_ENTRY = 1 46ACTION_NO_ENTRY_WARN = 2 47ACTION_NO_CHANGE = 3 48 49COLOR_BLACK = '0;30' 50COLOR_RED = '0;31' 51COLOR_GREEN = '0;32' 52COLOR_BROWN = '0;33' 53COLOR_BLUE = '0;34' 54COLOR_PURPLE = '0;35' 55COLOR_CYAN = '0;36' 56COLOR_LIGHT_GRAY = '0;37' 57COLOR_DARK_GRAY = '1;30' 58COLOR_LIGHT_RED = '1;31' 59COLOR_LIGHT_GREEN = '1;32' 60COLOR_YELLOW = '1;33' 61COLOR_LIGHT_BLUE = '1;34' 62COLOR_LIGHT_PURPLE = '1;35' 63COLOR_LIGHT_CYAN = '1;36' 64COLOR_WHITE = '1;37' 65 66AUTO_CONF_PATH = 'include/config/auto.conf' 67CONFIG_DATABASE = 'moveconfig.db' 68 69CONFIG_LEN = len('CONFIG_') 70 71SIZES = { 72 "SZ_1": 0x00000001, "SZ_2": 0x00000002, 73 "SZ_4": 0x00000004, "SZ_8": 0x00000008, 74 "SZ_16": 0x00000010, "SZ_32": 0x00000020, 75 "SZ_64": 0x00000040, "SZ_128": 0x00000080, 76 "SZ_256": 0x00000100, "SZ_512": 0x00000200, 77 "SZ_1K": 0x00000400, "SZ_2K": 0x00000800, 78 "SZ_4K": 0x00001000, "SZ_8K": 0x00002000, 79 "SZ_16K": 0x00004000, "SZ_32K": 0x00008000, 80 "SZ_64K": 0x00010000, "SZ_128K": 0x00020000, 81 "SZ_256K": 0x00040000, "SZ_512K": 0x00080000, 82 "SZ_1M": 0x00100000, "SZ_2M": 0x00200000, 83 "SZ_4M": 0x00400000, "SZ_8M": 0x00800000, 84 "SZ_16M": 0x01000000, "SZ_32M": 0x02000000, 85 "SZ_64M": 0x04000000, "SZ_128M": 0x08000000, 86 "SZ_256M": 0x10000000, "SZ_512M": 0x20000000, 87 "SZ_1G": 0x40000000, "SZ_2G": 0x80000000, 88 "SZ_4G": 0x100000000 89} 90 91### helper functions ### 92def get_devnull(): 93 """Get the file object of '/dev/null' device.""" 94 try: 95 devnull = subprocess.DEVNULL # py3k 96 except AttributeError: 97 devnull = open(os.devnull, 'wb') 98 return devnull 99 100def check_top_directory(): 101 """Exit if we are not at the top of source directory.""" 102 for f in ('README', 'Licenses'): 103 if not os.path.exists(f): 104 sys.exit('Please run at the top of source directory.') 105 106def check_clean_directory(): 107 """Exit if the source tree is not clean.""" 108 for f in ('.config', 'include/config'): 109 if os.path.exists(f): 110 sys.exit("source tree is not clean, please run 'make mrproper'") 111 112def get_make_cmd(): 113 """Get the command name of GNU Make. 114 115 U-Boot needs GNU Make for building, but the command name is not 116 necessarily "make". (for example, "gmake" on FreeBSD). 117 Returns the most appropriate command name on your system. 118 """ 119 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 120 ret = process.communicate() 121 if process.returncode: 122 sys.exit('GNU Make not found') 123 return ret[0].rstrip() 124 125def get_matched_defconfig(line): 126 """Get the defconfig files that match a pattern 127 128 Args: 129 line: Path or filename to match, e.g. 'configs/snow_defconfig' or 130 'k2*_defconfig'. If no directory is provided, 'configs/' is 131 prepended 132 133 Returns: 134 a list of matching defconfig files 135 """ 136 dirname = os.path.dirname(line) 137 if dirname: 138 pattern = line 139 else: 140 pattern = os.path.join('configs', line) 141 return glob.glob(pattern) + glob.glob(pattern + '_defconfig') 142 143def get_matched_defconfigs(defconfigs_file): 144 """Get all the defconfig files that match the patterns in a file. 145 146 Args: 147 defconfigs_file: File containing a list of defconfigs to process, or 148 '-' to read the list from stdin 149 150 Returns: 151 A list of paths to defconfig files, with no duplicates 152 """ 153 defconfigs = [] 154 if defconfigs_file == '-': 155 fd = sys.stdin 156 defconfigs_file = 'stdin' 157 else: 158 fd = open(defconfigs_file) 159 for i, line in enumerate(fd): 160 line = line.strip() 161 if not line: 162 continue # skip blank lines silently 163 if ' ' in line: 164 line = line.split(' ')[0] # handle 'git log' input 165 matched = get_matched_defconfig(line) 166 if not matched: 167 print("warning: %s:%d: no defconfig matched '%s'" % \ 168 (defconfigs_file, i + 1, line), file=sys.stderr) 169 170 defconfigs += matched 171 172 # use set() to drop multiple matching 173 return [ defconfig[len('configs') + 1:] for defconfig in set(defconfigs) ] 174 175def get_all_defconfigs(): 176 """Get all the defconfig files under the configs/ directory.""" 177 defconfigs = [] 178 for (dirpath, dirnames, filenames) in os.walk('configs'): 179 dirpath = dirpath[len('configs') + 1:] 180 for filename in fnmatch.filter(filenames, '*_defconfig'): 181 defconfigs.append(os.path.join(dirpath, filename)) 182 183 return defconfigs 184 185def color_text(color_enabled, color, string): 186 """Return colored string.""" 187 if color_enabled: 188 # LF should not be surrounded by the escape sequence. 189 # Otherwise, additional whitespace or line-feed might be printed. 190 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 191 for s in string.split('\n') ]) 192 else: 193 return string 194 195def show_diff(a, b, file_path, color_enabled): 196 """Show unidified diff. 197 198 Arguments: 199 a: A list of lines (before) 200 b: A list of lines (after) 201 file_path: Path to the file 202 color_enabled: Display the diff in color 203 """ 204 205 diff = difflib.unified_diff(a, b, 206 fromfile=os.path.join('a', file_path), 207 tofile=os.path.join('b', file_path)) 208 209 for line in diff: 210 if line[0] == '-' and line[1] != '-': 211 print(color_text(color_enabled, COLOR_RED, line), end=' ') 212 elif line[0] == '+' and line[1] != '+': 213 print(color_text(color_enabled, COLOR_GREEN, line), end=' ') 214 else: 215 print(line, end=' ') 216 217def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre, 218 extend_post): 219 """Extend matched lines if desired patterns are found before/after already 220 matched lines. 221 222 Arguments: 223 lines: A list of lines handled. 224 matched: A list of line numbers that have been already matched. 225 (will be updated by this function) 226 pre_patterns: A list of regular expression that should be matched as 227 preamble. 228 post_patterns: A list of regular expression that should be matched as 229 postamble. 230 extend_pre: Add the line number of matched preamble to the matched list. 231 extend_post: Add the line number of matched postamble to the matched list. 232 """ 233 extended_matched = [] 234 235 j = matched[0] 236 237 for i in matched: 238 if i == 0 or i < j: 239 continue 240 j = i 241 while j in matched: 242 j += 1 243 if j >= len(lines): 244 break 245 246 for p in pre_patterns: 247 if p.search(lines[i - 1]): 248 break 249 else: 250 # not matched 251 continue 252 253 for p in post_patterns: 254 if p.search(lines[j]): 255 break 256 else: 257 # not matched 258 continue 259 260 if extend_pre: 261 extended_matched.append(i - 1) 262 if extend_post: 263 extended_matched.append(j) 264 265 matched += extended_matched 266 matched.sort() 267 268def confirm(options, prompt): 269 if not options.yes: 270 while True: 271 choice = input('{} [y/n]: '.format(prompt)) 272 choice = choice.lower() 273 print(choice) 274 if choice == 'y' or choice == 'n': 275 break 276 277 if choice == 'n': 278 return False 279 280 return True 281 282def cleanup_empty_blocks(header_path, options): 283 """Clean up empty conditional blocks 284 285 Arguments: 286 header_path: path to the cleaned file. 287 options: option flags. 288 """ 289 pattern = re.compile(r'^\s*#\s*if.*$\n^\s*#\s*endif.*$\n*', flags=re.M) 290 with open(header_path) as f: 291 try: 292 data = f.read() 293 except UnicodeDecodeError as e: 294 print("Failed on file %s': %s" % (header_path, e)) 295 return 296 297 new_data = pattern.sub('\n', data) 298 299 show_diff(data.splitlines(True), new_data.splitlines(True), header_path, 300 options.color) 301 302 if options.dry_run: 303 return 304 305 with open(header_path, 'w') as f: 306 f.write(new_data) 307 308def cleanup_one_header(header_path, patterns, options): 309 """Clean regex-matched lines away from a file. 310 311 Arguments: 312 header_path: path to the cleaned file. 313 patterns: list of regex patterns. Any lines matching to these 314 patterns are deleted. 315 options: option flags. 316 """ 317 with open(header_path) as f: 318 try: 319 lines = f.readlines() 320 except UnicodeDecodeError as e: 321 print("Failed on file %s': %s" % (header_path, e)) 322 return 323 324 matched = [] 325 for i, line in enumerate(lines): 326 if i - 1 in matched and lines[i - 1][-2:] == '\\\n': 327 matched.append(i) 328 continue 329 for pattern in patterns: 330 if pattern.search(line): 331 matched.append(i) 332 break 333 334 if not matched: 335 return 336 337 # remove empty #ifdef ... #endif, successive blank lines 338 pattern_if = re.compile(r'#\s*if(def|ndef)?\W') # #if, #ifdef, #ifndef 339 pattern_elif = re.compile(r'#\s*el(if|se)\W') # #elif, #else 340 pattern_endif = re.compile(r'#\s*endif\W') # #endif 341 pattern_blank = re.compile(r'^\s*$') # empty line 342 343 while True: 344 old_matched = copy.copy(matched) 345 extend_matched_lines(lines, matched, [pattern_if], 346 [pattern_endif], True, True) 347 extend_matched_lines(lines, matched, [pattern_elif], 348 [pattern_elif, pattern_endif], True, False) 349 extend_matched_lines(lines, matched, [pattern_if, pattern_elif], 350 [pattern_blank], False, True) 351 extend_matched_lines(lines, matched, [pattern_blank], 352 [pattern_elif, pattern_endif], True, False) 353 extend_matched_lines(lines, matched, [pattern_blank], 354 [pattern_blank], True, False) 355 if matched == old_matched: 356 break 357 358 tolines = copy.copy(lines) 359 360 for i in reversed(matched): 361 tolines.pop(i) 362 363 show_diff(lines, tolines, header_path, options.color) 364 365 if options.dry_run: 366 return 367 368 with open(header_path, 'w') as f: 369 for line in tolines: 370 f.write(line) 371 372def cleanup_headers(configs, options): 373 """Delete config defines from board headers. 374 375 Arguments: 376 configs: A list of CONFIGs to remove. 377 options: option flags. 378 """ 379 if not confirm(options, 'Clean up headers?'): 380 return 381 382 patterns = [] 383 for config in configs: 384 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 385 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 386 387 for dir in 'include', 'arch', 'board': 388 for (dirpath, dirnames, filenames) in os.walk(dir): 389 if dirpath == os.path.join('include', 'generated'): 390 continue 391 for filename in filenames: 392 if not filename.endswith(('~', '.dts', '.dtsi', '.bin', 393 '.elf','.aml','.dat')): 394 header_path = os.path.join(dirpath, filename) 395 # This file contains UTF-16 data and no CONFIG symbols 396 if header_path == 'include/video_font_data.h': 397 continue 398 cleanup_one_header(header_path, patterns, options) 399 cleanup_empty_blocks(header_path, options) 400 401def cleanup_one_extra_option(defconfig_path, configs, options): 402 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file. 403 404 Arguments: 405 defconfig_path: path to the cleaned defconfig file. 406 configs: A list of CONFIGs to remove. 407 options: option flags. 408 """ 409 410 start = 'CONFIG_SYS_EXTRA_OPTIONS="' 411 end = '"\n' 412 413 with open(defconfig_path) as f: 414 lines = f.readlines() 415 416 for i, line in enumerate(lines): 417 if line.startswith(start) and line.endswith(end): 418 break 419 else: 420 # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig 421 return 422 423 old_tokens = line[len(start):-len(end)].split(',') 424 new_tokens = [] 425 426 for token in old_tokens: 427 pos = token.find('=') 428 if not (token[:pos] if pos >= 0 else token) in configs: 429 new_tokens.append(token) 430 431 if new_tokens == old_tokens: 432 return 433 434 tolines = copy.copy(lines) 435 436 if new_tokens: 437 tolines[i] = start + ','.join(new_tokens) + end 438 else: 439 tolines.pop(i) 440 441 show_diff(lines, tolines, defconfig_path, options.color) 442 443 if options.dry_run: 444 return 445 446 with open(defconfig_path, 'w') as f: 447 for line in tolines: 448 f.write(line) 449 450def cleanup_extra_options(configs, options): 451 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files. 452 453 Arguments: 454 configs: A list of CONFIGs to remove. 455 options: option flags. 456 """ 457 if not confirm(options, 'Clean up CONFIG_SYS_EXTRA_OPTIONS?'): 458 return 459 460 configs = [ config[len('CONFIG_'):] for config in configs ] 461 462 defconfigs = get_all_defconfigs() 463 464 for defconfig in defconfigs: 465 cleanup_one_extra_option(os.path.join('configs', defconfig), configs, 466 options) 467 468def cleanup_whitelist(configs, options): 469 """Delete config whitelist entries 470 471 Arguments: 472 configs: A list of CONFIGs to remove. 473 options: option flags. 474 """ 475 if not confirm(options, 'Clean up whitelist entries?'): 476 return 477 478 with open(os.path.join('scripts', 'config_whitelist.txt')) as f: 479 lines = f.readlines() 480 481 lines = [x for x in lines if x.strip() not in configs] 482 483 with open(os.path.join('scripts', 'config_whitelist.txt'), 'w') as f: 484 f.write(''.join(lines)) 485 486def find_matching(patterns, line): 487 for pat in patterns: 488 if pat.search(line): 489 return True 490 return False 491 492def cleanup_readme(configs, options): 493 """Delete config description in README 494 495 Arguments: 496 configs: A list of CONFIGs to remove. 497 options: option flags. 498 """ 499 if not confirm(options, 'Clean up README?'): 500 return 501 502 patterns = [] 503 for config in configs: 504 patterns.append(re.compile(r'^\s+%s' % config)) 505 506 with open('README') as f: 507 lines = f.readlines() 508 509 found = False 510 newlines = [] 511 for line in lines: 512 if not found: 513 found = find_matching(patterns, line) 514 if found: 515 continue 516 517 if found and re.search(r'^\s+CONFIG', line): 518 found = False 519 520 if not found: 521 newlines.append(line) 522 523 with open('README', 'w') as f: 524 f.write(''.join(newlines)) 525 526def try_expand(line): 527 """If value looks like an expression, try expanding it 528 Otherwise just return the existing value 529 """ 530 if line.find('=') == -1: 531 return line 532 533 try: 534 aeval = asteval.Interpreter( usersyms=SIZES, minimal=True ) 535 cfg, val = re.split("=", line) 536 val= val.strip('\"') 537 if re.search("[*+-/]|<<|SZ_+|\(([^\)]+)\)", val): 538 newval = hex(aeval(val)) 539 print("\tExpanded expression %s to %s" % (val, newval)) 540 return cfg+'='+newval 541 except: 542 print("\tFailed to expand expression in %s" % line) 543 544 return line 545 546 547### classes ### 548class Progress: 549 550 """Progress Indicator""" 551 552 def __init__(self, total): 553 """Create a new progress indicator. 554 555 Arguments: 556 total: A number of defconfig files to process. 557 """ 558 self.current = 0 559 self.total = total 560 561 def inc(self): 562 """Increment the number of processed defconfig files.""" 563 564 self.current += 1 565 566 def show(self): 567 """Display the progress.""" 568 print(' %d defconfigs out of %d\r' % (self.current, self.total), end=' ') 569 sys.stdout.flush() 570 571 572class KconfigScanner: 573 """Kconfig scanner.""" 574 575 def __init__(self): 576 """Scan all the Kconfig files and create a Config object.""" 577 # Define environment variables referenced from Kconfig 578 os.environ['srctree'] = os.getcwd() 579 os.environ['UBOOTVERSION'] = 'dummy' 580 os.environ['KCONFIG_OBJDIR'] = '' 581 self.conf = kconfiglib.Kconfig() 582 583 584class KconfigParser: 585 586 """A parser of .config and include/autoconf.mk.""" 587 588 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 589 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 590 591 def __init__(self, configs, options, build_dir): 592 """Create a new parser. 593 594 Arguments: 595 configs: A list of CONFIGs to move. 596 options: option flags. 597 build_dir: Build directory. 598 """ 599 self.configs = configs 600 self.options = options 601 self.dotconfig = os.path.join(build_dir, '.config') 602 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 603 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', 604 'autoconf.mk') 605 self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH) 606 self.defconfig = os.path.join(build_dir, 'defconfig') 607 608 def get_arch(self): 609 """Parse .config file and return the architecture. 610 611 Returns: 612 Architecture name (e.g. 'arm'). 613 """ 614 arch = '' 615 cpu = '' 616 for line in open(self.dotconfig): 617 m = self.re_arch.match(line) 618 if m: 619 arch = m.group(1) 620 continue 621 m = self.re_cpu.match(line) 622 if m: 623 cpu = m.group(1) 624 625 if not arch: 626 return None 627 628 # fix-up for aarch64 629 if arch == 'arm' and cpu == 'armv8': 630 arch = 'aarch64' 631 632 return arch 633 634 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 635 """Parse .config, defconfig, include/autoconf.mk for one config. 636 637 This function looks for the config options in the lines from 638 defconfig, .config, and include/autoconf.mk in order to decide 639 which action should be taken for this defconfig. 640 641 Arguments: 642 config: CONFIG name to parse. 643 dotconfig_lines: lines from the .config file. 644 autoconf_lines: lines from the include/autoconf.mk file. 645 646 Returns: 647 A tupple of the action for this defconfig and the line 648 matched for the config. 649 """ 650 not_set = '# %s is not set' % config 651 652 for line in autoconf_lines: 653 line = line.rstrip() 654 if line.startswith(config + '='): 655 new_val = line 656 break 657 else: 658 new_val = not_set 659 660 new_val = try_expand(new_val) 661 662 for line in dotconfig_lines: 663 line = line.rstrip() 664 if line.startswith(config + '=') or line == not_set: 665 old_val = line 666 break 667 else: 668 if new_val == not_set: 669 return (ACTION_NO_ENTRY, config) 670 else: 671 return (ACTION_NO_ENTRY_WARN, config) 672 673 # If this CONFIG is neither bool nor trisate 674 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 675 # tools/scripts/define2mk.sed changes '1' to 'y'. 676 # This is a problem if the CONFIG is int type. 677 # Check the type in Kconfig and handle it correctly. 678 if new_val[-2:] == '=y': 679 new_val = new_val[:-1] + '1' 680 681 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE, 682 new_val) 683 684 def update_dotconfig(self): 685 """Parse files for the config options and update the .config. 686 687 This function parses the generated .config and include/autoconf.mk 688 searching the target options. 689 Move the config option(s) to the .config as needed. 690 691 Arguments: 692 defconfig: defconfig name. 693 694 Returns: 695 Return a tuple of (updated flag, log string). 696 The "updated flag" is True if the .config was updated, False 697 otherwise. The "log string" shows what happend to the .config. 698 """ 699 700 results = [] 701 updated = False 702 suspicious = False 703 rm_files = [self.config_autoconf, self.autoconf] 704 705 if self.options.spl: 706 if os.path.exists(self.spl_autoconf): 707 autoconf_path = self.spl_autoconf 708 rm_files.append(self.spl_autoconf) 709 else: 710 for f in rm_files: 711 os.remove(f) 712 return (updated, suspicious, 713 color_text(self.options.color, COLOR_BROWN, 714 "SPL is not enabled. Skipped.") + '\n') 715 else: 716 autoconf_path = self.autoconf 717 718 with open(self.dotconfig) as f: 719 dotconfig_lines = f.readlines() 720 721 with open(autoconf_path) as f: 722 autoconf_lines = f.readlines() 723 724 for config in self.configs: 725 result = self.parse_one_config(config, dotconfig_lines, 726 autoconf_lines) 727 results.append(result) 728 729 log = '' 730 731 for (action, value) in results: 732 if action == ACTION_MOVE: 733 actlog = "Move '%s'" % value 734 log_color = COLOR_LIGHT_GREEN 735 elif action == ACTION_NO_ENTRY: 736 actlog = "%s is not defined in Kconfig. Do nothing." % value 737 log_color = COLOR_LIGHT_BLUE 738 elif action == ACTION_NO_ENTRY_WARN: 739 actlog = "%s is not defined in Kconfig (suspicious). Do nothing." % value 740 log_color = COLOR_YELLOW 741 suspicious = True 742 elif action == ACTION_NO_CHANGE: 743 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 744 % value 745 log_color = COLOR_LIGHT_PURPLE 746 elif action == ACTION_SPL_NOT_EXIST: 747 actlog = "SPL is not enabled for this defconfig. Skip." 748 log_color = COLOR_PURPLE 749 else: 750 sys.exit("Internal Error. This should not happen.") 751 752 log += color_text(self.options.color, log_color, actlog) + '\n' 753 754 with open(self.dotconfig, 'a') as f: 755 for (action, value) in results: 756 if action == ACTION_MOVE: 757 f.write(value + '\n') 758 updated = True 759 760 self.results = results 761 for f in rm_files: 762 os.remove(f) 763 764 return (updated, suspicious, log) 765 766 def check_defconfig(self): 767 """Check the defconfig after savedefconfig 768 769 Returns: 770 Return additional log if moved CONFIGs were removed again by 771 'make savedefconfig'. 772 """ 773 774 log = '' 775 776 with open(self.defconfig) as f: 777 defconfig_lines = f.readlines() 778 779 for (action, value) in self.results: 780 if action != ACTION_MOVE: 781 continue 782 if not value + '\n' in defconfig_lines: 783 log += color_text(self.options.color, COLOR_YELLOW, 784 "'%s' was removed by savedefconfig.\n" % 785 value) 786 787 return log 788 789 790class DatabaseThread(threading.Thread): 791 """This thread processes results from Slot threads. 792 793 It collects the data in the master config directary. There is only one 794 result thread, and this helps to serialise the build output. 795 """ 796 def __init__(self, config_db, db_queue): 797 """Set up a new result thread 798 799 Args: 800 builder: Builder which will be sent each result 801 """ 802 threading.Thread.__init__(self) 803 self.config_db = config_db 804 self.db_queue= db_queue 805 806 def run(self): 807 """Called to start up the result thread. 808 809 We collect the next result job and pass it on to the build. 810 """ 811 while True: 812 defconfig, configs = self.db_queue.get() 813 self.config_db[defconfig] = configs 814 self.db_queue.task_done() 815 816 817class Slot: 818 819 """A slot to store a subprocess. 820 821 Each instance of this class handles one subprocess. 822 This class is useful to control multiple threads 823 for faster processing. 824 """ 825 826 def __init__(self, toolchains, configs, options, progress, devnull, 827 make_cmd, reference_src_dir, db_queue): 828 """Create a new process slot. 829 830 Arguments: 831 toolchains: Toolchains object containing toolchains. 832 configs: A list of CONFIGs to move. 833 options: option flags. 834 progress: A progress indicator. 835 devnull: A file object of '/dev/null'. 836 make_cmd: command name of GNU Make. 837 reference_src_dir: Determine the true starting config state from this 838 source tree. 839 db_queue: output queue to write config info for the database 840 """ 841 self.toolchains = toolchains 842 self.options = options 843 self.progress = progress 844 self.build_dir = tempfile.mkdtemp() 845 self.devnull = devnull 846 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 847 self.reference_src_dir = reference_src_dir 848 self.db_queue = db_queue 849 self.parser = KconfigParser(configs, options, self.build_dir) 850 self.state = STATE_IDLE 851 self.failed_boards = set() 852 self.suspicious_boards = set() 853 854 def __del__(self): 855 """Delete the working directory 856 857 This function makes sure the temporary directory is cleaned away 858 even if Python suddenly dies due to error. It should be done in here 859 because it is guaranteed the destructor is always invoked when the 860 instance of the class gets unreferenced. 861 862 If the subprocess is still running, wait until it finishes. 863 """ 864 if self.state != STATE_IDLE: 865 while self.ps.poll() == None: 866 pass 867 shutil.rmtree(self.build_dir) 868 869 def add(self, defconfig): 870 """Assign a new subprocess for defconfig and add it to the slot. 871 872 If the slot is vacant, create a new subprocess for processing the 873 given defconfig and add it to the slot. Just returns False if 874 the slot is occupied (i.e. the current subprocess is still running). 875 876 Arguments: 877 defconfig: defconfig name. 878 879 Returns: 880 Return True on success or False on failure 881 """ 882 if self.state != STATE_IDLE: 883 return False 884 885 self.defconfig = defconfig 886 self.log = '' 887 self.current_src_dir = self.reference_src_dir 888 self.do_defconfig() 889 return True 890 891 def poll(self): 892 """Check the status of the subprocess and handle it as needed. 893 894 Returns True if the slot is vacant (i.e. in idle state). 895 If the configuration is successfully finished, assign a new 896 subprocess to build include/autoconf.mk. 897 If include/autoconf.mk is generated, invoke the parser to 898 parse the .config and the include/autoconf.mk, moving 899 config options to the .config as needed. 900 If the .config was updated, run "make savedefconfig" to sync 901 it, update the original defconfig, and then set the slot back 902 to the idle state. 903 904 Returns: 905 Return True if the subprocess is terminated, False otherwise 906 """ 907 if self.state == STATE_IDLE: 908 return True 909 910 if self.ps.poll() == None: 911 return False 912 913 if self.ps.poll() != 0: 914 self.handle_error() 915 elif self.state == STATE_DEFCONFIG: 916 if self.reference_src_dir and not self.current_src_dir: 917 self.do_savedefconfig() 918 else: 919 self.do_autoconf() 920 elif self.state == STATE_AUTOCONF: 921 if self.current_src_dir: 922 self.current_src_dir = None 923 self.do_defconfig() 924 elif self.options.build_db: 925 self.do_build_db() 926 else: 927 self.do_savedefconfig() 928 elif self.state == STATE_SAVEDEFCONFIG: 929 self.update_defconfig() 930 else: 931 sys.exit("Internal Error. This should not happen.") 932 933 return True if self.state == STATE_IDLE else False 934 935 def handle_error(self): 936 """Handle error cases.""" 937 938 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 939 "Failed to process.\n") 940 if self.options.verbose: 941 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 942 self.ps.stderr.read().decode()) 943 self.finish(False) 944 945 def do_defconfig(self): 946 """Run 'make <board>_defconfig' to create the .config file.""" 947 948 cmd = list(self.make_cmd) 949 cmd.append(self.defconfig) 950 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 951 stderr=subprocess.PIPE, 952 cwd=self.current_src_dir) 953 self.state = STATE_DEFCONFIG 954 955 def do_autoconf(self): 956 """Run 'make AUTO_CONF_PATH'.""" 957 958 arch = self.parser.get_arch() 959 try: 960 toolchain = self.toolchains.Select(arch) 961 except ValueError: 962 self.log += color_text(self.options.color, COLOR_YELLOW, 963 "Tool chain for '%s' is missing. Do nothing.\n" % arch) 964 self.finish(False) 965 return 966 env = toolchain.MakeEnvironment(False) 967 968 cmd = list(self.make_cmd) 969 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 970 cmd.append(AUTO_CONF_PATH) 971 self.ps = subprocess.Popen(cmd, stdout=self.devnull, env=env, 972 stderr=subprocess.PIPE, 973 cwd=self.current_src_dir) 974 self.state = STATE_AUTOCONF 975 976 def do_build_db(self): 977 """Add the board to the database""" 978 configs = {} 979 with open(os.path.join(self.build_dir, AUTO_CONF_PATH)) as fd: 980 for line in fd.readlines(): 981 if line.startswith('CONFIG'): 982 config, value = line.split('=', 1) 983 configs[config] = value.rstrip() 984 self.db_queue.put([self.defconfig, configs]) 985 self.finish(True) 986 987 def do_savedefconfig(self): 988 """Update the .config and run 'make savedefconfig'.""" 989 990 (updated, suspicious, log) = self.parser.update_dotconfig() 991 if suspicious: 992 self.suspicious_boards.add(self.defconfig) 993 self.log += log 994 995 if not self.options.force_sync and not updated: 996 self.finish(True) 997 return 998 if updated: 999 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 1000 "Syncing by savedefconfig...\n")
1001 else: 1002 self.log += "Syncing by savedefconfig (forced by option)...\n" 1003 1004 cmd = list(self.make_cmd) 1005 cmd.append('savedefconfig') 1006 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1007 stderr=subprocess.PIPE) 1008 self.state = STATE_SAVEDEFCONFIG 1009 1010 def update_defconfig(self): 1011 """Update the input defconfig and go back to the idle state.""" 1012 1013 log = self.parser.check_defconfig() 1014 if log: 1015 self.suspicious_boards.add(self.defconfig) 1016 self.log += log 1017 orig_defconfig = os.path.join('configs', self.defconfig) 1018 new_defconfig = os.path.join(self.build_dir, 'defconfig') 1019 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 1020 1021 if updated: 1022 self.log += color_text(self.options.color, COLOR_LIGHT_BLUE, 1023 "defconfig was updated.\n") 1024 1025 if not self.options.dry_run and updated: 1026 shutil.move(new_defconfig, orig_defconfig) 1027 self.finish(True) 1028 1029 def finish(self, success): 1030 """Display log along with progress and go to the idle state. 1031 1032 Arguments: 1033 success: Should be True when the defconfig was processed 1034 successfully, or False when it fails. 1035 """ 1036 # output at least 30 characters to hide the "* defconfigs out of *". 1037 log = self.defconfig.ljust(30) + '\n' 1038 1039 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 1040 # Some threads are running in parallel. 1041 # Print log atomically to not mix up logs from different threads. 1042 print(log, file=(sys.stdout if success else sys.stderr)) 1043 1044 if not success: 1045 if self.options.exit_on_error: 1046 sys.exit("Exit on error.") 1047 # If --exit-on-error flag is not set, skip this board and continue. 1048 # Record the failed board. 1049 self.failed_boards.add(self.defconfig) 1050 1051 self.progress.inc() 1052 self.progress.show() 1053 self.state = STATE_IDLE 1054 1055 def get_failed_boards(self): 1056 """Returns a set of failed boards (defconfigs) in this slot. 1057 """ 1058 return self.failed_boards 1059 1060 def get_suspicious_boards(self): 1061 """Returns a set of boards (defconfigs) with possible misconversion. 1062 """ 1063 return self.suspicious_boards - self.failed_boards 1064 1065class Slots: 1066 1067 """Controller of the array of subprocess slots.""" 1068 1069 def __init__(self, toolchains, configs, options, progress, 1070 reference_src_dir, db_queue): 1071 """Create a new slots controller. 1072 1073 Arguments: 1074 toolchains: Toolchains object containing toolchains. 1075 configs: A list of CONFIGs to move. 1076 options: option flags. 1077 progress: A progress indicator. 1078 reference_src_dir: Determine the true starting config state from this 1079 source tree. 1080 db_queue: output queue to write config info for the database 1081 """ 1082 self.options = options 1083 self.slots = [] 1084 devnull = get_devnull() 1085 make_cmd = get_make_cmd() 1086 for i in range(options.jobs): 1087 self.slots.append(Slot(toolchains, configs, options, progress, 1088 devnull, make_cmd, reference_src_dir, 1089 db_queue)) 1090 1091 def add(self, defconfig): 1092 """Add a new subprocess if a vacant slot is found. 1093 1094 Arguments: 1095 defconfig: defconfig name to be put into. 1096 1097 Returns: 1098 Return True on success or False on failure 1099 """ 1100 for slot in self.slots: 1101 if slot.add(defconfig): 1102 return True 1103 return False 1104 1105 def available(self): 1106 """Check if there is a vacant slot. 1107 1108 Returns: 1109 Return True if at lease one vacant slot is found, False otherwise. 1110 """ 1111 for slot in self.slots: 1112 if slot.poll(): 1113 return True 1114 return False 1115 1116 def empty(self): 1117 """Check if all slots are vacant. 1118 1119 Returns: 1120 Return True if all the slots are vacant, False otherwise. 1121 """ 1122 ret = True 1123 for slot in self.slots: 1124 if not slot.poll(): 1125 ret = False 1126 return ret 1127 1128 def show_failed_boards(self): 1129 """Display all of the failed boards (defconfigs).""" 1130 boards = set() 1131 output_file = 'moveconfig.failed' 1132 1133 for slot in self.slots: 1134 boards |= slot.get_failed_boards() 1135 1136 if boards: 1137 boards = '\n'.join(boards) + '\n' 1138 msg = "The following boards were not processed due to error:\n" 1139 msg += boards 1140 msg += "(the list has been saved in %s)\n" % output_file 1141 print(color_text(self.options.color, COLOR_LIGHT_RED, 1142 msg), file=sys.stderr) 1143 1144 with open(output_file, 'w') as f: 1145 f.write(boards) 1146 1147 def show_suspicious_boards(self): 1148 """Display all boards (defconfigs) with possible misconversion.""" 1149 boards = set() 1150 output_file = 'moveconfig.suspicious' 1151 1152 for slot in self.slots: 1153 boards |= slot.get_suspicious_boards() 1154 1155 if boards: 1156 boards = '\n'.join(boards) + '\n' 1157 msg = "The following boards might have been converted incorrectly.\n" 1158 msg += "It is highly recommended to check them manually:\n" 1159 msg += boards 1160 msg += "(the list has been saved in %s)\n" % output_file 1161 print(color_text(self.options.color, COLOR_YELLOW, 1162 msg), file=sys.stderr) 1163 1164 with open(output_file, 'w') as f: 1165 f.write(boards) 1166 1167class ReferenceSource: 1168 1169 """Reference source against which original configs should be parsed.""" 1170 1171 def __init__(self, commit): 1172 """Create a reference source directory based on a specified commit. 1173 1174 Arguments: 1175 commit: commit to git-clone 1176 """ 1177 self.src_dir = tempfile.mkdtemp() 1178 print("Cloning git repo to a separate work directory...") 1179 subprocess.check_output(['git', 'clone', os.getcwd(), '.'], 1180 cwd=self.src_dir) 1181 print("Checkout '%s' to build the original autoconf.mk." % \ 1182 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip()) 1183 subprocess.check_output(['git', 'checkout', commit], 1184 stderr=subprocess.STDOUT, cwd=self.src_dir) 1185 1186 def __del__(self): 1187 """Delete the reference source directory 1188 1189 This function makes sure the temporary directory is cleaned away 1190 even if Python suddenly dies due to error. It should be done in here 1191 because it is guaranteed the destructor is always invoked when the 1192 instance of the class gets unreferenced. 1193 """ 1194 shutil.rmtree(self.src_dir) 1195 1196 def get_dir(self): 1197 """Return the absolute path to the reference source directory.""" 1198 1199 return self.src_dir 1200 1201def move_config(toolchains, configs, options, db_queue): 1202 """Move config options to defconfig files. 1203 1204 Arguments: 1205 configs: A list of CONFIGs to move. 1206 options: option flags 1207 """ 1208 if len(configs) == 0: 1209 if options.force_sync: 1210 print('No CONFIG is specified. You are probably syncing defconfigs.', end=' ') 1211 elif options.build_db: 1212 print('Building %s database' % CONFIG_DATABASE) 1213 else: 1214 print('Neither CONFIG nor --force-sync is specified. Nothing will happen.', end=' ') 1215 else: 1216 print('Move ' + ', '.join(configs), end=' ') 1217 print('(jobs: %d)\n' % options.jobs) 1218 1219 if options.git_ref: 1220 reference_src = ReferenceSource(options.git_ref) 1221 reference_src_dir = reference_src.get_dir() 1222 else: 1223 reference_src_dir = None 1224 1225 if options.defconfigs: 1226 defconfigs = get_matched_defconfigs(options.defconfigs) 1227 else: 1228 defconfigs = get_all_defconfigs() 1229 1230 progress = Progress(len(defconfigs)) 1231 slots = Slots(toolchains, configs, options, progress, reference_src_dir, 1232 db_queue) 1233 1234 # Main loop to process defconfig files: 1235 # Add a new subprocess into a vacant slot. 1236 # Sleep if there is no available slot. 1237 for defconfig in defconfigs: 1238 while not slots.add(defconfig): 1239 while not slots.available(): 1240 # No available slot: sleep for a while 1241 time.sleep(SLEEP_TIME) 1242 1243 # wait until all the subprocesses finish 1244 while not slots.empty(): 1245 time.sleep(SLEEP_TIME) 1246 1247 print('') 1248 slots.show_failed_boards() 1249 slots.show_suspicious_boards() 1250 1251def find_kconfig_rules(kconf, config, imply_config): 1252 """Check whether a config has a 'select' or 'imply' keyword 1253 1254 Args: 1255 kconf: Kconfiglib.Kconfig object 1256 config: Name of config to check (without CONFIG_ prefix) 1257 imply_config: Implying config (without CONFIG_ prefix) which may or 1258 may not have an 'imply' for 'config') 1259 1260 Returns: 1261 Symbol object for 'config' if found, else None 1262 """ 1263 sym = kconf.syms.get(imply_config) 1264 if sym: 1265 for sel, cond in (sym.selects + sym.implies): 1266 if sel == config: 1267 return sym 1268 return None 1269 1270def check_imply_rule(kconf, config, imply_config): 1271 """Check if we can add an 'imply' option 1272 1273 This finds imply_config in the Kconfig and looks to see if it is possible 1274 to add an 'imply' for 'config' to that part of the Kconfig. 1275 1276 Args: 1277 kconf: Kconfiglib.Kconfig object 1278 config: Name of config to check (without CONFIG_ prefix) 1279 imply_config: Implying config (without CONFIG_ prefix) which may or 1280 may not have an 'imply' for 'config') 1281 1282 Returns: 1283 tuple: 1284 filename of Kconfig file containing imply_config, or None if none 1285 line number within the Kconfig file, or 0 if none 1286 message indicating the result 1287 """ 1288 sym = kconf.syms.get(imply_config) 1289 if not sym: 1290 return 'cannot find sym' 1291 nodes = sym.nodes 1292 if len(nodes) != 1: 1293 return '%d locations' % len(nodes) 1294 fname, linenum = nodes[0].filename, nodes[0].linern 1295 cwd = os.getcwd() 1296 if cwd and fname.startswith(cwd): 1297 fname = fname[len(cwd) + 1:] 1298 file_line = ' at %s:%d' % (fname, linenum) 1299 with open(fname) as fd: 1300 data = fd.read().splitlines() 1301 if data[linenum - 1] != 'config %s' % imply_config: 1302 return None, 0, 'bad sym format %s%s' % (data[linenum], file_line) 1303 return fname, linenum, 'adding%s' % file_line 1304 1305def add_imply_rule(config, fname, linenum): 1306 """Add a new 'imply' option to a Kconfig 1307 1308 Args: 1309 config: config option to add an imply for (without CONFIG_ prefix) 1310 fname: Kconfig filename to update 1311 linenum: Line number to place the 'imply' before 1312 1313 Returns: 1314 Message indicating the result 1315 """ 1316 file_line = ' at %s:%d' % (fname, linenum) 1317 data = open(fname).read().splitlines() 1318 linenum -= 1 1319 1320 for offset, line in enumerate(data[linenum:]): 1321 if line.strip().startswith('help') or not line: 1322 data.insert(linenum + offset, '\timply %s' % config) 1323 with open(fname, 'w') as fd: 1324 fd.write('\n'.join(data) + '\n') 1325 return 'added%s' % file_line 1326 1327 return 'could not insert%s' 1328 1329(IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = ( 1330 1, 2, 4, 8) 1331 1332IMPLY_FLAGS = { 1333 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'], 1334 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'], 1335 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'], 1336 'non-arch-board': [ 1337 IMPLY_NON_ARCH_BOARD, 1338 'Allow Kconfig options outside arch/ and /board/ to imply'], 1339}; 1340 1341def do_imply_config(config_list, add_imply, imply_flags, skip_added, 1342 check_kconfig=True, find_superset=False): 1343 """Find CONFIG options which imply those in the list 1344 1345 Some CONFIG options can be implied by others and this can help to reduce 1346 the size of the defconfig files. For example, CONFIG_X86 implies 1347 CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and 1348 all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to 1349 each of the x86 defconfig files. 1350 1351 This function uses the moveconfig database to find such options. It 1352 displays a list of things that could possibly imply those in the list. 1353 The algorithm ignores any that start with CONFIG_TARGET since these 1354 typically refer to only a few defconfigs (often one). It also does not 1355 display a config with less than 5 defconfigs. 1356 1357 The algorithm works using sets. For each target config in config_list: 1358 - Get the set 'defconfigs' which use that target config 1359 - For each config (from a list of all configs): 1360 - Get the set 'imply_defconfig' of defconfigs which use that config 1361 - 1362 - If imply_defconfigs contains anything not in defconfigs then 1363 this config does not imply the target config 1364 1365 Params: 1366 config_list: List of CONFIG options to check (each a string) 1367 add_imply: Automatically add an 'imply' for each config. 1368 imply_flags: Flags which control which implying configs are allowed 1369 (IMPLY_...) 1370 skip_added: Don't show options which already have an imply added. 1371 check_kconfig: Check if implied symbols already have an 'imply' or 1372 'select' for the target config, and show this information if so. 1373 find_superset: True to look for configs which are a superset of those 1374 already found. So for example if CONFIG_EXYNOS5 implies an option, 1375 but CONFIG_EXYNOS covers a larger set of defconfigs and also 1376 implies that option, this will drop the former in favour of the 1377 latter. In practice this option has not proved very used. 1378 1379 Note the terminoloy: 1380 config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM') 1381 defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig') 1382 """ 1383 kconf = KconfigScanner().conf if check_kconfig else None 1384 if add_imply and add_imply != 'all': 1385 add_imply = add_imply.split() 1386 1387 # key is defconfig name, value is dict of (CONFIG_xxx, value) 1388 config_db = {} 1389 1390 # Holds a dict containing the set of defconfigs that contain each config 1391 # key is config, value is set of defconfigs using that config 1392 defconfig_db = collections.defaultdict(set) 1393 1394 # Set of all config options we have seen 1395 all_configs = set() 1396 1397 # Set of all defconfigs we have seen 1398 all_defconfigs = set() 1399 1400 # Read in the database 1401 configs = {} 1402 with open(CONFIG_DATABASE) as fd: 1403 for line in fd.readlines(): 1404 line = line.rstrip() 1405 if not line: # Separator between defconfigs 1406 config_db[defconfig] = configs 1407 all_defconfigs.add(defconfig) 1408 configs = {} 1409 elif line[0] == ' ': # CONFIG line 1410 config, value = line.strip().split('=', 1) 1411 configs[config] = value 1412 defconfig_db[config].add(defconfig) 1413 all_configs.add(config) 1414 else: # New defconfig 1415 defconfig = line 1416 1417 # Work through each target config option in tern, independently 1418 for config in config_list: 1419 defconfigs = defconfig_db.get(config) 1420 if not defconfigs: 1421 print('%s not found in any defconfig' % config) 1422 continue 1423 1424 # Get the set of defconfigs without this one (since a config cannot 1425 # imply itself) 1426 non_defconfigs = all_defconfigs - defconfigs 1427 num_defconfigs = len(defconfigs) 1428 print('%s found in %d/%d defconfigs' % (config, num_defconfigs, 1429 len(all_configs))) 1430 1431 # This will hold the results: key=config, value=defconfigs containing it 1432 imply_configs = {} 1433 rest_configs = all_configs - set([config]) 1434 1435 # Look at every possible config, except the target one 1436 for imply_config in rest_configs: 1437 if 'ERRATUM' in imply_config: 1438 continue 1439 if not (imply_flags & IMPLY_CMD): 1440 if 'CONFIG_CMD' in imply_config: 1441 continue 1442 if not (imply_flags & IMPLY_TARGET): 1443 if 'CONFIG_TARGET' in imply_config: 1444 continue 1445 1446 # Find set of defconfigs that have this config 1447 imply_defconfig = defconfig_db[imply_config] 1448 1449 # Get the intersection of this with defconfigs containing the 1450 # target config 1451 common_defconfigs = imply_defconfig & defconfigs 1452 1453 # Get the set of defconfigs containing this config which DO NOT 1454 # also contain the taret config. If this set is non-empty it means 1455 # that this config affects other defconfigs as well as (possibly) 1456 # the ones affected by the target config. This means it implies 1457 # things we don't want to imply. 1458 not_common_defconfigs = imply_defconfig & non_defconfigs 1459 if not_common_defconfigs: 1460 continue 1461 1462 # If there are common defconfigs, imply_config may be useful 1463 if common_defconfigs: 1464 skip = False 1465 if find_superset: 1466 for prev in list(imply_configs.keys()): 1467 prev_count = len(imply_configs[prev]) 1468 count = len(common_defconfigs) 1469 if (prev_count > count and 1470 (imply_configs[prev] & common_defconfigs == 1471 common_defconfigs)): 1472 # skip imply_config because prev is a superset 1473 skip = True 1474 break 1475 elif count > prev_count: 1476 # delete prev because imply_config is a superset 1477 del imply_configs[prev] 1478 if not skip: 1479 imply_configs[imply_config] = common_defconfigs 1480 1481 # Now we have a dict imply_configs of configs which imply each config 1482 # The value of each dict item is the set of defconfigs containing that 1483 # config. Rank them so that we print the configs that imply the largest 1484 # number of defconfigs first. 1485 ranked_iconfigs = sorted(imply_configs, 1486 key=lambda k: len(imply_configs[k]), reverse=True) 1487 kconfig_info = '' 1488 cwd = os.getcwd() 1489 add_list = collections.defaultdict(list) 1490 for iconfig in ranked_iconfigs: 1491 num_common = len(imply_configs[iconfig]) 1492 1493 # Don't bother if there are less than 5 defconfigs affected. 1494 if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5): 1495 continue 1496 missing = defconfigs - imply_configs[iconfig] 1497 missing_str = ', '.join(missing) if missing else 'all' 1498 missing_str = '' 1499 show = True 1500 if kconf: 1501 sym = find_kconfig_rules(kconf, config[CONFIG_LEN:], 1502 iconfig[CONFIG_LEN:]) 1503 kconfig_info = '' 1504 if sym: 1505 nodes = sym.nodes 1506 if len(nodes) == 1: 1507 fname, linenum = nodes[0].filename, nodes[0].linenr 1508 if cwd and fname.startswith(cwd): 1509 fname = fname[len(cwd) + 1:] 1510 kconfig_info = '%s:%d' % (fname, linenum) 1511 if skip_added: 1512 show = False 1513 else: 1514 sym = kconf.syms.get(iconfig[CONFIG_LEN:]) 1515 fname = '' 1516 if sym: 1517 nodes = sym.nodes 1518 if len(nodes) == 1: 1519 fname, linenum = nodes[0].filename, nodes[0].linenr 1520 if cwd and fname.startswith(cwd): 1521 fname = fname[len(cwd) + 1:] 1522 in_arch_board = not sym or (fname.startswith('arch') or 1523 fname.startswith('board')) 1524 if (not in_arch_board and 1525 not (imply_flags & IMPLY_NON_ARCH_BOARD)): 1526 continue 1527 1528 if add_imply and (add_imply == 'all' or 1529 iconfig in add_imply): 1530 fname, linenum, kconfig_info = (check_imply_rule(kconf, 1531 config[CONFIG_LEN:], iconfig[CONFIG_LEN:])) 1532 if fname: 1533 add_list[fname].append(linenum) 1534 1535 if show and kconfig_info != 'skip': 1536 print('%5d : %-30s%-25s %s' % (num_common, iconfig.ljust(30), 1537 kconfig_info, missing_str)) 1538 1539 # Having collected a list of things to add, now we add them. We process 1540 # each file from the largest line number to the smallest so that 1541 # earlier additions do not affect our line numbers. E.g. if we added an 1542 # imply at line 20 it would change the position of each line after 1543 # that. 1544 for fname, linenums in add_list.items(): 1545 for linenum in sorted(linenums, reverse=True): 1546 add_imply_rule(config[CONFIG_LEN:], fname, linenum) 1547 1548 1549def main(): 1550 try: 1551 cpu_count = multiprocessing.cpu_count() 1552 except NotImplementedError: 1553 cpu_count = 1 1554 1555 parser = optparse.OptionParser() 1556 # Add options here 1557 parser.add_option('-a', '--add-imply', type='string', default='', 1558 help='comma-separated list of CONFIG options to add ' 1559 "an 'imply' statement to for the CONFIG in -i") 1560 parser.add_option('-A', '--skip-added', action='store_true', default=False, 1561 help="don't show options which are already marked as " 1562 'implying others') 1563 parser.add_option('-b', '--build-db', action='store_true', default=False, 1564 help='build a CONFIG database') 1565 parser.add_option('-c', '--color', action='store_true', default=False, 1566 help='display the log in color') 1567 parser.add_option('-C', '--commit', action='store_true', default=False, 1568 help='Create a git commit for the operation') 1569 parser.add_option('-d', '--defconfigs', type='string', 1570 help='a file containing a list of defconfigs to move, ' 1571 "one per line (for example 'snow_defconfig') " 1572 "or '-' to read from stdin") 1573 parser.add_option('-i', '--imply', action='store_true', default=False, 1574 help='find options which imply others') 1575 parser.add_option('-I', '--imply-flags', type='string', default='', 1576 help="control the -i option ('help' for help") 1577 parser.add_option('-n', '--dry-run', action='store_true', default=False, 1578 help='perform a trial run (show log with no changes)') 1579 parser.add_option('-e', '--exit-on-error', action='store_true', 1580 default=False, 1581 help='exit immediately on any error') 1582 parser.add_option('-s', '--force-sync', action='store_true', default=False, 1583 help='force sync by savedefconfig') 1584 parser.add_option('-S', '--spl', action='store_true', default=False, 1585 help='parse config options defined for SPL build') 1586 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 1587 action='store_true', default=False, 1588 help='only cleanup the headers') 1589 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 1590 help='the number of jobs to run simultaneously') 1591 parser.add_option('-r', '--git-ref', type='string', 1592 help='the git ref to clone for building the autoconf.mk') 1593 parser.add_option('-y', '--yes', action='store_true', default=False, 1594 help="respond 'yes' to any prompts") 1595 parser.add_option('-v', '--verbose', action='store_true', default=False, 1596 help='show any build errors as boards are built') 1597 parser.usage += ' CONFIG ...' 1598 1599 (options, configs) = parser.parse_args() 1600 1601 if len(configs) == 0 and not any((options.force_sync, options.build_db, 1602 options.imply)): 1603 parser.print_usage() 1604 sys.exit(1) 1605 1606 # prefix the option name with CONFIG_ if missing 1607 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 1608 for config in configs ] 1609 1610 check_top_directory() 1611 1612 if options.imply: 1613 imply_flags = 0 1614 if options.imply_flags == 'all': 1615 imply_flags = -1 1616 1617 elif options.imply_flags: 1618 for flag in options.imply_flags.split(','): 1619 bad = flag not in IMPLY_FLAGS 1620 if bad: 1621 print("Invalid flag '%s'" % flag) 1622 if flag == 'help' or bad: 1623 print("Imply flags: (separate with ',')") 1624 for name, info in IMPLY_FLAGS.items(): 1625 print(' %-15s: %s' % (name, info[1])) 1626 parser.print_usage() 1627 sys.exit(1) 1628 imply_flags |= IMPLY_FLAGS[flag][0] 1629 1630 do_imply_config(configs, options.add_imply, imply_flags, 1631 options.skip_added) 1632 return 1633 1634 config_db = {} 1635 db_queue = queue.Queue() 1636 t = DatabaseThread(config_db, db_queue) 1637 t.setDaemon(True) 1638 t.start() 1639 1640 if not options.cleanup_headers_only: 1641 check_clean_directory() 1642 bsettings.Setup('') 1643 toolchains = toolchain.Toolchains() 1644 toolchains.GetSettings() 1645 toolchains.Scan(verbose=False) 1646 move_config(toolchains, configs, options, db_queue) 1647 db_queue.join() 1648 1649 if configs: 1650 cleanup_headers(configs, options) 1651 cleanup_extra_options(configs, options) 1652 cleanup_whitelist(configs, options) 1653 cleanup_readme(configs, options) 1654 1655 if options.commit: 1656 subprocess.call(['git', 'add', '-u']) 1657 if configs: 1658 msg = 'Convert %s %sto Kconfig' % (configs[0], 1659 'et al ' if len(configs) > 1 else '') 1660 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % 1661 '\n '.join(configs)) 1662 else: 1663 msg = 'configs: Resync with savedefconfig' 1664 msg += '\n\nRsync all defconfig files using moveconfig.py' 1665 subprocess.call(['git', 'commit', '-s', '-m', msg]) 1666 1667 if options.build_db: 1668 with open(CONFIG_DATABASE, 'w') as fd: 1669 for defconfig, configs in config_db.items(): 1670 fd.write('%s\n' % defconfig) 1671 for config in sorted(configs.keys()): 1672 fd.write(' %s=%s\n' % (config, configs[config])) 1673 fd.write('\n') 1674 1675if __name__ == '__main__': 1676 main() 1677