1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2016 Google, Inc 3# Written by Simon Glass <sjg@chromium.org> 4# 5# Creates binary images from input files controlled by a description 6# 7 8from collections import OrderedDict 9import glob 10import os 11import pkg_resources 12import re 13 14import sys 15from patman import tools 16 17from binman import cbfs_util 18from binman import elf 19from patman import command 20from patman import tout 21 22# List of images we plan to create 23# Make this global so that it can be referenced from tests 24images = OrderedDict() 25 26# Help text for each type of missing blob, dict: 27# key: Value of the entry's 'missing-msg' or entry name 28# value: Text for the help 29missing_blob_help = {} 30 31def _ReadImageDesc(binman_node, use_expanded): 32 """Read the image descriptions from the /binman node 33 34 This normally produces a single Image object called 'image'. But if 35 multiple images are present, they will all be returned. 36 37 Args: 38 binman_node: Node object of the /binman node 39 use_expanded: True if the FDT will be updated with the entry information 40 Returns: 41 OrderedDict of Image objects, each of which describes an image 42 """ 43 images = OrderedDict() 44 if 'multiple-images' in binman_node.props: 45 for node in binman_node.subnodes: 46 images[node.name] = Image(node.name, node, 47 use_expanded=use_expanded) 48 else: 49 images['image'] = Image('image', binman_node, use_expanded=use_expanded) 50 return images 51 52def _FindBinmanNode(dtb): 53 """Find the 'binman' node in the device tree 54 55 Args: 56 dtb: Fdt object to scan 57 Returns: 58 Node object of /binman node, or None if not found 59 """ 60 for node in dtb.GetRoot().subnodes: 61 if node.name == 'binman': 62 return node 63 return None 64 65def _ReadMissingBlobHelp(): 66 """Read the missing-blob-help file 67 68 This file containins help messages explaining what to do when external blobs 69 are missing. 70 71 Returns: 72 Dict: 73 key: Message tag (str) 74 value: Message text (str) 75 """ 76 77 def _FinishTag(tag, msg, result): 78 if tag: 79 result[tag] = msg.rstrip() 80 tag = None 81 msg = '' 82 return tag, msg 83 84 my_data = pkg_resources.resource_string(__name__, 'missing-blob-help') 85 re_tag = re.compile('^([-a-z0-9]+):$') 86 result = {} 87 tag = None 88 msg = '' 89 for line in my_data.decode('utf-8').splitlines(): 90 if not line.startswith('#'): 91 m_tag = re_tag.match(line) 92 if m_tag: 93 _, msg = _FinishTag(tag, msg, result) 94 tag = m_tag.group(1) 95 elif tag: 96 msg += line + '\n' 97 _FinishTag(tag, msg, result) 98 return result 99 100def _ShowBlobHelp(path, text): 101 tout.Warning('\n%s:' % path) 102 for line in text.splitlines(): 103 tout.Warning(' %s' % line) 104 105def _ShowHelpForMissingBlobs(missing_list): 106 """Show help for each missing blob to help the user take action 107 108 Args: 109 missing_list: List of Entry objects to show help for 110 """ 111 global missing_blob_help 112 113 if not missing_blob_help: 114 missing_blob_help = _ReadMissingBlobHelp() 115 116 for entry in missing_list: 117 tags = entry.GetHelpTags() 118 119 # Show the first match help message 120 for tag in tags: 121 if tag in missing_blob_help: 122 _ShowBlobHelp(entry._node.path, missing_blob_help[tag]) 123 break 124 125def GetEntryModules(include_testing=True): 126 """Get a set of entry class implementations 127 128 Returns: 129 Set of paths to entry class filenames 130 """ 131 glob_list = pkg_resources.resource_listdir(__name__, 'etype') 132 glob_list = [fname for fname in glob_list if fname.endswith('.py')] 133 return set([os.path.splitext(os.path.basename(item))[0] 134 for item in glob_list 135 if include_testing or '_testing' not in item]) 136 137def WriteEntryDocs(modules, test_missing=None): 138 """Write out documentation for all entries 139 140 Args: 141 modules: List of Module objects to get docs for 142 test_missing: Used for testing only, to force an entry's documeentation 143 to show as missing even if it is present. Should be set to None in 144 normal use. 145 """ 146 from binman.entry import Entry 147 Entry.WriteDocs(modules, test_missing) 148 149 150def ListEntries(image_fname, entry_paths): 151 """List the entries in an image 152 153 This decodes the supplied image and displays a table of entries from that 154 image, preceded by a header. 155 156 Args: 157 image_fname: Image filename to process 158 entry_paths: List of wildcarded paths (e.g. ['*dtb*', 'u-boot*', 159 'section/u-boot']) 160 """ 161 image = Image.FromFile(image_fname) 162 163 entries, lines, widths = image.GetListEntries(entry_paths) 164 165 num_columns = len(widths) 166 for linenum, line in enumerate(lines): 167 if linenum == 1: 168 # Print header line 169 print('-' * (sum(widths) + num_columns * 2)) 170 out = '' 171 for i, item in enumerate(line): 172 width = -widths[i] 173 if item.startswith('>'): 174 width = -width 175 item = item[1:] 176 txt = '%*s ' % (width, item) 177 out += txt 178 print(out.rstrip()) 179 180 181def ReadEntry(image_fname, entry_path, decomp=True): 182 """Extract an entry from an image 183 184 This extracts the data from a particular entry in an image 185 186 Args: 187 image_fname: Image filename to process 188 entry_path: Path to entry to extract 189 decomp: True to return uncompressed data, if the data is compress 190 False to return the raw data 191 192 Returns: 193 data extracted from the entry 194 """ 195 global Image 196 from binman.image import Image 197 198 image = Image.FromFile(image_fname) 199 entry = image.FindEntryPath(entry_path) 200 return entry.ReadData(decomp) 201 202 203def ExtractEntries(image_fname, output_fname, outdir, entry_paths, 204 decomp=True): 205 """Extract the data from one or more entries and write it to files 206 207 Args: 208 image_fname: Image filename to process 209 output_fname: Single output filename to use if extracting one file, None 210 otherwise 211 outdir: Output directory to use (for any number of files), else None 212 entry_paths: List of entry paths to extract 213 decomp: True to decompress the entry data 214 215 Returns: 216 List of EntryInfo records that were written 217 """ 218 image = Image.FromFile(image_fname) 219 220 # Output an entry to a single file, as a special case 221 if output_fname: 222 if not entry_paths: 223 raise ValueError('Must specify an entry path to write with -f') 224 if len(entry_paths) != 1: 225 raise ValueError('Must specify exactly one entry path to write with -f') 226 entry = image.FindEntryPath(entry_paths[0]) 227 data = entry.ReadData(decomp) 228 tools.WriteFile(output_fname, data) 229 tout.Notice("Wrote %#x bytes to file '%s'" % (len(data), output_fname)) 230 return 231 232 # Otherwise we will output to a path given by the entry path of each entry. 233 # This means that entries will appear in subdirectories if they are part of 234 # a sub-section. 235 einfos = image.GetListEntries(entry_paths)[0] 236 tout.Notice('%d entries match and will be written' % len(einfos)) 237 for einfo in einfos: 238 entry = einfo.entry 239 data = entry.ReadData(decomp) 240 path = entry.GetPath()[1:] 241 fname = os.path.join(outdir, path) 242 243 # If this entry has children, create a directory for it and put its 244 # data in a file called 'root' in that directory 245 if entry.GetEntries(): 246 if fname and not os.path.exists(fname): 247 os.makedirs(fname) 248 fname = os.path.join(fname, 'root') 249 tout.Notice("Write entry '%s' size %x to '%s'" % 250 (entry.GetPath(), len(data), fname)) 251 tools.WriteFile(fname, data) 252 return einfos 253 254 255def BeforeReplace(image, allow_resize): 256 """Handle getting an image ready for replacing entries in it 257 258 Args: 259 image: Image to prepare 260 """ 261 state.PrepareFromLoadedData(image) 262 image.LoadData() 263 264 # If repacking, drop the old offset/size values except for the original 265 # ones, so we are only left with the constraints. 266 if allow_resize: 267 image.ResetForPack() 268 269 270def ReplaceOneEntry(image, entry, data, do_compress, allow_resize): 271 """Handle replacing a single entry an an image 272 273 Args: 274 image: Image to update 275 entry: Entry to write 276 data: Data to replace with 277 do_compress: True to compress the data if needed, False if data is 278 already compressed so should be used as is 279 allow_resize: True to allow entries to change size (this does a re-pack 280 of the entries), False to raise an exception 281 """ 282 if not entry.WriteData(data, do_compress): 283 if not image.allow_repack: 284 entry.Raise('Entry data size does not match, but allow-repack is not present for this image') 285 if not allow_resize: 286 entry.Raise('Entry data size does not match, but resize is disabled') 287 288 289def AfterReplace(image, allow_resize, write_map): 290 """Handle write out an image after replacing entries in it 291 292 Args: 293 image: Image to write 294 allow_resize: True to allow entries to change size (this does a re-pack 295 of the entries), False to raise an exception 296 write_map: True to write a map file 297 """ 298 tout.Info('Processing image') 299 ProcessImage(image, update_fdt=True, write_map=write_map, 300 get_contents=False, allow_resize=allow_resize) 301 302 303def WriteEntryToImage(image, entry, data, do_compress=True, allow_resize=True, 304 write_map=False): 305 BeforeReplace(image, allow_resize) 306 tout.Info('Writing data to %s' % entry.GetPath()) 307 ReplaceOneEntry(image, entry, data, do_compress, allow_resize) 308 AfterReplace(image, allow_resize=allow_resize, write_map=write_map) 309 310 311def WriteEntry(image_fname, entry_path, data, do_compress=True, 312 allow_resize=True, write_map=False): 313 """Replace an entry in an image 314 315 This replaces the data in a particular entry in an image. This size of the 316 new data must match the size of the old data unless allow_resize is True. 317 318 Args: 319 image_fname: Image filename to process 320 entry_path: Path to entry to extract 321 data: Data to replace with 322 do_compress: True to compress the data if needed, False if data is 323 already compressed so should be used as is 324 allow_resize: True to allow entries to change size (this does a re-pack 325 of the entries), False to raise an exception 326 write_map: True to write a map file 327 328 Returns: 329 Image object that was updated 330 """ 331 tout.Info("Write entry '%s', file '%s'" % (entry_path, image_fname)) 332 image = Image.FromFile(image_fname) 333 entry = image.FindEntryPath(entry_path) 334 WriteEntryToImage(image, entry, data, do_compress=do_compress, 335 allow_resize=allow_resize, write_map=write_map) 336 337 return image 338 339 340def ReplaceEntries(image_fname, input_fname, indir, entry_paths, 341 do_compress=True, allow_resize=True, write_map=False): 342 """Replace the data from one or more entries from input files 343 344 Args: 345 image_fname: Image filename to process 346 input_fname: Single input filename to use if replacing one file, None 347 otherwise 348 indir: Input directory to use (for any number of files), else None 349 entry_paths: List of entry paths to replace 350 do_compress: True if the input data is uncompressed and may need to be 351 compressed if the entry requires it, False if the data is already 352 compressed. 353 write_map: True to write a map file 354 355 Returns: 356 List of EntryInfo records that were written 357 """ 358 image = Image.FromFile(image_fname) 359 360 # Replace an entry from a single file, as a special case 361 if input_fname: 362 if not entry_paths: 363 raise ValueError('Must specify an entry path to read with -f') 364 if len(entry_paths) != 1: 365 raise ValueError('Must specify exactly one entry path to write with -f') 366 entry = image.FindEntryPath(entry_paths[0]) 367 data = tools.ReadFile(input_fname) 368 tout.Notice("Read %#x bytes from file '%s'" % (len(data), input_fname)) 369 WriteEntryToImage(image, entry, data, do_compress=do_compress, 370 allow_resize=allow_resize, write_map=write_map) 371 return 372 373 # Otherwise we will input from a path given by the entry path of each entry. 374 # This means that files must appear in subdirectories if they are part of 375 # a sub-section. 376 einfos = image.GetListEntries(entry_paths)[0] 377 tout.Notice("Replacing %d matching entries in image '%s'" % 378 (len(einfos), image_fname)) 379 380 BeforeReplace(image, allow_resize) 381 382 for einfo in einfos: 383 entry = einfo.entry 384 if entry.GetEntries(): 385 tout.Info("Skipping section entry '%s'" % entry.GetPath()) 386 continue 387 388 path = entry.GetPath()[1:] 389 fname = os.path.join(indir, path) 390 391 if os.path.exists(fname): 392 tout.Notice("Write entry '%s' from file '%s'" % 393 (entry.GetPath(), fname)) 394 data = tools.ReadFile(fname) 395 ReplaceOneEntry(image, entry, data, do_compress, allow_resize) 396 else: 397 tout.Warning("Skipping entry '%s' from missing file '%s'" % 398 (entry.GetPath(), fname)) 399 400 AfterReplace(image, allow_resize=allow_resize, write_map=write_map) 401 return image 402 403 404def PrepareImagesAndDtbs(dtb_fname, select_images, update_fdt, use_expanded): 405 """Prepare the images to be processed and select the device tree 406 407 This function: 408 - reads in the device tree 409 - finds and scans the binman node to create all entries 410 - selects which images to build 411 - Updates the device tress with placeholder properties for offset, 412 image-pos, etc. 413 414 Args: 415 dtb_fname: Filename of the device tree file to use (.dts or .dtb) 416 selected_images: List of images to output, or None for all 417 update_fdt: True to update the FDT wth entry offsets, etc. 418 use_expanded: True to use expanded versions of entries, if available. 419 So if 'u-boot' is called for, we use 'u-boot-expanded' instead. This 420 is needed if update_fdt is True (although tests may disable it) 421 422 Returns: 423 OrderedDict of images: 424 key: Image name (str) 425 value: Image object 426 """ 427 # Import these here in case libfdt.py is not available, in which case 428 # the above help option still works. 429 from dtoc import fdt 430 from dtoc import fdt_util 431 global images 432 433 # Get the device tree ready by compiling it and copying the compiled 434 # output into a file in our output directly. Then scan it for use 435 # in binman. 436 dtb_fname = fdt_util.EnsureCompiled(dtb_fname) 437 fname = tools.GetOutputFilename('u-boot.dtb.out') 438 tools.WriteFile(fname, tools.ReadFile(dtb_fname)) 439 dtb = fdt.FdtScan(fname) 440 441 node = _FindBinmanNode(dtb) 442 if not node: 443 raise ValueError("Device tree '%s' does not have a 'binman' " 444 "node" % dtb_fname) 445 446 images = _ReadImageDesc(node, use_expanded) 447 448 if select_images: 449 skip = [] 450 new_images = OrderedDict() 451 for name, image in images.items(): 452 if name in select_images: 453 new_images[name] = image 454 else: 455 skip.append(name) 456 images = new_images 457 tout.Notice('Skipping images: %s' % ', '.join(skip)) 458 459 state.Prepare(images, dtb) 460 461 # Prepare the device tree by making sure that any missing 462 # properties are added (e.g. 'pos' and 'size'). The values of these 463 # may not be correct yet, but we add placeholders so that the 464 # size of the device tree is correct. Later, in 465 # SetCalculatedProperties() we will insert the correct values 466 # without changing the device-tree size, thus ensuring that our 467 # entry offsets remain the same. 468 for image in images.values(): 469 image.ExpandEntries() 470 if update_fdt: 471 image.AddMissingProperties(True) 472 image.ProcessFdt(dtb) 473 474 for dtb_item in state.GetAllFdts(): 475 dtb_item.Sync(auto_resize=True) 476 dtb_item.Pack() 477 dtb_item.Flush() 478 return images 479 480 481def ProcessImage(image, update_fdt, write_map, get_contents=True, 482 allow_resize=True, allow_missing=False, 483 allow_fake_blobs=False): 484 """Perform all steps for this image, including checking and # writing it. 485 486 This means that errors found with a later image will be reported after 487 earlier images are already completed and written, but that does not seem 488 important. 489 490 Args: 491 image: Image to process 492 update_fdt: True to update the FDT wth entry offsets, etc. 493 write_map: True to write a map file 494 get_contents: True to get the image contents from files, etc., False if 495 the contents is already present 496 allow_resize: True to allow entries to change size (this does a re-pack 497 of the entries), False to raise an exception 498 allow_missing: Allow blob_ext objects to be missing 499 allow_fake_blobs: Allow blob_ext objects to be faked with dummy files 500 501 Returns: 502 True if one or more external blobs are missing or faked, 503 False if all are present 504 """ 505 if get_contents: 506 image.SetAllowMissing(allow_missing) 507 image.SetAllowFakeBlob(allow_fake_blobs) 508 image.GetEntryContents() 509 image.GetEntryOffsets() 510 511 # We need to pack the entries to figure out where everything 512 # should be placed. This sets the offset/size of each entry. 513 # However, after packing we call ProcessEntryContents() which 514 # may result in an entry changing size. In that case we need to 515 # do another pass. Since the device tree often contains the 516 # final offset/size information we try to make space for this in 517 # AddMissingProperties() above. However, if the device is 518 # compressed we cannot know this compressed size in advance, 519 # since changing an offset from 0x100 to 0x104 (for example) can 520 # alter the compressed size of the device tree. So we need a 521 # third pass for this. 522 passes = 5 523 for pack_pass in range(passes): 524 try: 525 image.PackEntries() 526 except Exception as e: 527 if write_map: 528 fname = image.WriteMap() 529 print("Wrote map file '%s' to show errors" % fname) 530 raise 531 image.SetImagePos() 532 if update_fdt: 533 image.SetCalculatedProperties() 534 for dtb_item in state.GetAllFdts(): 535 dtb_item.Sync() 536 dtb_item.Flush() 537 image.WriteSymbols() 538 sizes_ok = image.ProcessEntryContents() 539 if sizes_ok: 540 break 541 image.ResetForPack() 542 tout.Info('Pack completed after %d pass(es)' % (pack_pass + 1)) 543 if not sizes_ok: 544 image.Raise('Entries changed size after packing (tried %s passes)' % 545 passes) 546 547 image.BuildImage() 548 if write_map: 549 image.WriteMap() 550 missing_list = [] 551 image.CheckMissing(missing_list) 552 if missing_list: 553 tout.Warning("Image '%s' is missing external blobs and is non-functional: %s" % 554 (image.name, ' '.join([e.name for e in missing_list]))) 555 _ShowHelpForMissingBlobs(missing_list) 556 faked_list = [] 557 image.CheckFakedBlobs(faked_list) 558 if faked_list: 559 tout.Warning("Image '%s:%s' has faked external blobs and is non-functional: %s" % 560 (image.name, image.image_name, 561 ' '.join([e.GetDefaultFilename() for e in faked_list]))) 562 return bool(missing_list) or bool(faked_list) 563 564 565def Binman(args): 566 """The main control code for binman 567 568 This assumes that help and test options have already been dealt with. It 569 deals with the core task of building images. 570 571 Args: 572 args: Command line arguments Namespace object 573 """ 574 global Image 575 global state 576 577 if args.full_help: 578 tools.PrintFullHelp( 579 os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'README.rst') 580 ) 581 return 0 582 583 # Put these here so that we can import this module without libfdt 584 from binman.image import Image 585 from binman import state 586 587 if args.cmd in ['ls', 'extract', 'replace']: 588 try: 589 tout.Init(args.verbosity) 590 tools.PrepareOutputDir(None) 591 if args.cmd == 'ls': 592 ListEntries(args.image, args.paths) 593 594 if args.cmd == 'extract': 595 ExtractEntries(args.image, args.filename, args.outdir, args.paths, 596 not args.uncompressed) 597 598 if args.cmd == 'replace': 599 ReplaceEntries(args.image, args.filename, args.indir, args.paths, 600 do_compress=not args.compressed, 601 allow_resize=not args.fix_size, write_map=args.map) 602 except: 603 raise 604 finally: 605 tools.FinaliseOutputDir() 606 return 0 607 608 elf_params = None 609 if args.update_fdt_in_elf: 610 elf_params = args.update_fdt_in_elf.split(',') 611 if len(elf_params) != 4: 612 raise ValueError('Invalid args %s to --update-fdt-in-elf: expected infile,outfile,begin_sym,end_sym' % 613 elf_params) 614 615 # Try to figure out which device tree contains our image description 616 if args.dt: 617 dtb_fname = args.dt 618 else: 619 board = args.board 620 if not board: 621 raise ValueError('Must provide a board to process (use -b <board>)') 622 board_pathname = os.path.join(args.build_dir, board) 623 dtb_fname = os.path.join(board_pathname, 'u-boot.dtb') 624 if not args.indir: 625 args.indir = ['.'] 626 args.indir.append(board_pathname) 627 628 try: 629 tout.Init(args.verbosity) 630 elf.debug = args.debug 631 cbfs_util.VERBOSE = args.verbosity > 2 632 state.use_fake_dtb = args.fake_dtb 633 634 # Normally we replace the 'u-boot' etype with 'u-boot-expanded', etc. 635 # When running tests this can be disabled using this flag. When not 636 # updating the FDT in image, it is not needed by binman, but we use it 637 # for consistency, so that the images look the same to U-Boot at 638 # runtime. 639 use_expanded = not args.no_expanded 640 try: 641 tools.SetInputDirs(args.indir) 642 tools.PrepareOutputDir(args.outdir, args.preserve) 643 tools.SetToolPaths(args.toolpath) 644 state.SetEntryArgs(args.entry_arg) 645 state.SetThreads(args.threads) 646 647 images = PrepareImagesAndDtbs(dtb_fname, args.image, 648 args.update_fdt, use_expanded) 649 650 if args.test_section_timeout: 651 # Set the first image to timeout, used in testThreadTimeout() 652 images[list(images.keys())[0]].test_section_timeout = True 653 invalid = False 654 for image in images.values(): 655 invalid |= ProcessImage(image, args.update_fdt, args.map, 656 allow_missing=args.allow_missing, 657 allow_fake_blobs=args.fake_ext_blobs) 658 659 # Write the updated FDTs to our output files 660 for dtb_item in state.GetAllFdts(): 661 tools.WriteFile(dtb_item._fname, dtb_item.GetContents()) 662 663 if elf_params: 664 data = state.GetFdtForEtype('u-boot-dtb').GetContents() 665 elf.UpdateFile(*elf_params, data) 666 667 if invalid: 668 tout.Warning("\nSome images are invalid") 669 670 # Use this to debug the time take to pack the image 671 #state.TimingShow() 672 finally: 673 tools.FinaliseOutputDir() 674 finally: 675 tout.Uninit() 676 677 return 0 678