Merge git://git.denx.de/u-boot-dm

board/sandbox/README.sandbox

@@ -320,6 +320,25 @@ CONFIG_SPI_IDLE_VAL
 	The idle value on the SPI bus
 
 
+Block Device Emulation
+----------------------
+
+U-Boot can use raw disk images for block device emulation. To e.g. list
+the contents of the root directory on the second partion of the image
+"disk.raw", you can use the following commands:
+
+=>host bind 0 ./disk.raw
+=>ls host 0:2
+
+A disk image can be created using the following commands:
+
+$> truncate -s 1200M ./disk.raw
+$> echo -e "label: gpt\n,64M,U\n,,L" | /usr/sbin/sfdisk  ./disk.raw
+$> lodev=`sudo losetup -P -f --show ./disk.raw`
+$> sudo mkfs.vfat -n EFI -v ${lodev}p1
+$> sudo mkfs.ext4 -L ROOT -v ${lodev}p2
+
+
 Writing Sandbox Drivers
 -----------------------
 

cmd/host.c

@@ -25,6 +25,12 @@ static int do_host_ls(cmd_tbl_t *cmdtp, int flag, int argc,
 	return do_ls(cmdtp, flag, argc, argv, FS_TYPE_SANDBOX);
 }
 
+static int do_host_size(cmd_tbl_t *cmdtp, int flag, int argc,
+			   char * const argv[])
+{
+	return do_size(cmdtp, flag, argc, argv, FS_TYPE_SANDBOX);
+}
+
 static int do_host_save(cmd_tbl_t *cmdtp, int flag, int argc,
 			   char * const argv[])
 {
@@ -138,6 +144,7 @@ static cmd_tbl_t cmd_host_sub[] = {
 	U_BOOT_CMD_MKENT(load, 7, 0, do_host_load, "", ""),
 	U_BOOT_CMD_MKENT(ls, 3, 0, do_host_ls, "", ""),
 	U_BOOT_CMD_MKENT(save, 6, 0, do_host_save, "", ""),
+	U_BOOT_CMD_MKENT(size, 3, 0, do_host_size, "", ""),
 	U_BOOT_CMD_MKENT(bind, 3, 0, do_host_bind, "", ""),
 	U_BOOT_CMD_MKENT(info, 3, 0, do_host_info, "", ""),
 	U_BOOT_CMD_MKENT(dev, 0, 1, do_host_dev, "", ""),
@@ -174,6 +181,7 @@ U_BOOT_CMD(
 	"host ls hostfs - <filename>                    - list files on host\n"
 	"host save hostfs - <addr> <filename> <bytes> [<offset>] - "
 		"save a file to host\n"
+	"host size hostfs - <filename> - determine size of file on host"
 	"host bind <dev> [<filename>] - bind \"host\" device to file\n"
 	"host info [<dev>]            - show device binding & info\n"
 	"host dev [<dev>] - Set or retrieve the current host device\n"

lib/libfdt/libfdt.swig

@@ -75,6 +75,14 @@ struct fdt_property {
     }
 %}
 
+%typemap(in) (const void *) {
+  if (!PyByteArray_Check($input)) {
+    SWIG_exception_fail(SWIG_TypeError, "in method '" "$symname" "', argument "
+                       "$argnum"" of type '" "$type""'");
+  }
+  $1 = (void *) PyByteArray_AsString($input);
+}
+
 const void *fdt_offset_ptr(const void *fdt, int offset, unsigned int checklen);
 int fdt_path_offset(const void *fdt, const char *path);
 int fdt_first_property_offset(const void *fdt, int nodeoffset);
@@ -87,3 +95,19 @@ const char *fdt_get_name(const void *fdt, int nodeoffset, int *OUTPUT);
 const char *fdt_string(const void *fdt, int stroffset);
 int fdt_first_subnode(const void *fdt, int offset);
 int fdt_next_subnode(const void *fdt, int offset);
+
+%typemap(in) (void *) {
+  if (!PyByteArray_Check($input)) {
+    SWIG_exception_fail(SWIG_TypeError, "in method '" "$symname" "', argument "
+                       "$argnum"" of type '" "$type""'");
+  }
+  $1 = PyByteArray_AsString($input);
+}
+
+int fdt_delprop(void *fdt, int nodeoffset, const char *name);
+
+const char *fdt_strerror(int errval);
+int fdt_pack(void *fdt);
+
+int fdt_totalsize(const void *fdt);
+int fdt_off_dt_struct(const void *fdt);

tools/buildman/control.py

@@ -237,7 +237,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
         options.step = len(series.commits) - 1
 
     gnu_make = command.Output(os.path.join(options.git,
-                                           'scripts/show-gnu-make')).rstrip()
+            'scripts/show-gnu-make'), raise_on_error=False).rstrip()
     if not gnu_make:
         sys.exit('GNU Make not found')
 

tools/dtoc/dtoc.py

@@ -9,27 +9,16 @@
 import copy
 from optparse import OptionError, OptionParser
 import os
+import struct
 import sys
 
-import fdt_util
-
 # Bring in the patman libraries
 our_path = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(os.path.join(our_path, '../patman'))
 
-# Bring in either the normal fdt library (which relies on libfdt) or the
+import fdt
-# fallback one (which uses fdtget and is slower). Both provide the same
+import fdt_select
-# interfface for this file to use.
+import fdt_util
-try:
-    from fdt import Fdt
-    import fdt
-    have_libfdt = True
-except ImportError:
-    have_libfdt = False
-    from fdt_fallback import Fdt
-    import fdt_fallback as fdt
-
-import struct
 
 # When we see these properties we ignore them - i.e. do not create a structure member
 PROP_IGNORE_LIST = [
@@ -45,10 +34,10 @@ PROP_IGNORE_LIST = [
 
 # C type declarations for the tyues we support
 TYPE_NAMES = {
-    fdt_util.TYPE_INT: 'fdt32_t',
+    fdt.TYPE_INT: 'fdt32_t',
-    fdt_util.TYPE_BYTE: 'unsigned char',
+    fdt.TYPE_BYTE: 'unsigned char',
-    fdt_util.TYPE_STRING: 'const char *',
+    fdt.TYPE_STRING: 'const char *',
-    fdt_util.TYPE_BOOL: 'bool',
+    fdt.TYPE_BOOL: 'bool',
 };
 
 STRUCT_PREFIX = 'dtd_'
@@ -150,13 +139,13 @@ class DtbPlatdata:
             type: Data type (fdt_util)
             value: Data value, as a string of bytes
         """
-        if type == fdt_util.TYPE_INT:
+        if type == fdt.TYPE_INT:
             return '%#x' % fdt_util.fdt32_to_cpu(value)
-        elif type == fdt_util.TYPE_BYTE:
+        elif type == fdt.TYPE_BYTE:
             return '%#x' % ord(value[0])
-        elif type == fdt_util.TYPE_STRING:
+        elif type == fdt.TYPE_STRING:
             return '"%s"' % value
-        elif type == fdt_util.TYPE_BOOL:
+        elif type == fdt.TYPE_BOOL:
             return 'true'
 
     def GetCompatName(self, node):
@@ -178,8 +167,7 @@ class DtbPlatdata:
         Once this is done, self.fdt.GetRoot() can be called to obtain the
         device tree root node, and progress from there.
         """
-        self.fdt = Fdt(self._dtb_fname)
+        self.fdt = fdt_select.FdtScan(self._dtb_fname)
-        self.fdt.Scan()
 
     def ScanTree(self):
         """Scan the device tree for useful information

tools/dtoc/fdt.py

@@ -6,17 +6,26 @@
 # SPDX-License-Identifier:      GPL-2.0+
 #
 
-import fdt_util
+import struct
-import libfdt
 import sys
 
-# This deals with a device tree, presenting it as a list of Node and Prop
+import fdt_util
-# objects, representing nodes and properties, respectively.
-#
-# This implementation uses a libfdt Python library to access the device tree,
-# so it is fairly efficient.
 
-class Prop:
+# This deals with a device tree, presenting it as an assortment of Node and
+# Prop objects, representing nodes and properties, respectively. This file
+# contains the base classes and defines the high-level API. Most of the
+# implementation is in the FdtFallback and FdtNormal subclasses. See
+# fdt_select.py for how to create an Fdt object.
+
+# A list of types we support
+(TYPE_BYTE, TYPE_INT, TYPE_STRING, TYPE_BOOL) = range(4)
+
+def CheckErr(errnum, msg):
+    if errnum:
+        raise ValueError('Error %d: %s: %s' %
+            (errnum, libfdt.fdt_strerror(errnum), msg))
+
+class PropBase:
     """A device tree property
 
     Properties:
@@ -25,14 +34,11 @@ class Prop:
             bytes
         type: Value type
     """
-    def __init__(self, name, bytes):
+    def __init__(self, node, offset, name):
+        self._node = node
+        self._offset = offset
         self.name = name
         self.value = None
-        if not bytes:
-            self.type = fdt_util.TYPE_BOOL
-            self.value = True
-            return
-        self.type, self.value = fdt_util.BytesToValue(bytes)
 
     def GetPhandle(self):
         """Get a (single) phandle value from a property
@@ -71,12 +77,85 @@ class Prop:
             self.value = [self.value]
 
         if type(self.value) == list and len(newprop.value) > len(self.value):
-            val = fdt_util.GetEmpty(self.type)
+            val = self.GetEmpty(self.type)
             while len(self.value) < len(newprop.value):
                 self.value.append(val)
 
+    def BytesToValue(self, bytes):
+        """Converts a string of bytes into a type and value
 
-class Node:
+        Args:
+            A string containing bytes
+
+        Return:
+            A tuple:
+                Type of data
+                Data, either a single element or a list of elements. Each element
+                is one of:
+                    TYPE_STRING: string value from the property
+                    TYPE_INT: a byte-swapped integer stored as a 4-byte string
+                    TYPE_BYTE: a byte stored as a single-byte string
+        """
+        size = len(bytes)
+        strings = bytes.split('\0')
+        is_string = True
+        count = len(strings) - 1
+        if count > 0 and not strings[-1]:
+            for string in strings[:-1]:
+                if not string:
+                    is_string = False
+                    break
+                for ch in string:
+                    if ch < ' ' or ch > '~':
+                        is_string = False
+                        break
+        else:
+            is_string = False
+        if is_string:
+            if count == 1:
+                return TYPE_STRING, strings[0]
+            else:
+                return TYPE_STRING, strings[:-1]
+        if size % 4:
+            if size == 1:
+                return TYPE_BYTE, bytes[0]
+            else:
+                return TYPE_BYTE, list(bytes)
+        val = []
+        for i in range(0, size, 4):
+            val.append(bytes[i:i + 4])
+        if size == 4:
+            return TYPE_INT, val[0]
+        else:
+            return TYPE_INT, val
+
+    def GetEmpty(self, type):
+        """Get an empty / zero value of the given type
+
+        Returns:
+            A single value of the given type
+        """
+        if type == TYPE_BYTE:
+            return chr(0)
+        elif type == TYPE_INT:
+            return struct.pack('<I', 0);
+        elif type == TYPE_STRING:
+            return ''
+        else:
+            return True
+
+    def GetOffset(self):
+        """Get the offset of a property
+
+        This can be implemented by subclasses.
+
+        Returns:
+            The offset of the property (struct fdt_property) within the
+            file, or None if not known.
+        """
+        return None
+
+class NodeBase:
     """A device tree node
 
     Properties:
@@ -89,32 +168,42 @@ class Node:
             Keyed by property name
     """
     def __init__(self, fdt, offset, name, path):
-        self.offset = offset
+        self._fdt = fdt
+        self._offset = offset
         self.name = name
         self.path = path
-        self._fdt = fdt
         self.subnodes = []
         self.props = {}
 
-    def Scan(self):
+    def _FindNode(self, name):
-        """Scan a node's properties and subnodes
+        """Find a node given its name
 
-        This fills in the props and subnodes properties, recursively
+        Args:
-        searching into subnodes so that the entire tree is built.
+            name: Node name to look for
+        Returns:
+            Node object if found, else None
         """
-        self.props = self._fdt.GetProps(self.path)
+        for subnode in self.subnodes:
+            if subnode.name == name:
+                return subnode
+        return None
 
-        offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self.offset)
+    def Scan(self):
-        while offset >= 0:
+        """Scan the subnodes of a node
-            sep = '' if self.path[-1] == '/' else '/'
-            name = libfdt.Name(self._fdt.GetFdt(), offset)
-            path = self.path + sep + name
-            node = Node(self._fdt, offset, name, path)
-            self.subnodes.append(node)
 
-            node.Scan()
+        This should be implemented by subclasses
-            offset = libfdt.fdt_next_subnode(self._fdt.GetFdt(), offset)
+        """
+        raise NotImplementedError()
 
+    def DeleteProp(self, prop_name):
+        """Delete a property of a node
+
+        This should be implemented by subclasses
+
+        Args:
+            prop_name: Name of the property to delete
+        """
+        raise NotImplementedError()
 
 class Fdt:
     """Provides simple access to a flat device tree blob.
@@ -123,26 +212,20 @@ class Fdt:
       fname: Filename of fdt
       _root: Root of device tree (a Node object)
     """
-
     def __init__(self, fname):
-        self.fname = fname
+        self._fname = fname
-        with open(fname) as fd:
-            self._fdt = fd.read()
 
-    def GetFdt(self):
+    def Scan(self, root='/'):
-        """Get the contents of the FDT
-
-        Returns:
-            The FDT contents as a string of bytes
-        """
-        return self._fdt
-
-    def Scan(self):
         """Scan a device tree, building up a tree of Node objects
 
         This fills in the self._root property
+
+        Args:
+            root: Ignored
+
+        TODO(sjg@chromium.org): Implement the 'root' parameter
         """
-        self._root = Node(self, 0, '/', '/')
+        self._root = self.Node(self, 0, '/', '/')
         self._root.Scan()
 
     def GetRoot(self):
@@ -153,28 +236,34 @@ class Fdt:
         """
         return self._root
 
-    def GetProps(self, node):
+    def GetNode(self, path):
-        """Get all properties from a node.
+        """Look up a node from its path
 
         Args:
-            node: Full path to node name to look in.
+            path: Path to look up, e.g. '/microcode/update@0'
-
         Returns:
-            A dictionary containing all the properties, indexed by node name.
+            Node object, or None if not found
-            The entries are Prop objects.
-
-        Raises:
-            ValueError: if the node does not exist.
         """
-        offset = libfdt.fdt_path_offset(self._fdt, node)
+        node = self._root
-        if offset < 0:
+        for part in path.split('/')[1:]:
-            libfdt.Raise(offset)
+            node = node._FindNode(part)
-        props_dict = {}
+            if not node:
-        poffset = libfdt.fdt_first_property_offset(self._fdt, offset)
+                return None
-        while poffset >= 0:
+        return node
-            dprop, plen = libfdt.fdt_get_property_by_offset(self._fdt, poffset)
-            prop = Prop(libfdt.String(self._fdt, dprop.nameoff), libfdt.Data(dprop))
-            props_dict[prop.name] = prop
 
-            poffset = libfdt.fdt_next_property_offset(self._fdt, poffset)
+    def Flush(self):
-        return props_dict
+        """Flush device tree changes back to the file
+
+        If the device tree has changed in memory, write it back to the file.
+        Subclasses can implement this if needed.
+        """
+        pass
+
+    def Pack(self):
+        """Pack the device tree down to its minimum size
+
+        When nodes and properties shrink or are deleted, wasted space can
+        build up in the device tree binary. Subclasses can implement this
+        to remove that spare space.
+        """
+        pass

tools/dtoc/fdt_fallback.py

@@ -7,6 +7,8 @@
 #
 
 import command
+import fdt
+from fdt import Fdt, NodeBase, PropBase
 import fdt_util
 import sys
 
@@ -17,7 +19,7 @@ import sys
 # is not very efficient for larger trees. The tool is called once for each
 # node and property in the tree.
 
-class Prop:
+class Prop(PropBase):
     """A device tree property
 
     Properties:
@@ -26,58 +28,17 @@ class Prop:
             bytes
         type: Value type
     """
-    def __init__(self, name, byte_list_str):
+    def __init__(self, node, name, byte_list_str):
-        self.name = name
+        PropBase.__init__(self, node, 0, name)
-        self.value = None
         if not byte_list_str.strip():
-            self.type = fdt_util.TYPE_BOOL
+            self.type = fdt.TYPE_BOOL
             return
-        bytes = [chr(int(byte, 16)) for byte in byte_list_str.strip().split(' ')]
+        self.bytes = [chr(int(byte, 16))
-        self.type, self.value = fdt_util.BytesToValue(''.join(bytes))
+                      for byte in byte_list_str.strip().split(' ')]
-
+        self.type, self.value = self.BytesToValue(''.join(self.bytes))
-    def GetPhandle(self):
-        """Get a (single) phandle value from a property
-
-        Gets the phandle valuie from a property and returns it as an integer
-        """
-        return fdt_util.fdt32_to_cpu(self.value[:4])
-
-    def Widen(self, newprop):
-        """Figure out which property type is more general
-
-        Given a current property and a new property, this function returns the
-        one that is less specific as to type. The less specific property will
-        be ble to represent the data in the more specific property. This is
-        used for things like:
-
-            node1 {
-                compatible = "fred";
-                value = <1>;
-            };
-            node1 {
-                compatible = "fred";
-                value = <1 2>;
-            };
-
-        He we want to use an int array for 'value'. The first property
-        suggests that a single int is enough, but the second one shows that
-        it is not. Calling this function with these two propertes would
-        update the current property to be like the second, since it is less
-        specific.
-        """
-        if newprop.type < self.type:
-            self.type = newprop.type
-
-        if type(newprop.value) == list and type(self.value) != list:
-            self.value = newprop.value
-
-        if type(self.value) == list and len(newprop.value) > len(self.value):
-            val = fdt_util.GetEmpty(self.type)
-            while len(self.value) < len(newprop.value):
-                self.value.append(val)
 
 
-class Node:
+class Node(NodeBase):
     """A device tree node
 
     Properties:
@@ -88,12 +49,8 @@ class Node:
         props: A dict of properties for this node, each a Prop object.
             Keyed by property name
     """
-    def __init__(self, fdt, name, path):
+    def __init__(self, fdt, offset, name, path):
-        self.name = name
+        NodeBase.__init__(self, fdt, offset, name, path)
-        self.path = path
-        self._fdt = fdt
-        self.subnodes = []
-        self.props = {}
 
     def Scan(self):
         """Scan a node's properties and subnodes
@@ -102,44 +59,42 @@ class Node:
         searching into subnodes so that the entire tree is built.
         """
         for name, byte_list_str in self._fdt.GetProps(self.path).iteritems():
-            prop = Prop(name, byte_list_str)
+            prop = Prop(self, name, byte_list_str)
             self.props[name] = prop
 
         for name in self._fdt.GetSubNodes(self.path):
             sep = '' if self.path[-1] == '/' else '/'
             path = self.path + sep + name
-            node = Node(self._fdt, name, path)
+            node = Node(self._fdt, 0, name, path)
             self.subnodes.append(node)
 
             node.Scan()
 
+    def DeleteProp(self, prop_name):
+        """Delete a property of a node
 
-class Fdt:
+        The property is deleted using fdtput.
-    """Provides simple access to a flat device tree blob.
+
+        Args:
+            prop_name: Name of the property to delete
+        Raises:
+            CommandError if the property does not exist
+        """
+        args = [self._fdt._fname, '-d', self.path, prop_name]
+        command.Output('fdtput', *args)
+        del self.props[prop_name]
+
+class FdtFallback(Fdt):
+    """Provides simple access to a flat device tree blob using fdtget/fdtput
 
     Properties:
-      fname: Filename of fdt
+        See superclass
-      _root: Root of device tree (a Node object)
     """
 
     def __init__(self, fname):
-        self.fname = fname
+        Fdt.__init__(self, fname)
-
+        if self._fname:
-    def Scan(self):
+            self._fname = fdt_util.EnsureCompiled(self._fname)
-        """Scan a device tree, building up a tree of Node objects
-
-        This fills in the self._root property
-        """
-        self._root = Node(self, '/', '/')
-        self._root.Scan()
-
-    def GetRoot(self):
-        """Get the root Node of the device tree
-
-        Returns:
-            The root Node object
-        """
-        return self._root
 
     def GetSubNodes(self, node):
         """Returns a list of sub-nodes of a given node
@@ -153,15 +108,14 @@ class Fdt:
         Raises:
             CmdError: if the node does not exist.
         """
-        out = command.Output('fdtget', self.fname, '-l', node)
+        out = command.Output('fdtget', self._fname, '-l', node)
         return out.strip().splitlines()
 
-    def GetProps(self, node, convert_dashes=False):
+    def GetProps(self, node):
         """Get all properties from a node
 
         Args:
             node: full path to node name to look in
-            convert_dashes: True to convert - to _ in node names
 
         Returns:
             A dictionary containing all the properties, indexed by node name.
@@ -171,13 +125,11 @@ class Fdt:
         Raises:
             CmdError: if the node does not exist.
         """
-        out = command.Output('fdtget', self.fname, node, '-p')
+        out = command.Output('fdtget', self._fname, node, '-p')
         props = out.strip().splitlines()
         props_dict = {}
         for prop in props:
             name = prop
-            if convert_dashes:
-                prop = re.sub('-', '_', prop)
             props_dict[prop] = self.GetProp(node, name)
         return props_dict
 
@@ -204,10 +156,26 @@ class Fdt:
         Raises:
             CmdError: if the property does not exist and no default is provided.
         """
-        args = [self.fname, node, prop, '-t', 'bx']
+        args = [self._fname, node, prop, '-t', 'bx']
         if default is not None:
           args += ['-d', str(default)]
         if typespec is not None:
           args += ['-t%s' % typespec]
         out = command.Output('fdtget', *args)
         return out.strip()
+
+    @classmethod
+    def Node(self, fdt, offset, name, path):
+        """Create a new node
+
+        This is used by Fdt.Scan() to create a new node using the correct
+        class.
+
+        Args:
+            fdt: Fdt object
+            offset: Offset of node
+            name: Node name
+            path: Full path to node
+        """
+        node = Node(fdt, offset, name, path)
+        return node

tools/dtoc/fdt_normal.py

@@ -0,0 +1,228 @@
+#!/usr/bin/python
+#
+# Copyright (C) 2016 Google, Inc
+# Written by Simon Glass <sjg@chromium.org>
+#
+# SPDX-License-Identifier:      GPL-2.0+
+#
+
+import struct
+import sys
+
+import fdt
+from fdt import Fdt, NodeBase, PropBase
+import fdt_util
+import libfdt
+
+# This deals with a device tree, presenting it as a list of Node and Prop
+# objects, representing nodes and properties, respectively.
+#
+# This implementation uses a libfdt Python library to access the device tree,
+# so it is fairly efficient.
+
+def CheckErr(errnum, msg):
+    if errnum:
+        raise ValueError('Error %d: %s: %s' %
+            (errnum, libfdt.fdt_strerror(errnum), msg))
+
+class Prop(PropBase):
+    """A device tree property
+
+    Properties:
+        name: Property name (as per the device tree)
+        value: Property value as a string of bytes, or a list of strings of
+            bytes
+        type: Value type
+    """
+    def __init__(self, node, offset, name, bytes):
+        PropBase.__init__(self, node, offset, name)
+        self.bytes = bytes
+        if not bytes:
+            self.type = fdt.TYPE_BOOL
+            self.value = True
+            return
+        self.type, self.value = self.BytesToValue(bytes)
+
+    def GetOffset(self):
+        """Get the offset of a property
+
+        Returns:
+            The offset of the property (struct fdt_property) within the file
+        """
+        return self._node._fdt.GetStructOffset(self._offset)
+
+class Node(NodeBase):
+    """A device tree node
+
+    Properties:
+        offset: Integer offset in the device tree
+        name: Device tree node tname
+        path: Full path to node, along with the node name itself
+        _fdt: Device tree object
+        subnodes: A list of subnodes for this node, each a Node object
+        props: A dict of properties for this node, each a Prop object.
+            Keyed by property name
+    """
+    def __init__(self, fdt, offset, name, path):
+        NodeBase.__init__(self, fdt, offset, name, path)
+
+    def Offset(self):
+        """Returns the offset of a node, after checking the cache
+
+        This should be used instead of self._offset directly, to ensure that
+        the cache does not contain invalid offsets.
+        """
+        self._fdt.CheckCache()
+        return self._offset
+
+    def Scan(self):
+        """Scan a node's properties and subnodes
+
+        This fills in the props and subnodes properties, recursively
+        searching into subnodes so that the entire tree is built.
+        """
+        self.props = self._fdt.GetProps(self, self.path)
+
+        offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self.Offset())
+        while offset >= 0:
+            sep = '' if self.path[-1] == '/' else '/'
+            name = libfdt.Name(self._fdt.GetFdt(), offset)
+            path = self.path + sep + name
+            node = Node(self._fdt, offset, name, path)
+            self.subnodes.append(node)
+
+            node.Scan()
+            offset = libfdt.fdt_next_subnode(self._fdt.GetFdt(), offset)
+
+    def Refresh(self, my_offset):
+        """Fix up the _offset for each node, recursively
+
+        Note: This does not take account of property offsets - these will not
+        be updated.
+        """
+        if self._offset != my_offset:
+            #print '%s: %d -> %d\n' % (self.path, self._offset, my_offset)
+            self._offset = my_offset
+        offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self._offset)
+        for subnode in self.subnodes:
+            subnode.Refresh(offset)
+            offset = libfdt.fdt_next_subnode(self._fdt.GetFdt(), offset)
+
+    def DeleteProp(self, prop_name):
+        """Delete a property of a node
+
+        The property is deleted and the offset cache is invalidated.
+
+        Args:
+            prop_name: Name of the property to delete
+        Raises:
+            ValueError if the property does not exist
+        """
+        CheckErr(libfdt.fdt_delprop(self._fdt.GetFdt(), self.Offset(), prop_name),
+                 "Node '%s': delete property: '%s'" % (self.path, prop_name))
+        del self.props[prop_name]
+        self._fdt.Invalidate()
+
+class FdtNormal(Fdt):
+    """Provides simple access to a flat device tree blob using libfdt.
+
+    Properties:
+        _fdt: Device tree contents (bytearray)
+        _cached_offsets: True if all the nodes have a valid _offset property,
+            False if something has changed to invalidate the offsets
+    """
+    def __init__(self, fname):
+        Fdt.__init__(self, fname)
+        self._cached_offsets = False
+        if self._fname:
+            self._fname = fdt_util.EnsureCompiled(self._fname)
+
+            with open(self._fname) as fd:
+                self._fdt = bytearray(fd.read())
+
+    def GetFdt(self):
+        """Get the contents of the FDT
+
+        Returns:
+            The FDT contents as a string of bytes
+        """
+        return self._fdt
+
+    def Flush(self):
+        """Flush device tree changes back to the file"""
+        with open(self._fname, 'wb') as fd:
+            fd.write(self._fdt)
+
+    def Pack(self):
+        """Pack the device tree down to its minimum size"""
+        CheckErr(libfdt.fdt_pack(self._fdt), 'pack')
+        fdt_len = libfdt.fdt_totalsize(self._fdt)
+        del self._fdt[fdt_len:]
+
+    def GetProps(self, node, path):
+        """Get all properties from a node.
+
+        Args:
+            node: Full path to node name to look in.
+
+        Returns:
+            A dictionary containing all the properties, indexed by node name.
+            The entries are Prop objects.
+
+        Raises:
+            ValueError: if the node does not exist.
+        """
+        offset = libfdt.fdt_path_offset(self._fdt, path)
+        if offset < 0:
+            libfdt.Raise(offset)
+        props_dict = {}
+        poffset = libfdt.fdt_first_property_offset(self._fdt, offset)
+        while poffset >= 0:
+            dprop, plen = libfdt.fdt_get_property_by_offset(self._fdt, poffset)
+            prop = Prop(node, poffset, libfdt.String(self._fdt, dprop.nameoff),
+                        libfdt.Data(dprop))
+            props_dict[prop.name] = prop
+
+            poffset = libfdt.fdt_next_property_offset(self._fdt, poffset)
+        return props_dict
+
+    def Invalidate(self):
+        """Mark our offset cache as invalid"""
+        self._cached_offsets = False
+
+    def CheckCache(self):
+        """Refresh the offset cache if needed"""
+        if self._cached_offsets:
+            return
+        self.Refresh()
+        self._cached_offsets = True
+
+    def Refresh(self):
+        """Refresh the offset cache"""
+        self._root.Refresh(0)
+
+    def GetStructOffset(self, offset):
+        """Get the file offset of a given struct offset
+
+        Args:
+            offset: Offset within the 'struct' region of the device tree
+        Returns:
+            Position of @offset within the device tree binary
+        """
+        return libfdt.fdt_off_dt_struct(self._fdt) + offset
+
+    @classmethod
+    def Node(self, fdt, offset, name, path):
+        """Create a new node
+
+        This is used by Fdt.Scan() to create a new node using the correct
+        class.
+
+        Args:
+            fdt: Fdt object
+            offset: Offset of node
+            name: Node name
+            path: Full path to node
+        """
+        node = Node(fdt, offset, name, path)
+        return node

tools/dtoc/fdt_select.py

@@ -0,0 +1,26 @@
+#!/usr/bin/python
+#
+# Copyright (C) 2016 Google, Inc
+# Written by Simon Glass <sjg@chromium.org>
+#
+# SPDX-License-Identifier:      GPL-2.0+
+#
+
+# Bring in either the normal fdt library (which relies on libfdt) or the
+# fallback one (which uses fdtget and is slower). Both provide the same
+# interface for this file to use.
+try:
+    import fdt_normal
+    have_libfdt = True
+except ImportError:
+    have_libfdt = False
+    import fdt_fallback
+
+def FdtScan(fname):
+    """Returns a new Fdt object from the implementation we are using"""
+    if have_libfdt:
+        dtb = fdt_normal.FdtNormal(fname)
+    else:
+        dtb = fdt_fallback.FdtFallback(fname)
+    dtb.Scan()
+    return dtb

tools/dtoc/fdt_util.py

@@ -6,73 +6,12 @@
 # SPDX-License-Identifier:      GPL-2.0+
 #
 
+import os
 import struct
+import tempfile
 
-# A list of types we support
+import command
-(TYPE_BYTE, TYPE_INT, TYPE_STRING, TYPE_BOOL) = range(4)
+import tools
-
-def BytesToValue(bytes):
-    """Converts a string of bytes into a type and value
-
-    Args:
-        A string containing bytes
-
-    Return:
-        A tuple:
-            Type of data
-            Data, either a single element or a list of elements. Each element
-            is one of:
-                TYPE_STRING: string value from the property
-                TYPE_INT: a byte-swapped integer stored as a 4-byte string
-                TYPE_BYTE: a byte stored as a single-byte string
-    """
-    size = len(bytes)
-    strings = bytes.split('\0')
-    is_string = True
-    count = len(strings) - 1
-    if count > 0 and not strings[-1]:
-        for string in strings[:-1]:
-            if not string:
-                is_string = False
-                break
-            for ch in string:
-                if ch < ' ' or ch > '~':
-                    is_string = False
-                    break
-    else:
-        is_string = False
-    if is_string:
-        if count == 1:
-            return TYPE_STRING, strings[0]
-        else:
-            return TYPE_STRING, strings[:-1]
-    if size % 4:
-        if size == 1:
-            return TYPE_BYTE, bytes[0]
-        else:
-            return TYPE_BYTE, list(bytes)
-    val = []
-    for i in range(0, size, 4):
-        val.append(bytes[i:i + 4])
-    if size == 4:
-        return TYPE_INT, val[0]
-    else:
-        return TYPE_INT, val
-
-def GetEmpty(type):
-    """Get an empty / zero value of the given type
-
-    Returns:
-        A single value of the given type
-    """
-    if type == TYPE_BYTE:
-        return chr(0)
-    elif type == TYPE_INT:
-        return struct.pack('<I', 0);
-    elif type == TYPE_STRING:
-        return ''
-    else:
-        return True
 
 def fdt32_to_cpu(val):
     """Convert a device tree cell to an integer
@@ -83,4 +22,65 @@ def fdt32_to_cpu(val):
     Return:
         A native-endian integer value
     """
-    return struct.unpack(">I", val)[0]
+    return struct.unpack('>I', val)[0]
+
+def EnsureCompiled(fname):
+    """Compile an fdt .dts source file into a .dtb binary blob if needed.
+
+    Args:
+        fname: Filename (if .dts it will be compiled). It not it will be
+            left alone
+
+    Returns:
+        Filename of resulting .dtb file
+    """
+    _, ext = os.path.splitext(fname)
+    if ext != '.dts':
+        return fname
+
+    dts_input = tools.GetOutputFilename('source.dts')
+    dtb_output = tools.GetOutputFilename('source.dtb')
+
+    search_paths = [os.path.join(os.getcwd(), 'include')]
+    root, _ = os.path.splitext(fname)
+    args = ['-E', '-P', '-x', 'assembler-with-cpp', '-D__ASSEMBLY__']
+    args += ['-Ulinux']
+    for path in search_paths:
+        args.extend(['-I', path])
+    args += ['-o', dts_input, fname]
+    command.Run('cc', *args)
+
+    # If we don't have a directory, put it in the tools tempdir
+    search_list = []
+    for path in search_paths:
+        search_list.extend(['-i', path])
+    args = ['-I', 'dts', '-o', dtb_output, '-O', 'dtb']
+    args.extend(search_list)
+    args.append(dts_input)
+    command.Run('dtc', *args)
+    return dtb_output
+
+def GetInt(node, propname, default=None):
+    prop = node.props.get(propname)
+    if not prop:
+        return default
+    value = fdt32_to_cpu(prop.value)
+    if type(value) == type(list):
+        raise ValueError("Node '%s' property '%' has list value: expecting"
+                         "a single integer" % (node.name, propname))
+    return value
+
+def GetString(node, propname, default=None):
+    prop = node.props.get(propname)
+    if not prop:
+        return default
+    value = prop.value
+    if type(value) == type(list):
+        raise ValueError("Node '%s' property '%' has list value: expecting"
+                         "a single string" % (node.name, propname))
+    return value
+
+def GetBool(node, propname, default=False):
+    if propname in node.props:
+        return True
+    return default

tools/patman/checkpatch.py

@@ -63,7 +63,8 @@ def CheckPatch(fname, verbose=False):
     result.problems = []
     chk = FindCheckPatch()
     item = {}
-    result.stdout = command.Output(chk, '--no-tree', fname)
+    result.stdout = command.Output(chk, '--no-tree', fname,
+                                   raise_on_error=False)
     #pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE)
     #stdout, stderr = pipe.communicate()
 

tools/patman/command.py

@@ -104,8 +104,9 @@ def RunPipe(pipe_list, infile=None, outfile=None,
         raise Exception("Error running '%s'" % user_pipestr)
     return result
 
-def Output(*cmd):
+def Output(*cmd, **kwargs):
-    return RunPipe([cmd], capture=True, raise_on_error=False).stdout
+    raise_on_error = kwargs.get('raise_on_error', True)
+    return RunPipe([cmd], capture=True, raise_on_error=raise_on_error).stdout
 
 def OutputOneLine(*cmd, **kwargs):
     raise_on_error = kwargs.pop('raise_on_error', True)

tools/patman/gitutil.py

@@ -391,7 +391,8 @@ def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname,
     """
     to = BuildEmailList(series.get('to'), '--to', alias, raise_on_error)
     if not to:
-        git_config_to = command.Output('git', 'config', 'sendemail.to')
+        git_config_to = command.Output('git', 'config', 'sendemail.to',
+                                       raise_on_error=False)
         if not git_config_to:
             print ("No recipient.\n"
                    "Please add something like this to a commit\n"

tools/patman/tools.py

@@ -0,0 +1,120 @@
+#
+# Copyright (c) 2016 Google, Inc
+#
+# SPDX-License-Identifier:      GPL-2.0+
+#
+
+import os
+import shutil
+import tempfile
+
+import tout
+
+outdir = None
+indirs = None
+preserve_outdir = False
+
+def PrepareOutputDir(dirname, preserve=False):
+    """Select an output directory, ensuring it exists.
+
+    This either creates a temporary directory or checks that the one supplied
+    by the user is valid. For a temporary directory, it makes a note to
+    remove it later if required.
+
+    Args:
+        dirname: a string, name of the output directory to use to store
+                intermediate and output files. If is None - create a temporary
+                directory.
+        preserve: a Boolean. If outdir above is None and preserve is False, the
+                created temporary directory will be destroyed on exit.
+
+    Raises:
+        OSError: If it cannot create the output directory.
+    """
+    global outdir, preserve_outdir
+
+    preserve_outdir = dirname or preserve
+    if dirname:
+        outdir = dirname
+        if not os.path.isdir(outdir):
+            try:
+                os.makedirs(outdir)
+            except OSError as err:
+                raise CmdError("Cannot make output directory '%s': '%s'" %
+                                (outdir, err.strerror))
+        tout.Debug("Using output directory '%s'" % outdir)
+    else:
+        outdir = tempfile.mkdtemp(prefix='binman.')
+        tout.Debug("Using temporary directory '%s'" % outdir)
+
+def _RemoveOutputDir():
+    global outdir
+
+    shutil.rmtree(outdir)
+    tout.Debug("Deleted temporary directory '%s'" % outdir)
+    outdir = None
+
+def FinaliseOutputDir():
+    global outdir, preserve_outdir
+
+    """Tidy up: delete output directory if temporary and not preserved."""
+    if outdir and not preserve_outdir:
+        _RemoveOutputDir()
+
+def GetOutputFilename(fname):
+    """Return a filename within the output directory.
+
+    Args:
+        fname: Filename to use for new file
+
+    Returns:
+        The full path of the filename, within the output directory
+    """
+    return os.path.join(outdir, fname)
+
+def _FinaliseForTest():
+    """Remove the output directory (for use by tests)"""
+    global outdir
+
+    if outdir:
+        _RemoveOutputDir()
+
+def SetInputDirs(dirname):
+    """Add a list of input directories, where input files are kept.
+
+    Args:
+        dirname: a list of paths to input directories to use for obtaining
+                files needed by binman to place in the image.
+    """
+    global indir
+
+    indir = dirname
+    tout.Debug("Using input directories %s" % indir)
+
+def GetInputFilename(fname):
+    """Return a filename for use as input.
+
+    Args:
+        fname: Filename to use for new file
+
+    Returns:
+        The full path of the filename, within the input directory
+    """
+    if not indir:
+        return fname
+    for dirname in indir:
+        pathname = os.path.join(dirname, fname)
+        if os.path.exists(pathname):
+            return pathname
+
+    raise ValueError("Filename '%s' not found in input path (%s)" %
+                     (fname, ','.join(indir)))
+
+def Align(pos, align):
+    if align:
+        mask = align - 1
+        pos = (pos + mask) & ~mask
+    return pos
+
+def NotPowerOfTwo(num):
+    return num and (num & (num - 1))

tools/patman/tout.py

@@ -0,0 +1,166 @@
+# Copyright (c) 2016 Google, Inc
+#
+# SPDX-License-Identifier:            GPL-2.0+
+#
+# Terminal output logging.
+#
+
+import sys
+
+import terminal
+
+# Output verbosity levels that we support
+ERROR = 0
+WARNING = 1
+NOTICE = 2
+INFO = 3
+DEBUG = 4
+
+"""
+This class handles output of progress and other useful information
+to the user. It provides for simple verbosity level control and can
+output nothing but errors at verbosity zero.
+
+The idea is that modules set up an Output object early in their years and pass
+it around to other modules that need it. This keeps the output under control
+of a single class.
+
+Public properties:
+    verbose: Verbosity level: 0=silent, 1=progress, 3=full, 4=debug
+"""
+def __enter__():
+    return
+
+def __exit__(unused1, unused2, unused3):
+    """Clean up and remove any progress message."""
+    ClearProgress()
+    return False
+
+def UserIsPresent():
+    """This returns True if it is likely that a user is present.
+
+    Sometimes we want to prompt the user, but if no one is there then this
+    is a waste of time, and may lock a script which should otherwise fail.
+
+    Returns:
+        True if it thinks the user is there, and False otherwise
+    """
+    return stdout_is_tty and verbose > 0
+
+def ClearProgress():
+    """Clear any active progress message on the terminal."""
+    if verbose > 0 and stdout_is_tty:
+        _stdout.write('\r%s\r' % (" " * len (_progress)))
+        _stdout.flush()
+
+def Progress(msg, warning=False, trailer='...'):
+    """Display progress information.
+
+    Args:
+        msg: Message to display.
+        warning: True if this is a warning."""
+    ClearProgress()
+    if verbose > 0:
+        _progress = msg + trailer
+        if stdout_is_tty:
+            col = _color.YELLOW if warning else _color.GREEN
+            _stdout.write('\r' + _color.Color(col, _progress))
+            _stdout.flush()
+        else:
+            _stdout.write(_progress + '\n')
+
+def _Output(level, msg, color=None):
+    """Output a message to the terminal.
+
+    Args:
+        level: Verbosity level for this message. It will only be displayed if
+                this as high as the currently selected level.
+        msg; Message to display.
+        error: True if this is an error message, else False.
+    """
+    if verbose >= level:
+        ClearProgress()
+        if color:
+            msg = _color.Color(color, msg)
+        _stdout.write(msg + '\n')
+
+def DoOutput(level, msg):
+    """Output a message to the terminal.
+
+    Args:
+        level: Verbosity level for this message. It will only be displayed if
+                this as high as the currently selected level.
+        msg; Message to display.
+    """
+    _Output(level, msg)
+
+def Error(msg):
+    """Display an error message
+
+    Args:
+        msg; Message to display.
+    """
+    _Output(0, msg, _color.RED)
+
+def Warning(msg):
+    """Display a warning message
+
+    Args:
+        msg; Message to display.
+    """
+    _Output(1, msg, _color.YELLOW)
+
+def Notice(msg):
+    """Display an important infomation message
+
+    Args:
+        msg; Message to display.
+    """
+    _Output(2, msg)
+
+def Info(msg):
+    """Display an infomation message
+
+    Args:
+        msg; Message to display.
+    """
+    _Output(3, msg)
+
+def Debug(msg):
+    """Display a debug message
+
+    Args:
+        msg; Message to display.
+    """
+    _Output(4, msg)
+
+def UserOutput(msg):
+    """Display a message regardless of the current output level.
+
+    This is used when the output was specifically requested by the user.
+    Args:
+        msg; Message to display.
+    """
+    _Output(0, msg)
+
+def Init(_verbose=WARNING, stdout=sys.stdout):
+    """Initialize a new output object.
+
+    Args:
+        verbose: Verbosity level (0-4).
+        stdout: File to use for stdout.
+    """
+    global verbose, _progress, _color, _stdout, stdout_is_tty
+
+    verbose = _verbose
+    _progress = ''                    # Our last progress message
+    _color = terminal.Color()
+    _stdout = stdout
+
+    # TODO(sjg): Move this into Chromite libraries when we have them
+    stdout_is_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
+
+def Uninit():
+    ClearProgress()
+
+Init()