scripts: create meta-tool package, "west"

We have agreed to develop a meta-tool named "west", which will be a
swiss-army knife of Zephyr development. It will support use cases like
building, flashing and debugging; bootloader integration; emulator
support; and integration with multiple git repositories.

The basic usage for the tool is similar to git(1):

    west [common opts] <command-name> [command opts] [<command args>]

There are common options, such as verbosity control, followed by a
mandatory sub-command. The sub-command then takes its own options and
arguments.

This patch adds the basic framework for this tool, as follows:

- a Python 3 package named 'west', in scripts/meta. There is no PyPI
  integration for now; the tool will be improving quickly, so we need
  to keep users up to date by having it in tree.
- an main entry point, main.py, and a package-level shim, __main__.py
- a cmd subpackage, which defines the abstract base class for commands
- logging (log.py)
- catch-all utilities (util.py)
- Windows and Unix launchers so users can type "west" to run the tool
  after sourcing the appropriate zephyr-env script for their
  environment.

Subsequent patches will start to add individual commands.

Signed-off-by: Marti Bolivar <marti@opensourcefoundries.com>

scripts/meta/west/__init__.py

@@ -0,0 +1,5 @@
+# Copyright 2018 Open Source Foundries Limited.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Nothing here for now.

scripts/meta/west/__main__.py

@@ -0,0 +1,12 @@
+# Copyright 2018 Open Source Foundries Limited.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+'''Zephyr RTOS meta-tool (west)
+'''
+
+from .main import main
+import sys
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

scripts/meta/west/cmd/__init__.py

@@ -0,0 +1,74 @@
+# Copyright 2018 Open Source Foundries Limited.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+'''West's commands subpackage.
+
+All commands should be implemented within modules in this package.
+'''
+
+from abc import ABC, abstractmethod
+
+__all__ = ['CommandContextError', 'WestCommand']
+
+
+class CommandContextError(RuntimeError):
+    '''Indicates that a context-dependent command could not be run.'''
+
+
+class WestCommand(ABC):
+    '''Abstract superclass for a west command.
+
+    All top-level commands supported by west implement this interface.'''
+
+    def __init__(self, name, description, accepts_unknown_args=False):
+        '''Create a command instance.
+
+        `name`: the command's name, as entered by the user.
+        `description`: one-line command description to show to the user.
+
+        `accepts_unknown_args`: if true, the command can handle
+        arbitrary unknown command line arguments in its run()
+        method. Otherwise, passing unknown arguments will cause
+        UnknownArgumentsError to be raised.
+        '''
+        self.name = name
+        self.description = description
+        self._accept_unknown = accepts_unknown_args
+
+    def run(self, args, unknown):
+        '''Run the command.
+
+        `args`: known arguments parsed via `register_arguments()`
+        `unknown`: unknown arguments present on the command line
+        '''
+        if unknown and not self._accept_unknown:
+            self.parser.error('unexpected arguments: {}'.format(unknown))
+        self.do_run(args, unknown)
+
+    def add_parser(self, parser_adder):
+        '''Registers a parser for this command, and returns it.
+        '''
+        self.parser = self.do_add_parser(parser_adder)
+        return self.parser
+
+    #
+    # Mandatory subclass hooks
+    #
+
+    @abstractmethod
+    def do_add_parser(self, parser_adder):
+        '''Subclass method for registering command line arguments.
+
+        `parser_adder` is an argparse argument subparsers adder.'''
+
+    @abstractmethod
+    def do_run(self, args, unknown):
+        '''Subclasses must implement; called when the command is run.
+
+        `args` is the namespace of parsed known arguments.
+
+        If `accepts_unknown_args` was False when constructing this
+        object, `unknown` will be empty. Otherwise, it is an iterable
+        containing all unknown arguments present on the command line.
+        '''

scripts/meta/west/log.py

@@ -0,0 +1,66 @@
+# Copyright 2018 Open Source Foundries Limited.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+'''Logging module for west
+
+Provides common methods for logging messages to display to the user.'''
+
+import sys
+
+VERBOSE_NONE = 0
+'''Base verbosity level (zero), no verbose messages printed.'''
+
+VERBOSE_NORMAL = 1
+'''Base verbosity level, some verbose messages printed.'''
+
+VERBOSE_VERY = 2
+'''Very verbose output messages will be printed.'''
+
+VERBOSE_EXTREME = 3
+'''Extremely verbose output messages will be printed.'''
+
+VERBOSE = VERBOSE_NONE
+'''Global verbosity level. VERBOSE_NONE is the default.'''
+
+
+def set_verbosity(value):
+    '''Set the logging verbosity level.'''
+    global VERBOSE
+    VERBOSE = int(value)
+
+
+def dbg(*args, level=VERBOSE_NORMAL):
+    '''Print a verbose debug logging message.
+
+    The message is only printed if level is at least the current
+    verbosity level.'''
+    if level > VERBOSE:
+        return
+    print(*args)
+
+
+def inf(*args):
+    '''Print an informational message.'''
+    print(*args)
+
+
+def wrn(*args):
+    '''Print a warning.'''
+    print('warning:', end=' ', file=sys.stderr, flush=False)
+    print(*args, file=sys.stderr)
+
+
+def err(*args, fatal=False):
+    '''Print an error.'''
+    if fatal:
+        print('fatal', end=' ', file=sys.stderr, flush=False)
+    print('error:', end=' ', file=sys.stderr, flush=False)
+    print(*args, file=sys.stderr)
+
+
+def die(*args, exit_code=1):
+    '''Print a fatal error, and abort with the given exit code.'''
+    print('fatal error:', end=' ', file=sys.stderr, flush=False)
+    print(*args, file=sys.stderr)
+    sys.exit(exit_code)

scripts/meta/west/main.py

@@ -0,0 +1,109 @@
+# Copyright 2018 Open Source Foundries Limited.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+'''Zephyr RTOS meta-tool (west) main module
+'''
+
+
+import argparse
+from functools import partial
+import os
+import sys
+from subprocess import CalledProcessError
+
+from . import log
+from .cmd import CommandContextError
+from .util import quote_sh_list
+
+
+COMMANDS = ()
+'''Supported top-level commands.'''
+
+
+class InvalidWestContext(RuntimeError):
+    pass
+
+
+def command_handler(command, known_args, unknown_args):
+    command.run(known_args, unknown_args)
+
+
+def validate_context(args, unknown):
+    '''Validate the run-time context expected by west.'''
+    if args.zephyr_base:
+        os.environ['ZEPHYR_BASE'] = args.zephyr_base
+    else:
+        if 'ZEPHYR_BASE' not in os.environ:
+            raise InvalidWestContext(
+                '--zephyr-base missing and no ZEPHYR_BASE '
+                'in the environment')
+        else:
+            args.zephyr_base = os.environ['ZEPHYR_BASE']
+
+
+def parse_args(argv):
+    west_parser = argparse.ArgumentParser(
+        prog='west', description='The Zephyr RTOS meta-tool.',
+        epilog='Run "west <command> -h" for help on each command.')
+    west_parser.add_argument('-z', '--zephyr-base', default=None,
+                             help='''Path to the Zephyr base directory. If not
+                             given, ZEPHYR_BASE must be defined in the
+                             environment, and will be used instead.''')
+    west_parser.add_argument('-v', '--verbose', default=0, action='count',
+                             help='''Display verbose output. May be given
+                             multiple times to increase verbosity.''')
+    subparser_gen = west_parser.add_subparsers(title='commands',
+                                               dest='command')
+
+    for command in COMMANDS:
+        parser = command.add_parser(subparser_gen)
+        parser.set_defaults(handler=partial(command_handler, command))
+
+    args, unknown = west_parser.parse_known_args(args=argv)
+
+    # Set up logging verbosity before doing anything else, so
+    # e.g. verbose messages related to argument handling errors
+    # work properly.
+    log.set_verbosity(args.verbose)
+
+    try:
+        validate_context(args, unknown)
+    except InvalidWestContext as iwc:
+        log.err(*iwc.args, fatal=True)
+        west_parser.print_usage(file=sys.stderr)
+        sys.exit(1)
+
+    if 'handler' not in args:
+        log.err('you must specify a command', fatal=True)
+        west_parser.print_usage(file=sys.stderr)
+        sys.exit(1)
+
+    return args, unknown
+
+
+def main(argv):
+    args, unknown = parse_args(argv)
+
+    for_stack_trace = 'run as "west -v ... {} ..." for a stack trace'.format(
+        args.command)
+    try:
+        args.handler(args, unknown)
+    except KeyboardInterrupt:
+        sys.exit(0)
+    except CalledProcessError as cpe:
+        log.err('command exited with status {}: {}'.format(
+            cpe.args[0], quote_sh_list(cpe.args[1])))
+        if args.verbose:
+            raise
+        else:
+            log.inf(for_stack_trace)
+    except CommandContextError as cce:
+        log.die('command', args.command, 'cannot be run in this context:',
+                *cce.args)
+    except Exception as exc:
+        log.err(*exc.args, fatal=True)
+        if args.verbose:
+            raise
+        else:
+            log.inf(for_stack_trace)

scripts/meta/west/util.py

@@ -0,0 +1,22 @@
+# Copyright 2018 Open Source Foundries Limited.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+'''Miscellaneous utilities used by west
+'''
+
+import shlex
+import textwrap
+
+
+def quote_sh_list(cmd):
+    '''Transform a command from list into shell string form.'''
+    fmt = ' '.join('{}' for _ in cmd)
+    args = [shlex.quote(s) for s in cmd]
+    return fmt.format(*args)
+
+
+def wrap(text, indent):
+    '''Convenience routine for wrapping text to a consistent indent.'''
+    return textwrap.wrap(text, initial_indent=indent,
+                         subsequent_indent=indent)

scripts/west

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# UNIX operating system entry point to the west tool.
+export "PYTHONPATH=${PYTHONPATH:+${PYTHONPATH}:}$ZEPHYR_BASE/scripts/meta"
+python3 -m west $@

scripts/west-win.py

@@ -0,0 +1,11 @@
+# Windows-specific launcher alias for west (west wind?).
+
+import os
+import sys
+
+zephyr_base = os.environ['ZEPHYR_BASE']
+sys.path.append(os.path.join(zephyr_base, 'scripts', 'meta'))
+
+from west.main import main  # noqa E402 (silence flake8 warning)
+
+main(sys.argv[1:])

zephyr-env.cmd

@@ -4,3 +4,6 @@ set ZEPHYR_BASE=%~dp0
 if exist "%userprofile%\zephyrrc.cmd" (
 	call "%userprofile%\zephyrrc.cmd"
 )
+
+rem Zephyr meta-tool (west) launcher alias
+doskey west=py -3 %ZEPHYR_BASE%\scripts\west-win.py $*