init

object/object_pkg/install/_local_setup_util_sh.py

@@ -0,0 +1,407 @@
+# Copyright 2016-2019 Dirk Thomas
+# Licensed under the Apache License, Version 2.0
+
+import argparse
+from collections import OrderedDict
+import os
+from pathlib import Path
+import sys
+
+
+FORMAT_STR_COMMENT_LINE = '# {comment}'
+FORMAT_STR_SET_ENV_VAR = 'export {name}="{value}"'
+FORMAT_STR_USE_ENV_VAR = '${name}'
+FORMAT_STR_INVOKE_SCRIPT = 'COLCON_CURRENT_PREFIX="{prefix}" _colcon_prefix_sh_source_script "{script_path}"'  # noqa: E501
+FORMAT_STR_REMOVE_LEADING_SEPARATOR = 'if [ "$(echo -n ${name} | head -c 1)" = ":" ]; then export {name}=${{{name}#?}} ; fi'  # noqa: E501
+FORMAT_STR_REMOVE_TRAILING_SEPARATOR = 'if [ "$(echo -n ${name} | tail -c 1)" = ":" ]; then export {name}=${{{name}%?}} ; fi'  # noqa: E501
+
+DSV_TYPE_APPEND_NON_DUPLICATE = 'append-non-duplicate'
+DSV_TYPE_PREPEND_NON_DUPLICATE = 'prepend-non-duplicate'
+DSV_TYPE_PREPEND_NON_DUPLICATE_IF_EXISTS = 'prepend-non-duplicate-if-exists'
+DSV_TYPE_SET = 'set'
+DSV_TYPE_SET_IF_UNSET = 'set-if-unset'
+DSV_TYPE_SOURCE = 'source'
+
+
+def main(argv=sys.argv[1:]):  # noqa: D103
+    parser = argparse.ArgumentParser(
+        description='Output shell commands for the packages in topological '
+                    'order')
+    parser.add_argument(
+        'primary_extension',
+        help='The file extension of the primary shell')
+    parser.add_argument(
+        'additional_extension', nargs='?',
+        help='The additional file extension to be considered')
+    parser.add_argument(
+        '--merged-install', action='store_true',
+        help='All install prefixes are merged into a single location')
+    args = parser.parse_args(argv)
+
+    packages = get_packages(Path(__file__).parent, args.merged_install)
+
+    ordered_packages = order_packages(packages)
+    for pkg_name in ordered_packages:
+        if _include_comments():
+            print(
+                FORMAT_STR_COMMENT_LINE.format_map(
+                    {'comment': 'Package: ' + pkg_name}))
+        prefix = os.path.abspath(os.path.dirname(__file__))
+        if not args.merged_install:
+            prefix = os.path.join(prefix, pkg_name)
+        for line in get_commands(
+            pkg_name, prefix, args.primary_extension,
+            args.additional_extension
+        ):
+            print(line)
+
+    for line in _remove_ending_separators():
+        print(line)
+
+
+def get_packages(prefix_path, merged_install):
+    """
+    Find packages based on colcon-specific files created during installation.
+
+    :param Path prefix_path: The install prefix path of all packages
+    :param bool merged_install: The flag if the packages are all installed
+      directly in the prefix or if each package is installed in a subdirectory
+      named after the package
+    :returns: A mapping from the package name to the set of runtime
+      dependencies
+    :rtype: dict
+    """
+    packages = {}
+    # since importing colcon_core isn't feasible here the following constant
+    # must match colcon_core.location.get_relative_package_index_path()
+    subdirectory = 'share/colcon-core/packages'
+    if merged_install:
+        # return if workspace is empty
+        if not (prefix_path / subdirectory).is_dir():
+            return packages
+        # find all files in the subdirectory
+        for p in (prefix_path / subdirectory).iterdir():
+            if not p.is_file():
+                continue
+            if p.name.startswith('.'):
+                continue
+            add_package_runtime_dependencies(p, packages)
+    else:
+        # for each subdirectory look for the package specific file
+        for p in prefix_path.iterdir():
+            if not p.is_dir():
+                continue
+            if p.name.startswith('.'):
+                continue
+            p = p / subdirectory / p.name
+            if p.is_file():
+                add_package_runtime_dependencies(p, packages)
+
+    # remove unknown dependencies
+    pkg_names = set(packages.keys())
+    for k in packages.keys():
+        packages[k] = {d for d in packages[k] if d in pkg_names}
+
+    return packages
+
+
+def add_package_runtime_dependencies(path, packages):
+    """
+    Check the path and if it exists extract the packages runtime dependencies.
+
+    :param Path path: The resource file containing the runtime dependencies
+    :param dict packages: A mapping from package names to the sets of runtime
+      dependencies to add to
+    """
+    content = path.read_text()
+    dependencies = set(content.split(os.pathsep) if content else [])
+    packages[path.name] = dependencies
+
+
+def order_packages(packages):
+    """
+    Order packages topologically.
+
+    :param dict packages: A mapping from package name to the set of runtime
+      dependencies
+    :returns: The package names
+    :rtype: list
+    """
+    # select packages with no dependencies in alphabetical order
+    to_be_ordered = list(packages.keys())
+    ordered = []
+    while to_be_ordered:
+        pkg_names_without_deps = [
+            name for name in to_be_ordered if not packages[name]]
+        if not pkg_names_without_deps:
+            reduce_cycle_set(packages)
+            raise RuntimeError(
+                'Circular dependency between: ' + ', '.join(sorted(packages)))
+        pkg_names_without_deps.sort()
+        pkg_name = pkg_names_without_deps[0]
+        to_be_ordered.remove(pkg_name)
+        ordered.append(pkg_name)
+        # remove item from dependency lists
+        for k in list(packages.keys()):
+            if pkg_name in packages[k]:
+                packages[k].remove(pkg_name)
+    return ordered
+
+
+def reduce_cycle_set(packages):
+    """
+    Reduce the set of packages to the ones part of the circular dependency.
+
+    :param dict packages: A mapping from package name to the set of runtime
+      dependencies which is modified in place
+    """
+    last_depended = None
+    while len(packages) > 0:
+        # get all remaining dependencies
+        depended = set()
+        for pkg_name, dependencies in packages.items():
+            depended = depended.union(dependencies)
+        # remove all packages which are not dependent on
+        for name in list(packages.keys()):
+            if name not in depended:
+                del packages[name]
+        if last_depended:
+            # if remaining packages haven't changed return them
+            if last_depended == depended:
+                return packages.keys()
+        # otherwise reduce again
+        last_depended = depended
+
+
+def _include_comments():
+    # skipping comment lines when COLCON_TRACE is not set speeds up the
+    # processing especially on Windows
+    return bool(os.environ.get('COLCON_TRACE'))
+
+
+def get_commands(pkg_name, prefix, primary_extension, additional_extension):
+    commands = []
+    package_dsv_path = os.path.join(prefix, 'share', pkg_name, 'package.dsv')
+    if os.path.exists(package_dsv_path):
+        commands += process_dsv_file(
+            package_dsv_path, prefix, primary_extension, additional_extension)
+    return commands
+
+
+def process_dsv_file(
+    dsv_path, prefix, primary_extension=None, additional_extension=None
+):
+    commands = []
+    if _include_comments():
+        commands.append(FORMAT_STR_COMMENT_LINE.format_map({'comment': dsv_path}))
+    with open(dsv_path, 'r') as h:
+        content = h.read()
+    lines = content.splitlines()
+
+    basenames = OrderedDict()
+    for i, line in enumerate(lines):
+        # skip over empty or whitespace-only lines
+        if not line.strip():
+            continue
+        # skip over comments
+        if line.startswith('#'):
+            continue
+        try:
+            type_, remainder = line.split(';', 1)
+        except ValueError:
+            raise RuntimeError(
+                "Line %d in '%s' doesn't contain a semicolon separating the "
+                'type from the arguments' % (i + 1, dsv_path))
+        if type_ != DSV_TYPE_SOURCE:
+            # handle non-source lines
+            try:
+                commands += handle_dsv_types_except_source(
+                    type_, remainder, prefix)
+            except RuntimeError as e:
+                raise RuntimeError(
+                    "Line %d in '%s' %s" % (i + 1, dsv_path, e)) from e
+        else:
+            # group remaining source lines by basename
+            path_without_ext, ext = os.path.splitext(remainder)
+            if path_without_ext not in basenames:
+                basenames[path_without_ext] = set()
+            assert ext.startswith('.')
+            ext = ext[1:]
+            if ext in (primary_extension, additional_extension):
+                basenames[path_without_ext].add(ext)
+
+    # add the dsv extension to each basename if the file exists
+    for basename, extensions in basenames.items():
+        if not os.path.isabs(basename):
+            basename = os.path.join(prefix, basename)
+        if os.path.exists(basename + '.dsv'):
+            extensions.add('dsv')
+
+    for basename, extensions in basenames.items():
+        if not os.path.isabs(basename):
+            basename = os.path.join(prefix, basename)
+        if 'dsv' in extensions:
+            # process dsv files recursively
+            commands += process_dsv_file(
+                basename + '.dsv', prefix, primary_extension=primary_extension,
+                additional_extension=additional_extension)
+        elif primary_extension in extensions and len(extensions) == 1:
+            # source primary-only files
+            commands += [
+                FORMAT_STR_INVOKE_SCRIPT.format_map({
+                    'prefix': prefix,
+                    'script_path': basename + '.' + primary_extension})]
+        elif additional_extension in extensions:
+            # source non-primary files
+            commands += [
+                FORMAT_STR_INVOKE_SCRIPT.format_map({
+                    'prefix': prefix,
+                    'script_path': basename + '.' + additional_extension})]
+
+    return commands
+
+
+def handle_dsv_types_except_source(type_, remainder, prefix):
+    commands = []
+    if type_ in (DSV_TYPE_SET, DSV_TYPE_SET_IF_UNSET):
+        try:
+            env_name, value = remainder.split(';', 1)
+        except ValueError:
+            raise RuntimeError(
+                "doesn't contain a semicolon separating the environment name "
+                'from the value')
+        try_prefixed_value = os.path.join(prefix, value) if value else prefix
+        if os.path.exists(try_prefixed_value):
+            value = try_prefixed_value
+        if type_ == DSV_TYPE_SET:
+            commands += _set(env_name, value)
+        elif type_ == DSV_TYPE_SET_IF_UNSET:
+            commands += _set_if_unset(env_name, value)
+        else:
+            assert False
+    elif type_ in (
+        DSV_TYPE_APPEND_NON_DUPLICATE,
+        DSV_TYPE_PREPEND_NON_DUPLICATE,
+        DSV_TYPE_PREPEND_NON_DUPLICATE_IF_EXISTS
+    ):
+        try:
+            env_name_and_values = remainder.split(';')
+        except ValueError:
+            raise RuntimeError(
+                "doesn't contain a semicolon separating the environment name "
+                'from the values')
+        env_name = env_name_and_values[0]
+        values = env_name_and_values[1:]
+        for value in values:
+            if not value:
+                value = prefix
+            elif not os.path.isabs(value):
+                value = os.path.join(prefix, value)
+            if (
+                type_ == DSV_TYPE_PREPEND_NON_DUPLICATE_IF_EXISTS and
+                not os.path.exists(value)
+            ):
+                comment = f'skip extending {env_name} with not existing ' \
+                    f'path: {value}'
+                if _include_comments():
+                    commands.append(
+                        FORMAT_STR_COMMENT_LINE.format_map({'comment': comment}))
+            elif type_ == DSV_TYPE_APPEND_NON_DUPLICATE:
+                commands += _append_unique_value(env_name, value)
+            else:
+                commands += _prepend_unique_value(env_name, value)
+    else:
+        raise RuntimeError(
+            'contains an unknown environment hook type: ' + type_)
+    return commands
+
+
+env_state = {}
+
+
+def _append_unique_value(name, value):
+    global env_state
+    if name not in env_state:
+        if os.environ.get(name):
+            env_state[name] = set(os.environ[name].split(os.pathsep))
+        else:
+            env_state[name] = set()
+    # append even if the variable has not been set yet, in case a shell script sets the
+    # same variable without the knowledge of this Python script.
+    # later _remove_ending_separators() will cleanup any unintentional leading separator
+    extend = FORMAT_STR_USE_ENV_VAR.format_map({'name': name}) + os.pathsep
+    line = FORMAT_STR_SET_ENV_VAR.format_map(
+        {'name': name, 'value': extend + value})
+    if value not in env_state[name]:
+        env_state[name].add(value)
+    else:
+        if not _include_comments():
+            return []
+        line = FORMAT_STR_COMMENT_LINE.format_map({'comment': line})
+    return [line]
+
+
+def _prepend_unique_value(name, value):
+    global env_state
+    if name not in env_state:
+        if os.environ.get(name):
+            env_state[name] = set(os.environ[name].split(os.pathsep))
+        else:
+            env_state[name] = set()
+    # prepend even if the variable has not been set yet, in case a shell script sets the
+    # same variable without the knowledge of this Python script.
+    # later _remove_ending_separators() will cleanup any unintentional trailing separator
+    extend = os.pathsep + FORMAT_STR_USE_ENV_VAR.format_map({'name': name})
+    line = FORMAT_STR_SET_ENV_VAR.format_map(
+        {'name': name, 'value': value + extend})
+    if value not in env_state[name]:
+        env_state[name].add(value)
+    else:
+        if not _include_comments():
+            return []
+        line = FORMAT_STR_COMMENT_LINE.format_map({'comment': line})
+    return [line]
+
+
+# generate commands for removing prepended underscores
+def _remove_ending_separators():
+    # do nothing if the shell extension does not implement the logic
+    if FORMAT_STR_REMOVE_TRAILING_SEPARATOR is None:
+        return []
+
+    global env_state
+    commands = []
+    for name in env_state:
+        # skip variables that already had values before this script started prepending
+        if name in os.environ:
+            continue
+        commands += [
+            FORMAT_STR_REMOVE_LEADING_SEPARATOR.format_map({'name': name}),
+            FORMAT_STR_REMOVE_TRAILING_SEPARATOR.format_map({'name': name})]
+    return commands
+
+
+def _set(name, value):
+    global env_state
+    env_state[name] = value
+    line = FORMAT_STR_SET_ENV_VAR.format_map(
+        {'name': name, 'value': value})
+    return [line]
+
+
+def _set_if_unset(name, value):
+    global env_state
+    line = FORMAT_STR_SET_ENV_VAR.format_map(
+        {'name': name, 'value': value})
+    if env_state.get(name, os.environ.get(name)):
+        line = FORMAT_STR_COMMENT_LINE.format_map({'comment': line})
+    return [line]
+
+
+if __name__ == '__main__':  # pragma: no cover
+    try:
+        rc = main()
+    except RuntimeError as e:
+        print(str(e), file=sys.stderr)
+        rc = 1
+    sys.exit(rc)